"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
Implement Gofile backend - fixes #4632
Nick Craig-Woodc1a98768bc6f
diff --git a/README.md b/README.mdindex 833902ebc..fe3604e4e 100644--- a/README.md+++ b/README.md@@ -56,6 +56,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and* Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)* Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)* FTP [:page_facing_up:](https://rclone.org/ftp/)+ * GoFile [:page_facing_up:](https://rclone.org/gofile/)* Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)* Google Drive [:page_facing_up:](https://rclone.org/drive/)* Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)diff --git a/backend/all/all.go b/backend/all/all.goindex 9800df87b..b9ba25509 100644--- a/backend/all/all.go+++ b/backend/all/all.go@@ -18,6 +18,7 @@ import (_ "github.com/rclone/rclone/backend/fichier"_ "github.com/rclone/rclone/backend/filefabric"_ "github.com/rclone/rclone/backend/ftp"+ _ "github.com/rclone/rclone/backend/gofile"_ "github.com/rclone/rclone/backend/googlecloudstorage"_ "github.com/rclone/rclone/backend/googlephotos"_ "github.com/rclone/rclone/backend/hasher"diff --git a/backend/gofile/api/types.go b/backend/gofile/api/types.gonew file mode 100644index 000000000..4799687d0--- /dev/null+++ b/backend/gofile/api/types.go@@ -0,0 +1,311 @@+// Package api has type definitions for gofile+//+// Converted from the API docs with help from https://mholt.github.io/json-to-go/+package api++import (+ "fmt"+ "time"+)++const (+ // 2017-05-03T07:26:10-07:00+ timeFormat = `"` + time.RFC3339 + `"`+)++// Time represents date and time information for the+// gofile API, by using RFC3339+type Time time.Time++// MarshalJSON turns a Time into JSON (in UTC)+func (t *Time) MarshalJSON() (out []byte, err error) {+ timeString := (*time.Time)(t).Format(timeFormat)+ return []byte(timeString), nil+}++// UnmarshalJSON turns JSON into a Time+func (t *Time) UnmarshalJSON(data []byte) error {+ newT, err := time.Parse(timeFormat, string(data))+ if err != nil {+ return err+ }+ *t = Time(newT)+ return nil+}++// Error is returned from gofile when things go wrong+type Error struct {+ Status string `json:"status"`+}++// Error returns a string for the error and satisfies the error interface+func (e Error) Error() string {+ out := fmt.Sprintf("Error %q", e.Status)+ return out+}++// IsError returns true if there is an error+func (e Error) IsError() bool {+ return e.Status != "ok"+}++// Err returns err if not nil, or e if IsError or nil+func (e Error) Err(err error) error {+ if err != nil {+ return err+ }+ if e.IsError() {+ return e+ }+ return nil+}++// Check Error satisfies the error interface+var _ error = (*Error)(nil)++// Types of things in Item+const (+ ItemTypeFolder = "folder"+ ItemTypeFile = "file"+)++// Item describes a folder or a file as returned by /contents+type Item struct {+ ID string `json:"id"`+ ParentFolder string `json:"parentFolder"`+ Type string `json:"type"`+ Name string `json:"name"`+ Size int64 `json:"size"`+ Code string `json:"code"`+ CreateTime int64 `json:"createTime"`+ ModTime int64 `json:"modTime"`+ Link string `json:"link"`+ MD5 string `json:"md5"`+ MimeType string `json:"mimetype"`+ ChildrenCount int `json:"childrenCount"`+ DirectLinks map[string]*DirectLink `json:"directLinks"`+ //Public bool `json:"public"`+ //ServerSelected string `json:"serverSelected"`+ //Thumbnail string `json:"thumbnail"`+ //DownloadCount int `json:"downloadCount"`+ //TotalDownloadCount int64 `json:"totalDownloadCount"`+ //TotalSize int64 `json:"totalSize"`+ //ChildrenIDs []string `json:"childrenIds"`+ Children map[string]*Item `json:"children"`+}++// ToNativeTime converts a go time to a native time+func ToNativeTime(t time.Time) int64 {+ return t.Unix()+}++// FromNativeTime converts native time to a go time+func FromNativeTime(t int64) time.Time {+ return time.Unix(t, 0)+}++// DirectLink describes a direct link to a file so it can be+// downloaded by third parties.+type DirectLink struct {+ ExpireTime int64 `json:"expireTime"`+ SourceIpsAllowed []any `json:"sourceIpsAllowed"`+ DomainsAllowed []any `json:"domainsAllowed"`+ Auth []any `json:"auth"`+ IsReqLink bool `json:"isReqLink"`+ DirectLink string `json:"directLink"`+}++// Contents is returned from the /contents call+type Contents struct {+ Error+ Data struct {+ Item+ } `json:"data"`+ Metadata Metadata `json:"metadata"`+}++// Metadata is returned when paging is in use+type Metadata struct {+ TotalCount int `json:"totalCount"`+ TotalPages int `json:"totalPages"`+ Page int `json:"page"`+ PageSize int `json:"pageSize"`+ HasNextPage bool `json:"hasNextPage"`+}++// AccountsGetID is the result of /accounts/getid+type AccountsGetID struct {+ Error+ Data struct {+ ID string `json:"id"`+ } `json:"data"`+}++// Stats of storage and traffic+type Stats struct {+ FolderCount int64 `json:"folderCount"`+ FileCount int64 `json:"fileCount"`+ Storage int64 `json:"storage"`+ TrafficDirectGenerated int64 `json:"trafficDirectGenerated"`+ TrafficReqDownloaded int64 `json:"trafficReqDownloaded"`+ TrafficWebDownloaded int64 `json:"trafficWebDownloaded"`+}++// AccountsGet is the result of /accounts/{id}+type AccountsGet struct {+ Error+ Data struct {+ ID string `json:"id"`+ Email string `json:"email"`+ Tier string `json:"tier"`+ PremiumType string `json:"premiumType"`+ Token string `json:"token"`+ RootFolder string `json:"rootFolder"`+ SubscriptionProvider string `json:"subscriptionProvider"`+ SubscriptionEndDate int `json:"subscriptionEndDate"`+ SubscriptionLimitDirectTraffic int64 `json:"subscriptionLimitDirectTraffic"`+ SubscriptionLimitStorage int64 `json:"subscriptionLimitStorage"`+ StatsCurrent Stats `json:"statsCurrent"`+ // StatsHistory map[int]map[int]map[int]Stats `json:"statsHistory"`+ } `json:"data"`+}++// CreateFolderRequest is the input to /contents/createFolder+type CreateFolderRequest struct {+ ParentFolderID string `json:"parentFolderId"`+ FolderName string `json:"folderName"`+ ModTime int64 `json:"modTime,omitempty"`+}++// CreateFolderResponse is the output from /contents/createFolder+type CreateFolderResponse struct {+ Error+ Data Item `json:"data"`+}++// DeleteRequest is the input to DELETE /contents+type DeleteRequest struct {+ ContentsID string `json:"contentsId"` // comma separated list of IDs+}++// DeleteResponse is the input to DELETE /contents+type DeleteResponse struct {+ Error+ Data map[string]Error+}++// Server is an upload server+type Server struct {+ Name string `json:"name"`+ Zone string `json:"zone"`+}++// String returns a string representation of the Server+func (s *Server) String() string {+ return fmt.Sprintf("%s (%s)", s.Name, s.Zone)+}++// Root returns the root URL for the server+func (s *Server) Root() string {+ return fmt.Sprintf("https://%s.gofile.io/", s.Name)+}++// URL returns the upload URL for the server+func (s *Server) URL() string {+ return fmt.Sprintf("https://%s.gofile.io/contents/uploadfile", s.Name)+}++// ServersResponse is the output from /servers+type ServersResponse struct {+ Error+ Data struct {+ Servers []Server `json:"servers"`+ } `json:"data"`+}++// UploadResponse is returned by POST /contents/uploadfile+type UploadResponse struct {+ Error+ Data Item `json:"data"`+}++// DirectLinksRequest specifies the parameters for the direct link+type DirectLinksRequest struct {+ ExpireTime int64 `json:"expireTime,omitempty"`+ SourceIpsAllowed []any `json:"sourceIpsAllowed,omitempty"`+ DomainsAllowed []any `json:"domainsAllowed,omitempty"`+ Auth []any `json:"auth,omitempty"`+}++// DirectLinksResult is returned from POST /contents/{id}/directlinks+type DirectLinksResult struct {+ Error+ Data struct {+ ExpireTime int64 `json:"expireTime"`+ SourceIpsAllowed []any `json:"sourceIpsAllowed"`+ DomainsAllowed []any `json:"domainsAllowed"`+ Auth []any `json:"auth"`+ IsReqLink bool `json:"isReqLink"`+ ID string `json:"id"`+ DirectLink string `json:"directLink"`+ } `json:"data"`+}++// UpdateItemRequest describes the updates to be done to an item for PUT /contents/{id}/update+//+// The Value of the attribute to define :+// For Attribute "name" : The name of the content (file or folder)+// For Attribute "description" : The description displayed on the download page (folder only)+// For Attribute "tags" : A comma-separated list of tags (folder only)+// For Attribute "public" : either true or false (folder only)+// For Attribute "expiry" : A unix timestamp of the expiration date (folder only)+// For Attribute "password" : The password to set (folder only)+type UpdateItemRequest struct {+ Attribute string `json:"attribute"`+ Value any `json:"attributeValue"`+}++// UpdateItemResponse is returned by PUT /contents/{id}/update+type UpdateItemResponse struct {+ Error+ Data Item `json:"data"`+}++// MoveRequest is the input to /contents/move+type MoveRequest struct {+ FolderID string `json:"folderId"`+ ContentsID string `json:"contentsId"` // comma separated list of IDs+}++// MoveResponse is returned by POST /contents/move+type MoveResponse struct {+ Error+ Data map[string]struct {+ Error+ Item `json:"data"`+ } `json:"data"`+}++// CopyRequest is the input to /contents/copy+type CopyRequest struct {+ FolderID string `json:"folderId"`+ ContentsID string `json:"contentsId"` // comma separated list of IDs+}++// CopyResponse is returned by POST /contents/copy+type CopyResponse struct {+ Error+ Data map[string]struct {+ Error+ Item `json:"data"`+ } `json:"data"`+}++// UploadServerStatus is returned when fetching the root of an upload server+type UploadServerStatus struct {+ Error+ Data struct {+ Server string `json:"server"`+ Test string `json:"test"`+ } `json:"data"`+}diff --git a/backend/gofile/gofile.go b/backend/gofile/gofile.gonew file mode 100644index 000000000..4b009e8cb--- /dev/null+++ b/backend/gofile/gofile.go@@ -0,0 +1,1633 @@+// Package gofile provides an interface to the Gofile+// object storage system.+package gofile++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "math/rand"+ "net/http"+ "net/url"+ "path"+ "strconv"+ "strings"+ "sync"+ "time"++ "github.com/rclone/rclone/backend/gofile/api"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config"+ "github.com/rclone/rclone/fs/config/configmap"+ "github.com/rclone/rclone/fs/config/configstruct"+ "github.com/rclone/rclone/fs/fserrors"+ "github.com/rclone/rclone/fs/fshttp"+ "github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/lib/dircache"+ "github.com/rclone/rclone/lib/encoder"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/rest"+)++const (+ minSleep = 10 * time.Millisecond+ maxSleep = 20 * time.Second+ decayConstant = 1 // bigger for slower decay, exponential+ rootURL = "https://api.gofile.io"+ serversExpiry = 60 * time.Second // check for new upload servers this often+ serversActive = 2 // choose this many closest upload servers to use+ rateLimitSleep = 5 * time.Second // penalise a goroutine by this long for making a rate limit error+ maxDepth = 4 // in ListR recursive list this deep (maximum is 16)+)++/*+ // TestGoFile{sb0-v}+stringNeedsEscaping = []rune{+ '!', '*', '.', '/', ':', '<', '>', '?', '\"', '\\', '\a', '\b', '\f', '\n', '\r', '\t', '\v', '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x0e', '\x0f', '\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', '\x18', '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f', '\x7f', '\xbf', '\xfe', '|'+}+maxFileLength = 255 // for 1 byte unicode characters+maxFileLength = 255 // for 2 byte unicode characters+maxFileLength = 255 // for 3 byte unicode characters+maxFileLength = 255 // for 4 byte unicode characters+canWriteUnnormalized = true+canReadUnnormalized = true+canReadRenormalized = false+canStream = true+base32768isOK = false // make sure maxFileLength for 2 byte unicode chars is the same as for 1 byte characters+*/++// Register with Fs+func init() {+ fs.Register(&fs.RegInfo{+ Name: "gofile",+ Description: "Gofile",+ NewFs: NewFs,+ Options: []fs.Option{{+ Name: "access_token",+ Help: `API Access token++You can get this from the web control panel.`,+ Sensitive: true,+ }, {+ Name: "root_folder_id",+ Help: `ID of the root folder++Leave this blank normally, rclone will fill it in automatically.++If you want rclone to be restricted to a particular folder you can+fill it in - see the docs for more info.+`,+ Default: "",+ Advanced: true,+ Sensitive: true,+ }, {+ Name: "account_id",+ Help: `Account ID++Leave this blank normally, rclone will fill it in automatically.+`,+ Default: "",+ Advanced: true,+ Sensitive: true,+ }, {+ Name: "list_chunk",+ Help: `Number of items to list in each call`,+ Default: 1000,+ Advanced: true,+ }, {+ Name: config.ConfigEncoding,+ Help: config.ConfigEncodingHelp,+ Advanced: true,+ Default: (encoder.Display | // Slash Control Delete Dot+ encoder.EncodeDoubleQuote |+ encoder.EncodeAsterisk |+ encoder.EncodeColon |+ encoder.EncodeLtGt |+ encoder.EncodeQuestion |+ encoder.EncodeBackSlash |+ encoder.EncodePipe |+ encoder.EncodeExclamation |+ encoder.EncodeLeftPeriod |+ encoder.EncodeRightPeriod |+ encoder.EncodeInvalidUtf8),+ }},+ })+}++// Options defines the configuration for this backend+type Options struct {+ AccessToken string `config:"access_token"`+ RootFolderID string `config:"root_folder_id"`+ AccountID string `config:"account_id"`+ ListChunk int `config:"list_chunk"`+ Enc encoder.MultiEncoder `config:"encoding"`+}++// Fs represents a remote gofile+type Fs struct {+ name string // name of this remote+ root string // the path we are working on+ opt Options // parsed options+ features *fs.Features // optional features+ srv *rest.Client // the connection to the server+ dirCache *dircache.DirCache // Map of directory path to directory id+ pacer *fs.Pacer // pacer for API calls+ serversMu *sync.Mutex // protect the servers info below+ servers []api.Server // upload servers we can use+ serversChecked time.Time // time the servers were refreshed+}++// Object describes a gofile object+//+// The full set of metadata will always be present+type Object struct {+ fs *Fs // what this object is part of+ remote string // The remote path+ size int64 // size of the object+ modTime time.Time // modification time of the object+ id string // ID of the object+ dirID string // ID of the object's directory+ mimeType string // mime type of the object+ md5 string // MD5 of the object content+ url string // where to download this object+}++// Directory describes a gofile directory+type Directory struct {+ Object+ items int64 // number of items in the directory+}++// ------------------------------------------------------------++// Name of the remote (as passed into NewFs)+func (f *Fs) Name() string {+ return f.name+}++// Root of the remote (as passed into NewFs)+func (f *Fs) Root() string {+ return f.root+}++// String converts this Fs to a string+func (f *Fs) String() string {+ return fmt.Sprintf("gofile root '%s'", f.root)+}++// Features returns the optional features of this Fs+func (f *Fs) Features() *fs.Features {+ return f.features+}++// parsePath parses a gofile 'url'+func parsePath(path string) (root string) {+ root = strings.Trim(path, "/")+ return+}++// retryErrorCodes is a slice of error codes that we will retry+var retryErrorCodes = []int{+ 429, // Too Many Requests.+ 500, // Internal Server Error+ 502, // Bad Gateway+ 503, // Service Unavailable+ 504, // Gateway Timeout+ 509, // Bandwidth Limit Exceeded+}++// Return true if the api error has the status given+func isAPIErr(err error, status string) bool {+ var apiErr api.Error+ if errors.As(err, &apiErr) {+ return apiErr.Status == status+ }+ return false+}++// shouldRetry returns a boolean as to whether this resp and err+// deserve to be retried. It returns the err as a convenience+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {+ if fserrors.ContextError(ctx, &err) {+ return false, err+ }+ if isAPIErr(err, "error-rateLimit") {+ // Give an immediate penalty to rate limits+ fs.Debugf(nil, "Rate limited, sleep for %v", rateLimitSleep)+ time.Sleep(rateLimitSleep)+ //return true, pacer.RetryAfterError(err, 2*time.Second)+ return true, err+ }+ return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err+}++// readMetaDataForPath reads the metadata from the path+func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {+ // defer log.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)+ leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)+ if err != nil {+ if err == fs.ErrorDirNotFound {+ return nil, fs.ErrorObjectNotFound+ }+ return nil, err+ }++ found, err := f.listAll(ctx, directoryID, false, true, leaf, func(item *api.Item) bool {+ if item.Name == leaf {+ info = item+ return true+ }+ return false+ })+ if err != nil {+ return nil, err+ }+ if !found {+ return nil, fs.ErrorObjectNotFound+ }+ return info, nil+}++// readMetaDataForID reads the metadata for the ID given+func (f *Fs) readMetaDataForID(ctx context.Context, id string) (info *api.Item, err error) {+ opts := rest.Opts{+ Method: "GET",+ Path: "/contents/" + id,+ Parameters: url.Values{+ "page": {"1"},+ "pageSize": {"1"}, // not interested in children so just ask for 1+ },+ }+ var result api.Contents+ err = f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)+ // Retry not found errors - when looking for an ID it should really exist+ if isAPIErr(err, "error-notFound") {+ return true, err+ }+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ return nil, fmt.Errorf("failed to get item info: %w", err)+ }+ return &result.Data.Item, nil+}++// errorHandler parses a non 2xx error response into an error+func errorHandler(resp *http.Response) error {+ body, err := rest.ReadBody(resp)+ if err != nil {+ fs.Debugf(nil, "Couldn't read error out of body: %v", err)+ body = nil+ }+ // Decode error response if there was one - they can be blank+ var errResponse api.Error+ if len(body) > 0 {+ err = json.Unmarshal(body, &errResponse)+ if err != nil {+ fs.Debugf(nil, "Couldn't decode error response: %v", err)+ }+ }+ if errResponse.Status == "" {+ errResponse.Status = fmt.Sprintf("%s (%d): %s", resp.Status, resp.StatusCode, string(body))+ }+ return errResponse+}++// NewFs constructs an Fs from the path, container:path+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {+ // Parse config into Options struct+ opt := new(Options)+ err := configstruct.Set(m, opt)+ if err != nil {+ return nil, err+ }++ root = parsePath(root)++ client := fshttp.NewClient(ctx)++ f := &Fs{+ name: name,+ root: root,+ opt: *opt,+ srv: rest.NewClient(client).SetRoot(rootURL),+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),+ serversMu: new(sync.Mutex),+ }+ f.features = (&fs.Features{+ CaseInsensitive: false,+ CanHaveEmptyDirectories: true,+ DuplicateFiles: true,+ ReadMimeType: true,+ WriteMimeType: false,+ WriteDirSetModTime: true,+ DirModTimeUpdatesOnWrite: true,+ }).Fill(ctx, f)+ f.srv.SetErrorHandler(errorHandler)+ f.srv.SetHeader("Authorization", "Bearer "+f.opt.AccessToken)++ // Read account ID if not present+ err = f.readAccountID(ctx, m)+ if err != nil {+ return nil, err+ }++ // Read Root Folder ID if not present+ err = f.readRootFolderID(ctx, m)+ if err != nil {+ return nil, err+ }++ // Get rootFolderID+ rootID := f.opt.RootFolderID+ f.dirCache = dircache.New(root, rootID, f)++ // Find the current root+ err = f.dirCache.FindRoot(ctx, false)+ if err != nil {+ // Assume it is a file+ newRoot, remote := dircache.SplitPath(root)+ tempF := *f+ tempF.dirCache = dircache.New(newRoot, rootID, &tempF)+ tempF.root = newRoot+ // Make new Fs which is the parent+ err = tempF.dirCache.FindRoot(ctx, false)+ if err != nil {+ // No root so return old f+ return f, nil+ }+ _, err := tempF.newObjectWithInfo(ctx, remote, nil)+ if err != nil {+ if err == fs.ErrorObjectNotFound {+ // File doesn't exist so return old f+ return f, nil+ }+ return nil, err+ }+ f.features.Fill(ctx, &tempF)+ // XXX: update the old f here instead of returning tempF, since+ // `features` were already filled with functions having *f as a receiver.+ // See https://github.com/rclone/rclone/issues/2182+ f.dirCache = tempF.dirCache+ f.root = tempF.root+ // return an error with an fs which points to the parent+ return f, fs.ErrorIsFile+ }+ return f, nil+}++// Read the AccountID into f.opt if not set and cache in the config file as account_id+func (f *Fs) readAccountID(ctx context.Context, m configmap.Mapper) (err error) {+ if f.opt.AccountID != "" {+ return nil+ }+ opts := rest.Opts{+ Method: "GET",+ Path: "/accounts/getid",+ }+ var result api.AccountsGetID+ var resp *http.Response+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ return fmt.Errorf("failed to read account ID: %w", err)+ }+ f.opt.AccountID = result.Data.ID+ m.Set("account_id", f.opt.AccountID)+ return nil+}++// Read the Accounts info+func (f *Fs) getAccounts(ctx context.Context) (result *api.AccountsGet, err error) {+ opts := rest.Opts{+ Method: "GET",+ Path: "/accounts/" + f.opt.AccountID,+ }+ result = new(api.AccountsGet)+ var resp *http.Response+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ return nil, fmt.Errorf("failed to read accountd info: %w", err)+ }+ return result, nil+}++// Read the RootFolderID into f.opt if not set and cache in the config file as root_folder_id+func (f *Fs) readRootFolderID(ctx context.Context, m configmap.Mapper) (err error) {+ if f.opt.RootFolderID != "" {+ return nil+ }+ result, err := f.getAccounts(ctx)+ if err != nil {+ return err+ }+ f.opt.RootFolderID = result.Data.RootFolder+ m.Set("root_folder_id", f.opt.RootFolderID)+ return nil+}++// Find the top n servers measured by response time+func (f *Fs) bestServers(ctx context.Context, servers []api.Server, n int) (newServers []api.Server) {+ ctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Second))+ defer cancel()++ if n > len(servers) {+ n = len(servers)+ }+ results := make(chan int, len(servers))++ // Test how long the servers take to respond+ for i := range servers {+ i := i // for closure+ go func() {+ opts := rest.Opts{+ Method: "GET",+ RootURL: servers[i].Root(),+ }+ var result api.UploadServerStatus+ start := time.Now()+ _, err := f.srv.CallJSON(ctx, &opts, nil, &result)+ ping := time.Since(start)+ err = result.Err(err)+ if err != nil {+ results <- -1 // send a -ve number on error+ return+ }+ fs.Debugf(nil, "Upload server %v responded in %v", &servers[i], ping)+ results <- i+ }()+ }++ // Wait for n servers to respond+ newServers = make([]api.Server, 0, n)+ for range servers {+ i := <-results+ if i >= 0 {+ newServers = append(newServers, servers[i])+ }+ if len(newServers) >= n {+ break+ }+ }+ return newServers+}++// Clear all the upload servers - call on an error+func (f *Fs) clearServers() {+ f.serversMu.Lock()+ defer f.serversMu.Unlock()++ fs.Debugf(f, "Clearing upload servers")+ f.servers = nil+}++// Gets an upload server+func (f *Fs) getServer(ctx context.Context) (server *api.Server, err error) {+ f.serversMu.Lock()+ defer f.serversMu.Unlock()++ if len(f.servers) == 0 || time.Since(f.serversChecked) >= serversExpiry {+ opts := rest.Opts{+ Method: "GET",+ Path: "/servers",+ }+ var result api.ServersResponse+ var resp *http.Response+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ if len(f.servers) == 0 {+ return nil, fmt.Errorf("failed to read upload servers: %w", err)+ }+ fs.Errorf(f, "failed to read new upload servers: %v", err)+ } else {+ // Find the top servers measured by response time+ f.servers = f.bestServers(ctx, result.Data.Servers, serversActive)+ f.serversChecked = time.Now()+ }+ }++ if len(f.servers) == 0 {+ return nil, errors.New("no upload servers found")+ }++ // Pick a server at random since we've already found the top ones+ i := rand.Intn(len(f.servers))+ return &f.servers[i], nil+}++// rootSlash returns root with a slash on if it is empty, otherwise empty string+func (f *Fs) rootSlash() string {+ if f.root == "" {+ return f.root+ }+ return f.root + "/"+}++// Return an Object from a path+//+// If it can't be found it returns the error fs.ErrorObjectNotFound.+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) {+ o := &Object{+ fs: f,+ remote: remote,+ }+ var err error+ if info != nil {+ // Set info+ err = o.setMetaData(info)+ } else {+ err = o.readMetaData(ctx) // reads info and meta, returning an error+ }+ if err != nil {+ return nil, err+ }+ return o, nil+}++// NewObject finds the Object at remote. If it can't be found+// it returns the error fs.ErrorObjectNotFound.+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {+ return f.newObjectWithInfo(ctx, remote, nil)+}++// FindLeaf finds a directory of name leaf in the folder with ID pathID+func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {+ // Find the leaf in pathID+ found, err = f.listAll(ctx, pathID, true, false, leaf, func(item *api.Item) bool {+ if item.Name == leaf {+ pathIDOut = item.ID+ return true+ }+ return false+ })+ return pathIDOut, found, err+}++// createDir makes a directory with pathID as parent and name leaf and modTime+func (f *Fs) createDir(ctx context.Context, pathID, leaf string, modTime time.Time) (item *api.Item, err error) {+ var resp *http.Response+ var result api.CreateFolderResponse+ opts := rest.Opts{+ Method: "POST",+ Path: "/contents/createFolder",+ }+ mkdir := api.CreateFolderRequest{+ FolderName: f.opt.Enc.FromStandardName(leaf),+ ParentFolderID: pathID,+ ModTime: api.ToNativeTime(modTime),+ }+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ return nil, fmt.Errorf("failed to create folder: %w", err)+ }+ return &result.Data, nil+}++// CreateDir makes a directory with pathID as parent and name leaf+func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {+ // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf)+ item, err := f.createDir(ctx, pathID, leaf, time.Now())+ if err != nil {+ return "", err+ }+ return item.ID, nil+}++// list the objects into the function supplied+//+// If directories is set it only sends directories+// User function to process a File item from listAll+//+// Should return true to finish processing+type listAllFn func(*api.Item) bool++// Lists the directory required calling the user function on each item found+//+// If name is set then the server will limit the returned items to those+// with that name.+//+// If the user fn ever returns true then it early exits with found = true+func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, name string, fn listAllFn) (found bool, err error) {+ opts := rest.Opts{+ Method: "GET",+ Path: "/contents/" + dirID,+ Parameters: url.Values{},+ }+ if name != "" {+ opts.Parameters.Add("contentname", f.opt.Enc.FromStandardName(name))+ }+ page := 1+OUTER:+ for {+ opts.Parameters.Set("page", strconv.Itoa(page))+ opts.Parameters.Set("pageSize", strconv.Itoa(f.opt.ListChunk))+ var result api.Contents+ var resp *http.Response+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ if isAPIErr(err, "error-notFound") {+ return found, fs.ErrorDirNotFound+ }+ return found, fmt.Errorf("couldn't list files: %w", err)+ }+ for id, item := range result.Data.Children {+ _ = id+ if item.Type == api.ItemTypeFolder {+ if filesOnly {+ continue+ }+ } else if item.Type == api.ItemTypeFile {+ if directoriesOnly {+ continue+ }+ } else {+ fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type)+ continue+ }+ item.Name = f.opt.Enc.ToStandardName(item.Name)+ if fn(item) {+ found = true+ break OUTER+ }+ }+ if !result.Metadata.HasNextPage {+ break+ }+ page += 1+ }+ return found, err+}++// Convert a list item into a DirEntry+func (f *Fs) itemToDirEntry(ctx context.Context, remote string, info *api.Item) (entry fs.DirEntry, err error) {+ if info.Type == api.ItemTypeFolder {+ // cache the directory ID for later lookups+ f.dirCache.Put(remote, info.ID)+ d := &Directory{+ Object: Object{+ fs: f,+ remote: remote,+ },+ items: int64(info.ChildrenCount),+ }+ d.setMetaDataAny(info)+ entry = d+ } else if info.Type == api.ItemTypeFile {+ entry, err = f.newObjectWithInfo(ctx, remote, info)+ if err != nil {+ return nil, err+ }+ }+ return entry, nil+}++// List the objects and directories in dir into entries. The+// entries can be returned in any order but should be for a+// complete directory.+//+// dir should be "" to list the root, and should not have+// trailing slashes.+//+// This should return ErrDirNotFound if the directory isn't+// found.+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {+ directoryID, err := f.dirCache.FindDir(ctx, dir, false)+ if err != nil {+ return nil, err+ }+ var iErr error+ _, err = f.listAll(ctx, directoryID, false, false, "", func(info *api.Item) bool {+ remote := path.Join(dir, info.Name)+ entry, err := f.itemToDirEntry(ctx, remote, info)+ if err != nil {+ iErr = err+ return true+ }+ entries = append(entries, entry)+ return false+ })+ if err != nil {+ return nil, err+ }+ if iErr != nil {+ return nil, iErr+ }+ return entries, nil+}++// implementation of ListR+func (f *Fs) listR(ctx context.Context, dir string, list *walk.ListRHelper) (err error) {+ directoryID, err := f.dirCache.FindDir(ctx, dir, false)+ if err != nil {+ return err+ }+ opts := rest.Opts{+ Method: "GET",+ Path: "/contents/" + directoryID,+ Parameters: url.Values{"maxdepth": {strconv.Itoa(maxDepth)}},+ }+ page := 1+ for {+ opts.Parameters.Set("page", strconv.Itoa(page))+ opts.Parameters.Set("pageSize", strconv.Itoa(f.opt.ListChunk))+ var result api.Contents+ var resp *http.Response+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ if isAPIErr(err, "error-notFound") {+ return fs.ErrorDirNotFound+ }+ return fmt.Errorf("couldn't recursively list files: %w", err)+ }+ // Result.Data.Item now contains a recursive listing so we will have to decode recursively+ var decode func(string, *api.Item) error+ decode = func(dir string, dirItem *api.Item) error {+ // If we have ChildrenCount but no Children this means the recursion stopped here+ if dirItem.ChildrenCount > 0 && len(dirItem.Children) == 0 {+ return f.listR(ctx, dir, list)+ }+ for _, item := range dirItem.Children {+ if item.Type != api.ItemTypeFolder && item.Type != api.ItemTypeFile {+ fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type)+ continue+ }+ item.Name = f.opt.Enc.ToStandardName(item.Name)+ remote := path.Join(dir, item.Name)+ entry, err := f.itemToDirEntry(ctx, remote, item)+ if err != nil {+ return err+ }+ err = list.Add(entry)+ if err != nil {+ return err+ }+ if item.Type == api.ItemTypeFolder {+ err := decode(remote, item)+ if err != nil {+ return err+ }+ }+ }+ return nil+ }+ err = decode(dir, &result.Data.Item)+ if err != nil {+ return err+ }+ if !result.Metadata.HasNextPage {+ break+ }+ page += 1+ }+ return err+}++// ListR lists the objects and directories of the Fs starting+// from dir recursively into out.+//+// dir should be "" to start from the root, and should not+// have trailing slashes.+//+// This should return ErrDirNotFound if the directory isn't+// found.+//+// It should call callback for each tranche of entries read.+// These need not be returned in any particular order. If+// callback returns an error then the listing will stop+// immediately.+//+// Don't implement this unless you have a more efficient way+// of listing recursively than doing a directory traversal.+func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {+ list := walk.NewListRHelper(callback)+ err = f.listR(ctx, dir, list)+ if err != nil {+ return err+ }+ return list.Flush()+}++// Creates from the parameters passed in a half finished Object which+// must have setMetaData called on it+//+// Returns the object, leaf, directoryID and error.+//+// Used to create new objects+func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {+ // Create the directory for the object if it doesn't exist+ leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)+ if err != nil {+ return+ }+ // Temporary Object under construction+ o = &Object{+ fs: f,+ remote: remote,+ }+ return o, leaf, directoryID, nil+}++// Put the object+//+// Copy the reader in to the new object which is returned.+//+// The new object may have been created if an error is returned+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {+ existingObj, err := f.NewObject(ctx, src.Remote())+ switch err {+ case nil:+ return existingObj, existingObj.Update(ctx, in, src, options...)+ case fs.ErrorObjectNotFound:+ // Not found so create it+ return f.PutUnchecked(ctx, in, src, options...)+ default:+ return nil, err+ }+}++// PutStream uploads to the remote path with the modTime given of indeterminate size+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {+ return f.Put(ctx, in, src, options...)+}++// PutUnchecked the object into the container+//+// This will produce a duplicate if the object already exists.+//+// Copy the reader in to the new object which is returned.+//+// The new object may have been created if an error is returned+func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {+ remote := src.Remote()+ size := src.Size()+ modTime := src.ModTime(ctx)++ o, _, _, err := f.createObject(ctx, remote, modTime, size)+ if err != nil {+ return nil, err+ }+ return o, o.Update(ctx, in, src, options...)+}++// Mkdir creates the container if it doesn't exist+func (f *Fs) Mkdir(ctx context.Context, dir string) error {+ _, err := f.dirCache.FindDir(ctx, dir, true)+ return err+}++// DirSetModTime sets the directory modtime for dir+func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {+ dirID, err := f.dirCache.FindDir(ctx, dir, false)+ if err != nil {+ return err+ }+ _, err = f.setModTime(ctx, dirID, modTime)+ return err+}++// deleteObject removes an object by ID+func (f *Fs) deleteObject(ctx context.Context, id string) error {+ opts := rest.Opts{+ Method: "DELETE",+ Path: "/contents/",+ }+ request := api.DeleteRequest{+ ContentsID: id,+ }+ var result api.DeleteResponse+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ return fmt.Errorf("failed to delete item: %w", err)+ }+ // Check the individual result codes also+ for _, err := range result.Data {+ if err.IsError() {+ return fmt.Errorf("failed to delete item: %w", err)+ }+ }+ return nil+}++// purgeCheck removes the root directory, if check is set then it+// refuses to do so if it has anything in+func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {+ root := path.Join(f.root, dir)+ if root == "" {+ return errors.New("can't purge root directory")+ }+ dc := f.dirCache+ rootID, err := dc.FindDir(ctx, dir, false)+ if err != nil {+ return err+ }++ // Check to see if there is contents in the directory+ if check {+ found, err := f.listAll(ctx, rootID, false, false, "", func(item *api.Item) bool {+ return true+ })+ if err != nil {+ return err+ }+ if found {+ return fs.ErrorDirectoryNotEmpty+ }+ }++ // Delete the directory+ err = f.deleteObject(ctx, rootID)+ if err != nil {+ return err+ }++ f.dirCache.FlushDir(dir)+ return nil+}++// Rmdir deletes the root folder+//+// Returns an error if it isn't empty+func (f *Fs) Rmdir(ctx context.Context, dir string) error {+ return f.purgeCheck(ctx, dir, true)+}++// Precision return the precision of this Fs+func (f *Fs) Precision() time.Duration {+ return time.Second+}++// Purge deletes all the files and the container+//+// Optional interface: Only implement this if you have a way of+// deleting all the files quicker than just running Remove() on the+// result of List()+func (f *Fs) Purge(ctx context.Context, dir string) error {+ return f.purgeCheck(ctx, dir, false)+}++// About gets quota information+func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {+ result, err := f.getAccounts(ctx)+ if err != nil {+ return nil, err+ }+ used := result.Data.StatsCurrent.Storage+ files := result.Data.StatsCurrent.FileCount+ total := result.Data.SubscriptionLimitStorage+ usage = &fs.Usage{+ Used: fs.NewUsageValue(used), // bytes in use+ Total: fs.NewUsageValue(total), // bytes total+ Free: fs.NewUsageValue(total - used), // bytes free+ Objects: fs.NewUsageValue(files), // total objects+ }+ return usage, nil+}++// patch an attribute on an object to value+func (f *Fs) patch(ctx context.Context, id, attribute string, value any) (item *api.Item, err error) {+ var resp *http.Response+ var request = api.UpdateItemRequest{+ Attribute: attribute,+ Value: value,+ }+ var result api.UpdateItemResponse+ opts := rest.Opts{+ Method: "PUT",+ Path: "/contents/" + id + "/update",+ }+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ return nil, fmt.Errorf("failed to patch item %q to %v: %w", attribute, value, err)+ }+ return &result.Data, nil+}++// rename a file or a folder+func (f *Fs) rename(ctx context.Context, id, newLeaf string) (item *api.Item, err error) {+ return f.patch(ctx, id, "name", f.opt.Enc.FromStandardName(newLeaf))+}++// setModTime sets the modification time of a file or folder+func (f *Fs) setModTime(ctx context.Context, id string, modTime time.Time) (item *api.Item, err error) {+ return f.patch(ctx, id, "modTime", api.ToNativeTime(modTime))+}++// move a file or a folder to a new directory+func (f *Fs) move(ctx context.Context, id, newDirID string) (item *api.Item, err error) {+ var resp *http.Response+ var request = api.MoveRequest{+ FolderID: newDirID,+ ContentsID: id,+ }+ var result api.MoveResponse+ opts := rest.Opts{+ Method: "PUT",+ Path: "/contents/move",+ }+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ return nil, fmt.Errorf("failed to move item: %w", err)+ }+ itemResult, ok := result.Data[id]+ if !ok || itemResult.Item.ID == "" {+ return nil, errors.New("failed to read result of move")+ }+ return &itemResult.Item, nil+}++// move and rename a file or folder to directoryID with leaf+func (f *Fs) moveTo(ctx context.Context, id, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID string) (info *api.Item, err error) {+ // Can have duplicates so don't have to be careful here++ // Rename if required+ if srcLeaf != dstLeaf {+ info, err = f.rename(ctx, id, dstLeaf)+ if err != nil {+ return nil, err+ }+ }+ // Move if required+ if srcDirectoryID != dstDirectoryID {+ info, err = f.move(ctx, id, dstDirectoryID)+ if err != nil {+ return nil, err+ }+ }+ if info == nil {+ return f.readMetaDataForID(ctx, id)+ }+ return info, nil+}++// Move src to this remote using server-side move operations.+//+// This is stored with the remote path given.+//+// It returns the destination Object and a possible error.+//+// Will only be called if src.Fs().Name() == f.Name()+//+// If it isn't possible then return fs.ErrorCantMove+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {+ srcObj, ok := src.(*Object)+ if !ok {+ fs.Debugf(src, "Can't move - not same remote type")+ return nil, fs.ErrorCantMove+ }++ // Create temporary object+ dstObj, dstLeaf, dstDirectoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size)+ if err != nil {+ return nil, err+ }++ // Do the move+ info, err := f.moveTo(ctx, srcObj.id, path.Base(srcObj.remote), dstLeaf, srcObj.dirID, dstDirectoryID)+ if err != nil {+ return nil, err+ }++ err = dstObj.setMetaData(info)+ if err != nil {+ return nil, err+ }+ return dstObj, nil+}++// DirMove moves src, srcRemote to this remote at dstRemote+// using server-side move operations.+//+// Will only be called if src.Fs().Name() == f.Name()+//+// If it isn't possible then return fs.ErrorCantDirMove+//+// If destination exists then return fs.ErrorDirExists+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {+ srcFs, ok := src.(*Fs)+ if !ok {+ fs.Debugf(srcFs, "Can't move directory - not same remote type")+ return fs.ErrorCantDirMove+ }++ srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)+ if err != nil {+ return err+ }++ // Do the move+ _, err = f.moveTo(ctx, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)+ if err != nil {+ return err+ }+ srcFs.dirCache.FlushDir(srcRemote)+ return nil+}++// copy a file or a folder to a new directory+func (f *Fs) copy(ctx context.Context, id, newDirID string) (item *api.Item, err error) {+ var resp *http.Response+ var request = api.CopyRequest{+ FolderID: newDirID,+ ContentsID: id,+ }+ var result api.CopyResponse+ opts := rest.Opts{+ Method: "POST",+ Path: "/contents/copy",+ }+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ return nil, fmt.Errorf("failed to copy item: %w", err)+ }+ itemResult, ok := result.Data[id]+ if !ok || itemResult.Item.ID == "" {+ return nil, errors.New("failed to read result of copy")+ }+ return &itemResult.Item, nil+}++// copy and rename a file or folder to directoryID with leaf+func (f *Fs) copyTo(ctx context.Context, srcID, srcLeaf, dstLeaf, dstDirectoryID string) (info *api.Item, err error) {+ // Can have duplicates so don't have to be careful here++ // Copy to dstDirectoryID first+ info, err = f.copy(ctx, srcID, dstDirectoryID)+ if err != nil {+ return nil, err+ }++ // Rename if required+ if srcLeaf != dstLeaf {+ info, err = f.rename(ctx, info.ID, dstLeaf)+ if err != nil {+ return nil, err+ }+ }+ return info, nil+}++// Copy src to this remote using server-side copy operations.+//+// This is stored with the remote path given.+//+// It returns the destination Object and a possible error.+//+// Will only be called if src.Fs().Name() == f.Name()+//+// If it isn't possible then return fs.ErrorCantCopy+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {+ srcObj, ok := src.(*Object)+ if !ok {+ fs.Debugf(src, "Can't copy - not same remote type")+ return nil, fs.ErrorCantCopy+ }+ srcLeaf := path.Base(srcObj.remote)++ srcPath := srcObj.fs.rootSlash() + srcObj.remote+ dstPath := f.rootSlash() + remote+ if srcPath == dstPath {+ return nil, fmt.Errorf("can't copy %q -> %q as are same name", srcPath, dstPath)+ }++ // Create temporary object+ dstObj, dstLeaf, dstDirectoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size)+ if err != nil {+ return nil, err+ }++ // Copy the object+ info, err := f.copyTo(ctx, srcObj.id, srcLeaf, dstLeaf, dstDirectoryID)+ if err != nil {+ return nil, err+ }+ err = dstObj.setMetaData(info)+ if err != nil {+ return nil, err+ }++ // Reset the modification time as copy does not preserve it+ err = dstObj.SetModTime(ctx, srcObj.modTime)+ if err != nil {+ return nil, err+ }++ return dstObj, nil+}++// unLink a file or directory+func (f *Fs) unLink(ctx context.Context, remote string, id string, info *api.Item) (err error) {+ if info == nil {+ info, err = f.readMetaDataForID(ctx, id)+ if err != nil {+ return err+ }+ }+ for linkID, link := range info.DirectLinks {+ fs.Debugf(remote, "Removing direct link %s", link.DirectLink)+ opts := rest.Opts{+ Method: "DELETE",+ Path: "/contents/" + id + "/directlinks/" + linkID,+ }+ var result api.Error+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ return fmt.Errorf("failed to unlink: %s", link.DirectLink)+ }+ }+ return nil+}++// PublicLink adds a "readable by anyone with link" permission on the given file or folder.+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {+ id, err := f.dirCache.FindDir(ctx, remote, false)+ var info *api.Item+ if err == nil {+ fs.Debugf(f, "attempting to share directory '%s'", remote)+ } else {+ fs.Debugf(f, "attempting to share single file '%s'", remote)+ info, err = f.readMetaDataForPath(ctx, remote)+ if err != nil {+ return "", err+ }+ id = info.ID+ }+ if unlink {+ return "", f.unLink(ctx, remote, id, info)+ }+ var resp *http.Response+ var request api.DirectLinksRequest+ var result api.DirectLinksResult+ opts := rest.Opts{+ Method: "POST",+ Path: "/contents/" + id + "/directlinks",+ }+ if expire != fs.DurationOff {+ when := time.Now().Add(time.Duration(expire))+ fs.Debugf(f, "Link expires at %v (duration %v)", when, expire)+ request.ExpireTime = api.ToNativeTime(when)+ }+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ return "", fmt.Errorf("failed to create direct link: %w", err)+ }+ return result.Data.DirectLink, err+}++// DirCacheFlush resets the directory cache - used in testing as an+// optional interface+func (f *Fs) DirCacheFlush() {+ f.dirCache.ResetRoot()+}++// Hashes returns the supported hash sets.+func (f *Fs) Hashes() hash.Set {+ return hash.Set(hash.MD5)+}++// MergeDirs merges the contents of all the directories passed+// in into the first one and rmdirs the other directories.+func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {+ if len(dirs) < 2 {+ return nil+ }+ dstDir := dirs[0]+ for _, srcDir := range dirs[1:] {+ // list the objects+ infos := []*api.Item{}+ _, err := f.listAll(ctx, srcDir.ID(), false, false, "", func(info *api.Item) bool {+ infos = append(infos, info)+ return false+ })+ if err != nil {+ return fmt.Errorf("MergeDirs list failed on %v: %w", srcDir, err)+ }+ // move them into place+ for _, info := range infos {+ fs.Infof(srcDir, "merging %q", info.Name)+ // Move the file into the destination+ _, err = f.move(ctx, info.ID, dstDir.ID())+ if err != nil {+ return fmt.Errorf("MergeDirs move failed on %q in %v: %w", info.Name, srcDir, err)+ }+ }+ // rmdir the now empty source directory+ fs.Infof(srcDir, "removing empty directory")+ err = f.deleteObject(ctx, srcDir.ID())+ if err != nil {+ return fmt.Errorf("MergeDirs move failed to rmdir %q: %w", srcDir, err)+ }+ }+ return nil+}++// ------------------------------------------------------------++// Fs returns the parent Fs+func (o *Object) Fs() fs.Info {+ return o.fs+}++// Return a string version+func (o *Object) String() string {+ if o == nil {+ return "<nil>"+ }+ return o.remote+}++// Remote returns the remote path+func (o *Object) Remote() string {+ return o.remote+}++// Hash returns the SHA-1 of an object returning a lowercase hex string+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {+ if t != hash.MD5 {+ return "", hash.ErrUnsupported+ }+ return o.md5, nil+}++// Size returns the size of an object in bytes+func (o *Object) Size() int64 {+ return o.size+}++// setMetaDataAny sets the metadata from info but doesn't check the type+func (o *Object) setMetaDataAny(info *api.Item) {+ o.size = info.Size+ o.modTime = api.FromNativeTime(info.ModTime)+ o.id = info.ID+ o.dirID = info.ParentFolder+ o.mimeType = info.MimeType+ o.md5 = info.MD5+ o.url = info.Link+}++// setMetaData sets the metadata from info+func (o *Object) setMetaData(info *api.Item) (err error) {+ if info.Type == api.ItemTypeFolder {+ return fs.ErrorIsDir+ }+ if info.Type != api.ItemTypeFile {+ return fmt.Errorf("%q is %q: %w", o.remote, info.Type, fs.ErrorNotAFile)+ }+ if info.ID == "" {+ return fmt.Errorf("ID not found in response")+ }+ o.setMetaDataAny(info)+ return nil+}++// readMetaData gets the metadata unconditionally as we expect Object+// to always have the full set of metadata+func (o *Object) readMetaData(ctx context.Context) (err error) {+ var info *api.Item+ if o.id != "" {+ info, err = o.fs.readMetaDataForID(ctx, o.id)+ } else {+ info, err = o.fs.readMetaDataForPath(ctx, o.remote)+ }+ if err != nil {+ if isAPIErr(err, "error-notFound") {+ return fs.ErrorObjectNotFound+ }+ return err+ }+ return o.setMetaData(info)+}++// ModTime returns the modification time of the object+//+// It attempts to read the objects mtime and if that isn't present the+// LastModified returned in the http headers+func (o *Object) ModTime(ctx context.Context) time.Time {+ return o.modTime+}++// SetModTime sets the modification time of the local fs object+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {+ info, err := o.fs.setModTime(ctx, o.id, modTime)+ if err != nil {+ return err+ }+ return o.setMetaData(info)+}++// Storable returns a boolean showing whether this object storable+func (o *Object) Storable() bool {+ return true+}++// Open an object for read+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {+ if o.id == "" {+ return nil, errors.New("can't download - no id")+ }+ fs.FixRangeOption(options, o.size)+ var resp *http.Response+ opts := rest.Opts{+ Method: "GET",+ RootURL: o.url,+ Options: options,+ // Workaround for bug in content servers - no longer needed+ // ExtraHeaders: map[string]string{+ // "Cookie": "accountToken=" + o.fs.opt.AccessToken,+ // },+ }++ err = o.fs.pacer.Call(func() (bool, error) {+ resp, err = o.fs.srv.Call(ctx, &opts)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, err+ }+ return resp.Body, err+}++// Update the object with the contents of the io.Reader, modTime and size+//+// If existing is set then it updates the object rather than creating a new one.+//+// The new object may have been created if an error is returned.+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {+ remote := o.Remote()+ modTime := src.ModTime(ctx)++ // Create the directory for the object if it doesn't exist+ leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)+ if err != nil {+ return err+ }++ // Find an upload server+ server, err := o.fs.getServer(ctx)+ if err != nil {+ return err+ }+ fs.Debugf(o, "Using upload server %v", server)++ // If the file exists, delete it after a successful upload+ if o.id != "" {+ id := o.id+ o.id = ""+ defer func() {+ if err != nil {+ return+ }+ fs.Debugf(o, "Removing old object on successful upload")+ deleteErr := o.fs.deleteObject(ctx, id)+ if deleteErr != nil {+ err = fmt.Errorf("failed to delete existing object: %w", deleteErr)+ }+ }()+ }++ // Do the upload+ var resp *http.Response+ var result api.UploadResponse+ opts := rest.Opts{+ Method: "POST",+ Body: in,+ MultipartParams: url.Values{+ "folderId": {directoryID},+ "modTime": {strconv.FormatInt(modTime.Unix(), 10)},+ },+ MultipartContentName: "file",+ MultipartFileName: o.fs.opt.Enc.FromStandardName(leaf),+ RootURL: server.URL(),+ Options: options,+ }+ err = o.fs.pacer.CallNoRetry(func() (bool, error) {+ resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err = result.Err(err); err != nil {+ if isAPIErr(err, "error-freespace") {+ fs.Errorf(o, "Upload server out of space - need to retry upload")+ }+ o.fs.clearServers()+ return fmt.Errorf("failed to upload file: %w", err)+ }+ return o.setMetaData(&result.Data)+}++// Remove an object+func (o *Object) Remove(ctx context.Context) error {+ return o.fs.deleteObject(ctx, o.id)+}++// ID returns the ID of the Object if known, or "" if not+func (o *Object) ID() string {+ return o.id+}++// MimeType returns the content type of the Object if known, or "" if not+func (o *Object) MimeType(ctx context.Context) string {+ return o.mimeType+}++// ParentID returns the ID of the Object parent if known, or "" if not+func (o *Object) ParentID() string {+ return o.dirID+}++// setMetaData sets the metadata from info for a directory+func (d *Directory) setMetaData(info *api.Item) (err error) {+ if info.Type == api.ItemTypeFile {+ return fs.ErrorIsFile+ }+ if info.Type != api.ItemTypeFolder {+ return fmt.Errorf("%q is not a directory (type %q)", d.remote, info.Type)+ }+ if info.ID == "" {+ return fmt.Errorf("ID not found in response")+ }+ d.setMetaDataAny(info)+ return nil+}++// SetModTime sets the modification time of the directory+func (d *Directory) SetModTime(ctx context.Context, modTime time.Time) error {+ info, err := d.fs.setModTime(ctx, d.id, modTime)+ if err != nil {+ return err+ }+ return d.setMetaData(info)+}++// Items returns the count of items in this directory or this+// directory and subdirectories if known, -1 for unknown+func (d *Directory) Items() int64 {+ return d.items+}++// Hash does nothing on a directory+//+// This method is implemented with the incorrect type signature to+// stop the Directory type asserting to fs.Object or fs.ObjectInfo+func (d *Directory) Hash() {+ // Does nothing+}++// Check the interfaces are satisfied+var (+ _ fs.Fs = (*Fs)(nil)+ _ fs.Purger = (*Fs)(nil)+ _ fs.PutStreamer = (*Fs)(nil)+ _ fs.Copier = (*Fs)(nil)+ _ fs.Abouter = (*Fs)(nil)+ _ fs.Mover = (*Fs)(nil)+ _ fs.DirMover = (*Fs)(nil)+ _ fs.DirCacheFlusher = (*Fs)(nil)+ _ fs.PublicLinker = (*Fs)(nil)+ _ fs.MergeDirser = (*Fs)(nil)+ _ fs.DirSetModTimer = (*Fs)(nil)+ _ fs.ListRer = (*Fs)(nil)+ _ fs.Object = (*Object)(nil)+ _ fs.IDer = (*Object)(nil)+ _ fs.MimeTyper = (*Object)(nil)+ _ fs.Directory = (*Directory)(nil)+ _ fs.SetModTimer = (*Directory)(nil)+ _ fs.ParentIDer = (*Directory)(nil)+)diff --git a/backend/gofile/gofile_test.go b/backend/gofile/gofile_test.gonew file mode 100644index 000000000..2f8816e61--- /dev/null+++ b/backend/gofile/gofile_test.go@@ -0,0 +1,17 @@+// Test Gofile filesystem interface+package gofile_test++import (+ "testing"++ "github.com/rclone/rclone/backend/gofile"+ "github.com/rclone/rclone/fstest/fstests"+)++// TestIntegration runs integration tests against the remote+func TestIntegration(t *testing.T) {+ fstests.Run(t, &fstests.Opt{+ RemoteName: "TestGoFile:",+ NilObject: (*gofile.Object)(nil),+ })+}diff --git a/bin/make_manual.py b/bin/make_manual.pyindex 41ddccf7c..b5bffecb1 100755--- a/bin/make_manual.py+++ b/bin/make_manual.py@@ -42,6 +42,7 @@ docs = ["dropbox.md","filefabric.md","ftp.md",+ "gofile.md","googlecloudstorage.md","drive.md","googlephotos.md",diff --git a/docs/content/_index.md b/docs/content/_index.mdindex 714cce51a..3152ae46f 100644--- a/docs/content/_index.md+++ b/docs/content/_index.md@@ -123,6 +123,7 @@ WebDAV or S3, that work out of the box.){{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}}{{< provider name="Fastmail Files" home="https://www.fastmail.com/" config="/webdav/#fastmail-files" >}}{{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}}+{{< provider name="Gofile" home="https://gofile.io/" config="/gofile/" >}}{{< provider name="Google Cloud Storage" home="https://cloud.google.com/storage/" config="/googlecloudstorage/" >}}{{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}}{{< provider name="Google Photos" home="https://www.google.com/photos/about/" config="/googlephotos/" >}}diff --git a/docs/content/docs.md b/docs/content/docs.mdindex 5eaaee51e..ade70c4c8 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -43,6 +43,7 @@ See the following for detailed instructions for* [Dropbox](/dropbox/)* [Enterprise File Fabric](/filefabric/)* [FTP](/ftp/)+ * [Gofile](/gofile/)* [Google Cloud Storage](/googlecloudstorage/)* [Google Drive](/drive/)* [Google Photos](/googlephotos/)diff --git a/docs/content/gofile.md b/docs/content/gofile.mdnew file mode 100644index 000000000..8ce0d7036--- /dev/null+++ b/docs/content/gofile.md@@ -0,0 +1,256 @@+---+title: "Gofile"+description: "Rclone docs for Gofile"+versionIntroduced: "v1.68"+---++# {{< icon "fa fa-folder" >}} Gofile++[Gofile](https://gofile.io) is a content storage and distribution+platform. Its aim is to provide as much service as possible for free+or at a very low price.++The initial setup for Gofile involves logging in to the web interface+and going to the "My Profile" section. Copy the "Account API token"+for use in the config file.++Note that if you wish to connect rclone to Gofile you will need a+premium account.++## Configuration++Here is an example of how to make a remote called `remote`. First run:++ rclone config++This will guide you through an interactive setup process:++```+No remotes found, make a new one?+n) New remote+s) Set configuration password+q) Quit config+n/s/q> n++Enter name for new remote.+name> remote++Option Storage.+Type of storage to configure.+Choose a number from below, or type in your own value.+XX / Gofile+ \ (gofile)+Storage> gofile++Option access_token.+API Access token+You can get this from the web control panel.+Enter a value. Press Enter to leave empty.+access_token> YOURACCESSTOKEN++Edit advanced config?+y) Yes+n) No (default)+y/n> n++Configuration complete.+Options:+- type: gofile+- access_token: YOURACCESSTOKEN+Keep this "remote" remote?+y) Yes this is OK (default)+e) Edit this remote+d) Delete this remote+y/e/d> y+```++Once configured you can then use `rclone` like this,++List directories and files in the top level of your Gofile++ rclone lsf remote:++To copy a local directory to an Gofile directory called backup++ rclone copy /home/source remote:backup++### Modification times and hashes++Gofile supports modification times with a resolution of 1 second.++Gofile supports MD5 hashes, so you can use the `--checksum` flag.++### Restricted filename characters++In addition to the [default restricted characters set](/overview/#restricted-characters)+the following characters are also replaced:++| Character | Value | Replacement |+| --------- |:-----:|:-----------:|+| ! | 0x21 | ! |+| " | 0x22 | " |+| * | 0x2A | * |+| : | 0x3A | : |+| < | 0x3C | < |+| > | 0x3E | > |+| ? | 0x3F | ? |+| \ | 0x5C | \ |+| \| | 0x7C | | |+++File names can also not start or end with the following characters.+These only get replaced if they are the first or last character in the+name:++| Character | Value | Replacement |+| --------- |:-----:|:-----------:|+| . | 0x2E | . |++Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),+as they can't be used in JSON strings.++### Public Links++Gofile supports `rclone link` to make public links to files or+directories. If you specify a directory it will download as a `zip`+file. You can use the `--expire` flag to specify the time the link+should be valid. Note that `rclone link --unlink` removes all the+public links for a file.++### Root folder ID++You can set the `root_folder_id` for rclone. This is the directory+(identified by its `Folder ID`) that rclone considers to be the root+of your Gofile drive.++Normally you will leave this blank and rclone will determine the+correct root to use itself and fill in the value in the config file.++However you can set this to restrict rclone to a specific folder+hierarchy.++In order to do this you will have to find the `Folder ID` of the+directory you wish rclone to display.++You can do this with rclone++```+$ rclone lsf -Fip --dirs-only remote:+d6341f53-ee65-4f29-9f59-d11e8070b2a0;Files/+f4f5c9b8-6ece-478b-b03e-4538edfe5a1c;Photos/+d50e356c-29ca-4b27-a3a7-494d91026e04;Videos/+```++The ID to use is the part before the `;` so you could set++```+root_folder_id = d6341f53-ee65-4f29-9f59-d11e8070b2a0+```++To restrict rclone to the `Files` directory.++{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/gofile/gofile.go then run make backenddocs" >}}+### Standard options++Here are the Standard options specific to gofile (Gofile).++#### --gofile-access-token++API Access token++You can get this from the web control panel.++Properties:++- Config: access_token+- Env Var: RCLONE_GOFILE_ACCESS_TOKEN+- Type: string+- Required: false++### Advanced options++Here are the Advanced options specific to gofile (Gofile).++#### --gofile-root-folder-id++ID of the root folder++Leave this blank normally, rclone will fill it in automatically.++If you want rclone to be restricted to a particular folder you can+fill it in - see the docs for more info.+++Properties:++- Config: root_folder_id+- Env Var: RCLONE_GOFILE_ROOT_FOLDER_ID+- Type: string+- Required: false++#### --gofile-account-id++Account ID++Leave this blank normally, rclone will fill it in automatically.+++Properties:++- Config: account_id+- Env Var: RCLONE_GOFILE_ACCOUNT_ID+- Type: string+- Required: false++#### --gofile-encoding++The encoding for the backend.++See the [encoding section in the overview](/overview/#encoding) for more info.++Properties:++- Config: encoding+- Env Var: RCLONE_GOFILE_ENCODING+- Type: Encoding+- Default: Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftPeriod,RightPeriod,InvalidUtf8,Dot,Exclamation++#### --gofile-description++Description of the remote.++Properties:++- Config: description+- Env Var: RCLONE_GOFILE_DESCRIPTION+- Type: string+- Required: false++{{< rem autogenerated options stop >}}++## Limitations++Gofile only supports filenames up to 255 characters in length, where a+character is a unicode character.++Directories should not be cached for more than 24h otherwise files in+the directory may not be downloadable. In practice this means when+using a VFS based rclone command such as `rclone mount` you should+make sure `--dir-cache-time` is less than `24h`.++Note that Gofile is currently limited to a total of 100,000 items. If+you attempt to upload more than that you will get+`error-limit-100000`. This limit may be lifted in the future.++### Duplicated files++Gofile is capable of having files with duplicated file names. For+instance two files called `hello.txt` in the same directory.++Rclone cannot sync that to a normal file system but it can be fixed+with the `rclone dedupe` command.++Duplicated files cause problems with the syncing and you will see+messages in the log about duplicates.++Use `rclone dedupe` to fix duplicated files.diff --git a/docs/content/overview.md b/docs/content/overview.mdindex 813367c1f..cdf27f7b8 100644--- a/docs/content/overview.md+++ b/docs/content/overview.md@@ -25,6 +25,7 @@ Here is an overview of the major features of each cloud storage system.| Dropbox | DBHASH ¹ | R | Yes | No | - | - || Enterprise File Fabric | - | R/W | Yes | No | R/W | - || FTP | - | R/W ¹⁰ | No | No | - | - |+| Gofile | MD5 | DR/W | No | Yes | R | - || Google Cloud Storage | MD5 | R/W | No | No | R/W | - || Google Drive | MD5, SHA1, SHA256 | DR/W | No | Yes | R/W | DRWU || Google Photos | - | - | No | Yes | R | - |@@ -501,6 +502,7 @@ upon backend-specific capabilities.| Dropbox | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes || Enterprise File Fabric | Yes | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes || FTP | No | No | Yes | Yes | No | No | Yes | No | No | No | Yes |+| Gofile | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes || Google Cloud Storage | Yes | Yes | No | No | No | Yes | Yes | No | No | No | No || Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes || Google Photos | No | No | No | No | No | No | No | No | No | No | No |diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.htmlindex 1c67b098b..f4b928da5 100644--- a/docs/layouts/chrome/navbar.html+++ b/docs/layouts/chrome/navbar.html@@ -66,6 +66,7 @@<a class="dropdown-item" href="/dropbox/"><i class="fab fa-dropbox fa-fw"></i> Dropbox</a><a class="dropdown-item" href="/filefabric/"><i class="fa fa-cloud fa-fw"></i> Enterprise File Fabric</a><a class="dropdown-item" href="/ftp/"><i class="fa fa-file fa-fw"></i> FTP</a>+ <a class="dropdown-item" href="/gofile/"><i class="fa fa-folder fa-fw"></i> Gofile</a><a class="dropdown-item" href="/googlecloudstorage/"><i class="fab fa-google fa-fw"></i> Google Cloud Storage</a><a class="dropdown-item" href="/drive/"><i class="fab fa-google fa-fw"></i> Google Drive</a><a class="dropdown-item" href="/googlephotos/"><i class="fas fa-images fa-fw"></i> Google Photos</a>diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yamlindex 155f3093f..ded4f10ad 100644--- a/fstest/test_all/config.yaml+++ b/fstest/test_all/config.yaml@@ -137,6 +137,9 @@ backends:# remote: "TestFileFabric:"# fastlist: false# extratime: 2.0+ - backend: "gofile"+ remote: "TestGoFile:"+ fastlist: true- backend: "googlecloudstorage"remote: "TestGoogleCloudStorage:"fastlist: true