"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
backend/gofile: update to use new direct upload endpoint
wbulot359433017774
diff --git a/backend/gofile/api/types.go b/backend/gofile/api/types.goindex 4799687d0..0a0d63ab9 100644--- a/backend/gofile/api/types.go+++ b/backend/gofile/api/types.go@@ -194,33 +194,9 @@ type DeleteResponse struct {Data map[string]Error}-// Server is an upload server-type Server struct {- Name string `json:"name"`- Zone string `json:"zone"`-}--// String returns a string representation of the Server-func (s *Server) String() string {- return fmt.Sprintf("%s (%s)", s.Name, s.Zone)-}--// Root returns the root URL for the server-func (s *Server) Root() string {- return fmt.Sprintf("https://%s.gofile.io/", s.Name)-}--// URL returns the upload URL for the server-func (s *Server) URL() string {- return fmt.Sprintf("https://%s.gofile.io/contents/uploadfile", s.Name)-}--// ServersResponse is the output from /servers-type ServersResponse struct {- Error- Data struct {- Servers []Server `json:"servers"`- } `json:"data"`+// DirectUploadURL returns the direct upload URL for Gofile+func DirectUploadURL() string {+ return "https://upload.gofile.io/uploadfile"}// UploadResponse is returned by POST /contents/uploadfilediff --git a/backend/gofile/gofile.go b/backend/gofile/gofile.goindex d8981b9c6..fc48b7c59 100644--- a/backend/gofile/gofile.go+++ b/backend/gofile/gofile.go@@ -8,13 +8,11 @@ import ("errors""fmt""io"- "math/rand""net/http""net/url""path""strconv""strings"- "sync""time""github.com/rclone/rclone/backend/gofile/api"@@ -37,10 +35,8 @@ const (maxSleep = 20 * time.SeconddecayConstant = 1 // bigger for slower decay, exponentialrootURL = "https://api.gofile.io"- serversExpiry = 60 * time.Second // check for new upload servers this often- serversActive = 2 // choose this many closest upload servers to use- rateLimitSleep = 5 * time.Second // penalise a goroutine by this long for making a rate limit error- maxDepth = 4 // in ListR recursive list this deep (maximum is 16)+ rateLimitSleep = 5 * time.Second // penalise a goroutine by this long for making a rate limit error+ maxDepth = 4 // in ListR recursive list this deep (maximum is 16))/*@@ -128,16 +124,13 @@ type Options struct {// Fs represents a remote gofiletype Fs struct {- name string // name of this remote- root string // the path we are working on- opt Options // parsed options- features *fs.Features // optional features- srv *rest.Client // the connection to the server- dirCache *dircache.DirCache // Map of directory path to directory id- pacer *fs.Pacer // pacer for API calls- serversMu *sync.Mutex // protect the servers info below- servers []api.Server // upload servers we can use- serversChecked time.Time // time the servers were refreshed+ name string // name of this remote+ root string // the path we are working on+ opt Options // parsed options+ features *fs.Features // optional features+ srv *rest.Client // the connection to the server+ dirCache *dircache.DirCache // Map of directory path to directory id+ pacer *fs.Pacer // pacer for API calls}// Object describes a gofile object@@ -311,12 +304,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, eclient := fshttp.NewClient(ctx)f := &Fs{- name: name,- root: root,- opt: *opt,- srv: rest.NewClient(client).SetRoot(rootURL),- pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),- serversMu: new(sync.Mutex),+ name: name,+ root: root,+ opt: *opt,+ srv: rest.NewClient(client).SetRoot(rootURL),+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),}f.features = (&fs.Features{CaseInsensitive: false,@@ -435,98 +427,6 @@ func (f *Fs) readRootFolderID(ctx context.Context, m configmap.Mapper) (err erroreturn nil}-// Find the top n servers measured by response time-func (f *Fs) bestServers(ctx context.Context, servers []api.Server, n int) (newServers []api.Server) {- ctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Second))- defer cancel()-- if n > len(servers) {- n = len(servers)- }- results := make(chan int, len(servers))-- // Test how long the servers take to respond- for i := range servers {- i := i // for closure- go func() {- opts := rest.Opts{- Method: "GET",- RootURL: servers[i].Root(),- }- var result api.UploadServerStatus- start := time.Now()- _, err := f.srv.CallJSON(ctx, &opts, nil, &result)- ping := time.Since(start)- err = result.Err(err)- if err != nil {- results <- -1 // send a -ve number on error- return- }- fs.Debugf(nil, "Upload server %v responded in %v", &servers[i], ping)- results <- i- }()- }-- // Wait for n servers to respond- newServers = make([]api.Server, 0, n)- for range servers {- i := <-results- if i >= 0 {- newServers = append(newServers, servers[i])- }- if len(newServers) >= n {- break- }- }- return newServers-}--// Clear all the upload servers - call on an error-func (f *Fs) clearServers() {- f.serversMu.Lock()- defer f.serversMu.Unlock()-- fs.Debugf(f, "Clearing upload servers")- f.servers = nil-}--// Gets an upload server-func (f *Fs) getServer(ctx context.Context) (server *api.Server, err error) {- f.serversMu.Lock()- defer f.serversMu.Unlock()-- if len(f.servers) == 0 || time.Since(f.serversChecked) >= serversExpiry {- opts := rest.Opts{- Method: "GET",- Path: "/servers",- }- var result api.ServersResponse- var resp *http.Response- err = f.pacer.Call(func() (bool, error) {- resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)- return shouldRetry(ctx, resp, err)- })- if err = result.Err(err); err != nil {- if len(f.servers) == 0 {- return nil, fmt.Errorf("failed to read upload servers: %w", err)- }- fs.Errorf(f, "failed to read new upload servers: %v", err)- } else {- // Find the top servers measured by response time- f.servers = f.bestServers(ctx, result.Data.Servers, serversActive)- f.serversChecked = time.Now()- }- }-- if len(f.servers) == 0 {- return nil, errors.New("no upload servers found")- }-- // Pick a server at random since we've already found the top ones- i := rand.Intn(len(f.servers))- return &f.servers[i], nil-}-// rootSlash returns root with a slash on if it is empty, otherwise empty stringfunc (f *Fs) rootSlash() string {if f.root == "" {@@ -1526,13 +1426,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opreturn err}- // Find an upload server- server, err := o.fs.getServer(ctx)- if err != nil {- return err- }- fs.Debugf(o, "Using upload server %v", server)-// If the file exists, delete it after a successful uploadif o.id != "" {id := o.id@@ -1561,7 +1454,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op},MultipartContentName: "file",MultipartFileName: o.fs.opt.Enc.FromStandardName(leaf),- RootURL: server.URL(),+ RootURL: api.DirectUploadURL(),Options: options,}err = o.fs.pacer.CallNoRetry(func() (bool, error) {@@ -1569,10 +1462,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opreturn shouldRetry(ctx, resp, err)})if err = result.Err(err); err != nil {- if isAPIErr(err, "error-freespace") {- fs.Errorf(o, "Upload server out of space - need to retry upload")- }- o.fs.clearServers()return fmt.Errorf("failed to upload file: %w", err)}return o.setMetaData(&result.Data)