"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
linkbox: fix downloading files by using web API - fixes #8665
Nick Craig-Wood20eaad4b6d25
diff --git a/backend/linkbox/linkbox.go b/backend/linkbox/linkbox.goindex ad33a16fc..311fd8c3d 100644--- a/backend/linkbox/linkbox.go+++ b/backend/linkbox/linkbox.go@@ -14,19 +14,21 @@ import ("bytes""context""crypto/md5"+ "crypto/tls""fmt""io""net/http""net/url""path"- "regexp""strconv""strings"+ "sync""time""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/config/configstruct"+ "github.com/rclone/rclone/fs/config/obscure""github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"@@ -41,6 +43,7 @@ const (maxSleep = 2 * time.SecondpacerBurst = 1linkboxAPIURL = "https://www.linkbox.to/api/open/"+ linkboxWebAPIURL = "https://www.linkbox.to/api/"rootID = "0" // ID of root directory)@@ -54,6 +57,20 @@ func init() {Help: "Token from https://www.linkbox.to/admin/account",Sensitive: true,Required: true,+ }, {+ Name: "email",+ Help: "Email for login",+ Sensitive: true,+ Required: true,+ }, {+ Name: "password",+ Help: "Password for login",+ IsPassword: true,+ Required: true,+ }, {+ Name: "web_token",+ Help: "Web API login token - set automatically.",+ Hide: fs.OptionHideBoth,}},}fs.Register(fsi)@@ -61,19 +78,26 @@ func init() {// Options defines the configuration for this backendtype Options struct {- Token string `config:"token"`+ Token string `config:"token"`+ Email string `config:"email"`+ Password string `config:"password"`+ WebToken string `config:"web_token"`}// Fs stores the interface to the remote Linkbox filestype Fs struct {- name string- root string- opt Options // options for this backend- features *fs.Features // optional features- ci *fs.ConfigInfo // global config- srv *rest.Client // the connection to the server- dirCache *dircache.DirCache // Map of directory path to directory id- pacer *fs.Pacer+ name string+ root string+ opt Options // options for this backend+ features *fs.Features // optional features+ ci *fs.ConfigInfo // global config+ srv *rest.Client // the connection to the server+ cdnSrv *rest.Client // HTTP client for CDN downloads (HTTP/2 disabled)+ dirCache *dircache.DirCache // Map of directory path to directory id+ pacer *fs.Pacer+ m configmap.Mapper // config mapper for saving tokens+ webToken string // token from web API login+ webTokenMu sync.Mutex // protects webToken}// Object is a remote object that has been stat'd (so it exists, but is not necessarily open for reading)@@ -104,14 +128,27 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, eci := fs.GetConfig(ctx)- f := &Fs{- name: name,- opt: *opt,- root: root,- ci: ci,- srv: rest.NewClient(fshttp.NewClient(ctx)),+ // Create a separate HTTP client for CDN downloads with HTTP/2 disabled.+ // The Linkbox CDN intermittently blocks Go's HTTP/2 fingerprint and+ // non-browser User-Agents.+ cdnClient := fshttp.NewClientCustom(ctx, func(t *http.Transport) {+ t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}+ })+ if tr, ok := cdnClient.Transport.(*fshttp.Transport); ok {+ tr.SetRequestFilter(func(req *http.Request) {+ req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")+ })+ }- pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep))),+ f := &Fs{+ name: name,+ opt: *opt,+ root: root,+ ci: ci,+ srv: rest.NewClient(fshttp.NewClient(ctx)),+ cdnSrv: rest.NewClient(cdnClient),+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep))),+ m: m,}f.dirCache = dircache.New(root, rootID, f)@@ -120,12 +157,27 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, eCaseInsensitive: true,}).Fill(ctx, f)+ if f.opt.Email == "" || f.opt.Password == "" {+ return nil, fmt.Errorf("email and password are required - run `rclone config`")+ }++ // Load cached web token or login to get a new one+ if opt.WebToken != "" {+ f.webToken = opt.WebToken+ fs.Debugf(f, "Using cached web token")+ } else {+ err = f.login(ctx)+ if err != nil {+ return nil, err+ }+ }+// Find the current rooterr = f.dirCache.FindRoot(ctx, false)if err != nil {// Assume it is a filenewRoot, remote := dircache.SplitPath(root)- tempF := *f+ tempF := *f //nolint:govet // copying mutex is OK here as it is a new FstempF.dirCache = dircache.New(newRoot, rootID, &tempF)tempF.root = newRoot// Make new Fs which is the parent@@ -154,6 +206,78 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, ereturn f, nil}+// loginResponse is the response from the web API login+type loginResponse struct {+ response+ Data struct {+ Token string `json:"token"`+ } `json:"data"`+}++// login authenticates with the web API using email and password+// and saves the token in the config for reuse.+func (f *Fs) login(ctx context.Context) error {+ password, err := obscure.Reveal(f.opt.Password)+ if err != nil {+ return fmt.Errorf("failed to decrypt password: %w", err)+ }+ opts := &rest.Opts{+ Method: "GET",+ RootURL: linkboxWebAPIURL,+ Path: "user/login_email",+ Parameters: url.Values{+ "email": {f.opt.Email},+ "pwd": {password},+ "platform": {"web"},+ "pf": {"web"},+ "lan": {"en"},+ },+ }+ var result loginResponse+ err = getUnmarshaledResponse(ctx, f, opts, &result)+ if err != nil {+ return fmt.Errorf("login failed: %w", err)+ }+ if result.Data.Token == "" {+ return fmt.Errorf("login returned empty token")+ }+ f.webTokenMu.Lock()+ f.webToken = result.Data.Token+ f.webTokenMu.Unlock()+ // Save token to config for reuse across invocations+ f.m.Set("web_token", f.webToken)+ fs.Debugf(f, "Login successful")+ return nil+}++// shouldRetryWeb determines whether a web API error should be retried.+//+// If the API returns an error it may be because the token has expired,+// so this refreshes the token and updates the request parameters for+// the retry.+func (f *Fs) shouldRetryWeb(ctx context.Context, resp *http.Response, err error, result responser, opts *rest.Opts) (bool, error) {+ if fserrors.ContextError(ctx, &err) {+ return false, err+ }+ if fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes) {+ return true, err+ }+ // If the web API returned an error, it may be due to an expired token.+ // Refresh the token and retry.+ if result.IsError() {+ fs.Debugf(f, "Web API returned error %q - refreshing token", result.Error())+ loginErr := f.login(ctx)+ if loginErr != nil {+ return false, loginErr+ }+ f.webTokenMu.Lock()+ opts.Parameters.Set("token", f.webToken)+ f.webTokenMu.Unlock()+ return true, nil+ }+ return false, nil+}+type entity struct {Type string `json:"type"`Name string `json:"name"`@@ -216,40 +340,28 @@ func getUnmarshaledResponse(ctx context.Context, f *Fs, opts *rest.Opts, result// Should return true to finish processingtype listAllFn func(*entity) bool-// Search is a bit fussy about which characters match-//-// If the name doesn't match this then do an dir list instead-// N.B.: Linkbox doesn't support search by name that is longer than 50 chars-var searchOK = regexp.MustCompile(`^[a-zA-Z0-9_ -.]{1,50}$`)-// Lists the directory required calling the user function on each item found//// If the user fn ever returns true then it early exits with found = true-//-// If you set name then search ignores dirID. name is a substring-// search also so name="dir" matches "sub dir" also. This filters it-// down so it only returns items in dirIDfunc (f *Fs) listAll(ctx context.Context, dirID string, name string, fn listAllFn) (found bool, err error) {+ _ = name // name parameter kept for call-site compatibility but unused with web APIvar (pageNumber = 0numberOfEntities = maxEntitiesPerPage)- name = strings.TrimSpace(name) // search doesn't like spaces- if !searchOK.MatchString(name) {- // If name isn't good then do an unbounded search- name = ""- }OUTER:for numberOfEntities == maxEntitiesPerPage {pageNumber+++ f.webTokenMu.Lock()+ webToken := f.webToken+ f.webTokenMu.Unlock()opts := &rest.Opts{Method: "GET",- RootURL: linkboxAPIURL,- Path: "file_search",+ RootURL: linkboxWebAPIURL,+ Path: "file/my_file_list/web",Parameters: url.Values{- "token": {f.opt.Token},- "name": {name},+ "token": {webToken},"pid": {dirID},"pageNo": {itoa(pageNumber)},"pageSize": {itoa64(maxEntitiesPerPage)},@@ -257,7 +369,10 @@ OUTER:}var responseResult fileSearchRes- err = getUnmarshaledResponse(ctx, f, opts, &responseResult)+ err = f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(ctx, opts, nil, &responseResult)+ return f.shouldRetryWeb(ctx, resp, err, &responseResult, opts)+ })if err != nil {return false, fmt.Errorf("getting files failed: %w", err)}@@ -403,7 +518,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e}// get an entity with leaf from dirID-func getEntity(ctx context.Context, f *Fs, leaf string, directoryID string, token string) (*entity, error) {+func getEntity(ctx context.Context, f *Fs, leaf string, directoryID string) (*entity, error) {var result *entityvar resultErr = fs.ErrorObjectNotFound_, err := f.listAll(ctx, directoryID, leaf, func(entity *entity) bool {@@ -441,7 +556,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {return nil, err}- entity, err := getEntity(ctx, f, leaf, dirID, f.opt.Token)+ entity, err := getEntity(ctx, f, leaf, dirID)if err != nil {return nil, err}@@ -518,7 +633,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClodownloadURL := o.fullURLif downloadURL == "" {_, name := splitDirAndName(o.Remote())- newObject, err := getEntity(ctx, o.fs, name, itoa64(o.dirID), o.fs.opt.Token)+ newObject, err := getEntity(ctx, o.fs, name, itoa64(o.dirID))if err != nil {return nil, err}@@ -530,6 +645,10 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClodownloadURL = newObject.URL}+ if downloadURL == "" {+ return nil, fmt.Errorf("no download URL for %q - try re-running `rclone config reconnect`", o.Remote())+ }+opts := &rest.Opts{Method: "GET",RootURL: downloadURL,@@ -538,7 +657,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloerr := o.fs.pacer.Call(func() (bool, error) {var err error- res, err = o.fs.srv.Call(ctx, opts)+ res, err = o.fs.cdnSrv.Call(ctx, opts)return o.fs.shouldRetry(ctx, res, err)})@@ -697,7 +816,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opvar sleepTime = 100 * time.Millisecondvar entity *entityfor try := 1; try <= maxTries; try++ {- entity, err = getEntity(ctx, o.fs, leaf, dirID, o.fs.opt.Token)+ entity, err = getEntity(ctx, o.fs, leaf, dirID)if err == nil {break}diff --git a/docs/content/linkbox.md b/docs/content/linkbox.mdindex 1703a520c..ea2947e70 100644--- a/docs/content/linkbox.md+++ b/docs/content/linkbox.md@@ -12,6 +12,17 @@ Linkbox is [a private cloud drive](https://linkbox.to/).Here is an example of making a remote for Linkbox.+You will need:+- An API token from https://www.linkbox.to/admin/account+- The email and password for your Linkbox account++**Note:** As of v1.74, email and password are required in addition to+the API token. The API token is used for uploads and modifications,+while email/password authentication is used for listing and+downloading files. If you have an existing Linkbox remote, you will+need to reconfigure it with `rclone config` to edit the remote and add+the email and password.+First run:```console@@ -40,12 +51,24 @@ Storage> XXOption token.Token from https://www.linkbox.to/admin/accountEnter a value.-token> testFromCLToken+token> YOUR_API_TOKEN++Option email.+Email for login+Enter a value.+email> [email protected]++Option password.+Password for login+Enter a value.+password> YOUR_PASSWORDConfiguration complete.Options:- type: linkbox- token: XXXXXXXXXXX+- email: [email protected]+- password: *** ENCRYPTED ***Keep this "linkbox" remote?y) Yes this is OK (default)e) Edit this remote