Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

linkbox: fix downloading files by using web API - fixes #8665

Nick Craig-Wood20eaad4b6d25
diff --git a/backend/linkbox/linkbox.go b/backend/linkbox/linkbox.go
index ad33a16fc..311fd8c3d 100644
--- a/backend/linkbox/linkbox.go
+++ b/backend/linkbox/linkbox.go
@@ -14,19 +14,21 @@ import (
"bytes"
"context"
"crypto/md5"
+ "crypto/tls"
"fmt"
"io"
"net/http"
"net/url"
"path"
- "regexp"
"strconv"
"strings"
+ "sync"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
+ "github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@@ -41,6 +43,7 @@ const (
maxSleep = 2 * time.Second
pacerBurst = 1
linkboxAPIURL = "https://www.linkbox.to/api/open/"
+ linkboxWebAPIURL = "https://www.linkbox.to/api/"
rootID = "0" // ID of root directory
)
@@ -54,6 +57,20 @@ func init() {
Help: "Token from https://www.linkbox.to/admin/account",
Sensitive: true,
Required: true,
+ }, {
+ Name: "email",
+ Help: "Email for login",
+ Sensitive: true,
+ Required: true,
+ }, {
+ Name: "password",
+ Help: "Password for login",
+ IsPassword: true,
+ Required: true,
+ }, {
+ Name: "web_token",
+ Help: "Web API login token - set automatically.",
+ Hide: fs.OptionHideBoth,
}},
}
fs.Register(fsi)
@@ -61,19 +78,26 @@ func init() {
// Options defines the configuration for this backend
type Options struct {
- Token string `config:"token"`
+ Token string `config:"token"`
+ Email string `config:"email"`
+ Password string `config:"password"`
+ WebToken string `config:"web_token"`
}
// Fs stores the interface to the remote Linkbox files
type Fs struct {
- name string
- root string
- opt Options // options for this backend
- features *fs.Features // optional features
- ci *fs.ConfigInfo // global config
- srv *rest.Client // the connection to the server
- dirCache *dircache.DirCache // Map of directory path to directory id
- pacer *fs.Pacer
+ name string
+ root string
+ opt Options // options for this backend
+ features *fs.Features // optional features
+ ci *fs.ConfigInfo // global config
+ srv *rest.Client // the connection to the server
+ cdnSrv *rest.Client // HTTP client for CDN downloads (HTTP/2 disabled)
+ dirCache *dircache.DirCache // Map of directory path to directory id
+ pacer *fs.Pacer
+ m configmap.Mapper // config mapper for saving tokens
+ webToken string // token from web API login
+ webTokenMu sync.Mutex // protects webToken
}
// Object is a remote object that has been stat'd (so it exists, but is not necessarily open for reading)
@@ -104,14 +128,27 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ci := fs.GetConfig(ctx)
- f := &Fs{
- name: name,
- opt: *opt,
- root: root,
- ci: ci,
- srv: rest.NewClient(fshttp.NewClient(ctx)),
+ // Create a separate HTTP client for CDN downloads with HTTP/2 disabled.
+ // The Linkbox CDN intermittently blocks Go's HTTP/2 fingerprint and
+ // non-browser User-Agents.
+ cdnClient := fshttp.NewClientCustom(ctx, func(t *http.Transport) {
+ t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
+ })
+ if tr, ok := cdnClient.Transport.(*fshttp.Transport); ok {
+ tr.SetRequestFilter(func(req *http.Request) {
+ req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
+ })
+ }
- pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep))),
+ f := &Fs{
+ name: name,
+ opt: *opt,
+ root: root,
+ ci: ci,
+ srv: rest.NewClient(fshttp.NewClient(ctx)),
+ cdnSrv: rest.NewClient(cdnClient),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep))),
+ m: m,
}
f.dirCache = dircache.New(root, rootID, f)
@@ -120,12 +157,27 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
CaseInsensitive: true,
}).Fill(ctx, f)
+ if f.opt.Email == "" || f.opt.Password == "" {
+ return nil, fmt.Errorf("email and password are required - run `rclone config`")
+ }
+
+ // Load cached web token or login to get a new one
+ if opt.WebToken != "" {
+ f.webToken = opt.WebToken
+ fs.Debugf(f, "Using cached web token")
+ } else {
+ err = f.login(ctx)
+ if err != nil {
+ return nil, err
+ }
+ }
+
// Find the current root
err = f.dirCache.FindRoot(ctx, false)
if err != nil {
// Assume it is a file
newRoot, remote := dircache.SplitPath(root)
- tempF := *f
+ tempF := *f //nolint:govet // copying mutex is OK here as it is a new Fs
tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
tempF.root = newRoot
// Make new Fs which is the parent
@@ -154,6 +206,78 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return f, nil
}
+// loginResponse is the response from the web API login
+type loginResponse struct {
+ response
+ Data struct {
+ Token string `json:"token"`
+ } `json:"data"`
+}
+
+// login authenticates with the web API using email and password
+// and saves the token in the config for reuse.
+func (f *Fs) login(ctx context.Context) error {
+ password, err := obscure.Reveal(f.opt.Password)
+ if err != nil {
+ return fmt.Errorf("failed to decrypt password: %w", err)
+ }
+ opts := &rest.Opts{
+ Method: "GET",
+ RootURL: linkboxWebAPIURL,
+ Path: "user/login_email",
+ Parameters: url.Values{
+ "email": {f.opt.Email},
+ "pwd": {password},
+ "platform": {"web"},
+ "pf": {"web"},
+ "lan": {"en"},
+ },
+ }
+ var result loginResponse
+ err = getUnmarshaledResponse(ctx, f, opts, &result)
+ if err != nil {
+ return fmt.Errorf("login failed: %w", err)
+ }
+ if result.Data.Token == "" {
+ return fmt.Errorf("login returned empty token")
+ }
+ f.webTokenMu.Lock()
+ f.webToken = result.Data.Token
+ f.webTokenMu.Unlock()
+ // Save token to config for reuse across invocations
+ f.m.Set("web_token", f.webToken)
+ fs.Debugf(f, "Login successful")
+ return nil
+}
+
+// shouldRetryWeb determines whether a web API error should be retried.
+//
+// If the API returns an error it may be because the token has expired,
+// so this refreshes the token and updates the request parameters for
+// the retry.
+func (f *Fs) shouldRetryWeb(ctx context.Context, resp *http.Response, err error, result responser, opts *rest.Opts) (bool, error) {
+ if fserrors.ContextError(ctx, &err) {
+ return false, err
+ }
+ if fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes) {
+ return true, err
+ }
+ // If the web API returned an error, it may be due to an expired token.
+ // Refresh the token and retry.
+ if result.IsError() {
+ fs.Debugf(f, "Web API returned error %q - refreshing token", result.Error())
+ loginErr := f.login(ctx)
+ if loginErr != nil {
+ return false, loginErr
+ }
+ f.webTokenMu.Lock()
+ opts.Parameters.Set("token", f.webToken)
+ f.webTokenMu.Unlock()
+ return true, nil
+ }
+ return false, nil
+}
+
type entity struct {
Type string `json:"type"`
Name string `json:"name"`
@@ -216,40 +340,28 @@ func getUnmarshaledResponse(ctx context.Context, f *Fs, opts *rest.Opts, result
// Should return true to finish processing
type listAllFn func(*entity) bool
-// Search is a bit fussy about which characters match
-//
-// If the name doesn't match this then do an dir list instead
-// N.B.: Linkbox doesn't support search by name that is longer than 50 chars
-var searchOK = regexp.MustCompile(`^[a-zA-Z0-9_ -.]{1,50}$`)
-
// Lists the directory required calling the user function on each item found
//
// If the user fn ever returns true then it early exits with found = true
-//
-// If you set name then search ignores dirID. name is a substring
-// search also so name="dir" matches "sub dir" also. This filters it
-// down so it only returns items in dirID
func (f *Fs) listAll(ctx context.Context, dirID string, name string, fn listAllFn) (found bool, err error) {
+ _ = name // name parameter kept for call-site compatibility but unused with web API
var (
pageNumber = 0
numberOfEntities = maxEntitiesPerPage
)
- name = strings.TrimSpace(name) // search doesn't like spaces
- if !searchOK.MatchString(name) {
- // If name isn't good then do an unbounded search
- name = ""
- }
OUTER:
for numberOfEntities == maxEntitiesPerPage {
pageNumber++
+ f.webTokenMu.Lock()
+ webToken := f.webToken
+ f.webTokenMu.Unlock()
opts := &rest.Opts{
Method: "GET",
- RootURL: linkboxAPIURL,
- Path: "file_search",
+ RootURL: linkboxWebAPIURL,
+ Path: "file/my_file_list/web",
Parameters: url.Values{
- "token": {f.opt.Token},
- "name": {name},
+ "token": {webToken},
"pid": {dirID},
"pageNo": {itoa(pageNumber)},
"pageSize": {itoa64(maxEntitiesPerPage)},
@@ -257,7 +369,10 @@ OUTER:
}
var responseResult fileSearchRes
- err = getUnmarshaledResponse(ctx, f, opts, &responseResult)
+ err = f.pacer.Call(func() (bool, error) {
+ resp, err := f.srv.CallJSON(ctx, opts, nil, &responseResult)
+ return f.shouldRetryWeb(ctx, resp, err, &responseResult, opts)
+ })
if err != nil {
return false, fmt.Errorf("getting files failed: %w", err)
}
@@ -403,7 +518,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
}
// get an entity with leaf from dirID
-func getEntity(ctx context.Context, f *Fs, leaf string, directoryID string, token string) (*entity, error) {
+func getEntity(ctx context.Context, f *Fs, leaf string, directoryID string) (*entity, error) {
var result *entity
var resultErr = fs.ErrorObjectNotFound
_, err := f.listAll(ctx, directoryID, leaf, func(entity *entity) bool {
@@ -441,7 +556,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return nil, err
}
- entity, err := getEntity(ctx, f, leaf, dirID, f.opt.Token)
+ entity, err := getEntity(ctx, f, leaf, dirID)
if err != nil {
return nil, err
}
@@ -518,7 +633,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
downloadURL := o.fullURL
if downloadURL == "" {
_, name := splitDirAndName(o.Remote())
- newObject, err := getEntity(ctx, o.fs, name, itoa64(o.dirID), o.fs.opt.Token)
+ newObject, err := getEntity(ctx, o.fs, name, itoa64(o.dirID))
if err != nil {
return nil, err
}
@@ -530,6 +645,10 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
downloadURL = newObject.URL
}
+ if downloadURL == "" {
+ return nil, fmt.Errorf("no download URL for %q - try re-running `rclone config reconnect`", o.Remote())
+ }
+
opts := &rest.Opts{
Method: "GET",
RootURL: downloadURL,
@@ -538,7 +657,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
err := o.fs.pacer.Call(func() (bool, error) {
var err error
- res, err = o.fs.srv.Call(ctx, opts)
+ res, err = o.fs.cdnSrv.Call(ctx, opts)
return o.fs.shouldRetry(ctx, res, err)
})
@@ -697,7 +816,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
var sleepTime = 100 * time.Millisecond
var entity *entity
for try := 1; try <= maxTries; try++ {
- entity, err = getEntity(ctx, o.fs, leaf, dirID, o.fs.opt.Token)
+ entity, err = getEntity(ctx, o.fs, leaf, dirID)
if err == nil {
break
}
diff --git a/docs/content/linkbox.md b/docs/content/linkbox.md
index 1703a520c..ea2947e70 100644
--- a/docs/content/linkbox.md
+++ b/docs/content/linkbox.md
@@ -12,6 +12,17 @@ Linkbox is [a private cloud drive](https://linkbox.to/).
Here is an example of making a remote for Linkbox.
+You will need:
+- An API token from https://www.linkbox.to/admin/account
+- The email and password for your Linkbox account
+
+**Note:** As of v1.74, email and password are required in addition to
+the API token. The API token is used for uploads and modifications,
+while email/password authentication is used for listing and
+downloading files. If you have an existing Linkbox remote, you will
+need to reconfigure it with `rclone config` to edit the remote and add
+the email and password.
+
First run:
```console
@@ -40,12 +51,24 @@ Storage> XX
Option token.
Token from https://www.linkbox.to/admin/account
Enter a value.
-token> testFromCLToken
+token> YOUR_API_TOKEN
+
+Option email.
+Email for login
+Enter a value.
+
+Option password.
+Password for login
+Enter a value.
+password> YOUR_PASSWORD
Configuration complete.
Options:
- type: linkbox
- token: XXXXXXXXXXX
+- password: *** ENCRYPTED ***
Keep this "linkbox" remote?
y) Yes this is OK (default)
e) Edit this remote