Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

pikpak: fix login issue where token retrieval fails

wiserained84553dc1ef
diff --git a/backend/pikpak/api/types.go b/backend/pikpak/api/types.go
index 96c16ac24..a81bda3c1 100644
--- a/backend/pikpak/api/types.go
+++ b/backend/pikpak/api/types.go
@@ -513,6 +513,72 @@ type RequestDecompress struct {
DefaultParent bool `json:"default_parent,omitempty"`
}
+// ------------------------------------------------------------ authorization
+
+// CaptchaToken is a response to requestCaptchaToken api call
+type CaptchaToken struct {
+ CaptchaToken string `json:"captcha_token"`
+ ExpiresIn int64 `json:"expires_in"` // currently 300s
+ // API doesn't provide Expiry field and thus it should be populated from ExpiresIn on retrieval
+ Expiry time.Time `json:"expiry,omitempty"`
+ URL string `json:"url,omitempty"` // a link for users to solve captcha
+}
+
+// expired reports whether the token is expired.
+// t must be non-nil.
+func (t *CaptchaToken) expired() bool {
+ if t.Expiry.IsZero() {
+ return false
+ }
+
+ expiryDelta := time.Duration(10) * time.Second // same as oauth2's defaultExpiryDelta
+ return t.Expiry.Round(0).Add(-expiryDelta).Before(time.Now())
+}
+
+// Valid reports whether t is non-nil, has an AccessToken, and is not expired.
+func (t *CaptchaToken) Valid() bool {
+ return t != nil && t.CaptchaToken != "" && !t.expired()
+}
+
+// CaptchaTokenRequest is to request for captcha token
+type CaptchaTokenRequest struct {
+ Action string `json:"action,omitempty"`
+ CaptchaToken string `json:"captcha_token,omitempty"`
+ ClientID string `json:"client_id,omitempty"`
+ DeviceID string `json:"device_id,omitempty"`
+ Meta *CaptchaTokenMeta `json:"meta,omitempty"`
+}
+
+// CaptchaTokenMeta contains meta info for CaptchaTokenRequest
+type CaptchaTokenMeta struct {
+ CaptchaSign string `json:"captcha_sign,omitempty"`
+ ClientVersion string `json:"client_version,omitempty"`
+ PackageName string `json:"package_name,omitempty"`
+ Timestamp string `json:"timestamp,omitempty"`
+ UserID string `json:"user_id,omitempty"` // webdrive uses this instead of UserName
+ UserName string `json:"username,omitempty"`
+ Email string `json:"email,omitempty"`
+ PhoneNumber string `json:"phone_number,omitempty"`
+}
+
+// Token represents oauth2 token used for pikpak which needs to be converted to be compatible with oauth2.Token
+type Token struct {
+ TokenType string `json:"token_type"`
+ AccessToken string `json:"access_token"`
+ RefreshToken string `json:"refresh_token"`
+ ExpiresIn int `json:"expires_in"`
+ Sub string `json:"sub"`
+}
+
+// Expiry returns expiry from expires in, so it should be called on retrieval
+// e must be non-nil.
+func (e *Token) Expiry() (t time.Time) {
+ if v := e.ExpiresIn; v != 0 {
+ return time.Now().Add(time.Duration(v) * time.Second)
+ }
+ return
+}
+
// ------------------------------------------------------------
// NOT implemented YET
diff --git a/backend/pikpak/helper.go b/backend/pikpak/helper.go
index fd1561d7a..ee2815d94 100644
--- a/backend/pikpak/helper.go
+++ b/backend/pikpak/helper.go
@@ -3,8 +3,10 @@ package pikpak
import (
"bytes"
"context"
+ "crypto/md5"
"crypto/sha1"
"encoding/hex"
+ "encoding/json"
"errors"
"fmt"
"io"
@@ -14,10 +16,13 @@ import (
"os"
"strconv"
"strings"
+ "sync"
"time"
"github.com/rclone/rclone/backend/pikpak/api"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config/configmap"
+ "github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/lib/rest"
)
@@ -262,15 +267,20 @@ func (f *Fs) getGcid(ctx context.Context, src fs.ObjectInfo) (gcid string, err e
if err != nil {
return
}
+ if src.Size() == 0 {
+ // If src is zero-length, the API will return
+ // Error "cid and file_size is required" (400)
+ // In this case, we can simply return cid == gcid
+ return cid, nil
+ }
params := url.Values{}
params.Set("cid", cid)
params.Set("file_size", strconv.FormatInt(src.Size(), 10))
opts := rest.Opts{
- Method: "GET",
- Path: "/drive/v1/resource/cid",
- Parameters: params,
- ExtraHeaders: map[string]string{"x-device-id": f.deviceID},
+ Method: "GET",
+ Path: "/drive/v1/resource/cid",
+ Parameters: params,
}
info := struct {
@@ -408,6 +418,8 @@ func calcCid(ctx context.Context, src fs.ObjectInfo) (cid string, err error) {
return
}
+// ------------------------------------------------------------ authorization
+
// randomly generates device id used for request header 'x-device-id'
//
// original javascript implementation
@@ -428,3 +440,206 @@ func genDeviceID() string {
}
return string(base)
}
+
+var md5Salt = []string{
+ "C9qPpZLN8ucRTaTiUMWYS9cQvWOE",
+ "+r6CQVxjzJV6LCV",
+ "F",
+ "pFJRC",
+ "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt",
+ "/750aCr4lm/Sly/c",
+ "RB+DT/gZCrbV",
+ "",
+ "CyLsf7hdkIRxRm215hl",
+ "7xHvLi2tOYP0Y92b",
+ "ZGTXXxu8E/MIWaEDB+Sm/",
+ "1UI3",
+ "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO",
+ "ihtqpG6FMt65+Xk+tWUH2",
+ "NhXXU9rg4XXdzo7u5o",
+}
+
+func md5Sum(text string) string {
+ hash := md5.Sum([]byte(text))
+ return hex.EncodeToString(hash[:])
+}
+
+func calcCaptchaSign(deviceID string) (timestamp, sign string) {
+ timestamp = fmt.Sprint(time.Now().UnixMilli())
+ str := fmt.Sprint(clientID, clientVersion, packageName, deviceID, timestamp)
+ for _, salt := range md5Salt {
+ str = md5Sum(str + salt)
+ }
+ sign = "1." + str
+ return
+}
+
+func newCaptchaTokenRequest(action, oldToken string, opt *Options) (req *api.CaptchaTokenRequest) {
+ req = &api.CaptchaTokenRequest{
+ Action: action,
+ CaptchaToken: oldToken, // can be empty initially
+ ClientID: clientID,
+ DeviceID: opt.DeviceID,
+ Meta: new(api.CaptchaTokenMeta),
+ }
+ switch action {
+ case "POST:/v1/auth/signin":
+ req.Meta.UserName = opt.Username
+ default:
+ timestamp, captchaSign := calcCaptchaSign(opt.DeviceID)
+ req.Meta.CaptchaSign = captchaSign
+ req.Meta.Timestamp = timestamp
+ req.Meta.ClientVersion = clientVersion
+ req.Meta.PackageName = packageName
+ req.Meta.UserID = opt.UserID
+ }
+ return
+}
+
+// CaptchaTokenSource stores updated captcha tokens in the config file
+type CaptchaTokenSource struct {
+ mu sync.Mutex
+ m configmap.Mapper
+ opt *Options
+ token *api.CaptchaToken
+ ctx context.Context
+ rst *pikpakClient
+}
+
+// initialize CaptchaTokenSource from rclone.conf if possible
+func newCaptchaTokenSource(ctx context.Context, opt *Options, m configmap.Mapper) *CaptchaTokenSource {
+ token := new(api.CaptchaToken)
+ tokenString, ok := m.Get("captcha_token")
+ if !ok || tokenString == "" {
+ fs.Debugf(nil, "failed to read captcha token out of config file")
+ } else {
+ if err := json.Unmarshal([]byte(tokenString), token); err != nil {
+ fs.Debugf(nil, "failed to parse captcha token out of config file: %v", err)
+ }
+ }
+ return &CaptchaTokenSource{
+ m: m,
+ opt: opt,
+ token: token,
+ ctx: ctx,
+ rst: newPikpakClient(getClient(ctx, opt), opt),
+ }
+}
+
+// requestToken retrieves captcha token from API
+func (cts *CaptchaTokenSource) requestToken(ctx context.Context, req *api.CaptchaTokenRequest) (err error) {
+ opts := rest.Opts{
+ Method: "POST",
+ RootURL: "https://user.mypikpak.com/v1/shield/captcha/init",
+ }
+ var info *api.CaptchaToken
+ _, err = cts.rst.CallJSON(ctx, &opts, &req, &info)
+ if err == nil && info.ExpiresIn != 0 {
+ // populate to Expiry
+ info.Expiry = time.Now().Add(time.Duration(info.ExpiresIn) * time.Second)
+ cts.token = info // update with a new one
+ }
+ return
+}
+
+func (cts *CaptchaTokenSource) refreshToken(opts *rest.Opts) (string, error) {
+ oldToken := ""
+ if cts.token != nil {
+ oldToken = cts.token.CaptchaToken
+ }
+ action := "GET:/drive/v1/about"
+ if opts.RootURL == "" && opts.Path != "" {
+ action = fmt.Sprintf("%s:%s", opts.Method, opts.Path)
+ } else if u, err := url.Parse(opts.RootURL); err == nil {
+ action = fmt.Sprintf("%s:%s", opts.Method, u.Path)
+ }
+ req := newCaptchaTokenRequest(action, oldToken, cts.opt)
+ if err := cts.requestToken(cts.ctx, req); err != nil {
+ return "", fmt.Errorf("failed to retrieve captcha token from api: %w", err)
+ }
+
+ // put it into rclone.conf
+ tokenBytes, err := json.Marshal(cts.token)
+ if err != nil {
+ return "", fmt.Errorf("failed to marshal captcha token: %w", err)
+ }
+ cts.m.Set("captcha_token", string(tokenBytes))
+ return cts.token.CaptchaToken, nil
+}
+
+// Invalidate resets existing captcha token for a forced refresh
+func (cts *CaptchaTokenSource) Invalidate() {
+ cts.mu.Lock()
+ cts.token.CaptchaToken = ""
+ cts.mu.Unlock()
+}
+
+// Token returns a valid captcha token
+func (cts *CaptchaTokenSource) Token(opts *rest.Opts) (string, error) {
+ cts.mu.Lock()
+ defer cts.mu.Unlock()
+ if cts.token.Valid() {
+ return cts.token.CaptchaToken, nil
+ }
+ return cts.refreshToken(opts)
+}
+
+// pikpakClient wraps rest.Client with a handle of captcha token
+type pikpakClient struct {
+ opt *Options
+ client *rest.Client
+ captcha *CaptchaTokenSource
+}
+
+// newPikpakClient takes an (oauth) http.Client and makes a new api instance for pikpak with
+// * error handler
+// * root url
+// * default headers
+func newPikpakClient(c *http.Client, opt *Options) *pikpakClient {
+ client := rest.NewClient(c).SetErrorHandler(errorHandler).SetRoot(rootURL)
+ for key, val := range map[string]string{
+ "Referer": "https://mypikpak.com/",
+ "x-client-id": clientID,
+ "x-client-version": clientVersion,
+ "x-device-id": opt.DeviceID,
+ // "x-device-model": "firefox%2F129.0",
+ // "x-device-name": "PC-Firefox",
+ // "x-device-sign": fmt.Sprintf("wdi10.%sxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", opt.DeviceID),
+ // "x-net-work-type": "NONE",
+ // "x-os-version": "Win32",
+ // "x-platform-version": "1",
+ // "x-protocol-version": "301",
+ // "x-provider-name": "NONE",
+ // "x-sdk-version": "8.0.3",
+ } {
+ client.SetHeader(key, val)
+ }
+ return &pikpakClient{
+ client: client,
+ opt: opt,
+ }
+}
+
+// This should be called right after pikpakClient initialized
+func (c *pikpakClient) SetCaptchaTokener(ctx context.Context, m configmap.Mapper) *pikpakClient {
+ c.captcha = newCaptchaTokenSource(ctx, c.opt, m)
+ return c
+}
+
+func (c *pikpakClient) CallJSON(ctx context.Context, opts *rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
+ if c.captcha != nil {
+ token, err := c.captcha.Token(opts)
+ if err != nil || token == "" {
+ return nil, fserrors.FatalError(fmt.Errorf("couldn't get captcha token: %v", err))
+ }
+ if opts.ExtraHeaders == nil {
+ opts.ExtraHeaders = make(map[string]string)
+ }
+ opts.ExtraHeaders["x-captcha-token"] = token
+ }
+ return c.client.CallJSON(ctx, opts, request, response)
+}
+
+func (c *pikpakClient) Call(ctx context.Context, opts *rest.Opts) (resp *http.Response, err error) {
+ return c.client.Call(ctx, opts)
+}
diff --git a/backend/pikpak/pikpak.go b/backend/pikpak/pikpak.go
index 1a5d82a33..25c27f1c4 100644
--- a/backend/pikpak/pikpak.go
+++ b/backend/pikpak/pikpak.go
@@ -23,6 +23,7 @@ package pikpak
import (
"bytes"
"context"
+ "encoding/base64"
"encoding/json"
"errors"
"fmt"
@@ -51,6 +52,7 @@ import (
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fserrors"
+ "github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/dircache"
@@ -64,15 +66,17 @@ import (
// Constants
const (
- rcloneClientID = "YNxT9w7GMdWvEOKa"
- rcloneEncryptedClientSecret = "aqrmB6M1YJ1DWCBxVxFSjFo7wzWEky494YMmkqgAl1do1WKOe2E"
- minSleep = 100 * time.Millisecond
- maxSleep = 2 * time.Second
- taskWaitTime = 500 * time.Millisecond
- decayConstant = 2 // bigger for slower decay, exponential
- rootURL = "https://api-drive.mypikpak.com"
- minChunkSize = fs.SizeSuffix(manager.MinUploadPartSize)
- defaultUploadConcurrency = manager.DefaultUploadConcurrency
+ clientID = "YUMx5nI8ZU8Ap8pm"
+ clientVersion = "2.0.0"
+ packageName = "mypikpak.com"
+ defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0"
+ minSleep = 100 * time.Millisecond
+ maxSleep = 2 * time.Second
+ taskWaitTime = 500 * time.Millisecond
+ decayConstant = 2 // bigger for slower decay, exponential
+ rootURL = "https://api-drive.mypikpak.com"
+ minChunkSize = fs.SizeSuffix(manager.MinUploadPartSize)
+ defaultUploadConcurrency = manager.DefaultUploadConcurrency
)
// Globals
@@ -85,43 +89,53 @@ var (
TokenURL: "https://user.mypikpak.com/v1/auth/token",
AuthStyle: oauth2.AuthStyleInParams,
},
- ClientID: rcloneClientID,
- ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
- RedirectURL: oauthutil.RedirectURL,
+ ClientID: clientID,
+ RedirectURL: oauthutil.RedirectURL,
}
)
-// Returns OAuthOptions modified for pikpak
-func pikpakOAuthOptions() []fs.Option {
- opts := []fs.Option{}
- for _, opt := range oauthutil.SharedOptions {
- if opt.Name == config.ConfigClientID {
- opt.Advanced = true
- } else if opt.Name == config.ConfigClientSecret {
- opt.Advanced = true
- }
- opts = append(opts, opt)
- }
- return opts
-}
-
// pikpakAutorize retrieves OAuth token using user/pass and save it to rclone.conf
func pikpakAuthorize(ctx context.Context, opt *Options, name string, m configmap.Mapper) error {
- // override default client id/secret
- if id, ok := m.Get("client_id"); ok && id != "" {
- oauthConfig.ClientID = id
- }
- if secret, ok := m.Get("client_secret"); ok && secret != "" {
- oauthConfig.ClientSecret = secret
+ if opt.Username == "" {
+ return errors.New("no username")
}
pass, err := obscure.Reveal(opt.Password)
if err != nil {
return fmt.Errorf("failed to decode password - did you obscure it?: %w", err)
}
- t, err := oauthConfig.PasswordCredentialsToken(ctx, opt.Username, pass)
+ // new device id if necessary
+ if len(opt.DeviceID) != 32 {
+ opt.DeviceID = genDeviceID()
+ m.Set("device_id", opt.DeviceID)
+ fs.Infof(nil, "Using new device id %q", opt.DeviceID)
+ }
+ opts := rest.Opts{
+ Method: "POST",
+ RootURL: "https://user.mypikpak.com/v1/auth/signin",
+ }
+ req := map[string]string{
+ "username": opt.Username,
+ "password": pass,
+ "client_id": clientID,
+ }
+ var token api.Token
+ rst := newPikpakClient(getClient(ctx, opt), opt).SetCaptchaTokener(ctx, m)
+ _, err = rst.CallJSON(ctx, &opts, req, &token)
+ if apiErr, ok := err.(*api.Error); ok {
+ if apiErr.Reason == "captcha_invalid" && apiErr.Code == 4002 {
+ rst.captcha.Invalidate()
+ _, err = rst.CallJSON(ctx, &opts, req, &token)
+ }
+ }
if err != nil {
return fmt.Errorf("failed to retrieve token using username/password: %w", err)
}
+ t := &oauth2.Token{
+ AccessToken: token.AccessToken,
+ TokenType: token.TokenType,
+ RefreshToken: token.RefreshToken,
+ Expiry: token.Expiry(),
+ }
return oauthutil.PutToken(name, m, t, false)
}
@@ -160,7 +174,7 @@ func init() {
}
return nil, fmt.Errorf("unknown state %q", config.State)
},
- Options: append(pikpakOAuthOptions(), []fs.Option{{
+ Options: []fs.Option{{
Name: "user",
Help: "Pikpak username.",
Required: true,
@@ -170,6 +184,18 @@ func init() {
Help: "Pikpak password.",
Required: true,
IsPassword: true,
+ }, {
+ Name: "device_id",
+ Help: "Device ID used for authorization.",
+ Advanced: true,
+ Sensitive: true,
+ }, {
+ Name: "user_agent",
+ Default: defaultUserAgent,
+ Advanced: true,
+ Help: fmt.Sprintf(`HTTP user agent for pikpak.
+
+Defaults to "%s" or "--pikpak-user-agent" provided on command line.`, defaultUserAgent),
}, {
Name: "root_folder_id",
Help: `ID of the root folder.
@@ -248,7 +274,7 @@ this may help to speed up the transfers.`,
encoder.EncodeRightSpace |
encoder.EncodeRightPeriod |
encoder.EncodeInvalidUtf8),
- }}...),
+ }},
})
}
@@ -256,6 +282,9 @@ this may help to speed up the transfers.`,
type Options struct {
Username string `config:"user"`
Password string `config:"pass"`
+ UserID string `config:"user_id"` // only available during runtime
+ DeviceID string `config:"device_id"`
+ UserAgent string `config:"user_agent"`
RootFolderID string `config:"root_folder_id"`
UseTrash bool `config:"use_trash"`
TrashedOnly bool `config:"trashed_only"`
@@ -271,11 +300,10 @@ type Fs struct {
root string // the path we are working on
opt Options // parsed options
features *fs.Features // optional features
- rst *rest.Client // the connection to the server
+ rst *pikpakClient // the connection to the server
dirCache *dircache.DirCache // Map of directory path to directory id
pacer *fs.Pacer // pacer for API calls
rootFolderID string // the id of the root folder
- deviceID string // device id used for api requests
client *http.Client // authorized client
m configmap.Mapper
tokenMu *sync.Mutex // when renewing tokens
@@ -429,6 +457,12 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
} else if apiErr.Reason == "file_space_not_enough" {
// "file_space_not_enough" (8): Storage space is not enough
return false, fserrors.FatalError(err)
+ } else if apiErr.Reason == "captcha_invalid" && apiErr.Code == 9 {
+ // "captcha_invalid" (9): Verification code is invalid
+ // This error occurred on the POST:/drive/v1/files endpoint
+ // when a zero-byte file was uploaded with an invalid captcha token
+ f.rst.captcha.Invalidate()
+ return true, err
}
}
@@ -452,13 +486,36 @@ func errorHandler(resp *http.Response) error {
return errResponse
}
+// getClient makes an http client according to the options
+func getClient(ctx context.Context, opt *Options) *http.Client {
+ // Override few config settings and create a client
+ newCtx, ci := fs.AddConfig(ctx)
+ ci.UserAgent = opt.UserAgent
+ return fshttp.NewClient(newCtx)
+}
+
// newClientWithPacer sets a new http/rest client with a pacer to Fs
func (f *Fs) newClientWithPacer(ctx context.Context) (err error) {
- f.client, _, err = oauthutil.NewClient(ctx, f.name, f.m, oauthConfig)
+ var ts *oauthutil.TokenSource
+ f.client, ts, err = oauthutil.NewClientWithBaseClient(ctx, f.name, f.m, oauthConfig, getClient(ctx, &f.opt))
if err != nil {
return fmt.Errorf("failed to create oauth client: %w", err)
}
- f.rst = rest.NewClient(f.client).SetRoot(rootURL).SetErrorHandler(errorHandler)
+ token, err := ts.Token()
+ if err != nil {
+ return err
+ }
+ // parse user_id from oauth access token for later use
+ if parts := strings.Split(token.AccessToken, "."); len(parts) > 1 {
+ jsonStr, _ := base64.URLEncoding.DecodeString(parts[1] + "===")
+ info := struct {
+ UserID string `json:"sub,omitempty"`
+ }{}
+ if jsonErr := json.Unmarshal(jsonStr, &info); jsonErr == nil {
+ f.opt.UserID = info.UserID
+ }
+ }
+ f.rst = newPikpakClient(f.client, &f.opt).SetCaptchaTokener(ctx, f.m)
f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
return nil
}
@@ -491,10 +548,19 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
CanHaveEmptyDirectories: true, // can have empty directories
NoMultiThreading: true, // can't have multiple threads downloading
}).Fill(ctx, f)
- f.deviceID = genDeviceID()
+
+ // new device id if necessary
+ if len(f.opt.DeviceID) != 32 {
+ f.opt.DeviceID = genDeviceID()
+ m.Set("device_id", f.opt.DeviceID)
+ fs.Infof(nil, "Using new device id %q", f.opt.DeviceID)
+ }
if err := f.newClientWithPacer(ctx); err != nil {
- return nil, err
+ // re-authorize if necessary
+ if strings.Contains(err.Error(), "invalid_grant") {
+ return f, f.reAuthorize(ctx)
+ }
}
return f, nil