"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
pikpak: fix login issue where token retrieval fails
wiserained84553dc1ef
diff --git a/backend/pikpak/api/types.go b/backend/pikpak/api/types.goindex 96c16ac24..a81bda3c1 100644--- a/backend/pikpak/api/types.go+++ b/backend/pikpak/api/types.go@@ -513,6 +513,72 @@ type RequestDecompress struct {DefaultParent bool `json:"default_parent,omitempty"`}+// ------------------------------------------------------------ authorization++// CaptchaToken is a response to requestCaptchaToken api call+type CaptchaToken struct {+ CaptchaToken string `json:"captcha_token"`+ ExpiresIn int64 `json:"expires_in"` // currently 300s+ // API doesn't provide Expiry field and thus it should be populated from ExpiresIn on retrieval+ Expiry time.Time `json:"expiry,omitempty"`+ URL string `json:"url,omitempty"` // a link for users to solve captcha+}++// expired reports whether the token is expired.+// t must be non-nil.+func (t *CaptchaToken) expired() bool {+ if t.Expiry.IsZero() {+ return false+ }++ expiryDelta := time.Duration(10) * time.Second // same as oauth2's defaultExpiryDelta+ return t.Expiry.Round(0).Add(-expiryDelta).Before(time.Now())+}++// Valid reports whether t is non-nil, has an AccessToken, and is not expired.+func (t *CaptchaToken) Valid() bool {+ return t != nil && t.CaptchaToken != "" && !t.expired()+}++// CaptchaTokenRequest is to request for captcha token+type CaptchaTokenRequest struct {+ Action string `json:"action,omitempty"`+ CaptchaToken string `json:"captcha_token,omitempty"`+ ClientID string `json:"client_id,omitempty"`+ DeviceID string `json:"device_id,omitempty"`+ Meta *CaptchaTokenMeta `json:"meta,omitempty"`+}++// CaptchaTokenMeta contains meta info for CaptchaTokenRequest+type CaptchaTokenMeta struct {+ CaptchaSign string `json:"captcha_sign,omitempty"`+ ClientVersion string `json:"client_version,omitempty"`+ PackageName string `json:"package_name,omitempty"`+ Timestamp string `json:"timestamp,omitempty"`+ UserID string `json:"user_id,omitempty"` // webdrive uses this instead of UserName+ UserName string `json:"username,omitempty"`+ Email string `json:"email,omitempty"`+ PhoneNumber string `json:"phone_number,omitempty"`+}++// Token represents oauth2 token used for pikpak which needs to be converted to be compatible with oauth2.Token+type Token struct {+ TokenType string `json:"token_type"`+ AccessToken string `json:"access_token"`+ RefreshToken string `json:"refresh_token"`+ ExpiresIn int `json:"expires_in"`+ Sub string `json:"sub"`+}++// Expiry returns expiry from expires in, so it should be called on retrieval+// e must be non-nil.+func (e *Token) Expiry() (t time.Time) {+ if v := e.ExpiresIn; v != 0 {+ return time.Now().Add(time.Duration(v) * time.Second)+ }+ return+}+// ------------------------------------------------------------// NOT implemented YETdiff --git a/backend/pikpak/helper.go b/backend/pikpak/helper.goindex fd1561d7a..ee2815d94 100644--- a/backend/pikpak/helper.go+++ b/backend/pikpak/helper.go@@ -3,8 +3,10 @@ package pikpakimport ("bytes""context"+ "crypto/md5""crypto/sha1""encoding/hex"+ "encoding/json""errors""fmt""io"@@ -14,10 +16,13 @@ import ("os""strconv""strings"+ "sync""time""github.com/rclone/rclone/backend/pikpak/api""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config/configmap"+ "github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/lib/rest")@@ -262,15 +267,20 @@ func (f *Fs) getGcid(ctx context.Context, src fs.ObjectInfo) (gcid string, err eif err != nil {return}+ if src.Size() == 0 {+ // If src is zero-length, the API will return+ // Error "cid and file_size is required" (400)+ // In this case, we can simply return cid == gcid+ return cid, nil+ }params := url.Values{}params.Set("cid", cid)params.Set("file_size", strconv.FormatInt(src.Size(), 10))opts := rest.Opts{- Method: "GET",- Path: "/drive/v1/resource/cid",- Parameters: params,- ExtraHeaders: map[string]string{"x-device-id": f.deviceID},+ Method: "GET",+ Path: "/drive/v1/resource/cid",+ Parameters: params,}info := struct {@@ -408,6 +418,8 @@ func calcCid(ctx context.Context, src fs.ObjectInfo) (cid string, err error) {return}+// ------------------------------------------------------------ authorization+// randomly generates device id used for request header 'x-device-id'//// original javascript implementation@@ -428,3 +440,206 @@ func genDeviceID() string {}return string(base)}++var md5Salt = []string{+ "C9qPpZLN8ucRTaTiUMWYS9cQvWOE",+ "+r6CQVxjzJV6LCV",+ "F",+ "pFJRC",+ "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt",+ "/750aCr4lm/Sly/c",+ "RB+DT/gZCrbV",+ "",+ "CyLsf7hdkIRxRm215hl",+ "7xHvLi2tOYP0Y92b",+ "ZGTXXxu8E/MIWaEDB+Sm/",+ "1UI3",+ "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO",+ "ihtqpG6FMt65+Xk+tWUH2",+ "NhXXU9rg4XXdzo7u5o",+}++func md5Sum(text string) string {+ hash := md5.Sum([]byte(text))+ return hex.EncodeToString(hash[:])+}++func calcCaptchaSign(deviceID string) (timestamp, sign string) {+ timestamp = fmt.Sprint(time.Now().UnixMilli())+ str := fmt.Sprint(clientID, clientVersion, packageName, deviceID, timestamp)+ for _, salt := range md5Salt {+ str = md5Sum(str + salt)+ }+ sign = "1." + str+ return+}++func newCaptchaTokenRequest(action, oldToken string, opt *Options) (req *api.CaptchaTokenRequest) {+ req = &api.CaptchaTokenRequest{+ Action: action,+ CaptchaToken: oldToken, // can be empty initially+ ClientID: clientID,+ DeviceID: opt.DeviceID,+ Meta: new(api.CaptchaTokenMeta),+ }+ switch action {+ case "POST:/v1/auth/signin":+ req.Meta.UserName = opt.Username+ default:+ timestamp, captchaSign := calcCaptchaSign(opt.DeviceID)+ req.Meta.CaptchaSign = captchaSign+ req.Meta.Timestamp = timestamp+ req.Meta.ClientVersion = clientVersion+ req.Meta.PackageName = packageName+ req.Meta.UserID = opt.UserID+ }+ return+}++// CaptchaTokenSource stores updated captcha tokens in the config file+type CaptchaTokenSource struct {+ mu sync.Mutex+ m configmap.Mapper+ opt *Options+ token *api.CaptchaToken+ ctx context.Context+ rst *pikpakClient+}++// initialize CaptchaTokenSource from rclone.conf if possible+func newCaptchaTokenSource(ctx context.Context, opt *Options, m configmap.Mapper) *CaptchaTokenSource {+ token := new(api.CaptchaToken)+ tokenString, ok := m.Get("captcha_token")+ if !ok || tokenString == "" {+ fs.Debugf(nil, "failed to read captcha token out of config file")+ } else {+ if err := json.Unmarshal([]byte(tokenString), token); err != nil {+ fs.Debugf(nil, "failed to parse captcha token out of config file: %v", err)+ }+ }+ return &CaptchaTokenSource{+ m: m,+ opt: opt,+ token: token,+ ctx: ctx,+ rst: newPikpakClient(getClient(ctx, opt), opt),+ }+}++// requestToken retrieves captcha token from API+func (cts *CaptchaTokenSource) requestToken(ctx context.Context, req *api.CaptchaTokenRequest) (err error) {+ opts := rest.Opts{+ Method: "POST",+ RootURL: "https://user.mypikpak.com/v1/shield/captcha/init",+ }+ var info *api.CaptchaToken+ _, err = cts.rst.CallJSON(ctx, &opts, &req, &info)+ if err == nil && info.ExpiresIn != 0 {+ // populate to Expiry+ info.Expiry = time.Now().Add(time.Duration(info.ExpiresIn) * time.Second)+ cts.token = info // update with a new one+ }+ return+}++func (cts *CaptchaTokenSource) refreshToken(opts *rest.Opts) (string, error) {+ oldToken := ""+ if cts.token != nil {+ oldToken = cts.token.CaptchaToken+ }+ action := "GET:/drive/v1/about"+ if opts.RootURL == "" && opts.Path != "" {+ action = fmt.Sprintf("%s:%s", opts.Method, opts.Path)+ } else if u, err := url.Parse(opts.RootURL); err == nil {+ action = fmt.Sprintf("%s:%s", opts.Method, u.Path)+ }+ req := newCaptchaTokenRequest(action, oldToken, cts.opt)+ if err := cts.requestToken(cts.ctx, req); err != nil {+ return "", fmt.Errorf("failed to retrieve captcha token from api: %w", err)+ }++ // put it into rclone.conf+ tokenBytes, err := json.Marshal(cts.token)+ if err != nil {+ return "", fmt.Errorf("failed to marshal captcha token: %w", err)+ }+ cts.m.Set("captcha_token", string(tokenBytes))+ return cts.token.CaptchaToken, nil+}++// Invalidate resets existing captcha token for a forced refresh+func (cts *CaptchaTokenSource) Invalidate() {+ cts.mu.Lock()+ cts.token.CaptchaToken = ""+ cts.mu.Unlock()+}++// Token returns a valid captcha token+func (cts *CaptchaTokenSource) Token(opts *rest.Opts) (string, error) {+ cts.mu.Lock()+ defer cts.mu.Unlock()+ if cts.token.Valid() {+ return cts.token.CaptchaToken, nil+ }+ return cts.refreshToken(opts)+}++// pikpakClient wraps rest.Client with a handle of captcha token+type pikpakClient struct {+ opt *Options+ client *rest.Client+ captcha *CaptchaTokenSource+}++// newPikpakClient takes an (oauth) http.Client and makes a new api instance for pikpak with+// * error handler+// * root url+// * default headers+func newPikpakClient(c *http.Client, opt *Options) *pikpakClient {+ client := rest.NewClient(c).SetErrorHandler(errorHandler).SetRoot(rootURL)+ for key, val := range map[string]string{+ "Referer": "https://mypikpak.com/",+ "x-client-id": clientID,+ "x-client-version": clientVersion,+ "x-device-id": opt.DeviceID,+ // "x-device-model": "firefox%2F129.0",+ // "x-device-name": "PC-Firefox",+ // "x-device-sign": fmt.Sprintf("wdi10.%sxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", opt.DeviceID),+ // "x-net-work-type": "NONE",+ // "x-os-version": "Win32",+ // "x-platform-version": "1",+ // "x-protocol-version": "301",+ // "x-provider-name": "NONE",+ // "x-sdk-version": "8.0.3",+ } {+ client.SetHeader(key, val)+ }+ return &pikpakClient{+ client: client,+ opt: opt,+ }+}++// This should be called right after pikpakClient initialized+func (c *pikpakClient) SetCaptchaTokener(ctx context.Context, m configmap.Mapper) *pikpakClient {+ c.captcha = newCaptchaTokenSource(ctx, c.opt, m)+ return c+}++func (c *pikpakClient) CallJSON(ctx context.Context, opts *rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {+ if c.captcha != nil {+ token, err := c.captcha.Token(opts)+ if err != nil || token == "" {+ return nil, fserrors.FatalError(fmt.Errorf("couldn't get captcha token: %v", err))+ }+ if opts.ExtraHeaders == nil {+ opts.ExtraHeaders = make(map[string]string)+ }+ opts.ExtraHeaders["x-captcha-token"] = token+ }+ return c.client.CallJSON(ctx, opts, request, response)+}++func (c *pikpakClient) Call(ctx context.Context, opts *rest.Opts) (resp *http.Response, err error) {+ return c.client.Call(ctx, opts)+}diff --git a/backend/pikpak/pikpak.go b/backend/pikpak/pikpak.goindex 1a5d82a33..25c27f1c4 100644--- a/backend/pikpak/pikpak.go+++ b/backend/pikpak/pikpak.go@@ -23,6 +23,7 @@ package pikpakimport ("bytes""context"+ "encoding/base64""encoding/json""errors""fmt"@@ -51,6 +52,7 @@ import ("github.com/rclone/rclone/fs/config/configstruct""github.com/rclone/rclone/fs/config/obscure""github.com/rclone/rclone/fs/fserrors"+ "github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/lib/atexit""github.com/rclone/rclone/lib/dircache"@@ -64,15 +66,17 @@ import (// Constantsconst (- rcloneClientID = "YNxT9w7GMdWvEOKa"- rcloneEncryptedClientSecret = "aqrmB6M1YJ1DWCBxVxFSjFo7wzWEky494YMmkqgAl1do1WKOe2E"- minSleep = 100 * time.Millisecond- maxSleep = 2 * time.Second- taskWaitTime = 500 * time.Millisecond- decayConstant = 2 // bigger for slower decay, exponential- rootURL = "https://api-drive.mypikpak.com"- minChunkSize = fs.SizeSuffix(manager.MinUploadPartSize)- defaultUploadConcurrency = manager.DefaultUploadConcurrency+ clientID = "YUMx5nI8ZU8Ap8pm"+ clientVersion = "2.0.0"+ packageName = "mypikpak.com"+ defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0"+ minSleep = 100 * time.Millisecond+ maxSleep = 2 * time.Second+ taskWaitTime = 500 * time.Millisecond+ decayConstant = 2 // bigger for slower decay, exponential+ rootURL = "https://api-drive.mypikpak.com"+ minChunkSize = fs.SizeSuffix(manager.MinUploadPartSize)+ defaultUploadConcurrency = manager.DefaultUploadConcurrency)// Globals@@ -85,43 +89,53 @@ var (TokenURL: "https://user.mypikpak.com/v1/auth/token",AuthStyle: oauth2.AuthStyleInParams,},- ClientID: rcloneClientID,- ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),- RedirectURL: oauthutil.RedirectURL,+ ClientID: clientID,+ RedirectURL: oauthutil.RedirectURL,})-// Returns OAuthOptions modified for pikpak-func pikpakOAuthOptions() []fs.Option {- opts := []fs.Option{}- for _, opt := range oauthutil.SharedOptions {- if opt.Name == config.ConfigClientID {- opt.Advanced = true- } else if opt.Name == config.ConfigClientSecret {- opt.Advanced = true- }- opts = append(opts, opt)- }- return opts-}-// pikpakAutorize retrieves OAuth token using user/pass and save it to rclone.conffunc pikpakAuthorize(ctx context.Context, opt *Options, name string, m configmap.Mapper) error {- // override default client id/secret- if id, ok := m.Get("client_id"); ok && id != "" {- oauthConfig.ClientID = id- }- if secret, ok := m.Get("client_secret"); ok && secret != "" {- oauthConfig.ClientSecret = secret+ if opt.Username == "" {+ return errors.New("no username")}pass, err := obscure.Reveal(opt.Password)if err != nil {return fmt.Errorf("failed to decode password - did you obscure it?: %w", err)}- t, err := oauthConfig.PasswordCredentialsToken(ctx, opt.Username, pass)+ // new device id if necessary+ if len(opt.DeviceID) != 32 {+ opt.DeviceID = genDeviceID()+ m.Set("device_id", opt.DeviceID)+ fs.Infof(nil, "Using new device id %q", opt.DeviceID)+ }+ opts := rest.Opts{+ Method: "POST",+ RootURL: "https://user.mypikpak.com/v1/auth/signin",+ }+ req := map[string]string{+ "username": opt.Username,+ "password": pass,+ "client_id": clientID,+ }+ var token api.Token+ rst := newPikpakClient(getClient(ctx, opt), opt).SetCaptchaTokener(ctx, m)+ _, err = rst.CallJSON(ctx, &opts, req, &token)+ if apiErr, ok := err.(*api.Error); ok {+ if apiErr.Reason == "captcha_invalid" && apiErr.Code == 4002 {+ rst.captcha.Invalidate()+ _, err = rst.CallJSON(ctx, &opts, req, &token)+ }+ }if err != nil {return fmt.Errorf("failed to retrieve token using username/password: %w", err)}+ t := &oauth2.Token{+ AccessToken: token.AccessToken,+ TokenType: token.TokenType,+ RefreshToken: token.RefreshToken,+ Expiry: token.Expiry(),+ }return oauthutil.PutToken(name, m, t, false)}@@ -160,7 +174,7 @@ func init() {}return nil, fmt.Errorf("unknown state %q", config.State)},- Options: append(pikpakOAuthOptions(), []fs.Option{{+ Options: []fs.Option{{Name: "user",Help: "Pikpak username.",Required: true,@@ -170,6 +184,18 @@ func init() {Help: "Pikpak password.",Required: true,IsPassword: true,+ }, {+ Name: "device_id",+ Help: "Device ID used for authorization.",+ Advanced: true,+ Sensitive: true,+ }, {+ Name: "user_agent",+ Default: defaultUserAgent,+ Advanced: true,+ Help: fmt.Sprintf(`HTTP user agent for pikpak.++Defaults to "%s" or "--pikpak-user-agent" provided on command line.`, defaultUserAgent),}, {Name: "root_folder_id",Help: `ID of the root folder.@@ -248,7 +274,7 @@ this may help to speed up the transfers.`,encoder.EncodeRightSpace |encoder.EncodeRightPeriod |encoder.EncodeInvalidUtf8),- }}...),+ }},})}@@ -256,6 +282,9 @@ this may help to speed up the transfers.`,type Options struct {Username string `config:"user"`Password string `config:"pass"`+ UserID string `config:"user_id"` // only available during runtime+ DeviceID string `config:"device_id"`+ UserAgent string `config:"user_agent"`RootFolderID string `config:"root_folder_id"`UseTrash bool `config:"use_trash"`TrashedOnly bool `config:"trashed_only"`@@ -271,11 +300,10 @@ type Fs struct {root string // the path we are working onopt Options // parsed optionsfeatures *fs.Features // optional features- rst *rest.Client // the connection to the server+ rst *pikpakClient // the connection to the serverdirCache *dircache.DirCache // Map of directory path to directory idpacer *fs.Pacer // pacer for API callsrootFolderID string // the id of the root folder- deviceID string // device id used for api requestsclient *http.Client // authorized clientm configmap.MappertokenMu *sync.Mutex // when renewing tokens@@ -429,6 +457,12 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b} else if apiErr.Reason == "file_space_not_enough" {// "file_space_not_enough" (8): Storage space is not enoughreturn false, fserrors.FatalError(err)+ } else if apiErr.Reason == "captcha_invalid" && apiErr.Code == 9 {+ // "captcha_invalid" (9): Verification code is invalid+ // This error occurred on the POST:/drive/v1/files endpoint+ // when a zero-byte file was uploaded with an invalid captcha token+ f.rst.captcha.Invalidate()+ return true, err}}@@ -452,13 +486,36 @@ func errorHandler(resp *http.Response) error {return errResponse}+// getClient makes an http client according to the options+func getClient(ctx context.Context, opt *Options) *http.Client {+ // Override few config settings and create a client+ newCtx, ci := fs.AddConfig(ctx)+ ci.UserAgent = opt.UserAgent+ return fshttp.NewClient(newCtx)+}+// newClientWithPacer sets a new http/rest client with a pacer to Fsfunc (f *Fs) newClientWithPacer(ctx context.Context) (err error) {- f.client, _, err = oauthutil.NewClient(ctx, f.name, f.m, oauthConfig)+ var ts *oauthutil.TokenSource+ f.client, ts, err = oauthutil.NewClientWithBaseClient(ctx, f.name, f.m, oauthConfig, getClient(ctx, &f.opt))if err != nil {return fmt.Errorf("failed to create oauth client: %w", err)}- f.rst = rest.NewClient(f.client).SetRoot(rootURL).SetErrorHandler(errorHandler)+ token, err := ts.Token()+ if err != nil {+ return err+ }+ // parse user_id from oauth access token for later use+ if parts := strings.Split(token.AccessToken, "."); len(parts) > 1 {+ jsonStr, _ := base64.URLEncoding.DecodeString(parts[1] + "===")+ info := struct {+ UserID string `json:"sub,omitempty"`+ }{}+ if jsonErr := json.Unmarshal(jsonStr, &info); jsonErr == nil {+ f.opt.UserID = info.UserID+ }+ }+ f.rst = newPikpakClient(f.client, &f.opt).SetCaptchaTokener(ctx, f.m)f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))return nil}@@ -491,10 +548,19 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, errCanHaveEmptyDirectories: true, // can have empty directoriesNoMultiThreading: true, // can't have multiple threads downloading}).Fill(ctx, f)- f.deviceID = genDeviceID()++ // new device id if necessary+ if len(f.opt.DeviceID) != 32 {+ f.opt.DeviceID = genDeviceID()+ m.Set("device_id", f.opt.DeviceID)+ fs.Infof(nil, "Using new device id %q", f.opt.DeviceID)+ }if err := f.newClientWithPacer(ctx); err != nil {- return nil, err+ // re-authorize if necessary+ if strings.Contains(err.Error(), "invalid_grant") {+ return f, f.reAuthorize(ctx)+ }}return f, nil