"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
iclouddrive: fix ADP/PCS cookie acquisition for iCloud Drive
Yakov Till2dbad62a11db
diff --git a/backend/iclouddrive/api/client.go b/backend/iclouddrive/api/client.goindex 1bd96c918..feff7a745 100644--- a/backend/iclouddrive/api/client.go+++ b/backend/iclouddrive/api/client.go@@ -25,6 +25,13 @@ const (authEndpoint = "https://idmsa.apple.com/appleauth/auth")+// Webservice keys in AccountInfo.Webservices map+const (+ WsDrive = "drivews"+ WsDocs = "docws"+ WsPhotos = "ckdatabasews"+)+type sessionSave func(*Session)// Client defines the client configuration@@ -32,6 +39,7 @@ type Client struct {appleID stringpassword stringremoteName string // rclone remote name, used for cache namespacing+ pcsWSKey string // webservice key for PCS cookie scoping (e.g. WsDrive, WsPhotos)srv *rest.ClientSession *SessionsessionSaveCallback sessionSave@@ -41,11 +49,14 @@ type Client struct {}// New creates a new iCloud API client and initializes its HTTP session-func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave, remoteName string) (*Client, error) {+// pcsWSKey scopes PCS cookie acquisition to the caller's webservice (WsDrive, WsPhotos);+// empty string skips PCS entirely+func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave, remoteName string, pcsWSKey string) (*Client, error) {icloud := &Client{appleID: strings.ToLower(appleID), // Apple SRP requires lowercase in client-side proofpassword: password,remoteName: filepath.Base(remoteName),+ pcsWSKey: pcsWSKey,srv: rest.NewClient(fshttp.NewClient(context.Background())),Session: NewSession(),sessionSaveCallback: sessionSaveCallback,@@ -75,8 +86,8 @@ func (c *Client) DriveService() (*DriveService, error) {func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {resp, err = c.Session.Request(ctx, opts, request, response)if err != nil && resp != nil {- // try to reauth- if resp.StatusCode == 401 || resp.StatusCode == 421 {+ // 401/421 = session expired, 423 = missing PCS cookies (ADP)+ if resp.StatusCode == 401 || resp.StatusCode == 421 || resp.StatusCode == 423 {err = c.Authenticate(ctx)if err != nil {return nil, err@@ -95,6 +106,24 @@ func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, respofunc (c *Client) Authenticate(ctx context.Context) error {c.mu.Lock()defer c.mu.Unlock()++ if err := c.authenticateSession(ctx); err != nil {+ return err+ }++ // Ensure PCS cookies after any successful auth path (ADP accounts)+ acquired, err := c.Session.ensurePCSCookies(ctx, c.pcsWSKey)+ if err != nil {+ return err+ }+ if acquired && c.sessionSaveCallback != nil {+ c.sessionSaveCallback(c.Session)+ }+ return nil+}++// authenticateSession establishes a valid session via the cheapest available path+func (c *Client) authenticateSession(ctx context.Context) error {// Skip /validate round-trip when saved session has cookies + service endpoints// Native client behavior: use cached session, reauth lazily on 401/421if c.Session.Cookies != nil && len(c.Session.AccountInfo.Webservices) > 0 {diff --git a/backend/iclouddrive/api/drive.go b/backend/iclouddrive/api/drive.goindex fa98362f4..1fe9d907a 100644--- a/backend/iclouddrive/api/drive.go+++ b/backend/iclouddrive/api/drive.go@@ -33,7 +33,7 @@ type DriveService struct {// NewDriveService creates a new DriveService instance.func NewDriveService(icloud *Client) (*DriveService, error) {- return &DriveService{icloud: icloud, RootID: "FOLDER::com.apple.CloudDocs::root", endpoint: icloud.Session.AccountInfo.Webservices["drivews"].URL, docsEndpoint: icloud.Session.AccountInfo.Webservices["docws"].URL}, nil+ return &DriveService{icloud: icloud, RootID: "FOLDER::com.apple.CloudDocs::root", endpoint: icloud.Session.AccountInfo.Webservices[WsDrive].URL, docsEndpoint: icloud.Session.AccountInfo.Webservices[WsDocs].URL}, nil}// GetItemByDriveID retrieves a DriveItem by its Drive ID.diff --git a/backend/iclouddrive/api/photos.go b/backend/iclouddrive/api/photos.goindex c014b50ce..4c3e2457d 100644--- a/backend/iclouddrive/api/photos.go+++ b/backend/iclouddrive/api/photos.go@@ -511,7 +511,7 @@ func (album *Album) SetTestPhotoCache(cache map[string]*Photo) {// NewPhotosService creates a new PhotosService instancefunc NewPhotosService(ctx context.Context, client *Client, pacer *fs.Pacer, shouldRetry ShouldRetryFunc) (*PhotosService, error) {- service, exists := client.Session.AccountInfo.Webservices["ckdatabasews"]+ service, exists := client.Session.AccountInfo.Webservices[WsPhotos]if !exists || service.Status != "active" {return nil, fmt.Errorf("ckdatabasews service not available")}@@ -2311,7 +2311,7 @@ func (ps *PhotosService) requestWithReauth(ctx context.Context, makeOpts func()reauthDone := falsereturn ps.pacer.Call(func() (bool, error) {resp, err := ps.client.Session.Request(ctx, makeOpts(), data, response)- if !reauthDone && err != nil && resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 421) {+ if !reauthDone && err != nil && resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 421 || resp.StatusCode == 423) {reauthDone = trueif authErr := ps.client.Authenticate(ctx); authErr != nil {return false, authErrdiff --git a/backend/iclouddrive/api/session.go b/backend/iclouddrive/api/session.goindex 89e47a937..fa3d2d9f0 100644--- a/backend/iclouddrive/api/session.go+++ b/backend/iclouddrive/api/session.go@@ -495,7 +495,7 @@ func (s *Session) getSRPAuthHeaders() map[string]string {return headers}-// AuthWithToken authenticates the session+// AuthWithToken authenticates the session with the account login endpointfunc (s *Session) AuthWithToken(ctx context.Context) error {values := map[string]any{"accountCountryCode": s.AccountCountry,@@ -522,86 +522,110 @@ func (s *Session) AuthWithToken(ctx context.Context) error {fs.Debugf(nil, "iclouddrive: accountLogin response cookies: %v", cookieDebugSummaries(resp.Cookies()))fs.Debugf(nil, "iclouddrive: session cookie jar after accountLogin: %v", cookieJarDebugSummaries(s.Cookies))- // Acquire PCS cookies if Advanced Data Protection is enabled- if ws := s.AccountInfo.Webservices["ckdatabasews"]; ws != nil && ws.PcsRequired {- fs.Debugf(nil, "iclouddrive: ADP detected (pcsRequired=true)")- if s.hasPCSCookies() {- fs.Debugf(nil, "iclouddrive: PCS cookies already present, skipping acquisition")- } else {- if err := s.acquirePCSCookies(ctx); err != nil {- return err- }+ return nil+}++type pcsService struct {+ wsKey string+ appName string+ cookies []string+}++// pcsServices lists all services that need PCS cookies+// appName values from icloud.com JS (Build 2616/19)+var pcsServices = []pcsService{+ {WsPhotos, "photos", []string{"X-APPLE-WEBAUTH-PCS-Photos", "X-APPLE-WEBAUTH-PCS-Sharing"}},+ {WsDrive, "iclouddrive", []string{"X-APPLE-WEBAUTH-PCS-Documents"}},+}++func (s *Session) hasPCSCookiesFor(names []string) bool {+ for _, name := range names {+ if !slices.ContainsFunc(s.Cookies, func(c *http.Cookie) bool { return c.Name == name }) {+ return false}- } else {- fs.Debugf(nil, "iclouddrive: no ADP (pcsRequired=false)")}-- return nil+ return true}-// hasPCSCookies checks if the required PCS cookies for Photos are already present-func (s *Session) hasPCSCookies() bool {- var hasPhotos, hasSharing bool- for _, c := range s.Cookies {- switch c.Name {- case "X-APPLE-WEBAUTH-PCS-Photos":- hasPhotos = true- case "X-APPLE-WEBAUTH-PCS-Sharing":- hasSharing = true+// ensurePCSCookies checks whether the session needs PCS cookies for the given+// webservice and acquires them if missing, returning true if new cookies were acquired+func (s *Session) ensurePCSCookies(ctx context.Context, pcsWSKey string) (bool, error) {+ if pcsWSKey == "" {+ return false, nil+ }+ for _, pcs := range pcsServices {+ if pcs.wsKey != pcsWSKey {+ continue}+ ws := s.AccountInfo.Webservices[pcs.wsKey]+ if ws == nil || !ws.PcsRequired {+ return false, nil+ }+ if s.hasPCSCookiesFor(pcs.cookies) {+ return false, nil+ }+ fs.Debugf(nil, "iclouddrive: ADP detected, acquiring PCS cookies for %s", pcs.appName)+ if err := s.acquirePCSCookiesFor(ctx, pcs.appName, pcs.cookies); err != nil {+ return false, err+ }+ return true, nil}- return hasPhotos && hasSharing+ return false, nil}-// acquirePCSCookies requests PCS cookies for ADP-enabled accounts+// acquirePCSCookiesFor requests PCS cookies for a specific service on ADP-enabled accounts// May require user approval on a trusted device (polls every 10s, max 5 min)-func (s *Session) acquirePCSCookies(ctx context.Context) error {- fs.Logf(nil, "iclouddrive: Advanced Data Protection enabled, requesting PCS cookies")- const maxAttempts = 30 // 30 * 10s = 5 minutes max- for attempt := 0; attempt < maxAttempts; attempt++ {- fs.Debugf(nil, "iclouddrive: requestPCS outgoing cookies: %v", cookieJarDebugSummaries(s.Cookies))+func (s *Session) acquirePCSCookiesFor(ctx context.Context, appName string, cookies []string) error {+ fs.Logf(nil, "iclouddrive: Advanced Data Protection enabled, requesting PCS cookies for %s", appName)+ const maxAttempts = 30+ for range maxAttempts {+ fs.Debugf(nil, "iclouddrive: requestPCS(%s) outgoing cookies: %v", appName, cookieJarDebugSummaries(s.Cookies))values := map[string]any{- "appName": "photos",+ "appName": appName,"derivedFromUserAction": true,}body, err := IntoReader(values)if err != nil {- return fmt.Errorf("requestPCS: %w", err)+ return fmt.Errorf("requestPCS(%s): %w", appName, err)}opts := rest.Opts{Method: "POST",Path: "/requestPCS",ExtraHeaders: s.GetHeaders(map[string]string{}),RootURL: setupEndpoint,+ Body: body,}- opts.Body = bodyvar pcsResp struct {Status string `json:"status"`Message string `json:"message"`}resp, err := s.Request(ctx, opts, nil, &pcsResp)if err != nil {- return fmt.Errorf("requestPCS: %w", err)+ return fmt.Errorf("requestPCS(%s): %w", appName, err)}- fs.Debugf(nil, "iclouddrive: requestPCS response cookies: %v", cookieDebugSummaries(resp.Cookies()))- fs.Debugf(nil, "iclouddrive: requestPCS response: status=%q message=%q cookies=%d",- pcsResp.Status, pcsResp.Message, len(resp.Cookies()))+ fs.Debugf(nil, "iclouddrive: requestPCS(%s) response: status=%q message=%q cookies=%d %v",+ appName, pcsResp.Status, pcsResp.Message, len(resp.Cookies()), cookieDebugSummaries(resp.Cookies()))if pcsResp.Status == "success" {- if !s.hasPCSCookies() {- return fmt.Errorf("requestPCS: server returned success but PCS cookies missing")+ if !s.hasPCSCookiesFor(cookies) {+ var missing []string+ for _, name := range cookies {+ if !slices.ContainsFunc(s.Cookies, func(c *http.Cookie) bool { return c.Name == name }) {+ missing = append(missing, name)+ }+ }+ return fmt.Errorf("requestPCS(%s): server returned success but cookies still missing: %v", appName, missing)}- fs.Logf(nil, "iclouddrive: PCS cookies acquired")+ fs.Logf(nil, "iclouddrive: PCS cookies acquired for %s", appName)return nil}- // Device consent required - poll until approved- fs.Logf(nil, "iclouddrive: waiting for device approval for PCS (%s)", pcsResp.Message)+ fs.Logf(nil, "iclouddrive: waiting for device approval for PCS/%s (%s)", appName, pcsResp.Message)select {case <-ctx.Done():return ctx.Err()case <-time.After(10 * time.Second):}}- return fmt.Errorf("requestPCS: timed out waiting for device approval after 5 minutes")+ return fmt.Errorf("requestPCS(%s): timed out waiting for device approval after 5 minutes", appName)}// RequestPushNotification explicitly requests a push notification to trusted devicesdiff --git a/backend/iclouddrive/icloud.go b/backend/iclouddrive/icloud.goindex 40ee6c949..244ec4b89 100644--- a/backend/iclouddrive/icloud.go+++ b/backend/iclouddrive/icloud.go@@ -96,7 +96,7 @@ func resumeConfigClient(m configmap.Mapper, appleid, password, trustToken, clienif err != nil {return nil, nil, err}- icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil, "_config")+ icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil, "_config", "")if err != nil {return nil, nil, err}@@ -280,7 +280,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf// Force fresh SRP authentication - ignore stale trust token and cookies// so that reconnect always prompts for 2FAm.Set(configAuthSession, "")- icloud, err := api.New(appleid, password, "", clientID, nil, nil, "_config")+ icloud, err := api.New(appleid, password, "", clientID, nil, nil, "_config", "")if err != nil {return nil, err}@@ -410,7 +410,8 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf// newICloudClient parses options, authenticates, and returns a ready client// Shared between NewFs (Drive) and NewFsPhotos (Photos) to avoid duplication-func newICloudClient(ctx context.Context, name string, m configmap.Mapper) (*api.Client, *Options, error) {+// pcsWSKey scopes PCS cookie acquisition to the caller's service+func newICloudClient(ctx context.Context, name string, m configmap.Mapper, pcsWSKey string) (*api.Client, *Options, error) {opt := new(Options)err := configstruct.Set(m, opt)if err != nil {@@ -442,6 +443,7 @@ func newICloudClient(ctx context.Context, name string, m configmap.Mapper) (*apicookies,callback,name,+ pcsWSKey,)if err != nil {return nil, nil, errdiff --git a/backend/iclouddrive/iclouddrive.go b/backend/iclouddrive/iclouddrive.goindex 135ec75e9..90be7a712 100644--- a/backend/iclouddrive/iclouddrive.go+++ b/backend/iclouddrive/iclouddrive.go@@ -677,7 +677,7 @@ func retryResultUnknown(ctx context.Context, resp *http.Response, err error) (bo// NewFs constructs an Fs from the path, container:pathfunc NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {- icloud, opt, err := newICloudClient(ctx, name, m)+ icloud, opt, err := newICloudClient(ctx, name, m, api.WsDrive)if err != nil {return nil, err}diff --git a/backend/iclouddrive/icloudphotos.go b/backend/iclouddrive/icloudphotos.goindex 922333f98..b3d80462c 100644--- a/backend/iclouddrive/icloudphotos.go+++ b/backend/iclouddrive/icloudphotos.go@@ -59,7 +59,7 @@ type PhotosObject struct {// NewFsPhotos constructs an Fs for Photos from the path, container:pathfunc NewFsPhotos(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {- icloud, opt, err := newICloudClient(ctx, name, m)+ icloud, opt, err := newICloudClient(ctx, name, m, api.WsPhotos)if err != nil {return nil, err}