"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
iclouddrive: add read only iCloud Photos support and SRP authentication
Yakov Tilld0c469c3c05e
diff --git a/backend/iclouddrive/api/client.go b/backend/iclouddrive/api/client.goindex 350a4fc33..1bd96c918 100644--- a/backend/iclouddrive/api/client.go+++ b/backend/iclouddrive/api/client.go@@ -1,4 +1,4 @@-// Package api provides functionality for interacting with the iCloud API.+// Package api provides functionality for interacting with the iCloud APIpackage apiimport (@@ -8,16 +8,19 @@ import ("errors""fmt""net/http"+ "os"+ "path/filepath""strings"+ "sync""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/lib/rest")const (baseEndpoint = "https://www.icloud.com"- homeEndpoint = "https://www.icloud.com"setupEndpoint = "https://setup.icloud.com/setup/ws/1"authEndpoint = "https://idmsa.apple.com/appleauth/auth")@@ -28,26 +31,21 @@ type sessionSave func(*Session)type Client struct {appleID stringpassword string+ remoteName string // rclone remote name, used for cache namespacingsrv *rest.ClientSession *SessionsessionSaveCallback sessionSavedrive *DriveService+ mu sync.Mutex // protects drive and Authenticate}-// New creates a new Client instance with the provided Apple ID, password, trust token, cookies, and session save callback.-//-// Parameters:-// - appleID: the Apple ID of the user.-// - password: the password of the user.-// - trustToken: the trust token for the session.-// - clientID: the client id for the session.-// - cookies: the cookies for the session.-// - sessionSaveCallback: the callback function to save the session.-func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave) (*Client, error) {+// New creates a new iCloud API client and initializes its HTTP session+func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave, remoteName string) (*Client, error) {icloud := &Client{- appleID: appleID,+ appleID: strings.ToLower(appleID), // Apple SRP requires lowercase in client-side proofpassword: password,+ remoteName: filepath.Base(remoteName),srv: rest.NewClient(fshttp.NewClient(context.Background())),Session: NewSession(),sessionSaveCallback: sessionSaveCallback,@@ -59,10 +57,12 @@ func New(appleID, password, trustToken string, clientID string, cookies []*http.return icloud, nil}-// DriveService returns the DriveService instance associated with the Client.+// DriveService returns the DriveService instance, creating it on first callfunc (c *Client) DriveService() (*DriveService, error) {- var err error+ c.mu.Lock()+ defer c.mu.Unlock()if c.drive == nil {+ var err errorc.drive, err = NewDriveService(c)if err != nil {return nil, err@@ -71,11 +71,7 @@ func (c *Client) DriveService() (*DriveService, error) {return c.drive, nil}-// Request makes a request and retries it if the session is invalid.-//-// This function is the main entry point for making requests to the iCloud-// API. If the initial request returns a 401 (Unauthorized), it will try to-// reauthenticate and retry the request.+// Request makes a request to the iCloud API, re-authenticating on 401/421func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {resp, err = c.Session.Request(ctx, opts, request, response)if err != nil && resp != nil {@@ -89,55 +85,93 @@ func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, respoif c.Session.Requires2FA() {return nil, errors.New("trust token expired, please reauth")}- return c.RequestNoReAuth(ctx, opts, request, response)+ return c.Session.Request(ctx, opts, request, response)}}return resp, err}-// RequestNoReAuth makes a request without re-authenticating.-//-// This function is useful when you have a session that is already-// authenticated, but you need to make a request without triggering-// a re-authentication.-func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {- // Make the request without re-authenticating- resp, err = c.Session.Request(ctx, opts, request, response)- return resp, err-}--// Authenticate authenticates the client with the iCloud API.+// Authenticate authenticates the client, reusing existing session if validfunc (c *Client) Authenticate(ctx context.Context) error {+ c.mu.Lock()+ defer c.mu.Unlock()+ // Skip /validate round-trip when saved session has cookies + service endpoints+ // Native client behavior: use cached session, reauth lazily on 401/421+ if c.Session.Cookies != nil && len(c.Session.AccountInfo.Webservices) > 0 {+ fs.Debugf(nil, "iclouddrive: reusing saved session")+ return nil+ }+ // Try loading cached service endpoints to avoid /validate round-trip (~5s)+ if c.Session.Cookies != nil && c.loadCachedWebservices() {+ fs.Debugf(nil, "iclouddrive: reusing session with cached endpoints")+ return nil+ }if c.Session.Cookies != nil {if err := c.Session.ValidateSession(ctx); err == nil {- fs.Debugf("icloud", "Valid session, no need to reauth")+ fs.Debugf(nil, "iclouddrive: valid session, no need to reauth")+ c.saveCachedWebservices()return nil}c.Session.Cookies = nil}- fs.Debugf("icloud", "Authenticating as %s\n", c.appleID)+ fs.Debugf(nil, "iclouddrive: authenticating")err := c.Session.SignIn(ctx, c.appleID, c.password)if err != nil {return err}- // If 2FA is required, don't try AuthWithToken yet — the caller- // must complete 2FA first, then call AuthWithToken.+ // If 2FA is required, skip AuthWithToken - caller must complete 2FA firstif c.Session.Requires2FA() {return nil}err = c.Session.AuthWithToken(ctx)- if err == nil && c.sessionSaveCallback != nil {- c.sessionSaveCallback(c.Session)+ if err == nil {+ c.saveCachedWebservices()+ if c.sessionSaveCallback != nil {+ c.sessionSaveCallback(c.Session)+ }}return err}-// SignIn signs in the client using the provided context and credentials.-func (c *Client) SignIn(ctx context.Context) error {- return c.Session.SignIn(ctx, c.appleID, c.password)+// loadCachedWebservices loads service endpoints from disk cache+func (c *Client) loadCachedWebservices() bool {+ data, err := os.ReadFile(filepath.Join(config.GetCacheDir(), cacheSubdir, c.remoteName, "webservices.json"))+ if err != nil {+ return false+ }+ var ws map[string]*webService+ if err := json.Unmarshal(data, &ws); err != nil {+ return false+ }+ if len(ws) == 0 {+ return false+ }+ c.Session.AccountInfo.Webservices = ws+ return true+}++// saveCachedWebservices persists service endpoints to disk+func (c *Client) saveCachedWebservices() {+ if len(c.Session.AccountInfo.Webservices) == 0 {+ return+ }+ saveJSONCache(filepath.Join(config.GetCacheDir(), cacheSubdir, c.remoteName), "webservices.json", c.Session.AccountInfo.Webservices)+}++// CacheDir returns the disk cache directory for this remote+func (c *Client) CacheDir() string {+ return filepath.Join(config.GetCacheDir(), cacheSubdir, c.remoteName)+}++// ClearCacheDir removes all disk cache files for a remote+func ClearCacheDir(remoteName string) {+ dir := filepath.Join(config.GetCacheDir(), cacheSubdir, filepath.Base(remoteName))+ if err := os.RemoveAll(dir); err != nil {+ fs.Debugf(nil, "iclouddrive: failed to clear cache: %v", err)+ }}// IntoReader marshals the provided values into a JSON encoded reader@@ -155,19 +189,19 @@ type RequestError struct {Text string}-// Error satisfy the error interface.+// Error satisfies the error interfacefunc (e *RequestError) Error() string {return fmt.Sprintf("%s: %s", e.Text, e.Status)}-func newRequestError(Status string, Text string) *RequestError {+func newRequestError(status string, text string) *RequestError {return &RequestError{- Status: strings.ToLower(Status),- Text: Text,+ Status: strings.ToLower(status),+ Text: text,}}-// newErr orf makes a new error from sprintf parameters.-func newRequestErrorf(Status string, Text string, Parameters ...any) *RequestError {- return newRequestError(strings.ToLower(Status), fmt.Sprintf(Text, Parameters...))+// newRequestErrorf makes a new error from sprintf parameters+func newRequestErrorf(status string, text string, params ...any) *RequestError {+ return newRequestError(status, fmt.Sprintf(text, params...))}diff --git a/backend/iclouddrive/api/photos.go b/backend/iclouddrive/api/photos.gonew file mode 100644index 000000000..06cdc226e--- /dev/null+++ b/backend/iclouddrive/api/photos.go@@ -0,0 +1,2341 @@+package api++import (+ "context"+ "crypto/sha256"+ "encoding/base64"+ "encoding/hex"+ "encoding/json"+ "errors"+ "fmt"+ "net/http"+ "os"+ "path"+ "path/filepath"+ "strings"+ "sync"+ "sync/atomic"++ "github.com/rclone/rclone/fs"++ "github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/rest"++ "golang.org/x/text/unicode/norm"+)++const (+ // CloudKit returns CPLMaster+CPLAsset pairs, so 200 records = 100 photos+ // Server caps at 200 regardless of requested value (tested: 500-5000 all return 200)+ photosQueryLimit = 200+ rootFolderRecord = "----Root-Folder----"+ projectRootFolderRecord = "----Project-Root-Folder----"+ indexingStateReady = "FINISHED"++ // cacheSubdir is the subdirectory under rclone's cache dir for all iCloud Photos state+ cacheSubdir = "iclouddrive-photos"++ // Album type constants from CloudKit CPLAlbum records+ albumTypeFolder = 3++ // CPLAsset.assetSubtype values+ subtypePanorama = 1+ subtypeSloMo = 100+ subtypeTimeLapse = 101++ // CPLAsset.assetSubtypeV2 values+ subtypeV2Live = 2+ subtypeV2Screenshot = 3++ // CPLAsset.adjustmentRenderType bitmask values+ adjustPortrait = 2+ adjustLongExposure = 4++ // CloudKit record type names+ recordTypeAlbum = "CPLAlbumByPositionLive"+ recordTypeCountIndex = "HyperionIndexCountLookup"++ // CloudKit endpoint area names+ areaPrivate = "private"+ areaShared = "shared"++ // Slo-mo adjustment type (metadata-only edit, no separate rendered resource)+ adjustSloMo = "com.apple.video.slomo"+)++// utiExtensions maps common Apple UTI descriptors to file extensions+// Used as fallback when filenameEnc is missing from a CPLMaster record+var utiExtensions = map[string]string{+ "public.jpeg": ".jpg",+ "public.png": ".png",+ "public.heic": ".heic",+ "public.heif": ".heif",+ "public.tiff": ".tiff",+ "public.mpeg-4": ".mp4",+ "com.apple.quicktime-movie": ".mov",+ "com.compuserve.gif": ".gif",+ "com.adobe.raw-image": ".dng",+ "com.canon.cr2-raw-image": ".cr2",+ "com.canon.cr3-raw-image": ".cr3",+ "com.nikon.raw-image": ".nef",+ "com.sony.arw-raw-image": ".arw",+ "public.avif": ".avif",+ "org.webmproject.webp": ".webp",+ "public.mpeg-2-video": ".m2v",+ "com.apple.m4v-video": ".m4v",+ "public.avi": ".avi",+ "public.mp3": ".mp3",+ "com.apple.m4a-audio": ".m4a",+ "public.image": ".heic",+ "com.fuji.raw-image": ".raf",+ "com.panasonic.rw2-raw-image": ".rw2",+ "com.olympus.raw-image": ".orf",+ "com.pentax.raw-image": ".pef",+ "com.nikon.nrw-raw-image": ".nrw",+ "com.canon.crw-raw-image": ".crw",+}++// extFromUTI returns the file extension for an Apple UTI string field,+// falling back to the provided default if the field is nil or unknown+func extFromUTI(field *ckStringField, fallback string) string {+ if field != nil {+ if mapped, ok := utiExtensions[field.Value]; ok {+ return mapped+ }+ }+ return fallback+}++// buildPhotoCache creates a filename-keyed map from a photo slice+func buildPhotoCache(photos []*Photo) map[string]*Photo {+ m := make(map[string]*Photo, len(photos))+ for _, p := range photos {+ if p.Filename != "" {+ m[p.Filename] = p+ }+ }+ return m+}++// saveJSONCache marshals v to JSON and writes it atomically to dir/filename+func saveJSONCache(dir, filename string, v any) {+ if err := os.MkdirAll(dir, 0700); err != nil {+ fs.Debugf(nil, "iclouddrive: failed to create cache dir: %v", err)+ return+ }+ data, err := json.Marshal(v)+ if err != nil {+ fs.Debugf(nil, "iclouddrive: failed to marshal cache %s: %v", filename, err)+ return+ }+ if err := atomicWriteFile(filepath.Join(dir, filename), data); err != nil {+ fs.Debugf(nil, "iclouddrive: failed to write cache %s: %v", filename, err)+ }+}++// atomicWriteFile writes data to target via atomic temp+rename+func atomicWriteFile(target string, data []byte) error {+ tmp := target + ".tmp"+ if err := os.WriteFile(tmp, data, 0600); err != nil {+ return err+ }+ if err := os.Rename(tmp, target); err != nil {+ _ = os.Remove(tmp)+ return err+ }+ return nil+}++// ShouldRetryFunc classifies whether an HTTP response/error is retryable+type ShouldRetryFunc func(ctx context.Context, resp *http.Response, err error) (bool, error)++// PhotosService manages iCloud Photos API interactions+type PhotosService struct {+ client *Client+ endpoint string+ pacer *fs.Pacer+ shouldRetry ShouldRetryFunc+ mu sync.Mutex+ libraries map[string]*Library+}++type libraryDiscovery struct {+ libraries map[string]*Library+ refreshedAreas map[string]bool+}++// FlushCaches clears all cached libraries, albums, and photos+func (ps *PhotosService) FlushCaches() {+ ps.mu.Lock()+ defer ps.mu.Unlock()+ for _, lib := range ps.libraries {+ lib.deltaMu.Lock()+ lib.mu.Lock()+ lib.cacheValid.Store(false)+ lib.pendingDelta = nil+ lib.clearDiskCache()+ for _, album := range lib.albums {+ album.mu.Lock()+ album.photoCache = nil+ album.mu.Unlock()+ }+ lib.albums = make(map[string]*Album)+ lib.mu.Unlock()+ lib.deltaMu.Unlock()+ }+ ps.libraries = make(map[string]*Library)+ // Also remove the libraries cache file+ _ = os.Remove(filepath.Join(ps.client.CacheDir(), "libraries.json"))+}++// deltaPayload holds a buffered changes/zone response waiting to be applied+// after albums are populated+type deltaPayload struct {+ records []json.RawMessage+ syncToken string+ moreComing bool+}++// deltaContainsAlbumChanges checks if any record in the delta is a CPLAlbum+// (album create/rename/delete) requiring eager album cache invalidation+func deltaContainsAlbumChanges(records []json.RawMessage) bool {+ for _, raw := range records {+ var header struct {+ RecordType string `json:"recordType"`+ }+ if json.Unmarshal(raw, &header) == nil && header.RecordType == "CPLAlbum" {+ return true+ }+ }+ return false+}++// Library represents a photo library in a specific zone+type Library struct {+ service *PhotosService+ zoneID string+ area string // "private" or "shared" - determines API endpoint path+ ownerRecordName string // zone owner's _UUID for full zoneID in requests+ zoneType string // "REGULAR_CUSTOM_ZONE" for full zoneID in requests+ mu sync.Mutex // protects albums map+ albums map[string]*Album+ deltaMu sync.Mutex // serializes delta checks+apply; lock order: ps.mu before deltaMu+ cacheValid atomic.Bool // true = album/photo disk caches are loadable+ pendingDelta *deltaPayload // buffered delta waiting for albums to be populated, protected by deltaMu+ notifyToken string // separate changes/zone token for ChangeNotify polling (memory-only)+}++// zoneIDMap returns the full zoneID object for CloudKit API requests+func (lib *Library) zoneIDMap() map[string]any {+ m := map[string]any{"zoneName": lib.zoneID}+ if lib.ownerRecordName != "" {+ m["ownerRecordName"] = lib.ownerRecordName+ }+ if lib.zoneType != "" {+ m["zoneType"] = lib.zoneType+ }+ return m+}++// invalidateAlbumCache clears in-memory albums and removes disk cache+func (lib *Library) invalidateAlbumCache() {+ lib.mu.Lock()+ lib.albums = make(map[string]*Album)+ lib.mu.Unlock()+ _ = os.Remove(filepath.Join(lib.zoneCacheDir(), "albums.json"))+}++// bufferDelta stores a pending delta for later application, eagerly+// invalidating album cache if the delta contains CPLAlbum records+// Must be called under deltaMu+func (lib *Library) bufferDelta(records []json.RawMessage, syncToken string, moreComing bool) {+ lib.pendingDelta = &deltaPayload{records: records, syncToken: syncToken, moreComing: moreComing}+ lib.cacheValid.Store(true)+ fs.Debugf(nil, "iclouddrive photos: zone %s has pending changes, buffered for later application", lib.zoneID)+ if deltaContainsAlbumChanges(records) {+ lib.invalidateAlbumCache()+ fs.Debugf(nil, "iclouddrive photos: zone %s delta contains album changes, invalidated album cache", lib.zoneID)+ }+}++// request makes an API call routed to this library's area (private or shared)+func (lib *Library) request(ctx context.Context, endpoint string, data, response any) error {+ return lib.service.requestForArea(ctx, lib.area, endpoint, data, response)+}++func (lib *Library) isSharedLibrary() bool {+ return strings.HasPrefix(lib.zoneID, "SharedSync")+}++func isSharedAlbumIndexError(err error) bool {+ if err == nil {+ return false+ }+ msg := err.Error()+ return strings.Contains(msg, "HTTP error 400") &&+ strings.Contains(msg, "BAD_REQUEST") &&+ strings.Contains(msg, "Index has invalid data")+}++func isZoneNotFoundError(err error) bool {+ if err == nil {+ return false+ }+ msg := err.Error()+ return strings.Contains(msg, "HTTP error 404") &&+ strings.Contains(msg, "ZONE_NOT_FOUND") &&+ strings.Contains(msg, "Zone does not exist")+}++func (lib *Library) probeZoneExists(ctx context.Context) (bool, error) {+ query := map[string]any{+ "query": map[string]any{+ "recordType": "CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted",+ "filterBy": []map[string]any{+ {+ "fieldName": "startRank",+ "comparator": "EQUALS",+ "fieldValue": map[string]any{"type": "INT64", "value": 0},+ },+ {+ "fieldName": "direction",+ "comparator": "EQUALS",+ "fieldValue": map[string]any{"type": "STRING", "value": "ASCENDING"},+ },+ },+ },+ "resultsLimit": 1,+ "desiredKeys": []string{"masterRef"},+ "zoneID": lib.zoneIDMap(),+ }+ var response struct {+ Records []json.RawMessage `json:"records"`+ }+ if err := lib.request(ctx, "records/query", query, &response); err != nil {+ if isZoneNotFoundError(err) {+ return false, nil+ }+ return false, err+ }+ return true, nil+}++// newUserAlbum creates a standard user album with CPLContainerRelation query config+func (lib *Library) newUserAlbum(name, recordName string) *Album {+ return &Album{+ Name: name,+ ObjectType: fmt.Sprintf("CPLContainerRelationNotDeletedByAssetDate:%s", recordName),+ ListType: "CPLContainerRelationLiveByAssetDate",+ Direction: "ASCENDING",+ RecordName: recordName,+ lib: lib,+ Filters: []Filter{{+ FieldName: "parentId",+ Comparator: "EQUALS",+ FieldValue: map[string]string{"type": "STRING", "value": recordName},+ }},+ }+}++// Album represents a photo album with its metadata and query configuration+type Album struct {+ Name string `json:"name"`+ ObjectType string `json:"objectType"`+ ListType string `json:"listType"`+ Direction string `json:"direction"`+ Filters []Filter `json:"filters,omitempty"`+ RecordName string `json:"recordName,omitempty"`+ IsFolder bool `json:"isFolder,omitempty"`+ Children map[string]*Album `json:"children,omitempty"`+ lib *Library `json:"-"`+ mu sync.Mutex `json:"-"`+ photoCache map[string]*Photo `json:"-"`+}++// Photo represents a photo or video with its metadata+type Photo struct {+ ID string+ Filename string+ Size int64+ AssetDate int64 // Unix timestamp in milliseconds+ AddedDate int64+ Width int+ Height int+ IsFavorite bool+ IsHidden bool+ SmartAlbums []string // smart album names this photo belongs to (for delta sync routing)+ ResourceKey string // CloudKit field name for download (default: resOriginalRes)+}++// companion creates a derivative Photo entry sharing metadata with the parent+// Used for Live Photo MOV, edited versions, and RAW alternatives+func (p *Photo) companion(id, filename, resourceKey string, size int64) *Photo {+ return &Photo{+ ID: id,+ Filename: filename,+ Size: size,+ AssetDate: p.AssetDate,+ AddedDate: p.AddedDate,+ IsFavorite: p.IsFavorite,+ IsHidden: p.IsHidden,+ ResourceKey: resourceKey,+ SmartAlbums: p.SmartAlbums,+ }+}++// Filter represents a CloudKit query filter+type Filter struct {+ FieldName string `json:"fieldName"`+ Comparator string `json:"comparator"`+ FieldValue any `json:"fieldValue"`+}++// smartAlbumFilter defines a filter-based smart album generated from a short table+// Each entry maps: display name -> ObjectType suffix + filter tag+// All share ListType "CPLAssetAndMasterInSmartAlbumByAssetDate", direction ASCENDING,+// and a single smartAlbum EQUALS filter+type smartAlbumFilter struct {+ suffix string // appended to "CPLAssetInSmartAlbumByAssetDate:"+ tag string // smartAlbum filter value (e.g. "VIDEO", "FAVORITE")+}++// smartAlbumFilters is the data table for the 10 filter-based smart albums+var smartAlbumFilters = map[string]smartAlbumFilter{+ "Time-lapse": {suffix: "Timelapse", tag: "TIMELAPSE"},+ "Videos": {suffix: "Video", tag: "VIDEO"},+ "Slo-mo": {suffix: "Slomo", tag: "SLOMO"},+ "Favorites": {suffix: "Favorite", tag: "FAVORITE"},+ "Panoramas": {suffix: "Panorama", tag: "PANORAMA"},+ "Screenshots": {suffix: "Screenshot", tag: "SCREENSHOT"},+ "Live": {suffix: "Live", tag: "LIVE"},+ "Portrait": {suffix: "Depth", tag: "DEPTH"},+ "Long Exposure": {suffix: "Exposure", tag: "EXPOSURE"},+ "Animated": {suffix: "Animated", tag: "ANIMATED"},+ // SELFIE filter exists server-side but the index is never populated via web API -+ // selfie classification is on-device only (iOS reads LensModel EXIF for "front camera")+ // Apple's own icloud.com doesn't show it either - omitted to avoid an always-empty album+}++// SmartAlbums defines the built-in smart album types available in iCloud Photos+// 10 filter-based albums are generated from smartAlbumFilters; 4 special albums+// (All Photos, Bursts, Hidden, Recently Deleted) use unique recordTypes+var SmartAlbums = buildSmartAlbums()++func buildSmartAlbums() map[string]*Album {+ albums := map[string]*Album{+ "All Photos": {+ Name: "All Photos",+ ObjectType: "CPLAssetByAssetDateWithoutHiddenOrDeleted",+ ListType: "CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted",+ Direction: "ASCENDING",+ },+ "Bursts": {+ Name: "Bursts",+ ObjectType: "CPLAssetBurstStackAssetByAssetDate",+ ListType: "CPLBurstStackAssetAndMasterByAssetDate",+ Direction: "ASCENDING",+ },+ "Hidden": {+ Name: "Hidden",+ ObjectType: "CPLAssetHiddenByAssetDate",+ ListType: "CPLAssetAndMasterHiddenByAssetDate",+ Direction: "ASCENDING",+ },+ "Recently Deleted": {+ Name: "Recently Deleted",+ ObjectType: "CPLAssetDeletedByExpungedDate",+ ListType: "CPLAssetAndMasterDeletedByExpungedDate",+ Direction: "DESCENDING",+ },+ }+ for name, f := range smartAlbumFilters {+ albums[name] = &Album{+ Name: name,+ ObjectType: "CPLAssetInSmartAlbumByAssetDate:" + f.suffix,+ ListType: "CPLAssetAndMasterInSmartAlbumByAssetDate",+ Direction: "ASCENDING",+ Filters: []Filter{{+ FieldName: "smartAlbum",+ Comparator: "EQUALS",+ FieldValue: map[string]string{"type": "STRING", "value": f.tag},+ }},+ }+ }+ return albums+}++type errorRoundTripper struct{}++func (errorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) {+ return nil, errors.New("test photos service has no network transport")+}++// NewTestPhotosService creates a PhotosService with pre-populated libraries for testing+func NewTestPhotosService(libs map[string]map[string]*Album) *PhotosService {+ httpClient := &http.Client{Transport: errorRoundTripper{}}+ session := &Session{srv: rest.NewClient(httpClient)}+ ps := &PhotosService{+ client: &Client{+ remoteName: "_test",+ Session: session,+ },+ endpoint: "http://test.invalid/database/1/com.apple.photos.cloud/production",+ pacer: fs.NewPacer(context.Background(), pacer.NewDefault()),+ shouldRetry: func(ctx context.Context, resp *http.Response, err error) (bool, error) { return false, err },+ libraries: make(map[string]*Library),+ }+ for zoneName, albums := range libs {+ lib := &Library{+ service: ps,+ zoneID: zoneName,+ area: areaPrivate,+ albums: make(map[string]*Album),+ }+ lib.cacheValid.Store(true)+ for name, album := range albums {+ lib.restoreAlbumLinks(album)+ lib.albums[name] = album+ }+ ps.libraries[zoneName] = lib+ }+ return ps+}++// SetTestPhotoCache populates an album's photo cache for testing+func (album *Album) SetTestPhotoCache(cache map[string]*Photo) {+ album.mu.Lock()+ album.photoCache = cache+ album.mu.Unlock()+}++// NewPhotosService creates a new PhotosService instance+func NewPhotosService(ctx context.Context, client *Client, pacer *fs.Pacer, shouldRetry ShouldRetryFunc) (*PhotosService, error) {+ service, exists := client.Session.AccountInfo.Webservices["ckdatabasews"]+ if !exists || service.Status != "active" {+ return nil, fmt.Errorf("ckdatabasews service not available")+ }+ endpoint := fmt.Sprintf("%s/database/1/com.apple.photos.cloud/production", service.URL)++ ps := &PhotosService{+ client: client,+ endpoint: endpoint,+ pacer: pacer,+ shouldRetry: shouldRetry,+ libraries: make(map[string]*Library),+ }++ ps.checkIndexingState(ctx, "PrimarySync")++ return ps, nil+}++// GetLibraries returns all available photo libraries+func (ps *PhotosService) GetLibraries(ctx context.Context) (map[string]*Library, error) {+ ps.mu.Lock()+ defer ps.mu.Unlock()++ if len(ps.libraries) > 0 {+ if discovered, err := ps.discoverLibraries(ctx); err == nil {+ ps.libraries = mergeDiscoveredLibraries(ctx, ps.libraries, discovered)+ ps.saveCachedLibraries()+ fs.Debugf(nil, "iclouddrive photos: refreshed %d in-memory libraries from API", len(ps.libraries))+ } else {+ fs.Debugf(nil, "iclouddrive photos: in-memory library rediscovery failed, using cached state: %v", err)+ }+ return ps.libraries, nil+ }++ // Try loading cached zone names from disk+ if cached := ps.loadCachedLibraries(); cached != nil {+ ps.batchCheckForChanges(ctx, cached)+ ps.libraries = cached+ if discovered, err := ps.discoverLibraries(ctx); err == nil {+ ps.libraries = mergeDiscoveredLibraries(ctx, cached, discovered)+ ps.saveCachedLibraries()+ fs.Debugf(nil, "iclouddrive photos: refreshed %d libraries from API", len(ps.libraries))+ } else {+ fs.Debugf(nil, "iclouddrive photos: library rediscovery failed, using cached zones: %v", err)+ fs.Debugf(nil, "iclouddrive photos: %d libraries from cache", len(cached))+ }+ return ps.libraries, nil+ }++ discovered, err := ps.discoverLibraries(ctx)+ if err != nil {+ return nil, err+ }+ ps.libraries = discovered.libraries+ ps.saveCachedLibraries()+ return ps.libraries, nil+}++func (ps *PhotosService) discoverLibraries(ctx context.Context) (*libraryDiscovery, error) {+ result := &libraryDiscovery{+ libraries: make(map[string]*Library),+ refreshedAreas: make(map[string]bool),+ }++ // Discover zones from API - probe both private and shared databases+ // Private zones: owned by the current user (PrimarySync + owned SharedSync)+ // Shared zones: owned by another user (non-owner SharedSync)+ type zoneResponse struct {+ Zones []struct {+ ZoneID struct {+ ZoneName string `json:"zoneName"`+ OwnerRecordName string `json:"ownerRecordName"`+ ZoneType string `json:"zoneType"`+ } `json:"zoneID"`+ Deleted bool `json:"deleted,omitempty"`+ } `json:"zones"`+ }++ for _, area := range []string{areaPrivate, areaShared} {+ var response zoneResponse+ if err := ps.requestForArea(ctx, area, "changes/database", map[string]any{}, &response); err != nil {+ if area == areaShared {+ // Shared database may not exist for all accounts+ fs.Debugf(nil, "iclouddrive photos: shared zone discovery failed (expected if no shared library): %v", err)+ continue+ }+ return nil, fmt.Errorf("failed to discover zones: %w", err)+ }+ result.refreshedAreas[area] = true+ for _, zone := range response.Zones {+ if zone.Deleted {+ continue+ }+ name := zone.ZoneID.ZoneName+ // SharedSync-* found in private takes precedence over shared+ if _, exists := result.libraries[name]; exists {+ continue+ }+ result.libraries[name] = &Library{+ service: ps,+ zoneID: name,+ area: area,+ ownerRecordName: zone.ZoneID.OwnerRecordName,+ zoneType: zone.ZoneID.ZoneType,+ albums: make(map[string]*Album),+ }+ }+ }+ return result, nil+}++func mergeDiscoveredLibraries(ctx context.Context, existing map[string]*Library, discovered *libraryDiscovery) map[string]*Library {+ merged := make(map[string]*Library, len(existing)+len(discovered.libraries))+ for name, cached := range existing {+ if !discovered.refreshedAreas[cached.area] {+ if cached.area == areaShared {+ exists, err := cached.probeZoneExists(ctx)+ if err != nil {+ fs.Debugf(nil, "iclouddrive photos: shared zone probe failed for %q, keeping cached zone: %v", cached.zoneID, err)+ } else if !exists {+ fs.Debugf(nil, "iclouddrive photos: dropping cached shared zone %q after authoritative ZONE_NOT_FOUND probe", cached.zoneID)+ continue+ }+ }+ merged[name] = cached+ }+ }+ for name, fresh := range discovered.libraries {+ if cached, ok := existing[name]; ok {+ cached.service = fresh.service+ cached.area = fresh.area+ cached.ownerRecordName = fresh.ownerRecordName+ cached.zoneType = fresh.zoneType+ merged[name] = cached+ continue+ }+ merged[name] = fresh+ }+ return merged+}++// albumRecord represents a CPLAlbum record from CloudKit+type albumRecord struct {+ RecordName string `json:"recordName"`+ Fields struct {+ AlbumNameEnc *ckStringField `json:"albumNameEnc,omitempty"`+ AlbumType *ckIntField `json:"albumType,omitempty"`+ ParentID *ckStringField `json:"parentId,omitempty"`+ IsDeleted *ckBoolField `json:"isDeleted,omitempty"`+ } `json:"fields"`+}++// albumQueryResponse wraps a paginated list of album records+type albumQueryResponse struct {+ Records []albumRecord `json:"records"`+ ContinuationMarker string `json:"continuationMarker"`+}++// cachedLibraryEntry stores zone metadata for disk cache persistence+type cachedLibraryEntry struct {+ ZoneName string `json:"zoneName"`+ Area string `json:"area,omitempty"`+ OwnerRecordName string `json:"ownerRecordName,omitempty"`+ ZoneType string `json:"zoneType,omitempty"`+}++// loadCachedLibraries loads zone metadata from disk cache+func (ps *PhotosService) loadCachedLibraries() map[string]*Library {+ cacheFile := filepath.Join(ps.client.CacheDir(), "libraries.json")+ data, err := os.ReadFile(cacheFile)+ if err != nil {+ return nil+ }++ var entries []cachedLibraryEntry+ if err := json.Unmarshal(data, &entries); err != nil {+ return nil+ }++ libs := make(map[string]*Library, len(entries))+ for _, entry := range entries {+ area := entry.Area+ if area == "" {+ area = areaPrivate+ }+ libs[entry.ZoneName] = &Library{+ service: ps,+ zoneID: entry.ZoneName,+ area: area,+ ownerRecordName: entry.OwnerRecordName,+ zoneType: entry.ZoneType,+ albums: make(map[string]*Album),+ }+ }+ return libs+}++// saveCachedLibraries persists zone metadata to disk via atomic rename+func (ps *PhotosService) saveCachedLibraries() {+ var entries []cachedLibraryEntry+ for _, lib := range ps.libraries {+ entries = append(entries, cachedLibraryEntry{+ ZoneName: lib.zoneID,+ Area: lib.area,+ OwnerRecordName: lib.ownerRecordName,+ ZoneType: lib.zoneType,+ })+ }+ saveJSONCache(ps.client.CacheDir(), "libraries.json", entries)+}++// GetAlbums returns all albums for this library+func (lib *Library) GetAlbums(ctx context.Context) (map[string]*Album, error) {+ lib.mu.Lock()+ defer lib.mu.Unlock()++ if len(lib.albums) > 0 {+ return lib.albums, nil+ }++ // Try loading cached albums if zone is unchanged+ if lib.cacheValid.Load() {+ if cached := lib.loadCachedAlbums(); cached != nil {+ lib.albums = cached+ fs.Debugf(nil, "iclouddrive photos: %d albums from cache for zone %s", len(cached), lib.zoneID)+ return lib.albums, nil+ }+ }++ // Build albums into a local map first so that a transient user album+ // query failure leaves lib.albums empty (retried on next call) rather+ // than permanently caching the smart-album-only subset+ albums := make(map[string]*Album, len(SmartAlbums))++ // Add smart albums+ for name, template := range SmartAlbums {+ albums[name] = &Album{+ Name: template.Name,+ ObjectType: template.ObjectType,+ ListType: template.ListType,+ Direction: template.Direction,+ Filters: append([]Filter{}, template.Filters...),+ RecordName: template.RecordName,+ lib: lib,+ }+ }++ // Add user albums and folders (paginated - CloudKit caps at 200 per page)+ var allRecords []albumRecord+ var continuationMarker string++ for {+ query := map[string]any{+ "query": map[string]any{"recordType": recordTypeAlbum},+ "zoneID": lib.zoneIDMap(),+ "desiredKeys": []string{"albumNameEnc", "albumType", "parentId", "isDeleted"},+ }+ if continuationMarker != "" {+ query["continuationMarker"] = continuationMarker+ }++ var response albumQueryResponse++ if err := lib.request(ctx, "records/query", query, &response); err != nil {+ // SharedSync libraries return BAD_REQUEST / "Index has invalid data"+ // on CPLAlbumByPositionLive in live probes, so fall back to smart+ // albums there while surfacing PrimarySync failures directly+ fs.Debugf(nil, "iclouddrive photos: user album query failed for zone %q: %v", lib.zoneID, err)+ if lib.isSharedLibrary() && isSharedAlbumIndexError(err) {+ return albums, nil+ }+ return nil, fmt.Errorf("query user albums for zone %q: %w", lib.zoneID, err)+ }++ allRecords = append(allRecords, response.Records...)++ if response.ContinuationMarker == "" {+ break+ }+ continuationMarker = response.ContinuationMarker+ }++ for _, record := range allRecords {+ if record.Fields.AlbumNameEnc == nil ||+ record.RecordName == rootFolderRecord ||+ record.RecordName == projectRootFolderRecord ||+ (record.Fields.IsDeleted != nil && record.Fields.IsDeleted.Value) {+ continue+ }++ nameBytes, err := base64.StdEncoding.DecodeString(record.Fields.AlbumNameEnc.Value)+ if err != nil {+ fs.Debugf(nil, "iclouddrive photos: skipping album %q: base64 decode: %v", record.RecordName, err)+ continue+ }++ isFolder := record.Fields.AlbumType != nil && record.Fields.AlbumType.Value == albumTypeFolder+ albumName := norm.NFC.String(string(nameBytes))++ // User album with same name as a smart album - smart album has special+ // server-side query semantics that can't be replicated by the user album,+ // so we keep the smart album and skip the user album with a warning+ if _, isSmart := SmartAlbums[albumName]; isSmart && !isFolder {+ fs.Logf(nil, "iclouddrive photos: user album %q shadows smart album, using smart album", albumName)+ continue+ }++ if isFolder {+ folder := &Album{+ Name: albumName,+ RecordName: record.RecordName,+ lib: lib,+ IsFolder: true,+ Children: make(map[string]*Album),+ }+ if err := lib.fetchFolderChildren(ctx, folder); err != nil {+ return nil, fmt.Errorf("fetch children of folder %q: %w", albumName, err)+ }+ albums[albumName] = folder+ } else {+ albums[albumName] = lib.newUserAlbum(albumName, record.RecordName)+ }+ }++ lib.albums = albums+ lib.saveCachedAlbums()+ return lib.albums, nil+}++// loadCachedAlbums loads album metadata from disk cache+func (lib *Library) loadCachedAlbums() map[string]*Album {+ cacheFile := filepath.Join(lib.zoneCacheDir(), "albums.json")+ data, err := os.ReadFile(cacheFile)+ if err != nil {+ return nil+ }+ var albums map[string]*Album+ if err := json.Unmarshal(data, &albums); err != nil {+ return nil+ }+ // Restore runtime-only fields after deserialization+ for _, a := range albums {+ lib.restoreAlbumLinks(a)+ }+ return albums+}++// restoreAlbumLinks sets runtime-only fields (lib pointer) on an album+// and its children after deserialization from disk cache+func (lib *Library) restoreAlbumLinks(a *Album) {+ a.lib = lib+ for _, child := range a.Children {+ lib.restoreAlbumLinks(child)+ }+}++func flattenAlbumTree(dst []*Album, albums map[string]*Album) []*Album {+ for _, album := range albums {+ dst = append(dst, album)+ if album.IsFolder {+ dst = flattenAlbumTree(dst, album.Children)+ }+ }+ return dst+}++// saveCachedAlbums persists album metadata to disk via atomic rename+func (lib *Library) saveCachedAlbums() {+ saveJSONCache(lib.zoneCacheDir(), "albums.json", lib.albums)+}++// fetchFolderChildren queries child albums inside a folder by parentId+func (lib *Library) fetchFolderChildren(ctx context.Context, folder *Album) error {+ query := map[string]any{+ "query": map[string]any{+ "recordType": recordTypeAlbum,+ "filterBy": []map[string]any{{+ "fieldName": "parentId",+ "comparator": "EQUALS",+ "fieldValue": map[string]string{"type": "STRING", "value": folder.RecordName},+ }},+ },+ "zoneID": lib.zoneIDMap(),+ "desiredKeys": []string{"albumNameEnc", "albumType", "isDeleted"},+ }++ var continuationMarker string+ for {+ if continuationMarker != "" {+ query["continuationMarker"] = continuationMarker+ }++ var response albumQueryResponse++ if err := lib.request(ctx, "records/query", query, &response); err != nil {+ return err+ }++ for _, record := range response.Records {+ if record.Fields.AlbumNameEnc == nil ||+ (record.Fields.IsDeleted != nil && record.Fields.IsDeleted.Value) {+ continue+ }++ nameBytes, err := base64.StdEncoding.DecodeString(record.Fields.AlbumNameEnc.Value)+ if err != nil {+ fs.Debugf(nil, "iclouddrive photos: skipping child album %q: base64 decode: %v", record.RecordName, err)+ continue+ }++ childName := norm.NFC.String(string(nameBytes))+ isFolder := record.Fields.AlbumType != nil && record.Fields.AlbumType.Value == albumTypeFolder++ if isFolder {+ childFolder := &Album{+ Name: childName,+ RecordName: record.RecordName,+ lib: lib,+ IsFolder: true,+ Children: make(map[string]*Album),+ }+ if err := lib.fetchFolderChildren(ctx, childFolder); err != nil {+ return err+ }+ folder.Children[childName] = childFolder+ } else {+ folder.Children[childName] = lib.newUserAlbum(childName, record.RecordName)+ }+ }++ if response.ContinuationMarker == "" {+ break+ }+ continuationMarker = response.ContinuationMarker+ }++ return nil+}++// albumCacheKey returns a stable filename-safe key for an album's disk cache+func albumCacheKey(objectType string) string {+ h := sha256.Sum256([]byte(objectType))+ return hex.EncodeToString(h[:8])+}++// zoneCacheDir returns the disk cache directory for this zone+// Path follows rclone convention: <cacheDir>/<backend>/<remoteName>/<zone>/+func (lib *Library) zoneCacheDir() string {+ return filepath.Join(lib.service.client.CacheDir(), lib.zoneID)+}++// checkForChanges detects whether the zone has been modified since the last+// sync. If unchanged (0 records), sets cacheValid=true. If changed, buffers+// the first page as pendingDelta and still sets cacheValid=true (album disk+// cache is valid under the old token - delta hasn't been applied yet)+// The buffered delta is consumed later by applyPendingDelta when albums exist+func (lib *Library) checkForChanges(ctx context.Context) {+ lib.deltaMu.Lock()+ defer lib.deltaMu.Unlock()++ // Already have a buffered delta waiting to be applied+ if lib.pendingDelta != nil {+ return+ }++ token := lib.readSyncToken()+ if token == "" {+ return+ }++ var response changesZoneResponse+ if err := lib.request(ctx, "changes/zone", lib.changesZoneBody(token), &response); err != nil {+ fs.Debugf(nil, "iclouddrive photos: delta check failed for zone %s: %v", lib.zoneID, err)+ return+ }+ if len(response.Zones) == 0 {+ return+ }++ zone := response.Zones[0]++ if len(zone.Records) == 0 && !zone.MoreComing {+ // No changes - advance token, all caches valid+ fs.Debugf(nil, "iclouddrive photos: zone %s unchanged, using cached listings", lib.zoneID)+ lib.saveSyncToken(zone.SyncToken)+ lib.cacheValid.Store(true)+ return+ }++ // Buffer the delta for later application (after albums are populated)+ // Album disk cache is still valid under the old token+ lib.bufferDelta(zone.Records, zone.SyncToken, zone.MoreComing)+}++// zoneEntry pairs a changes/zone request body with its library for batched zone operations+type zoneEntry struct {+ zone map[string]any+ lib *Library+}++// flattenZoneEntries extracts the zone request bodies and builds a zoneID→Library lookup+func flattenZoneEntries(entries []zoneEntry) ([]map[string]any, map[string]*Library) {+ zones := make([]map[string]any, len(entries))+ libByZone := make(map[string]*Library, len(entries))+ for i, e := range entries {+ zones[i] = e.zone+ libByZone[e.lib.zoneID] = e.lib+ }+ return zones, libByZone+}++// batchCheckForChanges checks all zones for changes in a single API call+// Each zone with a syncToken gets checked; zones without tokens are skipped+func (ps *PhotosService) batchCheckForChanges(ctx context.Context, libs map[string]*Library) {+ // Group zones by area for separate API calls (private and shared use different endpoints)+ byArea := make(map[string][]zoneEntry)+ for _, lib := range libs {+ lib.deltaMu.Lock()+ hasPending := lib.pendingDelta != nil+ lib.deltaMu.Unlock()+ if hasPending {+ continue+ }+ token := lib.readSyncToken()+ if token == "" {+ continue+ }+ zone := map[string]any{+ "zoneID": lib.zoneIDMap(),+ "desiredKeys": changesZoneDesiredKeys,+ "syncToken": token,+ }+ byArea[lib.area] = append(byArea[lib.area], zoneEntry{zone: zone, lib: lib})+ }++ for area, entries := range byArea {+ zones, libByZone := flattenZoneEntries(entries)+ var response changesZoneResponse+ if err := ps.requestForArea(ctx, area, "changes/zone", map[string]any{"zones": zones}, &response); err != nil {+ fs.Debugf(nil, "iclouddrive photos: batch delta check (%s) failed: %v", area, err)+ continue+ }+ for _, zone := range response.Zones {+ lib := libByZone[zone.ZoneID.ZoneName]+ if lib == nil {+ continue+ }+ lib.deltaMu.Lock()+ if len(zone.Records) == 0 && !zone.MoreComing {+ fs.Debugf(nil, "iclouddrive photos: zone %s unchanged, using cached listings", lib.zoneID)+ lib.saveSyncToken(zone.SyncToken)+ lib.cacheValid.Store(true)+ } else {+ lib.bufferDelta(zone.Records, zone.SyncToken, zone.MoreComing)+ }+ lib.deltaMu.Unlock()+ }+ }+}++// PollForChanges checks all zones for changes in a single API call using+// separate notification tokens, returns zone names that have been modified+// Used by ChangeNotify - does not consume or interfere with listing delta sync+func (ps *PhotosService) PollForChanges(ctx context.Context) []string {+ ps.mu.Lock()+ libs := make([]*Library, 0, len(ps.libraries))+ for _, lib := range ps.libraries {+ libs = append(libs, lib)+ }+ ps.mu.Unlock()++ // Group zones by area for separate API calls+ byArea := make(map[string][]zoneEntry)+ for _, lib := range libs {+ lib.deltaMu.Lock()+ token := lib.notifyToken+ lib.deltaMu.Unlock()+ if token == "" {+ token = lib.readSyncToken()+ }+ if token == "" {+ continue+ }+ byArea[lib.area] = append(byArea[lib.area], zoneEntry{+ zone: map[string]any{+ "zoneID": lib.zoneIDMap(),+ "desiredKeys": changesZoneDesiredKeys,+ "syncToken": token,+ },+ lib: lib,+ })+ }++ var changed []string+ for area, entries := range byArea {+ zones, libByZone := flattenZoneEntries(entries)+ var response changesZoneResponse+ if err := ps.requestForArea(ctx, area, "changes/zone", map[string]any{"zones": zones}, &response); err != nil {+ continue+ }++ for _, zone := range response.Zones {+ lib := libByZone[zone.ZoneID.ZoneName]+ if lib == nil {+ continue+ }+ lib.deltaMu.Lock()+ lib.notifyToken = zone.SyncToken+ lib.deltaMu.Unlock()++ if len(zone.Records) == 0 && !zone.MoreComing {+ continue+ }++ // Drain remaining pages per zone individually+ for zone.MoreComing {+ lib.deltaMu.Lock()+ token := lib.notifyToken+ lib.deltaMu.Unlock()+ var next changesZoneResponse+ body := map[string]any{"zones": []map[string]any{{+ "zoneID": lib.zoneIDMap(),+ "desiredKeys": changesZoneDesiredKeys,+ "syncToken": token,+ }}}+ if err := lib.request(ctx, "changes/zone", body, &next); err != nil {+ break+ }+ if len(next.Zones) == 0 {+ break+ }+ lib.deltaMu.Lock()+ lib.notifyToken = next.Zones[0].SyncToken+ lib.deltaMu.Unlock()+ zone.MoreComing = next.Zones[0].MoreComing+ }++ changed = append(changed, lib.zoneID)+ fs.Debugf(nil, "iclouddrive photos: ChangeNotify detected changes in zone %s", lib.zoneID)+ }+ }+ return changed+}++// readSyncToken loads the sync token from disk+func (lib *Library) readSyncToken() string {+ data, err := os.ReadFile(filepath.Join(lib.zoneCacheDir(), "syncToken"))+ if err != nil {+ return ""+ }+ return strings.TrimSpace(string(data))+}++// changesZoneResponse is the structure returned by changes/zone+type changesZoneResponse struct {+ Zones []changesZoneResult `json:"zones"`+}++// changesZoneResult is a single zone entry within a changesZoneResponse+type changesZoneResult struct {+ ZoneID struct {+ ZoneName string `json:"zoneName"`+ } `json:"zoneID"`+ Records []json.RawMessage `json:"records"`+ SyncToken string `json:"syncToken"`+ MoreComing bool `json:"moreComing"`+}++// changesZoneDesiredKeys are the fields requested from changes/zone for+// delta sync classification into smart albums+var changesZoneDesiredKeys = []string{+ // CPLMaster fields+ "filenameEnc", "itemType", "resOriginalRes", "resOriginalWidth", "resOriginalHeight",+ "resOriginalFileType", "resOriginalVidComplRes",+ "resOriginalAltRes", "resOriginalAltFileType",+ // CPLAsset classification + metadata fields+ "masterRef", "assetDate", "addedDate",+ "isFavorite", "isHidden", "isDeleted", "assetSubtype", "assetSubtypeV2", "burstId",+ "adjustmentRenderType",+ "adjustmentType", "resJPEGFullRes", "resJPEGFullFileType",+ "resVidFullRes", "resVidFullFileType",+ // CPLContainerRelation field for user album membership invalidation+ "containerId",+}++// changesZoneBody builds the request body for a changes/zone call+func (lib *Library) changesZoneBody(syncToken string) map[string]any {+ zone := map[string]any{+ "zoneID": lib.zoneIDMap(),+ "desiredKeys": changesZoneDesiredKeys,+ }+ if syncToken != "" {+ zone["syncToken"] = syncToken+ }+ return map[string]any{"zones": []map[string]any{zone}}+}++// deltaParseResult holds the classified output of parseDeltaRecords+type deltaParseResult struct {+ deletedIDs map[string]bool+ newMasters map[string]*photoRecord+ newAssets map[string]*photoRecord+ changedAlbumRecords map[string]bool+ albumMetadataChanged bool+ hasAssetOnlyUpdates bool+}++func relationAlbumRecordFromRecordName(recordName string) (string, bool) {+ parts := strings.SplitN(recordName, "-IN-", 2)+ if len(parts) != 2 || parts[1] == "" {+ return "", false+ }+ return parts[1], true+}++// shouldInvalidate returns true if an album's cache should be invalidated+// rather than incrementally updated from this delta+func (r *deltaParseResult) shouldInvalidate(recordName string, isSmart bool) bool {+ return (!isSmart && r.changedAlbumRecords[recordName]) ||+ (isSmart && r.hasAssetOnlyUpdates)+}++// parseDeltaRecords classifies raw delta records from changes/zone into+// deletions, new masters/assets, album membership changes, and metadata flags+func parseDeltaRecords(records []json.RawMessage) *deltaParseResult {+ r := &deltaParseResult{+ deletedIDs: map[string]bool{},+ newMasters: map[string]*photoRecord{},+ newAssets: map[string]*photoRecord{},+ changedAlbumRecords: map[string]bool{},+ }++ for _, raw := range records {+ var header struct {+ RecordName string `json:"recordName"`+ RecordType string `json:"recordType"`+ Deleted bool `json:"deleted"`+ }+ if err := json.Unmarshal(raw, &header); err != nil {+ continue+ }+ if header.Deleted && header.RecordType == "" {+ if albumRecord, ok := relationAlbumRecordFromRecordName(header.RecordName); ok {+ r.changedAlbumRecords[albumRecord] = true+ continue+ }+ }+ switch header.RecordType {+ case "CPLMaster":+ if header.Deleted {+ r.deletedIDs[header.RecordName] = true+ } else {+ var rec photoRecord+ if err := json.Unmarshal(raw, &rec); err == nil {+ r.newMasters[rec.RecordName] = &rec+ }+ }+ case "CPLAsset":+ if header.Deleted {+ // Asset deletion - mark both the asset itself and its master for removal+ // The asset ID is needed because edited entries use asset.RecordName as ID+ // (not master), so filtering by master alone leaves ghost -edited entries+ r.deletedIDs[header.RecordName] = true+ var rec photoRecord+ if err := json.Unmarshal(raw, &rec); err == nil && rec.Fields.MasterRef != nil {+ r.deletedIDs[rec.Fields.MasterRef.Value.RecordName] = true+ }+ } else {+ var rec photoRecord+ if err := json.Unmarshal(raw, &rec); err == nil && rec.Fields.MasterRef != nil {+ r.newAssets[rec.Fields.MasterRef.Value.RecordName] = &rec+ }+ }+ case "CPLAlbum":+ r.albumMetadataChanged = true+ case "CPLContainerRelation":+ var rel struct {+ Fields struct {+ ContainerID *struct {+ Value string `json:"value"`+ } `json:"containerId"`+ } `json:"fields"`+ }+ if err := json.Unmarshal(raw, &rel); err == nil {+ if rel.Fields.ContainerID != nil && rel.Fields.ContainerID.Value != "" {+ r.changedAlbumRecords[rel.Fields.ContainerID.Value] = true+ continue+ }+ }+ // Deleted relation records and some changes/zone entries omit fields,+ // the deterministic recordName still encodes the target album record+ if albumRecord, ok := relationAlbumRecordFromRecordName(header.RecordName); ok {+ r.changedAlbumRecords[albumRecord] = true+ } else if header.Deleted {+ fs.Debugf(nil, "iclouddrive photos: deleted CPLContainerRelation %q has unexpected recordName format", header.RecordName)+ }+ }+ }++ // Detect asset-only metadata updates (favorite/hide/soft-delete toggle)+ for masterID := range r.newAssets {+ if _, hasMaster := r.newMasters[masterID]; !hasMaster && !r.deletedIDs[masterID] {+ r.hasAssetOnlyUpdates = true+ break+ }+ }++ return r+}++// applyPendingDelta consumes a buffered delta and applies it to album caches+// Called from GetPhotos after albums are guaranteed populated+// Returns true if cache is current (no pending delta, or delta applied successfully)+func (lib *Library) applyPendingDelta(ctx context.Context) bool {+ lib.deltaMu.Lock()+ defer lib.deltaMu.Unlock()++ pending := lib.pendingDelta+ if pending == nil {+ return true // nothing pending, cache is current+ }+ failPendingDelta := func() bool {+ lib.pendingDelta = nil+ lib.cacheValid.Store(false)+ return false+ }++ // Verify albums are populated - if not (e.g. eager album invalidation+ // cleared the map before GetAlbums ran), clear pendingDelta so+ // checkForChanges can re-detect it on the next call. SyncToken was not+ // advanced so the same delta will be returned by changes/zone+ lib.mu.Lock()+ hasAlbums := len(lib.albums) > 0+ lib.mu.Unlock()+ if !hasAlbums {+ lib.pendingDelta = nil+ return false+ }++ // Collect all delta records (first page from buffer + remaining pages from API)+ allRecords := pending.records+ syncToken := pending.syncToken+ moreComing := pending.moreComing+ for moreComing {+ var response changesZoneResponse+ if err := lib.request(ctx, "changes/zone", lib.changesZoneBody(syncToken), &response); err != nil {+ return failPendingDelta()+ }+ if len(response.Zones) == 0 {+ return failPendingDelta()+ }+ allRecords = append(allRecords, response.Zones[0].Records...)+ syncToken = response.Zones[0].SyncToken+ moreComing = response.Zones[0].MoreComing+ }++ result := parseDeltaRecords(allRecords)++ // Build new Photo entries from delta master+asset pairs+ var addedPhotos []*Photo+ for masterID, master := range result.newMasters {+ built := buildPhotos(master, result.newAssets[masterID])+ addedPhotos = append(addedPhotos, built...)+ }++ fs.Debugf(nil, "iclouddrive photos: zone %s delta: %d deleted, %d added, %d album membership changes from %d records",+ lib.zoneID, len(result.deletedIDs), len(addedPhotos), len(result.changedAlbumRecords), len(allRecords))++ // Apply delta to each album's disk cache+ // Pre-resolve cache dir from lib to avoid re-acquiring ps.mu+ // under deltaMu (lock ordering: deltaMu must not precede ps.mu)+ cacheDir := lib.zoneCacheDir()+ lib.mu.Lock()+ albums := make([]*Album, 0, len(lib.albums))+ for _, album := range flattenAlbumTree(nil, lib.albums) {+ if album.ObjectType != "" {+ albums = append(albums, album)+ }+ }+ lib.mu.Unlock()++ for _, album := range albums {+ cached, ok := album.loadDiskCacheFrom(cacheDir)+ if !ok {+ continue+ }++ _, isSmart := SmartAlbums[album.Name]++ // Skip albums that will be invalidated below (avoids stale-data+ // window for concurrent readers and wasted disk I/O)+ if result.shouldInvalidate(album.RecordName, isSmart) {+ continue+ }++ // Remove deleted/changed entries+ filtered := make([]*Photo, 0, len(cached))+ for _, p := range cached {+ if !result.deletedIDs[p.ID] {+ filtered = append(filtered, p)+ }+ }++ // Route new photos to smart albums based on classifySmartAlbums()+ if isSmart {+ for _, p := range addedPhotos {+ for _, sa := range p.SmartAlbums {+ if sa == album.Name {+ filtered = append(filtered, p)+ break+ }+ }+ }+ }++ album.saveDiskCacheTo(cacheDir, filtered)++ // Update in-memory cache if populated+ album.mu.Lock()+ if album.photoCache != nil {+ // Deep copy before dedup so shared *Photo pointers across albums+ // don't get cross-contaminated by filename suffix mutations+ deduped := make([]*Photo, len(filtered))+ for i, p := range filtered {+ cp := *p+ deduped[i] = &cp+ }+ deduplicateFilenames(deduped)+ album.photoCache = buildPhotoCache(deduped)+ }+ album.mu.Unlock()+ }++ // Invalidate caches for albums affected by membership or metadata changes+ if len(result.changedAlbumRecords) > 0 || result.hasAssetOnlyUpdates {+ lib.mu.Lock()+ var invalidated []*Album+ for _, album := range flattenAlbumTree(nil, lib.albums) {+ _, isSmart := SmartAlbums[album.Name]+ if result.shouldInvalidate(album.RecordName, isSmart) {+ invalidated = append(invalidated, album)+ }+ }+ lib.mu.Unlock()+ for _, album := range invalidated {+ album.mu.Lock()+ album.photoCache = nil+ album.mu.Unlock()+ if album.ObjectType != "" {+ _ = os.Remove(filepath.Join(cacheDir, albumCacheKey(album.ObjectType)+".json"))+ }+ fs.Debugf(nil, "iclouddrive photos: invalidated album %q cache", album.Name)+ }+ }++ // Album metadata change (CPLAlbum created/renamed/deleted) - clear album+ // list so GetAlbums re-fetches from API on next call+ if result.albumMetadataChanged {+ lib.invalidateAlbumCache()+ fs.Debugf(nil, "iclouddrive photos: zone %s album metadata changed, will re-fetch album list", lib.zoneID)+ }++ lib.pendingDelta = nil+ lib.saveSyncToken(syncToken)+ return true+}++// saveSyncToken persists the zone sync token to disk via atomic rename+func (lib *Library) saveSyncToken(token string) {+ dir := lib.zoneCacheDir()+ if err := os.MkdirAll(dir, 0700); err != nil {+ fs.Debugf(nil, "iclouddrive photos: failed to create cache dir: %v", err)+ return+ }+ if err := atomicWriteFile(filepath.Join(dir, "syncToken"), []byte(token)); err != nil {+ fs.Debugf(nil, "iclouddrive photos: failed to write sync token: %v", err)+ }+}++// clearDiskCache removes all cached album data and sync token for this zone+func (lib *Library) clearDiskCache() {+ dir := lib.zoneCacheDir()+ entries, err := os.ReadDir(dir)+ if err != nil {+ return+ }+ for _, e := range entries {+ _ = os.Remove(filepath.Join(dir, e.Name()))+ }+}++// request makes an API call routed through the album's library area+func (album *Album) request(ctx context.Context, endpoint string, data, response any) error {+ if album.lib != nil {+ return album.lib.request(ctx, endpoint, data, response)+ }+ return fmt.Errorf("album %q has no library", album.Name)+}++// zoneIDMap returns the full zoneID for this album's zone+func (album *Album) zoneIDMap() map[string]any {+ if album.lib == nil {+ return map[string]any{}+ }+ return album.lib.zoneIDMap()+}++// loadDiskCacheFrom loads cached photo data from a specific cache directory+func (album *Album) loadDiskCacheFrom(cacheDir string) ([]*Photo, bool) {+ cacheFile := filepath.Join(cacheDir, albumCacheKey(album.ObjectType)+".json")+ data, err := os.ReadFile(cacheFile)+ if err != nil {+ return nil, false+ }+ var photos []*Photo+ if err := json.Unmarshal(data, &photos); err != nil {+ return nil, false+ }+ return photos, true+}++// saveDiskCache persists photo data to disk for delta sync via atomic rename+func (album *Album) saveDiskCache(photos []*Photo) {+ if album.ObjectType == "" || album.lib == nil {+ return+ }+ album.saveDiskCacheTo(album.lib.zoneCacheDir(), photos)+}++// saveDiskCacheTo persists photo data to a specific cache directory+func (album *Album) saveDiskCacheTo(dir string, photos []*Photo) {+ saveJSONCache(dir, albumCacheKey(album.ObjectType)+".json", photos)+}++// countQuery builds a HyperionIndexCountLookup query for a single object type+func countQuery(objectType string, zoneID map[string]any) map[string]any {+ return map[string]any{+ "resultsLimit": 1,+ "query": map[string]any{+ "filterBy": map[string]any{+ "fieldName": "indexCountID",+ "fieldValue": map[string]any{"type": "STRING_LIST", "value": []string{objectType}},+ "comparator": "IN",+ },+ "recordType": recordTypeCountIndex,+ },+ "zoneWide": true,+ "zoneID": zoneID,+ }+}++// countBatchResponse is the response shape for batched count queries+type countBatchResponse struct {+ Batch []struct {+ Records []struct {+ Fields struct {+ ItemCount struct {+ Value int64 `json:"value"`+ } `json:"itemCount"`+ } `json:"fields"`+ } `json:"records"`+ } `json:"batch"`+}++// toCounts maps an ordered list of names to their counts from the batch response+func (r *countBatchResponse) toCounts(names []string) map[string]int64 {+ counts := make(map[string]int64, len(names))+ for i, name := range names {+ if i < len(r.Batch) && len(r.Batch[i].Records) > 0 {+ counts[name] = r.Batch[i].Records[0].Fields.ItemCount.Value+ }+ }+ return counts+}++// GetLibraryAlbumCounts returns the album count for each library in a single+// batched request, keyed by zone name (e.g. "PrimarySync")+func (ps *PhotosService) GetLibraryAlbumCounts(ctx context.Context) (map[string]int64, error) {+ ps.mu.Lock()+ type libEntry struct {+ name string+ lib *Library+ }+ byArea := make(map[string][]libEntry)+ for name, lib := range ps.libraries {+ byArea[lib.area] = append(byArea[lib.area], libEntry{name: name, lib: lib})+ }+ ps.mu.Unlock()++ counts := make(map[string]int64)+ for area, entries := range byArea {+ var batch []map[string]any+ var order []string+ for _, e := range entries {+ order = append(order, e.name)+ batch = append(batch, countQuery(recordTypeAlbum, e.lib.zoneIDMap()))+ }+ var response countBatchResponse+ if err := ps.requestForArea(ctx, area, "internal/records/query/batch", map[string]any{"batch": batch}, &response); err != nil {+ return nil, fmt.Errorf("failed to get library album counts: %w", err)+ }+ for k, v := range response.toCounts(order) {+ counts[k] = v+ }+ }+ return counts, nil+}++// CloudKit field types for record deserialization+type ckStringField struct {+ Value string `json:"value"`+ Type string `json:"type,omitempty"` // present on filenameEnc (ENCRYPTED_BYTES vs STRING)+}++type ckIntField struct {+ Value int `json:"value"`+}++type ckTimestampField struct {+ Value int64 `json:"value"`+}++type ckResourceField struct {+ Value struct {+ Size int64 `json:"size"`+ DownloadURL string `json:"downloadURL"`+ } `json:"value"`+}++type ckBoolField struct {+ Value bool `json:"value"`+}++type ckReferenceField struct {+ Value struct {+ RecordName string `json:"recordName"`+ } `json:"value"`+}++// photoRecord represents a CloudKit record (CPLAsset or CPLMaster)+type photoRecord struct {+ RecordName string `json:"recordName"`+ RecordType string `json:"recordType"`+ Fields struct {+ FilenameEnc *ckStringField `json:"filenameEnc,omitempty"`+ ItemType *ckStringField `json:"itemType,omitempty"`+ ResOriginalRes *ckResourceField `json:"resOriginalRes,omitempty"`+ ResOriginalWidth *ckIntField `json:"resOriginalWidth,omitempty"`+ ResOriginalHeight *ckIntField `json:"resOriginalHeight,omitempty"`+ ResOriginalFileType *ckStringField `json:"resOriginalFileType,omitempty"`+ ResOriginalVidComplRes *ckResourceField `json:"resOriginalVidComplRes,omitempty"`+ ResOriginalAltRes *ckResourceField `json:"resOriginalAltRes,omitempty"`+ ResOriginalAltFileType *ckStringField `json:"resOriginalAltFileType,omitempty"`+ MasterRef *ckReferenceField `json:"masterRef,omitempty"`+ AssetDate *ckTimestampField `json:"assetDate,omitempty"`+ AddedDate *ckTimestampField `json:"addedDate,omitempty"`+ IsFavorite *ckIntField `json:"isFavorite,omitempty"`+ IsHidden *ckIntField `json:"isHidden,omitempty"`+ AssetSubtype *ckIntField `json:"assetSubtype,omitempty"`+ AssetSubtypeV2 *ckIntField `json:"assetSubtypeV2,omitempty"`+ BurstID *ckStringField `json:"burstId,omitempty"`+ AdjustmentRenderType *ckIntField `json:"adjustmentRenderType,omitempty"`+ IsDeleted *ckIntField `json:"isDeleted,omitempty"`+ AdjustmentType *ckStringField `json:"adjustmentType,omitempty"`+ ResJPEGFullRes *ckResourceField `json:"resJPEGFullRes,omitempty"`+ ResJPEGFullFileType *ckStringField `json:"resJPEGFullFileType,omitempty"`+ ResVidFullRes *ckResourceField `json:"resVidFullRes,omitempty"`+ ResVidFullFileType *ckStringField `json:"resVidFullFileType,omitempty"`+ } `json:"fields"`+}++// classifySmartAlbums determines which smart albums a photo belongs to+// based on CPLMaster and CPLAsset fields from CloudKit+func classifySmartAlbums(master *photoRecord, asset *photoRecord) []string {+ isVideo := false+ if master.Fields.ResOriginalFileType != nil {+ uti := master.Fields.ResOriginalFileType.Value+ isVideo = uti == "public.mpeg-4" || uti == "com.apple.quicktime-movie"+ }++ var subtype, subtypeV2, favorite, hidden, deleted int+ if asset != nil {+ if asset.Fields.AssetSubtype != nil {+ subtype = asset.Fields.AssetSubtype.Value+ }+ if asset.Fields.AssetSubtypeV2 != nil {+ subtypeV2 = asset.Fields.AssetSubtypeV2.Value+ }+ if asset.Fields.IsFavorite != nil {+ favorite = asset.Fields.IsFavorite.Value+ }+ if asset.Fields.IsHidden != nil {+ hidden = asset.Fields.IsHidden.Value+ }+ if asset.Fields.IsDeleted != nil {+ deleted = asset.Fields.IsDeleted.Value+ }+ }++ // Soft-deleted assets (isDeleted=1) go to Recently Deleted only+ if deleted == 1 {+ return []string{"Recently Deleted"}+ }++ var albums []string+ if hidden == 0 {+ albums = append(albums, "All Photos")+ }+ if hidden == 1 {+ albums = append(albums, "Hidden")+ }+ if favorite == 1 {+ albums = append(albums, "Favorites")+ }+ if isVideo && subtype == 0 {+ albums = append(albums, "Videos")+ }+ if subtype == subtypeSloMo {+ albums = append(albums, "Slo-mo")+ }+ if subtype == subtypeTimeLapse {+ albums = append(albums, "Time-lapse")+ }+ if subtype == subtypePanorama {+ albums = append(albums, "Panoramas")+ }+ if subtypeV2 == subtypeV2Live {+ albums = append(albums, "Live")+ }+ if subtypeV2 == subtypeV2Screenshot {+ albums = append(albums, "Screenshots")+ }+ if asset != nil && asset.Fields.BurstID != nil && asset.Fields.BurstID.Value != "" {+ albums = append(albums, "Bursts")+ }+ // adjustmentRenderType is a bitmask: PORTRAIT=2, LONG_EXPOSURE=4+ if asset != nil && asset.Fields.AdjustmentRenderType != nil {+ art := asset.Fields.AdjustmentRenderType.Value+ if art&adjustPortrait != 0 {+ albums = append(albums, "Portrait")+ }+ if art&adjustLongExposure != 0 {+ albums = append(albums, "Long Exposure")+ }+ }+ // Animated (GIFs) detected by file type on master record+ if master.Fields.ResOriginalFileType != nil && master.Fields.ResOriginalFileType.Value == "com.compuserve.gif" {+ albums = append(albums, "Animated")+ }+ // Selfies: no reliable field available from delta records - server query handles it+ return albums+}++// deduplicateFilenames renames ALL photos with colliding filenames by appending+// the full masterID (CloudKit recordName) before the extension. Every duplicate+// gets the suffix so filenames are stable when photos are added or removed+// Unique filenames are untouched. Collision-free by construction since CloudKit+// recordNames are unique. Same pattern as googlephotos which embeds the full+// media item ID ({55+ chars}) in every filename+func deduplicateFilenames(photos []*Photo) {+ counts := make(map[string]int, len(photos))+ for _, p := range photos {+ counts[p.Filename]+++ }+ for _, p := range photos {+ if counts[p.Filename] <= 1 {+ continue+ }+ ext := path.Ext(p.Filename)+ base := strings.TrimSuffix(p.Filename, ext)+ p.Filename = base + "_" + p.ID + ext+ }+}++// buildPhotos creates Photo entries from a CPLMaster record and its paired CPLAsset+// Returns 1-2 entries: the photo itself, plus a .MOV companion for Live Photos+func buildPhotos(master *photoRecord, asset *photoRecord) []*Photo {+ photo := &Photo{ID: master.RecordName}++ if master.Fields.FilenameEnc != nil {+ if master.Fields.FilenameEnc.Type == "STRING" {+ photo.Filename = norm.NFC.String(master.Fields.FilenameEnc.Value)+ } else if decoded, err := base64.StdEncoding.DecodeString(master.Fields.FilenameEnc.Value); err == nil {+ photo.Filename = norm.NFC.String(string(decoded))+ }+ }+ // Fallback: synthesize filename from recordName + itemType UTI when filenameEnc is missing+ if photo.Filename == "" && master.Fields.ItemType != nil {+ if ext, ok := utiExtensions[master.Fields.ItemType.Value]; ok {+ photo.Filename = master.RecordName + ext+ }+ }++ if master.Fields.ResOriginalRes != nil {+ photo.Size = master.Fields.ResOriginalRes.Value.Size+ }++ if master.Fields.ResOriginalWidth != nil {+ photo.Width = master.Fields.ResOriginalWidth.Value+ }+ if master.Fields.ResOriginalHeight != nil {+ photo.Height = master.Fields.ResOriginalHeight.Value+ }++ var liveVideoSize int64+ var hasLiveVideo bool+ if master.Fields.ResOriginalVidComplRes != nil && master.Fields.ResOriginalVidComplRes.Value.DownloadURL != "" {+ liveVideoSize = master.Fields.ResOriginalVidComplRes.Value.Size+ hasLiveVideo = true+ }++ if asset != nil {+ if asset.Fields.AssetDate != nil {+ photo.AssetDate = asset.Fields.AssetDate.Value+ }+ if asset.Fields.AddedDate != nil {+ photo.AddedDate = asset.Fields.AddedDate.Value+ }+ photo.IsFavorite = asset.Fields.IsFavorite != nil && asset.Fields.IsFavorite.Value == 1+ photo.IsHidden = asset.Fields.IsHidden != nil && asset.Fields.IsHidden.Value == 1+ }++ photo.SmartAlbums = classifySmartAlbums(master, asset)++ hasDownloadURL := master.Fields.ResOriginalRes != nil && master.Fields.ResOriginalRes.Value.DownloadURL != ""+ if !hasDownloadURL || photo.Filename == "" {+ return nil+ }++ photo.ResourceKey = "resOriginalRes"+ result := []*Photo{photo}++ ext := path.Ext(photo.Filename)+ stem := strings.TrimSuffix(photo.Filename, ext)++ if hasLiveVideo {+ result = append(result, photo.companion(photo.ID, stem+".MOV", "resOriginalVidComplRes", liveVideoSize))+ }++ // Edited photo version (Photos.app adjustments)+ // Slo-mo edits are metadata-only (playback speed) with no separate rendered resource+ if asset != nil && asset.Fields.AdjustmentType != nil &&+ asset.Fields.AdjustmentType.Value != "" &&+ asset.Fields.AdjustmentType.Value != adjustSloMo {+ if asset.Fields.ResJPEGFullRes != nil && asset.Fields.ResJPEGFullRes.Value.DownloadURL != "" {+ editExt := extFromUTI(asset.Fields.ResJPEGFullFileType, ext)+ result = append(result, photo.companion(asset.RecordName, stem+"-edited"+editExt, "resJPEGFullRes", asset.Fields.ResJPEGFullRes.Value.Size))+ } else if asset.Fields.ResVidFullRes != nil && asset.Fields.ResVidFullRes.Value.DownloadURL != "" {+ editExt := extFromUTI(asset.Fields.ResVidFullFileType, ext)+ result = append(result, photo.companion(asset.RecordName, stem+"-edited"+editExt, "resVidFullRes", asset.Fields.ResVidFullRes.Value.Size))+ }+ }++ // RAW alternative (RAW+JPEG pairs where both originals are stored)+ if master.Fields.ResOriginalAltRes != nil && master.Fields.ResOriginalAltRes.Value.DownloadURL != "" {+ altExt := extFromUTI(master.Fields.ResOriginalAltFileType, ext)+ altFilename := stem + altExt+ if strings.EqualFold(altFilename, photo.Filename) {+ altFilename = stem + "-alt" + altExt+ }+ alt := photo.companion(master.RecordName, altFilename, "resOriginalAltRes", master.Fields.ResOriginalAltRes.Value.Size)+ alt.Width = photo.Width // same sensor capture, same dimensions+ alt.Height = photo.Height // same sensor capture, same dimensions+ result = append(result, alt)+ }++ return result+}++// photosDesiredKeys are the fields requested for photo listing+var photosDesiredKeys = []string{+ "resOriginalRes", "resOriginalVidComplRes", "resOriginalFileType",+ "resOriginalWidth", "resOriginalHeight",+ "resOriginalAltRes", "resOriginalAltFileType",+ "filenameEnc", "itemType", "assetDate", "addedDate", "masterRef",+ "isFavorite", "isHidden", "isDeleted",+ "assetSubtype", "assetSubtypeV2", "burstId", "adjustmentRenderType",+ "adjustmentType", "resJPEGFullRes", "resJPEGFullFileType",+ "resVidFullRes", "resVidFullFileType",+}++// fetchPhotoCount returns the photo count for this album via HyperionIndexCountLookup+func (album *Album) fetchPhotoCount(ctx context.Context) (int64, error) {+ if album.ObjectType == "" {+ return 0, nil+ }+ var response struct {+ Records []struct {+ Fields struct {+ ItemCount struct {+ Value int64 `json:"value"`+ } `json:"itemCount"`+ } `json:"fields"`+ } `json:"records"`+ }+ if err := album.request(ctx, "records/query", countQuery(album.ObjectType, album.zoneIDMap()), &response); err != nil {+ return 0, err+ }+ if len(response.Records) > 0 {+ return response.Records[0].Fields.ItemCount.Value, nil+ }+ return 0, nil+}++// parsePhotoRecords extracts Photo entries from a batch of CloudKit records+func parsePhotoRecords(records []photoRecord) []*Photo {+ if len(records) == 0 {+ return nil+ }+ half := len(records)/2 + 1+ assetMap := make(map[string]*photoRecord, half)+ masters := make([]*photoRecord, 0, half)+ for i := range records {+ record := &records[i]+ switch record.RecordType {+ case "CPLAsset":+ if record.Fields.MasterRef != nil {+ assetMap[record.Fields.MasterRef.Value.RecordName] = record+ }+ case "CPLMaster":+ masters = append(masters, record)+ }+ }+ photos := make([]*Photo, 0, len(masters))+ for _, master := range masters {+ built := buildPhotos(master, assetMap[master.RecordName])+ photos = append(photos, built...)+ }+ return photos+}++// buildPartitionQuery constructs a CloudKit records/query body for a single+// startRank EQUALS partition, including album-specific direction and filters+func (album *Album) buildPartitionQuery(startRank int) map[string]any {+ filters := []map[string]any{+ {+ "fieldName": "startRank",+ "comparator": "EQUALS",+ "fieldValue": map[string]any{"type": "INT64", "value": startRank},+ },+ {+ "fieldName": "direction",+ "fieldValue": map[string]any{"type": "STRING", "value": album.Direction},+ "comparator": "EQUALS",+ },+ }+ for _, filter := range album.Filters {+ filters = append(filters, map[string]any{+ "fieldName": filter.FieldName,+ "comparator": filter.Comparator,+ "fieldValue": filter.FieldValue,+ })+ }+ return map[string]any{+ "query": map[string]any{+ "filterBy": filters,+ "recordType": album.ListType,+ },+ "resultsLimit": photosQueryLimit,+ "desiredKeys": photosDesiredKeys,+ "zoneID": album.zoneIDMap(),+ }+}++// fetchPhotosParallel fetches all photos using parallel startRank partitions+// Each partition is one API call with startRank EQUALS, no continuationMarker+// Stride = photosQueryLimit/2 photos per partition (200 records = 100 photos)+func (album *Album) fetchPhotosParallel(ctx context.Context, totalPhotos int64) ([]*Photo, string, error) {+ stride := photosQueryLimit / 2 // 100 photos per partition+ numPartitions := int((totalPhotos + int64(stride) - 1) / int64(stride))+ workers := fs.GetConfig(ctx).Checkers+ if workers < 1 {+ workers = 8+ }++ fs.Logf(nil, "iclouddrive photos: parallel cold listing %d photos in %d partitions (%d workers)",+ totalPhotos, numPartitions, workers)++ type partitionResult struct {+ photos []*Photo+ syncToken string+ recordCount int // raw record count to detect full pages+ err error+ }++ // Cancel all remaining goroutines on first error+ ctx, cancel := context.WithCancel(ctx)+ defer cancel()++ results := make([]partitionResult, numPartitions)+ sem := make(chan struct{}, workers)+ var wg sync.WaitGroup++ for i := 0; i < numPartitions; i++ {+ wg.Add(1)+ go func(idx int) {+ defer wg.Done()+ select {+ case sem <- struct{}{}:+ case <-ctx.Done():+ results[idx] = partitionResult{err: ctx.Err()}+ return+ }+ defer func() { <-sem }()++ startRank := idx * stride+ query := album.buildPartitionQuery(startRank)++ var response struct {+ Records []photoRecord `json:"records"`+ SyncToken string `json:"syncToken"`+ }+ if err := album.request(ctx, "records/query", query, &response); err != nil {+ cancel() // stop remaining partitions+ results[idx] = partitionResult{err: fmt.Errorf("partition %d (rank=%d): %w", idx, startRank, err)}+ return+ }+ results[idx] = partitionResult{+ photos: parsePhotoRecords(response.Records),+ syncToken: response.SyncToken,+ recordCount: len(response.Records),+ }+ }(i)+ }+ wg.Wait()++ // Merge results in order+ var allPhotos []*Photo+ var lastSyncToken string+ var lastRecordCount int+ for _, r := range results {+ if r.err != nil {+ return nil, "", r.err+ }+ allPhotos = append(allPhotos, r.photos...)+ if r.syncToken != "" {+ lastSyncToken = r.syncToken+ }+ lastRecordCount = r.recordCount+ }++ // Completeness: keep fetching until a partition returns fewer than+ // resultsLimit records (partial page = last page). This handles stale+ // counts AND count=0 (count query failed - discover all photos here)+ needsTailCheck := lastRecordCount >= photosQueryLimit || numPartitions == 0+ nextRank := numPartitions * stride+ for needsTailCheck {+ fs.Debugf(nil, "iclouddrive photos: fetching tail partition at rank=%d", nextRank)+ query := album.buildPartitionQuery(nextRank)+ var response struct {+ Records []photoRecord `json:"records"`+ SyncToken string `json:"syncToken"`+ }+ if err := album.request(ctx, "records/query", query, &response); err != nil {+ return nil, "", fmt.Errorf("tail partition (rank=%d): %w", nextRank, err)+ }+ lastRecordCount = len(response.Records)+ allPhotos = append(allPhotos, parsePhotoRecords(response.Records)...)+ if response.SyncToken != "" {+ lastSyncToken = response.SyncToken+ }+ nextRank += stride+ needsTailCheck = lastRecordCount >= photosQueryLimit+ }++ fs.Logf(nil, "iclouddrive photos: parallel fetch complete, %d photos", len(allPhotos))+ return allPhotos, lastSyncToken, nil+}++// GetPhotos retrieves photos from this album using parallel partitions with disk cache+func (album *Album) GetPhotos(ctx context.Context) ([]*Photo, error) {+ // No library configured - return pre-populated cache (test path)+ if album.lib == nil {+ album.mu.Lock()+ defer album.mu.Unlock()+ result := make([]*Photo, 0, len(album.photoCache))+ for _, p := range album.photoCache {+ result = append(result, p)+ }+ return result, nil+ }++ // Check for changes, apply any buffered delta, serve from cache+ if album.ObjectType != "" {+ album.lib.checkForChanges(ctx)+ if album.lib.applyPendingDelta(ctx) && album.lib.cacheValid.Load() {+ // Serve from in-memory cache if populated (avoids disk I/O + JSON parse + dedup)+ album.mu.Lock()+ if album.photoCache != nil {+ result := make([]*Photo, 0, len(album.photoCache))+ for _, p := range album.photoCache {+ result = append(result, p)+ }+ album.mu.Unlock()+ return result, nil+ }+ album.mu.Unlock()+ // Fall back to disk cache+ if cached, ok := album.loadDiskCacheFrom(album.lib.zoneCacheDir()); ok {+ deduplicateFilenames(cached)+ album.mu.Lock()+ album.photoCache = buildPhotoCache(cached)+ album.mu.Unlock()+ fs.Debugf(nil, "iclouddrive photos: %d items from cache for %q", len(cached), album.Name)+ return cached, nil+ }+ }+ }++ // Fetch photo count for parallel partition calculation+ // If count unavailable, use 0 - the tail-fetch loop handles completeness+ var count int64+ if album.ObjectType != "" {+ count, _ = album.fetchPhotoCount(ctx)+ }+ photos, lastSyncToken, err := album.fetchPhotosParallel(ctx, count)+ if err != nil {+ return nil, err+ }++ // Persist original filenames for delta sync (dedup is applied on read)+ album.saveDiskCache(photos)++ deduplicateFilenames(photos)++ // Populate filename cache for NewObject lookups+ album.mu.Lock()+ album.photoCache = buildPhotoCache(photos)+ album.mu.Unlock()+ if lastSyncToken != "" && album.lib != nil {+ album.lib.saveSyncToken(lastSyncToken)+ }++ return photos, nil+}++// GetPhotoByName looks up a photo by filename, using cache if available+// CloudKit has no filterBy on filename fields - the only queryable fields+// are rank, date, smartAlbum, etc. Apple's own icloud.com UI paginates+// the full album and indexes client-side. On cache miss we must enumerate+// the entire album via GetPhotos before lookup+func (album *Album) GetPhotoByName(ctx context.Context, filename string) (*Photo, error) {+ album.mu.Lock()+ if album.photoCache != nil {+ photo, exists := album.photoCache[filename]+ album.mu.Unlock()+ if exists {+ return photo, nil+ }+ return nil, fmt.Errorf("photo %q not found in album %q", filename, album.Name)+ }+ album.mu.Unlock()++ // Cache miss - fetch all photos to populate cache+ photos, err := album.GetPhotos(ctx)+ if err != nil {+ return nil, err+ }+ for _, photo := range photos {+ if photo.Filename == filename {+ return photo, nil+ }+ }+ return nil, fmt.Errorf("photo %q not found in album %q", filename, album.Name)+}++// GetAlbumCounts returns photo counts for all albums in a single batch request+func (lib *Library) GetAlbumCounts(ctx context.Context) (map[string]int64, error) {+ // Snapshot under lock to avoid racing with GetAlbums+ type albumEntry struct {+ name string+ objectType string+ }+ lib.mu.Lock()+ entries := make([]albumEntry, 0, len(lib.albums))+ for name, album := range lib.albums {+ if album.ObjectType == "" {+ continue // skip folders (albumType=3), they have no photo count+ }+ entries = append(entries, albumEntry{name: name, objectType: album.ObjectType})+ }+ lib.mu.Unlock()++ if len(entries) == 0 {+ return nil, nil+ }++ zoneIDAny := lib.zoneIDMap()++ var batch []map[string]any+ var albumOrder []string+ for _, entry := range entries {+ albumOrder = append(albumOrder, entry.name)+ batch = append(batch, countQuery(entry.objectType, zoneIDAny))+ }+ var response countBatchResponse+ if err := lib.request(ctx, "internal/records/query/batch", map[string]any{"batch": batch}, &response); err != nil {+ return nil, err+ }+ return response.toCounts(albumOrder), nil+}++// resolveZone returns the area and full zoneID for a zone name,+// using the library metadata if available or falling back to private+func (ps *PhotosService) resolveZone(zoneName string) (area string, zoneID map[string]any) {+ ps.mu.Lock()+ lib := ps.libraries[zoneName]+ ps.mu.Unlock()+ if lib != nil {+ return lib.area, lib.zoneIDMap()+ }+ return areaPrivate, map[string]any{"zoneName": zoneName}+}++// LookupDownloadURL fetches a fresh download URL for a record+// recordName is the CPLMaster or CPLAsset recordName depending on the resource+// resourceKey selects which resource to look up (e.g. "resOriginalRes",+// "resOriginalVidComplRes" for Live Photo video, "resJPEGFullRes" for edited)+func (ps *PhotosService) LookupDownloadURL(ctx context.Context, recordName, zone, resourceKey string) (string, error) {+ area, zoneID := ps.resolveZone(zone)++ query := map[string]any{+ "records": []map[string]any{+ {"recordName": recordName},+ },+ "zoneID": zoneID,+ }++ var response struct {+ Records []json.RawMessage `json:"records"`+ }++ if err := ps.requestForArea(ctx, area, "records/lookup", query, &response); err != nil {+ return "", fmt.Errorf("failed to look up record %q: %w", recordName, err)+ }++ if len(response.Records) == 0 {+ return "", fmt.Errorf("no records in lookup response for %q", recordName)+ }++ // Parse fields as raw JSON to extract the requested resource key+ var record struct {+ Fields map[string]json.RawMessage `json:"fields"`+ }+ if err := json.Unmarshal(response.Records[0], &record); err != nil {+ return "", fmt.Errorf("failed to parse lookup response: %w", err)+ }++ rawField, exists := record.Fields[resourceKey]+ if !exists {+ return "", fmt.Errorf("no %q field in record %q", resourceKey, recordName)+ }++ var res struct {+ Value struct {+ DownloadURL string `json:"downloadURL"`+ } `json:"value"`+ }+ if err := json.Unmarshal(rawField, &res); err != nil {+ return "", fmt.Errorf("failed to parse %q field: %w", resourceKey, err)+ }+ if res.Value.DownloadURL == "" {+ return "", fmt.Errorf("no download URL for %q in record %q", resourceKey, recordName)+ }++ return res.Value.DownloadURL, nil+}++// checkIndexingState warns if the iCloud Photo Library is still indexing+func (ps *PhotosService) checkIndexingState(ctx context.Context, zoneName string) {+ area, zoneID := ps.resolveZone(zoneName)++ query := map[string]any{+ "query": map[string]any{"recordType": "CheckIndexingState"},+ "zoneID": zoneID,+ }++ var response struct {+ Records []struct {+ Fields struct {+ State struct {+ Value string `json:"value"`+ } `json:"state"`+ } `json:"fields"`+ } `json:"records"`+ }++ if err := ps.requestForArea(ctx, area, "records/query", query, &response); err != nil {+ fs.Logf(nil, "iclouddrive photos: could not check indexing state: %v", err)+ return+ }++ if len(response.Records) == 0 || response.Records[0].Fields.State.Value != indexingStateReady {+ fs.Logf(nil, "iclouddrive photos: library is still indexing, listings may be incomplete")+ }+}++// requestWithReauth makes a CloudKit request with pacer retry and reauth on 401/421+func (ps *PhotosService) requestWithReauth(ctx context.Context, makeOpts func() rest.Opts, data, response any) error {+ reauthDone := false+ return ps.pacer.Call(func() (bool, error) {+ resp, err := ps.client.Session.Request(ctx, makeOpts(), data, response)+ if !reauthDone && err != nil && resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 421) {+ reauthDone = true+ if authErr := ps.client.Authenticate(ctx); authErr != nil {+ return false, authErr+ }+ if ps.client.Session.Requires2FA() {+ return false, errors.New("trust token expired, please reauth")+ }+ resp, err = ps.client.Session.Request(ctx, makeOpts(), data, response)+ }+ return ps.shouldRetry(ctx, resp, err)+ })+}++// requestForArea makes a request to the given area (private or shared) endpoint+func (ps *PhotosService) requestForArea(ctx context.Context, area, endpoint string, data, response any) error {+ rootURL := fmt.Sprintf("%s/%s/%s?remapEnums=true&getCurrentSyncToken=true", ps.endpoint, area, endpoint)++ return ps.requestWithReauth(ctx, func() rest.Opts {+ return rest.Opts{+ Method: "POST",+ RootURL: rootURL,+ ExtraHeaders: ps.client.Session.GetHeaders(map[string]string{"Content-Type": "text/plain"}), // text/plain matches icloud.com (CORS preflight bypass)+ }+ }, data, response)+}diff --git a/backend/iclouddrive/api/photos_test.go b/backend/iclouddrive/api/photos_test.gonew file mode 100644index 000000000..59ec73168--- /dev/null+++ b/backend/iclouddrive/api/photos_test.go@@ -0,0 +1,1285 @@+package api++import (+ "context"+ "encoding/base64"+ "encoding/json"+ "io"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "sync/atomic"+ "testing"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/rest"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func setTestCacheDir(t *testing.T) string {+ t.Helper()+ oldCacheDir := config.GetCacheDir()+ cacheDir := t.TempDir()+ require.NoError(t, config.SetCacheDir(cacheDir))+ t.Cleanup(func() {+ _ = config.SetCacheDir(oldCacheDir)+ })+ return cacheDir+}++func newHTTPTestPhotosService(t *testing.T, remoteName string, handler http.HandlerFunc) *PhotosService {+ t.Helper()+ server := httptest.NewServer(handler)+ t.Cleanup(server.Close)++ session := NewSession()+ session.srv = rest.NewClient(server.Client())++ return &PhotosService{+ client: &Client{+ remoteName: remoteName,+ Session: session,+ },+ endpoint: server.URL + "/database/1/com.apple.photos.cloud/production",+ pacer: fs.NewPacer(context.Background(), pacer.NewDefault()),+ shouldRetry: func(ctx context.Context, resp *http.Response, err error) (bool, error) { return false, err },+ libraries: make(map[string]*Library),+ }+}++func writeJSON(t *testing.T, w http.ResponseWriter, v any) {+ t.Helper()+ w.Header().Set("Content-Type", "application/json")+ if err := json.NewEncoder(w).Encode(v); err != nil {+ panic(err)+ }+}++func readJSONBody(r *http.Request, out any) error {+ data, err := io.ReadAll(r.Body)+ if err != nil {+ return err+ }+ return json.Unmarshal(data, out)+}++func testAlbumRecordJSON(name, recordName string, albumType int, parentID string) map[string]any {+ fields := map[string]any{+ "albumNameEnc": map[string]any{"value": base64.StdEncoding.EncodeToString([]byte(name))},+ }+ if albumType != 0 {+ fields["albumType"] = map[string]any{"value": albumType}+ }+ if parentID != "" {+ fields["parentId"] = map[string]any{"value": parentID}+ }+ return map[string]any{+ "recordName": recordName,+ "fields": fields,+ }+}++func newUserAlbumForTest(lib *Library, name, recordName string) *Album {+ album := lib.newUserAlbum(name, recordName)+ album.lib = lib+ return album+}++func TestGetPhotos_DoesNotServeStaleCacheOnPagedDeltaFailure(t *testing.T) {+ setTestCacheDir(t)+ ctx := context.Background()+ var changesZoneCalls atomic.Int32++ ps := newHTTPTestPhotosService(t, "paged-delta-failure", func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/database/1/com.apple.photos.cloud/production/private/changes/zone":+ if changesZoneCalls.Add(1) > 1 {+ http.Error(w, "paged delta failure", http.StatusInternalServerError)+ return+ }+ writeJSON(t, w, map[string]any{+ "zones": []map[string]any{{+ "zoneID": map[string]any{"zoneName": "PrimarySync"},+ "records": []map[string]any{{"recordName": "asset1-IN-album-record", "deleted": true}},+ "syncToken": "mid-token",+ "moreComing": true,+ }},+ })+ default:+ http.Error(w, "unexpected request", http.StatusInternalServerError)+ }+ })++ lib := &Library{service: ps, zoneID: "PrimarySync", area: areaPrivate, albums: map[string]*Album{}}+ album := newUserAlbumForTest(lib, "Exported", "album-record")+ lib.albums[album.Name] = album+ album.saveDiskCache([]*Photo{{ID: "stale-id", Filename: "stale.jpg"}})+ require.NoError(t, os.WriteFile(filepath.Join(lib.zoneCacheDir(), "syncToken"), []byte("old-token"), 0600))++ photos, err := album.GetPhotos(ctx)+ assert.Error(t, err)+ assert.Nil(t, photos)+ assert.Nil(t, lib.pendingDelta, "failed delta apply must not leave pendingDelta stuck forever")+}++func TestGetLibraries_RediscoversZonesWhenCacheExists(t *testing.T) {+ setTestCacheDir(t)+ ctx := context.Background()+ var changesDatabaseCalls atomic.Int32++ ps := newHTTPTestPhotosService(t, "library-rediscovery", func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == "/database/1/com.apple.photos.cloud/production/private/changes/database" {+ changesDatabaseCalls.Add(1)+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})+ return+ }+ if r.URL.Path == "/database/1/com.apple.photos.cloud/production/shared/changes/database" {+ changesDatabaseCalls.Add(1)+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "SharedSync-test", "ownerRecordName": "other", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})+ return+ }+ http.Error(w, "unexpected request", http.StatusInternalServerError)+ })++ saveJSONCache(ps.client.CacheDir(), "libraries.json", []cachedLibraryEntry{{ZoneName: "PrimarySync", Area: areaPrivate}})++ libs, err := ps.GetLibraries(ctx)+ require.NoError(t, err)+ assert.Contains(t, libs, "PrimarySync")+ assert.Contains(t, libs, "SharedSync-test")+ assert.GreaterOrEqual(t, changesDatabaseCalls.Load(), int32(2), "must rediscover zones even when cache exists")+}++func TestGetLibraries_RediscoveryPreservesBufferedDelta(t *testing.T) {+ setTestCacheDir(t)+ ctx := context.Background()++ ps := newHTTPTestPhotosService(t, "library-merge", func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/database/1/com.apple.photos.cloud/production/private/changes/zone":+ writeJSON(t, w, map[string]any{+ "zones": []map[string]any{{+ "zoneID": map[string]any{"zoneName": "PrimarySync"},+ "records": []map[string]any{{"recordName": "asset123-IN-album-uuid", "deleted": true}},+ "syncToken": "new-token",+ "moreComing": false,+ }},+ })+ case "/database/1/com.apple.photos.cloud/production/private/changes/database":+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})+ case "/database/1/com.apple.photos.cloud/production/shared/changes/database":+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "SharedSync-test", "ownerRecordName": "other", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})+ default:+ http.Error(w, "unexpected request", http.StatusInternalServerError)+ }+ })++ saveJSONCache(ps.client.CacheDir(), "libraries.json", []cachedLibraryEntry{{ZoneName: "PrimarySync", Area: areaPrivate}})+ require.NoError(t, os.MkdirAll(filepath.Join(ps.client.CacheDir(), "PrimarySync"), 0700))+ require.NoError(t, os.WriteFile(filepath.Join(ps.client.CacheDir(), "PrimarySync", "syncToken"), []byte("cached-token"), 0600))++ libs, err := ps.GetLibraries(ctx)+ require.NoError(t, err)+ primary := libs["PrimarySync"]+ require.NotNil(t, primary)+ require.NotNil(t, primary.pendingDelta, "rediscovery must not discard pending delta buffered from cached zone state")+ assert.Contains(t, libs, "SharedSync-test")+}++func TestGetLibraries_RefreshesInMemoryLibraries(t *testing.T) {+ setTestCacheDir(t)+ ctx := context.Background()+ var changesDatabaseCalls atomic.Int32++ ps := newHTTPTestPhotosService(t, "library-refresh-memory", func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/database/1/com.apple.photos.cloud/production/private/changes/database":+ changesDatabaseCalls.Add(1)+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})+ case "/database/1/com.apple.photos.cloud/production/shared/changes/database":+ changesDatabaseCalls.Add(1)+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "SharedSync-live", "ownerRecordName": "other", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})+ default:+ http.Error(w, "unexpected request", http.StatusInternalServerError)+ }+ })++ ps.libraries = map[string]*Library{+ "PrimarySync": {service: ps, zoneID: "PrimarySync", area: areaPrivate, albums: make(map[string]*Album)},+ }++ libs, err := ps.GetLibraries(ctx)+ require.NoError(t, err)+ assert.Contains(t, libs, "PrimarySync")+ assert.Contains(t, libs, "SharedSync-live")+ assert.GreaterOrEqual(t, changesDatabaseCalls.Load(), int32(2), "in-memory libraries must still rediscover zones")+}++func TestGetLibraries_KeepsCachedSharedZonesOnTransientSharedRediscoveryFailure(t *testing.T) {+ setTestCacheDir(t)+ ctx := context.Background()++ ps := newHTTPTestPhotosService(t, "library-shared-preserve", func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/database/1/com.apple.photos.cloud/production/private/changes/database":+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})+ case "/database/1/com.apple.photos.cloud/production/shared/changes/database":+ http.Error(w, "transient shared discovery failure", http.StatusInternalServerError)+ case "/database/1/com.apple.photos.cloud/production/shared/records/query":+ http.Error(w, "transient per-zone probe failure", http.StatusInternalServerError)+ default:+ http.Error(w, "unexpected request", http.StatusInternalServerError)+ }+ })++ saveJSONCache(ps.client.CacheDir(), "libraries.json", []cachedLibraryEntry{{ZoneName: "PrimarySync", Area: areaPrivate}, {ZoneName: "SharedSync-cached", Area: areaShared}})++ libs, err := ps.GetLibraries(ctx)+ require.NoError(t, err)+ assert.Contains(t, libs, "PrimarySync")+ assert.Contains(t, libs, "SharedSync-cached")++ data, err := os.ReadFile(filepath.Join(ps.client.CacheDir(), "libraries.json"))+ require.NoError(t, err)+ assert.Contains(t, string(data), "SharedSync-cached")+}++func TestGetLibraries_DropsCachedSharedZoneOnZoneNotFoundProbe(t *testing.T) {+ setTestCacheDir(t)+ ctx := context.Background()++ ps := newHTTPTestPhotosService(t, "library-shared-drop", func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/database/1/com.apple.photos.cloud/production/private/changes/database":+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})+ case "/database/1/com.apple.photos.cloud/production/shared/changes/database":+ http.Error(w, "transient shared discovery failure", http.StatusInternalServerError)+ case "/database/1/com.apple.photos.cloud/production/shared/records/query":+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusNotFound)+ _, _ = w.Write([]byte(`{+ "serverErrorCode": "ZONE_NOT_FOUND",+ "reason": "Zone does not exist",+ "errorClass": "ZONE_NOT_FOUND",+ "error": "ZONE_NOT_FOUND"+ }`))+ default:+ http.Error(w, "unexpected request", http.StatusInternalServerError)+ }+ })++ saveJSONCache(ps.client.CacheDir(), "libraries.json", []cachedLibraryEntry{{ZoneName: "PrimarySync", Area: areaPrivate}, {ZoneName: "SharedSync-cached", Area: areaShared, OwnerRecordName: "owner", ZoneType: "REGULAR_CUSTOM_ZONE"}})++ libs, err := ps.GetLibraries(ctx)+ require.NoError(t, err)+ assert.Contains(t, libs, "PrimarySync")+ assert.NotContains(t, libs, "SharedSync-cached")++ data, err := os.ReadFile(filepath.Join(ps.client.CacheDir(), "libraries.json"))+ require.NoError(t, err)+ assert.NotContains(t, string(data), "SharedSync-cached")+}++func TestGetAlbums_RetriesAfterTransientUserAlbumQueryFailure(t *testing.T) {+ setTestCacheDir(t)+ ctx := context.Background()+ var topQueries atomic.Int32++ ps := newHTTPTestPhotosService(t, "albums-retry-top", func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/database/1/com.apple.photos.cloud/production/private/records/query" {+ http.Error(w, "unexpected request", http.StatusInternalServerError)+ return+ }+ var body struct {+ Query struct {+ FilterBy []struct {+ FieldName string `json:"fieldName"`+ Comparator string `json:"comparator"`+ FieldValue struct {+ Value string `json:"value"`+ } `json:"fieldValue"`+ } `json:"filterBy"`+ } `json:"query"`+ }+ if err := readJSONBody(r, &body); err != nil {+ http.Error(w, err.Error(), http.StatusInternalServerError)+ return+ }+ if len(body.Query.FilterBy) != 0 {+ http.Error(w, "unexpected child query", http.StatusInternalServerError)+ return+ }+ if topQueries.Add(1) == 1 {+ http.Error(w, "transient top-level album failure", http.StatusInternalServerError)+ return+ }+ writeJSON(t, w, map[string]any{+ "records": []map[string]any{testAlbumRecordJSON("User Album", "user-record", 0, "")},+ })+ })++ lib := &Library{service: ps, zoneID: "PrimarySync", area: areaPrivate, albums: map[string]*Album{}}+ _, err := lib.GetAlbums(ctx)+ assert.Error(t, err, "private library failure should surface as error")+ albums, err := lib.GetAlbums(ctx)+ require.NoError(t, err)+ assert.Contains(t, albums, "User Album")+ assert.GreaterOrEqual(t, topQueries.Load(), int32(2), "second GetAlbums call must retry transient top-level failures")+}++func TestGetAlbums_SharedSyncQueryFailureReturnsSmartAlbumsOnly(t *testing.T) {+ setTestCacheDir(t)+ ctx := context.Background()++ ps := newHTTPTestPhotosService(t, "albums-shared-fallback", func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusBadRequest)+ _, _ = w.Write([]byte(`{+ "serverErrorCode": "BAD_REQUEST",+ "reason": "Index has invalid data",+ "errorClass": "BAD_REQUEST",+ "error": "BAD_REQUEST"+ }`))+ })++ lib := &Library{service: ps, zoneID: "SharedSync-test", area: areaPrivate, albums: map[string]*Album{}}+ albums, err := lib.GetAlbums(ctx)+ require.NoError(t, err)+ assert.Contains(t, albums, "All Photos")+ assert.NotContains(t, albums, "User Album")+ assert.Empty(t, lib.albums, "SharedSync fallback should not cache partial album results")+}++func TestGetAlbums_SharedSyncUnexpectedFailureReturnsError(t *testing.T) {+ setTestCacheDir(t)+ ctx := context.Background()++ ps := newHTTPTestPhotosService(t, "albums-shared-error", func(w http.ResponseWriter, r *http.Request) {+ http.Error(w, "unexpected shared failure", http.StatusInternalServerError)+ })++ lib := &Library{service: ps, zoneID: "SharedSync-test", area: areaPrivate, albums: map[string]*Album{}}+ _, err := lib.GetAlbums(ctx)+ assert.Error(t, err)+ assert.Empty(t, lib.albums, "SharedSync unexpected failures must not cache partial album results")+}++func TestGetAlbums_RetriesAfterTransientFolderChildQueryFailure(t *testing.T) {+ setTestCacheDir(t)+ ctx := context.Background()+ var childQueries atomic.Int32++ ps := newHTTPTestPhotosService(t, "albums-retry-folder", func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/database/1/com.apple.photos.cloud/production/private/records/query" {+ http.Error(w, "unexpected request", http.StatusInternalServerError)+ return+ }+ var body struct {+ Query struct {+ FilterBy []struct {+ FieldName string `json:"fieldName"`+ Comparator string `json:"comparator"`+ FieldValue struct {+ Value string `json:"value"`+ } `json:"fieldValue"`+ } `json:"filterBy"`+ } `json:"query"`+ }+ if err := readJSONBody(r, &body); err != nil {+ http.Error(w, err.Error(), http.StatusInternalServerError)+ return+ }+ parentID := ""+ for _, filter := range body.Query.FilterBy {+ if filter.FieldName == "parentId" {+ parentID = filter.FieldValue.Value+ }+ }+ switch parentID {+ case "":+ writeJSON(t, w, map[string]any{+ "records": []map[string]any{testAlbumRecordJSON("Folder", "folder-record", albumTypeFolder, "")},+ })+ case "folder-record":+ if childQueries.Add(1) == 1 {+ http.Error(w, "transient child query failure", http.StatusInternalServerError)+ return+ }+ writeJSON(t, w, map[string]any{+ "records": []map[string]any{testAlbumRecordJSON("Child", "child-record", 0, "folder-record")},+ })+ default:+ http.Error(w, "unexpected parentId", http.StatusInternalServerError)+ }+ })++ lib := &Library{service: ps, zoneID: "PrimarySync", area: areaPrivate, albums: map[string]*Album{}}+ _, _ = lib.GetAlbums(ctx)+ albums, err := lib.GetAlbums(ctx)+ require.NoError(t, err)+ folder, ok := albums["Folder"]+ require.True(t, ok)+ require.True(t, folder.IsFolder)+ assert.Contains(t, folder.Children, "Child")+ assert.GreaterOrEqual(t, childQueries.Load(), int32(2), "second GetAlbums call must retry transient child query failures")+}++func TestBuildPhotos(t *testing.T) {+ filename := base64.StdEncoding.EncodeToString([]byte("IMG_0001.JPG"))++ master := &photoRecord{+ RecordName: "master123",+ RecordType: "CPLMaster",+ }+ master.Fields.FilenameEnc = &ckStringField{Value: filename, Type: "ENCRYPTED_BYTES"}+ master.Fields.ResOriginalRes = &ckResourceField{Value: struct {+ Size int64 `json:"size"`+ DownloadURL string `json:"downloadURL"`+ }{Size: 1024, DownloadURL: "https://example.com/photo"}}+ master.Fields.ResOriginalWidth = &ckIntField{Value: 4032}+ master.Fields.ResOriginalHeight = &ckIntField{Value: 3024}++ asset := &photoRecord{}+ asset.Fields.AssetDate = &ckTimestampField{Value: 1700000000000}+ asset.Fields.AddedDate = &ckTimestampField{Value: 1700000001000}++ t.Run("basic photo", func(t *testing.T) {+ photos := buildPhotos(master, asset)+ require.Len(t, photos, 1)+ p := photos[0]+ assert.Equal(t, "master123", p.ID)+ assert.Equal(t, "IMG_0001.JPG", p.Filename)+ assert.Equal(t, int64(1024), p.Size)+ assert.Equal(t, 4032, p.Width)+ assert.Equal(t, 3024, p.Height)+ assert.Equal(t, int64(1700000000000), p.AssetDate)+ assert.Equal(t, int64(1700000001000), p.AddedDate)+ assert.Equal(t, "resOriginalRes", p.ResourceKey)+ })++ t.Run("live photo with MOV companion", func(t *testing.T) {+ liveMaster := *master // copy to avoid mutating shared fixture+ liveMaster.Fields.ResOriginalVidComplRes = &ckResourceField{Value: struct {+ Size int64 `json:"size"`+ DownloadURL string `json:"downloadURL"`+ }{Size: 2048, DownloadURL: "https://example.com/video"}}++ photos := buildPhotos(&liveMaster, asset)+ require.Len(t, photos, 2)++ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)+ assert.Equal(t, "resOriginalRes", photos[0].ResourceKey)++ assert.Equal(t, "IMG_0001.MOV", photos[1].Filename)+ assert.Equal(t, "resOriginalVidComplRes", photos[1].ResourceKey)+ assert.Equal(t, int64(2048), photos[1].Size)+ assert.Equal(t, "master123", photos[1].ID)+ })++ t.Run("no download URL returns nil", func(t *testing.T) {+ noURL := &photoRecord{RecordName: "no-url"}+ noURL.Fields.FilenameEnc = master.Fields.FilenameEnc+ photos := buildPhotos(noURL, nil)+ assert.Nil(t, photos)+ })++ t.Run("no filename returns nil", func(t *testing.T) {+ noName := &photoRecord{RecordName: "no-name"}+ noName.Fields.ResOriginalRes = master.Fields.ResOriginalRes+ photos := buildPhotos(noName, nil)+ assert.Nil(t, photos)+ })++ t.Run("nil asset is handled", func(t *testing.T) {+ photos := buildPhotos(master, nil)+ require.Len(t, photos, 1)+ assert.Equal(t, int64(0), photos[0].AssetDate)+ })++ t.Run("invalid base64 filename returns nil", func(t *testing.T) {+ bad := &photoRecord{RecordName: "bad-b64"}+ bad.Fields.FilenameEnc = &ckStringField{Value: "!!!not-base64!!!", Type: "ENCRYPTED_BYTES"}+ bad.Fields.ResOriginalRes = master.Fields.ResOriginalRes+ photos := buildPhotos(bad, nil)+ assert.Nil(t, photos)+ })++ t.Run("STRING type filenameEnc used as-is", func(t *testing.T) {+ rec := &photoRecord{RecordName: "str-name"}+ rec.Fields.FilenameEnc = &ckStringField{Value: "plain_photo.heic", Type: "STRING"}+ rec.Fields.ResOriginalRes = master.Fields.ResOriginalRes+ photos := buildPhotos(rec, nil)+ require.Len(t, photos, 1)+ assert.Equal(t, "plain_photo.heic", photos[0].Filename)+ })++ t.Run("NFD filename normalized to NFC", func(t *testing.T) {+ // "Café.heic" in NFD: e + combining acute accent (U+0301)+ nfd := "Cafe\u0301.heic"+ nfc := "Caf\u00e9.heic"++ rec := &photoRecord{RecordName: "nfd-enc"}+ rec.Fields.FilenameEnc = &ckStringField{Value: base64.StdEncoding.EncodeToString([]byte(nfd)), Type: "ENCRYPTED_BYTES"}+ rec.Fields.ResOriginalRes = master.Fields.ResOriginalRes+ photos := buildPhotos(rec, nil)+ require.Len(t, photos, 1)+ assert.Equal(t, nfc, photos[0].Filename)++ rec2 := &photoRecord{RecordName: "nfd-str"}+ rec2.Fields.FilenameEnc = &ckStringField{Value: nfd, Type: "STRING"}+ rec2.Fields.ResOriginalRes = master.Fields.ResOriginalRes+ photos2 := buildPhotos(rec2, nil)+ require.Len(t, photos2, 1)+ assert.Equal(t, nfc, photos2[0].Filename)+ })++ t.Run("itemType fallback when filenameEnc missing", func(t *testing.T) {+ rec := &photoRecord{RecordName: "AaBbCcDd1234"}+ rec.Fields.ItemType = &ckStringField{Value: "public.heic"}+ rec.Fields.ResOriginalRes = master.Fields.ResOriginalRes+ photos := buildPhotos(rec, nil)+ require.Len(t, photos, 1)+ assert.Equal(t, "AaBbCcDd1234.heic", photos[0].Filename)+ })++ t.Run("unknown itemType no fallback returns nil", func(t *testing.T) {+ rec := &photoRecord{RecordName: "unknown-uti"}+ rec.Fields.ItemType = &ckStringField{Value: "com.unknown.weird-format"}+ rec.Fields.ResOriginalRes = master.Fields.ResOriginalRes+ photos := buildPhotos(rec, nil)+ assert.Nil(t, photos)+ })++ t.Run("edited photo version from adjustmentType", func(t *testing.T) {+ editAsset := &photoRecord{RecordName: "asset-edited-123"}+ editAsset.Fields.AssetDate = asset.Fields.AssetDate+ editAsset.Fields.AdjustmentType = &ckStringField{Value: "com.apple.photos"}+ editAsset.Fields.ResJPEGFullRes = &ckResourceField{Value: struct {+ Size int64 `json:"size"`+ DownloadURL string `json:"downloadURL"`+ }{Size: 512, DownloadURL: "https://example.com/edited"}}+ editAsset.Fields.ResJPEGFullFileType = &ckStringField{Value: "public.jpeg"}++ photos := buildPhotos(master, editAsset)+ require.Len(t, photos, 2)+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)+ assert.Equal(t, "resOriginalRes", photos[0].ResourceKey)+ assert.Equal(t, "IMG_0001-edited.jpg", photos[1].Filename)+ assert.Equal(t, "resJPEGFullRes", photos[1].ResourceKey)+ assert.Equal(t, int64(512), photos[1].Size)+ assert.Equal(t, "asset-edited-123", photos[1].ID)+ })++ t.Run("slo-mo skips edited version", func(t *testing.T) {+ slomoAsset := &photoRecord{RecordName: "asset-slomo"}+ slomoAsset.Fields.AssetDate = asset.Fields.AssetDate+ slomoAsset.Fields.AdjustmentType = &ckStringField{Value: "com.apple.video.slomo"}+ photos := buildPhotos(master, slomoAsset)+ require.Len(t, photos, 1)+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)+ })++ t.Run("RAW alternative from resOriginalAltRes", func(t *testing.T) {+ rawMaster := &photoRecord{RecordName: "master-raw"}+ rawMaster.Fields.FilenameEnc = master.Fields.FilenameEnc+ rawMaster.Fields.ResOriginalRes = master.Fields.ResOriginalRes+ rawMaster.Fields.ResOriginalWidth = master.Fields.ResOriginalWidth+ rawMaster.Fields.ResOriginalHeight = master.Fields.ResOriginalHeight+ rawMaster.Fields.ResOriginalAltRes = &ckResourceField{Value: struct {+ Size int64 `json:"size"`+ DownloadURL string `json:"downloadURL"`+ }{Size: 8192, DownloadURL: "https://example.com/raw"}}+ rawMaster.Fields.ResOriginalAltFileType = &ckStringField{Value: "com.canon.cr2-raw-image"}++ photos := buildPhotos(rawMaster, nil)+ require.Len(t, photos, 2)+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)+ assert.Equal(t, "resOriginalRes", photos[0].ResourceKey)+ assert.Equal(t, "IMG_0001.cr2", photos[1].Filename)+ assert.Equal(t, "resOriginalAltRes", photos[1].ResourceKey)+ assert.Equal(t, int64(8192), photos[1].Size)+ assert.Equal(t, 4032, photos[1].Width, "RAW alt must inherit width from original")+ assert.Equal(t, 3024, photos[1].Height, "RAW alt must inherit height from original")+ })++ t.Run("RAW alt same extension gets -alt suffix", func(t *testing.T) {+ dupMaster := &photoRecord{RecordName: "master-dup-ext"}+ dupMaster.Fields.FilenameEnc = master.Fields.FilenameEnc+ dupMaster.Fields.ResOriginalRes = master.Fields.ResOriginalRes+ dupMaster.Fields.ResOriginalAltRes = &ckResourceField{Value: struct {+ Size int64 `json:"size"`+ DownloadURL string `json:"downloadURL"`+ }{Size: 4096, DownloadURL: "https://example.com/alt"}}+ dupMaster.Fields.ResOriginalAltFileType = &ckStringField{Value: "public.jpeg"}++ photos := buildPhotos(dupMaster, nil)+ require.Len(t, photos, 2)+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)+ // Same extension would collide, so -alt suffix added+ assert.Equal(t, "IMG_0001-alt.jpg", photos[1].Filename)+ })++ t.Run("edited video via resVidFullRes", func(t *testing.T) {+ vidAsset := &photoRecord{RecordName: "asset-vid-edit"}+ vidAsset.Fields.AssetDate = asset.Fields.AssetDate+ vidAsset.Fields.AdjustmentType = &ckStringField{Value: "com.apple.photos"}+ vidAsset.Fields.ResVidFullRes = &ckResourceField{Value: struct {+ Size int64 `json:"size"`+ DownloadURL string `json:"downloadURL"`+ }{Size: 2048, DownloadURL: "https://example.com/vid-edit"}}+ vidAsset.Fields.ResVidFullFileType = &ckStringField{Value: "public.mpeg-4"}++ photos := buildPhotos(master, vidAsset)+ require.Len(t, photos, 2)+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)+ assert.Equal(t, "IMG_0001-edited.mp4", photos[1].Filename)+ assert.Equal(t, "resVidFullRes", photos[1].ResourceKey)+ })++ t.Run("edited + RAW on same photo", func(t *testing.T) {+ rawMaster := &photoRecord{RecordName: "master-combo"}+ rawMaster.Fields.FilenameEnc = master.Fields.FilenameEnc+ rawMaster.Fields.ResOriginalRes = master.Fields.ResOriginalRes+ rawMaster.Fields.ResOriginalAltRes = &ckResourceField{Value: struct {+ Size int64 `json:"size"`+ DownloadURL string `json:"downloadURL"`+ }{Size: 8192, DownloadURL: "https://example.com/raw"}}+ rawMaster.Fields.ResOriginalAltFileType = &ckStringField{Value: "com.nikon.raw-image"}++ comboAsset := &photoRecord{RecordName: "asset-combo"}+ comboAsset.Fields.AssetDate = asset.Fields.AssetDate+ comboAsset.Fields.AdjustmentType = &ckStringField{Value: "com.apple.photos"}+ comboAsset.Fields.ResJPEGFullRes = &ckResourceField{Value: struct {+ Size int64 `json:"size"`+ DownloadURL string `json:"downloadURL"`+ }{Size: 512, DownloadURL: "https://example.com/edited"}}+ comboAsset.Fields.ResJPEGFullFileType = &ckStringField{Value: "public.jpeg"}++ photos := buildPhotos(rawMaster, comboAsset)+ require.Len(t, photos, 3)+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)+ assert.Equal(t, "IMG_0001-edited.jpg", photos[1].Filename)+ assert.Equal(t, "IMG_0001.nef", photos[2].Filename)+ })++ t.Run("RAW alt with nil file type uses original extension", func(t *testing.T) {+ noTypeMaster := &photoRecord{RecordName: "master-notype"}+ noTypeMaster.Fields.FilenameEnc = master.Fields.FilenameEnc+ noTypeMaster.Fields.ResOriginalRes = master.Fields.ResOriginalRes+ noTypeMaster.Fields.ResOriginalAltRes = &ckResourceField{Value: struct {+ Size int64 `json:"size"`+ DownloadURL string `json:"downloadURL"`+ }{Size: 4096, DownloadURL: "https://example.com/alt-notype"}}+ // No ResOriginalAltFileType set++ photos := buildPhotos(noTypeMaster, nil)+ require.Len(t, photos, 2)+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)+ // Same extension collision triggers -alt suffix+ assert.Equal(t, "IMG_0001-alt.JPG", photos[1].Filename)+ })++ t.Run("extensionless filename gets .MOV companion", func(t *testing.T) {+ noExt := &photoRecord{RecordName: "no-ext"}+ noExt.Fields.FilenameEnc = &ckStringField{Value: base64.StdEncoding.EncodeToString([]byte("IMG_NOEXT")), Type: "ENCRYPTED_BYTES"}+ noExt.Fields.ResOriginalRes = master.Fields.ResOriginalRes+ noExt.Fields.ResOriginalVidComplRes = &ckResourceField{Value: struct {+ Size int64 `json:"size"`+ DownloadURL string `json:"downloadURL"`+ }{Size: 512, DownloadURL: "https://example.com/v"}}+ photos := buildPhotos(noExt, nil)+ require.Len(t, photos, 2)+ assert.Equal(t, "IMG_NOEXT", photos[0].Filename)+ assert.Equal(t, "IMG_NOEXT.MOV", photos[1].Filename)+ })+}++func TestParsePhotoRecords(t *testing.T) {+ filename := base64.StdEncoding.EncodeToString([]byte("IMG_0001.JPG"))++ records := []photoRecord{+ {RecordName: "master1", RecordType: "CPLMaster"},+ {RecordName: "asset1", RecordType: "CPLAsset"},+ {RecordName: "master2", RecordType: "CPLMaster"},+ {RecordName: "asset2", RecordType: "CPLAsset"},+ }+ // Set up master fields+ for i := range records {+ if records[i].RecordType == "CPLMaster" {+ records[i].Fields.FilenameEnc = &ckStringField{Value: filename, Type: "ENCRYPTED_BYTES"}+ records[i].Fields.ResOriginalRes = &ckResourceField{Value: struct {+ Size int64 `json:"size"`+ DownloadURL string `json:"downloadURL"`+ }{Size: 1024, DownloadURL: "https://example.com/dl"}}+ }+ }+ // Link assets to masters+ records[1].Fields.MasterRef = &ckReferenceField{Value: struct {+ RecordName string `json:"recordName"`+ }{RecordName: "master1"}}+ records[3].Fields.MasterRef = &ckReferenceField{Value: struct {+ RecordName string `json:"recordName"`+ }{RecordName: "master2"}}++ photos := parsePhotoRecords(records)+ require.Len(t, photos, 2)+ assert.Equal(t, "master1", photos[0].ID)+ assert.Equal(t, "master2", photos[1].ID)+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)++ t.Run("orphan master without asset", func(t *testing.T) {+ orphan := []photoRecord{+ {RecordName: "orphan-master", RecordType: "CPLMaster"},+ }+ orphan[0].Fields.FilenameEnc = &ckStringField{Value: filename, Type: "ENCRYPTED_BYTES"}+ orphan[0].Fields.ResOriginalRes = records[0].Fields.ResOriginalRes+ photos := parsePhotoRecords(orphan)+ require.Len(t, photos, 1)+ assert.Equal(t, "orphan-master", photos[0].ID)+ })++ t.Run("empty records", func(t *testing.T) {+ photos := parsePhotoRecords(nil)+ assert.Nil(t, photos)+ })+}++func TestClassifySmartAlbums(t *testing.T) {+ makeMaster := func(fileType string) *photoRecord {+ m := &photoRecord{}+ if fileType != "" {+ m.Fields.ResOriginalFileType = &ckStringField{Value: fileType}+ }+ return m+ }+ makeAsset := func(subtype, subtypeV2, favorite, hidden int) *photoRecord {+ a := &photoRecord{}+ a.Fields.AssetSubtype = &ckIntField{Value: subtype}+ a.Fields.AssetSubtypeV2 = &ckIntField{Value: subtypeV2}+ a.Fields.IsFavorite = &ckIntField{Value: favorite}+ a.Fields.IsHidden = &ckIntField{Value: hidden}+ return a+ }++ tests := []struct {+ name string+ master *photoRecord+ asset *photoRecord+ expected []string+ }{+ {"regular photo", makeMaster("public.jpeg"), makeAsset(0, 0, 0, 0), []string{"All Photos"}},+ {"video mpeg4", makeMaster("public.mpeg-4"), makeAsset(0, 0, 0, 0), []string{"All Photos", "Videos"}},+ {"video quicktime", makeMaster("com.apple.quicktime-movie"), makeAsset(0, 0, 0, 0), []string{"All Photos", "Videos"}},+ {"favorite photo", makeMaster("public.jpeg"), makeAsset(0, 0, 1, 0), []string{"All Photos", "Favorites"}},+ {"hidden photo", makeMaster("public.jpeg"), makeAsset(0, 0, 0, 1), []string{"Hidden"}},+ {"screenshot", makeMaster("public.jpeg"), makeAsset(0, 3, 0, 0), []string{"All Photos", "Screenshots"}},+ {"favorited screenshot", makeMaster("public.jpeg"), makeAsset(0, 3, 1, 0), []string{"All Photos", "Favorites", "Screenshots"}},+ {"slo-mo", makeMaster("public.mpeg-4"), makeAsset(100, 0, 0, 0), []string{"All Photos", "Slo-mo"}},+ {"time-lapse", makeMaster("public.mpeg-4"), makeAsset(101, 0, 0, 0), []string{"All Photos", "Time-lapse"}},+ {"panorama", makeMaster("public.jpeg"), makeAsset(1, 0, 0, 0), []string{"All Photos", "Panoramas"}},+ {"live photo", makeMaster("public.jpeg"), makeAsset(0, 2, 0, 0), []string{"All Photos", "Live"}},+ {"nil asset", makeMaster("public.jpeg"), nil, []string{"All Photos"}},+ {"no file type", makeMaster(""), makeAsset(0, 0, 0, 0), []string{"All Photos"}},+ {"hidden favorite", makeMaster("public.jpeg"), makeAsset(0, 0, 1, 1), []string{"Hidden", "Favorites"}},+ {"burst photo", makeMaster("public.jpeg"), func() *photoRecord {+ a := makeAsset(0, 0, 0, 0)+ a.Fields.BurstID = &ckStringField{Value: "B7A3F2E1-4D5C-6789-ABCD-EF0123456789"}+ return a+ }(), []string{"All Photos", "Bursts"}},+ {"portrait photo", makeMaster("public.jpeg"), func() *photoRecord {+ a := makeAsset(0, 0, 0, 0)+ a.Fields.AdjustmentRenderType = &ckIntField{Value: 2} // PORTRAIT bit+ return a+ }(), []string{"All Photos", "Portrait"}},+ {"long exposure", makeMaster("public.jpeg"), func() *photoRecord {+ a := makeAsset(0, 0, 0, 0)+ a.Fields.AdjustmentRenderType = &ckIntField{Value: 4} // LONG_EXPOSURE bit+ return a+ }(), []string{"All Photos", "Long Exposure"}},+ {"portrait + long exposure", makeMaster("public.jpeg"), func() *photoRecord {+ a := makeAsset(0, 0, 0, 0)+ a.Fields.AdjustmentRenderType = &ckIntField{Value: 6} // both bits+ return a+ }(), []string{"All Photos", "Portrait", "Long Exposure"}},+ {"animated gif", makeMaster("com.compuserve.gif"), makeAsset(0, 0, 0, 0), []string{"All Photos", "Animated"}},+ {"soft-deleted photo", makeMaster("public.jpeg"), func() *photoRecord {+ a := makeAsset(0, 0, 1, 0) // favorite=1 should be ignored when deleted+ a.Fields.IsDeleted = &ckIntField{Value: 1}+ return a+ }(), []string{"Recently Deleted"}},+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := classifySmartAlbums(tt.master, tt.asset)+ assert.Equal(t, tt.expected, got)+ })+ }+}++func TestGetPhotoByName_CacheHit(t *testing.T) {+ album := &Album{+ Name: "Videos",+ ObjectType: "CPLAssetInSmartAlbumByAssetDate:Video",+ }+ album.photoCache = map[string]*Photo{+ "IMG_0001.JPG": {ID: "m1", Filename: "IMG_0001.JPG", Size: 1024},+ "IMG_0002.MOV": {ID: "m2", Filename: "IMG_0002.MOV", Size: 2048},+ }++ t.Run("found in cache", func(t *testing.T) {+ photo, err := album.GetPhotoByName(context.Background(), "IMG_0001.JPG")+ require.NoError(t, err)+ assert.Equal(t, "m1", photo.ID)+ assert.Equal(t, int64(1024), photo.Size)+ })++ t.Run("not in cache", func(t *testing.T) {+ // GetPhotoByName with a populated cache but missing filename+ // should return error (cache exists but name not found)+ _, err := album.GetPhotoByName(context.Background(), "MISSING.JPG")+ assert.Error(t, err)+ assert.Contains(t, err.Error(), "MISSING.JPG")+ })+}++func TestDeduplicateFilenames(t *testing.T) {+ t.Run("no duplicates unchanged", func(t *testing.T) {+ photos := []*Photo{+ {ID: "master1", Filename: "IMG_0001.JPG"},+ {ID: "master2", Filename: "IMG_0002.JPG"},+ }+ deduplicateFilenames(photos)+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)+ assert.Equal(t, "IMG_0002.JPG", photos[1].Filename)+ })++ t.Run("all duplicates get full ID suffix", func(t *testing.T) {+ photos := []*Photo{+ {ID: "AQJ2Fq0Px7pGM", Filename: "camphoto_001.jpg"},+ {ID: "BRK3Gr1Qy8qHN", Filename: "camphoto_001.jpg"},+ {ID: "CSL4Hs2Rz9rIO", Filename: "camphoto_001.jpg"},+ }+ deduplicateFilenames(photos)+ assert.Equal(t, "camphoto_001_AQJ2Fq0Px7pGM.jpg", photos[0].Filename)+ assert.Equal(t, "camphoto_001_BRK3Gr1Qy8qHN.jpg", photos[1].Filename)+ assert.Equal(t, "camphoto_001_CSL4Hs2Rz9rIO.jpg", photos[2].Filename)+ })++ t.Run("no extension handled", func(t *testing.T) {+ photos := []*Photo{+ {ID: "masterA", Filename: "noext"},+ {ID: "masterB", Filename: "noext"},+ }+ deduplicateFilenames(photos)+ assert.Equal(t, "noext_masterA", photos[0].Filename)+ assert.Equal(t, "noext_masterB", photos[1].Filename)+ })++ t.Run("deterministic across runs", func(t *testing.T) {+ mk := func() []*Photo {+ return []*Photo{+ {ID: "AQJ2Fq0Px7pGM", Filename: "dup.jpg"},+ {ID: "BRK3Gr1Qy8qHN", Filename: "dup.jpg"},+ }+ }+ p1 := mk()+ p2 := mk()+ deduplicateFilenames(p1)+ deduplicateFilenames(p2)+ assert.Equal(t, p1[0].Filename, p2[0].Filename)+ assert.Equal(t, p1[1].Filename, p2[1].Filename)+ })++ t.Run("stable when new duplicate added", func(t *testing.T) {+ // Before: two duplicates+ before := []*Photo{+ {ID: "AQJ2Fq0Px7pGM", Filename: "dup.jpg"},+ {ID: "BRK3Gr1Qy8qHN", Filename: "dup.jpg"},+ }+ deduplicateFilenames(before)+ nameA := before[0].Filename+ nameB := before[1].Filename++ // After: third duplicate added - existing names must not change+ after := []*Photo{+ {ID: "AQJ2Fq0Px7pGM", Filename: "dup.jpg"},+ {ID: "BRK3Gr1Qy8qHN", Filename: "dup.jpg"},+ {ID: "CSL4Hs2Rz9rIO", Filename: "dup.jpg"},+ }+ deduplicateFilenames(after)+ assert.Equal(t, nameA, after[0].Filename, "existing file A must keep same name")+ assert.Equal(t, nameB, after[1].Filename, "existing file B must keep same name")+ })+}++func TestDeduplicateFilenamesSharedPointers(t *testing.T) {+ // Photos classified into multiple smart albums share *Photo pointers+ // deduplicateFilenames mutates Filename in place, so shared pointers+ // across albums get cross-contaminated unless deep-copied first+ shared := &Photo{ID: "master1", Filename: "dup.jpg"}+ other := &Photo{ID: "master2", Filename: "dup.jpg"}++ albumA := []*Photo{shared, other}+ albumB := []*Photo{shared} // same pointer, unique in this album++ // Simulate the fix: deep-copy before dedup (same as applyPendingDelta)+ dedupedA := make([]*Photo, len(albumA))+ for i, p := range albumA {+ cp := *p+ dedupedA[i] = &cp+ }+ deduplicateFilenames(dedupedA)++ // albumA's dedup copies got suffixes+ assert.Contains(t, dedupedA[0].Filename, "_master1")+ assert.Contains(t, dedupedA[1].Filename, "_master2")++ // Original shared pointer is untouched - albumB sees clean filename+ assert.Equal(t, "dup.jpg", albumB[0].Filename, "deep copy must prevent cross-album contamination")+ assert.Equal(t, "dup.jpg", shared.Filename, "original pointer must be unmodified")+}++func TestDeltaContainsAlbumChanges(t *testing.T) {+ t.Run("CPLAlbum record triggers invalidation", func(t *testing.T) {+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"master1","recordType":"CPLMaster"}`),+ json.RawMessage(`{"recordName":"asset1","recordType":"CPLAsset"}`),+ json.RawMessage(`{"recordName":"album1","recordType":"CPLAlbum"}`),+ }+ assert.True(t, deltaContainsAlbumChanges(records))+ })++ t.Run("no CPLAlbum does not trigger", func(t *testing.T) {+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"master1","recordType":"CPLMaster"}`),+ json.RawMessage(`{"recordName":"asset1","recordType":"CPLAsset"}`),+ json.RawMessage(`{"recordName":"rel1","recordType":"CPLContainerRelation"}`),+ }+ assert.False(t, deltaContainsAlbumChanges(records))+ })++ t.Run("empty records does not trigger", func(t *testing.T) {+ assert.False(t, deltaContainsAlbumChanges(nil))+ })++ t.Run("malformed JSON skipped", func(t *testing.T) {+ records := []json.RawMessage{+ json.RawMessage(`{invalid`),+ json.RawMessage(`{"recordName":"album1","recordType":"CPLAlbum"}`),+ }+ assert.True(t, deltaContainsAlbumChanges(records))+ })+}++func TestParseDeltaRecords(t *testing.T) {+ t.Run("deleted CPLAsset marks both asset and master IDs", func(t *testing.T) {+ // Edited entries use asset.RecordName as ID; ghost entries survive+ // if only masterID is in deletedIDs+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"master-AAA","recordType":"CPLMaster","deleted":true}`),+ json.RawMessage(`{"recordName":"asset-BBB","recordType":"CPLAsset","deleted":true,"fields":{"masterRef":{"value":{"recordName":"master-AAA"}}}}`),+ }+ r := parseDeltaRecords(records)+ assert.True(t, r.deletedIDs["master-AAA"], "master ID from CPLMaster deletion")+ assert.True(t, r.deletedIDs["asset-BBB"], "asset ID from CPLAsset deletion (for -edited entries)")+ })++ t.Run("deleted CPLAsset without masterRef still tracks asset ID", func(t *testing.T) {+ // CloudKit may strip fields from deleted records+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"asset-CCC","recordType":"CPLAsset","deleted":true}`),+ }+ r := parseDeltaRecords(records)+ assert.True(t, r.deletedIDs["asset-CCC"], "asset ID tracked even without masterRef")+ assert.Equal(t, 1, len(r.deletedIDs), "only asset ID, no master")+ })++ t.Run("deleted CPLContainerRelation extracts album from recordName", func(t *testing.T) {+ // Deleted relations lack fields; album parsed from "assetID-IN-albumRecordName"+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"assetXYZ-IN-album-uuid-456","recordType":"CPLContainerRelation","deleted":true}`),+ }+ r := parseDeltaRecords(records)+ assert.True(t, r.changedAlbumRecords["album-uuid-456"], "album extracted from deleted relation recordName")+ })++ t.Run("deleted relation without recordType still extracts album from recordName", func(t *testing.T) {+ // changes/zone omits recordType for deleted CPLContainerRelation records+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"assetXYZ-IN-album-uuid-654","deleted":true}`),+ }+ r := parseDeltaRecords(records)+ assert.True(t, r.changedAlbumRecords["album-uuid-654"], "album extracted from deleted relation with missing recordType")+ })++ t.Run("non-deleted CPLContainerRelation uses containerId STRING field", func(t *testing.T) {+ // CloudKit relation records expose containerId as the canonical album field+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"rel1","recordType":"CPLContainerRelation","fields":{"containerId":{"value":"album-uuid-789"}}}`),+ }+ r := parseDeltaRecords(records)+ assert.True(t, r.changedAlbumRecords["album-uuid-789"], "album extracted from containerId STRING field")+ })++ t.Run("non-deleted CPLContainerRelation falls back to recordName when fields missing", func(t *testing.T) {+ // changes/zone can return live relation records with empty fields+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"assetXYZ-IN-album-uuid-987","recordType":"CPLContainerRelation","fields":{},"deleted":false}`),+ }+ r := parseDeltaRecords(records)+ assert.True(t, r.changedAlbumRecords["album-uuid-987"], "album extracted from live relation recordName fallback")+ })++ t.Run("deleted relation with unexpected recordName format is safe", func(t *testing.T) {+ // No "-IN-" in recordName - must not panic or add garbage+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"weirdformat","recordType":"CPLContainerRelation","deleted":true}`),+ }+ r := parseDeltaRecords(records)+ assert.Empty(t, r.changedAlbumRecords, "no album invalidated for malformed recordName")+ })++ t.Run("CPLAlbum sets albumMetadataChanged", func(t *testing.T) {+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"album1","recordType":"CPLAlbum"}`),+ }+ r := parseDeltaRecords(records)+ assert.True(t, r.albumMetadataChanged)+ })++ t.Run("asset-only update detected", func(t *testing.T) {+ // CPLAsset changed but no CPLMaster in delta - triggers smart album invalidation+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"asset-DDD","recordType":"CPLAsset","fields":{"masterRef":{"value":{"recordName":"master-EEE"}}}}`),+ }+ r := parseDeltaRecords(records)+ assert.True(t, r.hasAssetOnlyUpdates, "asset without matching master = asset-only update")+ })++ t.Run("asset with matching master is not asset-only", func(t *testing.T) {+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"master-FFF","recordType":"CPLMaster","fields":{"resOriginalRes":{"value":{"downloadURL":"https://example.com"}}}}`),+ json.RawMessage(`{"recordName":"asset-GGG","recordType":"CPLAsset","fields":{"masterRef":{"value":{"recordName":"master-FFF"}}}}`),+ }+ r := parseDeltaRecords(records)+ assert.False(t, r.hasAssetOnlyUpdates)+ assert.Equal(t, 1, len(r.newMasters))+ assert.Equal(t, 1, len(r.newAssets))+ })++ t.Run("unknown record types are silently skipped", func(t *testing.T) {+ records := []json.RawMessage{+ json.RawMessage(`{"recordName":"x","recordType":"CPLSomethingNew"}`),+ }+ r := parseDeltaRecords(records)+ assert.Empty(t, r.deletedIDs)+ assert.Empty(t, r.newMasters)+ assert.Empty(t, r.newAssets)+ assert.Empty(t, r.changedAlbumRecords)+ assert.False(t, r.albumMetadataChanged)+ })++ t.Run("malformed JSON records skipped without panic", func(t *testing.T) {+ records := []json.RawMessage{+ json.RawMessage(`{invalid`),+ json.RawMessage(`{"recordName":"master1","recordType":"CPLMaster","deleted":true}`),+ }+ r := parseDeltaRecords(records)+ assert.True(t, r.deletedIDs["master1"], "valid record after malformed one still parsed")+ })++ t.Run("empty records produces empty result", func(t *testing.T) {+ r := parseDeltaRecords(nil)+ assert.Empty(t, r.deletedIDs)+ assert.False(t, r.albumMetadataChanged)+ assert.False(t, r.hasAssetOnlyUpdates)+ })+}++func TestApplyPendingDelta_EmptyAlbumsClearsPending(t *testing.T) {+ // When checkForChanges eagerly clears lib.albums (CPLAlbum delta),+ // applyPendingDelta must clear pendingDelta instead of leaving it stuck+ // Without this fix, pendingDelta stays non-nil forever and all future+ // checkForChanges calls skip (line 912: pendingDelta != nil)+ lib := &Library{+ albums: make(map[string]*Album), // empty - simulates eager clear+ pendingDelta: &deltaPayload{+ records: []json.RawMessage{json.RawMessage(`{"recordName":"m1","recordType":"CPLMaster"}`)},+ syncToken: "old-token",+ },+ }+ result := lib.applyPendingDelta(context.Background())+ assert.False(t, result, "must return false when albums empty")+ assert.Nil(t, lib.pendingDelta, "must clear pendingDelta to avoid permanent stuck state")+}++func TestApplyPendingDelta_InvalidatesNestedAlbum(t *testing.T) {+ oldCacheDir := config.GetCacheDir()+ t.Cleanup(func() {+ _ = config.SetCacheDir(oldCacheDir)+ })+ require.NoError(t, config.SetCacheDir(t.TempDir()))++ ps := &PhotosService{client: &Client{remoteName: "delta-nested-test"}}+ lib := &Library{+ service: ps,+ zoneID: "PrimarySync",+ area: areaPrivate,+ albums: make(map[string]*Album),+ }+ child := lib.newUserAlbum("Child", "child-record")+ child.SetTestPhotoCache(map[string]*Photo{+ "one.jpg": {ID: "master1", Filename: "one.jpg"},+ })+ child.saveDiskCache([]*Photo{{ID: "master1", Filename: "one.jpg"}})++ folder := &Album{+ Name: "Folder",+ RecordName: "folder-record",+ lib: lib,+ IsFolder: true,+ Children: map[string]*Album{+ "Child": child,+ },+ }+ lib.albums[folder.Name] = folder+ lib.pendingDelta = &deltaPayload{+ records: []json.RawMessage{json.RawMessage(`{"recordName":"asset123-IN-child-record","deleted":true}`)},+ syncToken: "next-token",+ }++ cacheFile := filepath.Join(lib.zoneCacheDir(), albumCacheKey(child.ObjectType)+".json")+ _, err := os.Stat(cacheFile)+ require.NoError(t, err, "nested child album cache file should exist before invalidation")++ result := lib.applyPendingDelta(context.Background())+ assert.True(t, result)+ assert.Nil(t, lib.pendingDelta)++ child.mu.Lock()+ assert.Nil(t, child.photoCache, "nested child album memory cache should be invalidated")+ child.mu.Unlock()+ _, err = os.Stat(cacheFile)+ assert.ErrorIs(t, err, os.ErrNotExist, "nested child album disk cache should be removed")+ data, err := os.ReadFile(filepath.Join(lib.zoneCacheDir(), "syncToken"))+ require.NoError(t, err)+ assert.Equal(t, "next-token", string(data))+}++func TestFlushCaches_NoPendingDeltaRace(t *testing.T) {+ // Verify FlushCaches and applyPendingDelta don't race on pendingDelta.+ // FlushCaches must acquire deltaMu before writing pendingDelta.+ ps := &PhotosService{+ client: &Client{remoteName: "race-test"},+ libraries: make(map[string]*Library),+ }+ lib := &Library{+ service: ps,+ zoneID: "PrimarySync",+ area: areaPrivate,+ albums: make(map[string]*Album),+ }+ ps.libraries["PrimarySync"] = lib++ // Run concurrent FlushCaches + applyPendingDelta+ // The race detector will flag unsynchronized access to pendingDelta+ done := make(chan struct{})+ go func() {+ defer close(done)+ for i := 0; i < 100; i++ {+ lib.deltaMu.Lock()+ lib.pendingDelta = &deltaPayload{+ records: []json.RawMessage{json.RawMessage(`{"recordName":"m1","recordType":"CPLMaster"}`)},+ syncToken: "tok",+ }+ lib.deltaMu.Unlock()+ lib.applyPendingDelta(context.Background())+ }+ }()+ for i := 0; i < 100; i++ {+ ps.FlushCaches()+ // Re-add the library since FlushCaches clears it+ ps.mu.Lock()+ ps.libraries["PrimarySync"] = lib+ ps.mu.Unlock()+ }+ <-done+}++func TestBuildSmartAlbums(t *testing.T) {+ albums := buildSmartAlbums()++ // Must produce exactly 14 smart albums+ assert.Equal(t, 14, len(albums), "expected 14 smart albums")++ // All expected names present+ expected := []string{+ "All Photos", "Favorites", "Videos", "Screenshots", "Live",+ "Slo-mo", "Time-lapse", "Panoramas", "Portrait", "Long Exposure",+ "Animated", "Bursts", "Hidden", "Recently Deleted",+ }+ for _, name := range expected {+ a, ok := albums[name]+ require.True(t, ok, "missing smart album %q", name)+ assert.NotEmpty(t, a.ObjectType, "album %q has no ObjectType", name)+ assert.NotEmpty(t, a.ListType, "album %q has no ListType", name)+ assert.NotEmpty(t, a.Direction, "album %q has no Direction", name)+ }++ // Verify special albums have correct configuration+ assert.Equal(t, "DESCENDING", albums["Recently Deleted"].Direction)+ assert.Empty(t, albums["All Photos"].Filters, "All Photos should have no filters")+ assert.Empty(t, albums["Bursts"].Filters, "Bursts should have no filters")+ assert.Empty(t, albums["Hidden"].Filters, "Hidden should have no filters")++ // Verify filter-based albums have exactly one smartAlbum filter+ for _, name := range []string{"Favorites", "Videos", "Screenshots", "Live", "Slo-mo", "Time-lapse", "Panoramas", "Portrait", "Long Exposure", "Animated"} {+ a := albums[name]+ require.Len(t, a.Filters, 1, "album %q should have exactly one filter", name)+ assert.Equal(t, "smartAlbum", a.Filters[0].FieldName)+ assert.Equal(t, "EQUALS", a.Filters[0].Comparator)+ }+}++func TestAtomicWriteFile(t *testing.T) {+ dir := t.TempDir()+ target := filepath.Join(dir, "test.json")++ // Successful write+ err := atomicWriteFile(target, []byte(`{"key":"value"}`))+ require.NoError(t, err)+ data, err := os.ReadFile(target)+ require.NoError(t, err)+ assert.Equal(t, `{"key":"value"}`, string(data))++ // Overwrite existing+ err = atomicWriteFile(target, []byte(`{"key":"updated"}`))+ require.NoError(t, err)+ data, err = os.ReadFile(target)+ require.NoError(t, err)+ assert.Equal(t, `{"key":"updated"}`, string(data))++ // No .tmp file left behind+ _, err = os.Stat(target + ".tmp")+ assert.True(t, os.IsNotExist(err), "temp file should not persist")+}++func TestAlbumCacheKey(t *testing.T) {+ k1 := albumCacheKey("CPLAssetByAssetDateWithoutHiddenOrDeleted")+ k2 := albumCacheKey("CPLAssetInSmartAlbumByAssetDate:Video")+ k3 := albumCacheKey("CPLAssetByAssetDateWithoutHiddenOrDeleted")++ assert.Len(t, k1, 16, "cache key should be 16 hex chars")+ assert.Equal(t, k1, k3, "same input should produce same key")+ assert.NotEqual(t, k1, k2, "different input should produce different key")++ // Verify filename-safe+ for _, c := range k1 {+ assert.True(t, (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'), "key should be hex: %c", c)+ }+}diff --git a/backend/iclouddrive/api/session.go b/backend/iclouddrive/api/session.goindex 004faaa37..89e47a937 100644--- a/backend/iclouddrive/api/session.go+++ b/backend/iclouddrive/api/session.go@@ -2,16 +2,21 @@ package apiimport ("context"+ "crypto/sha256""encoding/base64"+ "encoding/hex"+ "encoding/json""fmt"+ "io""maps""net/http""net/url""slices""strings"+ "sync"+ "time""github.com/google/uuid"- "github.com/oracle/oci-go-sdk/v65/common""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/lib/rest"@@ -32,6 +37,7 @@ type Session struct {Cookies []*http.Cookie `json:"cookies"`AccountInfo AccountInfo `json:"account_info"`+ mu sync.Mutex `json:"-"` // protects session fields during concurrent Request callssrv *rest.Client `json:"-"`needs2FA bool `json:"-"` // set when SRP signin returns 409}@@ -45,20 +51,117 @@ type srpInitResponse struct {C string `json:"c"`}-// String returns the session as a string-// func (s *Session) String() string {-// jsession, _ := json.Marshal(s)-// return string(jsession)-// }+func cookieValueFingerprint(value string) string {+ if value == "" {+ return ""+ }+ sum := sha256.Sum256([]byte(value))+ return hex.EncodeToString(sum[:6])+}-// Request makes a request-func (s *Session) Request(ctx context.Context, opts rest.Opts, request any, response any) (*http.Response, error) {- resp, err := s.srv.CallJSON(ctx, &opts, &request, &response)+func authStateBodySummary(body []byte) string {+ summary := []string{fmt.Sprintf("bytes=%d", len(body)), "hash=" + cookieValueFingerprint(string(body))}+ var top map[string]json.RawMessage+ if err := json.Unmarshal(body, &top); err != nil {+ return strings.Join(summary, " ")+ }+ keys := make([]string, 0, len(top))+ for key := range top {+ keys = append(keys, key)+ }+ slices.Sort(keys)+ summary = append(summary, fmt.Sprintf("keys=%v", keys))+ if _, ok := top["phoneNumberVerification"]; ok {+ summary = append(summary, "wrapped=true")+ }+ return strings.Join(summary, " ")+}- if err != nil {- return resp, err+func cookieDebugSummary(c *http.Cookie) string {+ parts := []string{c.Name}+ if c.Value == "" {+ parts = append(parts, "empty", "len=0")+ } else {+ parts = append(parts, "set", fmt.Sprintf("len=%d", len(c.Value)), "hash="+cookieValueFingerprint(c.Value))+ }+ if c.Path != "" {+ parts = append(parts, "path="+c.Path)+ }+ if c.Domain != "" {+ parts = append(parts, "domain="+c.Domain)+ }+ if c.MaxAge != 0 {+ parts = append(parts, fmt.Sprintf("maxAge=%d", c.MaxAge))+ }+ if !c.Expires.IsZero() {+ parts = append(parts, "expires")+ }+ return strings.Join(parts, ",")+}++func cookieDebugSummaries(cookies []*http.Cookie) []string {+ if len(cookies) == 0 {+ return nil+ }+ out := make([]string, 0, len(cookies))+ for _, c := range cookies {+ out = append(out, cookieDebugSummary(c))+ }+ return out+}++func cookieJarDebugSummaries(cookies []*http.Cookie) []string {+ if len(cookies) == 0 {+ return nil+ }+ out := make([]string, 0, len(cookies))+ for _, c := range cookies {+ summary := c.Name+ if c.Value == "" {+ summary += ",empty,len=0"+ } else {+ summary += fmt.Sprintf(",len=%d,hash=%s", len(c.Value), cookieValueFingerprint(c.Value))+ }+ out = append(out, summary)+ }+ return out+}++func (s *Session) mergeCookies(cookies []*http.Cookie) {+ if len(cookies) == 0 {+ return+ }+ existing := make(map[string]int, len(s.Cookies))+ for i, c := range s.Cookies {+ existing[c.Name] = i+ }+ for _, c := range cookies {+ if c.Value == "" {+ if idx, ok := existing[c.Name]; ok {+ fs.Debugf(nil, "iclouddrive: deleting empty auth cookie: %s", cookieDebugSummary(c))+ s.Cookies = append(s.Cookies[:idx], s.Cookies[idx+1:]...)+ delete(existing, c.Name)+ for j := idx; j < len(s.Cookies); j++ {+ existing[s.Cookies[j].Name] = j+ }+ } else {+ fs.Debugf(nil, "iclouddrive: ignoring empty auth cookie tombstone for missing cookie: %s", cookieDebugSummary(c))+ }+ continue+ }+ if idx, ok := existing[c.Name]; ok {+ s.Cookies[idx] = c+ } else {+ s.Cookies = append(s.Cookies, c)+ existing[c.Name] = len(s.Cookies) - 1+ }}+}+// extractHeaders reads Apple session headers from an HTTP response into the session+// Does NOT acquire s.mu - caller is responsible for locking if needed+func (s *Session) extractHeaders(resp *http.Response) {+ s.mergeCookies(resp.Cookies())if val := resp.Header.Get("X-Apple-ID-Account-Country"); val != "" {s.AccountCountry = val}@@ -77,7 +180,17 @@ func (s *Session) Request(ctx context.Context, opts rest.Opts, request any, respif val := resp.Header.Get("X-Apple-Auth-Attributes"); val != "" {s.AuthAttributes = val}+}+// Request makes a JSON API request and extracts session headers+func (s *Session) Request(ctx context.Context, opts rest.Opts, request any, response any) (*http.Response, error) {+ resp, err := s.srv.CallJSON(ctx, &opts, &request, &response)+ if err != nil {+ return resp, err+ }+ s.mu.Lock()+ s.extractHeaders(resp)+ s.mu.Unlock()return resp, nil}@@ -89,13 +202,8 @@ func (s *Session) Requires2FA() bool {return s.AccountInfo.DsInfo != nil && s.AccountInfo.DsInfo.HsaVersion == 2 && s.AccountInfo.HsaChallengeRequired}-// SignIn performs SRP-based authentication against Apple's idmsa endpoint.+// SignIn performs SRP-based authentication against Apple's idmsa endpointfunc (s *Session) SignIn(ctx context.Context, appleID, password string) error {- // Apple's SRP implementation expects a lowercase account name.- // The old plaintext flow didn't need this because the server normalized- // it, but SRP uses the username in client-side proof computation (M1).- appleID = strings.ToLower(appleID)-// Step 1: Initialize the auth sessionif err := s.authStart(ctx); err != nil {return fmt.Errorf("authStart: %w", err)@@ -106,8 +214,11 @@ func (s *Session) SignIn(ctx context.Context, appleID, password string) error {return fmt.Errorf("authFederate: %w", err)}- // Step 3: SRP init — send client public value A, get salt + B- client := newSRPClient()+ // Step 3: SRP init - send client public value A, get salt + B+ client, err := newSRPClient()+ if err != nil {+ return fmt.Errorf("newSRPClient: %w", err)+ }aBase64 := base64.StdEncoding.EncodeToString(client.getABytes())initResp, err := s.authSRPInit(ctx, aBase64, appleID)@@ -130,9 +241,11 @@ func (s *Session) SignIn(ctx context.Context, appleID, password string) error {if err != nil {return fmt.Errorf("derivePassword: %w", err)}- client.processChallenge([]byte(appleID), derivedKey, salt, serverB)+ if err := client.processChallenge([]byte(appleID), derivedKey, salt, serverB); err != nil {+ return fmt.Errorf("processChallenge: %w", err)+ }- // Step 5: Complete — send M1, M2 proofs+ // Step 5: Complete - send M1, M2 proofsm1Base64 := base64.StdEncoding.EncodeToString(client.M1)m2Base64 := base64.StdEncoding.EncodeToString(client.M2)@@ -143,7 +256,7 @@ func (s *Session) SignIn(ctx context.Context, appleID, password string) error {return nil}-// authStart initializes the SRP auth session by hitting the authorize/signin endpoint.+// authStart initializes the SRP auth session by hitting the authorize/signin endpointfunc (s *Session) authStart(ctx context.Context) error {if s.FrameID == "" {s.FrameID = strings.ToLower(uuid.New().String())@@ -184,20 +297,11 @@ func (s *Session) authStart(ctx context.Context) error {return fmt.Errorf("authStart: unexpected status %s", resp.Status)}- if val := resp.Header.Get("X-Apple-Auth-Attributes"); val != "" {- s.AuthAttributes = val- }- if val := resp.Header.Get("scnt"); val != "" {- s.Scnt = val- }- if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" {- s.SessionID = val- }-+ s.extractHeaders(resp)return nil}-// authFederate submits the account name to Apple's federate endpoint.+// authFederate submits the account name to Apple's federate endpointfunc (s *Session) authFederate(ctx context.Context, accountName string) error {values := map[string]any{"accountName": accountName,@@ -224,16 +328,7 @@ func (s *Session) authFederate(ctx context.Context, accountName string) error {}_ = resp.Body.Close()- if val := resp.Header.Get("X-Apple-Auth-Attributes"); val != "" {- s.AuthAttributes = val- }- if val := resp.Header.Get("scnt"); val != "" {- s.Scnt = val- }- if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" {- s.SessionID = val- }-+ s.extractHeaders(resp)if resp.StatusCode != http.StatusOK {return fmt.Errorf("authFederate: unexpected status %s", resp.Status)}@@ -241,7 +336,7 @@ func (s *Session) authFederate(ctx context.Context, accountName string) error {}// authSRPInit sends the client's public value A to the server and retrieves-// the salt, server public value B, iteration count, protocol, and challenge.+// the salt, server public value B, iteration count, protocol, and challengefunc (s *Session) authSRPInit(ctx context.Context, aBase64, accountName string) (*srpInitResponse, error) {values := map[string]any{"a": aBase64,@@ -267,18 +362,12 @@ func (s *Session) authSRPInit(ctx context.Context, aBase64, accountName string)return nil, err}- if val := resp.Header.Get("scnt"); val != "" {- s.Scnt = val- }- if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" {- s.SessionID = val- }-+ s.extractHeaders(resp)return &initResp, nil}-// authSRPComplete sends the SRP proofs M1 and M2 to complete authentication.-// Returns nil on success (200 or 409/2FA needed).+// authSRPComplete sends the SRP proofs M1 and M2 to complete authentication+// Returns nil on success (200 or 409/2FA needed)func (s *Session) authSRPComplete(ctx context.Context, accountName, m1Base64, m2Base64, c string) error {trustTokens := []string{}if s.TrustToken != "" {@@ -305,7 +394,6 @@ func (s *Session) authSRPComplete(ctx context.Context, accountName, m1Base64, m2ExtraHeaders: s.getSRPAuthHeaders(),RootURL: authEndpoint,IgnoreStatus: true,- NoResponse: true,Body: body,}@@ -313,51 +401,65 @@ func (s *Session) authSRPComplete(ctx context.Context, accountName, m1Base64, m2if err != nil {return err}+ respBody, _ := io.ReadAll(resp.Body)_ = resp.Body.Close()- // Extract updated headers- if val := resp.Header.Get("X-Apple-Auth-Attributes"); val != "" {- s.AuthAttributes = val- }- if val := resp.Header.Get("X-Apple-Session-Token"); val != "" {- s.SessionToken = val- }- if val := resp.Header.Get("scnt"); val != "" {- s.Scnt = val- }- if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" {- s.SessionID = val- }- if val := resp.Header.Get("X-Apple-ID-Account-Country"); val != "" {- s.AccountCountry = val- }- if val := resp.Header.Get("X-Apple-TwoSV-Trust-Token"); val != "" {- s.TrustToken = val- }+ s.extractHeaders(resp)switch resp.StatusCode {case http.StatusOK:- fs.Debugf("icloud", "SRP sign in successful")+ fs.Debugf(nil, "iclouddrive: SRP sign in successful")return nilcase http.StatusConflict:- // 409 = 2FA required, this is expected- fs.Debugf("icloud", "SRP sign in requires 2FA")+ // 409 = 2FA required+ fs.Debugf(nil, "iclouddrive: SRP sign in requires 2FA, response: %s", respBody)s.needs2FA = truereturn nil+ case http.StatusPreconditionFailed:+ // 412 = non-2FA account needs repair/complete step+ fs.Debugf(nil, "iclouddrive: SRP sign in returned 412, attempting repair/complete")+ return s.authRepairComplete(ctx)case http.StatusForbidden:return fmt.Errorf("sign in failed: incorrect username or password")default:- return fmt.Errorf("sign in failed: %s", resp.Status)+ return fmt.Errorf("sign in failed: %s: %s", resp.Status, respBody)}}-// getAuthOrigin returns the origin URL for auth requests.-// Supports both global (idmsa.apple.com) and China (idmsa.apple.com.cn) endpoints.+// authRepairComplete handles the repair flow for non-2FA accounts that return 412+func (s *Session) authRepairComplete(ctx context.Context) error {+ body, err := IntoReader(map[string]any{})+ if err != nil {+ return err+ }+ opts := rest.Opts{+ Method: "POST",+ Path: "/repair/complete",+ ExtraHeaders: s.getSRPAuthHeaders(),+ RootURL: authEndpoint,+ IgnoreStatus: true,+ NoResponse: true,+ Body: body,+ }+ resp, err := s.srv.Call(ctx, &opts)+ if err != nil {+ return fmt.Errorf("repair/complete failed: %w", err)+ }+ if resp.StatusCode != http.StatusOK {+ return fmt.Errorf("repair/complete returned %s", resp.Status)+ }+ s.extractHeaders(resp)+ fs.Debugf(nil, "iclouddrive: repair/complete successful")+ return nil+}++// getAuthOrigin returns the origin URL for auth requests+// Supports both global (idmsa.apple.com) and China (idmsa.apple.com.cn) endpointsfunc getAuthOrigin() string {return strings.TrimSuffix(authEndpoint, "/appleauth/auth")}-// getSRPAuthHeaders returns headers needed for SRP auth requests.+// getSRPAuthHeaders returns headers needed for SRP auth requestsfunc (s *Session) getSRPAuthHeaders() map[string]string {frameTag := "auth-" + s.FrameIDauthOrigin := getAuthOrigin()@@ -414,14 +516,110 @@ func (s *Session) AuthWithToken(ctx context.Context) error {}resp, err := s.Request(ctx, opts, nil, &s.AccountInfo)- if err == nil {- s.Cookies = resp.Cookies()+ if err != nil {+ return err+ }+ fs.Debugf(nil, "iclouddrive: accountLogin response cookies: %v", cookieDebugSummaries(resp.Cookies()))+ fs.Debugf(nil, "iclouddrive: session cookie jar after accountLogin: %v", cookieJarDebugSummaries(s.Cookies))++ // Acquire PCS cookies if Advanced Data Protection is enabled+ if ws := s.AccountInfo.Webservices["ckdatabasews"]; ws != nil && ws.PcsRequired {+ fs.Debugf(nil, "iclouddrive: ADP detected (pcsRequired=true)")+ if s.hasPCSCookies() {+ fs.Debugf(nil, "iclouddrive: PCS cookies already present, skipping acquisition")+ } else {+ if err := s.acquirePCSCookies(ctx); err != nil {+ return err+ }+ }+ } else {+ fs.Debugf(nil, "iclouddrive: no ADP (pcsRequired=false)")+ }++ return nil+}++// hasPCSCookies checks if the required PCS cookies for Photos are already present+func (s *Session) hasPCSCookies() bool {+ var hasPhotos, hasSharing bool+ for _, c := range s.Cookies {+ switch c.Name {+ case "X-APPLE-WEBAUTH-PCS-Photos":+ hasPhotos = true+ case "X-APPLE-WEBAUTH-PCS-Sharing":+ hasSharing = true+ }}+ return hasPhotos && hasSharing+}+// acquirePCSCookies requests PCS cookies for ADP-enabled accounts+// May require user approval on a trusted device (polls every 10s, max 5 min)+func (s *Session) acquirePCSCookies(ctx context.Context) error {+ fs.Logf(nil, "iclouddrive: Advanced Data Protection enabled, requesting PCS cookies")+ const maxAttempts = 30 // 30 * 10s = 5 minutes max+ for attempt := 0; attempt < maxAttempts; attempt++ {+ fs.Debugf(nil, "iclouddrive: requestPCS outgoing cookies: %v", cookieJarDebugSummaries(s.Cookies))+ values := map[string]any{+ "appName": "photos",+ "derivedFromUserAction": true,+ }+ body, err := IntoReader(values)+ if err != nil {+ return fmt.Errorf("requestPCS: %w", err)+ }+ opts := rest.Opts{+ Method: "POST",+ Path: "/requestPCS",+ ExtraHeaders: s.GetHeaders(map[string]string{}),+ RootURL: setupEndpoint,+ }+ opts.Body = body+ var pcsResp struct {+ Status string `json:"status"`+ Message string `json:"message"`+ }+ resp, err := s.Request(ctx, opts, nil, &pcsResp)+ if err != nil {+ return fmt.Errorf("requestPCS: %w", err)+ }+ fs.Debugf(nil, "iclouddrive: requestPCS response cookies: %v", cookieDebugSummaries(resp.Cookies()))+ fs.Debugf(nil, "iclouddrive: requestPCS response: status=%q message=%q cookies=%d",+ pcsResp.Status, pcsResp.Message, len(resp.Cookies()))+ if pcsResp.Status == "success" {+ if !s.hasPCSCookies() {+ return fmt.Errorf("requestPCS: server returned success but PCS cookies missing")+ }+ fs.Logf(nil, "iclouddrive: PCS cookies acquired")+ return nil+ }+ // Device consent required - poll until approved+ fs.Logf(nil, "iclouddrive: waiting for device approval for PCS (%s)", pcsResp.Message)+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-time.After(10 * time.Second):+ }+ }+ return fmt.Errorf("requestPCS: timed out waiting for device approval after 5 minutes")+}++// RequestPushNotification explicitly requests a push notification to trusted devices+// Required for iOS 26.4+ where the SRP 409 response no longer auto-pushes+func (s *Session) RequestPushNotification(ctx context.Context) error {+ opts := rest.Opts{+ Method: "PUT",+ Path: "/verify/trusteddevice/securitycode",+ ExtraHeaders: s.GetAuthHeaders(map[string]string{}),+ RootURL: authEndpoint,+ NoResponse: true,+ }++ _, err := s.Request(ctx, opts, nil, nil)return err}-// Validate2FACode validates the 2FA code+// Validate2FACode validates the 2FA code from a trusted device push notificationfunc (s *Session) Validate2FACode(ctx context.Context, code string) error {values := map[string]any{"securityCode": map[string]string{"code": code}}body, err := IntoReader(values)@@ -450,6 +648,132 @@ func (s *Session) Validate2FACode(ctx context.Context, code string) error {return fmt.Errorf("validate2FACode failed: %w", err)}+// TrustedPhoneNumber represents a phone number that can receive SMS verification codes+type TrustedPhoneNumber struct {+ ID int `json:"id"`+ NumberWithDialCode string `json:"numberWithDialCode"`+ ObfuscatedNumber string `json:"obfuscatedNumber"`+ PushMode string `json:"pushMode"`+ NonFTEU bool `json:"nonFTEU"`+}++// AuthStateResponse is the response from GET /appleauth/auth after sign-in+// Some accounts return fields at top level, others nest them under phoneNumberVerification+type AuthStateResponse struct {+ TrustedPhoneNumbers []TrustedPhoneNumber `json:"trustedPhoneNumbers"`+ TrustedPhoneNumber *TrustedPhoneNumber `json:"trustedPhoneNumber"`+ NoTrustedDevices bool `json:"noTrustedDevices"`+ AuthenticationType string `json:"authenticationType"`+ Hsa2Account bool `json:"hsa2Account"`+ PhoneNumberVerification *AuthStateResponse `json:"phoneNumberVerification"`+}++// GetAuthState retrieves the current auth state including trusted phone numbers for SMS 2FA+func (s *Session) GetAuthState(ctx context.Context) (*AuthStateResponse, error) {+ opts := rest.Opts{+ Method: "GET",+ Path: "",+ ExtraHeaders: s.GetAuthHeaders(map[string]string{}),+ RootURL: authEndpoint,+ ContentLength: int64Ptr(0),+ }+ // Use srv.Call directly to capture the raw response body for debugging+ resp, err := s.srv.Call(ctx, &opts)+ if err != nil {+ return nil, fmt.Errorf("getAuthState: %w", err)+ }+ s.mu.Lock()+ s.extractHeaders(resp)+ s.mu.Unlock()+ body, readErr := io.ReadAll(resp.Body)+ _ = resp.Body.Close()+ if readErr != nil {+ return nil, fmt.Errorf("getAuthState read: %w", readErr)+ }+ fs.Debugf(nil, "iclouddrive: auth state response summary: %s", authStateBodySummary(body))+ var state AuthStateResponse+ if err := json.Unmarshal(body, &state); err != nil {+ fs.Debugf(nil, "iclouddrive: auth state parse failed: %s", authStateBodySummary(body))+ return nil, fmt.Errorf("getAuthState unmarshal: %w", err)+ }+ // Some accounts nest auth data under phoneNumberVerification+ if state.PhoneNumberVerification != nil && state.AuthenticationType == "" {+ fs.Debugf(nil, "iclouddrive: unwrapping phoneNumberVerification envelope")+ n := state.PhoneNumberVerification+ state.TrustedPhoneNumbers = n.TrustedPhoneNumbers+ state.TrustedPhoneNumber = n.TrustedPhoneNumber+ state.NoTrustedDevices = n.NoTrustedDevices+ state.AuthenticationType = n.AuthenticationType+ state.Hsa2Account = n.Hsa2Account+ state.PhoneNumberVerification = nil+ }+ fs.Debugf(nil, "iclouddrive: auth state: type=%s hsa2=%v noTrustedDevices=%v phones=%d phoneSingular=%v",+ state.AuthenticationType, state.Hsa2Account, state.NoTrustedDevices,+ len(state.TrustedPhoneNumbers), state.TrustedPhoneNumber != nil)+ // Fall back to singular trustedPhoneNumber when plural array is empty+ // Some accounts return only the singular form (SMS-only, no trusted devices)+ if len(state.TrustedPhoneNumbers) == 0 && state.TrustedPhoneNumber != nil {+ fs.Debugf(nil, "iclouddrive: using singular trustedPhoneNumber (id=%d) as fallback",+ state.TrustedPhoneNumber.ID)+ state.TrustedPhoneNumbers = []TrustedPhoneNumber{*state.TrustedPhoneNumber}+ }+ return &state, nil+}++// RequestSMSCode triggers SMS code delivery to a trusted phone number+func (s *Session) RequestSMSCode(ctx context.Context, phoneID int, mode string) error {+ values := map[string]any{+ "phoneNumber": map[string]any{"id": phoneID},+ "mode": mode,+ }+ body, err := IntoReader(values)+ if err != nil {+ return err+ }+ opts := rest.Opts{+ Method: "PUT",+ Path: "/verify/phone",+ ExtraHeaders: s.GetAuthHeaders(map[string]string{}),+ RootURL: authEndpoint,+ Body: body,+ NoResponse: true,+ }+ _, err = s.Request(ctx, opts, nil, nil)+ if err != nil {+ return fmt.Errorf("requestSMSCode: %w", err)+ }+ return nil+}++// ValidateSMSCode validates a 2FA code received via SMS+func (s *Session) ValidateSMSCode(ctx context.Context, code string, phoneID int, mode string) error {+ values := map[string]any{+ "securityCode": map[string]string{"code": code},+ "phoneNumber": map[string]any{"id": phoneID},+ "mode": mode,+ }+ body, err := IntoReader(values)+ if err != nil {+ return err+ }+ opts := rest.Opts{+ Method: "POST",+ Path: "/verify/phone/securitycode",+ ExtraHeaders: s.GetAuthHeaders(map[string]string{}),+ RootURL: authEndpoint,+ Body: body,+ NoResponse: true,+ }+ _, err = s.Request(ctx, opts, nil, nil)+ if err == nil {+ if err := s.TrustSession(ctx); err != nil {+ return err+ }+ return nil+ }+ return fmt.Errorf("validateSMSCode: %w", err)+}+// TrustSession trusts the sessionfunc (s *Session) TrustSession(ctx context.Context) error {opts := rest.Opts{@@ -458,7 +782,7 @@ func (s *Session) TrustSession(ctx context.Context) error {ExtraHeaders: s.GetAuthHeaders(map[string]string{}),RootURL: authEndpoint,NoResponse: true,- ContentLength: common.Int64(0),+ ContentLength: int64Ptr(0),}_, err := s.Request(ctx, opts, nil, nil)@@ -476,25 +800,26 @@ func (s *Session) ValidateSession(ctx context.Context) error {Path: "/validate",ExtraHeaders: s.GetHeaders(map[string]string{}),RootURL: setupEndpoint,- ContentLength: common.Int64(0),+ ContentLength: int64Ptr(0),}_, err := s.Request(ctx, opts, nil, &s.AccountInfo)if err != nil {return fmt.Errorf("validateSession failed: %w", err)}+ s.needs2FA = falsereturn nil}-// GetAuthHeaders returns the authentication headers for the session.-// Used for 2FA validation and trust requests to idmsa.apple.com.+// GetAuthHeaders returns the authentication headers for the session+// Used for 2FA validation and trust requests to idmsa.apple.comfunc (s *Session) GetAuthHeaders(overwrite map[string]string) map[string]string {headers := s.getSRPAuthHeaders()maps.Copy(headers, overwrite)return headers}-// GetHeaders Gets the authentication headers required for a request+// GetHeaders returns the authentication headers required for a requestfunc (s *Session) GetHeaders(overwrite map[string]string) map[string]string {headers := GetCommonHeaders(map[string]string{})headers["Cookie"] = s.GetCookieString()@@ -502,17 +827,26 @@ func (s *Session) GetHeaders(overwrite map[string]string) map[string]string {return headers}-// GetCookieString returns the cookie header string for the session.+// GetCookieString returns the cookie header string for the sessionfunc (s *Session) GetCookieString() string {- cookieHeader := ""- // we only care about name and value.+ var b strings.Builder+ first := truefor _, cookie := range s.Cookies {- cookieHeader = cookieHeader + cookie.Name + "=" + cookie.Value + ";"+ if cookie.Value == "" {+ continue+ }+ if !first {+ b.WriteString("; ")+ }+ first = false+ b.WriteString(cookie.Name)+ b.WriteByte('=')+ b.WriteString(cookie.Value)}- return cookieHeader+ return b.String()}-// GetCommonHeaders generates common HTTP headers with optional overwrite.+// GetCommonHeaders generates common HTTP headers with optional overwritefunc GetCommonHeaders(overwrite map[string]string) map[string]string {headers := map[string]string{"Content-Type": "application/json",@@ -524,32 +858,9 @@ func GetCommonHeaders(overwrite map[string]string) map[string]string {return headers}-// MergeCookies merges two slices of http.Cookies, ensuring no duplicates are added.-func MergeCookies(left []*http.Cookie, right []*http.Cookie) ([]*http.Cookie, error) {- var hashes []string- for _, cookie := range right {- hashes = append(hashes, cookie.Raw)- }- for _, cookie := range left {- if !slices.Contains(hashes, cookie.Raw) {- right = append(right, cookie)- }- }- return right, nil-}--// GetCookiesForDomain filters the provided cookies based on the domain of the given URL.-func GetCookiesForDomain(url *url.URL, cookies []*http.Cookie) ([]*http.Cookie, error) {- var domainCookies []*http.Cookie- for _, cookie := range cookies {- if strings.HasSuffix(url.Host, cookie.Domain) {- domainCookies = append(domainCookies, cookie)- }- }- return domainCookies, nil-}+func int64Ptr(v int64) *int64 { return &v }-// NewSession creates a new Session instance with default values.+// NewSession creates a new Session instance with default valuesfunc NewSession() *Session {session := &Session{FrameID: strings.ToLower(uuid.New().String()),@@ -564,117 +875,20 @@ func NewSession() *Session {return session}-// AccountInfo represents an account info+// AccountInfo represents the subset of Apple's account response we actually use+// json.Unmarshal silently ignores extra fields in the responsetype AccountInfo struct {- DsInfo *ValidateDataDsInfo `json:"dsInfo"`- HasMinimumDeviceForPhotosWeb bool `json:"hasMinimumDeviceForPhotosWeb"`- ICDPEnabled bool `json:"iCDPEnabled"`- Webservices map[string]*webService `json:"webservices"`- PcsEnabled bool `json:"pcsEnabled"`- TermsUpdateNeeded bool `json:"termsUpdateNeeded"`- ConfigBag struct {- Urls struct {- AccountCreateUI string `json:"accountCreateUI"`- AccountLoginUI string `json:"accountLoginUI"`- AccountLogin string `json:"accountLogin"`- AccountRepairUI string `json:"accountRepairUI"`- DownloadICloudTerms string `json:"downloadICloudTerms"`- RepairDone string `json:"repairDone"`- AccountAuthorizeUI string `json:"accountAuthorizeUI"`- VettingURLForEmail string `json:"vettingUrlForEmail"`- AccountCreate string `json:"accountCreate"`- GetICloudTerms string `json:"getICloudTerms"`- VettingURLForPhone string `json:"vettingUrlForPhone"`- } `json:"urls"`- AccountCreateEnabled bool `json:"accountCreateEnabled"`- } `json:"configBag"`- HsaTrustedBrowser bool `json:"hsaTrustedBrowser"`- AppsOrder []string `json:"appsOrder"`- Version int `json:"version"`- IsExtendedLogin bool `json:"isExtendedLogin"`- PcsServiceIdentitiesIncluded bool `json:"pcsServiceIdentitiesIncluded"`- IsRepairNeeded bool `json:"isRepairNeeded"`- HsaChallengeRequired bool `json:"hsaChallengeRequired"`- RequestInfo struct {- Country string `json:"country"`- TimeZone string `json:"timeZone"`- Region string `json:"region"`- } `json:"requestInfo"`- PcsDeleted bool `json:"pcsDeleted"`- ICloudInfo struct {- SafariBookmarksHasMigratedToCloudKit bool `json:"SafariBookmarksHasMigratedToCloudKit"`- } `json:"iCloudInfo"`- Apps map[string]*ValidateDataApp `json:"apps"`-}--// ValidateDataDsInfo represents an validation info-type ValidateDataDsInfo struct {- HsaVersion int `json:"hsaVersion"`- LastName string `json:"lastName"`- ICDPEnabled bool `json:"iCDPEnabled"`- TantorMigrated bool `json:"tantorMigrated"`- Dsid string `json:"dsid"`- HsaEnabled bool `json:"hsaEnabled"`- IsHideMyEmailSubscriptionActive bool `json:"isHideMyEmailSubscriptionActive"`- IroncadeMigrated bool `json:"ironcadeMigrated"`- Locale string `json:"locale"`- BrZoneConsolidated bool `json:"brZoneConsolidated"`- ICDRSCapableDeviceList string `json:"ICDRSCapableDeviceList"`- IsManagedAppleID bool `json:"isManagedAppleID"`- IsCustomDomainsFeatureAvailable bool `json:"isCustomDomainsFeatureAvailable"`- IsHideMyEmailFeatureAvailable bool `json:"isHideMyEmailFeatureAvailable"`- ContinueOnDeviceEligibleDeviceInfo []string `json:"ContinueOnDeviceEligibleDeviceInfo"`- Gilligvited bool `json:"gilligvited"`- AppleIDAliases []any `json:"appleIdAliases"`- UbiquityEOLEnabled bool `json:"ubiquityEOLEnabled"`- IsPaidDeveloper bool `json:"isPaidDeveloper"`- CountryCode string `json:"countryCode"`- NotificationID string `json:"notificationId"`- PrimaryEmailVerified bool `json:"primaryEmailVerified"`- ADsID string `json:"aDsID"`- Locked bool `json:"locked"`- ICDRSCapableDeviceCount int `json:"ICDRSCapableDeviceCount"`- HasICloudQualifyingDevice bool `json:"hasICloudQualifyingDevice"`- PrimaryEmail string `json:"primaryEmail"`- AppleIDEntries []struct {- IsPrimary bool `json:"isPrimary"`- Type string `json:"type"`- Value string `json:"value"`- } `json:"appleIdEntries"`- GilliganEnabled bool `json:"gilligan-enabled"`- IsWebAccessAllowed bool `json:"isWebAccessAllowed"`- FullName string `json:"fullName"`- MailFlags struct {- IsThreadingAvailable bool `json:"isThreadingAvailable"`- IsSearchV2Provisioned bool `json:"isSearchV2Provisioned"`- SCKMail bool `json:"sCKMail"`- IsMppSupportedInCurrentCountry bool `json:"isMppSupportedInCurrentCountry"`- } `json:"mailFlags"`- LanguageCode string `json:"languageCode"`- AppleID string `json:"appleId"`- HasUnreleasedOS bool `json:"hasUnreleasedOS"`- AnalyticsOptInStatus bool `json:"analyticsOptInStatus"`- FirstName string `json:"firstName"`- ICloudAppleIDAlias string `json:"iCloudAppleIdAlias"`- NotesMigrated bool `json:"notesMigrated"`- BeneficiaryInfo struct {- IsBeneficiary bool `json:"isBeneficiary"`- } `json:"beneficiaryInfo"`- HasPaymentInfo bool `json:"hasPaymentInfo"`- PcsDelet bool `json:"pcsDelet"`- AppleIDAlias string `json:"appleIdAlias"`- BrMigrated bool `json:"brMigrated"`- StatusCode int `json:"statusCode"`- FamilyEligible bool `json:"familyEligible"`-}--// ValidateDataApp represents an app-type ValidateDataApp struct {- CanLaunchWithOneFactor bool `json:"canLaunchWithOneFactor"`- IsQualifiedForBeta bool `json:"isQualifiedForBeta"`-}--// WebService represents a web service+ DsInfo *dsInfo `json:"dsInfo"`+ Webservices map[string]*webService `json:"webservices"`+ HsaChallengeRequired bool `json:"hsaChallengeRequired"`+}++// dsInfo holds the account metadata fields we read+type dsInfo struct {+ HsaVersion int `json:"hsaVersion"`+}++// webService represents a web servicetype webService struct {PcsRequired bool `json:"pcsRequired"`URL string `json:"url"`diff --git a/backend/iclouddrive/api/session_test.go b/backend/iclouddrive/api/session_test.gonew file mode 100644index 000000000..c3d06a37b--- /dev/null+++ b/backend/iclouddrive/api/session_test.go@@ -0,0 +1,40 @@+package api++import (+ "net/http"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestExtractHeadersMergesCookies(t *testing.T) {+ s := NewSession()+ s.Cookies = []*http.Cookie{{Name: "existing", Value: "old"}}++ resp := &http.Response{Header: make(http.Header)}+ resp.Header.Add("Set-Cookie", (&http.Cookie{Name: "existing", Value: "new"}).String())+ resp.Header.Add("Set-Cookie", (&http.Cookie{Name: "fresh", Value: "value"}).String())+ resp.Header.Set("X-Apple-Session-Token", "session-token")++ s.extractHeaders(resp)++ require.Len(t, s.Cookies, 2)+ assert.Equal(t, "new", s.Cookies[0].Value)+ assert.Equal(t, "fresh", s.Cookies[1].Name)+ assert.Equal(t, "session-token", s.SessionToken)+}++func TestExtractHeadersDeletesEmptyCookies(t *testing.T) {+ s := NewSession()+ s.Cookies = []*http.Cookie{{Name: "X-APPLE-WEBAUTH-HSA-LOGIN", Value: "stale"}, {Name: "keep", Value: "value"}}++ resp := &http.Response{Header: make(http.Header)}+ resp.Header.Add("Set-Cookie", (&http.Cookie{Name: "X-APPLE-WEBAUTH-HSA-LOGIN", Value: ""}).String())++ s.extractHeaders(resp)++ require.Len(t, s.Cookies, 1)+ assert.Equal(t, "keep", s.Cookies[0].Name)+ assert.Equal(t, "keep=value", s.GetCookieString())+}diff --git a/backend/iclouddrive/api/srp.go b/backend/iclouddrive/api/srp.goindex e2f7c7f66..9ca50eea4 100644--- a/backend/iclouddrive/api/srp.go+++ b/backend/iclouddrive/api/srp.go@@ -46,10 +46,10 @@ type srpClient struct {}// newSRPClient generates a random 32-byte secret and computes the public value A.-func newSRPClient() *srpClient {+func newSRPClient() (*srpClient, error) {secret := make([]byte, 32)if _, err := rand.Read(secret); err != nil {- panic(fmt.Sprintf("srp: rand.Read failed: %v", err))+ return nil, fmt.Errorf("srp: rand.Read failed: %w", err)}a := new(big.Int).SetBytes(secret)@@ -60,7 +60,7 @@ func newSRPClient() *srpClient {a: a,A: A,k: k,- }+ }, nil}// getABytes returns the padded public value A.@@ -71,16 +71,21 @@ func (c *srpClient) getABytes() []byte {// processChallenge computes the session key and proof values from the server challenge.// username is the Apple ID, derivedKey is the output of derivePassword, salt and B come// from the server's init response (raw bytes, already base64-decoded).-func (c *srpClient) processChallenge(username, derivedKey, salt, serverB []byte) {+func (c *srpClient) processChallenge(username, derivedKey, salt, serverB []byte) error {B := new(big.Int).SetBytes(serverB)- // Validate B+ // Validate B (RFC 5054)if B.Cmp(big.NewInt(0)) <= 0 || B.Cmp(srpN) >= 0 {- panic("srp: invalid server-supplied B, must be 1..N-1")+ return fmt.Errorf("srp: invalid server-supplied B, must be 1..N-1")}x := calculateX(salt, derivedKey)u := calculateU(c.A, B)++ // Validate u (RFC 5054)+ if u.Sign() == 0 {+ return fmt.Errorf("srp: calculated u is zero, aborting")+ }S := calculateS(c.k, x, c.a, B, u)c.K = calculateK(S)@@ -88,6 +93,7 @@ func (c *srpClient) processChallenge(username, derivedKey, salt, serverB []byte)bBytes := padToN(B)c.M1 = calculateM1(username, salt, aBytes, bBytes, c.K)c.M2 = calculateM2(aBytes, c.M1, c.K)+ return nil}// derivePassword performs Apple's password key derivation.@@ -98,10 +104,12 @@ func derivePassword(password string, salt []byte, iterations int, protocol strinvar passInput stringswitch protocol {+ case "s2k":+ passInput = string(passHash[:])case "s2k_fo":passInput = hex.EncodeToString(passHash[:])- default: // "s2k"- passInput = string(passHash[:])+ default:+ return nil, fmt.Errorf("unsupported SRP protocol: %q", protocol)}return pbkdf2.Key(sha256.New, passInput, salt, iterations, 32)}@@ -142,7 +150,7 @@ func getMultiplier() *big.Int {}// calculateX computes x = H(salt | H(":" | password))-// Apple's variant: NoUserNameInX — username is omitted from the inner hash.+// Apple's variant: NoUserNameInX - username is omitted from the inner hash.// The "password" here is actually the derived key from derivePassword.func calculateX(salt, derivedKey []byte) *big.Int {h := srpHashFunc()diff --git a/backend/iclouddrive/api/srp_test.go b/backend/iclouddrive/api/srp_test.goindex 3136eda55..5eec3c011 100644--- a/backend/iclouddrive/api/srp_test.go+++ b/backend/iclouddrive/api/srp_test.go@@ -27,7 +27,7 @@ func TestPadToN(t *testing.T) {})t.Run("preserves value that fills N", func(t *testing.T) {- // Use N itself — already 256 bytes+ // Use N itself - already 256 bytesresult := padToN(srpN)assert.Equal(t, srpNLenBytes, len(result))assert.Equal(t, srpN.Bytes(), result)@@ -52,7 +52,7 @@ func TestGetMultiplier(t *testing.T) {k2 := getMultiplier()assert.Equal(t, 0, k.Cmp(k2), "multiplier must be deterministic")- // k = H(N | pad(g)) — verify by manual computation+ // k = H(N | pad(g)) - verify by manual computationh := srpHashFunc()nBytes := srpN.Bytes()gBytes := srpG.Bytes()@@ -123,6 +123,12 @@ func TestDerivePassword(t *testing.T) {require.NoError(t, err)assert.Equal(t, 32, len(key))})++ t.Run("unknown protocol returns error", func(t *testing.T) {+ _, err := derivePassword(password, salt, 1000, "unknown")+ assert.Error(t, err)+ assert.Contains(t, err.Error(), "unsupported SRP protocol")+ })}func TestCalculateX(t *testing.T) {@@ -256,7 +262,8 @@ func TestCalculateM2(t *testing.T) {}func TestNewSRPClient(t *testing.T) {- client := newSRPClient()+ client, err := newSRPClient()+ require.NoError(t, err)// A must be non-zero and in range [1, N-1]assert.True(t, client.A.Sign() > 0, "A must be positive")@@ -273,7 +280,8 @@ func TestNewSRPClient(t *testing.T) {assert.Equal(t, 0, client.A.Cmp(recoveredA))// Two clients should have different secrets (probabilistic, but 2^256 collision chance)- client2 := newSRPClient()+ client2, err := newSRPClient()+ require.NoError(t, err)assert.NotEqual(t, 0, client.a.Cmp(client2.a), "two clients must have different secrets")}@@ -315,7 +323,8 @@ func TestProcessChallenge(t *testing.T) {serverB := padToN(B)- client.processChallenge(username, derivedKey, salt, serverB)+ err := client.processChallenge(username, derivedKey, salt, serverB)+ require.NoError(t, err)// M1, M2, K must be populatedassert.NotNil(t, client.M1)@@ -328,13 +337,14 @@ func TestProcessChallenge(t *testing.T) {// M1 and M2 must be differentassert.NotEqual(t, client.M1, client.M2)- // Results must be deterministic — run again with same inputs+ // Results must be deterministic - run again with same inputsclient2 := &srpClient{a: new(big.Int).Set(a),A: new(big.Int).Set(A),k: new(big.Int).Set(k),}- client2.processChallenge(username, derivedKey, salt, serverB)+ err = client2.processChallenge(username, derivedKey, salt, serverB)+ require.NoError(t, err)assert.Equal(t, client.M1, client2.M1, "M1 must be deterministic")assert.Equal(t, client.M2, client2.M2, "M2 must be deterministic")@@ -352,6 +362,28 @@ func TestProcessChallenge(t *testing.T) {assert.Equal(t, client.K, kServer, "client and server must derive the same session key")}+func TestProcessChallenge_InvalidB(t *testing.T) {+ k := getMultiplier()+ client := &srpClient{+ a: big.NewInt(42),+ A: new(big.Int).Exp(srpG, big.NewInt(42), srpN),+ k: k,+ }+ username := []byte("[email protected]")+ derivedKey := make([]byte, 32)+ salt := []byte("salt")++ t.Run("B=0 rejected", func(t *testing.T) {+ err := client.processChallenge(username, derivedKey, salt, padToN(big.NewInt(0)))+ assert.Error(t, err)+ })++ t.Run("B=N rejected", func(t *testing.T) {+ err := client.processChallenge(username, derivedKey, salt, padToN(srpN))+ assert.Error(t, err)+ })+}+func TestSRPGroupParameters(t *testing.T) {// Verify the 2048-bit prime starts with known hex prefixnHex := hex.EncodeToString(srpN.Bytes())diff --git a/backend/iclouddrive/icloud.go b/backend/iclouddrive/icloud.gonew file mode 100644index 000000000..40ee6c949--- /dev/null+++ b/backend/iclouddrive/icloud.go@@ -0,0 +1,495 @@+//go:build !plan9 && !solaris++// Package iclouddrive provides access to iCloud Drive and Photos+package iclouddrive++import (+ "context"+ "encoding/base64"+ "encoding/json"+ "errors"+ "fmt"+ "net/http"+ "os"+ "strconv"+ "strings"++ "github.com/rclone/rclone/backend/iclouddrive/api"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config"+ "github.com/rclone/rclone/fs/config/configmap"+ "github.com/rclone/rclone/fs/config/configstruct"+ "github.com/rclone/rclone/fs/config/obscure"+ "github.com/rclone/rclone/lib/encoder"+)++const configAuthSession = "_auth_session"++// configAuthState holds session fields that must be preserved across+// Config() state machine steps (push 2FA and SMS both need it)+type configAuthState struct {+ Scnt string `json:"s"`+ SessionID string `json:"i"`+ AuthAttributes string `json:"a"`+ FrameID string `json:"f"`+ SessionToken string `json:"t"`+ ClientID string `json:"c"`+ AccountCountry string `json:"ac"`+ Phones []smsPhone `json:"p,omitempty"`+}++type smsPhone struct {+ ID int `json:"id"`+ Num string `json:"n"`+ Mode string `json:"m"`+}++func saveAuthSession(m configmap.Mapper, s *api.Session, phones []smsPhone) {+ st := configAuthState{+ Scnt: s.Scnt,+ SessionID: s.SessionID,+ AuthAttributes: s.AuthAttributes,+ FrameID: s.FrameID,+ SessionToken: s.SessionToken,+ ClientID: s.ClientID,+ AccountCountry: s.AccountCountry,+ Phones: phones,+ }+ data, err := json.Marshal(st)+ if err != nil {+ fs.Debugf(nil, "iclouddrive: failed to marshal auth session: %v", err)+ return+ }+ m.Set(configAuthSession, base64.StdEncoding.EncodeToString(data))+}++func loadAuthSession(m configmap.Mapper) (*configAuthState, error) {+ raw, _ := m.Get(configAuthSession)+ if raw == "" {+ return nil, errors.New("auth session state lost, please reconfigure")+ }+ data, err := base64.StdEncoding.DecodeString(raw)+ if err != nil {+ return nil, fmt.Errorf("corrupt auth session state: %w", err)+ }+ var st configAuthState+ if err := json.Unmarshal(data, &st); err != nil {+ return nil, fmt.Errorf("invalid auth session state: %w", err)+ }+ return &st, nil+}++func restoreAuthSession(icloud *api.Client, st *configAuthState) {+ icloud.Session.Scnt = st.Scnt+ icloud.Session.SessionID = st.SessionID+ icloud.Session.AuthAttributes = st.AuthAttributes+ icloud.Session.FrameID = st.FrameID+ icloud.Session.SessionToken = st.SessionToken+ icloud.Session.ClientID = st.ClientID+ icloud.Session.AccountCountry = st.AccountCountry+}++// resumeConfigClient recreates an API client and restores session state saved+// from a previous Config() step. Used across 2FA states to avoid re-authenticating+func resumeConfigClient(m configmap.Mapper, appleid, password, trustToken, clientID string, cookies []*http.Cookie) (*api.Client, *configAuthState, error) {+ st, err := loadAuthSession(m)+ if err != nil {+ return nil, nil, err+ }+ icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil, "_config")+ if err != nil {+ return nil, nil, err+ }+ restoreAuthSession(icloud, st)+ return icloud, st, nil+}++// saveAuthCredentials persists trust token, cookies, and clears session state+// after successful 2FA validation+func saveAuthCredentials(m configmap.Mapper, icloud *api.Client, name string) {+ m.Set(configTrustToken, icloud.Session.TrustToken)+ m.Set(configCookies, icloud.Session.GetCookieString())+ m.Set(configAuthSession, "")+ api.ClearCacheDir(name)+}++// triggerSMSFlow handles phone selection and SMS triggering+// Single phone: triggers SMS immediately, returns code prompt+// Multiple phones: saves session + phone list, returns phone picker+func triggerSMSFlow(ctx context.Context, icloud *api.Client, phones []api.TrustedPhoneNumber, m configmap.Mapper) (*fs.ConfigOut, error) {+ smsPhones := make([]smsPhone, len(phones))+ for i, p := range phones {+ mode := p.PushMode+ if mode == "" {+ mode = "sms"+ }+ smsPhones[i] = smsPhone{ID: p.ID, Num: p.NumberWithDialCode, Mode: mode}+ }++ if len(phones) == 1 {+ p := smsPhones[0]+ if err := icloud.Session.RequestSMSCode(ctx, p.ID, p.Mode); err != nil {+ return nil, fmt.Errorf("failed to send SMS code: %w", err)+ }+ saveAuthSession(m, icloud.Session, nil)+ nextState := fmt.Sprintf("2fa_sms_%d_%s", p.ID, p.Mode)+ return fs.ConfigInput(nextState, "config_2fa_sms", fmt.Sprintf("Enter the verification code sent to %s", p.Num))+ }++ // Multiple phones - save session and present picker+ saveAuthSession(m, icloud.Session, smsPhones)+ items := make([]fs.OptionExample, len(smsPhones))+ for i, p := range smsPhones {+ items[i] = fs.OptionExample{+ Value: fmt.Sprintf("%d_%s", p.ID, p.Mode),+ Help: p.Num,+ }+ }+ return fs.ConfigChooseExclusiveFixed("2fa_sms_select", "config_2fa_phone", "Select phone number for SMS verification", items)+}++const (+ configService = "service"++ // Service types+ serviceDrive = "drive"+ servicePhotos = "photos"+)++// ServiceOptions defines the configuration for service selection+type ServiceOptions struct {+ Service string `config:"service"`+}++// Register with rclone+func init() {+ fs.Register(&fs.RegInfo{+ Name: "iclouddrive",+ Description: "iCloud Drive and Photos",+ Config: Config,+ NewFs: NewServiceFs,+ MetadataInfo: &fs.MetadataInfo{+ System: map[string]fs.MetadataHelp{+ "width": {+ Help: "Image width in pixels",+ Type: "int",+ ReadOnly: true,+ },+ "height": {+ Help: "Image height in pixels",+ Type: "int",+ ReadOnly: true,+ },+ "added-time": {+ Help: "Time the item was added to the iCloud library",+ Type: "RFC 3339",+ Example: "2006-01-02T15:04:05Z",+ ReadOnly: true,+ },+ "favorite": {+ Help: "Whether the item is marked as favorite",+ Type: "bool",+ ReadOnly: true,+ },+ "hidden": {+ Help: "Whether the item is hidden",+ Type: "bool",+ ReadOnly: true,+ },+ },+ Help: "Metadata is read-only and available for the Photos service only.",+ },+ Options: []fs.Option{{+ Name: configService,+ Help: "iCloud service to use.",+ Required: true,+ Default: serviceDrive,+ Examples: []fs.OptionExample{{+ Value: serviceDrive,+ Help: "iCloud Drive",+ }, {+ Value: servicePhotos,+ Help: "iCloud Photos",+ }},+ }, {+ Name: configAppleID,+ Help: "Apple ID.",+ Required: true,+ Sensitive: true,+ }, {+ Name: configPassword,+ Help: "Password.",+ Required: true,+ IsPassword: true,+ Sensitive: true,+ }, {+ Name: configTrustToken,+ Help: "Trust token for session authentication.",+ IsPassword: false,+ Required: false,+ Sensitive: true,+ Hide: fs.OptionHideBoth,+ }, {+ Name: configCookies,+ Help: "Session cookies.",+ Required: false,+ Advanced: false,+ Sensitive: true,+ Hide: fs.OptionHideBoth,+ }, {+ Name: configClientID,+ Help: "Client ID for iCloud API access.",+ Required: false,+ Advanced: true,+ Default: "d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d",+ }, {+ Name: config.ConfigEncoding,+ Help: config.ConfigEncodingHelp,+ Advanced: true,+ Default: (encoder.Display |+ encoder.EncodeBackSlash |+ encoder.EncodeSlash |+ encoder.EncodeInvalidUtf8),+ }},+ })+}++// Config handles the authentication and configuration flow+func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {+ var err error+ appleid, _ := m.Get(configAppleID)+ if appleid == "" {+ return nil, errors.New("an Apple ID is required")+ }++ password, _ := m.Get(configPassword)+ if password != "" {+ password, err = obscure.Reveal(password)+ if err != nil {+ return nil, err+ }+ }++ trustToken, _ := m.Get(configTrustToken)+ cookieRaw, _ := m.Get(configCookies)+ clientID, _ := m.Get(configClientID)+ cookies := ReadCookies(cookieRaw)++ switch {+ case config.State == "":+ // Force fresh SRP authentication - ignore stale trust token and cookies+ // so that reconnect always prompts for 2FA+ m.Set(configAuthSession, "")+ icloud, err := api.New(appleid, password, "", clientID, nil, nil, "_config")+ if err != nil {+ return nil, err+ }+ if err := icloud.Authenticate(ctx); err != nil {+ return nil, err+ }+ m.Set(configCookies, icloud.Session.GetCookieString())+ if icloud.Session.Requires2FA() {+ // Check if user has no trusted devices - auto-trigger SMS if so+ authState, err := icloud.Session.GetAuthState(ctx)+ if err == nil && authState.NoTrustedDevices && len(authState.TrustedPhoneNumbers) > 0 {+ return triggerSMSFlow(ctx, icloud, authState.TrustedPhoneNumbers, m)+ }+ // Explicitly request push to trusted devices - required for iOS 26.4++ // where the SRP 409 no longer auto-pushes. GET /appleauth/auth above+ // may also trigger a push (cosmetic double on pre-26.4, harmless)+ if err := icloud.Session.RequestPushNotification(ctx); err != nil {+ fs.Debugf(nil, "iclouddrive: push notification request failed (SMS fallback available): %v", err)+ } else {+ fs.Debugf(nil, "iclouddrive: push notification requested to trusted devices")+ }+ // Save session state so 2fa_do can validate without re-authenticating+ // Push codes are account-scoped so session reuse is not strictly required,+ // but it avoids a redundant SRP roundtrip and a second push on pre-26.4+ saveAuthSession(m, icloud.Session, nil)+ return fs.ConfigInput("2fa_do", "config_2fa", "Two-factor authentication: enter your 2FA code or type 'sms' for a text message")+ }+ // Auth succeeded without 2FA - save updated credentials and clear stale cache+ m.Set(configTrustToken, icloud.Session.TrustToken)+ api.ClearCacheDir(name)+ return nil, nil++ case config.State == "2fa_do":+ code := config.Result+ if code == "" {+ return fs.ConfigError("authenticate", "2FA codes can't be blank")+ }++ // Restore session from initial sign-in instead of re-authenticating+ // This avoids a redundant SRP roundtrip and extra push on pre-26.4+ icloud, _, err := resumeConfigClient(m, appleid, password, trustToken, clientID, cookies)+ if err != nil {+ return nil, err+ }++ if strings.EqualFold(code, "sms") {+ authState, err := icloud.Session.GetAuthState(ctx)+ if err != nil {+ return nil, fmt.Errorf("failed to get trusted phone numbers: %w", err)+ }+ if len(authState.TrustedPhoneNumbers) == 0 {+ return nil, errors.New("no trusted phone numbers on this account")+ }+ return triggerSMSFlow(ctx, icloud, authState.TrustedPhoneNumbers, m)+ }++ if err := icloud.Session.Validate2FACode(ctx, code); err != nil {+ return nil, err+ }+ saveAuthCredentials(m, icloud, name)+ return nil, nil++ case config.State == "2fa_sms_select":+ // User selected a phone from ConfigChooseExclusiveFixed (value: "ID_mode")+ idStr, mode, _ := strings.Cut(config.Result, "_")+ phoneID, err := strconv.Atoi(idStr)+ if err != nil {+ m.Set(configAuthSession, "")+ return nil, fmt.Errorf("invalid phone selection %q", config.Result)+ }+ if mode == "" {+ mode = "sms"+ }+ icloud, smsState, err := resumeConfigClient(m, appleid, password, trustToken, clientID, cookies)+ if err != nil {+ return nil, err+ }+ // Find the selected phone's display number+ var phoneNum string+ for _, p := range smsState.Phones {+ if p.ID == phoneID {+ phoneNum = p.Num+ break+ }+ }++ if err := icloud.Session.RequestSMSCode(ctx, phoneID, mode); err != nil {+ m.Set(configAuthSession, "")+ return nil, fmt.Errorf("failed to send SMS code: %w", err)+ }+ saveAuthSession(m, icloud.Session, nil)+ nextState := fmt.Sprintf("2fa_sms_%d_%s", phoneID, mode)+ return fs.ConfigInput(nextState, "config_2fa_sms", fmt.Sprintf("Enter the verification code sent to %s", phoneNum))++ case strings.HasPrefix(config.State, "2fa_sms_"):+ code := config.Result+ if code == "" {+ return fs.ConfigError("authenticate", "SMS code can't be blank")+ }+ // State encodes phone ID and mode: "2fa_sms_<ID>_<mode>"+ suffix := strings.TrimPrefix(config.State, "2fa_sms_")+ idStr, mode, _ := strings.Cut(suffix, "_")+ phoneID, err := strconv.Atoi(idStr)+ if err != nil {+ return nil, fmt.Errorf("invalid phone ID in state %q: %w", config.State, err)+ }+ if mode == "" {+ mode = "sms"+ }++ icloud, _, err := resumeConfigClient(m, appleid, password, trustToken, clientID, cookies)+ if err != nil {+ return nil, err+ }++ if err := icloud.Session.ValidateSMSCode(ctx, code, phoneID, mode); err != nil {+ m.Set(configAuthSession, "")+ return nil, err+ }+ saveAuthCredentials(m, icloud, name)+ return nil, nil++ default:+ return nil, fmt.Errorf("unknown state %q", config.State)+ }+}++// newICloudClient parses options, authenticates, and returns a ready client+// Shared between NewFs (Drive) and NewFsPhotos (Photos) to avoid duplication+func newICloudClient(ctx context.Context, name string, m configmap.Mapper) (*api.Client, *Options, error) {+ opt := new(Options)+ err := configstruct.Set(m, opt)+ if err != nil {+ return nil, nil, err+ }++ if opt.Password != "" {+ opt.Password, err = obscure.Reveal(opt.Password)+ if err != nil {+ return nil, nil, fmt.Errorf("couldn't decrypt user password: %w", err)+ }+ }++ if opt.TrustToken == "" {+ return nil, nil, fmt.Errorf("missing icloud trust token: try refreshing it with \"rclone config reconnect %s:\"", name)+ }++ cookies := ReadCookies(opt.Cookies)++ callback := func(session *api.Session) {+ m.Set(configCookies, session.GetCookieString())+ }++ icloud, err := api.New(+ opt.AppleID,+ opt.Password,+ opt.TrustToken,+ opt.ClientID,+ cookies,+ callback,+ name,+ )+ if err != nil {+ return nil, nil, err+ }++ if err := icloud.Authenticate(ctx); err != nil {+ return nil, nil, err+ }++ if icloud.Session.Requires2FA() {+ return nil, nil, errors.New("trust token expired, please reauth")+ }++ return icloud, opt, nil+}++// disconnectClient clears authentication state and removes disk caches+// Shared between Drive Fs and Photos Fs Disconnect() implementations+func disconnectClient(m configmap.Mapper, icloud *api.Client) error {+ m.Set(configTrustToken, "")+ m.Set(configCookies, "")+ m.Set(configAuthSession, "")+ return os.RemoveAll(icloud.CacheDir())+}++// NewServiceFs creates a filesystem instance for the selected service+func NewServiceFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {+ // Parse the service selection+ opt := new(ServiceOptions)+ err := configstruct.Set(m, opt)+ if err != nil {+ return nil, err+ }++ // Set default service if not specified+ if opt.Service == "" {+ opt.Service = serviceDrive+ }++ // Route to the appropriate backend+ switch opt.Service {+ case serviceDrive:+ // Create Drive filesystem+ return NewFs(ctx, name, root, m)+ case servicePhotos:+ // Create Photos filesystem+ return NewFsPhotos(ctx, name, root, m)+ default:+ return nil, fmt.Errorf("invalid service selection: %s (must be 'drive' or 'photos')", opt.Service)+ }+}diff --git a/backend/iclouddrive/iclouddrive.go b/backend/iclouddrive/iclouddrive.goindex 1a51b9ab8..135ec75e9 100644--- a/backend/iclouddrive/iclouddrive.go+++ b/backend/iclouddrive/iclouddrive.go@@ -9,17 +9,13 @@ import ("path""errors"- "fmt""io""net/http""strings""time""github.com/rclone/rclone/fs"- "github.com/rclone/rclone/fs/config""github.com/rclone/rclone/fs/config/configmap"- "github.com/rclone/rclone/fs/config/configstruct"- "github.com/rclone/rclone/fs/config/obscure""github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/backend/iclouddrive/api"@@ -51,61 +47,10 @@ const (decayConstant = 2)-// Register with Fs-func init() {- fs.Register(&fs.RegInfo{- Name: "iclouddrive",- Description: "iCloud Drive",- Config: Config,- NewFs: NewFs,- Options: []fs.Option{{- Name: configAppleID,- Help: "Apple ID.",- Required: true,- Sensitive: true,- }, {- Name: configPassword,- Help: "Password.",- Required: true,- IsPassword: true,- Sensitive: true,- }, {- Name: configTrustToken,- Help: "Trust token (internal use)",- IsPassword: false,- Required: false,- Sensitive: true,- Hide: fs.OptionHideBoth,- }, {- Name: configCookies,- Help: "cookies (internal use only)",- Required: false,- Advanced: false,- Sensitive: true,- Hide: fs.OptionHideBoth,- }, {- Name: configClientID,- Help: "Client id",- Required: false,- Advanced: true,- Default: "d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d",- }, {- Name: config.ConfigEncoding,- Help: config.ConfigEncodingHelp,- Advanced: true,- Default: (encoder.Display |- //encoder.EncodeDot |- encoder.EncodeBackSlash |- encoder.EncodeInvalidUtf8),- }},- })-}-// Options defines the configuration for this backendtype Options struct {AppleID string `config:"apple_id"`Password string `config:"password"`- Photos bool `config:"photos"`TrustToken string `config:"trust_token"`Cookies string `config:"cookies"`ClientID string `config:"client_id"`@@ -118,6 +63,7 @@ type Fs struct {root string // the path we are working on.rootID stringopt Options // parsed config options+ m configmap.Mapper // config map for persisting auth statefeatures *fs.Features // optional featuresdirCache *dircache.DirCache // Map of directory path to directory idicloud *api.Client@@ -139,72 +85,6 @@ type Object struct {downloadURL string}-// Config configures the iCloud remote.-func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {- var err error- appleid, _ := m.Get(configAppleID)- if appleid == "" {- return nil, errors.New("a apple ID is required")- }-- password, _ := m.Get(configPassword)- if password != "" {- password, err = obscure.Reveal(password)- if err != nil {- return nil, err- }- }-- trustToken, _ := m.Get(configTrustToken)- cookieRaw, _ := m.Get(configCookies)- clientID, _ := m.Get(configClientID)- cookies := ReadCookies(cookieRaw)-- switch config.State {- case "":- icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil)- if err != nil {- return nil, err- }- if err := icloud.Authenticate(ctx); err != nil {- return nil, err- }- m.Set(configCookies, icloud.Session.GetCookieString())- if icloud.Session.Requires2FA() {- return fs.ConfigInput("2fa_do", "config_2fa", "Two-factor authentication: please enter your 2FA code")- }- return nil, nil- case "2fa_do":- code := config.Result- if code == "" {- return fs.ConfigError("authenticate", "2FA codes can't be blank")- }-- icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil)- if err != nil {- return nil, err- }- if err := icloud.SignIn(ctx); err != nil {- return nil, err- }-- if err := icloud.Session.Validate2FACode(ctx, code); err != nil {- return nil, err- }-- m.Set(configTrustToken, icloud.Session.TrustToken)- m.Set(configCookies, icloud.Session.GetCookieString())- return nil, nil-- case "2fa_error":- if config.Result == "true" {- return fs.ConfigGoto("2fa")- }- return nil, errors.New("2fa authentication failed")- }- return nil, fmt.Errorf("unknown state %q", config.State)-}-// find item by path. Will not return any children for the itemfunc (f *Fs) findItem(ctx context.Context, dir string) (item *api.DriveItem, found bool, err error) {var resp *http.Response@@ -797,51 +677,11 @@ func retryResultUnknown(ctx context.Context, resp *http.Response, err error) (bo// NewFs constructs an Fs from the path, container:pathfunc NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {- // Parse config into Options struct- opt := new(Options)- err := configstruct.Set(m, opt)+ icloud, opt, err := newICloudClient(ctx, name, m)if err != nil {return nil, err}- if opt.Password != "" {- var err error- opt.Password, err = obscure.Reveal(opt.Password)- if err != nil {- return nil, fmt.Errorf("couldn't decrypt user password: %w", err)- }- }-- if opt.TrustToken == "" {- return nil, fmt.Errorf("missing icloud trust token: try refreshing it with \"rclone config reconnect %s:\"", name)- }-- cookies := ReadCookies(opt.Cookies)-- callback := func(session *api.Session) {- m.Set(configCookies, session.GetCookieString())- }-- icloud, err := api.New(- opt.AppleID,- opt.Password,- opt.TrustToken,- opt.ClientID,- cookies,- callback,- )- if err != nil {- return nil, err- }-- if err := icloud.Authenticate(ctx); err != nil {- return nil, err- }-- if icloud.Session.Requires2FA() {- return nil, errors.New("trust token expired, please reauth")- }-root = strings.Trim(root, "/")f := &Fs{@@ -850,6 +690,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, eicloud: icloud,rootID: "FOLDER::com.apple.CloudDocs::root",opt: *opt,+ m: m,pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),}f.features = (&fs.Features{@@ -1164,9 +1005,15 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opreturn nil}+// Disconnect clears authentication state and removes disk caches+func (f *Fs) Disconnect(ctx context.Context) error {+ return disconnectClient(f.m, f.icloud)+}+// Check interfaces are satisfiedvar (_ fs.Fs = &Fs{}+ _ fs.Disconnecter = (*Fs)(nil)_ fs.Mover = (*Fs)(nil)_ fs.Purger = (*Fs)(nil)_ fs.DirMover = (*Fs)(nil)diff --git a/backend/iclouddrive/icloudphotos.go b/backend/iclouddrive/icloudphotos.gonew file mode 100644index 000000000..922333f98--- /dev/null+++ b/backend/iclouddrive/icloudphotos.go@@ -0,0 +1,892 @@+//go:build !plan9 && !solaris++package iclouddrive++import (+ "context"+ "fmt"+ "io"+ "net/http"+ "path"+ "strconv"+ "strings"+ "sync"+ "time"++ "github.com/rclone/rclone/backend/iclouddrive/api"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config/configmap"+ "github.com/rclone/rclone/fs/fshttp"+ "github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/list"+ "github.com/rclone/rclone/lib/dircache"+ "github.com/rclone/rclone/lib/pacer"+)++const rootID = "photos-root"++// PhotosFs represents a remote iCloud Photos server+type PhotosFs struct {+ name string+ root string+ opt Options+ features *fs.Features+ icloud *api.Client+ m configmap.Mapper+ pacer *fs.Pacer+ dirCache *dircache.DirCache+ httpClient *http.Client+ startTime time.Time+ mu sync.Mutex+ photos *api.PhotosService+}++// PhotosObject describes an iCloud Photos object+type PhotosObject struct {+ fs *PhotosFs+ remote string+ size int64+ modTime time.Time+ masterID string // CloudKit recordName for fresh URL lookup (master or asset depending on resource)+ zone string // CloudKit zone (e.g. "PrimarySync")+ resourceKey string // CloudKit resource field (resOriginalRes or resOriginalVidComplRes)+ width int+ height int+ addedDate int64+ isFavorite bool+ isHidden bool+}++// NewFsPhotos constructs an Fs for Photos from the path, container:path+func NewFsPhotos(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {+ icloud, opt, err := newICloudClient(ctx, name, m)+ if err != nil {+ return nil, err+ }++ root = strings.Trim(root, "/")++ f := &PhotosFs{+ name: name,+ root: root,+ opt: *opt,+ icloud: icloud,+ m: m,+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),+ httpClient: fshttp.NewClient(ctx),+ startTime: time.Now(),+ }++ f.features = (&fs.Features{+ CanHaveEmptyDirectories: true,+ PartialUploads: false,+ ReadMimeType: false,+ ReadMetadata: true,+ }).Fill(ctx, f)++ f.dirCache = dircache.New(root, rootID, f)++ err = f.dirCache.FindRoot(ctx, false)+ if err != nil {+ // Check if root points to a file (e.g. PrimarySync/Videos/file.mp4)+ newRoot, remote := dircache.SplitPath(root)+ tempF := &PhotosFs{+ name: f.name,+ root: newRoot,+ opt: f.opt,+ features: f.features,+ icloud: f.icloud,+ pacer: f.pacer,+ httpClient: f.httpClient,+ startTime: f.startTime,+ photos: f.photos, // nil here; photosService() creates lazily on first use+ }+ tempF.dirCache = dircache.New(newRoot, rootID, tempF)+ if err2 := tempF.dirCache.FindRoot(ctx, false); err2 != nil {+ return f, nil+ }+ _, err2 := tempF.NewObject(ctx, remote)+ if err2 != nil {+ return f, nil+ }+ // Root is a file - adjust f to parent dir, signal ErrorIsFile+ f.root = newRoot+ f.dirCache = tempF.dirCache+ return f, fs.ErrorIsFile+ }++ return f, nil+}++// photosService returns the PhotosService, creating it on first call+func (f *PhotosFs) photosService(ctx context.Context) (*api.PhotosService, error) {+ f.mu.Lock()+ defer f.mu.Unlock()+ if f.photos == nil {+ var err error+ f.photos, err = api.NewPhotosService(ctx, f.icloud, f.pacer, shouldRetry)+ if err != nil {+ return nil, err+ }+ }+ return f.photos, nil+}++// Name of the remote (as passed into NewFs)+func (f *PhotosFs) Name() string {+ return f.name+}++// Root of the remote (as passed into NewFs)+func (f *PhotosFs) Root() string {+ return f.opt.Enc.ToStandardPath(f.root)+}++// String converts this Fs to a string+func (f *PhotosFs) String() string {+ return fmt.Sprintf("iCloud Photos root '%s'", f.root)+}++// Precision of the object storage system+func (f *PhotosFs) Precision() time.Duration {+ return time.Second+}++// Hashes returns the supported hash sets+func (f *PhotosFs) Hashes() hash.Set {+ return hash.Set(hash.None)+}++// Features returns the optional features of this Fs+func (f *PhotosFs) Features() *fs.Features {+ return f.features+}++// List the objects and directories in dir into entries+func (f *PhotosFs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {+ photosService, err := f.photosService(ctx)+ if err != nil {+ return nil, fmt.Errorf("failed to get photos service: %w", err)+ }++ dirID, err := f.dirCache.FindDir(ctx, dir, false)+ if err != nil {+ return nil, err+ }++ switch {+ case dirID == rootID:+ // List libraries+ libraries, err := photosService.GetLibraries(ctx)+ if err != nil {+ return nil, err+ }++ albumCounts, err := photosService.GetLibraryAlbumCounts(ctx)+ if err != nil {+ fs.Debugf(f, "Failed to get library album counts: %v", err)+ }++ for libraryName := range libraries {+ count := int64(-1)+ if albumCounts != nil {+ if c, ok := albumCounts[libraryName]; ok {+ count = c+ }+ }+ d := fs.NewDir(path.Join(dir, f.opt.Enc.ToStandardName(libraryName)), f.startTime).SetItems(count)+ entries = append(entries, d)+ }++ case strings.HasPrefix(dirID, "lib:"):+ // List albums in a library+ libraryName := strings.TrimPrefix(dirID, "lib:")+ libraries, err := photosService.GetLibraries(ctx)+ if err != nil {+ return nil, err+ }++ library, exists := libraries[libraryName]+ if !exists {+ return nil, fs.ErrorDirNotFound+ }++ albums, err := library.GetAlbums(ctx)+ if err != nil {+ return nil, err+ }++ counts, err := library.GetAlbumCounts(ctx)+ if err != nil {+ fs.Debugf(f, "Failed to get album counts for library %q: %v", libraryName, err)+ }++ for albumName := range albums {+ count := int64(-1)+ if counts != nil {+ if c, ok := counts[albumName]; ok {+ count = c+ }+ }+ d := fs.NewDir(path.Join(dir, f.opt.Enc.ToStandardName(albumName)), f.startTime).SetItems(count)+ entries = append(entries, d)+ }++ case strings.HasPrefix(dirID, "album:"):+ // List contents of an album or folder+ libraryName, albumPath, ok := parseAlbumDirID(dirID)+ if !ok {+ return nil, fs.ErrorDirNotFound+ }++ album, err := f.resolveAlbum(ctx, photosService, libraryName, albumPath)+ if err != nil {+ return nil, err+ }++ // Folders list their child albums as subdirectories+ if album.IsFolder {+ for childName, child := range album.Children {+ d := fs.NewDir(path.Join(dir, f.opt.Enc.ToStandardName(childName)), f.startTime)+ if child.IsFolder {+ d.SetItems(int64(len(child.Children)))+ }+ entries = append(entries, d)+ }+ return entries, nil+ }++ photos, err := album.GetPhotos(ctx)+ if err != nil {+ return nil, err+ }++ for _, photo := range photos {+ if photo.Filename == "" {+ continue+ }+ encodedName := f.opt.Enc.ToStandardName(photo.Filename)+ remotePath := encodedName+ if dir != "" {+ remotePath = path.Join(dir, encodedName)+ }+ o := f.newPhotosObject(remotePath, photo, libraryName)+ entries = append(entries, o)+ }++ default:+ return nil, fs.ErrorDirNotFound+ }++ return entries, nil+}++// NewObject finds the Object at remote. If it can't be found+// it returns the error fs.ErrorObjectNotFound+func (f *PhotosFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {+ dir, leaf := dircache.SplitPath(remote)+ filename := f.opt.Enc.FromStandardName(leaf)++ dirID, err := f.dirCache.FindDir(ctx, dir, false)+ if err != nil {+ fs.Debugf(f, "NewObject(%s): FindDir: %v", remote, err)+ return nil, fs.ErrorObjectNotFound+ }++ if !strings.HasPrefix(dirID, "album:") {+ return nil, fs.ErrorObjectNotFound+ }++ libraryName, albumPath, ok := parseAlbumDirID(dirID)+ if !ok {+ return nil, fs.ErrorObjectNotFound+ }++ photosService, err := f.photosService(ctx)+ if err != nil {+ return nil, fmt.Errorf("failed to get photos service: %w", err)+ }++ album, err := f.resolveAlbum(ctx, photosService, libraryName, albumPath)+ if err != nil || album.IsFolder {+ fs.Debugf(f, "NewObject(%s): album resolve failed or is folder", remote)+ return nil, fs.ErrorObjectNotFound+ }++ photo, err := album.GetPhotoByName(ctx, filename)+ if err != nil {+ fs.Debugf(f, "NewObject(%s): %v", remote, err)+ return nil, fs.ErrorObjectNotFound+ }++ return f.newPhotosObject(remote, photo, libraryName), nil+}++// newPhotosObject creates a PhotosObject from a Photo and zone+func (f *PhotosFs) newPhotosObject(remote string, photo *api.Photo, zone string) *PhotosObject {+ return &PhotosObject{+ fs: f,+ remote: remote,+ size: photo.Size,+ modTime: time.UnixMilli(photo.AssetDate),+ masterID: photo.ID,+ zone: zone,+ resourceKey: photo.ResourceKey,+ width: photo.Width,+ height: photo.Height,+ addedDate: photo.AddedDate,+ isFavorite: photo.IsFavorite,+ isHidden: photo.IsHidden,+ }+}++// Put is not supported for this read-only backend+func (f *PhotosFs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {+ return nil, fs.ErrorNotImplemented+}++// Mkdir is not supported for this read-only backend+func (f *PhotosFs) Mkdir(ctx context.Context, dir string) error {+ return fs.ErrorNotImplemented+}++// Rmdir is not supported for this read-only backend+func (f *PhotosFs) Rmdir(ctx context.Context, dir string) error {+ return fs.ErrorNotImplemented+}++// FindLeaf finds a directory of name leaf in the folder with ID pathID+func (f *PhotosFs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {+ decodedLeaf := f.opt.Enc.FromStandardName(leaf)++ photosService, err := f.photosService(ctx)+ if err != nil {+ return "", false, fmt.Errorf("failed to get photos service: %w", err)+ }++ if pathID == rootID {+ libraries, err := photosService.GetLibraries(ctx)+ if err != nil {+ return "", false, err+ }++ if _, exists := libraries[decodedLeaf]; exists {+ return "lib:" + decodedLeaf, true, nil+ }+ return "", false, nil+ }++ if strings.HasPrefix(pathID, "lib:") {+ libraryName := strings.TrimPrefix(pathID, "lib:")+ libraries, err := photosService.GetLibraries(ctx)+ if err != nil {+ return "", false, err+ }++ library, exists := libraries[libraryName]+ if !exists {+ return "", false, fs.ErrorDirNotFound+ }++ albums, err := library.GetAlbums(ctx)+ if err != nil {+ return "", false, err+ }++ if _, exists := albums[decodedLeaf]; exists {+ albumID := "album:" + libraryName + ":" + decodedLeaf+ return albumID, true, nil+ }+ return "", false, nil+ }++ if strings.HasPrefix(pathID, "album:") {+ // Check if this is a folder containing child albums+ libraryName, albumPath, ok := parseAlbumDirID(pathID)+ if !ok {+ return "", false, nil+ }++ album, err := f.resolveAlbum(ctx, photosService, libraryName, albumPath)+ if err != nil {+ return "", false, nil+ }++ if album.IsFolder {+ if _, exists := album.Children[decodedLeaf]; exists {+ childID := "album:" + libraryName + ":" + albumPath + "/" + decodedLeaf+ return childID, true, nil+ }+ }+ return "", false, nil+ }++ return "", false, nil+}++// CreateDir makes a directory with pathID as parent and name leaf+func (f *PhotosFs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {+ return "", fs.ErrorNotImplemented+}++// Fs returns the parent Fs+func (o *PhotosObject) Fs() fs.Info {+ return o.fs+}++// Return a string version+func (o *PhotosObject) String() string {+ if o == nil {+ return "<nil>"+ }+ return o.remote+}++// Remote returns the remote path+func (o *PhotosObject) Remote() string {+ return o.remote+}++// ModTime returns the modification time of the object+func (o *PhotosObject) ModTime(ctx context.Context) time.Time {+ return o.modTime+}++// Size returns the size of an object in bytes+func (o *PhotosObject) Size() int64 {+ return o.size+}++// Storable returns a boolean as to whether this object is storable+func (o *PhotosObject) Storable() bool {+ return true+}++// Hash returns the hash of an object returning a lowercase hex string+func (o *PhotosObject) Hash(ctx context.Context, ty hash.Type) (string, error) {+ return "", hash.ErrUnsupported+}++// SetModTime sets the modification time of the object+func (o *PhotosObject) SetModTime(ctx context.Context, modTime time.Time) error {+ return fs.ErrorCantSetModTime+}++// Metadata returns metadata for the photo object+func (o *PhotosObject) Metadata(ctx context.Context) (fs.Metadata, error) {+ metadata := make(fs.Metadata, 5)+ if o.width > 0 {+ metadata["width"] = strconv.Itoa(o.width)+ }+ if o.height > 0 {+ metadata["height"] = strconv.Itoa(o.height)+ }+ if o.addedDate > 0 {+ metadata["added-time"] = time.UnixMilli(o.addedDate).UTC().Format(time.RFC3339)+ }+ metadata["favorite"] = strconv.FormatBool(o.isFavorite)+ metadata["hidden"] = strconv.FormatBool(o.isHidden)+ return metadata, nil+}++// Open an object for read, fetching a fresh download URL via records/lookup+func (o *PhotosObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {+ photosService, err := o.fs.photosService(ctx)+ if err != nil {+ return nil, fmt.Errorf("failed to get photos service: %w", err)+ }++ downloadURL, err := photosService.LookupDownloadURL(ctx, o.masterID, o.zone, o.resourceKey)+ if err != nil {+ return nil, fmt.Errorf("failed to get download URL for %q: %w", o.remote, err)+ }++ fs.FixRangeOption(options, o.size)++ var resp *http.Response+ err = o.fs.pacer.Call(func() (bool, error) {+ if resp != nil {+ _ = resp.Body.Close()+ resp = nil+ }+ req, err := http.NewRequestWithContext(ctx, "GET", downloadURL, nil)+ if err != nil {+ return false, fmt.Errorf("failed to create request: %w", err)+ }+ fs.OpenOptionAddHTTPHeaders(req.Header, options)++ resp, err = o.fs.httpClient.Do(req)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, err+ }++ if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {+ _ = resp.Body.Close()+ return nil, fmt.Errorf("download %q failed: %s", o.remote, resp.Status)+ }++ return resp.Body, nil+}++// Update is not supported for this read-only backend+func (o *PhotosObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {+ return fs.ErrorNotImplemented+}++// Remove is not supported for this read-only backend+func (o *PhotosObject) Remove(ctx context.Context) error {+ return fs.ErrorNotImplemented+}++// resolveAlbum finds an album by path, traversing folder hierarchy+// albumPath can be "AlbumName" or "FolderName/AlbumName" for nested albums+func (f *PhotosFs) resolveAlbum(ctx context.Context, ps *api.PhotosService, libraryName, albumPath string) (*api.Album, error) {+ libraries, err := ps.GetLibraries(ctx)+ if err != nil {+ return nil, err+ }++ library, exists := libraries[libraryName]+ if !exists {+ return nil, fs.ErrorDirNotFound+ }++ albums, err := library.GetAlbums(ctx)+ if err != nil {+ return nil, err+ }++ return resolveAlbumPath(albums, albumPath)+}++// resolveAlbumPath finds an album inside an album tree by slash-separated path+func resolveAlbumPath(albums map[string]*api.Album, albumPath string) (*api.Album, error) {+ parts := strings.Split(albumPath, "/")+ current := albums+ for i, part := range parts {+ album, exists := current[part]+ if !exists {+ return nil, fs.ErrorDirNotFound+ }+ if i == len(parts)-1 {+ return album, nil+ }+ if !album.IsFolder || album.Children == nil {+ return nil, fs.ErrorDirNotFound+ }+ current = album.Children+ }+ return nil, fs.ErrorDirNotFound+}++// parseAlbumDirID extracts library and album names from "album:lib:album" dirID+func parseAlbumDirID(dirID string) (libraryName, albumName string, ok bool) {+ parts := strings.SplitN(strings.TrimPrefix(dirID, "album:"), ":", 2)+ if len(parts) != 2 {+ return "", "", false+ }+ return parts[0], parts[1], true+}++// DirCacheFlush resets the directory cache - used in testing as an+// optional interface+func (f *PhotosFs) DirCacheFlush() {+ f.dirCache.ResetRoot()+ // Also flush API-layer caches so albums/photos are re-fetched+ // DirCacheFlusher interface has no ctx param; use a bounded timeout+ // to avoid hanging indefinitely if photosService triggers first init+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()+ if ps, err := f.photosService(ctx); err == nil {+ ps.FlushCaches()+ }+}++// listRAlbumJob represents an album to be listed by a ListR worker+type listRAlbumJob struct {+ album *api.Album+ zone string+ dirPath string // path relative to Fs root (e.g. "Videos" when f.root="PrimarySync")+}++// ListR lists the objects and directories of the Fs starting from dir+// recursively into out, calling callback for each batch of entries+// Albums are listed in parallel using a goroutine pool+func (f *PhotosFs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) error {+ ps, err := f.photosService(ctx)+ if err != nil {+ return err+ }++ helper := list.NewHelper(callback)+ var mu sync.Mutex // protects helper (not thread-safe)++ addEntry := func(entry fs.DirEntry) error {+ mu.Lock()+ defer mu.Unlock()+ return helper.Add(entry)+ }++ // Resolve dir through dirCache - this respects f.root+ dirID, err := f.dirCache.FindDir(ctx, dir, false)+ if err != nil {+ return err+ }++ // Collect album jobs, scoped to the resolved directory+ var jobs []listRAlbumJob++ // collectAlbumJobs emits directory entries for albums/folders under basePath+ // and collects leaf albums as jobs for parallel photo listing+ // Recurses into folders to handle arbitrary nesting depth+ var collectAlbumJobs func(albums map[string]*api.Album, zone, basePath string) error+ collectAlbumJobs = func(albums map[string]*api.Album, zone, basePath string) error {+ for albumName, album := range albums {+ albumPath := path.Join(basePath, f.opt.Enc.ToStandardName(albumName))+ if err := addEntry(fs.NewDir(albumPath, f.startTime)); err != nil {+ return err+ }+ if album.IsFolder {+ if err := collectAlbumJobs(album.Children, zone, albumPath); err != nil {+ return err+ }+ } else {+ jobs = append(jobs, listRAlbumJob{album: album, zone: zone, dirPath: albumPath})+ }+ }+ return nil+ }++ switch {+ case dirID == rootID:+ // Root level - emit libraries, then all albums+ libraries, err := ps.GetLibraries(ctx)+ if err != nil {+ return err+ }+ for libName, lib := range libraries {+ libPath := path.Join(dir, f.opt.Enc.ToStandardName(libName))+ if err := addEntry(fs.NewDir(libPath, f.startTime)); err != nil {+ return err+ }+ albums, err := lib.GetAlbums(ctx)+ if err != nil {+ return err+ }+ if err := collectAlbumJobs(albums, libName, libPath); err != nil {+ return err+ }+ }++ case strings.HasPrefix(dirID, "lib:"):+ // Library level - emit and collect all albums+ libraryName := strings.TrimPrefix(dirID, "lib:")+ libraries, err := ps.GetLibraries(ctx)+ if err != nil {+ return err+ }+ library, exists := libraries[libraryName]+ if !exists {+ return fs.ErrorDirNotFound+ }+ albums, err := library.GetAlbums(ctx)+ if err != nil {+ return err+ }+ if err := collectAlbumJobs(albums, libraryName, dir); err != nil {+ return err+ }++ case strings.HasPrefix(dirID, "album:"):+ // Album or folder level+ libraryName, albumPath, ok := parseAlbumDirID(dirID)+ if !ok {+ return fs.ErrorDirNotFound+ }+ album, err := f.resolveAlbum(ctx, ps, libraryName, albumPath)+ if err != nil {+ return err+ }+ if album.IsFolder {+ if err := collectAlbumJobs(album.Children, libraryName, dir); err != nil {+ return err+ }+ } else {+ // Single album - just list its photos+ jobs = append(jobs, listRAlbumJob{album: album, zone: libraryName, dirPath: dir})+ }++ default:+ return fs.ErrorDirNotFound+ }++ if len(jobs) == 0 {+ return helper.Flush()+ }++ // Fan out album photo listing across goroutines+ ctx, cancel := context.WithCancel(ctx)+ defer cancel()++ jobCh := make(chan listRAlbumJob, len(jobs))+ for _, job := range jobs {+ jobCh <- job+ }+ close(jobCh)++ workers := fs.GetConfig(ctx).Checkers+ if len(jobs) < workers {+ workers = len(jobs)+ }+ errs := make(chan error, workers)++ for range workers {+ go func() {+ for job := range jobCh {+ photos, err := job.album.GetPhotos(ctx)+ if err != nil {+ errs <- err+ cancel()+ return+ }+ for _, photo := range photos {+ if photo.Filename == "" {+ continue+ }+ remotePath := path.Join(job.dirPath, f.opt.Enc.ToStandardName(photo.Filename))+ o := f.newPhotosObject(remotePath, photo, job.zone)+ if err := addEntry(o); err != nil {+ errs <- err+ cancel()+ return+ }+ }+ }+ errs <- nil+ }()+ }++ for range workers {+ if e := <-errs; e != nil && err == nil {+ err = e+ }+ }+ if err != nil {+ return err+ }++ return helper.Flush()+}++func notifyAlbumTree(albums map[string]*api.Album, base string, notifyFunc func(string, fs.EntryType)) {+ for name, album := range albums {+ albumPath := name+ if base != "" {+ albumPath = path.Join(base, name)+ }+ notifyFunc(albumPath, fs.EntryDirectory)+ if album.IsFolder {+ notifyAlbumTree(album.Children, albumPath, notifyFunc)+ }+ }+}++func (f *PhotosFs) notifyZoneChange(ctx context.Context, ps *api.PhotosService, zone string, notifyFunc func(string, fs.EntryType)) {+ if f.root == "" {+ notifyFunc(zone, fs.EntryDirectory)+ return+ }++ libraryName, albumPath, _ := strings.Cut(f.root, "/")+ if libraryName != zone {+ return+ }++ // Always invalidate the mounted root when its backing zone changes+ notifyFunc("", fs.EntryDirectory)++ libraries, err := ps.GetLibraries(ctx)+ if err != nil {+ return+ }+ library, ok := libraries[zone]+ if !ok {+ return+ }+ albums, err := library.GetAlbums(ctx)+ if err != nil {+ return+ }+ if albumPath == "" {+ notifyAlbumTree(albums, "", notifyFunc)+ return+ }++ album, err := resolveAlbumPath(albums, albumPath)+ if err != nil {+ return+ }+ if album.IsFolder {+ notifyAlbumTree(album.Children, "", notifyFunc)+ }+}++// ChangeNotify polls for changes and notifies the VFS when directories are modified+func (f *PhotosFs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {+ go func() {+ var ticker *time.Ticker+ var tickerC <-chan time.Time+ for {+ select {+ case pollInterval, ok := <-pollIntervalChan:+ if !ok {+ if ticker != nil {+ ticker.Stop()+ }+ return+ }+ if ticker != nil {+ ticker.Stop()+ tickerC = nil+ }+ if pollInterval > 0 {+ ticker = time.NewTicker(pollInterval)+ tickerC = ticker.C+ }+ case <-tickerC:+ ps, err := f.photosService(ctx)+ if err != nil {+ fs.Debugf(f, "ChangeNotify: failed to get photos service: %v", err)+ continue+ }+ changedZones := ps.PollForChanges(ctx)+ for _, zone := range changedZones {+ f.notifyZoneChange(ctx, ps, zone, notifyFunc)+ }+ case <-ctx.Done():+ if ticker != nil {+ ticker.Stop()+ }+ return+ }+ }+ }()+}++// Disconnect clears authentication state and removes disk caches+func (f *PhotosFs) Disconnect(ctx context.Context) error {+ return disconnectClient(f.m, f.icloud)+}++// Check interfaces are satisfied+var (+ _ fs.Fs = (*PhotosFs)(nil)+ _ fs.Disconnecter = (*PhotosFs)(nil)+ _ fs.ListRer = (*PhotosFs)(nil)+ _ fs.ChangeNotifier = (*PhotosFs)(nil)+ _ fs.DirCacheFlusher = (*PhotosFs)(nil)+ _ fs.Object = (*PhotosObject)(nil)+ _ fs.Metadataer = (*PhotosObject)(nil)+)diff --git a/backend/iclouddrive/icloudphotos_test.go b/backend/iclouddrive/icloudphotos_test.gonew file mode 100644index 000000000..1aa32146d--- /dev/null+++ b/backend/iclouddrive/icloudphotos_test.go@@ -0,0 +1,516 @@+//go:build !plan9 && !solaris++package iclouddrive++import (+ "context"+ "sort"+ "testing"+ "time"++ "github.com/rclone/rclone/backend/iclouddrive/api"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/dircache"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func newTestPhotosFs() *PhotosFs {+ return &PhotosFs{+ name: "test-icp",+ root: "",+ opt: Options{},+ }+}++func TestSmartAlbumDefinitions(t *testing.T) {+ expected := map[string]struct {+ ObjectType string+ ListType string+ }{+ "All Photos": {"CPLAssetByAssetDateWithoutHiddenOrDeleted", "CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted"},+ "Time-lapse": {"CPLAssetInSmartAlbumByAssetDate:Timelapse", "CPLAssetAndMasterInSmartAlbumByAssetDate"},+ "Videos": {"CPLAssetInSmartAlbumByAssetDate:Video", "CPLAssetAndMasterInSmartAlbumByAssetDate"},+ "Slo-mo": {"CPLAssetInSmartAlbumByAssetDate:Slomo", "CPLAssetAndMasterInSmartAlbumByAssetDate"},+ "Bursts": {"CPLAssetBurstStackAssetByAssetDate", "CPLBurstStackAssetAndMasterByAssetDate"},+ "Favorites": {"CPLAssetInSmartAlbumByAssetDate:Favorite", "CPLAssetAndMasterInSmartAlbumByAssetDate"},+ "Panoramas": {"CPLAssetInSmartAlbumByAssetDate:Panorama", "CPLAssetAndMasterInSmartAlbumByAssetDate"},+ "Screenshots": {"CPLAssetInSmartAlbumByAssetDate:Screenshot", "CPLAssetAndMasterInSmartAlbumByAssetDate"},+ "Live": {"CPLAssetInSmartAlbumByAssetDate:Live", "CPLAssetAndMasterInSmartAlbumByAssetDate"},+ "Recently Deleted": {"CPLAssetDeletedByExpungedDate", "CPLAssetAndMasterDeletedByExpungedDate"},+ "Hidden": {"CPLAssetHiddenByAssetDate", "CPLAssetAndMasterHiddenByAssetDate"},+ "Portrait": {"CPLAssetInSmartAlbumByAssetDate:Depth", "CPLAssetAndMasterInSmartAlbumByAssetDate"},+ "Long Exposure": {"CPLAssetInSmartAlbumByAssetDate:Exposure", "CPLAssetAndMasterInSmartAlbumByAssetDate"},+ "Animated": {"CPLAssetInSmartAlbumByAssetDate:Animated", "CPLAssetAndMasterInSmartAlbumByAssetDate"},+ }++ for name, exp := range expected {+ t.Run(name, func(t *testing.T) {+ album, exists := api.SmartAlbums[name]+ require.True(t, exists, "smart album %q must be defined", name)+ assert.Equal(t, exp.ObjectType, album.ObjectType)+ assert.Equal(t, exp.ListType, album.ListType)+ })+ }++ assert.Equal(t, len(expected), len(api.SmartAlbums))+}++func TestSmartAlbumFilters(t *testing.T) {+ filtered := map[string]string{+ "Time-lapse": "TIMELAPSE",+ "Videos": "VIDEO",+ "Slo-mo": "SLOMO",+ "Favorites": "FAVORITE",+ "Panoramas": "PANORAMA",+ "Screenshots": "SCREENSHOT",+ "Live": "LIVE",+ "Portrait": "DEPTH",+ "Long Exposure": "EXPOSURE",+ "Animated": "ANIMATED",+ }++ for name, filterVal := range filtered {+ t.Run(name, func(t *testing.T) {+ album := api.SmartAlbums[name]+ require.NotEmpty(t, album.Filters, "album %q must have filters", name)+ assert.Equal(t, "smartAlbum", album.Filters[0].FieldName)+ assert.Equal(t, "EQUALS", album.Filters[0].Comparator)+ fv, ok := album.Filters[0].FieldValue.(map[string]string)+ require.True(t, ok, "filter value must be map[string]string")+ assert.Equal(t, filterVal, fv["value"])+ })+ }++ unfiltered := []string{"All Photos", "Bursts", "Recently Deleted", "Hidden"}+ for _, name := range unfiltered {+ t.Run(name+"_no_filter", func(t *testing.T) {+ album := api.SmartAlbums[name]+ assert.Empty(t, album.Filters, "album %q must not have filters", name)+ })+ }+}++func TestParseAlbumDirID(t *testing.T) {+ lib, album, ok := parseAlbumDirID("album:PrimarySync:Favorites")+ assert.True(t, ok)+ assert.Equal(t, "PrimarySync", lib)+ assert.Equal(t, "Favorites", album)++ // Colon in album name+ lib, album, ok = parseAlbumDirID("album:PrimarySync:My:Album")+ assert.True(t, ok)+ assert.Equal(t, "PrimarySync", lib)+ assert.Equal(t, "My:Album", album)++ // Missing album+ _, _, ok = parseAlbumDirID("album:PrimarySync")+ assert.False(t, ok)++ // Empty+ _, _, ok = parseAlbumDirID("album:")+ assert.False(t, ok)+}++func TestPhotosObject_Metadata(t *testing.T) {+ o := &PhotosObject{+ fs: &PhotosFs{name: "test"},+ remote: "PrimarySync/All Photos/IMG_0001.HEIC",+ size: 4200000,+ modTime: time.Date(2025, 6, 15, 14, 30, 0, 0, time.UTC),+ masterID: "master-001",+ zone: "PrimarySync",+ width: 4032,+ height: 3024,+ addedDate: 1718459400000, // 2024-06-15T13:50:00Z in millis+ isFavorite: true,+ isHidden: false,+ }++ metadata, err := o.Metadata(context.Background())+ require.NoError(t, err)+ assert.Equal(t, "4032", metadata["width"])+ assert.Equal(t, "3024", metadata["height"])+ assert.Equal(t, "2024-06-15T13:50:00Z", metadata["added-time"])+ assert.Equal(t, "true", metadata["favorite"])+ assert.Equal(t, "false", metadata["hidden"])+}++func TestPhotosObject_MetadataZeroDimensions(t *testing.T) {+ // Live Photo .MOV companion - no width/height, not favorite+ o := &PhotosObject{+ fs: &PhotosFs{name: "test"},+ remote: "PrimarySync/Live/IMG_3031.MOV",+ }++ metadata, err := o.Metadata(context.Background())+ require.NoError(t, err)+ _, hasWidth := metadata["width"]+ _, hasHeight := metadata["height"]+ assert.False(t, hasWidth, "zero width should be omitted")+ assert.False(t, hasHeight, "zero height should be omitted")+ assert.Equal(t, "false", metadata["favorite"])+ assert.Equal(t, "false", metadata["hidden"])+}++// newTestPhotosService builds a PhotosService with pre-populated libraries and albums+// for testing resolveAlbum and FindLeaf without HTTP calls+func newTestPhotosService() *api.PhotosService {+ return api.NewTestPhotosService(map[string]map[string]*api.Album{+ "PrimarySync": {+ "All Photos": {Name: "All Photos", ObjectType: "CPLAssetByAssetDateWithoutHiddenOrDeleted"},+ "Videos": {Name: "Videos", ObjectType: "CPLAssetInSmartAlbumByAssetDate:Video"},+ "UserAlbum": {Name: "UserAlbum", ObjectType: "CPLContainerRelationNotDeletedByAssetDate:rec1", RecordName: "rec1"},+ "Folder": {+ Name: "Folder", RecordName: "folder1", IsFolder: true,+ Children: map[string]*api.Album{+ "ChildAlbum": {Name: "ChildAlbum", ObjectType: "CPLContainerRelationNotDeletedByAssetDate:rec2", RecordName: "rec2"},+ "NestedFolder": {+ Name: "NestedFolder", RecordName: "folder2", IsFolder: true,+ Children: map[string]*api.Album{+ "LeafAlbum": {Name: "LeafAlbum", ObjectType: "CPLContainerRelationNotDeletedByAssetDate:rec3", RecordName: "rec3"},+ },+ },+ },+ },+ },+ })+}++func TestResolveAlbum(t *testing.T) {+ f := newTestPhotosFs()+ f.photos = newTestPhotosService()+ ctx := context.Background()++ t.Run("simple album", func(t *testing.T) {+ album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Videos")+ require.NoError(t, err)+ assert.Equal(t, "Videos", album.Name)+ })++ t.Run("nested folder child", func(t *testing.T) {+ album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder/ChildAlbum")+ require.NoError(t, err)+ assert.Equal(t, "ChildAlbum", album.Name)+ })++ t.Run("two-level nesting", func(t *testing.T) {+ album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder/NestedFolder/LeafAlbum")+ require.NoError(t, err)+ assert.Equal(t, "LeafAlbum", album.Name)+ })++ t.Run("folder itself", func(t *testing.T) {+ album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder")+ require.NoError(t, err)+ assert.True(t, album.IsFolder)+ })++ t.Run("missing album", func(t *testing.T) {+ _, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "NoSuchAlbum")+ assert.ErrorIs(t, err, fs.ErrorDirNotFound)+ })++ t.Run("missing library", func(t *testing.T) {+ _, err := f.resolveAlbum(ctx, f.photos, "NoSuchLib", "Videos")+ assert.ErrorIs(t, err, fs.ErrorDirNotFound)+ })++ t.Run("traverse into non-folder", func(t *testing.T) {+ _, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Videos/SubAlbum")+ assert.ErrorIs(t, err, fs.ErrorDirNotFound)+ })+}++func TestFindLeaf(t *testing.T) {+ f := newTestPhotosFs()+ f.photos = newTestPhotosService()+ ctx := context.Background()++ t.Run("root to library", func(t *testing.T) {+ id, found, err := f.FindLeaf(ctx, rootID, "PrimarySync")+ require.NoError(t, err)+ assert.True(t, found)+ assert.Equal(t, "lib:PrimarySync", id)+ })++ t.Run("root to missing library", func(t *testing.T) {+ _, found, err := f.FindLeaf(ctx, rootID, "NoSuchLib")+ require.NoError(t, err)+ assert.False(t, found)+ })++ t.Run("library to album", func(t *testing.T) {+ id, found, err := f.FindLeaf(ctx, "lib:PrimarySync", "Videos")+ require.NoError(t, err)+ assert.True(t, found)+ assert.Equal(t, "album:PrimarySync:Videos", id)+ })++ t.Run("library to missing album", func(t *testing.T) {+ _, found, err := f.FindLeaf(ctx, "lib:PrimarySync", "NoSuchAlbum")+ require.NoError(t, err)+ assert.False(t, found)+ })++ t.Run("missing library returns ErrorDirNotFound", func(t *testing.T) {+ _, _, err := f.FindLeaf(ctx, "lib:NoSuchLib", "Videos")+ assert.ErrorIs(t, err, fs.ErrorDirNotFound)+ })++ t.Run("folder to child album", func(t *testing.T) {+ id, found, err := f.FindLeaf(ctx, "album:PrimarySync:Folder", "ChildAlbum")+ require.NoError(t, err)+ assert.True(t, found)+ assert.Equal(t, "album:PrimarySync:Folder/ChildAlbum", id)+ })++ t.Run("non-folder album returns not found", func(t *testing.T) {+ _, found, err := f.FindLeaf(ctx, "album:PrimarySync:Videos", "SubAlbum")+ require.NoError(t, err)+ assert.False(t, found)+ })++ t.Run("unknown prefix returns not found", func(t *testing.T) {+ _, found, err := f.FindLeaf(ctx, "unknown:prefix", "leaf")+ require.NoError(t, err)+ assert.False(t, found)+ })+}++func TestResolveAlbum_EmptyPath(t *testing.T) {+ f := newTestPhotosFs()+ f.photos = newTestPhotosService()+ ctx := context.Background()++ // parseAlbumDirID("album:PrimarySync:") returns ("PrimarySync", "", true)+ // This empty album path feeds into resolveAlbum+ _, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "")+ assert.ErrorIs(t, err, fs.ErrorDirNotFound, "empty album path should return ErrorDirNotFound")+}++func TestNewObject_ErrorPaths(t *testing.T) {+ f := newTestPhotosFs()+ f.photos = newTestPhotosService()+ ctx := context.Background()++ // Pre-populate photoCache on the "Videos" album so GetPhotoByName works+ // without HTTP. Access via the test PhotosService's internal structure+ libs, _ := f.photos.GetLibraries(ctx)+ albums, _ := libs["PrimarySync"].GetAlbums(ctx)+ videos := albums["Videos"]+ videos.SetTestPhotoCache(map[string]*api.Photo{+ "existing.mp4": {ID: "m1", Filename: "existing.mp4", Size: 1024, ResourceKey: "resOriginalRes"},+ })++ // NewObject depends on dircache.FindDir which requires FindRoot + HTTP+ // Test the reachable paths through resolveAlbum and GetPhotoByName++ // Test resolveAlbum → folder → GetPhotoByName on folder (should fail)+ album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder")+ require.NoError(t, err)+ assert.True(t, album.IsFolder, "Folder should be a folder")+ // NewObject would return ErrorObjectNotFound for a folder path++ // Test GetPhotoByName on album with populated cache+ photo, err := videos.GetPhotoByName(ctx, "existing.mp4")+ require.NoError(t, err)+ assert.Equal(t, "m1", photo.ID)++ // Test GetPhotoByName cache miss+ _, err = videos.GetPhotoByName(ctx, "nonexistent.mp4")+ assert.Error(t, err)+ assert.Contains(t, err.Error(), "nonexistent.mp4")+}++func TestParseAlbumDirID_Exhaustive(t *testing.T) {+ tests := []struct {+ input string+ lib, album string+ ok bool+ }{+ {"album:PrimarySync:Videos", "PrimarySync", "Videos", true},+ {"album:PrimarySync:Folder/Child", "PrimarySync", "Folder/Child", true},+ {"album:PrimarySync:Name:With:Colons", "PrimarySync", "Name:With:Colons", true},+ {"album:PrimarySync:", "PrimarySync", "", true},+ {"album:", "", "", false},+ {"notalbum:foo:bar", "notalbum", "foo:bar", true}, // strips "album:" prefix literally, "notalbum:" stays+ {"", "", "", false},+ }++ for _, tt := range tests {+ lib, album, ok := parseAlbumDirID(tt.input)+ if tt.ok {+ assert.True(t, ok, "input=%q", tt.input)+ assert.Equal(t, tt.lib, lib, "input=%q", tt.input)+ assert.Equal(t, tt.album, album, "input=%q", tt.input)+ } else {+ assert.False(t, ok, "input=%q", tt.input)+ }+ }+}++// setEmptyPhotoCaches recursively sets empty photo caches on all leaf albums+// so GetPhotos returns without HTTP (service=nil test fast path)+func setEmptyPhotoCaches(albums map[string]*api.Album) {+ for _, album := range albums {+ if album.IsFolder {+ setEmptyPhotoCaches(album.Children)+ } else {+ album.SetTestPhotoCache(map[string]*api.Photo{})+ }+ }+}++func TestListR_NestedFolderRecursion(t *testing.T) {+ f := newTestPhotosFs()+ f.photos = newTestPhotosService()+ f.dirCache = dircache.New("", rootID, f)+ f.features = (&fs.Features{}).Fill(context.Background(), f)+ f.startTime = time.Now()+ ctx := context.Background()++ err := f.dirCache.FindRoot(ctx, false)+ require.NoError(t, err)++ // Pre-populate empty photo caches on all leaf albums+ libs, err := f.photos.GetLibraries(ctx)+ require.NoError(t, err)+ for _, lib := range libs {+ albums, err := lib.GetAlbums(ctx)+ require.NoError(t, err)+ setEmptyPhotoCaches(albums)+ }++ var dirs []string+ err = f.ListR(ctx, "", func(entries fs.DirEntries) error {+ for _, entry := range entries {+ if _, ok := entry.(fs.Directory); ok {+ dirs = append(dirs, entry.Remote())+ }+ }+ return nil+ })+ require.NoError(t, err)+ sort.Strings(dirs)++ // All directories must be present, including deeply nested ones+ assert.Contains(t, dirs, "PrimarySync")+ assert.Contains(t, dirs, "PrimarySync/All Photos")+ assert.Contains(t, dirs, "PrimarySync/Videos")+ assert.Contains(t, dirs, "PrimarySync/Folder")+ assert.Contains(t, dirs, "PrimarySync/Folder/ChildAlbum")+ assert.Contains(t, dirs, "PrimarySync/Folder/NestedFolder")+ assert.Contains(t, dirs, "PrimarySync/Folder/NestedFolder/LeafAlbum")+}++func TestListR_FolderLevel(t *testing.T) {+ f := &PhotosFs{+ name: "test",+ root: "PrimarySync/Folder",+ opt: Options{},+ startTime: time.Now(),+ }+ f.features = (&fs.Features{}).Fill(context.Background(), f)+ f.photos = newTestPhotosService()+ f.dirCache = dircache.New("PrimarySync/Folder", rootID, f)+ ctx := context.Background()++ err := f.dirCache.FindRoot(ctx, false)+ require.NoError(t, err)++ libs, _ := f.photos.GetLibraries(ctx)+ for _, lib := range libs {+ albums, _ := lib.GetAlbums(ctx)+ setEmptyPhotoCaches(albums)+ }++ var dirs []string+ err = f.ListR(ctx, "", func(entries fs.DirEntries) error {+ for _, entry := range entries {+ if _, ok := entry.(fs.Directory); ok {+ dirs = append(dirs, entry.Remote())+ }+ }+ return nil+ })+ require.NoError(t, err)++ // When rooted at a folder, should see children recursively+ assert.Contains(t, dirs, "ChildAlbum")+ assert.Contains(t, dirs, "NestedFolder")+ assert.Contains(t, dirs, "NestedFolder/LeafAlbum")+}++func TestNotifyZoneChange(t *testing.T) {+ ctx := context.Background()+ ps := newTestPhotosService()++ tests := []struct {+ name string+ root string+ zone string+ want []string+ }{+ {+ name: "top level root",+ root: "",+ zone: "PrimarySync",+ want: []string{"PrimarySync"},+ },+ {+ name: "library root",+ root: "PrimarySync",+ zone: "PrimarySync",+ want: []string{"", "All Photos", "Folder", "Folder/ChildAlbum", "Folder/NestedFolder", "Folder/NestedFolder/LeafAlbum", "UserAlbum", "Videos"},+ },+ {+ name: "folder root",+ root: "PrimarySync/Folder",+ zone: "PrimarySync",+ want: []string{"", "ChildAlbum", "NestedFolder", "NestedFolder/LeafAlbum"},+ },+ {+ name: "nested folder root",+ root: "PrimarySync/Folder/NestedFolder",+ zone: "PrimarySync",+ want: []string{"", "LeafAlbum"},+ },+ {+ name: "leaf album root",+ root: "PrimarySync/Folder/NestedFolder/LeafAlbum",+ zone: "PrimarySync",+ want: []string{""},+ },+ {+ name: "missing nested album still invalidates root",+ root: "PrimarySync/Folder/NoSuchAlbum",+ zone: "PrimarySync",+ want: []string{""},+ },+ {+ name: "different zone ignored",+ root: "PrimarySync/Folder/NestedFolder",+ zone: "SharedSync",+ want: nil,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ f := newTestPhotosFs()+ f.root = tt.root++ var got []string+ f.notifyZoneChange(ctx, ps, tt.zone, func(remote string, entryType fs.EntryType) {+ assert.Equal(t, fs.EntryDirectory, entryType)+ got = append(got, remote)+ })++ sort.Strings(got)+ want := append([]string(nil), tt.want...)+ sort.Strings(want)+ assert.Equal(t, want, got)+ })+ }+}diff --git a/docs/content/iclouddrive.md b/docs/content/iclouddrive.mdindex af18796fd..22c488757 100644--- a/docs/content/iclouddrive.md+++ b/docs/content/iclouddrive.md@@ -1,20 +1,25 @@----title: "iCloud Drive"-description: "Rclone docs for iCloud Drive"+title: "iCloud Drive and Photos"+description: "Rclone docs for iCloud Drive and Photos"versionIntroduced: "v1.69"----# iCloud Drive+# {{< icon "fa fa-cloud" >}} iCloud Drive and Photos## Configuration-The initial setup for an iCloud Drive backend involves getting a trust token/session.-This can be done by simply using the regular iCloud password, and accepting the code-prompt on another iCloud connected device.+The initial setup for an iCloud backend involves getting a trust token/session.+This uses your regular Apple ID password plus 2FA, either from a trusted+device prompt or an SMS code sent to a trusted phone number.**IMPORTANT**: App-specific passwords are not accepted. Only use yourregular Apple ID password and 2FA.+This backend serves two Apple services:++- `drive` - iCloud Drive (default)+- `photos` - iCloud Photos+`rclone config` walks you through the token creation. The trust token is validfor 30 days. After which you will have to reauthenticate with `rclone reconnect`or `rclone config`.@@ -30,10 +35,12 @@ The authentication flow is:1. rclone initiates a session with Apple's identity service2. An SRP key exchange takes place (your password is used locally to derive a key)-3. Apple sends a 2FA prompt to your trusted devices+3. Apple sends a 2FA prompt to your trusted devices, or lets you request an+ SMS code4. After you enter the 2FA code, rclone receives a trust token for future sessions-Here is an example of how to make a remote called `iclouddrive`. First run:+Here is an example of how to make a Photos remote called `icloudphotos`.+For iCloud Drive, leave `service` at its default `drive` value. First run:```consolerclone config@@ -47,7 +54,7 @@ n) New remotes) Set configuration passwordq) Quit confign/s/q> n-name> iclouddrive+name> icloudphotosOption Storage.Type of storage to configure.Choose a number from below, or type in your own value.@@ -56,6 +63,15 @@ XX / iCloud Drive\ (iclouddrive)[snip]Storage> iclouddrive+Option service.+iCloud service to use.+Choose a number from below, or type in your own value of type string.+Press Enter for the default (drive).+ 1 / iCloud Drive+ \ (drive)+ 2 / iCloud Photos+ \ (photos)+service> 2Option apple_id.Apple ID.Enter a value.@@ -75,13 +91,14 @@ y) Yesn) No (default)y/n> nOption config_2fa.-Two-factor authentication: please enter your 2FA code+Two-factor authentication: enter your 2FA code or type 'sms' for a text messageEnter a value.config_2fa> 2FACODERemote config---------------------[iclouddrive]+[icloudphotos]- type: iclouddrive+- service: photos- apple_id: APPLEID- password: *** ENCRYPTED ***- cookies: ****************************@@ -93,30 +110,124 @@ d) Delete this remotey/e/d> y```+## iCloud Photos++The iCloud Drive backend also supports accessing iCloud Photos by setting the+`service` option to `photos`:++```console+rclone lsd iclouddrive: --iclouddrive-service photos+```++This presents a read-only hierarchy rooted at photo libraries:++- **Level 1**: Photo libraries — your personal library (`PrimarySync`) and+ any Shared Photo Library (`SharedSync-XXXX`)+- **Level 2+**: Albums and folders within a library, nested recursively as in+ Apple Photos+- **Leaf level**: Photos/videos within an album, including Live Photo `.MOV`+ companions++Examples:++```console+# List libraries+rclone lsd iclouddrive: --iclouddrive-service photos++# List albums in your primary library+rclone lsd iclouddrive:PrimarySync/ --iclouddrive-service photos++# List photos in an album+rclone ls iclouddrive:PrimarySync/All\ Photos/ --iclouddrive-service photos++# Download a photo+rclone copy iclouddrive:PrimarySync/Favorites/IMG_0001.HEIC /tmp/ --iclouddrive-service photos+```++You can either:++- set `service = photos` in `rclone config` for a dedicated Photos remote+- keep `service = drive` and pass `--iclouddrive-service photos` when needed++### Metadata++With `--metadata`, Photos entries expose these read-only metadata keys:++- `width`+- `height`+- `added-time`+- `favorite`+- `hidden`++These metadata keys are only available when `service = photos`.++### Caching++iCloud Photos caches album listings to disk for fast subsequent access.+On the first run, listing a large album uses parallel `startRank`+partitions to fetch pages concurrently. After that, a lightweight change+check (~200ms) determines whether the cache is still valid.++Cache location: `~/.cache/rclone/iclouddrive-photos/<remote>/<zone>/`++To clear the cache: delete that directory or run `rclone config reconnect`.++### FUSE mounts++For mounting iCloud Photos via `rclone mount`, the following flags+are recommended:++ rclone mount remote: /mnt/photos \+ --iclouddrive-service photos \+ --vfs-refresh \+ --dir-cache-time 1h \+ --vfs-cache-mode full \+ --attr-timeout 1m \+ --read-only++- `--vfs-refresh` pre-warms directory caches in the background on mount+ start so that albums are ready when you browse them+- `--dir-cache-time 1h` extends the in-memory cache lifetime beyond the+ default 5 minutes (change detection is fast, so this is safe)+- `--vfs-cache-mode full` caches downloaded photos and videos to local+ disk for fast repeated access+- `--attr-timeout 1m` reduces kernel attribute lookups (safe because+ the backend is read-only)+- `--read-only` prevents confusing write errors++The first listing of a very large album (e.g. 75,000 items in "All+Photos") can take several minutes due to API pagination limits. This+happens once — subsequent listings use the disk cache.++### Limitations++iCloud Photos is read-only. Upload, delete, rename, and move operations+are not supported.+## Advanced Data Protection-ADP is currently unsupported and need to be disabled+Advanced Data Protection is supported.On iPhone, Settings `>` Apple Account `>` iCloud `>` 'Access iCloud Data on the Web'-must be ON, and 'Advanced Data Protection' OFF.+must be ON.++If ADP is enabled on your account, rclone requests PCS cookies after 2FA.+Apple may require approval on a trusted device before those cookies are issued.## Troubleshooting-### Missing PCS cookies from the request+### PCS cookie errors with ADP-This means you have Advanced Data Protection (ADP) turned on. This is not supported-at the moment. If you want to use rclone you will have to turn it off. See above-for how to turn it off.+If you see `Missing PCS cookies from the request` or a `requestPCS:` error,+the ADP approval flow did not complete successfully.-You will need to clear the `cookies` and the `trust_token` fields in the config.-Or you can delete the remote config and start again.+Check that 'Access iCloud Data on the Web' is enabled and approve any prompt+on your trusted device.-You should then run `rclone reconnect remote:`.+Then run `rclone reconnect remote:`.-Note that changing the ADP setting may not take effect immediately - you may-need to wait a few hours or a day before you can get rclone to work - keep-clearing the config entry and running `rclone reconnect remote:` until rclone-functions properly.+If the remote still has stale auth state, clear the `cookies` and+`trust_token` fields in the config, or delete and recreate the remote.<!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/iclouddrive/iclouddrive.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->### Standard options