Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

iclouddrive: add read only iCloud Photos support and SRP authentication

Yakov Tilld0c469c3c05e
diff --git a/backend/iclouddrive/api/client.go b/backend/iclouddrive/api/client.go
index 350a4fc33..1bd96c918 100644
--- a/backend/iclouddrive/api/client.go
+++ b/backend/iclouddrive/api/client.go
@@ -1,4 +1,4 @@
-// Package api provides functionality for interacting with the iCloud API.
+// Package api provides functionality for interacting with the iCloud API
package api
import (
@@ -8,16 +8,19 @@ import (
"errors"
"fmt"
"net/http"
+ "os"
+ "path/filepath"
"strings"
+ "sync"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/lib/rest"
)
const (
baseEndpoint = "https://www.icloud.com"
- homeEndpoint = "https://www.icloud.com"
setupEndpoint = "https://setup.icloud.com/setup/ws/1"
authEndpoint = "https://idmsa.apple.com/appleauth/auth"
)
@@ -28,26 +31,21 @@ type sessionSave func(*Session)
type Client struct {
appleID string
password string
+ remoteName string // rclone remote name, used for cache namespacing
srv *rest.Client
Session *Session
sessionSaveCallback sessionSave
drive *DriveService
+ mu sync.Mutex // protects drive and Authenticate
}
-// New creates a new Client instance with the provided Apple ID, password, trust token, cookies, and session save callback.
-//
-// Parameters:
-// - appleID: the Apple ID of the user.
-// - password: the password of the user.
-// - trustToken: the trust token for the session.
-// - clientID: the client id for the session.
-// - cookies: the cookies for the session.
-// - sessionSaveCallback: the callback function to save the session.
-func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave) (*Client, error) {
+// New creates a new iCloud API client and initializes its HTTP session
+func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave, remoteName string) (*Client, error) {
icloud := &Client{
- appleID: appleID,
+ appleID: strings.ToLower(appleID), // Apple SRP requires lowercase in client-side proof
password: password,
+ remoteName: filepath.Base(remoteName),
srv: rest.NewClient(fshttp.NewClient(context.Background())),
Session: NewSession(),
sessionSaveCallback: sessionSaveCallback,
@@ -59,10 +57,12 @@ func New(appleID, password, trustToken string, clientID string, cookies []*http.
return icloud, nil
}
-// DriveService returns the DriveService instance associated with the Client.
+// DriveService returns the DriveService instance, creating it on first call
func (c *Client) DriveService() (*DriveService, error) {
- var err error
+ c.mu.Lock()
+ defer c.mu.Unlock()
if c.drive == nil {
+ var err error
c.drive, err = NewDriveService(c)
if err != nil {
return nil, err
@@ -71,11 +71,7 @@ func (c *Client) DriveService() (*DriveService, error) {
return c.drive, nil
}
-// Request makes a request and retries it if the session is invalid.
-//
-// This function is the main entry point for making requests to the iCloud
-// API. If the initial request returns a 401 (Unauthorized), it will try to
-// reauthenticate and retry the request.
+// Request makes a request to the iCloud API, re-authenticating on 401/421
func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {
resp, err = c.Session.Request(ctx, opts, request, response)
if err != nil && resp != nil {
@@ -89,55 +85,93 @@ func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, respo
if c.Session.Requires2FA() {
return nil, errors.New("trust token expired, please reauth")
}
- return c.RequestNoReAuth(ctx, opts, request, response)
+ return c.Session.Request(ctx, opts, request, response)
}
}
return resp, err
}
-// RequestNoReAuth makes a request without re-authenticating.
-//
-// This function is useful when you have a session that is already
-// authenticated, but you need to make a request without triggering
-// a re-authentication.
-func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {
- // Make the request without re-authenticating
- resp, err = c.Session.Request(ctx, opts, request, response)
- return resp, err
-}
-
-// Authenticate authenticates the client with the iCloud API.
+// Authenticate authenticates the client, reusing existing session if valid
func (c *Client) Authenticate(ctx context.Context) error {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ // Skip /validate round-trip when saved session has cookies + service endpoints
+ // Native client behavior: use cached session, reauth lazily on 401/421
+ if c.Session.Cookies != nil && len(c.Session.AccountInfo.Webservices) > 0 {
+ fs.Debugf(nil, "iclouddrive: reusing saved session")
+ return nil
+ }
+ // Try loading cached service endpoints to avoid /validate round-trip (~5s)
+ if c.Session.Cookies != nil && c.loadCachedWebservices() {
+ fs.Debugf(nil, "iclouddrive: reusing session with cached endpoints")
+ return nil
+ }
if c.Session.Cookies != nil {
if err := c.Session.ValidateSession(ctx); err == nil {
- fs.Debugf("icloud", "Valid session, no need to reauth")
+ fs.Debugf(nil, "iclouddrive: valid session, no need to reauth")
+ c.saveCachedWebservices()
return nil
}
c.Session.Cookies = nil
}
- fs.Debugf("icloud", "Authenticating as %s\n", c.appleID)
+ fs.Debugf(nil, "iclouddrive: authenticating")
err := c.Session.SignIn(ctx, c.appleID, c.password)
if err != nil {
return err
}
- // If 2FA is required, don't try AuthWithToken yet — the caller
- // must complete 2FA first, then call AuthWithToken.
+ // If 2FA is required, skip AuthWithToken - caller must complete 2FA first
if c.Session.Requires2FA() {
return nil
}
err = c.Session.AuthWithToken(ctx)
- if err == nil && c.sessionSaveCallback != nil {
- c.sessionSaveCallback(c.Session)
+ if err == nil {
+ c.saveCachedWebservices()
+ if c.sessionSaveCallback != nil {
+ c.sessionSaveCallback(c.Session)
+ }
}
return err
}
-// SignIn signs in the client using the provided context and credentials.
-func (c *Client) SignIn(ctx context.Context) error {
- return c.Session.SignIn(ctx, c.appleID, c.password)
+// loadCachedWebservices loads service endpoints from disk cache
+func (c *Client) loadCachedWebservices() bool {
+ data, err := os.ReadFile(filepath.Join(config.GetCacheDir(), cacheSubdir, c.remoteName, "webservices.json"))
+ if err != nil {
+ return false
+ }
+ var ws map[string]*webService
+ if err := json.Unmarshal(data, &ws); err != nil {
+ return false
+ }
+ if len(ws) == 0 {
+ return false
+ }
+ c.Session.AccountInfo.Webservices = ws
+ return true
+}
+
+// saveCachedWebservices persists service endpoints to disk
+func (c *Client) saveCachedWebservices() {
+ if len(c.Session.AccountInfo.Webservices) == 0 {
+ return
+ }
+ saveJSONCache(filepath.Join(config.GetCacheDir(), cacheSubdir, c.remoteName), "webservices.json", c.Session.AccountInfo.Webservices)
+}
+
+// CacheDir returns the disk cache directory for this remote
+func (c *Client) CacheDir() string {
+ return filepath.Join(config.GetCacheDir(), cacheSubdir, c.remoteName)
+}
+
+// ClearCacheDir removes all disk cache files for a remote
+func ClearCacheDir(remoteName string) {
+ dir := filepath.Join(config.GetCacheDir(), cacheSubdir, filepath.Base(remoteName))
+ if err := os.RemoveAll(dir); err != nil {
+ fs.Debugf(nil, "iclouddrive: failed to clear cache: %v", err)
+ }
}
// IntoReader marshals the provided values into a JSON encoded reader
@@ -155,19 +189,19 @@ type RequestError struct {
Text string
}
-// Error satisfy the error interface.
+// Error satisfies the error interface
func (e *RequestError) Error() string {
return fmt.Sprintf("%s: %s", e.Text, e.Status)
}
-func newRequestError(Status string, Text string) *RequestError {
+func newRequestError(status string, text string) *RequestError {
return &RequestError{
- Status: strings.ToLower(Status),
- Text: Text,
+ Status: strings.ToLower(status),
+ Text: text,
}
}
-// newErr orf makes a new error from sprintf parameters.
-func newRequestErrorf(Status string, Text string, Parameters ...any) *RequestError {
- return newRequestError(strings.ToLower(Status), fmt.Sprintf(Text, Parameters...))
+// newRequestErrorf makes a new error from sprintf parameters
+func newRequestErrorf(status string, text string, params ...any) *RequestError {
+ return newRequestError(status, fmt.Sprintf(text, params...))
}
diff --git a/backend/iclouddrive/api/photos.go b/backend/iclouddrive/api/photos.go
new file mode 100644
index 000000000..06cdc226e
--- /dev/null
+++ b/backend/iclouddrive/api/photos.go
@@ -0,0 +1,2341 @@
+package api
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+ "sync"
+ "sync/atomic"
+
+ "github.com/rclone/rclone/fs"
+
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/rclone/rclone/lib/rest"
+
+ "golang.org/x/text/unicode/norm"
+)
+
+const (
+ // CloudKit returns CPLMaster+CPLAsset pairs, so 200 records = 100 photos
+ // Server caps at 200 regardless of requested value (tested: 500-5000 all return 200)
+ photosQueryLimit = 200
+ rootFolderRecord = "----Root-Folder----"
+ projectRootFolderRecord = "----Project-Root-Folder----"
+ indexingStateReady = "FINISHED"
+
+ // cacheSubdir is the subdirectory under rclone's cache dir for all iCloud Photos state
+ cacheSubdir = "iclouddrive-photos"
+
+ // Album type constants from CloudKit CPLAlbum records
+ albumTypeFolder = 3
+
+ // CPLAsset.assetSubtype values
+ subtypePanorama = 1
+ subtypeSloMo = 100
+ subtypeTimeLapse = 101
+
+ // CPLAsset.assetSubtypeV2 values
+ subtypeV2Live = 2
+ subtypeV2Screenshot = 3
+
+ // CPLAsset.adjustmentRenderType bitmask values
+ adjustPortrait = 2
+ adjustLongExposure = 4
+
+ // CloudKit record type names
+ recordTypeAlbum = "CPLAlbumByPositionLive"
+ recordTypeCountIndex = "HyperionIndexCountLookup"
+
+ // CloudKit endpoint area names
+ areaPrivate = "private"
+ areaShared = "shared"
+
+ // Slo-mo adjustment type (metadata-only edit, no separate rendered resource)
+ adjustSloMo = "com.apple.video.slomo"
+)
+
+// utiExtensions maps common Apple UTI descriptors to file extensions
+// Used as fallback when filenameEnc is missing from a CPLMaster record
+var utiExtensions = map[string]string{
+ "public.jpeg": ".jpg",
+ "public.png": ".png",
+ "public.heic": ".heic",
+ "public.heif": ".heif",
+ "public.tiff": ".tiff",
+ "public.mpeg-4": ".mp4",
+ "com.apple.quicktime-movie": ".mov",
+ "com.compuserve.gif": ".gif",
+ "com.adobe.raw-image": ".dng",
+ "com.canon.cr2-raw-image": ".cr2",
+ "com.canon.cr3-raw-image": ".cr3",
+ "com.nikon.raw-image": ".nef",
+ "com.sony.arw-raw-image": ".arw",
+ "public.avif": ".avif",
+ "org.webmproject.webp": ".webp",
+ "public.mpeg-2-video": ".m2v",
+ "com.apple.m4v-video": ".m4v",
+ "public.avi": ".avi",
+ "public.mp3": ".mp3",
+ "com.apple.m4a-audio": ".m4a",
+ "public.image": ".heic",
+ "com.fuji.raw-image": ".raf",
+ "com.panasonic.rw2-raw-image": ".rw2",
+ "com.olympus.raw-image": ".orf",
+ "com.pentax.raw-image": ".pef",
+ "com.nikon.nrw-raw-image": ".nrw",
+ "com.canon.crw-raw-image": ".crw",
+}
+
+// extFromUTI returns the file extension for an Apple UTI string field,
+// falling back to the provided default if the field is nil or unknown
+func extFromUTI(field *ckStringField, fallback string) string {
+ if field != nil {
+ if mapped, ok := utiExtensions[field.Value]; ok {
+ return mapped
+ }
+ }
+ return fallback
+}
+
+// buildPhotoCache creates a filename-keyed map from a photo slice
+func buildPhotoCache(photos []*Photo) map[string]*Photo {
+ m := make(map[string]*Photo, len(photos))
+ for _, p := range photos {
+ if p.Filename != "" {
+ m[p.Filename] = p
+ }
+ }
+ return m
+}
+
+// saveJSONCache marshals v to JSON and writes it atomically to dir/filename
+func saveJSONCache(dir, filename string, v any) {
+ if err := os.MkdirAll(dir, 0700); err != nil {
+ fs.Debugf(nil, "iclouddrive: failed to create cache dir: %v", err)
+ return
+ }
+ data, err := json.Marshal(v)
+ if err != nil {
+ fs.Debugf(nil, "iclouddrive: failed to marshal cache %s: %v", filename, err)
+ return
+ }
+ if err := atomicWriteFile(filepath.Join(dir, filename), data); err != nil {
+ fs.Debugf(nil, "iclouddrive: failed to write cache %s: %v", filename, err)
+ }
+}
+
+// atomicWriteFile writes data to target via atomic temp+rename
+func atomicWriteFile(target string, data []byte) error {
+ tmp := target + ".tmp"
+ if err := os.WriteFile(tmp, data, 0600); err != nil {
+ return err
+ }
+ if err := os.Rename(tmp, target); err != nil {
+ _ = os.Remove(tmp)
+ return err
+ }
+ return nil
+}
+
+// ShouldRetryFunc classifies whether an HTTP response/error is retryable
+type ShouldRetryFunc func(ctx context.Context, resp *http.Response, err error) (bool, error)
+
+// PhotosService manages iCloud Photos API interactions
+type PhotosService struct {
+ client *Client
+ endpoint string
+ pacer *fs.Pacer
+ shouldRetry ShouldRetryFunc
+ mu sync.Mutex
+ libraries map[string]*Library
+}
+
+type libraryDiscovery struct {
+ libraries map[string]*Library
+ refreshedAreas map[string]bool
+}
+
+// FlushCaches clears all cached libraries, albums, and photos
+func (ps *PhotosService) FlushCaches() {
+ ps.mu.Lock()
+ defer ps.mu.Unlock()
+ for _, lib := range ps.libraries {
+ lib.deltaMu.Lock()
+ lib.mu.Lock()
+ lib.cacheValid.Store(false)
+ lib.pendingDelta = nil
+ lib.clearDiskCache()
+ for _, album := range lib.albums {
+ album.mu.Lock()
+ album.photoCache = nil
+ album.mu.Unlock()
+ }
+ lib.albums = make(map[string]*Album)
+ lib.mu.Unlock()
+ lib.deltaMu.Unlock()
+ }
+ ps.libraries = make(map[string]*Library)
+ // Also remove the libraries cache file
+ _ = os.Remove(filepath.Join(ps.client.CacheDir(), "libraries.json"))
+}
+
+// deltaPayload holds a buffered changes/zone response waiting to be applied
+// after albums are populated
+type deltaPayload struct {
+ records []json.RawMessage
+ syncToken string
+ moreComing bool
+}
+
+// deltaContainsAlbumChanges checks if any record in the delta is a CPLAlbum
+// (album create/rename/delete) requiring eager album cache invalidation
+func deltaContainsAlbumChanges(records []json.RawMessage) bool {
+ for _, raw := range records {
+ var header struct {
+ RecordType string `json:"recordType"`
+ }
+ if json.Unmarshal(raw, &header) == nil && header.RecordType == "CPLAlbum" {
+ return true
+ }
+ }
+ return false
+}
+
+// Library represents a photo library in a specific zone
+type Library struct {
+ service *PhotosService
+ zoneID string
+ area string // "private" or "shared" - determines API endpoint path
+ ownerRecordName string // zone owner's _UUID for full zoneID in requests
+ zoneType string // "REGULAR_CUSTOM_ZONE" for full zoneID in requests
+ mu sync.Mutex // protects albums map
+ albums map[string]*Album
+ deltaMu sync.Mutex // serializes delta checks+apply; lock order: ps.mu before deltaMu
+ cacheValid atomic.Bool // true = album/photo disk caches are loadable
+ pendingDelta *deltaPayload // buffered delta waiting for albums to be populated, protected by deltaMu
+ notifyToken string // separate changes/zone token for ChangeNotify polling (memory-only)
+}
+
+// zoneIDMap returns the full zoneID object for CloudKit API requests
+func (lib *Library) zoneIDMap() map[string]any {
+ m := map[string]any{"zoneName": lib.zoneID}
+ if lib.ownerRecordName != "" {
+ m["ownerRecordName"] = lib.ownerRecordName
+ }
+ if lib.zoneType != "" {
+ m["zoneType"] = lib.zoneType
+ }
+ return m
+}
+
+// invalidateAlbumCache clears in-memory albums and removes disk cache
+func (lib *Library) invalidateAlbumCache() {
+ lib.mu.Lock()
+ lib.albums = make(map[string]*Album)
+ lib.mu.Unlock()
+ _ = os.Remove(filepath.Join(lib.zoneCacheDir(), "albums.json"))
+}
+
+// bufferDelta stores a pending delta for later application, eagerly
+// invalidating album cache if the delta contains CPLAlbum records
+// Must be called under deltaMu
+func (lib *Library) bufferDelta(records []json.RawMessage, syncToken string, moreComing bool) {
+ lib.pendingDelta = &deltaPayload{records: records, syncToken: syncToken, moreComing: moreComing}
+ lib.cacheValid.Store(true)
+ fs.Debugf(nil, "iclouddrive photos: zone %s has pending changes, buffered for later application", lib.zoneID)
+ if deltaContainsAlbumChanges(records) {
+ lib.invalidateAlbumCache()
+ fs.Debugf(nil, "iclouddrive photos: zone %s delta contains album changes, invalidated album cache", lib.zoneID)
+ }
+}
+
+// request makes an API call routed to this library's area (private or shared)
+func (lib *Library) request(ctx context.Context, endpoint string, data, response any) error {
+ return lib.service.requestForArea(ctx, lib.area, endpoint, data, response)
+}
+
+func (lib *Library) isSharedLibrary() bool {
+ return strings.HasPrefix(lib.zoneID, "SharedSync")
+}
+
+func isSharedAlbumIndexError(err error) bool {
+ if err == nil {
+ return false
+ }
+ msg := err.Error()
+ return strings.Contains(msg, "HTTP error 400") &&
+ strings.Contains(msg, "BAD_REQUEST") &&
+ strings.Contains(msg, "Index has invalid data")
+}
+
+func isZoneNotFoundError(err error) bool {
+ if err == nil {
+ return false
+ }
+ msg := err.Error()
+ return strings.Contains(msg, "HTTP error 404") &&
+ strings.Contains(msg, "ZONE_NOT_FOUND") &&
+ strings.Contains(msg, "Zone does not exist")
+}
+
+func (lib *Library) probeZoneExists(ctx context.Context) (bool, error) {
+ query := map[string]any{
+ "query": map[string]any{
+ "recordType": "CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted",
+ "filterBy": []map[string]any{
+ {
+ "fieldName": "startRank",
+ "comparator": "EQUALS",
+ "fieldValue": map[string]any{"type": "INT64", "value": 0},
+ },
+ {
+ "fieldName": "direction",
+ "comparator": "EQUALS",
+ "fieldValue": map[string]any{"type": "STRING", "value": "ASCENDING"},
+ },
+ },
+ },
+ "resultsLimit": 1,
+ "desiredKeys": []string{"masterRef"},
+ "zoneID": lib.zoneIDMap(),
+ }
+ var response struct {
+ Records []json.RawMessage `json:"records"`
+ }
+ if err := lib.request(ctx, "records/query", query, &response); err != nil {
+ if isZoneNotFoundError(err) {
+ return false, nil
+ }
+ return false, err
+ }
+ return true, nil
+}
+
+// newUserAlbum creates a standard user album with CPLContainerRelation query config
+func (lib *Library) newUserAlbum(name, recordName string) *Album {
+ return &Album{
+ Name: name,
+ ObjectType: fmt.Sprintf("CPLContainerRelationNotDeletedByAssetDate:%s", recordName),
+ ListType: "CPLContainerRelationLiveByAssetDate",
+ Direction: "ASCENDING",
+ RecordName: recordName,
+ lib: lib,
+ Filters: []Filter{{
+ FieldName: "parentId",
+ Comparator: "EQUALS",
+ FieldValue: map[string]string{"type": "STRING", "value": recordName},
+ }},
+ }
+}
+
+// Album represents a photo album with its metadata and query configuration
+type Album struct {
+ Name string `json:"name"`
+ ObjectType string `json:"objectType"`
+ ListType string `json:"listType"`
+ Direction string `json:"direction"`
+ Filters []Filter `json:"filters,omitempty"`
+ RecordName string `json:"recordName,omitempty"`
+ IsFolder bool `json:"isFolder,omitempty"`
+ Children map[string]*Album `json:"children,omitempty"`
+ lib *Library `json:"-"`
+ mu sync.Mutex `json:"-"`
+ photoCache map[string]*Photo `json:"-"`
+}
+
+// Photo represents a photo or video with its metadata
+type Photo struct {
+ ID string
+ Filename string
+ Size int64
+ AssetDate int64 // Unix timestamp in milliseconds
+ AddedDate int64
+ Width int
+ Height int
+ IsFavorite bool
+ IsHidden bool
+ SmartAlbums []string // smart album names this photo belongs to (for delta sync routing)
+ ResourceKey string // CloudKit field name for download (default: resOriginalRes)
+}
+
+// companion creates a derivative Photo entry sharing metadata with the parent
+// Used for Live Photo MOV, edited versions, and RAW alternatives
+func (p *Photo) companion(id, filename, resourceKey string, size int64) *Photo {
+ return &Photo{
+ ID: id,
+ Filename: filename,
+ Size: size,
+ AssetDate: p.AssetDate,
+ AddedDate: p.AddedDate,
+ IsFavorite: p.IsFavorite,
+ IsHidden: p.IsHidden,
+ ResourceKey: resourceKey,
+ SmartAlbums: p.SmartAlbums,
+ }
+}
+
+// Filter represents a CloudKit query filter
+type Filter struct {
+ FieldName string `json:"fieldName"`
+ Comparator string `json:"comparator"`
+ FieldValue any `json:"fieldValue"`
+}
+
+// smartAlbumFilter defines a filter-based smart album generated from a short table
+// Each entry maps: display name -> ObjectType suffix + filter tag
+// All share ListType "CPLAssetAndMasterInSmartAlbumByAssetDate", direction ASCENDING,
+// and a single smartAlbum EQUALS filter
+type smartAlbumFilter struct {
+ suffix string // appended to "CPLAssetInSmartAlbumByAssetDate:"
+ tag string // smartAlbum filter value (e.g. "VIDEO", "FAVORITE")
+}
+
+// smartAlbumFilters is the data table for the 10 filter-based smart albums
+var smartAlbumFilters = map[string]smartAlbumFilter{
+ "Time-lapse": {suffix: "Timelapse", tag: "TIMELAPSE"},
+ "Videos": {suffix: "Video", tag: "VIDEO"},
+ "Slo-mo": {suffix: "Slomo", tag: "SLOMO"},
+ "Favorites": {suffix: "Favorite", tag: "FAVORITE"},
+ "Panoramas": {suffix: "Panorama", tag: "PANORAMA"},
+ "Screenshots": {suffix: "Screenshot", tag: "SCREENSHOT"},
+ "Live": {suffix: "Live", tag: "LIVE"},
+ "Portrait": {suffix: "Depth", tag: "DEPTH"},
+ "Long Exposure": {suffix: "Exposure", tag: "EXPOSURE"},
+ "Animated": {suffix: "Animated", tag: "ANIMATED"},
+ // SELFIE filter exists server-side but the index is never populated via web API -
+ // selfie classification is on-device only (iOS reads LensModel EXIF for "front camera")
+ // Apple's own icloud.com doesn't show it either - omitted to avoid an always-empty album
+}
+
+// SmartAlbums defines the built-in smart album types available in iCloud Photos
+// 10 filter-based albums are generated from smartAlbumFilters; 4 special albums
+// (All Photos, Bursts, Hidden, Recently Deleted) use unique recordTypes
+var SmartAlbums = buildSmartAlbums()
+
+func buildSmartAlbums() map[string]*Album {
+ albums := map[string]*Album{
+ "All Photos": {
+ Name: "All Photos",
+ ObjectType: "CPLAssetByAssetDateWithoutHiddenOrDeleted",
+ ListType: "CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted",
+ Direction: "ASCENDING",
+ },
+ "Bursts": {
+ Name: "Bursts",
+ ObjectType: "CPLAssetBurstStackAssetByAssetDate",
+ ListType: "CPLBurstStackAssetAndMasterByAssetDate",
+ Direction: "ASCENDING",
+ },
+ "Hidden": {
+ Name: "Hidden",
+ ObjectType: "CPLAssetHiddenByAssetDate",
+ ListType: "CPLAssetAndMasterHiddenByAssetDate",
+ Direction: "ASCENDING",
+ },
+ "Recently Deleted": {
+ Name: "Recently Deleted",
+ ObjectType: "CPLAssetDeletedByExpungedDate",
+ ListType: "CPLAssetAndMasterDeletedByExpungedDate",
+ Direction: "DESCENDING",
+ },
+ }
+ for name, f := range smartAlbumFilters {
+ albums[name] = &Album{
+ Name: name,
+ ObjectType: "CPLAssetInSmartAlbumByAssetDate:" + f.suffix,
+ ListType: "CPLAssetAndMasterInSmartAlbumByAssetDate",
+ Direction: "ASCENDING",
+ Filters: []Filter{{
+ FieldName: "smartAlbum",
+ Comparator: "EQUALS",
+ FieldValue: map[string]string{"type": "STRING", "value": f.tag},
+ }},
+ }
+ }
+ return albums
+}
+
+type errorRoundTripper struct{}
+
+func (errorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) {
+ return nil, errors.New("test photos service has no network transport")
+}
+
+// NewTestPhotosService creates a PhotosService with pre-populated libraries for testing
+func NewTestPhotosService(libs map[string]map[string]*Album) *PhotosService {
+ httpClient := &http.Client{Transport: errorRoundTripper{}}
+ session := &Session{srv: rest.NewClient(httpClient)}
+ ps := &PhotosService{
+ client: &Client{
+ remoteName: "_test",
+ Session: session,
+ },
+ endpoint: "http://test.invalid/database/1/com.apple.photos.cloud/production",
+ pacer: fs.NewPacer(context.Background(), pacer.NewDefault()),
+ shouldRetry: func(ctx context.Context, resp *http.Response, err error) (bool, error) { return false, err },
+ libraries: make(map[string]*Library),
+ }
+ for zoneName, albums := range libs {
+ lib := &Library{
+ service: ps,
+ zoneID: zoneName,
+ area: areaPrivate,
+ albums: make(map[string]*Album),
+ }
+ lib.cacheValid.Store(true)
+ for name, album := range albums {
+ lib.restoreAlbumLinks(album)
+ lib.albums[name] = album
+ }
+ ps.libraries[zoneName] = lib
+ }
+ return ps
+}
+
+// SetTestPhotoCache populates an album's photo cache for testing
+func (album *Album) SetTestPhotoCache(cache map[string]*Photo) {
+ album.mu.Lock()
+ album.photoCache = cache
+ album.mu.Unlock()
+}
+
+// NewPhotosService creates a new PhotosService instance
+func NewPhotosService(ctx context.Context, client *Client, pacer *fs.Pacer, shouldRetry ShouldRetryFunc) (*PhotosService, error) {
+ service, exists := client.Session.AccountInfo.Webservices["ckdatabasews"]
+ if !exists || service.Status != "active" {
+ return nil, fmt.Errorf("ckdatabasews service not available")
+ }
+ endpoint := fmt.Sprintf("%s/database/1/com.apple.photos.cloud/production", service.URL)
+
+ ps := &PhotosService{
+ client: client,
+ endpoint: endpoint,
+ pacer: pacer,
+ shouldRetry: shouldRetry,
+ libraries: make(map[string]*Library),
+ }
+
+ ps.checkIndexingState(ctx, "PrimarySync")
+
+ return ps, nil
+}
+
+// GetLibraries returns all available photo libraries
+func (ps *PhotosService) GetLibraries(ctx context.Context) (map[string]*Library, error) {
+ ps.mu.Lock()
+ defer ps.mu.Unlock()
+
+ if len(ps.libraries) > 0 {
+ if discovered, err := ps.discoverLibraries(ctx); err == nil {
+ ps.libraries = mergeDiscoveredLibraries(ctx, ps.libraries, discovered)
+ ps.saveCachedLibraries()
+ fs.Debugf(nil, "iclouddrive photos: refreshed %d in-memory libraries from API", len(ps.libraries))
+ } else {
+ fs.Debugf(nil, "iclouddrive photos: in-memory library rediscovery failed, using cached state: %v", err)
+ }
+ return ps.libraries, nil
+ }
+
+ // Try loading cached zone names from disk
+ if cached := ps.loadCachedLibraries(); cached != nil {
+ ps.batchCheckForChanges(ctx, cached)
+ ps.libraries = cached
+ if discovered, err := ps.discoverLibraries(ctx); err == nil {
+ ps.libraries = mergeDiscoveredLibraries(ctx, cached, discovered)
+ ps.saveCachedLibraries()
+ fs.Debugf(nil, "iclouddrive photos: refreshed %d libraries from API", len(ps.libraries))
+ } else {
+ fs.Debugf(nil, "iclouddrive photos: library rediscovery failed, using cached zones: %v", err)
+ fs.Debugf(nil, "iclouddrive photos: %d libraries from cache", len(cached))
+ }
+ return ps.libraries, nil
+ }
+
+ discovered, err := ps.discoverLibraries(ctx)
+ if err != nil {
+ return nil, err
+ }
+ ps.libraries = discovered.libraries
+ ps.saveCachedLibraries()
+ return ps.libraries, nil
+}
+
+func (ps *PhotosService) discoverLibraries(ctx context.Context) (*libraryDiscovery, error) {
+ result := &libraryDiscovery{
+ libraries: make(map[string]*Library),
+ refreshedAreas: make(map[string]bool),
+ }
+
+ // Discover zones from API - probe both private and shared databases
+ // Private zones: owned by the current user (PrimarySync + owned SharedSync)
+ // Shared zones: owned by another user (non-owner SharedSync)
+ type zoneResponse struct {
+ Zones []struct {
+ ZoneID struct {
+ ZoneName string `json:"zoneName"`
+ OwnerRecordName string `json:"ownerRecordName"`
+ ZoneType string `json:"zoneType"`
+ } `json:"zoneID"`
+ Deleted bool `json:"deleted,omitempty"`
+ } `json:"zones"`
+ }
+
+ for _, area := range []string{areaPrivate, areaShared} {
+ var response zoneResponse
+ if err := ps.requestForArea(ctx, area, "changes/database", map[string]any{}, &response); err != nil {
+ if area == areaShared {
+ // Shared database may not exist for all accounts
+ fs.Debugf(nil, "iclouddrive photos: shared zone discovery failed (expected if no shared library): %v", err)
+ continue
+ }
+ return nil, fmt.Errorf("failed to discover zones: %w", err)
+ }
+ result.refreshedAreas[area] = true
+ for _, zone := range response.Zones {
+ if zone.Deleted {
+ continue
+ }
+ name := zone.ZoneID.ZoneName
+ // SharedSync-* found in private takes precedence over shared
+ if _, exists := result.libraries[name]; exists {
+ continue
+ }
+ result.libraries[name] = &Library{
+ service: ps,
+ zoneID: name,
+ area: area,
+ ownerRecordName: zone.ZoneID.OwnerRecordName,
+ zoneType: zone.ZoneID.ZoneType,
+ albums: make(map[string]*Album),
+ }
+ }
+ }
+ return result, nil
+}
+
+func mergeDiscoveredLibraries(ctx context.Context, existing map[string]*Library, discovered *libraryDiscovery) map[string]*Library {
+ merged := make(map[string]*Library, len(existing)+len(discovered.libraries))
+ for name, cached := range existing {
+ if !discovered.refreshedAreas[cached.area] {
+ if cached.area == areaShared {
+ exists, err := cached.probeZoneExists(ctx)
+ if err != nil {
+ fs.Debugf(nil, "iclouddrive photos: shared zone probe failed for %q, keeping cached zone: %v", cached.zoneID, err)
+ } else if !exists {
+ fs.Debugf(nil, "iclouddrive photos: dropping cached shared zone %q after authoritative ZONE_NOT_FOUND probe", cached.zoneID)
+ continue
+ }
+ }
+ merged[name] = cached
+ }
+ }
+ for name, fresh := range discovered.libraries {
+ if cached, ok := existing[name]; ok {
+ cached.service = fresh.service
+ cached.area = fresh.area
+ cached.ownerRecordName = fresh.ownerRecordName
+ cached.zoneType = fresh.zoneType
+ merged[name] = cached
+ continue
+ }
+ merged[name] = fresh
+ }
+ return merged
+}
+
+// albumRecord represents a CPLAlbum record from CloudKit
+type albumRecord struct {
+ RecordName string `json:"recordName"`
+ Fields struct {
+ AlbumNameEnc *ckStringField `json:"albumNameEnc,omitempty"`
+ AlbumType *ckIntField `json:"albumType,omitempty"`
+ ParentID *ckStringField `json:"parentId,omitempty"`
+ IsDeleted *ckBoolField `json:"isDeleted,omitempty"`
+ } `json:"fields"`
+}
+
+// albumQueryResponse wraps a paginated list of album records
+type albumQueryResponse struct {
+ Records []albumRecord `json:"records"`
+ ContinuationMarker string `json:"continuationMarker"`
+}
+
+// cachedLibraryEntry stores zone metadata for disk cache persistence
+type cachedLibraryEntry struct {
+ ZoneName string `json:"zoneName"`
+ Area string `json:"area,omitempty"`
+ OwnerRecordName string `json:"ownerRecordName,omitempty"`
+ ZoneType string `json:"zoneType,omitempty"`
+}
+
+// loadCachedLibraries loads zone metadata from disk cache
+func (ps *PhotosService) loadCachedLibraries() map[string]*Library {
+ cacheFile := filepath.Join(ps.client.CacheDir(), "libraries.json")
+ data, err := os.ReadFile(cacheFile)
+ if err != nil {
+ return nil
+ }
+
+ var entries []cachedLibraryEntry
+ if err := json.Unmarshal(data, &entries); err != nil {
+ return nil
+ }
+
+ libs := make(map[string]*Library, len(entries))
+ for _, entry := range entries {
+ area := entry.Area
+ if area == "" {
+ area = areaPrivate
+ }
+ libs[entry.ZoneName] = &Library{
+ service: ps,
+ zoneID: entry.ZoneName,
+ area: area,
+ ownerRecordName: entry.OwnerRecordName,
+ zoneType: entry.ZoneType,
+ albums: make(map[string]*Album),
+ }
+ }
+ return libs
+}
+
+// saveCachedLibraries persists zone metadata to disk via atomic rename
+func (ps *PhotosService) saveCachedLibraries() {
+ var entries []cachedLibraryEntry
+ for _, lib := range ps.libraries {
+ entries = append(entries, cachedLibraryEntry{
+ ZoneName: lib.zoneID,
+ Area: lib.area,
+ OwnerRecordName: lib.ownerRecordName,
+ ZoneType: lib.zoneType,
+ })
+ }
+ saveJSONCache(ps.client.CacheDir(), "libraries.json", entries)
+}
+
+// GetAlbums returns all albums for this library
+func (lib *Library) GetAlbums(ctx context.Context) (map[string]*Album, error) {
+ lib.mu.Lock()
+ defer lib.mu.Unlock()
+
+ if len(lib.albums) > 0 {
+ return lib.albums, nil
+ }
+
+ // Try loading cached albums if zone is unchanged
+ if lib.cacheValid.Load() {
+ if cached := lib.loadCachedAlbums(); cached != nil {
+ lib.albums = cached
+ fs.Debugf(nil, "iclouddrive photos: %d albums from cache for zone %s", len(cached), lib.zoneID)
+ return lib.albums, nil
+ }
+ }
+
+ // Build albums into a local map first so that a transient user album
+ // query failure leaves lib.albums empty (retried on next call) rather
+ // than permanently caching the smart-album-only subset
+ albums := make(map[string]*Album, len(SmartAlbums))
+
+ // Add smart albums
+ for name, template := range SmartAlbums {
+ albums[name] = &Album{
+ Name: template.Name,
+ ObjectType: template.ObjectType,
+ ListType: template.ListType,
+ Direction: template.Direction,
+ Filters: append([]Filter{}, template.Filters...),
+ RecordName: template.RecordName,
+ lib: lib,
+ }
+ }
+
+ // Add user albums and folders (paginated - CloudKit caps at 200 per page)
+ var allRecords []albumRecord
+ var continuationMarker string
+
+ for {
+ query := map[string]any{
+ "query": map[string]any{"recordType": recordTypeAlbum},
+ "zoneID": lib.zoneIDMap(),
+ "desiredKeys": []string{"albumNameEnc", "albumType", "parentId", "isDeleted"},
+ }
+ if continuationMarker != "" {
+ query["continuationMarker"] = continuationMarker
+ }
+
+ var response albumQueryResponse
+
+ if err := lib.request(ctx, "records/query", query, &response); err != nil {
+ // SharedSync libraries return BAD_REQUEST / "Index has invalid data"
+ // on CPLAlbumByPositionLive in live probes, so fall back to smart
+ // albums there while surfacing PrimarySync failures directly
+ fs.Debugf(nil, "iclouddrive photos: user album query failed for zone %q: %v", lib.zoneID, err)
+ if lib.isSharedLibrary() && isSharedAlbumIndexError(err) {
+ return albums, nil
+ }
+ return nil, fmt.Errorf("query user albums for zone %q: %w", lib.zoneID, err)
+ }
+
+ allRecords = append(allRecords, response.Records...)
+
+ if response.ContinuationMarker == "" {
+ break
+ }
+ continuationMarker = response.ContinuationMarker
+ }
+
+ for _, record := range allRecords {
+ if record.Fields.AlbumNameEnc == nil ||
+ record.RecordName == rootFolderRecord ||
+ record.RecordName == projectRootFolderRecord ||
+ (record.Fields.IsDeleted != nil && record.Fields.IsDeleted.Value) {
+ continue
+ }
+
+ nameBytes, err := base64.StdEncoding.DecodeString(record.Fields.AlbumNameEnc.Value)
+ if err != nil {
+ fs.Debugf(nil, "iclouddrive photos: skipping album %q: base64 decode: %v", record.RecordName, err)
+ continue
+ }
+
+ isFolder := record.Fields.AlbumType != nil && record.Fields.AlbumType.Value == albumTypeFolder
+ albumName := norm.NFC.String(string(nameBytes))
+
+ // User album with same name as a smart album - smart album has special
+ // server-side query semantics that can't be replicated by the user album,
+ // so we keep the smart album and skip the user album with a warning
+ if _, isSmart := SmartAlbums[albumName]; isSmart && !isFolder {
+ fs.Logf(nil, "iclouddrive photos: user album %q shadows smart album, using smart album", albumName)
+ continue
+ }
+
+ if isFolder {
+ folder := &Album{
+ Name: albumName,
+ RecordName: record.RecordName,
+ lib: lib,
+ IsFolder: true,
+ Children: make(map[string]*Album),
+ }
+ if err := lib.fetchFolderChildren(ctx, folder); err != nil {
+ return nil, fmt.Errorf("fetch children of folder %q: %w", albumName, err)
+ }
+ albums[albumName] = folder
+ } else {
+ albums[albumName] = lib.newUserAlbum(albumName, record.RecordName)
+ }
+ }
+
+ lib.albums = albums
+ lib.saveCachedAlbums()
+ return lib.albums, nil
+}
+
+// loadCachedAlbums loads album metadata from disk cache
+func (lib *Library) loadCachedAlbums() map[string]*Album {
+ cacheFile := filepath.Join(lib.zoneCacheDir(), "albums.json")
+ data, err := os.ReadFile(cacheFile)
+ if err != nil {
+ return nil
+ }
+ var albums map[string]*Album
+ if err := json.Unmarshal(data, &albums); err != nil {
+ return nil
+ }
+ // Restore runtime-only fields after deserialization
+ for _, a := range albums {
+ lib.restoreAlbumLinks(a)
+ }
+ return albums
+}
+
+// restoreAlbumLinks sets runtime-only fields (lib pointer) on an album
+// and its children after deserialization from disk cache
+func (lib *Library) restoreAlbumLinks(a *Album) {
+ a.lib = lib
+ for _, child := range a.Children {
+ lib.restoreAlbumLinks(child)
+ }
+}
+
+func flattenAlbumTree(dst []*Album, albums map[string]*Album) []*Album {
+ for _, album := range albums {
+ dst = append(dst, album)
+ if album.IsFolder {
+ dst = flattenAlbumTree(dst, album.Children)
+ }
+ }
+ return dst
+}
+
+// saveCachedAlbums persists album metadata to disk via atomic rename
+func (lib *Library) saveCachedAlbums() {
+ saveJSONCache(lib.zoneCacheDir(), "albums.json", lib.albums)
+}
+
+// fetchFolderChildren queries child albums inside a folder by parentId
+func (lib *Library) fetchFolderChildren(ctx context.Context, folder *Album) error {
+ query := map[string]any{
+ "query": map[string]any{
+ "recordType": recordTypeAlbum,
+ "filterBy": []map[string]any{{
+ "fieldName": "parentId",
+ "comparator": "EQUALS",
+ "fieldValue": map[string]string{"type": "STRING", "value": folder.RecordName},
+ }},
+ },
+ "zoneID": lib.zoneIDMap(),
+ "desiredKeys": []string{"albumNameEnc", "albumType", "isDeleted"},
+ }
+
+ var continuationMarker string
+ for {
+ if continuationMarker != "" {
+ query["continuationMarker"] = continuationMarker
+ }
+
+ var response albumQueryResponse
+
+ if err := lib.request(ctx, "records/query", query, &response); err != nil {
+ return err
+ }
+
+ for _, record := range response.Records {
+ if record.Fields.AlbumNameEnc == nil ||
+ (record.Fields.IsDeleted != nil && record.Fields.IsDeleted.Value) {
+ continue
+ }
+
+ nameBytes, err := base64.StdEncoding.DecodeString(record.Fields.AlbumNameEnc.Value)
+ if err != nil {
+ fs.Debugf(nil, "iclouddrive photos: skipping child album %q: base64 decode: %v", record.RecordName, err)
+ continue
+ }
+
+ childName := norm.NFC.String(string(nameBytes))
+ isFolder := record.Fields.AlbumType != nil && record.Fields.AlbumType.Value == albumTypeFolder
+
+ if isFolder {
+ childFolder := &Album{
+ Name: childName,
+ RecordName: record.RecordName,
+ lib: lib,
+ IsFolder: true,
+ Children: make(map[string]*Album),
+ }
+ if err := lib.fetchFolderChildren(ctx, childFolder); err != nil {
+ return err
+ }
+ folder.Children[childName] = childFolder
+ } else {
+ folder.Children[childName] = lib.newUserAlbum(childName, record.RecordName)
+ }
+ }
+
+ if response.ContinuationMarker == "" {
+ break
+ }
+ continuationMarker = response.ContinuationMarker
+ }
+
+ return nil
+}
+
+// albumCacheKey returns a stable filename-safe key for an album's disk cache
+func albumCacheKey(objectType string) string {
+ h := sha256.Sum256([]byte(objectType))
+ return hex.EncodeToString(h[:8])
+}
+
+// zoneCacheDir returns the disk cache directory for this zone
+// Path follows rclone convention: <cacheDir>/<backend>/<remoteName>/<zone>/
+func (lib *Library) zoneCacheDir() string {
+ return filepath.Join(lib.service.client.CacheDir(), lib.zoneID)
+}
+
+// checkForChanges detects whether the zone has been modified since the last
+// sync. If unchanged (0 records), sets cacheValid=true. If changed, buffers
+// the first page as pendingDelta and still sets cacheValid=true (album disk
+// cache is valid under the old token - delta hasn't been applied yet)
+// The buffered delta is consumed later by applyPendingDelta when albums exist
+func (lib *Library) checkForChanges(ctx context.Context) {
+ lib.deltaMu.Lock()
+ defer lib.deltaMu.Unlock()
+
+ // Already have a buffered delta waiting to be applied
+ if lib.pendingDelta != nil {
+ return
+ }
+
+ token := lib.readSyncToken()
+ if token == "" {
+ return
+ }
+
+ var response changesZoneResponse
+ if err := lib.request(ctx, "changes/zone", lib.changesZoneBody(token), &response); err != nil {
+ fs.Debugf(nil, "iclouddrive photos: delta check failed for zone %s: %v", lib.zoneID, err)
+ return
+ }
+ if len(response.Zones) == 0 {
+ return
+ }
+
+ zone := response.Zones[0]
+
+ if len(zone.Records) == 0 && !zone.MoreComing {
+ // No changes - advance token, all caches valid
+ fs.Debugf(nil, "iclouddrive photos: zone %s unchanged, using cached listings", lib.zoneID)
+ lib.saveSyncToken(zone.SyncToken)
+ lib.cacheValid.Store(true)
+ return
+ }
+
+ // Buffer the delta for later application (after albums are populated)
+ // Album disk cache is still valid under the old token
+ lib.bufferDelta(zone.Records, zone.SyncToken, zone.MoreComing)
+}
+
+// zoneEntry pairs a changes/zone request body with its library for batched zone operations
+type zoneEntry struct {
+ zone map[string]any
+ lib *Library
+}
+
+// flattenZoneEntries extracts the zone request bodies and builds a zoneID→Library lookup
+func flattenZoneEntries(entries []zoneEntry) ([]map[string]any, map[string]*Library) {
+ zones := make([]map[string]any, len(entries))
+ libByZone := make(map[string]*Library, len(entries))
+ for i, e := range entries {
+ zones[i] = e.zone
+ libByZone[e.lib.zoneID] = e.lib
+ }
+ return zones, libByZone
+}
+
+// batchCheckForChanges checks all zones for changes in a single API call
+// Each zone with a syncToken gets checked; zones without tokens are skipped
+func (ps *PhotosService) batchCheckForChanges(ctx context.Context, libs map[string]*Library) {
+ // Group zones by area for separate API calls (private and shared use different endpoints)
+ byArea := make(map[string][]zoneEntry)
+ for _, lib := range libs {
+ lib.deltaMu.Lock()
+ hasPending := lib.pendingDelta != nil
+ lib.deltaMu.Unlock()
+ if hasPending {
+ continue
+ }
+ token := lib.readSyncToken()
+ if token == "" {
+ continue
+ }
+ zone := map[string]any{
+ "zoneID": lib.zoneIDMap(),
+ "desiredKeys": changesZoneDesiredKeys,
+ "syncToken": token,
+ }
+ byArea[lib.area] = append(byArea[lib.area], zoneEntry{zone: zone, lib: lib})
+ }
+
+ for area, entries := range byArea {
+ zones, libByZone := flattenZoneEntries(entries)
+ var response changesZoneResponse
+ if err := ps.requestForArea(ctx, area, "changes/zone", map[string]any{"zones": zones}, &response); err != nil {
+ fs.Debugf(nil, "iclouddrive photos: batch delta check (%s) failed: %v", area, err)
+ continue
+ }
+ for _, zone := range response.Zones {
+ lib := libByZone[zone.ZoneID.ZoneName]
+ if lib == nil {
+ continue
+ }
+ lib.deltaMu.Lock()
+ if len(zone.Records) == 0 && !zone.MoreComing {
+ fs.Debugf(nil, "iclouddrive photos: zone %s unchanged, using cached listings", lib.zoneID)
+ lib.saveSyncToken(zone.SyncToken)
+ lib.cacheValid.Store(true)
+ } else {
+ lib.bufferDelta(zone.Records, zone.SyncToken, zone.MoreComing)
+ }
+ lib.deltaMu.Unlock()
+ }
+ }
+}
+
+// PollForChanges checks all zones for changes in a single API call using
+// separate notification tokens, returns zone names that have been modified
+// Used by ChangeNotify - does not consume or interfere with listing delta sync
+func (ps *PhotosService) PollForChanges(ctx context.Context) []string {
+ ps.mu.Lock()
+ libs := make([]*Library, 0, len(ps.libraries))
+ for _, lib := range ps.libraries {
+ libs = append(libs, lib)
+ }
+ ps.mu.Unlock()
+
+ // Group zones by area for separate API calls
+ byArea := make(map[string][]zoneEntry)
+ for _, lib := range libs {
+ lib.deltaMu.Lock()
+ token := lib.notifyToken
+ lib.deltaMu.Unlock()
+ if token == "" {
+ token = lib.readSyncToken()
+ }
+ if token == "" {
+ continue
+ }
+ byArea[lib.area] = append(byArea[lib.area], zoneEntry{
+ zone: map[string]any{
+ "zoneID": lib.zoneIDMap(),
+ "desiredKeys": changesZoneDesiredKeys,
+ "syncToken": token,
+ },
+ lib: lib,
+ })
+ }
+
+ var changed []string
+ for area, entries := range byArea {
+ zones, libByZone := flattenZoneEntries(entries)
+ var response changesZoneResponse
+ if err := ps.requestForArea(ctx, area, "changes/zone", map[string]any{"zones": zones}, &response); err != nil {
+ continue
+ }
+
+ for _, zone := range response.Zones {
+ lib := libByZone[zone.ZoneID.ZoneName]
+ if lib == nil {
+ continue
+ }
+ lib.deltaMu.Lock()
+ lib.notifyToken = zone.SyncToken
+ lib.deltaMu.Unlock()
+
+ if len(zone.Records) == 0 && !zone.MoreComing {
+ continue
+ }
+
+ // Drain remaining pages per zone individually
+ for zone.MoreComing {
+ lib.deltaMu.Lock()
+ token := lib.notifyToken
+ lib.deltaMu.Unlock()
+ var next changesZoneResponse
+ body := map[string]any{"zones": []map[string]any{{
+ "zoneID": lib.zoneIDMap(),
+ "desiredKeys": changesZoneDesiredKeys,
+ "syncToken": token,
+ }}}
+ if err := lib.request(ctx, "changes/zone", body, &next); err != nil {
+ break
+ }
+ if len(next.Zones) == 0 {
+ break
+ }
+ lib.deltaMu.Lock()
+ lib.notifyToken = next.Zones[0].SyncToken
+ lib.deltaMu.Unlock()
+ zone.MoreComing = next.Zones[0].MoreComing
+ }
+
+ changed = append(changed, lib.zoneID)
+ fs.Debugf(nil, "iclouddrive photos: ChangeNotify detected changes in zone %s", lib.zoneID)
+ }
+ }
+ return changed
+}
+
+// readSyncToken loads the sync token from disk
+func (lib *Library) readSyncToken() string {
+ data, err := os.ReadFile(filepath.Join(lib.zoneCacheDir(), "syncToken"))
+ if err != nil {
+ return ""
+ }
+ return strings.TrimSpace(string(data))
+}
+
+// changesZoneResponse is the structure returned by changes/zone
+type changesZoneResponse struct {
+ Zones []changesZoneResult `json:"zones"`
+}
+
+// changesZoneResult is a single zone entry within a changesZoneResponse
+type changesZoneResult struct {
+ ZoneID struct {
+ ZoneName string `json:"zoneName"`
+ } `json:"zoneID"`
+ Records []json.RawMessage `json:"records"`
+ SyncToken string `json:"syncToken"`
+ MoreComing bool `json:"moreComing"`
+}
+
+// changesZoneDesiredKeys are the fields requested from changes/zone for
+// delta sync classification into smart albums
+var changesZoneDesiredKeys = []string{
+ // CPLMaster fields
+ "filenameEnc", "itemType", "resOriginalRes", "resOriginalWidth", "resOriginalHeight",
+ "resOriginalFileType", "resOriginalVidComplRes",
+ "resOriginalAltRes", "resOriginalAltFileType",
+ // CPLAsset classification + metadata fields
+ "masterRef", "assetDate", "addedDate",
+ "isFavorite", "isHidden", "isDeleted", "assetSubtype", "assetSubtypeV2", "burstId",
+ "adjustmentRenderType",
+ "adjustmentType", "resJPEGFullRes", "resJPEGFullFileType",
+ "resVidFullRes", "resVidFullFileType",
+ // CPLContainerRelation field for user album membership invalidation
+ "containerId",
+}
+
+// changesZoneBody builds the request body for a changes/zone call
+func (lib *Library) changesZoneBody(syncToken string) map[string]any {
+ zone := map[string]any{
+ "zoneID": lib.zoneIDMap(),
+ "desiredKeys": changesZoneDesiredKeys,
+ }
+ if syncToken != "" {
+ zone["syncToken"] = syncToken
+ }
+ return map[string]any{"zones": []map[string]any{zone}}
+}
+
+// deltaParseResult holds the classified output of parseDeltaRecords
+type deltaParseResult struct {
+ deletedIDs map[string]bool
+ newMasters map[string]*photoRecord
+ newAssets map[string]*photoRecord
+ changedAlbumRecords map[string]bool
+ albumMetadataChanged bool
+ hasAssetOnlyUpdates bool
+}
+
+func relationAlbumRecordFromRecordName(recordName string) (string, bool) {
+ parts := strings.SplitN(recordName, "-IN-", 2)
+ if len(parts) != 2 || parts[1] == "" {
+ return "", false
+ }
+ return parts[1], true
+}
+
+// shouldInvalidate returns true if an album's cache should be invalidated
+// rather than incrementally updated from this delta
+func (r *deltaParseResult) shouldInvalidate(recordName string, isSmart bool) bool {
+ return (!isSmart && r.changedAlbumRecords[recordName]) ||
+ (isSmart && r.hasAssetOnlyUpdates)
+}
+
+// parseDeltaRecords classifies raw delta records from changes/zone into
+// deletions, new masters/assets, album membership changes, and metadata flags
+func parseDeltaRecords(records []json.RawMessage) *deltaParseResult {
+ r := &deltaParseResult{
+ deletedIDs: map[string]bool{},
+ newMasters: map[string]*photoRecord{},
+ newAssets: map[string]*photoRecord{},
+ changedAlbumRecords: map[string]bool{},
+ }
+
+ for _, raw := range records {
+ var header struct {
+ RecordName string `json:"recordName"`
+ RecordType string `json:"recordType"`
+ Deleted bool `json:"deleted"`
+ }
+ if err := json.Unmarshal(raw, &header); err != nil {
+ continue
+ }
+ if header.Deleted && header.RecordType == "" {
+ if albumRecord, ok := relationAlbumRecordFromRecordName(header.RecordName); ok {
+ r.changedAlbumRecords[albumRecord] = true
+ continue
+ }
+ }
+ switch header.RecordType {
+ case "CPLMaster":
+ if header.Deleted {
+ r.deletedIDs[header.RecordName] = true
+ } else {
+ var rec photoRecord
+ if err := json.Unmarshal(raw, &rec); err == nil {
+ r.newMasters[rec.RecordName] = &rec
+ }
+ }
+ case "CPLAsset":
+ if header.Deleted {
+ // Asset deletion - mark both the asset itself and its master for removal
+ // The asset ID is needed because edited entries use asset.RecordName as ID
+ // (not master), so filtering by master alone leaves ghost -edited entries
+ r.deletedIDs[header.RecordName] = true
+ var rec photoRecord
+ if err := json.Unmarshal(raw, &rec); err == nil && rec.Fields.MasterRef != nil {
+ r.deletedIDs[rec.Fields.MasterRef.Value.RecordName] = true
+ }
+ } else {
+ var rec photoRecord
+ if err := json.Unmarshal(raw, &rec); err == nil && rec.Fields.MasterRef != nil {
+ r.newAssets[rec.Fields.MasterRef.Value.RecordName] = &rec
+ }
+ }
+ case "CPLAlbum":
+ r.albumMetadataChanged = true
+ case "CPLContainerRelation":
+ var rel struct {
+ Fields struct {
+ ContainerID *struct {
+ Value string `json:"value"`
+ } `json:"containerId"`
+ } `json:"fields"`
+ }
+ if err := json.Unmarshal(raw, &rel); err == nil {
+ if rel.Fields.ContainerID != nil && rel.Fields.ContainerID.Value != "" {
+ r.changedAlbumRecords[rel.Fields.ContainerID.Value] = true
+ continue
+ }
+ }
+ // Deleted relation records and some changes/zone entries omit fields,
+ // the deterministic recordName still encodes the target album record
+ if albumRecord, ok := relationAlbumRecordFromRecordName(header.RecordName); ok {
+ r.changedAlbumRecords[albumRecord] = true
+ } else if header.Deleted {
+ fs.Debugf(nil, "iclouddrive photos: deleted CPLContainerRelation %q has unexpected recordName format", header.RecordName)
+ }
+ }
+ }
+
+ // Detect asset-only metadata updates (favorite/hide/soft-delete toggle)
+ for masterID := range r.newAssets {
+ if _, hasMaster := r.newMasters[masterID]; !hasMaster && !r.deletedIDs[masterID] {
+ r.hasAssetOnlyUpdates = true
+ break
+ }
+ }
+
+ return r
+}
+
+// applyPendingDelta consumes a buffered delta and applies it to album caches
+// Called from GetPhotos after albums are guaranteed populated
+// Returns true if cache is current (no pending delta, or delta applied successfully)
+func (lib *Library) applyPendingDelta(ctx context.Context) bool {
+ lib.deltaMu.Lock()
+ defer lib.deltaMu.Unlock()
+
+ pending := lib.pendingDelta
+ if pending == nil {
+ return true // nothing pending, cache is current
+ }
+ failPendingDelta := func() bool {
+ lib.pendingDelta = nil
+ lib.cacheValid.Store(false)
+ return false
+ }
+
+ // Verify albums are populated - if not (e.g. eager album invalidation
+ // cleared the map before GetAlbums ran), clear pendingDelta so
+ // checkForChanges can re-detect it on the next call. SyncToken was not
+ // advanced so the same delta will be returned by changes/zone
+ lib.mu.Lock()
+ hasAlbums := len(lib.albums) > 0
+ lib.mu.Unlock()
+ if !hasAlbums {
+ lib.pendingDelta = nil
+ return false
+ }
+
+ // Collect all delta records (first page from buffer + remaining pages from API)
+ allRecords := pending.records
+ syncToken := pending.syncToken
+ moreComing := pending.moreComing
+ for moreComing {
+ var response changesZoneResponse
+ if err := lib.request(ctx, "changes/zone", lib.changesZoneBody(syncToken), &response); err != nil {
+ return failPendingDelta()
+ }
+ if len(response.Zones) == 0 {
+ return failPendingDelta()
+ }
+ allRecords = append(allRecords, response.Zones[0].Records...)
+ syncToken = response.Zones[0].SyncToken
+ moreComing = response.Zones[0].MoreComing
+ }
+
+ result := parseDeltaRecords(allRecords)
+
+ // Build new Photo entries from delta master+asset pairs
+ var addedPhotos []*Photo
+ for masterID, master := range result.newMasters {
+ built := buildPhotos(master, result.newAssets[masterID])
+ addedPhotos = append(addedPhotos, built...)
+ }
+
+ fs.Debugf(nil, "iclouddrive photos: zone %s delta: %d deleted, %d added, %d album membership changes from %d records",
+ lib.zoneID, len(result.deletedIDs), len(addedPhotos), len(result.changedAlbumRecords), len(allRecords))
+
+ // Apply delta to each album's disk cache
+ // Pre-resolve cache dir from lib to avoid re-acquiring ps.mu
+ // under deltaMu (lock ordering: deltaMu must not precede ps.mu)
+ cacheDir := lib.zoneCacheDir()
+ lib.mu.Lock()
+ albums := make([]*Album, 0, len(lib.albums))
+ for _, album := range flattenAlbumTree(nil, lib.albums) {
+ if album.ObjectType != "" {
+ albums = append(albums, album)
+ }
+ }
+ lib.mu.Unlock()
+
+ for _, album := range albums {
+ cached, ok := album.loadDiskCacheFrom(cacheDir)
+ if !ok {
+ continue
+ }
+
+ _, isSmart := SmartAlbums[album.Name]
+
+ // Skip albums that will be invalidated below (avoids stale-data
+ // window for concurrent readers and wasted disk I/O)
+ if result.shouldInvalidate(album.RecordName, isSmart) {
+ continue
+ }
+
+ // Remove deleted/changed entries
+ filtered := make([]*Photo, 0, len(cached))
+ for _, p := range cached {
+ if !result.deletedIDs[p.ID] {
+ filtered = append(filtered, p)
+ }
+ }
+
+ // Route new photos to smart albums based on classifySmartAlbums()
+ if isSmart {
+ for _, p := range addedPhotos {
+ for _, sa := range p.SmartAlbums {
+ if sa == album.Name {
+ filtered = append(filtered, p)
+ break
+ }
+ }
+ }
+ }
+
+ album.saveDiskCacheTo(cacheDir, filtered)
+
+ // Update in-memory cache if populated
+ album.mu.Lock()
+ if album.photoCache != nil {
+ // Deep copy before dedup so shared *Photo pointers across albums
+ // don't get cross-contaminated by filename suffix mutations
+ deduped := make([]*Photo, len(filtered))
+ for i, p := range filtered {
+ cp := *p
+ deduped[i] = &cp
+ }
+ deduplicateFilenames(deduped)
+ album.photoCache = buildPhotoCache(deduped)
+ }
+ album.mu.Unlock()
+ }
+
+ // Invalidate caches for albums affected by membership or metadata changes
+ if len(result.changedAlbumRecords) > 0 || result.hasAssetOnlyUpdates {
+ lib.mu.Lock()
+ var invalidated []*Album
+ for _, album := range flattenAlbumTree(nil, lib.albums) {
+ _, isSmart := SmartAlbums[album.Name]
+ if result.shouldInvalidate(album.RecordName, isSmart) {
+ invalidated = append(invalidated, album)
+ }
+ }
+ lib.mu.Unlock()
+ for _, album := range invalidated {
+ album.mu.Lock()
+ album.photoCache = nil
+ album.mu.Unlock()
+ if album.ObjectType != "" {
+ _ = os.Remove(filepath.Join(cacheDir, albumCacheKey(album.ObjectType)+".json"))
+ }
+ fs.Debugf(nil, "iclouddrive photos: invalidated album %q cache", album.Name)
+ }
+ }
+
+ // Album metadata change (CPLAlbum created/renamed/deleted) - clear album
+ // list so GetAlbums re-fetches from API on next call
+ if result.albumMetadataChanged {
+ lib.invalidateAlbumCache()
+ fs.Debugf(nil, "iclouddrive photos: zone %s album metadata changed, will re-fetch album list", lib.zoneID)
+ }
+
+ lib.pendingDelta = nil
+ lib.saveSyncToken(syncToken)
+ return true
+}
+
+// saveSyncToken persists the zone sync token to disk via atomic rename
+func (lib *Library) saveSyncToken(token string) {
+ dir := lib.zoneCacheDir()
+ if err := os.MkdirAll(dir, 0700); err != nil {
+ fs.Debugf(nil, "iclouddrive photos: failed to create cache dir: %v", err)
+ return
+ }
+ if err := atomicWriteFile(filepath.Join(dir, "syncToken"), []byte(token)); err != nil {
+ fs.Debugf(nil, "iclouddrive photos: failed to write sync token: %v", err)
+ }
+}
+
+// clearDiskCache removes all cached album data and sync token for this zone
+func (lib *Library) clearDiskCache() {
+ dir := lib.zoneCacheDir()
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return
+ }
+ for _, e := range entries {
+ _ = os.Remove(filepath.Join(dir, e.Name()))
+ }
+}
+
+// request makes an API call routed through the album's library area
+func (album *Album) request(ctx context.Context, endpoint string, data, response any) error {
+ if album.lib != nil {
+ return album.lib.request(ctx, endpoint, data, response)
+ }
+ return fmt.Errorf("album %q has no library", album.Name)
+}
+
+// zoneIDMap returns the full zoneID for this album's zone
+func (album *Album) zoneIDMap() map[string]any {
+ if album.lib == nil {
+ return map[string]any{}
+ }
+ return album.lib.zoneIDMap()
+}
+
+// loadDiskCacheFrom loads cached photo data from a specific cache directory
+func (album *Album) loadDiskCacheFrom(cacheDir string) ([]*Photo, bool) {
+ cacheFile := filepath.Join(cacheDir, albumCacheKey(album.ObjectType)+".json")
+ data, err := os.ReadFile(cacheFile)
+ if err != nil {
+ return nil, false
+ }
+ var photos []*Photo
+ if err := json.Unmarshal(data, &photos); err != nil {
+ return nil, false
+ }
+ return photos, true
+}
+
+// saveDiskCache persists photo data to disk for delta sync via atomic rename
+func (album *Album) saveDiskCache(photos []*Photo) {
+ if album.ObjectType == "" || album.lib == nil {
+ return
+ }
+ album.saveDiskCacheTo(album.lib.zoneCacheDir(), photos)
+}
+
+// saveDiskCacheTo persists photo data to a specific cache directory
+func (album *Album) saveDiskCacheTo(dir string, photos []*Photo) {
+ saveJSONCache(dir, albumCacheKey(album.ObjectType)+".json", photos)
+}
+
+// countQuery builds a HyperionIndexCountLookup query for a single object type
+func countQuery(objectType string, zoneID map[string]any) map[string]any {
+ return map[string]any{
+ "resultsLimit": 1,
+ "query": map[string]any{
+ "filterBy": map[string]any{
+ "fieldName": "indexCountID",
+ "fieldValue": map[string]any{"type": "STRING_LIST", "value": []string{objectType}},
+ "comparator": "IN",
+ },
+ "recordType": recordTypeCountIndex,
+ },
+ "zoneWide": true,
+ "zoneID": zoneID,
+ }
+}
+
+// countBatchResponse is the response shape for batched count queries
+type countBatchResponse struct {
+ Batch []struct {
+ Records []struct {
+ Fields struct {
+ ItemCount struct {
+ Value int64 `json:"value"`
+ } `json:"itemCount"`
+ } `json:"fields"`
+ } `json:"records"`
+ } `json:"batch"`
+}
+
+// toCounts maps an ordered list of names to their counts from the batch response
+func (r *countBatchResponse) toCounts(names []string) map[string]int64 {
+ counts := make(map[string]int64, len(names))
+ for i, name := range names {
+ if i < len(r.Batch) && len(r.Batch[i].Records) > 0 {
+ counts[name] = r.Batch[i].Records[0].Fields.ItemCount.Value
+ }
+ }
+ return counts
+}
+
+// GetLibraryAlbumCounts returns the album count for each library in a single
+// batched request, keyed by zone name (e.g. "PrimarySync")
+func (ps *PhotosService) GetLibraryAlbumCounts(ctx context.Context) (map[string]int64, error) {
+ ps.mu.Lock()
+ type libEntry struct {
+ name string
+ lib *Library
+ }
+ byArea := make(map[string][]libEntry)
+ for name, lib := range ps.libraries {
+ byArea[lib.area] = append(byArea[lib.area], libEntry{name: name, lib: lib})
+ }
+ ps.mu.Unlock()
+
+ counts := make(map[string]int64)
+ for area, entries := range byArea {
+ var batch []map[string]any
+ var order []string
+ for _, e := range entries {
+ order = append(order, e.name)
+ batch = append(batch, countQuery(recordTypeAlbum, e.lib.zoneIDMap()))
+ }
+ var response countBatchResponse
+ if err := ps.requestForArea(ctx, area, "internal/records/query/batch", map[string]any{"batch": batch}, &response); err != nil {
+ return nil, fmt.Errorf("failed to get library album counts: %w", err)
+ }
+ for k, v := range response.toCounts(order) {
+ counts[k] = v
+ }
+ }
+ return counts, nil
+}
+
+// CloudKit field types for record deserialization
+type ckStringField struct {
+ Value string `json:"value"`
+ Type string `json:"type,omitempty"` // present on filenameEnc (ENCRYPTED_BYTES vs STRING)
+}
+
+type ckIntField struct {
+ Value int `json:"value"`
+}
+
+type ckTimestampField struct {
+ Value int64 `json:"value"`
+}
+
+type ckResourceField struct {
+ Value struct {
+ Size int64 `json:"size"`
+ DownloadURL string `json:"downloadURL"`
+ } `json:"value"`
+}
+
+type ckBoolField struct {
+ Value bool `json:"value"`
+}
+
+type ckReferenceField struct {
+ Value struct {
+ RecordName string `json:"recordName"`
+ } `json:"value"`
+}
+
+// photoRecord represents a CloudKit record (CPLAsset or CPLMaster)
+type photoRecord struct {
+ RecordName string `json:"recordName"`
+ RecordType string `json:"recordType"`
+ Fields struct {
+ FilenameEnc *ckStringField `json:"filenameEnc,omitempty"`
+ ItemType *ckStringField `json:"itemType,omitempty"`
+ ResOriginalRes *ckResourceField `json:"resOriginalRes,omitempty"`
+ ResOriginalWidth *ckIntField `json:"resOriginalWidth,omitempty"`
+ ResOriginalHeight *ckIntField `json:"resOriginalHeight,omitempty"`
+ ResOriginalFileType *ckStringField `json:"resOriginalFileType,omitempty"`
+ ResOriginalVidComplRes *ckResourceField `json:"resOriginalVidComplRes,omitempty"`
+ ResOriginalAltRes *ckResourceField `json:"resOriginalAltRes,omitempty"`
+ ResOriginalAltFileType *ckStringField `json:"resOriginalAltFileType,omitempty"`
+ MasterRef *ckReferenceField `json:"masterRef,omitempty"`
+ AssetDate *ckTimestampField `json:"assetDate,omitempty"`
+ AddedDate *ckTimestampField `json:"addedDate,omitempty"`
+ IsFavorite *ckIntField `json:"isFavorite,omitempty"`
+ IsHidden *ckIntField `json:"isHidden,omitempty"`
+ AssetSubtype *ckIntField `json:"assetSubtype,omitempty"`
+ AssetSubtypeV2 *ckIntField `json:"assetSubtypeV2,omitempty"`
+ BurstID *ckStringField `json:"burstId,omitempty"`
+ AdjustmentRenderType *ckIntField `json:"adjustmentRenderType,omitempty"`
+ IsDeleted *ckIntField `json:"isDeleted,omitempty"`
+ AdjustmentType *ckStringField `json:"adjustmentType,omitempty"`
+ ResJPEGFullRes *ckResourceField `json:"resJPEGFullRes,omitempty"`
+ ResJPEGFullFileType *ckStringField `json:"resJPEGFullFileType,omitempty"`
+ ResVidFullRes *ckResourceField `json:"resVidFullRes,omitempty"`
+ ResVidFullFileType *ckStringField `json:"resVidFullFileType,omitempty"`
+ } `json:"fields"`
+}
+
+// classifySmartAlbums determines which smart albums a photo belongs to
+// based on CPLMaster and CPLAsset fields from CloudKit
+func classifySmartAlbums(master *photoRecord, asset *photoRecord) []string {
+ isVideo := false
+ if master.Fields.ResOriginalFileType != nil {
+ uti := master.Fields.ResOriginalFileType.Value
+ isVideo = uti == "public.mpeg-4" || uti == "com.apple.quicktime-movie"
+ }
+
+ var subtype, subtypeV2, favorite, hidden, deleted int
+ if asset != nil {
+ if asset.Fields.AssetSubtype != nil {
+ subtype = asset.Fields.AssetSubtype.Value
+ }
+ if asset.Fields.AssetSubtypeV2 != nil {
+ subtypeV2 = asset.Fields.AssetSubtypeV2.Value
+ }
+ if asset.Fields.IsFavorite != nil {
+ favorite = asset.Fields.IsFavorite.Value
+ }
+ if asset.Fields.IsHidden != nil {
+ hidden = asset.Fields.IsHidden.Value
+ }
+ if asset.Fields.IsDeleted != nil {
+ deleted = asset.Fields.IsDeleted.Value
+ }
+ }
+
+ // Soft-deleted assets (isDeleted=1) go to Recently Deleted only
+ if deleted == 1 {
+ return []string{"Recently Deleted"}
+ }
+
+ var albums []string
+ if hidden == 0 {
+ albums = append(albums, "All Photos")
+ }
+ if hidden == 1 {
+ albums = append(albums, "Hidden")
+ }
+ if favorite == 1 {
+ albums = append(albums, "Favorites")
+ }
+ if isVideo && subtype == 0 {
+ albums = append(albums, "Videos")
+ }
+ if subtype == subtypeSloMo {
+ albums = append(albums, "Slo-mo")
+ }
+ if subtype == subtypeTimeLapse {
+ albums = append(albums, "Time-lapse")
+ }
+ if subtype == subtypePanorama {
+ albums = append(albums, "Panoramas")
+ }
+ if subtypeV2 == subtypeV2Live {
+ albums = append(albums, "Live")
+ }
+ if subtypeV2 == subtypeV2Screenshot {
+ albums = append(albums, "Screenshots")
+ }
+ if asset != nil && asset.Fields.BurstID != nil && asset.Fields.BurstID.Value != "" {
+ albums = append(albums, "Bursts")
+ }
+ // adjustmentRenderType is a bitmask: PORTRAIT=2, LONG_EXPOSURE=4
+ if asset != nil && asset.Fields.AdjustmentRenderType != nil {
+ art := asset.Fields.AdjustmentRenderType.Value
+ if art&adjustPortrait != 0 {
+ albums = append(albums, "Portrait")
+ }
+ if art&adjustLongExposure != 0 {
+ albums = append(albums, "Long Exposure")
+ }
+ }
+ // Animated (GIFs) detected by file type on master record
+ if master.Fields.ResOriginalFileType != nil && master.Fields.ResOriginalFileType.Value == "com.compuserve.gif" {
+ albums = append(albums, "Animated")
+ }
+ // Selfies: no reliable field available from delta records - server query handles it
+ return albums
+}
+
+// deduplicateFilenames renames ALL photos with colliding filenames by appending
+// the full masterID (CloudKit recordName) before the extension. Every duplicate
+// gets the suffix so filenames are stable when photos are added or removed
+// Unique filenames are untouched. Collision-free by construction since CloudKit
+// recordNames are unique. Same pattern as googlephotos which embeds the full
+// media item ID ({55+ chars}) in every filename
+func deduplicateFilenames(photos []*Photo) {
+ counts := make(map[string]int, len(photos))
+ for _, p := range photos {
+ counts[p.Filename]++
+ }
+ for _, p := range photos {
+ if counts[p.Filename] <= 1 {
+ continue
+ }
+ ext := path.Ext(p.Filename)
+ base := strings.TrimSuffix(p.Filename, ext)
+ p.Filename = base + "_" + p.ID + ext
+ }
+}
+
+// buildPhotos creates Photo entries from a CPLMaster record and its paired CPLAsset
+// Returns 1-2 entries: the photo itself, plus a .MOV companion for Live Photos
+func buildPhotos(master *photoRecord, asset *photoRecord) []*Photo {
+ photo := &Photo{ID: master.RecordName}
+
+ if master.Fields.FilenameEnc != nil {
+ if master.Fields.FilenameEnc.Type == "STRING" {
+ photo.Filename = norm.NFC.String(master.Fields.FilenameEnc.Value)
+ } else if decoded, err := base64.StdEncoding.DecodeString(master.Fields.FilenameEnc.Value); err == nil {
+ photo.Filename = norm.NFC.String(string(decoded))
+ }
+ }
+ // Fallback: synthesize filename from recordName + itemType UTI when filenameEnc is missing
+ if photo.Filename == "" && master.Fields.ItemType != nil {
+ if ext, ok := utiExtensions[master.Fields.ItemType.Value]; ok {
+ photo.Filename = master.RecordName + ext
+ }
+ }
+
+ if master.Fields.ResOriginalRes != nil {
+ photo.Size = master.Fields.ResOriginalRes.Value.Size
+ }
+
+ if master.Fields.ResOriginalWidth != nil {
+ photo.Width = master.Fields.ResOriginalWidth.Value
+ }
+ if master.Fields.ResOriginalHeight != nil {
+ photo.Height = master.Fields.ResOriginalHeight.Value
+ }
+
+ var liveVideoSize int64
+ var hasLiveVideo bool
+ if master.Fields.ResOriginalVidComplRes != nil && master.Fields.ResOriginalVidComplRes.Value.DownloadURL != "" {
+ liveVideoSize = master.Fields.ResOriginalVidComplRes.Value.Size
+ hasLiveVideo = true
+ }
+
+ if asset != nil {
+ if asset.Fields.AssetDate != nil {
+ photo.AssetDate = asset.Fields.AssetDate.Value
+ }
+ if asset.Fields.AddedDate != nil {
+ photo.AddedDate = asset.Fields.AddedDate.Value
+ }
+ photo.IsFavorite = asset.Fields.IsFavorite != nil && asset.Fields.IsFavorite.Value == 1
+ photo.IsHidden = asset.Fields.IsHidden != nil && asset.Fields.IsHidden.Value == 1
+ }
+
+ photo.SmartAlbums = classifySmartAlbums(master, asset)
+
+ hasDownloadURL := master.Fields.ResOriginalRes != nil && master.Fields.ResOriginalRes.Value.DownloadURL != ""
+ if !hasDownloadURL || photo.Filename == "" {
+ return nil
+ }
+
+ photo.ResourceKey = "resOriginalRes"
+ result := []*Photo{photo}
+
+ ext := path.Ext(photo.Filename)
+ stem := strings.TrimSuffix(photo.Filename, ext)
+
+ if hasLiveVideo {
+ result = append(result, photo.companion(photo.ID, stem+".MOV", "resOriginalVidComplRes", liveVideoSize))
+ }
+
+ // Edited photo version (Photos.app adjustments)
+ // Slo-mo edits are metadata-only (playback speed) with no separate rendered resource
+ if asset != nil && asset.Fields.AdjustmentType != nil &&
+ asset.Fields.AdjustmentType.Value != "" &&
+ asset.Fields.AdjustmentType.Value != adjustSloMo {
+ if asset.Fields.ResJPEGFullRes != nil && asset.Fields.ResJPEGFullRes.Value.DownloadURL != "" {
+ editExt := extFromUTI(asset.Fields.ResJPEGFullFileType, ext)
+ result = append(result, photo.companion(asset.RecordName, stem+"-edited"+editExt, "resJPEGFullRes", asset.Fields.ResJPEGFullRes.Value.Size))
+ } else if asset.Fields.ResVidFullRes != nil && asset.Fields.ResVidFullRes.Value.DownloadURL != "" {
+ editExt := extFromUTI(asset.Fields.ResVidFullFileType, ext)
+ result = append(result, photo.companion(asset.RecordName, stem+"-edited"+editExt, "resVidFullRes", asset.Fields.ResVidFullRes.Value.Size))
+ }
+ }
+
+ // RAW alternative (RAW+JPEG pairs where both originals are stored)
+ if master.Fields.ResOriginalAltRes != nil && master.Fields.ResOriginalAltRes.Value.DownloadURL != "" {
+ altExt := extFromUTI(master.Fields.ResOriginalAltFileType, ext)
+ altFilename := stem + altExt
+ if strings.EqualFold(altFilename, photo.Filename) {
+ altFilename = stem + "-alt" + altExt
+ }
+ alt := photo.companion(master.RecordName, altFilename, "resOriginalAltRes", master.Fields.ResOriginalAltRes.Value.Size)
+ alt.Width = photo.Width // same sensor capture, same dimensions
+ alt.Height = photo.Height // same sensor capture, same dimensions
+ result = append(result, alt)
+ }
+
+ return result
+}
+
+// photosDesiredKeys are the fields requested for photo listing
+var photosDesiredKeys = []string{
+ "resOriginalRes", "resOriginalVidComplRes", "resOriginalFileType",
+ "resOriginalWidth", "resOriginalHeight",
+ "resOriginalAltRes", "resOriginalAltFileType",
+ "filenameEnc", "itemType", "assetDate", "addedDate", "masterRef",
+ "isFavorite", "isHidden", "isDeleted",
+ "assetSubtype", "assetSubtypeV2", "burstId", "adjustmentRenderType",
+ "adjustmentType", "resJPEGFullRes", "resJPEGFullFileType",
+ "resVidFullRes", "resVidFullFileType",
+}
+
+// fetchPhotoCount returns the photo count for this album via HyperionIndexCountLookup
+func (album *Album) fetchPhotoCount(ctx context.Context) (int64, error) {
+ if album.ObjectType == "" {
+ return 0, nil
+ }
+ var response struct {
+ Records []struct {
+ Fields struct {
+ ItemCount struct {
+ Value int64 `json:"value"`
+ } `json:"itemCount"`
+ } `json:"fields"`
+ } `json:"records"`
+ }
+ if err := album.request(ctx, "records/query", countQuery(album.ObjectType, album.zoneIDMap()), &response); err != nil {
+ return 0, err
+ }
+ if len(response.Records) > 0 {
+ return response.Records[0].Fields.ItemCount.Value, nil
+ }
+ return 0, nil
+}
+
+// parsePhotoRecords extracts Photo entries from a batch of CloudKit records
+func parsePhotoRecords(records []photoRecord) []*Photo {
+ if len(records) == 0 {
+ return nil
+ }
+ half := len(records)/2 + 1
+ assetMap := make(map[string]*photoRecord, half)
+ masters := make([]*photoRecord, 0, half)
+ for i := range records {
+ record := &records[i]
+ switch record.RecordType {
+ case "CPLAsset":
+ if record.Fields.MasterRef != nil {
+ assetMap[record.Fields.MasterRef.Value.RecordName] = record
+ }
+ case "CPLMaster":
+ masters = append(masters, record)
+ }
+ }
+ photos := make([]*Photo, 0, len(masters))
+ for _, master := range masters {
+ built := buildPhotos(master, assetMap[master.RecordName])
+ photos = append(photos, built...)
+ }
+ return photos
+}
+
+// buildPartitionQuery constructs a CloudKit records/query body for a single
+// startRank EQUALS partition, including album-specific direction and filters
+func (album *Album) buildPartitionQuery(startRank int) map[string]any {
+ filters := []map[string]any{
+ {
+ "fieldName": "startRank",
+ "comparator": "EQUALS",
+ "fieldValue": map[string]any{"type": "INT64", "value": startRank},
+ },
+ {
+ "fieldName": "direction",
+ "fieldValue": map[string]any{"type": "STRING", "value": album.Direction},
+ "comparator": "EQUALS",
+ },
+ }
+ for _, filter := range album.Filters {
+ filters = append(filters, map[string]any{
+ "fieldName": filter.FieldName,
+ "comparator": filter.Comparator,
+ "fieldValue": filter.FieldValue,
+ })
+ }
+ return map[string]any{
+ "query": map[string]any{
+ "filterBy": filters,
+ "recordType": album.ListType,
+ },
+ "resultsLimit": photosQueryLimit,
+ "desiredKeys": photosDesiredKeys,
+ "zoneID": album.zoneIDMap(),
+ }
+}
+
+// fetchPhotosParallel fetches all photos using parallel startRank partitions
+// Each partition is one API call with startRank EQUALS, no continuationMarker
+// Stride = photosQueryLimit/2 photos per partition (200 records = 100 photos)
+func (album *Album) fetchPhotosParallel(ctx context.Context, totalPhotos int64) ([]*Photo, string, error) {
+ stride := photosQueryLimit / 2 // 100 photos per partition
+ numPartitions := int((totalPhotos + int64(stride) - 1) / int64(stride))
+ workers := fs.GetConfig(ctx).Checkers
+ if workers < 1 {
+ workers = 8
+ }
+
+ fs.Logf(nil, "iclouddrive photos: parallel cold listing %d photos in %d partitions (%d workers)",
+ totalPhotos, numPartitions, workers)
+
+ type partitionResult struct {
+ photos []*Photo
+ syncToken string
+ recordCount int // raw record count to detect full pages
+ err error
+ }
+
+ // Cancel all remaining goroutines on first error
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ results := make([]partitionResult, numPartitions)
+ sem := make(chan struct{}, workers)
+ var wg sync.WaitGroup
+
+ for i := 0; i < numPartitions; i++ {
+ wg.Add(1)
+ go func(idx int) {
+ defer wg.Done()
+ select {
+ case sem <- struct{}{}:
+ case <-ctx.Done():
+ results[idx] = partitionResult{err: ctx.Err()}
+ return
+ }
+ defer func() { <-sem }()
+
+ startRank := idx * stride
+ query := album.buildPartitionQuery(startRank)
+
+ var response struct {
+ Records []photoRecord `json:"records"`
+ SyncToken string `json:"syncToken"`
+ }
+ if err := album.request(ctx, "records/query", query, &response); err != nil {
+ cancel() // stop remaining partitions
+ results[idx] = partitionResult{err: fmt.Errorf("partition %d (rank=%d): %w", idx, startRank, err)}
+ return
+ }
+ results[idx] = partitionResult{
+ photos: parsePhotoRecords(response.Records),
+ syncToken: response.SyncToken,
+ recordCount: len(response.Records),
+ }
+ }(i)
+ }
+ wg.Wait()
+
+ // Merge results in order
+ var allPhotos []*Photo
+ var lastSyncToken string
+ var lastRecordCount int
+ for _, r := range results {
+ if r.err != nil {
+ return nil, "", r.err
+ }
+ allPhotos = append(allPhotos, r.photos...)
+ if r.syncToken != "" {
+ lastSyncToken = r.syncToken
+ }
+ lastRecordCount = r.recordCount
+ }
+
+ // Completeness: keep fetching until a partition returns fewer than
+ // resultsLimit records (partial page = last page). This handles stale
+ // counts AND count=0 (count query failed - discover all photos here)
+ needsTailCheck := lastRecordCount >= photosQueryLimit || numPartitions == 0
+ nextRank := numPartitions * stride
+ for needsTailCheck {
+ fs.Debugf(nil, "iclouddrive photos: fetching tail partition at rank=%d", nextRank)
+ query := album.buildPartitionQuery(nextRank)
+ var response struct {
+ Records []photoRecord `json:"records"`
+ SyncToken string `json:"syncToken"`
+ }
+ if err := album.request(ctx, "records/query", query, &response); err != nil {
+ return nil, "", fmt.Errorf("tail partition (rank=%d): %w", nextRank, err)
+ }
+ lastRecordCount = len(response.Records)
+ allPhotos = append(allPhotos, parsePhotoRecords(response.Records)...)
+ if response.SyncToken != "" {
+ lastSyncToken = response.SyncToken
+ }
+ nextRank += stride
+ needsTailCheck = lastRecordCount >= photosQueryLimit
+ }
+
+ fs.Logf(nil, "iclouddrive photos: parallel fetch complete, %d photos", len(allPhotos))
+ return allPhotos, lastSyncToken, nil
+}
+
+// GetPhotos retrieves photos from this album using parallel partitions with disk cache
+func (album *Album) GetPhotos(ctx context.Context) ([]*Photo, error) {
+ // No library configured - return pre-populated cache (test path)
+ if album.lib == nil {
+ album.mu.Lock()
+ defer album.mu.Unlock()
+ result := make([]*Photo, 0, len(album.photoCache))
+ for _, p := range album.photoCache {
+ result = append(result, p)
+ }
+ return result, nil
+ }
+
+ // Check for changes, apply any buffered delta, serve from cache
+ if album.ObjectType != "" {
+ album.lib.checkForChanges(ctx)
+ if album.lib.applyPendingDelta(ctx) && album.lib.cacheValid.Load() {
+ // Serve from in-memory cache if populated (avoids disk I/O + JSON parse + dedup)
+ album.mu.Lock()
+ if album.photoCache != nil {
+ result := make([]*Photo, 0, len(album.photoCache))
+ for _, p := range album.photoCache {
+ result = append(result, p)
+ }
+ album.mu.Unlock()
+ return result, nil
+ }
+ album.mu.Unlock()
+ // Fall back to disk cache
+ if cached, ok := album.loadDiskCacheFrom(album.lib.zoneCacheDir()); ok {
+ deduplicateFilenames(cached)
+ album.mu.Lock()
+ album.photoCache = buildPhotoCache(cached)
+ album.mu.Unlock()
+ fs.Debugf(nil, "iclouddrive photos: %d items from cache for %q", len(cached), album.Name)
+ return cached, nil
+ }
+ }
+ }
+
+ // Fetch photo count for parallel partition calculation
+ // If count unavailable, use 0 - the tail-fetch loop handles completeness
+ var count int64
+ if album.ObjectType != "" {
+ count, _ = album.fetchPhotoCount(ctx)
+ }
+ photos, lastSyncToken, err := album.fetchPhotosParallel(ctx, count)
+ if err != nil {
+ return nil, err
+ }
+
+ // Persist original filenames for delta sync (dedup is applied on read)
+ album.saveDiskCache(photos)
+
+ deduplicateFilenames(photos)
+
+ // Populate filename cache for NewObject lookups
+ album.mu.Lock()
+ album.photoCache = buildPhotoCache(photos)
+ album.mu.Unlock()
+ if lastSyncToken != "" && album.lib != nil {
+ album.lib.saveSyncToken(lastSyncToken)
+ }
+
+ return photos, nil
+}
+
+// GetPhotoByName looks up a photo by filename, using cache if available
+// CloudKit has no filterBy on filename fields - the only queryable fields
+// are rank, date, smartAlbum, etc. Apple's own icloud.com UI paginates
+// the full album and indexes client-side. On cache miss we must enumerate
+// the entire album via GetPhotos before lookup
+func (album *Album) GetPhotoByName(ctx context.Context, filename string) (*Photo, error) {
+ album.mu.Lock()
+ if album.photoCache != nil {
+ photo, exists := album.photoCache[filename]
+ album.mu.Unlock()
+ if exists {
+ return photo, nil
+ }
+ return nil, fmt.Errorf("photo %q not found in album %q", filename, album.Name)
+ }
+ album.mu.Unlock()
+
+ // Cache miss - fetch all photos to populate cache
+ photos, err := album.GetPhotos(ctx)
+ if err != nil {
+ return nil, err
+ }
+ for _, photo := range photos {
+ if photo.Filename == filename {
+ return photo, nil
+ }
+ }
+ return nil, fmt.Errorf("photo %q not found in album %q", filename, album.Name)
+}
+
+// GetAlbumCounts returns photo counts for all albums in a single batch request
+func (lib *Library) GetAlbumCounts(ctx context.Context) (map[string]int64, error) {
+ // Snapshot under lock to avoid racing with GetAlbums
+ type albumEntry struct {
+ name string
+ objectType string
+ }
+ lib.mu.Lock()
+ entries := make([]albumEntry, 0, len(lib.albums))
+ for name, album := range lib.albums {
+ if album.ObjectType == "" {
+ continue // skip folders (albumType=3), they have no photo count
+ }
+ entries = append(entries, albumEntry{name: name, objectType: album.ObjectType})
+ }
+ lib.mu.Unlock()
+
+ if len(entries) == 0 {
+ return nil, nil
+ }
+
+ zoneIDAny := lib.zoneIDMap()
+
+ var batch []map[string]any
+ var albumOrder []string
+ for _, entry := range entries {
+ albumOrder = append(albumOrder, entry.name)
+ batch = append(batch, countQuery(entry.objectType, zoneIDAny))
+ }
+ var response countBatchResponse
+ if err := lib.request(ctx, "internal/records/query/batch", map[string]any{"batch": batch}, &response); err != nil {
+ return nil, err
+ }
+ return response.toCounts(albumOrder), nil
+}
+
+// resolveZone returns the area and full zoneID for a zone name,
+// using the library metadata if available or falling back to private
+func (ps *PhotosService) resolveZone(zoneName string) (area string, zoneID map[string]any) {
+ ps.mu.Lock()
+ lib := ps.libraries[zoneName]
+ ps.mu.Unlock()
+ if lib != nil {
+ return lib.area, lib.zoneIDMap()
+ }
+ return areaPrivate, map[string]any{"zoneName": zoneName}
+}
+
+// LookupDownloadURL fetches a fresh download URL for a record
+// recordName is the CPLMaster or CPLAsset recordName depending on the resource
+// resourceKey selects which resource to look up (e.g. "resOriginalRes",
+// "resOriginalVidComplRes" for Live Photo video, "resJPEGFullRes" for edited)
+func (ps *PhotosService) LookupDownloadURL(ctx context.Context, recordName, zone, resourceKey string) (string, error) {
+ area, zoneID := ps.resolveZone(zone)
+
+ query := map[string]any{
+ "records": []map[string]any{
+ {"recordName": recordName},
+ },
+ "zoneID": zoneID,
+ }
+
+ var response struct {
+ Records []json.RawMessage `json:"records"`
+ }
+
+ if err := ps.requestForArea(ctx, area, "records/lookup", query, &response); err != nil {
+ return "", fmt.Errorf("failed to look up record %q: %w", recordName, err)
+ }
+
+ if len(response.Records) == 0 {
+ return "", fmt.Errorf("no records in lookup response for %q", recordName)
+ }
+
+ // Parse fields as raw JSON to extract the requested resource key
+ var record struct {
+ Fields map[string]json.RawMessage `json:"fields"`
+ }
+ if err := json.Unmarshal(response.Records[0], &record); err != nil {
+ return "", fmt.Errorf("failed to parse lookup response: %w", err)
+ }
+
+ rawField, exists := record.Fields[resourceKey]
+ if !exists {
+ return "", fmt.Errorf("no %q field in record %q", resourceKey, recordName)
+ }
+
+ var res struct {
+ Value struct {
+ DownloadURL string `json:"downloadURL"`
+ } `json:"value"`
+ }
+ if err := json.Unmarshal(rawField, &res); err != nil {
+ return "", fmt.Errorf("failed to parse %q field: %w", resourceKey, err)
+ }
+ if res.Value.DownloadURL == "" {
+ return "", fmt.Errorf("no download URL for %q in record %q", resourceKey, recordName)
+ }
+
+ return res.Value.DownloadURL, nil
+}
+
+// checkIndexingState warns if the iCloud Photo Library is still indexing
+func (ps *PhotosService) checkIndexingState(ctx context.Context, zoneName string) {
+ area, zoneID := ps.resolveZone(zoneName)
+
+ query := map[string]any{
+ "query": map[string]any{"recordType": "CheckIndexingState"},
+ "zoneID": zoneID,
+ }
+
+ var response struct {
+ Records []struct {
+ Fields struct {
+ State struct {
+ Value string `json:"value"`
+ } `json:"state"`
+ } `json:"fields"`
+ } `json:"records"`
+ }
+
+ if err := ps.requestForArea(ctx, area, "records/query", query, &response); err != nil {
+ fs.Logf(nil, "iclouddrive photos: could not check indexing state: %v", err)
+ return
+ }
+
+ if len(response.Records) == 0 || response.Records[0].Fields.State.Value != indexingStateReady {
+ fs.Logf(nil, "iclouddrive photos: library is still indexing, listings may be incomplete")
+ }
+}
+
+// requestWithReauth makes a CloudKit request with pacer retry and reauth on 401/421
+func (ps *PhotosService) requestWithReauth(ctx context.Context, makeOpts func() rest.Opts, data, response any) error {
+ reauthDone := false
+ return ps.pacer.Call(func() (bool, error) {
+ resp, err := ps.client.Session.Request(ctx, makeOpts(), data, response)
+ if !reauthDone && err != nil && resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 421) {
+ reauthDone = true
+ if authErr := ps.client.Authenticate(ctx); authErr != nil {
+ return false, authErr
+ }
+ if ps.client.Session.Requires2FA() {
+ return false, errors.New("trust token expired, please reauth")
+ }
+ resp, err = ps.client.Session.Request(ctx, makeOpts(), data, response)
+ }
+ return ps.shouldRetry(ctx, resp, err)
+ })
+}
+
+// requestForArea makes a request to the given area (private or shared) endpoint
+func (ps *PhotosService) requestForArea(ctx context.Context, area, endpoint string, data, response any) error {
+ rootURL := fmt.Sprintf("%s/%s/%s?remapEnums=true&getCurrentSyncToken=true", ps.endpoint, area, endpoint)
+
+ return ps.requestWithReauth(ctx, func() rest.Opts {
+ return rest.Opts{
+ Method: "POST",
+ RootURL: rootURL,
+ ExtraHeaders: ps.client.Session.GetHeaders(map[string]string{"Content-Type": "text/plain"}), // text/plain matches icloud.com (CORS preflight bypass)
+ }
+ }, data, response)
+}
diff --git a/backend/iclouddrive/api/photos_test.go b/backend/iclouddrive/api/photos_test.go
new file mode 100644
index 000000000..59ec73168
--- /dev/null
+++ b/backend/iclouddrive/api/photos_test.go
@@ -0,0 +1,1285 @@
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "sync/atomic"
+ "testing"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/rclone/rclone/lib/rest"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func setTestCacheDir(t *testing.T) string {
+ t.Helper()
+ oldCacheDir := config.GetCacheDir()
+ cacheDir := t.TempDir()
+ require.NoError(t, config.SetCacheDir(cacheDir))
+ t.Cleanup(func() {
+ _ = config.SetCacheDir(oldCacheDir)
+ })
+ return cacheDir
+}
+
+func newHTTPTestPhotosService(t *testing.T, remoteName string, handler http.HandlerFunc) *PhotosService {
+ t.Helper()
+ server := httptest.NewServer(handler)
+ t.Cleanup(server.Close)
+
+ session := NewSession()
+ session.srv = rest.NewClient(server.Client())
+
+ return &PhotosService{
+ client: &Client{
+ remoteName: remoteName,
+ Session: session,
+ },
+ endpoint: server.URL + "/database/1/com.apple.photos.cloud/production",
+ pacer: fs.NewPacer(context.Background(), pacer.NewDefault()),
+ shouldRetry: func(ctx context.Context, resp *http.Response, err error) (bool, error) { return false, err },
+ libraries: make(map[string]*Library),
+ }
+}
+
+func writeJSON(t *testing.T, w http.ResponseWriter, v any) {
+ t.Helper()
+ w.Header().Set("Content-Type", "application/json")
+ if err := json.NewEncoder(w).Encode(v); err != nil {
+ panic(err)
+ }
+}
+
+func readJSONBody(r *http.Request, out any) error {
+ data, err := io.ReadAll(r.Body)
+ if err != nil {
+ return err
+ }
+ return json.Unmarshal(data, out)
+}
+
+func testAlbumRecordJSON(name, recordName string, albumType int, parentID string) map[string]any {
+ fields := map[string]any{
+ "albumNameEnc": map[string]any{"value": base64.StdEncoding.EncodeToString([]byte(name))},
+ }
+ if albumType != 0 {
+ fields["albumType"] = map[string]any{"value": albumType}
+ }
+ if parentID != "" {
+ fields["parentId"] = map[string]any{"value": parentID}
+ }
+ return map[string]any{
+ "recordName": recordName,
+ "fields": fields,
+ }
+}
+
+func newUserAlbumForTest(lib *Library, name, recordName string) *Album {
+ album := lib.newUserAlbum(name, recordName)
+ album.lib = lib
+ return album
+}
+
+func TestGetPhotos_DoesNotServeStaleCacheOnPagedDeltaFailure(t *testing.T) {
+ setTestCacheDir(t)
+ ctx := context.Background()
+ var changesZoneCalls atomic.Int32
+
+ ps := newHTTPTestPhotosService(t, "paged-delta-failure", func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/database/1/com.apple.photos.cloud/production/private/changes/zone":
+ if changesZoneCalls.Add(1) > 1 {
+ http.Error(w, "paged delta failure", http.StatusInternalServerError)
+ return
+ }
+ writeJSON(t, w, map[string]any{
+ "zones": []map[string]any{{
+ "zoneID": map[string]any{"zoneName": "PrimarySync"},
+ "records": []map[string]any{{"recordName": "asset1-IN-album-record", "deleted": true}},
+ "syncToken": "mid-token",
+ "moreComing": true,
+ }},
+ })
+ default:
+ http.Error(w, "unexpected request", http.StatusInternalServerError)
+ }
+ })
+
+ lib := &Library{service: ps, zoneID: "PrimarySync", area: areaPrivate, albums: map[string]*Album{}}
+ album := newUserAlbumForTest(lib, "Exported", "album-record")
+ lib.albums[album.Name] = album
+ album.saveDiskCache([]*Photo{{ID: "stale-id", Filename: "stale.jpg"}})
+ require.NoError(t, os.WriteFile(filepath.Join(lib.zoneCacheDir(), "syncToken"), []byte("old-token"), 0600))
+
+ photos, err := album.GetPhotos(ctx)
+ assert.Error(t, err)
+ assert.Nil(t, photos)
+ assert.Nil(t, lib.pendingDelta, "failed delta apply must not leave pendingDelta stuck forever")
+}
+
+func TestGetLibraries_RediscoversZonesWhenCacheExists(t *testing.T) {
+ setTestCacheDir(t)
+ ctx := context.Background()
+ var changesDatabaseCalls atomic.Int32
+
+ ps := newHTTPTestPhotosService(t, "library-rediscovery", func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/database/1/com.apple.photos.cloud/production/private/changes/database" {
+ changesDatabaseCalls.Add(1)
+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})
+ return
+ }
+ if r.URL.Path == "/database/1/com.apple.photos.cloud/production/shared/changes/database" {
+ changesDatabaseCalls.Add(1)
+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "SharedSync-test", "ownerRecordName": "other", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})
+ return
+ }
+ http.Error(w, "unexpected request", http.StatusInternalServerError)
+ })
+
+ saveJSONCache(ps.client.CacheDir(), "libraries.json", []cachedLibraryEntry{{ZoneName: "PrimarySync", Area: areaPrivate}})
+
+ libs, err := ps.GetLibraries(ctx)
+ require.NoError(t, err)
+ assert.Contains(t, libs, "PrimarySync")
+ assert.Contains(t, libs, "SharedSync-test")
+ assert.GreaterOrEqual(t, changesDatabaseCalls.Load(), int32(2), "must rediscover zones even when cache exists")
+}
+
+func TestGetLibraries_RediscoveryPreservesBufferedDelta(t *testing.T) {
+ setTestCacheDir(t)
+ ctx := context.Background()
+
+ ps := newHTTPTestPhotosService(t, "library-merge", func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/database/1/com.apple.photos.cloud/production/private/changes/zone":
+ writeJSON(t, w, map[string]any{
+ "zones": []map[string]any{{
+ "zoneID": map[string]any{"zoneName": "PrimarySync"},
+ "records": []map[string]any{{"recordName": "asset123-IN-album-uuid", "deleted": true}},
+ "syncToken": "new-token",
+ "moreComing": false,
+ }},
+ })
+ case "/database/1/com.apple.photos.cloud/production/private/changes/database":
+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})
+ case "/database/1/com.apple.photos.cloud/production/shared/changes/database":
+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "SharedSync-test", "ownerRecordName": "other", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})
+ default:
+ http.Error(w, "unexpected request", http.StatusInternalServerError)
+ }
+ })
+
+ saveJSONCache(ps.client.CacheDir(), "libraries.json", []cachedLibraryEntry{{ZoneName: "PrimarySync", Area: areaPrivate}})
+ require.NoError(t, os.MkdirAll(filepath.Join(ps.client.CacheDir(), "PrimarySync"), 0700))
+ require.NoError(t, os.WriteFile(filepath.Join(ps.client.CacheDir(), "PrimarySync", "syncToken"), []byte("cached-token"), 0600))
+
+ libs, err := ps.GetLibraries(ctx)
+ require.NoError(t, err)
+ primary := libs["PrimarySync"]
+ require.NotNil(t, primary)
+ require.NotNil(t, primary.pendingDelta, "rediscovery must not discard pending delta buffered from cached zone state")
+ assert.Contains(t, libs, "SharedSync-test")
+}
+
+func TestGetLibraries_RefreshesInMemoryLibraries(t *testing.T) {
+ setTestCacheDir(t)
+ ctx := context.Background()
+ var changesDatabaseCalls atomic.Int32
+
+ ps := newHTTPTestPhotosService(t, "library-refresh-memory", func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/database/1/com.apple.photos.cloud/production/private/changes/database":
+ changesDatabaseCalls.Add(1)
+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})
+ case "/database/1/com.apple.photos.cloud/production/shared/changes/database":
+ changesDatabaseCalls.Add(1)
+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "SharedSync-live", "ownerRecordName": "other", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})
+ default:
+ http.Error(w, "unexpected request", http.StatusInternalServerError)
+ }
+ })
+
+ ps.libraries = map[string]*Library{
+ "PrimarySync": {service: ps, zoneID: "PrimarySync", area: areaPrivate, albums: make(map[string]*Album)},
+ }
+
+ libs, err := ps.GetLibraries(ctx)
+ require.NoError(t, err)
+ assert.Contains(t, libs, "PrimarySync")
+ assert.Contains(t, libs, "SharedSync-live")
+ assert.GreaterOrEqual(t, changesDatabaseCalls.Load(), int32(2), "in-memory libraries must still rediscover zones")
+}
+
+func TestGetLibraries_KeepsCachedSharedZonesOnTransientSharedRediscoveryFailure(t *testing.T) {
+ setTestCacheDir(t)
+ ctx := context.Background()
+
+ ps := newHTTPTestPhotosService(t, "library-shared-preserve", func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/database/1/com.apple.photos.cloud/production/private/changes/database":
+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})
+ case "/database/1/com.apple.photos.cloud/production/shared/changes/database":
+ http.Error(w, "transient shared discovery failure", http.StatusInternalServerError)
+ case "/database/1/com.apple.photos.cloud/production/shared/records/query":
+ http.Error(w, "transient per-zone probe failure", http.StatusInternalServerError)
+ default:
+ http.Error(w, "unexpected request", http.StatusInternalServerError)
+ }
+ })
+
+ saveJSONCache(ps.client.CacheDir(), "libraries.json", []cachedLibraryEntry{{ZoneName: "PrimarySync", Area: areaPrivate}, {ZoneName: "SharedSync-cached", Area: areaShared}})
+
+ libs, err := ps.GetLibraries(ctx)
+ require.NoError(t, err)
+ assert.Contains(t, libs, "PrimarySync")
+ assert.Contains(t, libs, "SharedSync-cached")
+
+ data, err := os.ReadFile(filepath.Join(ps.client.CacheDir(), "libraries.json"))
+ require.NoError(t, err)
+ assert.Contains(t, string(data), "SharedSync-cached")
+}
+
+func TestGetLibraries_DropsCachedSharedZoneOnZoneNotFoundProbe(t *testing.T) {
+ setTestCacheDir(t)
+ ctx := context.Background()
+
+ ps := newHTTPTestPhotosService(t, "library-shared-drop", func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/database/1/com.apple.photos.cloud/production/private/changes/database":
+ writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}})
+ case "/database/1/com.apple.photos.cloud/production/shared/changes/database":
+ http.Error(w, "transient shared discovery failure", http.StatusInternalServerError)
+ case "/database/1/com.apple.photos.cloud/production/shared/records/query":
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusNotFound)
+ _, _ = w.Write([]byte(`{
+ "serverErrorCode": "ZONE_NOT_FOUND",
+ "reason": "Zone does not exist",
+ "errorClass": "ZONE_NOT_FOUND",
+ "error": "ZONE_NOT_FOUND"
+ }`))
+ default:
+ http.Error(w, "unexpected request", http.StatusInternalServerError)
+ }
+ })
+
+ saveJSONCache(ps.client.CacheDir(), "libraries.json", []cachedLibraryEntry{{ZoneName: "PrimarySync", Area: areaPrivate}, {ZoneName: "SharedSync-cached", Area: areaShared, OwnerRecordName: "owner", ZoneType: "REGULAR_CUSTOM_ZONE"}})
+
+ libs, err := ps.GetLibraries(ctx)
+ require.NoError(t, err)
+ assert.Contains(t, libs, "PrimarySync")
+ assert.NotContains(t, libs, "SharedSync-cached")
+
+ data, err := os.ReadFile(filepath.Join(ps.client.CacheDir(), "libraries.json"))
+ require.NoError(t, err)
+ assert.NotContains(t, string(data), "SharedSync-cached")
+}
+
+func TestGetAlbums_RetriesAfterTransientUserAlbumQueryFailure(t *testing.T) {
+ setTestCacheDir(t)
+ ctx := context.Background()
+ var topQueries atomic.Int32
+
+ ps := newHTTPTestPhotosService(t, "albums-retry-top", func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/database/1/com.apple.photos.cloud/production/private/records/query" {
+ http.Error(w, "unexpected request", http.StatusInternalServerError)
+ return
+ }
+ var body struct {
+ Query struct {
+ FilterBy []struct {
+ FieldName string `json:"fieldName"`
+ Comparator string `json:"comparator"`
+ FieldValue struct {
+ Value string `json:"value"`
+ } `json:"fieldValue"`
+ } `json:"filterBy"`
+ } `json:"query"`
+ }
+ if err := readJSONBody(r, &body); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if len(body.Query.FilterBy) != 0 {
+ http.Error(w, "unexpected child query", http.StatusInternalServerError)
+ return
+ }
+ if topQueries.Add(1) == 1 {
+ http.Error(w, "transient top-level album failure", http.StatusInternalServerError)
+ return
+ }
+ writeJSON(t, w, map[string]any{
+ "records": []map[string]any{testAlbumRecordJSON("User Album", "user-record", 0, "")},
+ })
+ })
+
+ lib := &Library{service: ps, zoneID: "PrimarySync", area: areaPrivate, albums: map[string]*Album{}}
+ _, err := lib.GetAlbums(ctx)
+ assert.Error(t, err, "private library failure should surface as error")
+ albums, err := lib.GetAlbums(ctx)
+ require.NoError(t, err)
+ assert.Contains(t, albums, "User Album")
+ assert.GreaterOrEqual(t, topQueries.Load(), int32(2), "second GetAlbums call must retry transient top-level failures")
+}
+
+func TestGetAlbums_SharedSyncQueryFailureReturnsSmartAlbumsOnly(t *testing.T) {
+ setTestCacheDir(t)
+ ctx := context.Background()
+
+ ps := newHTTPTestPhotosService(t, "albums-shared-fallback", func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusBadRequest)
+ _, _ = w.Write([]byte(`{
+ "serverErrorCode": "BAD_REQUEST",
+ "reason": "Index has invalid data",
+ "errorClass": "BAD_REQUEST",
+ "error": "BAD_REQUEST"
+ }`))
+ })
+
+ lib := &Library{service: ps, zoneID: "SharedSync-test", area: areaPrivate, albums: map[string]*Album{}}
+ albums, err := lib.GetAlbums(ctx)
+ require.NoError(t, err)
+ assert.Contains(t, albums, "All Photos")
+ assert.NotContains(t, albums, "User Album")
+ assert.Empty(t, lib.albums, "SharedSync fallback should not cache partial album results")
+}
+
+func TestGetAlbums_SharedSyncUnexpectedFailureReturnsError(t *testing.T) {
+ setTestCacheDir(t)
+ ctx := context.Background()
+
+ ps := newHTTPTestPhotosService(t, "albums-shared-error", func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "unexpected shared failure", http.StatusInternalServerError)
+ })
+
+ lib := &Library{service: ps, zoneID: "SharedSync-test", area: areaPrivate, albums: map[string]*Album{}}
+ _, err := lib.GetAlbums(ctx)
+ assert.Error(t, err)
+ assert.Empty(t, lib.albums, "SharedSync unexpected failures must not cache partial album results")
+}
+
+func TestGetAlbums_RetriesAfterTransientFolderChildQueryFailure(t *testing.T) {
+ setTestCacheDir(t)
+ ctx := context.Background()
+ var childQueries atomic.Int32
+
+ ps := newHTTPTestPhotosService(t, "albums-retry-folder", func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/database/1/com.apple.photos.cloud/production/private/records/query" {
+ http.Error(w, "unexpected request", http.StatusInternalServerError)
+ return
+ }
+ var body struct {
+ Query struct {
+ FilterBy []struct {
+ FieldName string `json:"fieldName"`
+ Comparator string `json:"comparator"`
+ FieldValue struct {
+ Value string `json:"value"`
+ } `json:"fieldValue"`
+ } `json:"filterBy"`
+ } `json:"query"`
+ }
+ if err := readJSONBody(r, &body); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ parentID := ""
+ for _, filter := range body.Query.FilterBy {
+ if filter.FieldName == "parentId" {
+ parentID = filter.FieldValue.Value
+ }
+ }
+ switch parentID {
+ case "":
+ writeJSON(t, w, map[string]any{
+ "records": []map[string]any{testAlbumRecordJSON("Folder", "folder-record", albumTypeFolder, "")},
+ })
+ case "folder-record":
+ if childQueries.Add(1) == 1 {
+ http.Error(w, "transient child query failure", http.StatusInternalServerError)
+ return
+ }
+ writeJSON(t, w, map[string]any{
+ "records": []map[string]any{testAlbumRecordJSON("Child", "child-record", 0, "folder-record")},
+ })
+ default:
+ http.Error(w, "unexpected parentId", http.StatusInternalServerError)
+ }
+ })
+
+ lib := &Library{service: ps, zoneID: "PrimarySync", area: areaPrivate, albums: map[string]*Album{}}
+ _, _ = lib.GetAlbums(ctx)
+ albums, err := lib.GetAlbums(ctx)
+ require.NoError(t, err)
+ folder, ok := albums["Folder"]
+ require.True(t, ok)
+ require.True(t, folder.IsFolder)
+ assert.Contains(t, folder.Children, "Child")
+ assert.GreaterOrEqual(t, childQueries.Load(), int32(2), "second GetAlbums call must retry transient child query failures")
+}
+
+func TestBuildPhotos(t *testing.T) {
+ filename := base64.StdEncoding.EncodeToString([]byte("IMG_0001.JPG"))
+
+ master := &photoRecord{
+ RecordName: "master123",
+ RecordType: "CPLMaster",
+ }
+ master.Fields.FilenameEnc = &ckStringField{Value: filename, Type: "ENCRYPTED_BYTES"}
+ master.Fields.ResOriginalRes = &ckResourceField{Value: struct {
+ Size int64 `json:"size"`
+ DownloadURL string `json:"downloadURL"`
+ }{Size: 1024, DownloadURL: "https://example.com/photo"}}
+ master.Fields.ResOriginalWidth = &ckIntField{Value: 4032}
+ master.Fields.ResOriginalHeight = &ckIntField{Value: 3024}
+
+ asset := &photoRecord{}
+ asset.Fields.AssetDate = &ckTimestampField{Value: 1700000000000}
+ asset.Fields.AddedDate = &ckTimestampField{Value: 1700000001000}
+
+ t.Run("basic photo", func(t *testing.T) {
+ photos := buildPhotos(master, asset)
+ require.Len(t, photos, 1)
+ p := photos[0]
+ assert.Equal(t, "master123", p.ID)
+ assert.Equal(t, "IMG_0001.JPG", p.Filename)
+ assert.Equal(t, int64(1024), p.Size)
+ assert.Equal(t, 4032, p.Width)
+ assert.Equal(t, 3024, p.Height)
+ assert.Equal(t, int64(1700000000000), p.AssetDate)
+ assert.Equal(t, int64(1700000001000), p.AddedDate)
+ assert.Equal(t, "resOriginalRes", p.ResourceKey)
+ })
+
+ t.Run("live photo with MOV companion", func(t *testing.T) {
+ liveMaster := *master // copy to avoid mutating shared fixture
+ liveMaster.Fields.ResOriginalVidComplRes = &ckResourceField{Value: struct {
+ Size int64 `json:"size"`
+ DownloadURL string `json:"downloadURL"`
+ }{Size: 2048, DownloadURL: "https://example.com/video"}}
+
+ photos := buildPhotos(&liveMaster, asset)
+ require.Len(t, photos, 2)
+
+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)
+ assert.Equal(t, "resOriginalRes", photos[0].ResourceKey)
+
+ assert.Equal(t, "IMG_0001.MOV", photos[1].Filename)
+ assert.Equal(t, "resOriginalVidComplRes", photos[1].ResourceKey)
+ assert.Equal(t, int64(2048), photos[1].Size)
+ assert.Equal(t, "master123", photos[1].ID)
+ })
+
+ t.Run("no download URL returns nil", func(t *testing.T) {
+ noURL := &photoRecord{RecordName: "no-url"}
+ noURL.Fields.FilenameEnc = master.Fields.FilenameEnc
+ photos := buildPhotos(noURL, nil)
+ assert.Nil(t, photos)
+ })
+
+ t.Run("no filename returns nil", func(t *testing.T) {
+ noName := &photoRecord{RecordName: "no-name"}
+ noName.Fields.ResOriginalRes = master.Fields.ResOriginalRes
+ photos := buildPhotos(noName, nil)
+ assert.Nil(t, photos)
+ })
+
+ t.Run("nil asset is handled", func(t *testing.T) {
+ photos := buildPhotos(master, nil)
+ require.Len(t, photos, 1)
+ assert.Equal(t, int64(0), photos[0].AssetDate)
+ })
+
+ t.Run("invalid base64 filename returns nil", func(t *testing.T) {
+ bad := &photoRecord{RecordName: "bad-b64"}
+ bad.Fields.FilenameEnc = &ckStringField{Value: "!!!not-base64!!!", Type: "ENCRYPTED_BYTES"}
+ bad.Fields.ResOriginalRes = master.Fields.ResOriginalRes
+ photos := buildPhotos(bad, nil)
+ assert.Nil(t, photos)
+ })
+
+ t.Run("STRING type filenameEnc used as-is", func(t *testing.T) {
+ rec := &photoRecord{RecordName: "str-name"}
+ rec.Fields.FilenameEnc = &ckStringField{Value: "plain_photo.heic", Type: "STRING"}
+ rec.Fields.ResOriginalRes = master.Fields.ResOriginalRes
+ photos := buildPhotos(rec, nil)
+ require.Len(t, photos, 1)
+ assert.Equal(t, "plain_photo.heic", photos[0].Filename)
+ })
+
+ t.Run("NFD filename normalized to NFC", func(t *testing.T) {
+ // "Café.heic" in NFD: e + combining acute accent (U+0301)
+ nfd := "Cafe\u0301.heic"
+ nfc := "Caf\u00e9.heic"
+
+ rec := &photoRecord{RecordName: "nfd-enc"}
+ rec.Fields.FilenameEnc = &ckStringField{Value: base64.StdEncoding.EncodeToString([]byte(nfd)), Type: "ENCRYPTED_BYTES"}
+ rec.Fields.ResOriginalRes = master.Fields.ResOriginalRes
+ photos := buildPhotos(rec, nil)
+ require.Len(t, photos, 1)
+ assert.Equal(t, nfc, photos[0].Filename)
+
+ rec2 := &photoRecord{RecordName: "nfd-str"}
+ rec2.Fields.FilenameEnc = &ckStringField{Value: nfd, Type: "STRING"}
+ rec2.Fields.ResOriginalRes = master.Fields.ResOriginalRes
+ photos2 := buildPhotos(rec2, nil)
+ require.Len(t, photos2, 1)
+ assert.Equal(t, nfc, photos2[0].Filename)
+ })
+
+ t.Run("itemType fallback when filenameEnc missing", func(t *testing.T) {
+ rec := &photoRecord{RecordName: "AaBbCcDd1234"}
+ rec.Fields.ItemType = &ckStringField{Value: "public.heic"}
+ rec.Fields.ResOriginalRes = master.Fields.ResOriginalRes
+ photos := buildPhotos(rec, nil)
+ require.Len(t, photos, 1)
+ assert.Equal(t, "AaBbCcDd1234.heic", photos[0].Filename)
+ })
+
+ t.Run("unknown itemType no fallback returns nil", func(t *testing.T) {
+ rec := &photoRecord{RecordName: "unknown-uti"}
+ rec.Fields.ItemType = &ckStringField{Value: "com.unknown.weird-format"}
+ rec.Fields.ResOriginalRes = master.Fields.ResOriginalRes
+ photos := buildPhotos(rec, nil)
+ assert.Nil(t, photos)
+ })
+
+ t.Run("edited photo version from adjustmentType", func(t *testing.T) {
+ editAsset := &photoRecord{RecordName: "asset-edited-123"}
+ editAsset.Fields.AssetDate = asset.Fields.AssetDate
+ editAsset.Fields.AdjustmentType = &ckStringField{Value: "com.apple.photos"}
+ editAsset.Fields.ResJPEGFullRes = &ckResourceField{Value: struct {
+ Size int64 `json:"size"`
+ DownloadURL string `json:"downloadURL"`
+ }{Size: 512, DownloadURL: "https://example.com/edited"}}
+ editAsset.Fields.ResJPEGFullFileType = &ckStringField{Value: "public.jpeg"}
+
+ photos := buildPhotos(master, editAsset)
+ require.Len(t, photos, 2)
+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)
+ assert.Equal(t, "resOriginalRes", photos[0].ResourceKey)
+ assert.Equal(t, "IMG_0001-edited.jpg", photos[1].Filename)
+ assert.Equal(t, "resJPEGFullRes", photos[1].ResourceKey)
+ assert.Equal(t, int64(512), photos[1].Size)
+ assert.Equal(t, "asset-edited-123", photos[1].ID)
+ })
+
+ t.Run("slo-mo skips edited version", func(t *testing.T) {
+ slomoAsset := &photoRecord{RecordName: "asset-slomo"}
+ slomoAsset.Fields.AssetDate = asset.Fields.AssetDate
+ slomoAsset.Fields.AdjustmentType = &ckStringField{Value: "com.apple.video.slomo"}
+ photos := buildPhotos(master, slomoAsset)
+ require.Len(t, photos, 1)
+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)
+ })
+
+ t.Run("RAW alternative from resOriginalAltRes", func(t *testing.T) {
+ rawMaster := &photoRecord{RecordName: "master-raw"}
+ rawMaster.Fields.FilenameEnc = master.Fields.FilenameEnc
+ rawMaster.Fields.ResOriginalRes = master.Fields.ResOriginalRes
+ rawMaster.Fields.ResOriginalWidth = master.Fields.ResOriginalWidth
+ rawMaster.Fields.ResOriginalHeight = master.Fields.ResOriginalHeight
+ rawMaster.Fields.ResOriginalAltRes = &ckResourceField{Value: struct {
+ Size int64 `json:"size"`
+ DownloadURL string `json:"downloadURL"`
+ }{Size: 8192, DownloadURL: "https://example.com/raw"}}
+ rawMaster.Fields.ResOriginalAltFileType = &ckStringField{Value: "com.canon.cr2-raw-image"}
+
+ photos := buildPhotos(rawMaster, nil)
+ require.Len(t, photos, 2)
+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)
+ assert.Equal(t, "resOriginalRes", photos[0].ResourceKey)
+ assert.Equal(t, "IMG_0001.cr2", photos[1].Filename)
+ assert.Equal(t, "resOriginalAltRes", photos[1].ResourceKey)
+ assert.Equal(t, int64(8192), photos[1].Size)
+ assert.Equal(t, 4032, photos[1].Width, "RAW alt must inherit width from original")
+ assert.Equal(t, 3024, photos[1].Height, "RAW alt must inherit height from original")
+ })
+
+ t.Run("RAW alt same extension gets -alt suffix", func(t *testing.T) {
+ dupMaster := &photoRecord{RecordName: "master-dup-ext"}
+ dupMaster.Fields.FilenameEnc = master.Fields.FilenameEnc
+ dupMaster.Fields.ResOriginalRes = master.Fields.ResOriginalRes
+ dupMaster.Fields.ResOriginalAltRes = &ckResourceField{Value: struct {
+ Size int64 `json:"size"`
+ DownloadURL string `json:"downloadURL"`
+ }{Size: 4096, DownloadURL: "https://example.com/alt"}}
+ dupMaster.Fields.ResOriginalAltFileType = &ckStringField{Value: "public.jpeg"}
+
+ photos := buildPhotos(dupMaster, nil)
+ require.Len(t, photos, 2)
+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)
+ // Same extension would collide, so -alt suffix added
+ assert.Equal(t, "IMG_0001-alt.jpg", photos[1].Filename)
+ })
+
+ t.Run("edited video via resVidFullRes", func(t *testing.T) {
+ vidAsset := &photoRecord{RecordName: "asset-vid-edit"}
+ vidAsset.Fields.AssetDate = asset.Fields.AssetDate
+ vidAsset.Fields.AdjustmentType = &ckStringField{Value: "com.apple.photos"}
+ vidAsset.Fields.ResVidFullRes = &ckResourceField{Value: struct {
+ Size int64 `json:"size"`
+ DownloadURL string `json:"downloadURL"`
+ }{Size: 2048, DownloadURL: "https://example.com/vid-edit"}}
+ vidAsset.Fields.ResVidFullFileType = &ckStringField{Value: "public.mpeg-4"}
+
+ photos := buildPhotos(master, vidAsset)
+ require.Len(t, photos, 2)
+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)
+ assert.Equal(t, "IMG_0001-edited.mp4", photos[1].Filename)
+ assert.Equal(t, "resVidFullRes", photos[1].ResourceKey)
+ })
+
+ t.Run("edited + RAW on same photo", func(t *testing.T) {
+ rawMaster := &photoRecord{RecordName: "master-combo"}
+ rawMaster.Fields.FilenameEnc = master.Fields.FilenameEnc
+ rawMaster.Fields.ResOriginalRes = master.Fields.ResOriginalRes
+ rawMaster.Fields.ResOriginalAltRes = &ckResourceField{Value: struct {
+ Size int64 `json:"size"`
+ DownloadURL string `json:"downloadURL"`
+ }{Size: 8192, DownloadURL: "https://example.com/raw"}}
+ rawMaster.Fields.ResOriginalAltFileType = &ckStringField{Value: "com.nikon.raw-image"}
+
+ comboAsset := &photoRecord{RecordName: "asset-combo"}
+ comboAsset.Fields.AssetDate = asset.Fields.AssetDate
+ comboAsset.Fields.AdjustmentType = &ckStringField{Value: "com.apple.photos"}
+ comboAsset.Fields.ResJPEGFullRes = &ckResourceField{Value: struct {
+ Size int64 `json:"size"`
+ DownloadURL string `json:"downloadURL"`
+ }{Size: 512, DownloadURL: "https://example.com/edited"}}
+ comboAsset.Fields.ResJPEGFullFileType = &ckStringField{Value: "public.jpeg"}
+
+ photos := buildPhotos(rawMaster, comboAsset)
+ require.Len(t, photos, 3)
+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)
+ assert.Equal(t, "IMG_0001-edited.jpg", photos[1].Filename)
+ assert.Equal(t, "IMG_0001.nef", photos[2].Filename)
+ })
+
+ t.Run("RAW alt with nil file type uses original extension", func(t *testing.T) {
+ noTypeMaster := &photoRecord{RecordName: "master-notype"}
+ noTypeMaster.Fields.FilenameEnc = master.Fields.FilenameEnc
+ noTypeMaster.Fields.ResOriginalRes = master.Fields.ResOriginalRes
+ noTypeMaster.Fields.ResOriginalAltRes = &ckResourceField{Value: struct {
+ Size int64 `json:"size"`
+ DownloadURL string `json:"downloadURL"`
+ }{Size: 4096, DownloadURL: "https://example.com/alt-notype"}}
+ // No ResOriginalAltFileType set
+
+ photos := buildPhotos(noTypeMaster, nil)
+ require.Len(t, photos, 2)
+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)
+ // Same extension collision triggers -alt suffix
+ assert.Equal(t, "IMG_0001-alt.JPG", photos[1].Filename)
+ })
+
+ t.Run("extensionless filename gets .MOV companion", func(t *testing.T) {
+ noExt := &photoRecord{RecordName: "no-ext"}
+ noExt.Fields.FilenameEnc = &ckStringField{Value: base64.StdEncoding.EncodeToString([]byte("IMG_NOEXT")), Type: "ENCRYPTED_BYTES"}
+ noExt.Fields.ResOriginalRes = master.Fields.ResOriginalRes
+ noExt.Fields.ResOriginalVidComplRes = &ckResourceField{Value: struct {
+ Size int64 `json:"size"`
+ DownloadURL string `json:"downloadURL"`
+ }{Size: 512, DownloadURL: "https://example.com/v"}}
+ photos := buildPhotos(noExt, nil)
+ require.Len(t, photos, 2)
+ assert.Equal(t, "IMG_NOEXT", photos[0].Filename)
+ assert.Equal(t, "IMG_NOEXT.MOV", photos[1].Filename)
+ })
+}
+
+func TestParsePhotoRecords(t *testing.T) {
+ filename := base64.StdEncoding.EncodeToString([]byte("IMG_0001.JPG"))
+
+ records := []photoRecord{
+ {RecordName: "master1", RecordType: "CPLMaster"},
+ {RecordName: "asset1", RecordType: "CPLAsset"},
+ {RecordName: "master2", RecordType: "CPLMaster"},
+ {RecordName: "asset2", RecordType: "CPLAsset"},
+ }
+ // Set up master fields
+ for i := range records {
+ if records[i].RecordType == "CPLMaster" {
+ records[i].Fields.FilenameEnc = &ckStringField{Value: filename, Type: "ENCRYPTED_BYTES"}
+ records[i].Fields.ResOriginalRes = &ckResourceField{Value: struct {
+ Size int64 `json:"size"`
+ DownloadURL string `json:"downloadURL"`
+ }{Size: 1024, DownloadURL: "https://example.com/dl"}}
+ }
+ }
+ // Link assets to masters
+ records[1].Fields.MasterRef = &ckReferenceField{Value: struct {
+ RecordName string `json:"recordName"`
+ }{RecordName: "master1"}}
+ records[3].Fields.MasterRef = &ckReferenceField{Value: struct {
+ RecordName string `json:"recordName"`
+ }{RecordName: "master2"}}
+
+ photos := parsePhotoRecords(records)
+ require.Len(t, photos, 2)
+ assert.Equal(t, "master1", photos[0].ID)
+ assert.Equal(t, "master2", photos[1].ID)
+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)
+
+ t.Run("orphan master without asset", func(t *testing.T) {
+ orphan := []photoRecord{
+ {RecordName: "orphan-master", RecordType: "CPLMaster"},
+ }
+ orphan[0].Fields.FilenameEnc = &ckStringField{Value: filename, Type: "ENCRYPTED_BYTES"}
+ orphan[0].Fields.ResOriginalRes = records[0].Fields.ResOriginalRes
+ photos := parsePhotoRecords(orphan)
+ require.Len(t, photos, 1)
+ assert.Equal(t, "orphan-master", photos[0].ID)
+ })
+
+ t.Run("empty records", func(t *testing.T) {
+ photos := parsePhotoRecords(nil)
+ assert.Nil(t, photos)
+ })
+}
+
+func TestClassifySmartAlbums(t *testing.T) {
+ makeMaster := func(fileType string) *photoRecord {
+ m := &photoRecord{}
+ if fileType != "" {
+ m.Fields.ResOriginalFileType = &ckStringField{Value: fileType}
+ }
+ return m
+ }
+ makeAsset := func(subtype, subtypeV2, favorite, hidden int) *photoRecord {
+ a := &photoRecord{}
+ a.Fields.AssetSubtype = &ckIntField{Value: subtype}
+ a.Fields.AssetSubtypeV2 = &ckIntField{Value: subtypeV2}
+ a.Fields.IsFavorite = &ckIntField{Value: favorite}
+ a.Fields.IsHidden = &ckIntField{Value: hidden}
+ return a
+ }
+
+ tests := []struct {
+ name string
+ master *photoRecord
+ asset *photoRecord
+ expected []string
+ }{
+ {"regular photo", makeMaster("public.jpeg"), makeAsset(0, 0, 0, 0), []string{"All Photos"}},
+ {"video mpeg4", makeMaster("public.mpeg-4"), makeAsset(0, 0, 0, 0), []string{"All Photos", "Videos"}},
+ {"video quicktime", makeMaster("com.apple.quicktime-movie"), makeAsset(0, 0, 0, 0), []string{"All Photos", "Videos"}},
+ {"favorite photo", makeMaster("public.jpeg"), makeAsset(0, 0, 1, 0), []string{"All Photos", "Favorites"}},
+ {"hidden photo", makeMaster("public.jpeg"), makeAsset(0, 0, 0, 1), []string{"Hidden"}},
+ {"screenshot", makeMaster("public.jpeg"), makeAsset(0, 3, 0, 0), []string{"All Photos", "Screenshots"}},
+ {"favorited screenshot", makeMaster("public.jpeg"), makeAsset(0, 3, 1, 0), []string{"All Photos", "Favorites", "Screenshots"}},
+ {"slo-mo", makeMaster("public.mpeg-4"), makeAsset(100, 0, 0, 0), []string{"All Photos", "Slo-mo"}},
+ {"time-lapse", makeMaster("public.mpeg-4"), makeAsset(101, 0, 0, 0), []string{"All Photos", "Time-lapse"}},
+ {"panorama", makeMaster("public.jpeg"), makeAsset(1, 0, 0, 0), []string{"All Photos", "Panoramas"}},
+ {"live photo", makeMaster("public.jpeg"), makeAsset(0, 2, 0, 0), []string{"All Photos", "Live"}},
+ {"nil asset", makeMaster("public.jpeg"), nil, []string{"All Photos"}},
+ {"no file type", makeMaster(""), makeAsset(0, 0, 0, 0), []string{"All Photos"}},
+ {"hidden favorite", makeMaster("public.jpeg"), makeAsset(0, 0, 1, 1), []string{"Hidden", "Favorites"}},
+ {"burst photo", makeMaster("public.jpeg"), func() *photoRecord {
+ a := makeAsset(0, 0, 0, 0)
+ a.Fields.BurstID = &ckStringField{Value: "B7A3F2E1-4D5C-6789-ABCD-EF0123456789"}
+ return a
+ }(), []string{"All Photos", "Bursts"}},
+ {"portrait photo", makeMaster("public.jpeg"), func() *photoRecord {
+ a := makeAsset(0, 0, 0, 0)
+ a.Fields.AdjustmentRenderType = &ckIntField{Value: 2} // PORTRAIT bit
+ return a
+ }(), []string{"All Photos", "Portrait"}},
+ {"long exposure", makeMaster("public.jpeg"), func() *photoRecord {
+ a := makeAsset(0, 0, 0, 0)
+ a.Fields.AdjustmentRenderType = &ckIntField{Value: 4} // LONG_EXPOSURE bit
+ return a
+ }(), []string{"All Photos", "Long Exposure"}},
+ {"portrait + long exposure", makeMaster("public.jpeg"), func() *photoRecord {
+ a := makeAsset(0, 0, 0, 0)
+ a.Fields.AdjustmentRenderType = &ckIntField{Value: 6} // both bits
+ return a
+ }(), []string{"All Photos", "Portrait", "Long Exposure"}},
+ {"animated gif", makeMaster("com.compuserve.gif"), makeAsset(0, 0, 0, 0), []string{"All Photos", "Animated"}},
+ {"soft-deleted photo", makeMaster("public.jpeg"), func() *photoRecord {
+ a := makeAsset(0, 0, 1, 0) // favorite=1 should be ignored when deleted
+ a.Fields.IsDeleted = &ckIntField{Value: 1}
+ return a
+ }(), []string{"Recently Deleted"}},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := classifySmartAlbums(tt.master, tt.asset)
+ assert.Equal(t, tt.expected, got)
+ })
+ }
+}
+
+func TestGetPhotoByName_CacheHit(t *testing.T) {
+ album := &Album{
+ Name: "Videos",
+ ObjectType: "CPLAssetInSmartAlbumByAssetDate:Video",
+ }
+ album.photoCache = map[string]*Photo{
+ "IMG_0001.JPG": {ID: "m1", Filename: "IMG_0001.JPG", Size: 1024},
+ "IMG_0002.MOV": {ID: "m2", Filename: "IMG_0002.MOV", Size: 2048},
+ }
+
+ t.Run("found in cache", func(t *testing.T) {
+ photo, err := album.GetPhotoByName(context.Background(), "IMG_0001.JPG")
+ require.NoError(t, err)
+ assert.Equal(t, "m1", photo.ID)
+ assert.Equal(t, int64(1024), photo.Size)
+ })
+
+ t.Run("not in cache", func(t *testing.T) {
+ // GetPhotoByName with a populated cache but missing filename
+ // should return error (cache exists but name not found)
+ _, err := album.GetPhotoByName(context.Background(), "MISSING.JPG")
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "MISSING.JPG")
+ })
+}
+
+func TestDeduplicateFilenames(t *testing.T) {
+ t.Run("no duplicates unchanged", func(t *testing.T) {
+ photos := []*Photo{
+ {ID: "master1", Filename: "IMG_0001.JPG"},
+ {ID: "master2", Filename: "IMG_0002.JPG"},
+ }
+ deduplicateFilenames(photos)
+ assert.Equal(t, "IMG_0001.JPG", photos[0].Filename)
+ assert.Equal(t, "IMG_0002.JPG", photos[1].Filename)
+ })
+
+ t.Run("all duplicates get full ID suffix", func(t *testing.T) {
+ photos := []*Photo{
+ {ID: "AQJ2Fq0Px7pGM", Filename: "camphoto_001.jpg"},
+ {ID: "BRK3Gr1Qy8qHN", Filename: "camphoto_001.jpg"},
+ {ID: "CSL4Hs2Rz9rIO", Filename: "camphoto_001.jpg"},
+ }
+ deduplicateFilenames(photos)
+ assert.Equal(t, "camphoto_001_AQJ2Fq0Px7pGM.jpg", photos[0].Filename)
+ assert.Equal(t, "camphoto_001_BRK3Gr1Qy8qHN.jpg", photos[1].Filename)
+ assert.Equal(t, "camphoto_001_CSL4Hs2Rz9rIO.jpg", photos[2].Filename)
+ })
+
+ t.Run("no extension handled", func(t *testing.T) {
+ photos := []*Photo{
+ {ID: "masterA", Filename: "noext"},
+ {ID: "masterB", Filename: "noext"},
+ }
+ deduplicateFilenames(photos)
+ assert.Equal(t, "noext_masterA", photos[0].Filename)
+ assert.Equal(t, "noext_masterB", photos[1].Filename)
+ })
+
+ t.Run("deterministic across runs", func(t *testing.T) {
+ mk := func() []*Photo {
+ return []*Photo{
+ {ID: "AQJ2Fq0Px7pGM", Filename: "dup.jpg"},
+ {ID: "BRK3Gr1Qy8qHN", Filename: "dup.jpg"},
+ }
+ }
+ p1 := mk()
+ p2 := mk()
+ deduplicateFilenames(p1)
+ deduplicateFilenames(p2)
+ assert.Equal(t, p1[0].Filename, p2[0].Filename)
+ assert.Equal(t, p1[1].Filename, p2[1].Filename)
+ })
+
+ t.Run("stable when new duplicate added", func(t *testing.T) {
+ // Before: two duplicates
+ before := []*Photo{
+ {ID: "AQJ2Fq0Px7pGM", Filename: "dup.jpg"},
+ {ID: "BRK3Gr1Qy8qHN", Filename: "dup.jpg"},
+ }
+ deduplicateFilenames(before)
+ nameA := before[0].Filename
+ nameB := before[1].Filename
+
+ // After: third duplicate added - existing names must not change
+ after := []*Photo{
+ {ID: "AQJ2Fq0Px7pGM", Filename: "dup.jpg"},
+ {ID: "BRK3Gr1Qy8qHN", Filename: "dup.jpg"},
+ {ID: "CSL4Hs2Rz9rIO", Filename: "dup.jpg"},
+ }
+ deduplicateFilenames(after)
+ assert.Equal(t, nameA, after[0].Filename, "existing file A must keep same name")
+ assert.Equal(t, nameB, after[1].Filename, "existing file B must keep same name")
+ })
+}
+
+func TestDeduplicateFilenamesSharedPointers(t *testing.T) {
+ // Photos classified into multiple smart albums share *Photo pointers
+ // deduplicateFilenames mutates Filename in place, so shared pointers
+ // across albums get cross-contaminated unless deep-copied first
+ shared := &Photo{ID: "master1", Filename: "dup.jpg"}
+ other := &Photo{ID: "master2", Filename: "dup.jpg"}
+
+ albumA := []*Photo{shared, other}
+ albumB := []*Photo{shared} // same pointer, unique in this album
+
+ // Simulate the fix: deep-copy before dedup (same as applyPendingDelta)
+ dedupedA := make([]*Photo, len(albumA))
+ for i, p := range albumA {
+ cp := *p
+ dedupedA[i] = &cp
+ }
+ deduplicateFilenames(dedupedA)
+
+ // albumA's dedup copies got suffixes
+ assert.Contains(t, dedupedA[0].Filename, "_master1")
+ assert.Contains(t, dedupedA[1].Filename, "_master2")
+
+ // Original shared pointer is untouched - albumB sees clean filename
+ assert.Equal(t, "dup.jpg", albumB[0].Filename, "deep copy must prevent cross-album contamination")
+ assert.Equal(t, "dup.jpg", shared.Filename, "original pointer must be unmodified")
+}
+
+func TestDeltaContainsAlbumChanges(t *testing.T) {
+ t.Run("CPLAlbum record triggers invalidation", func(t *testing.T) {
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"master1","recordType":"CPLMaster"}`),
+ json.RawMessage(`{"recordName":"asset1","recordType":"CPLAsset"}`),
+ json.RawMessage(`{"recordName":"album1","recordType":"CPLAlbum"}`),
+ }
+ assert.True(t, deltaContainsAlbumChanges(records))
+ })
+
+ t.Run("no CPLAlbum does not trigger", func(t *testing.T) {
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"master1","recordType":"CPLMaster"}`),
+ json.RawMessage(`{"recordName":"asset1","recordType":"CPLAsset"}`),
+ json.RawMessage(`{"recordName":"rel1","recordType":"CPLContainerRelation"}`),
+ }
+ assert.False(t, deltaContainsAlbumChanges(records))
+ })
+
+ t.Run("empty records does not trigger", func(t *testing.T) {
+ assert.False(t, deltaContainsAlbumChanges(nil))
+ })
+
+ t.Run("malformed JSON skipped", func(t *testing.T) {
+ records := []json.RawMessage{
+ json.RawMessage(`{invalid`),
+ json.RawMessage(`{"recordName":"album1","recordType":"CPLAlbum"}`),
+ }
+ assert.True(t, deltaContainsAlbumChanges(records))
+ })
+}
+
+func TestParseDeltaRecords(t *testing.T) {
+ t.Run("deleted CPLAsset marks both asset and master IDs", func(t *testing.T) {
+ // Edited entries use asset.RecordName as ID; ghost entries survive
+ // if only masterID is in deletedIDs
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"master-AAA","recordType":"CPLMaster","deleted":true}`),
+ json.RawMessage(`{"recordName":"asset-BBB","recordType":"CPLAsset","deleted":true,"fields":{"masterRef":{"value":{"recordName":"master-AAA"}}}}`),
+ }
+ r := parseDeltaRecords(records)
+ assert.True(t, r.deletedIDs["master-AAA"], "master ID from CPLMaster deletion")
+ assert.True(t, r.deletedIDs["asset-BBB"], "asset ID from CPLAsset deletion (for -edited entries)")
+ })
+
+ t.Run("deleted CPLAsset without masterRef still tracks asset ID", func(t *testing.T) {
+ // CloudKit may strip fields from deleted records
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"asset-CCC","recordType":"CPLAsset","deleted":true}`),
+ }
+ r := parseDeltaRecords(records)
+ assert.True(t, r.deletedIDs["asset-CCC"], "asset ID tracked even without masterRef")
+ assert.Equal(t, 1, len(r.deletedIDs), "only asset ID, no master")
+ })
+
+ t.Run("deleted CPLContainerRelation extracts album from recordName", func(t *testing.T) {
+ // Deleted relations lack fields; album parsed from "assetID-IN-albumRecordName"
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"assetXYZ-IN-album-uuid-456","recordType":"CPLContainerRelation","deleted":true}`),
+ }
+ r := parseDeltaRecords(records)
+ assert.True(t, r.changedAlbumRecords["album-uuid-456"], "album extracted from deleted relation recordName")
+ })
+
+ t.Run("deleted relation without recordType still extracts album from recordName", func(t *testing.T) {
+ // changes/zone omits recordType for deleted CPLContainerRelation records
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"assetXYZ-IN-album-uuid-654","deleted":true}`),
+ }
+ r := parseDeltaRecords(records)
+ assert.True(t, r.changedAlbumRecords["album-uuid-654"], "album extracted from deleted relation with missing recordType")
+ })
+
+ t.Run("non-deleted CPLContainerRelation uses containerId STRING field", func(t *testing.T) {
+ // CloudKit relation records expose containerId as the canonical album field
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"rel1","recordType":"CPLContainerRelation","fields":{"containerId":{"value":"album-uuid-789"}}}`),
+ }
+ r := parseDeltaRecords(records)
+ assert.True(t, r.changedAlbumRecords["album-uuid-789"], "album extracted from containerId STRING field")
+ })
+
+ t.Run("non-deleted CPLContainerRelation falls back to recordName when fields missing", func(t *testing.T) {
+ // changes/zone can return live relation records with empty fields
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"assetXYZ-IN-album-uuid-987","recordType":"CPLContainerRelation","fields":{},"deleted":false}`),
+ }
+ r := parseDeltaRecords(records)
+ assert.True(t, r.changedAlbumRecords["album-uuid-987"], "album extracted from live relation recordName fallback")
+ })
+
+ t.Run("deleted relation with unexpected recordName format is safe", func(t *testing.T) {
+ // No "-IN-" in recordName - must not panic or add garbage
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"weirdformat","recordType":"CPLContainerRelation","deleted":true}`),
+ }
+ r := parseDeltaRecords(records)
+ assert.Empty(t, r.changedAlbumRecords, "no album invalidated for malformed recordName")
+ })
+
+ t.Run("CPLAlbum sets albumMetadataChanged", func(t *testing.T) {
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"album1","recordType":"CPLAlbum"}`),
+ }
+ r := parseDeltaRecords(records)
+ assert.True(t, r.albumMetadataChanged)
+ })
+
+ t.Run("asset-only update detected", func(t *testing.T) {
+ // CPLAsset changed but no CPLMaster in delta - triggers smart album invalidation
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"asset-DDD","recordType":"CPLAsset","fields":{"masterRef":{"value":{"recordName":"master-EEE"}}}}`),
+ }
+ r := parseDeltaRecords(records)
+ assert.True(t, r.hasAssetOnlyUpdates, "asset without matching master = asset-only update")
+ })
+
+ t.Run("asset with matching master is not asset-only", func(t *testing.T) {
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"master-FFF","recordType":"CPLMaster","fields":{"resOriginalRes":{"value":{"downloadURL":"https://example.com"}}}}`),
+ json.RawMessage(`{"recordName":"asset-GGG","recordType":"CPLAsset","fields":{"masterRef":{"value":{"recordName":"master-FFF"}}}}`),
+ }
+ r := parseDeltaRecords(records)
+ assert.False(t, r.hasAssetOnlyUpdates)
+ assert.Equal(t, 1, len(r.newMasters))
+ assert.Equal(t, 1, len(r.newAssets))
+ })
+
+ t.Run("unknown record types are silently skipped", func(t *testing.T) {
+ records := []json.RawMessage{
+ json.RawMessage(`{"recordName":"x","recordType":"CPLSomethingNew"}`),
+ }
+ r := parseDeltaRecords(records)
+ assert.Empty(t, r.deletedIDs)
+ assert.Empty(t, r.newMasters)
+ assert.Empty(t, r.newAssets)
+ assert.Empty(t, r.changedAlbumRecords)
+ assert.False(t, r.albumMetadataChanged)
+ })
+
+ t.Run("malformed JSON records skipped without panic", func(t *testing.T) {
+ records := []json.RawMessage{
+ json.RawMessage(`{invalid`),
+ json.RawMessage(`{"recordName":"master1","recordType":"CPLMaster","deleted":true}`),
+ }
+ r := parseDeltaRecords(records)
+ assert.True(t, r.deletedIDs["master1"], "valid record after malformed one still parsed")
+ })
+
+ t.Run("empty records produces empty result", func(t *testing.T) {
+ r := parseDeltaRecords(nil)
+ assert.Empty(t, r.deletedIDs)
+ assert.False(t, r.albumMetadataChanged)
+ assert.False(t, r.hasAssetOnlyUpdates)
+ })
+}
+
+func TestApplyPendingDelta_EmptyAlbumsClearsPending(t *testing.T) {
+ // When checkForChanges eagerly clears lib.albums (CPLAlbum delta),
+ // applyPendingDelta must clear pendingDelta instead of leaving it stuck
+ // Without this fix, pendingDelta stays non-nil forever and all future
+ // checkForChanges calls skip (line 912: pendingDelta != nil)
+ lib := &Library{
+ albums: make(map[string]*Album), // empty - simulates eager clear
+ pendingDelta: &deltaPayload{
+ records: []json.RawMessage{json.RawMessage(`{"recordName":"m1","recordType":"CPLMaster"}`)},
+ syncToken: "old-token",
+ },
+ }
+ result := lib.applyPendingDelta(context.Background())
+ assert.False(t, result, "must return false when albums empty")
+ assert.Nil(t, lib.pendingDelta, "must clear pendingDelta to avoid permanent stuck state")
+}
+
+func TestApplyPendingDelta_InvalidatesNestedAlbum(t *testing.T) {
+ oldCacheDir := config.GetCacheDir()
+ t.Cleanup(func() {
+ _ = config.SetCacheDir(oldCacheDir)
+ })
+ require.NoError(t, config.SetCacheDir(t.TempDir()))
+
+ ps := &PhotosService{client: &Client{remoteName: "delta-nested-test"}}
+ lib := &Library{
+ service: ps,
+ zoneID: "PrimarySync",
+ area: areaPrivate,
+ albums: make(map[string]*Album),
+ }
+ child := lib.newUserAlbum("Child", "child-record")
+ child.SetTestPhotoCache(map[string]*Photo{
+ "one.jpg": {ID: "master1", Filename: "one.jpg"},
+ })
+ child.saveDiskCache([]*Photo{{ID: "master1", Filename: "one.jpg"}})
+
+ folder := &Album{
+ Name: "Folder",
+ RecordName: "folder-record",
+ lib: lib,
+ IsFolder: true,
+ Children: map[string]*Album{
+ "Child": child,
+ },
+ }
+ lib.albums[folder.Name] = folder
+ lib.pendingDelta = &deltaPayload{
+ records: []json.RawMessage{json.RawMessage(`{"recordName":"asset123-IN-child-record","deleted":true}`)},
+ syncToken: "next-token",
+ }
+
+ cacheFile := filepath.Join(lib.zoneCacheDir(), albumCacheKey(child.ObjectType)+".json")
+ _, err := os.Stat(cacheFile)
+ require.NoError(t, err, "nested child album cache file should exist before invalidation")
+
+ result := lib.applyPendingDelta(context.Background())
+ assert.True(t, result)
+ assert.Nil(t, lib.pendingDelta)
+
+ child.mu.Lock()
+ assert.Nil(t, child.photoCache, "nested child album memory cache should be invalidated")
+ child.mu.Unlock()
+ _, err = os.Stat(cacheFile)
+ assert.ErrorIs(t, err, os.ErrNotExist, "nested child album disk cache should be removed")
+ data, err := os.ReadFile(filepath.Join(lib.zoneCacheDir(), "syncToken"))
+ require.NoError(t, err)
+ assert.Equal(t, "next-token", string(data))
+}
+
+func TestFlushCaches_NoPendingDeltaRace(t *testing.T) {
+ // Verify FlushCaches and applyPendingDelta don't race on pendingDelta.
+ // FlushCaches must acquire deltaMu before writing pendingDelta.
+ ps := &PhotosService{
+ client: &Client{remoteName: "race-test"},
+ libraries: make(map[string]*Library),
+ }
+ lib := &Library{
+ service: ps,
+ zoneID: "PrimarySync",
+ area: areaPrivate,
+ albums: make(map[string]*Album),
+ }
+ ps.libraries["PrimarySync"] = lib
+
+ // Run concurrent FlushCaches + applyPendingDelta
+ // The race detector will flag unsynchronized access to pendingDelta
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ for i := 0; i < 100; i++ {
+ lib.deltaMu.Lock()
+ lib.pendingDelta = &deltaPayload{
+ records: []json.RawMessage{json.RawMessage(`{"recordName":"m1","recordType":"CPLMaster"}`)},
+ syncToken: "tok",
+ }
+ lib.deltaMu.Unlock()
+ lib.applyPendingDelta(context.Background())
+ }
+ }()
+ for i := 0; i < 100; i++ {
+ ps.FlushCaches()
+ // Re-add the library since FlushCaches clears it
+ ps.mu.Lock()
+ ps.libraries["PrimarySync"] = lib
+ ps.mu.Unlock()
+ }
+ <-done
+}
+
+func TestBuildSmartAlbums(t *testing.T) {
+ albums := buildSmartAlbums()
+
+ // Must produce exactly 14 smart albums
+ assert.Equal(t, 14, len(albums), "expected 14 smart albums")
+
+ // All expected names present
+ expected := []string{
+ "All Photos", "Favorites", "Videos", "Screenshots", "Live",
+ "Slo-mo", "Time-lapse", "Panoramas", "Portrait", "Long Exposure",
+ "Animated", "Bursts", "Hidden", "Recently Deleted",
+ }
+ for _, name := range expected {
+ a, ok := albums[name]
+ require.True(t, ok, "missing smart album %q", name)
+ assert.NotEmpty(t, a.ObjectType, "album %q has no ObjectType", name)
+ assert.NotEmpty(t, a.ListType, "album %q has no ListType", name)
+ assert.NotEmpty(t, a.Direction, "album %q has no Direction", name)
+ }
+
+ // Verify special albums have correct configuration
+ assert.Equal(t, "DESCENDING", albums["Recently Deleted"].Direction)
+ assert.Empty(t, albums["All Photos"].Filters, "All Photos should have no filters")
+ assert.Empty(t, albums["Bursts"].Filters, "Bursts should have no filters")
+ assert.Empty(t, albums["Hidden"].Filters, "Hidden should have no filters")
+
+ // Verify filter-based albums have exactly one smartAlbum filter
+ for _, name := range []string{"Favorites", "Videos", "Screenshots", "Live", "Slo-mo", "Time-lapse", "Panoramas", "Portrait", "Long Exposure", "Animated"} {
+ a := albums[name]
+ require.Len(t, a.Filters, 1, "album %q should have exactly one filter", name)
+ assert.Equal(t, "smartAlbum", a.Filters[0].FieldName)
+ assert.Equal(t, "EQUALS", a.Filters[0].Comparator)
+ }
+}
+
+func TestAtomicWriteFile(t *testing.T) {
+ dir := t.TempDir()
+ target := filepath.Join(dir, "test.json")
+
+ // Successful write
+ err := atomicWriteFile(target, []byte(`{"key":"value"}`))
+ require.NoError(t, err)
+ data, err := os.ReadFile(target)
+ require.NoError(t, err)
+ assert.Equal(t, `{"key":"value"}`, string(data))
+
+ // Overwrite existing
+ err = atomicWriteFile(target, []byte(`{"key":"updated"}`))
+ require.NoError(t, err)
+ data, err = os.ReadFile(target)
+ require.NoError(t, err)
+ assert.Equal(t, `{"key":"updated"}`, string(data))
+
+ // No .tmp file left behind
+ _, err = os.Stat(target + ".tmp")
+ assert.True(t, os.IsNotExist(err), "temp file should not persist")
+}
+
+func TestAlbumCacheKey(t *testing.T) {
+ k1 := albumCacheKey("CPLAssetByAssetDateWithoutHiddenOrDeleted")
+ k2 := albumCacheKey("CPLAssetInSmartAlbumByAssetDate:Video")
+ k3 := albumCacheKey("CPLAssetByAssetDateWithoutHiddenOrDeleted")
+
+ assert.Len(t, k1, 16, "cache key should be 16 hex chars")
+ assert.Equal(t, k1, k3, "same input should produce same key")
+ assert.NotEqual(t, k1, k2, "different input should produce different key")
+
+ // Verify filename-safe
+ for _, c := range k1 {
+ assert.True(t, (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'), "key should be hex: %c", c)
+ }
+}
diff --git a/backend/iclouddrive/api/session.go b/backend/iclouddrive/api/session.go
index 004faaa37..89e47a937 100644
--- a/backend/iclouddrive/api/session.go
+++ b/backend/iclouddrive/api/session.go
@@ -2,16 +2,21 @@ package api
import (
"context"
+ "crypto/sha256"
"encoding/base64"
+ "encoding/hex"
+ "encoding/json"
"fmt"
+ "io"
"maps"
"net/http"
"net/url"
"slices"
"strings"
+ "sync"
+ "time"
"github.com/google/uuid"
- "github.com/oracle/oci-go-sdk/v65/common"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/lib/rest"
@@ -32,6 +37,7 @@ type Session struct {
Cookies []*http.Cookie `json:"cookies"`
AccountInfo AccountInfo `json:"account_info"`
+ mu sync.Mutex `json:"-"` // protects session fields during concurrent Request calls
srv *rest.Client `json:"-"`
needs2FA bool `json:"-"` // set when SRP signin returns 409
}
@@ -45,20 +51,117 @@ type srpInitResponse struct {
C string `json:"c"`
}
-// String returns the session as a string
-// func (s *Session) String() string {
-// jsession, _ := json.Marshal(s)
-// return string(jsession)
-// }
+func cookieValueFingerprint(value string) string {
+ if value == "" {
+ return ""
+ }
+ sum := sha256.Sum256([]byte(value))
+ return hex.EncodeToString(sum[:6])
+}
-// Request makes a request
-func (s *Session) Request(ctx context.Context, opts rest.Opts, request any, response any) (*http.Response, error) {
- resp, err := s.srv.CallJSON(ctx, &opts, &request, &response)
+func authStateBodySummary(body []byte) string {
+ summary := []string{fmt.Sprintf("bytes=%d", len(body)), "hash=" + cookieValueFingerprint(string(body))}
+ var top map[string]json.RawMessage
+ if err := json.Unmarshal(body, &top); err != nil {
+ return strings.Join(summary, " ")
+ }
+ keys := make([]string, 0, len(top))
+ for key := range top {
+ keys = append(keys, key)
+ }
+ slices.Sort(keys)
+ summary = append(summary, fmt.Sprintf("keys=%v", keys))
+ if _, ok := top["phoneNumberVerification"]; ok {
+ summary = append(summary, "wrapped=true")
+ }
+ return strings.Join(summary, " ")
+}
- if err != nil {
- return resp, err
+func cookieDebugSummary(c *http.Cookie) string {
+ parts := []string{c.Name}
+ if c.Value == "" {
+ parts = append(parts, "empty", "len=0")
+ } else {
+ parts = append(parts, "set", fmt.Sprintf("len=%d", len(c.Value)), "hash="+cookieValueFingerprint(c.Value))
+ }
+ if c.Path != "" {
+ parts = append(parts, "path="+c.Path)
+ }
+ if c.Domain != "" {
+ parts = append(parts, "domain="+c.Domain)
+ }
+ if c.MaxAge != 0 {
+ parts = append(parts, fmt.Sprintf("maxAge=%d", c.MaxAge))
+ }
+ if !c.Expires.IsZero() {
+ parts = append(parts, "expires")
+ }
+ return strings.Join(parts, ",")
+}
+
+func cookieDebugSummaries(cookies []*http.Cookie) []string {
+ if len(cookies) == 0 {
+ return nil
+ }
+ out := make([]string, 0, len(cookies))
+ for _, c := range cookies {
+ out = append(out, cookieDebugSummary(c))
+ }
+ return out
+}
+
+func cookieJarDebugSummaries(cookies []*http.Cookie) []string {
+ if len(cookies) == 0 {
+ return nil
+ }
+ out := make([]string, 0, len(cookies))
+ for _, c := range cookies {
+ summary := c.Name
+ if c.Value == "" {
+ summary += ",empty,len=0"
+ } else {
+ summary += fmt.Sprintf(",len=%d,hash=%s", len(c.Value), cookieValueFingerprint(c.Value))
+ }
+ out = append(out, summary)
+ }
+ return out
+}
+
+func (s *Session) mergeCookies(cookies []*http.Cookie) {
+ if len(cookies) == 0 {
+ return
+ }
+ existing := make(map[string]int, len(s.Cookies))
+ for i, c := range s.Cookies {
+ existing[c.Name] = i
+ }
+ for _, c := range cookies {
+ if c.Value == "" {
+ if idx, ok := existing[c.Name]; ok {
+ fs.Debugf(nil, "iclouddrive: deleting empty auth cookie: %s", cookieDebugSummary(c))
+ s.Cookies = append(s.Cookies[:idx], s.Cookies[idx+1:]...)
+ delete(existing, c.Name)
+ for j := idx; j < len(s.Cookies); j++ {
+ existing[s.Cookies[j].Name] = j
+ }
+ } else {
+ fs.Debugf(nil, "iclouddrive: ignoring empty auth cookie tombstone for missing cookie: %s", cookieDebugSummary(c))
+ }
+ continue
+ }
+ if idx, ok := existing[c.Name]; ok {
+ s.Cookies[idx] = c
+ } else {
+ s.Cookies = append(s.Cookies, c)
+ existing[c.Name] = len(s.Cookies) - 1
+ }
}
+}
+// extractHeaders reads Apple session headers from an HTTP response into the session
+// Does NOT acquire s.mu - caller is responsible for locking if needed
+func (s *Session) extractHeaders(resp *http.Response) {
+ s.mergeCookies(resp.Cookies())
if val := resp.Header.Get("X-Apple-ID-Account-Country"); val != "" {
s.AccountCountry = val
}
@@ -77,7 +180,17 @@ func (s *Session) Request(ctx context.Context, opts rest.Opts, request any, resp
if val := resp.Header.Get("X-Apple-Auth-Attributes"); val != "" {
s.AuthAttributes = val
}
+}
+// Request makes a JSON API request and extracts session headers
+func (s *Session) Request(ctx context.Context, opts rest.Opts, request any, response any) (*http.Response, error) {
+ resp, err := s.srv.CallJSON(ctx, &opts, &request, &response)
+ if err != nil {
+ return resp, err
+ }
+ s.mu.Lock()
+ s.extractHeaders(resp)
+ s.mu.Unlock()
return resp, nil
}
@@ -89,13 +202,8 @@ func (s *Session) Requires2FA() bool {
return s.AccountInfo.DsInfo != nil && s.AccountInfo.DsInfo.HsaVersion == 2 && s.AccountInfo.HsaChallengeRequired
}
-// SignIn performs SRP-based authentication against Apple's idmsa endpoint.
+// SignIn performs SRP-based authentication against Apple's idmsa endpoint
func (s *Session) SignIn(ctx context.Context, appleID, password string) error {
- // Apple's SRP implementation expects a lowercase account name.
- // The old plaintext flow didn't need this because the server normalized
- // it, but SRP uses the username in client-side proof computation (M1).
- appleID = strings.ToLower(appleID)
-
// Step 1: Initialize the auth session
if err := s.authStart(ctx); err != nil {
return fmt.Errorf("authStart: %w", err)
@@ -106,8 +214,11 @@ func (s *Session) SignIn(ctx context.Context, appleID, password string) error {
return fmt.Errorf("authFederate: %w", err)
}
- // Step 3: SRP init — send client public value A, get salt + B
- client := newSRPClient()
+ // Step 3: SRP init - send client public value A, get salt + B
+ client, err := newSRPClient()
+ if err != nil {
+ return fmt.Errorf("newSRPClient: %w", err)
+ }
aBase64 := base64.StdEncoding.EncodeToString(client.getABytes())
initResp, err := s.authSRPInit(ctx, aBase64, appleID)
@@ -130,9 +241,11 @@ func (s *Session) SignIn(ctx context.Context, appleID, password string) error {
if err != nil {
return fmt.Errorf("derivePassword: %w", err)
}
- client.processChallenge([]byte(appleID), derivedKey, salt, serverB)
+ if err := client.processChallenge([]byte(appleID), derivedKey, salt, serverB); err != nil {
+ return fmt.Errorf("processChallenge: %w", err)
+ }
- // Step 5: Complete — send M1, M2 proofs
+ // Step 5: Complete - send M1, M2 proofs
m1Base64 := base64.StdEncoding.EncodeToString(client.M1)
m2Base64 := base64.StdEncoding.EncodeToString(client.M2)
@@ -143,7 +256,7 @@ func (s *Session) SignIn(ctx context.Context, appleID, password string) error {
return nil
}
-// authStart initializes the SRP auth session by hitting the authorize/signin endpoint.
+// authStart initializes the SRP auth session by hitting the authorize/signin endpoint
func (s *Session) authStart(ctx context.Context) error {
if s.FrameID == "" {
s.FrameID = strings.ToLower(uuid.New().String())
@@ -184,20 +297,11 @@ func (s *Session) authStart(ctx context.Context) error {
return fmt.Errorf("authStart: unexpected status %s", resp.Status)
}
- if val := resp.Header.Get("X-Apple-Auth-Attributes"); val != "" {
- s.AuthAttributes = val
- }
- if val := resp.Header.Get("scnt"); val != "" {
- s.Scnt = val
- }
- if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" {
- s.SessionID = val
- }
-
+ s.extractHeaders(resp)
return nil
}
-// authFederate submits the account name to Apple's federate endpoint.
+// authFederate submits the account name to Apple's federate endpoint
func (s *Session) authFederate(ctx context.Context, accountName string) error {
values := map[string]any{
"accountName": accountName,
@@ -224,16 +328,7 @@ func (s *Session) authFederate(ctx context.Context, accountName string) error {
}
_ = resp.Body.Close()
- if val := resp.Header.Get("X-Apple-Auth-Attributes"); val != "" {
- s.AuthAttributes = val
- }
- if val := resp.Header.Get("scnt"); val != "" {
- s.Scnt = val
- }
- if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" {
- s.SessionID = val
- }
-
+ s.extractHeaders(resp)
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("authFederate: unexpected status %s", resp.Status)
}
@@ -241,7 +336,7 @@ func (s *Session) authFederate(ctx context.Context, accountName string) error {
}
// authSRPInit sends the client's public value A to the server and retrieves
-// the salt, server public value B, iteration count, protocol, and challenge.
+// the salt, server public value B, iteration count, protocol, and challenge
func (s *Session) authSRPInit(ctx context.Context, aBase64, accountName string) (*srpInitResponse, error) {
values := map[string]any{
"a": aBase64,
@@ -267,18 +362,12 @@ func (s *Session) authSRPInit(ctx context.Context, aBase64, accountName string)
return nil, err
}
- if val := resp.Header.Get("scnt"); val != "" {
- s.Scnt = val
- }
- if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" {
- s.SessionID = val
- }
-
+ s.extractHeaders(resp)
return &initResp, nil
}
-// authSRPComplete sends the SRP proofs M1 and M2 to complete authentication.
-// Returns nil on success (200 or 409/2FA needed).
+// authSRPComplete sends the SRP proofs M1 and M2 to complete authentication
+// Returns nil on success (200 or 409/2FA needed)
func (s *Session) authSRPComplete(ctx context.Context, accountName, m1Base64, m2Base64, c string) error {
trustTokens := []string{}
if s.TrustToken != "" {
@@ -305,7 +394,6 @@ func (s *Session) authSRPComplete(ctx context.Context, accountName, m1Base64, m2
ExtraHeaders: s.getSRPAuthHeaders(),
RootURL: authEndpoint,
IgnoreStatus: true,
- NoResponse: true,
Body: body,
}
@@ -313,51 +401,65 @@ func (s *Session) authSRPComplete(ctx context.Context, accountName, m1Base64, m2
if err != nil {
return err
}
+ respBody, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
- // Extract updated headers
- if val := resp.Header.Get("X-Apple-Auth-Attributes"); val != "" {
- s.AuthAttributes = val
- }
- if val := resp.Header.Get("X-Apple-Session-Token"); val != "" {
- s.SessionToken = val
- }
- if val := resp.Header.Get("scnt"); val != "" {
- s.Scnt = val
- }
- if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" {
- s.SessionID = val
- }
- if val := resp.Header.Get("X-Apple-ID-Account-Country"); val != "" {
- s.AccountCountry = val
- }
- if val := resp.Header.Get("X-Apple-TwoSV-Trust-Token"); val != "" {
- s.TrustToken = val
- }
+ s.extractHeaders(resp)
switch resp.StatusCode {
case http.StatusOK:
- fs.Debugf("icloud", "SRP sign in successful")
+ fs.Debugf(nil, "iclouddrive: SRP sign in successful")
return nil
case http.StatusConflict:
- // 409 = 2FA required, this is expected
- fs.Debugf("icloud", "SRP sign in requires 2FA")
+ // 409 = 2FA required
+ fs.Debugf(nil, "iclouddrive: SRP sign in requires 2FA, response: %s", respBody)
s.needs2FA = true
return nil
+ case http.StatusPreconditionFailed:
+ // 412 = non-2FA account needs repair/complete step
+ fs.Debugf(nil, "iclouddrive: SRP sign in returned 412, attempting repair/complete")
+ return s.authRepairComplete(ctx)
case http.StatusForbidden:
return fmt.Errorf("sign in failed: incorrect username or password")
default:
- return fmt.Errorf("sign in failed: %s", resp.Status)
+ return fmt.Errorf("sign in failed: %s: %s", resp.Status, respBody)
}
}
-// getAuthOrigin returns the origin URL for auth requests.
-// Supports both global (idmsa.apple.com) and China (idmsa.apple.com.cn) endpoints.
+// authRepairComplete handles the repair flow for non-2FA accounts that return 412
+func (s *Session) authRepairComplete(ctx context.Context) error {
+ body, err := IntoReader(map[string]any{})
+ if err != nil {
+ return err
+ }
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/repair/complete",
+ ExtraHeaders: s.getSRPAuthHeaders(),
+ RootURL: authEndpoint,
+ IgnoreStatus: true,
+ NoResponse: true,
+ Body: body,
+ }
+ resp, err := s.srv.Call(ctx, &opts)
+ if err != nil {
+ return fmt.Errorf("repair/complete failed: %w", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("repair/complete returned %s", resp.Status)
+ }
+ s.extractHeaders(resp)
+ fs.Debugf(nil, "iclouddrive: repair/complete successful")
+ return nil
+}
+
+// getAuthOrigin returns the origin URL for auth requests
+// Supports both global (idmsa.apple.com) and China (idmsa.apple.com.cn) endpoints
func getAuthOrigin() string {
return strings.TrimSuffix(authEndpoint, "/appleauth/auth")
}
-// getSRPAuthHeaders returns headers needed for SRP auth requests.
+// getSRPAuthHeaders returns headers needed for SRP auth requests
func (s *Session) getSRPAuthHeaders() map[string]string {
frameTag := "auth-" + s.FrameID
authOrigin := getAuthOrigin()
@@ -414,14 +516,110 @@ func (s *Session) AuthWithToken(ctx context.Context) error {
}
resp, err := s.Request(ctx, opts, nil, &s.AccountInfo)
- if err == nil {
- s.Cookies = resp.Cookies()
+ if err != nil {
+ return err
+ }
+ fs.Debugf(nil, "iclouddrive: accountLogin response cookies: %v", cookieDebugSummaries(resp.Cookies()))
+ fs.Debugf(nil, "iclouddrive: session cookie jar after accountLogin: %v", cookieJarDebugSummaries(s.Cookies))
+
+ // Acquire PCS cookies if Advanced Data Protection is enabled
+ if ws := s.AccountInfo.Webservices["ckdatabasews"]; ws != nil && ws.PcsRequired {
+ fs.Debugf(nil, "iclouddrive: ADP detected (pcsRequired=true)")
+ if s.hasPCSCookies() {
+ fs.Debugf(nil, "iclouddrive: PCS cookies already present, skipping acquisition")
+ } else {
+ if err := s.acquirePCSCookies(ctx); err != nil {
+ return err
+ }
+ }
+ } else {
+ fs.Debugf(nil, "iclouddrive: no ADP (pcsRequired=false)")
+ }
+
+ return nil
+}
+
+// hasPCSCookies checks if the required PCS cookies for Photos are already present
+func (s *Session) hasPCSCookies() bool {
+ var hasPhotos, hasSharing bool
+ for _, c := range s.Cookies {
+ switch c.Name {
+ case "X-APPLE-WEBAUTH-PCS-Photos":
+ hasPhotos = true
+ case "X-APPLE-WEBAUTH-PCS-Sharing":
+ hasSharing = true
+ }
}
+ return hasPhotos && hasSharing
+}
+// acquirePCSCookies requests PCS cookies for ADP-enabled accounts
+// May require user approval on a trusted device (polls every 10s, max 5 min)
+func (s *Session) acquirePCSCookies(ctx context.Context) error {
+ fs.Logf(nil, "iclouddrive: Advanced Data Protection enabled, requesting PCS cookies")
+ const maxAttempts = 30 // 30 * 10s = 5 minutes max
+ for attempt := 0; attempt < maxAttempts; attempt++ {
+ fs.Debugf(nil, "iclouddrive: requestPCS outgoing cookies: %v", cookieJarDebugSummaries(s.Cookies))
+ values := map[string]any{
+ "appName": "photos",
+ "derivedFromUserAction": true,
+ }
+ body, err := IntoReader(values)
+ if err != nil {
+ return fmt.Errorf("requestPCS: %w", err)
+ }
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/requestPCS",
+ ExtraHeaders: s.GetHeaders(map[string]string{}),
+ RootURL: setupEndpoint,
+ }
+ opts.Body = body
+ var pcsResp struct {
+ Status string `json:"status"`
+ Message string `json:"message"`
+ }
+ resp, err := s.Request(ctx, opts, nil, &pcsResp)
+ if err != nil {
+ return fmt.Errorf("requestPCS: %w", err)
+ }
+ fs.Debugf(nil, "iclouddrive: requestPCS response cookies: %v", cookieDebugSummaries(resp.Cookies()))
+ fs.Debugf(nil, "iclouddrive: requestPCS response: status=%q message=%q cookies=%d",
+ pcsResp.Status, pcsResp.Message, len(resp.Cookies()))
+ if pcsResp.Status == "success" {
+ if !s.hasPCSCookies() {
+ return fmt.Errorf("requestPCS: server returned success but PCS cookies missing")
+ }
+ fs.Logf(nil, "iclouddrive: PCS cookies acquired")
+ return nil
+ }
+ // Device consent required - poll until approved
+ fs.Logf(nil, "iclouddrive: waiting for device approval for PCS (%s)", pcsResp.Message)
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-time.After(10 * time.Second):
+ }
+ }
+ return fmt.Errorf("requestPCS: timed out waiting for device approval after 5 minutes")
+}
+
+// RequestPushNotification explicitly requests a push notification to trusted devices
+// Required for iOS 26.4+ where the SRP 409 response no longer auto-pushes
+func (s *Session) RequestPushNotification(ctx context.Context) error {
+ opts := rest.Opts{
+ Method: "PUT",
+ Path: "/verify/trusteddevice/securitycode",
+ ExtraHeaders: s.GetAuthHeaders(map[string]string{}),
+ RootURL: authEndpoint,
+ NoResponse: true,
+ }
+
+ _, err := s.Request(ctx, opts, nil, nil)
return err
}
-// Validate2FACode validates the 2FA code
+// Validate2FACode validates the 2FA code from a trusted device push notification
func (s *Session) Validate2FACode(ctx context.Context, code string) error {
values := map[string]any{"securityCode": map[string]string{"code": code}}
body, err := IntoReader(values)
@@ -450,6 +648,132 @@ func (s *Session) Validate2FACode(ctx context.Context, code string) error {
return fmt.Errorf("validate2FACode failed: %w", err)
}
+// TrustedPhoneNumber represents a phone number that can receive SMS verification codes
+type TrustedPhoneNumber struct {
+ ID int `json:"id"`
+ NumberWithDialCode string `json:"numberWithDialCode"`
+ ObfuscatedNumber string `json:"obfuscatedNumber"`
+ PushMode string `json:"pushMode"`
+ NonFTEU bool `json:"nonFTEU"`
+}
+
+// AuthStateResponse is the response from GET /appleauth/auth after sign-in
+// Some accounts return fields at top level, others nest them under phoneNumberVerification
+type AuthStateResponse struct {
+ TrustedPhoneNumbers []TrustedPhoneNumber `json:"trustedPhoneNumbers"`
+ TrustedPhoneNumber *TrustedPhoneNumber `json:"trustedPhoneNumber"`
+ NoTrustedDevices bool `json:"noTrustedDevices"`
+ AuthenticationType string `json:"authenticationType"`
+ Hsa2Account bool `json:"hsa2Account"`
+ PhoneNumberVerification *AuthStateResponse `json:"phoneNumberVerification"`
+}
+
+// GetAuthState retrieves the current auth state including trusted phone numbers for SMS 2FA
+func (s *Session) GetAuthState(ctx context.Context) (*AuthStateResponse, error) {
+ opts := rest.Opts{
+ Method: "GET",
+ Path: "",
+ ExtraHeaders: s.GetAuthHeaders(map[string]string{}),
+ RootURL: authEndpoint,
+ ContentLength: int64Ptr(0),
+ }
+ // Use srv.Call directly to capture the raw response body for debugging
+ resp, err := s.srv.Call(ctx, &opts)
+ if err != nil {
+ return nil, fmt.Errorf("getAuthState: %w", err)
+ }
+ s.mu.Lock()
+ s.extractHeaders(resp)
+ s.mu.Unlock()
+ body, readErr := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+ if readErr != nil {
+ return nil, fmt.Errorf("getAuthState read: %w", readErr)
+ }
+ fs.Debugf(nil, "iclouddrive: auth state response summary: %s", authStateBodySummary(body))
+ var state AuthStateResponse
+ if err := json.Unmarshal(body, &state); err != nil {
+ fs.Debugf(nil, "iclouddrive: auth state parse failed: %s", authStateBodySummary(body))
+ return nil, fmt.Errorf("getAuthState unmarshal: %w", err)
+ }
+ // Some accounts nest auth data under phoneNumberVerification
+ if state.PhoneNumberVerification != nil && state.AuthenticationType == "" {
+ fs.Debugf(nil, "iclouddrive: unwrapping phoneNumberVerification envelope")
+ n := state.PhoneNumberVerification
+ state.TrustedPhoneNumbers = n.TrustedPhoneNumbers
+ state.TrustedPhoneNumber = n.TrustedPhoneNumber
+ state.NoTrustedDevices = n.NoTrustedDevices
+ state.AuthenticationType = n.AuthenticationType
+ state.Hsa2Account = n.Hsa2Account
+ state.PhoneNumberVerification = nil
+ }
+ fs.Debugf(nil, "iclouddrive: auth state: type=%s hsa2=%v noTrustedDevices=%v phones=%d phoneSingular=%v",
+ state.AuthenticationType, state.Hsa2Account, state.NoTrustedDevices,
+ len(state.TrustedPhoneNumbers), state.TrustedPhoneNumber != nil)
+ // Fall back to singular trustedPhoneNumber when plural array is empty
+ // Some accounts return only the singular form (SMS-only, no trusted devices)
+ if len(state.TrustedPhoneNumbers) == 0 && state.TrustedPhoneNumber != nil {
+ fs.Debugf(nil, "iclouddrive: using singular trustedPhoneNumber (id=%d) as fallback",
+ state.TrustedPhoneNumber.ID)
+ state.TrustedPhoneNumbers = []TrustedPhoneNumber{*state.TrustedPhoneNumber}
+ }
+ return &state, nil
+}
+
+// RequestSMSCode triggers SMS code delivery to a trusted phone number
+func (s *Session) RequestSMSCode(ctx context.Context, phoneID int, mode string) error {
+ values := map[string]any{
+ "phoneNumber": map[string]any{"id": phoneID},
+ "mode": mode,
+ }
+ body, err := IntoReader(values)
+ if err != nil {
+ return err
+ }
+ opts := rest.Opts{
+ Method: "PUT",
+ Path: "/verify/phone",
+ ExtraHeaders: s.GetAuthHeaders(map[string]string{}),
+ RootURL: authEndpoint,
+ Body: body,
+ NoResponse: true,
+ }
+ _, err = s.Request(ctx, opts, nil, nil)
+ if err != nil {
+ return fmt.Errorf("requestSMSCode: %w", err)
+ }
+ return nil
+}
+
+// ValidateSMSCode validates a 2FA code received via SMS
+func (s *Session) ValidateSMSCode(ctx context.Context, code string, phoneID int, mode string) error {
+ values := map[string]any{
+ "securityCode": map[string]string{"code": code},
+ "phoneNumber": map[string]any{"id": phoneID},
+ "mode": mode,
+ }
+ body, err := IntoReader(values)
+ if err != nil {
+ return err
+ }
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/verify/phone/securitycode",
+ ExtraHeaders: s.GetAuthHeaders(map[string]string{}),
+ RootURL: authEndpoint,
+ Body: body,
+ NoResponse: true,
+ }
+ _, err = s.Request(ctx, opts, nil, nil)
+ if err == nil {
+ if err := s.TrustSession(ctx); err != nil {
+ return err
+ }
+ return nil
+ }
+ return fmt.Errorf("validateSMSCode: %w", err)
+}
+
// TrustSession trusts the session
func (s *Session) TrustSession(ctx context.Context) error {
opts := rest.Opts{
@@ -458,7 +782,7 @@ func (s *Session) TrustSession(ctx context.Context) error {
ExtraHeaders: s.GetAuthHeaders(map[string]string{}),
RootURL: authEndpoint,
NoResponse: true,
- ContentLength: common.Int64(0),
+ ContentLength: int64Ptr(0),
}
_, err := s.Request(ctx, opts, nil, nil)
@@ -476,25 +800,26 @@ func (s *Session) ValidateSession(ctx context.Context) error {
Path: "/validate",
ExtraHeaders: s.GetHeaders(map[string]string{}),
RootURL: setupEndpoint,
- ContentLength: common.Int64(0),
+ ContentLength: int64Ptr(0),
}
_, err := s.Request(ctx, opts, nil, &s.AccountInfo)
if err != nil {
return fmt.Errorf("validateSession failed: %w", err)
}
+ s.needs2FA = false
return nil
}
-// GetAuthHeaders returns the authentication headers for the session.
-// Used for 2FA validation and trust requests to idmsa.apple.com.
+// GetAuthHeaders returns the authentication headers for the session
+// Used for 2FA validation and trust requests to idmsa.apple.com
func (s *Session) GetAuthHeaders(overwrite map[string]string) map[string]string {
headers := s.getSRPAuthHeaders()
maps.Copy(headers, overwrite)
return headers
}
-// GetHeaders Gets the authentication headers required for a request
+// GetHeaders returns the authentication headers required for a request
func (s *Session) GetHeaders(overwrite map[string]string) map[string]string {
headers := GetCommonHeaders(map[string]string{})
headers["Cookie"] = s.GetCookieString()
@@ -502,17 +827,26 @@ func (s *Session) GetHeaders(overwrite map[string]string) map[string]string {
return headers
}
-// GetCookieString returns the cookie header string for the session.
+// GetCookieString returns the cookie header string for the session
func (s *Session) GetCookieString() string {
- cookieHeader := ""
- // we only care about name and value.
+ var b strings.Builder
+ first := true
for _, cookie := range s.Cookies {
- cookieHeader = cookieHeader + cookie.Name + "=" + cookie.Value + ";"
+ if cookie.Value == "" {
+ continue
+ }
+ if !first {
+ b.WriteString("; ")
+ }
+ first = false
+ b.WriteString(cookie.Name)
+ b.WriteByte('=')
+ b.WriteString(cookie.Value)
}
- return cookieHeader
+ return b.String()
}
-// GetCommonHeaders generates common HTTP headers with optional overwrite.
+// GetCommonHeaders generates common HTTP headers with optional overwrite
func GetCommonHeaders(overwrite map[string]string) map[string]string {
headers := map[string]string{
"Content-Type": "application/json",
@@ -524,32 +858,9 @@ func GetCommonHeaders(overwrite map[string]string) map[string]string {
return headers
}
-// MergeCookies merges two slices of http.Cookies, ensuring no duplicates are added.
-func MergeCookies(left []*http.Cookie, right []*http.Cookie) ([]*http.Cookie, error) {
- var hashes []string
- for _, cookie := range right {
- hashes = append(hashes, cookie.Raw)
- }
- for _, cookie := range left {
- if !slices.Contains(hashes, cookie.Raw) {
- right = append(right, cookie)
- }
- }
- return right, nil
-}
-
-// GetCookiesForDomain filters the provided cookies based on the domain of the given URL.
-func GetCookiesForDomain(url *url.URL, cookies []*http.Cookie) ([]*http.Cookie, error) {
- var domainCookies []*http.Cookie
- for _, cookie := range cookies {
- if strings.HasSuffix(url.Host, cookie.Domain) {
- domainCookies = append(domainCookies, cookie)
- }
- }
- return domainCookies, nil
-}
+func int64Ptr(v int64) *int64 { return &v }
-// NewSession creates a new Session instance with default values.
+// NewSession creates a new Session instance with default values
func NewSession() *Session {
session := &Session{
FrameID: strings.ToLower(uuid.New().String()),
@@ -564,117 +875,20 @@ func NewSession() *Session {
return session
}
-// AccountInfo represents an account info
+// AccountInfo represents the subset of Apple's account response we actually use
+// json.Unmarshal silently ignores extra fields in the response
type AccountInfo struct {
- DsInfo *ValidateDataDsInfo `json:"dsInfo"`
- HasMinimumDeviceForPhotosWeb bool `json:"hasMinimumDeviceForPhotosWeb"`
- ICDPEnabled bool `json:"iCDPEnabled"`
- Webservices map[string]*webService `json:"webservices"`
- PcsEnabled bool `json:"pcsEnabled"`
- TermsUpdateNeeded bool `json:"termsUpdateNeeded"`
- ConfigBag struct {
- Urls struct {
- AccountCreateUI string `json:"accountCreateUI"`
- AccountLoginUI string `json:"accountLoginUI"`
- AccountLogin string `json:"accountLogin"`
- AccountRepairUI string `json:"accountRepairUI"`
- DownloadICloudTerms string `json:"downloadICloudTerms"`
- RepairDone string `json:"repairDone"`
- AccountAuthorizeUI string `json:"accountAuthorizeUI"`
- VettingURLForEmail string `json:"vettingUrlForEmail"`
- AccountCreate string `json:"accountCreate"`
- GetICloudTerms string `json:"getICloudTerms"`
- VettingURLForPhone string `json:"vettingUrlForPhone"`
- } `json:"urls"`
- AccountCreateEnabled bool `json:"accountCreateEnabled"`
- } `json:"configBag"`
- HsaTrustedBrowser bool `json:"hsaTrustedBrowser"`
- AppsOrder []string `json:"appsOrder"`
- Version int `json:"version"`
- IsExtendedLogin bool `json:"isExtendedLogin"`
- PcsServiceIdentitiesIncluded bool `json:"pcsServiceIdentitiesIncluded"`
- IsRepairNeeded bool `json:"isRepairNeeded"`
- HsaChallengeRequired bool `json:"hsaChallengeRequired"`
- RequestInfo struct {
- Country string `json:"country"`
- TimeZone string `json:"timeZone"`
- Region string `json:"region"`
- } `json:"requestInfo"`
- PcsDeleted bool `json:"pcsDeleted"`
- ICloudInfo struct {
- SafariBookmarksHasMigratedToCloudKit bool `json:"SafariBookmarksHasMigratedToCloudKit"`
- } `json:"iCloudInfo"`
- Apps map[string]*ValidateDataApp `json:"apps"`
-}
-
-// ValidateDataDsInfo represents an validation info
-type ValidateDataDsInfo struct {
- HsaVersion int `json:"hsaVersion"`
- LastName string `json:"lastName"`
- ICDPEnabled bool `json:"iCDPEnabled"`
- TantorMigrated bool `json:"tantorMigrated"`
- Dsid string `json:"dsid"`
- HsaEnabled bool `json:"hsaEnabled"`
- IsHideMyEmailSubscriptionActive bool `json:"isHideMyEmailSubscriptionActive"`
- IroncadeMigrated bool `json:"ironcadeMigrated"`
- Locale string `json:"locale"`
- BrZoneConsolidated bool `json:"brZoneConsolidated"`
- ICDRSCapableDeviceList string `json:"ICDRSCapableDeviceList"`
- IsManagedAppleID bool `json:"isManagedAppleID"`
- IsCustomDomainsFeatureAvailable bool `json:"isCustomDomainsFeatureAvailable"`
- IsHideMyEmailFeatureAvailable bool `json:"isHideMyEmailFeatureAvailable"`
- ContinueOnDeviceEligibleDeviceInfo []string `json:"ContinueOnDeviceEligibleDeviceInfo"`
- Gilligvited bool `json:"gilligvited"`
- AppleIDAliases []any `json:"appleIdAliases"`
- UbiquityEOLEnabled bool `json:"ubiquityEOLEnabled"`
- IsPaidDeveloper bool `json:"isPaidDeveloper"`
- CountryCode string `json:"countryCode"`
- NotificationID string `json:"notificationId"`
- PrimaryEmailVerified bool `json:"primaryEmailVerified"`
- ADsID string `json:"aDsID"`
- Locked bool `json:"locked"`
- ICDRSCapableDeviceCount int `json:"ICDRSCapableDeviceCount"`
- HasICloudQualifyingDevice bool `json:"hasICloudQualifyingDevice"`
- PrimaryEmail string `json:"primaryEmail"`
- AppleIDEntries []struct {
- IsPrimary bool `json:"isPrimary"`
- Type string `json:"type"`
- Value string `json:"value"`
- } `json:"appleIdEntries"`
- GilliganEnabled bool `json:"gilligan-enabled"`
- IsWebAccessAllowed bool `json:"isWebAccessAllowed"`
- FullName string `json:"fullName"`
- MailFlags struct {
- IsThreadingAvailable bool `json:"isThreadingAvailable"`
- IsSearchV2Provisioned bool `json:"isSearchV2Provisioned"`
- SCKMail bool `json:"sCKMail"`
- IsMppSupportedInCurrentCountry bool `json:"isMppSupportedInCurrentCountry"`
- } `json:"mailFlags"`
- LanguageCode string `json:"languageCode"`
- AppleID string `json:"appleId"`
- HasUnreleasedOS bool `json:"hasUnreleasedOS"`
- AnalyticsOptInStatus bool `json:"analyticsOptInStatus"`
- FirstName string `json:"firstName"`
- ICloudAppleIDAlias string `json:"iCloudAppleIdAlias"`
- NotesMigrated bool `json:"notesMigrated"`
- BeneficiaryInfo struct {
- IsBeneficiary bool `json:"isBeneficiary"`
- } `json:"beneficiaryInfo"`
- HasPaymentInfo bool `json:"hasPaymentInfo"`
- PcsDelet bool `json:"pcsDelet"`
- AppleIDAlias string `json:"appleIdAlias"`
- BrMigrated bool `json:"brMigrated"`
- StatusCode int `json:"statusCode"`
- FamilyEligible bool `json:"familyEligible"`
-}
-
-// ValidateDataApp represents an app
-type ValidateDataApp struct {
- CanLaunchWithOneFactor bool `json:"canLaunchWithOneFactor"`
- IsQualifiedForBeta bool `json:"isQualifiedForBeta"`
-}
-
-// WebService represents a web service
+ DsInfo *dsInfo `json:"dsInfo"`
+ Webservices map[string]*webService `json:"webservices"`
+ HsaChallengeRequired bool `json:"hsaChallengeRequired"`
+}
+
+// dsInfo holds the account metadata fields we read
+type dsInfo struct {
+ HsaVersion int `json:"hsaVersion"`
+}
+
+// webService represents a web service
type webService struct {
PcsRequired bool `json:"pcsRequired"`
URL string `json:"url"`
diff --git a/backend/iclouddrive/api/session_test.go b/backend/iclouddrive/api/session_test.go
new file mode 100644
index 000000000..c3d06a37b
--- /dev/null
+++ b/backend/iclouddrive/api/session_test.go
@@ -0,0 +1,40 @@
+package api
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestExtractHeadersMergesCookies(t *testing.T) {
+ s := NewSession()
+ s.Cookies = []*http.Cookie{{Name: "existing", Value: "old"}}
+
+ resp := &http.Response{Header: make(http.Header)}
+ resp.Header.Add("Set-Cookie", (&http.Cookie{Name: "existing", Value: "new"}).String())
+ resp.Header.Add("Set-Cookie", (&http.Cookie{Name: "fresh", Value: "value"}).String())
+ resp.Header.Set("X-Apple-Session-Token", "session-token")
+
+ s.extractHeaders(resp)
+
+ require.Len(t, s.Cookies, 2)
+ assert.Equal(t, "new", s.Cookies[0].Value)
+ assert.Equal(t, "fresh", s.Cookies[1].Name)
+ assert.Equal(t, "session-token", s.SessionToken)
+}
+
+func TestExtractHeadersDeletesEmptyCookies(t *testing.T) {
+ s := NewSession()
+ s.Cookies = []*http.Cookie{{Name: "X-APPLE-WEBAUTH-HSA-LOGIN", Value: "stale"}, {Name: "keep", Value: "value"}}
+
+ resp := &http.Response{Header: make(http.Header)}
+ resp.Header.Add("Set-Cookie", (&http.Cookie{Name: "X-APPLE-WEBAUTH-HSA-LOGIN", Value: ""}).String())
+
+ s.extractHeaders(resp)
+
+ require.Len(t, s.Cookies, 1)
+ assert.Equal(t, "keep", s.Cookies[0].Name)
+ assert.Equal(t, "keep=value", s.GetCookieString())
+}
diff --git a/backend/iclouddrive/api/srp.go b/backend/iclouddrive/api/srp.go
index e2f7c7f66..9ca50eea4 100644
--- a/backend/iclouddrive/api/srp.go
+++ b/backend/iclouddrive/api/srp.go
@@ -46,10 +46,10 @@ type srpClient struct {
}
// newSRPClient generates a random 32-byte secret and computes the public value A.
-func newSRPClient() *srpClient {
+func newSRPClient() (*srpClient, error) {
secret := make([]byte, 32)
if _, err := rand.Read(secret); err != nil {
- panic(fmt.Sprintf("srp: rand.Read failed: %v", err))
+ return nil, fmt.Errorf("srp: rand.Read failed: %w", err)
}
a := new(big.Int).SetBytes(secret)
@@ -60,7 +60,7 @@ func newSRPClient() *srpClient {
a: a,
A: A,
k: k,
- }
+ }, nil
}
// getABytes returns the padded public value A.
@@ -71,16 +71,21 @@ func (c *srpClient) getABytes() []byte {
// processChallenge computes the session key and proof values from the server challenge.
// username is the Apple ID, derivedKey is the output of derivePassword, salt and B come
// from the server's init response (raw bytes, already base64-decoded).
-func (c *srpClient) processChallenge(username, derivedKey, salt, serverB []byte) {
+func (c *srpClient) processChallenge(username, derivedKey, salt, serverB []byte) error {
B := new(big.Int).SetBytes(serverB)
- // Validate B
+ // Validate B (RFC 5054)
if B.Cmp(big.NewInt(0)) <= 0 || B.Cmp(srpN) >= 0 {
- panic("srp: invalid server-supplied B, must be 1..N-1")
+ return fmt.Errorf("srp: invalid server-supplied B, must be 1..N-1")
}
x := calculateX(salt, derivedKey)
u := calculateU(c.A, B)
+
+ // Validate u (RFC 5054)
+ if u.Sign() == 0 {
+ return fmt.Errorf("srp: calculated u is zero, aborting")
+ }
S := calculateS(c.k, x, c.a, B, u)
c.K = calculateK(S)
@@ -88,6 +93,7 @@ func (c *srpClient) processChallenge(username, derivedKey, salt, serverB []byte)
bBytes := padToN(B)
c.M1 = calculateM1(username, salt, aBytes, bBytes, c.K)
c.M2 = calculateM2(aBytes, c.M1, c.K)
+ return nil
}
// derivePassword performs Apple's password key derivation.
@@ -98,10 +104,12 @@ func derivePassword(password string, salt []byte, iterations int, protocol strin
var passInput string
switch protocol {
+ case "s2k":
+ passInput = string(passHash[:])
case "s2k_fo":
passInput = hex.EncodeToString(passHash[:])
- default: // "s2k"
- passInput = string(passHash[:])
+ default:
+ return nil, fmt.Errorf("unsupported SRP protocol: %q", protocol)
}
return pbkdf2.Key(sha256.New, passInput, salt, iterations, 32)
}
@@ -142,7 +150,7 @@ func getMultiplier() *big.Int {
}
// calculateX computes x = H(salt | H(":" | password))
-// Apple's variant: NoUserNameInX — username is omitted from the inner hash.
+// Apple's variant: NoUserNameInX - username is omitted from the inner hash.
// The "password" here is actually the derived key from derivePassword.
func calculateX(salt, derivedKey []byte) *big.Int {
h := srpHashFunc()
diff --git a/backend/iclouddrive/api/srp_test.go b/backend/iclouddrive/api/srp_test.go
index 3136eda55..5eec3c011 100644
--- a/backend/iclouddrive/api/srp_test.go
+++ b/backend/iclouddrive/api/srp_test.go
@@ -27,7 +27,7 @@ func TestPadToN(t *testing.T) {
})
t.Run("preserves value that fills N", func(t *testing.T) {
- // Use N itself — already 256 bytes
+ // Use N itself - already 256 bytes
result := padToN(srpN)
assert.Equal(t, srpNLenBytes, len(result))
assert.Equal(t, srpN.Bytes(), result)
@@ -52,7 +52,7 @@ func TestGetMultiplier(t *testing.T) {
k2 := getMultiplier()
assert.Equal(t, 0, k.Cmp(k2), "multiplier must be deterministic")
- // k = H(N | pad(g)) — verify by manual computation
+ // k = H(N | pad(g)) - verify by manual computation
h := srpHashFunc()
nBytes := srpN.Bytes()
gBytes := srpG.Bytes()
@@ -123,6 +123,12 @@ func TestDerivePassword(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, 32, len(key))
})
+
+ t.Run("unknown protocol returns error", func(t *testing.T) {
+ _, err := derivePassword(password, salt, 1000, "unknown")
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "unsupported SRP protocol")
+ })
}
func TestCalculateX(t *testing.T) {
@@ -256,7 +262,8 @@ func TestCalculateM2(t *testing.T) {
}
func TestNewSRPClient(t *testing.T) {
- client := newSRPClient()
+ client, err := newSRPClient()
+ require.NoError(t, err)
// A must be non-zero and in range [1, N-1]
assert.True(t, client.A.Sign() > 0, "A must be positive")
@@ -273,7 +280,8 @@ func TestNewSRPClient(t *testing.T) {
assert.Equal(t, 0, client.A.Cmp(recoveredA))
// Two clients should have different secrets (probabilistic, but 2^256 collision chance)
- client2 := newSRPClient()
+ client2, err := newSRPClient()
+ require.NoError(t, err)
assert.NotEqual(t, 0, client.a.Cmp(client2.a), "two clients must have different secrets")
}
@@ -315,7 +323,8 @@ func TestProcessChallenge(t *testing.T) {
serverB := padToN(B)
- client.processChallenge(username, derivedKey, salt, serverB)
+ err := client.processChallenge(username, derivedKey, salt, serverB)
+ require.NoError(t, err)
// M1, M2, K must be populated
assert.NotNil(t, client.M1)
@@ -328,13 +337,14 @@ func TestProcessChallenge(t *testing.T) {
// M1 and M2 must be different
assert.NotEqual(t, client.M1, client.M2)
- // Results must be deterministic — run again with same inputs
+ // Results must be deterministic - run again with same inputs
client2 := &srpClient{
a: new(big.Int).Set(a),
A: new(big.Int).Set(A),
k: new(big.Int).Set(k),
}
- client2.processChallenge(username, derivedKey, salt, serverB)
+ err = client2.processChallenge(username, derivedKey, salt, serverB)
+ require.NoError(t, err)
assert.Equal(t, client.M1, client2.M1, "M1 must be deterministic")
assert.Equal(t, client.M2, client2.M2, "M2 must be deterministic")
@@ -352,6 +362,28 @@ func TestProcessChallenge(t *testing.T) {
assert.Equal(t, client.K, kServer, "client and server must derive the same session key")
}
+func TestProcessChallenge_InvalidB(t *testing.T) {
+ k := getMultiplier()
+ client := &srpClient{
+ a: big.NewInt(42),
+ A: new(big.Int).Exp(srpG, big.NewInt(42), srpN),
+ k: k,
+ }
+ username := []byte("[email protected]")
+ derivedKey := make([]byte, 32)
+ salt := []byte("salt")
+
+ t.Run("B=0 rejected", func(t *testing.T) {
+ err := client.processChallenge(username, derivedKey, salt, padToN(big.NewInt(0)))
+ assert.Error(t, err)
+ })
+
+ t.Run("B=N rejected", func(t *testing.T) {
+ err := client.processChallenge(username, derivedKey, salt, padToN(srpN))
+ assert.Error(t, err)
+ })
+}
+
func TestSRPGroupParameters(t *testing.T) {
// Verify the 2048-bit prime starts with known hex prefix
nHex := hex.EncodeToString(srpN.Bytes())
diff --git a/backend/iclouddrive/icloud.go b/backend/iclouddrive/icloud.go
new file mode 100644
index 000000000..40ee6c949
--- /dev/null
+++ b/backend/iclouddrive/icloud.go
@@ -0,0 +1,495 @@
+//go:build !plan9 && !solaris
+
+// Package iclouddrive provides access to iCloud Drive and Photos
+package iclouddrive
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+
+ "github.com/rclone/rclone/backend/iclouddrive/api"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config"
+ "github.com/rclone/rclone/fs/config/configmap"
+ "github.com/rclone/rclone/fs/config/configstruct"
+ "github.com/rclone/rclone/fs/config/obscure"
+ "github.com/rclone/rclone/lib/encoder"
+)
+
+const configAuthSession = "_auth_session"
+
+// configAuthState holds session fields that must be preserved across
+// Config() state machine steps (push 2FA and SMS both need it)
+type configAuthState struct {
+ Scnt string `json:"s"`
+ SessionID string `json:"i"`
+ AuthAttributes string `json:"a"`
+ FrameID string `json:"f"`
+ SessionToken string `json:"t"`
+ ClientID string `json:"c"`
+ AccountCountry string `json:"ac"`
+ Phones []smsPhone `json:"p,omitempty"`
+}
+
+type smsPhone struct {
+ ID int `json:"id"`
+ Num string `json:"n"`
+ Mode string `json:"m"`
+}
+
+func saveAuthSession(m configmap.Mapper, s *api.Session, phones []smsPhone) {
+ st := configAuthState{
+ Scnt: s.Scnt,
+ SessionID: s.SessionID,
+ AuthAttributes: s.AuthAttributes,
+ FrameID: s.FrameID,
+ SessionToken: s.SessionToken,
+ ClientID: s.ClientID,
+ AccountCountry: s.AccountCountry,
+ Phones: phones,
+ }
+ data, err := json.Marshal(st)
+ if err != nil {
+ fs.Debugf(nil, "iclouddrive: failed to marshal auth session: %v", err)
+ return
+ }
+ m.Set(configAuthSession, base64.StdEncoding.EncodeToString(data))
+}
+
+func loadAuthSession(m configmap.Mapper) (*configAuthState, error) {
+ raw, _ := m.Get(configAuthSession)
+ if raw == "" {
+ return nil, errors.New("auth session state lost, please reconfigure")
+ }
+ data, err := base64.StdEncoding.DecodeString(raw)
+ if err != nil {
+ return nil, fmt.Errorf("corrupt auth session state: %w", err)
+ }
+ var st configAuthState
+ if err := json.Unmarshal(data, &st); err != nil {
+ return nil, fmt.Errorf("invalid auth session state: %w", err)
+ }
+ return &st, nil
+}
+
+func restoreAuthSession(icloud *api.Client, st *configAuthState) {
+ icloud.Session.Scnt = st.Scnt
+ icloud.Session.SessionID = st.SessionID
+ icloud.Session.AuthAttributes = st.AuthAttributes
+ icloud.Session.FrameID = st.FrameID
+ icloud.Session.SessionToken = st.SessionToken
+ icloud.Session.ClientID = st.ClientID
+ icloud.Session.AccountCountry = st.AccountCountry
+}
+
+// resumeConfigClient recreates an API client and restores session state saved
+// from a previous Config() step. Used across 2FA states to avoid re-authenticating
+func resumeConfigClient(m configmap.Mapper, appleid, password, trustToken, clientID string, cookies []*http.Cookie) (*api.Client, *configAuthState, error) {
+ st, err := loadAuthSession(m)
+ if err != nil {
+ return nil, nil, err
+ }
+ icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil, "_config")
+ if err != nil {
+ return nil, nil, err
+ }
+ restoreAuthSession(icloud, st)
+ return icloud, st, nil
+}
+
+// saveAuthCredentials persists trust token, cookies, and clears session state
+// after successful 2FA validation
+func saveAuthCredentials(m configmap.Mapper, icloud *api.Client, name string) {
+ m.Set(configTrustToken, icloud.Session.TrustToken)
+ m.Set(configCookies, icloud.Session.GetCookieString())
+ m.Set(configAuthSession, "")
+ api.ClearCacheDir(name)
+}
+
+// triggerSMSFlow handles phone selection and SMS triggering
+// Single phone: triggers SMS immediately, returns code prompt
+// Multiple phones: saves session + phone list, returns phone picker
+func triggerSMSFlow(ctx context.Context, icloud *api.Client, phones []api.TrustedPhoneNumber, m configmap.Mapper) (*fs.ConfigOut, error) {
+ smsPhones := make([]smsPhone, len(phones))
+ for i, p := range phones {
+ mode := p.PushMode
+ if mode == "" {
+ mode = "sms"
+ }
+ smsPhones[i] = smsPhone{ID: p.ID, Num: p.NumberWithDialCode, Mode: mode}
+ }
+
+ if len(phones) == 1 {
+ p := smsPhones[0]
+ if err := icloud.Session.RequestSMSCode(ctx, p.ID, p.Mode); err != nil {
+ return nil, fmt.Errorf("failed to send SMS code: %w", err)
+ }
+ saveAuthSession(m, icloud.Session, nil)
+ nextState := fmt.Sprintf("2fa_sms_%d_%s", p.ID, p.Mode)
+ return fs.ConfigInput(nextState, "config_2fa_sms", fmt.Sprintf("Enter the verification code sent to %s", p.Num))
+ }
+
+ // Multiple phones - save session and present picker
+ saveAuthSession(m, icloud.Session, smsPhones)
+ items := make([]fs.OptionExample, len(smsPhones))
+ for i, p := range smsPhones {
+ items[i] = fs.OptionExample{
+ Value: fmt.Sprintf("%d_%s", p.ID, p.Mode),
+ Help: p.Num,
+ }
+ }
+ return fs.ConfigChooseExclusiveFixed("2fa_sms_select", "config_2fa_phone", "Select phone number for SMS verification", items)
+}
+
+const (
+ configService = "service"
+
+ // Service types
+ serviceDrive = "drive"
+ servicePhotos = "photos"
+)
+
+// ServiceOptions defines the configuration for service selection
+type ServiceOptions struct {
+ Service string `config:"service"`
+}
+
+// Register with rclone
+func init() {
+ fs.Register(&fs.RegInfo{
+ Name: "iclouddrive",
+ Description: "iCloud Drive and Photos",
+ Config: Config,
+ NewFs: NewServiceFs,
+ MetadataInfo: &fs.MetadataInfo{
+ System: map[string]fs.MetadataHelp{
+ "width": {
+ Help: "Image width in pixels",
+ Type: "int",
+ ReadOnly: true,
+ },
+ "height": {
+ Help: "Image height in pixels",
+ Type: "int",
+ ReadOnly: true,
+ },
+ "added-time": {
+ Help: "Time the item was added to the iCloud library",
+ Type: "RFC 3339",
+ Example: "2006-01-02T15:04:05Z",
+ ReadOnly: true,
+ },
+ "favorite": {
+ Help: "Whether the item is marked as favorite",
+ Type: "bool",
+ ReadOnly: true,
+ },
+ "hidden": {
+ Help: "Whether the item is hidden",
+ Type: "bool",
+ ReadOnly: true,
+ },
+ },
+ Help: "Metadata is read-only and available for the Photos service only.",
+ },
+ Options: []fs.Option{{
+ Name: configService,
+ Help: "iCloud service to use.",
+ Required: true,
+ Default: serviceDrive,
+ Examples: []fs.OptionExample{{
+ Value: serviceDrive,
+ Help: "iCloud Drive",
+ }, {
+ Value: servicePhotos,
+ Help: "iCloud Photos",
+ }},
+ }, {
+ Name: configAppleID,
+ Help: "Apple ID.",
+ Required: true,
+ Sensitive: true,
+ }, {
+ Name: configPassword,
+ Help: "Password.",
+ Required: true,
+ IsPassword: true,
+ Sensitive: true,
+ }, {
+ Name: configTrustToken,
+ Help: "Trust token for session authentication.",
+ IsPassword: false,
+ Required: false,
+ Sensitive: true,
+ Hide: fs.OptionHideBoth,
+ }, {
+ Name: configCookies,
+ Help: "Session cookies.",
+ Required: false,
+ Advanced: false,
+ Sensitive: true,
+ Hide: fs.OptionHideBoth,
+ }, {
+ Name: configClientID,
+ Help: "Client ID for iCloud API access.",
+ Required: false,
+ Advanced: true,
+ Default: "d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d",
+ }, {
+ Name: config.ConfigEncoding,
+ Help: config.ConfigEncodingHelp,
+ Advanced: true,
+ Default: (encoder.Display |
+ encoder.EncodeBackSlash |
+ encoder.EncodeSlash |
+ encoder.EncodeInvalidUtf8),
+ }},
+ })
+}
+
+// Config handles the authentication and configuration flow
+func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
+ var err error
+ appleid, _ := m.Get(configAppleID)
+ if appleid == "" {
+ return nil, errors.New("an Apple ID is required")
+ }
+
+ password, _ := m.Get(configPassword)
+ if password != "" {
+ password, err = obscure.Reveal(password)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ trustToken, _ := m.Get(configTrustToken)
+ cookieRaw, _ := m.Get(configCookies)
+ clientID, _ := m.Get(configClientID)
+ cookies := ReadCookies(cookieRaw)
+
+ switch {
+ case config.State == "":
+ // Force fresh SRP authentication - ignore stale trust token and cookies
+ // so that reconnect always prompts for 2FA
+ m.Set(configAuthSession, "")
+ icloud, err := api.New(appleid, password, "", clientID, nil, nil, "_config")
+ if err != nil {
+ return nil, err
+ }
+ if err := icloud.Authenticate(ctx); err != nil {
+ return nil, err
+ }
+ m.Set(configCookies, icloud.Session.GetCookieString())
+ if icloud.Session.Requires2FA() {
+ // Check if user has no trusted devices - auto-trigger SMS if so
+ authState, err := icloud.Session.GetAuthState(ctx)
+ if err == nil && authState.NoTrustedDevices && len(authState.TrustedPhoneNumbers) > 0 {
+ return triggerSMSFlow(ctx, icloud, authState.TrustedPhoneNumbers, m)
+ }
+ // Explicitly request push to trusted devices - required for iOS 26.4+
+ // where the SRP 409 no longer auto-pushes. GET /appleauth/auth above
+ // may also trigger a push (cosmetic double on pre-26.4, harmless)
+ if err := icloud.Session.RequestPushNotification(ctx); err != nil {
+ fs.Debugf(nil, "iclouddrive: push notification request failed (SMS fallback available): %v", err)
+ } else {
+ fs.Debugf(nil, "iclouddrive: push notification requested to trusted devices")
+ }
+ // Save session state so 2fa_do can validate without re-authenticating
+ // Push codes are account-scoped so session reuse is not strictly required,
+ // but it avoids a redundant SRP roundtrip and a second push on pre-26.4
+ saveAuthSession(m, icloud.Session, nil)
+ return fs.ConfigInput("2fa_do", "config_2fa", "Two-factor authentication: enter your 2FA code or type 'sms' for a text message")
+ }
+ // Auth succeeded without 2FA - save updated credentials and clear stale cache
+ m.Set(configTrustToken, icloud.Session.TrustToken)
+ api.ClearCacheDir(name)
+ return nil, nil
+
+ case config.State == "2fa_do":
+ code := config.Result
+ if code == "" {
+ return fs.ConfigError("authenticate", "2FA codes can't be blank")
+ }
+
+ // Restore session from initial sign-in instead of re-authenticating
+ // This avoids a redundant SRP roundtrip and extra push on pre-26.4
+ icloud, _, err := resumeConfigClient(m, appleid, password, trustToken, clientID, cookies)
+ if err != nil {
+ return nil, err
+ }
+
+ if strings.EqualFold(code, "sms") {
+ authState, err := icloud.Session.GetAuthState(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get trusted phone numbers: %w", err)
+ }
+ if len(authState.TrustedPhoneNumbers) == 0 {
+ return nil, errors.New("no trusted phone numbers on this account")
+ }
+ return triggerSMSFlow(ctx, icloud, authState.TrustedPhoneNumbers, m)
+ }
+
+ if err := icloud.Session.Validate2FACode(ctx, code); err != nil {
+ return nil, err
+ }
+ saveAuthCredentials(m, icloud, name)
+ return nil, nil
+
+ case config.State == "2fa_sms_select":
+ // User selected a phone from ConfigChooseExclusiveFixed (value: "ID_mode")
+ idStr, mode, _ := strings.Cut(config.Result, "_")
+ phoneID, err := strconv.Atoi(idStr)
+ if err != nil {
+ m.Set(configAuthSession, "")
+ return nil, fmt.Errorf("invalid phone selection %q", config.Result)
+ }
+ if mode == "" {
+ mode = "sms"
+ }
+ icloud, smsState, err := resumeConfigClient(m, appleid, password, trustToken, clientID, cookies)
+ if err != nil {
+ return nil, err
+ }
+ // Find the selected phone's display number
+ var phoneNum string
+ for _, p := range smsState.Phones {
+ if p.ID == phoneID {
+ phoneNum = p.Num
+ break
+ }
+ }
+
+ if err := icloud.Session.RequestSMSCode(ctx, phoneID, mode); err != nil {
+ m.Set(configAuthSession, "")
+ return nil, fmt.Errorf("failed to send SMS code: %w", err)
+ }
+ saveAuthSession(m, icloud.Session, nil)
+ nextState := fmt.Sprintf("2fa_sms_%d_%s", phoneID, mode)
+ return fs.ConfigInput(nextState, "config_2fa_sms", fmt.Sprintf("Enter the verification code sent to %s", phoneNum))
+
+ case strings.HasPrefix(config.State, "2fa_sms_"):
+ code := config.Result
+ if code == "" {
+ return fs.ConfigError("authenticate", "SMS code can't be blank")
+ }
+ // State encodes phone ID and mode: "2fa_sms_<ID>_<mode>"
+ suffix := strings.TrimPrefix(config.State, "2fa_sms_")
+ idStr, mode, _ := strings.Cut(suffix, "_")
+ phoneID, err := strconv.Atoi(idStr)
+ if err != nil {
+ return nil, fmt.Errorf("invalid phone ID in state %q: %w", config.State, err)
+ }
+ if mode == "" {
+ mode = "sms"
+ }
+
+ icloud, _, err := resumeConfigClient(m, appleid, password, trustToken, clientID, cookies)
+ if err != nil {
+ return nil, err
+ }
+
+ if err := icloud.Session.ValidateSMSCode(ctx, code, phoneID, mode); err != nil {
+ m.Set(configAuthSession, "")
+ return nil, err
+ }
+ saveAuthCredentials(m, icloud, name)
+ return nil, nil
+
+ default:
+ return nil, fmt.Errorf("unknown state %q", config.State)
+ }
+}
+
+// newICloudClient parses options, authenticates, and returns a ready client
+// Shared between NewFs (Drive) and NewFsPhotos (Photos) to avoid duplication
+func newICloudClient(ctx context.Context, name string, m configmap.Mapper) (*api.Client, *Options, error) {
+ opt := new(Options)
+ err := configstruct.Set(m, opt)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if opt.Password != "" {
+ opt.Password, err = obscure.Reveal(opt.Password)
+ if err != nil {
+ return nil, nil, fmt.Errorf("couldn't decrypt user password: %w", err)
+ }
+ }
+
+ if opt.TrustToken == "" {
+ return nil, nil, fmt.Errorf("missing icloud trust token: try refreshing it with \"rclone config reconnect %s:\"", name)
+ }
+
+ cookies := ReadCookies(opt.Cookies)
+
+ callback := func(session *api.Session) {
+ m.Set(configCookies, session.GetCookieString())
+ }
+
+ icloud, err := api.New(
+ opt.AppleID,
+ opt.Password,
+ opt.TrustToken,
+ opt.ClientID,
+ cookies,
+ callback,
+ name,
+ )
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if err := icloud.Authenticate(ctx); err != nil {
+ return nil, nil, err
+ }
+
+ if icloud.Session.Requires2FA() {
+ return nil, nil, errors.New("trust token expired, please reauth")
+ }
+
+ return icloud, opt, nil
+}
+
+// disconnectClient clears authentication state and removes disk caches
+// Shared between Drive Fs and Photos Fs Disconnect() implementations
+func disconnectClient(m configmap.Mapper, icloud *api.Client) error {
+ m.Set(configTrustToken, "")
+ m.Set(configCookies, "")
+ m.Set(configAuthSession, "")
+ return os.RemoveAll(icloud.CacheDir())
+}
+
+// NewServiceFs creates a filesystem instance for the selected service
+func NewServiceFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
+ // Parse the service selection
+ opt := new(ServiceOptions)
+ err := configstruct.Set(m, opt)
+ if err != nil {
+ return nil, err
+ }
+
+ // Set default service if not specified
+ if opt.Service == "" {
+ opt.Service = serviceDrive
+ }
+
+ // Route to the appropriate backend
+ switch opt.Service {
+ case serviceDrive:
+ // Create Drive filesystem
+ return NewFs(ctx, name, root, m)
+ case servicePhotos:
+ // Create Photos filesystem
+ return NewFsPhotos(ctx, name, root, m)
+ default:
+ return nil, fmt.Errorf("invalid service selection: %s (must be 'drive' or 'photos')", opt.Service)
+ }
+}
diff --git a/backend/iclouddrive/iclouddrive.go b/backend/iclouddrive/iclouddrive.go
index 1a51b9ab8..135ec75e9 100644
--- a/backend/iclouddrive/iclouddrive.go
+++ b/backend/iclouddrive/iclouddrive.go
@@ -9,17 +9,13 @@ import (
"path"
"errors"
- "fmt"
"io"
"net/http"
"strings"
"time"
"github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
- "github.com/rclone/rclone/fs/config/configstruct"
- "github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/backend/iclouddrive/api"
@@ -51,61 +47,10 @@ const (
decayConstant = 2
)
-// Register with Fs
-func init() {
- fs.Register(&fs.RegInfo{
- Name: "iclouddrive",
- Description: "iCloud Drive",
- Config: Config,
- NewFs: NewFs,
- Options: []fs.Option{{
- Name: configAppleID,
- Help: "Apple ID.",
- Required: true,
- Sensitive: true,
- }, {
- Name: configPassword,
- Help: "Password.",
- Required: true,
- IsPassword: true,
- Sensitive: true,
- }, {
- Name: configTrustToken,
- Help: "Trust token (internal use)",
- IsPassword: false,
- Required: false,
- Sensitive: true,
- Hide: fs.OptionHideBoth,
- }, {
- Name: configCookies,
- Help: "cookies (internal use only)",
- Required: false,
- Advanced: false,
- Sensitive: true,
- Hide: fs.OptionHideBoth,
- }, {
- Name: configClientID,
- Help: "Client id",
- Required: false,
- Advanced: true,
- Default: "d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d",
- }, {
- Name: config.ConfigEncoding,
- Help: config.ConfigEncodingHelp,
- Advanced: true,
- Default: (encoder.Display |
- //encoder.EncodeDot |
- encoder.EncodeBackSlash |
- encoder.EncodeInvalidUtf8),
- }},
- })
-}
-
// Options defines the configuration for this backend
type Options struct {
AppleID string `config:"apple_id"`
Password string `config:"password"`
- Photos bool `config:"photos"`
TrustToken string `config:"trust_token"`
Cookies string `config:"cookies"`
ClientID string `config:"client_id"`
@@ -118,6 +63,7 @@ type Fs struct {
root string // the path we are working on.
rootID string
opt Options // parsed config options
+ m configmap.Mapper // config map for persisting auth state
features *fs.Features // optional features
dirCache *dircache.DirCache // Map of directory path to directory id
icloud *api.Client
@@ -139,72 +85,6 @@ type Object struct {
downloadURL string
}
-// Config configures the iCloud remote.
-func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
- var err error
- appleid, _ := m.Get(configAppleID)
- if appleid == "" {
- return nil, errors.New("a apple ID is required")
- }
-
- password, _ := m.Get(configPassword)
- if password != "" {
- password, err = obscure.Reveal(password)
- if err != nil {
- return nil, err
- }
- }
-
- trustToken, _ := m.Get(configTrustToken)
- cookieRaw, _ := m.Get(configCookies)
- clientID, _ := m.Get(configClientID)
- cookies := ReadCookies(cookieRaw)
-
- switch config.State {
- case "":
- icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil)
- if err != nil {
- return nil, err
- }
- if err := icloud.Authenticate(ctx); err != nil {
- return nil, err
- }
- m.Set(configCookies, icloud.Session.GetCookieString())
- if icloud.Session.Requires2FA() {
- return fs.ConfigInput("2fa_do", "config_2fa", "Two-factor authentication: please enter your 2FA code")
- }
- return nil, nil
- case "2fa_do":
- code := config.Result
- if code == "" {
- return fs.ConfigError("authenticate", "2FA codes can't be blank")
- }
-
- icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil)
- if err != nil {
- return nil, err
- }
- if err := icloud.SignIn(ctx); err != nil {
- return nil, err
- }
-
- if err := icloud.Session.Validate2FACode(ctx, code); err != nil {
- return nil, err
- }
-
- m.Set(configTrustToken, icloud.Session.TrustToken)
- m.Set(configCookies, icloud.Session.GetCookieString())
- return nil, nil
-
- case "2fa_error":
- if config.Result == "true" {
- return fs.ConfigGoto("2fa")
- }
- return nil, errors.New("2fa authentication failed")
- }
- return nil, fmt.Errorf("unknown state %q", config.State)
-}
-
// find item by path. Will not return any children for the item
func (f *Fs) findItem(ctx context.Context, dir string) (item *api.DriveItem, found bool, err error) {
var resp *http.Response
@@ -797,51 +677,11 @@ func retryResultUnknown(ctx context.Context, resp *http.Response, err error) (bo
// NewFs constructs an Fs from the path, container:path
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
- // Parse config into Options struct
- opt := new(Options)
- err := configstruct.Set(m, opt)
+ icloud, opt, err := newICloudClient(ctx, name, m)
if err != nil {
return nil, err
}
- if opt.Password != "" {
- var err error
- opt.Password, err = obscure.Reveal(opt.Password)
- if err != nil {
- return nil, fmt.Errorf("couldn't decrypt user password: %w", err)
- }
- }
-
- if opt.TrustToken == "" {
- return nil, fmt.Errorf("missing icloud trust token: try refreshing it with \"rclone config reconnect %s:\"", name)
- }
-
- cookies := ReadCookies(opt.Cookies)
-
- callback := func(session *api.Session) {
- m.Set(configCookies, session.GetCookieString())
- }
-
- icloud, err := api.New(
- opt.AppleID,
- opt.Password,
- opt.TrustToken,
- opt.ClientID,
- cookies,
- callback,
- )
- if err != nil {
- return nil, err
- }
-
- if err := icloud.Authenticate(ctx); err != nil {
- return nil, err
- }
-
- if icloud.Session.Requires2FA() {
- return nil, errors.New("trust token expired, please reauth")
- }
-
root = strings.Trim(root, "/")
f := &Fs{
@@ -850,6 +690,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
icloud: icloud,
rootID: "FOLDER::com.apple.CloudDocs::root",
opt: *opt,
+ m: m,
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.features = (&fs.Features{
@@ -1164,9 +1005,15 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return nil
}
+// Disconnect clears authentication state and removes disk caches
+func (f *Fs) Disconnect(ctx context.Context) error {
+ return disconnectClient(f.m, f.icloud)
+}
+
// Check interfaces are satisfied
var (
_ fs.Fs = &Fs{}
+ _ fs.Disconnecter = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
diff --git a/backend/iclouddrive/icloudphotos.go b/backend/iclouddrive/icloudphotos.go
new file mode 100644
index 000000000..922333f98
--- /dev/null
+++ b/backend/iclouddrive/icloudphotos.go
@@ -0,0 +1,892 @@
+//go:build !plan9 && !solaris
+
+package iclouddrive
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "path"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/rclone/rclone/backend/iclouddrive/api"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config/configmap"
+ "github.com/rclone/rclone/fs/fshttp"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/list"
+ "github.com/rclone/rclone/lib/dircache"
+ "github.com/rclone/rclone/lib/pacer"
+)
+
+const rootID = "photos-root"
+
+// PhotosFs represents a remote iCloud Photos server
+type PhotosFs struct {
+ name string
+ root string
+ opt Options
+ features *fs.Features
+ icloud *api.Client
+ m configmap.Mapper
+ pacer *fs.Pacer
+ dirCache *dircache.DirCache
+ httpClient *http.Client
+ startTime time.Time
+ mu sync.Mutex
+ photos *api.PhotosService
+}
+
+// PhotosObject describes an iCloud Photos object
+type PhotosObject struct {
+ fs *PhotosFs
+ remote string
+ size int64
+ modTime time.Time
+ masterID string // CloudKit recordName for fresh URL lookup (master or asset depending on resource)
+ zone string // CloudKit zone (e.g. "PrimarySync")
+ resourceKey string // CloudKit resource field (resOriginalRes or resOriginalVidComplRes)
+ width int
+ height int
+ addedDate int64
+ isFavorite bool
+ isHidden bool
+}
+
+// NewFsPhotos constructs an Fs for Photos from the path, container:path
+func NewFsPhotos(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
+ icloud, opt, err := newICloudClient(ctx, name, m)
+ if err != nil {
+ return nil, err
+ }
+
+ root = strings.Trim(root, "/")
+
+ f := &PhotosFs{
+ name: name,
+ root: root,
+ opt: *opt,
+ icloud: icloud,
+ m: m,
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ httpClient: fshttp.NewClient(ctx),
+ startTime: time.Now(),
+ }
+
+ f.features = (&fs.Features{
+ CanHaveEmptyDirectories: true,
+ PartialUploads: false,
+ ReadMimeType: false,
+ ReadMetadata: true,
+ }).Fill(ctx, f)
+
+ f.dirCache = dircache.New(root, rootID, f)
+
+ err = f.dirCache.FindRoot(ctx, false)
+ if err != nil {
+ // Check if root points to a file (e.g. PrimarySync/Videos/file.mp4)
+ newRoot, remote := dircache.SplitPath(root)
+ tempF := &PhotosFs{
+ name: f.name,
+ root: newRoot,
+ opt: f.opt,
+ features: f.features,
+ icloud: f.icloud,
+ pacer: f.pacer,
+ httpClient: f.httpClient,
+ startTime: f.startTime,
+ photos: f.photos, // nil here; photosService() creates lazily on first use
+ }
+ tempF.dirCache = dircache.New(newRoot, rootID, tempF)
+ if err2 := tempF.dirCache.FindRoot(ctx, false); err2 != nil {
+ return f, nil
+ }
+ _, err2 := tempF.NewObject(ctx, remote)
+ if err2 != nil {
+ return f, nil
+ }
+ // Root is a file - adjust f to parent dir, signal ErrorIsFile
+ f.root = newRoot
+ f.dirCache = tempF.dirCache
+ return f, fs.ErrorIsFile
+ }
+
+ return f, nil
+}
+
+// photosService returns the PhotosService, creating it on first call
+func (f *PhotosFs) photosService(ctx context.Context) (*api.PhotosService, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if f.photos == nil {
+ var err error
+ f.photos, err = api.NewPhotosService(ctx, f.icloud, f.pacer, shouldRetry)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return f.photos, nil
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *PhotosFs) Name() string {
+ return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *PhotosFs) Root() string {
+ return f.opt.Enc.ToStandardPath(f.root)
+}
+
+// String converts this Fs to a string
+func (f *PhotosFs) String() string {
+ return fmt.Sprintf("iCloud Photos root '%s'", f.root)
+}
+
+// Precision of the object storage system
+func (f *PhotosFs) Precision() time.Duration {
+ return time.Second
+}
+
+// Hashes returns the supported hash sets
+func (f *PhotosFs) Hashes() hash.Set {
+ return hash.Set(hash.None)
+}
+
+// Features returns the optional features of this Fs
+func (f *PhotosFs) Features() *fs.Features {
+ return f.features
+}
+
+// List the objects and directories in dir into entries
+func (f *PhotosFs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+ photosService, err := f.photosService(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get photos service: %w", err)
+ }
+
+ dirID, err := f.dirCache.FindDir(ctx, dir, false)
+ if err != nil {
+ return nil, err
+ }
+
+ switch {
+ case dirID == rootID:
+ // List libraries
+ libraries, err := photosService.GetLibraries(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ albumCounts, err := photosService.GetLibraryAlbumCounts(ctx)
+ if err != nil {
+ fs.Debugf(f, "Failed to get library album counts: %v", err)
+ }
+
+ for libraryName := range libraries {
+ count := int64(-1)
+ if albumCounts != nil {
+ if c, ok := albumCounts[libraryName]; ok {
+ count = c
+ }
+ }
+ d := fs.NewDir(path.Join(dir, f.opt.Enc.ToStandardName(libraryName)), f.startTime).SetItems(count)
+ entries = append(entries, d)
+ }
+
+ case strings.HasPrefix(dirID, "lib:"):
+ // List albums in a library
+ libraryName := strings.TrimPrefix(dirID, "lib:")
+ libraries, err := photosService.GetLibraries(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ library, exists := libraries[libraryName]
+ if !exists {
+ return nil, fs.ErrorDirNotFound
+ }
+
+ albums, err := library.GetAlbums(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ counts, err := library.GetAlbumCounts(ctx)
+ if err != nil {
+ fs.Debugf(f, "Failed to get album counts for library %q: %v", libraryName, err)
+ }
+
+ for albumName := range albums {
+ count := int64(-1)
+ if counts != nil {
+ if c, ok := counts[albumName]; ok {
+ count = c
+ }
+ }
+ d := fs.NewDir(path.Join(dir, f.opt.Enc.ToStandardName(albumName)), f.startTime).SetItems(count)
+ entries = append(entries, d)
+ }
+
+ case strings.HasPrefix(dirID, "album:"):
+ // List contents of an album or folder
+ libraryName, albumPath, ok := parseAlbumDirID(dirID)
+ if !ok {
+ return nil, fs.ErrorDirNotFound
+ }
+
+ album, err := f.resolveAlbum(ctx, photosService, libraryName, albumPath)
+ if err != nil {
+ return nil, err
+ }
+
+ // Folders list their child albums as subdirectories
+ if album.IsFolder {
+ for childName, child := range album.Children {
+ d := fs.NewDir(path.Join(dir, f.opt.Enc.ToStandardName(childName)), f.startTime)
+ if child.IsFolder {
+ d.SetItems(int64(len(child.Children)))
+ }
+ entries = append(entries, d)
+ }
+ return entries, nil
+ }
+
+ photos, err := album.GetPhotos(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, photo := range photos {
+ if photo.Filename == "" {
+ continue
+ }
+ encodedName := f.opt.Enc.ToStandardName(photo.Filename)
+ remotePath := encodedName
+ if dir != "" {
+ remotePath = path.Join(dir, encodedName)
+ }
+ o := f.newPhotosObject(remotePath, photo, libraryName)
+ entries = append(entries, o)
+ }
+
+ default:
+ return nil, fs.ErrorDirNotFound
+ }
+
+ return entries, nil
+}
+
+// NewObject finds the Object at remote. If it can't be found
+// it returns the error fs.ErrorObjectNotFound
+func (f *PhotosFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+ dir, leaf := dircache.SplitPath(remote)
+ filename := f.opt.Enc.FromStandardName(leaf)
+
+ dirID, err := f.dirCache.FindDir(ctx, dir, false)
+ if err != nil {
+ fs.Debugf(f, "NewObject(%s): FindDir: %v", remote, err)
+ return nil, fs.ErrorObjectNotFound
+ }
+
+ if !strings.HasPrefix(dirID, "album:") {
+ return nil, fs.ErrorObjectNotFound
+ }
+
+ libraryName, albumPath, ok := parseAlbumDirID(dirID)
+ if !ok {
+ return nil, fs.ErrorObjectNotFound
+ }
+
+ photosService, err := f.photosService(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get photos service: %w", err)
+ }
+
+ album, err := f.resolveAlbum(ctx, photosService, libraryName, albumPath)
+ if err != nil || album.IsFolder {
+ fs.Debugf(f, "NewObject(%s): album resolve failed or is folder", remote)
+ return nil, fs.ErrorObjectNotFound
+ }
+
+ photo, err := album.GetPhotoByName(ctx, filename)
+ if err != nil {
+ fs.Debugf(f, "NewObject(%s): %v", remote, err)
+ return nil, fs.ErrorObjectNotFound
+ }
+
+ return f.newPhotosObject(remote, photo, libraryName), nil
+}
+
+// newPhotosObject creates a PhotosObject from a Photo and zone
+func (f *PhotosFs) newPhotosObject(remote string, photo *api.Photo, zone string) *PhotosObject {
+ return &PhotosObject{
+ fs: f,
+ remote: remote,
+ size: photo.Size,
+ modTime: time.UnixMilli(photo.AssetDate),
+ masterID: photo.ID,
+ zone: zone,
+ resourceKey: photo.ResourceKey,
+ width: photo.Width,
+ height: photo.Height,
+ addedDate: photo.AddedDate,
+ isFavorite: photo.IsFavorite,
+ isHidden: photo.IsHidden,
+ }
+}
+
+// Put is not supported for this read-only backend
+func (f *PhotosFs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+ return nil, fs.ErrorNotImplemented
+}
+
+// Mkdir is not supported for this read-only backend
+func (f *PhotosFs) Mkdir(ctx context.Context, dir string) error {
+ return fs.ErrorNotImplemented
+}
+
+// Rmdir is not supported for this read-only backend
+func (f *PhotosFs) Rmdir(ctx context.Context, dir string) error {
+ return fs.ErrorNotImplemented
+}
+
+// FindLeaf finds a directory of name leaf in the folder with ID pathID
+func (f *PhotosFs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
+ decodedLeaf := f.opt.Enc.FromStandardName(leaf)
+
+ photosService, err := f.photosService(ctx)
+ if err != nil {
+ return "", false, fmt.Errorf("failed to get photos service: %w", err)
+ }
+
+ if pathID == rootID {
+ libraries, err := photosService.GetLibraries(ctx)
+ if err != nil {
+ return "", false, err
+ }
+
+ if _, exists := libraries[decodedLeaf]; exists {
+ return "lib:" + decodedLeaf, true, nil
+ }
+ return "", false, nil
+ }
+
+ if strings.HasPrefix(pathID, "lib:") {
+ libraryName := strings.TrimPrefix(pathID, "lib:")
+ libraries, err := photosService.GetLibraries(ctx)
+ if err != nil {
+ return "", false, err
+ }
+
+ library, exists := libraries[libraryName]
+ if !exists {
+ return "", false, fs.ErrorDirNotFound
+ }
+
+ albums, err := library.GetAlbums(ctx)
+ if err != nil {
+ return "", false, err
+ }
+
+ if _, exists := albums[decodedLeaf]; exists {
+ albumID := "album:" + libraryName + ":" + decodedLeaf
+ return albumID, true, nil
+ }
+ return "", false, nil
+ }
+
+ if strings.HasPrefix(pathID, "album:") {
+ // Check if this is a folder containing child albums
+ libraryName, albumPath, ok := parseAlbumDirID(pathID)
+ if !ok {
+ return "", false, nil
+ }
+
+ album, err := f.resolveAlbum(ctx, photosService, libraryName, albumPath)
+ if err != nil {
+ return "", false, nil
+ }
+
+ if album.IsFolder {
+ if _, exists := album.Children[decodedLeaf]; exists {
+ childID := "album:" + libraryName + ":" + albumPath + "/" + decodedLeaf
+ return childID, true, nil
+ }
+ }
+ return "", false, nil
+ }
+
+ return "", false, nil
+}
+
+// CreateDir makes a directory with pathID as parent and name leaf
+func (f *PhotosFs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
+ return "", fs.ErrorNotImplemented
+}
+
+// Fs returns the parent Fs
+func (o *PhotosObject) Fs() fs.Info {
+ return o.fs
+}
+
+// Return a string version
+func (o *PhotosObject) String() string {
+ if o == nil {
+ return "<nil>"
+ }
+ return o.remote
+}
+
+// Remote returns the remote path
+func (o *PhotosObject) Remote() string {
+ return o.remote
+}
+
+// ModTime returns the modification time of the object
+func (o *PhotosObject) ModTime(ctx context.Context) time.Time {
+ return o.modTime
+}
+
+// Size returns the size of an object in bytes
+func (o *PhotosObject) Size() int64 {
+ return o.size
+}
+
+// Storable returns a boolean as to whether this object is storable
+func (o *PhotosObject) Storable() bool {
+ return true
+}
+
+// Hash returns the hash of an object returning a lowercase hex string
+func (o *PhotosObject) Hash(ctx context.Context, ty hash.Type) (string, error) {
+ return "", hash.ErrUnsupported
+}
+
+// SetModTime sets the modification time of the object
+func (o *PhotosObject) SetModTime(ctx context.Context, modTime time.Time) error {
+ return fs.ErrorCantSetModTime
+}
+
+// Metadata returns metadata for the photo object
+func (o *PhotosObject) Metadata(ctx context.Context) (fs.Metadata, error) {
+ metadata := make(fs.Metadata, 5)
+ if o.width > 0 {
+ metadata["width"] = strconv.Itoa(o.width)
+ }
+ if o.height > 0 {
+ metadata["height"] = strconv.Itoa(o.height)
+ }
+ if o.addedDate > 0 {
+ metadata["added-time"] = time.UnixMilli(o.addedDate).UTC().Format(time.RFC3339)
+ }
+ metadata["favorite"] = strconv.FormatBool(o.isFavorite)
+ metadata["hidden"] = strconv.FormatBool(o.isHidden)
+ return metadata, nil
+}
+
+// Open an object for read, fetching a fresh download URL via records/lookup
+func (o *PhotosObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+ photosService, err := o.fs.photosService(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get photos service: %w", err)
+ }
+
+ downloadURL, err := photosService.LookupDownloadURL(ctx, o.masterID, o.zone, o.resourceKey)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get download URL for %q: %w", o.remote, err)
+ }
+
+ fs.FixRangeOption(options, o.size)
+
+ var resp *http.Response
+ err = o.fs.pacer.Call(func() (bool, error) {
+ if resp != nil {
+ _ = resp.Body.Close()
+ resp = nil
+ }
+ req, err := http.NewRequestWithContext(ctx, "GET", downloadURL, nil)
+ if err != nil {
+ return false, fmt.Errorf("failed to create request: %w", err)
+ }
+ fs.OpenOptionAddHTTPHeaders(req.Header, options)
+
+ resp, err = o.fs.httpClient.Do(req)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
+ _ = resp.Body.Close()
+ return nil, fmt.Errorf("download %q failed: %s", o.remote, resp.Status)
+ }
+
+ return resp.Body, nil
+}
+
+// Update is not supported for this read-only backend
+func (o *PhotosObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+ return fs.ErrorNotImplemented
+}
+
+// Remove is not supported for this read-only backend
+func (o *PhotosObject) Remove(ctx context.Context) error {
+ return fs.ErrorNotImplemented
+}
+
+// resolveAlbum finds an album by path, traversing folder hierarchy
+// albumPath can be "AlbumName" or "FolderName/AlbumName" for nested albums
+func (f *PhotosFs) resolveAlbum(ctx context.Context, ps *api.PhotosService, libraryName, albumPath string) (*api.Album, error) {
+ libraries, err := ps.GetLibraries(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ library, exists := libraries[libraryName]
+ if !exists {
+ return nil, fs.ErrorDirNotFound
+ }
+
+ albums, err := library.GetAlbums(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ return resolveAlbumPath(albums, albumPath)
+}
+
+// resolveAlbumPath finds an album inside an album tree by slash-separated path
+func resolveAlbumPath(albums map[string]*api.Album, albumPath string) (*api.Album, error) {
+ parts := strings.Split(albumPath, "/")
+ current := albums
+ for i, part := range parts {
+ album, exists := current[part]
+ if !exists {
+ return nil, fs.ErrorDirNotFound
+ }
+ if i == len(parts)-1 {
+ return album, nil
+ }
+ if !album.IsFolder || album.Children == nil {
+ return nil, fs.ErrorDirNotFound
+ }
+ current = album.Children
+ }
+ return nil, fs.ErrorDirNotFound
+}
+
+// parseAlbumDirID extracts library and album names from "album:lib:album" dirID
+func parseAlbumDirID(dirID string) (libraryName, albumName string, ok bool) {
+ parts := strings.SplitN(strings.TrimPrefix(dirID, "album:"), ":", 2)
+ if len(parts) != 2 {
+ return "", "", false
+ }
+ return parts[0], parts[1], true
+}
+
+// DirCacheFlush resets the directory cache - used in testing as an
+// optional interface
+func (f *PhotosFs) DirCacheFlush() {
+ f.dirCache.ResetRoot()
+ // Also flush API-layer caches so albums/photos are re-fetched
+ // DirCacheFlusher interface has no ctx param; use a bounded timeout
+ // to avoid hanging indefinitely if photosService triggers first init
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ if ps, err := f.photosService(ctx); err == nil {
+ ps.FlushCaches()
+ }
+}
+
+// listRAlbumJob represents an album to be listed by a ListR worker
+type listRAlbumJob struct {
+ album *api.Album
+ zone string
+ dirPath string // path relative to Fs root (e.g. "Videos" when f.root="PrimarySync")
+}
+
+// ListR lists the objects and directories of the Fs starting from dir
+// recursively into out, calling callback for each batch of entries
+// Albums are listed in parallel using a goroutine pool
+func (f *PhotosFs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) error {
+ ps, err := f.photosService(ctx)
+ if err != nil {
+ return err
+ }
+
+ helper := list.NewHelper(callback)
+ var mu sync.Mutex // protects helper (not thread-safe)
+
+ addEntry := func(entry fs.DirEntry) error {
+ mu.Lock()
+ defer mu.Unlock()
+ return helper.Add(entry)
+ }
+
+ // Resolve dir through dirCache - this respects f.root
+ dirID, err := f.dirCache.FindDir(ctx, dir, false)
+ if err != nil {
+ return err
+ }
+
+ // Collect album jobs, scoped to the resolved directory
+ var jobs []listRAlbumJob
+
+ // collectAlbumJobs emits directory entries for albums/folders under basePath
+ // and collects leaf albums as jobs for parallel photo listing
+ // Recurses into folders to handle arbitrary nesting depth
+ var collectAlbumJobs func(albums map[string]*api.Album, zone, basePath string) error
+ collectAlbumJobs = func(albums map[string]*api.Album, zone, basePath string) error {
+ for albumName, album := range albums {
+ albumPath := path.Join(basePath, f.opt.Enc.ToStandardName(albumName))
+ if err := addEntry(fs.NewDir(albumPath, f.startTime)); err != nil {
+ return err
+ }
+ if album.IsFolder {
+ if err := collectAlbumJobs(album.Children, zone, albumPath); err != nil {
+ return err
+ }
+ } else {
+ jobs = append(jobs, listRAlbumJob{album: album, zone: zone, dirPath: albumPath})
+ }
+ }
+ return nil
+ }
+
+ switch {
+ case dirID == rootID:
+ // Root level - emit libraries, then all albums
+ libraries, err := ps.GetLibraries(ctx)
+ if err != nil {
+ return err
+ }
+ for libName, lib := range libraries {
+ libPath := path.Join(dir, f.opt.Enc.ToStandardName(libName))
+ if err := addEntry(fs.NewDir(libPath, f.startTime)); err != nil {
+ return err
+ }
+ albums, err := lib.GetAlbums(ctx)
+ if err != nil {
+ return err
+ }
+ if err := collectAlbumJobs(albums, libName, libPath); err != nil {
+ return err
+ }
+ }
+
+ case strings.HasPrefix(dirID, "lib:"):
+ // Library level - emit and collect all albums
+ libraryName := strings.TrimPrefix(dirID, "lib:")
+ libraries, err := ps.GetLibraries(ctx)
+ if err != nil {
+ return err
+ }
+ library, exists := libraries[libraryName]
+ if !exists {
+ return fs.ErrorDirNotFound
+ }
+ albums, err := library.GetAlbums(ctx)
+ if err != nil {
+ return err
+ }
+ if err := collectAlbumJobs(albums, libraryName, dir); err != nil {
+ return err
+ }
+
+ case strings.HasPrefix(dirID, "album:"):
+ // Album or folder level
+ libraryName, albumPath, ok := parseAlbumDirID(dirID)
+ if !ok {
+ return fs.ErrorDirNotFound
+ }
+ album, err := f.resolveAlbum(ctx, ps, libraryName, albumPath)
+ if err != nil {
+ return err
+ }
+ if album.IsFolder {
+ if err := collectAlbumJobs(album.Children, libraryName, dir); err != nil {
+ return err
+ }
+ } else {
+ // Single album - just list its photos
+ jobs = append(jobs, listRAlbumJob{album: album, zone: libraryName, dirPath: dir})
+ }
+
+ default:
+ return fs.ErrorDirNotFound
+ }
+
+ if len(jobs) == 0 {
+ return helper.Flush()
+ }
+
+ // Fan out album photo listing across goroutines
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ jobCh := make(chan listRAlbumJob, len(jobs))
+ for _, job := range jobs {
+ jobCh <- job
+ }
+ close(jobCh)
+
+ workers := fs.GetConfig(ctx).Checkers
+ if len(jobs) < workers {
+ workers = len(jobs)
+ }
+ errs := make(chan error, workers)
+
+ for range workers {
+ go func() {
+ for job := range jobCh {
+ photos, err := job.album.GetPhotos(ctx)
+ if err != nil {
+ errs <- err
+ cancel()
+ return
+ }
+ for _, photo := range photos {
+ if photo.Filename == "" {
+ continue
+ }
+ remotePath := path.Join(job.dirPath, f.opt.Enc.ToStandardName(photo.Filename))
+ o := f.newPhotosObject(remotePath, photo, job.zone)
+ if err := addEntry(o); err != nil {
+ errs <- err
+ cancel()
+ return
+ }
+ }
+ }
+ errs <- nil
+ }()
+ }
+
+ for range workers {
+ if e := <-errs; e != nil && err == nil {
+ err = e
+ }
+ }
+ if err != nil {
+ return err
+ }
+
+ return helper.Flush()
+}
+
+func notifyAlbumTree(albums map[string]*api.Album, base string, notifyFunc func(string, fs.EntryType)) {
+ for name, album := range albums {
+ albumPath := name
+ if base != "" {
+ albumPath = path.Join(base, name)
+ }
+ notifyFunc(albumPath, fs.EntryDirectory)
+ if album.IsFolder {
+ notifyAlbumTree(album.Children, albumPath, notifyFunc)
+ }
+ }
+}
+
+func (f *PhotosFs) notifyZoneChange(ctx context.Context, ps *api.PhotosService, zone string, notifyFunc func(string, fs.EntryType)) {
+ if f.root == "" {
+ notifyFunc(zone, fs.EntryDirectory)
+ return
+ }
+
+ libraryName, albumPath, _ := strings.Cut(f.root, "/")
+ if libraryName != zone {
+ return
+ }
+
+ // Always invalidate the mounted root when its backing zone changes
+ notifyFunc("", fs.EntryDirectory)
+
+ libraries, err := ps.GetLibraries(ctx)
+ if err != nil {
+ return
+ }
+ library, ok := libraries[zone]
+ if !ok {
+ return
+ }
+ albums, err := library.GetAlbums(ctx)
+ if err != nil {
+ return
+ }
+ if albumPath == "" {
+ notifyAlbumTree(albums, "", notifyFunc)
+ return
+ }
+
+ album, err := resolveAlbumPath(albums, albumPath)
+ if err != nil {
+ return
+ }
+ if album.IsFolder {
+ notifyAlbumTree(album.Children, "", notifyFunc)
+ }
+}
+
+// ChangeNotify polls for changes and notifies the VFS when directories are modified
+func (f *PhotosFs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
+ go func() {
+ var ticker *time.Ticker
+ var tickerC <-chan time.Time
+ for {
+ select {
+ case pollInterval, ok := <-pollIntervalChan:
+ if !ok {
+ if ticker != nil {
+ ticker.Stop()
+ }
+ return
+ }
+ if ticker != nil {
+ ticker.Stop()
+ tickerC = nil
+ }
+ if pollInterval > 0 {
+ ticker = time.NewTicker(pollInterval)
+ tickerC = ticker.C
+ }
+ case <-tickerC:
+ ps, err := f.photosService(ctx)
+ if err != nil {
+ fs.Debugf(f, "ChangeNotify: failed to get photos service: %v", err)
+ continue
+ }
+ changedZones := ps.PollForChanges(ctx)
+ for _, zone := range changedZones {
+ f.notifyZoneChange(ctx, ps, zone, notifyFunc)
+ }
+ case <-ctx.Done():
+ if ticker != nil {
+ ticker.Stop()
+ }
+ return
+ }
+ }
+ }()
+}
+
+// Disconnect clears authentication state and removes disk caches
+func (f *PhotosFs) Disconnect(ctx context.Context) error {
+ return disconnectClient(f.m, f.icloud)
+}
+
+// Check interfaces are satisfied
+var (
+ _ fs.Fs = (*PhotosFs)(nil)
+ _ fs.Disconnecter = (*PhotosFs)(nil)
+ _ fs.ListRer = (*PhotosFs)(nil)
+ _ fs.ChangeNotifier = (*PhotosFs)(nil)
+ _ fs.DirCacheFlusher = (*PhotosFs)(nil)
+ _ fs.Object = (*PhotosObject)(nil)
+ _ fs.Metadataer = (*PhotosObject)(nil)
+)
diff --git a/backend/iclouddrive/icloudphotos_test.go b/backend/iclouddrive/icloudphotos_test.go
new file mode 100644
index 000000000..1aa32146d
--- /dev/null
+++ b/backend/iclouddrive/icloudphotos_test.go
@@ -0,0 +1,516 @@
+//go:build !plan9 && !solaris
+
+package iclouddrive
+
+import (
+ "context"
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/backend/iclouddrive/api"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/lib/dircache"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func newTestPhotosFs() *PhotosFs {
+ return &PhotosFs{
+ name: "test-icp",
+ root: "",
+ opt: Options{},
+ }
+}
+
+func TestSmartAlbumDefinitions(t *testing.T) {
+ expected := map[string]struct {
+ ObjectType string
+ ListType string
+ }{
+ "All Photos": {"CPLAssetByAssetDateWithoutHiddenOrDeleted", "CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted"},
+ "Time-lapse": {"CPLAssetInSmartAlbumByAssetDate:Timelapse", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
+ "Videos": {"CPLAssetInSmartAlbumByAssetDate:Video", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
+ "Slo-mo": {"CPLAssetInSmartAlbumByAssetDate:Slomo", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
+ "Bursts": {"CPLAssetBurstStackAssetByAssetDate", "CPLBurstStackAssetAndMasterByAssetDate"},
+ "Favorites": {"CPLAssetInSmartAlbumByAssetDate:Favorite", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
+ "Panoramas": {"CPLAssetInSmartAlbumByAssetDate:Panorama", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
+ "Screenshots": {"CPLAssetInSmartAlbumByAssetDate:Screenshot", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
+ "Live": {"CPLAssetInSmartAlbumByAssetDate:Live", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
+ "Recently Deleted": {"CPLAssetDeletedByExpungedDate", "CPLAssetAndMasterDeletedByExpungedDate"},
+ "Hidden": {"CPLAssetHiddenByAssetDate", "CPLAssetAndMasterHiddenByAssetDate"},
+ "Portrait": {"CPLAssetInSmartAlbumByAssetDate:Depth", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
+ "Long Exposure": {"CPLAssetInSmartAlbumByAssetDate:Exposure", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
+ "Animated": {"CPLAssetInSmartAlbumByAssetDate:Animated", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
+ }
+
+ for name, exp := range expected {
+ t.Run(name, func(t *testing.T) {
+ album, exists := api.SmartAlbums[name]
+ require.True(t, exists, "smart album %q must be defined", name)
+ assert.Equal(t, exp.ObjectType, album.ObjectType)
+ assert.Equal(t, exp.ListType, album.ListType)
+ })
+ }
+
+ assert.Equal(t, len(expected), len(api.SmartAlbums))
+}
+
+func TestSmartAlbumFilters(t *testing.T) {
+ filtered := map[string]string{
+ "Time-lapse": "TIMELAPSE",
+ "Videos": "VIDEO",
+ "Slo-mo": "SLOMO",
+ "Favorites": "FAVORITE",
+ "Panoramas": "PANORAMA",
+ "Screenshots": "SCREENSHOT",
+ "Live": "LIVE",
+ "Portrait": "DEPTH",
+ "Long Exposure": "EXPOSURE",
+ "Animated": "ANIMATED",
+ }
+
+ for name, filterVal := range filtered {
+ t.Run(name, func(t *testing.T) {
+ album := api.SmartAlbums[name]
+ require.NotEmpty(t, album.Filters, "album %q must have filters", name)
+ assert.Equal(t, "smartAlbum", album.Filters[0].FieldName)
+ assert.Equal(t, "EQUALS", album.Filters[0].Comparator)
+ fv, ok := album.Filters[0].FieldValue.(map[string]string)
+ require.True(t, ok, "filter value must be map[string]string")
+ assert.Equal(t, filterVal, fv["value"])
+ })
+ }
+
+ unfiltered := []string{"All Photos", "Bursts", "Recently Deleted", "Hidden"}
+ for _, name := range unfiltered {
+ t.Run(name+"_no_filter", func(t *testing.T) {
+ album := api.SmartAlbums[name]
+ assert.Empty(t, album.Filters, "album %q must not have filters", name)
+ })
+ }
+}
+
+func TestParseAlbumDirID(t *testing.T) {
+ lib, album, ok := parseAlbumDirID("album:PrimarySync:Favorites")
+ assert.True(t, ok)
+ assert.Equal(t, "PrimarySync", lib)
+ assert.Equal(t, "Favorites", album)
+
+ // Colon in album name
+ lib, album, ok = parseAlbumDirID("album:PrimarySync:My:Album")
+ assert.True(t, ok)
+ assert.Equal(t, "PrimarySync", lib)
+ assert.Equal(t, "My:Album", album)
+
+ // Missing album
+ _, _, ok = parseAlbumDirID("album:PrimarySync")
+ assert.False(t, ok)
+
+ // Empty
+ _, _, ok = parseAlbumDirID("album:")
+ assert.False(t, ok)
+}
+
+func TestPhotosObject_Metadata(t *testing.T) {
+ o := &PhotosObject{
+ fs: &PhotosFs{name: "test"},
+ remote: "PrimarySync/All Photos/IMG_0001.HEIC",
+ size: 4200000,
+ modTime: time.Date(2025, 6, 15, 14, 30, 0, 0, time.UTC),
+ masterID: "master-001",
+ zone: "PrimarySync",
+ width: 4032,
+ height: 3024,
+ addedDate: 1718459400000, // 2024-06-15T13:50:00Z in millis
+ isFavorite: true,
+ isHidden: false,
+ }
+
+ metadata, err := o.Metadata(context.Background())
+ require.NoError(t, err)
+ assert.Equal(t, "4032", metadata["width"])
+ assert.Equal(t, "3024", metadata["height"])
+ assert.Equal(t, "2024-06-15T13:50:00Z", metadata["added-time"])
+ assert.Equal(t, "true", metadata["favorite"])
+ assert.Equal(t, "false", metadata["hidden"])
+}
+
+func TestPhotosObject_MetadataZeroDimensions(t *testing.T) {
+ // Live Photo .MOV companion - no width/height, not favorite
+ o := &PhotosObject{
+ fs: &PhotosFs{name: "test"},
+ remote: "PrimarySync/Live/IMG_3031.MOV",
+ }
+
+ metadata, err := o.Metadata(context.Background())
+ require.NoError(t, err)
+ _, hasWidth := metadata["width"]
+ _, hasHeight := metadata["height"]
+ assert.False(t, hasWidth, "zero width should be omitted")
+ assert.False(t, hasHeight, "zero height should be omitted")
+ assert.Equal(t, "false", metadata["favorite"])
+ assert.Equal(t, "false", metadata["hidden"])
+}
+
+// newTestPhotosService builds a PhotosService with pre-populated libraries and albums
+// for testing resolveAlbum and FindLeaf without HTTP calls
+func newTestPhotosService() *api.PhotosService {
+ return api.NewTestPhotosService(map[string]map[string]*api.Album{
+ "PrimarySync": {
+ "All Photos": {Name: "All Photos", ObjectType: "CPLAssetByAssetDateWithoutHiddenOrDeleted"},
+ "Videos": {Name: "Videos", ObjectType: "CPLAssetInSmartAlbumByAssetDate:Video"},
+ "UserAlbum": {Name: "UserAlbum", ObjectType: "CPLContainerRelationNotDeletedByAssetDate:rec1", RecordName: "rec1"},
+ "Folder": {
+ Name: "Folder", RecordName: "folder1", IsFolder: true,
+ Children: map[string]*api.Album{
+ "ChildAlbum": {Name: "ChildAlbum", ObjectType: "CPLContainerRelationNotDeletedByAssetDate:rec2", RecordName: "rec2"},
+ "NestedFolder": {
+ Name: "NestedFolder", RecordName: "folder2", IsFolder: true,
+ Children: map[string]*api.Album{
+ "LeafAlbum": {Name: "LeafAlbum", ObjectType: "CPLContainerRelationNotDeletedByAssetDate:rec3", RecordName: "rec3"},
+ },
+ },
+ },
+ },
+ },
+ })
+}
+
+func TestResolveAlbum(t *testing.T) {
+ f := newTestPhotosFs()
+ f.photos = newTestPhotosService()
+ ctx := context.Background()
+
+ t.Run("simple album", func(t *testing.T) {
+ album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Videos")
+ require.NoError(t, err)
+ assert.Equal(t, "Videos", album.Name)
+ })
+
+ t.Run("nested folder child", func(t *testing.T) {
+ album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder/ChildAlbum")
+ require.NoError(t, err)
+ assert.Equal(t, "ChildAlbum", album.Name)
+ })
+
+ t.Run("two-level nesting", func(t *testing.T) {
+ album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder/NestedFolder/LeafAlbum")
+ require.NoError(t, err)
+ assert.Equal(t, "LeafAlbum", album.Name)
+ })
+
+ t.Run("folder itself", func(t *testing.T) {
+ album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder")
+ require.NoError(t, err)
+ assert.True(t, album.IsFolder)
+ })
+
+ t.Run("missing album", func(t *testing.T) {
+ _, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "NoSuchAlbum")
+ assert.ErrorIs(t, err, fs.ErrorDirNotFound)
+ })
+
+ t.Run("missing library", func(t *testing.T) {
+ _, err := f.resolveAlbum(ctx, f.photos, "NoSuchLib", "Videos")
+ assert.ErrorIs(t, err, fs.ErrorDirNotFound)
+ })
+
+ t.Run("traverse into non-folder", func(t *testing.T) {
+ _, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Videos/SubAlbum")
+ assert.ErrorIs(t, err, fs.ErrorDirNotFound)
+ })
+}
+
+func TestFindLeaf(t *testing.T) {
+ f := newTestPhotosFs()
+ f.photos = newTestPhotosService()
+ ctx := context.Background()
+
+ t.Run("root to library", func(t *testing.T) {
+ id, found, err := f.FindLeaf(ctx, rootID, "PrimarySync")
+ require.NoError(t, err)
+ assert.True(t, found)
+ assert.Equal(t, "lib:PrimarySync", id)
+ })
+
+ t.Run("root to missing library", func(t *testing.T) {
+ _, found, err := f.FindLeaf(ctx, rootID, "NoSuchLib")
+ require.NoError(t, err)
+ assert.False(t, found)
+ })
+
+ t.Run("library to album", func(t *testing.T) {
+ id, found, err := f.FindLeaf(ctx, "lib:PrimarySync", "Videos")
+ require.NoError(t, err)
+ assert.True(t, found)
+ assert.Equal(t, "album:PrimarySync:Videos", id)
+ })
+
+ t.Run("library to missing album", func(t *testing.T) {
+ _, found, err := f.FindLeaf(ctx, "lib:PrimarySync", "NoSuchAlbum")
+ require.NoError(t, err)
+ assert.False(t, found)
+ })
+
+ t.Run("missing library returns ErrorDirNotFound", func(t *testing.T) {
+ _, _, err := f.FindLeaf(ctx, "lib:NoSuchLib", "Videos")
+ assert.ErrorIs(t, err, fs.ErrorDirNotFound)
+ })
+
+ t.Run("folder to child album", func(t *testing.T) {
+ id, found, err := f.FindLeaf(ctx, "album:PrimarySync:Folder", "ChildAlbum")
+ require.NoError(t, err)
+ assert.True(t, found)
+ assert.Equal(t, "album:PrimarySync:Folder/ChildAlbum", id)
+ })
+
+ t.Run("non-folder album returns not found", func(t *testing.T) {
+ _, found, err := f.FindLeaf(ctx, "album:PrimarySync:Videos", "SubAlbum")
+ require.NoError(t, err)
+ assert.False(t, found)
+ })
+
+ t.Run("unknown prefix returns not found", func(t *testing.T) {
+ _, found, err := f.FindLeaf(ctx, "unknown:prefix", "leaf")
+ require.NoError(t, err)
+ assert.False(t, found)
+ })
+}
+
+func TestResolveAlbum_EmptyPath(t *testing.T) {
+ f := newTestPhotosFs()
+ f.photos = newTestPhotosService()
+ ctx := context.Background()
+
+ // parseAlbumDirID("album:PrimarySync:") returns ("PrimarySync", "", true)
+ // This empty album path feeds into resolveAlbum
+ _, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "")
+ assert.ErrorIs(t, err, fs.ErrorDirNotFound, "empty album path should return ErrorDirNotFound")
+}
+
+func TestNewObject_ErrorPaths(t *testing.T) {
+ f := newTestPhotosFs()
+ f.photos = newTestPhotosService()
+ ctx := context.Background()
+
+ // Pre-populate photoCache on the "Videos" album so GetPhotoByName works
+ // without HTTP. Access via the test PhotosService's internal structure
+ libs, _ := f.photos.GetLibraries(ctx)
+ albums, _ := libs["PrimarySync"].GetAlbums(ctx)
+ videos := albums["Videos"]
+ videos.SetTestPhotoCache(map[string]*api.Photo{
+ "existing.mp4": {ID: "m1", Filename: "existing.mp4", Size: 1024, ResourceKey: "resOriginalRes"},
+ })
+
+ // NewObject depends on dircache.FindDir which requires FindRoot + HTTP
+ // Test the reachable paths through resolveAlbum and GetPhotoByName
+
+ // Test resolveAlbum → folder → GetPhotoByName on folder (should fail)
+ album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder")
+ require.NoError(t, err)
+ assert.True(t, album.IsFolder, "Folder should be a folder")
+ // NewObject would return ErrorObjectNotFound for a folder path
+
+ // Test GetPhotoByName on album with populated cache
+ photo, err := videos.GetPhotoByName(ctx, "existing.mp4")
+ require.NoError(t, err)
+ assert.Equal(t, "m1", photo.ID)
+
+ // Test GetPhotoByName cache miss
+ _, err = videos.GetPhotoByName(ctx, "nonexistent.mp4")
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "nonexistent.mp4")
+}
+
+func TestParseAlbumDirID_Exhaustive(t *testing.T) {
+ tests := []struct {
+ input string
+ lib, album string
+ ok bool
+ }{
+ {"album:PrimarySync:Videos", "PrimarySync", "Videos", true},
+ {"album:PrimarySync:Folder/Child", "PrimarySync", "Folder/Child", true},
+ {"album:PrimarySync:Name:With:Colons", "PrimarySync", "Name:With:Colons", true},
+ {"album:PrimarySync:", "PrimarySync", "", true},
+ {"album:", "", "", false},
+ {"notalbum:foo:bar", "notalbum", "foo:bar", true}, // strips "album:" prefix literally, "notalbum:" stays
+ {"", "", "", false},
+ }
+
+ for _, tt := range tests {
+ lib, album, ok := parseAlbumDirID(tt.input)
+ if tt.ok {
+ assert.True(t, ok, "input=%q", tt.input)
+ assert.Equal(t, tt.lib, lib, "input=%q", tt.input)
+ assert.Equal(t, tt.album, album, "input=%q", tt.input)
+ } else {
+ assert.False(t, ok, "input=%q", tt.input)
+ }
+ }
+}
+
+// setEmptyPhotoCaches recursively sets empty photo caches on all leaf albums
+// so GetPhotos returns without HTTP (service=nil test fast path)
+func setEmptyPhotoCaches(albums map[string]*api.Album) {
+ for _, album := range albums {
+ if album.IsFolder {
+ setEmptyPhotoCaches(album.Children)
+ } else {
+ album.SetTestPhotoCache(map[string]*api.Photo{})
+ }
+ }
+}
+
+func TestListR_NestedFolderRecursion(t *testing.T) {
+ f := newTestPhotosFs()
+ f.photos = newTestPhotosService()
+ f.dirCache = dircache.New("", rootID, f)
+ f.features = (&fs.Features{}).Fill(context.Background(), f)
+ f.startTime = time.Now()
+ ctx := context.Background()
+
+ err := f.dirCache.FindRoot(ctx, false)
+ require.NoError(t, err)
+
+ // Pre-populate empty photo caches on all leaf albums
+ libs, err := f.photos.GetLibraries(ctx)
+ require.NoError(t, err)
+ for _, lib := range libs {
+ albums, err := lib.GetAlbums(ctx)
+ require.NoError(t, err)
+ setEmptyPhotoCaches(albums)
+ }
+
+ var dirs []string
+ err = f.ListR(ctx, "", func(entries fs.DirEntries) error {
+ for _, entry := range entries {
+ if _, ok := entry.(fs.Directory); ok {
+ dirs = append(dirs, entry.Remote())
+ }
+ }
+ return nil
+ })
+ require.NoError(t, err)
+ sort.Strings(dirs)
+
+ // All directories must be present, including deeply nested ones
+ assert.Contains(t, dirs, "PrimarySync")
+ assert.Contains(t, dirs, "PrimarySync/All Photos")
+ assert.Contains(t, dirs, "PrimarySync/Videos")
+ assert.Contains(t, dirs, "PrimarySync/Folder")
+ assert.Contains(t, dirs, "PrimarySync/Folder/ChildAlbum")
+ assert.Contains(t, dirs, "PrimarySync/Folder/NestedFolder")
+ assert.Contains(t, dirs, "PrimarySync/Folder/NestedFolder/LeafAlbum")
+}
+
+func TestListR_FolderLevel(t *testing.T) {
+ f := &PhotosFs{
+ name: "test",
+ root: "PrimarySync/Folder",
+ opt: Options{},
+ startTime: time.Now(),
+ }
+ f.features = (&fs.Features{}).Fill(context.Background(), f)
+ f.photos = newTestPhotosService()
+ f.dirCache = dircache.New("PrimarySync/Folder", rootID, f)
+ ctx := context.Background()
+
+ err := f.dirCache.FindRoot(ctx, false)
+ require.NoError(t, err)
+
+ libs, _ := f.photos.GetLibraries(ctx)
+ for _, lib := range libs {
+ albums, _ := lib.GetAlbums(ctx)
+ setEmptyPhotoCaches(albums)
+ }
+
+ var dirs []string
+ err = f.ListR(ctx, "", func(entries fs.DirEntries) error {
+ for _, entry := range entries {
+ if _, ok := entry.(fs.Directory); ok {
+ dirs = append(dirs, entry.Remote())
+ }
+ }
+ return nil
+ })
+ require.NoError(t, err)
+
+ // When rooted at a folder, should see children recursively
+ assert.Contains(t, dirs, "ChildAlbum")
+ assert.Contains(t, dirs, "NestedFolder")
+ assert.Contains(t, dirs, "NestedFolder/LeafAlbum")
+}
+
+func TestNotifyZoneChange(t *testing.T) {
+ ctx := context.Background()
+ ps := newTestPhotosService()
+
+ tests := []struct {
+ name string
+ root string
+ zone string
+ want []string
+ }{
+ {
+ name: "top level root",
+ root: "",
+ zone: "PrimarySync",
+ want: []string{"PrimarySync"},
+ },
+ {
+ name: "library root",
+ root: "PrimarySync",
+ zone: "PrimarySync",
+ want: []string{"", "All Photos", "Folder", "Folder/ChildAlbum", "Folder/NestedFolder", "Folder/NestedFolder/LeafAlbum", "UserAlbum", "Videos"},
+ },
+ {
+ name: "folder root",
+ root: "PrimarySync/Folder",
+ zone: "PrimarySync",
+ want: []string{"", "ChildAlbum", "NestedFolder", "NestedFolder/LeafAlbum"},
+ },
+ {
+ name: "nested folder root",
+ root: "PrimarySync/Folder/NestedFolder",
+ zone: "PrimarySync",
+ want: []string{"", "LeafAlbum"},
+ },
+ {
+ name: "leaf album root",
+ root: "PrimarySync/Folder/NestedFolder/LeafAlbum",
+ zone: "PrimarySync",
+ want: []string{""},
+ },
+ {
+ name: "missing nested album still invalidates root",
+ root: "PrimarySync/Folder/NoSuchAlbum",
+ zone: "PrimarySync",
+ want: []string{""},
+ },
+ {
+ name: "different zone ignored",
+ root: "PrimarySync/Folder/NestedFolder",
+ zone: "SharedSync",
+ want: nil,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ f := newTestPhotosFs()
+ f.root = tt.root
+
+ var got []string
+ f.notifyZoneChange(ctx, ps, tt.zone, func(remote string, entryType fs.EntryType) {
+ assert.Equal(t, fs.EntryDirectory, entryType)
+ got = append(got, remote)
+ })
+
+ sort.Strings(got)
+ want := append([]string(nil), tt.want...)
+ sort.Strings(want)
+ assert.Equal(t, want, got)
+ })
+ }
+}
diff --git a/docs/content/iclouddrive.md b/docs/content/iclouddrive.md
index af18796fd..22c488757 100644
--- a/docs/content/iclouddrive.md
+++ b/docs/content/iclouddrive.md
@@ -1,20 +1,25 @@
---
-title: "iCloud Drive"
-description: "Rclone docs for iCloud Drive"
+title: "iCloud Drive and Photos"
+description: "Rclone docs for iCloud Drive and Photos"
versionIntroduced: "v1.69"
---
-# iCloud Drive
+# {{< icon "fa fa-cloud" >}} iCloud Drive and Photos
## Configuration
-The initial setup for an iCloud Drive backend involves getting a trust token/session.
-This can be done by simply using the regular iCloud password, and accepting the code
-prompt on another iCloud connected device.
+The initial setup for an iCloud backend involves getting a trust token/session.
+This uses your regular Apple ID password plus 2FA, either from a trusted
+device prompt or an SMS code sent to a trusted phone number.
**IMPORTANT**: App-specific passwords are not accepted. Only use your
regular Apple ID password and 2FA.
+This backend serves two Apple services:
+
+- `drive` - iCloud Drive (default)
+- `photos` - iCloud Photos
+
`rclone config` walks you through the token creation. The trust token is valid
for 30 days. After which you will have to reauthenticate with `rclone reconnect`
or `rclone config`.
@@ -30,10 +35,12 @@ The authentication flow is:
1. rclone initiates a session with Apple's identity service
2. An SRP key exchange takes place (your password is used locally to derive a key)
-3. Apple sends a 2FA prompt to your trusted devices
+3. Apple sends a 2FA prompt to your trusted devices, or lets you request an
+ SMS code
4. After you enter the 2FA code, rclone receives a trust token for future sessions
-Here is an example of how to make a remote called `iclouddrive`. First run:
+Here is an example of how to make a Photos remote called `icloudphotos`.
+For iCloud Drive, leave `service` at its default `drive` value. First run:
```console
rclone config
@@ -47,7 +54,7 @@ n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
-name> iclouddrive
+name> icloudphotos
Option Storage.
Type of storage to configure.
Choose a number from below, or type in your own value.
@@ -56,6 +63,15 @@ XX / iCloud Drive
\ (iclouddrive)
[snip]
Storage> iclouddrive
+Option service.
+iCloud service to use.
+Choose a number from below, or type in your own value of type string.
+Press Enter for the default (drive).
+ 1 / iCloud Drive
+ \ (drive)
+ 2 / iCloud Photos
+ \ (photos)
+service> 2
Option apple_id.
Apple ID.
Enter a value.
@@ -75,13 +91,14 @@ y) Yes
n) No (default)
y/n> n
Option config_2fa.
-Two-factor authentication: please enter your 2FA code
+Two-factor authentication: enter your 2FA code or type 'sms' for a text message
Enter a value.
config_2fa> 2FACODE
Remote config
--------------------
-[iclouddrive]
+[icloudphotos]
- type: iclouddrive
+- service: photos
- apple_id: APPLEID
- password: *** ENCRYPTED ***
- cookies: ****************************
@@ -93,30 +110,124 @@ d) Delete this remote
y/e/d> y
```
+## iCloud Photos
+
+The iCloud Drive backend also supports accessing iCloud Photos by setting the
+`service` option to `photos`:
+
+```console
+rclone lsd iclouddrive: --iclouddrive-service photos
+```
+
+This presents a read-only hierarchy rooted at photo libraries:
+
+- **Level 1**: Photo libraries — your personal library (`PrimarySync`) and
+ any Shared Photo Library (`SharedSync-XXXX`)
+- **Level 2+**: Albums and folders within a library, nested recursively as in
+ Apple Photos
+- **Leaf level**: Photos/videos within an album, including Live Photo `.MOV`
+ companions
+
+Examples:
+
+```console
+# List libraries
+rclone lsd iclouddrive: --iclouddrive-service photos
+
+# List albums in your primary library
+rclone lsd iclouddrive:PrimarySync/ --iclouddrive-service photos
+
+# List photos in an album
+rclone ls iclouddrive:PrimarySync/All\ Photos/ --iclouddrive-service photos
+
+# Download a photo
+rclone copy iclouddrive:PrimarySync/Favorites/IMG_0001.HEIC /tmp/ --iclouddrive-service photos
+```
+
+You can either:
+
+- set `service = photos` in `rclone config` for a dedicated Photos remote
+- keep `service = drive` and pass `--iclouddrive-service photos` when needed
+
+### Metadata
+
+With `--metadata`, Photos entries expose these read-only metadata keys:
+
+- `width`
+- `height`
+- `added-time`
+- `favorite`
+- `hidden`
+
+These metadata keys are only available when `service = photos`.
+
+### Caching
+
+iCloud Photos caches album listings to disk for fast subsequent access.
+On the first run, listing a large album uses parallel `startRank`
+partitions to fetch pages concurrently. After that, a lightweight change
+check (~200ms) determines whether the cache is still valid.
+
+Cache location: `~/.cache/rclone/iclouddrive-photos/<remote>/<zone>/`
+
+To clear the cache: delete that directory or run `rclone config reconnect`.
+
+### FUSE mounts
+
+For mounting iCloud Photos via `rclone mount`, the following flags
+are recommended:
+
+ rclone mount remote: /mnt/photos \
+ --iclouddrive-service photos \
+ --vfs-refresh \
+ --dir-cache-time 1h \
+ --vfs-cache-mode full \
+ --attr-timeout 1m \
+ --read-only
+
+- `--vfs-refresh` pre-warms directory caches in the background on mount
+ start so that albums are ready when you browse them
+- `--dir-cache-time 1h` extends the in-memory cache lifetime beyond the
+ default 5 minutes (change detection is fast, so this is safe)
+- `--vfs-cache-mode full` caches downloaded photos and videos to local
+ disk for fast repeated access
+- `--attr-timeout 1m` reduces kernel attribute lookups (safe because
+ the backend is read-only)
+- `--read-only` prevents confusing write errors
+
+The first listing of a very large album (e.g. 75,000 items in "All
+Photos") can take several minutes due to API pagination limits. This
+happens once — subsequent listings use the disk cache.
+
+### Limitations
+
+iCloud Photos is read-only. Upload, delete, rename, and move operations
+are not supported.
+
## Advanced Data Protection
-ADP is currently unsupported and need to be disabled
+Advanced Data Protection is supported.
On iPhone, Settings `>` Apple Account `>` iCloud `>` 'Access iCloud Data on the Web'
-must be ON, and 'Advanced Data Protection' OFF.
+must be ON.
+
+If ADP is enabled on your account, rclone requests PCS cookies after 2FA.
+Apple may require approval on a trusted device before those cookies are issued.
## Troubleshooting
-### Missing PCS cookies from the request
+### PCS cookie errors with ADP
-This means you have Advanced Data Protection (ADP) turned on. This is not supported
-at the moment. If you want to use rclone you will have to turn it off. See above
-for how to turn it off.
+If you see `Missing PCS cookies from the request` or a `requestPCS:` error,
+the ADP approval flow did not complete successfully.
-You will need to clear the `cookies` and the `trust_token` fields in the config.
-Or you can delete the remote config and start again.
+Check that 'Access iCloud Data on the Web' is enabled and approve any prompt
+on your trusted device.
-You should then run `rclone reconnect remote:`.
+Then run `rclone reconnect remote:`.
-Note that changing the ADP setting may not take effect immediately - you may
-need to wait a few hours or a day before you can get rclone to work - keep
-clearing the config entry and running `rclone reconnect remote:` until rclone
-functions properly.
+If the remote still has stale auth state, clear the `cookies` and
+`trust_token` fields in the config, or delete and recreate the remote.
<!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/iclouddrive/iclouddrive.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->
### Standard options