"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
Add ICloud Drive backend
lostb1t51db76fd4754
diff --git a/README.md b/README.mdindex 9cbd74cb5..866986479 100644--- a/README.md+++ b/README.md@@ -66,6 +66,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and* HiDrive [:page_facing_up:](https://rclone.org/hidrive/)* HTTP [:page_facing_up:](https://rclone.org/http/)* Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)+ * iCloud Drive [:page_facing_up:](https://rclone.org/iclouddrive/)* ImageKit [:page_facing_up:](https://rclone.org/imagekit/)* Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)* Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)diff --git a/backend/all/all.go b/backend/all/all.goindex a9191c98d..1abd6770f 100644--- a/backend/all/all.go+++ b/backend/all/all.go@@ -26,6 +26,7 @@ import (_ "github.com/rclone/rclone/backend/hdfs"_ "github.com/rclone/rclone/backend/hidrive"_ "github.com/rclone/rclone/backend/http"+ _ "github.com/rclone/rclone/backend/iclouddrive"_ "github.com/rclone/rclone/backend/imagekit"_ "github.com/rclone/rclone/backend/internetarchive"_ "github.com/rclone/rclone/backend/jottacloud"diff --git a/backend/iclouddrive/api/client.go b/backend/iclouddrive/api/client.gonew file mode 100644index 000000000..7cdf74baf--- /dev/null+++ b/backend/iclouddrive/api/client.go@@ -0,0 +1,166 @@+// Package api provides functionality for interacting with the iCloud API.+package api++import (+ "bytes"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "net/http"+ "strings"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/fshttp"+ "github.com/rclone/rclone/lib/rest"+)++const (+ baseEndpoint = "https://www.icloud.com"+ homeEndpoint = "https://www.icloud.com"+ setupEndpoint = "https://setup.icloud.com/setup/ws/1"+ authEndpoint = "https://idmsa.apple.com/appleauth/auth"+)++type sessionSave func(*Session)++// Client defines the client configuration+type Client struct {+ appleID string+ password string+ srv *rest.Client+ Session *Session+ sessionSaveCallback sessionSave++ drive *DriveService+}++// New creates a new Client instance with the provided Apple ID, password, trust token, cookies, and session save callback.+//+// Parameters:+// - appleID: the Apple ID of the user.+// - password: the password of the user.+// - trustToken: the trust token for the session.+// - clientID: the client id for the session.+// - cookies: the cookies for the session.+// - sessionSaveCallback: the callback function to save the session.+func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave) (*Client, error) {+ icloud := &Client{+ appleID: appleID,+ password: password,+ srv: rest.NewClient(fshttp.NewClient(context.Background())),+ Session: NewSession(),+ sessionSaveCallback: sessionSaveCallback,+ }++ icloud.Session.TrustToken = trustToken+ icloud.Session.Cookies = cookies+ icloud.Session.ClientID = clientID+ return icloud, nil+}++// DriveService returns the DriveService instance associated with the Client.+func (c *Client) DriveService() (*DriveService, error) {+ var err error+ if c.drive == nil {+ c.drive, err = NewDriveService(c)+ if err != nil {+ return nil, err+ }+ }+ return c.drive, nil+}++// Request makes a request and retries it if the session is invalid.+//+// This function is the main entry point for making requests to the iCloud+// API. If the initial request returns a 401 (Unauthorized), it will try to+// reauthenticate and retry the request.+func (c *Client) Request(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {+ resp, err = c.Session.Request(ctx, opts, request, response)+ if err != nil && resp != nil {+ // try to reauth+ if resp.StatusCode == 401 || resp.StatusCode == 421 {+ err = c.Authenticate(ctx)+ if err != nil {+ return nil, err+ }++ if c.Session.Requires2FA() {+ return nil, errors.New("trust token expired, please reauth")+ }+ return c.RequestNoReAuth(ctx, opts, request, response)+ }+ }+ return resp, err+}++// RequestNoReAuth makes a request without re-authenticating.+//+// This function is useful when you have a session that is already+// authenticated, but you need to make a request without triggering+// a re-authentication.+func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {+ // Make the request without re-authenticating+ resp, err = c.Session.Request(ctx, opts, request, response)+ return resp, err+}++// Authenticate authenticates the client with the iCloud API.+func (c *Client) Authenticate(ctx context.Context) error {+ if c.Session.Cookies != nil {+ if err := c.Session.ValidateSession(ctx); err == nil {+ fs.Debugf("icloud", "Valid session, no need to reauth")+ return nil+ }+ c.Session.Cookies = nil+ }++ fs.Debugf("icloud", "Authenticating as %s\n", c.appleID)+ err := c.Session.SignIn(ctx, c.appleID, c.password)++ if err == nil {+ err = c.Session.AuthWithToken(ctx)+ if err == nil && c.sessionSaveCallback != nil {+ c.sessionSaveCallback(c.Session)+ }+ }+ return err+}++// SignIn signs in the client using the provided context and credentials.+func (c *Client) SignIn(ctx context.Context) error {+ return c.Session.SignIn(ctx, c.appleID, c.password)+}++// IntoReader marshals the provided values into a JSON encoded reader+func IntoReader(values any) (*bytes.Reader, error) {+ m, err := json.Marshal(values)+ if err != nil {+ return nil, err+ }+ return bytes.NewReader(m), nil+}++// RequestError holds info on a result state, icloud can return a 200 but the result is unknown+type RequestError struct {+ Status string+ Text string+}++// Error satisfy the error interface.+func (e *RequestError) Error() string {+ return fmt.Sprintf("%s: %s", e.Text, e.Status)+}++func newRequestError(Status string, Text string) *RequestError {+ return &RequestError{+ Status: strings.ToLower(Status),+ Text: Text,+ }+}++// newErr orf makes a new error from sprintf parameters.+func newRequestErrorf(Status string, Text string, Parameters ...interface{}) *RequestError {+ return newRequestError(strings.ToLower(Status), fmt.Sprintf(Text, Parameters...))+}diff --git a/backend/iclouddrive/api/drive.go b/backend/iclouddrive/api/drive.gonew file mode 100644index 000000000..5d735ebbd--- /dev/null+++ b/backend/iclouddrive/api/drive.go@@ -0,0 +1,913 @@+package api++import (+ "bytes"+ "context"+ "io"+ "mime"+ "net/http"+ "net/url"+ "path/filepath"+ "strconv"+ "strings"+ "time"++ "github.com/google/uuid"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/rest"+)++const (+ defaultZone = "com.apple.CloudDocs"+ statusOk = "OK"+ statusEtagConflict = "ETAG_CONFLICT"+)++// DriveService represents an iCloud Drive service.+type DriveService struct {+ icloud *Client+ RootID string+ endpoint string+ docsEndpoint string+}++// NewDriveService creates a new DriveService instance.+func NewDriveService(icloud *Client) (*DriveService, error) {+ return &DriveService{icloud: icloud, RootID: "FOLDER::com.apple.CloudDocs::root", endpoint: icloud.Session.AccountInfo.Webservices["drivews"].URL, docsEndpoint: icloud.Session.AccountInfo.Webservices["docws"].URL}, nil+}++// GetItemByDriveID retrieves a DriveItem by its Drive ID.+func (d *DriveService) GetItemByDriveID(ctx context.Context, id string, includeChildren bool) (*DriveItem, *http.Response, error) {+ items, resp, err := d.GetItemsByDriveID(ctx, []string{id}, includeChildren)+ if err != nil {+ return nil, resp, err+ }+ return items[0], resp, err+}++// GetItemsByDriveID retrieves DriveItems by their Drive IDs.+func (d *DriveService) GetItemsByDriveID(ctx context.Context, ids []string, includeChildren bool) ([]*DriveItem, *http.Response, error) {+ var err error+ _items := []map[string]any{}+ for _, id := range ids {+ _items = append(_items, map[string]any{+ "drivewsid": id,+ "partialData": false,+ "includeHierarchy": false,+ })+ }++ var body *bytes.Reader+ var path string+ if !includeChildren {+ values := []map[string]any{{+ "items": _items,+ }}+ body, err = IntoReader(values)+ if err != nil {+ return nil, nil, err+ }+ path = "/retrieveItemDetails"+ } else {+ values := _items+ body, err = IntoReader(values)+ if err != nil {+ return nil, nil, err+ }+ path = "/retrieveItemDetailsInFolders"+ }++ opts := rest.Opts{+ Method: "POST",+ Path: path,+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.endpoint,+ Body: body,+ }+ var items []*DriveItem+ resp, err := d.icloud.Request(ctx, opts, nil, &items)+ if err != nil {+ return nil, resp, err+ }++ return items, resp, err+}++// GetDocByPath retrieves a document by its path.+func (d *DriveService) GetDocByPath(ctx context.Context, path string) (*Document, *http.Response, error) {+ values := url.Values{}+ values.Set("unified_format", "false")+ body, err := IntoReader(path)+ if err != nil {+ return nil, nil, err+ }+ opts := rest.Opts{+ Method: "POST",+ Path: "/ws/" + defaultZone + "/list/lookup_by_path",+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.docsEndpoint,+ Parameters: values,+ Body: body,+ }+ var item []*Document+ resp, err := d.icloud.Request(ctx, opts, nil, &item)+ if err != nil {+ return nil, resp, err+ }++ return item[0], resp, err+}++// GetItemByPath retrieves a DriveItem by its path.+func (d *DriveService) GetItemByPath(ctx context.Context, path string) (*DriveItem, *http.Response, error) {+ values := url.Values{}+ values.Set("unified_format", "true")++ body, err := IntoReader(path)+ if err != nil {+ return nil, nil, err+ }+ opts := rest.Opts{+ Method: "POST",+ Path: "/ws/" + defaultZone + "/list/lookup_by_path",+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.docsEndpoint,+ Parameters: values,+ Body: body,+ }+ var item []*DriveItem+ resp, err := d.icloud.Request(ctx, opts, nil, &item)+ if err != nil {+ return nil, resp, err+ }++ return item[0], resp, err+}++// GetDocByItemID retrieves a document by its item ID.+func (d *DriveService) GetDocByItemID(ctx context.Context, id string) (*Document, *http.Response, error) {+ values := url.Values{}+ values.Set("document_id", id)+ values.Set("unified_format", "false") // important+ opts := rest.Opts{+ Method: "GET",+ Path: "/ws/" + defaultZone + "/list/lookup_by_id",+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.docsEndpoint,+ Parameters: values,+ }+ var item *Document+ resp, err := d.icloud.Request(ctx, opts, nil, &item)+ if err != nil {+ return nil, resp, err+ }++ return item, resp, err+}++// GetItemRawByItemID retrieves a DriveItemRaw by its item ID.+func (d *DriveService) GetItemRawByItemID(ctx context.Context, id string) (*DriveItemRaw, *http.Response, error) {+ opts := rest.Opts{+ Method: "GET",+ Path: "/v1/item/" + id,+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.docsEndpoint,+ }+ var item *DriveItemRaw+ resp, err := d.icloud.Request(ctx, opts, nil, &item)+ if err != nil {+ return nil, resp, err+ }++ return item, resp, err+}++// GetItemsInFolder retrieves a list of DriveItemRaw objects in a folder with the given ID.+func (d *DriveService) GetItemsInFolder(ctx context.Context, id string, limit int64) ([]*DriveItemRaw, *http.Response, error) {+ values := url.Values{}+ values.Set("limit", strconv.FormatInt(limit, 10))++ opts := rest.Opts{+ Method: "GET",+ Path: "/v1/enumerate/" + id,+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.docsEndpoint,+ Parameters: values,+ }++ items := struct {+ Items []*DriveItemRaw `json:"drive_item"`+ }{}++ resp, err := d.icloud.Request(ctx, opts, nil, &items)+ if err != nil {+ return nil, resp, err+ }++ return items.Items, resp, err+}++// GetDownloadURLByDriveID retrieves the download URL for a file in the DriveService.+func (d *DriveService) GetDownloadURLByDriveID(ctx context.Context, id string) (string, *http.Response, error) {+ _, zone, docid := DeconstructDriveID(id)+ values := url.Values{}+ values.Set("document_id", docid)++ if zone == "" {+ zone = defaultZone+ }++ opts := rest.Opts{+ Method: "GET",+ Path: "/ws/" + zone + "/download/by_id",+ Parameters: values,+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.docsEndpoint,+ }++ var filer *FileRequest+ resp, err := d.icloud.Request(ctx, opts, nil, &filer)++ if err != nil {+ return "", resp, err+ }++ var url string+ if filer.DataToken != nil {+ url = filer.DataToken.URL+ } else {+ url = filer.PackageToken.URL+ }++ return url, resp, err+}++// DownloadFile downloads a file from the given URL using the provided options.+func (d *DriveService) DownloadFile(ctx context.Context, url string, opt []fs.OpenOption) (*http.Response, error) {+ opts := &rest.Opts{+ Method: "GET",+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: url,+ Options: opt,+ }++ resp, err := d.icloud.srv.Call(ctx, opts)+ if err != nil {+ // icloud has some weird http codes+ if resp.StatusCode == 330 {+ loc, err := resp.Location()+ if err == nil {+ return d.DownloadFile(ctx, loc.String(), opt)+ }+ }++ return resp, err+ }+ return d.icloud.srv.Call(ctx, opts)+}++// MoveItemToTrashByItemID moves an item to the trash based on the item ID.+func (d *DriveService) MoveItemToTrashByItemID(ctx context.Context, id, etag string, force bool) (*DriveItem, *http.Response, error) {+ doc, resp, err := d.GetDocByItemID(ctx, id)+ if err != nil {+ return nil, resp, err+ }+ return d.MoveItemToTrashByID(ctx, doc.DriveID(), etag, force)+}++// MoveItemToTrashByID moves an item to the trash based on the item ID.+func (d *DriveService) MoveItemToTrashByID(ctx context.Context, drivewsid, etag string, force bool) (*DriveItem, *http.Response, error) {+ values := map[string]any{+ "items": []map[string]any{{+ "drivewsid": drivewsid,+ "etag": etag,+ "clientId": drivewsid,+ }}}++ body, err := IntoReader(values)+ if err != nil {+ return nil, nil, err+ }++ opts := rest.Opts{+ Method: "POST",+ Path: "/moveItemsToTrash",+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.endpoint,+ Body: body,+ }++ item := struct {+ Items []*DriveItem `json:"items"`+ }{}+ resp, err := d.icloud.Request(ctx, opts, nil, &item)++ if err != nil {+ return nil, resp, err+ }++ if item.Items[0].Status != statusOk {+ // rerun with latest etag+ if force && item.Items[0].Status == "ETAG_CONFLICT" {+ return d.MoveItemToTrashByID(ctx, drivewsid, item.Items[0].Etag, false)+ }++ err = newRequestError(item.Items[0].Status, "unknown request status")+ }++ return item.Items[0], resp, err+}++// CreateNewFolderByItemID creates a new folder by item ID.+func (d *DriveService) CreateNewFolderByItemID(ctx context.Context, id, name string) (*DriveItem, *http.Response, error) {+ doc, resp, err := d.GetDocByItemID(ctx, id)+ if err != nil {+ return nil, resp, err+ }+ return d.CreateNewFolderByDriveID(ctx, doc.DriveID(), name)+}++// CreateNewFolderByDriveID creates a new folder by its Drive ID.+func (d *DriveService) CreateNewFolderByDriveID(ctx context.Context, drivewsid, name string) (*DriveItem, *http.Response, error) {+ values := map[string]any{+ "destinationDrivewsId": drivewsid,+ "folders": []map[string]any{{+ "clientId": "FOLDER::UNKNOWN_ZONE::TempId-" + uuid.New().String(),+ "name": name,+ }},+ }++ body, err := IntoReader(values)+ if err != nil {+ return nil, nil, err+ }++ opts := rest.Opts{+ Method: "POST",+ Path: "/createFolders",+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.endpoint,+ Body: body,+ }+ var fResp *CreateFoldersResponse+ resp, err := d.icloud.Request(ctx, opts, nil, &fResp)+ if err != nil {+ return nil, resp, err+ }+ status := fResp.Folders[0].Status+ if status != statusOk {+ err = newRequestError(status, "unknown request status")+ }++ return fResp.Folders[0], resp, err+}++// RenameItemByItemID renames a DriveItem by its item ID.+func (d *DriveService) RenameItemByItemID(ctx context.Context, id, etag, name string, force bool) (*DriveItem, *http.Response, error) {+ doc, resp, err := d.GetDocByItemID(ctx, id)+ if err != nil {+ return nil, resp, err+ }+ return d.RenameItemByDriveID(ctx, doc.DriveID(), doc.Etag, name, force)+}++// RenameItemByDriveID renames a DriveItem by its drive ID.+func (d *DriveService) RenameItemByDriveID(ctx context.Context, id, etag, name string, force bool) (*DriveItem, *http.Response, error) {+ values := map[string]any{+ "items": []map[string]any{{+ "drivewsid": id,+ "name": name,+ "etag": etag,+ // "extension": split[1],+ }},+ }++ body, err := IntoReader(values)+ if err != nil {+ return nil, nil, err+ }++ opts := rest.Opts{+ Method: "POST",+ Path: "/renameItems",+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.endpoint,+ Body: body,+ }+ var items *DriveItem+ resp, err := d.icloud.Request(ctx, opts, nil, &items)++ if err != nil {+ return nil, resp, err+ }++ status := items.Items[0].Status+ if status != statusOk {+ // rerun with latest etag+ if force && status == "ETAG_CONFLICT" {+ return d.RenameItemByDriveID(ctx, id, items.Items[0].Etag, name, false)+ }+ err = newRequestErrorf(status, "unknown inner status for: %s %s", opts.Method, resp.Request.URL)+ }++ return items.Items[0], resp, err+}++// MoveItemByItemID moves an item by its item ID to a destination item ID.+func (d *DriveService) MoveItemByItemID(ctx context.Context, id, etag, dstID string, force bool) (*DriveItem, *http.Response, error) {+ docSrc, resp, err := d.GetDocByItemID(ctx, id)+ if err != nil {+ return nil, resp, err+ }+ docDst, resp, err := d.GetDocByItemID(ctx, dstID)+ if err != nil {+ return nil, resp, err+ }+ return d.MoveItemByDriveID(ctx, docSrc.DriveID(), docSrc.Etag, docDst.DriveID(), force)+}++// MoveItemByDocID moves an item by its doc ID.+// func (d *DriveService) MoveItemByDocID(ctx context.Context, srcDocID, srcEtag, dstDocID string, force bool) (*DriveItem, *http.Response, error) {+// return d.MoveItemByDriveID(ctx, srcDocID, srcEtag, docDst.DriveID(), force)+// }++// MoveItemByDriveID moves an item by its drive ID.+func (d *DriveService) MoveItemByDriveID(ctx context.Context, id, etag, dstID string, force bool) (*DriveItem, *http.Response, error) {+ values := map[string]any{+ "destinationDrivewsId": dstID,+ "items": []map[string]any{{+ "drivewsid": id,+ "etag": etag,+ "clientId": id,+ }},+ }++ body, err := IntoReader(values)+ if err != nil {+ return nil, nil, err+ }++ opts := rest.Opts{+ Method: "POST",+ Path: "/moveItems",+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.endpoint,+ Body: body,+ }++ var items *DriveItem+ resp, err := d.icloud.Request(ctx, opts, nil, &items)++ if err != nil {+ return nil, resp, err+ }++ status := items.Items[0].Status+ if status != statusOk {+ // rerun with latest etag+ if force && status == "ETAG_CONFLICT" {+ return d.MoveItemByDriveID(ctx, id, items.Items[0].Etag, dstID, false)+ }+ err = newRequestErrorf(status, "unknown inner status for: %s %s", opts.Method, resp.Request.URL)+ }++ return items.Items[0], resp, err+}++// CopyDocByItemID copies a document by its item ID.+func (d *DriveService) CopyDocByItemID(ctx context.Context, itemID string) (*DriveItemRaw, *http.Response, error) {+ // putting name in info doesnt work. extension does work so assume this is a bug in the endpoint+ values := map[string]any{+ "info_to_update": map[string]any{},+ }++ body, err := IntoReader(values)+ if err != nil {+ return nil, nil, err+ }+ opts := rest.Opts{+ Method: "POST",+ Path: "/v1/item/copy/" + itemID,+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.docsEndpoint,+ Body: body,+ }++ var info *DriveItemRaw+ resp, err := d.icloud.Request(ctx, opts, nil, &info)+ if err != nil {+ return nil, resp, err+ }+ return info, resp, err+}++// CreateUpload creates an url for an upload.+func (d *DriveService) CreateUpload(ctx context.Context, size int64, name string) (*UploadResponse, *http.Response, error) {+ // first we need to request an upload url+ values := map[string]any{+ "filename": name,+ "type": "FILE",+ "size": strconv.FormatInt(size, 10),+ "content_type": GetContentTypeForFile(name),+ }+ body, err := IntoReader(values)+ if err != nil {+ return nil, nil, err+ }++ opts := rest.Opts{+ Method: "POST",+ Path: "/ws/" + defaultZone + "/upload/web",+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.docsEndpoint,+ Body: body,+ }+ var responseInfo []*UploadResponse+ resp, err := d.icloud.Request(ctx, opts, nil, &responseInfo)+ if err != nil {+ return nil, resp, err+ }+ return responseInfo[0], resp, err+}++// Upload uploads a file to the given url+func (d *DriveService) Upload(ctx context.Context, in io.Reader, size int64, name, uploadURL string) (*SingleFileResponse, *http.Response, error) {+ // TODO: implement multipart upload+ opts := rest.Opts{+ Method: "POST",+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: uploadURL,+ Body: in,+ ContentLength: &size,+ ContentType: GetContentTypeForFile(name),+ // MultipartContentName: "files",+ MultipartFileName: name,+ }+ var singleFileResponse *SingleFileResponse+ resp, err := d.icloud.Request(ctx, opts, nil, &singleFileResponse)+ if err != nil {+ return nil, resp, err+ }+ return singleFileResponse, resp, err+}++// UpdateFile updates a file in the DriveService.+//+// ctx: the context.Context object for the request.+// r: a pointer to the UpdateFileInfo struct containing the information for the file update.+// Returns a pointer to the DriveItem struct representing the updated file, the http.Response object, and an error if any.+func (d *DriveService) UpdateFile(ctx context.Context, r *UpdateFileInfo) (*DriveItem, *http.Response, error) {+ body, err := IntoReader(r)+ if err != nil {+ return nil, nil, err+ }+ opts := rest.Opts{+ Method: "POST",+ Path: "/ws/" + defaultZone + "/update/documents",+ ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),+ RootURL: d.docsEndpoint,+ Body: body,+ }+ var responseInfo *DocumentUpdateResponse+ resp, err := d.icloud.Request(ctx, opts, nil, &responseInfo)+ if err != nil {+ return nil, resp, err+ }++ doc := responseInfo.Results[0].Document+ item := DriveItem{+ Drivewsid: "FILE::com.apple.CloudDocs::" + doc.DocumentID,+ Docwsid: doc.DocumentID,+ Itemid: doc.ItemID,+ Etag: doc.Etag,+ ParentID: doc.ParentID,+ DateModified: time.Unix(r.Mtime, 0),+ DateCreated: time.Unix(r.Mtime, 0),+ Type: doc.Type,+ Name: doc.Name,+ Size: doc.Size,+ }++ return &item, resp, err+}++// UpdateFileInfo represents the information for an update to a file in the DriveService.+type UpdateFileInfo struct {+ AllowConflict bool `json:"allow_conflict"`+ Btime int64 `json:"btime"`+ Command string `json:"command"`+ CreateShortGUID bool `json:"create_short_guid"`+ Data struct {+ Receipt string `json:"receipt,omitempty"`+ ReferenceSignature string `json:"reference_signature,omitempty"`+ Signature string `json:"signature,omitempty"`+ Size int64 `json:"size,omitempty"`+ WrappingKey string `json:"wrapping_key,omitempty"`+ } `json:"data,omitempty"`+ DocumentID string `json:"document_id"`+ FileFlags FileFlags `json:"file_flags"`+ Mtime int64 `json:"mtime"`+ Path struct {+ Path string `json:"path"`+ StartingDocumentID string `json:"starting_document_id"`+ } `json:"path"`+}++// FileFlags defines the file flags for a document.+type FileFlags struct {+ IsExecutable bool `json:"is_executable"`+ IsHidden bool `json:"is_hidden"`+ IsWritable bool `json:"is_writable"`+}++// NewUpdateFileInfo creates a new UpdateFileInfo object with default values.+//+// Returns an UpdateFileInfo object.+func NewUpdateFileInfo() UpdateFileInfo {+ return UpdateFileInfo{+ Command: "add_file",+ CreateShortGUID: true,+ AllowConflict: true,+ FileFlags: FileFlags{+ IsExecutable: true,+ IsHidden: false,+ IsWritable: false,+ },+ }+}++// DriveItemRaw is a raw drive item.+// not suure what to call this but there seems to be a "unified" and non "unified" drive item response. This is the non unified.+type DriveItemRaw struct {+ ItemID string `json:"item_id"`+ ItemInfo *DriveItemRawInfo `json:"item_info"`+}++// SplitName splits the name of a DriveItemRaw into its name and extension.+//+// It returns the name and extension as separate strings. If the name ends with a dot,+// it means there is no extension, so an empty string is returned for the extension.+// If the name does not contain a dot, it means+func (d *DriveItemRaw) SplitName() (string, string) {+ name := d.ItemInfo.Name+ // ends with a dot, no extension+ if strings.HasSuffix(name, ".") {+ return name, ""+ }+ lastInd := strings.LastIndex(name, ".")++ if lastInd == -1 {+ return name, ""+ }+ return name[:lastInd], name[lastInd+1:]+}++// ModTime returns the modification time of the DriveItemRaw.+//+// It parses the ModifiedAt field of the ItemInfo struct and converts it to a time.Time value.+// If the parsing fails, it returns the zero value of time.Time.+// The returned time.Time value represents the modification time of the DriveItemRaw.+func (d *DriveItemRaw) ModTime() time.Time {+ i, err := strconv.ParseInt(d.ItemInfo.ModifiedAt, 10, 64)+ if err != nil {+ return time.Time{}+ }+ return time.UnixMilli(i)+}++// CreatedTime returns the creation time of the DriveItemRaw.+//+// It parses the CreatedAt field of the ItemInfo struct and converts it to a time.Time value.+// If the parsing fails, it returns the zero value of time.Time.+// The returned time.Time+func (d *DriveItemRaw) CreatedTime() time.Time {+ i, err := strconv.ParseInt(d.ItemInfo.CreatedAt, 10, 64)+ if err != nil {+ return time.Time{}+ }+ return time.UnixMilli(i)+}++// DriveItemRawInfo is the raw information about a drive item.+type DriveItemRawInfo struct {+ Name string `json:"name"`+ // Extension is absolutely borked on endpoints so dont use it.+ Extension string `json:"extension"`+ Size int64 `json:"size,string"`+ Type string `json:"type"`+ Version string `json:"version"`+ ModifiedAt string `json:"modified_at"`+ CreatedAt string `json:"created_at"`+ Urls struct {+ URLDownload string `json:"url_download"`+ } `json:"urls"`+}++// IntoDriveItem converts a DriveItemRaw into a DriveItem.+//+// It takes no parameters.+// It returns a pointer to a DriveItem.+func (d *DriveItemRaw) IntoDriveItem() *DriveItem {+ name, extension := d.SplitName()+ return &DriveItem{+ Itemid: d.ItemID,+ Name: name,+ Extension: extension,+ Type: d.ItemInfo.Type,+ Etag: d.ItemInfo.Version,+ DateModified: d.ModTime(),+ DateCreated: d.CreatedTime(),+ Size: d.ItemInfo.Size,+ Urls: d.ItemInfo.Urls,+ }+}++// DocumentUpdateResponse is the response of a document update request.+type DocumentUpdateResponse struct {+ Status struct {+ StatusCode int `json:"status_code"`+ ErrorMessage string `json:"error_message"`+ } `json:"status"`+ Results []struct {+ Status struct {+ StatusCode int `json:"status_code"`+ ErrorMessage string `json:"error_message"`+ } `json:"status"`+ OperationID interface{} `json:"operation_id"`+ Document *Document `json:"document"`+ } `json:"results"`+}++// Document represents a document on iCloud.+type Document struct {+ Status struct {+ StatusCode int `json:"status_code"`+ ErrorMessage string `json:"error_message"`+ } `json:"status"`+ DocumentID string `json:"document_id"`+ ItemID string `json:"item_id"`+ Urls struct {+ URLDownload string `json:"url_download"`+ } `json:"urls"`+ Etag string `json:"etag"`+ ParentID string `json:"parent_id"`+ Name string `json:"name"`+ Type string `json:"type"`+ Deleted bool `json:"deleted"`+ Mtime int64 `json:"mtime"`+ LastEditorName string `json:"last_editor_name"`+ Data DocumentData `json:"data"`+ Size int64 `json:"size"`+ Btime int64 `json:"btime"`+ Zone string `json:"zone"`+ FileFlags struct {+ IsExecutable bool `json:"is_executable"`+ IsWritable bool `json:"is_writable"`+ IsHidden bool `json:"is_hidden"`+ } `json:"file_flags"`+ LastOpenedTime int64 `json:"lastOpenedTime"`+ RestorePath interface{} `json:"restorePath"`+ HasChainedParent bool `json:"hasChainedParent"`+}++// DriveID returns the drive ID of the Document.+func (d *Document) DriveID() string {+ if d.Zone == "" {+ d.Zone = defaultZone+ }+ return d.Type + "::" + d.Zone + "::" + d.DocumentID+}++// DocumentData represents the data of a document.+type DocumentData struct {+ Signature string `json:"signature"`+ Owner string `json:"owner"`+ Size int64 `json:"size"`+ ReferenceSignature string `json:"reference_signature"`+ WrappingKey string `json:"wrapping_key"`+ PcsInfo string `json:"pcsInfo"`+}++// SingleFileResponse is the response of a single file request.+type SingleFileResponse struct {+ SingleFile *SingleFileInfo `json:"singleFile"`+}++// SingleFileInfo represents the information of a single file.+type SingleFileInfo struct {+ ReferenceSignature string `json:"referenceChecksum"`+ Size int64 `json:"size"`+ Signature string `json:"fileChecksum"`+ WrappingKey string `json:"wrappingKey"`+ Receipt string `json:"receipt"`+}++// UploadResponse is the response of an upload request.+type UploadResponse struct {+ URL string `json:"url"`+ DocumentID string `json:"document_id"`+}++// FileRequestToken represents the token of a file request.+type FileRequestToken struct {+ URL string `json:"url"`+ Token string `json:"token"`+ Signature string `json:"signature"`+ WrappingKey string `json:"wrapping_key"`+ ReferenceSignature string `json:"reference_signature"`+}++// FileRequest represents the request of a file.+type FileRequest struct {+ DocumentID string `json:"document_id"`+ ItemID string `json:"item_id"`+ OwnerDsid int64 `json:"owner_dsid"`+ DataToken *FileRequestToken `json:"data_token,omitempty"`+ PackageToken *FileRequestToken `json:"package_token,omitempty"`+ DoubleEtag string `json:"double_etag"`+}++// CreateFoldersResponse is the response of a create folders request.+type CreateFoldersResponse struct {+ Folders []*DriveItem `json:"folders"`+}++// DriveItem represents an item on iCloud.+type DriveItem struct {+ DateCreated time.Time `json:"dateCreated"`+ Drivewsid string `json:"drivewsid"`+ Docwsid string `json:"docwsid"`+ Itemid string `json:"item_id"`+ Zone string `json:"zone"`+ Name string `json:"name"`+ ParentID string `json:"parentId"`+ Hierarchy []DriveItem `json:"hierarchy"`+ Etag string `json:"etag"`+ Type string `json:"type"`+ AssetQuota int64 `json:"assetQuota"`+ FileCount int64 `json:"fileCount"`+ ShareCount int64 `json:"shareCount"`+ ShareAliasCount int64 `json:"shareAliasCount"`+ DirectChildrenCount int64 `json:"directChildrenCount"`+ Items []*DriveItem `json:"items"`+ NumberOfItems int64 `json:"numberOfItems"`+ Status string `json:"status"`+ Extension string `json:"extension,omitempty"`+ DateModified time.Time `json:"dateModified,omitempty"`+ DateChanged time.Time `json:"dateChanged,omitempty"`+ Size int64 `json:"size,omitempty"`+ LastOpenTime time.Time `json:"lastOpenTime,omitempty"`+ Urls struct {+ URLDownload string `json:"url_download"`+ } `json:"urls"`+}++// IsFolder returns true if the item is a folder.+func (d *DriveItem) IsFolder() bool {+ return d.Type == "FOLDER" || d.Type == "APP_CONTAINER" || d.Type == "APP_LIBRARY"+}++// DownloadURL returns the download URL of the item.+func (d *DriveItem) DownloadURL() string {+ return d.Urls.URLDownload+}++// FullName returns the full name of the item.+// name + extension+func (d *DriveItem) FullName() string {+ if d.Extension != "" {+ return d.Name + "." + d.Extension+ }+ return d.Name+}++// GetDocIDFromDriveID returns the DocumentID from the drive ID.+func GetDocIDFromDriveID(id string) string {+ split := strings.Split(id, "::")+ return split[len(split)-1]+}++// DeconstructDriveID returns the document type, zone, and document ID from the drive ID.+func DeconstructDriveID(id string) (docType, zone, docid string) {+ split := strings.Split(id, "::")+ if len(split) < 3 {+ return "", "", id+ }+ return split[0], split[1], split[2]+}++// ConstructDriveID constructs a drive ID from the given components.+func ConstructDriveID(id string, zone string, t string) string {+ return strings.Join([]string{t, zone, id}, "::")+}++// GetContentTypeForFile detects content type for given file name.+func GetContentTypeForFile(name string) string {+ // detect MIME type by looking at the filename only+ mimeType := mime.TypeByExtension(filepath.Ext(name))+ if mimeType == "" {+ // api requires a mime type passed in+ mimeType = "text/plain"+ }+ return strings.Split(mimeType, ";")[0]+}diff --git a/backend/iclouddrive/api/session.go b/backend/iclouddrive/api/session.gonew file mode 100644index 000000000..7ee350675--- /dev/null+++ b/backend/iclouddrive/api/session.go@@ -0,0 +1,412 @@+package api++import (+ "context"+ "fmt"+ "net/http"+ "net/url"+ "slices"+ "strings"++ "github.com/oracle/oci-go-sdk/v65/common"++ "github.com/rclone/rclone/fs/fshttp"+ "github.com/rclone/rclone/lib/rest"+)++// Session represents an iCloud session+type Session struct {+ SessionToken string `json:"session_token"`+ Scnt string `json:"scnt"`+ SessionID string `json:"session_id"`+ AccountCountry string `json:"account_country"`+ TrustToken string `json:"trust_token"`+ ClientID string `json:"client_id"`+ Cookies []*http.Cookie `json:"cookies"`+ AccountInfo AccountInfo `json:"account_info"`++ srv *rest.Client `json:"-"`+}++// String returns the session as a string+// func (s *Session) String() string {+// jsession, _ := json.Marshal(s)+// return string(jsession)+// }++// Request makes a request+func (s *Session) Request(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (*http.Response, error) {+ resp, err := s.srv.CallJSON(ctx, &opts, &request, &response)++ if err != nil {+ return resp, err+ }++ if val := resp.Header.Get("X-Apple-ID-Account-Country"); val != "" {+ s.AccountCountry = val+ }+ if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" {+ s.SessionID = val+ }+ if val := resp.Header.Get("X-Apple-Session-Token"); val != "" {+ s.SessionToken = val+ }+ if val := resp.Header.Get("X-Apple-TwoSV-Trust-Token"); val != "" {+ s.TrustToken = val+ }+ if val := resp.Header.Get("scnt"); val != "" {+ s.Scnt = val+ }++ return resp, nil+}++// Requires2FA returns true if the session requires 2FA+func (s *Session) Requires2FA() bool {+ return s.AccountInfo.DsInfo.HsaVersion == 2 && s.AccountInfo.HsaChallengeRequired+}++// SignIn signs in the session+func (s *Session) SignIn(ctx context.Context, appleID, password string) error {+ trustTokens := []string{}+ if s.TrustToken != "" {+ trustTokens = []string{s.TrustToken}+ }+ values := map[string]any{+ "accountName": appleID,+ "password": password,+ "rememberMe": true,+ "trustTokens": trustTokens,+ }+ body, err := IntoReader(values)+ if err != nil {+ return err+ }+ opts := rest.Opts{+ Method: "POST",+ Path: "/signin",+ Parameters: url.Values{},+ ExtraHeaders: s.GetAuthHeaders(map[string]string{}),+ RootURL: authEndpoint,+ IgnoreStatus: true, // need to handle 409 for hsa2+ NoResponse: true,+ Body: body,+ }+ opts.Parameters.Set("isRememberMeEnabled", "true")+ _, err = s.Request(ctx, opts, nil, nil)++ return err++}++// AuthWithToken authenticates the session+func (s *Session) AuthWithToken(ctx context.Context) error {+ values := map[string]any{+ "accountCountryCode": s.AccountCountry,+ "dsWebAuthToken": s.SessionToken,+ "extended_login": true,+ "trustToken": s.TrustToken,+ }+ body, err := IntoReader(values)+ if err != nil {+ return err+ }+ opts := rest.Opts{+ Method: "POST",+ Path: "/accountLogin",+ ExtraHeaders: GetCommonHeaders(map[string]string{}),+ RootURL: setupEndpoint,+ Body: body,+ }++ resp, err := s.Request(ctx, opts, nil, &s.AccountInfo)+ if err == nil {+ s.Cookies = resp.Cookies()+ }++ return err+}++// Validate2FACode validates the 2FA code+func (s *Session) Validate2FACode(ctx context.Context, code string) error {+ values := map[string]interface{}{"securityCode": map[string]string{"code": code}}+ body, err := IntoReader(values)+ if err != nil {+ return err+ }++ headers := s.GetAuthHeaders(map[string]string{})+ headers["scnt"] = s.Scnt+ headers["X-Apple-ID-Session-Id"] = s.SessionID++ opts := rest.Opts{+ Method: "POST",+ Path: "/verify/trusteddevice/securitycode",+ ExtraHeaders: headers,+ RootURL: authEndpoint,+ Body: body,+ NoResponse: true,+ }++ _, err = s.Request(ctx, opts, nil, nil)+ if err == nil {+ if err := s.TrustSession(ctx); err != nil {+ return err+ }++ return nil+ }++ return fmt.Errorf("validate2FACode failed: %w", err)+}++// TrustSession trusts the session+func (s *Session) TrustSession(ctx context.Context) error {+ headers := s.GetAuthHeaders(map[string]string{})+ headers["scnt"] = s.Scnt+ headers["X-Apple-ID-Session-Id"] = s.SessionID++ opts := rest.Opts{+ Method: "GET",+ Path: "/2sv/trust",+ ExtraHeaders: headers,+ RootURL: authEndpoint,+ NoResponse: true,+ ContentLength: common.Int64(0),+ }++ _, err := s.Request(ctx, opts, nil, nil)+ if err != nil {+ return fmt.Errorf("trustSession failed: %w", err)+ }++ return s.AuthWithToken(ctx)+}++// ValidateSession validates the session+func (s *Session) ValidateSession(ctx context.Context) error {+ opts := rest.Opts{+ Method: "POST",+ Path: "/validate",+ ExtraHeaders: s.GetHeaders(map[string]string{}),+ RootURL: setupEndpoint,+ ContentLength: common.Int64(0),+ }+ _, err := s.Request(ctx, opts, nil, &s.AccountInfo)+ if err != nil {+ return fmt.Errorf("validateSession failed: %w", err)+ }++ return nil+}++// GetAuthHeaders returns the authentication headers for the session.+//+// It takes an `overwrite` map[string]string parameter which allows+// overwriting the default headers. It returns a map[string]string.+func (s *Session) GetAuthHeaders(overwrite map[string]string) map[string]string {+ headers := map[string]string{+ "Accept": "application/json",+ "Content-Type": "application/json",+ "X-Apple-OAuth-Client-Id": s.ClientID,+ "X-Apple-OAuth-Client-Type": "firstPartyAuth",+ "X-Apple-OAuth-Redirect-URI": "https://www.icloud.com",+ "X-Apple-OAuth-Require-Grant-Code": "true",+ "X-Apple-OAuth-Response-Mode": "web_message",+ "X-Apple-OAuth-Response-Type": "code",+ "X-Apple-OAuth-State": s.ClientID,+ "X-Apple-Widget-Key": s.ClientID,+ "Origin": homeEndpoint,+ "Referer": fmt.Sprintf("%s/", homeEndpoint),+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",+ }+ for k, v := range overwrite {+ headers[k] = v+ }+ return headers+}++// GetHeaders Gets the authentication headers required for a request+func (s *Session) GetHeaders(overwrite map[string]string) map[string]string {+ headers := GetCommonHeaders(map[string]string{})+ headers["Cookie"] = s.GetCookieString()+ for k, v := range overwrite {+ headers[k] = v+ }+ return headers+}++// GetCookieString returns the cookie header string for the session.+func (s *Session) GetCookieString() string {+ cookieHeader := ""+ // we only care about name and value.+ for _, cookie := range s.Cookies {+ cookieHeader = cookieHeader + cookie.Name + "=" + cookie.Value + ";"+ }+ return cookieHeader+}++// GetCommonHeaders generates common HTTP headers with optional overwrite.+func GetCommonHeaders(overwrite map[string]string) map[string]string {+ headers := map[string]string{+ "Content-Type": "application/json",+ "Origin": baseEndpoint,+ "Referer": fmt.Sprintf("%s/", baseEndpoint),+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",+ }+ for k, v := range overwrite {+ headers[k] = v+ }+ return headers+}++// MergeCookies merges two slices of http.Cookies, ensuring no duplicates are added.+func MergeCookies(left []*http.Cookie, right []*http.Cookie) ([]*http.Cookie, error) {+ var hashes []string+ for _, cookie := range right {+ hashes = append(hashes, cookie.Raw)+ }+ for _, cookie := range left {+ if !slices.Contains(hashes, cookie.Raw) {+ right = append(right, cookie)+ }+ }+ return right, nil+}++// GetCookiesForDomain filters the provided cookies based on the domain of the given URL.+func GetCookiesForDomain(url *url.URL, cookies []*http.Cookie) ([]*http.Cookie, error) {+ var domainCookies []*http.Cookie+ for _, cookie := range cookies {+ if strings.HasSuffix(url.Host, cookie.Domain) {+ domainCookies = append(domainCookies, cookie)+ }+ }+ return domainCookies, nil+}++// NewSession creates a new Session instance with default values.+func NewSession() *Session {+ session := &Session{}+ session.srv = rest.NewClient(fshttp.NewClient(context.Background())).SetRoot(baseEndpoint)+ //session.ClientID = "auth-" + uuid.New().String()+ return session+}++// AccountInfo represents an account info+type AccountInfo struct {+ DsInfo *ValidateDataDsInfo `json:"dsInfo"`+ HasMinimumDeviceForPhotosWeb bool `json:"hasMinimumDeviceForPhotosWeb"`+ ICDPEnabled bool `json:"iCDPEnabled"`+ Webservices map[string]*webService `json:"webservices"`+ PcsEnabled bool `json:"pcsEnabled"`+ TermsUpdateNeeded bool `json:"termsUpdateNeeded"`+ ConfigBag struct {+ Urls struct {+ AccountCreateUI string `json:"accountCreateUI"`+ AccountLoginUI string `json:"accountLoginUI"`+ AccountLogin string `json:"accountLogin"`+ AccountRepairUI string `json:"accountRepairUI"`+ DownloadICloudTerms string `json:"downloadICloudTerms"`+ RepairDone string `json:"repairDone"`+ AccountAuthorizeUI string `json:"accountAuthorizeUI"`+ VettingURLForEmail string `json:"vettingUrlForEmail"`+ AccountCreate string `json:"accountCreate"`+ GetICloudTerms string `json:"getICloudTerms"`+ VettingURLForPhone string `json:"vettingUrlForPhone"`+ } `json:"urls"`+ AccountCreateEnabled bool `json:"accountCreateEnabled"`+ } `json:"configBag"`+ HsaTrustedBrowser bool `json:"hsaTrustedBrowser"`+ AppsOrder []string `json:"appsOrder"`+ Version int `json:"version"`+ IsExtendedLogin bool `json:"isExtendedLogin"`+ PcsServiceIdentitiesIncluded bool `json:"pcsServiceIdentitiesIncluded"`+ IsRepairNeeded bool `json:"isRepairNeeded"`+ HsaChallengeRequired bool `json:"hsaChallengeRequired"`+ RequestInfo struct {+ Country string `json:"country"`+ TimeZone string `json:"timeZone"`+ Region string `json:"region"`+ } `json:"requestInfo"`+ PcsDeleted bool `json:"pcsDeleted"`+ ICloudInfo struct {+ SafariBookmarksHasMigratedToCloudKit bool `json:"SafariBookmarksHasMigratedToCloudKit"`+ } `json:"iCloudInfo"`+ Apps map[string]*ValidateDataApp `json:"apps"`+}++// ValidateDataDsInfo represents an validation info+type ValidateDataDsInfo struct {+ HsaVersion int `json:"hsaVersion"`+ LastName string `json:"lastName"`+ ICDPEnabled bool `json:"iCDPEnabled"`+ TantorMigrated bool `json:"tantorMigrated"`+ Dsid string `json:"dsid"`+ HsaEnabled bool `json:"hsaEnabled"`+ IsHideMyEmailSubscriptionActive bool `json:"isHideMyEmailSubscriptionActive"`+ IroncadeMigrated bool `json:"ironcadeMigrated"`+ Locale string `json:"locale"`+ BrZoneConsolidated bool `json:"brZoneConsolidated"`+ ICDRSCapableDeviceList string `json:"ICDRSCapableDeviceList"`+ IsManagedAppleID bool `json:"isManagedAppleID"`+ IsCustomDomainsFeatureAvailable bool `json:"isCustomDomainsFeatureAvailable"`+ IsHideMyEmailFeatureAvailable bool `json:"isHideMyEmailFeatureAvailable"`+ ContinueOnDeviceEligibleDeviceInfo []string `json:"ContinueOnDeviceEligibleDeviceInfo"`+ Gilligvited bool `json:"gilligvited"`+ AppleIDAliases []interface{} `json:"appleIdAliases"`+ UbiquityEOLEnabled bool `json:"ubiquityEOLEnabled"`+ IsPaidDeveloper bool `json:"isPaidDeveloper"`+ CountryCode string `json:"countryCode"`+ NotificationID string `json:"notificationId"`+ PrimaryEmailVerified bool `json:"primaryEmailVerified"`+ ADsID string `json:"aDsID"`+ Locked bool `json:"locked"`+ ICDRSCapableDeviceCount int `json:"ICDRSCapableDeviceCount"`+ HasICloudQualifyingDevice bool `json:"hasICloudQualifyingDevice"`+ PrimaryEmail string `json:"primaryEmail"`+ AppleIDEntries []struct {+ IsPrimary bool `json:"isPrimary"`+ Type string `json:"type"`+ Value string `json:"value"`+ } `json:"appleIdEntries"`+ GilliganEnabled bool `json:"gilligan-enabled"`+ IsWebAccessAllowed bool `json:"isWebAccessAllowed"`+ FullName string `json:"fullName"`+ MailFlags struct {+ IsThreadingAvailable bool `json:"isThreadingAvailable"`+ IsSearchV2Provisioned bool `json:"isSearchV2Provisioned"`+ SCKMail bool `json:"sCKMail"`+ IsMppSupportedInCurrentCountry bool `json:"isMppSupportedInCurrentCountry"`+ } `json:"mailFlags"`+ LanguageCode string `json:"languageCode"`+ AppleID string `json:"appleId"`+ HasUnreleasedOS bool `json:"hasUnreleasedOS"`+ AnalyticsOptInStatus bool `json:"analyticsOptInStatus"`+ FirstName string `json:"firstName"`+ ICloudAppleIDAlias string `json:"iCloudAppleIdAlias"`+ NotesMigrated bool `json:"notesMigrated"`+ BeneficiaryInfo struct {+ IsBeneficiary bool `json:"isBeneficiary"`+ } `json:"beneficiaryInfo"`+ HasPaymentInfo bool `json:"hasPaymentInfo"`+ PcsDelet bool `json:"pcsDelet"`+ AppleIDAlias string `json:"appleIdAlias"`+ BrMigrated bool `json:"brMigrated"`+ StatusCode int `json:"statusCode"`+ FamilyEligible bool `json:"familyEligible"`+}++// ValidateDataApp represents an app+type ValidateDataApp struct {+ CanLaunchWithOneFactor bool `json:"canLaunchWithOneFactor"`+ IsQualifiedForBeta bool `json:"isQualifiedForBeta"`+}++// WebService represents a web service+type webService struct {+ PcsRequired bool `json:"pcsRequired"`+ URL string `json:"url"`+ UploadURL string `json:"uploadUrl"`+ Status string `json:"status"`+}diff --git a/backend/iclouddrive/iclouddrive.go b/backend/iclouddrive/iclouddrive.gonew file mode 100644index 000000000..591d81718--- /dev/null+++ b/backend/iclouddrive/iclouddrive.go@@ -0,0 +1,1174 @@+//go:build !plan9 && !solaris++// Package iclouddrive implements the iCloud Drive backend+package iclouddrive++import (+ "bytes"+ "context"+ "path"++ "errors"+ "fmt"+ "io"+ "net/http"+ "strings"+ "time"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config"+ "github.com/rclone/rclone/fs/config/configmap"+ "github.com/rclone/rclone/fs/config/configstruct"+ "github.com/rclone/rclone/fs/config/obscure"+ "github.com/rclone/rclone/fs/fserrors"++ "github.com/rclone/rclone/backend/iclouddrive/api"+ "github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/lib/dircache"+ "github.com/rclone/rclone/lib/encoder"+ "github.com/rclone/rclone/lib/pacer"+)++/*+- dirCache operates on relative path to root+- path sanitization+ - rule of thumb: sanitize before use, but store things as-is+ - the paths cached in dirCache are after sanitizing+ - the remote/dir passed in aren't, and are stored as-is+*/++const (+ configAppleID = "apple_id"+ configPassword = "password"+ configClientID = "client_id"+ configCookies = "cookies"+ configTrustToken = "trust_token"++ minSleep = 10 * time.Millisecond+ maxSleep = 2 * time.Second+ decayConstant = 2+)++// Register with Fs+func init() {+ fs.Register(&fs.RegInfo{+ Name: "iclouddrive",+ Description: "iCloud Drive",+ Config: Config,+ NewFs: NewFs,+ Options: []fs.Option{{+ Name: configAppleID,+ Help: "Apple ID.",+ Required: true,+ Sensitive: true,+ }, {+ Name: configPassword,+ Help: "Password.",+ Required: true,+ IsPassword: true,+ Sensitive: true,+ }, {+ Name: configTrustToken,+ Help: "Trust token (internal use)",+ IsPassword: false,+ Required: false,+ Sensitive: true,+ Hide: fs.OptionHideBoth,+ }, {+ Name: configCookies,+ Help: "cookies (internal use only)",+ Required: false,+ Advanced: false,+ Sensitive: true,+ Hide: fs.OptionHideBoth,+ }, {+ Name: configClientID,+ Help: "Client id",+ Required: false,+ Advanced: true,+ Default: "d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d",+ }, {+ Name: config.ConfigEncoding,+ Help: config.ConfigEncodingHelp,+ Advanced: true,+ Default: (encoder.Display |+ //encoder.EncodeDot |+ encoder.EncodeBackSlash |+ encoder.EncodeInvalidUtf8),+ }},+ })+}++// Options defines the configuration for this backend+type Options struct {+ AppleID string `config:"apple_id"`+ Password string `config:"password"`+ Photos bool `config:"photos"`+ TrustToken string `config:"trust_token"`+ Cookies string `config:"cookies"`+ ClientID string `config:"client_id"`+ Enc encoder.MultiEncoder `config:"encoding"`+}++// Fs represents a remote icloud drive+type Fs struct {+ name string // name of this remote+ root string // the path we are working on.+ rootID string+ opt Options // parsed config options+ features *fs.Features // optional features+ dirCache *dircache.DirCache // Map of directory path to directory id+ icloud *api.Client+ service *api.DriveService+ pacer *fs.Pacer // pacer for API calls+}++// Object describes an icloud drive object+type Object struct {+ fs *Fs // what this object is part of+ remote string // The remote path (relative to the fs.root)+ size int64 // size of the object (on server, after encryption)+ modTime time.Time // modification time of the object+ createdTime time.Time // creation time of the object+ driveID string // item ID of the object+ docID string // document ID of the object+ itemID string // item ID of the object+ etag string+ downloadURL string+}++// Config configures the iCloud remote.+func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {+ var err error+ appleid, _ := m.Get(configAppleID)+ if appleid == "" {+ return nil, errors.New("a apple ID is required")+ }++ password, _ := m.Get(configPassword)+ if password != "" {+ password, err = obscure.Reveal(password)+ if err != nil {+ return nil, err+ }+ }++ trustToken, _ := m.Get(configTrustToken)+ cookieRaw, _ := m.Get(configCookies)+ clientID, _ := m.Get(configClientID)+ cookies := ReadCookies(cookieRaw)++ switch config.State {+ case "":+ icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil)+ if err != nil {+ return nil, err+ }+ if err := icloud.Authenticate(ctx); err != nil {+ return nil, err+ }+ m.Set(configCookies, icloud.Session.GetCookieString())+ if icloud.Session.Requires2FA() {+ return fs.ConfigInput("2fa_do", "config_2fa", "Two-factor authentication: please enter your 2FA code")+ }+ return nil, nil+ case "2fa_do":+ code := config.Result+ if code == "" {+ return fs.ConfigError("authenticate", "2FA codes can't be blank")+ }++ icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil)+ if err != nil {+ return nil, err+ }+ if err := icloud.SignIn(ctx); err != nil {+ return nil, err+ }++ if err := icloud.Session.Validate2FACode(ctx, code); err != nil {+ return nil, err+ }++ m.Set(configTrustToken, icloud.Session.TrustToken)+ m.Set(configCookies, icloud.Session.GetCookieString())+ return nil, nil++ case "2fa_error":+ if config.Result == "true" {+ return fs.ConfigGoto("2fa")+ }+ return nil, errors.New("2fa authentication failed")+ }+ return nil, fmt.Errorf("unknown state %q", config.State)+}++// find item by path. Will not return any children for the item+func (f *Fs) findItem(ctx context.Context, dir string) (item *api.DriveItem, found bool, err error) {+ var resp *http.Response+ if err = f.pacer.Call(func() (bool, error) {+ item, resp, err = f.service.GetItemByPath(ctx, path.Join(f.root, dir))+ return shouldRetry(ctx, resp, err)+ }); err != nil {+ if item == nil && resp.StatusCode == 404 {+ return nil, false, nil+ }+ return nil, false, err+ }++ return item, true, nil+}++func (f *Fs) findLeafItem(ctx context.Context, pathID string, leaf string) (item *api.DriveItem, found bool, err error) {+ items, err := f.listAll(ctx, pathID)+ if err != nil {+ return nil, false, err+ }+ for _, item := range items {+ if strings.EqualFold(item.FullName(), leaf) {+ return item, true, nil+ }+ }++ return nil, false, nil++}++// FindLeaf finds a directory of name leaf in the folder with ID pathID+func (f *Fs) FindLeaf(ctx context.Context, pathID string, leaf string) (pathIDOut string, found bool, err error) {+ item, found, err := f.findLeafItem(ctx, pathID, leaf)++ if err != nil {+ return "", found, err+ }++ if !found {+ return "", false, err+ }++ if !item.IsFolder() {+ return "", false, fs.ErrorIsFile+ }++ return f.IDJoin(item.Drivewsid, item.Etag), true, nil+}++// Features implements fs.Fs.+func (f *Fs) Features() *fs.Features {+ return f.features+}++// Hashes are not exposed anywhere+func (f *Fs) Hashes() hash.Set {+ return hash.Set(hash.None)+}++func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {+ root := path.Join(f.root, dir)+ if root == "" {+ return errors.New("can't purge root directory")+ }++ directoryID, etag, err := f.FindDir(ctx, dir, false)+ if err != nil {+ return err+ }++ if check {+ item, found, err := f.findItem(ctx, dir)+ if err != nil {+ return err+ }++ if found && item.DirectChildrenCount > 0 {+ return fs.ErrorDirectoryNotEmpty+ }+ }++ var _ *api.DriveItem+ var resp *http.Response+ if err = f.pacer.Call(func() (bool, error) {+ _, resp, err = f.service.MoveItemToTrashByID(ctx, directoryID, etag, true)+ return retryResultUnknown(ctx, resp, err)+ }); err != nil {+ return err+ }++ // flush everything from the left of the dir+ f.dirCache.FlushDir(dir)++ return nil+}++// Purge all files in the directory specified+//+// Implement this if you have a way of deleting all the files+// quicker than just running Remove() on the result of List()+//+// Return an error if it doesn't exist+func (f *Fs) Purge(ctx context.Context, dir string) error {+ if dir == "" {+ return fs.ErrorCantPurge+ }+ return f.purgeCheck(ctx, dir, false)+}++func (f *Fs) listAll(ctx context.Context, dirID string) (items []*api.DriveItem, err error) {+ var item *api.DriveItem+ var resp *http.Response++ if err = f.pacer.Call(func() (bool, error) {+ id, _ := f.parseNormalizedID(dirID)+ item, resp, err = f.service.GetItemByDriveID(ctx, id, true)+ return shouldRetry(ctx, resp, err)+ }); err != nil {+ return nil, err+ }++ items = item.Items++ for i, item := range items {+ item.Name = f.opt.Enc.ToStandardName(item.Name)+ item.Extension = f.opt.Enc.ToStandardName(item.Extension)+ items[i] = item+ }++ return items, nil+}++// List implements fs.Fs.+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {+ dirRemoteID, err := f.dirCache.FindDir(ctx, dir, false)+ if err != nil {+ return nil, err+ }++ entries = make(fs.DirEntries, 0)+ items, err := f.listAll(ctx, dirRemoteID)++ if err != nil {+ return nil, err+ }++ for _, item := range items {+ id := item.Drivewsid+ name := item.FullName()+ remote := path.Join(dir, name)+ if item.IsFolder() {+ jid := f.putFolderCache(id, item.Etag, remote)+ d := fs.NewDir(remote, item.DateModified).SetID(jid).SetSize(item.AssetQuota)+ entries = append(entries, d)+ } else {+ o, err := f.NewObjectFromDriveItem(ctx, remote, item)+ if err != nil {+ return nil, err+ }+ entries = append(entries, o)+ }+ }++ return entries, nil+}++// Mkdir implements fs.Fs.+func (f *Fs) Mkdir(ctx context.Context, dir string) error {+ _, _, err := f.FindDir(ctx, dir, true)+ return err+}++// Name implements fs.Fs.+func (f *Fs) Name() string {+ return f.name+}++// Precision implements fs.Fs.+func (f *Fs) Precision() time.Duration {+ return time.Second+}++// Copy src to this remote using server-side copy operations.+//+// This is stored with the remote path given.+//+// It returns the destination Object and a possible error.+//+// Will only be called if src.Fs().Name() == f.Name()+//+// If it isn't possible then return fs.ErrorCantCopy+//+//nolint:all+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {+ // ICloud cooy endpoint is broken. Once they fixed it this can be re-enabled.+ return nil, fs.ErrorCantCopy++ // note: so many calls its only just faster then a reupload for big files.+ srcObj, ok := src.(*Object)+ if !ok {+ fs.Debugf(src, "Can't copy - not same remote type")+ return nil, fs.ErrorCantCopy+ }++ file, pathID, _, err := f.FindPath(ctx, remote, true)+ if err != nil {+ return nil, err+ }++ var resp *http.Response+ var info *api.DriveItemRaw++ // make a copy+ if err = f.pacer.Call(func() (bool, error) {+ info, resp, err = f.service.CopyDocByItemID(ctx, srcObj.itemID)+ return retryResultUnknown(ctx, resp, err)+ }); err != nil {+ return nil, err+ }++ // renaming in CopyDocByID endpoint does not work :/ so do it the hard way++ // get new document+ var doc *api.Document+ if err = f.pacer.Call(func() (bool, error) {+ doc, resp, err = f.service.GetDocByItemID(ctx, info.ItemID)+ return shouldRetry(ctx, resp, err)+ }); err != nil {+ return nil, err+ }++ // get parentdrive id+ var dirDoc *api.Document+ if err = f.pacer.Call(func() (bool, error) {+ dirDoc, resp, err = f.service.GetDocByItemID(ctx, pathID)+ return shouldRetry(ctx, resp, err)+ }); err != nil {+ return nil, err+ }++ // build request+ // cant use normal rename as file needs to be "activated" first++ r := api.NewUpdateFileInfo()+ r.DocumentID = doc.DocumentID+ r.Path.Path = file+ r.Path.StartingDocumentID = dirDoc.DocumentID+ r.Data.Signature = doc.Data.Signature+ r.Data.ReferenceSignature = doc.Data.ReferenceSignature+ r.Data.WrappingKey = doc.Data.WrappingKey+ r.Data.Size = doc.Data.Size+ r.Mtime = srcObj.modTime.UnixMilli()+ r.Btime = srcObj.modTime.UnixMilli()++ var item *api.DriveItem+ if err = f.pacer.Call(func() (bool, error) {+ item, resp, err = f.service.UpdateFile(ctx, &r)+ return retryResultUnknown(ctx, resp, err)+ }); err != nil {+ return nil, err+ }++ o, err := f.NewObjectFromDriveItem(ctx, remote, item)+ if err != nil {+ return nil, err+ }+ obj := o.(*Object)++ // cheat unit tests+ obj.modTime = srcObj.modTime+ obj.createdTime = srcObj.createdTime++ return obj, nil+}++// Put in to the remote path with the modTime given of the given size+//+// When called from outside an Fs by rclone, src.Size() will always be >= 0.+// But for unknown-sized objects (indicated by src.Size() == -1), Put should either+// return an error or upload it properly (rather than e.g. calling panic).+//+// May create the object even if it returns an error - if so+// will return the object and the error, otherwise will return+// nil and the error+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {+ size := src.Size()+ if size < 0 {+ return nil, errors.New("file size unknown")+ }+ existingObj, err := f.NewObject(ctx, src.Remote())+ switch err {+ case nil:+ // object is found+ return existingObj, existingObj.Update(ctx, in, src, options...)+ case fs.ErrorObjectNotFound:+ // object not found, so we need to create it+ remote := src.Remote()+ size := src.Size()+ modTime := src.ModTime(ctx)++ obj, err := f.createObject(ctx, remote, modTime, size)+ if err != nil {+ return nil, err+ }+ return obj, obj.Update(ctx, in, src, options...)+ default:+ // real error caught+ return nil, err+ }+}++// DirCacheFlush resets the directory cache - used in testing as an+// optional interface+func (f *Fs) DirCacheFlush() {+ f.dirCache.ResetRoot()+}++// parseNormalizedID parses a normalized ID (may be in the form `driveID#itemID` or just `itemID`)+// and returns itemID, driveID, rootURL.+// Such a normalized ID can come from (*Item).GetID()+//+// Parameters:+// - rid: the normalized ID to be parsed+//+// Returns:+// - id: the itemID extracted from the normalized ID+// - etag: the driveID extracted from the normalized ID, or an empty string if not present+func (f *Fs) parseNormalizedID(rid string) (id string, etag string) {+ split := strings.Split(rid, "#")+ if len(split) == 1 {+ return split[0], ""+ }+ return split[0], split[1]+}++// FindPath finds the leaf and directoryID from a normalized path+func (f *Fs) FindPath(ctx context.Context, remote string, create bool) (leaf, directoryID, etag string, err error) {+ leaf, jDirectoryID, err := f.dirCache.FindPath(ctx, remote, create)+ if err != nil {+ return "", "", "", err+ }+ directoryID, etag = f.parseNormalizedID(jDirectoryID)+ return leaf, directoryID, etag, nil+}++// FindDir finds the directory passed in returning the directory ID+// starting from pathID+func (f *Fs) FindDir(ctx context.Context, path string, create bool) (pathID string, etag string, err error) {+ jDirectoryID, err := f.dirCache.FindDir(ctx, path, create)+ if err != nil {+ return "", "", err+ }+ directoryID, etag := f.parseNormalizedID(jDirectoryID)+ return directoryID, etag, nil+}++// IDJoin joins the given ID and ETag into a single string with a "#" delimiter.+func (f *Fs) IDJoin(id string, etag string) string {+ if strings.Contains(id, "#") {+ // already contains an etag, replace+ id, _ = f.parseNormalizedID(id)+ }++ return strings.Join([]string{id, etag}, "#")+}++func (f *Fs) putFolderCache(id, etag, remote string) string {+ jid := f.IDJoin(id, etag)+ f.dirCache.Put(remote, f.IDJoin(id, etag))+ return jid+}++// Rmdir implements fs.Fs.+func (f *Fs) Rmdir(ctx context.Context, dir string) error {+ return f.purgeCheck(ctx, dir, true)+}++// Root implements fs.Fs.+func (f *Fs) Root() string {+ return f.opt.Enc.ToStandardPath(f.root)+}++// String implements fs.Fs.+func (f *Fs) String() string {+ return f.root+}++// CreateDir makes a directory with pathID as parent and name leaf+//+// This should be implemented by the backend and will be called by the+// dircache package when appropriate.+func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (string, error) {+ var item *api.DriveItem+ var err error+ var found bool+ var resp *http.Response+ if err = f.pacer.Call(func() (bool, error) {+ id, _ := f.parseNormalizedID(pathID)+ item, resp, err = f.service.CreateNewFolderByDriveID(ctx, id, f.opt.Enc.FromStandardName(leaf))++ // check if it went oke+ if requestError, ok := err.(*api.RequestError); ok {+ if requestError.Status == "unknown" {+ fs.Debugf(requestError, " checking if dir is created with separate call.")+ time.Sleep(1 * time.Second) // sleep to give icloud time to clear up its mind+ item, found, err = f.findLeafItem(ctx, pathID, leaf)+ if err != nil {+ return false, err+ }++ if !found {+ // lets assume it failed and retry+ return true, err+ }++ // success, clear err+ err = nil+ }+ }++ return ignoreResultUnknown(ctx, resp, err)+ }); err != nil {+ return "", err+ }++ return f.IDJoin(item.Drivewsid, item.Etag), err+}++// DirMove moves src, srcRemote to this remote at dstRemote+// using server-side move operations.+//+// Will only be called if src.Fs().Name() == f.Name()+//+// If it isn't possible then return fs.ErrorCantDirMove+//+// If destination exists then return fs.ErrorDirExists+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {+ srcFs, ok := src.(*Fs)+ if !ok {+ fs.Debugf(srcFs, "Can't move directory - not same remote type")+ return fs.ErrorCantDirMove+ }++ srcID, jsrcDirectoryID, srcLeaf, jdstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)+ if err != nil {+ return err+ }++ srcDirectoryID, srcEtag := f.parseNormalizedID(jsrcDirectoryID)+ dstDirectoryID, _ := f.parseNormalizedID(jdstDirectoryID)++ _, err = f.move(ctx, srcID, srcDirectoryID, srcLeaf, srcEtag, dstDirectoryID, dstLeaf)+ if err != nil {+ return err+ }++ srcFs.dirCache.FlushDir(srcRemote)++ return nil+}++func (f *Fs) move(ctx context.Context, ID, srcDirectoryID, srcLeaf, srcEtag, dstDirectoryID, dstLeaf string) (*api.DriveItem, error) {+ var resp *http.Response+ var item *api.DriveItem+ var err error++ // move+ if srcDirectoryID != dstDirectoryID {+ if err = f.pacer.Call(func() (bool, error) {+ id, _ := f.parseNormalizedID(ID)+ item, resp, err = f.service.MoveItemByDriveID(ctx, id, srcEtag, dstDirectoryID, true)+ return ignoreResultUnknown(ctx, resp, err)+ }); err != nil {+ return nil, err+ }+ ID = item.Drivewsid+ srcEtag = item.Etag+ }++ // rename+ if srcLeaf != dstLeaf {+ if err = f.pacer.Call(func() (bool, error) {+ id, _ := f.parseNormalizedID(ID)+ item, resp, err = f.service.RenameItemByDriveID(ctx, id, srcEtag, dstLeaf, true)+ return ignoreResultUnknown(ctx, resp, err)+ }); err != nil {+ return item, err+ }+ }++ return item, err+}++// Move moves the src object to the specified remote.+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {+ srcObj, ok := src.(*Object)+ if !ok {+ fs.Debugf(src, "Can't move - not same remote type")+ return nil, fs.ErrorCantMove+ }++ srcLeaf, srcDirectoryID, _, err := srcObj.fs.FindPath(ctx, srcObj.remote, true)+ if err != nil {+ return nil, err+ }++ dstLeaf, dstDirectoryID, _, err := f.FindPath(ctx, remote, true)+ if err != nil {+ return nil, err+ }++ item, err := f.move(ctx, srcObj.driveID, srcDirectoryID, srcLeaf, srcObj.etag, dstDirectoryID, dstLeaf)+ if err != nil {+ return src, err+ }++ return f.NewObjectFromDriveItem(ctx, remote, item)+}++// Creates from the parameters passed in a half finished Object which+// must have setMetaData called on it+//+// Returns the object, leaf, directoryID and error.+//+// Used to create new objects+func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, err error) {+ // Create the directory for the object if it doesn't exist+ _, _, _, err = f.FindPath(ctx, remote, true)+ if err != nil {+ return+ }+ // Temporary Object under construction+ o = &Object{+ fs: f,+ remote: remote,+ modTime: modTime,+ size: size,+ }+ return o, nil+}++// ReadCookies parses the raw cookie string and returns an array of http.Cookie objects.+func ReadCookies(raw string) []*http.Cookie {+ header := http.Header{}+ header.Add("Cookie", raw)+ request := http.Request{Header: header}+ return request.Cookies()+}++var retryErrorCodes = []int{+ 400, // icloud is a mess, sometimes returns 400 on a perfectly fine request. So just retry+ 408, // Request Timeout+ 409, // Conflict, retry could fix it.+ 429, // Rate exceeded.+ 500, // Get occasional 500 Internal Server Error+ 502, // Server overload+ 503, // Service Unavailable+ 504, // Gateway Time-out+}++func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {+ if fserrors.ContextError(ctx, &err) {+ return false, err+ }++ return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err+}++func ignoreResultUnknown(ctx context.Context, resp *http.Response, err error) (bool, error) {+ if requestError, ok := err.(*api.RequestError); ok {+ if requestError.Status == "unknown" {+ fs.Debugf(requestError, " ignoring.")+ return false, nil+ }+ }+ return shouldRetry(ctx, resp, err)+}++func retryResultUnknown(ctx context.Context, resp *http.Response, err error) (bool, error) {+ if requestError, ok := err.(*api.RequestError); ok {+ if requestError.Status == "unknown" {+ fs.Debugf(requestError, " retrying.")+ return true, err+ }+ }+ return shouldRetry(ctx, resp, err)+}++// NewFs constructs an Fs from the path, container:path+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {+ // Parse config into Options struct+ opt := new(Options)+ err := configstruct.Set(m, opt)+ if err != nil {+ return nil, err+ }++ if opt.Password != "" {+ var err error+ opt.Password, err = obscure.Reveal(opt.Password)+ if err != nil {+ return nil, fmt.Errorf("couldn't decrypt user password: %w", err)+ }+ }++ if opt.TrustToken == "" {+ return nil, fmt.Errorf("missing icloud trust token: try refreshing it with \"rclone config reconnect %s:\"", name)+ }++ cookies := ReadCookies(opt.Cookies)++ callback := func(session *api.Session) {+ m.Set(configCookies, session.GetCookieString())+ }++ icloud, err := api.New(+ opt.AppleID,+ opt.Password,+ opt.TrustToken,+ opt.ClientID,+ cookies,+ callback,+ )+ if err != nil {+ return nil, err+ }++ if err := icloud.Authenticate(ctx); err != nil {+ return nil, err+ }++ if icloud.Session.Requires2FA() {+ return nil, errors.New("trust token expired, please reauth")+ }++ root = strings.Trim(root, "/")++ f := &Fs{+ name: name,+ root: root,+ icloud: icloud,+ rootID: "FOLDER::com.apple.CloudDocs::root",+ opt: *opt,+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),+ }+ f.features = (&fs.Features{+ CanHaveEmptyDirectories: true,+ PartialUploads: false,+ }).Fill(ctx, f)++ rootID := f.rootID+ f.service, err = icloud.DriveService()+ if err != nil {+ return nil, err+ }++ f.dirCache = dircache.New(+ root,+ rootID,+ f,+ )++ err = f.dirCache.FindRoot(ctx, false)+ if err != nil {+ // Assume it is a file+ newRoot, remote := dircache.SplitPath(root)+ tempF := *f+ tempF.dirCache = dircache.New(newRoot, rootID, &tempF)+ tempF.root = newRoot+ // Make new Fs which is the parent+ err = tempF.dirCache.FindRoot(ctx, false)+ if err != nil {+ // No root so return old f+ return f, nil+ }++ _, err := tempF.NewObject(ctx, remote)+ if err != nil {+ if err == fs.ErrorObjectNotFound {+ // File doesn't exist so return old f+ return f, nil+ }++ return nil, err+ }++ f.dirCache = tempF.dirCache+ f.root = tempF.root+ // return an error with an fs which points to the parent+ return f, fs.ErrorIsFile+ }++ return f, nil+}++// NewObject creates a new fs.Object from a given remote string.+//+// ctx: The context.Context for the function.+// remote: The remote string representing the object's location.+// Returns an fs.Object and an error.+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {+ return f.NewObjectFromDriveItem(ctx, remote, nil)+}++// NewObjectFromDriveItem creates a new fs.Object from a given remote string and DriveItem.+//+// ctx: The context.Context for the function.+// remote: The remote string representing the object's location.+// item: The optional DriveItem to use for initializing the Object. If nil, the function will read the metadata from the remote location.+// Returns an fs.Object and an error.+func (f *Fs) NewObjectFromDriveItem(ctx context.Context, remote string, item *api.DriveItem) (fs.Object, error) {+ o := &Object{+ fs: f,+ remote: remote,+ }+ if item != nil {+ err := o.setMetaData(item)+ if err != nil {+ return nil, err+ }+ } else {+ item, err := f.readMetaData(ctx, remote)++ if err != nil {+ return nil, err+ }++ err = o.setMetaData(item)+ if err != nil {+ return nil, err+ }+ }++ return o, nil+}++func (f *Fs) readMetaData(ctx context.Context, path string) (item *api.DriveItem, err error) {+ leaf, ID, _, err := f.FindPath(ctx, path, false)++ if err != nil {+ if err == fs.ErrorDirNotFound {+ return nil, fs.ErrorObjectNotFound+ }+ return nil, err+ }++ item, found, err := f.findLeafItem(ctx, ID, leaf)++ if err != nil {+ return nil, err+ }++ if !found {+ return nil, fs.ErrorObjectNotFound+ }++ return item, nil+}++func (o *Object) setMetaData(item *api.DriveItem) (err error) {+ if item.IsFolder() {+ return fs.ErrorIsDir+ }+ o.size = item.Size+ o.modTime = item.DateModified+ o.createdTime = item.DateCreated+ o.driveID = item.Drivewsid+ o.docID = item.Docwsid+ o.itemID = item.Itemid+ o.etag = item.Etag+ o.downloadURL = item.DownloadURL()+ return nil+}++// ID returns the ID of the Object if known, or "" if not+func (o *Object) ID() string {+ return o.driveID+}++// Fs implements fs.Object.+func (o *Object) Fs() fs.Info {+ return o.fs+}++// Hash implements fs.Object.+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {+ return "", hash.ErrUnsupported+}++// ModTime implements fs.Object.+func (o *Object) ModTime(context.Context) time.Time {+ return o.modTime+}++// Open implements fs.Object.+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {+ fs.FixRangeOption(options, o.size)++ // Drive does not support empty files, so we cheat+ if o.size == 0 {+ return io.NopCloser(bytes.NewBufferString("")), nil+ }++ var resp *http.Response+ var err error++ if err = o.fs.pacer.Call(func() (bool, error) {+ var url string++ //var doc *api.Document+ //if o.docID == "" {+ //doc, resp, err = o.fs.service.GetDocByItemID(ctx, o.itemID)+ //}++ // Can not get the download url on a item to work, so do it the hard way.+ url, _, err = o.fs.service.GetDownloadURLByDriveID(ctx, o.driveID)++ resp, err = o.fs.service.DownloadFile(ctx, url, options)+ return shouldRetry(ctx, resp, err)+ }); err != nil {+ return nil, err+ }++ return resp.Body, err+}++// Remote implements fs.Object.+func (o *Object) Remote() string {+ return o.remote+}++// Remove implements fs.Object.+func (o *Object) Remove(ctx context.Context) error {+ if o.itemID == "" {+ return nil+ }++ var resp *http.Response+ var err error+ if err = o.fs.pacer.Call(func() (bool, error) {+ _, resp, err = o.fs.service.MoveItemToTrashByID(ctx, o.driveID, o.etag, true)+ return retryResultUnknown(ctx, resp, err)+ }); err != nil {+ return err+ }++ return nil+}++// SetModTime implements fs.Object.+func (o *Object) SetModTime(ctx context.Context, t time.Time) error {+ return fs.ErrorCantSetModTime+}++// Size implements fs.Object.+func (o *Object) Size() int64 {+ return o.size+}++// Storable implements fs.Object.+func (o *Object) Storable() bool {+ return true+}++// String implements fs.Object.+func (o *Object) String() string {+ if o == nil {+ return "<nil>"+ }+ return o.remote+}++// Update implements fs.Object.+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {+ size := src.Size()+ if size < 0 {+ return errors.New("file size unknown")+ }++ remote := o.Remote()+ modTime := src.ModTime(ctx)++ leaf, dirID, _, err := o.fs.FindPath(ctx, path.Clean(remote), true)+ if err != nil {+ return err+ }++ // Move current file to trash+ if o.driveID != "" {+ err = o.Remove(ctx)+ if err != nil {+ return err+ }+ }++ name := o.fs.opt.Enc.FromStandardName(leaf)+ var resp *http.Response++ // Create document+ var uploadInfo *api.UploadResponse+ if err = o.fs.pacer.Call(func() (bool, error) {+ uploadInfo, resp, err = o.fs.service.CreateUpload(ctx, size, name)+ return ignoreResultUnknown(ctx, resp, err)+ }); err != nil {+ return err+ }++ // Upload content+ var upload *api.SingleFileResponse+ if err = o.fs.pacer.Call(func() (bool, error) {+ upload, resp, err = o.fs.service.Upload(ctx, in, size, name, uploadInfo.URL)+ return ignoreResultUnknown(ctx, resp, err)+ }); err != nil {+ return err+ }++ //var doc *api.Document+ //if err = o.fs.pacer.Call(func() (bool, error) {+ // doc, resp, err = o.fs.service.GetDocByItemID(ctx, dirID)+ // return ignoreResultUnknown(ctx, resp, err)+ //}); err != nil {+ // return err+ //}++ r := api.NewUpdateFileInfo()+ r.DocumentID = uploadInfo.DocumentID+ r.Path.Path = name+ r.Path.StartingDocumentID = api.GetDocIDFromDriveID(dirID)+ //r.Path.StartingDocumentID = doc.DocumentID+ r.Data.Receipt = upload.SingleFile.Receipt+ r.Data.Signature = upload.SingleFile.Signature+ r.Data.ReferenceSignature = upload.SingleFile.ReferenceSignature+ r.Data.WrappingKey = upload.SingleFile.WrappingKey+ r.Data.Size = upload.SingleFile.Size+ r.Mtime = modTime.Unix() * 1000+ r.Btime = modTime.Unix() * 1000++ // Update metadata+ var item *api.DriveItem+ if err = o.fs.pacer.Call(func() (bool, error) {+ item, resp, err = o.fs.service.UpdateFile(ctx, &r)+ return ignoreResultUnknown(ctx, resp, err)+ }); err != nil {+ return err+ }++ err = o.setMetaData(item)+ if err != nil {+ return err+ }++ o.modTime = modTime+ o.size = src.Size()++ return nil+}++// Check interfaces are satisfied+var (+ _ fs.Fs = &Fs{}+ _ fs.Mover = (*Fs)(nil)+ _ fs.Purger = (*Fs)(nil)+ _ fs.DirMover = (*Fs)(nil)+ _ fs.DirCacheFlusher = (*Fs)(nil)+ _ fs.Copier = (*Fs)(nil)+ _ fs.Object = &Object{}+ _ fs.IDer = (*Object)(nil)+)diff --git a/backend/iclouddrive/iclouddrive_test.go b/backend/iclouddrive/iclouddrive_test.gonew file mode 100644index 000000000..13db708ea--- /dev/null+++ b/backend/iclouddrive/iclouddrive_test.go@@ -0,0 +1,18 @@+//go:build !plan9 && !solaris++package iclouddrive_test++import (+ "testing"++ "github.com/rclone/rclone/backend/iclouddrive"+ "github.com/rclone/rclone/fstest/fstests"+)++// TestIntegration runs integration tests against the remote+func TestIntegration(t *testing.T) {+ fstests.Run(t, &fstests.Opt{+ RemoteName: "TestICloudDrive:",+ NilObject: (*iclouddrive.Object)(nil),+ })+}diff --git a/backend/iclouddrive/iclouddrive_unsupported.go b/backend/iclouddrive/iclouddrive_unsupported.gonew file mode 100644index 000000000..2eeb6b639--- /dev/null+++ b/backend/iclouddrive/iclouddrive_unsupported.go@@ -0,0 +1,7 @@+// Build for iclouddrive for unsupported platforms to stop go complaining+// about "no buildable Go source files "++//go:build plan9 || solaris++// Package iclouddrive implements the iCloud Drive backend+package iclouddrive\ No newline at end of filediff --git a/bin/make_manual.py b/bin/make_manual.pyindex c28398dfd..990ab7130 100755--- a/bin/make_manual.py+++ b/bin/make_manual.py@@ -52,6 +52,7 @@ docs = ["hidrive.md","http.md","imagekit.md",+ "iclouddrive.md","internetarchive.md","jottacloud.md","koofr.md",diff --git a/docs/content/_index.md b/docs/content/_index.mdindex 2fa03fb06..5bb813a0b 100644--- a/docs/content/_index.md+++ b/docs/content/_index.md@@ -132,6 +132,7 @@ WebDAV or S3, that work out of the box.){{< provider name="Hetzner Storage Box" home="https://www.hetzner.com/storage/storage-box" config="/sftp/#hetzner-storage-box" >}}{{< provider name="HiDrive" home="https://www.strato.de/cloud-speicher/" config="/hidrive/" >}}{{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}}+{{< provider name="iCloud Drive" home="https://icloud.com/" config="/iclouddrive/" >}}{{< provider name="ImageKit" home="https://imagekit.io" config="/imagekit/" >}}{{< provider name="Internet Archive" home="https://archive.org/" config="/internetarchive/" >}}{{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}}diff --git a/docs/content/docs.md b/docs/content/docs.mdindex 1d1bdaa76..cb05455b6 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -53,6 +53,7 @@ See the following for detailed instructions for* [Hetzner Storage Box](/sftp/#hetzner-storage-box)* [HiDrive](/hidrive/)* [HTTP](/http/)+ * [iCloud Drive](/iclouddrive/)* [Internet Archive](/internetarchive/)* [Jottacloud](/jottacloud/)* [Koofr](/koofr/)diff --git a/docs/content/iclouddrive.md b/docs/content/iclouddrive.mdnew file mode 100644index 000000000..e4982cc4b--- /dev/null+++ b/docs/content/iclouddrive.md@@ -0,0 +1,156 @@+---+title: "iCloud Drive"+description: "Rclone docs for iCloud Drive"+versionIntroduced: "v1.69"+---++# {{< icon "fa fa-cloud" >}} iCloud Drive+++## Configuration++The initial setup for an iCloud Drive backend involves getting a trust token/session.+`rclone config` walks you through it. The trust token is valid for 30 days. After which you will have to reauthenticate with rclone reconnect or rclone config.++Here is an example of how to make a remote called `iclouddrive`. First run:++ rclone config++This will guide you through an interactive setup process:++```+No remotes found, make a new one?+n) New remote+s) Set configuration password+q) Quit config+n/s/q> n+name> iclouddrive+Option Storage.+Type of storage to configure.+Choose a number from below, or type in your own value.+[snip]+XX / iCloud Drive+ \ (iclouddrive)+[snip]+Storage> iclouddrive+Option apple_id.+Apple ID.+Enter a value.+apple_id> APPLEID+Option password.+Password.+Choose an alternative below.+y) Yes, type in my own password+g) Generate random password+y/g> y+Enter the password:+password:+Confirm the password:+password:+Edit advanced config?+y) Yes+n) No (default)+y/n> n+Option config_2fa.+Two-factor authentication: please enter your 2FA code+Enter a value.+config_2fa> 2FACODE+Remote config+--------------------+[koofr]+- type: iclouddrive+- apple_id: APPLEID+- password: *** ENCRYPTED ***+- cookies: ****************************+- trust_token: ****************************+--------------------+y) Yes this is OK (default)+e) Edit this remote+d) Delete this remote+y/e/d> y+```++## Advanced Data Protection++ADP is currently unsupported and need to be disabled++{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/iclouddrive/iclouddrive.go then run make backenddocs" >}}+### Standard options++Here are the Standard options specific to iclouddrive (iCloud Drive).++#### --iclouddrive-apple-id++Apple ID.++Properties:++- Config: apple_id+- Env Var: RCLONE_ICLOUDDRIVE_APPLE_ID+- Type: string+- Required: true++#### --iclouddrive-password++Password.++**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).++Properties:++- Config: password+- Env Var: RCLONE_ICLOUDDRIVE_PASSWORD+- Type: string+- Required: true++#### --iclouddrive-trust-token++trust token (internal use)++Properties:++- Config: trust_token+- Env Var: RCLONE_ICLOUDDRIVE_TRUST_TOKEN+- Type: string+- Required: false++#### --iclouddrive-cookies++cookies (internal use only)++Properties:++- Config: cookies+- Env Var: RCLONE_ICLOUDDRIVE_COOKIES+- Type: string+- Required: false++### Advanced options++Here are the Advanced options specific to iclouddrive (iCloud Drive).++#### --iclouddrive-encoding++The encoding for the backend.++See the [encoding section in the overview](/overview/#encoding) for more info.++Properties:++- Config: encoding+- Env Var: RCLONE_ICLOUDDRIVE_ENCODING+- Type: Encoding+- Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot++#### --iclouddrive-description++Description of the remote.++Properties:++- Config: description+- Env Var: RCLONE_ICLOUDDRIVE_DESCRIPTION+- Type: string+- Required: false++{{< rem autogenerated options stop >}}diff --git a/docs/content/overview.md b/docs/content/overview.mdindex 089889540..6131ff438 100644--- a/docs/content/overview.md+++ b/docs/content/overview.md@@ -33,6 +33,7 @@ Here is an overview of the major features of each cloud storage system.| HDFS | - | R/W | No | No | - | - || HiDrive | HiDrive ¹² | R/W | No | No | - | - || HTTP | - | R | No | No | R | - |+| iCloud Drive | - | R | No | No | - | - || Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU || Jottacloud | MD5 | R/W | Yes | No | R | RW || Koofr | MD5 | - | Yes | No | - | - |@@ -505,12 +506,13 @@ upon backend-specific capabilities.| Files.com | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | No | Yes || FTP | No | No | Yes | Yes | No | No | Yes | No | No | No | Yes || Gofile | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes |-| Google Cloud Storage | Yes | Yes | No | No | No | Yes | Yes | No | No | No | No |+| Google Cloud Storage | Yes | Yes | No | No | No | No | Yes | No | No | No | No || Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes || Google Photos | No | No | No | No | No | No | No | No | No | No | No || HDFS | Yes | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes || HiDrive | Yes | Yes | Yes | Yes | No | No | Yes | No | No | No | Yes || HTTP | No | No | No | No | No | No | No | No | No | No | Yes |+| iCloud Drive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes || ImageKit | Yes | Yes | Yes | No | No | No | No | No | No | No | Yes || Internet Archive | No | Yes | No | No | Yes | Yes | No | No | Yes | Yes | No || Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.htmlindex 6b2938c75..c6fa0db6f 100644--- a/docs/layouts/chrome/navbar.html+++ b/docs/layouts/chrome/navbar.html@@ -75,6 +75,7 @@<a class="dropdown-item" href="/hdfs/"><i class="fa fa-globe fa-fw"></i> HDFS (Hadoop Distributed Filesystem)</a><a class="dropdown-item" href="/hidrive/"><i class="fa fa-cloud fa-fw"></i> HiDrive</a><a class="dropdown-item" href="/http/"><i class="fa fa-globe fa-fw"></i> HTTP</a>+ <a class="dropdown-item" href="/iclouddrive/"><i class="fa fa-archive fa-fw"></i> iCloud Drive</a><a class="dropdown-item" href="/imagekit/"><i class="fa fa-cloud fa-fw"></i> ImageKit</a><a class="dropdown-item" href="/internetarchive/"><i class="fa fa-archive fa-fw"></i> Internet Archive</a><a class="dropdown-item" href="/jottacloud/"><i class="fa fa-cloud fa-fw"></i> Jottacloud</a>diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yamlindex 7c5ed0feb..3aa68afab 100644--- a/fstest/test_all/config.yaml+++ b/fstest/test_all/config.yaml@@ -503,3 +503,6 @@ backends:- backend: "ulozto"remote: "TestUlozto:"fastlist: false+ - backend: "iclouddrive"+ remote: "TestICloudDrive:"+ fastlist: false