Native iCloud apps for Windows and Linux, built in Rust. Notes, Reminders, Calendar, Contacts, Drive and Photos, offline and in sync with your iPhone.
Rust apple-notesapple-reminderscloudkitcrdtdesktop-appicloudicloud-calendaricloud-driveicloud-notesicloud-photosiphonelinuxlinux-desktoplocal-firstoffline-firstprivacyrusttauritauri-appwindows
Drive: an audit of the folder's writes and reads, and its fixes
Yashwanth Kamireddia039f26f8a8e
diff --git a/crates/quince-icloud/src/drive.rs b/crates/quince-icloud/src/drive.rsindex 9c032bf..07a783f 100644--- a/crates/quince-icloud/src/drive.rs+++ b/crates/quince-icloud/src/drive.rs@@ -157,6 +157,27 @@ pub async fn download_url(docws_root: &str,params: Vec<(String, String)>,item: &DriveItem,+) -> Result<String> {+ link(http, docws_root, params, item, false).await+}++/// [`download_url`] for exactly the version `item` names: refused ([`Error::Conflict`]) when+/// Apple has a newer one. For reading a file in parts, kept part by part under its version.+pub async fn download_url_of_version(+ http: &reqwest::Client,+ docws_root: &str,+ params: Vec<(String, String)>,+ item: &DriveItem,+) -> Result<String> {+ link(http, docws_root, params, item, true).await+}++async fn link(+ http: &reqwest::Client,+ docws_root: &str,+ params: Vec<(String, String)>,+ item: &DriveItem,+ exact: bool,) -> Result<String> {// The zone in the path is the file's own. An app library — Keynote's, Shortcuts', a// third party's — is a zone of its own, and asking for one of its files under the@@ -189,6 +210,18 @@ pub async fn download_url(let body: Value = serde_json::from_str(&text).map_err(|e| Error::Protocol(format!("download reply was not JSON: {e}")))?;+ // The link is to the version Apple has now (measured: `double_etag` is the item's own+ // etag). A file changed elsewhere since it was listed would be read in parts of two+ // versions and kept as one: refused instead, and read again once listed afresh.+ if let (true, Some(known), Some(serving)) = (+ exact,+ item.etag.as_deref(),+ body.get("double_etag").and_then(Value::as_str),+ ) {+ if known != serving {+ return Err(Error::Conflict);+ }+ }body.pointer("/data_token/url").and_then(Value::as_str).map(ToString::to_string)@@ -2348,7 +2381,19 @@ async fn upload_once(// itself, by id, as icloud.com reads an item (`retrieveItemDetails`) — it must be there,// in this folder, at this size. Reading the whole folder back instead took six of the// nine seconds "Checking" lasted for a 40 MB file in a full folder, for every file sent.- let found = item_details(http, drivews_root, ¶ms, &stored).await?;+ // A blip while reading it back is asked again, not taken for a failed send: retried+ // whole, a file Apple had filed came back as a conflicted copy beside itself.+ let mut tries = 0u64;+ let found = loop {+ match item_details(http, drivews_root, ¶ms, &stored).await {+ Ok(found) => break found,+ Err(e) if tries >= 2 => return Err(e),+ Err(_) => {+ tries += 1;+ tokio::time::sleep(std::time::Duration::from_millis(700 * tries)).await;+ }+ }+ };match found {Some(item)if item.parent_id.as_deref() == Some(folder_id) && item.size == Some(size as i64) =>diff --git a/crates/quince-vfs/src/fuse.rs b/crates/quince-vfs/src/fuse.rsindex e0f7422..f66389c 100644--- a/crates/quince-vfs/src/fuse.rs+++ b/crates/quince-vfs/src/fuse.rs@@ -464,6 +464,33 @@ impl Inner {.block_on(async move { tokio::time::timeout(LIST_WAIT, remote.list(&id)).await });}+ /// A folder by its id, found in what the store holds by walking down from the top: for a+ /// folder the table has not met since a restart.+ fn find_folder(&self, id: &str) -> Option<DriveItem> {+ let root = self.item(1)?;+ if root.drivewsid == id {+ return Some(root);+ }+ let mut queue = std::collections::VecDeque::from([root.drivewsid]);+ let mut seen = 0usize;+ while let Some(at) = queue.pop_front() {+ for child in self.store.children(&at).unwrap_or_default() {+ if !child.is_container() {+ continue;+ }+ if child.drivewsid == id {+ return Some(child);+ }+ seen += 1;+ if seen > 20_000 {+ return None;+ }+ queue.push_back(child.drivewsid);+ }+ }+ None+ }+fn item(&self, ino: u64) -> Option<DriveItem> {self.inodes.lock()@@ -752,9 +779,14 @@ impl Inner {.ino_of(&p.folder).and_then(|f| inodes.item(f).cloned())};+ // A folder not browsed since a restart is not in the table yet: found in what+ // the library holds, from the top down. Not found at all, it is looked for+ // again later — given up, a file waiting there from before was never sent.+ let folder = folder.or_else(|| me.find_folder(&p.folder));let Some(folder) = folder else {- me.with_pending(ino, |p| p.sender = false);- return;+ tokio::time::sleep(wait).await;+ wait = (wait * 2).min(LONGEST_RETRY);+ continue;};let key = format!("{}/{}", p.folder, p.name);// Behind any send of the same name still under way, so this one finds it@@ -814,25 +846,45 @@ impl Inner {.unwrap_or_else(std::sync::PoisonError::into_inner).insert(key, sent.item.clone());}- let done = me- .with_pending(ino, |now| {- if now.generation == generation {- true- } else {- // Written again while on its way: what Apple has now is- // what the next send replaces.- now.replaces = Some(sent.item.clone());- false- }- })- .unwrap_or(true);+ // Deleted while on its way: what was just sent goes too, into Recently+ // Deleted — kept, it came back after the delete had said it was gone.+ if me.pending(ino).is_none() {+ let _ = me.remote.delete(&sent.item).await;+ let _ = me.remote.list(&folder.drivewsid).await;+ return;+ }let _ = me.remote.list(&folder.drivewsid).await;+ // Written again — or renamed, or deleted — while on its way, the wait for+ // the listing included: decided and let go of in one step, so a save+ // landing in that wait is never dropped with the staged copy.+ // None: deleted in that wait.+ let done: Option<bool> = match &me.writes {+ None => Some(true),+ Some(w) => {+ let mut items = w+ .items+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner);+ match items.get_mut(&ino) {+ Some(now) if now.generation != generation => {+ // What Apple has now is what the next send replaces.+ now.replaces = Some(sent.item.clone());+ Some(false)+ }+ Some(_) => {+ items.remove(&ino);+ Some(true)+ }+ None => None,+ }+ }+ };+ let Some(done) = done else {+ let _ = me.remote.delete(&sent.item).await;+ return;+ };if done {if let Some(w) = &me.writes {- w.items- .lock()- .unwrap_or_else(std::sync::PoisonError::into_inner)- .remove(&ino);w.drop_staging(ino);w.save();}@@ -841,6 +893,18 @@ impl Inner {.lock().unwrap_or_else(std::sync::PoisonError::into_inner).bind(ino, &sent.item);+ // A replace is a new document at Apple, with a new id: a file+ // chosen to be kept on this computer stays kept as its new self.+ if let Some(old) = p.replaces.as_ref() {+ if old.drivewsid != sent.item.drivewsid+ && me.cache.kept().chosen.contains(&old.drivewsid)+ {+ me.cache.set_kept(|k| {+ k.chosen.remove(&old.drivewsid);+ k.chosen.insert(sent.item.drivewsid.clone());+ });+ }+ }}return;}@@ -1017,6 +1081,13 @@ impl Inner {me.with_pending(ino, |now| {now.folder = to.drivewsid.clone();now.name = newname.to_string();+ // A change, when there is something to send or a send on its way: one+ // under the old name is followed by one under the new, which replaces it+ // — finished as it was, the rename was lost.+ if now.dirty || now.sending {+ now.generation += 1;+ now.dirty = true;+ }});}(None, None) => me.move_and_rename(&item, &from, &to, name, newname)?,@@ -1471,15 +1542,31 @@ impl Filesystem for DriveFs {reply: ReplyEmpty,) {let inner = &self.0;+ // Some(true): last close of a changed file, to send. Some(false): last close of a file+ // opened for writing and never written — its copy is let go of, or every read went on+ // being served from that copy and changes made on the phone stayed unseen.let last = inner.with_pending(ino.0, |p| {p.open = p.open.saturating_sub(1);- p.open == 0 && p.dirty+ (p.open == 0).then_some(p.dirty || p.sending || p.sender)});- if last == Some(true) {- if let Some(w) = &inner.writes {- w.save();+ match last {+ Some(Some(true)) => {+ if let Some(w) = &inner.writes {+ w.save();+ }+ Inner::schedule(inner, ino.0, writes::SETTLE);+ }+ Some(Some(false)) => {+ if let Some(w) = &inner.writes {+ w.items+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner)+ .remove(&ino.0);+ w.drop_staging(ino.0);+ w.save();+ }}- Inner::schedule(inner, ino.0, writes::SETTLE);+ _ => {}}reply.ok();}diff --git a/crates/quince-vfs/src/lib.rs b/crates/quince-vfs/src/lib.rsindex 469b36d..670c310 100644--- a/crates/quince-vfs/src/lib.rs+++ b/crates/quince-vfs/src/lib.rs@@ -536,6 +536,13 @@ impl Cache {{// One fetch at a time for a file: two readers of the same part wait for one fetch.let _one = lock.lock().await;+ // In use from now, before any part is fetched: marked only after the read, a long+ // read of a file last read long ago could have it let go of mid-way by the+ // eviction its own fetches set off — and the parts already found read back as+ // zeros.+ if let Ok(f) = std::fs::File::options().write(true).open(&path) {+ let _ = f.set_modified(std::time::SystemTime::now());+ }let first = usize::try_from(offset / CHUNK).unwrap_or(0);let last = usize::try_from((end - 1) / CHUNK).unwrap_or(0);for chunk in first..=last {diff --git a/crates/quince-vfs/src/writes.rs b/crates/quince-vfs/src/writes.rsindex 959fb77..e861bf3 100644--- a/crates/quince-vfs/src/writes.rs+++ b/crates/quince-vfs/src/writes.rs@@ -282,15 +282,33 @@ pub async fn send(let new = remote.upload(folder, staged).await?;if let Some(old) = current {// The old one into Recently Deleted, then the new one given its name: if anything- // fails between, both are still there to be found — nothing is ever lost.- remote.delete(&old).await?;+ // fails between, both are still there to be found — nothing is ever lost. Once the new+ // bytes are at Apple, a step after that is tried again where it stands: failing the+ // whole send uploaded them again, and left a "name 2" beside the result each time.+ let mut tries = 0u64;+ while let Err(e) = remote.delete(&old).await {+ tries += 1;+ if tries >= 3 {+ return Err(e);+ }+ tokio::time::sleep(std::time::Duration::from_millis(800 * tries)).await;+ }if new.display_name() != name {- remote.rename(&new, &name).await?;+ let mut tries = 0u64;+ while let Err(e) = remote.rename(&new, &name).await {+ tries += 1;+ if tries >= 3 {+ return Err(e);+ }+ tokio::time::sleep(std::time::Duration::from_millis(800 * tries)).await;+ }}- let after = remote.list(&folder.drivewsid).await?;- let item = after- .into_iter()- .find(|i| i.drivewsid == new.drivewsid)+ // Read back for its new version; sent all the same if the listing fails.+ let item = remote+ .list(&folder.drivewsid)+ .await+ .ok()+ .and_then(|after| after.into_iter().find(|i| i.drivewsid == new.drivewsid)).unwrap_or(new);return Ok(Sent {item,diff --git a/crates/quince-vfs/tests/writes.rs b/crates/quince-vfs/tests/writes.rsindex 40372dc..2932a67 100644--- a/crates/quince-vfs/tests/writes.rs+++ b/crates/quince-vfs/tests/writes.rs@@ -491,3 +491,80 @@ fn what_is_waiting_is_told() {);drop(session);}++/// Starts a mount over a world whose Apple is slow, as `a_save_during_a_send_waits_its_turn`.+fn slow_world() -> Arc<World> {+ let world = Arc::new(World::default());+ world+ .folders+ .lock()+ .unwrap()+ .insert(ROOT.into(), Vec::new());+ *world.slow.lock().unwrap() = Duration::from_millis(2500);+ world+}++fn upload_started(world: &World) {+ for _ in 0..80 {+ if !world.uploads.lock().unwrap().is_empty() {+ return;+ }+ std::thread::sleep(Duration::from_millis(50));+ }+}++/// A file deleted while it is on its way to Apple stays deleted: what arrives goes straight+/// to Recently Deleted (kept, it came back after `rm` had said it was gone).+#[test]+fn a_file_deleted_while_sending_stays_deleted() {+ if !Path::new("/dev/fuse").exists() {+ return;+ }+ let world = slow_world();+ let rt = tokio::runtime::Runtime::new().unwrap();+ let (cache, staging, at) = (+ tempfile::tempdir().unwrap(),+ tempfile::tempdir().unwrap(),+ tempfile::tempdir().unwrap(),+ );+ let root = at.path();+ let session = mount(&world, &rt, root, cache.path(), staging.path());+ std::fs::write(root.join("Big.mov"), b"frames\n").unwrap();+ upload_started(&world);+ std::fs::remove_file(root.join("Big.mov")).unwrap();+ std::thread::sleep(Duration::from_millis(5000));+ assert!(+ !world.names(ROOT).iter().any(|n| n == "Big.mov"),+ "deleted while sending, yet at Apple: {:?}",+ world.names(ROOT)+ );+ drop(session);+}++/// A file renamed while it is on its way to Apple ends under its new name only.+#[test]+fn a_file_renamed_while_sending_ends_under_its_new_name() {+ if !Path::new("/dev/fuse").exists() {+ return;+ }+ let world = slow_world();+ let rt = tokio::runtime::Runtime::new().unwrap();+ let (cache, staging, at) = (+ tempfile::tempdir().unwrap(),+ tempfile::tempdir().unwrap(),+ tempfile::tempdir().unwrap(),+ );+ let root = at.path();+ let session = mount(&world, &rt, root, cache.path(), staging.path());+ std::fs::write(root.join("a.txt"), b"words\n").unwrap();+ upload_started(&world);+ std::fs::rename(root.join("a.txt"), root.join("b.txt")).unwrap();+ assert!(+ settled(&world, "b.txt", b"words\n"),+ "{:?}",+ world.names(ROOT)+ );+ std::thread::sleep(Duration::from_millis(1500));+ assert_eq!(world.names(ROOT), ["b.txt"], "only the new name");+ drop(session);+}diff --git a/probe/examples/drive_download_reply.rs b/probe/examples/drive_download_reply.rsnew file mode 100644index 0000000..00f3d99--- /dev/null+++ b/probe/examples/drive_download_reply.rs@@ -0,0 +1,66 @@+//! Read-only: the shape of Apple's reply when a download is prepared — its field names, and the+//! version markers beside the item's own — for one file in the Drive root. No file is fetched.+use anyhow::{anyhow, Result};+const ROOT: &str = "FOLDER::com.apple.CloudDocs::root";+#[tokio::main]+async fn main() -> Result<()> {+ let dir = std::path::PathBuf::from(std::env::var("HOME")?).join(".local/share/quince-probe");+ let mut client = quince_auth::Client::new(&dir)?;+ if !client.resume().await? {+ return Err(anyhow!("no session"));+ }+ let drivews = client+ .session()+ .service_url("drivews")+ .ok_or_else(|| anyhow!("no drivews"))?;+ let docws = client+ .session()+ .service_url("docws")+ .ok_or_else(|| anyhow!("no docws"))?;+ let (http, params) = (client.http().clone(), client.service_params());+ let items = quince_icloud::drive::fetch_folder(&http, &drivews, params.clone(), ROOT).await?;+ let file = items+ .iter()+ .find(|i| i.kind == "FILE")+ .ok_or_else(|| anyhow!("no file in the root"))?;+ println!("item: etag {:?} size {:?}", file.etag, file.size);+ let id = file.drivewsid.rsplit("::").next().unwrap_or_default();+ let url = format!(+ "{}/ws/com.apple.CloudDocs/download/by_id",+ docws.trim_end_matches('/')+ );+ let mut q = params;+ q.push(("document_id".into(), id.to_string()));+ let body: serde_json::Value = http.get(url).query(&q).send().await?.json().await?;+ let keys = |v: &serde_json::Value| v.as_object().map(|o| o.keys().cloned().collect::<Vec<_>>());+ println!("reply keys: {:?}", keys(&body));+ println!("data_token keys: {:?}", keys(&body["data_token"]));+ for k in ["double_etag", "etag", "size"] {+ println!("{k}: {}", body[k]);+ }+ println!("data_token.size: {}", body["data_token"]["size"]);+ // The app's own check: this version is served; one Apple has moved on from is refused.+ let ok = quince_icloud::drive::download_url_of_version(+ &http,+ &docws,+ q[..q.len() - 1].to_vec(),+ file,+ )+ .await;+ let mut stale = file.clone();+ stale.etag = Some("1a::19".into());+ let refused = quince_icloud::drive::download_url_of_version(+ &http,+ &docws,+ q[..q.len() - 1].to_vec(),+ &stale,+ )+ .await;+ println!(+ "same version: {}; stale version refused: {}",+ ok.is_ok(),+ matches!(refused, Err(quince_icloud::Error::Conflict))+ );+ anyhow::ensure!(ok.is_ok() && matches!(refused, Err(quince_icloud::Error::Conflict)));+ Ok(())+}diff --git a/src-tauri/src/mount.rs b/src-tauri/src/mount.rsindex dc7e07d..a6c812f 100644--- a/src-tauri/src/mount.rs+++ b/src-tauri/src/mount.rs@@ -182,9 +182,11 @@ impl Remote for AppRemote {(Some(u), false) => u.clone(),_ => {let (http, docws, params) = context(&app, "docws").await?;- let u = quince_icloud::drive::download_url(&http, &docws, params, &item)- .await- .map_err(|e| e.to_string())?;+ let u = quince_icloud::drive::download_url_of_version(+ &http, &docws, params, &item,+ )+ .await+ .map_err(|e| e.to_string())?;links.lock().unwrap_or_else(std::sync::PoisonError::into_inner)