Skip to content

Native iCloud apps for Windows and Linux, built in Rust. Notes, Reminders, Calendar, Contacts, Drive and Photos, offline and in sync with your iPhone.

Drive: an audit of the folder's writes and reads, and its fixes

Yashwanth Kamireddia039f26f8a8e
diff --git a/crates/quince-icloud/src/drive.rs b/crates/quince-icloud/src/drive.rs
index 9c032bf..07a783f 100644
--- a/crates/quince-icloud/src/drive.rs
+++ b/crates/quince-icloud/src/drive.rs
@@ -157,6 +157,27 @@ pub async fn download_url(
docws_root: &str,
params: Vec<(String, String)>,
item: &DriveItem,
+) -> Result<String> {
+ link(http, docws_root, params, item, false).await
+}
+
+/// [`download_url`] for exactly the version `item` names: refused ([`Error::Conflict`]) when
+/// Apple has a newer one. For reading a file in parts, kept part by part under its version.
+pub async fn download_url_of_version(
+ http: &reqwest::Client,
+ docws_root: &str,
+ params: Vec<(String, String)>,
+ item: &DriveItem,
+) -> Result<String> {
+ link(http, docws_root, params, item, true).await
+}
+
+async fn link(
+ http: &reqwest::Client,
+ docws_root: &str,
+ params: Vec<(String, String)>,
+ item: &DriveItem,
+ exact: bool,
) -> Result<String> {
// The zone in the path is the file's own. An app library — Keynote's, Shortcuts', a
// third party's — is a zone of its own, and asking for one of its files under the
@@ -189,6 +210,18 @@ pub async fn download_url(
let body: Value = serde_json::from_str(&text)
.map_err(|e| Error::Protocol(format!("download reply was not JSON: {e}")))?;
+ // The link is to the version Apple has now (measured: `double_etag` is the item's own
+ // etag). A file changed elsewhere since it was listed would be read in parts of two
+ // versions and kept as one: refused instead, and read again once listed afresh.
+ if let (true, Some(known), Some(serving)) = (
+ exact,
+ item.etag.as_deref(),
+ body.get("double_etag").and_then(Value::as_str),
+ ) {
+ if known != serving {
+ return Err(Error::Conflict);
+ }
+ }
body.pointer("/data_token/url")
.and_then(Value::as_str)
.map(ToString::to_string)
@@ -2348,7 +2381,19 @@ async fn upload_once(
// itself, by id, as icloud.com reads an item (`retrieveItemDetails`) — it must be there,
// in this folder, at this size. Reading the whole folder back instead took six of the
// nine seconds "Checking" lasted for a 40 MB file in a full folder, for every file sent.
- let found = item_details(http, drivews_root, &params, &stored).await?;
+ // A blip while reading it back is asked again, not taken for a failed send: retried
+ // whole, a file Apple had filed came back as a conflicted copy beside itself.
+ let mut tries = 0u64;
+ let found = loop {
+ match item_details(http, drivews_root, &params, &stored).await {
+ Ok(found) => break found,
+ Err(e) if tries >= 2 => return Err(e),
+ Err(_) => {
+ tries += 1;
+ tokio::time::sleep(std::time::Duration::from_millis(700 * tries)).await;
+ }
+ }
+ };
match found {
Some(item)
if item.parent_id.as_deref() == Some(folder_id) && item.size == Some(size as i64) =>
diff --git a/crates/quince-vfs/src/fuse.rs b/crates/quince-vfs/src/fuse.rs
index e0f7422..f66389c 100644
--- a/crates/quince-vfs/src/fuse.rs
+++ b/crates/quince-vfs/src/fuse.rs
@@ -464,6 +464,33 @@ impl Inner {
.block_on(async move { tokio::time::timeout(LIST_WAIT, remote.list(&id)).await });
}
+ /// A folder by its id, found in what the store holds by walking down from the top: for a
+ /// folder the table has not met since a restart.
+ fn find_folder(&self, id: &str) -> Option<DriveItem> {
+ let root = self.item(1)?;
+ if root.drivewsid == id {
+ return Some(root);
+ }
+ let mut queue = std::collections::VecDeque::from([root.drivewsid]);
+ let mut seen = 0usize;
+ while let Some(at) = queue.pop_front() {
+ for child in self.store.children(&at).unwrap_or_default() {
+ if !child.is_container() {
+ continue;
+ }
+ if child.drivewsid == id {
+ return Some(child);
+ }
+ seen += 1;
+ if seen > 20_000 {
+ return None;
+ }
+ queue.push_back(child.drivewsid);
+ }
+ }
+ None
+ }
+
fn item(&self, ino: u64) -> Option<DriveItem> {
self.inodes
.lock()
@@ -752,9 +779,14 @@ impl Inner {
.ino_of(&p.folder)
.and_then(|f| inodes.item(f).cloned())
};
+ // A folder not browsed since a restart is not in the table yet: found in what
+ // the library holds, from the top down. Not found at all, it is looked for
+ // again later — given up, a file waiting there from before was never sent.
+ let folder = folder.or_else(|| me.find_folder(&p.folder));
let Some(folder) = folder else {
- me.with_pending(ino, |p| p.sender = false);
- return;
+ tokio::time::sleep(wait).await;
+ wait = (wait * 2).min(LONGEST_RETRY);
+ continue;
};
let key = format!("{}/{}", p.folder, p.name);
// Behind any send of the same name still under way, so this one finds it
@@ -814,25 +846,45 @@ impl Inner {
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(key, sent.item.clone());
}
- let done = me
- .with_pending(ino, |now| {
- if now.generation == generation {
- true
- } else {
- // Written again while on its way: what Apple has now is
- // what the next send replaces.
- now.replaces = Some(sent.item.clone());
- false
- }
- })
- .unwrap_or(true);
+ // Deleted while on its way: what was just sent goes too, into Recently
+ // Deleted — kept, it came back after the delete had said it was gone.
+ if me.pending(ino).is_none() {
+ let _ = me.remote.delete(&sent.item).await;
+ let _ = me.remote.list(&folder.drivewsid).await;
+ return;
+ }
let _ = me.remote.list(&folder.drivewsid).await;
+ // Written again — or renamed, or deleted — while on its way, the wait for
+ // the listing included: decided and let go of in one step, so a save
+ // landing in that wait is never dropped with the staged copy.
+ // None: deleted in that wait.
+ let done: Option<bool> = match &me.writes {
+ None => Some(true),
+ Some(w) => {
+ let mut items = w
+ .items
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ match items.get_mut(&ino) {
+ Some(now) if now.generation != generation => {
+ // What Apple has now is what the next send replaces.
+ now.replaces = Some(sent.item.clone());
+ Some(false)
+ }
+ Some(_) => {
+ items.remove(&ino);
+ Some(true)
+ }
+ None => None,
+ }
+ }
+ };
+ let Some(done) = done else {
+ let _ = me.remote.delete(&sent.item).await;
+ return;
+ };
if done {
if let Some(w) = &me.writes {
- w.items
- .lock()
- .unwrap_or_else(std::sync::PoisonError::into_inner)
- .remove(&ino);
w.drop_staging(ino);
w.save();
}
@@ -841,6 +893,18 @@ impl Inner {
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.bind(ino, &sent.item);
+ // A replace is a new document at Apple, with a new id: a file
+ // chosen to be kept on this computer stays kept as its new self.
+ if let Some(old) = p.replaces.as_ref() {
+ if old.drivewsid != sent.item.drivewsid
+ && me.cache.kept().chosen.contains(&old.drivewsid)
+ {
+ me.cache.set_kept(|k| {
+ k.chosen.remove(&old.drivewsid);
+ k.chosen.insert(sent.item.drivewsid.clone());
+ });
+ }
+ }
}
return;
}
@@ -1017,6 +1081,13 @@ impl Inner {
me.with_pending(ino, |now| {
now.folder = to.drivewsid.clone();
now.name = newname.to_string();
+ // A change, when there is something to send or a send on its way: one
+ // under the old name is followed by one under the new, which replaces it
+ // — finished as it was, the rename was lost.
+ if now.dirty || now.sending {
+ now.generation += 1;
+ now.dirty = true;
+ }
});
}
(None, None) => me.move_and_rename(&item, &from, &to, name, newname)?,
@@ -1471,15 +1542,31 @@ impl Filesystem for DriveFs {
reply: ReplyEmpty,
) {
let inner = &self.0;
+ // Some(true): last close of a changed file, to send. Some(false): last close of a file
+ // opened for writing and never written — its copy is let go of, or every read went on
+ // being served from that copy and changes made on the phone stayed unseen.
let last = inner.with_pending(ino.0, |p| {
p.open = p.open.saturating_sub(1);
- p.open == 0 && p.dirty
+ (p.open == 0).then_some(p.dirty || p.sending || p.sender)
});
- if last == Some(true) {
- if let Some(w) = &inner.writes {
- w.save();
+ match last {
+ Some(Some(true)) => {
+ if let Some(w) = &inner.writes {
+ w.save();
+ }
+ Inner::schedule(inner, ino.0, writes::SETTLE);
+ }
+ Some(Some(false)) => {
+ if let Some(w) = &inner.writes {
+ w.items
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .remove(&ino.0);
+ w.drop_staging(ino.0);
+ w.save();
+ }
}
- Inner::schedule(inner, ino.0, writes::SETTLE);
+ _ => {}
}
reply.ok();
}
diff --git a/crates/quince-vfs/src/lib.rs b/crates/quince-vfs/src/lib.rs
index 469b36d..670c310 100644
--- a/crates/quince-vfs/src/lib.rs
+++ b/crates/quince-vfs/src/lib.rs
@@ -536,6 +536,13 @@ impl Cache {
{
// One fetch at a time for a file: two readers of the same part wait for one fetch.
let _one = lock.lock().await;
+ // In use from now, before any part is fetched: marked only after the read, a long
+ // read of a file last read long ago could have it let go of mid-way by the
+ // eviction its own fetches set off — and the parts already found read back as
+ // zeros.
+ if let Ok(f) = std::fs::File::options().write(true).open(&path) {
+ let _ = f.set_modified(std::time::SystemTime::now());
+ }
let first = usize::try_from(offset / CHUNK).unwrap_or(0);
let last = usize::try_from((end - 1) / CHUNK).unwrap_or(0);
for chunk in first..=last {
diff --git a/crates/quince-vfs/src/writes.rs b/crates/quince-vfs/src/writes.rs
index 959fb77..e861bf3 100644
--- a/crates/quince-vfs/src/writes.rs
+++ b/crates/quince-vfs/src/writes.rs
@@ -282,15 +282,33 @@ pub async fn send(
let new = remote.upload(folder, staged).await?;
if let Some(old) = current {
// The old one into Recently Deleted, then the new one given its name: if anything
- // fails between, both are still there to be found — nothing is ever lost.
- remote.delete(&old).await?;
+ // fails between, both are still there to be found — nothing is ever lost. Once the new
+ // bytes are at Apple, a step after that is tried again where it stands: failing the
+ // whole send uploaded them again, and left a "name 2" beside the result each time.
+ let mut tries = 0u64;
+ while let Err(e) = remote.delete(&old).await {
+ tries += 1;
+ if tries >= 3 {
+ return Err(e);
+ }
+ tokio::time::sleep(std::time::Duration::from_millis(800 * tries)).await;
+ }
if new.display_name() != name {
- remote.rename(&new, &name).await?;
+ let mut tries = 0u64;
+ while let Err(e) = remote.rename(&new, &name).await {
+ tries += 1;
+ if tries >= 3 {
+ return Err(e);
+ }
+ tokio::time::sleep(std::time::Duration::from_millis(800 * tries)).await;
+ }
}
- let after = remote.list(&folder.drivewsid).await?;
- let item = after
- .into_iter()
- .find(|i| i.drivewsid == new.drivewsid)
+ // Read back for its new version; sent all the same if the listing fails.
+ let item = remote
+ .list(&folder.drivewsid)
+ .await
+ .ok()
+ .and_then(|after| after.into_iter().find(|i| i.drivewsid == new.drivewsid))
.unwrap_or(new);
return Ok(Sent {
item,
diff --git a/crates/quince-vfs/tests/writes.rs b/crates/quince-vfs/tests/writes.rs
index 40372dc..2932a67 100644
--- a/crates/quince-vfs/tests/writes.rs
+++ b/crates/quince-vfs/tests/writes.rs
@@ -491,3 +491,80 @@ fn what_is_waiting_is_told() {
);
drop(session);
}
+
+/// Starts a mount over a world whose Apple is slow, as `a_save_during_a_send_waits_its_turn`.
+fn slow_world() -> Arc<World> {
+ let world = Arc::new(World::default());
+ world
+ .folders
+ .lock()
+ .unwrap()
+ .insert(ROOT.into(), Vec::new());
+ *world.slow.lock().unwrap() = Duration::from_millis(2500);
+ world
+}
+
+fn upload_started(world: &World) {
+ for _ in 0..80 {
+ if !world.uploads.lock().unwrap().is_empty() {
+ return;
+ }
+ std::thread::sleep(Duration::from_millis(50));
+ }
+}
+
+/// A file deleted while it is on its way to Apple stays deleted: what arrives goes straight
+/// to Recently Deleted (kept, it came back after `rm` had said it was gone).
+#[test]
+fn a_file_deleted_while_sending_stays_deleted() {
+ if !Path::new("/dev/fuse").exists() {
+ return;
+ }
+ let world = slow_world();
+ let rt = tokio::runtime::Runtime::new().unwrap();
+ let (cache, staging, at) = (
+ tempfile::tempdir().unwrap(),
+ tempfile::tempdir().unwrap(),
+ tempfile::tempdir().unwrap(),
+ );
+ let root = at.path();
+ let session = mount(&world, &rt, root, cache.path(), staging.path());
+ std::fs::write(root.join("Big.mov"), b"frames\n").unwrap();
+ upload_started(&world);
+ std::fs::remove_file(root.join("Big.mov")).unwrap();
+ std::thread::sleep(Duration::from_millis(5000));
+ assert!(
+ !world.names(ROOT).iter().any(|n| n == "Big.mov"),
+ "deleted while sending, yet at Apple: {:?}",
+ world.names(ROOT)
+ );
+ drop(session);
+}
+
+/// A file renamed while it is on its way to Apple ends under its new name only.
+#[test]
+fn a_file_renamed_while_sending_ends_under_its_new_name() {
+ if !Path::new("/dev/fuse").exists() {
+ return;
+ }
+ let world = slow_world();
+ let rt = tokio::runtime::Runtime::new().unwrap();
+ let (cache, staging, at) = (
+ tempfile::tempdir().unwrap(),
+ tempfile::tempdir().unwrap(),
+ tempfile::tempdir().unwrap(),
+ );
+ let root = at.path();
+ let session = mount(&world, &rt, root, cache.path(), staging.path());
+ std::fs::write(root.join("a.txt"), b"words\n").unwrap();
+ upload_started(&world);
+ std::fs::rename(root.join("a.txt"), root.join("b.txt")).unwrap();
+ assert!(
+ settled(&world, "b.txt", b"words\n"),
+ "{:?}",
+ world.names(ROOT)
+ );
+ std::thread::sleep(Duration::from_millis(1500));
+ assert_eq!(world.names(ROOT), ["b.txt"], "only the new name");
+ drop(session);
+}
diff --git a/probe/examples/drive_download_reply.rs b/probe/examples/drive_download_reply.rs
new file mode 100644
index 0000000..00f3d99
--- /dev/null
+++ b/probe/examples/drive_download_reply.rs
@@ -0,0 +1,66 @@
+//! Read-only: the shape of Apple's reply when a download is prepared — its field names, and the
+//! version markers beside the item's own — for one file in the Drive root. No file is fetched.
+use anyhow::{anyhow, Result};
+const ROOT: &str = "FOLDER::com.apple.CloudDocs::root";
+#[tokio::main]
+async fn main() -> Result<()> {
+ let dir = std::path::PathBuf::from(std::env::var("HOME")?).join(".local/share/quince-probe");
+ let mut client = quince_auth::Client::new(&dir)?;
+ if !client.resume().await? {
+ return Err(anyhow!("no session"));
+ }
+ let drivews = client
+ .session()
+ .service_url("drivews")
+ .ok_or_else(|| anyhow!("no drivews"))?;
+ let docws = client
+ .session()
+ .service_url("docws")
+ .ok_or_else(|| anyhow!("no docws"))?;
+ let (http, params) = (client.http().clone(), client.service_params());
+ let items = quince_icloud::drive::fetch_folder(&http, &drivews, params.clone(), ROOT).await?;
+ let file = items
+ .iter()
+ .find(|i| i.kind == "FILE")
+ .ok_or_else(|| anyhow!("no file in the root"))?;
+ println!("item: etag {:?} size {:?}", file.etag, file.size);
+ let id = file.drivewsid.rsplit("::").next().unwrap_or_default();
+ let url = format!(
+ "{}/ws/com.apple.CloudDocs/download/by_id",
+ docws.trim_end_matches('/')
+ );
+ let mut q = params;
+ q.push(("document_id".into(), id.to_string()));
+ let body: serde_json::Value = http.get(url).query(&q).send().await?.json().await?;
+ let keys = |v: &serde_json::Value| v.as_object().map(|o| o.keys().cloned().collect::<Vec<_>>());
+ println!("reply keys: {:?}", keys(&body));
+ println!("data_token keys: {:?}", keys(&body["data_token"]));
+ for k in ["double_etag", "etag", "size"] {
+ println!("{k}: {}", body[k]);
+ }
+ println!("data_token.size: {}", body["data_token"]["size"]);
+ // The app's own check: this version is served; one Apple has moved on from is refused.
+ let ok = quince_icloud::drive::download_url_of_version(
+ &http,
+ &docws,
+ q[..q.len() - 1].to_vec(),
+ file,
+ )
+ .await;
+ let mut stale = file.clone();
+ stale.etag = Some("1a::19".into());
+ let refused = quince_icloud::drive::download_url_of_version(
+ &http,
+ &docws,
+ q[..q.len() - 1].to_vec(),
+ &stale,
+ )
+ .await;
+ println!(
+ "same version: {}; stale version refused: {}",
+ ok.is_ok(),
+ matches!(refused, Err(quince_icloud::Error::Conflict))
+ );
+ anyhow::ensure!(ok.is_ok() && matches!(refused, Err(quince_icloud::Error::Conflict)));
+ Ok(())
+}
diff --git a/src-tauri/src/mount.rs b/src-tauri/src/mount.rs
index dc7e07d..a6c812f 100644
--- a/src-tauri/src/mount.rs
+++ b/src-tauri/src/mount.rs
@@ -182,9 +182,11 @@ impl Remote for AppRemote {
(Some(u), false) => u.clone(),
_ => {
let (http, docws, params) = context(&app, "docws").await?;
- let u = quince_icloud::drive::download_url(&http, &docws, params, &item)
- .await
- .map_err(|e| e.to_string())?;
+ let u = quince_icloud::drive::download_url_of_version(
+ &http, &docws, params, &item,
+ )
+ .await
+ .map_err(|e| e.to_string())?;
links
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)