Skip to content

Native iCloud apps for Windows and Linux, built in Rust. Notes, Reminders, Calendar, Contacts, Drive and Photos, offline and in sync with your iPhone.

Download whole folders and selections from iCloud Drive into Downloads

Yashwanth Kamireddi9b735b07375f
diff --git a/Cargo.lock b/Cargo.lock
index cc5ee98..ac4249d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4035,6 +4035,7 @@ dependencies = [
"glib 0.18.5",
"gstreamer",
"gtk",
+ "icu_normalizer",
"image",
"jiff",
"ksni",
diff --git a/SECURITY.md b/SECURITY.md
index 48f80c8..95ec82a 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -117,6 +117,18 @@ never handed to the account signing in.
save for one write only, and every command that reads or writes a file (uploads, imports,
attachments, exports) refuses a path that is not one of them, compared with `..` and links
resolved. So even script running in the window could not name a file of yours on its own.
+- **Download a Copy** in iCloud Drive — a file, a folder, or several at once — saves into your
+ Downloads folder only: the window says what to download, never where. Every name in it, a
+ shared folder's included, is made one plain name first (no `..`, folder separator, drive,
+ stream or Windows device name), a folder arrives as a new folder there ("Photos 2" beside a
+ "Photos"), nothing that was in Downloads before is written over or into, and no link found
+ on the way is followed. Each file is written under a name of its own and moved into place
+ only once it is whole; on Windows it is marked as come from the internet, so SmartScreen and
+ Office's Protected View still look at it. A download that does not finish is remembered in
+ Skykeep's own data folder (what was chosen, and where it went) until it does; downloading
+ the same again goes on inside the folder it made — only if that is still a folder in
+ Downloads — keeping each file already there at iCloud's size and replacing any other copy of
+ its own.
- **A note's attachment** you open is saved under a cleaned-up name (never a path, a hidden
file, or a name that reverses itself to hide its type) in your private data folder, not
executable, and handed to the app this computer uses for it. A program, script, launcher or
diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml
index 5671785..0b498ea 100644
--- a/src-tauri/Cargo.toml
+++ b/src-tauri/Cargo.toml
@@ -23,7 +23,9 @@ quince-icloud = { path = "../crates/quince-icloud" }
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
-tokio = { workspace = true, features = ["signal"] }
+# "sync": a folder download hands files from the reading to the fetching through a bounded
+# queue (folder_download.rs).
+tokio = { workspace = true, features = ["signal", "sync"] }
anyhow.workspace = true
jiff = "0.2.37"
uuid.workspace = true
@@ -46,6 +48,9 @@ semver = "1"
# A merged card's picture made small enough for Apple (contacts_merge.rs): the same build of
# the crate quince-media already uses, so nothing new is compiled.
image = { version = "0.25", default-features = false, features = ["png", "jpeg"] }
+# Names from a Mac saved composed (NFC), and compared so, in a folder download (folder_download.rs):
+# the build reqwest's URL parsing already compiles, through idna, so nothing new is built.
+icu_normalizer = { version = "2.3", default-features = false, features = ["compiled_data"] }
[target.'cfg(not(target_os = "linux"))'.dependencies]
tauri = { version = "2", features = ["tray-icon"] }
@@ -60,6 +65,7 @@ windows = { version = "0.62", features = [
"ApplicationModel",
"Win32_Foundation",
"Win32_Storage_EnhancedStorage",
+ "Win32_Storage_FileSystem",
"Win32_Storage_Packaging_Appx",
"Win32_System_Com",
"Win32_System_Com_StructuredStorage",
diff --git a/src-tauri/src/folder_download.rs b/src-tauri/src/folder_download.rs
new file mode 100644
index 0000000..f4dbd5b
--- /dev/null
+++ b/src-tauri/src/folder_download.rs
@@ -0,0 +1,1385 @@
+//! A folder from iCloud Drive — or a selection of files and folders — saved into Downloads as
+//! the same tree of folders and files it is in Drive, at any size: a terabyte, a hundred
+//! thousand files, a tree hundreds of folders deep.
+//!
+//! - **Where it goes.** Every name in the tree is whoever shared the folder's to choose, so each
+//! is made one plain name before it touches a path ([`crate::export::file_name`]: no `..`, no
+//! separator, no drive, no stream, no Windows device name, no trailing dot or space), and the
+//! path it makes is checked to sit directly in the folder it was made for
+//! ([`crate::saving::directly_in`]). Nothing is written outside the folder made for the
+//! download in Downloads — or, for a file chosen on its own, anywhere but Downloads itself.
+//! - **Names that clash.** Nothing that was in Downloads before is written over or into: a
+//! folder is made new and numbered ("Photos 2"), as Finder numbers one. Inside, two names
+//! that come out the same once cleaned, that differ only in case, or only in how an accent is
+//! encoded (a Mac's decomposed "é" beside a composed one), are numbered too — Windows, macOS
+//! and an exFAT drive would take them for one file. Names are saved composed (NFC).
+//! - **Links.** None is followed: a name taken by anything at all, a link included, counts as
+//! taken, and every folder between the download's own and a file is checked to be a real
+//! folder before the file is written into it.
+//! - **Two passes, bounded memory.** The tree is read once to count it — files, bytes, and on a
+//! resumed download what is already here — keeping only the folders still to read, so the
+//! room it needs is known before a byte is written. Then it is read again and fetched as it
+//! is read: the reading stays no more than [`AHEAD`] files ahead of the fetching, four files
+//! and four folders at a time. A folder of a hundred thousand files takes the memory of a few
+//! hundred. (`retrieveItemDetailsInFolders` answers with a folder whole: there is no page to
+//! ask for next.)
+//! - **Whole files only.** Each file goes into a partial file of its own and is moved into
+//! place once every byte Apple promised is there (`quince_icloud::drive`), Windows' virus
+//! scanner waited out (`quince_core::files`). A dropped connection or a busy Drive is asked
+//! again ([`quince_icloud::drive::patiently`]).
+//! - **Resumable.** What was chosen and where it went is kept in a small journal ([`Journal`])
+//! until the download finishes cleanly. Downloading the same thing again — after Cancel, a
+//! lost connection, a full disk, or a restart — goes on in the same folder: a file already
+//! there at iCloud's size is kept, any other is fetched again in its place, and partial files
+//! left by a crash are cleared.
+//! - **When it can't.** A fresh sign-in needed, a disk that is full or would be (checked before
+//! it starts and as it goes), or the connection lost for good, stops the lot and says why —
+//! resumable. A file that cannot be saved — its path too long for this computer, say — is
+//! named with the reason, and the rest go on.
+//! - **Stopping.** Cancel stops everything at once: what is arriving is dropped with its partial
+//! file, what has arrived stays, and a folder made that is left empty goes again.
+//!
+//! The window hears how far it has got (`DOWNLOAD`): a count while the tree is read, then each
+//! file as it lands — and, for a file of a megabyte or more, its bytes as they arrive.
+
+use std::collections::{HashSet, VecDeque};
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex, PoisonError};
+use std::time::{Duration, Instant};
+
+use futures_util::future::{AbortHandle, Abortable};
+use futures_util::StreamExt;
+use quince_core::drive::DriveItem;
+use quince_icloud::drive::Placing;
+use quince_icloud::Error;
+
+/// Where a download's progress is announced.
+pub(crate) const DOWNLOAD: &str = "quince://download";
+
+/// How many folders are read, and how many files fetched, at once.
+pub(crate) const AT_ONCE: usize = 4;
+
+/// How many files found may wait for their turn: how far the reading gets ahead of the fetching.
+pub(crate) const AHEAD: usize = 256;
+
+/// Folders deeper than this are not gone into: far past any real tree, and a guard against a
+/// listing that names a folder inside itself under a new id.
+pub(crate) const DEEPEST: usize = 256;
+
+/// How many failures are named one by one; past this they are counted.
+const NAMED_FAILURES: usize = 200;
+
+/// Files this big or bigger tell their bytes as they arrive; smaller ones only that they landed.
+const LARGE: u64 = 1 << 20;
+
+/// How often the count of what has been found is told while the tree is read.
+const FOUND_EVERY: Duration = Duration::from_millis(100);
+
+/// How often the disk's free space is looked at again while files arrive, in the app.
+pub(crate) const ROOM_EVERY: Duration = Duration::from_secs(2);
+
+/// How many files in a row may fail on a dropped connection before the connection counts as
+/// lost and the download pauses, rather than marking every file after them as failed.
+const LOST_AFTER: usize = 8;
+
+/// One step of a download, as the window hears it — the same fields as an upload's.
+#[derive(serde::Serialize, Clone, Debug, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct DownloadEvent {
+ /// Unique within the batch the window started.
+ pub id: String,
+ pub batch: String,
+ pub name: String,
+ /// The folders it sits in, inside Downloads, as they were saved.
+ pub within: String,
+ pub sent: u64,
+ pub total: u64,
+ /// For a file: `downloading`, `done`, `failed` or `cancelled`. For the batch: `found` while
+ /// the tree is read and `planned` once all of it is known — their `sent` the files, their
+ /// `total` the bytes.
+ pub state: &'static str,
+ /// A folder rather than a file: only ever one that could not be read or made.
+ pub folder: bool,
+ pub error: Option<String>,
+}
+
+/// What a download came to.
+#[derive(serde::Serialize, Debug, Default, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct DownloadReport {
+ /// What it is known by in the journal (see [`Journal::key`]).
+ pub key: String,
+ /// What is now in Downloads: one path for each thing chosen that is there, whole or in part.
+ pub saved: Vec<String>,
+ /// What did not arrive, one line each: where it would be inside Downloads, and why.
+ pub failed: Vec<String>,
+ /// Failures past those named.
+ pub more_failed: usize,
+ /// Stopped with Cancel.
+ pub cancelled: bool,
+ /// Why it stopped short on its own: the disk full or about to be, the connection lost.
+ pub problem: Option<String>,
+ /// It can be gone on with: downloading the same again resumes it.
+ pub resumable: bool,
+ /// The most files that were found and waiting at once: what the reading's lead cost.
+ #[serde(skip)]
+ pub peak_waiting: usize,
+}
+
+/// How events reach the window.
+pub(crate) type Emit = Arc<dyn Fn(DownloadEvent) + Send + Sync>;
+
+/// Drive, as one download asks it, and the computer it saves to.
+pub(crate) struct Drive {
+ pub http: reqwest::Client,
+ /// `drivews`: the listings.
+ pub drivews: String,
+ /// `docws`: the bytes.
+ pub docws: String,
+ pub params: Vec<(String, String)>,
+ /// The first wait before a request that met a busy Drive is made again.
+ pub retry_after: Duration,
+ /// How much room a folder's disk has (see [`free_space`]).
+ pub free_space: fn(&Path) -> Option<u64>,
+ /// How often that is looked at again while files arrive ([`ROOM_EVERY`] in the app).
+ pub room_every: Duration,
+}
+
+/// One download asked for.
+pub(crate) struct Request<'a> {
+ pub batch: &'a str,
+ /// What the window calls it — a folder's name, "3 items" — kept for resuming it later.
+ pub title: &'a str,
+ pub items: Vec<DriveItem>,
+ /// Downloads.
+ pub into: &'a Path,
+ /// Where unfinished downloads are remembered; none, and nothing is resumable.
+ pub journal: Option<&'a Journal>,
+}
+
+// ── Stopping ────────────────────────────────────────────────────────────────────────────────
+
+/// A download's Cancel. Pressed once, everything still running is dropped and nothing more
+/// begins.
+#[derive(Clone, Default)]
+pub(crate) struct Stop(Arc<StopInner>);
+
+#[derive(Default)]
+struct StopInner {
+ stopped: AtomicBool,
+ /// What is running now, by a number of its own: let go of as each finishes, so a download
+ /// of thousands of files keeps no more than the few running.
+ running: Mutex<std::collections::HashMap<u64, AbortHandle>>,
+ next: AtomicU64,
+}
+
+impl Stop {
+ pub fn stop(&self) {
+ self.0.stopped.store(true, Ordering::SeqCst);
+ let running = std::mem::take(&mut *self.running());
+ for handle in running.into_values() {
+ handle.abort();
+ }
+ }
+
+ pub fn stopped(&self) -> bool {
+ self.0.stopped.load(Ordering::SeqCst)
+ }
+
+ fn running(&self) -> std::sync::MutexGuard<'_, std::collections::HashMap<u64, AbortHandle>> {
+ self.0
+ .running
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ }
+
+ /// `work`, unless Cancel is pressed first or while it runs (`None` then). Registered under
+ /// the lock `stop` takes, after the flag is read: work that starts as Cancel is pressed is
+ /// either stopped by it or never starts.
+ async fn guard<F: std::future::Future>(&self, work: F) -> Option<F::Output> {
+ let (handle, registration) = AbortHandle::new_pair();
+ let id = self.0.next.fetch_add(1, Ordering::Relaxed);
+ {
+ let mut running = self.running();
+ if self.stopped() {
+ return None;
+ }
+ running.insert(id, handle);
+ }
+ let outcome = Abortable::new(work, registration).await.ok();
+ self.running().remove(&id);
+ outcome
+ }
+}
+
+// ── The journal ─────────────────────────────────────────────────────────────────────────────
+
+/// Downloads not yet finished cleanly, one small file each in Skykeep's own data folder: what
+/// was chosen, and where in Downloads each part of it went. Read when the same is downloaded
+/// again, so it goes on where it was rather than starting over beside itself.
+pub(crate) struct Journal {
+ dir: PathBuf,
+}
+
+/// An unfinished download, as the journal keeps it.
+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct Record {
+ pub key: String,
+ pub title: String,
+ pub items: Vec<DriveItem>,
+ pub placed: Vec<Placed>,
+ pub started_at: i64,
+}
+
+/// Where one thing chosen went.
+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct Placed {
+ pub drivewsid: String,
+ pub path: PathBuf,
+ pub folder: bool,
+}
+
+impl Journal {
+ pub fn new(dir: PathBuf) -> Self {
+ Self { dir }
+ }
+
+ /// What a choice of items is known by: the same items, in any order, are the same download.
+ pub fn key(items: &[DriveItem]) -> String {
+ use sha2::Digest;
+ let mut ids: Vec<&str> = items.iter().map(|i| i.drivewsid.as_str()).collect();
+ ids.sort_unstable();
+ ids.dedup();
+ let digest = sha2::Sha256::digest(ids.join("\n").as_bytes());
+ digest[..16].iter().map(|b| format!("{b:02x}")).collect()
+ }
+
+ /// The file a key is kept in — only ever a key's own shape, so no key names another path.
+ fn file(&self, key: &str) -> Option<PathBuf> {
+ (key.len() == 32 && key.bytes().all(|b| b.is_ascii_hexdigit()))
+ .then(|| self.dir.join(format!("{key}.json")))
+ }
+
+ pub fn load(&self, key: &str) -> Option<Record> {
+ let text = std::fs::read(self.file(key)?).ok()?;
+ serde_json::from_slice::<Record>(&text)
+ .ok()
+ .filter(|r| r.key == key)
+ }
+
+ /// Kept whole or not at all: written beside itself and moved into place.
+ pub fn save(&self, record: &Record) -> std::io::Result<()> {
+ let file = self
+ .file(&record.key)
+ .ok_or_else(|| std::io::Error::other("not a download's key"))?;
+ std::fs::create_dir_all(&self.dir)?;
+ let fresh = file.with_extension("json.new");
+ std::fs::write(
+ &fresh,
+ serde_json::to_vec(record).map_err(std::io::Error::other)?,
+ )?;
+ quince_core::files::rename(&fresh, &file)
+ }
+
+ pub fn forget(&self, key: &str) {
+ if let Some(file) = self.file(key) {
+ let _ = std::fs::remove_file(file);
+ }
+ }
+
+ /// Every download not yet finished, oldest first.
+ pub fn unfinished(&self) -> Vec<Record> {
+ let Ok(entries) = std::fs::read_dir(&self.dir) else {
+ return Vec::new();
+ };
+ let mut records: Vec<Record> = entries
+ .flatten()
+ .filter_map(|e| {
+ let name = e.file_name().to_string_lossy().into_owned();
+ self.load(name.strip_suffix(".json")?)
+ })
+ .collect();
+ records.sort_by_key(|r| r.started_at);
+ records
+ }
+}
+
+// ── Names ───────────────────────────────────────────────────────────────────────────────────
+
+/// `s` composed (NFC): "é" as one character, the way Linux and Windows programs write it, where
+/// a Mac may have stored it as "e" and an accent.
+fn composed(s: &str) -> String {
+ icu_normalizer::ComposingNormalizerBorrowed::new_nfc()
+ .normalize(s)
+ .into_owned()
+}
+
+/// Names given out in one folder of the copy, compared composed and with case ignored — as
+/// Windows, macOS and exFAT compare them.
+#[derive(Default)]
+pub(crate) struct Taken(HashSet<String>);
+
+impl Taken {
+ /// Where `given` is saved in `dir`, a folder that was here before the download: one plain
+ /// name, numbered — "Photos 2", "report 2.pdf" — past anything at all on disk under it (a
+ /// link too, even one leading nowhere) and past what this download has given out there.
+ pub fn name(&mut self, dir: &Path, given: &str, folder: bool) -> std::io::Result<PathBuf> {
+ self.pick(dir, given, folder, true)
+ }
+
+ /// Where `given` is saved in `dir`, a folder the download made itself: numbered past what
+ /// this download has given out there only, so the same tree is named the same way every
+ /// time it is read — what makes a download resumable.
+ pub fn ours(&mut self, dir: &Path, given: &str, folder: bool) -> std::io::Result<PathBuf> {
+ self.pick(dir, given, folder, false)
+ }
+
+ /// A name this download already holds in the folder (a resumed one's).
+ pub fn hold(&mut self, path: &Path) {
+ if let Some(name) = path.file_name() {
+ self.0
+ .insert(composed(&name.to_string_lossy().to_lowercase()));
+ }
+ }
+
+ fn pick(
+ &mut self,
+ dir: &Path,
+ given: &str,
+ folder: bool,
+ disk: bool,
+ ) -> std::io::Result<PathBuf> {
+ let clean =
+ crate::export::file_name(&composed(given), if folder { "Folder" } else { "Download" });
+ // A folder's name is all stem: "v1.2" numbered is "v1.2 2", not "v1 2.2".
+ let (stem, ext) = if folder {
+ (clean.as_str(), "")
+ } else {
+ crate::name_parts(&clean)
+ };
+ for n in 1..=10_000u32 {
+ let name = if n == 1 {
+ clean.clone()
+ } else {
+ format!("{stem} {n}{ext}")
+ };
+ let key = composed(&name.to_lowercase());
+ if self.0.contains(&key) {
+ continue;
+ }
+ let path = dir.join(&name);
+ if !crate::saving::directly_in(dir, &path) {
+ return Err(std::io::Error::other(
+ "that name can\u{2019}t be saved here",
+ ));
+ }
+ if disk && path.symlink_metadata().is_ok() {
+ continue;
+ }
+ self.0.insert(key);
+ return Ok(path);
+ }
+ Err(std::io::Error::other(
+ "too many things of that name are already there",
+ ))
+ }
+}
+
+/// A new folder for `given` in `dir`, numbered past anything already there, and made — never
+/// one that was there before, so nothing of anyone else's is written into.
+pub(crate) fn make_folder(taken: &mut Taken, dir: &Path, given: &str) -> std::io::Result<PathBuf> {
+ loop {
+ let path = taken.name(dir, given, true)?;
+ match std::fs::create_dir(&path) {
+ Ok(()) => return Ok(path),
+ // Taken between the look and the making: the next number.
+ Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {}
+ Err(e) => return Err(e),
+ }
+ }
+}
+
+/// The folder for `given` inside one the download made: made, or — resumed — the one made for
+/// it before. Whether it was made now.
+fn ensure_folder(taken: &mut Taken, dir: &Path, given: &str) -> std::io::Result<(PathBuf, bool)> {
+ let path = taken.ours(dir, given, true)?;
+ match std::fs::create_dir(&path) {
+ Ok(()) => Ok((path, true)),
+ Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
+ if is_real_dir(&path) {
+ Ok((path, false))
+ } else {
+ Err(std::io::Error::other(
+ "something else is already where it would go",
+ ))
+ }
+ }
+ Err(e) => Err(e),
+ }
+}
+
+fn is_real_dir(path: &Path) -> bool {
+ std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
+}
+
+/// Whether every folder from `top` down to `dir` is a real folder, not a link: what this
+/// download made, still as it made it.
+pub(crate) fn real_folders(top: &Path, dir: &Path) -> bool {
+ let Ok(below) = dir.strip_prefix(top) else {
+ return false;
+ };
+ let mut at = top.to_path_buf();
+ if !is_real_dir(&at) {
+ return false;
+ }
+ for part in below.components() {
+ let std::path::Component::Normal(name) = part else {
+ return false;
+ };
+ at.push(name);
+ if !is_real_dir(&at) {
+ return false;
+ }
+ }
+ true
+}
+
+/// The partial files a download of `path` may have left — `name.part`, `name.part.1` … — as
+/// `quince_icloud::drive` names them.
+fn partials(path: &Path) -> impl Iterator<Item = PathBuf> {
+ let base = path.with_extension(match path.extension() {
+ Some(ext) => format!("{}.part", ext.to_string_lossy()),
+ None => "part".into(),
+ });
+ (0..=100).map(move |n| {
+ if n == 0 {
+ base.clone()
+ } else {
+ let mut name = base.as_os_str().to_owned();
+ name.push(format!(".{n}"));
+ PathBuf::from(name)
+ }
+ })
+}
+
+// ── The disk ────────────────────────────────────────────────────────────────────────────────
+
+/// The error codes that mean the disk (or the person's share of it) is full.
+#[cfg(unix)]
+const NO_ROOM: &[i32] = &[libc::ENOSPC, libc::EDQUOT];
+/// ERROR_HANDLE_DISK_FULL, ERROR_DISK_FULL.
+#[cfg(windows)]
+const NO_ROOM: &[i32] = &[39, 112];
+#[cfg(not(any(unix, windows)))]
+const NO_ROOM: &[i32] = &[];
+
+/// The error codes that mean a path is longer than this computer takes.
+#[cfg(unix)]
+const TOO_LONG: &[i32] = &[libc::ENAMETOOLONG];
+/// ERROR_FILENAME_EXCED_RANGE.
+#[cfg(windows)]
+const TOO_LONG: &[i32] = &[206];
+#[cfg(not(any(unix, windows)))]
+const TOO_LONG: &[i32] = &[];
+
+fn no_room(e: &std::io::Error) -> bool {
+ e.kind() == std::io::ErrorKind::StorageFull
+ || e.kind() == std::io::ErrorKind::QuotaExceeded
+ || e.raw_os_error().is_some_and(|c| NO_ROOM.contains(&c))
+}
+
+/// What went wrong writing to the disk, in words a person can act on.
+fn disk_words(e: &std::io::Error) -> String {
+ if no_room(e) {
+ "there isn\u{2019}t enough space left on the disk".into()
+ } else if e.raw_os_error().is_some_and(|c| TOO_LONG.contains(&c)) {
+ "its path is too long for this computer".into()
+ } else if e.kind() == std::io::ErrorKind::PermissionDenied {
+ "Skykeep isn\u{2019}t allowed to write there".into()
+ } else {
+ e.to_string()
+ }
+}
+
+/// How many bytes can still be written to the disk `dir` is on, by this user.
+pub(crate) fn free_space(dir: &Path) -> Option<u64> {
+ #[cfg(unix)]
+ {
+ use std::os::unix::ffi::OsStrExt;
+ let c = std::ffi::CString::new(dir.as_os_str().as_bytes()).ok()?;
+ // SAFETY: a zeroed statvfs is a valid value to be written over; `c` is a NUL-ended
+ // path that outlives the call.
+ let mut s: libc::statvfs = unsafe { std::mem::zeroed() };
+ if unsafe { libc::statvfs(c.as_ptr(), &mut s) } != 0 {
+ return None;
+ }
+ #[allow(clippy::useless_conversion)]
+ Some(u64::from(s.f_bavail).saturating_mul(u64::from(s.f_frsize)))
+ }
+ #[cfg(windows)]
+ {
+ use std::os::windows::ffi::OsStrExt;
+ let wide: Vec<u16> = dir.as_os_str().encode_wide().chain(Some(0)).collect();
+ let mut free = 0u64;
+ // SAFETY: `wide` is a NUL-ended path that outlives the call; Windows writes one u64.
+ unsafe {
+ windows::Win32::Storage::FileSystem::GetDiskFreeSpaceExW(
+ windows::core::PCWSTR(wide.as_ptr()),
+ Some(&mut free),
+ None,
+ None,
+ )
+ }
+ .ok()?;
+ Some(free)
+ }
+ #[cfg(not(any(unix, windows)))]
+ {
+ let _ = dir;
+ None
+ }
+}
+
+/// A size as a person reads it: "3.2 GB".
+fn size_words(bytes: u64) -> String {
+ const UNITS: [&str; 5] = ["bytes", "KB", "MB", "GB", "TB"];
+ let mut value = bytes as f64;
+ let mut unit = 0;
+ while value >= 1000.0 && unit < UNITS.len() - 1 {
+ value /= 1000.0;
+ unit += 1;
+ }
+ if unit == 0 {
+ format!("{bytes} bytes")
+ } else {
+ format!("{value:.1} {}", UNITS[unit])
+ }
+}
+
+// ── The download ────────────────────────────────────────────────────────────────────────────
+
+/// `within` and a name below it, as a path is shown: `Photos/Trip`.
+fn below(within: &str, name: &str) -> String {
+ if within.is_empty() {
+ name.to_string()
+ } else {
+ format!("{within}/{name}")
+ }
+}
+
+fn shown(path: &Path) -> String {
+ path.file_name()
+ .map(|n| n.to_string_lossy().into_owned())
+ .unwrap_or_default()
+}
+
+fn size_of(item: &DriveItem) -> Option<u64> {
+ item.size.and_then(|n| u64::try_from(n).ok())
+}
+
+/// A folder of Drive's to read, and where it is saved.
+struct Folder {
+ item: DriveItem,
+ dir: PathBuf,
+ /// Where it is inside Downloads, as saved: `Photos/Trip`.
+ within: String,
+ /// Which of the things chosen it is in.
+ top: usize,
+ depth: usize,
+ /// Already on disk, from an earlier run: what is in it is looked at while counting.
+ on_disk: bool,
+}
+
+/// A file to fetch, and where it goes.
+struct Fetch {
+ item: DriveItem,
+ path: PathBuf,
+ within: String,
+ /// The folder this download made that it goes into, checked down to the file's own before
+ /// it is written; `None` for a file chosen on its own, which goes into Downloads itself.
+ top_dir: Option<PathBuf>,
+ top: usize,
+ placing: Placing,
+ id: u64,
+}
+
+/// One thing chosen, and where in Downloads it goes.
+struct Top {
+ item: DriveItem,
+ path: Option<PathBuf>,
+ /// From an earlier run of this download.
+ resumed: bool,
+ arrived: AtomicUsize,
+}
+
+/// Which reading of the tree this is.
+#[derive(Clone, Copy)]
+enum Pass<'a> {
+ /// Counting it: what there is, and what of it is already here. Nothing is written.
+ Count,
+ /// Saving it: folders made, files handed on to be fetched.
+ Save(&'a tokio::sync::mpsc::Sender<Fetch>),
+}
+
+/// Everything a running download shares.
+struct Run<'a> {
+ drive: &'a Drive,
+ batch: &'a str,
+ into: &'a Path,
+ emit: Emit,
+ stop: &'a Stop,
+ tops: Vec<Top>,
+ failed: Mutex<(Vec<String>, usize)>,
+ problem: Mutex<Option<String>>,
+ /// Set once Apple asks for a fresh sign-in: nothing more can arrive.
+ reauth: AtomicBool,
+ folders_failed: AtomicU64,
+ /// Files and bytes found by the reading under way.
+ found: AtomicU64,
+ found_bytes: AtomicU64,
+ /// Counting: the bytes of files already here, whole, from an earlier run.
+ have_bytes: AtomicU64,
+ /// Saving: the bytes still to come when it began, and those that have come since.
+ needed: AtomicU64,
+ arrived_bytes: AtomicU64,
+ last_found: Mutex<Instant>,
+ last_room: Mutex<Instant>,
+ /// Files in a row that failed on a dropped connection.
+ lost: AtomicUsize,
+ next_id: AtomicU64,
+ waiting: AtomicUsize,
+ peak_waiting: AtomicUsize,
+}
+
+impl Run<'_> {
+ fn note_failure(&self, line: String) {
+ let mut failed = self.failed.lock().unwrap_or_else(PoisonError::into_inner);
+ if failed.0.len() < NAMED_FAILURES {
+ failed.0.push(line);
+ } else {
+ failed.1 += 1;
+ }
+ }
+
+ fn event(&self, id: String, name: &str, within: &str, state: &'static str) -> DownloadEvent {
+ DownloadEvent {
+ id,
+ batch: self.batch.to_string(),
+ name: name.to_string(),
+ within: within.to_string(),
+ sent: 0,
+ total: 0,
+ state,
+ folder: false,
+ error: None,
+ }
+ }
+
+ fn fail_folder(&self, name: &str, within: &str, error: String) {
+ let n = self.folders_failed.fetch_add(1, Ordering::Relaxed);
+ let mut e = self.event(format!("{}:x:{n}", self.batch), name, within, "failed");
+ e.folder = true;
+ e.error = Some(error.clone());
+ (self.emit)(e);
+ self.note_failure(format!("{}: {error}", below(within, name)));
+ }
+
+ /// Stopped short on its own, for a reason said once: everything else stops.
+ fn give_up(&self, why: String) {
+ self.problem
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .get_or_insert(why);
+ self.stop.stop();
+ }
+
+ /// Apple wants a fresh sign-in: nothing more can arrive.
+ fn reauth(&self) {
+ self.reauth.store(true, Ordering::SeqCst);
+ self.stop.stop();
+ }
+
+ /// A disk error placing something: said in words, and the lot stopped when the disk is full.
+ fn disk_error(&self, e: &std::io::Error) -> String {
+ if no_room(e) {
+ self.give_up(
+ "Downloads ran out of space, so the download stopped. What arrived is kept; nothing half-saved was."
+ .into(),
+ );
+ }
+ disk_words(e)
+ }
+
+ /// The tally of what the reading has found, told at most every FOUND_EVERY (and at once
+ /// when `now`).
+ fn tell_found(&self, now: bool) {
+ let mut last = self
+ .last_found
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
+ if !now && last.elapsed() < FOUND_EVERY {
+ return;
+ }
+ *last = Instant::now();
+ let mut e = self.event(format!("{}:found", self.batch), "", "", "found");
+ e.folder = true;
+ e.sent = self.found.load(Ordering::Relaxed);
+ e.total = self.found_bytes.load(Ordering::Relaxed);
+ (self.emit)(e);
+ }
+
+ /// Whether there is still room for what is still to come, looked at every `room_every`: other
+ /// programs write to the same disk while a long download runs.
+ fn check_room(&self) {
+ {
+ let mut last = self
+ .last_room
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
+ if last.elapsed() < self.drive.room_every {
+ return;
+ }
+ *last = Instant::now();
+ }
+ let still = self
+ .needed
+ .load(Ordering::Relaxed)
+ .saturating_sub(self.arrived_bytes.load(Ordering::Relaxed));
+ if let Some(free) = (self.drive.free_space)(self.into).filter(|free| *free < still) {
+ self.give_up(format!(
+ "Downloads is running out of space, so the download paused: about {} more is needed and {} is free. What arrived is kept.",
+ size_words(still),
+ size_words(free),
+ ));
+ }
+ }
+
+ async fn list(&self, folder: &DriveItem) -> Option<quince_icloud::Result<Vec<DriveItem>>> {
+ let d = self.drive;
+ let mut op = || {
+ quince_icloud::drive::fetch_folder(
+ &d.http,
+ &d.drivews,
+ d.params.clone(),
+ &folder.drivewsid,
+ )
+ };
+ self.stop
+ .guard(quince_icloud::drive::patiently_after(
+ d.retry_after,
+ &mut op,
+ ))
+ .await
+ }
+
+ /// Read the tree from `frontier` down, four folders at a time, keeping only the folders
+ /// still to read. Counting, it tallies; saving, it makes folders and hands each file on to
+ /// be fetched as it is found. The folders it made, in the order made.
+ async fn walk(&self, mut frontier: VecDeque<Folder>, pass: Pass<'_>) -> Vec<PathBuf> {
+ let mut made = Vec::new();
+ // Folders already gone into: a listing that named one again is not followed round.
+ let mut seen: HashSet<String> = frontier.iter().map(|f| f.item.drivewsid.clone()).collect();
+ while !frontier.is_empty() && !self.stop.stopped() {
+ let now: Vec<Folder> = frontier.drain(..frontier.len().min(AT_ONCE)).collect();
+ // Answered in order: the same tree is named the same way every time it is read.
+ let listed = futures_util::future::join_all(now.into_iter().map(|folder| async move {
+ let listing = self.list(&folder.item).await;
+ (folder, listing)
+ }))
+ .await;
+ for (folder, listing) in listed {
+ let mut children = match listing {
+ None => return made,
+ Some(Ok(children)) => children,
+ Some(Err(Error::ReauthRequired)) => {
+ self.reauth();
+ return made;
+ }
+ // Counting, a folder that cannot be read is passed over: the saving pass
+ // asks again, and says so if it still cannot.
+ Some(Err(_)) if matches!(pass, Pass::Count) => continue,
+ Some(Err(e)) => {
+ // Not left behind empty, as if that were what it held.
+ let _ = std::fs::remove_dir(&folder.dir);
+ let parent = folder
+ .within
+ .rsplit_once('/')
+ .map_or(String::new(), |(p, _)| p.to_string());
+ self.fail_folder(&shown(&folder.dir), &parent, e.to_string());
+ continue;
+ }
+ };
+ // In name order, as Finder lists them: which of two clashing names is numbered
+ // is then the same every time.
+ children.sort_by_cached_key(|c| {
+ (
+ composed(&c.display_name().to_lowercase()),
+ c.drivewsid.clone(),
+ )
+ });
+ let mut here = Taken::default();
+ for child in children {
+ if self.stop.stopped() {
+ return made;
+ }
+ let name = child.display_name();
+ if child.is_container() {
+ if !seen.insert(child.drivewsid.clone()) {
+ continue;
+ }
+ if folder.depth + 1 >= DEEPEST {
+ if let Pass::Save(_) = pass {
+ self.fail_folder(
+ &name,
+ &folder.within,
+ "it is too many folders deep to save".into(),
+ );
+ }
+ continue;
+ }
+ let placed = match pass {
+ Pass::Count => here.ours(&folder.dir, &name, true).map(|dir| {
+ let on_disk = folder.on_disk && is_real_dir(&dir);
+ (dir, on_disk, false)
+ }),
+ Pass::Save(_) => ensure_folder(&mut here, &folder.dir, &name)
+ .map(|(dir, fresh)| (dir, !fresh, fresh)),
+ };
+ match placed {
+ Ok((dir, on_disk, fresh)) => {
+ if fresh {
+ made.push(dir.clone());
+ }
+ frontier.push_back(Folder {
+ within: below(&folder.within, &shown(&dir)),
+ item: child,
+ dir,
+ top: folder.top,
+ depth: folder.depth + 1,
+ on_disk,
+ });
+ }
+ Err(_) if matches!(pass, Pass::Count) => {}
+ Err(e) => {
+ let words = self.disk_error(&e);
+ self.fail_folder(&name, &folder.within, words);
+ }
+ }
+ } else {
+ let path = match here.ours(&folder.dir, &name, false) {
+ Ok(path) => path,
+ Err(_) if matches!(pass, Pass::Count) => continue,
+ Err(e) => {
+ self.note_failure(format!(
+ "{}: {}",
+ below(&folder.within, &name),
+ disk_words(&e)
+ ));
+ continue;
+ }
+ };
+ if !self.found_file(&child, &path, folder.on_disk, pass) {
+ return made;
+ }
+ if let Pass::Save(files) = pass {
+ let job = Fetch {
+ item: child,
+ path,
+ within: folder.within.clone(),
+ top_dir: self.tops[folder.top].path.clone(),
+ top: folder.top,
+ placing: Placing::Replace,
+ id: self.next_id.fetch_add(1, Ordering::Relaxed),
+ };
+ if !self.hand_on(files, job).await {
+ return made;
+ }
+ }
+ }
+ }
+ if matches!(pass, Pass::Count) {
+ self.tell_found(false);
+ }
+ }
+ }
+ made
+ }
+
+ /// A file found: counted, and — counting a resumed download — what of it is here already.
+ /// False once there is nothing more to do.
+ fn found_file(&self, item: &DriveItem, path: &Path, on_disk: bool, pass: Pass<'_>) -> bool {
+ let size = size_of(item).unwrap_or(0);
+ self.found.fetch_add(1, Ordering::Relaxed);
+ self.found_bytes.fetch_add(size, Ordering::Relaxed);
+ if matches!(pass, Pass::Count) && on_disk && whole_here(path, item) {
+ self.have_bytes.fetch_add(size, Ordering::Relaxed);
+ }
+ !self.stop.stopped()
+ }
+
+ /// Hand a file on to be fetched — waiting while AHEAD are waiting already.
+ async fn hand_on(&self, files: &tokio::sync::mpsc::Sender<Fetch>, job: Fetch) -> bool {
+ let now = self.waiting.fetch_add(1, Ordering::Relaxed) + 1;
+ self.peak_waiting.fetch_max(now, Ordering::Relaxed);
+ files.send(job).await.is_ok()
+ }
+
+ /// Fetch one file.
+ async fn fetch(&self, job: Fetch) {
+ self.fetch_one(&job).await;
+ self.waiting.fetch_sub(1, Ordering::Relaxed);
+ }
+
+ async fn fetch_one(&self, job: &Fetch) {
+ if self.stop.stopped() {
+ return;
+ }
+ let name = shown(&job.path);
+ let id = format!("{}:{}", self.batch, job.id);
+ let size = size_of(&job.item);
+ let done = |sent: u64| {
+ let mut e = self.event(id.clone(), &name, &job.within, "done");
+ e.sent = sent;
+ e.total = sent.max(size.unwrap_or(0));
+ (self.emit)(e);
+ self.tops[job.top].arrived.fetch_add(1, Ordering::Relaxed);
+ };
+ let fail = |words: String| {
+ let mut e = self.event(id.clone(), &name, &job.within, "failed");
+ e.total = size.unwrap_or(0);
+ e.error = Some(words.clone());
+ (self.emit)(e);
+ self.note_failure(format!("{}: {words}", below(&job.within, &name)));
+ };
+ if job.placing == Placing::Replace {
+ // Resumed: a file already here at iCloud's size is the one that arrived before.
+ if whole_here(&job.path, &job.item) {
+ done(size.unwrap_or(0));
+ return;
+ }
+ if is_real_dir(&job.path) {
+ fail("a folder is already where it would go".into());
+ return;
+ }
+ // Partial files a crash left behind are cleared, so they cannot pile up.
+ for partial in partials(&job.path) {
+ if std::fs::symlink_metadata(&partial).is_ok_and(|m| m.is_file()) {
+ let _ = std::fs::remove_file(&partial);
+ }
+ }
+ }
+ let large = size.is_none_or(|s| s >= LARGE);
+ let progress: Option<quince_icloud::drive::Progress> = large.then(|| {
+ let mut start = self.event(id.clone(), &name, &job.within, "downloading");
+ start.total = size.unwrap_or(0);
+ (self.emit)(start.clone());
+ // Told about every chunk, passed on at every whole percent: a smooth bar without a
+ // flood of messages for a large file.
+ let (emit, last) = (self.emit.clone(), AtomicU64::new(0));
+ let progress: quince_icloud::drive::Progress =
+ Arc::new(move |sent: u64, total: u64| {
+ let step = (total / 100).max(1);
+ let before = last.load(Ordering::Relaxed);
+ if sent / step != before / step || sent == total {
+ last.store(sent, Ordering::Relaxed);
+ let mut e = start.clone();
+ e.sent = sent;
+ e.total = total.max(e.total);
+ emit(e);
+ }
+ });
+ progress
+ });
+ let stopped = || {
+ if large {
+ let mut e = self.event(id.clone(), &name, &job.within, "cancelled");
+ e.total = size.unwrap_or(0);
+ (self.emit)(e);
+ }
+ };
+ let d = self.drive;
+ let (item, path, top_dir, placing) =
+ (&job.item, &job.path, job.top_dir.as_deref(), job.placing);
+ let mut op = || {
+ let progress = progress.clone();
+ async move {
+ // Checked before every try: a folder swapped for a link meanwhile is not
+ // written through.
+ let unchanged = match (top_dir, path.parent()) {
+ (Some(top), Some(dir)) => real_folders(top, dir),
+ (None, _) => true,
+ (Some(_), None) => false,
+ };
+ if !unchanged {
+ return Err(Error::Refused(
+ "the folder it was going into has changed, so it wasn\u{2019}t saved"
+ .into(),
+ ));
+ }
+ quince_icloud::drive::download_with_progress(
+ &d.http,
+ &d.docws,
+ d.params.clone(),
+ item,
+ path,
+ progress,
+ placing,
+ )
+ .await
+ }
+ };
+ let outcome = self
+ .stop
+ .guard(quince_icloud::drive::patiently_after(
+ d.retry_after,
+ &mut op,
+ ))
+ .await;
+ match outcome {
+ Some(Ok(())) => {
+ crate::saving::mark_from_internet(path);
+ let len = std::fs::metadata(path).map_or(size.unwrap_or(0), |m| m.len());
+ self.arrived_bytes.fetch_add(len, Ordering::Relaxed);
+ self.lost.store(0, Ordering::Relaxed);
+ done(len);
+ self.check_room();
+ }
+ Some(Err(Error::ReauthRequired)) => {
+ self.reauth();
+ stopped();
+ }
+ Some(Err(e)) => {
+ if quince_icloud::drive::transient(&e)
+ && self.lost.fetch_add(1, Ordering::Relaxed) + 1 >= LOST_AFTER
+ {
+ self.give_up(
+ "The connection to iCloud was lost, so the download paused. What arrived is kept; download it again to go on."
+ .into(),
+ );
+ }
+ let words = match &e {
+ Error::Io(io) => self.disk_error(io),
+ other => other.to_string(),
+ };
+ fail(words);
+ }
+ None => stopped(),
+ }
+ }
+}
+
+/// Whether the file at `path` is `item` whole: a plain file, at the size iCloud has it.
+fn whole_here(path: &Path, item: &DriveItem) -> bool {
+ size_of(item).is_some_and(|size| {
+ std::fs::symlink_metadata(path).is_ok_and(|m| m.is_file() && m.len() == size)
+ })
+}
+
+fn now_seconds() -> i64 {
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX))
+}
+
+/// Save what was asked for from Drive into Downloads: each file chosen as a file there, each
+/// folder as a folder there with everything in it — or, when the same was downloaded before
+/// and not finished, on into what it saved then. Progress goes to `emit` as it happens; `stop`
+/// cancels. Fails as a whole only when Apple wants a fresh sign-in; anything else that did not
+/// arrive is in the report.
+pub(crate) async fn run(
+ drive: &Drive,
+ request: Request<'_>,
+ emit: Emit,
+ stop: &Stop,
+) -> Result<DownloadReport, String> {
+ let Request {
+ batch,
+ title,
+ items,
+ into,
+ journal,
+ } = request;
+ if !into.is_dir() {
+ return Err("Couldn\u{2019}t find your Downloads folder.".into());
+ }
+ let mut chosen = HashSet::new();
+ let items: Vec<DriveItem> = items
+ .into_iter()
+ .filter(|i| chosen.insert(i.drivewsid.clone()))
+ .collect();
+ let key = Journal::key(&items);
+ let before = journal.and_then(|j| j.load(&key));
+
+ // Where an earlier run put each thing chosen, if it is still there as it left it — and in
+ // Downloads: a journal edited to point elsewhere is not followed.
+ let resumed_at = |item: &DriveItem| -> Option<PathBuf> {
+ let placed = before
+ .as_ref()?
+ .placed
+ .iter()
+ .find(|p| p.drivewsid == item.drivewsid && p.folder == item.is_container())?;
+ if !crate::saving::directly_in(into, &placed.path) {
+ return None;
+ }
+ let fits = if placed.folder {
+ is_real_dir(&placed.path)
+ } else {
+ std::fs::symlink_metadata(&placed.path).map_or(true, |m| m.is_file())
+ };
+ fits.then(|| placed.path.clone())
+ };
+ let tops: Vec<Top> = items
+ .iter()
+ .map(|item| {
+ let path = resumed_at(item);
+ Top {
+ item: item.clone(),
+ resumed: path.is_some(),
+ path,
+ arrived: AtomicUsize::new(0),
+ }
+ })
+ .collect();
+
+ let mut run = Run {
+ drive,
+ batch,
+ into,
+ emit: emit.clone(),
+ stop,
+ tops,
+ failed: Mutex::new((Vec::new(), 0)),
+ problem: Mutex::new(None),
+ reauth: AtomicBool::new(false),
+ folders_failed: AtomicU64::new(0),
+ found: AtomicU64::new(0),
+ found_bytes: AtomicU64::new(0),
+ have_bytes: AtomicU64::new(0),
+ needed: AtomicU64::new(0),
+ arrived_bytes: AtomicU64::new(0),
+ last_found: Mutex::new(Instant::now()),
+ last_room: Mutex::new(Instant::now()),
+ lost: AtomicUsize::new(0),
+ next_id: AtomicU64::new(0),
+ waiting: AtomicUsize::new(0),
+ peak_waiting: AtomicUsize::new(0),
+ };
+ let report = |run: &Run<'_>, resumable: bool| {
+ let (failed, more_failed) =
+ std::mem::take(&mut *run.failed.lock().unwrap_or_else(PoisonError::into_inner));
+ let problem = run
+ .problem
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .clone();
+ DownloadReport {
+ key: key.clone(),
+ saved: run
+ .tops
+ .iter()
+ .filter_map(|t| t.path.as_ref())
+ .filter(|p| std::fs::symlink_metadata(p).is_ok())
+ .map(|p| p.to_string_lossy().into_owned())
+ .collect(),
+ failed,
+ more_failed,
+ cancelled: stop.stopped() && problem.is_none() && !run.reauth.load(Ordering::SeqCst),
+ problem,
+ resumable,
+ peak_waiting: run.peak_waiting.load(Ordering::Relaxed),
+ }
+ };
+
+ // ── First, count it: nothing is written until it is known there is room. ──
+ let mut frontier = VecDeque::new();
+ for (top, t) in run.tops.iter().enumerate() {
+ if t.item.is_container() {
+ frontier.push_back(Folder {
+ item: t.item.clone(),
+ dir: t.path.clone().unwrap_or_default(),
+ within: String::new(),
+ top,
+ depth: 0,
+ on_disk: t.resumed,
+ });
+ } else if let Some(path) = &t.path {
+ run.found_file(&t.item, path, true, Pass::Count);
+ } else {
+ run.found_file(&t.item, Path::new(""), false, Pass::Count);
+ }
+ }
+ run.walk(frontier, Pass::Count).await;
+ if run.reauth.load(Ordering::SeqCst) {
+ return Err(Error::ReauthRequired.to_string());
+ }
+ run.tell_found(true);
+ if stop.stopped() {
+ return Ok(report(&run, before.is_some()));
+ }
+ let total_files = run.found.swap(0, Ordering::Relaxed);
+ let total_bytes = run.found_bytes.swap(0, Ordering::Relaxed);
+ let needed = total_bytes.saturating_sub(run.have_bytes.load(Ordering::Relaxed));
+ run.needed.store(needed, Ordering::Relaxed);
+ if let Some(free) = (drive.free_space)(into).filter(|free| needed > *free) {
+ run.give_up(format!(
+ "There isn\u{2019}t room in Downloads for this: it needs {} and {} is free.",
+ size_words(needed),
+ size_words(free),
+ ));
+ return Ok(report(&run, before.is_some()));
+ }
+ let planned = move |files: u64, bytes: u64| DownloadEvent {
+ id: format!("{batch}:planned"),
+ batch: batch.to_string(),
+ name: String::new(),
+ within: String::new(),
+ sent: files,
+ total: bytes,
+ state: "planned",
+ folder: true,
+ error: None,
+ };
+ emit(planned(total_files, total_bytes));
+
+ // ── Then place each thing chosen in Downloads, and remember where. ──
+ let mut made = Vec::new();
+ let mut in_downloads = Taken::default();
+ for t in run.tops.iter().filter_map(|t| t.path.as_ref()) {
+ in_downloads.hold(t);
+ }
+ let mut early = Vec::new();
+ for t in run.tops.iter_mut().filter(|t| t.path.is_none()) {
+ let name = t.item.display_name();
+ let placed = if t.item.is_container() {
+ make_folder(&mut in_downloads, into, &name).inspect(|p| made.push(p.clone()))
+ } else {
+ in_downloads.name(into, &name, false)
+ };
+ match placed {
+ Ok(path) => t.path = Some(path),
+ Err(e) => early.push((name, t.item.is_container(), e)),
+ }
+ }
+ for (name, folder, e) in early {
+ let words = run.disk_error(&e);
+ if folder {
+ run.fail_folder(&name, "", words);
+ } else {
+ run.note_failure(format!("{name}: {words}"));
+ }
+ }
+ let placed: Vec<Placed> = run
+ .tops
+ .iter()
+ .filter_map(|t| {
+ Some(Placed {
+ drivewsid: t.item.drivewsid.clone(),
+ path: t.path.clone()?,
+ folder: t.item.is_container(),
+ })
+ })
+ .collect();
+ let remembered = match journal {
+ Some(j) if !placed.is_empty() => j
+ .save(&Record {
+ key: key.clone(),
+ title: title.to_string(),
+ items: items.clone(),
+ placed,
+ started_at: before.as_ref().map_or_else(now_seconds, |b| b.started_at),
+ })
+ .is_ok(),
+ _ => false,
+ };
+
+ // ── Then save it: read again, and fetched as it is read. ──
+ let (files, queue) = tokio::sync::mpsc::channel::<Fetch>(AHEAD);
+ let run = &run;
+ let reading = async move {
+ let files = files;
+ let mut frontier = VecDeque::new();
+ for (top, t) in run.tops.iter().enumerate() {
+ let Some(path) = t.path.clone() else { continue };
+ if t.item.is_container() {
+ frontier.push_back(Folder {
+ item: t.item.clone(),
+ within: shown(&path),
+ dir: path,
+ top,
+ depth: 0,
+ on_disk: true,
+ });
+ } else {
+ run.found_file(&t.item, &path, false, Pass::Save(&files));
+ let job = Fetch {
+ item: t.item.clone(),
+ path,
+ within: String::new(),
+ top_dir: None,
+ top,
+ placing: if t.resumed {
+ Placing::Replace
+ } else {
+ Placing::New
+ },
+ id: run.next_id.fetch_add(1, Ordering::Relaxed),
+ };
+ if !run.hand_on(&files, job).await {
+ return Vec::new();
+ }
+ }
+ }
+ let made = run.walk(frontier, Pass::Save(&files)).await;
+ // All of it is known: the count it ends at, in case Drive changed since it was counted.
+ if !stop.stopped() {
+ emit(planned(
+ run.found.load(Ordering::Relaxed),
+ run.found_bytes.load(Ordering::Relaxed),
+ ));
+ }
+ made
+ };
+ let fetching = futures_util::stream::unfold(queue, |mut queue| async move {
+ queue.recv().await.map(|job| (job, queue))
+ })
+ .map(|job| run.fetch(job))
+ .buffer_unordered(AT_ONCE)
+ .for_each(|_| async {});
+ let (below_made, ()) = futures_util::future::join(reading, fetching).await;
+ made.extend(below_made);
+
+ let stopped = stop.stopped();
+ // Stopped part-way: the folders made now that hold nothing are taken away again, deepest
+ // first. A folder only ever goes while empty (remove_dir takes nothing else), so a file
+ // that arrived — or anything put there meanwhile — keeps its folder.
+ if stopped {
+ for dir in made.iter().rev() {
+ if is_real_dir(dir) {
+ let _ = std::fs::remove_dir(dir);
+ }
+ }
+ }
+ if run.reauth.load(Ordering::SeqCst) {
+ return Err(Error::ReauthRequired.to_string());
+ }
+ let clean = !stopped && {
+ let failed = run.failed.lock().unwrap_or_else(PoisonError::into_inner);
+ failed.0.is_empty() && failed.1 == 0
+ };
+ let anything_there = run
+ .tops
+ .iter()
+ .filter_map(|t| t.path.as_ref())
+ .any(|p| std::fs::symlink_metadata(p).is_ok());
+ let resumable = remembered && !clean && anything_there;
+ if let Some(j) = journal.filter(|_| !resumable) {
+ j.forget(&key);
+ }
+ Ok(report(run, resumable))
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/src-tauri/src/folder_download/tests.rs b/src-tauri/src/folder_download/tests.rs
new file mode 100644
index 0000000..6cc5e0d
--- /dev/null
+++ b/src-tauri/src/folder_download/tests.rs
@@ -0,0 +1,1453 @@
+use super::*;
+use serde_json::{json, Value};
+use std::collections::{BTreeMap, HashMap};
+use wiremock::matchers::{method, path, path_regex};
+use wiremock::{Mock, MockServer, Request as Asked, Respond, ResponseTemplate};
+
+// ── Names, paths, links ─────────────────────────────────────────────────────────────────────
+
+fn name_in(dir: &Path, taken: &mut Taken, given: &str, folder: bool) -> String {
+ let at = taken.name(dir, given, folder).unwrap();
+ assert_eq!(at.parent(), Some(dir), "{given:?}");
+ assert!(crate::saving::directly_in(dir, &at), "{given:?}");
+ at.file_name().unwrap().to_string_lossy().into_owned()
+}
+
+/// Whatever a sharer called it, a name is one plain name in its own folder — on Linux and on
+/// Windows alike, where `..\`, `C:`, a stream, a device or a trailing dot reach further.
+#[test]
+fn every_name_is_plain_and_stays_in_its_folder() {
+ let dir = tempfile::tempdir().unwrap();
+ for (given, want) in [
+ ("Report.pdf", "Report.pdf"),
+ ("..", "Download"),
+ (".", "Download"),
+ ("", "Download"),
+ (
+ "../../.config/autostart/x.desktop",
+ "configautostartx.desktop",
+ ),
+ (
+ "..\\..\\AppData\\Roaming\\evil.bat",
+ "AppDataRoamingevil.bat",
+ ),
+ ("/etc/passwd", "etcpasswd"),
+ ("C:\\Windows\\evil.dll", "CWindowsevil.dll"),
+ ("C:evil.txt", "Cevil.txt"),
+ ("report.pdf:evil.exe", "report.pdfevil.exe"),
+ ("a*b?c\"d<e>f|g.txt", "abcdefg.txt"),
+ ("CON", "_CON"),
+ ("con.txt", "_con.txt"),
+ ("NUL .txt", "_NUL .txt"),
+ ("COM1", "_COM1"),
+ ("LPT\u{b9}.log", "_LPT\u{b9}.log"),
+ ("trailing. . ", "trailing"),
+ (" .hidden", "hidden"),
+ ("\u{202E}fdp.exe", "fdp.exe"),
+ ("写真 🎉.png", "写真 🎉.png"),
+ ("Привет мир.docx", "Привет мир.docx"),
+ ] {
+ assert_eq!(
+ name_in(dir.path(), &mut Taken::default(), given, false),
+ want,
+ "{given:?}"
+ );
+ }
+ assert_eq!(
+ name_in(dir.path(), &mut Taken::default(), "..", true),
+ "Folder"
+ );
+ // Long names are cut to a length every file system takes, the extension kept, never ending
+ // in a dot or a space Windows would drop.
+ let long = name_in(
+ dir.path(),
+ &mut Taken::default(),
+ &format!("{}. x.pdf", "あ".repeat(200)),
+ false,
+ );
+ assert!(long.len() <= 120, "{} bytes", long.len());
+ assert!(
+ long.ends_with(".pdf") && !long.ends_with([' ', '.']),
+ "{long}"
+ );
+}
+
+#[test]
+fn a_macs_decomposed_accents_are_saved_composed() {
+ let dir = tempfile::tempdir().unwrap();
+ let name = name_in(
+ dir.path(),
+ &mut Taken::default(),
+ "Cafe\u{301} cre\u{300}me.txt",
+ false,
+ );
+ assert_eq!(name, "Caf\u{e9} cr\u{e8}me.txt");
+}
+
+/// Two names one Windows, macOS or exFAT folder would take for one file — the same but for
+/// case, or for how an accent is written, or once cleaned — are numbered.
+#[test]
+fn names_one_folder_would_take_for_one_are_numbered() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut taken = Taken::default();
+ let mut name = |given: &str, folder: bool| name_in(dir.path(), &mut taken, given, folder);
+ assert_eq!(name("Photo.JPG", false), "Photo.JPG");
+ assert_eq!(name("photo.jpg", false), "photo 2.jpg");
+ assert_eq!(name("PHOTO.jpg", false), "PHOTO 3.jpg");
+ assert_eq!(name("Caf\u{e9}.txt", false), "Caf\u{e9}.txt");
+ assert_eq!(name("Cafe\u{301}.txt", false), "Caf\u{e9} 2.txt");
+ assert_eq!(name("a:b.txt", false), "ab.txt");
+ assert_eq!(name("ab.txt", false), "ab 2.txt");
+ assert_eq!(name("v1.2", true), "v1.2");
+ assert_eq!(name("V1.2", true), "V1.2 2");
+ assert_eq!(name("archive.tar.gz", false), "archive.tar.gz");
+ assert_eq!(name("Archive.tar.gz", false), "Archive 2.tar.gz");
+}
+
+/// Inside the download's own folder, names come from the tree alone — not from what is on disk
+/// — so reading the same tree again names it the same way: what resuming depends on.
+#[test]
+fn inside_its_own_folder_the_same_tree_is_named_the_same_way() {
+ let dir = tempfile::tempdir().unwrap();
+ std::fs::write(dir.path().join("a.txt"), b"from before").unwrap();
+ let first = Taken::default().ours(dir.path(), "a.txt", false).unwrap();
+ assert_eq!(first, dir.path().join("a.txt"));
+ let mut taken = Taken::default();
+ taken.ours(dir.path(), "A.txt", false).unwrap();
+ assert_eq!(
+ taken.ours(dir.path(), "a.txt", false).unwrap(),
+ dir.path().join("a 2.txt")
+ );
+}
+
+#[test]
+fn a_folder_is_always_made_new_and_numbered_past_what_is_there() {
+ let dir = tempfile::tempdir().unwrap();
+ std::fs::create_dir(dir.path().join("Photos")).unwrap();
+ std::fs::write(dir.path().join("Photos 2"), b"a file").unwrap();
+ let made = make_folder(&mut Taken::default(), dir.path(), "Photos").unwrap();
+ assert_eq!(made, dir.path().join("Photos 3"));
+ assert!(made.is_dir());
+}
+
+#[cfg(unix)]
+#[test]
+fn a_link_in_the_way_is_never_followed() {
+ let dir = tempfile::tempdir().unwrap();
+ let elsewhere = tempfile::tempdir().unwrap();
+ std::os::unix::fs::symlink(elsewhere.path(), dir.path().join("Photos")).unwrap();
+ std::os::unix::fs::symlink("/nonexistent/target", dir.path().join("Photos 2")).unwrap();
+ let made = make_folder(&mut Taken::default(), dir.path(), "Photos").unwrap();
+ assert_eq!(made, dir.path().join("Photos 3"));
+ assert_eq!(std::fs::read_dir(elsewhere.path()).unwrap().count(), 0);
+
+ // A folder of the download's own swapped for a link is noticed before a file goes in, and
+ // a link is never taken for a folder made before.
+ let top = dir.path().join("Photos 3");
+ std::fs::create_dir(top.join("Trip")).unwrap();
+ assert!(real_folders(&top, &top.join("Trip")));
+ std::fs::remove_dir(top.join("Trip")).unwrap();
+ std::os::unix::fs::symlink(elsewhere.path(), top.join("Trip")).unwrap();
+ assert!(!real_folders(&top, &top.join("Trip")));
+ assert!(!real_folders(&top, Path::new("/etc")));
+ assert!(ensure_folder(&mut Taken::default(), &top, "Trip").is_err());
+}
+
+#[test]
+fn disk_errors_are_said_in_words() {
+ #[cfg(unix)]
+ let (full, long) = (libc::ENOSPC, libc::ENAMETOOLONG);
+ #[cfg(windows)]
+ let (full, long) = (112, 206);
+ let full = std::io::Error::from_raw_os_error(full);
+ assert!(no_room(&full));
+ assert!(disk_words(&full).contains("enough space"));
+ let long = std::io::Error::from_raw_os_error(long);
+ assert!(!no_room(&long));
+ assert!(disk_words(&long).contains("too long"));
+ assert!(no_room(&std::io::Error::from(
+ std::io::ErrorKind::StorageFull
+ )));
+ assert_eq!(size_words(512), "512 bytes");
+ assert_eq!(size_words(3_200_000_000), "3.2 GB");
+}
+
+#[test]
+fn this_computers_free_space_can_be_read() {
+ let dir = tempfile::tempdir().unwrap();
+ assert!(free_space(dir.path()).is_some_and(|n| n > 0));
+}
+
+#[tokio::test]
+async fn cancel_stops_what_runs_and_what_is_to_come() {
+ let stop = Stop::default();
+ let running = {
+ let stop = stop.clone();
+ tokio::spawn(async move {
+ stop.guard(tokio::time::sleep(Duration::from_secs(30)))
+ .await
+ })
+ };
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ stop.stop();
+ let stopped = tokio::time::timeout(Duration::from_secs(2), running)
+ .await
+ .expect("stopped at once")
+ .unwrap();
+ assert!(stopped.is_none());
+ assert!(
+ stop.guard(async { 1 }).await.is_none(),
+ "nothing begins once stopped"
+ );
+ assert!(stop.0.running.lock().unwrap().is_empty());
+ let fresh = Stop::default();
+ assert_eq!(fresh.guard(async { 2 }).await, Some(2));
+ assert!(
+ fresh.0.running.lock().unwrap().is_empty(),
+ "what finishes lets go of its stop"
+ );
+}
+
+/// The journal: the same items in any order are one download; only a key's own shape names a
+/// file; a record comes back as it went in.
+#[test]
+fn the_journal_keeps_what_it_is_given_and_nothing_else() {
+ let dir = tempfile::tempdir().unwrap();
+ let journal = Journal::new(dir.path().join("downloads"));
+ let (a, b) = (
+ item(&file("a", "a", None, b"")),
+ item(&folder("b", "B", vec![])),
+ );
+ assert_eq!(
+ Journal::key(&[a.clone(), b.clone()]),
+ Journal::key(&[b.clone(), a.clone()])
+ );
+ assert_ne!(
+ Journal::key(std::slice::from_ref(&a)),
+ Journal::key(std::slice::from_ref(&b))
+ );
+ let key = Journal::key(&[a.clone(), b.clone()]);
+ let record = Record {
+ key: key.clone(),
+ title: "2 items".into(),
+ items: vec![a, b],
+ placed: vec![Placed {
+ drivewsid: "x".into(),
+ path: "/d/B".into(),
+ folder: true,
+ }],
+ started_at: 7,
+ };
+ journal.save(&record).unwrap();
+ assert_eq!(journal.load(&key), Some(record.clone()));
+ assert_eq!(journal.unfinished(), vec![record]);
+ assert!(journal.load("../../etc/passwd").is_none());
+ assert!(journal.file("0123").is_none());
+ journal.forget(&key);
+ assert!(journal.unfinished().is_empty());
+}
+
+// ── A Drive of its own ──────────────────────────────────────────────────────────────────────
+
+const ZONE: &str = "com.apple.CloudDocs";
+
+#[derive(Clone)]
+struct Node {
+ id: String,
+ name: String,
+ ext: Option<String>,
+ folder: bool,
+ bytes: Vec<u8>,
+ children: Vec<Node>,
+}
+
+fn file(id: &str, name: &str, ext: Option<&str>, bytes: &[u8]) -> Node {
+ Node {
+ id: id.into(),
+ name: name.into(),
+ ext: ext.map(Into::into),
+ folder: false,
+ bytes: bytes.to_vec(),
+ children: vec![],
+ }
+}
+
+fn folder(id: &str, name: &str, children: Vec<Node>) -> Node {
+ Node {
+ id: id.into(),
+ name: name.into(),
+ ext: None,
+ folder: true,
+ bytes: vec![],
+ children,
+ }
+}
+
+fn wsid(n: &Node) -> String {
+ format!(
+ "{}::{ZONE}::{}",
+ if n.folder { "FOLDER" } else { "FILE" },
+ n.id
+ )
+}
+
+fn as_json(n: &Node, parent: &str) -> Value {
+ json!({
+ "drivewsid": wsid(n), "docwsid": n.id, "parentId": parent, "name": n.name,
+ "extension": n.ext, "type": if n.folder { "FOLDER" } else { "FILE" },
+ "size": if n.folder { -1 } else { n.bytes.len() as i64 }, "etag": "1",
+ })
+}
+
+fn item(n: &Node) -> DriveItem {
+ DriveItem {
+ drivewsid: wsid(n),
+ parent_id: None,
+ name: n.name.clone(),
+ extension: n.ext.clone(),
+ kind: if n.folder { "FOLDER" } else { "FILE" }.into(),
+ size: (!n.folder).then_some(n.bytes.len() as i64),
+ child_count: None,
+ created_at: None,
+ modified_at: None,
+ etag: None,
+ }
+}
+
+/// `retrieveItemDetailsInFolders` for the pretend tree.
+struct Listings {
+ folders: HashMap<String, Vec<Value>>,
+ broken: HashSet<String>,
+ signed_out: bool,
+}
+
+impl Respond for Listings {
+ fn respond(&self, request: &Asked) -> ResponseTemplate {
+ if self.signed_out {
+ return ResponseTemplate::new(450);
+ }
+ let asked: Value = serde_json::from_slice(&request.body).unwrap();
+ let id = asked[0]["drivewsid"].as_str().unwrap().to_string();
+ if self.broken.contains(&id) {
+ return ResponseTemplate::new(500);
+ }
+ match self.folders.get(&id) {
+ Some(items) => ResponseTemplate::new(200)
+ .set_body_json(json!([{ "drivewsid": id, "status": "OK", "items": items }])),
+ None => ResponseTemplate::new(200)
+ .set_body_json(json!([{ "items": [], "status": "ID_INVALID" }])),
+ }
+ }
+}
+
+/// `download/by_id`: a link to the bytes.
+struct Links(String);
+
+impl Respond for Links {
+ fn respond(&self, request: &Asked) -> ResponseTemplate {
+ let id = request
+ .url
+ .query_pairs()
+ .find(|(k, _)| k == "document_id")
+ .map(|(_, v)| v.into_owned())
+ .unwrap();
+ ResponseTemplate::new(200)
+ .set_body_json(json!({ "data_token": { "url": format!("{}/bytes/{id}", self.0) } }))
+ }
+}
+
+type Hits = Arc<Mutex<HashMap<String, u32>>>;
+
+/// The bytes themselves: some busy for a while, some very slow, every fetch counted.
+struct Bytes {
+ files: HashMap<String, Vec<u8>>,
+ busy: Mutex<HashMap<String, u32>>,
+ slow: HashSet<String>,
+ /// After this many fetches, every one is busy: a connection that goes and stays gone.
+ busy_after: Option<u32>,
+ hits: Hits,
+}
+
+impl Respond for Bytes {
+ fn respond(&self, request: &Asked) -> ResponseTemplate {
+ let id = request.url.path().rsplit('/').next().unwrap().to_string();
+ let asked: u32 = {
+ let mut hits = self.hits.lock().unwrap();
+ *hits.entry(id.clone()).or_default() += 1;
+ hits.values().sum()
+ };
+ if self.busy_after.is_some_and(|n| asked > n) {
+ return ResponseTemplate::new(503);
+ }
+ if let Some(left) = self.busy.lock().unwrap().get_mut(&id).filter(|n| **n > 0) {
+ *left -= 1;
+ return ResponseTemplate::new(503);
+ }
+ let reply = ResponseTemplate::new(200).set_body_bytes(self.files[&id].clone());
+ if self.slow.contains(&id) {
+ reply.set_delay(Duration::from_secs(30))
+ } else {
+ reply
+ }
+ }
+}
+
+#[derive(Default)]
+struct Setup {
+ broken: Vec<&'static str>,
+ busy: Vec<(&'static str, u32)>,
+ slow: Vec<&'static str>,
+ busy_after: Option<u32>,
+ signed_out: bool,
+ free_space: Option<fn(&Path) -> Option<u64>>,
+}
+
+fn index(
+ n: &Node,
+ folders: &mut HashMap<String, Vec<Value>>,
+ files: &mut HashMap<String, Vec<u8>>,
+) {
+ if n.folder {
+ folders.insert(
+ wsid(n),
+ n.children.iter().map(|c| as_json(c, &wsid(n))).collect(),
+ );
+ for c in &n.children {
+ index(c, folders, files);
+ }
+ } else {
+ files.insert(n.id.clone(), n.bytes.clone());
+ }
+}
+
+async fn drive_of(roots: &[Node], setup: Setup) -> (MockServer, Drive, Hits) {
+ let server = MockServer::builder()
+ .disable_request_recording()
+ .start()
+ .await;
+ let (mut folders, mut files) = (HashMap::new(), HashMap::new());
+ for r in roots {
+ index(r, &mut folders, &mut files);
+ }
+ Mock::given(method("POST"))
+ .and(path("/retrieveItemDetailsInFolders"))
+ .respond_with(Listings {
+ folders,
+ broken: setup
+ .broken
+ .iter()
+ .map(|id| format!("FOLDER::{ZONE}::{id}"))
+ .collect(),
+ signed_out: setup.signed_out,
+ })
+ .mount(&server)
+ .await;
+ Mock::given(method("GET"))
+ .and(path(format!("/ws/{ZONE}/download/by_id")))
+ .respond_with(Links(server.uri()))
+ .mount(&server)
+ .await;
+ let hits: Hits = Arc::default();
+ Mock::given(method("GET"))
+ .and(path_regex("^/bytes/"))
+ .respond_with(Bytes {
+ files,
+ busy: Mutex::new(
+ setup
+ .busy
+ .iter()
+ .map(|(id, n)| (id.to_string(), *n))
+ .collect(),
+ ),
+ slow: setup.slow.iter().map(ToString::to_string).collect(),
+ busy_after: setup.busy_after,
+ hits: hits.clone(),
+ })
+ .mount(&server)
+ .await;
+ let drive = Drive {
+ http: reqwest::Client::new(),
+ drivews: server.uri(),
+ docws: server.uri(),
+ params: vec![],
+ retry_after: Duration::from_millis(2),
+ free_space: setup.free_space.unwrap_or(|_| None),
+ room_every: Duration::ZERO,
+ };
+ (server, drive, hits)
+}
+
+type Heard = Arc<Mutex<Vec<DownloadEvent>>>;
+
+fn hearing() -> (Heard, Emit) {
+ let heard: Heard = Arc::default();
+ let emit: Emit = {
+ let heard = heard.clone();
+ Arc::new(move |e| heard.lock().unwrap().push(e))
+ };
+ (heard, emit)
+}
+
+async fn download(
+ drive: &Drive,
+ roots: &[&Node],
+ into: &Path,
+ journal: Option<&Journal>,
+ emit: Emit,
+ stop: &Stop,
+) -> Result<DownloadReport, String> {
+ let request = Request {
+ batch: "b",
+ title: "t",
+ items: roots.iter().map(|n| item(n)).collect(),
+ into,
+ journal,
+ };
+ run(drive, request, emit, stop).await
+}
+
+/// Every file and folder under `dir`, by its path inside it, with a file's bytes.
+fn on_disk(dir: &Path) -> BTreeMap<String, Option<Vec<u8>>> {
+ fn walk(root: &Path, at: &Path, out: &mut BTreeMap<String, Option<Vec<u8>>>) {
+ for entry in std::fs::read_dir(at).unwrap() {
+ let path = entry.unwrap().path();
+ let rel = path
+ .strip_prefix(root)
+ .unwrap()
+ .to_string_lossy()
+ .replace('\\', "/");
+ if std::fs::symlink_metadata(&path).unwrap().is_dir() {
+ out.insert(format!("{rel}/"), None);
+ walk(root, &path, out);
+ } else {
+ out.insert(rel, Some(std::fs::read(&path).unwrap()));
+ }
+ }
+ }
+ let mut out = BTreeMap::new();
+ walk(dir, dir, &mut out);
+ out
+}
+
+fn photos() -> Node {
+ folder(
+ "photos",
+ "Photos",
+ vec![
+ file("beach", "beach", Some("jpg"), b"sand"),
+ file("rc", "../../.bashrc", None, b"rm -rf"),
+ file("con", "CON", Some("txt"), b"device"),
+ file("p1", "Photo", Some("JPG"), b"upper"),
+ file("p2", "photo", Some("jpg"), b"lower"),
+ file("nfd", "Cafe\u{301}", Some("txt"), b"decomposed"),
+ file("nfc", "Caf\u{e9}", Some("txt"), b"composed"),
+ file("cjk", "写真 🎉", Some("png"), b"emoji"),
+ file("flaky", "flaky", Some("bin"), b"second time lucky"),
+ file("long", &"长".repeat(150), Some("pdf"), b"long"),
+ folder(
+ "trip",
+ "Trip",
+ vec![
+ file("map", "map", Some("pdf"), b"%PDF"),
+ folder("empty", "Empty", vec![]),
+ folder("up", "..", vec![file("x", "x", Some("txt"), b"x")]),
+ folder("odd", "a:b*c?", vec![]),
+ folder("dots", "trailing. . ", vec![]),
+ ],
+ ),
+ ],
+ )
+}
+
+/// A folder arrives as the same tree, every name made safe, clashes numbered, a busy moment
+/// ridden out — beside a folder of the same name already in Downloads, untouched — and once it
+/// is all there, nothing of it is kept to resume.
+#[tokio::test]
+async fn a_folder_arrives_as_the_same_tree() {
+ let tree = photos();
+ let (_server, drive, _) = drive_of(
+ std::slice::from_ref(&tree),
+ Setup {
+ busy: vec![("flaky", 2)],
+ ..Setup::default()
+ },
+ )
+ .await;
+ let downloads = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ let journal = Journal::new(state.path().to_path_buf());
+ std::fs::create_dir(downloads.path().join("Photos")).unwrap();
+ std::fs::write(downloads.path().join("Photos/mine.txt"), b"mine").unwrap();
+ let (heard, emit) = hearing();
+ let report = download(
+ &drive,
+ &[&tree],
+ downloads.path(),
+ Some(&journal),
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap();
+
+ let long = format!("{}.pdf", "长".repeat(38));
+ let mut want: BTreeMap<String, Option<Vec<u8>>> = [
+ ("Photos/", None),
+ ("Photos/mine.txt", Some(&b"mine"[..])),
+ ("Photos 2/", None),
+ ("Photos 2/beach.jpg", Some(b"sand")),
+ ("Photos 2/bashrc", Some(b"rm -rf")),
+ ("Photos 2/_CON.txt", Some(b"device")),
+ ("Photos 2/Photo.JPG", Some(b"upper")),
+ ("Photos 2/photo 2.jpg", Some(b"lower")),
+ ("Photos 2/Caf\u{e9}.txt", Some(b"composed")),
+ ("Photos 2/Caf\u{e9} 2.txt", Some(b"decomposed")),
+ ("Photos 2/写真 🎉.png", Some(b"emoji")),
+ ("Photos 2/flaky.bin", Some(b"second time lucky")),
+ ("Photos 2/Trip/", None),
+ ("Photos 2/Trip/map.pdf", Some(b"%PDF")),
+ ("Photos 2/Trip/Empty/", None),
+ ("Photos 2/Trip/Folder/", None),
+ ("Photos 2/Trip/Folder/x.txt", Some(b"x")),
+ ("Photos 2/Trip/abc/", None),
+ ("Photos 2/Trip/trailing/", None),
+ ]
+ .into_iter()
+ .map(|(k, v)| (k.to_string(), v.map(<[u8]>::to_vec)))
+ .collect();
+ want.insert(format!("Photos 2/{long}"), Some(b"long".to_vec()));
+ assert_eq!(on_disk(downloads.path()), want);
+ assert_eq!(
+ report.saved,
+ vec![downloads
+ .path()
+ .join("Photos 2")
+ .to_string_lossy()
+ .into_owned()]
+ );
+ assert!(
+ report.failed.is_empty() && !report.cancelled && report.problem.is_none(),
+ "{report:?}"
+ );
+ assert!(
+ !report.resumable && journal.unfinished().is_empty(),
+ "nothing kept once it is all there"
+ );
+
+ let heard = heard.lock().unwrap();
+ let bytes: u64 = want.values().flatten().map(|b| b.len() as u64).sum::<u64>() - 4;
+ let planned: Vec<_> = heard.iter().filter(|e| e.state == "planned").collect();
+ assert_eq!(
+ planned.first().map(|e| (e.sent, e.total)),
+ Some((12, bytes)),
+ "the total is known before a byte arrives"
+ );
+ let first_done = heard.iter().position(|e| e.state == "done").unwrap();
+ assert!(heard.iter().position(|e| e.state == "planned").unwrap() < first_done);
+ assert_eq!(heard.iter().filter(|e| e.state == "done").count(), 12);
+ let map = heard
+ .iter()
+ .find(|e| e.name == "map.pdf" && e.state == "done")
+ .unwrap();
+ assert_eq!(
+ (map.within.as_str(), map.sent, map.total),
+ ("Photos 2/Trip", 4, 4)
+ );
+}
+
+/// Files and folders chosen together land side by side in Downloads — here a Downloads that is
+/// a link to a folder on another disk, as a moved Downloads often is.
+#[cfg(unix)]
+#[tokio::test]
+async fn a_selection_lands_side_by_side_in_a_downloads_kept_elsewhere() {
+ let notes = file("notes", "notes", Some("txt"), b"new notes");
+ let docs = folder("docs", "Docs", vec![file("a", "a", Some("txt"), b"a")]);
+ let (_server, drive, _) = drive_of(&[notes.clone(), docs.clone()], Setup::default()).await;
+ let disk = tempfile::tempdir().unwrap();
+ let other = disk.path().join("other-disk/Downloads");
+ std::fs::create_dir_all(&other).unwrap();
+ std::fs::write(other.join("notes.txt"), b"old notes").unwrap();
+ let downloads = disk.path().join("Downloads");
+ std::os::unix::fs::symlink(&other, &downloads).unwrap();
+
+ let (_heard, emit) = hearing();
+ let report = download(
+ &drive,
+ &[&notes, &docs],
+ &downloads,
+ None,
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ std::fs::read(other.join("notes.txt")).unwrap(),
+ b"old notes"
+ );
+ assert_eq!(
+ std::fs::read(other.join("notes 2.txt")).unwrap(),
+ b"new notes"
+ );
+ assert_eq!(std::fs::read(other.join("Docs/a.txt")).unwrap(), b"a");
+ assert_eq!(
+ report.saved,
+ vec![
+ downloads.join("notes 2.txt").to_string_lossy().into_owned(),
+ downloads.join("Docs").to_string_lossy().into_owned(),
+ ]
+ );
+}
+
+fn big() -> Node {
+ folder(
+ "big",
+ "Big",
+ vec![
+ file("fast", "fast", Some("txt"), b"here"),
+ folder(
+ "later",
+ "Later",
+ vec![file("slow", "slow", Some("bin"), &[9; 2 << 20])],
+ ),
+ ],
+ )
+}
+
+/// Cancel drops what is arriving and keeps what arrived; folders it made that are left empty
+/// go again. Downloaded again — after a restart, read from the journal afresh — it goes on in
+/// the same folder: what arrived is not fetched twice.
+#[tokio::test]
+async fn cancelled_it_keeps_what_arrived_and_resumes_where_it_was() {
+ let tree = big();
+ let downloads = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ {
+ let (_server, drive, _) = drive_of(
+ std::slice::from_ref(&tree),
+ Setup {
+ slow: vec!["slow"],
+ ..Setup::default()
+ },
+ )
+ .await;
+ let stop = Stop::default();
+ let (heard, emit) = hearing();
+ // Cancel pressed while the slow one is on its way: its request is out, its bytes not back.
+ let emit: Emit = {
+ let stop = stop.clone();
+ Arc::new(move |e: DownloadEvent| {
+ let go = e.name == "slow.bin" && e.state == "downloading";
+ emit(e);
+ if go {
+ let stop = stop.clone();
+ tokio::spawn(async move {
+ tokio::time::sleep(Duration::from_millis(300)).await;
+ stop.stop();
+ });
+ }
+ })
+ };
+ let journal = Journal::new(state.path().to_path_buf());
+ let began = Instant::now();
+ let report = download(
+ &drive,
+ &[&tree],
+ downloads.path(),
+ Some(&journal),
+ emit,
+ &stop,
+ )
+ .await
+ .unwrap();
+ assert!(began.elapsed() < Duration::from_secs(10), "stopped at once");
+ assert!(
+ report.cancelled && report.problem.is_none() && report.resumable,
+ "{report:?}"
+ );
+ let disk = on_disk(downloads.path());
+ assert_eq!(disk.get("Big/fast.txt"), Some(&Some(b"here".to_vec())));
+ assert!(
+ !disk
+ .keys()
+ .any(|k| k.contains("Later") || k.contains(".part")),
+ "{disk:?}"
+ );
+ let slow: Vec<_> = heard
+ .lock()
+ .unwrap()
+ .iter()
+ .filter(|e| e.name == "slow.bin")
+ .map(|e| e.state)
+ .collect();
+ assert_eq!(slow.last(), Some(&"cancelled"));
+ }
+ // Skykeep quit and opened again: only the journal on disk is left of it.
+ let journal = Journal::new(state.path().to_path_buf());
+ let unfinished = journal.unfinished();
+ assert_eq!(unfinished.len(), 1);
+ assert_eq!(unfinished[0].title, "t");
+ let (_server, drive, hits) = drive_of(std::slice::from_ref(&tree), Setup::default()).await;
+ let (_heard, emit) = hearing();
+ let report = download(
+ &drive,
+ &[&tree],
+ downloads.path(),
+ Some(&journal),
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap();
+ assert!(report.failed.is_empty() && !report.resumable, "{report:?}");
+ let disk = on_disk(downloads.path());
+ assert_eq!(
+ disk.keys().filter(|k| k.starts_with("Big")).count(),
+ 4,
+ "{:?}",
+ disk.keys()
+ );
+ assert_eq!(
+ disk.get("Big/Later/slow.bin")
+ .map(|b| b.as_ref().map(Vec::len)),
+ Some(Some(2 << 20))
+ );
+ assert!(
+ !disk.keys().any(|k| k.starts_with("Big 2")),
+ "the same folder, not a second one"
+ );
+ assert_eq!(
+ hits.lock().unwrap().get("fast"),
+ None,
+ "what arrived before is not fetched again"
+ );
+ assert!(journal.unfinished().is_empty());
+}
+
+/// Resumed, a file there at another size than iCloud's — an older copy, or cut short by a
+/// crash with no partial file of its own — is fetched again in its place, and partial files a
+/// crash left behind are cleared.
+#[tokio::test]
+async fn resumed_a_wrong_sized_copy_is_replaced_and_leftovers_cleared() {
+ let tree = folder(
+ "r",
+ "R",
+ vec![
+ file("a", "a", Some("txt"), b"right"),
+ file("b", "b", Some("txt"), b"also right"),
+ ],
+ );
+ let downloads = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ let journal = Journal::new(state.path().to_path_buf());
+ let top = downloads.path().join("R");
+ std::fs::create_dir(&top).unwrap();
+ std::fs::write(top.join("a.txt"), b"wrong!!").unwrap();
+ std::fs::write(top.join("b.txt"), b"also right").unwrap();
+ std::fs::write(top.join("a.txt.part"), b"crash").unwrap();
+ std::fs::write(top.join("a.txt.part.1"), b"crash").unwrap();
+ journal
+ .save(&Record {
+ key: Journal::key(&[item(&tree)]),
+ title: "R".into(),
+ items: vec![item(&tree)],
+ placed: vec![Placed {
+ drivewsid: wsid(&tree),
+ path: top.clone(),
+ folder: true,
+ }],
+ started_at: 1,
+ })
+ .unwrap();
+ let (_server, drive, hits) = drive_of(std::slice::from_ref(&tree), Setup::default()).await;
+ let (_heard, emit) = hearing();
+ let report = download(
+ &drive,
+ &[&tree],
+ downloads.path(),
+ Some(&journal),
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap();
+ assert!(report.failed.is_empty(), "{report:?}");
+ assert_eq!(
+ on_disk(downloads.path()).into_iter().collect::<Vec<_>>(),
+ vec![
+ ("R/".into(), None),
+ ("R/a.txt".into(), Some(b"right".to_vec())),
+ ("R/b.txt".into(), Some(b"also right".to_vec())),
+ ]
+ );
+ assert_eq!(hits.lock().unwrap().get("b"), None);
+}
+
+/// A journal edited to point outside Downloads is not followed: the download starts afresh.
+#[tokio::test]
+async fn a_journal_pointing_outside_downloads_is_not_followed() {
+ let tree = folder("r", "R", vec![file("a", "a", Some("txt"), b"x")]);
+ let downloads = tempfile::tempdir().unwrap();
+ let elsewhere = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ let journal = Journal::new(state.path().to_path_buf());
+ journal
+ .save(&Record {
+ key: Journal::key(&[item(&tree)]),
+ title: "R".into(),
+ items: vec![item(&tree)],
+ placed: vec![Placed {
+ drivewsid: wsid(&tree),
+ path: elsewhere.path().to_path_buf(),
+ folder: true,
+ }],
+ started_at: 1,
+ })
+ .unwrap();
+ let (_server, drive, _) = drive_of(std::slice::from_ref(&tree), Setup::default()).await;
+ let (_heard, emit) = hearing();
+ download(
+ &drive,
+ &[&tree],
+ downloads.path(),
+ Some(&journal),
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(std::fs::read_dir(elsewhere.path()).unwrap().count(), 0);
+ assert_eq!(
+ std::fs::read(downloads.path().join("R/a.txt")).unwrap(),
+ b"x"
+ );
+}
+
+/// A folder that cannot be read is said, and not left behind empty as if that were what it
+/// held; everything else arrives — and once it can be read, resuming fetches only it.
+#[tokio::test]
+async fn a_folder_that_cannot_be_read_is_said_and_can_be_gone_back_for() {
+ let tree = folder(
+ "top",
+ "Top",
+ vec![
+ file("ok", "ok", Some("txt"), b"fine"),
+ folder(
+ "broken",
+ "Broken",
+ vec![file("lost", "lost", Some("txt"), b"found")],
+ ),
+ ],
+ );
+ let downloads = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ let journal = Journal::new(state.path().to_path_buf());
+ {
+ let (_server, drive, _) = drive_of(
+ std::slice::from_ref(&tree),
+ Setup {
+ broken: vec!["broken"],
+ ..Setup::default()
+ },
+ )
+ .await;
+ let (heard, emit) = hearing();
+ let report = download(
+ &drive,
+ &[&tree],
+ downloads.path(),
+ Some(&journal),
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap();
+ let disk = on_disk(downloads.path());
+ assert_eq!(disk.keys().collect::<Vec<_>>(), ["Top/", "Top/ok.txt"]);
+ assert_eq!(report.failed.len(), 1, "{:?}", report.failed);
+ assert!(
+ report.failed[0].starts_with("Top/Broken: ") && report.failed[0].contains("busy"),
+ "{:?}",
+ report.failed
+ );
+ assert!(report.resumable);
+ assert!(heard
+ .lock()
+ .unwrap()
+ .iter()
+ .any(|e| e.folder && e.state == "failed" && e.name == "Broken"));
+ }
+ let (_server, drive, hits) = drive_of(std::slice::from_ref(&tree), Setup::default()).await;
+ let (_heard, emit) = hearing();
+ let report = download(
+ &drive,
+ &[&tree],
+ downloads.path(),
+ Some(&journal),
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap();
+ assert!(report.failed.is_empty() && !report.resumable, "{report:?}");
+ assert_eq!(
+ std::fs::read(downloads.path().join("Top/Broken/lost.txt")).unwrap(),
+ b"found"
+ );
+ assert_eq!(hits.lock().unwrap().get("ok"), None);
+}
+
+/// A fresh sign-in needed stops the lot before anything is written.
+#[tokio::test]
+async fn a_lapsed_sign_in_stops_everything() {
+ let tree = photos();
+ let (_server, drive, _) = drive_of(
+ std::slice::from_ref(&tree),
+ Setup {
+ signed_out: true,
+ ..Setup::default()
+ },
+ )
+ .await;
+ let downloads = tempfile::tempdir().unwrap();
+ let (_heard, emit) = hearing();
+ let err = download(
+ &drive,
+ &[&tree],
+ downloads.path(),
+ None,
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap_err();
+ assert!(err.contains("sign-in"), "{err}");
+ assert!(on_disk(downloads.path()).is_empty());
+}
+
+/// A download bigger than the room there is is refused before a byte — or a folder — is
+/// written, and says what it needs.
+#[tokio::test]
+async fn a_download_too_big_for_the_disk_is_refused_before_anything_is_written() {
+ let tree = folder(
+ "big",
+ "Big",
+ vec![folder(
+ "in",
+ "In",
+ vec![file("f", "film", Some("mov"), &[7; 100])],
+ )],
+ );
+ let (_server, drive, hits) = drive_of(
+ std::slice::from_ref(&tree),
+ Setup {
+ free_space: Some(|_| Some(10)),
+ ..Setup::default()
+ },
+ )
+ .await;
+ let downloads = tempfile::tempdir().unwrap();
+ let (_heard, emit) = hearing();
+ let report = download(
+ &drive,
+ &[&tree],
+ downloads.path(),
+ None,
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap();
+ let problem = report.problem.clone().unwrap_or_default();
+ assert!(
+ problem.contains("isn\u{2019}t room")
+ && problem.contains("100 bytes")
+ && problem.contains("10 bytes"),
+ "{problem}"
+ );
+ assert!(!report.cancelled && report.saved.is_empty(), "{report:?}");
+ assert!(on_disk(downloads.path()).is_empty());
+ assert!(hits.lock().unwrap().is_empty(), "not a byte asked for");
+}
+
+static ROOM: AtomicU64 = AtomicU64::new(u64::MAX);
+
+/// Room that runs out while it downloads — something else filling the disk — pauses it, with
+/// what arrived kept and the rest resumable.
+#[tokio::test]
+async fn room_running_out_as_it_goes_pauses_it() {
+ let tree = folder(
+ "f",
+ "F",
+ (0..40)
+ .map(|n| {
+ file(
+ &format!("f{n}"),
+ &format!("{n:02}"),
+ Some("bin"),
+ &[1; 1000],
+ )
+ })
+ .collect(),
+ );
+ let (_server, drive, _) = drive_of(
+ std::slice::from_ref(&tree),
+ Setup {
+ free_space: Some(|_| Some(ROOM.load(Ordering::SeqCst))),
+ ..Setup::default()
+ },
+ )
+ .await;
+ let downloads = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ let journal = Journal::new(state.path().to_path_buf());
+ let (_heard, emit) = hearing();
+ // The disk fills up the moment the first file lands; the next look at it notices.
+ let emit: Emit = Arc::new(move |e: DownloadEvent| {
+ if e.state == "done" {
+ ROOM.store(500, Ordering::SeqCst);
+ }
+ emit(e);
+ });
+ let stop = Stop::default();
+ let report = tokio::time::timeout(
+ Duration::from_secs(30),
+ download(
+ &drive,
+ &[&tree],
+ downloads.path(),
+ Some(&journal),
+ emit,
+ &stop,
+ ),
+ )
+ .await
+ .expect("it ends")
+ .unwrap();
+ let problem = report.problem.clone().unwrap_or_default();
+ assert!(problem.contains("running out of space"), "{report:?}");
+ assert!(report.resumable && !report.cancelled);
+}
+
+/// A connection that stays lost pauses the download — it does not go on to fail every file —
+/// and it can be gone on with.
+#[tokio::test]
+async fn a_lost_connection_pauses_rather_than_failing_everything() {
+ let tree = folder(
+ "f",
+ "F",
+ (0..60)
+ .map(|n| file(&format!("f{n}"), &format!("{n:02}"), Some("bin"), b"x"))
+ .collect(),
+ );
+ let (_server, drive, hits) = drive_of(
+ std::slice::from_ref(&tree),
+ Setup {
+ busy_after: Some(5),
+ ..Setup::default()
+ },
+ )
+ .await;
+ let downloads = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ let journal = Journal::new(state.path().to_path_buf());
+ let (_heard, emit) = hearing();
+ let report = download(
+ &drive,
+ &[&tree],
+ downloads.path(),
+ Some(&journal),
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap();
+ let problem = report.problem.clone().unwrap_or_default();
+ assert!(problem.contains("connection"), "{report:?}");
+ assert!(report.resumable);
+ let asked = hits.lock().unwrap().len();
+ assert!(asked < 60, "stopped asking: {asked} of 60 files were tried");
+}
+
+/// Thousands of files: every one arrives; the reading never gets more than AHEAD files ahead of
+/// the fetching; and a small file costs the window one event.
+#[tokio::test]
+async fn a_folder_of_thousands_arrives_whole_in_bounded_memory() {
+ const MANY: usize = 1500;
+ let tree = folder(
+ "many",
+ "Many",
+ (0..MANY)
+ .map(|n| file(&format!("f{n}"), &format!("file {n:04}"), Some("txt"), b"x"))
+ .collect(),
+ );
+ let (_server, drive, _) = drive_of(std::slice::from_ref(&tree), Setup::default()).await;
+ let downloads = tempfile::tempdir().unwrap();
+ let (heard, emit) = hearing();
+ let report = download(
+ &drive,
+ &[&tree],
+ downloads.path(),
+ None,
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap();
+ assert!(report.failed.is_empty(), "{:?}", report.failed);
+ assert_eq!(
+ std::fs::read_dir(downloads.path().join("Many"))
+ .unwrap()
+ .count(),
+ MANY
+ );
+ assert!(
+ report.peak_waiting <= AHEAD + AT_ONCE + 1,
+ "{} files waited at once",
+ report.peak_waiting
+ );
+ let events = heard.lock().unwrap().len();
+ assert!(
+ events <= MANY + 50,
+ "{events} events for {MANY} small files"
+ );
+}
+
+/// A tree deeper than any real one is saved as deep as DEEPEST, the rest said.
+#[tokio::test]
+async fn a_tree_too_deep_is_saved_as_far_as_it_sensibly_goes() {
+ let levels = DEEPEST + 4;
+ let mut node = folder(
+ &format!("d{levels}"),
+ "z",
+ vec![file("bottom", "bottom", Some("txt"), b"b")],
+ );
+ for n in (0..levels).rev() {
+ let mut children = vec![node];
+ if n == 10 {
+ children.push(file("mid", "mid", Some("txt"), b"m"));
+ }
+ node = folder(&format!("d{n}"), "z", children);
+ }
+ let (_server, drive, _) = drive_of(std::slice::from_ref(&node), Setup::default()).await;
+ let downloads = tempfile::tempdir().unwrap();
+ let (_heard, emit) = hearing();
+ let report = download(
+ &drive,
+ &[&node],
+ downloads.path(),
+ None,
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(report.failed.len(), 1, "{:?}", report.failed);
+ assert!(report.failed[0].contains("too many folders deep"));
+ let disk = on_disk(downloads.path());
+ assert!(disk
+ .keys()
+ .any(|k| k.ends_with("/mid.txt") && k.matches('/').count() == 11));
+ assert!(!disk.keys().any(|k| k.ends_with("bottom.txt")));
+ let deepest = disk.keys().map(|k| k.matches('/').count()).max().unwrap();
+ assert_eq!(deepest, DEEPEST, "{deepest} folders deep");
+}
+
+// ── At scale ────────────────────────────────────────────────────────────────────────────────
+
+/// A Drive that keeps nothing: every listing and every file made up as it is asked for, so what
+/// the scale test measures is the download's own memory. (wiremock keeps every request it
+/// answers — a gigabyte by the hundred-thousandth file.) `folders` folders in `Archive`, each
+/// of `each` two-byte files.
+async fn synthetic_drive(folders: usize, each: usize) -> String {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let base = format!("http://{}", listener.local_addr().unwrap());
+ let uri = base.clone();
+ tokio::spawn(async move {
+ while let Ok((socket, _)) = listener.accept().await {
+ let _ = socket.set_nodelay(true);
+ tokio::spawn(answer(socket, base.clone(), folders, each));
+ }
+ });
+ uri
+}
+
+/// One connection, kept alive: HTTP/1.1 as reqwest speaks it, request after request.
+async fn answer(mut socket: tokio::net::TcpStream, base: String, folders: usize, each: usize) {
+ use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};
+ let (reading, mut writing) = socket.split();
+ let mut reading = tokio::io::BufReader::new(reading);
+ loop {
+ let mut line = String::new();
+ if reading.read_line(&mut line).await.unwrap_or(0) == 0 {
+ return;
+ }
+ let target = line
+ .split_whitespace()
+ .nth(1)
+ .unwrap_or_default()
+ .to_string();
+ let mut length = 0usize;
+ loop {
+ let mut header = String::new();
+ if reading.read_line(&mut header).await.unwrap_or(0) == 0 {
+ return;
+ }
+ if header == "\r\n" {
+ break;
+ }
+ if let Some(v) = header.to_ascii_lowercase().strip_prefix("content-length:") {
+ length = v.trim().parse().unwrap_or(0);
+ }
+ }
+ let mut body = vec![0; length];
+ if reading.read_exact(&mut body).await.is_err() {
+ return;
+ }
+ let reply = synthetic_reply(&target, &body, &base, folders, each);
+ // One write: a head and a body sent apart wait on each other's acknowledgement.
+ let mut whole = format!(
+ "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n",
+ reply.len()
+ )
+ .into_bytes();
+ whole.extend_from_slice(&reply);
+ if writing.write_all(&whole).await.is_err() {
+ return;
+ }
+ }
+}
+
+fn synthetic_reply(target: &str, body: &[u8], base: &str, folders: usize, each: usize) -> Vec<u8> {
+ let entry = |kind: &str, id: String, name: String, ext: Option<&str>, size: i64| {
+ json!({ "drivewsid": format!("{kind}::{ZONE}::{id}"), "name": name, "extension": ext,
+ "type": kind, "size": size, "etag": "1" })
+ };
+ if target.starts_with("/retrieveItemDetailsInFolders") {
+ let asked: Value = serde_json::from_slice(body).unwrap();
+ let id = asked[0]["drivewsid"].as_str().unwrap().to_string();
+ let items: Vec<Value> = match id.rsplit("::").next().unwrap() {
+ "archive" => (0..folders)
+ .map(|f| entry("FOLDER", format!("d{f}"), format!("{f:04}"), None, -1))
+ .collect(),
+ d => {
+ let f: usize = d.trim_start_matches('d').parse().unwrap();
+ (0..each)
+ .map(|n| {
+ entry(
+ "FILE",
+ format!("f{f}-{n}"),
+ format!("{n:03}"),
+ Some("bin"),
+ 2,
+ )
+ })
+ .collect()
+ }
+ };
+ serde_json::to_vec(&json!([{ "drivewsid": id, "status": "OK", "items": items }])).unwrap()
+ } else if let Some(query) = target.strip_prefix(&format!("/ws/{ZONE}/download/by_id?")) {
+ let doc = query
+ .split('&')
+ .find_map(|kv| kv.strip_prefix("document_id="))
+ .unwrap();
+ serde_json::to_vec(&json!({ "data_token": { "url": format!("{base}/bytes/{doc}") } }))
+ .unwrap()
+ } else {
+ b"ab".to_vec()
+ }
+}
+
+/// This process's resident memory, in kilobytes, where the system says.
+fn resident_kb() -> Option<u64> {
+ std::fs::read_to_string("/proc/self/status")
+ .ok()?
+ .lines()
+ .find_map(|l| l.strip_prefix("VmRSS:"))?
+ .trim()
+ .trim_end_matches(" kB")
+ .parse()
+ .ok()
+}
+
+/// The largest folders people keep: a hundred thousand files, in a thousand folders. Every one
+/// arrives; the reading's lead stays bounded, the window hears about one event a file, and the
+/// download's memory does not grow with the files it has fetched.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn a_hundred_thousand_files_in_bounded_memory() {
+ const FOLDERS: usize = 1000;
+ const EACH: usize = 100;
+ let uri = synthetic_drive(FOLDERS, EACH).await;
+ let drive = Drive {
+ http: reqwest::Client::new(),
+ drivews: uri.clone(),
+ docws: uri,
+ params: vec![],
+ retry_after: Duration::from_millis(2),
+ free_space,
+ room_every: ROOM_EVERY,
+ };
+ let archive = folder("archive", "Archive", vec![]);
+ let downloads = tempfile::tempdir().unwrap();
+ let events = Arc::new(AtomicUsize::new(0));
+ // Memory once the first ten thousand files are in, when everything a download holds for
+ // its whole length has been made.
+ let early = Arc::new(AtomicU64::new(0));
+ let emit: Emit = {
+ let (events, early) = (events.clone(), early.clone());
+ let done = AtomicUsize::new(0);
+ Arc::new(move |e: DownloadEvent| {
+ events.fetch_add(1, Ordering::Relaxed);
+ if e.state == "done" && done.fetch_add(1, Ordering::Relaxed) + 1 == 10_000 {
+ early.store(resident_kb().unwrap_or(0), Ordering::Relaxed);
+ }
+ })
+ };
+ let began = Instant::now();
+ let report = download(
+ &drive,
+ &[&archive],
+ downloads.path(),
+ None,
+ emit,
+ &Stop::default(),
+ )
+ .await
+ .unwrap();
+ let took = began.elapsed();
+ let late = resident_kb().unwrap_or(0);
+ assert!(
+ report.failed.is_empty(),
+ "{:?}",
+ &report.failed[..report.failed.len().min(5)]
+ );
+ let mut files = 0;
+ for dir in std::fs::read_dir(downloads.path().join("Archive")).unwrap() {
+ files += std::fs::read_dir(dir.unwrap().path()).unwrap().count();
+ }
+ assert_eq!(files, FOLDERS * EACH);
+ assert!(
+ report.peak_waiting <= AHEAD + AT_ONCE + 1,
+ "{} files waited at once",
+ report.peak_waiting
+ );
+ let heard = events.load(Ordering::Relaxed);
+ assert!(
+ heard <= FOLDERS * EACH + 100,
+ "{heard} events for {} files",
+ FOLDERS * EACH
+ );
+ let early = early.load(Ordering::Relaxed);
+ eprintln!(
+ "100k files in {took:?}: {heard} events, at most {} waiting, {early} kB resident after 10k files, {late} kB after 100k",
+ report.peak_waiting
+ );
+ if early > 0 && late > 0 {
+ // Ninety thousand more files fetched, and next to nothing more held: memory that grew
+ // with the files would be hundreds of megabytes by now.
+ assert!(
+ late.saturating_sub(early) < 48 * 1024,
+ "grew from {early} kB to {late} kB"
+ );
+ }
+}
diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs
index 3b4073f..9481fe8 100644
--- a/src-tauri/src/main.rs
+++ b/src-tauri/src/main.rs
@@ -23,6 +23,7 @@ mod data_access;
mod diagnostics;
mod drive_watch;
mod export;
+mod folder_download;
mod grants;
mod intake;
mod library_lock;
@@ -4484,6 +4485,181 @@ async fn drive_download(
Ok(destination.to_string_lossy().into_owned())
}
+/// Downloads that can still be stopped, by batch.
+fn downloads_running(
+) -> &'static std::sync::Mutex<std::collections::HashMap<String, folder_download::Stop>> {
+ static RUNNING: std::sync::OnceLock<
+ std::sync::Mutex<std::collections::HashMap<String, folder_download::Stop>>,
+ > = std::sync::OnceLock::new();
+ RUNNING.get_or_init(Default::default)
+}
+
+/// The downloads running now, by what they are known by (`Journal::key`): the same one twice
+/// at once would write into the one folder from two sides.
+fn downloads_by_key() -> &'static std::sync::Mutex<std::collections::HashSet<String>> {
+ static RUNNING: std::sync::OnceLock<std::sync::Mutex<std::collections::HashSet<String>>> =
+ std::sync::OnceLock::new();
+ RUNNING.get_or_init(Default::default)
+}
+
+/// Where unfinished folder downloads are remembered, so downloading the same again resumes.
+fn download_journal(app: &tauri::AppHandle) -> folder_download::Journal {
+ folder_download::Journal::new(state_dir(app).join("downloads"))
+}
+
+/// An unfinished download from before, for the window to offer to go on with.
+#[derive(serde::Serialize)]
+#[serde(rename_all = "camelCase")]
+struct UnfinishedDownload {
+ key: String,
+ title: String,
+ items: Vec<quince_core::drive::DriveItem>,
+ /// What of it is in Downloads now.
+ saved: Vec<String>,
+}
+
+/// Downloads into Downloads that did not finish — cancelled, stopped short, or cut off by
+/// quitting — oldest first.
+#[tauri::command]
+fn drive_downloads_unfinished(app: tauri::AppHandle) -> Vec<UnfinishedDownload> {
+ let running = downloads_by_key()
+ .lock()
+ .map(|k| k.clone())
+ .unwrap_or_default();
+ download_journal(&app)
+ .unfinished()
+ .into_iter()
+ .filter(|r| !running.contains(&r.key))
+ .map(|r| UnfinishedDownload {
+ saved: r
+ .placed
+ .iter()
+ .filter(|p| std::fs::symlink_metadata(&p.path).is_ok())
+ .map(|p| p.path.to_string_lossy().into_owned())
+ .collect(),
+ key: r.key,
+ title: r.title,
+ items: r.items,
+ })
+ .collect()
+}
+
+/// Let an unfinished download go: it is no longer offered, and downloading the same again
+/// starts afresh. What it saved stays where it is.
+#[tauri::command]
+fn drive_download_forget(app: tauri::AppHandle, key: String) {
+ download_journal(&app).forget(&key);
+}
+
+/// Save folders — and a selection of files and folders — from Drive into Downloads, as the
+/// same tree (see `folder_download`), or go on with the same download where it was left.
+/// Progress arrives as `quince://download` events; the answer is what was saved, where, what
+/// was not, and whether it can be gone on with.
+///
+/// The items come from the window only to be found: the library's own copy of each wins when
+/// it has one, as a single download's does, and nothing the window says is ever a path here —
+/// every name is made a plain one, in a folder made for it in Downloads.
+#[tauri::command]
+async fn drive_download_items(
+ app: tauri::AppHandle,
+ batch: String,
+ title: String,
+ items: Vec<quince_core::drive::DriveItem>,
+ auth: State<'_, Auth>,
+ data: State<'_, Data>,
+) -> UiResult<folder_download::DownloadReport> {
+ if items.is_empty() {
+ return Err("Nothing was chosen to download.".into());
+ }
+ let chosen = {
+ let library = data.0.lock().await;
+ let mut chosen = Vec::with_capacity(items.len());
+ for item in items {
+ let stored = library
+ .drive_item(&item.drivewsid)
+ .map_err(|e| e.to_string())?;
+ chosen.push(stored.unwrap_or(item));
+ }
+ chosen
+ };
+ let (http, drivews, params) = drive_context(&auth).await?;
+ let docws = {
+ let client = auth.client.lock().await;
+ client
+ .session()
+ .service_url("docws")
+ .ok_or("Apple hasn't offered downloads for this account.")?
+ };
+ let downloads = app
+ .path()
+ .download_dir()
+ .map_err(|_| "Couldn't find your Downloads folder.".to_string())?;
+ tokio::fs::create_dir_all(&downloads)
+ .await
+ .map_err(|e| e.to_string())?;
+
+ let key = folder_download::Journal::key(&chosen);
+ let fresh = downloads_by_key()
+ .lock()
+ .is_ok_and(|mut k| k.insert(key.clone()));
+ if !fresh {
+ return Err("That is downloading already.".into());
+ }
+ let stop = folder_download::Stop::default();
+ downloads_running()
+ .lock()
+ .map(|mut m| m.insert(batch.clone(), stop.clone()))
+ .ok();
+ let drive = folder_download::Drive {
+ http,
+ drivews,
+ docws,
+ params,
+ retry_after: std::time::Duration::from_secs(1),
+ free_space: folder_download::free_space,
+ room_every: folder_download::ROOM_EVERY,
+ };
+ let emit: folder_download::Emit = {
+ let app = app.clone();
+ std::sync::Arc::new(move |event| {
+ app.emit(folder_download::DOWNLOAD, &event).ok();
+ })
+ };
+ let journal = download_journal(&app);
+ let request = folder_download::Request {
+ batch: &batch,
+ title: &title,
+ items: chosen,
+ into: &downloads,
+ journal: Some(&journal),
+ };
+ let report = folder_download::run(&drive, request, emit, &stop).await;
+ downloads_running()
+ .lock()
+ .map(|mut m| m.remove(&batch))
+ .ok();
+ downloads_by_key().lock().map(|mut k| k.remove(&key)).ok();
+ report
+}
+
+/// Stop downloads: a whole batch, or the batches these files are in. What has arrived stays;
+/// what is arriving is dropped with its partial file, and nothing more begins.
+#[tauri::command]
+fn drive_download_cancel(ids: Vec<String>, batch: Option<String>) {
+ let mut batches: Vec<String> = ids
+ .iter()
+ .filter_map(|id| id.split_once(':').map(|(b, _)| b.to_string()))
+ .collect();
+ batches.extend(batch);
+ if let Ok(running) = downloads_running().lock() {
+ for b in batches {
+ if let Some(stop) = running.get(&b) {
+ stop.stop();
+ }
+ }
+ }
+}
+
/// Where an upload's progress is announced, one event per file as it moves.
const UPLOAD: &str = "quince://upload";
@@ -7156,6 +7332,10 @@ fn main() {
drive_sync_folder,
drive_children,
drive_download,
+ drive_download_items,
+ drive_download_cancel,
+ drive_downloads_unfinished,
+ drive_download_forget,
drive_upload,
drive_create_folder,
drive_rename,