Native iCloud apps for Windows and Linux, built in Rust. Notes, Reminders, Calendar, Contacts, Drive and Photos, offline and in sync with your iPhone.
Rust apple-notesapple-reminderscloudkitcrdtdesktop-appicloudicloud-calendaricloud-driveicloud-notesicloud-photosiphonelinuxlinux-desktoplocal-firstoffline-firstprivacyrusttauritauri-appwindows
Every cache is held to a size; pictures are written whole
Yashwanth Kamireddic44c256d6e76
diff --git a/crates/quince-core/src/photos.rs b/crates/quince-core/src/photos.rsindex 550eb97..c4d89ad 100644--- a/crates/quince-core/src/photos.rs+++ b/crates/quince-core/src/photos.rs@@ -330,6 +330,14 @@ impl Library {self.put_photos(photos)}+ /// Every photo id the library holds, in any view: what a grid picture on disk may still+ /// be drawn for.+ pub fn photo_ids(&self) -> Result<std::collections::HashSet<String>> {+ let mut stmt = self.conn.prepare("SELECT id FROM photos")?;+ let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;+ Ok(rows.collect::<std::result::Result<_, _>>()?)+ }+/// A view's photos, newest first.pub fn photos(&self, view: View) -> Result<Vec<Photo>> {let (filter, order) = Self::view_query(view);diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rsindex a0207eb..5172cf5 100644--- a/src-tauri/src/main.rs+++ b/src-tauri/src/main.rs@@ -4674,7 +4674,7 @@ async fn folder_thumbnail_files(.map(|(_, b)| b.clone());let at = thumbnail_file(&cache, f);// An empty file records "Apple has no picture for this version", so it is not asked again.- let written = tokio::fs::write(&at, bytes.as_deref().unwrap_or(&[])).await;+ let written = write_whole(&at, bytes.as_deref().unwrap_or(&[])).await;if bytes.is_some() && written.is_ok() {out.insert(f.drivewsid.clone(), at);}@@ -4956,6 +4956,58 @@ fn photo_path(dir: &std::path::Path, guid: &str, signature: &str) -> PathBuf {dir.join("faces").join(format!("{guid_safe}-{safe}"))}+/// Write `bytes` to `path` whole or not at all: to a file of its own beside it, then renamed+/// over it. Written in place, a crash or a full disk part way left a cut-off picture that was+/// served from then on — the read side only asks whether the file is there.+async fn write_whole(path: &std::path::Path, bytes: &[u8]) -> std::io::Result<()> {+ let part = path.with_file_name(format!(+ ".{}.{}.part",+ path.file_name().unwrap_or_default().to_string_lossy(),+ uuid::Uuid::new_v4().simple()+ ));+ let written = async {+ tokio::fs::write(&part, bytes).await?;+ tokio::fs::rename(&part, path).await+ }+ .await;+ if written.is_err() {+ let _ = tokio::fs::remove_file(&part).await;+ }+ written+}++/// A contact's earlier pictures, once a new one is kept: one file per signature, and every+/// changed picture used to leave the last one behind for good.+fn forget_older_faces(kept: &std::path::Path) {+ let (Some(dir), Some(name)) = (kept.parent(), kept.file_name().and_then(|n| n.to_str())) else {+ return;+ };+ // `<guid>-<signature>`, and the signature can hold dashes of its own: the guid is told by+ // its length. A name not of that shape is left alone.+ let Some(guid) = guid_of_face(name) else {+ return;+ };+ let Ok(entries) = std::fs::read_dir(dir) else {+ return;+ };+ for e in entries.flatten() {+ let other = e.file_name();+ let Some(other) = other.to_str() else {+ continue;+ };+ if other != name && guid_of_face(other) == Some(guid) {+ let _ = std::fs::remove_file(e.path());+ }+ }+}++/// The contact part of a face file's name (see [`photo_path`]): a contact's guid is a UUID,+/// 36 characters.+fn guid_of_face(name: &str) -> Option<&str> {+ const UUID: usize = 36;+ (name.len() > UUID + 1 && name.as_bytes()[UUID] == b'-').then(|| &name[..UUID])+}+/// Serve a contact's picture to the interface.////// Through a scheme of our own rather than over IPC: the webview loads it the way it loads@@ -5026,7 +5078,9 @@ async fn contact_photo(if let Some(parent) = path.parent() {tokio::fs::create_dir_all(parent).await.ok();}- tokio::fs::write(path, &drawn).await.ok();+ if write_whole(path, &drawn).await.is_ok() {+ forget_older_faces(path);+ }}Ok(Photo {bytes: drawn,@@ -6025,6 +6079,8 @@ fn main() {// is brought back under its budget first thing.let trim = app.handle().clone();tauri::async_runtime::spawn_blocking(move || photos::trim_cache(&trim));+ // And the caches beside it, each to a size of its own (photo_cache::SIDE).+ photo_cache::keep_side_caches(app.handle());// Logging out or shutting down ends the app with a signal, not through its own// quit, and a crash ends it with neither: either way the iCloud Drive folder is// let go of first, never left behind answering "Transport endpoint is notdiff --git a/src-tauri/src/photo_cache.rs b/src-tauri/src/photo_cache.rsindex 02b4230..0113e97 100644--- a/src-tauri/src/photo_cache.rs+++ b/src-tauri/src/photo_cache.rs@@ -235,6 +235,141 @@ fn thumbnails(root: &Path) -> u64 {.sum()}+// ---------------------------------------------------------------------------------------------+// The other caches.++/// The caches beside Photos' own, each held to a size of its own, the longest unseen let go of+/// first: Drive's file thumbnails, contacts' faces, the pictures and files of notes and+/// reminders. Each is a picture or a file Apple still has, made again when next wanted — the+/// same ones "Clear Cached Pictures" lets go of. Before, nothing ever left them: every changed+/// face and every attachment opened stayed for good.+pub const SIDE: [(&str, u64); 5] = [+ ("thumbnails", 256 * MB),+ ("faces", 64 * MB),+ ("note-media", 512 * MB),+ ("note-files", 512 * MB),+ ("reminder-media", 256 * MB),+];++/// A grid picture of a photo the library no longer holds (deleted from iCloud), unseen this+/// long, is let go of: the grid's pictures are otherwise kept for good.+pub const GRID_ORPHAN: Duration = Duration::from_secs(30 * 24 * 3600);++/// When a file was last seen: read (where the disk keeps that) or written.+fn last_seen(meta: &std::fs::Metadata) -> SystemTime {+ let written = meta.modified().unwrap_or(std::time::UNIX_EPOCH);+ meta.accessed().map_or(written, |read| read.max(written))+}++/// `dir` held to `cap` bytes, the longest unseen first; nothing seen in the last+/// [`RECENT`] (on screen, being opened), and a download's `.part` only once abandoned. Answers+/// how many files were let go of.+pub fn trim_lru(dir: &Path, cap: u64, now: SystemTime) -> usize {+ let mut files = Vec::new();+ walk(dir, 2, &mut files);+ let mut removed = 0;+ let mut kept: Vec<(SystemTime, u64, PathBuf)> = Vec::new();+ for (path, meta) in files {+ let seen = last_seen(&meta);+ let age = now.duration_since(seen).unwrap_or_default();+ if is_part(&path) {+ if age > STALE_PART && std::fs::remove_file(&path).is_ok() {+ removed += 1;+ }+ continue;+ }+ kept.push((seen, on_disk(&meta), path));+ }+ let mut used: u64 = kept.iter().map(|(_, b, _)| b).sum();+ kept.sort_by_key(|(seen, ..)| *seen);+ for (seen, bytes, path) in kept {+ if used <= cap {+ break;+ }+ if now.duration_since(seen).unwrap_or_default() < RECENT {+ continue;+ }+ if std::fs::remove_file(&path).is_ok() {+ used = used.saturating_sub(bytes);+ removed += 1;+ }+ }+ removed+}++/// Every cache in [`SIDE`] under the data folder `dir`, held to its size.+pub fn trim_side(dir: &Path, now: SystemTime) -> usize {+ SIDE.iter()+ .map(|(name, cap)| trim_lru(&dir.join(name), *cap, now))+ .sum()+}++/// Grid pictures (`thumb/`, small `fit/` steps) of photos not in `known`, unseen for+/// [`GRID_ORPHAN`]: a photo deleted from iCloud keeps no picture here for good.+pub fn prune_grid(+ root: &Path,+ known: &std::collections::HashSet<String>,+ now: SystemTime,+) -> usize {+ let mut dirs = vec![root.join("thumb")];+ for e in std::fs::read_dir(root.join("fit"))+ .into_iter()+ .flatten()+ .flatten()+ {+ if !fit_is_preview(&e.file_name()) {+ dirs.push(e.path());+ }+ }+ let mut removed = 0;+ for dir in dirs {+ let mut files = Vec::new();+ walk(&dir, 0, &mut files);+ for (path, meta) in files {+ let Some(id) = path.file_name().and_then(|n| n.to_str()) else {+ continue;+ };+ if is_part(&path) || known.contains(id) {+ continue;+ }+ if now.duration_since(last_seen(&meta)).unwrap_or_default() > GRID_ORPHAN+ && std::fs::remove_file(&path).is_ok()+ {+ removed += 1;+ }+ }+ }+ removed+}++/// The other caches held to their sizes and the grid's orphans let go of: at start, then every+/// few hours for as long as Skykeep runs (a tray app can run for weeks). Off the window's+/// threads.+pub fn keep_side_caches(app: &AppHandle) {+ let app = app.clone();+ std::thread::spawn(move || loop {+ let dir = crate::state_dir(&app);+ let now = SystemTime::now();+ let side = trim_side(&dir, now);+ let known = {+ let data = app.state::<crate::Data>();+ let library = data.0.blocking_lock();+ library.photo_ids().ok()+ };+ // Only against a library that answered and holds photos: an empty one (signed out, a+ // first pass not in) says nothing about which pictures are gone.+ let grid = known+ .filter(|k| !k.is_empty())+ .map_or(0, |k| prune_grid(&root(&app), &k, now));+ if side + grid > 0 {+ quiet!(+ "caches: {side} files over their sizes and {grid} orphaned grid pictures let go of"+ );+ }+ std::thread::sleep(Duration::from_secs(6 * 3600));+ });+}+// ---------------------------------------------------------------------------------------------// Letting go.@@ -646,6 +781,62 @@ pub async fn photos_cache_clear(app: AppHandle) -> Result<Status, String> {.map_err(|e| e.to_string())}+#[cfg(test)]+mod side_tests {+ use super::*;++ fn put(dir: &Path, name: &str, bytes: usize, age: Duration) -> PathBuf {+ let path = dir.join(name);+ std::fs::create_dir_all(path.parent().unwrap()).unwrap();+ std::fs::write(&path, vec![7u8; bytes]).unwrap();+ let at = SystemTime::now() - age;+ let f = std::fs::File::options().write(true).open(&path).unwrap();+ f.set_times(std::fs::FileTimes::new().set_accessed(at).set_modified(at))+ .unwrap();+ path+ }++ /// Over its size, a cache lets go of the longest unseen first, never one just seen, and+ /// sweeps an abandoned download.+ #[test]+ fn a_cache_over_its_size_lets_the_longest_unseen_go_first() {+ let dir = tempfile::tempdir().unwrap();+ let hour = Duration::from_secs(3600);+ let oldest = put(dir.path(), "a/old", 40_000, hour * 30);+ let older = put(dir.path(), "b", 40_000, hour * 20);+ let newer = put(dir.path(), "c", 40_000, hour * 2);+ let on_screen = put(dir.path(), "d", 40_000, Duration::from_secs(10));+ let dead = put(dir.path(), ".e.1234.part", 10, hour * 2);+ let landing = put(dir.path(), ".f.5678.part", 10, Duration::from_secs(5));+ let used: u64 = [&oldest, &older, &newer, &on_screen]+ .iter()+ .map(|p| on_disk(&std::fs::metadata(p).unwrap()))+ .sum();+ // Room for two of the four.+ let removed = trim_lru(dir.path(), used / 2, SystemTime::now());+ assert_eq!(removed, 3, "two pictures and the dead download");+ assert!(!oldest.exists() && !older.exists() && !dead.exists());+ assert!(newer.exists() && on_screen.exists() && landing.exists());+ }++ /// Grid pictures of photos the library no longer holds go once long unseen; every other+ /// grid picture stays, as the grid is drawn from them.+ #[test]+ fn a_deleted_photos_grid_pictures_go_and_the_rest_stay() {+ let root = tempfile::tempdir().unwrap();+ let day = Duration::from_secs(24 * 3600);+ let kept = put(root.path(), "thumb/KEPT", 100, day * 90);+ let gone = put(root.path(), "thumb/GONE", 100, day * 90);+ let gone_fit = put(root.path(), "fit/256/GONE", 100, day * 90);+ let just_seen = put(root.path(), "thumb/NEW", 100, day);+ let preview = put(root.path(), "fit/2048/GONE", 100, day * 90);+ let known: std::collections::HashSet<String> = ["KEPT".to_string()].into();+ assert_eq!(prune_grid(root.path(), &known, SystemTime::now()), 2);+ assert!(kept.exists() && just_seen.exists() && preview.exists());+ assert!(!gone.exists() && !gone_fit.exists());+ }+}+#[cfg(test)]mod tests {use super::*;diff --git a/src-tauri/src/photos.rs b/src-tauri/src/photos.rsindex 07d353b..82f6756 100644--- a/src-tauri/src/photos.rs+++ b/src-tauri/src/photos.rs@@ -36,12 +36,22 @@ pub fn remember(links: HashMap<String, Links>) {let map = kept.get_or_insert_with(HashMap::new);// Links past their life are of no use (each is asked of Apple again), so they go: kept,// every photo's six signed links stayed in memory for as long as the app ran.- map.retain(|_, (_, at)| at.elapsed() < LINK_LIFE);+ keep_fresh(map);for (master, l) in links {map.insert(master, (l, now));}}+/// Links past their life let go of, and the room they took given back: a first pass over a+/// library of 100,000 photos brought links for every one — a few hundred megabytes — and a map+/// only pruned kept that size for as long as the app ran, long after the links had gone.+fn keep_fresh<T>(map: &mut HashMap<String, (T, Instant)>) {+ map.retain(|_, (_, at)| at.elapsed() < LINK_LIFE);+ if map.capacity() > 1024 && map.capacity() > map.len().saturating_mul(4) {+ map.shrink_to_fit();+ }+}+fn fresh_links(master: &str) -> Option<Links> {let kept = LINKS.lock()@@ -2719,6 +2729,31 @@ async fn refresh_photos(app: &AppHandle) {.ok();}+#[cfg(test)]+mod link_tests {+ use super::*;++ #[test]+ fn links_past_their_life_give_their_room_back() {+ let mut map: HashMap<String, ((), Instant)> = HashMap::new();+ let Some(old) = Instant::now().checked_sub(LINK_LIFE * 2) else {+ return;+ };+ for i in 0..100_000 {+ map.insert(format!("master-{i}"), ((), old));+ }+ map.insert("fresh".into(), ((), Instant::now()));+ let before = map.capacity();+ keep_fresh(&mut map);+ assert_eq!(map.len(), 1);+ assert!(+ map.capacity() < before / 100,+ "{} of {before}",+ map.capacity()+ );+ }+}+#[cfg(test)]mod tests {use super::*;