Skip to content

Native iCloud apps for Windows and Linux, built in Rust. Notes, Reminders, Calendar, Contacts, Drive and Photos, offline and in sync with your iPhone.

Every cache is held to a size; pictures are written whole

Yashwanth Kamireddic44c256d6e76
diff --git a/crates/quince-core/src/photos.rs b/crates/quince-core/src/photos.rs
index 550eb97..c4d89ad 100644
--- a/crates/quince-core/src/photos.rs
+++ b/crates/quince-core/src/photos.rs
@@ -330,6 +330,14 @@ impl Library {
self.put_photos(photos)
}
+ /// Every photo id the library holds, in any view: what a grid picture on disk may still
+ /// be drawn for.
+ pub fn photo_ids(&self) -> Result<std::collections::HashSet<String>> {
+ let mut stmt = self.conn.prepare("SELECT id FROM photos")?;
+ let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;
+ Ok(rows.collect::<std::result::Result<_, _>>()?)
+ }
+
/// A view's photos, newest first.
pub fn photos(&self, view: View) -> Result<Vec<Photo>> {
let (filter, order) = Self::view_query(view);
diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs
index a0207eb..5172cf5 100644
--- a/src-tauri/src/main.rs
+++ b/src-tauri/src/main.rs
@@ -4674,7 +4674,7 @@ async fn folder_thumbnail_files(
.map(|(_, b)| b.clone());
let at = thumbnail_file(&cache, f);
// An empty file records "Apple has no picture for this version", so it is not asked again.
- let written = tokio::fs::write(&at, bytes.as_deref().unwrap_or(&[])).await;
+ let written = write_whole(&at, bytes.as_deref().unwrap_or(&[])).await;
if bytes.is_some() && written.is_ok() {
out.insert(f.drivewsid.clone(), at);
}
@@ -4956,6 +4956,58 @@ fn photo_path(dir: &std::path::Path, guid: &str, signature: &str) -> PathBuf {
dir.join("faces").join(format!("{guid_safe}-{safe}"))
}
+/// Write `bytes` to `path` whole or not at all: to a file of its own beside it, then renamed
+/// over it. Written in place, a crash or a full disk part way left a cut-off picture that was
+/// served from then on — the read side only asks whether the file is there.
+async fn write_whole(path: &std::path::Path, bytes: &[u8]) -> std::io::Result<()> {
+ let part = path.with_file_name(format!(
+ ".{}.{}.part",
+ path.file_name().unwrap_or_default().to_string_lossy(),
+ uuid::Uuid::new_v4().simple()
+ ));
+ let written = async {
+ tokio::fs::write(&part, bytes).await?;
+ tokio::fs::rename(&part, path).await
+ }
+ .await;
+ if written.is_err() {
+ let _ = tokio::fs::remove_file(&part).await;
+ }
+ written
+}
+
+/// A contact's earlier pictures, once a new one is kept: one file per signature, and every
+/// changed picture used to leave the last one behind for good.
+fn forget_older_faces(kept: &std::path::Path) {
+ let (Some(dir), Some(name)) = (kept.parent(), kept.file_name().and_then(|n| n.to_str())) else {
+ return;
+ };
+ // `<guid>-<signature>`, and the signature can hold dashes of its own: the guid is told by
+ // its length. A name not of that shape is left alone.
+ let Some(guid) = guid_of_face(name) else {
+ return;
+ };
+ let Ok(entries) = std::fs::read_dir(dir) else {
+ return;
+ };
+ for e in entries.flatten() {
+ let other = e.file_name();
+ let Some(other) = other.to_str() else {
+ continue;
+ };
+ if other != name && guid_of_face(other) == Some(guid) {
+ let _ = std::fs::remove_file(e.path());
+ }
+ }
+}
+
+/// The contact part of a face file's name (see [`photo_path`]): a contact's guid is a UUID,
+/// 36 characters.
+fn guid_of_face(name: &str) -> Option<&str> {
+ const UUID: usize = 36;
+ (name.len() > UUID + 1 && name.as_bytes()[UUID] == b'-').then(|| &name[..UUID])
+}
+
/// Serve a contact's picture to the interface.
///
/// Through a scheme of our own rather than over IPC: the webview loads it the way it loads
@@ -5026,7 +5078,9 @@ async fn contact_photo(
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent).await.ok();
}
- tokio::fs::write(path, &drawn).await.ok();
+ if write_whole(path, &drawn).await.is_ok() {
+ forget_older_faces(path);
+ }
}
Ok(Photo {
bytes: drawn,
@@ -6025,6 +6079,8 @@ fn main() {
// is brought back under its budget first thing.
let trim = app.handle().clone();
tauri::async_runtime::spawn_blocking(move || photos::trim_cache(&trim));
+ // And the caches beside it, each to a size of its own (photo_cache::SIDE).
+ photo_cache::keep_side_caches(app.handle());
// Logging out or shutting down ends the app with a signal, not through its own
// quit, and a crash ends it with neither: either way the iCloud Drive folder is
// let go of first, never left behind answering "Transport endpoint is not
diff --git a/src-tauri/src/photo_cache.rs b/src-tauri/src/photo_cache.rs
index 02b4230..0113e97 100644
--- a/src-tauri/src/photo_cache.rs
+++ b/src-tauri/src/photo_cache.rs
@@ -235,6 +235,141 @@ fn thumbnails(root: &Path) -> u64 {
.sum()
}
+// ---------------------------------------------------------------------------------------------
+// The other caches.
+
+/// The caches beside Photos' own, each held to a size of its own, the longest unseen let go of
+/// first: Drive's file thumbnails, contacts' faces, the pictures and files of notes and
+/// reminders. Each is a picture or a file Apple still has, made again when next wanted — the
+/// same ones "Clear Cached Pictures" lets go of. Before, nothing ever left them: every changed
+/// face and every attachment opened stayed for good.
+pub const SIDE: [(&str, u64); 5] = [
+ ("thumbnails", 256 * MB),
+ ("faces", 64 * MB),
+ ("note-media", 512 * MB),
+ ("note-files", 512 * MB),
+ ("reminder-media", 256 * MB),
+];
+
+/// A grid picture of a photo the library no longer holds (deleted from iCloud), unseen this
+/// long, is let go of: the grid's pictures are otherwise kept for good.
+pub const GRID_ORPHAN: Duration = Duration::from_secs(30 * 24 * 3600);
+
+/// When a file was last seen: read (where the disk keeps that) or written.
+fn last_seen(meta: &std::fs::Metadata) -> SystemTime {
+ let written = meta.modified().unwrap_or(std::time::UNIX_EPOCH);
+ meta.accessed().map_or(written, |read| read.max(written))
+}
+
+/// `dir` held to `cap` bytes, the longest unseen first; nothing seen in the last
+/// [`RECENT`] (on screen, being opened), and a download's `.part` only once abandoned. Answers
+/// how many files were let go of.
+pub fn trim_lru(dir: &Path, cap: u64, now: SystemTime) -> usize {
+ let mut files = Vec::new();
+ walk(dir, 2, &mut files);
+ let mut removed = 0;
+ let mut kept: Vec<(SystemTime, u64, PathBuf)> = Vec::new();
+ for (path, meta) in files {
+ let seen = last_seen(&meta);
+ let age = now.duration_since(seen).unwrap_or_default();
+ if is_part(&path) {
+ if age > STALE_PART && std::fs::remove_file(&path).is_ok() {
+ removed += 1;
+ }
+ continue;
+ }
+ kept.push((seen, on_disk(&meta), path));
+ }
+ let mut used: u64 = kept.iter().map(|(_, b, _)| b).sum();
+ kept.sort_by_key(|(seen, ..)| *seen);
+ for (seen, bytes, path) in kept {
+ if used <= cap {
+ break;
+ }
+ if now.duration_since(seen).unwrap_or_default() < RECENT {
+ continue;
+ }
+ if std::fs::remove_file(&path).is_ok() {
+ used = used.saturating_sub(bytes);
+ removed += 1;
+ }
+ }
+ removed
+}
+
+/// Every cache in [`SIDE`] under the data folder `dir`, held to its size.
+pub fn trim_side(dir: &Path, now: SystemTime) -> usize {
+ SIDE.iter()
+ .map(|(name, cap)| trim_lru(&dir.join(name), *cap, now))
+ .sum()
+}
+
+/// Grid pictures (`thumb/`, small `fit/` steps) of photos not in `known`, unseen for
+/// [`GRID_ORPHAN`]: a photo deleted from iCloud keeps no picture here for good.
+pub fn prune_grid(
+ root: &Path,
+ known: &std::collections::HashSet<String>,
+ now: SystemTime,
+) -> usize {
+ let mut dirs = vec![root.join("thumb")];
+ for e in std::fs::read_dir(root.join("fit"))
+ .into_iter()
+ .flatten()
+ .flatten()
+ {
+ if !fit_is_preview(&e.file_name()) {
+ dirs.push(e.path());
+ }
+ }
+ let mut removed = 0;
+ for dir in dirs {
+ let mut files = Vec::new();
+ walk(&dir, 0, &mut files);
+ for (path, meta) in files {
+ let Some(id) = path.file_name().and_then(|n| n.to_str()) else {
+ continue;
+ };
+ if is_part(&path) || known.contains(id) {
+ continue;
+ }
+ if now.duration_since(last_seen(&meta)).unwrap_or_default() > GRID_ORPHAN
+ && std::fs::remove_file(&path).is_ok()
+ {
+ removed += 1;
+ }
+ }
+ }
+ removed
+}
+
+/// The other caches held to their sizes and the grid's orphans let go of: at start, then every
+/// few hours for as long as Skykeep runs (a tray app can run for weeks). Off the window's
+/// threads.
+pub fn keep_side_caches(app: &AppHandle) {
+ let app = app.clone();
+ std::thread::spawn(move || loop {
+ let dir = crate::state_dir(&app);
+ let now = SystemTime::now();
+ let side = trim_side(&dir, now);
+ let known = {
+ let data = app.state::<crate::Data>();
+ let library = data.0.blocking_lock();
+ library.photo_ids().ok()
+ };
+ // Only against a library that answered and holds photos: an empty one (signed out, a
+ // first pass not in) says nothing about which pictures are gone.
+ let grid = known
+ .filter(|k| !k.is_empty())
+ .map_or(0, |k| prune_grid(&root(&app), &k, now));
+ if side + grid > 0 {
+ quiet!(
+ "caches: {side} files over their sizes and {grid} orphaned grid pictures let go of"
+ );
+ }
+ std::thread::sleep(Duration::from_secs(6 * 3600));
+ });
+}
+
// ---------------------------------------------------------------------------------------------
// Letting go.
@@ -646,6 +781,62 @@ pub async fn photos_cache_clear(app: AppHandle) -> Result<Status, String> {
.map_err(|e| e.to_string())
}
+#[cfg(test)]
+mod side_tests {
+ use super::*;
+
+ fn put(dir: &Path, name: &str, bytes: usize, age: Duration) -> PathBuf {
+ let path = dir.join(name);
+ std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+ std::fs::write(&path, vec![7u8; bytes]).unwrap();
+ let at = SystemTime::now() - age;
+ let f = std::fs::File::options().write(true).open(&path).unwrap();
+ f.set_times(std::fs::FileTimes::new().set_accessed(at).set_modified(at))
+ .unwrap();
+ path
+ }
+
+ /// Over its size, a cache lets go of the longest unseen first, never one just seen, and
+ /// sweeps an abandoned download.
+ #[test]
+ fn a_cache_over_its_size_lets_the_longest_unseen_go_first() {
+ let dir = tempfile::tempdir().unwrap();
+ let hour = Duration::from_secs(3600);
+ let oldest = put(dir.path(), "a/old", 40_000, hour * 30);
+ let older = put(dir.path(), "b", 40_000, hour * 20);
+ let newer = put(dir.path(), "c", 40_000, hour * 2);
+ let on_screen = put(dir.path(), "d", 40_000, Duration::from_secs(10));
+ let dead = put(dir.path(), ".e.1234.part", 10, hour * 2);
+ let landing = put(dir.path(), ".f.5678.part", 10, Duration::from_secs(5));
+ let used: u64 = [&oldest, &older, &newer, &on_screen]
+ .iter()
+ .map(|p| on_disk(&std::fs::metadata(p).unwrap()))
+ .sum();
+ // Room for two of the four.
+ let removed = trim_lru(dir.path(), used / 2, SystemTime::now());
+ assert_eq!(removed, 3, "two pictures and the dead download");
+ assert!(!oldest.exists() && !older.exists() && !dead.exists());
+ assert!(newer.exists() && on_screen.exists() && landing.exists());
+ }
+
+ /// Grid pictures of photos the library no longer holds go once long unseen; every other
+ /// grid picture stays, as the grid is drawn from them.
+ #[test]
+ fn a_deleted_photos_grid_pictures_go_and_the_rest_stay() {
+ let root = tempfile::tempdir().unwrap();
+ let day = Duration::from_secs(24 * 3600);
+ let kept = put(root.path(), "thumb/KEPT", 100, day * 90);
+ let gone = put(root.path(), "thumb/GONE", 100, day * 90);
+ let gone_fit = put(root.path(), "fit/256/GONE", 100, day * 90);
+ let just_seen = put(root.path(), "thumb/NEW", 100, day);
+ let preview = put(root.path(), "fit/2048/GONE", 100, day * 90);
+ let known: std::collections::HashSet<String> = ["KEPT".to_string()].into();
+ assert_eq!(prune_grid(root.path(), &known, SystemTime::now()), 2);
+ assert!(kept.exists() && just_seen.exists() && preview.exists());
+ assert!(!gone.exists() && !gone_fit.exists());
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/src-tauri/src/photos.rs b/src-tauri/src/photos.rs
index 07d353b..82f6756 100644
--- a/src-tauri/src/photos.rs
+++ b/src-tauri/src/photos.rs
@@ -36,12 +36,22 @@ pub fn remember(links: HashMap<String, Links>) {
let map = kept.get_or_insert_with(HashMap::new);
// Links past their life are of no use (each is asked of Apple again), so they go: kept,
// every photo's six signed links stayed in memory for as long as the app ran.
- map.retain(|_, (_, at)| at.elapsed() < LINK_LIFE);
+ keep_fresh(map);
for (master, l) in links {
map.insert(master, (l, now));
}
}
+/// Links past their life let go of, and the room they took given back: a first pass over a
+/// library of 100,000 photos brought links for every one — a few hundred megabytes — and a map
+/// only pruned kept that size for as long as the app ran, long after the links had gone.
+fn keep_fresh<T>(map: &mut HashMap<String, (T, Instant)>) {
+ map.retain(|_, (_, at)| at.elapsed() < LINK_LIFE);
+ if map.capacity() > 1024 && map.capacity() > map.len().saturating_mul(4) {
+ map.shrink_to_fit();
+ }
+}
+
fn fresh_links(master: &str) -> Option<Links> {
let kept = LINKS
.lock()
@@ -2719,6 +2729,31 @@ async fn refresh_photos(app: &AppHandle) {
.ok();
}
+#[cfg(test)]
+mod link_tests {
+ use super::*;
+
+ #[test]
+ fn links_past_their_life_give_their_room_back() {
+ let mut map: HashMap<String, ((), Instant)> = HashMap::new();
+ let Some(old) = Instant::now().checked_sub(LINK_LIFE * 2) else {
+ return;
+ };
+ for i in 0..100_000 {
+ map.insert(format!("master-{i}"), ((), old));
+ }
+ map.insert("fresh".into(), ((), Instant::now()));
+ let before = map.capacity();
+ keep_fresh(&mut map);
+ assert_eq!(map.len(), 1);
+ assert!(
+ map.capacity() < before / 100,
+ "{} of {before}",
+ map.capacity()
+ );
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;