Native iCloud apps for Windows and Linux, built in Rust. Notes, Reminders, Calendar, Contacts, Drive and Photos, offline and in sync with your iPhone.
Rust apple-notesapple-reminderscloudkitcrdtdesktop-appicloudicloud-calendaricloud-driveicloud-notesicloud-photosiphonelinuxlinux-desktoplocal-firstoffline-firstprivacyrusttauritauri-appwindows
Scale: a library of a million photos at the speed of a small one
Yashwanth Kamireddi5521e9a88c6f
diff --git a/.uiverify/stub.mjs b/.uiverify/stub.mjsindex 4a00741..7c636a6 100644--- a/.uiverify/stub.mjs+++ b/.uiverify/stub.mjs@@ -676,7 +676,7 @@ export function installStub({ screen, services, twoFactor, calendarFixture, noteconst order = sortable ? args.order : null;const key = (p) => (order?.by === "added" || (!order && src.view === "recents") ? (p.addedAt ?? p.takenAt) : p.takenAt);if (order) list = [...list].sort((a, b) => (key(b) - key(a)) * (order.oldest ? -1 : 1) || (a.id < b.id ? -1 : 1));- if (cmd === "photos_page") return list.slice(args.offset, args.offset + args.limit);+ if (cmd === "photos_page") { const from = args.after ? list.findIndex((p) => p.id === args.after.id) + 1 : 0; return list.slice(from, from + args.limit); }(window.__indexReads ??= []).push(src);// A large library's index takes a moment: a test can make it take one.if (window.__indexDelay) await new Promise((r) => setTimeout(r, window.__indexDelay));diff --git a/crates/quince-core/src/lib.rs b/crates/quince-core/src/lib.rsindex abc910e..500ec94 100644--- a/crates/quince-core/src/lib.rs+++ b/crates/quince-core/src/lib.rs@@ -101,11 +101,14 @@ const ACCOUNT_KEY: &str = "account";const MADE_ON_OPEN: &str = "key IN ('documents.writer')";/// Every table holding what an account brought: all but `app_state`, which is sorted key by key,-/// and the search index's, which are not rows to delete one by one (`forget_account` empties it).+/// the search index's, which are not rows to delete one by one (`forget_account` empties it),+/// and the photo counts (migration 50), which are the photos' own triggers' to keep: one row a+/// view, always there, counted down to nothing as the photos go.fn account_tables(conn: &Connection) -> Result<Vec<String>> {let mut stmt = conn.prepare("SELECT name FROM sqlite_master WHERE type = 'table'AND name NOT LIKE 'sqlite_%' AND name != 'app_state'+ AND name != 'photo_view_counts'AND name NOT LIKE 'search\\_%' ESCAPE '\\'",)?;let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;@@ -220,6 +223,9 @@ pub struct Library {zone: Option<jiff::tz::TimeZone>,/// What a change being watched has written so far, row by row (`changing_items`).watch: changes::Watch,+ /// The Years and Months views' months, as last worked out, and the version of the library+ /// and the zone they were worked out for (see `photo_months`).+ months: std::sync::Mutex<Option<photos::MonthsKept>>,}impl Library {@@ -344,6 +350,7 @@ impl Library {conn,zone: None,watch: changes::Watch::default(),+ months: std::sync::Mutex::new(None),})}@@ -369,6 +376,7 @@ impl Library {conn,zone: None,watch: changes::Watch::default(),+ months: std::sync::Mutex::new(None),};library.migrate()?;changes::register(&library.conn, &library.watch)?;diff --git a/crates/quince-core/src/notes.rs b/crates/quince-core/src/notes.rsindex b42a75b..cd8c7e9 100644--- a/crates/quince-core/src/notes.rs+++ b/crates/quince-core/src/notes.rs@@ -223,7 +223,20 @@ impl Library {THEN editable ELSE excluded.editable END,locked = excluded.locked, pinned = excluded.pinned,created_at = excluded.created_at, modified_at = excluded.modified_at,- etag = excluded.etag, updated_at = excluded.updated_at",+ etag = excluded.etag, updated_at = excluded.updated_at+ -- Stored as it was: nothing written and no words indexed again. Fifty+ -- thousand notes stored again as they were took 27 s, rewriting the index.+ WHERE (notes.folder_guid, notes.title, notes.snippet, notes.locked,+ notes.pinned, notes.created_at, notes.modified_at, notes.etag,+ notes.body, notes.runs, notes.editable)+ IS NOT (excluded.folder_guid, excluded.title, excluded.snippet,+ excluded.locked, excluded.pinned, excluded.created_at,+ excluded.modified_at, excluded.etag,+ CASE WHEN excluded.locked THEN NULL ELSE COALESCE(excluded.body, notes.body) END,+ CASE WHEN excluded.locked THEN NULL+ WHEN excluded.body IS NULL THEN notes.runs ELSE excluded.runs END,+ CASE WHEN excluded.body IS NULL AND NOT excluded.locked AND notes.body IS NOT NULL+ THEN notes.editable ELSE excluded.editable END)",)?;let stamp = now();for n in notes {diff --git a/crates/quince-core/src/photos.rs b/crates/quince-core/src/photos.rsindex f7e5171..59f50b5 100644--- a/crates/quince-core/src/photos.rs+++ b/crates/quince-core/src/photos.rs@@ -167,9 +167,13 @@ const COLUMNS: &str = "id, master, taken_at, added_at, kind, width, height, dura/// A burst shows as its chosen photo, as Photos shows it (Apple's library says as much: its/// `burstsExcludedFromPhotosCount` is every burst's photos less one each). A burst whose chosen/// photo is not in the library shows whole rather than not at all.+///+/// The burst's own index, always: left to choose, SQLite took a covering index that starts with+/// `deleted` and walked the whole library once for every burst photo.pub(crate) const ONE_PER_BURST: &str =- "(burst IS NULL OR burst_pick = 1 OR NOT EXISTS (SELECT 1 FROM photos k \- WHERE k.burst = photos.burst AND k.burst_pick = 1 AND k.deleted = 0))";+ "(burst IS NULL OR burst_pick = 1 OR NOT EXISTS (SELECT 1 FROM photos k INDEXED BY photos_by_burst \+ WHERE k.burst = photos.burst AND k.burst IS NOT NULL \+ AND k.burst_pick = 1 AND k.deleted = 0))";fn row_to_photo(r: &rusqlite::Row<'_>) -> rusqlite::Result<Photo> {Ok(Photo {@@ -208,44 +212,82 @@ fn row_to_photo(r: &rusqlite::Row<'_>) -> rusqlite::Result<Photo> {})}-/// What is typed, as SQL: one condition per word (each its own parameter — what is typed is-/// never part of the SQL), every word to match the file's name, title, caption or a keyword, or-/// the year or month it was taken. `None` when nothing is typed.-fn search_terms(query: &str) -> Option<(String, Vec<String>)> {- let words: Vec<String> = query- .split_whitespace()- .map(|w| w.to_lowercase())- .filter(|w| !w.is_empty())- .take(8)- .collect();- if words.is_empty() {- return None;+/// What is typed, as SQL over the index of words every photo is kept in (migration 45: its+/// title or file name, caption, keywords, and the year and month it was taken), each word the+/// beginning of one of its words, case and accents aside — as Photos and this app's search find+/// them. What is typed is a parameter, never part of the SQL. `None` when nothing is typed.+/// Read through the index: a substring search read every photo (4.3 s at a million).+fn found_by(query: &str) -> Option<(String, Vec<String>)> {+ let words = crate::search::match_all(query)?;+ Some((+ "id IN (SELECT d.key FROM search_text CROSS JOIN search_docs d+ ON d.id = search_text.rowid+ WHERE search_text MATCH ?1 AND d.kind = 'photo')"+ .to_string(),+ vec![words],+ ))+}++/// The views whose counts are kept, in the order the sidebar shows them.+const COUNTED: [View; 9] = [+ View::Library,+ View::Favorites,+ View::Videos,+ View::Screenshots,+ View::Panoramas,+ View::Portrait,+ View::Hidden,+ View::Rated,+ View::RecentlyDeleted,+];++/// A view's name in `photo_view_counts`.+fn view_key(view: View) -> &'static str {+ match view {+ View::Library | View::Recents => "library",+ View::Favorites => "favorites",+ View::Videos => "videos",+ View::Screenshots => "screenshots",+ View::Panoramas => "panoramas",+ View::Portrait => "portrait",+ View::Hidden => "hidden",+ View::Rated => "rated",+ View::RecentlyDeleted => "recentlyDeleted",}- let one = "(LOWER(COALESCE(filename, '')) LIKE ?{n} ESCAPE '\\'- OR LOWER(COALESCE(title, '')) LIKE ?{n} ESCAPE '\\'- OR LOWER(COALESCE(caption, '')) LIKE ?{n} ESCAPE '\\'- OR LOWER(COALESCE(keywords, '')) LIKE ?{n} ESCAPE '\\'- OR strftime('%Y', taken_at, 'unixepoch', 'localtime') LIKE ?{n} ESCAPE '\\'- OR (CASE strftime('%m', taken_at, 'unixepoch', 'localtime')- WHEN '01' THEN 'january' WHEN '02' THEN 'february' WHEN '03' THEN 'march'- WHEN '04' THEN 'april' WHEN '05' THEN 'may' WHEN '06' THEN 'june'- WHEN '07' THEN 'july' WHEN '08' THEN 'august' WHEN '09' THEN 'september'- WHEN '10' THEN 'october' WHEN '11' THEN 'november' ELSE 'december' END)- LIKE ?{n} ESCAPE '\\')";- let conditions: Vec<String> = (1..=words.len())- .map(|n| one.replace("{n}", &n.to_string()))- .collect();- let patterns = words- .iter()- .map(|w| {- let escaped = w- .replace('\\', "\\\\")- .replace('%', "\\%")- .replace('_', "\\_");- format!("%{escaped}%")- })- .collect();- Some((conditions.join(" AND "), patterns))+}++/// Where a page carries on from: the last photo shown, by what its source is ordered by+/// (its [`PhotoIndex::keys`] value) and its id.+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+#[serde(rename_all = "camelCase")]+pub struct After {+ pub key: i64,+ pub id: String,+}++/// The little a picture of a photo is made from.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct PhotoShape {+ pub id: String,+ pub width: i64,+ pub height: i64,+ pub kind: Kind,+ pub taken_at: i64,+}++/// Where [`Library::grid_shapes`] carries on from.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct GridAfter {+ stage: usize,+ key: i64,+ id: String,+}++/// The months as last worked out, for the library's version and zone they were worked out for.+pub(crate) struct MonthsKept {+ version: (i64, u64),+ zone: String,+ months: Vec<PhotoMonth>,}impl Library {@@ -274,7 +316,24 @@ impl Library {keywords = excluded.keywords, latitude = excluded.latitude,longitude = excluded.longitude, rendered = excluded.rendered,burst = excluded.burst, burst_pick = excluded.burst_pick,- live_size = excluded.live_size",+ live_size = excluded.live_size+ -- Stored as it was: nothing written, no index touched, no count moved. A+ -- pass storing a large library again rewrote every row of it.+ WHERE (photos.master, photos.taken_at, photos.added_at, photos.kind,+ photos.width, photos.height, photos.duration, photos.filename,+ photos.size, photos.uti, photos.favorite, photos.hidden,+ photos.deleted, photos.deleted_at, photos.subtype, photos.change_tag,+ photos.expunge_at, photos.title, photos.caption, photos.rating,+ photos.keywords, photos.latitude, photos.longitude, photos.rendered,+ photos.burst, photos.burst_pick, photos.live_size)+ IS NOT (excluded.master, excluded.taken_at, excluded.added_at,+ excluded.kind, excluded.width, excluded.height, excluded.duration,+ excluded.filename, excluded.size, excluded.uti, excluded.favorite,+ excluded.hidden, excluded.deleted, excluded.deleted_at,+ excluded.subtype, excluded.change_tag, excluded.expunge_at,+ excluded.title, excluded.caption, excluded.rating, excluded.keywords,+ excluded.latitude, excluded.longitude, excluded.rendered,+ excluded.burst, excluded.burst_pick, excluded.live_size)",)?;for p in photos {stmt.execute(params![@@ -361,7 +420,7 @@ impl Library {// Counted for the bursts here only: a page of two hundred never counts the library's.let mut stmt = self.conn- .prepare_cached("SELECT COUNT(*) FROM photos WHERE burst = ?1 AND deleted = 0")?;+ .prepare_cached("SELECT COUNT(*) FROM photos INDEXED BY photos_by_burst WHERE burst = ?1 AND burst IS NOT NULL AND deleted = 0")?;let mut sizes: std::collections::HashMap<String, i64> = std::collections::HashMap::new();for b in photos.iter().filter_map(|p| p.burst.as_ref()) {if !sizes.contains_key(b) {@@ -380,7 +439,7 @@ impl Library {/// Every photo of one burst, in the order they were shot.pub fn burst_photos(&self, burst: &str) -> Result<Vec<Photo>> {let mut stmt = self.conn.prepare(&format!(- "SELECT {COLUMNS} FROM photos WHERE burst = ?1 AND deleted = 0 ORDER BY taken_at, id"+ "SELECT {COLUMNS} FROM photos INDEXED BY photos_by_burst WHERE burst = ?1 AND burst IS NOT NULL AND deleted = 0 ORDER BY taken_at, id"))?;let rows = stmt.query_map(params![burst], row_to_photo)?;self.with_bursts(rows.collect::<std::result::Result<_, _>>()?)@@ -390,11 +449,13 @@ impl Library {/// file's name, title, caption or a keyword, or the year or month it was taken ("2025",/// "march"). Deleted and hidden photos are not searched. Newest first, at most `limit`.pub fn search_photos(&self, query: &str, limit: usize) -> Result<Vec<Photo>> {- let Some((conditions, patterns)) = search_terms(query) else {+ let Some((conditions, patterns)) = found_by(query) else {return Ok(Vec::new());};+ // From the words to the photos (`+` keeps SQLite off the date index, which walked every+ // photo to find the few a word is in), then put in order.let sql = format!(- "SELECT {COLUMNS} FROM photos WHERE deleted = 0 AND hidden = 0 AND {ONE_PER_BURST} AND {conditions}+ "SELECT {COLUMNS} FROM photos WHERE +deleted = 0 AND +hidden = 0 AND {ONE_PER_BURST} AND {conditions}ORDER BY taken_at DESC, id LIMIT {}",limit.min(50_000));@@ -480,6 +541,34 @@ impl Library {/// How many photos each view holds.pub fn photo_counts(&self) -> Result<Vec<(View, i64)>> {+ // Kept by the photos' own triggers (migration 50): read, never counted. Less, for the+ // views that show a burst once, the burst's other photos — counted over the bursts'+ // own index, which holds bursts only.+ let mut kept = self+ .conn+ .prepare_cached("SELECT n FROM photo_view_counts WHERE view = ?1")?;+ let mut out = Vec::new();+ for view in COUNTED {+ let n: i64 = kept.query_row([view_key(view)], |r| r.get(0))?;+ let extra: i64 = if view == View::RecentlyDeleted {+ 0+ } else {+ let (filter, _) = Self::view_query(view);+ self.conn+ .prepare_cached(&format!(+ "SELECT COUNT(*) FROM photos INDEXED BY photos_by_burst+ WHERE burst IS NOT NULL AND {filter} AND NOT {ONE_PER_BURST}"+ ))?+ .query_row([], |r| r.get(0))?+ };+ out.push((view, n - extra));+ }+ Ok(out)+ }++ /// How many photos each view holds, each counted over the library: what the kept counts+ /// ([`Library::photo_counts`]) must always agree with.+ pub fn photo_counts_counted(&self) -> Result<Vec<(View, i64)>> {let mut out = Vec::new();for view in [View::Library,@@ -857,19 +946,23 @@ struct Query {order: String,key: &'static str,params: Vec<String>,+ /// Ordered by `key` then id ascending — newest first when `Some(true)` — so a page can+ /// carry on from a photo's place; `None` for an order of its own.+ keyset: Option<bool>,}impl Library {fn source_query(source: &Source, order: Option<Order>) -> Query {const TAKEN: &str = "taken_at";const ADDED: &str = "COALESCE(added_at, taken_at)";- let plain = |filter: String, order: String, key, params| Query {+ let plain = |filter: String, order: String, key, params, keyset| Query {from: "photos",pre: "",filter,order,key,params,+ keyset,};match source {Source::View(view) => {@@ -880,19 +973,26 @@ impl Library {ONE_PER_BURST};let sortable = !matches!(view, View::RecentlyDeleted | View::Rated);- let (order, key) = match order.filter(|_| sortable) {+ let (order, key, keyset) = match order.filter(|_| sortable) {Some(o) => {let key = if o.by == OrderBy::Added { ADDED } else { TAKEN };let dir = if o.oldest { "ASC" } else { "DESC" };- (format!("{key} {dir}, id"), key)+ (format!("{key} {dir}, id"), key, Some(!o.oldest))}// Grouped by what it is sorted by: Recents by when added.None => (format!("{own}, id"),if *view == View::Recents { ADDED } else { TAKEN },+ sortable.then_some(true),),};- plain(format!("{filter} AND {bursts}"), order, key, Vec::new())+ plain(+ format!("{filter} AND {bursts}"),+ order,+ key,+ Vec::new(),+ keyset,+ )}Source::Album(album) => Query {from: "photo_album_items i JOIN photos p ON p.id = i.asset",@@ -901,21 +1001,24 @@ impl Library {order: "i.position, p.taken_at DESC, p.id".into(),key: "p.taken_at",params: vec![album.clone()],+ keyset: None,},- Source::Query(typed) => match search_terms(typed) {+ Source::Query(typed) => match found_by(typed) {Some((conditions, patterns)) => plain(- format!("deleted = 0 AND hidden = 0 AND {ONE_PER_BURST} AND {conditions}"),+ format!("+deleted = 0 AND +hidden = 0 AND {ONE_PER_BURST} AND {conditions}"),"taken_at DESC, id".into(),TAKEN,patterns,+ Some(true),),- None => plain("0".into(), "id".into(), TAKEN, Vec::new()),+ None => plain("0".into(), "id".into(), TAKEN, Vec::new(), None),},Source::Burst(burst) => plain(- "burst = ?1 AND deleted = 0".into(),+ "burst = ?1 AND burst IS NOT NULL AND deleted = 0".into(),"taken_at, id".into(),TAKEN,vec![burst.clone()],+ Some(false),),}}@@ -949,12 +1052,15 @@ impl Library {Ok(out)}- /// `limit` photos of a source from `offset`, in the order its index has them.+ /// `limit` photos of a source, in the order its index has them: from its first, or from+ /// just after the photo `after` names. Carried on from that photo's place in the order+ /// (keyset) — never by counting past every photo before it, which a page near the end of a+ /// million cost a second.pub fn photo_page(&self,source: &Source,order: Option<Order>,- offset: usize,+ after: Option<&After>,limit: usize,) -> Result<Vec<Photo>> {let q = Self::source_query(source, order);@@ -963,17 +1069,135 @@ impl Library {.map(|c| format!("{}{}", q.pre, c.trim())).collect::<Vec<_>>().join(", ");+ let limit = limit.min(5_000);+ let mut params = q.params.clone();+ let filter = match (after, q.keyset) {+ (None, _) => q.filter.clone(),+ (Some(a), Some(desc)) => {+ let (k, i) = (params.len() + 1, params.len() + 2);+ params.push(a.key.to_string());+ params.push(a.id.clone());+ let past = if desc { "<" } else { ">" };+ format!(+ "{filter} AND ({key} {past} CAST(?{k} AS INTEGER) OR ({key} = CAST(?{k} AS INTEGER) AND {p}id > ?{i}))",+ filter = q.filter,+ key = q.key,+ p = q.pre,+ )+ }+ // An order of its own (an album's, Rated's): small, and walked to the photo named.+ (Some(a), None) => {+ let mut stmt = self.conn.prepare(&format!(+ "SELECT {columns} FROM {from} WHERE {filter} ORDER BY {order}",+ from = q.from,+ filter = q.filter,+ order = q.order,+ ))?;+ let rows =+ stmt.query_map(rusqlite::params_from_iter(q.params.iter()), row_to_photo)?;+ let all: Vec<Photo> = rows.collect::<std::result::Result<_, _>>()?;+ let from = all+ .iter()+ .position(|p| p.id == a.id)+ .map_or(all.len(), |i| i + 1);+ return self.with_bursts(all.into_iter().skip(from).take(limit).collect());+ }+ };let mut stmt = self.conn.prepare(&format!(- "SELECT {columns} FROM {from} WHERE {filter} ORDER BY {order} LIMIT {limit} OFFSET {offset}",+ "SELECT {columns} FROM {from} WHERE {filter} ORDER BY {order} LIMIT {limit}",from = q.from,- filter = q.filter,order = q.order,- limit = limit.min(5_000),))?;- let rows = stmt.query_map(rusqlite::params_from_iter(q.params.iter()), row_to_photo)?;+ let rows = stmt.query_map(rusqlite::params_from_iter(params.iter()), row_to_photo)?;self.with_bursts(rows.collect::<std::result::Result<_, _>>()?)}+ /// The photos a grid can show — the Library, Hidden and Recently Deleted, the one of those+ /// being looked at (`first`) first — as the little a picture is made from, newest first in+ /// each: `limit` at a time, from just after `after`, and where to carry on from. A page at a+ /// time, each a short read: the whole library held as photos was 665 MB at a million.+ pub fn grid_shapes(+ &self,+ first: Option<View>,+ after: Option<&GridAfter>,+ limit: usize,+ ) -> Result<(Vec<PhotoShape>, Option<GridAfter>)> {+ let mut stages = vec![+ format!("deleted = 0 AND hidden = 0 AND {ONE_PER_BURST}"),+ format!("deleted = 0 AND hidden = 1 AND {ONE_PER_BURST}"),+ "deleted = 1".to_string(),+ ];+ match first {+ Some(View::Hidden) => stages.swap(0, 1),+ Some(View::RecentlyDeleted) => {+ let d = stages.remove(2);+ stages.insert(0, d);+ }+ _ => {}+ }+ let mut out: Vec<PhotoShape> = Vec::new();+ let mut cursor = after.cloned();+ let from = after.map_or(0, |a| a.stage);+ for (stage, filter) in stages.iter().enumerate().skip(from) {+ if out.len() >= limit {+ break;+ }+ let carry = cursor.as_ref().filter(|c| c.stage == stage);+ let sql = format!(+ "SELECT id, width, height, kind, taken_at FROM photos+ WHERE {filter} {past} ORDER BY taken_at DESC, id LIMIT {n}",+ past = if carry.is_some() {+ "AND (taken_at < ?1 OR (taken_at = ?1 AND id > ?2))"+ } else {+ ""+ },+ n = limit - out.len(),+ );+ let mut stmt = self.conn.prepare_cached(&sql)?;+ let map = |r: &rusqlite::Row<'_>| {+ Ok(PhotoShape {+ id: r.get(0)?,+ width: r.get(1)?,+ height: r.get(2)?,+ kind: Kind::parse(&r.get::<_, String>(3)?),+ taken_at: r.get(4)?,+ })+ };+ let rows: Vec<PhotoShape> = match carry {+ Some(c) => stmt+ .query_map(params![c.key, c.id], map)?+ .collect::<std::result::Result<_, _>>()?,+ None => stmt+ .query_map([], map)?+ .collect::<std::result::Result<_, _>>()?,+ };+ if let Some(last) = rows.last() {+ cursor = Some(GridAfter {+ stage,+ key: last.taken_at,+ id: last.id.clone(),+ });+ }+ out.extend(rows);+ }+ let more = out.len() >= limit;+ Ok((out, cursor.filter(|_| more)))+ }++ /// The library's newest `limit` photos, newest first: what is kept ahead of being opened.+ pub fn newest_photos(&self, limit: usize) -> Result<Vec<Photo>> {+ self.photo_page(&Source::View(View::Library), None, None, limit)+ }++ /// Every video's id, the hidden and deleted ones too: their posters are drawn.+ pub fn video_ids(&self) -> Result<Vec<String>> {+ let mut stmt = self+ .conn+ .prepare("SELECT id FROM photos WHERE kind = 'video' ORDER BY taken_at DESC, id")?;+ let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;+ Ok(rows.collect::<std::result::Result<_, _>>()?)+ }+/// Photos by asset id, in the order asked; those gone are left out.pub fn photos_by_ids(&self, ids: &[String]) -> Result<Vec<Photo>> {let mut out = Vec::with_capacity(ids.len());@@ -998,6 +1222,40 @@ impl Library {/// rather than in the query: dating every photo in SQLite (`strftime … 'localtime'`) was/// most of the time on a large library, where a range over the index costs next to nothing.pub fn photo_months(&self) -> Result<Vec<PhotoMonth>> {+ // Worked out again only when the library has changed since — or the zone, whose+ // months they are. Asked again as nothing changed (every open of the view), they are+ // the ones worked out last.+ let version: (i64, u64) = (+ self.conn+ .query_row("PRAGMA data_version", [], |r| r.get(0))?,+ self.conn.total_changes(),+ );+ let zone = format!("{:?}", self.local_zone());+ {+ let kept = self+ .months+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner);+ if let Some(k) = kept+ .as_ref()+ .filter(|k| k.version == version && k.zone == zone)+ {+ return Ok(k.months.clone());+ }+ }+ let months = self.months_worked_out()?;+ *self+ .months+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(MonthsKept {+ version,+ zone,+ months: months.clone(),+ });+ Ok(months)+ }++ fn months_worked_out(&self) -> Result<Vec<PhotoMonth>> {use jiff::{civil::Date, Span, Timestamp};let (filter, _) = Self::view_query(View::Library);let shown = format!("{filter} AND {ONE_PER_BURST} AND taken_at >= ?1 AND taken_at < ?2");@@ -1444,10 +1702,18 @@ mod tests {("AL".into(), "p30".into(), 2),]).unwrap();- let pages = |source: &Source, order: Option<Order>, size: usize| {- let mut out = Vec::new();- for at in (0..).step_by(size) {- let page = lib.photo_page(source, order, at, size).unwrap();+ // Page after page, each carrying on from the last photo of the one before (keyset).+ let pages = |source: &Source, order: Option<Order>, index: &PhotoIndex, size: usize| {+ let mut out: Vec<Photo> = Vec::new();+ loop {+ let after = out.last().map(|last: &Photo| {+ let at = index.ids.iter().position(|i| *i == last.id).unwrap();+ After {+ key: index.keys[at],+ id: last.id.clone(),+ }+ });+ let page = lib.photo_page(source, order, after.as_ref(), size).unwrap();let n = page.len();out.extend(page);if n < size {@@ -1462,7 +1728,7 @@ mod tests {assert_eq!(index.ids, ids, "{source:?} {order:?}");for size in [1, 3, 7, 500] {assert_eq!(- pages(&source, order, size),+ pages(&source, order, &index, size),want,"{source:?} {order:?} by {size}");@@ -1720,7 +1986,7 @@ mod tests {.len()});let first = time(&|| {- serde_json::to_vec(&lib.photo_page(&source, None, 0, 240).unwrap())+ serde_json::to_vec(&lib.photo_page(&source, None, None, 240).unwrap()).unwrap().len()});@@ -1738,3 +2004,254 @@ mod tests {println!("page by id (mid):{:>10} bytes {:?}", deep.1, deep.0);}}++#[cfg(test)]+mod scale_tests {+ use super::*;+ use crate::notes::Note;++ fn photo(id: &str, taken: i64) -> Photo {+ Photo {+ id: id.into(),+ master: format!("m-{id}"),+ taken_at: taken,+ added_at: Some(taken),+ kind: Kind::Image,+ width: 4032,+ height: 3024,+ duration: None,+ filename: Some(format!("{id}.jpg")),+ size: Some(1),+ uti: Some("public.jpeg".into()),+ favorite: false,+ hidden: false,+ deleted: false,+ deleted_at: None,+ expunge_at: None,+ title: None,+ caption: None,+ rating: 0,+ keywords: Vec::new(),+ latitude: None,+ longitude: None,+ rendered: 0,+ subtype: 0,+ burst: None,+ burst_pick: false,+ burst_count: 0,+ live_size: None,+ change_tag: Some("t".into()),+ }+ }++ fn note(i: usize) -> Note {+ Note {+ guid: format!("n{i}"),+ folder_guid: None,+ title: format!("Note {i}"),+ snippet: None,+ body: Some(format!("Note {i}\nmilk")),+ locked: false,+ pinned: false,+ created_at: Some(1),+ modified_at: Some(2),+ editable: true,+ etag: Some("e".into()),+ runs: None,+ }+ }++ /// Stored again exactly as they are, photos and notes write nothing — no row, no word of+ /// the search index — and a real change still reaches both.+ #[test]+ fn storing_what_is_already_there_writes_nothing() {+ let mut lib = Library::in_memory().unwrap();+ let photos: Vec<Photo> = (0..50).map(|i| photo(&format!("p{i}"), i)).collect();+ let notes: Vec<Note> = (0..50).map(note).collect();+ lib.put_photos(&photos).unwrap();+ lib.put_notes(¬es).unwrap();+ let docs = |lib: &Library| -> Vec<i64> {+ let mut s = lib+ .conn+ .prepare("SELECT id FROM search_docs ORDER BY id")+ .unwrap();+ let rows = s.query_map([], |r| r.get(0)).unwrap();+ rows.collect::<rusqlite::Result<Vec<i64>>>().unwrap()+ };+ let before = (lib.conn.total_changes(), docs(&lib));+ lib.put_photos(&photos).unwrap();+ lib.put_notes(¬es).unwrap();+ assert_eq!(+ (lib.conn.total_changes(), docs(&lib)),+ before,+ "nothing written, nothing indexed again"+ );+ // A real change: written, and found by its new words.+ let mut moved = notes[3].clone();+ moved.body = Some("Note 3\nsaffron".into());+ lib.put_notes(&[moved]).unwrap();+ assert!(lib.conn.total_changes() > before.0);+ let hits: i64 = lib+ .conn+ .query_row(+ "SELECT COUNT(*) FROM search_text WHERE search_text MATCH 'saffron'",+ [],+ |r| r.get(0),+ )+ .unwrap();+ assert_eq!(hits, 1);+ // Pinning changes nothing the index holds: its words are not written again.+ let id_of = |lib: &Library| -> i64 {+ lib.conn+ .query_row(+ "SELECT id FROM search_docs WHERE kind = 'note' AND key = 'n5'",+ [],+ |r| r.get(0),+ )+ .unwrap()+ };+ let was = id_of(&lib);+ let mut pinned = notes[5].clone();+ pinned.pinned = true;+ lib.put_notes(&[pinned]).unwrap();+ assert_eq!(id_of(&lib), was, "indexed once, not again for a pin");+ }++ /// The kept counts are the counted ones through every kind of change: new photos, a+ /// favourite, a hide, a delete, a burst whose chosen photo comes and goes, an expunge.+ #[test]+ fn kept_counts_are_always_the_counted_ones() {+ let mut lib = Library::in_memory().unwrap();+ let agree = |lib: &Library, when: &str| {+ assert_eq!(+ lib.photo_counts().unwrap(),+ lib.photo_counts_counted().unwrap(),+ "{when}"+ );+ };+ agree(&lib, "empty");+ let mut all: Vec<Photo> = (0..40).map(|i| photo(&format!("p{i:02}"), i)).collect();+ for (i, p) in all.iter_mut().enumerate() {+ p.kind = if i % 5 == 0 { Kind::Video } else { Kind::Image };+ p.subtype = if i % 7 == 0 {+ 3+ } else if i % 11 == 0 {+ 1+ } else {+ 0+ };+ p.rendered = if i % 6 == 0 { 8 } else { 0 };+ p.rating = (i % 4) as i64;+ if (10..14).contains(&i) {+ p.burst = Some("B".into());+ p.burst_pick = i == 12;+ }+ }+ lib.put_photos(&all).unwrap();+ agree(&lib, "stored");+ for (i, change) in [+ (1usize, "favorite"),+ (2, "hidden"),+ (3, "deleted"),+ (12, "burst pick deleted"),+ (5, "kind"),+ (7, "subtype"),+ (8, "rating"),+ ] {+ let p = &mut all[i];+ match change {+ "favorite" => p.favorite = true,+ "hidden" => p.hidden = true,+ "deleted" | "burst pick deleted" => p.deleted = true,+ "kind" => p.kind = Kind::Image,+ "subtype" => p.subtype = 0,+ _ => p.rating = 5,+ }+ lib.put_photos(&all[i..=i]).unwrap();+ agree(&lib, change);+ }+ lib.forget_photos(&["p20".into(), "p11".into()]).unwrap();+ agree(&lib, "expunged");+ lib.replace_photos(&all[..5]).unwrap();+ agree(&lib, "replaced whole");+ // Signed out keeping nothing: every count to nothing, and kept again from there.+ lib.forget_account().unwrap();+ agree(&lib, "account forgotten");+ assert!(lib.photo_counts().unwrap().iter().all(|(_, n)| *n == 0));+ assert!(+ !lib.holds_account().unwrap(),+ "the counts are no record of anyone's"+ );+ lib.put_photos(&all[..3]).unwrap();+ agree(&lib, "a new account's");+ }++ /// A library made by 0.1.1–0.1.4 (schema 49), with photos and notes in it, opens at once:+ /// its counts kept from what it holds, and its index triggers firing only on a change.+ #[test]+ fn a_library_from_before_upgrades_cleanly() {+ let mut conn = rusqlite::Connection::open_in_memory().unwrap();+ crate::migrate_with(+ &mut conn,+ &crate::schema::MIGRATIONS[..49],+ crate::schema::FIXUPS,+ )+ .unwrap();+ conn.execute_batch(+ "INSERT INTO photos (id, master, taken_at, kind, width, height, favorite, hidden, deleted)+ VALUES ('a', 'ma', 10, 'image', 1, 1, 1, 0, 0), ('b', 'mb', 20, 'video', 1, 1, 0, 0, 0),+ ('c', 'mc', 30, 'image', 1, 1, 0, 1, 0), ('d', 'md', 40, 'image', 1, 1, 0, 0, 1);+ INSERT INTO notes (guid, title, body, locked, pinned, editable, updated_at)+ VALUES ('n', 'Groceries', 'Groceries\nmilk', 0, 0, 1, 0);",+ )+ .unwrap();+ crate::migrate_with(&mut conn, crate::schema::MIGRATIONS, crate::schema::FIXUPS).unwrap();+ let version: i64 = conn+ .query_row("PRAGMA user_version", [], |r| r.get(0))+ .unwrap();+ assert_eq!(version as usize, crate::schema::MIGRATIONS.len());+ let count = |view: &str| -> i64 {+ conn.query_row(+ "SELECT n FROM photo_view_counts WHERE view = ?1",+ [view],+ |r| r.get(0),+ )+ .unwrap()+ };+ assert_eq!(+ (+ count("library"),+ count("favorites"),+ count("videos"),+ count("hidden"),+ count("recentlyDeleted")+ ),+ (2, 1, 1, 1, 1)+ );+ let when: String = conn+ .query_row(+ "SELECT sql FROM sqlite_master WHERE name = 'search_notes_update'",+ [],+ |r| r.get(0),+ )+ .unwrap();+ assert!(+ when.contains("WHEN (old.folder_guid, old.title, old.body, old.modified_at) IS NOT"),+ "{when}"+ );+ // Still found by its words, and still re-indexed on a real change.+ conn.execute(+ "UPDATE notes SET body = 'Groceries\nsaffron' WHERE guid = 'n'",+ [],+ )+ .unwrap();+ let hits: i64 = conn+ .query_row(+ "SELECT COUNT(*) FROM search_text WHERE search_text MATCH 'saffron'",+ [],+ |r| r.get(0),+ )+ .unwrap();+ assert_eq!(hits, 1);+ }+}diff --git a/crates/quince-core/src/schema.rs b/crates/quince-core/src/schema.rsindex dd93d39..562d692 100644--- a/crates/quince-core/src/schema.rs+++ b/crates/quince-core/src/schema.rs@@ -858,10 +858,122 @@ pub const MIGRATIONS: &[&str] = &[r#"ALTER TABLE events ADD COLUMN floating INTEGER NOT NULL DEFAULT 0;"#,+ // 50 — a large library at the speed of a small one. A pass that stores rows unchanged+ // rewrites nothing the search index holds (`fts_when`, run with this migration, makes each+ // index trigger fire only on a real change); Recents, Favorites and Videos are read in+ // order from indexes of their own, as the Library is, each ending in the photo's id so a+ // page carries on from the last photo shown (keyset) instead of counting past the ones+ // before it; and how many photos each view holds is kept as the rows change, so the+ // sidebar's counts never count the library.+ r#"+ DROP INDEX IF EXISTS photos_by_taken;+ CREATE INDEX photos_by_taken ON photos (deleted, hidden, taken_at DESC, id);+ CREATE INDEX photos_by_added ON photos (deleted, hidden, COALESCE(added_at, taken_at) DESC, id);+ CREATE INDEX photos_favorites ON photos (deleted, hidden, taken_at DESC, id) WHERE favorite = 1;+ CREATE INDEX photos_videos ON photos (deleted, hidden, taken_at DESC, id) WHERE kind = 'video';+ CREATE INDEX photos_months ON photos (deleted, hidden, taken_at, id, kind, favorite, burst, burst_pick);+ CREATE TABLE photo_view_counts (+ view TEXT PRIMARY KEY,+ n INTEGER NOT NULL+ ) STRICT;+ INSERT INTO photo_view_counts (view, n) SELECT 'library', COUNT(*) FROM photos WHERE deleted = 0 AND hidden = 0;+ INSERT INTO photo_view_counts (view, n) SELECT 'favorites', COUNT(*) FROM photos WHERE deleted = 0 AND hidden = 0 AND favorite = 1;+ INSERT INTO photo_view_counts (view, n) SELECT 'videos', COUNT(*) FROM photos WHERE deleted = 0 AND hidden = 0 AND kind = 'video';+ INSERT INTO photo_view_counts (view, n) SELECT 'screenshots', COUNT(*) FROM photos WHERE deleted = 0 AND hidden = 0 AND subtype = 3;+ INSERT INTO photo_view_counts (view, n) SELECT 'panoramas', COUNT(*) FROM photos WHERE deleted = 0 AND hidden = 0 AND subtype = 1;+ INSERT INTO photo_view_counts (view, n) SELECT 'portrait', COUNT(*) FROM photos WHERE deleted = 0 AND hidden = 0 AND rendered = 8;+ INSERT INTO photo_view_counts (view, n) SELECT 'hidden', COUNT(*) FROM photos WHERE deleted = 0 AND hidden = 1;+ INSERT INTO photo_view_counts (view, n) SELECT 'rated', COUNT(*) FROM photos WHERE deleted = 0 AND hidden = 0 AND rating > 0;+ INSERT INTO photo_view_counts (view, n) SELECT 'recentlyDeleted', COUNT(*) FROM photos WHERE deleted = 1;+ CREATE TRIGGER photo_counts_insert AFTER INSERT ON photos BEGIN+ UPDATE photo_view_counts SET n = n + (CASE view WHEN 'library' THEN (new.deleted = 0 AND new.hidden = 0) WHEN 'favorites' THEN (new.deleted = 0 AND new.hidden = 0 AND new.favorite = 1) WHEN 'videos' THEN (new.deleted = 0 AND new.hidden = 0 AND new.kind = 'video') WHEN 'screenshots' THEN (new.deleted = 0 AND new.hidden = 0 AND new.subtype = 3) WHEN 'panoramas' THEN (new.deleted = 0 AND new.hidden = 0 AND new.subtype = 1) WHEN 'portrait' THEN (new.deleted = 0 AND new.hidden = 0 AND new.rendered = 8) WHEN 'hidden' THEN (new.deleted = 0 AND new.hidden = 1) WHEN 'rated' THEN (new.deleted = 0 AND new.hidden = 0 AND new.rating > 0) WHEN 'recentlyDeleted' THEN (new.deleted = 1) ELSE 0 END);+ END;+ CREATE TRIGGER photo_counts_delete AFTER DELETE ON photos BEGIN+ UPDATE photo_view_counts SET n = n - (CASE view WHEN 'library' THEN (old.deleted = 0 AND old.hidden = 0) WHEN 'favorites' THEN (old.deleted = 0 AND old.hidden = 0 AND old.favorite = 1) WHEN 'videos' THEN (old.deleted = 0 AND old.hidden = 0 AND old.kind = 'video') WHEN 'screenshots' THEN (old.deleted = 0 AND old.hidden = 0 AND old.subtype = 3) WHEN 'panoramas' THEN (old.deleted = 0 AND old.hidden = 0 AND old.subtype = 1) WHEN 'portrait' THEN (old.deleted = 0 AND old.hidden = 0 AND old.rendered = 8) WHEN 'hidden' THEN (old.deleted = 0 AND old.hidden = 1) WHEN 'rated' THEN (old.deleted = 0 AND old.hidden = 0 AND old.rating > 0) WHEN 'recentlyDeleted' THEN (old.deleted = 1) ELSE 0 END);+ END;+ CREATE TRIGGER photo_counts_update AFTER UPDATE OF deleted, hidden, favorite, kind, subtype, rendered, rating ON photos+ WHEN (old.deleted, old.hidden, old.favorite, old.kind, old.subtype, old.rendered, old.rating)+ IS NOT (new.deleted, new.hidden, new.favorite, new.kind, new.subtype, new.rendered, new.rating)+ BEGIN+ UPDATE photo_view_counts SET n = n - (CASE view WHEN 'library' THEN (old.deleted = 0 AND old.hidden = 0) WHEN 'favorites' THEN (old.deleted = 0 AND old.hidden = 0 AND old.favorite = 1) WHEN 'videos' THEN (old.deleted = 0 AND old.hidden = 0 AND old.kind = 'video') WHEN 'screenshots' THEN (old.deleted = 0 AND old.hidden = 0 AND old.subtype = 3) WHEN 'panoramas' THEN (old.deleted = 0 AND old.hidden = 0 AND old.subtype = 1) WHEN 'portrait' THEN (old.deleted = 0 AND old.hidden = 0 AND old.rendered = 8) WHEN 'hidden' THEN (old.deleted = 0 AND old.hidden = 1) WHEN 'rated' THEN (old.deleted = 0 AND old.hidden = 0 AND old.rating > 0) WHEN 'recentlyDeleted' THEN (old.deleted = 1) ELSE 0 END) + (CASE view WHEN 'library' THEN (new.deleted = 0 AND new.hidden = 0) WHEN 'favorites' THEN (new.deleted = 0 AND new.hidden = 0 AND new.favorite = 1) WHEN 'videos' THEN (new.deleted = 0 AND new.hidden = 0 AND new.kind = 'video') WHEN 'screenshots' THEN (new.deleted = 0 AND new.hidden = 0 AND new.subtype = 3) WHEN 'panoramas' THEN (new.deleted = 0 AND new.hidden = 0 AND new.subtype = 1) WHEN 'portrait' THEN (new.deleted = 0 AND new.hidden = 0 AND new.rendered = 8) WHEN 'hidden' THEN (new.deleted = 0 AND new.hidden = 1) WHEN 'rated' THEN (new.deleted = 0 AND new.hidden = 0 AND new.rating > 0) WHEN 'recentlyDeleted' THEN (new.deleted = 1) ELSE 0 END);+ END;+ "#,];/// Migrations that need more than SQL, run in the same transaction as the one numbered.-pub const FIXUPS: &[crate::Fixup] = &[(49, float_events)];+pub const FIXUPS: &[crate::Fixup] = &[(49, float_events), (50, fts_when)];++/// The search index's update triggers, and the columns whose change is a change to what they+/// index. Each fires on `UPDATE OF` those columns — which SQLite counts whenever an upsert+/// names them, changed or not: a pass storing fifty thousand notes as they were rewrote every+/// one's words (27 s, measured).+const FTS_UPDATES: &[(&str, &[&str])] = &[+ (+ "search_notes_update",+ &["folder_guid", "title", "body", "modified_at"],+ ),+ (+ "search_reminders_update",+ &["list_guid", "title", "notes", "due_at", "completed_at"],+ ),+ (+ "search_events_update",+ &["title", "location", "notes", "starts_at"],+ ),+ (+ "search_drive_items_update",+ &["name", "extension", "modified_at"],+ ),+ (+ "search_photos_update",+ &[+ "title", "filename", "caption", "keywords", "taken_at", "deleted", "hidden",+ ],+ ),+ (+ "search_contacts_update",+ &[+ "first_name",+ "middle_name",+ "last_name",+ "company",+ "sort_name",+ ],+ ),+ (+ "search_contact_methods_update",+ &["contact_guid", "kind", "value"],+ ),+];++/// Each search index update trigger made again exactly as it stands, with a `WHEN` that lets+/// it fire only when a column it indexes really changed.+fn fts_when(tx: &rusqlite::Transaction<'_>) -> crate::Result<()> {+ for (name, columns) in FTS_UPDATES {+ let sql: String = tx.query_row(+ "SELECT sql FROM sqlite_master WHERE type = 'trigger' AND name = ?1",+ [name],+ |r| r.get(0),+ )?;+ let olds = columns+ .iter()+ .map(|c| format!("old.{c}"))+ .collect::<Vec<_>>();+ let news = columns+ .iter()+ .map(|c| format!("new.{c}"))+ .collect::<Vec<_>>();+ let when = format!(+ " WHEN ({}) IS NOT ({}) BEGIN",+ olds.join(", "),+ news.join(", ")+ );+ let at = sql.find(" BEGIN").ok_or(rusqlite::Error::InvalidQuery)?;+ let made = format!("{}{}{}", &sql[..at], when, &sql[at + " BEGIN".len()..]);+ tx.execute_batch(&format!("DROP TRIGGER {name};\n{made};"))?;+ }+ Ok(())+}/// The all-day and floating events already stored, from instants to wall-clock time: read in/// the zone they were stored in — this computer's, by name, or UTC where it had none, as thediff --git a/crates/quince-core/tests/scale.rs b/crates/quince-core/tests/scale.rsnew file mode 100644index 0000000..070ef2b--- /dev/null+++ b/crates/quince-core/tests/scale.rs@@ -0,0 +1,293 @@+//! What a heavy library costs: synthetic photo libraries of 100,000 and 1,000,000 assets and+//! 50,000 notes, in a temporary folder, each call timed and the peak memory it took counted.+//! Run on purpose, not with every test:+//!+//! `SCALE=100000 cargo test -p quince-core --release --test scale -- --ignored --nocapture`+//!+//! `SCALE` is the number of photos (100,000 when unset); `SCALE_DIR` where the library is made+//! (a tmpfs is best; the system's temporary folder when unset).++use quince_core::photos::{Kind, Order, OrderBy, Photo, Source, View};+use quince_core::Library;+use std::alloc::{GlobalAlloc, Layout, System};+use std::sync::atomic::{AtomicUsize, Ordering};+use std::time::Instant;++/// Counts the bytes held now and the most held since the count was last reset.+struct Counting;+static NOW: AtomicUsize = AtomicUsize::new(0);+static PEAK: AtomicUsize = AtomicUsize::new(0);++unsafe impl GlobalAlloc for Counting {+ unsafe fn alloc(&self, layout: Layout) -> *mut u8 {+ // SAFETY: passed straight to the system allocator with the layout given.+ let p = unsafe { System.alloc(layout) };+ if !p.is_null() {+ let now = NOW.fetch_add(layout.size(), Ordering::Relaxed) + layout.size();+ PEAK.fetch_max(now, Ordering::Relaxed);+ }+ p+ }+ unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {+ // SAFETY: as allocated above.+ unsafe { System.dealloc(ptr, layout) };+ NOW.fetch_sub(layout.size(), Ordering::Relaxed);+ }+}++#[global_allocator]+static ALLOC: Counting = Counting;++/// Run `f`, and say how long it took and the most memory it held beyond what was held before.+fn measure<T>(what: &str, f: impl FnOnce() -> T) -> T {+ let before = NOW.load(Ordering::Relaxed);+ PEAK.store(before, Ordering::Relaxed);+ let started = Instant::now();+ let out = f();+ let took = started.elapsed();+ let peak = PEAK.load(Ordering::Relaxed).saturating_sub(before);+ println!(+ "{what:<44} {:>9.1} ms {:>9.1} MB peak",+ took.as_secs_f64() * 1000.0,+ peak as f64 / 1_048_576.0+ );+ out+}++fn photo(i: usize) -> Photo {+ let taken = 1_500_000_000 + (i as i64) * 300;+ Photo {+ id: format!("A{i:08}-7B1C-4E2F-9A3D-{i:012}"),+ master: format!("M{i:08}-1C2D-4E5F-8A9B-{i:012}"),+ taken_at: taken,+ added_at: Some(taken + if i % 7 == 0 { 86_400 * 30 } else { 60 }),+ kind: if i % 20 == 0 {+ Kind::Video+ } else if i % 9 == 0 {+ Kind::Live+ } else {+ Kind::Image+ },+ width: 4032,+ height: 3024,+ duration: (i % 20 == 0).then_some(12.5),+ filename: Some(format!("IMG_{:04}.HEIC", i % 10_000)),+ size: Some(2_400_000),+ uti: Some("public.heic".into()),+ favorite: i % 50 == 0,+ hidden: i % 997 == 0,+ deleted: i % 991 == 0,+ deleted_at: (i % 991 == 0).then_some(taken + 10),+ expunge_at: None,+ title: (i % 100 == 0).then(|| format!("Trip day {}", i % 31)),+ caption: (i % 250 == 0).then(|| "Sunset over the bay".to_string()),+ rating: (i % 333 == 0) as i64 * 4,+ keywords: if i % 400 == 0 {+ vec!["beach".into(), "family".into()]+ } else {+ Vec::new()+ },+ latitude: Some(17.7),+ longitude: Some(83.3),+ rendered: if i % 40 == 0 { 8 } else { 0 },+ subtype: if i % 60 == 0 { 3 } else { 0 },+ burst: (i % 1000 < 5).then(|| format!("B{}", i / 1000)),+ burst_pick: i % 1000 == 0,+ burst_count: 0,+ live_size: None,+ change_tag: Some(format!("t{i}")),+ }+}++fn note(i: usize) -> quince_core::notes::Note {+ quince_core::notes::Note {+ guid: format!("N{i:010}"),+ folder_guid: None,+ title: format!("Note {i}"),+ snippet: Some("Groceries and things".into()),+ body: Some(format!(+ "Note {i}\nmilk eggs bread\nThe quick brown fox jumps {i} times.\n"+ )),+ locked: false,+ pinned: false,+ created_at: Some(1_700_000_000),+ modified_at: Some(1_700_000_000 + i as i64),+ editable: true,+ etag: Some(format!("e{i}")),+ runs: None,+ }+}++#[test]+#[ignore = "a benchmark: run it on purpose"]+fn a_heavy_library() {+ let n: usize = std::env::var("SCALE")+ .ok()+ .and_then(|s| s.parse().ok())+ .unwrap_or(100_000);+ let base = std::env::var("SCALE_DIR").map_or_else(|_| std::env::temp_dir(), Into::into);+ let dir = tempfile::tempdir_in(base).unwrap();+ let mut lib = Library::open(dir.path()).unwrap();+ println!("\n{n} photos, 50000 notes, in {}", dir.path().display());++ measure("store every photo (first pass)", || {+ lib.atomically(|l| {+ for chunk in (0..n).collect::<Vec<_>>().chunks(5_000) {+ let batch: Vec<Photo> = chunk.iter().map(|&i| photo(i)).collect();+ l.put_photos(&batch)?;+ }+ Ok::<_, quince_core::Error>(())+ })+ .unwrap()+ });+ let again: Vec<Photo> = (0..50_000.min(n)).map(photo).collect();+ measure("store 50000 photos again, unchanged", || {+ lib.atomically(|l| l.put_photos(&again)).unwrap()+ });+ drop(again);++ let notes: Vec<_> = (0..50_000).map(note).collect();+ measure("store 50000 notes (first pass)", || {+ lib.atomically(|l| l.put_notes(¬es)).unwrap()+ });+ measure("store 50000 notes again, unchanged", || {+ lib.atomically(|l| l.put_notes(¬es)).unwrap()+ });+ drop(notes);++ let library = Source::View(View::Library);+ let taken = Some(Order {+ by: OrderBy::Taken,+ oldest: false,+ });+ let index = measure("photo_index (Library)", || {+ lib.photo_index(&library, taken).unwrap()+ });+ measure("photo_index (Recents)", || {+ lib.photo_index(&Source::View(View::Recents), None).unwrap()+ });+ measure("photo_index (Favorites)", || {+ lib.photo_index(&Source::View(View::Favorites), None)+ .unwrap()+ });+ measure("photo_index (Videos)", || {+ lib.photo_index(&Source::View(View::Videos), None).unwrap()+ });+ measure("photo_page first 200", || {+ lib.photo_page(&library, taken, None, 200).unwrap()+ });+ let mid = index.ids.len() / 2;+ let after = quince_core::photos::After {+ key: index.keys[mid],+ id: index.ids[mid].clone(),+ };+ measure("photo_page 200 from the middle", || {+ lib.photo_page(&library, taken, Some(&after), 200).unwrap()+ });+ let ids: Vec<String> = index.ids[mid..mid + 200].to_vec();+ measure("photos_by_ids 200", || lib.photos_by_ids(&ids).unwrap());+ drop(index);+ measure("photo_counts", || lib.photo_counts().unwrap());+ measure("photo_months", || lib.photo_months().unwrap());+ if std::env::var_os("SCALE_PLAN").is_some() {+ let mut st = lib+ .conn()+ .prepare(+ "EXPLAIN QUERY PLAN SELECT id FROM photos WHERE +deleted = 0 AND +hidden = 0 AND id IN+ (SELECT d.key FROM search_text CROSS JOIN search_docs d ON d.id = search_text.rowid+ WHERE search_text MATCH '\"beach\"*' AND d.kind = 'photo')+ ORDER BY taken_at DESC, id LIMIT 500",+ )+ .unwrap();+ let rows = st.query_map([], |r| r.get::<_, String>(3)).unwrap();+ println!("plan: {:?}", rows.collect::<Result<Vec<_>, _>>().unwrap());+ }+ measure("search_photos \"beach\"", || {+ lib.search_photos("beach", 500).unwrap()+ });+ measure("search_photos \"img_12\"", || {+ lib.search_photos("img_12", 500).unwrap()+ });+ measure("search_photos \"beach\" again", || {+ lib.search_photos("beach", 500).unwrap()+ });+ measure("search_photos \"sunset bay\"", || {+ lib.search_photos("sunset bay", 500).unwrap()+ });+ measure("whole library read as photos (old grid path)", || {+ lib.photos(View::Library).unwrap().len()+ });+ measure(+ "whole library read as shapes, a page at a time (grid path)",+ || {+ let (mut k, mut after) = (0usize, None);+ loop {+ let (page, next) = lib.grid_shapes(None, after.as_ref(), 2_000).unwrap();+ k += page.len();+ match next {+ Some(n) => after = Some(n),+ None => return k,+ }+ }+ },+ );+ measure("photo_months again, nothing changed", || {+ lib.photo_months().unwrap()+ });+ measure("newest 600 originals (keep_ahead)", || {+ lib.newest_photos(600).unwrap().len()+ });+ measure("every video's id (keep_ahead)", || {+ lib.video_ids().unwrap().len()+ });+ // A picture asked for while a pass stores: on its own read-only connection, as the app now+ // reads one, beside the pass's write (WAL) rather than behind it.+ let changed: Vec<Photo> = (0..50_000.min(n))+ .map(|i| {+ let mut p = photo(i);+ p.change_tag = Some(format!("u{i}"));+ p+ })+ .collect();+ let path = dir.path().to_path_buf();+ let done = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));+ let asking = {+ let done = done.clone();+ std::thread::spawn(move || {+ let reader = Library::open_reader(&path).unwrap();+ let mut waits = Vec::new();+ let mut i = 0usize;+ while !done.load(Ordering::Relaxed) || waits.len() < 50 {+ let started = Instant::now();+ reader.photo(&photo(i * 7919 % n).id).unwrap();+ waits.push(started.elapsed());+ i += 1;+ std::thread::sleep(std::time::Duration::from_millis(2));+ }+ waits.sort();+ waits+ })+ };+ measure("store 50000 changed photos, a reader asking", || {+ lib.atomically(|l| l.put_photos(&changed)).unwrap()+ });+ done.store(true, Ordering::Relaxed);+ let waits = asking.join().unwrap();+ let p = |q: f64| waits[((waits.len() as f64 - 1.0) * q) as usize].as_secs_f64() * 1000.0;+ println!(+ "{:<44} p50 {:.2} ms p99 {:.2} ms max {:.2} ms ({} asks)",+ "a picture's photo read during that store",+ p(0.5),+ p(0.99),+ p(1.0),+ waits.len()+ );+ println!(+ "library file: {:.0} MB",+ std::fs::metadata(dir.path().join("library.db")).map_or(0, |m| m.len()) as f64+ / 1_048_576.0+ );+ if std::env::var_os("SCALE_KEEP").is_some() {+ println!("kept at {}", dir.keep().display());+ }+}diff --git a/src-tauri/src/photos.rs b/src-tauri/src/photos.rsindex 0f2bae1..0cd76b4 100644--- a/src-tauri/src/photos.rs+++ b/src-tauri/src/photos.rs@@ -377,9 +377,10 @@ pub async fn cached(app: &AppHandle, id: &str, size: Size) -> Result<PathBuf, St/// Whether a photo's original is HEIC (or another HEIF).async fn is_heic(app: &AppHandle, id: &str) -> bool {let data = app.state::<Data>();- let library = data.0.lock().await;- library- .photo(id)+ // Read beside a sync pass, never behind it (see `LibraryLock::read`).+ data.0+ .read(|l| l.photo(id))+ .await.ok().flatten().and_then(|p| p.uti)@@ -435,9 +436,11 @@ async fn from_apple(}let photo = {let data = app.state::<Data>();- let library = data.0.lock().await;- library- .photo(id)+ // Beside a sync pass on a reader: a picture asked for never waits behind a pass's+ // store, which holds the writer for seconds on a large library.+ data.0+ .read(|l| l.photo(id))+ .await.map_err(|e| e.to_string())?.ok_or("That photo is no longer in the library.")?};@@ -597,15 +600,18 @@ pub fn keep_ahead(app: AppHandle) {}tauri::async_runtime::spawn(async move {use futures_util::StreamExt;+ // Read beside a pass, and only what is used: the newest 600 and the videos' ids. The+ // whole library read as photos, twice, under the writer's lock, was 1.3 GB at a+ // million and held up every pass behind it.let (newest, every_video) = {let data = app.state::<Data>();- let library = data.0.lock().await;- let videos: Vec<String> = grid_photos(&library, None)- .into_iter()- .filter(|p| p.kind == quince_core::photos::Kind::Video)- .map(|p| p.id)- .collect();- (library.photos(View::Library).unwrap_or_default(), videos)+ let newest = data+ .0+ .read(|l| l.newest_photos(600))+ .await+ .unwrap_or_default();+ let videos = data.0.read(|l| l.video_ids()).await.unwrap_or_default();+ (newest, videos)};// Videos whose black poster was kept before posters were remade: remade now, once —// once per video for the life of the process. Every pass decoded every video's poster@@ -707,23 +713,37 @@ pub struct Shown {pub view: Option<View>,}-/// Every photo a grid can show, the view being looked at first: the library, Hidden and-/// Recently Deleted, each once. Made for the library alone, Hidden's pictures were made only as-/// it was scrolled — every tile waited on one, and showed the soft thumbnail meanwhile.-fn grid_photos(library: &quince_core::Library, first: Option<View>) -> Vec<Photo> {- let mut seen = std::collections::HashSet::new();+/// How many of the grid's photos `prepare` reads at a time.+const GRID_PAGE: usize = 2_000;++/// Of one page of the grid's photos (`stills_before` stills came in earlier pages), the+/// pictures to make: each still's grid picture while it is thumbnail-sized (kept always), the+/// larger grid and viewer pictures of the newest 600 stills only — previews the cache's cap+/// lets go of, made for every photo they were let go and made again after every sync — and+/// of those only the ones `missing`. Answers them and how many stills the page held.+fn wanted_steps(+ page: &[quince_core::photos::PhotoShape],+ shown: Shown,+ stills_before: usize,+ missing: impl Fn(&str, u32) -> bool,+) -> (Vec<(String, u32)>, usize) {+ let stills: Vec<&quince_core::photos::PhotoShape> = page+ .iter()+ .filter(|p| p.kind != quince_core::photos::Kind::Video)+ .collect();let mut out = Vec::new();- let views = first- .into_iter()- .chain([View::Library, View::Hidden, View::RecentlyDeleted]);- for view in views {- for p in library.photos(view).unwrap_or_default() {- if seen.insert(p.id.clone()) {- out.push(p);- }+ for (i, p) in stills.iter().enumerate() {+ let (grid, viewer) = steps_of(p.width, p.height, shown);+ let newest = stills_before + i < 600;+ if grid <= crate::photo_cache::THUMB_STEP || newest {+ out.push((p.id.clone(), grid));+ }+ if newest {+ out.push((p.id.clone(), viewer));}}- out+ out.retain(|(id, step)| missing(id, *step));+ (out, stills.len())}/// The step a picture is made at for a longer side of `long` device pixels — the window's own@@ -737,8 +757,8 @@ fn step_for(long: f64) -> u32 {/// The steps one photo is shown at: in the grid (the whole cell when fitted, cropped to fill a/// square otherwise) and in the viewer (as large as the stage allows, never past 1.5×).-fn steps_of(p: &Photo, s: Shown) -> (u32, u32) {- let (w, h) = (p.width.max(1) as f64, p.height.max(1) as f64);+fn steps_of(width: i64, height: i64, s: Shown) -> (u32, u32) {+ let (w, h) = (width.max(1) as f64, height.max(1) as f64);let r = w / h;let tile = if s.fit {s.cell@@ -783,54 +803,54 @@ pub fn prepare(app: AppHandle) {let mine = PREPARING.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;tauri::async_runtime::spawn(async move {use futures_util::StreamExt;- let photos = {- let data = app.state::<Data>();- let library = data.0.lock().await;- grid_photos(&library, shown.view)- };let fit_dir = crate::state_dir(&app).join("photos").join("fit");- // A stat per picture, off the runtime's workers (see keep_ahead).- let wanted: Vec<(String, u32)> = tauri::async_runtime::spawn_blocking(move || {- // The grid's pictures for the whole library while they are thumbnail-sized (kept- // always); larger ones — a close zoom on a wide screen — are previews the cache's cap- // lets go of, so only the newest are made ahead: made for every photo, they were let- // go and made again after every sync.- let stills = || {- photos- .iter()- .filter(|p| p.kind != quince_core::photos::Kind::Video)+ // The grid's photos a page at a time, each read beside a pass on a reader and made+ // before the next is read: what is held at once is one page, however large the+ // library (the whole of it, read as photos under the writer's lock, was 665 MB and+ // seconds of every pass's wait at a million).+ let mut after = None;+ let mut stills_seen = 0usize;+ loop {+ if PREPARING.load(std::sync::atomic::Ordering::SeqCst) != mine {+ return;+ }+ let data = app.state::<Data>();+ let from = after.clone();+ let Ok((page, next)) = data+ .0+ .read(|l| l.grid_shapes(shown.view, from.as_ref(), GRID_PAGE))+ .await+ else {+ return;};- stills()- .map(|p| (p.id.clone(), steps_of(p, shown).0))- .enumerate()- .filter(|(i, (_, step))| *step <= crate::photo_cache::THUMB_STEP || *i < 600)- .map(|(_, w)| w)- .chain(- stills()- .take(600)- .map(|p| (p.id.clone(), steps_of(p, shown).1)),- )- .filter(|(id, step)| {- !fit_dir- .join(step.to_string())- .join(&*cache_name(id))- .exists()+ let seen = stills_seen;+ let dir = fit_dir.clone();+ // A stat per picture, off the runtime's workers (see keep_ahead).+ let (wanted, stills): (Vec<(String, u32)>, usize) =+ tauri::async_runtime::spawn_blocking(move || {+ wanted_steps(&page, shown, seen, |id, step| {+ !dir.join(step.to_string()).join(&*cache_name(id)).exists()+ })})- .collect()- })- .await- .unwrap_or_default();- futures_util::stream::iter(wanted)- .for_each_concurrent(4, |(id, step)| {- let app = app.clone();- async move {- if PREPARING.load(std::sync::atomic::Ordering::SeqCst) != mine {- return;+ .await+ .unwrap_or_default();+ stills_seen += stills;+ futures_util::stream::iter(wanted)+ .for_each_concurrent(4, |(id, step)| {+ let app = app.clone();+ async move {+ if PREPARING.load(std::sync::atomic::Ordering::SeqCst) != mine {+ return;+ }+ let _ = crate::cpu::ahead(fitted(&app, &id, step)).await;}- let _ = crate::cpu::ahead(fitted(&app, &id, step)).await;- }- })- .await;+ })+ .await;+ match next {+ Some(n) => after = Some(n),+ None => return,+ }+ }});}@@ -977,9 +997,9 @@ pub async fn film(app: &AppHandle, id: &str, size: Size) -> Result<Film, String>}let photo = {let data = app.state::<Data>();- let library = data.0.lock().await;- library- .photo(id)+ data.0+ .read(|l| l.photo(id))+ .await.map_err(|e| e.to_string())?.ok_or("That photo is no longer in the library.")?};@@ -1187,17 +1207,17 @@ pub async fn photos_months(.map_err(|e| e.to_string())}-/// `limit` photos from `offset` of a source, in its index's order.+/// `limit` photos of a source, in its index's order: its first, or those just after `after`.#[tauri::command]pub async fn photos_page(source: quince_core::photos::Source,order: Option<quince_core::photos::Order>,- offset: usize,+ after: Option<quince_core::photos::After>,limit: usize,data: State<'_, Data>,) -> UiResult<Vec<Photo>> {data.0- .read(|library| library.photo_page(&source, order, offset, limit))+ .read(|library| library.photo_page(&source, order, after.as_ref(), limit)).await.map_err(|e| e.to_string())}@@ -1402,7 +1422,11 @@ pub async fn photos_album_delete(// albums inside this folder"). Apple's server does not do that itself — checked live: an// album inside a deleted folder stayed, its folder gone — so the ones inside are deleted// here, the deepest first, and the folder last.- let all = data.0.lock().await.albums().map_err(|e| e.to_string())?;+ let all = data+ .0+ .read(|l| l.albums())+ .await+ .map_err(|e| e.to_string())?;let mut order: Vec<quince_core::photos::Album> = Vec::new();fn within(all: &[quince_core::photos::Album],@@ -3144,8 +3168,19 @@ mod tests {photo("D1", 5, false, true),]).unwrap();- let ids =- |first| -> Vec<String> { grid_photos(&lib, first).into_iter().map(|p| p.id).collect() };+ // Read a page at a time — one photo at a time here — as `prepare` reads them.+ let ids = |first| -> Vec<String> {+ let mut out = Vec::new();+ let mut after = None;+ loop {+ let (page, next) = lib.grid_shapes(first, after.as_ref(), 1).unwrap();+ out.extend(page.into_iter().map(|p| p.id));+ match next {+ Some(n) => after = Some(n),+ None => return out,+ }+ }+ };assert_eq!(ids(None), ["L1", "H1", "H2", "D1"]);assert_eq!(ids(Some(View::Hidden)), ["H1", "H2", "L1", "D1"]);assert_eq!(ids(Some(View::Library)), ["L1", "H1", "H2", "D1"]);@@ -3171,8 +3206,11 @@ mod tests {};// A square crop of a 4:3 photo reaches 240 along its long side; the stage draws it// 1066 wide (800 high × 4/3).- assert_eq!(steps_of(&p, shown), (256, 1536));- assert_eq!(steps_of(&p, Shown { fit: true, ..shown }).0, 256);+ assert_eq!(steps_of(p.width, p.height, shown), (256, 1536));+ assert_eq!(+ steps_of(p.width, p.height, Shown { fit: true, ..shown }).0,+ 256+ );assert_eq!(step_for(5000.0), 3840);assert_eq!(step_for(100.0), 128);}diff --git a/src-tauri/src/refresh.rs b/src-tauri/src/refresh.rsindex e85914d..b3821eb 100644--- a/src-tauri/src/refresh.rs+++ b/src-tauri/src/refresh.rs@@ -604,6 +604,7 @@ async fn run(client: &Arc<Mutex<Client>>, data: &LibraryLock, services: &[&str])ctx.account.as_deref(),fetched,move |library, f| {+ let records = f.records.len();let (_, links) = quince_icloud::photos::apply(library, f)?;// Marked, or every pass after walked the whole library again:// the daily full read is due when none was ever recorded.@@ -611,14 +612,14 @@ async fn run(client: &Arc<Mutex<Client>>, data: &LibraryLock, services: &[&str])library.mark_full_sync(quince_icloud::photos::SERVICE)?;}crate::photos::remember(links);- // Counted, not read: every row of the library loaded to say- // how many there were held the library's lock through each pass.- let n = library- .photo_counts()?- .into_iter()- .find(|(v, _)| *v == quince_core::photos::View::Library)- .map_or(0, |(_, n)| usize::try_from(n).unwrap_or(0));- Ok(format!("{n} photo{}", plural(n)))+ // Said from what the pass brought: counting the library to+ // say it held the writer's lock through each of a refresh's ten+ // rounds (4.4 s each at a million).+ Ok(if records == 0 {+ "Up to date".to_string()+ } else {+ format!("{records} change{} from iCloud", plural(records))+ })},).await;diff --git a/src/lib/ipc.ts b/src/lib/ipc.tsindex 6719829..961978c 100644--- a/src/lib/ipc.ts+++ b/src/lib/ipc.ts@@ -245,8 +245,10 @@ export const api = {/** Every photo a grid shows, as it lays them out: ids, grouping dates and marks, no photos. */photosIndex: (source: PhotoSource, order: PhotoOrder | null) => invoke<PhotoIndex>("photos_index", { source, order }),/** A source's photos from `offset`, in its index's order: the first screen, before the index. */- photosPage: (source: PhotoSource, order: PhotoOrder | null, offset: number, limit: number) =>- invoke<Photo[]>("photos_page", { source, order, offset, limit }),+ /** `limit` photos of a source: its first, or those just after `after` — the last photo+ * shown, by its index key and id. */+ photosPage: (source: PhotoSource, order: PhotoOrder | null, after: { key: number; id: string } | null, limit: number) =>+ invoke<Photo[]>("photos_page", { source, order, after, limit }),/** Photos by id, in the order asked; any gone are left out. */photosGet: (ids: string[]) => invoke<Photo[]>("photos_get", { ids }),photosCounts: () => invoke<[PhotoView, number][]>("photos_counts"),diff --git a/src/screens/PhotosView.tsx b/src/screens/PhotosView.tsxindex 255f312..9f40093 100644--- a/src/screens/PhotosView.tsx+++ b/src/screens/PhotosView.tsx@@ -970,7 +970,7 @@ export function PhotosView() {// Another place chosen: its first screen from its first page, which is small and comes at// once, painted before the whole index (a large library's takes a moment) is here.if (!again) {- api.photosPage(source, order, 0, PAGE).then(+ api.photosPage(source, order, null, PAGE).then((page) => {if (indexed || mine !== reads.current) return;setPhotos(PhotoList.of(page, store, (p) => sortKey(p, order?.by ?? "taken")));