LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
PERF-EDIT: a smart album's count follows an edit, asking only the photos it touched
Nathaniel Ledford43ba500c6e24
diff --git a/crates/catalog/src/lib.rs b/crates/catalog/src/lib.rsindex aa2f5745b..79b02f7dc 100644--- a/crates/catalog/src/lib.rs+++ b/crates/catalog/src/lib.rs@@ -25,6 +25,7 @@ pub mod rules;pub mod safe_file;pub mod stacks;pub mod store;+pub mod tally;use std::collections::BTreeMap;use std::sync::Arc;diff --git a/crates/catalog/src/tally.rs b/crates/catalog/src/tally.rsnew file mode 100644index 000000000..8f15e5ec4--- /dev/null+++ b/crates/catalog/src/tally.rs@@ -0,0 +1,234 @@+//! How many photos each album shows, kept up to date from what the latest ops touched.+//!+//! A smart album's count asks every photo whether its rules hold. Asked again after every edit, that is a pass over+//! the library for each smart album each time a keyword is given. [`AlbumTally`] remembers which photos each smart+//! album holds, and after a small edit asks only the photos the edit touched ([`Catalog::touched_since`]). An op+//! that changes more than photos of its own (an album's rules or photos, a stack, the keyword list) has it counted+//! whole again, as before.++use std::collections::{HashMap, HashSet};++use crate::{AlbumId, Aspects, Catalog, PhotoId};++/// The most photos an edit may touch for the counts to follow it photo by photo.+const MAX_FOLLOWED: usize = 256;+/// Bits are kept while they cost no more than this many words a photo held, and these to start with. Past that+/// (photo ids far apart: a library whose ids don't count up from 1) a set is kept instead, so what an album+/// remembers is always in proportion to the photos it holds, never to the largest id in the library.+const WORDS_A_MEMBER: usize = 4;+const WORDS_TO_START: usize = 1024;++/// The photos a smart album holds.+#[derive(Clone, Debug)]+enum Members {+ /// One bit a photo id, and how many are set.+ Bits {+ words: Vec<u64>,+ len: usize,+ },+ Set(HashSet<PhotoId>),+}++impl Members {+ fn none() -> Members {+ Members::Bits { words: Vec::new(), len: 0 }+ }++ fn len(&self) -> usize {+ match self {+ Members::Bits { len, .. } => *len,+ Members::Set(set) => set.len(),+ }+ }++ #[cfg(test)]+ fn holds(&self, id: PhotoId) -> bool {+ match self {+ Members::Bits { words, .. } => {+ usize::try_from(id.0 / 64).ok().and_then(|at| words.get(at)).is_some_and(|word| word & (1u64 << (id.0 % 64)) != 0)+ }+ Members::Set(set) => set.contains(&id),+ }+ }++ /// Put `id` in or take it out.+ fn set(&mut self, id: PhotoId, held: bool) {+ if let Members::Bits { words, len } = self {+ let bit = 1u64 << (id.0 % 64);+ match usize::try_from(id.0 / 64) {+ Ok(at) if at < words.len() => {+ if let Some(word) = words.get_mut(at) {+ match (*word & bit != 0, held) {+ (false, true) => {+ *word |= bit;+ *len = len.saturating_add(1);+ }+ (true, false) => {+ *word &= !bit;+ *len = len.saturating_sub(1);+ }+ _ => {}+ }+ }+ return;+ }+ // (past the end: not held, so there is nothing to take out)+ _ if !held => return,+ // room for it, as long as the bits stay in proportion to the photos held+ Ok(at) if at < len.saturating_mul(WORDS_A_MEMBER).saturating_add(WORDS_TO_START) => {+ words.resize(at + 1, 0);+ if let Some(word) = words.get_mut(at) {+ *word |= bit;+ *len = len.saturating_add(1);+ }+ return;+ }+ // ids too far apart for bits: a set from here on+ _ => {+ let ids = words+ .iter()+ .enumerate()+ .flat_map(|(at, word)| (0..64u64).filter(move |b| word & (1u64 << b) != 0).map(move |b| PhotoId(at as u64 * 64 + b)));+ *self = Members::Set(ids.collect());+ }+ }+ }+ if let Members::Set(set) = self {+ if held {+ set.insert(id);+ } else {+ set.remove(&id);+ }+ }+ }+}++/// [`Catalog::album_count`] of every album, as of one revision of the catalog.+#[derive(Clone, Debug)]+pub struct AlbumTally {+ revision: u64,+ counts: HashMap<AlbumId, usize>,+ /// The smart albums' photos.+ members: HashMap<AlbumId, Members>,+}++impl AlbumTally {+ /// Counted from every photo.+ pub fn of(cat: &Catalog) -> AlbumTally {+ let mut counts = HashMap::new();+ let mut members = HashMap::new();+ for album in cat.albums() {+ if album.is_smart() {+ let mut held = Members::none();+ for p in cat.photos().filter(|p| cat.album_contains(album.id, p)) {+ held.set(p.id, true);+ }+ counts.insert(album.id, held.len());+ members.insert(album.id, held);+ } else {+ counts.insert(album.id, cat.album_count(album.id));+ }+ }+ AlbumTally { revision: cat.revision, counts, members }+ }++ /// The count of each album.+ pub fn counts(&self) -> &HashMap<AlbumId, usize> {+ &self.counts+ }++ /// The catalog's revision the counts are of.+ pub fn revision(&self) -> u64 {+ self.revision+ }++ /// Bring the counts up to `cat` as it is now. `true` when that was done from the photos the ops since touched+ /// (or nothing had changed), `false` when every photo was asked again.+ ///+ /// `cat` has to be the catalog the counts were made of, further along. Another catalog is told apart only by+ /// its revisions: a library that is loaded takes a range of its own (and then everything is counted again), but+ /// two catalogs both counting from 0 look like one.+ pub fn update(&mut self, cat: &Catalog) -> bool {+ if self.revision == cat.revision {+ return true;+ }+ let followed = match (cat.touched_since(self.revision), cat.aspects_since(self.revision)) {+ (Some(touched), Some(changed)) if touched.len() <= MAX_FOLLOWED => {+ self.follow(cat, &touched, changed);+ true+ }+ _ => false,+ };+ if !followed {+ *self = AlbumTally::of(cat);+ }+ followed+ }++ /// The counts after ops that touched only `touched`: each smart album asks those photos again; an album of+ /// chosen photos counts again only when a photo came, went or was deleted (who is in it did not change: that+ /// is an op on the album, and not followed).+ fn follow(&mut self, cat: &Catalog, touched: &[PhotoId], changed: Aspects) {+ for (album, held) in &mut self.members {+ for id in touched {+ held.set(*id, cat.photo(*id).is_some_and(|p| cat.album_contains(*album, p)));+ }+ self.counts.insert(*album, held.len());+ }+ if changed.any_of(Aspects::PRESENCE) {+ for album in cat.albums().filter(|a| !a.is_smart()) {+ self.counts.insert(album.id, cat.album_count(album.id));+ }+ }+ self.revision = cat.revision;+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn ids(m: &Members, of: &[u64]) -> Vec<u64> {+ of.iter().copied().filter(|id| m.holds(PhotoId(*id))).collect()+ }++ #[test]+ fn members_are_kept_as_bits_at_the_edges_of_a_word_and_past_the_end() {+ let mut m = Members::none();+ let probe = [0, 1, 63, 64, 65, 127, 128, 5000];+ assert!(ids(&m, &probe).is_empty() && m.len() == 0);+ for id in [0, 63, 64, 5000] {+ m.set(PhotoId(id), true);+ }+ assert_eq!((ids(&m, &probe), m.len()), (vec![0, 63, 64, 5000], 4));+ // put in twice, taken out twice, taken out where nothing is, far past the end: counted once each way+ m.set(PhotoId(63), true);+ m.set(PhotoId(64), false);+ m.set(PhotoId(64), false);+ m.set(PhotoId(1), false);+ m.set(PhotoId(900_000), false);+ assert_eq!((ids(&m, &probe), m.len()), (vec![0, 63, 5000], 3));+ assert!(matches!(m, Members::Bits { .. }));+ }++ #[test]+ fn photo_ids_far_apart_are_kept_as_a_set_in_proportion_to_the_photos_held() {+ // one photo with an id in the thousands of millions: no bit for every id up to it+ let mut m = Members::none();+ for id in [3, 70, 9_000_000_000] {+ m.set(PhotoId(id), true);+ }+ assert!(matches!(m, Members::Set(_)), "a set once the ids are too far apart for bits");+ assert_eq!((ids(&m, &[3, 70, 71, 9_000_000_000, u64::MAX]), m.len()), (vec![3, 70, 9_000_000_000], 3));+ m.set(PhotoId(u64::MAX), true);+ m.set(PhotoId(3), false);+ assert_eq!((ids(&m, &[3, 70, 9_000_000_000, u64::MAX]), m.len()), (vec![70, 9_000_000_000, u64::MAX], 3));+ // an album of many photos with ids counting up stays bits, however many+ let mut dense = Members::none();+ for id in 1..=200_000u64 {+ dense.set(PhotoId(id), true);+ }+ assert!(matches!(&dense, Members::Bits { words, .. } if words.len() <= 200_000 / 64 + 1));+ assert_eq!(dense.len(), 200_000);+ }+}diff --git a/crates/engine/src/tests_view_follow.rs b/crates/engine/src/tests_view_follow.rsindex bf0656f86..391e38b3b 100644--- a/crates/engine/src/tests_view_follow.rs+++ b/crates/engine/src/tests_view_follow.rs@@ -219,3 +219,77 @@ fn an_edit_to_many_photos_or_to_more_than_photos_is_worked_out_whole() {let _ = s.visible_cloned();assert_eq!((s.view_followed, s.view_worked_out), (followed + 1, whole + 2));}++#[test]+fn the_albums_counts_followed_through_edits_are_the_counts_made_from_every_photo() {+ use lightcraft_catalog::tally::AlbumTally;+ let (mut s, mut ids, album, smart) = library(240);+ // more smart albums: picks, well rated or keyworded ones, one that looks into an album of chosen photos+ for rules in [+ json!({"ruleSet": {"match": "all", "rules": [{"field": "flag", "op": "is", "value": "pick"}]}}),+ json!({"ruleSet": {"match": "any", "rules": [{"field": "rating", "op": "gte", "value": 4}, {"field": "keywords", "op": "contains", "value": "pets"}]}}),+ json!({"album": album, "rating": 3}),+ // what the edits below change besides: a label, a slider, when a photo was taken, what its file is called+ json!({"label": "red"}),+ json!({"edited": true}),+ json!({"dateFrom": "2025-06-01"}),+ json!({"text": "REN"}),+ ] {+ let made = s.execute("album.createSmart", &json!({"name": "More", "rules": rules}));+ assert!(made.is_ok(), "{rules}: {made:?}");+ }+ assert!(s.catalog.album(lightcraft_catalog::AlbumId(smart)).is_some_and(|a| a.is_smart()));+ let fresh = |s: &Session| -> Vec<(u64, usize)> {+ let mut v: Vec<(u64, usize)> = s.catalog.albums().map(|a| (a.id.0, s.catalog.album_count(a.id))).collect();+ v.sort_unstable();+ v+ };+ let shown = |tally: &AlbumTally| -> Vec<(u64, usize)> {+ let mut v: Vec<(u64, usize)> = tally.counts().iter().map(|(id, n)| (id.0, *n)).collect();+ v.sort_unstable();+ v+ };+ let mut tally = AlbumTally::of(&s.catalog);+ assert_eq!(shown(&tally), fresh(&s));+ let mut dice = Dice(23);+ let (mut followed, mut whole) = (0, 0);+ for step in 0..900 {+ edit(&mut s, &mut dice, &mut ids, album);+ // (not every frame draws the albums: sometimes several edits pass between two looks)+ if dice.roll(4) == 0 {+ continue;+ }+ let moved = tally.revision() != s.catalog.revision;+ match (moved, tally.update(&s.catalog)) {+ (true, true) => followed += 1,+ (true, false) => whole += 1,+ // (an edit that was refused changed nothing: nothing to follow)+ (false, _) => {}+ }+ assert_eq!(shown(&tally), fresh(&s), "after edit {step}");+ }+ assert!(followed > 300 && whole > 20, "followed {followed} times, counted whole {whole} times");+ // a photo deleted: the album of chosen photos it is in counts one less, without every photo being asked+ let in_album = s+ .catalog+ .album(lightcraft_catalog::AlbumId(album))+ .and_then(|a| a.photos.iter().copied().find(|id| s.catalog.photo(*id).is_some_and(|p| !p.deleted)));+ if let Some(id) = in_album {+ let before = tally.counts()[&lightcraft_catalog::AlbumId(album)];+ s.execute("library.select", &json!({"ids": [id.0]})).unwrap();+ s.execute("photo.delete", &json!({})).unwrap();+ assert!(tally.update(&s.catalog), "followed");+ assert_eq!(tally.counts()[&lightcraft_catalog::AlbumId(album)], before - 1);+ }+ // three hundred photos rated at once: counted from every photo, and right+ let many: Vec<u64> = s.visible_cloned().iter().take(300).map(|id| id.0).collect();+ s.execute("library.source", &json!({"kind": "all"})).unwrap();+ s.execute("library.filter", &json!({"rating": 0, "text": ""})).unwrap();+ let many: Vec<u64> = if many.len() >= 300 { many } else { s.visible_cloned().iter().take(300).map(|id| id.0).collect() };+ if many.len() > 256 {+ s.execute("library.select", &json!({"ids": many})).unwrap();+ s.execute("photo.rate", &json!({"rating": 5})).unwrap();+ assert!(!tally.update(&s.catalog), "more photos than are followed one by one");+ assert_eq!(shown(&tally), fresh(&s));+ }+}diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rsindex b2fc4975e..f4ac68a00 100644--- a/crates/ui-egui/src/lib.rs+++ b/crates/ui-egui/src/lib.rs@@ -1525,8 +1525,12 @@ pub struct Caches {counts: Option<(u64, LibraryCounts)>,date_groups: Option<(u64, std::sync::Arc<Vec<lightcraft_catalog::DateGroup>>)>,filter_values: Option<(u64, std::sync::Arc<FilterValues>)>,- album_counts: Option<(u64, std::sync::Arc<std::collections::HashMap<lightcraft_catalog::AlbumId, usize>>)>,- /// How often the album counts were recomputed (tests check that unchanged frames don't).+ /// The albums' counts: the minute they are of (while a smart album asks about "the last N days"), the tally+ /// that follows edits, and the counts as the panels take them.+ album_counts:+ Option<(String, lightcraft_catalog::tally::AlbumTally, std::sync::Arc<std::collections::HashMap<lightcraft_catalog::AlbumId, usize>>)>,+ /// How often the album counts were made from every photo (tests check that unchanged frames, and edits to a+ /// photo or a few, don't).pub album_count_scans: usize,/// What the open smart-album rule dialog shows besides the rules ([`Caches::rules_view`]).rules_view: Option<(u64, std::sync::Arc<RulesView>)>,@@ -1829,21 +1833,37 @@ impl Caches {) -> std::sync::Arc<std::collections::HashMap<lightcraft_catalog::AlbumId, usize>> {let relative = cat.albums().any(|a| a.smart.as_ref().is_some_and(|f| f.depends_on_now()));let minute = if relative { now.get(..16).unwrap_or(now) } else { "" };- let k = key_of((cat.revision, minute));- match &self.album_counts {- Some((h, v)) if *h == k => v.clone(),- _ => {- if relative {- // "in the last N days" counts back from the session clock, as in the grid- lightcraft_catalog::rules::set_now(Some(now.to_string()));+ // the counts of this minute, if there are any: an edit to a photo or a few is followed from them, a smart+ // album asking only those photos again; anything more, or a new minute, is counted from every photo+ let kept = self.album_counts.take().filter(|(of, _, _)| of == minute);+ if let Some((_, tally, shown)) = &kept+ && tally.revision() == cat.revision+ {+ let shown = shown.clone();+ self.album_counts = kept;+ return shown;+ }+ if relative {+ // "in the last N days" counts back from the session clock, as in the grid. (Within a minute an edit asks+ // only the photos it touched, at this second; the others were asked earlier in the minute. That is as+ // far out as the counts already were between edits, and the next minute counts them all again.)+ lightcraft_catalog::rules::set_now(Some(now.to_string()));+ }+ let tally = match kept {+ Some((_, mut tally, _)) => {+ if !tally.update(cat) {+ self.album_count_scans += 1;}- let v: std::sync::Arc<std::collections::HashMap<_, _>> =- std::sync::Arc::new(cat.albums().map(|a| (a.id, cat.album_count(a.id))).collect());+ tally+ }+ None => {self.album_count_scans += 1;- self.album_counts = Some((k, v.clone()));- v+ lightcraft_catalog::tally::AlbumTally::of(cat)}- }+ };+ let shown = std::sync::Arc::new(tally.counts().clone());+ self.album_counts = Some((minute.to_string(), tally, shown.clone()));+ shown}}@@ -1884,16 +1904,17 @@ mod cache_tests {c.album_counts(&cat, "2026-09-30T12:00:40");}assert_eq!(c.album_count_scans, 1, "unchanged frames in the same minute don't rescan");- // metadata change+ // a photo rated: followed from that photo, not counted from every onecat.apply(Op::SetRating { id: PhotoId(2), rating: 4 }).unwrap();assert_eq!(c.album_counts(&cat, now)[&AlbumId(10)], 2);+ assert_eq!(c.album_count_scans, 1);// rule changelet rs: RuleSet = serde_json::from_value(serde_json::json!({"rules": [{"field": "rating", "op": "gte", "value": 5}]})).unwrap();cat.apply(Op::SetAlbumRules { id: AlbumId(10), rules: Box::new(Filter { rule_set: Some(rs), ..Default::default() }) }).unwrap();assert_eq!(c.album_counts(&cat, now)[&AlbumId(10)], 1);// an hour later the 11:59:30 photo has left "in the last hour", with no catalog changeassert_eq!(c.album_counts(&cat, "2026-09-30T13:00:10")[&AlbumId(11)], 0);- assert_eq!(c.album_count_scans, 4);+ assert_eq!(c.album_count_scans, 3, "new rules and a new minute are counted from every photo; the rating was not");lightcraft_catalog::rules::set_now(None);}diff --git a/docs/whats-new.md b/docs/whats-new.mdindex a0699f77a..654b285ad 100644--- a/docs/whats-new.md+++ b/docs/whats-new.md@@ -19,8 +19,12 @@keywords, dates, people and counts; a keyword leaves the folders, the dates and the counts as they are. With bothside panels open, the frame after a keyword went from about 156 ms to about 54 ms in that library (2 ms with thepanels closed, from 71 ms).-- Still tallied from every photo on each edit: the count beside each album and smart album in the left panel (about- 25 ms for a smart album in that library), and the filter bar's lists of cameras, lenses and keywords.+- The count beside each smart album in the left panel follows an edit the same way: the album asks only the photos+ the edit touched, where it asked every photo of the library after each edit (about 25 ms an album in that library).+ With that, the frame after a keyword is at about 29 ms there. New rules, a change to an album or the keyword list,+ an edit to more than 256 photos, and each new minute for an album that asks about "the last N days" are still+ counted from every photo.+- Still tallied from every photo on each edit: the filter bar's lists of cameras, lenses and keywords.### A view can hold its photos until you refresh it- View ▸ Hold View Until Refreshed (off unless you turn it on; kept with the library). With it on, an edit that makes