Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

PERF-EDIT: a smart album's count follows an edit, asking only the photos it touched

Nathaniel Ledford43ba500c6e24
diff --git a/crates/catalog/src/lib.rs b/crates/catalog/src/lib.rs
index aa2f5745b..79b02f7dc 100644
--- a/crates/catalog/src/lib.rs
+++ b/crates/catalog/src/lib.rs
@@ -25,6 +25,7 @@ pub mod rules;
pub mod safe_file;
pub mod stacks;
pub mod store;
+pub mod tally;
use std::collections::BTreeMap;
use std::sync::Arc;
diff --git a/crates/catalog/src/tally.rs b/crates/catalog/src/tally.rs
new file mode 100644
index 000000000..8f15e5ec4
--- /dev/null
+++ b/crates/catalog/src/tally.rs
@@ -0,0 +1,234 @@
+//! How many photos each album shows, kept up to date from what the latest ops touched.
+//!
+//! A smart album's count asks every photo whether its rules hold. Asked again after every edit, that is a pass over
+//! the library for each smart album each time a keyword is given. [`AlbumTally`] remembers which photos each smart
+//! album holds, and after a small edit asks only the photos the edit touched ([`Catalog::touched_since`]). An op
+//! that changes more than photos of its own (an album's rules or photos, a stack, the keyword list) has it counted
+//! whole again, as before.
+
+use std::collections::{HashMap, HashSet};
+
+use crate::{AlbumId, Aspects, Catalog, PhotoId};
+
+/// The most photos an edit may touch for the counts to follow it photo by photo.
+const MAX_FOLLOWED: usize = 256;
+/// Bits are kept while they cost no more than this many words a photo held, and these to start with. Past that
+/// (photo ids far apart: a library whose ids don't count up from 1) a set is kept instead, so what an album
+/// remembers is always in proportion to the photos it holds, never to the largest id in the library.
+const WORDS_A_MEMBER: usize = 4;
+const WORDS_TO_START: usize = 1024;
+
+/// The photos a smart album holds.
+#[derive(Clone, Debug)]
+enum Members {
+ /// One bit a photo id, and how many are set.
+ Bits {
+ words: Vec<u64>,
+ len: usize,
+ },
+ Set(HashSet<PhotoId>),
+}
+
+impl Members {
+ fn none() -> Members {
+ Members::Bits { words: Vec::new(), len: 0 }
+ }
+
+ fn len(&self) -> usize {
+ match self {
+ Members::Bits { len, .. } => *len,
+ Members::Set(set) => set.len(),
+ }
+ }
+
+ #[cfg(test)]
+ fn holds(&self, id: PhotoId) -> bool {
+ match self {
+ Members::Bits { words, .. } => {
+ usize::try_from(id.0 / 64).ok().and_then(|at| words.get(at)).is_some_and(|word| word & (1u64 << (id.0 % 64)) != 0)
+ }
+ Members::Set(set) => set.contains(&id),
+ }
+ }
+
+ /// Put `id` in or take it out.
+ fn set(&mut self, id: PhotoId, held: bool) {
+ if let Members::Bits { words, len } = self {
+ let bit = 1u64 << (id.0 % 64);
+ match usize::try_from(id.0 / 64) {
+ Ok(at) if at < words.len() => {
+ if let Some(word) = words.get_mut(at) {
+ match (*word & bit != 0, held) {
+ (false, true) => {
+ *word |= bit;
+ *len = len.saturating_add(1);
+ }
+ (true, false) => {
+ *word &= !bit;
+ *len = len.saturating_sub(1);
+ }
+ _ => {}
+ }
+ }
+ return;
+ }
+ // (past the end: not held, so there is nothing to take out)
+ _ if !held => return,
+ // room for it, as long as the bits stay in proportion to the photos held
+ Ok(at) if at < len.saturating_mul(WORDS_A_MEMBER).saturating_add(WORDS_TO_START) => {
+ words.resize(at + 1, 0);
+ if let Some(word) = words.get_mut(at) {
+ *word |= bit;
+ *len = len.saturating_add(1);
+ }
+ return;
+ }
+ // ids too far apart for bits: a set from here on
+ _ => {
+ let ids = words
+ .iter()
+ .enumerate()
+ .flat_map(|(at, word)| (0..64u64).filter(move |b| word & (1u64 << b) != 0).map(move |b| PhotoId(at as u64 * 64 + b)));
+ *self = Members::Set(ids.collect());
+ }
+ }
+ }
+ if let Members::Set(set) = self {
+ if held {
+ set.insert(id);
+ } else {
+ set.remove(&id);
+ }
+ }
+ }
+}
+
+/// [`Catalog::album_count`] of every album, as of one revision of the catalog.
+#[derive(Clone, Debug)]
+pub struct AlbumTally {
+ revision: u64,
+ counts: HashMap<AlbumId, usize>,
+ /// The smart albums' photos.
+ members: HashMap<AlbumId, Members>,
+}
+
+impl AlbumTally {
+ /// Counted from every photo.
+ pub fn of(cat: &Catalog) -> AlbumTally {
+ let mut counts = HashMap::new();
+ let mut members = HashMap::new();
+ for album in cat.albums() {
+ if album.is_smart() {
+ let mut held = Members::none();
+ for p in cat.photos().filter(|p| cat.album_contains(album.id, p)) {
+ held.set(p.id, true);
+ }
+ counts.insert(album.id, held.len());
+ members.insert(album.id, held);
+ } else {
+ counts.insert(album.id, cat.album_count(album.id));
+ }
+ }
+ AlbumTally { revision: cat.revision, counts, members }
+ }
+
+ /// The count of each album.
+ pub fn counts(&self) -> &HashMap<AlbumId, usize> {
+ &self.counts
+ }
+
+ /// The catalog's revision the counts are of.
+ pub fn revision(&self) -> u64 {
+ self.revision
+ }
+
+ /// Bring the counts up to `cat` as it is now. `true` when that was done from the photos the ops since touched
+ /// (or nothing had changed), `false` when every photo was asked again.
+ ///
+ /// `cat` has to be the catalog the counts were made of, further along. Another catalog is told apart only by
+ /// its revisions: a library that is loaded takes a range of its own (and then everything is counted again), but
+ /// two catalogs both counting from 0 look like one.
+ pub fn update(&mut self, cat: &Catalog) -> bool {
+ if self.revision == cat.revision {
+ return true;
+ }
+ let followed = match (cat.touched_since(self.revision), cat.aspects_since(self.revision)) {
+ (Some(touched), Some(changed)) if touched.len() <= MAX_FOLLOWED => {
+ self.follow(cat, &touched, changed);
+ true
+ }
+ _ => false,
+ };
+ if !followed {
+ *self = AlbumTally::of(cat);
+ }
+ followed
+ }
+
+ /// The counts after ops that touched only `touched`: each smart album asks those photos again; an album of
+ /// chosen photos counts again only when a photo came, went or was deleted (who is in it did not change: that
+ /// is an op on the album, and not followed).
+ fn follow(&mut self, cat: &Catalog, touched: &[PhotoId], changed: Aspects) {
+ for (album, held) in &mut self.members {
+ for id in touched {
+ held.set(*id, cat.photo(*id).is_some_and(|p| cat.album_contains(*album, p)));
+ }
+ self.counts.insert(*album, held.len());
+ }
+ if changed.any_of(Aspects::PRESENCE) {
+ for album in cat.albums().filter(|a| !a.is_smart()) {
+ self.counts.insert(album.id, cat.album_count(album.id));
+ }
+ }
+ self.revision = cat.revision;
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn ids(m: &Members, of: &[u64]) -> Vec<u64> {
+ of.iter().copied().filter(|id| m.holds(PhotoId(*id))).collect()
+ }
+
+ #[test]
+ fn members_are_kept_as_bits_at_the_edges_of_a_word_and_past_the_end() {
+ let mut m = Members::none();
+ let probe = [0, 1, 63, 64, 65, 127, 128, 5000];
+ assert!(ids(&m, &probe).is_empty() && m.len() == 0);
+ for id in [0, 63, 64, 5000] {
+ m.set(PhotoId(id), true);
+ }
+ assert_eq!((ids(&m, &probe), m.len()), (vec![0, 63, 64, 5000], 4));
+ // put in twice, taken out twice, taken out where nothing is, far past the end: counted once each way
+ m.set(PhotoId(63), true);
+ m.set(PhotoId(64), false);
+ m.set(PhotoId(64), false);
+ m.set(PhotoId(1), false);
+ m.set(PhotoId(900_000), false);
+ assert_eq!((ids(&m, &probe), m.len()), (vec![0, 63, 5000], 3));
+ assert!(matches!(m, Members::Bits { .. }));
+ }
+
+ #[test]
+ fn photo_ids_far_apart_are_kept_as_a_set_in_proportion_to_the_photos_held() {
+ // one photo with an id in the thousands of millions: no bit for every id up to it
+ let mut m = Members::none();
+ for id in [3, 70, 9_000_000_000] {
+ m.set(PhotoId(id), true);
+ }
+ assert!(matches!(m, Members::Set(_)), "a set once the ids are too far apart for bits");
+ assert_eq!((ids(&m, &[3, 70, 71, 9_000_000_000, u64::MAX]), m.len()), (vec![3, 70, 9_000_000_000], 3));
+ m.set(PhotoId(u64::MAX), true);
+ m.set(PhotoId(3), false);
+ assert_eq!((ids(&m, &[3, 70, 9_000_000_000, u64::MAX]), m.len()), (vec![70, 9_000_000_000, u64::MAX], 3));
+ // an album of many photos with ids counting up stays bits, however many
+ let mut dense = Members::none();
+ for id in 1..=200_000u64 {
+ dense.set(PhotoId(id), true);
+ }
+ assert!(matches!(&dense, Members::Bits { words, .. } if words.len() <= 200_000 / 64 + 1));
+ assert_eq!(dense.len(), 200_000);
+ }
+}
diff --git a/crates/engine/src/tests_view_follow.rs b/crates/engine/src/tests_view_follow.rs
index bf0656f86..391e38b3b 100644
--- a/crates/engine/src/tests_view_follow.rs
+++ b/crates/engine/src/tests_view_follow.rs
@@ -219,3 +219,77 @@ fn an_edit_to_many_photos_or_to_more_than_photos_is_worked_out_whole() {
let _ = s.visible_cloned();
assert_eq!((s.view_followed, s.view_worked_out), (followed + 1, whole + 2));
}
+
+#[test]
+fn the_albums_counts_followed_through_edits_are_the_counts_made_from_every_photo() {
+ use lightcraft_catalog::tally::AlbumTally;
+ let (mut s, mut ids, album, smart) = library(240);
+ // more smart albums: picks, well rated or keyworded ones, one that looks into an album of chosen photos
+ for rules in [
+ json!({"ruleSet": {"match": "all", "rules": [{"field": "flag", "op": "is", "value": "pick"}]}}),
+ json!({"ruleSet": {"match": "any", "rules": [{"field": "rating", "op": "gte", "value": 4}, {"field": "keywords", "op": "contains", "value": "pets"}]}}),
+ json!({"album": album, "rating": 3}),
+ // what the edits below change besides: a label, a slider, when a photo was taken, what its file is called
+ json!({"label": "red"}),
+ json!({"edited": true}),
+ json!({"dateFrom": "2025-06-01"}),
+ json!({"text": "REN"}),
+ ] {
+ let made = s.execute("album.createSmart", &json!({"name": "More", "rules": rules}));
+ assert!(made.is_ok(), "{rules}: {made:?}");
+ }
+ assert!(s.catalog.album(lightcraft_catalog::AlbumId(smart)).is_some_and(|a| a.is_smart()));
+ let fresh = |s: &Session| -> Vec<(u64, usize)> {
+ let mut v: Vec<(u64, usize)> = s.catalog.albums().map(|a| (a.id.0, s.catalog.album_count(a.id))).collect();
+ v.sort_unstable();
+ v
+ };
+ let shown = |tally: &AlbumTally| -> Vec<(u64, usize)> {
+ let mut v: Vec<(u64, usize)> = tally.counts().iter().map(|(id, n)| (id.0, *n)).collect();
+ v.sort_unstable();
+ v
+ };
+ let mut tally = AlbumTally::of(&s.catalog);
+ assert_eq!(shown(&tally), fresh(&s));
+ let mut dice = Dice(23);
+ let (mut followed, mut whole) = (0, 0);
+ for step in 0..900 {
+ edit(&mut s, &mut dice, &mut ids, album);
+ // (not every frame draws the albums: sometimes several edits pass between two looks)
+ if dice.roll(4) == 0 {
+ continue;
+ }
+ let moved = tally.revision() != s.catalog.revision;
+ match (moved, tally.update(&s.catalog)) {
+ (true, true) => followed += 1,
+ (true, false) => whole += 1,
+ // (an edit that was refused changed nothing: nothing to follow)
+ (false, _) => {}
+ }
+ assert_eq!(shown(&tally), fresh(&s), "after edit {step}");
+ }
+ assert!(followed > 300 && whole > 20, "followed {followed} times, counted whole {whole} times");
+ // a photo deleted: the album of chosen photos it is in counts one less, without every photo being asked
+ let in_album = s
+ .catalog
+ .album(lightcraft_catalog::AlbumId(album))
+ .and_then(|a| a.photos.iter().copied().find(|id| s.catalog.photo(*id).is_some_and(|p| !p.deleted)));
+ if let Some(id) = in_album {
+ let before = tally.counts()[&lightcraft_catalog::AlbumId(album)];
+ s.execute("library.select", &json!({"ids": [id.0]})).unwrap();
+ s.execute("photo.delete", &json!({})).unwrap();
+ assert!(tally.update(&s.catalog), "followed");
+ assert_eq!(tally.counts()[&lightcraft_catalog::AlbumId(album)], before - 1);
+ }
+ // three hundred photos rated at once: counted from every photo, and right
+ let many: Vec<u64> = s.visible_cloned().iter().take(300).map(|id| id.0).collect();
+ s.execute("library.source", &json!({"kind": "all"})).unwrap();
+ s.execute("library.filter", &json!({"rating": 0, "text": ""})).unwrap();
+ let many: Vec<u64> = if many.len() >= 300 { many } else { s.visible_cloned().iter().take(300).map(|id| id.0).collect() };
+ if many.len() > 256 {
+ s.execute("library.select", &json!({"ids": many})).unwrap();
+ s.execute("photo.rate", &json!({"rating": 5})).unwrap();
+ assert!(!tally.update(&s.catalog), "more photos than are followed one by one");
+ assert_eq!(shown(&tally), fresh(&s));
+ }
+}
diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs
index b2fc4975e..f4ac68a00 100644
--- a/crates/ui-egui/src/lib.rs
+++ b/crates/ui-egui/src/lib.rs
@@ -1525,8 +1525,12 @@ pub struct Caches {
counts: Option<(u64, LibraryCounts)>,
date_groups: Option<(u64, std::sync::Arc<Vec<lightcraft_catalog::DateGroup>>)>,
filter_values: Option<(u64, std::sync::Arc<FilterValues>)>,
- album_counts: Option<(u64, std::sync::Arc<std::collections::HashMap<lightcraft_catalog::AlbumId, usize>>)>,
- /// How often the album counts were recomputed (tests check that unchanged frames don't).
+ /// The albums' counts: the minute they are of (while a smart album asks about "the last N days"), the tally
+ /// that follows edits, and the counts as the panels take them.
+ album_counts:
+ Option<(String, lightcraft_catalog::tally::AlbumTally, std::sync::Arc<std::collections::HashMap<lightcraft_catalog::AlbumId, usize>>)>,
+ /// How often the album counts were made from every photo (tests check that unchanged frames, and edits to a
+ /// photo or a few, don't).
pub album_count_scans: usize,
/// What the open smart-album rule dialog shows besides the rules ([`Caches::rules_view`]).
rules_view: Option<(u64, std::sync::Arc<RulesView>)>,
@@ -1829,21 +1833,37 @@ impl Caches {
) -> std::sync::Arc<std::collections::HashMap<lightcraft_catalog::AlbumId, usize>> {
let relative = cat.albums().any(|a| a.smart.as_ref().is_some_and(|f| f.depends_on_now()));
let minute = if relative { now.get(..16).unwrap_or(now) } else { "" };
- let k = key_of((cat.revision, minute));
- match &self.album_counts {
- Some((h, v)) if *h == k => v.clone(),
- _ => {
- if relative {
- // "in the last N days" counts back from the session clock, as in the grid
- lightcraft_catalog::rules::set_now(Some(now.to_string()));
+ // the counts of this minute, if there are any: an edit to a photo or a few is followed from them, a smart
+ // album asking only those photos again; anything more, or a new minute, is counted from every photo
+ let kept = self.album_counts.take().filter(|(of, _, _)| of == minute);
+ if let Some((_, tally, shown)) = &kept
+ && tally.revision() == cat.revision
+ {
+ let shown = shown.clone();
+ self.album_counts = kept;
+ return shown;
+ }
+ if relative {
+ // "in the last N days" counts back from the session clock, as in the grid. (Within a minute an edit asks
+ // only the photos it touched, at this second; the others were asked earlier in the minute. That is as
+ // far out as the counts already were between edits, and the next minute counts them all again.)
+ lightcraft_catalog::rules::set_now(Some(now.to_string()));
+ }
+ let tally = match kept {
+ Some((_, mut tally, _)) => {
+ if !tally.update(cat) {
+ self.album_count_scans += 1;
}
- let v: std::sync::Arc<std::collections::HashMap<_, _>> =
- std::sync::Arc::new(cat.albums().map(|a| (a.id, cat.album_count(a.id))).collect());
+ tally
+ }
+ None => {
self.album_count_scans += 1;
- self.album_counts = Some((k, v.clone()));
- v
+ lightcraft_catalog::tally::AlbumTally::of(cat)
}
- }
+ };
+ let shown = std::sync::Arc::new(tally.counts().clone());
+ self.album_counts = Some((minute.to_string(), tally, shown.clone()));
+ shown
}
}
@@ -1884,16 +1904,17 @@ mod cache_tests {
c.album_counts(&cat, "2026-09-30T12:00:40");
}
assert_eq!(c.album_count_scans, 1, "unchanged frames in the same minute don't rescan");
- // metadata change
+ // a photo rated: followed from that photo, not counted from every one
cat.apply(Op::SetRating { id: PhotoId(2), rating: 4 }).unwrap();
assert_eq!(c.album_counts(&cat, now)[&AlbumId(10)], 2);
+ assert_eq!(c.album_count_scans, 1);
// rule change
let rs: RuleSet = serde_json::from_value(serde_json::json!({"rules": [{"field": "rating", "op": "gte", "value": 5}]})).unwrap();
cat.apply(Op::SetAlbumRules { id: AlbumId(10), rules: Box::new(Filter { rule_set: Some(rs), ..Default::default() }) }).unwrap();
assert_eq!(c.album_counts(&cat, now)[&AlbumId(10)], 1);
// an hour later the 11:59:30 photo has left "in the last hour", with no catalog change
assert_eq!(c.album_counts(&cat, "2026-09-30T13:00:10")[&AlbumId(11)], 0);
- assert_eq!(c.album_count_scans, 4);
+ assert_eq!(c.album_count_scans, 3, "new rules and a new minute are counted from every photo; the rating was not");
lightcraft_catalog::rules::set_now(None);
}
diff --git a/docs/whats-new.md b/docs/whats-new.md
index a0699f77a..654b285ad 100644
--- a/docs/whats-new.md
+++ b/docs/whats-new.md
@@ -19,8 +19,12 @@
keywords, dates, people and counts; a keyword leaves the folders, the dates and the counts as they are. With both
side panels open, the frame after a keyword went from about 156 ms to about 54 ms in that library (2 ms with the
panels closed, from 71 ms).
-- Still tallied from every photo on each edit: the count beside each album and smart album in the left panel (about
- 25 ms for a smart album in that library), and the filter bar's lists of cameras, lenses and keywords.
+- The count beside each smart album in the left panel follows an edit the same way: the album asks only the photos
+ the edit touched, where it asked every photo of the library after each edit (about 25 ms an album in that library).
+ With that, the frame after a keyword is at about 29 ms there. New rules, a change to an album or the keyword list,
+ an edit to more than 256 photos, and each new minute for an album that asks about "the last N days" are still
+ counted from every photo.
+- Still tallied from every photo on each edit: the filter bar's lists of cameras, lenses and keywords.
### A view can hold its photos until you refresh it
- View ▸ Hold View Until Refreshed (off unless you turn it on; kept with the library). With it on, an edit that makes