LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
PERF-SMART: count every smart album once per pass, not once per album that tests it
Nathaniel Ledford6b8cb9826c61
diff --git a/crates/catalog/src/lib.rs b/crates/catalog/src/lib.rsindex 1776c895c..a9b790811 100644--- a/crates/catalog/src/lib.rs+++ b/crates/catalog/src/lib.rs@@ -1164,4 +1164,6 @@ mod tests_repair;#[cfg(test)]mod tests_scale;#[cfg(test)]+mod tests_tally;+#[cfg(test)]mod tests_torn_append;diff --git a/crates/catalog/src/rules.rs b/crates/catalog/src/rules.rsindex 667509d6f..124125232 100644--- a/crates/catalog/src/rules.rs+++ b/crates/catalog/src/rules.rs@@ -432,12 +432,79 @@ thread_local! {std::cell::RefCell::new(std::collections::HashMap::new());}+thread_local! {+ /// While [`AlbumPass`] is alive on this thread: the photos of each smart album that was counted whole, with+ /// the longest chain of smart albums (itself included) its rules lead through.+ static PASS: std::cell::RefCell<Option<std::collections::HashMap<crate::AlbumId, (usize, crate::tally::Members)>>> =+ const { std::cell::RefCell::new(None) };+}++/// One pass over one catalog state that counts every smart album whole. Each album put in with `remember`+/// answers [`smart_album_holds`] from its photos, not from its rules, until the pass is dropped (it can't outlive+/// the state it was made of: it exists only inside the counting function).+///+/// Only an album whose rules lead to no loop may be remembered, and only with the height of its chain: its answer+/// then doesn't depend on which albums are being evaluated around it, as long as the chain still fits under+/// [`MAX_ALBUM_DEPTH`] from where it is asked (checked on every use; otherwise the rules are evaluated as before).+pub(crate) struct AlbumPass {+ outer: Option<std::collections::HashMap<crate::AlbumId, (usize, crate::tally::Members)>>,+}++impl AlbumPass {+ pub(crate) fn begin() -> AlbumPass {+ AlbumPass { outer: PASS.with_borrow_mut(|p| p.replace(std::collections::HashMap::new())) }+ }++ /// `id` holds exactly `members`, and its rules lead through a chain of `height` smart albums (itself included),+ /// none of them in a loop.+ pub(crate) fn remember(&self, id: crate::AlbumId, height: usize, members: crate::tally::Members) {+ PASS.with_borrow_mut(|p| {+ if let Some(p) = p {+ p.insert(id, (height, members));+ }+ });+ }++ /// Ends the pass, giving back what was remembered.+ pub(crate) fn finish(self) -> std::collections::HashMap<crate::AlbumId, crate::tally::Members> {+ PASS.with_borrow_mut(Option::take).unwrap_or_default().into_iter().map(|(id, (_, members))| (id, members)).collect()+ }+}++impl Drop for AlbumPass {+ fn drop(&mut self) {+ let outer = self.outer.take();+ PASS.with_borrow_mut(|p| *p = outer);+ }+}++/// What a counting pass knows of smart album `id` for `photo`, when asking now can't give another answer than+/// evaluating its rules would: the chain below it fits under [`MAX_ALBUM_DEPTH`] from the albums being evaluated.+fn pass_answer(id: crate::AlbumId, photo: crate::PhotoId) -> Option<bool> {+ PASS.with_borrow(|p| {+ let (height, members) = p.as_ref()?.get(&id)?;+ let depth = EVALUATING.with_borrow(Vec::len);+ (depth + height <= MAX_ALBUM_DEPTH).then(|| members.holds(photo))+ })+}++#[cfg(test)]+thread_local! {+ /// How many times a smart album's rules were worked out on this thread (tests count them).+ pub(crate) static EVALUATED: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };+}+/// Whether `photo` is in smart album `id`, by `matches` (its rules), remembered for the rest of the/// outermost question and guarded by [`evaluating`] (a loop or a chain too deep holds nothing).pub(crate) fn smart_album_holds(id: crate::AlbumId, photo: crate::PhotoId, matches: impl FnOnce() -> bool) -> bool {+ if let Some(known) = pass_answer(id, photo) {+ return known;+ }if let Some(known) = ANSWERS.with_borrow(|a| a.get(&(id, photo)).copied()) {return known;}+ #[cfg(test)]+ EVALUATED.with(|n| n.set(n.get() + 1));// the outermost question forgets the answers when it ends, however it endsstruct Forget(bool);impl Drop for Forget {@@ -456,7 +523,7 @@ pub(crate) fn smart_album_holds(id: crate::AlbumId, photo: crate::PhotoId, match}/// How deep smart albums may test smart albums before the innermost counts as holding nothing.-const MAX_ALBUM_DEPTH: usize = 32;+pub(crate) const MAX_ALBUM_DEPTH: usize = 32;/// Run `f` as the evaluation of smart album `id`'s rules: `None` (so it holds nothing) when `id`/// is already being evaluated (a loop the check would refuse, saved anyway) or the chain isdiff --git a/crates/catalog/src/tally.rs b/crates/catalog/src/tally.rsindex 8f15e5ec4..e8fdc6197 100644--- a/crates/catalog/src/tally.rs+++ b/crates/catalog/src/tally.rs@@ -8,7 +8,9 @@use std::collections::{HashMap, HashSet};-use crate::{AlbumId, Aspects, Catalog, PhotoId};+use crate::query::Filter;+use crate::rules::{AlbumPass, MAX_ALBUM_DEPTH};+use crate::{Album, AlbumId, Aspects, Catalog, PhotoId};/// The most photos an edit may touch for the counts to follow it photo by photo.const MAX_FOLLOWED: usize = 256;@@ -20,7 +22,7 @@ const WORDS_TO_START: usize = 1024;/// The photos a smart album holds.#[derive(Clone, Debug)]-enum Members {+pub(crate) enum Members {/// One bit a photo id, and how many are set.Bits {words: Vec<u64>,@@ -41,8 +43,7 @@ impl Members {}}- #[cfg(test)]- fn holds(&self, id: PhotoId) -> bool {+ pub(crate) fn holds(&self, id: PhotoId) -> bool {match self {Members::Bits { words, .. } => {usize::try_from(id.0 / 64).ok().and_then(|at| words.get(at)).is_some_and(|word| word & (1u64 << (id.0 % 64)) != 0)@@ -114,21 +115,46 @@ pub struct AlbumTally {impl AlbumTally {/// Counted from every photo.+ ///+ /// A smart album that tests another (`album is #14`) reads that album's photos, counted once for the whole+ /// pass, instead of working its rules out again for every photo: albums are counted shortest chain first.+ /// The photos are kept only until this function returns, so they can't be older than the catalog counted.+ /// An album whose rules lead into a loop, or through a chain too deep to be sure of, is counted by its rules+ /// as before: the pass ends before the first of them (they sort last), so nothing they evaluate reads it.pub fn of(cat: &Catalog) -> AlbumTally {+ let heights = chain_heights(cat);+ let height_of = |id: AlbumId| heights.get(&id).copied().flatten().filter(|h| *h <= MAX_ALBUM_DEPTH);+ let mut smart: Vec<AlbumId> = cat.albums().filter(|a| a.is_smart()).map(|a| a.id).collect();+ smart.sort_by_key(|id| height_of(*id).unwrap_or(usize::MAX));+ let held_by = |id: AlbumId| {+ let mut held = Members::none();+ for p in cat.photos().filter(|p| cat.album_contains(id, p)) {+ held.set(p.id, true);+ }+ held+ };+let mut counts = HashMap::new();let mut members = HashMap::new();- for album in cat.albums() {- if album.is_smart() {- let mut held = Members::none();- for p in cat.photos().filter(|p| cat.album_contains(album.id, p)) {- held.set(p.id, true);- }- counts.insert(album.id, held.len());- members.insert(album.id, held);- } else {- counts.insert(album.id, cat.album_count(album.id));+ let split = smart.partition_point(|id| height_of(*id).is_some());+ let (with_height, without) = smart.split_at(split);+ let pass = AlbumPass::begin();+ for id in with_height {+ let held = held_by(*id);+ counts.insert(*id, held.len());+ if let Some(height) = height_of(*id) {+ pass.remember(*id, height, held);}}+ members.extend(pass.finish());+ for id in without {+ let held = held_by(*id);+ counts.insert(*id, held.len());+ members.insert(*id, held);+ }+ for album in cat.albums().filter(|a| !a.is_smart()) {+ counts.insert(album.id, cat.album_count(album.id));+ }AlbumTally { revision: cat.revision, counts, members }}@@ -184,6 +210,57 @@ impl AlbumTally {}}+/// For each smart album, the most smart albums in a chain its rules lead through (itself included), or `None`+/// when a loop is reachable from it (its answer may then depend on which albums are being evaluated around it).+fn chain_heights(cat: &Catalog) -> HashMap<AlbumId, Option<usize>> {+ struct Frame {+ id: AlbumId,+ tests: Vec<AlbumId>,+ next: usize,+ tallest: usize,+ loops: bool,+ }+ let tests = |id: AlbumId| -> Vec<AlbumId> {+ let tested = cat.album(id).and_then(|a| a.smart.as_deref()).map(Filter::albums_tested).unwrap_or_default();+ tested.into_iter().filter(|t| cat.album(*t).is_some_and(Album::is_smart)).collect()+ };+ let frame = |id: AlbumId| Frame { id, tests: tests(id), next: 0, tallest: 0, loops: false };+ let mut done: HashMap<AlbumId, Option<usize>> = HashMap::new();+ let mut open: HashSet<AlbumId> = HashSet::new();+ for root in cat.albums().filter(|a| a.is_smart()).map(|a| a.id) {+ if done.contains_key(&root) {+ continue;+ }+ open.insert(root);+ let mut stack = vec![frame(root)];+ while let Some(top) = stack.last_mut() {+ if let Some(&kid) = top.tests.get(top.next) {+ top.next += 1;+ match done.get(&kid) {+ Some(Some(h)) => top.tallest = top.tallest.max(*h),+ Some(None) => top.loops = true,+ None if open.contains(&kid) => top.loops = true,+ None => {+ open.insert(kid);+ stack.push(frame(kid));+ }+ }+ } else if let Some(finished) = stack.pop() {+ open.remove(&finished.id);+ let height = (!finished.loops).then_some(finished.tallest + 1);+ done.insert(finished.id, height);+ if let Some(parent) = stack.last_mut() {+ match height {+ Some(h) => parent.tallest = parent.tallest.max(h),+ None => parent.loops = true,+ }+ }+ }+ }+ }+ done+}+#[cfg(test)]mod tests {use super::*;diff --git a/crates/catalog/src/tests_tally.rs b/crates/catalog/src/tests_tally.rsnew file mode 100644index 000000000..4953a8bd1--- /dev/null+++ b/crates/catalog/src/tests_tally.rs@@ -0,0 +1,172 @@+//! A whole count of every album (`AlbumTally::of`) keeps each smart album's photos for the pass, so an album testing+//! another doesn't work its rules out again. The counts must be the ones each album gets counted alone.++use serde_json::json;++use super::*;+use crate::tally::AlbumTally;++fn smart(id: u64, rules: serde_json::Value) -> Album {+ let rule_set = Some(serde_json::from_value(rules).unwrap());+ Album { smart: Some(Box::new(Filter { rule_set, ..Default::default() })), ..Album::new(AlbumId(id), format!("Smart {id}")) }+}++fn photos(n: u64) -> Vec<Op> {+ (0..n)+ .map(|i| {+ // folders with capitals, accents and a Windows path, so the text rules see every spelling+ let dir = ["Asian/Japanese", "Été/Ünï", "C:\\Users\\Me\\Pictures", "plain", "[SITES]/x"][(i % 5) as usize];+ let mut p = Photo::new(+ PhotoId(i + 1),+ Source::File { path: format!("/Volumes/share/photos/{dir}/name{}/IMG_{i:04}.jpg", i % 9) },+ &format!("IMG_{i:04}.jpg"),+ "JPEG",+ 6000,+ 4000,+ "2026-01-01T00:00:00",+ );+ p.rating = (i % 6) as u8;+ p.flag = [Flag::None, Flag::Pick, Flag::Reject][(i % 3) as usize];+ p.label = (i % 4 == 0).then_some(ColorLabel::Red);+ p.meta.keywords = (0..i % 4).map(|k| format!("kw{k}")).collect();+ p.deleted = i % 37 == 0;+ Op::AddPhoto { photo: Box::new(p) }+ })+ .collect()+}++fn is(album: u64) -> serde_json::Value {+ json!({"field": "album", "op": "is", "value": album})+}++fn is_not(album: u64) -> serde_json::Value {+ json!({"field": "album", "op": "isNot", "value": album})+}++fn path(op: &str, value: &str) -> serde_json::Value {+ json!({"field": "filePath", "op": op, "value": value})+}++/// Every album's count in `c`, each album counted alone (no pass, rules worked out per photo).+fn counted_alone(c: &Catalog) -> Vec<(AlbumId, usize)> {+ let mut v: Vec<_> = c.albums().map(|a| (a.id, c.album_count(a.id))).collect();+ v.sort_unstable();+ v+}++fn counted_together(c: &Catalog) -> Vec<(AlbumId, usize)> {+ let tally = AlbumTally::of(c);+ let mut v: Vec<_> = tally.counts().iter().map(|(id, n)| (*id, *n)).collect();+ v.sort_unstable();+ v+}++#[test]+fn a_whole_count_equals_each_album_counted_alone() {+ let mut c = Catalog::new();+ let mut ops = photos(240);+ let mut plain = Album::new(AlbumId(1), "Picks");+ plain.photos = (0..240).step_by(4).map(|i| PhotoId(i + 1)).collect();+ let folder = Album { folder: true, ..Album::new(AlbumId(2), "Trips") };+ let inside = Album { parent: Some(AlbumId(2)), photos: (0..240).step_by(3).map(|i| PhotoId(i + 1)).collect(), ..Album::new(AlbumId(3), "In") };+ ops.extend([plain, folder, inside].map(|album| Op::AddAlbum { album }));+ // the inner albums: #2-like, #14-like, then albums chained on it, the other way round, in groups and in different orders+ ops.push(Op::AddAlbum {+ album: smart(+ 10,+ json!({"match": "any", "rules": [{"field": "flag", "op": "is", "value": "reject"}, {"field": "label", "op": "is", "value": "red"}]}),+ ),+ });+ ops.push(Op::AddAlbum {+ album: smart(11, json!({"match": "all", "rules": [is_not(10), path("notContains", "Été"), path("notContains", "plain")]})),+ });+ for k in 0..9u64 {+ let tail = path("contains", &format!("NAME{k}"));+ let rules = match k % 3 {+ 0 => json!({"match": "all", "rules": [is(11), tail]}),+ 1 => json!({"match": "all", "rules": [json!({"field": "keywordCount", "op": "lte", "value": 2}), is(11), tail]}),+ _ => json!({"match": "all", "rules": [tail, is(11)]}),+ };+ ops.push(Op::AddAlbum { album: smart(12 + k, rules) });+ }+ ops.push(Op::AddAlbum {+ album: smart(+ 21,+ json!({"match": "all", "rules": [{"group": {"match": "any", "rules": [is(11), is(1)]}}, {"field": "rating", "op": "gte", "value": 2}]}),+ ),+ });+ ops.push(Op::AddAlbum { album: smart(22, json!({"match": "all", "rules": [is_not(21), is(3)]})) });+ ops.push(Op::AddAlbum { album: smart(23, json!({"match": "any", "rules": [is(2), is(999), is(1)]})) });+ // a loop saved anyway (30 ↔ 31), albums that test it from outside (32, 33), and one that tests both worlds+ ops.push(Op::AddAlbum { album: smart(30, json!({"rules": []})) });+ ops.push(Op::AddAlbum { album: smart(31, json!({"rules": [is_not(30)]})) });+ ops.push(Op::AddAlbum { album: smart(32, json!({"rules": [is(31), {"field": "rating", "op": "gte", "value": 1}]})) });+ ops.push(Op::AddAlbum { album: smart(33, json!({"rules": [is_not(32), is(11)]})) });+ ops.push(Op::AddAlbum { album: smart(34, json!({"rules": [is(34)]})) });+ // a chain deeper than the limit on how deep albums may test albums+ ops.push(Op::AddAlbum { album: smart(100, json!({"rules": [{"field": "rating", "op": "gte", "value": 1}]})) });+ for k in 101..=140u64 {+ ops.push(Op::AddAlbum { album: smart(k, json!({"rules": [is(k - 1)]})) });+ }+ ops.push(Op::AddAlbum { album: smart(141, json!({"rules": [is(140), is_not(11)]})) });+ // an album past the depth limit that also tests a memoized album directly: where the shared album is read from the+ // pass near the top, the chain below must still find it answered as it always did (200 ← 231 … 201, height 32)+ ops.push(Op::AddAlbum { album: smart(200, json!({"rules": [{"field": "rating", "op": "gte", "value": 1}]})) });+ for k in 201..=230u64 {+ ops.push(Op::AddAlbum { album: smart(k, json!({"rules": [is(k + 1)]})) });+ }+ ops.push(Op::AddAlbum { album: smart(231, json!({"rules": [is(200)]})) });+ ops.push(Op::AddAlbum { album: smart(250, json!({"match": "all", "rules": [is(200), is(201)]})) });+ c.apply(Op::Batch { ops }).unwrap();+ c.apply(Op::SetAlbumRules {+ id: AlbumId(30),+ rules: Box::new(Filter { rule_set: Some(serde_json::from_value(json!({"rules": [is_not(31)]})).unwrap()), ..Default::default() }),+ })+ .unwrap();++ let alone = counted_alone(&c);+ let count = |id: u64| alone.iter().find(|(a, _)| a.0 == id).map(|(_, n)| *n);+ assert!(count(11) > Some(0) && count(131) > Some(0), "{alone:?}");+ assert_eq!(count(132), Some(0), "the chain is too deep from here: holds nothing, as it did");+ assert!(count(250) > Some(0), "200 is answered near the top, so the chain finds it answered");+ assert_eq!(counted_together(&c), alone);+ // the pass leaves nothing behind: the same answers whichever album asks first, and again after an edit+ c.apply(Op::SetAlbumRules {+ id: AlbumId(10),+ rules: Box::new(Filter {+ rule_set: Some(serde_json::from_value(json!({"rules": [{"field": "rating", "op": "is", "value": 5}]})).unwrap()),+ ..Default::default()+ }),+ })+ .unwrap();+ let after = counted_alone(&c);+ assert_ne!(after, alone);+ assert_eq!(counted_together(&c), after);+}++/// The reason for the pass: with an album tested by many, its rules are worked out once per photo, not once per+/// photo for each album that tests it.+#[test]+fn an_album_tested_by_many_is_worked_out_once_a_photo() {+ let mut c = Catalog::new();+ let mut ops: Vec<Op> = (0..60u64)+ .map(|i| {+ let mut p = Photo::new(PhotoId(i + 1), Source::Demo { scene: 0 }, "a.jpg", "JPEG", 10, 10, "2026-01-01T00:00:00");+ p.rating = (i % 6) as u8;+ Op::AddPhoto { photo: Box::new(p) }+ })+ .collect();+ ops.push(Op::AddAlbum { album: smart(1, json!({"rules": [{"field": "rating", "op": "gte", "value": 1}]})) });+ for k in 0..10u64 {+ ops.push(Op::AddAlbum { album: smart(2 + k, json!({"rules": [is(1), {"field": "rating", "op": "lte", "value": k % 6}]})) });+ }+ c.apply(Op::Batch { ops }).unwrap();+ let alone = counted_alone(&c);+ crate::rules::EVALUATED.with(|n| n.set(0));+ let tally = AlbumTally::of(&c);+ let worked = crate::rules::EVALUATED.with(std::cell::Cell::get);+ assert_eq!(worked, 11 * 60, "each of the 11 smart albums asks each of the 60 photos once");+ let mut together: Vec<_> = tally.counts().iter().map(|(id, n)| (*id, *n)).collect();+ together.sort_unstable();+ assert_eq!(together, alone);+}diff --git a/crates/engine/src/cmd/query.rs b/crates/engine/src/cmd/query.rsindex dfbf4bfc5..a3926bc51 100644--- a/crates/engine/src/cmd/query.rs+++ b/crates/engine/src/cmd/query.rs@@ -1,6 +1,8 @@//! Read-only queries (not journaled): catalog, photos, develop state, controls, presets, albums.-use lightcraft_catalog::{Album, Photo, PhotoId};+use std::collections::HashMap;++use lightcraft_catalog::{Album, AlbumId, Photo, PhotoId};use lightcraft_develop::{CONTROLS, controls};use serde_json::{Value, json};@@ -36,9 +38,9 @@ pub fn photo_summary(p: &Photo) -> Value {/// instead of overflowing the stack.const MAX_ALBUM_DEPTH: usize = 64;-fn album_json(a: &Album, all: &[Album], cat: &lightcraft_catalog::Catalog, depth: usize) -> Value {+fn album_json(a: &Album, all: &[Album], counts: &HashMap<AlbumId, usize>, cat: &lightcraft_catalog::Catalog, depth: usize) -> Value {let children: Vec<Value> = if depth < MAX_ALBUM_DEPTH {- all.iter().filter(|c| c.parent == Some(a.id)).map(|c| album_json(c, all, cat, depth + 1)).collect()+ all.iter().filter(|c| c.parent == Some(a.id)).map(|c| album_json(c, all, counts, cat, depth + 1)).collect()} else {Vec::new()};@@ -46,7 +48,7 @@ fn album_json(a: &Album, all: &[Album], cat: &lightcraft_catalog::Catalog, depth"id": a.id.0,"name": a.name,"folder": a.folder,- "count": cat.album_count(a.id),+ "count": counts.get(&a.id).copied().unwrap_or(0),"cover": a.cover.map(|c| c.0),"children": children,});@@ -152,7 +154,9 @@ pub fn specs() -> Vec<CommandSpec> {}),cmd!(query "albums.list", "List Albums", [], None, "{}", always, |s, _| {let all: Vec<Album> = s.catalog.albums().cloned().collect();- Ok(Value::Array(all.iter().filter(|a| a.parent.is_none()).map(|a| album_json(a, &all, &s.catalog, 0)).collect()))+ // every album counted in one pass, so a smart album tested by others is worked out once+ let counts = lightcraft_catalog::tally::AlbumTally::of(&s.catalog);+ Ok(Value::Array(all.iter().filter(|a| a.parent.is_none()).map(|a| album_json(a, &all, counts.counts(), &s.catalog, 0)).collect()))}),cmd!(query "photo.inspect", "Inspect Photo", [], None, "{id?}", always, |s, p| {let id = photo_arg(s, p, "photo.inspect")?;diff --git a/docs/whats-new.md b/docs/whats-new.mdindex cffff83e2..017adef00 100644--- a/docs/whats-new.md+++ b/docs/whats-new.md@@ -2,6 +2,13 @@## 2026-10-21+### Counting many smart albums is faster+- Counting every smart album at once (the numbers in the Albums list, and the `albums.list` command) now works out an+ album that other smart albums test (“album is …”) once for the whole library, instead of again for every album+ that tests it. On a synthetic library of 312,000 photos with 77 smart albums, 51 of them testing one shared album,+ the full count took about 14 s instead of about 23 s on a heavily loaded machine; the rest is the text rules of+ the albums themselves. The counts are the same as before, including for albums that loop through each other.+### Sharper resampling in HDR and panorama merges- Merged photos (HDR and panorama) are resampled with a sharper filter (Lanczos-3 instead of bilinear). Moving aphoto by half a pixel used to lose a third of its finest detail; now it keeps most of it, without halos around