Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

Perf: big libraries — keyword tree 5.6× faster, per-frame library queries cached

Brandon Thomasec78cb823606
diff --git a/crates/catalog/src/keywords.rs b/crates/catalog/src/keywords.rs
index 4fc1b9c65..dde020157 100644
--- a/crates/catalog/src/keywords.rs
+++ b/crates/catalog/src/keywords.rs
@@ -51,38 +51,60 @@ pub struct KeywordNode {
impl Catalog {
/// The keyword tree: every keyword level with photo counts, sorted by name (case-insensitive).
pub fn keyword_tree(&self) -> Vec<KeywordNode> {
- fn insert(nodes: &mut Vec<KeywordNode>, parts: &[&str], prefix: &str, photo: u64, seen: &mut std::collections::HashSet<(String, u64)>) {
- let Some((first, rest)) = parts.split_first() else { return };
- let path = if prefix.is_empty() { first.to_string() } else { format!("{prefix}|{first}") };
- let i = match nodes.iter().position(|n| n.name.eq_ignore_ascii_case(first)) {
- Some(i) => i,
- None => {
- nodes.push(KeywordNode { name: first.to_string(), path: path.clone(), count: 0, children: Vec::new() });
- nodes.len() - 1
- }
- };
- if seen.insert((path.to_lowercase(), photo)) {
- nodes[i].count += 1;
- }
- let p = nodes[i].path.clone();
- insert(&mut nodes[i].children, rest, &p, photo, seen);
- }
- fn sort(nodes: &mut [KeywordNode]) {
- nodes.sort_by_key(|n| n.name.to_lowercase());
- for n in nodes {
- sort(&mut n.children);
- }
+ use std::collections::{HashMap, HashSet};
+ // every keyword level (`travel`, `travel|italy`…) by its lower-case path: the name and
+ // path as first written, and the photos counted (once per photo, even when two of its
+ // keywords share a parent)
+ struct Level {
+ name: String,
+ path: String,
+ count: usize,
+ parent: Option<String>,
}
- let mut roots = Vec::new();
- let mut seen = std::collections::HashSet::new();
+ let mut levels: HashMap<String, Level> = HashMap::new();
+ let mut this_photo: HashSet<String> = HashSet::new();
for p in self.photos().filter(|p| p.in_library()) {
+ this_photo.clear();
for k in &p.meta.keywords {
- let parts: Vec<&str> = k.split(SEP).map(str::trim).filter(|s| !s.is_empty()).collect();
- insert(&mut roots, &parts, "", p.id.0, &mut seen);
+ let mut path = String::new();
+ let mut lower = String::new();
+ let mut parent: Option<String> = None;
+ for part in k.split(SEP).map(str::trim).filter(|s| !s.is_empty()) {
+ if !path.is_empty() {
+ path.push(SEP);
+ lower.push(SEP);
+ }
+ path.push_str(part);
+ lower.push_str(&part.to_lowercase());
+ let l = levels.entry(lower.clone()).or_insert_with(|| Level {
+ name: part.to_string(),
+ path: path.clone(),
+ count: 0,
+ parent: parent.clone(),
+ });
+ if this_photo.insert(lower.clone()) {
+ l.count += 1;
+ }
+ parent = Some(lower.clone());
+ }
}
}
- sort(&mut roots);
- roots
+ let mut kids: HashMap<Option<String>, Vec<String>> = HashMap::new();
+ for (lower, l) in &levels {
+ kids.entry(l.parent.clone()).or_default().push(lower.clone());
+ }
+ fn build(at: Option<String>, levels: &HashMap<String, Level>, kids: &mut HashMap<Option<String>, Vec<String>>) -> Vec<KeywordNode> {
+ let Some(mut list) = kids.remove(&at) else { return Vec::new() };
+ list.sort_by_key(|l| levels[l].name.to_lowercase());
+ list.into_iter()
+ .map(|l| {
+ let lv = &levels[&l];
+ let children = build(Some(l.clone()), levels, kids);
+ KeywordNode { name: lv.name.clone(), path: lv.path.clone(), count: lv.count, children }
+ })
+ .collect()
+ }
+ build(None, &levels, &mut kids)
}
/// `SetMeta` ops for every photo whose keywords `f` changes.
diff --git a/crates/engine/src/tests_library.rs b/crates/engine/src/tests_library.rs
index 67bc95644..6b8ee8d92 100644
--- a/crates/engine/src/tests_library.rs
+++ b/crates/engine/src/tests_library.rs
@@ -237,3 +237,85 @@ fn quick_collection_and_target_album() {
s.execute("album.clearQuick", &serde_json::json!({})).unwrap();
assert_eq!(s.catalog.album_count(quick), 0);
}
+
+/// Scaling check (ignored: `cargo test --release -p lightcraft-engine -- --ignored scale --nocapture`):
+/// the per-frame / per-click library queries on a 100k-photo catalog.
+#[test]
+#[ignore]
+fn scale_100k_library_queries() {
+ use lightcraft_catalog::{Op, Photo, PhotoId, Source};
+ use std::time::Instant;
+ let mut s = crate::Session::new();
+ let n = 100_000u64;
+ let mut ops = Vec::with_capacity(n as usize);
+ for i in 0..n {
+ let mut p = Photo::new(
+ PhotoId(i + 1),
+ Source::Demo { scene: (i % 20) as u32 },
+ &format!("IMG_{i:06}.jpg"),
+ "JPEG",
+ 6000,
+ 4000,
+ "2026-01-01T00:00:00",
+ );
+ p.captured = Some(format!("20{:02}-{:02}-{:02}T{:02}:00:00", 10 + i % 16, 1 + i % 12, 1 + i % 28, i % 24));
+ p.rating = (i % 6) as u8;
+ p.meta.keywords = vec![format!("kw{}", i % 500), "travel|italy".into()];
+ p.meta.camera = format!("Model {}", i % 30);
+ ops.push(Op::AddPhoto { photo: Box::new(p) });
+ }
+ let t = Instant::now();
+ s.commit("Add", Op::Batch { ops }).unwrap();
+ eprintln!("add 100k: {:.0} ms", t.elapsed().as_secs_f64() * 1e3);
+ let time = |what: &str, f: &mut dyn FnMut()| {
+ let t = Instant::now();
+ f();
+ let ms = t.elapsed().as_secs_f64() * 1e3;
+ eprintln!("{what}: {ms:.1} ms");
+ ms
+ };
+ let mut worst: Vec<(String, f64)> = Vec::new();
+ let mut m = |w: &str, f: &mut dyn FnMut()| worst.push((w.to_string(), time(w, f)));
+ m("visible (all)", &mut || {
+ let _ = s.visible_cloned();
+ });
+ m("visible (cached)", &mut || {
+ let _ = s.visible_cloned();
+ });
+ m("filter rating ≥ 4", &mut || {
+ s.execute("library.filter", &serde_json::json!({"rating": 4})).unwrap();
+ let _ = s.visible_cloned();
+ });
+ m("filter ruleSet keyword + date", &mut || {
+ s.execute("library.filter", &serde_json::json!({"rating": 0, "ruleSet": {"rules": [{"field": "keywords", "op": "contains", "value": "kw42"}, {"field": "captureDate", "op": "is", "value": "2015"}]}})).unwrap();
+ let _ = s.visible_cloned();
+ });
+ s.execute("library.filter", &serde_json::json!({"ruleSet": null})).unwrap();
+ m("date groups", &mut || {
+ let _ = s.execute("library.groups", &serde_json::json!({})).unwrap();
+ });
+ m("keyword tree", &mut || {
+ let _ = s.execute("keyword.list", &serde_json::json!({})).unwrap();
+ });
+ m("keyword suggestions", &mut || {
+ let _ = s.catalog.keyword_suggestions(&["kw1".to_string()], "kw", 12);
+ });
+ let alb = s.execute("album.createSmart", &serde_json::json!({"name": "S", "rules": {"rating": 5}})).unwrap()["id"].as_u64().unwrap();
+ m("smart album count", &mut || {
+ let _ = s.catalog.album_count(lightcraft_catalog::AlbumId(alb));
+ });
+ m("albums.list", &mut || {
+ let _ = s.execute("albums.list", &serde_json::json!({})).unwrap();
+ });
+ m("select all", &mut || {
+ s.execute("library.selectAll", &serde_json::json!({})).unwrap();
+ });
+ m("rate 100k selected", &mut || {
+ s.execute("photo.rate", &serde_json::json!({"rating": 3})).unwrap();
+ });
+ m("undo", &mut || {
+ s.execute("edit.undo", &serde_json::json!({})).unwrap();
+ });
+ worst.sort_by(|a, b| b.1.total_cmp(&a.1));
+ eprintln!("slowest: {:?}", &worst[..3]);
+}
diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs
index 5de32bc30..35bfba5ac 100644
--- a/crates/ui-egui/src/lib.rs
+++ b/crates/ui-egui/src/lib.rs
@@ -80,6 +80,9 @@ pub struct Perf {
}
pub struct LightcraftApp {
+ /// Per-catalog-revision caches of library-wide results the panels show every frame
+ /// (expensive on big libraries).
+ pub caches: Caches,
pub session: Session,
pub ui: UiState,
pub services: Services,
@@ -147,6 +150,7 @@ impl LightcraftApp {
services,
renderer: render::Renderer::default(),
perf: Perf::default(),
+ caches: Caches::default(),
integrated_titlebar: false,
native_menu: false,
native_shortcuts: Default::default(),
@@ -620,3 +624,60 @@ mod drop_tests {
}
}
}
+
+/// Results recomputed only when the catalog (or their inputs) change.
+#[derive(Default)]
+pub struct Caches {
+ keyword_tree: Option<(u64, std::sync::Arc<Vec<lightcraft_catalog::KeywordNode>>)>,
+ date_runs: Option<(u64, std::sync::Arc<Vec<lightcraft_catalog::DateRun>>)>,
+ suggestions: Option<(u64, std::sync::Arc<Vec<String>>)>,
+}
+
+fn key_of(parts: impl std::hash::Hash) -> u64 {
+ use std::hash::{BuildHasher, BuildHasherDefault, DefaultHasher};
+ BuildHasherDefault::<DefaultHasher>::default().hash_one(parts)
+}
+
+impl Caches {
+ /// The library's keyword tree.
+ pub fn keyword_tree(&mut self, cat: &lightcraft_catalog::Catalog) -> std::sync::Arc<Vec<lightcraft_catalog::KeywordNode>> {
+ match &self.keyword_tree {
+ Some((r, t)) if *r == cat.revision => t.clone(),
+ _ => {
+ let t = std::sync::Arc::new(cat.keyword_tree());
+ self.keyword_tree = Some((cat.revision, t.clone()));
+ t
+ }
+ }
+ }
+ /// Date headers for `ids` in the grid.
+ pub fn date_runs(
+ &mut self,
+ cat: &lightcraft_catalog::Catalog,
+ ids: &[lightcraft_catalog::PhotoId],
+ key: lightcraft_catalog::SortKey,
+ by: lightcraft_catalog::GroupBy,
+ ) -> std::sync::Arc<Vec<lightcraft_catalog::DateRun>> {
+ let k = key_of((cat.revision, ids, format!("{key:?}{by:?}")));
+ match &self.date_runs {
+ Some((h, r)) if *h == k => r.clone(),
+ _ => {
+ let r = std::sync::Arc::new(cat.date_runs(ids, key, by));
+ self.date_runs = Some((k, r.clone()));
+ r
+ }
+ }
+ }
+ /// Keyword suggestions for a photo with `current` keywords and the typed `prefix`.
+ pub fn suggestions(&mut self, cat: &lightcraft_catalog::Catalog, current: &[String], prefix: &str, n: usize) -> std::sync::Arc<Vec<String>> {
+ let k = key_of((cat.revision, current, prefix, n));
+ match &self.suggestions {
+ Some((h, v)) if *h == k => v.clone(),
+ _ => {
+ let v = std::sync::Arc::new(cat.keyword_suggestions(current, prefix, n));
+ self.suggestions = Some((k, v.clone()));
+ v
+ }
+ }
+ }
+}
diff --git a/crates/ui-egui/src/panels/grid.rs b/crates/ui-egui/src/panels/grid.rs
index 8555786da..384c980bd 100644
--- a/crates/ui-egui/src/panels/grid.rs
+++ b/crates/ui-egui/src/panels/grid.rs
@@ -82,7 +82,7 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui) {
} else {
app.session.sort.key
};
- app.session.catalog.date_runs(&ids, key, by)
+ (*app.caches.date_runs(&app.session.catalog, &ids, key, by)).clone()
};
let spans: Vec<(usize, usize)> = runs.iter().map(|r| (r.start, r.count)).collect();
let lay = layout(&aspects, &spans, avail_w, target, square);
diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs
index 140186f88..730300777 100644
--- a/crates/ui-egui/src/panels/left.rs
+++ b/crates/ui-egui/src/panels/left.rs
@@ -449,7 +449,7 @@ fn is_within(app: &LightcraftApp, id: lightcraft_catalog::AlbumId, ancestor: lig
/// context menu renames, merges or deletes the keyword across the library.
fn keywords_section(app: &mut LightcraftApp, ui: &mut egui::Ui) {
let t = Tokens::get(ui.ctx());
- let tree = app.session.catalog.keyword_tree();
+ let tree = app.caches.keyword_tree(&app.session.catalog);
if tree.is_empty() {
return;
}
diff --git a/crates/ui-egui/src/panels/right.rs b/crates/ui-egui/src/panels/right.rs
index aa6a9473f..94574dd81 100644
--- a/crates/ui-egui/src/panels/right.rs
+++ b/crates/ui-egui/src/panels/right.rs
@@ -632,7 +632,7 @@ fn keywords(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
// photo's keywords, else the most used ones
let typed = ui.data(|d| d.get_temp::<String>(kid).unwrap_or_default());
let last = typed.rsplit(',').next().unwrap_or("").trim().to_string();
- let suggestions = app.session.catalog.keyword_suggestions(&p.meta.keywords, &last, 12);
+ let suggestions = (*app.caches.suggestions(&app.session.catalog, &p.meta.keywords, &last, 12)).clone();
if !suggestions.is_empty() {
ui.label(egui::RichText::new("Suggestions").color(t.text_dim));
ui.horizontal_wrapped(|ui| {