LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
fix(previews): shard smart preview storage and preserve flat layout compatibility (#795)
Nigel Ihza Firdausye90d83ecc3c6
diff --git a/crates/engine/src/cmd/browse.rs b/crates/engine/src/cmd/browse.rsindex 6406db479..738d4d515 100644--- a/crates/engine/src/cmd/browse.rs+++ b/crates/engine/src/cmd/browse.rs@@ -105,7 +105,7 @@ impl Session {let smart = self.media.smart_dir.clone();let plan = self.catalog- .forget_local_plan(&now, days, &|p| keep.contains(&p.id) || smart.as_ref().is_some_and(|d| d.join(crate::smart::file_name(p)).exists()));+ .forget_local_plan(&now, days, &|p| keep.contains(&p.id) || smart.as_ref().is_some_and(|d| crate::smart::file_path(d, p).exists()));if dry_run {return plan;}diff --git a/crates/engine/src/cmd/previews.rs b/crates/engine/src/cmd/previews.rsindex 50ccb4bd6..a1715d795 100644--- a/crates/engine/src/cmd/previews.rs+++ b/crates/engine/src/cmd/previews.rs@@ -192,6 +192,12 @@ fn smart_run(break;}let _one = SMART_WRITE.lock().unwrap_or_else(std::sync::PoisonError::into_inner);+ // Read either the new shard or a pre-upgrade flat file, without making new work+ // depend on having the original online.+ let path = if path.exists() { path } else {+ let legacy = path.file_name().map(|name| dir.join(name));+ legacy.filter(|old| old.exists()).unwrap_or(path)+ };if discard {if std::fs::remove_file(&path).is_ok() {n.removed += 1;@@ -215,7 +221,14 @@ fn smart_run(.ok_or_else(|| "nothing to build from".to_string()).and_then(|s| s.load_source()).and_then(|src| crate::smart::encode(&src.image, src.info_or(Default::default()).camera_tone.as_ref()))- .and_then(|b| lightcraft_catalog::safe_file::write_atomic(&path, &b).map_err(|e| format!("{}: {e}", path.display())));+ .and_then(|b| {+ let parent = path.parent().ok_or_else(|| format!("{}: no parent folder", path.display()))?;+ // Only create the leaf shard: do not recreate a missing external drive.+ if !parent.is_dir() {+ std::fs::create_dir(parent).map_err(|e| format!("{}: {e}", parent.display()))?;+ }+ lightcraft_catalog::safe_file::write_atomic(&path, &b).map_err(|e| format!("{}: {e}", path.display()))+ });match r {Ok(()) => {n.built += 1;@@ -270,7 +283,7 @@ fn smart(s: &mut Session, p: &Value) -> Result<Value> {if !matches!(ph.source, lightcraft_catalog::Source::File { .. }) {continue;}- let path = dir.join(crate::smart::file_name(&ph));+ let path = crate::smart::sharded_file_path(&dir, &ph);let source = (!discard).then(|| s.media.source_ref(&ph, crate::media::SourceLevel::Preview));jobs.push((id, path, source, ph.relative_wb()));}@@ -316,7 +329,7 @@ pub(crate) fn refresh_stale_smart_previews(s: &mut Session) -> Option<std::threalet max_edge = crate::media::SourceLevel::Preview.max_edge();let photos: Vec<_> = s.catalog.photos().filter(|p| matches!(p.source, lightcraft_catalog::Source::File { .. }) && p.relative_wb()).collect();let jobs: Vec<SmartJob> =- photos.iter().map(|p| (p.id, dir.join(crate::smart::file_name(p)), Some(s.media.origin_ref(&p.source, max_edge)), true)).collect();+ photos.iter().map(|p| (p.id, crate::smart::file_path(&dir, p), Some(s.media.origin_ref(&p.source, max_edge)), true)).collect();std::thread::Builder::new().name("lc-smart-look".into()).spawn(move || refresh_stale(&dir, jobs))@@ -410,7 +423,7 @@ fn smart_location(s: &mut Session, p: &Value) -> Result<Value> {/// Whether photo `id` has a smart preview.pub fn has_smart_preview(s: &Session, id: lightcraft_catalog::PhotoId) -> bool {match (&s.media.smart_dir, s.catalog.photo(id)) {- (Some(dir), Some(p)) => crate::smart::is_valid(&dir.join(crate::smart::file_name(p))),+ (Some(dir), Some(p)) => crate::smart::is_valid(&crate::smart::file_path(dir, p)),_ => false,}}diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rsindex 6d5b548f1..c345c0dcf 100644--- a/crates/engine/src/media.rs+++ b/crates/engine/src/media.rs@@ -613,7 +613,9 @@ impl MediaCache {// be offline is tried first and its smart preview is the render worker's fallback.#[cfg(not(target_arch = "wasm32"))]if let (Source::File { path }, Some(dir)) = (&p.source, &self.smart_dir) {- let sp = dir.join(crate::smart::file_name(p));+ let sharded = crate::smart::sharded_file_path(dir, p);+ let legacy = dir.join(crate::smart::file_name(p));+ let sp = if self.availability.exists(&sharded.to_string_lossy()) == Some(true) { sharded } else { legacy };// (a proxy cut short fails to load, like the offline original; the worker's fallback// in `SourceRef::load` only uses a proxy that passes `smart::is_valid`)if self.availability.is_offline(path) && self.availability.exists(&sp.to_string_lossy()) == Some(true) {diff --git a/crates/engine/src/smart.rs b/crates/engine/src/smart.rsindex cda0baa62..cd003be44 100644--- a/crates/engine/src/smart.rs+++ b/crates/engine/src/smart.rs@@ -21,6 +21,21 @@ pub fn file_name(p: &Photo) -> String {format!("{:032x}.lcsp", h.0)}+/// The default sharded location of a proxy. A two-character hex prefix bounds each+/// folder to about 1/256 of the library's smart previews.+pub fn sharded_file_path(dir: &Path, p: &Photo) -> PathBuf {+ let name = file_name(p);+ dir.join(&name[..2]).join(name)+}++/// Read an existing proxy in either layout. New proxies use the sharded location; previews+/// made by older releases still load and can be moved with smartPreviewsLocation.+pub fn file_path(dir: &Path, p: &Photo) -> PathBuf {+ let sharded = sharded_file_path(dir, p);+ let legacy = dir.join(file_name(p));+ if sharded.exists() || !legacy.exists() { sharded } else { legacy }+}+/// The most header [`is_valid`] reads (the JSON line before the JPEG).const HEADER_MAX: u64 = 4096;@@ -104,12 +119,45 @@ pub fn check_writable(dir: &Path) -> Result<(), String> {Ok(())}+/// The owned proxy files in both layouts. Do not follow symlinked directories or+/// traverse unrelated folders inside a user-selected smart preview directory.+fn files(dir: &Path) -> Vec<PathBuf> {+ let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };+ let mut found = Vec::new();+ for e in entries.flatten() {+ let Ok(kind) = e.file_type() else { continue };+ if kind.is_file() && e.path().extension().is_some_and(|ext| ext == "lcsp") {+ found.push(e.path()); // legacy layout+ } else if kind.is_dir() {+ let shard_name = e.file_name();+ let Some(shard) = shard_name.to_str() else { continue };+ if shard.len() != 2 || !shard.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) {+ continue;+ }+ let Ok(children) = std::fs::read_dir(e.path()) else { continue };+ for child in children.flatten() {+ let name = child.file_name();+ let Some(name) = name.to_str() else { continue };+ let Some(stem) = name.strip_suffix(".lcsp") else { continue };+ if stem.len() == 32+ && stem.starts_with(shard)+ && stem.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())+ && child.file_type().is_ok_and(|kind| kind.is_file())+ {+ found.push(child.path());+ }+ }+ }+ }+ found+}+/// The number of smart previews in `dir` and their total size in bytes.pub fn stats(dir: &Path) -> (usize, u64) {- let Ok(rd) = std::fs::read_dir(dir) else { return (0, 0) };- rd.flatten()- .filter(|e| e.path().extension().is_some_and(|x| x == "lcsp"))- .fold((0, 0), |(n, b), e| (n + 1, b + e.metadata().map_or(0, |m| m.len())))+ files(dir).into_iter().fold((0usize, 0u64), |(count, bytes), path| {+ let size = std::fs::metadata(path).map_or(0, |m| m.len());+ (count.saturating_add(1), bytes.saturating_add(size))+ })}/// What to do with the smart previews in the old folder when the location changes.@@ -140,24 +188,30 @@ pub fn migrate(from: &Path, to: &Path, what: Existing) -> (usize, Vec<String>) {if what == Existing::Leave {return (done, failed);}- let Ok(rd) = std::fs::read_dir(from) else { return (done, failed) };- for e in rd.flatten().filter(|e| e.path().extension().is_some_and(|x| x == "lcsp")) {- let src = e.path();+ for src in files(from) {+ let name = src.file_name().unwrap_or_default();+ let dst = name.to_str().and_then(|n| n.strip_suffix(".lcsp")).filter(|stem| {+ stem.len() == 32 && stem.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())+ }).map_or_else(|| to.join(name), |stem| to.join(&stem[..2]).join(name));let r = match what {Existing::Discard => std::fs::remove_file(&src),_ => {- let dst = to.join(e.file_name());- if is_valid(&dst) {- // named by content: the complete copy already there is the same preview- std::fs::remove_file(&src)- } else {- // across drives: copied atomically (never a partial proxy at the new place)- std::fs::rename(&src, &dst).or_else(|_| {- std::fs::read(&src)- .and_then(|b| lightcraft_catalog::safe_file::write_atomic(&dst, &b))- .and_then(|()| std::fs::remove_file(&src))- })- }+ let parent = dst.parent().unwrap_or(to);+ // Create only the shard, never missing parent directories on an unplugged drive.+ let prepare = if parent.is_dir() { Ok(()) } else { std::fs::create_dir(parent) };+ prepare.and_then(|()| {+ if is_valid(&dst) {+ // named by content: the complete copy already there is the same preview+ std::fs::remove_file(&src)+ } else {+ // across drives: copied atomically (never a partial proxy at the new place)+ std::fs::rename(&src, &dst).or_else(|_| {+ std::fs::read(&src)+ .and_then(|b| lightcraft_catalog::safe_file::write_atomic(&dst, &b))+ .and_then(|()| std::fs::remove_file(&src))+ })+ }+ })}};match r {@@ -288,6 +342,30 @@ mod tests {let _ = std::fs::remove_dir_all(&base);}+ #[test]+ fn sharded_and_legacy_proxies_are_counted_and_moved() {+ let base = temp("shards");+ let from = base.join("from");+ let to = base.join("to");+ std::fs::create_dir_all(from.join("a1")).unwrap();+ std::fs::create_dir(&to).unwrap();+ let old = "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2.lcsp";+ let new = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1.lcsp";+ std::fs::write(from.join(old), b"legacy").unwrap();+ std::fs::write(from.join("a1").join(new), b"sharded").unwrap();+ std::fs::write(from.join("a1").join("notes.txt"), b"user file").unwrap();+ std::fs::write(from.join("notes.lcsp.txt"), b"keep").unwrap();+ assert_eq!(stats(&from), (2, 13));+ let (moved, failed) = migrate(&from, &to, Existing::Move);+ assert_eq!((moved, failed), (2, Vec::<String>::new()));+ assert_eq!(stats(&from), (0, 0));+ assert_eq!(stats(&to), (2, 13));+ assert_eq!(std::fs::read(to.join("b2").join(old)).unwrap(), b"legacy");+ assert_eq!(std::fs::read(to.join("a1").join(new)).unwrap(), b"sharded");+ assert!(from.join("a1").join("notes.txt").exists());+ let _ = std::fs::remove_dir_all(&base);+ }+#[test]fn damaged_proxies_are_invalid_and_never_kept_over_good_ones() {let base = temp("valid");diff --git a/crates/engine/src/tests_import.rs b/crates/engine/src/tests_import.rsindex 670039ad7..193fdce88 100644--- a/crates/engine/src/tests_import.rs+++ b/crates/engine/src/tests_import.rs@@ -462,14 +462,14 @@ fn damaged_smart_previews_are_rebuilt_and_failed_writes_leave_none() {s.execute("library.import", &json!({"paths": [src.join("a.png").to_string_lossy()]})).unwrap();let id = s.active().unwrap();let dir = s.media.smart_dir.clone().unwrap();- let file = dir.join(crate::smart::file_name(s.catalog.photo(id).unwrap()));+ let file = crate::smart::sharded_file_path(&dir, s.catalog.photo(id).unwrap());{let _fault = lightcraft_catalog::safe_file::fail_writes_after(64);let r = s.execute("library.smartPreviews", &json!({})).unwrap();assert_eq!((r["built"].as_u64(), r["failed"].as_array().map(Vec::len)), (Some(0), Some(1)), "{r}");}assert!(!file.exists(), "no partial proxy");- assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0, "no temp file left");+ assert_eq!(std::fs::read_dir(file.parent().unwrap()).unwrap().count(), 0, "no temp file left in shard");assert_eq!(s.execute("photo.smartPreview", &json!({})).unwrap()["smartPreview"], false);let r = s.execute("library.smartPreviews", &json!({})).unwrap();@@ -1199,8 +1199,8 @@ fn smart_previews_from_an_older_look_fit_are_refreshed() {assert!(s.catalog.photo(id).unwrap().relative_wb() && !s.catalog.photo(png).unwrap().relative_wb());s.execute("library.select", &json!({"ids": [id.0, png.0], "active": id.0})).unwrap();let dir = s.media.smart_dir.clone().unwrap();- let file = dir.join(crate::smart::file_name(s.catalog.photo(id).unwrap()));- let png_file = dir.join(crate::smart::file_name(s.catalog.photo(png).unwrap()));+ let file = crate::smart::sharded_file_path(&dir, s.catalog.photo(id).unwrap());+ let png_file = crate::smart::sharded_file_path(&dir, s.catalog.photo(png).unwrap());let (old, new) = (test_tone(1.5), test_tone(3.5));let calls = std::sync::Arc::new(AtomicUsize::new(0));let plain = s.media.file_loader.clone().unwrap();diff --git a/crates/ui-egui/src/panels/right.rs b/crates/ui-egui/src/panels/right.rsindex ff6bc4013..f51ca6217 100644--- a/crates/ui-egui/src/panels/right.rs+++ b/crates/ui-egui/src/panels/right.rs@@ -697,7 +697,11 @@ fn info(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {.media.smart_dir.as_ref()- .is_some_and(|d| avail.exists(&d.join(lightcraft_engine::smart::file_name(&p)).to_string_lossy()) == Some(true));+ .is_some_and(|d| {+ let shard = lightcraft_engine::smart::sharded_file_path(d, &p);+ let legacy = d.join(lightcraft_engine::smart::file_name(&p));+ avail.exists(&shard.to_string_lossy()) == Some(true) || avail.exists(&legacy.to_string_lossy()) == Some(true)+ });if !online || smart {let text = match (online, smart) {(false, true) => "Original offline · editing the smart preview",