Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

Issue #104: keep file-system calls off the UI thread

Brandon Thomas7b59081d2240
diff --git a/crates/engine/src/availability.rs b/crates/engine/src/availability.rs
new file mode 100644
index 000000000..5976a3b2e
--- /dev/null
+++ b/crates/engine/src/availability.rs
@@ -0,0 +1,215 @@
+//! Is a file there? — answered without blocking the caller.
+//!
+//! On a sleeping NAS, a dropped share or a USB drive spinning up, one `stat` can take as long as
+//! the OS or SMB timeout. The UI asks about files every frame (grid cells, the Info panel, the
+//! Missing Photos view), so in the app the answers come from a cache that a worker thread fills
+//! ([`Availability::run_in_background`]): an unknown file is reported as `None` ("assume it is
+//! there"; the render worker finds out if it isn't), known answers are rechecked after
+//! [`RECHECK`]. Everywhere else (CLI, MCP, tests) the default is synchronous: every question is a
+//! fresh check, as before.
+
+use std::collections::{HashMap, HashSet, VecDeque};
+use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
+use std::time::Duration;
+
+/// Checks whether a path exists (injectable: tests use a slow one).
+pub type ExistsProbe = Arc<dyn Fn(&str) -> bool + Send + Sync>;
+/// Called (from the worker) when an answer changed, e.g. to repaint the UI.
+pub type Waker = Arc<dyn Fn() + Send + Sync>;
+
+/// How long a cached answer is used before it is checked again (it is still returned meanwhile).
+pub const RECHECK: Duration = Duration::from_secs(15);
+
+fn fs_exists(path: &str) -> bool {
+ // the browser build has no file system: its originals are always "there"
+ cfg!(target_arch = "wasm32") || std::path::Path::new(path).exists()
+}
+
+#[derive(Default)]
+struct State {
+ /// path → (exists, when checked)
+ known: HashMap<String, (bool, std::time::Instant)>,
+ queue: VecDeque<String>,
+ queued: HashSet<String>,
+ worker: bool,
+ /// Bumped whenever an answer changes (callers recompute what depends on it).
+ generation: u64,
+}
+
+fn lock(m: &Mutex<State>) -> MutexGuard<'_, State> {
+ m.lock().unwrap_or_else(PoisonError::into_inner)
+}
+
+/// File availability, synchronous or cached + background-checked. Cloning shares the cache.
+#[derive(Clone)]
+pub struct Availability {
+ probe: ExistsProbe,
+ background: Option<Waker>,
+ state: Arc<Mutex<State>>,
+}
+
+impl Default for Availability {
+ fn default() -> Self {
+ Self { probe: Arc::new(fs_exists), background: None, state: Arc::default() }
+ }
+}
+
+impl std::fmt::Debug for Availability {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Availability").field("background", &self.background.is_some()).finish()
+ }
+}
+
+impl Availability {
+ /// Replace the existence check (tests: a slow or counting probe). Forgets cached answers.
+ pub fn set_probe(&mut self, probe: ExistsProbe) {
+ self.probe = probe;
+ let mut st = lock(&self.state);
+ st.known.clear();
+ st.generation += 1;
+ }
+
+ /// Answer from the cache and check on a worker thread from now on; `wake` runs when an
+ /// answer changed. No effect in the browser build (no threads, no file system).
+ pub fn run_in_background(&mut self, wake: Waker) {
+ if cfg!(target_arch = "wasm32") {
+ return;
+ }
+ self.background = Some(wake);
+ }
+
+ /// The existence check itself (blocking: for worker threads, e.g. the Rename preview's).
+ pub fn probe(&self) -> ExistsProbe {
+ self.probe.clone()
+ }
+
+ pub fn is_background(&self) -> bool {
+ self.background.is_some()
+ }
+
+ /// Does `path` exist? Synchronous mode: checked now. Background mode: the cached answer
+ /// (`None` while unknown); unknown or stale answers are queued for the worker.
+ pub fn exists(&self, path: &str) -> Option<bool> {
+ let Some(wake) = &self.background else {
+ return Some((self.probe)(path));
+ };
+ let mut st = lock(&self.state);
+ let now = std::time::Instant::now();
+ let cached = st.known.get(path).copied();
+ if cached.is_none_or(|(_, at)| now.saturating_duration_since(at) >= RECHECK) && !st.queued.contains(path) {
+ st.queued.insert(path.to_string());
+ st.queue.push_back(path.to_string());
+ if !st.worker {
+ st.worker = true;
+ drop(st);
+ self.spawn_worker(wake.clone());
+ return cached.map(|c| c.0);
+ }
+ }
+ cached.map(|c| c.0)
+ }
+
+ /// Known to be missing (unknown counts as present).
+ pub fn is_offline(&self, path: &str) -> bool {
+ self.exists(path) == Some(false)
+ }
+
+ /// Changes whenever a cached answer changes.
+ pub fn generation(&self) -> u64 {
+ lock(&self.state).generation
+ }
+
+ /// Paths waiting for the worker.
+ pub fn pending(&self) -> usize {
+ lock(&self.state).queue.len()
+ }
+
+ fn spawn_worker(&self, wake: Waker) {
+ let (state, probe) = (self.state.clone(), self.probe.clone());
+ let started = std::thread::Builder::new().name("lc-availability".into()).spawn(move || {
+ loop {
+ let path = {
+ let mut st = lock(&state);
+ let Some(p) = st.queue.pop_front() else {
+ st.worker = false;
+ return;
+ };
+ p
+ };
+ let ok = probe(&path);
+ let changed = {
+ let mut st = lock(&state);
+ st.queued.remove(&path);
+ let prev = st.known.insert(path, (ok, std::time::Instant::now()));
+ let changed = prev.map(|p| p.0) != Some(ok);
+ if changed {
+ st.generation += 1;
+ }
+ changed
+ };
+ if changed {
+ wake();
+ }
+ }
+ });
+ if started.is_err() {
+ // try again on the next question
+ lock(&self.state).worker = false;
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ #[test]
+ fn synchronous_by_default() {
+ let mut a = Availability::default();
+ let n = Arc::new(AtomicUsize::new(0));
+ let c = n.clone();
+ a.set_probe(Arc::new(move |p| {
+ c.fetch_add(1, Ordering::SeqCst);
+ p == "here"
+ }));
+ assert_eq!(a.exists("here"), Some(true));
+ assert_eq!(a.exists("gone"), Some(false));
+ assert_eq!(a.exists("gone"), Some(false));
+ assert_eq!(n.load(Ordering::SeqCst), 3, "every question is a fresh check");
+ }
+
+ #[test]
+ fn background_answers_from_the_cache_without_blocking() {
+ let mut a = Availability::default();
+ let caller = std::thread::current().id();
+ let on_caller = Arc::new(AtomicUsize::new(0));
+ let c = on_caller.clone();
+ a.set_probe(Arc::new(move |p| {
+ if std::thread::current().id() == caller {
+ c.fetch_add(1, Ordering::SeqCst);
+ }
+ std::thread::sleep(Duration::from_millis(200)); // a sleeping NAS
+ p == "here"
+ }));
+ let woke = Arc::new(AtomicUsize::new(0));
+ let w = woke.clone();
+ a.run_in_background(Arc::new(move || {
+ w.fetch_add(1, Ordering::SeqCst);
+ }));
+ let t0 = std::time::Instant::now();
+ assert_eq!(a.exists("here"), None, "unknown at first");
+ assert_eq!(a.exists("gone"), None);
+ assert!(t0.elapsed() < Duration::from_millis(150), "the caller never waits for the probe");
+ let deadline = std::time::Instant::now() + Duration::from_secs(20);
+ while (a.exists("here").is_none() || a.exists("gone").is_none()) && std::time::Instant::now() < deadline {
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ assert_eq!(a.exists("here"), Some(true));
+ assert_eq!(a.exists("gone"), Some(false));
+ assert!(a.is_offline("gone") && !a.is_offline("here") && !a.is_offline("unknown"));
+ assert_eq!(on_caller.load(Ordering::SeqCst), 0, "never probed on the calling thread");
+ assert!(woke.load(Ordering::SeqCst) >= 2);
+ assert!(a.generation() >= 2);
+ }
+}
diff --git a/crates/engine/src/cmd/convert.rs b/crates/engine/src/cmd/convert.rs
index 900c5257d..c81267bfb 100644
--- a/crates/engine/src/cmd/convert.rs
+++ b/crates/engine/src/cmd/convert.rs
@@ -26,7 +26,12 @@ fn dng_names(path: &str) -> impl Iterator<Item = std::path::PathBuf> {
/// written to a temp file, synced, read back identical and only then given its name — never
/// replacing an existing file. On any failure no DNG is left behind and the raw stays in use.
pub(crate) fn write_dng_for(s: &Session, path: &str, packet: String) -> std::result::Result<String, String> {
- let bytes = match &s.media.file_bytes {
+ write_dng_with(s.media.file_bytes.as_ref(), path, packet)
+}
+
+/// [`write_dng_for`] without the session (imports convert on a worker thread).
+pub(crate) fn write_dng_with(file_bytes: Option<&crate::merge::ByteReader>, path: &str, packet: String) -> std::result::Result<String, String> {
+ let bytes = match file_bytes {
Some(r) => r(path)?,
None => std::fs::read(path).map_err(|e| format!("{path}: {e}"))?,
};
diff --git a/crates/engine/src/cmd/library.rs b/crates/engine/src/cmd/library.rs
index 7c81427c3..74e5d5b0f 100644
--- a/crates/engine/src/cmd/library.rs
+++ b/crates/engine/src/cmd/library.rs
@@ -7,6 +7,105 @@ use serde_json::{Value, json};
use super::{CommandSpec, always, bad, bool_or, cmd, has_active, has_selection, ok, str_param};
use crate::{LibrarySource, Result, Selection, Session};
+/// The auto-import folder's visible files with their sizes (the file-system half of
+/// `library.autoImportScan`; the app lists on a worker thread and passes `listing`).
+pub fn list_auto_import_folder(folder: &str) -> std::result::Result<Vec<(String, u64)>, String> {
+ let rd = std::fs::read_dir(folder).map_err(|e| format!("{folder}: {e}"))?;
+ Ok(rd
+ .flatten()
+ .filter(|e| !e.file_name().to_string_lossy().starts_with('.'))
+ .filter_map(|e| {
+ let m = std::fs::metadata(e.path()).ok()?;
+ m.is_file().then(|| (e.path().to_string_lossy().to_string(), m.len()))
+ })
+ .collect())
+}
+
+/// `library.import`'s params, checked (no file-system calls).
+pub struct ImportRequest {
+ pub paths: Vec<String>,
+ pub opts: crate::import::ImportOptions,
+ pub album: Option<u64>,
+ pub album_name: Option<String>,
+}
+
+/// Parse and check `library.import`'s params (the app's import task runs the import itself, on a
+/// worker thread, with the same options).
+pub fn import_params(s: &Session, p: &Value) -> Result<ImportRequest> {
+ let paths = strs(p, "paths");
+ let mode = match str_param(p, "mode").unwrap_or("add") {
+ "add" => crate::import::ImportMode::Add,
+ "copy" => crate::import::ImportMode::Copy,
+ "move" => crate::import::ImportMode::Move,
+ other => return Err(bad("library.import", format!("unknown mode `{other}` (add|copy|move)"))),
+ };
+ let preset = match str_param(p, "preset").filter(|x| !x.is_empty()) {
+ Some(id) => Some(s.presets.iter().find(|x| x.id == id).cloned().ok_or_else(|| bad("library.import", format!("unknown preset `{id}`")))?),
+ None => None,
+ };
+ let album = p.get("album").and_then(Value::as_u64);
+ if let Some(a) = album
+ && s.catalog.album(AlbumId(a)).is_none_or(|al| al.folder || al.is_smart())
+ {
+ return Err(bad("library.import", "album must be a regular album"));
+ }
+ let organize = match str_param(p, "organize") {
+ Some(o) => crate::import::Organize::parse(o).ok_or_else(|| {
+ bad("library.import", format!("unknown organize `{o}` (date|month|flat, or a folder template like {{date:%Y}}/{{date:%Y%m%d}})"))
+ })?,
+ None => Default::default(),
+ };
+ if let crate::import::Organize::Template(t) = &organize
+ && let Some(e) = crate::rename::folder_template_error(t)
+ {
+ return Err(bad("library.import", e));
+ }
+ let metadata_preset = str_param(p, "metadataPreset").map(str::trim).filter(|n| !n.is_empty()).map(str::to_string);
+ if let Some(n) = &metadata_preset
+ && !s.metadata_presets.iter().any(|m| m.name.eq_ignore_ascii_case(n))
+ {
+ return Err(bad("library.import", format!("unknown metadata preset `{n}`")));
+ }
+ let opts = crate::import::ImportOptions {
+ mode,
+ preset,
+ keywords: strs(p, "keywords"),
+ destination: str_param(p, "destination").map(str::to_string),
+ organize,
+ rename: str_param(p, "rename").map(str::to_string),
+ rename_start: p.get("renameStart").and_then(Value::as_u64).unwrap_or(1) as usize,
+ metadata_preset,
+ convert_dng: bool_or(p, "dng", false),
+ local: bool_or(p, "local", false),
+ };
+ let album_name = str_param(p, "albumName").map(str::trim).filter(|n| !n.is_empty()).map(str::to_string);
+ Ok(ImportRequest { paths, opts, album, album_name })
+}
+
+/// After a batch of an import was committed: the album (a new one named `album_name` when there
+/// is no `album` and something was imported) gets the photos. Returns the report as JSON (with
+/// `album`).
+pub fn import_batch_done(s: &mut Session, report: crate::import::ImportReport, mut album: Option<u64>, album_name: Option<&str>) -> Result<Value> {
+ let mut report = serde_json::to_value(report).unwrap_or_default();
+ let imported: Vec<u64> = report["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default();
+ if album.is_none()
+ && let Some(name) = album_name
+ && !imported.is_empty()
+ {
+ let r = s.execute("album.create", &json!({"name": name}))?;
+ album = r["id"].as_u64();
+ }
+ if let Some(a) = album
+ && !imported.is_empty()
+ {
+ // (album.addPhotos wants a selection: the new photos are about to be it)
+ s.selection = Selection::single(PhotoId(imported[0]));
+ s.execute("album.addPhotos", &json!({"id": a, "ids": imported}))?;
+ report["album"] = json!(a);
+ }
+ Ok(report)
+}
+
fn album_param(p: &Value, key: &str, c: &str) -> Result<AlbumId> {
p.get(key).and_then(Value::as_u64).map(AlbumId).ok_or_else(|| bad(c, format!("missing album `{key}`")))
}
@@ -657,9 +756,9 @@ pub fn specs() -> Vec<CommandSpec> {
"Auto Import Now",
[],
None,
- "{} — add the watched folder's new photos (files the library doesn't have yet; partial / still-copying files wait for the next scan) → {imported, folder}",
+ "{listing?: [[path, size]] (the folder as listed by the caller, e.g. on a worker thread), start?: bool (false: return the `library.import` params as `import` instead of importing)} — add the watched folder's new photos (files the library doesn't have yet; partial / still-copying files wait for the next scan) → {imported, folder, import?}",
always,
- |s, _| {
+ |s, p| {
let Some(folder) = s.import_defaults.auto_folder.clone() else { return Ok(json!({"imported": [], "folder": null})) };
// only files that stopped growing: a file still being written is left for later
let known: std::collections::HashSet<String> = s
@@ -670,18 +769,20 @@ pub fn specs() -> Vec<CommandSpec> {
_ => None,
})
.collect();
+ // the folder's files and sizes: listed here, or already listed on a worker thread (the app)
+ let listing: Vec<(String, u64)> = match p.get("listing").and_then(Value::as_array) {
+ Some(l) => l.iter().filter_map(|e| Some((e.get(0)?.as_str()?.to_string(), e.get(1)?.as_u64()?))).collect(),
+ None => list_auto_import_folder(&folder).map_err(|e| bad("library.autoImportScan", e))?,
+ };
let mut fresh = Vec::new();
- for e in std::fs::read_dir(&folder).map_err(|e| bad("library.autoImportScan", format!("{folder}: {e}")))?.flatten() {
- let path = e.path();
- let ps = path.to_string_lossy().to_string();
- if !path.is_file() || known.contains(&ps) || path.file_name().is_some_and(|n| n.to_string_lossy().starts_with('.')) {
+ for (ps, size) in listing {
+ if known.contains(&ps) {
continue;
}
// each file is tried once (a non-photo isn't retried every scan)
if s.auto_import_seen.get(&ps) == Some(&u64::MAX) {
continue;
}
- let size = e.metadata().map(|m| m.len()).unwrap_or(0);
let seen = s.auto_import_seen.insert(ps.clone(), size);
if size > 0 && seen == Some(size) {
fresh.push(ps);
@@ -701,6 +802,10 @@ pub fn specs() -> Vec<CommandSpec> {
for f in &fresh {
s.auto_import_seen.insert(f.clone(), u64::MAX);
}
+ if p.get("start").and_then(Value::as_bool) == Some(false) {
+ // the caller imports them (the app: on a worker thread)
+ return Ok(json!({"imported": [], "folder": folder, "import": params}));
+ }
let sel = s.selection.clone();
let r = s.execute("library.import", &params)?;
// arriving photos don't take over the selection
@@ -751,79 +856,15 @@ pub fn specs() -> Vec<CommandSpec> {
"{paths: [file or folder (recursive)], mode?: add|copy|move (add = reference the files in place; copy = into the library's Originals/YYYY/YYYY-MM-DD/; move = as copy, then each original and its XMP sidecars are removed from the source — only after the copy is verified (a hard link on the same volume, else copied, synced and compared byte for byte) and its catalog record is saved; failed, duplicate and unchecked files keep their sources; a taken name gets -1, -2…; undo removes the photos from the library but leaves the files at the destination), destination?: folder for copies / moves, organize?: date (YYYY/YYYY-MM-DD) | month (YYYY/YYYY-MM) | flat | a folder template, e.g. `{date:%Y}/{date:%Y%m%d}` → 2026/20260114 (the template's `/` make the folders, each level expanded with the rename tokens and made a safe folder name: never outside the destination; must be relative, no `..`; a level with missing metadata is `unknown`) — dated by capture time, else the import time, rename?: file-name template for copies, original extension added (tokens: {name} {num} {seq} {seq:N} {date} {date:%Y%m%d} {folder} {camera} {lens} {iso} {rating} {title} {creator} {ext}; photo.renameTokens explains each; blank = keep names), renameStart?: 1, metadataPreset?: name, dng?: bool (copy raws as DNG; copy only), local?: bool (browsing: the photos stay out of the library, like library.browse; not with move), album?: albumId, albumName?: new album, preset?: presetId, keywords?: [..]} → {imported, duplicates, failed, moved?: [{from, to, sidecars?}], kept?: [{path, reason}] (move: sources left in place and why), album?}",
always,
|s, p| {
- let paths = strs(p, "paths");
- if paths.is_empty() {
+ let req = import_params(s, p)?;
+ if req.paths.is_empty() {
return Err(bad("library.import", "no paths"));
}
- let mode = match str_param(p, "mode").unwrap_or("add") {
- "add" => crate::import::ImportMode::Add,
- "copy" => crate::import::ImportMode::Copy,
- "move" => crate::import::ImportMode::Move,
- other => return Err(bad("library.import", format!("unknown mode `{other}` (add|copy|move)"))),
- };
- let preset = match str_param(p, "preset").filter(|x| !x.is_empty()) {
- Some(id) => {
- Some(s.presets.iter().find(|x| x.id == id).cloned().ok_or_else(|| bad("library.import", format!("unknown preset `{id}`")))?)
- }
- None => None,
- };
- let mut album = p.get("album").and_then(Value::as_u64);
- if let Some(a) = album
- && s.catalog.album(AlbumId(a)).is_none_or(|al| al.folder || al.is_smart())
- {
- return Err(bad("library.import", "album must be a regular album"));
- }
- let organize = match str_param(p, "organize") {
- Some(o) => crate::import::Organize::parse(o).ok_or_else(|| {
- bad(
- "library.import",
- format!("unknown organize `{o}` (date|month|flat, or a folder template like {{date:%Y}}/{{date:%Y%m%d}})"),
- )
- })?,
- None => Default::default(),
- };
- if let crate::import::Organize::Template(t) = &organize
- && let Some(e) = crate::rename::folder_template_error(t)
- {
- return Err(bad("library.import", e));
- }
- let metadata_preset = str_param(p, "metadataPreset").map(str::trim).filter(|n| !n.is_empty()).map(str::to_string);
- if let Some(n) = &metadata_preset
- && !s.metadata_presets.iter().any(|m| m.name.eq_ignore_ascii_case(n))
- {
- return Err(bad("library.import", format!("unknown metadata preset `{n}`")));
- }
- let opts = crate::import::ImportOptions {
- mode,
- preset,
- keywords: strs(p, "keywords"),
- destination: str_param(p, "destination").map(str::to_string),
- organize,
- rename: str_param(p, "rename").map(str::to_string),
- rename_start: p.get("renameStart").and_then(Value::as_u64).unwrap_or(1) as usize,
- metadata_preset,
- convert_dng: super::bool_or(p, "dng", false),
- local: super::bool_or(p, "local", false),
- };
let undo0 = s.undo.len();
- let mut report = serde_json::to_value(crate::import::import_with(s, &paths, &opts)?).unwrap_or_default();
- let imported: Vec<u64> = report["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default();
- if album.is_none()
- && let Some(name) = str_param(p, "albumName").map(str::trim).filter(|n| !n.is_empty())
- && !imported.is_empty()
- {
- let r = s.execute("album.create", &json!({"name": name}))?;
- album = r["id"].as_u64();
- }
- if let Some(a) = album
- && !imported.is_empty()
- {
- // (album.addPhotos wants a selection: the new photos are about to be it)
- s.selection = Selection::single(PhotoId(imported[0]));
- s.execute("album.addPhotos", &json!({"id": a, "ids": imported}))?;
- report["album"] = json!(a);
- }
+ let report = crate::import::import_with(s, &req.paths, &req.opts)?;
+ let report = import_batch_done(s, report, req.album, req.album_name.as_deref())?;
// one undo step for the whole import
+ let imported: Vec<u64> = report["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default();
let n = s.undo.len().saturating_sub(undo0);
s.merge_undo(n, &format!("Add {} Photo{}", imported.len(), if imported.len() == 1 { "" } else { "s" }));
if let Some(f) = imported.first() {
diff --git a/crates/engine/src/cmd/missing.rs b/crates/engine/src/cmd/missing.rs
index bf35f22e0..3d5fb49a3 100644
--- a/crates/engine/src/cmd/missing.rs
+++ b/crates/engine/src/cmd/missing.rs
@@ -45,9 +45,10 @@ pub fn missing(s: &Session) -> Vec<(PhotoId, String)> {
}
/// Whether photo `id` is in scope and its file is gone (the Missing Photos view checks only
-/// the photos its query already narrowed to).
-pub fn is_missing(cat: &Catalog, id: PhotoId) -> bool {
- !cfg!(target_arch = "wasm32") && cat.photo(id).and_then(|p| checked_path(p)).is_some_and(|f| !Path::new(f).exists())
+/// the photos its query already narrowed to). In the app `avail` answers from its cache (files
+/// not checked yet count as present until the background check finds them gone).
+pub fn is_missing(cat: &Catalog, avail: &crate::availability::Availability, id: PhotoId) -> bool {
+ !cfg!(target_arch = "wasm32") && cat.photo(id).and_then(|p| checked_path(p)).is_some_and(|f| avail.is_offline(f))
}
fn relink_op(id: PhotoId, path: &str) -> Op {
@@ -94,8 +95,112 @@ fn file_hash(path: &str) -> Option<String> {
std::fs::read(path).ok().map(|b| lightcraft_preview::hash_bytes(&b).to_string())
}
-/// Search `folder` (recursively) for each missing photo's file and relink every match in one
-/// undoable step.
+/// An in-scope photo ([`checked_path`]) as Find Missing Photos sees it: its file, the size and
+/// content hash the library knows (0 / `None` = unknown).
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct FindCandidate {
+ pub id: PhotoId,
+ pub path: String,
+ pub size: u64,
+ pub hash: Option<String>,
+}
+
+/// Every in-scope photo, for [`plan_find_missing`]: the missing ones are looked for, and no
+/// photo's file is used for another. No file-system calls, so the app takes this snapshot on the
+/// UI thread and plans on a worker without holding the session.
+pub fn find_candidates(cat: &Catalog) -> Vec<FindCandidate> {
+ cat.photos()
+ .filter_map(|p| {
+ checked_path(p).map(|f| FindCandidate { id: p.id, path: f.to_string(), size: p.file_size, hash: stored_hash(p).map(str::to_string) })
+ })
+ .collect()
+}
+
+/// How a missing photo's file was recognised.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum MatchBy {
+ /// Same name and size (and content, when the library knows its hash).
+ Name,
+ /// Renamed: same size and content hash.
+ Content,
+}
+
+impl MatchBy {
+ pub fn as_str(self) -> &'static str {
+ match self {
+ MatchBy::Name => "name",
+ MatchBy::Content => "content",
+ }
+ }
+}
+
+/// A missing photo and the file found for it.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct FoundFile {
+ pub id: PhotoId,
+ pub from: String,
+ pub to: String,
+ pub by: MatchBy,
+}
+
+/// A missing photo with several look-alike files (same name and size, no hash to tell them
+/// apart): skipped, never guessed.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct AmbiguousFile {
+ pub id: PhotoId,
+ pub from: String,
+ pub candidates: Vec<String>,
+}
+
+/// What Find Missing Photos found ([`plan_find_missing`]), before relinking.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct FindPlan {
+ pub found: Vec<FoundFile>,
+ /// In-scope photos that stay missing (the ambiguous ones included).
+ pub missing: usize,
+ pub ambiguous: Vec<AmbiguousFile>,
+}
+
+impl FindPlan {
+ /// `{found: [{id, from, to, by}], missing, ambiguous: [{id, from, candidates}]}` — the
+ /// command's result, and what `library.findMissing {found, …}` takes back.
+ pub fn to_json(&self) -> Value {
+ let found: Vec<Value> = self.found.iter().map(|f| json!({"id": f.id.0, "from": f.from, "to": f.to, "by": f.by.as_str()})).collect();
+ let ambiguous: Vec<Value> = self.ambiguous.iter().map(|a| json!({"id": a.id.0, "from": a.from, "candidates": a.candidates})).collect();
+ json!({"found": found, "missing": self.missing, "ambiguous": ambiguous})
+ }
+
+ /// The inverse of [`FindPlan::to_json`] (entries it can't read are dropped).
+ pub fn from_json(v: &Value) -> FindPlan {
+ let list = |k: &str| v.get(k).and_then(Value::as_array).cloned().unwrap_or_default();
+ let found = list("found")
+ .iter()
+ .filter_map(|f| {
+ let by = if f.get("by").and_then(Value::as_str) == Some("content") { MatchBy::Content } else { MatchBy::Name };
+ Some(FoundFile {
+ id: PhotoId(f.get("id")?.as_u64()?),
+ from: str_param(f, "from")?.to_string(),
+ to: str_param(f, "to")?.to_string(),
+ by,
+ })
+ })
+ .collect();
+ let ambiguous = list("ambiguous")
+ .iter()
+ .filter_map(|a| {
+ let candidates = a.get("candidates")?.as_array()?.iter().filter_map(|c| c.as_str().map(str::to_string)).collect();
+ Some(AmbiguousFile { id: PhotoId(a.get("id")?.as_u64()?), from: str_param(a, "from")?.to_string(), candidates })
+ })
+ .collect();
+ let missing = v.get("missing").and_then(Value::as_u64).map_or(0, |n| usize::try_from(n).unwrap_or(usize::MAX));
+ FindPlan { found, missing, ambiguous }
+ }
+}
+
+/// The file-system half of Find Missing Photos: which `candidates` are gone, and which file in
+/// `folder` (recursively) is each one's. Touches only the disk — never the session — so the app
+/// runs it on a worker thread (walking a big folder on a network share and hashing look-alikes
+/// takes a while) and relinks the result with `library.findMissing {found, missing, ambiguous}`.
///
/// Matching, per photo:
/// 1. a file with the same name (any letter case), the same size and the same content hash;
@@ -107,20 +212,21 @@ fn file_hash(path: &str) -> Option<String> {
///
/// Files that are already some photo's original, or that another photo was just matched with,
/// are not used.
-fn find_missing(s: &mut Session, p: &Value) -> Result<Value> {
- const C: &str = "library.findMissing";
- let folder = str_param(p, "folder").ok_or_else(|| bad(C, "missing `folder`"))?;
+pub fn plan_find_missing(candidates: &[FindCandidate], folder: &str) -> std::result::Result<FindPlan, String> {
if !Path::new(folder).is_dir() {
- return Err(bad(C, format!("{folder}: not a folder")));
+ return Err(format!("{folder}: not a folder"));
+ }
+ if cfg!(target_arch = "wasm32") {
+ return Ok(FindPlan::default());
}
- let lost = missing(s);
+ let lost: Vec<&FindCandidate> = candidates.iter().filter(|c| !Path::new(&c.path).exists()).collect();
if lost.is_empty() {
- return Ok(json!({"found": [], "missing": 0, "ambiguous": []}));
+ return Ok(FindPlan::default());
}
- let in_use: HashSet<String> = s.catalog.photos().filter_map(|p| checked_path(p).map(str::to_string)).collect();
+ let in_use: HashSet<&str> = candidates.iter().map(|c| c.path.as_str()).collect();
// file name (lower case) → candidate paths; size → paths (filled lazily)
let mut by_name: HashMap<String, Vec<String>> = HashMap::new();
- let files: Vec<String> = crate::import::expand(&[folder.to_string()], None).into_iter().filter(|f| !in_use.contains(f)).collect();
+ let files: Vec<String> = crate::import::expand(&[folder.to_string()], None).into_iter().filter(|f| !in_use.contains(f.as_str())).collect();
for f in &files {
if let Some(n) = Path::new(f).file_name() {
by_name.entry(n.to_string_lossy().to_lowercase()).or_default().push(f.clone());
@@ -131,24 +237,21 @@ fn find_missing(s: &mut Session, p: &Value) -> Result<Value> {
let mut hash_of = |f: &str| hashes.entry(f.to_string()).or_insert_with(|| file_hash(f)).clone();
let size_of = |f: &str| std::fs::metadata(f).map(|m| m.len()).ok();
let mut used: HashSet<String> = HashSet::new();
- let mut ops = Vec::new();
- let mut found = Vec::new();
- let mut ambiguous = Vec::new();
- for (id, old) in &lost {
- let Some(ph) = s.catalog.photo(*id) else { continue };
- let name = Path::new(old).file_name().map(|n| n.to_string_lossy().to_lowercase()).unwrap_or_default();
- let size = ph.file_size;
- let hash = stored_hash(ph).map(str::to_string);
+ let mut plan = FindPlan::default();
+ for c in &lost {
+ let name = Path::new(&c.path).file_name().map(|n| n.to_string_lossy().to_lowercase()).unwrap_or_default();
+ let size = c.size;
+ let hash = c.hash.as_deref();
let named: Vec<&String> = by_name
.get(&name)
- .map(|c| c.iter().filter(|f| !used.contains(*f) && (size == 0 || size_of(f) == Some(size))).collect())
+ .map(|v| v.iter().filter(|f| !used.contains(*f) && (size == 0 || size_of(f) == Some(size))).collect())
.unwrap_or_default();
// 1. same name and size, confirmed by the content hash
- let mut hit: Option<(String, &str)> =
- hash.as_deref().and_then(|h| named.iter().find(|f| hash_of(f).as_deref() == Some(h))).map(|f| (f.to_string(), "name"));
+ let mut hit: Option<(String, MatchBy)> =
+ hash.and_then(|h| named.iter().find(|f| hash_of(f).as_deref() == Some(h))).map(|f| (f.to_string(), MatchBy::Name));
// 2. renamed: same size and content
if hit.is_none()
- && let Some(h) = &hash
+ && let Some(h) = hash
&& size > 0
{
let sizes = by_size.get_or_insert_with(|| {
@@ -162,36 +265,75 @@ fn find_missing(s: &mut Session, p: &Value) -> Result<Value> {
});
hit = sizes
.get(&size)
- .and_then(|c| c.iter().find(|f| !used.contains(*f) && hash_of(f).as_deref() == Some(h.as_str())))
- .map(|f| (f.clone(), "content"));
+ .and_then(|v| v.iter().find(|f| !used.contains(*f) && hash_of(f).as_deref() == Some(h)))
+ .map(|f| (f.clone(), MatchBy::Content));
}
// 3. same name and size only (no hash, or the file was edited in place): one candidate
// is taken, several are ambiguous and skipped rather than guessed
if hit.is_none() {
- match named.len() {
- 1 => hit = Some((named[0].clone(), "name")),
- 0 => {}
+ match named.as_slice() {
+ [one] => hit = Some(((*one).clone(), MatchBy::Name)),
+ [] => {}
_ => {
- ambiguous.push(json!({"id": id.0, "from": old, "candidates": named}));
+ plan.ambiguous.push(AmbiguousFile { id: c.id, from: c.path.clone(), candidates: named.iter().map(|f| (*f).clone()).collect() });
continue;
}
}
}
let Some((path, by)) = hit else { continue };
- ops.push(relink_op(*id, &path));
- found.push(json!({"id": id.0, "from": old, "to": path, "by": by}));
- used.insert(path);
+ used.insert(path.clone());
+ plan.found.push(FoundFile { id: c.id, from: c.path.clone(), to: path, by });
+ }
+ plan.missing = lost.len() - plan.found.len();
+ Ok(plan)
+}
+
+/// Relink what [`plan_find_missing`] found, in one undoable step, re-checked against the
+/// catalog as it is now (the search may have run on a worker while the library changed): a
+/// photo that no longer points at the path it was missing from (relinked, removed) is left
+/// alone, and a file that is now some photo's original — or that the plan names twice — is not
+/// used (that photo stays missing). `missing` is the plan's count, corrected for what changed
+/// among the photos the plan names.
+fn commit_find(s: &mut Session, plan: FindPlan) -> Result<Value> {
+ let still_at = |cat: &Catalog, id: PhotoId, from: &str| cat.photo(id).and_then(|p| checked_path(p)) == Some(from);
+ let mut in_use: HashSet<String> = candidates(&s.catalog).into_iter().collect();
+ let mut found = Vec::new();
+ let mut missing = plan.missing;
+ for f in plan.found {
+ if !still_at(&s.catalog, f.id, &f.from) {
+ continue;
+ }
+ if in_use.contains(&f.to) {
+ missing += 1;
+ continue;
+ }
+ in_use.insert(f.to.clone());
+ found.push(f);
}
- let still = lost.len() - found.len();
- if !ops.is_empty() {
- s.commit("Find Missing Photos", Op::Batch { ops })?;
+ let planned = plan.ambiguous.len();
+ let ambiguous: Vec<AmbiguousFile> = plan.ambiguous.into_iter().filter(|a| still_at(&s.catalog, a.id, &a.from)).collect();
+ // ambiguous photos relinked meanwhile aren't missing any more
+ missing = missing.saturating_sub(planned - ambiguous.len());
+ if !found.is_empty() {
+ s.commit("Find Missing Photos", Op::Batch { ops: found.iter().map(|f| relink_op(f.id, &f.to)).collect() })?;
for f in &found {
- if let Some(id) = f["id"].as_u64() {
- s.media.forget(PhotoId(id));
- }
+ s.media.forget(f.id);
}
}
- Ok(json!({"found": found, "missing": still, "ambiguous": ambiguous}))
+ Ok(FindPlan { found, missing, ambiguous }.to_json())
+}
+
+/// Search `folder` (recursively) for each missing photo's file ([`plan_find_missing`]) and
+/// relink every match in one undoable step. With `found` instead (what a search on a worker
+/// thread returned, as `{found, missing, ambiguous}`), only the relinking is done.
+fn find_missing(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "library.findMissing";
+ if p.get("found").is_some_and(Value::is_array) {
+ return commit_find(s, FindPlan::from_json(p));
+ }
+ let folder = str_param(p, "folder").ok_or_else(|| bad(C, "missing `folder`"))?;
+ let plan = plan_find_missing(&find_candidates(&s.catalog), folder).map_err(|e| bad(C, e))?;
+ commit_find(s, plan)
}
pub fn specs() -> Vec<CommandSpec> {
@@ -213,7 +355,7 @@ pub fn specs() -> Vec<CommandSpec> {
"Find Missing Photos",
[],
None,
- "{folder} — relink every missing photo whose file is somewhere in the folder: same name and size (and content, when the library knows its hash), or — renamed — same size and content → {found: [{id, from, to, by: name|content}], missing, ambiguous: [{id, from, candidates}] (several same-name candidates and no hash to tell them apart: skipped)}",
+ "{folder} — relink every missing photo whose file is somewhere in the folder: same name and size (and content, when the library knows its hash), or — renamed — same size and content; or {found, missing, ambiguous} — relink what a search already found (the app searches on a worker thread; photos relinked meanwhile and files now in use are skipped) → {found: [{id, from, to, by: name|content}], missing, ambiguous: [{id, from, candidates}] (several same-name candidates and no hash to tell them apart: skipped)}",
always,
find_missing
),
diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs
index ad7b03487..9fc831161 100644
--- a/crates/engine/src/cmd/mod.rs
+++ b/crates/engine/src/cmd/mod.rs
@@ -15,7 +15,7 @@ mod edit;
mod export;
pub mod filters;
pub mod keywords;
-mod library;
+pub mod library;
pub mod lut_profiles;
pub mod manage;
mod masks;
diff --git a/crates/engine/src/cmd/previews.rs b/crates/engine/src/cmd/previews.rs
index 44b2cebd9..8a2bf7e53 100644
--- a/crates/engine/src/cmd/previews.rs
+++ b/crates/engine/src/cmd/previews.rs
@@ -23,6 +23,12 @@ pub struct PreviewBuild {
pub failed: AtomicUsize,
pub cancel: AtomicBool,
pub finished: AtomicBool,
+ /// Damaged smart previews rebuilt (Build Smart Previews only).
+ pub repaired: AtomicUsize,
+ /// What is built ("" = previews; "smart previews" — Build Smart Previews runs the same way).
+ pub what: &'static str,
+ /// Why the whole build failed (e.g. the smart previews folder isn't there).
+ pub error: std::sync::Mutex<Option<String>>,
}
impl PreviewBuild {
@@ -33,8 +39,15 @@ impl PreviewBuild {
"failed": self.failed.load(Ordering::Relaxed),
"running": !self.finished.load(Ordering::Relaxed),
"cancelled": self.cancel.load(Ordering::Relaxed),
+ "what": if self.what.is_empty() { "previews" } else { self.what },
+ "repaired": self.repaired.load(Ordering::Relaxed),
+ "error": self.error(),
})
}
+
+ pub fn error(&self) -> Option<String> {
+ self.error.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone()
+ }
}
/// The jobs that build `id`'s previews: the largest grid thumbnail, then the view render at
@@ -94,56 +107,134 @@ fn build(s: &mut Session, p: &Value) -> Result<Value> {
Ok(state.json())
}
-/// Smart previews: build (from the original, at preview size) or discard the proxies that keep
-/// photos editable while their originals are offline.
+/// One photo's smart preview to build or discard: (photo, its file in the smart previews folder,
+/// how to load the original at preview size).
#[cfg(not(target_arch = "wasm32"))]
-fn smart(s: &mut Session, p: &Value) -> Result<Value> {
- const C: &str = "library.smartPreviews";
- let dir = s.media.smart_dir.clone().ok_or_else(|| bad(C, "smart previews need a library on disk"))?;
- let discard = p.get("discard").and_then(Value::as_bool).unwrap_or(false);
- let ids = if p.get("ids").is_some() || !s.selection.ids.is_empty() { s.targets(p) } else { s.visible_cloned() };
- let (mut built, mut repaired, mut removed, mut failed) = (0usize, 0usize, 0usize, Vec::new());
+type SmartJob = (lightcraft_catalog::PhotoId, std::path::PathBuf, Option<crate::media::SourceRef>);
+
+/// What Build / Discard Smart Previews did.
+#[cfg(not(target_arch = "wasm32"))]
+#[derive(Default)]
+struct SmartCounts {
+ /// Proxies there now (complete ones kept, missing or damaged ones written).
+ built: usize,
+ /// Of those, damaged ones (cut short by a crash or a full drive) that were rebuilt.
+ repaired: usize,
+ removed: usize,
+ /// `[[id, why]]`
+ failed: Vec<Value>,
+}
+
+/// The file-system half of Build / Discard Smart Previews (runs on a worker thread in the app).
+/// `custom`: the folder was chosen (on another drive, perhaps): it must be there, it is never
+/// recreated. With `state`, progress and cancel.
+#[cfg(not(target_arch = "wasm32"))]
+fn smart_run(
+ dir: &std::path::Path,
+ custom: bool,
+ discard: bool,
+ jobs: Vec<SmartJob>,
+ state: Option<&PreviewBuild>,
+) -> std::result::Result<SmartCounts, String> {
+ let mut n = SmartCounts::default();
if !discard {
// a chosen folder on another drive must be there: never recreate it on this one
- if s.smart_previews_dir.is_some() {
- crate::smart::check_writable(&dir).map_err(|e| bad(C, e))?;
+ if custom {
+ crate::smart::check_writable(dir)?;
} else {
- std::fs::create_dir_all(&dir).map_err(|e| bad(C, format!("{}: {e}", dir.display())))?;
+ std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
}
}
- for id in ids {
- let Some(ph) = s.catalog.photo(id).cloned() else { continue };
- if !matches!(ph.source, lightcraft_catalog::Source::File { .. }) {
- continue;
+ for (id, path, source) in jobs {
+ if state.is_some_and(|st| st.cancel.load(Ordering::Relaxed)) {
+ break;
}
- let path = dir.join(crate::smart::file_name(&ph));
if discard {
if std::fs::remove_file(&path).is_ok() {
- removed += 1;
+ n.removed += 1;
+ }
+ } else {
+ // a complete proxy is kept; a missing or damaged one (cut short by a crash or a full
+ // drive) is (re)built
+ let damaged = path.exists();
+ if damaged && crate::smart::is_valid(&path) {
+ n.built += 1;
+ } else {
+ // atomic: a failed write leaves no partial proxy that would pass for a built one
+ let r = source
+ .ok_or_else(|| "nothing to build from".to_string())
+ .and_then(|s| s.load())
+ .and_then(|src| crate::smart::encode(&src))
+ .and_then(|b| lightcraft_catalog::safe_file::write_atomic(&path, &b).map_err(|e| format!("{}: {e}", path.display())));
+ match r {
+ Ok(()) => {
+ n.built += 1;
+ if damaged {
+ n.repaired += 1;
+ if let Some(st) = state {
+ st.repaired.fetch_add(1, Ordering::Relaxed);
+ }
+ }
+ }
+ Err(e) => {
+ n.failed.push(json!([id.0, e]));
+ if let Some(st) = state {
+ st.failed.fetch_add(1, Ordering::Relaxed);
+ }
+ }
+ }
}
- continue;
}
- // a complete proxy is kept; a missing or damaged one (cut short by a crash or a full
- // drive) is (re)built
- let damaged = path.exists();
- if damaged && crate::smart::is_valid(&path) {
- built += 1;
- continue;
+ if let Some(st) = state {
+ st.done.fetch_add(1, Ordering::Relaxed);
}
- // atomic: a failed write leaves no partial proxy that would pass for a built one
- let r = s
- .source_now(id, crate::media::SourceLevel::Preview)
- .and_then(|src| crate::smart::encode(&src))
- .and_then(|b| lightcraft_catalog::safe_file::write_atomic(&path, &b).map_err(|e| format!("{}: {e}", path.display())));
- match r {
- Ok(()) => {
- built += 1;
- repaired += usize::from(damaged);
- }
- Err(e) => failed.push(json!([id.0, e])),
+ }
+ Ok(n)
+}
+
+/// Smart previews: build (from the original, at preview size) or discard the proxies that keep
+/// photos editable while their originals are offline. With `background` (the app) the files are
+/// read and written on a worker thread; progress like Build Previews (`library.previewProgress`).
+#[cfg(not(target_arch = "wasm32"))]
+fn smart(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "library.smartPreviews";
+ let dir = s.media.smart_dir.clone().ok_or_else(|| bad(C, "smart previews need a library on disk"))?;
+ let discard = p.get("discard").and_then(Value::as_bool).unwrap_or(false);
+ let background = p.get("background").and_then(Value::as_bool).unwrap_or(false);
+ if background && s.preview_build.as_ref().is_some_and(|b| !b.finished.load(Ordering::Relaxed)) {
+ return Err(bad(C, "a preview build is already running"));
+ }
+ let ids = if p.get("ids").is_some() || !s.selection.ids.is_empty() { s.targets(p) } else { s.visible_cloned() };
+ let mut jobs: Vec<SmartJob> = Vec::new();
+ for id in ids {
+ let Some(ph) = s.catalog.photo(id).cloned() else { continue };
+ if !matches!(ph.source, lightcraft_catalog::Source::File { .. }) {
+ continue;
}
+ let path = dir.join(crate::smart::file_name(&ph));
+ let source = (!discard).then(|| s.media.source_ref(&ph, crate::media::SourceLevel::Preview));
+ jobs.push((id, path, source));
+ }
+ let custom = s.smart_previews_dir.is_some();
+ if background {
+ let what = if discard { "discarding smart previews" } else { "smart previews" };
+ let state = Arc::new(PreviewBuild { total: jobs.len(), what, ..Default::default() });
+ s.preview_build = Some(state.clone());
+ let st = state.clone();
+ std::thread::Builder::new()
+ .name("lc-smart-previews".into())
+ .spawn(move || {
+ if let Err(e) = smart_run(&dir, custom, discard, jobs, Some(&st)) {
+ log::warn!("{C}: {e}");
+ *st.error.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(e);
+ }
+ st.finished.store(true, Ordering::Relaxed);
+ })
+ .map_err(|e| bad(C, format!("could not start: {e}")))?;
+ return Ok(state.json());
}
- Ok(json!({"built": built, "repaired": repaired, "removed": removed, "failed": failed}))
+ let r = smart_run(&dir, custom, discard, jobs, None).map_err(|e| bad(C, e))?;
+ Ok(json!({"built": r.built, "repaired": r.repaired, "removed": r.removed, "failed": r.failed}))
}
/// The smart previews folder: where it is, whether it is custom, what it holds, whether it can be
@@ -225,7 +316,7 @@ pub fn specs() -> Vec<CommandSpec> {
"Build Smart Previews",
[],
None,
- "{ids?, discard?: bool} — build (or discard) the smart previews of the selected photos (else all in view): compact proxies in the library that keep photos editable and exportable (at proxy size) while their originals are offline; a damaged proxy (cut short) is rebuilt → {built, repaired, removed, failed}",
+ "{ids?, discard?: bool, background?: bool} — build (or discard) the smart previews of the selected photos (else all in view): compact proxies in the library that keep photos editable and exportable (at proxy size) while their originals are offline; a damaged proxy (cut short) is rebuilt → {built, repaired, removed, failed}; with background (the app's menu) the files are read and written on a worker thread → {total, done, failed, repaired, running, what} like library.buildPreviews (progress: library.previewProgress, stop: library.cancelPreviews)",
always,
smart
),
@@ -256,7 +347,7 @@ pub fn specs() -> Vec<CommandSpec> {
always,
build
),
- cmd!(query "library.previewProgress", "Preview Build Progress", [], None, "{} → {total, done, failed, running, cancelled} | null", always, |s, _| {
+ cmd!(query "library.previewProgress", "Preview Build Progress", [], None, "{} → {total, done, failed, repaired, running, cancelled, what, error} | null", always, |s, _| {
Ok(s.preview_build.as_ref().map_or(Value::Null, |b| b.json()))
}),
cmd!("library.cancelPreviews", "Cancel Preview Build", [], None, "{}", always, |s, _| {
diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs
index 8eb4c0108..a85d62873 100644
--- a/crates/engine/src/import.rs
+++ b/crates/engine/src/import.rs
@@ -517,205 +517,392 @@ pub fn import(s: &mut Session, paths: &[String], mode: ImportMode) -> crate::Res
}
/// Import files/folders with options (preset, keywords). See the module docs.
+///
+/// The file-system half ([`ImportJob::prepare`]: expanding folders, probing, reading sidecars,
+/// copying / placing files) and the catalog half ([`commit_prepared`]) are separate so the app can
+/// run the first on a worker thread; here both run in turn.
pub fn import_with(s: &mut Session, paths: &[String], opts: &ImportOptions) -> crate::Result<ImportReport> {
- let mode = opts.mode;
- // Only a library on disk has an `Originals/` folder. The browser build keeps the bytes of every
- // added file in its own storage already, so "copy" there means "add".
- let transfer = matches!(mode, ImportMode::Copy | ImportMode::Move);
- let mode = if transfer && s.library.as_ref().is_some_and(|l| !l.on_disk) { ImportMode::Add } else { mode };
- if mode == ImportMode::Move && opts.local {
- return Err(crate::EngineError::Other("browsing a folder never moves its files".into()));
+ let mut job = ImportJob::new(s, opts.clone())?;
+ let prepared = job.prepare(paths, &std::sync::atomic::AtomicBool::new(false));
+ // probes a scan left for files not imported now stay for a later import
+ s.import_probes.extend(std::mem::take(&mut job.cache));
+ commit_prepared(s, &job.opts, &job.now, prepared)
+}
+
+/// Everything the file-system half of an import needs, detached from the session so it can run
+/// on another thread (a network share or a card can take minutes). Built by [`ImportJob::new`];
+/// [`ImportJob::prepare`] may be called for several batches in turn (renamed copies keep counting,
+/// duplicates are found across batches); each batch is then [`commit_prepared`] on the session.
+pub struct ImportJob {
+ pub opts: ImportOptions,
+ /// The mode in effect (Copy / Move become Add in a library without its own folder).
+ mode: ImportMode,
+ lib_dir: Option<PathBuf>,
+ copy_root: Option<PathBuf>,
+ /// path → (photo, a Local browse record?)
+ by_path: HashMap<String, (PhotoId, bool)>,
+ /// content hash → the photo that has it (`None`: a file earlier in this import)
+ by_hash: HashMap<String, Option<PhotoId>>,
+ probe: Option<crate::media::FileProbe>,
+ file_bytes: Option<crate::merge::ByteReader>,
+ /// Probes from a preceding scan (not read again).
+ cache: HashMap<String, ProbeInfo>,
+ naming: crate::sidecar::SidecarNaming,
+ now: String,
+ seq: usize,
+ /// Files found so far (folders expanded) and handled so far, for progress.
+ pub total: std::sync::Arc<std::sync::atomic::AtomicUsize>,
+ pub done: std::sync::Arc<std::sync::atomic::AtomicUsize>,
+}
+
+/// The outcome of [`ImportJob::prepare`] for one batch, in file order.
+#[derive(Debug, Default)]
+pub struct Prepared {
+ pub scanned: usize,
+ items: Vec<PreparedItem>,
+}
+
+#[derive(Debug)]
+enum PreparedItem {
+ /// A Local (browsed) record imported for real.
+ Promote(PhotoId),
+ Duplicate {
+ path: String,
+ existing: Option<PhotoId>,
+ reason: &'static str,
+ hash: Option<String>,
+ },
+ Failed(String, String),
+ Kept(Kept),
+ Ready(Box<ReadyFile>),
+}
+
+#[derive(Debug)]
+struct ReadyFile {
+ path: String,
+ stored: String,
+ info: ProbeInfo,
+ sidecar: Option<crate::sidecar::SidecarData>,
+ placed: Option<crate::import_move::Placed>,
+}
+
+impl Prepared {
+ /// Undo what the batch did on disk that only a commit would make permanent: files placed by a
+ /// Move (their sources were never touched). Copies stay, like copies of a failed commit.
+ pub fn rollback(&self) {
+ for it in &self.items {
+ if let PreparedItem::Ready(r) = it
+ && let Some(pl) = &r.placed
+ {
+ crate::import_move::rollback(pl);
+ }
+ }
}
- let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());
- // where copies go: the chosen folder, else the library's Originals/
- let copy_root = opts
- .destination
- .as_deref()
- .filter(|d| !d.trim().is_empty())
- .map(std::path::PathBuf::from)
- .or_else(|| lib_dir.as_ref().map(|l| l.join("Originals")));
- if matches!(mode, ImportMode::Copy | ImportMode::Move) && copy_root.is_none() {
- return Err(crate::EngineError::Other("copying or moving needs a destination folder or an open library".into()));
+
+ /// Entries in the batch (after expanding folders).
+ pub fn len(&self) -> usize {
+ self.items.len()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.items.is_empty()
}
- let mut seq = opts.rename_start.max(1);
- let files = expand(paths, lib_dir.as_deref());
- let mut report = ImportReport { scanned: files.len(), ..Default::default() };
-
- // existing paths and content hashes
- let mut by_path: HashMap<&str, PhotoId> = HashMap::new();
- let mut by_hash: HashMap<String, PhotoId> = HashMap::new();
- for p in s.catalog.photos() {
- if let Source::File { path } = &p.source {
- by_path.insert(path.as_str(), p.id);
+}
+
+impl ImportJob {
+ /// Snapshot what an import with `opts` needs from the session (no file-system calls). Takes
+ /// the probes a preceding scan left ([`Session::import_probes`]).
+ pub fn new(s: &mut Session, opts: ImportOptions) -> crate::Result<Self> {
+ let mode = opts.mode;
+ // Only a library on disk has an `Originals/` folder. The browser build keeps the bytes of
+ // every added file in its own storage already, so "copy" there means "add".
+ let transfer = matches!(mode, ImportMode::Copy | ImportMode::Move);
+ let mode = if transfer && s.library.as_ref().is_some_and(|l| !l.on_disk) { ImportMode::Add } else { mode };
+ if mode == ImportMode::Move && opts.local {
+ return Err(crate::EngineError::Other("browsing a folder never moves its files".into()));
}
- if let Some(h) = &p.content_hash {
- by_hash.insert(h.clone(), p.id);
+ let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());
+ // where copies go: the chosen folder, else the library's Originals/
+ let copy_root = opts
+ .destination
+ .as_deref()
+ .filter(|d| !d.trim().is_empty())
+ .map(std::path::PathBuf::from)
+ .or_else(|| lib_dir.as_ref().map(|l| l.join("Originals")));
+ if matches!(mode, ImportMode::Copy | ImportMode::Move) && copy_root.is_none() {
+ return Err(crate::EngineError::Other("copying or moving needs a destination folder or an open library".into()));
}
- }
- let mut todo = Vec::new();
- // a file that was only browsed (Local) joins the library when it is imported for real
- let mut promote = Vec::new();
- for f in files {
- match by_path.get(f.as_str()) {
- Some(id) if !opts.local && s.catalog.photo(*id).is_some_and(|p| p.local) => {
- promote.push(Op::SetLocal { id: *id, local: false });
- report.imported.push(id.0);
+ let mut by_path = HashMap::new();
+ let mut by_hash = HashMap::new();
+ for p in s.catalog.photos() {
+ if let Source::File { path } = &p.source {
+ by_path.insert(path.clone(), (p.id, p.local));
+ }
+ if let Some(h) = &p.content_hash {
+ by_hash.insert(h.clone(), Some(p.id));
}
- Some(id) => report.duplicates.push(Duplicate { path: f, existing: Some(id.0), reason: "path" }),
- None => todo.push(f),
}
+ Ok(ImportJob {
+ seq: opts.rename_start.max(1),
+ opts,
+ mode,
+ lib_dir,
+ copy_root,
+ by_path,
+ by_hash,
+ probe: s.media.file_probe.clone(),
+ file_bytes: s.media.file_bytes.clone(),
+ cache: std::mem::take(&mut s.import_probes),
+ naming: s.xmp.naming,
+ now: (s.clock)(),
+ total: Default::default(),
+ done: Default::default(),
+ })
}
- // files a preceding scan already probed are not read again (a network share is slow to read)
- let cached: Vec<Option<ProbeInfo>> = todo.iter().map(|f| s.import_probes.remove(f)).collect();
- let missing: Vec<String> = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect();
- let mut fresh = probe_all(s.media.file_probe.as_ref(), &missing, &ScanProgress::default()).into_iter();
- let probed: Vec<Result<ProbeInfo, String>> =
- cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect();
- crate::memory::release();
- let now = (s.clock)();
- let mut ops = promote;
- // Move: placed at the destination, sources untouched until the records are committed
- let mut placed: Vec<crate::import_move::Placed> = Vec::new();
- for (path, info) in todo.into_iter().zip(probed) {
- let info = match info {
- Ok(i) => i,
- Err(e) => {
- log::warn!("import {path}: {e}");
- report.failed.push((path, e));
- continue;
+ /// The file-system half of importing `paths`: expand folders, skip known paths, probe, skip
+ /// duplicates by content, read sidecars, copy or place (Move) the files. Nothing in the
+ /// catalog changes. Stops before the next file once `cancel` is set.
+ pub fn prepare(&mut self, paths: &[String], cancel: &std::sync::atomic::AtomicBool) -> Prepared {
+ let files = self.expand(paths);
+ self.prepare_files(files, cancel)
+ }
+
+ /// Expand `paths` (folders recursively, the library's own folder skipped) into the files an
+ /// import would handle; counted in [`ImportJob::total`].
+ pub fn expand(&self, paths: &[String]) -> Vec<String> {
+ let files = expand(paths, self.lib_dir.as_deref());
+ self.total.fetch_add(files.len(), std::sync::atomic::Ordering::Relaxed);
+ files
+ }
+
+ /// When the import started ([`Session::clock`]), for [`commit_prepared`].
+ pub fn now(&self) -> &str {
+ &self.now
+ }
+
+ /// [`ImportJob::prepare`] for files [`ImportJob::expand`] returned.
+ pub fn prepare_files(&mut self, files: Vec<String>, cancel: &std::sync::atomic::AtomicBool) -> Prepared {
+ use std::sync::atomic::Ordering::Relaxed;
+ let opts = self.opts.clone();
+ let mode = self.mode;
+ let mut out = Prepared { scanned: files.len(), items: Vec::new() };
+ let mut todo = Vec::new();
+ for f in files {
+ match self.by_path.get(f.as_str()).copied() {
+ Some((id, true)) if !opts.local => {
+ // a file that was only browsed (Local) joins the library when it is imported for real
+ self.by_path.insert(f, (id, false));
+ out.items.push(PreparedItem::Promote(id));
+ self.done.fetch_add(1, Relaxed);
+ }
+ Some((id, _)) => {
+ out.items.push(PreparedItem::Duplicate { path: f, existing: Some(id), reason: "path", hash: None });
+ self.done.fetch_add(1, Relaxed);
+ }
+ None => todo.push(f),
}
- };
- if let Some(h) = &info.content_hash
- && !opts.local
- && let Some(id) = by_hash.get(h)
- {
- report.duplicates.push(Duplicate { path, existing: Some(id.0), reason: "content" });
- continue;
}
- // the XMP sidecar (or a raw's embedded XMP), read before copying: its capture time files
- // and names the copy when the file itself has none
- let raw = info.kind == MediaKind::Raw;
- let packet = crate::sidecar::find_sidecar(&path, s.xmp.naming)
- .and_then(|f| std::fs::read_to_string(f).ok())
- .or_else(|| info.xmp.clone().filter(|_| raw));
- let sidecar = packet.and_then(|x| match crate::sidecar::parse_sidecar(&x, raw).map(|sc| sc.resolve_label(&s.catalog)) {
- Ok(sc) => (sc != crate::sidecar::SidecarData::default()).then_some(sc),
- Err(e) => {
- log::warn!("import {path}: XMP: {e}");
- None
+
+ // files a preceding scan already probed are not read again (a network share is slow to read)
+ let cached: Vec<Option<ProbeInfo>> = todo.iter().map(|f| self.cache.remove(f)).collect();
+ let missing: Vec<String> = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect();
+ let progress = ScanProgress::default();
+ let mut fresh = probe_all(self.probe.as_ref(), &missing, &progress).into_iter();
+ let probed: Vec<Result<ProbeInfo, String>> =
+ cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect();
+ crate::memory::release();
+ for (path, info) in todo.into_iter().zip(probed) {
+ if cancel.load(Relaxed) {
+ break;
}
- });
- let mut info = info;
- if info.captured.is_none() {
- info.captured = sidecar.as_ref().and_then(|sc| sc.captured.clone());
- }
- let stored = match (mode, &copy_root) {
- (ImportMode::Copy | ImportMode::Move, Some(root))
- if !crate::import_move::inside(Path::new(&path), root)
- && !lib_dir.as_ref().is_some_and(|l| crate::import_move::inside(Path::new(&path), l)) =>
+ self.done.fetch_add(1, Relaxed);
+ let mut info = match info {
+ Ok(i) => i,
+ Err(e) => {
+ log::warn!("import {path}: {e}");
+ out.items.push(PreparedItem::Failed(path, e));
+ continue;
+ }
+ };
+ if let Some(h) = &info.content_hash
+ && !opts.local
+ && let Some(existing) = self.by_hash.get(h)
{
- // the templates see the photo as it will be catalogued (undated: the import time)
- let own = Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
- let mut q = Photo::new(PhotoId(0), Source::File { path: path.clone() }, &own, &info.format, 0, 0, &now);
- q.captured = info.captured.clone();
- q.meta = info.meta.clone();
- let name = opts.rename.as_deref().filter(|t| !t.trim().is_empty()).map(|t| {
- let n = crate::rename::expand(t, &q, seq);
- seq += 1;
- n
- });
- let folders = opts.organize.folders(&q);
- if mode == ImportMode::Move && !crate::import_move::is_symlink(Path::new(&path)) {
- let dir = folders.iter().fold(root.to_path_buf(), |d, f| d.join(f));
- let name = name.unwrap_or(own);
- match crate::import_move::place(Path::new(&path), &dir, &name) {
- Ok(pl) => {
- let dst = pl.dst.to_string_lossy().to_string();
- placed.push(pl);
- dst
+ out.items.push(PreparedItem::Duplicate { path, existing: *existing, reason: "content", hash: Some(h.clone()) });
+ continue;
+ }
+ // the XMP sidecar (or a raw's embedded XMP), read before copying: its capture time files
+ // and names the copy when the file itself has none
+ let raw = info.kind == MediaKind::Raw;
+ let packet = crate::sidecar::find_sidecar(&path, self.naming)
+ .and_then(|f| std::fs::read_to_string(f).ok())
+ .or_else(|| info.xmp.clone().filter(|_| raw));
+ let sidecar = packet.and_then(|x| match crate::sidecar::parse_sidecar(&x, raw) {
+ Ok(sc) => Some(sc),
+ Err(e) => {
+ log::warn!("import {path}: XMP: {e}");
+ None
+ }
+ });
+ if info.captured.is_none() {
+ info.captured = sidecar.as_ref().and_then(|sc| sc.captured.clone());
+ }
+ let mut placed = None;
+ let stored = match (mode, &self.copy_root) {
+ (ImportMode::Copy | ImportMode::Move, Some(root))
+ if !crate::import_move::inside(Path::new(&path), root)
+ && !self.lib_dir.as_ref().is_some_and(|l| crate::import_move::inside(Path::new(&path), l)) =>
+ {
+ // the templates see the photo as it will be catalogued (undated: the import time)
+ let own = Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
+ let mut q = Photo::new(PhotoId(0), Source::File { path: path.clone() }, &own, &info.format, 0, 0, &self.now);
+ q.captured = info.captured.clone();
+ q.meta = info.meta.clone();
+ let name = opts.rename.as_deref().filter(|t| !t.trim().is_empty()).map(|t| {
+ let n = crate::rename::expand(t, &q, self.seq);
+ self.seq += 1;
+ n
+ });
+ let folders = opts.organize.folders(&q);
+ if mode == ImportMode::Move && !crate::import_move::is_symlink(Path::new(&path)) {
+ let dir = folders.iter().fold(root.to_path_buf(), |d, f| d.join(f));
+ let name = name.unwrap_or(own);
+ match crate::import_move::place(Path::new(&path), &dir, &name) {
+ Ok(pl) => {
+ let dst = pl.dst.to_string_lossy().to_string();
+ placed = Some(pl);
+ dst
+ }
+ Err(e) => {
+ out.items.push(PreparedItem::Failed(path, e));
+ continue;
+ }
}
- Err(e) => {
- report.failed.push((path, e));
- continue;
+ } else {
+ if mode == ImportMode::Move {
+ let reason = "a symbolic link: the file it points to was copied, the link and its target stay";
+ out.items.push(PreparedItem::Kept(Kept { path: path.clone(), reason: reason.into() }));
}
- }
- } else {
- if mode == ImportMode::Move {
- let reason = "a symbolic link: the file it points to was copied, the link and its target stay";
- report.kept.push(Kept { path: path.clone(), reason: reason.into() });
- }
- match copy_into(root, &path, &folders, name.as_deref()) {
- Ok(p) => p,
- Err(e) => {
- report.failed.push((path, e));
- continue;
+ match copy_into(root, &path, &folders, name.as_deref()) {
+ Ok(p) => p,
+ Err(e) => {
+ out.items.push(PreparedItem::Failed(path, e));
+ continue;
+ }
}
}
}
+ (ImportMode::Move, _) => {
+ let reason = "already inside the destination or the library: added where it is";
+ out.items.push(PreparedItem::Kept(Kept { path: path.clone(), reason: reason.into() }));
+ path.clone()
+ }
+ _ => path.clone(),
+ };
+ if let Some(h) = &info.content_hash {
+ self.by_hash.insert(h.clone(), None);
}
- (ImportMode::Move, _) => {
- let reason = "already inside the destination or the library: added where it is";
- report.kept.push(Kept { path: path.clone(), reason: reason.into() });
- path.clone()
+ // Copy as DNG: the copied raw becomes a DNG (the copy is ours to replace)
+ let mut stored = stored;
+ if opts.convert_dng
+ && mode == ImportMode::Copy
+ && stored != path
+ && info.kind == MediaKind::Raw
+ && !info.format.eq_ignore_ascii_case("DNG")
+ {
+ match crate::cmd::convert::write_dng_with(self.file_bytes.as_ref(), &stored, String::new()) {
+ Ok(dng) => {
+ let _ = std::fs::remove_file(&stored);
+ stored = dng;
+ info.format = "DNG".into();
+ }
+ Err(e) => log::warn!("import {path}: copy as DNG: {e}"),
+ }
}
- _ => path.clone(),
- };
- let id = s.catalog.alloc_photo_id();
- if let Some(h) = &info.content_hash {
- by_hash.insert(h.clone(), id);
+ out.items.push(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed })));
}
- // Copy as DNG: the copied raw becomes a DNG (the copy is ours to replace)
- let mut stored = stored;
- if opts.convert_dng && mode == ImportMode::Copy && stored != path && info.kind == MediaKind::Raw && !info.format.eq_ignore_ascii_case("DNG") {
- match crate::cmd::convert::write_dng_for(s, &stored, String::new()) {
- Ok(dng) => {
- let _ = std::fs::remove_file(&stored);
- stored = dng;
- info.format = "DNG".into();
+ out
+ }
+}
+
+/// The catalog half of an import: add the photos [`ImportJob::prepare`] readied (one undoable op;
+/// a Move's sources are removed once it is saved). On a failed commit a Move's placed files are
+/// taken back.
+pub fn commit_prepared(s: &mut Session, opts: &ImportOptions, now: &str, prepared: Prepared) -> crate::Result<ImportReport> {
+ let now = now.to_string();
+ let mut report = ImportReport { scanned: prepared.scanned, ..Default::default() };
+ let mut ops = Vec::new();
+ // Move: placed at the destination, sources untouched until the records are committed
+ let mut placed: Vec<crate::import_move::Placed> = Vec::new();
+ // photos added by this batch, by content (a duplicate of a file earlier in the import names it)
+ let mut new_hash: HashMap<String, PhotoId> = HashMap::new();
+ for it in prepared.items {
+ match it {
+ PreparedItem::Promote(id) => {
+ if s.catalog.photo(id).is_some_and(|p| p.local) {
+ ops.push(Op::SetLocal { id, local: false });
}
- Err(e) => log::warn!("import {path}: copy as DNG: {e}"),
+ report.imported.push(id.0);
}
- }
- // a copy is catalogued under its new name
- let name = Path::new(&stored).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.clone());
- let mut p = Photo::new(id, Source::File { path: stored }, &name, &info.format, info.width, info.height, &now);
- p.kind = info.kind;
- p.file_size = info.file_size;
- p.captured = info.captured;
- p.meta = info.meta;
- p.as_shot_wb = info.as_shot_wb;
- p.content_hash = info.content_hash;
- p.embedded_lens = info.embedded_lens;
- p.preview_only = info.preview_only.clone();
- apply_import_defaults(s, &mut p);
- if let Some(sc) = &sidecar {
- crate::sidecar::merge_into(&mut p, sc, &now);
- report.sidecars += 1;
- }
- if let Some(mp) = opts.metadata_preset.as_ref().and_then(|n| s.metadata_presets.iter().find(|m| m.name.eq_ignore_ascii_case(n))) {
- crate::cmd::metadata::apply_to(&mut p.meta, &mp.fields);
- }
- for k in &opts.keywords {
- let k = lightcraft_catalog::keywords::clean(k);
- if !k.is_empty() && !p.meta.keywords.iter().any(|x| x.eq_ignore_ascii_case(&k)) {
- p.meta.keywords.push(k);
+ PreparedItem::Duplicate { path, existing, reason, hash } => {
+ let existing = existing.or_else(|| {
+ let h = hash?;
+ new_hash.get(&h).copied().or_else(|| s.catalog.photos().find(|p| p.content_hash.as_deref() == Some(h.as_str())).map(|p| p.id))
+ });
+ report.duplicates.push(Duplicate { path, existing: existing.map(|i| i.0), reason });
+ }
+ PreparedItem::Failed(path, e) => report.failed.push((path, e)),
+ PreparedItem::Kept(k) => report.kept.push(k),
+ PreparedItem::Ready(r) => {
+ let ReadyFile { path, stored, info, sidecar, placed: pl } = *r;
+ placed.extend(pl);
+ let id = s.catalog.alloc_photo_id();
+ if let Some(h) = &info.content_hash {
+ new_hash.insert(h.clone(), id);
+ }
+ let sidecar = sidecar.map(|sc| sc.resolve_label(&s.catalog)).filter(|sc| *sc != crate::sidecar::SidecarData::default());
+ // a copy is catalogued under its new name
+ let name = Path::new(&stored).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.clone());
+ let mut p = Photo::new(id, Source::File { path: stored }, &name, &info.format, info.width, info.height, &now);
+ p.kind = info.kind;
+ p.file_size = info.file_size;
+ p.captured = info.captured;
+ p.meta = info.meta;
+ p.as_shot_wb = info.as_shot_wb;
+ p.content_hash = info.content_hash;
+ p.embedded_lens = info.embedded_lens;
+ p.preview_only = info.preview_only.clone();
+ apply_import_defaults(s, &mut p);
+ if let Some(sc) = &sidecar {
+ crate::sidecar::merge_into(&mut p, sc, &now);
+ report.sidecars += 1;
+ }
+ if let Some(mp) = opts.metadata_preset.as_ref().and_then(|n| s.metadata_presets.iter().find(|m| m.name.eq_ignore_ascii_case(n))) {
+ crate::cmd::metadata::apply_to(&mut p.meta, &mp.fields);
+ }
+ for k in &opts.keywords {
+ let k = lightcraft_catalog::keywords::clean(k);
+ if !k.is_empty() && !p.meta.keywords.iter().any(|x| x.eq_ignore_ascii_case(&k)) {
+ p.meta.keywords.push(k);
+ }
+ }
+ if let Some(preset) = &opts.preset {
+ let d = std::sync::Arc::new(preset.apply(&p.develop, 1.0));
+ let label = format!("Preset: {}", preset.name);
+ p.develop = d.clone();
+ p.edited = Some(now.clone());
+ p.history.push(lightcraft_catalog::HistoryStep { label, settings: d });
+ }
+ report.imported.push(id.0);
+ p.local = opts.local;
+ if opts.local {
+ // the state as browsed: a later change keeps the record from being forgotten
+ p.set_local_baseline();
+ }
+ ops.push(Op::AddPhoto { photo: Box::new(p) });
}
}
- if let Some(preset) = &opts.preset {
- let d = std::sync::Arc::new(preset.apply(&p.develop, 1.0));
- let label = format!("Preset: {}", preset.name);
- p.develop = d.clone();
- p.edited = Some(now.clone());
- p.history.push(lightcraft_catalog::HistoryStep { label, settings: d });
- }
- report.imported.push(id.0);
- p.local = opts.local;
- if opts.local {
- // the state as browsed: a later change keeps the record from being forgotten
- p.set_local_baseline();
- }
- ops.push(Op::AddPhoto { photo: Box::new(p) });
}
let log0 = s.pending_log.len();
if !ops.is_empty()
diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs
index bb4760ada..ecdb078d5 100644
--- a/crates/engine/src/lib.rs
+++ b/crates/engine/src/lib.rs
@@ -10,6 +10,7 @@
//! run them off the UI thread.
#![forbid(unsafe_code)]
+pub mod availability;
pub mod cmd;
pub mod crs;
pub mod crs_masks;
@@ -272,11 +273,22 @@ impl Session {
(spec.enabled)(self).map_err(|why| EngineError::Disabled(id.to_string(), why))?;
let empty = Value::Object(Default::default());
let params = if params.is_null() { &empty } else { params };
+ self.run_command(id, spec.journal.then_some(params), |s| (spec.run)(s, params))
+ }
+
+ /// Run `f` as command `id` with what [`Session::execute`] does around every command: the
+ /// panic guard, auto versions, XMP sidecar auto-write and the durable save (a failed save is
+ /// `NotSaved`). Not journaled. The app's import task commits its batches this way.
+ pub fn execute_fn(&mut self, id: &str, f: impl FnOnce(&mut Session) -> Result<Value>) -> Result<Value> {
+ self.run_command(id, None, f)
+ }
+
+ fn run_command(&mut self, id: &str, journal: Option<&Value>, f: impl FnOnce(&mut Session) -> Result<Value>) -> Result<Value> {
let log_start = self.pending_log.len();
let was_active = self.active();
self.depth += 1;
// last-resort guard: a panic in a command is that command's error, not a crash
- let r = guard::catch(&format!("`{id}`"), || (spec.run)(self, params)).unwrap_or_else(|e| Err(EngineError::Other(e)));
+ let r = guard::catch(&format!("`{id}`"), || f(self)).unwrap_or_else(|e| Err(EngineError::Other(e)));
self.depth -= 1;
if self.depth == 0 && was_active.is_some() && self.active() != was_active {
self.previous_active = was_active;
@@ -284,7 +296,10 @@ impl Session {
self.auto_version(left);
}
}
- if r.is_ok() && spec.journal && self.depth == 0 {
+ if r.is_ok()
+ && let Some(params) = journal
+ && self.depth == 0
+ {
self.journal.push((id.to_string(), params.clone()));
if self.journal.len() > 10_000 {
self.journal.drain(..1000);
@@ -560,7 +575,11 @@ impl Session {
/// Photos shown in the grid/filmstrip for the current source, filter and sort.
pub fn visible(&mut self) -> &[PhotoId] {
- let key = (self.catalog.revision, format!("{:?}|{:?}|{:?}|{:?}", self.source, self.filter, self.sort, self.browse));
+ let mut key = (self.catalog.revision, format!("{:?}|{:?}|{:?}|{:?}", self.source, self.filter, self.sort, self.browse));
+ if self.source == LibrarySource::Missing && self.media.availability.is_background() {
+ // the view fills in as the background checks find files gone
+ key.1.push_str(&format!("|{}", self.media.availability.generation()));
+ }
if self.visible_key.as_ref() != Some(&key) {
// "in the last N days" rules count back from the session's clock
lightcraft_catalog::rules::set_now(Some((self.clock)()));
@@ -595,8 +614,8 @@ impl Session {
}
if self.source == LibrarySource::Missing {
// only the photos the query kept (library photos, not Local browse records) are checked
- let cat = &self.catalog;
- visible.retain(|id| cmd::missing::is_missing(cat, *id));
+ let (cat, avail) = (&self.catalog, &self.media.availability);
+ visible.retain(|id| cmd::missing::is_missing(cat, avail, *id));
}
if self.source != LibrarySource::RecentlyDeleted {
visible = self.catalog.arrange_stacks(&visible);
diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs
index 18e67fafd..561b7b24d 100644
--- a/crates/engine/src/media.rs
+++ b/crates/engine/src/media.rs
@@ -82,6 +82,9 @@ pub enum SourceRef {
path: String,
max_edge: usize,
loader: Option<FileLoader>,
+ /// The photo's smart preview, used when the original can't be read (offline drive). The
+ /// UI thread never checks whether the original is there; the render worker finds out.
+ fallback: Option<std::path::PathBuf>,
},
/// A smart preview standing in for a missing original.
Smart {
@@ -94,10 +97,17 @@ impl SourceRef {
match self {
SourceRef::Loaded(a) => Ok(a.clone()),
SourceRef::Demo { scene, max_edge } => Ok(Arc::new(scene.render_fit(*max_edge))),
- SourceRef::File { path, max_edge, loader } => match loader {
- Some(l) => l(path, *max_edge).map(|(img, _)| Arc::new(img)),
- None => Err(format!("no decoder available for {path}")),
- },
+ SourceRef::File { path, max_edge, loader, fallback } => {
+ let r = match loader {
+ Some(l) => l(path, *max_edge).map(|(img, _)| Arc::new(img)),
+ None => Err(format!("no decoder available for {path}")),
+ };
+ match (r, fallback) {
+ #[cfg(not(target_arch = "wasm32"))]
+ (Err(e), Some(sp)) if crate::smart::is_valid(sp) => crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}")),
+ (r, _) => r,
+ }
+ }
#[cfg(not(target_arch = "wasm32"))]
SourceRef::Smart { path } => crate::smart::load(path),
#[cfg(target_arch = "wasm32")]
@@ -125,6 +135,9 @@ pub struct MediaCache {
pub file_bytes: Option<crate::merge::ByteReader>,
/// The library's smart previews folder (originals offline: render from the proxy).
pub smart_dir: Option<std::path::PathBuf>,
+ /// Whether originals (and smart previews) are on disk: cached and checked off the UI thread
+ /// in the app (see [`crate::availability`]).
+ pub availability: crate::availability::Availability,
scenes: Vec<lightcraft_scenes::Scene>,
/// Rendered thumbnails (memory, plus disk once a library is attached).
pub rendered: Arc<PreviewCache>,
@@ -144,6 +157,7 @@ impl Default for MediaCache {
preview_loader: None,
file_bytes: None,
smart_dir: None,
+ availability: Default::default(),
scenes: Vec::new(),
rendered: Arc::new(PreviewCache::memory(rendered_budget(budget))),
}
@@ -284,15 +298,22 @@ impl MediaCache {
(Source::Demo { .. }, SourceLevel::Full) => p.width.max(p.height).max(1) as usize,
_ => usize::MAX,
});
- // the original is offline: its smart preview, when there is one
+ // the original is offline: its smart preview, when there is one. Only cached answers are
+ // used here (no file-system call on the UI thread in the app); an original not known to
+ // be offline is tried first and its smart preview is the render worker's fallback.
#[cfg(not(target_arch = "wasm32"))]
- if let (Source::File { path }, Some(dir)) = (&p.source, &self.smart_dir)
- && !std::path::Path::new(path).exists()
- {
+ if let (Source::File { path }, Some(dir)) = (&p.source, &self.smart_dir) {
let sp = dir.join(crate::smart::file_name(p));
- if crate::smart::is_valid(&sp) {
+ // (a proxy cut short fails to load, like the offline original; the worker's fallback
+ // in `SourceRef::load` only uses a proxy that passes `smart::is_valid`)
+ if self.availability.is_offline(path) && self.availability.exists(&sp.to_string_lossy()) == Some(true) {
return SourceRef::Smart { path: sp };
}
+ let mut r = self.origin_ref(&p.source, max_edge);
+ if let SourceRef::File { fallback, .. } = &mut r {
+ *fallback = Some(sp);
+ }
+ return r;
}
self.origin_ref(&p.source, max_edge)
}
@@ -307,10 +328,10 @@ impl MediaCache {
}
match self.scenes.iter().find(|s| s.id == *scene) {
Some(s) => SourceRef::Demo { scene: Box::new(s.clone()), max_edge },
- None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None },
+ None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None },
}
}
- Source::File { path } => SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone() },
+ Source::File { path } => SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None },
}
}
}
diff --git a/crates/engine/src/rename.rs b/crates/engine/src/rename.rs
index 511aa8b49..27fc5fb5f 100644
--- a/crates/engine/src/rename.rs
+++ b/crates/engine/src/rename.rs
@@ -538,6 +538,92 @@ fn set_file_ops_to(op: &Op, to: &HashSet<&str>, out: &mut Vec<Op>) {
}
}
+/// The photos among `ids` a rename handles, in order: photos sharing one file (virtual copies)
+/// rename once. No file-system calls.
+pub fn rename_photos(cat: &lightcraft_catalog::Catalog, ids: &[PhotoId]) -> Vec<std::sync::Arc<Photo>> {
+ let mut seen_paths: HashSet<String> = HashSet::new();
+ ids.iter()
+ .filter_map(|id| cat.photo(*id))
+ .filter(|p| match &p.source {
+ Source::File { path } => seen_paths.insert(path.clone()),
+ _ => true,
+ })
+ .cloned()
+ .collect()
+}
+
+/// [`Session::plan_rename`] for [`rename_photos`], with the "is this name taken on disk?" check
+/// supplied: the Rename dialog plans its preview on a worker thread (a check can block on a slow
+/// drive). Whether a case-only change names this very file is decided on the disk.
+pub fn plan_rename_photos(photos: &[std::sync::Arc<Photo>], template: &str, start: usize, exists: &dyn Fn(&str) -> bool) -> Vec<RenamePlan> {
+ plan_rename_core(photos, template, start, &|p| exists(&p.to_string_lossy()), &|a, b| RealFs.same_file(a, b))
+}
+
+/// Plan renaming `photos`: `exists` says whether something answers to a path, `same_file` whether
+/// two spellings name one file (see [`MoveFs`]).
+fn plan_rename_core(
+ photos: &[std::sync::Arc<Photo>],
+ template: &str,
+ start: usize,
+ exists: &dyn Fn(&Path) -> bool,
+ same_file: &dyn Fn(&Path, &Path) -> bool,
+) -> Vec<RenamePlan> {
+ let mut taken: HashSet<String> = HashSet::new(); // lower-case target paths/names claimed in this batch
+ let mut out = Vec::new();
+ for (seq, p) in (start..).zip(photos) {
+ let want = expand(template, p, seq);
+ let (stem, ext) = split_ext(&want);
+ let (stem, ext) = (stem.to_string(), ext.to_string());
+ let candidate = |k: usize| {
+ if k == 0 {
+ want.clone()
+ } else if ext.is_empty() {
+ format!("{stem}-{k}")
+ } else {
+ format!("{stem}-{k}.{ext}")
+ }
+ };
+ let (to, to_path) = match &p.source {
+ Source::File { path } => {
+ let dir = Path::new(path).parent().map(Path::to_path_buf).unwrap_or_default();
+ let mut k = 0;
+ loop {
+ let name = candidate(k);
+ let tp = dir.join(&name).to_string_lossy().to_string();
+ let key = tp.to_lowercase();
+ // this very file, maybe spelled differently (case-insensitive volume)? By
+ // identity: on a case-sensitive volume `img_1.jpg` may be another photo
+ let (from, to) = (Path::new(path), Path::new(&tp));
+ let same = tp == *path || (from != to && key == path.to_lowercase() && exists(to) && same_file(from, to));
+ // free: not claimed in this batch and not on disk (unless it is this very file).
+ // A file this batch moves away still counts as taken: simple and safe.
+ if !taken.contains(&key) && (same || !exists(to)) {
+ taken.insert(key);
+ break (name, Some(tp));
+ }
+ k += 1;
+ }
+ }
+ Source::Demo { .. } => {
+ let mut k = 0;
+ loop {
+ let name = candidate(k);
+ if taken.insert(format!("demo:{}", name.to_lowercase())) {
+ break (name, None);
+ }
+ k += 1;
+ }
+ }
+ };
+ let from_path = match &p.source {
+ Source::File { path } => Some(path.clone()),
+ _ => None,
+ };
+ out.push(RenamePlan { id: p.id.0, from: p.file_name.clone(), to, from_path, to_path });
+ }
+ out
+}
+
impl Session {
/// Plan renaming `ids` with `template` (sequence numbers from `start`), resolving collisions.
pub fn plan_rename(&self, ids: &[PhotoId], template: &str, start: usize) -> Vec<RenamePlan> {
@@ -546,70 +632,7 @@ impl Session {
/// [`Session::plan_rename`] on the file system `fs`.
pub(crate) fn plan_rename_with(&self, fs: &dyn MoveFs, ids: &[PhotoId], template: &str, start: usize) -> Vec<RenamePlan> {
- // photos sharing one file (virtual copies) rename once
- let mut seen_paths: HashSet<String> = HashSet::new();
- let mut taken: HashSet<String> = HashSet::new(); // lower-case target paths/names claimed in this batch
- let mut out = Vec::new();
- let mut seq = start;
- for id in ids {
- let Some(p) = self.catalog.photo(*id) else { continue };
- if let Source::File { path } = &p.source
- && !seen_paths.insert(path.clone())
- {
- continue;
- }
- let want = expand(template, p, seq);
- seq += 1;
- let (stem, ext) = split_ext(&want);
- let (stem, ext) = (stem.to_string(), ext.to_string());
- let candidate = |k: usize| {
- if k == 0 {
- want.clone()
- } else if ext.is_empty() {
- format!("{stem}-{k}")
- } else {
- format!("{stem}-{k}.{ext}")
- }
- };
- let (to, to_path) = match &p.source {
- Source::File { path } => {
- let dir = Path::new(path).parent().map(Path::to_path_buf).unwrap_or_default();
- let mut k = 0;
- loop {
- let name = candidate(k);
- let tp = dir.join(&name).to_string_lossy().to_string();
- let key = tp.to_lowercase();
- // this very file, maybe spelled differently (case-insensitive volume)? By
- // identity: on a case-sensitive volume `img_1.jpg` may be another photo
- let same = tp == *path || is_respelling(fs, Path::new(path), Path::new(&tp));
- // free: not claimed in this batch and not on disk (unless it is this very file).
- // A file this batch moves away still counts as taken: simple and safe.
- let on_disk = fs.exists(Path::new(&tp)) && !same;
- if !taken.contains(&key) && !on_disk {
- taken.insert(key);
- break (name, Some(tp));
- }
- k += 1;
- }
- }
- Source::Demo { .. } => {
- let mut k = 0;
- loop {
- let name = candidate(k);
- if taken.insert(format!("demo:{}", name.to_lowercase())) {
- break (name, None);
- }
- k += 1;
- }
- }
- };
- let from_path = match &p.source {
- Source::File { path } => Some(path.clone()),
- _ => None,
- };
- out.push(RenamePlan { id: id.0, from: p.file_name.clone(), to, from_path, to_path });
- }
- out
+ plan_rename_core(&rename_photos(&self.catalog, ids), template, start, &|p| fs.exists(p), &|a, b| fs.same_file(a, b))
}
/// Carry out `plans`: move the files (rolled back on failure), then commit one undoable op
diff --git a/crates/engine/src/tests_import.rs b/crates/engine/src/tests_import.rs
index e83a4d344..dd3f7ca1f 100644
--- a/crates/engine/src/tests_import.rs
+++ b/crates/engine/src/tests_import.rs
@@ -144,11 +144,10 @@ fn browsing_a_folder_lists_its_photos_without_adding_them() {
let _ = std::fs::remove_dir_all(&dir);
}
-/// Issue #105: Find Missing finds renamed files by content, prefers the content match over a
-/// same-name same-size impostor, and skips (reports) photos it can't tell apart.
-#[test]
-fn find_missing_matches_renamed_files_by_content() {
- let dir = temp_dir("missing-content");
+/// Photos a.png, b.png, c.png imported from `old`, then: a renamed into `moved`; b renamed, with
+/// an impostor (same name and size, other bytes) under its old name; c gone, with two impostors.
+fn renamed_and_ambiguous_scene(tag: &str) -> (std::path::PathBuf, Session) {
+ let dir = temp_dir(tag);
std::fs::create_dir_all(dir.join("old")).unwrap();
std::fs::create_dir_all(dir.join("moved/real")).unwrap();
for (n, seed) in [("a.png", 1), ("b.png", 2), ("c.png", 3)] {
@@ -156,20 +155,26 @@ fn find_missing_matches_renamed_files_by_content() {
}
let mut s = Session::new().with_fs();
s.execute("library.import", &json!({"paths": [dir.join("old").to_string_lossy()]})).unwrap();
- // an impostor: same name and size, other bytes
let impostor = |src: &Path, dst: &Path| {
let mut b = std::fs::read(src).unwrap();
let k = b.len() - 20;
b[k] ^= 0xff;
std::fs::write(dst, b).unwrap();
};
- // a: renamed; b: renamed, with an impostor under its old name; c: gone, two impostors
std::fs::rename(dir.join("old/a.png"), dir.join("moved/Trip-001.png")).unwrap();
impostor(&dir.join("old/b.png"), &dir.join("moved/b.png"));
std::fs::rename(dir.join("old/b.png"), dir.join("moved/real/Trip-002.png")).unwrap();
impostor(&dir.join("old/c.png"), &dir.join("moved/c.png"));
impostor(&dir.join("old/c.png"), &dir.join("moved/real/c.png"));
std::fs::remove_file(dir.join("old/c.png")).unwrap();
+ (dir, s)
+}
+
+/// Issue #105: Find Missing finds renamed files by content, prefers the content match over a
+/// same-name same-size impostor, and skips (reports) photos it can't tell apart.
+#[test]
+fn find_missing_matches_renamed_files_by_content() {
+ let (dir, mut s) = renamed_and_ambiguous_scene("missing-content");
let r = s.execute("library.findMissing", &json!({"folder": dir.join("moved").to_string_lossy()})).unwrap();
let found = r["found"].as_array().unwrap();
assert_eq!(found.len(), 2, "{r}");
@@ -183,6 +188,55 @@ fn find_missing_matches_renamed_files_by_content() {
let _ = std::fs::remove_dir_all(&dir);
}
+/// Issue #104: the app plans Find Missing on a worker thread (no session) and relinks the plan
+/// afterwards; that gives exactly what the synchronous command gives — renamed files by content,
+/// impostors and look-alikes skipped (`ambiguous`).
+#[test]
+fn find_missing_planned_on_a_worker_matches_the_command() {
+ use crate::cmd::missing::{find_candidates, plan_find_missing};
+ let (dir, mut s) = renamed_and_ambiguous_scene("missing-worker");
+ let folder = dir.join("moved").to_string_lossy().to_string();
+ let candidates = find_candidates(&s.catalog);
+ let f = folder.clone();
+ let plan = std::thread::spawn(move || plan_find_missing(&candidates, &f)).join().unwrap().unwrap();
+ assert_eq!((plan.found.len(), plan.missing, plan.ambiguous.len()), (2, 1, 1), "{plan:?}");
+ let applied = s.execute("library.findMissing", &plan.to_json()).unwrap();
+ s.execute("edit.undo", &json!({})).unwrap();
+ assert_eq!(s.execute("library.missing", &json!({})).unwrap().as_array().map(Vec::len), Some(3), "undo restores all three");
+ let direct = s.execute("library.findMissing", &json!({"folder": folder})).unwrap();
+ assert_eq!(applied, direct);
+ assert_eq!(direct["found"].as_array().map(Vec::len), Some(2), "{direct}");
+ assert_eq!(direct["ambiguous"].as_array().map(Vec::len), Some(1), "{direct}");
+ let _ = std::fs::remove_dir_all(&dir);
+}
+
+/// Issue #104: a plan made on a worker is re-checked when it is applied: a photo relinked
+/// meanwhile is left alone, and a file that is now another photo's original isn't used.
+#[test]
+fn find_missing_plan_is_rechecked_when_applied() {
+ use crate::cmd::missing::{find_candidates, plan_find_missing};
+ let (dir, mut s) = renamed_and_ambiguous_scene("missing-recheck");
+ let plan = plan_find_missing(&find_candidates(&s.catalog), &dir.join("moved").to_string_lossy()).unwrap();
+ let to = |name: &str| plan.found.iter().find(|f| f.from.ends_with(name)).unwrap().clone();
+ let (a, b) = (to("a.png"), to("b.png"));
+ // meanwhile: a is relinked by hand (elsewhere), and the ambiguous c is pointed at b's file
+ std::fs::copy(&a.to, dir.join("a-elsewhere.png")).unwrap();
+ s.execute("photo.relink", &json!({"id": a.id.0, "path": dir.join("a-elsewhere.png").to_string_lossy()})).unwrap();
+ let c = plan.ambiguous[0].id;
+ s.execute("photo.relink", &json!({"id": c.0, "path": b.to})).unwrap();
+ let r = s.execute("library.findMissing", &plan.to_json()).unwrap();
+ assert_eq!(r["found"].as_array().map(Vec::len), Some(0), "{r}");
+ assert_eq!(r["ambiguous"].as_array().map(Vec::len), Some(0), "c isn't missing any more: {r}");
+ assert_eq!(r["missing"], 1, "b stays missing: its file is c's now: {r}");
+ let path_of = |id: lightcraft_catalog::PhotoId| match &s.catalog.photo(id).unwrap().source {
+ lightcraft_catalog::Source::File { path } => path.clone(),
+ _ => String::new(),
+ };
+ assert_eq!(path_of(a.id), dir.join("a-elsewhere.png").to_string_lossy());
+ assert_eq!(path_of(b.id), b.from);
+ let _ = std::fs::remove_dir_all(&dir);
+}
+
#[test]
fn missing_files_are_found_and_relinked() {
let dir = std::env::temp_dir().join(format!("lc-missing-{}", std::process::id()));
@@ -425,6 +479,20 @@ fn damaged_smart_previews_are_rebuilt_and_failed_writes_leave_none() {
let r = s.execute("library.smartPreviews", &json!({})).unwrap();
assert_eq!((r["built"].as_u64(), r["repaired"].as_u64()), (Some(1), Some(1)), "{r}");
assert_eq!(std::fs::read(&file).unwrap(), full, "rebuilt whole");
+ // the app's background build (issue #104) checks and repairs the same way, on a worker thread
+ std::fs::write(&file, &full[..full.len() / 2]).unwrap();
+ let r = s.execute("library.smartPreviews", &json!({"background": true})).unwrap();
+ assert_eq!(r["what"], "smart previews", "{r}");
+ let t0 = std::time::Instant::now();
+ let r = loop {
+ let r = s.execute("library.previewProgress", &json!({})).unwrap();
+ if r["running"] == false || t0.elapsed() > std::time::Duration::from_secs(60) {
+ break r;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(5));
+ };
+ assert_eq!((r["done"].as_u64(), r["failed"].as_u64(), r["repaired"].as_u64()), (Some(1), Some(0), Some(1)), "{r}");
+ assert_eq!(std::fs::read(&file).unwrap(), full, "rebuilt whole in the background");
let _ = std::fs::remove_dir_all(&src);
let _ = std::fs::remove_dir_all(&lib);
}
diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs
index feaa847c4..9e1f09252 100644
--- a/crates/ui-egui/src/control.rs
+++ b/crates/ui-egui/src/control.rs
@@ -88,6 +88,8 @@ pub fn inspect(app: &LightcraftApp, ctx: &egui::Context) -> Value {
"libraryProblem": app.library_problem.as_ref().map(crate::panels::library_problem::LibraryProblem::to_json),
"scan": app.scan.as_ref().map(crate::import::ScanTask::status),
"export": {"running": app.export.as_ref().map(crate::export_task::ExportTask::status), "last": app.last_export_result},
+ "import": app.import.as_ref().map(crate::import::ImportTask::status),
+ "tasks": app.tasks.labels(),
"memory": memory(app),
})
}
diff --git a/crates/ui-egui/src/headless.rs b/crates/ui-egui/src/headless.rs
index 893c3a408..a8c6e062d 100644
--- a/crates/ui-egui/src/headless.rs
+++ b/crates/ui-egui/src/headless.rs
@@ -176,6 +176,7 @@ impl Headless {
|| self.app.scan.is_some()
|| self.app.import.is_some()
|| self.app.export.is_some()
+ || !self.app.tasks.is_empty()
|| !self.app.synthetic.is_empty()
|| !self.events.is_empty()
}
diff --git a/crates/ui-egui/src/import.rs b/crates/ui-egui/src/import.rs
index a098d26f7..2058eb97f 100644
--- a/crates/ui-egui/src/import.rs
+++ b/crates/ui-egui/src/import.rs
@@ -3,8 +3,9 @@
//! a grid of thumbnails with checkboxes (duplicates marked and unchecked), the destination (add in
//! place / copy or move into the library's `Originals/` or a chosen folder, filed by day, by month,
//! into one folder or by a custom folder template, optionally renamed), an album
-//! (existing or new), a preset and keywords to apply. Importing runs in small batches, one per
-//! frame, with a progress window; the whole import is one undo step.
+//! (existing or new), a preset and keywords to apply. Importing runs on a worker thread (files are
+//! probed, copied or moved there) and its batches join the catalog between frames, with a progress
+//! window and Cancel; the whole import is one undo step.
use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2};
use lightcraft_engine::import::{ImportCandidate, ScanInput, ScanOutput, ScanProgress, scan_with};
@@ -17,7 +18,7 @@ use crate::render::Slot;
use crate::theme::Tokens;
use crate::widgets::register;
-/// Files per batch (one batch per frame, so the progress window updates).
+/// Files per batch (each batch joins the catalog as it is ready, so the progress window updates).
pub(crate) const BATCH: usize = 8;
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
@@ -91,9 +92,13 @@ impl ImportDialog {
}
}
-/// A running import (see the module docs).
+/// A running import (see the module docs). The file-system work — expanding folders, probing,
+/// reading sidecars, copying or placing files — runs on a worker thread
+/// ([`lightcraft_engine::import::ImportJob`]); each batch it readies is added to the catalog on the
+/// UI thread between frames, so a slow drive never stalls the window.
#[derive(Debug, Default)]
pub struct ImportTask {
+ /// Files and folders to import (handed to the worker when the task starts).
queue: Vec<String>,
pub total: usize,
pub done: usize,
@@ -109,6 +114,156 @@ pub struct ImportTask {
/// Reading a folder for the Local view: the photos stay out of the library, and nothing is
/// selected or announced as added.
browse: bool,
+ /// Cancel was pressed: no further files are started; batches already readied are added.
+ pub cancelled: bool,
+ /// Auto Import (the watched folder): the selection stays as it is, a short toast when done.
+ auto: bool,
+ /// Auto Import: the selection to keep.
+ keep_selection: Option<lightcraft_engine::Selection>,
+ run: Option<ImportRun>,
+}
+
+impl ImportTask {
+ /// An import of `paths` (files or folders) with `library.import` params (without `paths`).
+ pub fn new(paths: Vec<String>, params: Value, undo0: usize, browse: bool) -> Self {
+ ImportTask { total: paths.len(), queue: paths, params, undo0, browse, ..Default::default() }
+ }
+
+ /// An Auto Import (see [`ImportTask::auto`]).
+ pub fn auto(mut self) -> Self {
+ self.auto = true;
+ self
+ }
+
+ /// `{done, total, imported, cancelled}` for `ui.inspect`.
+ pub fn status(&self) -> Value {
+ json!({"done": self.done, "total": self.total, "imported": self.imported, "cancelled": self.cancelled})
+ }
+}
+
+/// Where an import's batches are readied: a worker thread, or (browser build: no threads, and no
+/// slow drives) inline, one batch per frame.
+enum Runner {
+ #[cfg(not(target_arch = "wasm32"))]
+ Thread(std::sync::mpsc::Receiver<lightcraft_engine::import::Prepared>),
+ #[cfg(target_arch = "wasm32")]
+ Inline { job: Box<lightcraft_engine::import::ImportJob>, queue: Vec<String>, files: Option<std::collections::VecDeque<String>> },
+}
+
+struct ImportRun {
+ runner: Runner,
+ cancel: std::sync::Arc<std::sync::atomic::AtomicBool>,
+ total: std::sync::Arc<std::sync::atomic::AtomicUsize>,
+ done: std::sync::Arc<std::sync::atomic::AtomicUsize>,
+ opts: lightcraft_engine::import::ImportOptions,
+ now: String,
+ album: Option<u64>,
+ album_name: Option<String>,
+}
+
+impl std::fmt::Debug for ImportRun {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("ImportRun").field("cancel", &self.cancel.load(Ordering::Relaxed)).finish()
+ }
+}
+
+impl Drop for ImportRun {
+ /// A dropped import (replaced, or the app closing) stops its worker at the next file.
+ fn drop(&mut self) {
+ self.cancel.store(true, Ordering::Relaxed);
+ }
+}
+
+impl ImportRun {
+ fn start(
+ job: lightcraft_engine::import::ImportJob,
+ queue: Vec<String>,
+ album: Option<u64>,
+ album_name: Option<String>,
+ ctx: &egui::Context,
+ ) -> Result<Self, String> {
+ let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let (total, done) = (job.total.clone(), job.done.clone());
+ let (opts, now) = (job.opts.clone(), job.now().to_string());
+ let runner = Self::spawn(job, queue, cancel.clone(), ctx)?;
+ Ok(ImportRun { runner, cancel, total, done, opts, now, album, album_name })
+ }
+
+ #[cfg(not(target_arch = "wasm32"))]
+ fn spawn(
+ mut job: lightcraft_engine::import::ImportJob,
+ queue: Vec<String>,
+ cancel: std::sync::Arc<std::sync::atomic::AtomicBool>,
+ ctx: &egui::Context,
+ ) -> Result<Runner, String> {
+ let (tx, rx) = std::sync::mpsc::channel();
+ let ctx = ctx.clone();
+ let work = move || {
+ let files = job.expand(&queue);
+ for chunk in files.chunks(BATCH) {
+ if cancel.load(Ordering::Relaxed) {
+ break;
+ }
+ let prepared = match lightcraft_engine::guard::catch("import", || job.prepare_files(chunk.to_vec(), &cancel)) {
+ Ok(p) => p,
+ Err(_) => break, // logged; the files readied so far are added
+ };
+ if let Err(unsent) = tx.send(prepared) {
+ // the import was dropped: nothing will add these, so take back what a Move placed
+ unsent.0.rollback();
+ break;
+ }
+ ctx.request_repaint();
+ }
+ ctx.request_repaint();
+ };
+ std::thread::Builder::new().name("lc-import".into()).spawn(work).map(|_| Runner::Thread(rx)).map_err(|e| format!("could not start: {e}"))
+ }
+
+ #[cfg(target_arch = "wasm32")]
+ fn spawn(
+ job: lightcraft_engine::import::ImportJob,
+ queue: Vec<String>,
+ _cancel: std::sync::Arc<std::sync::atomic::AtomicBool>,
+ _ctx: &egui::Context,
+ ) -> Result<Runner, String> {
+ Ok(Runner::Inline { job: Box::new(job), queue, files: None })
+ }
+
+ /// Readied batches (at most a few per frame). `None` once the worker is done.
+ fn next_batches(&mut self) -> Option<Vec<lightcraft_engine::import::Prepared>> {
+ match &mut self.runner {
+ #[cfg(not(target_arch = "wasm32"))]
+ Runner::Thread(rx) => {
+ let mut out = Vec::new();
+ while out.len() < 4 {
+ match rx.try_recv() {
+ Ok(p) => out.push(p),
+ Err(std::sync::mpsc::TryRecvError::Empty) => return Some(out),
+ Err(std::sync::mpsc::TryRecvError::Disconnected) => return if out.is_empty() { None } else { Some(out) },
+ }
+ }
+ Some(out)
+ }
+ #[cfg(target_arch = "wasm32")]
+ Runner::Inline { job, queue, files } => {
+ if self.cancel.load(Ordering::Relaxed) {
+ return None;
+ }
+ if files.is_none() {
+ *files = Some(job.expand(queue).into());
+ return Some(Vec::new());
+ }
+ let Some(files) = files.as_mut() else { return None };
+ if files.is_empty() {
+ return None;
+ }
+ let n = files.len().min(BATCH);
+ let batch: Vec<String> = files.drain(..n).collect();
+ Some(vec![job.prepare_files(batch, &self.cancel)])
+ }
+ }
+ }
}
/// A folder scan running on a worker thread (a network share can take minutes to read; the
@@ -232,7 +387,8 @@ pub fn poll_scan(app: &mut LightcraftApp, ctx: &egui::Context) {
if !queue.is_empty() {
let undo0 = app.session.undo.len();
let total = queue.len();
- app.import = Some(ImportTask { queue, total, params: json!({"mode": "add", "local": true}), undo0, browse: true, ..Default::default() });
+ let _ = total;
+ app.import = Some(ImportTask::new(queue, json!({"mode": "add", "local": true}), undo0, true));
}
return;
}
@@ -326,28 +482,82 @@ pub fn start(app: &mut LightcraftApp, d: &ImportDialog) -> Result<Value, String>
}
}
let total = queue.len();
- app.import = Some(ImportTask { queue, total, params, undo0, ..Default::default() });
+ app.import = Some(ImportTask::new(queue, params, undo0, false));
app.renderer.forget_imports();
Ok(json!({"importing": total}))
}
-/// Run one batch of the import in progress (called every frame).
+/// Start importing `paths` (files or folders) in the background, e.g. dropped on the window.
+pub fn start_paths(app: &mut LightcraftApp, paths: Vec<String>) -> Result<Value, String> {
+ if app.import.is_some() || app.scan.as_ref().is_some_and(|t| !t.browse) {
+ return Err("an import is running".into());
+ }
+ let undo0 = app.session.undo.len();
+ let total = paths.len();
+ app.import = Some(ImportTask::new(paths, json!({"mode": "add"}), undo0, false));
+ Ok(json!({"importing": total}))
+}
+
+/// Advance the import in progress (called every frame): start its worker, then add the batches it
+/// has readied to the catalog.
pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) {
- let Some(task) = app.import.as_mut() else { return };
- // (browsing reads a sidecar per file, slow on a network share: keep each frame short)
- let n = task.queue.len().min(if task.browse { 2 } else { BATCH });
- let batch: Vec<String> = task.queue.drain(..n).collect();
- let mut p = task.params.clone();
- p["paths"] = json!(batch);
- // renamed copies keep counting across batches
- if p.get("rename").is_some() {
- p["renameStart"] = json!(1 + task.imported);
- }
- let r = app.session.execute("library.import", &p);
- let Some(task) = app.import.as_mut() else { return };
- task.done += n;
+ let Some(mut task) = app.import.take() else { return };
+ if task.run.is_none() {
+ let mut p = task.params.clone();
+ p["paths"] = json!(task.queue);
+ let started = lightcraft_engine::cmd::library::import_params(&app.session, &p)
+ .and_then(|req| Ok((lightcraft_engine::import::ImportJob::new(&mut app.session, req.opts)?, req.album, req.album_name)))
+ .map_err(|e| e.to_string())
+ .and_then(|(job, album, name)| ImportRun::start(job, std::mem::take(&mut task.queue), album, name, ctx));
+ if task.auto {
+ task.keep_selection = Some(app.session.selection.clone());
+ }
+ match started {
+ Ok(run) => task.run = Some(run),
+ Err(e) => {
+ log::warn!("import: {e}");
+ app.toast(ctx, format!("Import failed: {e}"));
+ return;
+ }
+ }
+ }
+ ctx.request_repaint_after(std::time::Duration::from_millis(100));
+ let Some(run) = task.run.as_mut() else { return };
+ let batches = run.next_batches();
+ let finished = batches.is_none();
+ for prepared in batches.into_iter().flatten() {
+ commit_batch(app, &mut task, prepared);
+ // arriving photos don't take over the selection
+ if let Some(sel) = &task.keep_selection {
+ app.session.selection = sel.clone();
+ }
+ }
+ if let Some(run) = &task.run {
+ task.total = run.total.load(Ordering::Relaxed).max(task.total);
+ task.done = run.done.load(Ordering::Relaxed);
+ }
+ if !finished {
+ app.import = Some(task);
+ return;
+ }
+ finish(app, ctx, task);
+}
+
+/// Add one readied batch to the catalog (an undo step merged into the import's at the end).
+fn commit_batch(app: &mut LightcraftApp, task: &mut ImportTask, prepared: lightcraft_engine::import::Prepared) {
+ let Some(run) = task.run.as_ref() else { return };
+ let n = prepared.len();
+ let (opts, now, album, album_name) = (&run.opts, run.now.as_str(), run.album, run.album_name.clone());
+ let r = app.session.execute_fn("library.import", |s| {
+ let report = lightcraft_engine::import::commit_prepared(s, opts, now, prepared)?;
+ lightcraft_engine::cmd::library::import_batch_done(s, report, album, album_name.as_deref())
+ });
match r {
Ok(v) => {
+ // a new album (named in the params) is created with the first photos; later batches join it
+ if let (Some(a), Some(run)) = (v["album"].as_u64(), task.run.as_mut()) {
+ run.album = Some(a);
+ }
let len = |k: &str| v[k].as_array().map_or(0, Vec::len);
task.imported += len("imported");
task.duplicates += len("duplicates");
@@ -366,14 +576,19 @@ pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) {
task.failed += n;
}
}
- ctx.request_repaint();
- if !task.queue.is_empty() {
- return;
- }
- let Some(task) = app.import.take() else { return };
+}
+
+/// The import is done (or cancelled): one undo step, select the first photo, say what happened.
+fn finish(app: &mut LightcraftApp, ctx: &egui::Context, task: ImportTask) {
let steps = app.session.undo.len().saturating_sub(task.undo0);
let label = format!("Add {} Photo{}", task.imported, if task.imported == 1 { "" } else { "s" });
app.session.merge_undo(steps, &label);
+ if task.auto {
+ if task.imported > 0 {
+ app.toast(ctx, format!("Auto Import: added {} photo{}", task.imported, if task.imported == 1 { "" } else { "s" }));
+ }
+ return;
+ }
if task.browse {
if task.failed > 0 {
app.toast(ctx, format!("{} photo{} not readable", task.failed, if task.failed == 1 { "" } else { "s" }));
@@ -384,7 +599,11 @@ pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) {
let _ = app.run("library.select", json!({"ids": [f]}));
}
let plural = |n: usize| if n == 1 { "" } else { "s" };
- let mut msg = format!("Imported {} photo{}", task.imported, plural(task.imported));
+ let mut msg = if task.cancelled {
+ format!("Import cancelled · {} photo{} added", task.imported, plural(task.imported))
+ } else {
+ format!("Imported {} photo{}", task.imported, plural(task.imported))
+ };
if task.params["mode"] == "move" {
msg.push_str(&format!(" · {} moved", task.moved));
if task.kept > 0 {
@@ -400,19 +619,34 @@ pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) {
app.toast(ctx, msg);
}
-/// The progress window while an import runs.
+/// The progress window while an import runs, with Cancel (files already copied or added stay;
+/// nothing new is started).
pub fn progress(app: &mut LightcraftApp, ctx: &egui::Context) {
let Some(task) = &app.import else { return };
let t = Tokens::get(ctx);
let frac = task.done as f32 / task.total.max(1) as f32;
- let text = format!("{} photos… {} of {}", if task.browse { "Reading" } else { "Adding" }, task.done, task.total);
- egui::Window::new("Importing").title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([340.0, 60.0]).show(
+ let text = if task.cancelled {
+ "Stopping…".to_string()
+ } else {
+ format!("{} photos… {} of {}", if task.browse { "Reading" } else { "Adding" }, task.done, task.total)
+ };
+ let mut cancel = false;
+ egui::Window::new("Importing").title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([340.0, 80.0]).show(
ctx,
|ui| {
ui.label(egui::RichText::new(text).color(t.text));
ui.add(egui::ProgressBar::new(frac).desired_width(320.0));
+ let r = ui.add_enabled(!task.cancelled, egui::Button::new("Cancel"));
+ register(ui.ctx(), "button:importCancel", r.rect);
+ cancel = r.clicked();
},
);
+ if cancel && let Some(task) = app.import.as_mut() {
+ task.cancelled = true;
+ if let Some(run) = &task.run {
+ run.cancel.store(true, Ordering::Relaxed);
+ }
+ }
}
/// The dialog body: options, then the candidate grid.
@@ -458,7 +692,13 @@ pub fn body(app: &mut LightcraftApp, ui: &mut egui::Ui, d: &mut ImportDialog) {
// options
if !d.sources.is_empty() {
field(ui, "Source", |ui| {
- let r = ui.label(egui::RichText::new(source_summary(&d.sources)).color(t.text_label)).on_hover_text(d.sources.join("\n"));
+ // (whether each source is a folder is checked off the UI thread, once)
+ let joined = d.sources.join("\n");
+ let summary = crate::panels::left::fs_cached(ui, "import-source", &joined, f64::INFINITY, |s| {
+ source_summary(&s.split('\n').map(str::to_string).collect::<Vec<_>>())
+ })
+ .unwrap_or_else(|| format!("{} item{}", d.sources.len(), if d.sources.len() == 1 { "" } else { "s" }));
+ let r = ui.label(egui::RichText::new(summary).color(t.text_label)).on_hover_text(joined);
register(ui.ctx(), "label:importSource", r.rect);
});
ui.label(
diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs
index b6ea924c4..2e478d459 100644
--- a/crates/ui-egui/src/lib.rs
+++ b/crates/ui-egui/src/lib.rs
@@ -21,6 +21,7 @@ pub mod render;
pub mod shortcuts;
pub mod softpaint;
pub mod state;
+pub mod tasks;
pub mod theme;
pub mod widgets;
@@ -33,6 +34,8 @@ mod tests_library_problem;
#[cfg(test)]
mod tests_masking;
#[cfg(test)]
+mod tests_offline;
+#[cfg(test)]
mod tests_panels;
#[cfg(test)]
mod tests_quit_unsaved;
@@ -169,6 +172,8 @@ pub struct LightcraftApp {
pub scan: Option<import::ScanTask>,
/// A background export in progress.
pub export: Option<export_task::ExportTask>,
+ /// Background file-system work of other commands (Find Missing Photos, auto import…).
+ pub tasks: tasks::Tasks,
/// The files of the last finished background export (`ui.inspect` → `export.last`).
pub last_export_result: Option<Value>,
/// The look the loupe shows while the pointer rests on a preset or profile (set by the
@@ -224,6 +229,7 @@ impl LightcraftApp {
import: None,
scan: None,
export: None,
+ tasks: Default::default(),
last_export_result: None,
hover_preview: None,
window_is_fullscreen: false,
@@ -286,7 +292,12 @@ impl LightcraftApp {
if self.ui.preview_build_seen != Some((key, true)) {
self.ui.preview_build_seen = Some((key, true));
let (done, failed) = (b.done.load(Ordering::Relaxed), b.failed.load(Ordering::Relaxed));
- let mut msg = format!("Previews ready for {done} photo{}", if done == 1 { "" } else { "s" });
+ let plural = if done == 1 { "" } else { "s" };
+ let mut msg = match (b.error(), b.what) {
+ (Some(e), what) => format!("{}: {e}", if what.is_empty() { "previews" } else { what }),
+ (None, "") => format!("Previews ready for {done} photo{plural}"),
+ (None, what) => format!("Done ({what}): {done} photo{plural}"),
+ };
if failed > 0 {
msg.push_str(&format!(" · {failed} couldn't be rendered"));
}
@@ -295,7 +306,11 @@ impl LightcraftApp {
} else {
if self.ui.preview_build_seen != Some((key, false)) {
self.ui.preview_build_seen = Some((key, false));
- self.toast(ctx, format!("Building previews for {} photos…", b.total));
+ let msg = match b.what {
+ "" => format!("Building previews for {} photos…", b.total),
+ what => format!("Working on {what} for {} photos…", b.total),
+ };
+ self.toast(ctx, msg);
}
ctx.request_repaint_after(std::time::Duration::from_millis(250));
}
@@ -469,6 +484,10 @@ impl LightcraftApp {
// File → Add from Device lists cards scanned in the background: show hot-plugs
let repaint = ctx.clone();
lightcraft_engine::devices::on_change(move || repaint.request_repaint());
+ // "is the original there?" (grid, Info panel, Missing Photos) answers from a cache a
+ // worker fills: a sleeping NAS or a dropped share never stalls a frame
+ let repaint = ctx.clone();
+ self.session.media.availability.run_in_background(std::sync::Arc::new(move || repaint.request_repaint()));
self.styled = true;
} else {
self.fonts_ready = true;
@@ -493,6 +512,7 @@ impl LightcraftApp {
merge::poll(self, ctx);
import::poll_scan(self, ctx);
import::tick(self, ctx);
+ tasks::poll(self, ctx);
self.preview_build_status(ctx);
self.save_status(ctx);
self.slideshow_tick(ctx);
@@ -507,17 +527,33 @@ impl LightcraftApp {
}
}
self.ui.was_focused = focused;
- // auto import: scan the watched folder every few seconds
+ // auto import: list the watched folder every few seconds (on a worker thread: it may be on
+ // a network share), then import what's new like any import (also on a worker thread)
#[cfg(not(target_arch = "wasm32"))]
- if self.session.import_defaults.auto_folder.is_some() {
+ if let Some(folder) = self.session.import_defaults.auto_folder.clone() {
+ const LABEL: &str = "Auto Import";
let now = ctx.input(|i| i.time);
- if now - self.ui.auto_import_at >= 3.0 {
+ if now - self.ui.auto_import_at >= 3.0 && self.import.is_none() && !self.tasks.is_running(LABEL) {
self.ui.auto_import_at = now;
- if let Ok(r) = self.session.execute("library.autoImportScan", &serde_json::json!({})) {
- let n = r["imported"].as_array().map_or(0, Vec::len);
- if n > 0 {
- self.toast(ctx, format!("Auto Import: added {n} photo{}", if n == 1 { "" } else { "s" }));
+ let work = move || lightcraft_engine::cmd::library::list_auto_import_folder(&folder);
+ let done = |app: &mut LightcraftApp, _ctx: &egui::Context, listing: Result<Vec<(String, u64)>, String>| {
+ let listing = match listing {
+ Ok(l) => l,
+ Err(e) => return log::debug!("auto import: {e}"),
+ };
+ let p = serde_json::json!({"listing": listing, "start": false});
+ let Ok(r) = app.session.execute("library.autoImportScan", &p) else { return };
+ let Some(mut params) = r.get("import").cloned().filter(|_| app.import.is_none()) else { return };
+ let paths: Vec<String> =
+ params["paths"].as_array().into_iter().flatten().filter_map(|v| v.as_str().map(str::to_string)).collect();
+ if let Some(o) = params.as_object_mut() {
+ o.remove("paths");
}
+ let undo0 = app.session.undo.len();
+ app.import = Some(import::ImportTask::new(paths, params, undo0, false).auto());
+ };
+ if let Err(e) = tasks::spawn(self, LABEL, work, done) {
+ log::warn!("{e}");
}
}
ctx.request_repaint_after(std::time::Duration::from_secs(3));
@@ -537,8 +573,11 @@ impl LightcraftApp {
if !presets.is_empty() {
let _ = self.run("file.importPresets", serde_json::json!({"paths": presets}));
}
- if !photos.is_empty() {
- let _ = self.run("library.import", serde_json::json!({"paths": photos}));
+ // read and added on a worker thread (dropped folders can be large, or on a slow drive)
+ if !photos.is_empty()
+ && let Err(e) = import::start_paths(self, photos)
+ {
+ self.toast(ctx, e);
}
}
}
diff --git a/crates/ui-egui/src/menubar.rs b/crates/ui-egui/src/menubar.rs
index 71571d326..ab92acebb 100644
--- a/crates/ui-egui/src/menubar.rs
+++ b/crates/ui-egui/src/menubar.rs
@@ -325,20 +325,21 @@ fn expanded(app: &LightcraftApp, name: &str) -> Option<Vec<MenuNode>> {
item("library.buildPreviews", json!({"size": "full"}), format!("Build 1:1 Previews ({scope})"), None, !running, None),
item("library.cancelPreviews", Value::Null, "Stop Building Previews", None, running, None),
MenuNode::Separator,
+ // (read and written on a worker thread: the originals may be on a slow drive)
item(
"library.smartPreviews",
- Value::Null,
+ json!({"background": true}),
format!("Build Smart Previews ({scope})"),
None,
- app.session.media.smart_dir.is_some(),
+ app.session.media.smart_dir.is_some() && !running,
None,
),
item(
"library.smartPreviews",
- json!({"discard": true}),
+ json!({"discard": true, "background": true}),
format!("Discard Smart Previews ({scope})"),
None,
- app.session.media.smart_dir.is_some(),
+ app.session.media.smart_dir.is_some() && !running,
None,
),
MenuNode::Separator,
@@ -936,7 +937,7 @@ mod tests {
let mut app = LightcraftApp::new(lightcraft_engine::Session::new().with_fs(), Default::default());
app.session.execute("library.import", &json!({"paths": [dir.join("a").to_string_lossy()]})).unwrap();
std::fs::rename(dir.join("a/one.png"), dir.join("b/one.png")).unwrap();
- let r = run_item(&mut app, "file.findMissing", json!({"folder": dir.join("b").to_string_lossy()})).unwrap();
+ let r = run_item(&mut app, "file.findMissing", json!({"folder": dir.join("b").to_string_lossy(), "wait": true})).unwrap();
assert_eq!(r["found"].as_array().map(Vec::len), Some(1), "{r}");
// Locate: an explicit file
std::fs::rename(dir.join("b/one.png"), dir.join("one-renamed.png")).unwrap();
diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs
index b4ef0ab3c..32b1dfb3c 100644
--- a/crates/ui-egui/src/menus.rs
+++ b/crates/ui-egui/src/menus.rs
@@ -928,18 +928,40 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option<Re
None => app.services.pick_folder.as_mut().and_then(|f| f()),
};
let Some(folder) = folder else { return Some(Ok(Value::Null)) };
- let r = app.run("library.findMissing", json!({"folder": folder}));
- if let Ok(v) = &r {
- let n = v["found"].as_array().map_or(0, Vec::len);
- let left = v["missing"].as_u64().unwrap_or(0);
- let unsure = v["ambiguous"].as_array().map_or(0, Vec::len);
- let unsure = if unsure > 0 { format!(" ({unsure} with several look-alike files: use Locate)") } else { String::new() };
- app.toast(
- &egui::Context::default(),
- format!("Found {n} missing photo{}; {left} still missing{unsure}", if n == 1 { "" } else { "s" }),
- );
+ // the search (checking every photo's file, walking the folder) runs on a worker thread;
+ // the relinking happens back here, as one undo step
+ const LABEL: &str = "Find Missing Photos";
+ if app.tasks.is_running(LABEL) {
+ return Some(Err("Find Missing Photos is already searching".into()));
}
- r
+ let candidates = lightcraft_engine::cmd::missing::find_candidates(&app.session.catalog);
+ let work = move || lightcraft_engine::cmd::missing::plan_find_missing(&candidates, &folder);
+ let done = |app: &mut LightcraftApp, ctx: &egui::Context, plan: Result<lightcraft_engine::cmd::missing::FindPlan, String>| {
+ // relinked under the session as it is now: photos relinked meanwhile and files
+ // now in use are skipped
+ let r = plan.and_then(|plan| app.run("library.findMissing", plan.to_json()));
+ match r {
+ Ok(v) => {
+ let n = v["found"].as_array().map_or(0, Vec::len);
+ let left = v["missing"].as_u64().unwrap_or(0);
+ let unsure = v["ambiguous"].as_array().map_or(0, Vec::len);
+ let unsure = if unsure > 0 { format!(" ({unsure} with several look-alike files: use Locate)") } else { String::new() };
+ app.toast(ctx, format!("Found {n} missing photo{}; {left} still missing{unsure}", if n == 1 { "" } else { "s" }));
+ app.ui.last_find_missing = Some(v);
+ }
+ Err(e) => app.toast(ctx, e),
+ }
+ };
+ app.ui.last_find_missing = None;
+ if let Err(e) = crate::tasks::spawn(app, LABEL, work, done) {
+ return Some(Err(e));
+ }
+ if p.get("wait").and_then(Value::as_bool).unwrap_or(false) {
+ let ctx = app.tasks.repaint.clone().unwrap_or_default();
+ crate::tasks::wait(app, &ctx, std::time::Duration::from_secs(600));
+ return Some(Ok(app.ui.last_find_missing.clone().unwrap_or(Value::Null)));
+ }
+ Ok(json!({"background": true}))
}
"photo.tagFromTracklog" => {
// a GPX file → GPS for the selected photos by capture time (one undo step)
diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs
index da1b386dd..31ffb7865 100644
--- a/crates/ui-egui/src/panels/dialogs.rs
+++ b/crates/ui-egui/src/panels/dialogs.rs
@@ -7,6 +7,53 @@ use crate::LightcraftApp;
use crate::state::Dialog;
use crate::theme::Tokens;
+/// Rows the Rename dialog previews.
+const RENAME_PREVIEW_ROWS: usize = 6;
+
+/// The Rename dialog's preview: its first rows (`None` while being planned) and how many files
+/// are renamed. Which names are taken is checked on disk, which can block on a slow drive, so the
+/// rows are planned on a worker thread, and only when the template, start number, photos or
+/// catalog change — never once per frame.
+pub(crate) fn rename_preview(
+ app: &mut LightcraftApp,
+ ctx: &egui::Context,
+ template: &str,
+ start: usize,
+) -> (Option<Vec<lightcraft_engine::rename::RenamePlan>>, usize) {
+ use std::hash::{Hash, Hasher};
+ type Rows = std::sync::Arc<std::sync::Mutex<Option<Vec<lightcraft_engine::rename::RenamePlan>>>>;
+ let ids = app.session.targets(&json!({}));
+ let photos = lightcraft_engine::rename::rename_photos(&app.session.catalog, &ids);
+ let total = photos.len();
+ let mut h = std::collections::hash_map::DefaultHasher::new();
+ (template, start, app.session.catalog.revision, &ids).hash(&mut h);
+ let key = h.finish();
+ let id = egui::Id::new("rename-preview-rows");
+ let read = |rows: &Rows| rows.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone();
+ if let Some((k, rows)) = ctx.data(|d| d.get_temp::<(u64, Rows)>(id))
+ && k == key
+ {
+ return (read(&rows), total);
+ }
+ let rows: Rows = Default::default();
+ let first: Vec<_> = photos.into_iter().take(RENAME_PREVIEW_ROWS).collect();
+ let (out, template, repaint) = (rows.clone(), template.to_string(), ctx.clone());
+ let exists = app.session.media.availability.probe();
+ let work = move || {
+ let plans = lightcraft_engine::rename::plan_rename_photos(&first, &template, start, &|f| exists(f));
+ *out.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(plans);
+ repaint.request_repaint();
+ };
+ #[cfg(not(target_arch = "wasm32"))]
+ if let Err(e) = std::thread::Builder::new().name("lc-rename-preview".into()).spawn(work) {
+ log::warn!("rename preview: {e}");
+ }
+ #[cfg(target_arch = "wasm32")]
+ work();
+ ctx.data_mut(|d| d.insert_temp(id, (key, rows.clone())));
+ (read(&rows), total)
+}
+
/// Help ▸ What's New (docs/whats-new.md).
pub const WHATS_NEW: &str = include_str!("../../../../docs/whats-new.md");
@@ -316,16 +363,23 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {
egui::RichText::new("Tags: see Tags beside the template. Files are renamed on disk (with their XMP sidecars); existing names get -1, -2…")
.color(t.text_dim),
);
- let preview = app.session.execute("photo.renamePreview", &json!({"template": template, "start": start})).unwrap_or_default();
- egui::Grid::new("rename-preview").num_columns(3).spacing([8.0, 2.0]).show(ui, |ui| {
- for pl in preview.as_array().into_iter().flatten().take(6) {
- ui.label(egui::RichText::new(pl["from"].as_str().unwrap_or("")).color(t.text_dim));
- ui.label(egui::RichText::new("→").color(t.text_dim));
- ui.label(egui::RichText::new(pl["to"].as_str().unwrap_or("")).color(t.text));
- ui.end_row();
+ let (preview, total) = rename_preview(app, ui.ctx(), template, *start as usize);
+ match &preview {
+ Some(rows) => {
+ egui::Grid::new("rename-preview").num_columns(3).spacing([8.0, 2.0]).show(ui, |ui| {
+ for pl in rows {
+ ui.label(egui::RichText::new(&pl.from).color(t.text_dim));
+ ui.label(egui::RichText::new("→").color(t.text_dim));
+ ui.label(egui::RichText::new(&pl.to).color(t.text));
+ ui.end_row();
+ }
+ });
}
- });
- let more = preview.as_array().map_or(0, Vec::len).saturating_sub(6);
+ None => {
+ ui.label(egui::RichText::new("Checking names…").color(t.text_dim));
+ }
+ }
+ let more = total.saturating_sub(RENAME_PREVIEW_ROWS);
if more > 0 {
ui.label(egui::RichText::new(format!("… and {more} more")).color(t.text_dim));
}
diff --git a/crates/ui-egui/src/panels/grid.rs b/crates/ui-egui/src/panels/grid.rs
index eda9e6db6..dcebe199f 100644
--- a/crates/ui-egui/src/panels/grid.rs
+++ b/crates/ui-egui/src/panels/grid.rs
@@ -867,7 +867,8 @@ pub fn context_menu(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
}
ui.separator();
// the original moved or its drive is gone: point the photo at the file again
- let missing = matches!(&app.session.catalog.photo(id).map(|p| p.source.clone()), Some(lightcraft_catalog::Source::File { path }) if !std::path::Path::new(path).exists());
+ // (a cached answer, checked off the UI thread)
+ let missing = matches!(&app.session.catalog.photo(id).map(|p| p.source.clone()), Some(lightcraft_catalog::Source::File { path }) if app.session.media.availability.is_offline(path));
if missing && ui.button("Locate Missing File…").clicked() {
let _ = app.run("photo.locate", json!({}));
}
diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs
index 941e5070f..5b96941c8 100644
--- a/crates/ui-egui/src/panels/left.rs
+++ b/crates/ui-egui/src/panels/left.rs
@@ -200,7 +200,8 @@ fn local_section(app: &mut LightcraftApp, ui: &mut egui::Ui) {
for (name, sub) in [("Pictures", "Pictures"), ("Desktop", "Desktop"), ("Downloads", "Downloads"), ("Home", "")] {
// joined with the platform's separator, like the paths browsing gives back
let p = if sub.is_empty() { home.clone() } else { std::path::Path::new(&home).join(sub).to_string_lossy().to_string() };
- if std::path::Path::new(&p).is_dir() {
+ // (checked off the UI thread: a home folder can be on a network share)
+ if fs_cached(ui, "is-dir", &p, 5.0, |p| std::path::Path::new(p).is_dir()) == Some(true) {
builtin.push((name.to_string(), p));
}
}
@@ -305,15 +306,13 @@ pub(crate) fn local_places(
out
}
-/// The subfolders of `path` (not hidden ones), sorted; listed at most every 2 s per folder.
+/// The subfolders of `path` (not hidden ones), sorted; listed on a worker thread at most every
+/// 2 s per folder (empty until the first listing).
fn subfolders(ui: &egui::Ui, path: &str) -> Vec<(String, String)> {
- let id = egui::Id::new(("subfolders", path.to_string()));
- let now = ui.input(|i| i.time);
- if let Some((t, v)) = ui.data(|d| d.get_temp::<(f64, Vec<(String, String)>)>(id))
- && now - t < 2.0
- {
- return v;
- }
+ fs_cached(ui, "subfolders", path, 2.0, list_subfolders).unwrap_or_default()
+}
+
+fn list_subfolders(path: &str) -> Vec<(String, String)> {
let mut v: Vec<(String, String)> = std::fs::read_dir(path)
.map(|rd| {
rd.flatten()
@@ -326,10 +325,59 @@ fn subfolders(ui: &egui::Ui, path: &str) -> Vec<(String, String)> {
})
.unwrap_or_default();
v.sort_by_key(|(n, _)| n.to_lowercase());
- ui.data_mut(|d| d.insert_temp(id, (now, v.clone())));
v
}
+/// A file-system answer for `path` (`f(path)`), kept per `kind` and path and refreshed on a worker
+/// thread at most every `every` seconds: a folder on a sleeping NAS, a dropped share or a
+/// spinning-up drive never blocks a frame. `None` until the first answer arrives.
+pub(crate) fn fs_cached<T: Clone + Send + 'static>(ui: &egui::Ui, kind: &'static str, path: &str, every: f64, f: fn(&str) -> T) -> Option<T> {
+ struct Entry<T> {
+ value: Option<T>,
+ at: Option<f64>,
+ running: bool,
+ }
+ type Cell<T> = std::sync::Arc<std::sync::Mutex<Entry<T>>>;
+ let id = egui::Id::new(("fs-cached", kind, path.to_string()));
+ let now = ui.input(|i| i.time);
+ let cell: Cell<T> = match ui.data(|d| d.get_temp::<Cell<T>>(id)) {
+ Some(c) => c,
+ None => {
+ let c: Cell<T> = std::sync::Arc::new(std::sync::Mutex::new(Entry { value: None, at: None, running: false }));
+ ui.data_mut(|d| d.insert_temp(id, c.clone()));
+ c
+ }
+ };
+ let lock = |c: &Cell<T>| c.lock().unwrap_or_else(std::sync::PoisonError::into_inner).value.clone();
+ let start = {
+ let mut e = cell.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+ let due = !e.running && e.at.is_none_or(|at| now - at >= every || now < at);
+ if due {
+ e.running = true;
+ e.at = Some(now);
+ }
+ due
+ };
+ if start {
+ let (out, path, repaint) = (cell.clone(), path.to_string(), ui.ctx().clone());
+ let work = move || {
+ let v = f(&path);
+ let mut e = out.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+ e.value = Some(v);
+ e.running = false;
+ drop(e);
+ repaint.request_repaint();
+ };
+ #[cfg(not(target_arch = "wasm32"))]
+ if std::thread::Builder::new().name("lc-fs-list".into()).spawn(work).is_err() {
+ cell.lock().unwrap_or_else(std::sync::PoisonError::into_inner).running = false;
+ }
+ #[cfg(target_arch = "wasm32")]
+ work();
+ }
+ lock(&cell)
+}
+
/// A folder on disk with a disclosure triangle: click browses it, the triangle lists its
/// subfolders. In the tree that holds the folder being browsed (`reveal`), the folders on the
/// way down to it open whenever the browsed folder changes, so it shows highlighted in place.
diff --git a/crates/ui-egui/src/panels/right.rs b/crates/ui-egui/src/panels/right.rs
index b2cef17fc..a4b27e328 100644
--- a/crates/ui-egui/src/panels/right.rs
+++ b/crates/ui-egui/src/panels/right.rs
@@ -535,9 +535,16 @@ fn info(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
ui.label(egui::RichText::new(line).color(t.text_dim));
}
// offline originals / smart previews
+ // (cached answers, checked off the UI thread: unknown counts as online / no smart preview)
if let lightcraft_catalog::Source::File { path } = &p.source {
- let online = std::path::Path::new(path).exists();
- let smart = lightcraft_engine::cmd::previews::has_smart_preview(&app.session, p.id);
+ let avail = &app.session.media.availability;
+ let online = !avail.is_offline(path);
+ let smart = app
+ .session
+ .media
+ .smart_dir
+ .as_ref()
+ .is_some_and(|d| avail.exists(&d.join(lightcraft_engine::smart::file_name(&p)).to_string_lossy()) == Some(true));
if !online || smart {
let text = match (online, smart) {
(false, true) => "Original offline · editing the smart preview",
diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs
index c585c6a74..e295dee37 100644
--- a/crates/ui-egui/src/state.rs
+++ b/crates/ui-egui/src/state.rs
@@ -359,6 +359,9 @@ pub struct UiState {
/// Transient toast text and its expiry (seconds of app time).
#[serde(skip)]
pub toast: Option<(String, f64)>,
+ /// The result of the last Find Missing Photos (it searches in the background).
+ #[serde(skip)]
+ pub last_find_missing: Option<serde_json::Value>,
#[serde(skip)]
pub status: String,
#[serde(skip)]
@@ -589,6 +592,7 @@ impl Default for UiState {
compare: None,
reference: None,
toast: None,
+ last_find_missing: None,
status: String::new(),
dialog: None,
}
diff --git a/crates/ui-egui/src/tasks.rs b/crates/ui-egui/src/tasks.rs
new file mode 100644
index 000000000..2cf526acf
--- /dev/null
+++ b/crates/ui-egui/src/tasks.rs
@@ -0,0 +1,112 @@
+//! Background tasks for commands whose file-system work can take long on a slow or offline drive
+//! (Find Missing Photos walking a folder, listing the auto-import folder…): the work runs on a
+//! worker thread and its result is applied on the UI thread between frames, so the window keeps
+//! answering. Imports, scans, exports and preview builds have their own tasks with progress.
+
+use std::sync::mpsc::{Receiver, TryRecvError, channel};
+
+use crate::LightcraftApp;
+
+/// Applies a task's result to the app (on the UI thread).
+type Finish = Box<dyn FnOnce(&mut LightcraftApp, &egui::Context) + Send>;
+
+struct Task {
+ label: String,
+ rx: Receiver<Finish>,
+}
+
+/// The background tasks in flight.
+#[derive(Default)]
+pub struct Tasks {
+ running: Vec<Task>,
+ /// Repaints the window when a task finishes (set every frame).
+ pub(crate) repaint: Option<egui::Context>,
+}
+
+impl Tasks {
+ pub fn is_empty(&self) -> bool {
+ self.running.is_empty()
+ }
+
+ /// What is running (`ui.inspect` → `tasks`).
+ pub fn labels(&self) -> Vec<String> {
+ self.running.iter().map(|t| t.label.clone()).collect()
+ }
+
+ pub fn is_running(&self, label: &str) -> bool {
+ self.running.iter().any(|t| t.label == label)
+ }
+}
+
+/// Run `work` on a worker thread, then `done(app, ctx, result)` on the UI thread (in the browser
+/// build, which has no threads, `work` runs at once and `done` on the next frame).
+pub fn spawn<T: Send + 'static>(
+ app: &mut LightcraftApp,
+ label: &str,
+ work: impl FnOnce() -> T + Send + 'static,
+ done: impl FnOnce(&mut LightcraftApp, &egui::Context, T) + Send + 'static,
+) -> Result<(), String> {
+ let (tx, rx) = channel::<Finish>();
+ let repaint = app.tasks.repaint.clone();
+ let job = move || {
+ let t = work();
+ let _ = tx.send(Box::new(move |app: &mut LightcraftApp, ctx: &egui::Context| done(app, ctx, t)));
+ if let Some(c) = repaint {
+ c.request_repaint();
+ }
+ };
+ #[cfg(not(target_arch = "wasm32"))]
+ std::thread::Builder::new().name(format!("lc-task-{label}")).spawn(job).map_err(|e| format!("{label}: could not start: {e}"))?;
+ #[cfg(target_arch = "wasm32")]
+ job();
+ app.tasks.running.push(Task { label: label.to_string(), rx });
+ Ok(())
+}
+
+/// Apply finished tasks (called every frame).
+pub fn poll(app: &mut LightcraftApp, ctx: &egui::Context) {
+ app.tasks.repaint = Some(ctx.clone());
+ let mut i = 0;
+ while i < app.tasks.running.len() {
+ match app.tasks.running[i].rx.try_recv() {
+ Ok(finish) => {
+ app.tasks.running.remove(i);
+ finish(app, ctx);
+ }
+ Err(TryRecvError::Empty) => i += 1,
+ Err(TryRecvError::Disconnected) => {
+ // the worker died (a panic, already logged)
+ let t = app.tasks.running.remove(i);
+ app.toast(ctx, format!("{} failed", t.label));
+ }
+ }
+ }
+ if !app.tasks.is_empty() {
+ ctx.request_repaint_after(std::time::Duration::from_millis(100));
+ }
+}
+
+/// Wait for every task and apply it (tests, and commands asked to `wait`). `false` on timeout.
+pub fn wait(app: &mut LightcraftApp, ctx: &egui::Context, timeout: std::time::Duration) -> bool {
+ #[cfg(not(target_arch = "wasm32"))]
+ {
+ let t0 = std::time::Instant::now();
+ loop {
+ poll(app, ctx);
+ if app.tasks.is_empty() {
+ return true;
+ }
+ if t0.elapsed() > timeout {
+ return false;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(2));
+ }
+ }
+ #[cfg(target_arch = "wasm32")]
+ {
+ // tasks ran inline: their results are already waiting
+ let _ = timeout;
+ poll(app, ctx);
+ app.tasks.is_empty()
+ }
+}
diff --git a/crates/ui-egui/src/tests_offline.rs b/crates/ui-egui/src/tests_offline.rs
new file mode 100644
index 000000000..f682fba57
--- /dev/null
+++ b/crates/ui-egui/src/tests_offline.rs
@@ -0,0 +1,177 @@
+//! Slow or offline drives never stall a frame (issue #104): with an injected file-system probe
+//! that takes far longer than a frame (a sleeping NAS), the grid, the Info panel, the Missing
+//! Photos view, the Rename dialog and an import keep drawing frames quickly, the probe never runs
+//! on the UI thread, and the background answers still arrive.
+
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex};
+use std::thread::ThreadId;
+use std::time::{Duration, Instant};
+
+use lightcraft_catalog::{Op, Photo, PhotoId, Source};
+use lightcraft_engine::{LibrarySource, Session};
+use serde_json::json;
+
+use crate::headless::Headless;
+use crate::state::{RightPanel, ViewMode};
+use crate::{LightcraftApp, Services};
+
+/// What one file-system call takes on the "NAS".
+const SLOW: Duration = Duration::from_millis(400);
+/// A frame that waited for one such call would take at least this long.
+const FRAME_LIMIT: Duration = Duration::from_millis(300);
+const N: u64 = 12;
+
+/// Threads the probe ran on, and how often.
+#[derive(Clone, Default)]
+struct Calls {
+ threads: Arc<Mutex<Vec<ThreadId>>>,
+ n: Arc<AtomicUsize>,
+}
+
+impl Calls {
+ fn record(&self) {
+ self.threads.lock().unwrap().push(std::thread::current().id());
+ self.n.fetch_add(1, Ordering::SeqCst);
+ }
+ fn on(&self, t: ThreadId) -> usize {
+ self.threads.lock().unwrap().iter().filter(|x| **x == t).count()
+ }
+}
+
+/// `N` photos whose originals are on an unreachable share, a smart previews folder, and a probe
+/// that sleeps like a NAS waking up before saying "not there".
+fn offline_library(calls: &Calls) -> Session {
+ let mut s = Session::new();
+ for i in 0..N {
+ let path = format!("/lightcraft-offline-nas/IMG_{i:03}.jpg");
+ let p = Photo::new(PhotoId(i + 1), Source::File { path }, &format!("IMG_{i:03}.jpg"), "JPEG", 600, 400, "2026-01-01T00:00:00");
+ s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
+ }
+ s.media.smart_dir = Some(std::env::temp_dir().join("lightcraft-offline-smart-previews-none"));
+ let c = calls.clone();
+ s.media.availability.set_probe(Arc::new(move |_| {
+ c.record();
+ std::thread::sleep(SLOW);
+ false
+ }));
+ s
+}
+
+fn max_frame(h: &mut Headless, frames: usize) -> Duration {
+ let mut worst = Duration::ZERO;
+ for _ in 0..frames {
+ let t0 = Instant::now();
+ h.step();
+ worst = worst.max(t0.elapsed());
+ }
+ worst
+}
+
+#[test]
+fn offline_originals_never_block_a_frame() {
+ let calls = Calls::default();
+ let app = LightcraftApp::new(offline_library(&calls), Services { png: None, ..Default::default() });
+ let mut h = Headless::new(app, [1200.0, 800.0], 1.0);
+ let ui_thread = std::thread::current().id();
+ h.app.ui.view = ViewMode::PhotoGrid;
+ h.step();
+ assert!(h.app.session.media.availability.is_background(), "the app checks files off the UI thread");
+
+ // the grid: every cell asks for its thumbnail (which asks whether the original is there)
+ h.app.session.execute("library.select", &json!({"ids": [1]})).unwrap();
+ let worst = max_frame(&mut h, 20);
+ assert!(worst < FRAME_LIMIT, "grid frame took {worst:?}");
+
+ // the Info panel of the active photo, in the loupe
+ h.app.ui.view = ViewMode::Detail;
+ h.app.ui.right = RightPanel::Info;
+ let worst = max_frame(&mut h, 20);
+ assert!(worst < FRAME_LIMIT, "Info panel frame took {worst:?}");
+
+ // the Missing Photos view fills in as the background checks find the files gone
+ h.app.ui.view = ViewMode::PhotoGrid;
+ h.app.ui.right = RightPanel::None;
+ h.app.session.execute("library.source", &json!({"kind": "missing"})).unwrap();
+ assert_eq!(h.app.session.source, LibrarySource::Missing);
+ let t0 = Instant::now();
+ let mut worst = Duration::ZERO;
+ while h.app.session.visible().len() < N as usize && t0.elapsed() < Duration::from_secs(60) {
+ let f = Instant::now();
+ h.step();
+ worst = worst.max(f.elapsed());
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ assert_eq!(h.app.session.visible().len(), N as usize, "every photo found missing in the end");
+ assert!(worst < FRAME_LIMIT, "Missing Photos frame took {worst:?}");
+
+ // the Rename dialog's preview (it checks which names are taken on disk)
+ h.app.session.execute("library.select", &json!({"ids": [1, 2, 3]})).unwrap();
+ h.app.ui.dialog = Some(crate::state::Dialog::Rename { template: "Trip-{seq:3}".into(), start: 1 });
+ let worst = max_frame(&mut h, 20);
+ assert!(worst < FRAME_LIMIT, "Rename dialog frame took {worst:?}");
+
+ assert!(calls.n.load(Ordering::SeqCst) > 0, "the probe was asked");
+ assert_eq!(calls.on(ui_thread), 0, "the slow probe never ran on the UI thread");
+}
+
+/// An import whose files take long to read (probe: decoding headers and hashing on a slow drive)
+/// runs on a worker thread: frames stay quick, the probe never runs on the UI thread, and Cancel
+/// stops it with what was read so far added.
+#[test]
+fn import_reads_files_off_the_ui_thread_and_can_be_cancelled() {
+ let dir = std::env::temp_dir().join(format!("lc-ui-slow-import-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ let n = 40;
+ for i in 0..n {
+ std::fs::write(dir.join(format!("IMG_{i:03}.jpg")), format!("not really a jpeg {i}")).unwrap();
+ }
+ let calls = Calls::default();
+ let mut s = Session::new();
+ let c = calls.clone();
+ s.media.file_probe = Some(Arc::new(move |p: &str| {
+ c.record();
+ std::thread::sleep(Duration::from_millis(150));
+ Ok(lightcraft_engine::media::ProbeInfo {
+ width: 60,
+ height: 40,
+ format: "JPEG".into(),
+ content_hash: Some(p.to_string()),
+ ..Default::default()
+ })
+ }));
+ let app = LightcraftApp::new(s, Services { png: None, ..Default::default() });
+ let mut h = Headless::new(app, [1200.0, 800.0], 1.0);
+ let ui_thread = std::thread::current().id();
+ h.step();
+ let undo0 = h.app.session.undo.len();
+ crate::import::start_paths(&mut h.app, vec![dir.to_string_lossy().to_string()]).unwrap();
+ // some batches arrive; meanwhile every frame is quick
+ let t0 = Instant::now();
+ let mut worst = Duration::ZERO;
+ while h.app.import.as_ref().is_some_and(|t| t.imported < 8) && t0.elapsed() < Duration::from_secs(60) {
+ let f = Instant::now();
+ h.step();
+ worst = worst.max(f.elapsed());
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ assert!(worst < FRAME_LIMIT, "import frame took {worst:?}");
+ assert!(h.app.import.is_some(), "still importing");
+ let status = h.app.import.as_ref().map(crate::import::ImportTask::status).unwrap_or_default();
+ assert!(status["total"].as_u64().unwrap_or(0) >= 8, "{status}");
+ // Cancel: the files being read finish, nothing new starts
+ let r = h.request("ui.clickWidget", json!({"id": "button:importCancel"}), Duration::from_secs(10));
+ assert_eq!(r["ok"], true, "{r}");
+ let t0 = Instant::now();
+ while h.app.import.is_some() && t0.elapsed() < Duration::from_secs(60) {
+ h.step();
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ assert!(h.app.import.is_none(), "stopped");
+ let added = h.app.session.catalog.len();
+ assert!((8..n).contains(&added), "cancelled part-way: {added} of {n} added");
+ assert_eq!(h.app.session.undo.len(), undo0 + 1, "one undo step");
+ assert_eq!(calls.on(ui_thread), 0, "files were never read on the UI thread");
+ let _ = std::fs::remove_dir_all(&dir);
+}
diff --git a/docs/control-protocol.md b/docs/control-protocol.md
index 4723f7310..24fd2bdb4 100644
--- a/docs/control-protocol.md
+++ b/docs/control-protocol.md
@@ -30,7 +30,7 @@ mode ([mcp.md](mcp.md)) is a thin layer over this channel. Implementation:
| `engine.commands` | — | Engine + UI commands: id, label, menu, shortcut, params doc, enabled |
| `ui.menu.list` | — | Menu entries (flat: id, label, menu path, shortcut, enabled) |
| `ui.menu.tree` | — | The menu bar as shown (File … Help): items `{id, params?, label, shortcut?, enabled, checked?}`, separators, submenus — the model behind the native macOS menu bar and the in-window menus |
-| `ui.inspect` | — | UI state, window, canvas/image rects, `scroll: {grid, filmstrip}` (scroll offsets in points, `null` until drawn), active photo, selection, perf (`frameMs` = layout, `logicMs` = per-frame logic before it, `updateMs` = both, `maxUpdateMs`, `fps`, render queue …, `gpu` = adapter in use, `gpuReason` = why renders don't use the GPU, `gpuFallback` = latest render redone on the CPU and why — see `docs/gpu-pipeline.md`), status, memory (bytes per cache, see `library.memory`; plus stage caches and textures), `export: {running: {total, done, current} \| null, last}`, `notices` (warnings waiting to be shown, e.g. a damaged settings file; OK = `button:noticeOk`), `quitPrompt` (why quitting was stopped: unsaved changes; `button:quitRetry` / `button:quitAnyway` / `button:quitCancel`) |
+| `ui.inspect` | — | UI state, window, canvas/image rects, `scroll: {grid, filmstrip}` (scroll offsets in points, `null` until drawn), active photo, selection, perf (`frameMs` = layout, `logicMs` = per-frame logic before it, `updateMs` = both, `maxUpdateMs`, `fps`, render queue …, `gpu` = adapter in use, `gpuReason` = why renders don't use the GPU, `gpuFallback` = latest render redone on the CPU and why — see `docs/gpu-pipeline.md`), status, memory (bytes per cache, see `library.memory`; plus stage caches and textures), `export: {running: {total, done, current} \| null, last}`, `notices` (warnings waiting to be shown, e.g. a damaged settings file; OK = `button:noticeOk`), `quitPrompt` (why quitting was stopped: unsaved changes; `button:quitRetry` / `button:quitAnyway` / `button:quitCancel`), `import: {done, total, imported, cancelled} \| null` (an import runs on a worker thread), `tasks` (other background work: `Find Missing Photos`, `Auto Import`) |
| `ui.widgets` | `{filter?}` | On-screen widgets `{id, rect: [x, y, w, h]}` (screen points) |
| `ui.clickWidget` / `ui.dragWidget` / `ui.hoverWidget` | `{id, count?, fx?, fy?}` / `{id, toX?, toY?, dx?, dy?, steps?}` / `{id, fx?, fy?}` | Real egui input on a widget (hover: the pointer rests on it, e.g. for preset/profile previews) |
| `ui.move` / `ui.click` / `ui.drag` | `{x, y, count?, button?}` / `{x, y, toX, toY, steps?}` | Raw pointer input, screen points |
diff --git a/docs/parity.md b/docs/parity.md
index ac62ea34c..332c05e3b 100644
--- a/docs/parity.md
+++ b/docs/parity.md
@@ -113,7 +113,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins.
| Id | Feature | Tier | Status | Evidence | Missing / notes |
|---|---|---|---|---|---|
-| LR-IMP-ADD-DIALOG | Add photos/folders | P0 | ✅ | `cmd:file.addPhotos`, `cmd:library.importPreview`, `cmd:library.import` (`mode`, `album`, `albumName`, `preset`, `keywords`), `crates/ui-egui/src/import.rs` | review dialog (Import Photos): the scanned source (a scanned folder is not added to Local), candidate grid with thumbnails and checkboxes (duplicates by path/content marked and unchecked), transfer mode (add in place / copy / move, each explained), album (existing/new), preset, keywords; the folder scan runs on a background thread with a progress window and Cancel (a NAS folder no longer freezes the window); batched import with a progress window; one undo step. Drag-and-drop still imports directly |
+| LR-IMP-ADD-DIALOG | Add photos/folders | P0 | ✅ | `cmd:file.addPhotos`, `cmd:library.importPreview`, `cmd:library.import` (`mode`, `album`, `albumName`, `preset`, `keywords`), `crates/ui-egui/src/import.rs` | review dialog (Import Photos): the scanned source (a scanned folder is not added to Local), candidate grid with thumbnails and checkboxes (duplicates by path/content marked and unchecked), transfer mode (add in place / copy / move, each explained), album (existing/new), preset, keywords; the folder scan runs on a background thread with a progress window and Cancel (a NAS folder no longer freezes the window); the import itself (probing, sidecars, copying / moving files) runs on a worker thread and its batches join the library between frames, with a progress window and Cancel; one undo step. Drag-and-drop goes through the same background import |
| LR-IMP-DRAGDROP | Drop files/folders to import | P0 | ✅ | `crates/ui-egui/src/lib.rs` (dropped files → `cmd:library.import`) | dropping onto a specific album not supported |
| LR-IMP-DUPES | Skip duplicates by content | P1 | ✅ | `crates/engine/src/import.rs`, `crates/engine/src/tests_import.rs` | |
| LR-IMP-DEVICE | Import from camera/card | P1 | ✅ | `cmd:library.devices`, `cmd:file.addFromDevice`, `crates/engine/src/devices.rs` | mounted volumes with a DCIM folder (macOS /Volumes, Linux /media and /run/media, Windows drive letters); File → Import from Device → the import review, copying into the library by default; menus serve the last scan and rescan in the background (never blocking a frame; hot-plugs repaint); copies are verified like Move's (a new file, synced, compared byte for byte; a bad copy is removed and reported as failed, never counted as imported; taken names get -1, -2…, never replaced — issue #96, `crates/engine/src/import_move.rs`); no PTP/MTP (cameras that don't mount as a disk) |
@@ -480,7 +480,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins.
| Id | Feature | Tier | Status | Evidence | Missing / notes |
|---|---|---|---|---|---|
| LR-CLOUD-SYNC | Cloud sync | OOS | 🚫 | | |
-| LR-CLOUD-SMARTPREVIEW | Editable proxies | P2 | ✅ | `cmd:library.smartPreviews`, `cmd:library.smartPreviewsLocation`, `cmd:photo.smartPreview`, `crates/engine/src/smart.rs`, `crates/ui-egui/src/panels/settings.rs` | File ▸ Previews ▸ Build / Discard Smart Previews: ~1 MB proxies in the library; with the original offline the photo renders, edits and exports (at proxy size) from its proxy; Info shows the status; the proxy folder is chosen per library (Settings → Performance → Smart previews, saved in the library): the effective path, count and size are shown, changing it needs a choice for the proxies already built (move / leave / delete), and an unavailable or unwritable folder is an error, never a fallback to the library drive; proxies are written atomically and a damaged one (cut short by a crash or a full drive) doesn't count as built, is rebuilt (`repaired`) and is never moved over a good one (issue #106). The thumbnail cache stays in the library |
+| LR-CLOUD-SMARTPREVIEW | Editable proxies | P2 | ✅ | `cmd:library.smartPreviews`, `cmd:library.smartPreviewsLocation`, `cmd:photo.smartPreview`, `crates/engine/src/smart.rs`, `crates/ui-egui/src/panels/settings.rs` | File ▸ Previews ▸ Build / Discard Smart Previews (on a worker thread, progress and Stop like Build Previews): ~1 MB proxies in the library; with the original offline the photo renders, edits and exports (at proxy size) from its proxy; Info shows the status; the proxy folder is chosen per library (Settings → Performance → Smart previews, saved in the library): the effective path, count and size are shown, changing it needs a choice for the proxies already built (move / leave / delete), and an unavailable or unwritable folder is an error, never a fallback to the library drive; proxies are written atomically and a damaged one (cut short by a crash or a full drive) doesn't count as built, is rebuilt (`repaired`) and is never moved over a good one (issue #106). The thumbnail cache stays in the library |
| LR-AI-UPDATE-INDICATOR | AI-settings update indicator | P2 | ⬜ | | |
| LR-AI-CREDITS | Generative credits | OOS | 🚫 | | |
@@ -728,7 +728,7 @@ From `08-lightroom-classic-extras.md` (Classic-only features) and part 2 of `06-
| LRC-LIB-PEOPLE | People view | P2 | ⬜ | | |
| LRC-LIB-COMMENTS | Comments panel | P2 | ⬜ | | |
| LRC-LIB-VISUALSEARCH | Find similar photos | P2 | ✅ | `cmd:library.findSimilar`, `crates/pipeline/src/cull.rs` (`signature`) | photo menu / Photo ▸ Find Similar Photos: look-alike photos (composition and tones), most similar first, as a filter (`only`) |
-| LRC-LIB-MISSING | Missing files & relink | P1 | ✅ | `cmd:library.missing`, `cmd:photo.relink`, `cmd:library.findMissing`, `cmd:file.findMissing`, `cmd:photo.locate`, `crates/engine/src/cmd/missing.rs` (`checked_path`) | File → Find Missing Photos… (anywhere in a folder: same name, size and content hash; renamed files by size + content hash; same name and size alone only when unambiguous — look-alikes are skipped and reported; one undo step), photo menu → Locate Missing File…; unreadable files show "!" in the grid and a reason in the loupe; undo never moves files. Missing Photos in the sidebar (counted on a worker thread); it covers library photos only — Local browse records are never checked, for the count, the view, `library.missing` and Find Missing alike |
+| LRC-LIB-MISSING | Missing files & relink | P1 | ✅ | `cmd:library.missing`, `cmd:photo.relink`, `cmd:library.findMissing`, `cmd:file.findMissing`, `cmd:photo.locate`, `crates/engine/src/cmd/missing.rs` (`checked_path`) | File → Find Missing Photos… (anywhere in a folder: same name, size and content hash; renamed files by size + content hash; same name and size alone only when unambiguous — look-alikes are skipped and reported; searched on a worker thread, one undo step), photo menu → Locate Missing File…; unreadable files show "!" in the grid and a reason in the loupe; undo never moves files. Missing Photos in the sidebar (counted on a worker thread; the view, the Info panel, the grid's context menu and thumbnails read file availability from a cache a worker thread fills, so an offline NAS never stalls a frame); it covers library photos only — Local browse records are never checked, for the count, the view, `library.missing` and Find Missing alike |
| LRC-LIB-CONVERT | Convert to DNG | P2 | ✅ | `cmd:photo.convertToDng`, `crates/engine/src/cmd/convert.rs` | Photo ▸ Convert to DNG: lossless DNG next to the raw with the settings embedded, photo (and its virtual copies) relinked; originals kept; undoable; the DNG is decoded and compared with the raw data, written to a temp file, synced, read back and only then given a free name (never replacing a file) — a failed write leaves no DNG and the photo on its raw (issue #106) |
| LRC-LIB-PREVIEWS | Build / discard previews | P1 | ✅ | `cmd:library.buildPreviews`, `cmd:library.previewProgress`, `cmd:library.cancelPreviews`, `cmd:library.clearPreviews`, `crates/engine/src/cmd/previews.rs`, `crates/preview/src/lib.rs` | File ▸ Previews: build standard-sized (Settings → Performance size) or 1:1 previews of the selected / visible photos in the background (progress toasts, stop), discard the cache; disk thumbnail + view cache; no smart previews (offline proxies) |
| LRC-LIB-SLIDESHOW-IMPROMPTU | Impromptu slideshow | P2 | ✅ | `cmd:view.slideshow` | see LR-VIEW-SLIDESHOW |
diff --git a/docs/whats-new.md b/docs/whats-new.md
index ce7cd6c1b..84f28efc6 100644
--- a/docs/whats-new.md
+++ b/docs/whats-new.md
@@ -42,6 +42,10 @@
- The control port (`--control`) closes a connection as soon as it receives anything that isn't a JSON request
(issue #94): an HTTP request from a web page can no longer carry a command in its body. Lines are capped at
4 MiB and connections at 16.
+- A sleeping NAS, a dropped network share or a USB drive spinning up no longer freezes the window (issue #104):
+ whether originals are there is checked on a worker thread (grid thumbnails, the Info panel, the photo menu, Missing
+ Photos); importing (also by drag and drop), Find Missing Photos, Build / Discard Smart Previews, the Rename preview,
+ the Local folder tree and Auto Import read the disk on worker threads too. The import progress window has Cancel.
- Canon CR2 photos from the EOS 7D, 50D, 60D, 550D, 600D, 1200D, 1300D, 5D Mark II and 1D Mark IV (and other
models whose sensor starts on a green-blue row) no longer come out magenta (issue #85): the colour-filter
layout is read from each file instead of assumed.