LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
Import: scan folders on a background thread with progress and Cancel
Clément L5828a7d2d318
diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rsindex 37b5bac4d..dbea7fe44 100644--- a/crates/engine/src/import.rs+++ b/crates/engine/src/import.rs@@ -261,11 +261,12 @@ pub fn expand(paths: &[String], skip: Option<&Path>) -> Vec<String> {/// Probe files (headers + content hash) as an import would.pub(crate) fn probe_paths(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {- probe_all(s, paths)+ probe_all(s.media.file_probe.as_ref(), paths, &ScanProgress::default())}-fn probe_all(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {- let Some(probe) = s.media.file_probe.clone() else {+fn probe_all(probe: Option<&crate::media::FileProbe>, paths: &[String], progress: &ScanProgress) -> Vec<Result<ProbeInfo, String>> {+ use std::sync::atomic::Ordering::Relaxed;+ let Some(probe) = probe.cloned() else {return paths.iter().map(|p| {@@ -291,7 +292,11 @@ fn probe_all(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {if i >= paths.len() {break;}+ if progress.cancel.load(Relaxed) {+ break;+ }let r = probe(&paths[i]);+ progress.done.fetch_add(1, Relaxed);out.lock().unwrap_or_else(|e| e.into_inner())[i] = Some(r);}});@@ -300,7 +305,17 @@ fn probe_all(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {return results.into_iter().map(|r| r.unwrap_or_else(|| Err("not probed".into()))).collect();}}- paths.iter().map(|p| probe(p)).collect()+ paths+ .iter()+ .map(|p| {+ if progress.cancel.load(Relaxed) {+ return Err("cancelled".into());+ }+ let r = probe(p);+ progress.done.fetch_add(1, Relaxed);+ r+ })+ .collect()}/// Copy `src` into `root` (filed per `organize` by `date`) as `name` (default: its own name),@@ -331,53 +346,107 @@ fn copy_into(root: &Path, src: &str, date: &str, organize: Organize, name: OptioOk(dst.to_string_lossy().to_string())}+/// What a [`scan`] needs from the session, so it can run on another thread (a folder on a network+/// share can take minutes to read).+pub struct ScanInput {+ probe: Option<crate::media::FileProbe>,+ skip: Option<PathBuf>,+ /// path → (photo, its summary) for files already in the library.+ by_path: HashMap<String, (PhotoId, ImportCandidate)>,+ by_hash: HashMap<String, PhotoId>,+ cache: HashMap<String, ProbeInfo>,+}++/// Progress and cancellation of a running [`scan_with`].+#[derive(Default)]+pub struct ScanProgress {+ /// Files to probe, set once the folders are expanded.+ pub total: std::sync::atomic::AtomicUsize,+ pub done: std::sync::atomic::AtomicUsize,+ pub cancel: std::sync::atomic::AtomicBool,+}++/// The result of [`scan_with`]: the candidates, and the probes to keep for the import that follows.+pub struct ScanOutput {+ pub candidates: Vec<ImportCandidate>,+ pub probes: HashMap<String, ProbeInfo>,+}++impl ScanInput {+ pub fn new(s: &mut Session, paths: &[String]) -> (Self, Vec<String>) {+ let skip = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());+ let mut by_path = HashMap::new();+ let mut by_hash = HashMap::new();+ for p in s.catalog.photos() {+ if let Source::File { path } = &p.source {+ let c = ImportCandidate {+ path: path.clone(),+ format: p.format.clone(),+ kind: p.kind,+ width: p.width,+ height: p.height,+ file_size: p.file_size,+ captured: p.captured.clone(),+ duplicate: Some("path".into()),+ existing: Some(p.id.0),+ ..Default::default()+ };+ by_path.insert(path.clone(), (p.id, c));+ }+ if let Some(h) = &p.content_hash {+ by_hash.insert(h.clone(), p.id);+ }+ }+ let input = ScanInput { probe: s.media.file_probe.clone(), skip, by_path, by_hash, cache: std::mem::take(&mut s.import_probes) };+ (input, paths.to_vec())+ }+}+/// Find and probe what importing `paths` would add, marking duplicates (by path, or by content/// against the library and earlier candidates). Nothing is added; the probes are kept for the/// import that follows.pub fn scan(s: &mut Session, paths: &[String]) -> Vec<ImportCandidate> {- let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());- let files = expand(paths, lib_dir.as_deref());- let mut by_path: HashMap<String, PhotoId> = HashMap::new();- let mut by_hash: HashMap<String, PhotoId> = HashMap::new();- for p in s.catalog.photos() {- if let Source::File { path } = &p.source {- by_path.insert(path.clone(), p.id);- }- if let Some(h) = &p.content_hash {- by_hash.insert(h.clone(), p.id);- }- }- let todo: Vec<String> = files.iter().filter(|f| !by_path.contains_key(f.as_str())).cloned().collect();+ let (input, paths) = ScanInput::new(s, paths);+ let out = scan_with(input, &paths, &ScanProgress::default());+ s.import_probes = out.probes;+ out.candidates+}++/// [`scan`] without the session, so it can run on a worker thread. Stops early (returning what it+/// has) when `progress.cancel` is set.+pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput {+ use std::sync::atomic::Ordering::Relaxed;+ let files = expand(paths, input.skip.as_deref());+ let todo: Vec<String> = files.iter().filter(|f| !input.by_path.contains_key(f.as_str())).cloned().collect();+ progress.total.store(todo.len(), Relaxed);// probes from a preceding `scan` are reused when the file is unchanged (same size)let cached: Vec<Option<ProbeInfo>> = todo.iter().map(|f| {- let info = s.import_probes.remove(f)?;+ let info = input.cache.remove(f)?;let size = std::fs::metadata(f).map(|m| m.len()).ok();(size.is_none() || size == Some(info.file_size)).then_some(info)}).collect();let missing: Vec<String> = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect();- let mut fresh = probe_all(s, &missing).into_iter();+ progress.done.store(todo.len() - missing.len(), Relaxed);+ let mut fresh = probe_all(input.probe.as_ref(), &missing, progress).into_iter();let probed: Vec<Result<ProbeInfo, String>> =cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect();crate::memory::release();let mut probes: HashMap<String, Result<ProbeInfo, String>> = todo.into_iter().zip(probed).collect();- let mut seen_hash: HashMap<String, Option<u64>> = by_hash.into_iter().map(|(h, id)| (h, Some(id.0))).collect();+ let mut seen_hash: HashMap<String, Option<u64>> = input.by_hash.into_iter().map(|(h, id)| (h, Some(id.0))).collect();let mut out = Vec::with_capacity(files.len());+ let mut kept = HashMap::new();for f in files {- let name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone());- let mut c = ImportCandidate { path: f.clone(), name, ..Default::default() };- if let Some(id) = by_path.get(&f) {- c.duplicate = Some("path".into());- c.existing = Some(id.0);- if let Some(p) = s.catalog.photo(*id) {- (c.format, c.kind, c.width, c.height, c.file_size, c.captured) =- (p.format.clone(), p.kind, p.width, p.height, p.file_size, p.captured.clone());- }+ if let Some((_, c)) = input.by_path.get(&f) {+ let mut c = c.clone();+ c.name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone());out.push(c);continue;}+ let name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone());+ let mut c = ImportCandidate { path: f.clone(), name, ..Default::default() };match probes.remove(&f) {Some(Ok(info)) => {(c.format, c.kind, c.width, c.height, c.file_size, c.captured) =@@ -393,14 +462,14 @@ pub fn scan(s: &mut Session, paths: &[String]) -> Vec<ImportCandidate> {}}}- s.import_probes.insert(f, info);+ kept.insert(f, info);}Some(Err(e)) => c.error = Some(e),None => c.error = Some("not probed".into()),}out.push(c);}- out+ ScanOutput { candidates: out, probes: kept }}/// Import files/folders. See the module docs.@@ -454,7 +523,7 @@ pub fn import_with(s: &mut Session, paths: &[String], opts: &ImportOptions) -> c}}- let probed = probe_all(s, &todo);+ let probed = probe_all(s.media.file_probe.as_ref(), &todo, &ScanProgress::default());crate::memory::release();let now = (s.clock)();let mut ops = promote;diff --git a/crates/ui-egui/src/headless.rs b/crates/ui-egui/src/headless.rsindex c504a46f7..d2669ea6c 100644--- a/crates/ui-egui/src/headless.rs+++ b/crates/ui-egui/src/headless.rs@@ -799,11 +799,36 @@ mod tests {h.app.ui.dialog = None;let r = h.request("engine.execute", json!({"command": "file.addFromDevice", "params": {"path": sub.to_string_lossy()}}), t);assert_eq!(r["ok"], true, "{r}");+ h.settle(SETTLE);let Some(crate::state::Dialog::Import { opts }) = &h.app.ui.dialog else { panic!("no import review") };assert!(opts.copy);let _ = std::fs::remove_dir_all(&dir);}+ /// The folder scan runs in the background: the request returns at once and the review opens+ /// when the scan finishes (a folder on a network share must not freeze the window).+ #[test]+ fn add_folder_scans_in_the_background() {+ let mut h = demo([1200.0, 900.0]);+ let t = Duration::from_secs(10);+ let dir = std::env::temp_dir().join(format!("lc-scanbg-{}", std::process::id()));+ std::fs::create_dir_all(&dir).unwrap();+ let img = lightcraft_raster::Rgba8::from_fn(8, 8, |x, y| [(x * 9) as u8, (y * 12) as u8, 80, 255]);+ let png = lightcraft_engine::export::encode_image(+ &img,+ &lightcraft_engine::export::ExportOptions { format: lightcraft_engine::export::ExportFormat::Png, ..Default::default() },+ )+ .unwrap();+ std::fs::write(dir.join("a.png"), &png).unwrap();+ let r = h.request("engine.execute", json!({"command": "file.addFolder", "params": {"path": dir.to_string_lossy()}}), t);+ assert_eq!(r["result"]["scanning"], true, "{r}");+ assert!(h.app.scan.is_some() || h.app.ui.dialog.is_some());+ h.settle(SETTLE);+ assert!(h.app.scan.is_none());+ assert!(matches!(h.app.ui.dialog, Some(crate::state::Dialog::Import { .. })));+ let _ = std::fs::remove_dir_all(&dir);+ }+/// ⇧⌘V opens Paste Selected Settings (prefilled with the copied groups); unchecking a group/// leaves it alone. ⌘F puts typing into the search field.#[test]@@ -914,9 +939,11 @@ mod tests {let n0 = h.app.session.catalog.len();let undo0 = h.app.session.undo.len();let r = h.request("engine.execute", json!({"command": "file.addPhotos", "params": {"paths": [dir.to_string_lossy()]}}), t);- assert_eq!(r["result"]["candidates"], 6, "{r}");- assert_eq!(r["result"]["duplicates"], 1, "{r}");+ assert_eq!(r["result"]["scanning"], true, "{r}");h.settle(SETTLE);+ let Some(crate::state::Dialog::Import { opts }) = &h.app.ui.dialog else { panic!("no import review") };+ assert_eq!(opts.candidates.len(), 6);+ assert_eq!(opts.candidates.iter().filter(|c| c.duplicate.is_some()).count(), 1);assert!(h.app.renderer.textures.keys().filter(|s| matches!(s, crate::render::Slot::Import(_))).count() >= 5, "thumbnails");// uncheck the first photo; name a new album and keywordslet r = h.request("ui.clickWidget", json!({"id": "import:0"}), t);diff --git a/crates/ui-egui/src/import.rs b/crates/ui-egui/src/import.rsindex d7c5dfd3c..ed0a5bef8 100644--- a/crates/ui-egui/src/import.rs+++ b/crates/ui-egui/src/import.rs@@ -5,9 +5,10 @@//! frame, with a progress window; the whole import is one undo step.use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2};-use lightcraft_engine::import::ImportCandidate;+use lightcraft_engine::import::{ImportCandidate, ScanInput, ScanOutput, ScanProgress, scan_with};use serde::{Deserialize, Serialize};use serde_json::{Value, json};+use std::sync::atomic::Ordering;use crate::LightcraftApp;use crate::render::Slot;@@ -79,19 +80,82 @@ pub struct ImportTask {first: Option<u64>,}-/// Scan `paths` and open the review dialog. Returns the candidate counts.+/// A folder scan running on a worker thread (a network share can take minutes to read; the+/// window must keep answering meanwhile).+pub struct ScanTask {+ progress: std::sync::Arc<ScanProgress>,+ rx: std::sync::mpsc::Receiver<ScanOutput>,+ /// Open the review with "copy into the library" checked (a camera / card).+ pub copy: bool,+}++/// Scan `paths` in the background, then open the review dialog (see [`poll_scan`]).pub fn open(app: &mut LightcraftApp, paths: Vec<String>) -> Result<Value, String> {- let r = app.session.execute("library.importPreview", &json!({"paths": paths})).map_err(|e| e.to_string())?;- let candidates: Vec<ImportCandidate> = serde_json::from_value(r["candidates"].clone()).unwrap_or_default();- if candidates.is_empty() {- app.toast(&egui::Context::default(), "No photos found");- return Ok(json!({"candidates": 0}));+ if app.scan.is_some() {+ return Err("a scan is already running".into());+ }+ let (input, paths) = ScanInput::new(&mut app.session, &paths);+ let progress = std::sync::Arc::new(ScanProgress::default());+ let (tx, rx) = std::sync::mpsc::channel();+ let p = progress.clone();+ let job = move || {+ let _ = tx.send(scan_with(input, &paths, &p));+ };+ #[cfg(not(target_arch = "wasm32"))]+ std::thread::spawn(job);+ #[cfg(target_arch = "wasm32")]+ job();+ app.scan = Some(ScanTask { progress, rx, copy: false });+ Ok(json!({"scanning": true}))+}++/// Collect a finished scan and open the review (called every frame).+pub fn poll_scan(app: &mut LightcraftApp, ctx: &egui::Context) {+ let Some(task) = app.scan.as_ref() else { return };+ ctx.request_repaint_after(std::time::Duration::from_millis(100));+ let out = match task.rx.try_recv() {+ Ok(o) => o,+ Err(std::sync::mpsc::TryRecvError::Empty) => return,+ Err(_) => {+ app.scan = None;+ app.toast(ctx, "Scan failed");+ return;+ }+ };+ let task = app.scan.take().expect("scan task");+ if task.progress.cancel.load(Ordering::Relaxed) {+ return;+ }+ app.session.import_probes = out.probes;+ if out.candidates.is_empty() {+ app.toast(ctx, "No photos found");+ return;}app.renderer.forget_imports();- let d = ImportDialog::new(candidates);- let out = json!({"candidates": d.candidates.len(), "duplicates": r["duplicates"]});+ let mut d = ImportDialog::new(out.candidates);+ d.copy = task.copy;app.ui.dialog = Some(crate::state::Dialog::Import { opts: Box::new(d) });- Ok(out)+}++/// The progress window while a folder is being scanned.+pub fn scan_progress(app: &mut LightcraftApp, ctx: &egui::Context) {+ let Some(task) = &app.scan else { return };+ let t = Tokens::get(ctx);+ let total = task.progress.total.load(Ordering::Relaxed);+ let done = task.progress.done.load(Ordering::Relaxed);+ let text = if total == 0 { "Looking for photos…".to_string() } else { format!("Reading photos… {done} of {total}") };+ let mut cancel = false;+ egui::Window::new("Scanning").title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([340.0, 80.0]).show(+ ctx,+ |ui| {+ ui.label(egui::RichText::new(text).color(t.text));+ ui.add(egui::ProgressBar::new(done as f32 / total.max(1) as f32).desired_width(320.0));+ cancel = ui.button("Cancel").clicked();+ },+ );+ if cancel {+ task.progress.cancel.store(true, Ordering::Relaxed);+ }}/// Start importing the dialog's checked files (the dialog's OK / `ui.dialog.confirm`).diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rsindex 563417330..6a67bd16d 100644--- a/crates/ui-egui/src/lib.rs+++ b/crates/ui-egui/src/lib.rs@@ -125,6 +125,8 @@ pub struct LightcraftApp {pub merge: merge::MergeState,/// An import in progress (the import review dialog's batches).pub import: Option<import::ImportTask>,+ /// A folder scan in progress (feeds the import review).+ pub scan: Option<import::ScanTask>,/// A background export in progress.pub export: Option<export_task::ExportTask>,/// The files of the last finished background export (`ui.inspect` → `export.last`).@@ -172,6 +174,7 @@ impl LightcraftApp {loupe_shown: None,merge: merge::MergeState::default(),import: None,+ scan: None,export: None,last_export_result: None,hover_preview: None,@@ -404,6 +407,7 @@ impl LightcraftApp {}self.renderer.poll(ctx, &mut self.session);merge::poll(self, ctx);+ import::poll_scan(self, ctx);import::tick(self, ctx);self.preview_build_status(ctx);self.slideshow_tick(ctx);@@ -565,6 +569,7 @@ impl LightcraftApp {panels::second::show(self, &ctx);panels::dialogs::show(self, &ctx);import::progress(self, &ctx);+ import::scan_progress(self, &ctx);export_task::poll(self, &ctx);panels::grid::drag_feedback(self, &ctx);panels::toast(self, &ctx);diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rsindex 5112b531e..90b714cd0 100644--- a/crates/ui-egui/src/menus.rs+++ b/crates/ui-egui/src/menus.rs@@ -879,8 +879,8 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option<Re},};let r = crate::import::open(app, vec![path]);- if let Some(crate::state::Dialog::Import { opts }) = &mut app.ui.dialog {- opts.copy = true;+ if let Some(t) = &mut app.scan {+ t.copy = true;}r}diff --git a/docs/parity.md b/docs/parity.mdindex 0da044e7b..32b48fb85 100644--- a/docs/parity.md+++ b/docs/parity.md@@ -98,7 +98,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins.| Id | Feature | Tier | Status | Evidence | Missing / notes ||---|---|---|---|---|---|-| LR-IMP-ADD-DIALOG | Add photos/folders | P0 | ✅ | `cmd:file.addPhotos`, `cmd:library.importPreview`, `cmd:library.import` (`mode`, `album`, `albumName`, `preset`, `keywords`), `crates/ui-egui/src/import.rs` | review dialog: candidate grid with thumbnails and checkboxes (duplicates by path/content marked and unchecked), destination, album (existing/new), preset, keywords; batched import with a progress window; one undo step. Drag-and-drop still imports directly |+| LR-IMP-ADD-DIALOG | Add photos/folders | P0 | ✅ | `cmd:file.addPhotos`, `cmd:library.importPreview`, `cmd:library.import` (`mode`, `album`, `albumName`, `preset`, `keywords`), `crates/ui-egui/src/import.rs` | review dialog: candidate grid with thumbnails and checkboxes (duplicates by path/content marked and unchecked), destination, album (existing/new), preset, keywords; the folder scan runs on a background thread with a progress window and Cancel (a NAS folder no longer freezes the window); batched import with a progress window; one undo step. Drag-and-drop still imports directly || LR-IMP-DRAGDROP | Drop files/folders to import | P0 | ✅ | `crates/ui-egui/src/lib.rs` (dropped files → `cmd:library.import`) | dropping onto a specific album not supported || LR-IMP-DUPES | Skip duplicates by content | P1 | ✅ | `crates/engine/src/import.rs`, `crates/engine/src/tests_import.rs` | || LR-IMP-DEVICE | Import from camera/card | P1 | ✅ | `cmd:library.devices`, `cmd:file.addFromDevice`, `crates/engine/src/devices.rs` | mounted volumes with a DCIM folder (macOS /Volumes, Linux /media and /run/media, Windows drive letters); File → Add from Device → the import review, copying into the library by default; no PTP/MTP (cameras that don't mount as a disk) |