Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

Import: scan folders on a background thread with progress and Cancel

Clément L5828a7d2d318
diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs
index 37b5bac4d..dbea7fe44 100644
--- a/crates/engine/src/import.rs
+++ b/crates/engine/src/import.rs
@@ -261,11 +261,12 @@ pub fn expand(paths: &[String], skip: Option<&Path>) -> Vec<String> {
/// Probe files (headers + content hash) as an import would.
pub(crate) fn probe_paths(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {
- probe_all(s, paths)
+ probe_all(s.media.file_probe.as_ref(), paths, &ScanProgress::default())
}
-fn probe_all(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {
- let Some(probe) = s.media.file_probe.clone() else {
+fn probe_all(probe: Option<&crate::media::FileProbe>, paths: &[String], progress: &ScanProgress) -> Vec<Result<ProbeInfo, String>> {
+ use std::sync::atomic::Ordering::Relaxed;
+ let Some(probe) = probe.cloned() else {
return paths
.iter()
.map(|p| {
@@ -291,7 +292,11 @@ fn probe_all(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {
if i >= paths.len() {
break;
}
+ if progress.cancel.load(Relaxed) {
+ break;
+ }
let r = probe(&paths[i]);
+ progress.done.fetch_add(1, Relaxed);
out.lock().unwrap_or_else(|e| e.into_inner())[i] = Some(r);
}
});
@@ -300,7 +305,17 @@ fn probe_all(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {
return results.into_iter().map(|r| r.unwrap_or_else(|| Err("not probed".into()))).collect();
}
}
- paths.iter().map(|p| probe(p)).collect()
+ paths
+ .iter()
+ .map(|p| {
+ if progress.cancel.load(Relaxed) {
+ return Err("cancelled".into());
+ }
+ let r = probe(p);
+ progress.done.fetch_add(1, Relaxed);
+ r
+ })
+ .collect()
}
/// Copy `src` into `root` (filed per `organize` by `date`) as `name` (default: its own name),
@@ -331,53 +346,107 @@ fn copy_into(root: &Path, src: &str, date: &str, organize: Organize, name: Optio
Ok(dst.to_string_lossy().to_string())
}
+/// What a [`scan`] needs from the session, so it can run on another thread (a folder on a network
+/// share can take minutes to read).
+pub struct ScanInput {
+ probe: Option<crate::media::FileProbe>,
+ skip: Option<PathBuf>,
+ /// path → (photo, its summary) for files already in the library.
+ by_path: HashMap<String, (PhotoId, ImportCandidate)>,
+ by_hash: HashMap<String, PhotoId>,
+ cache: HashMap<String, ProbeInfo>,
+}
+
+/// Progress and cancellation of a running [`scan_with`].
+#[derive(Default)]
+pub struct ScanProgress {
+ /// Files to probe, set once the folders are expanded.
+ pub total: std::sync::atomic::AtomicUsize,
+ pub done: std::sync::atomic::AtomicUsize,
+ pub cancel: std::sync::atomic::AtomicBool,
+}
+
+/// The result of [`scan_with`]: the candidates, and the probes to keep for the import that follows.
+pub struct ScanOutput {
+ pub candidates: Vec<ImportCandidate>,
+ pub probes: HashMap<String, ProbeInfo>,
+}
+
+impl ScanInput {
+ pub fn new(s: &mut Session, paths: &[String]) -> (Self, Vec<String>) {
+ let skip = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());
+ let mut by_path = HashMap::new();
+ let mut by_hash = HashMap::new();
+ for p in s.catalog.photos() {
+ if let Source::File { path } = &p.source {
+ let c = ImportCandidate {
+ path: path.clone(),
+ format: p.format.clone(),
+ kind: p.kind,
+ width: p.width,
+ height: p.height,
+ file_size: p.file_size,
+ captured: p.captured.clone(),
+ duplicate: Some("path".into()),
+ existing: Some(p.id.0),
+ ..Default::default()
+ };
+ by_path.insert(path.clone(), (p.id, c));
+ }
+ if let Some(h) = &p.content_hash {
+ by_hash.insert(h.clone(), p.id);
+ }
+ }
+ let input = ScanInput { probe: s.media.file_probe.clone(), skip, by_path, by_hash, cache: std::mem::take(&mut s.import_probes) };
+ (input, paths.to_vec())
+ }
+}
+
/// Find and probe what importing `paths` would add, marking duplicates (by path, or by content
/// against the library and earlier candidates). Nothing is added; the probes are kept for the
/// import that follows.
pub fn scan(s: &mut Session, paths: &[String]) -> Vec<ImportCandidate> {
- let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());
- let files = expand(paths, lib_dir.as_deref());
- let mut by_path: HashMap<String, PhotoId> = HashMap::new();
- let mut by_hash: HashMap<String, PhotoId> = HashMap::new();
- for p in s.catalog.photos() {
- if let Source::File { path } = &p.source {
- by_path.insert(path.clone(), p.id);
- }
- if let Some(h) = &p.content_hash {
- by_hash.insert(h.clone(), p.id);
- }
- }
- let todo: Vec<String> = files.iter().filter(|f| !by_path.contains_key(f.as_str())).cloned().collect();
+ let (input, paths) = ScanInput::new(s, paths);
+ let out = scan_with(input, &paths, &ScanProgress::default());
+ s.import_probes = out.probes;
+ out.candidates
+}
+
+/// [`scan`] without the session, so it can run on a worker thread. Stops early (returning what it
+/// has) when `progress.cancel` is set.
+pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput {
+ use std::sync::atomic::Ordering::Relaxed;
+ let files = expand(paths, input.skip.as_deref());
+ let todo: Vec<String> = files.iter().filter(|f| !input.by_path.contains_key(f.as_str())).cloned().collect();
+ progress.total.store(todo.len(), Relaxed);
// probes from a preceding `scan` are reused when the file is unchanged (same size)
let cached: Vec<Option<ProbeInfo>> = todo
.iter()
.map(|f| {
- let info = s.import_probes.remove(f)?;
+ let info = input.cache.remove(f)?;
let size = std::fs::metadata(f).map(|m| m.len()).ok();
(size.is_none() || size == Some(info.file_size)).then_some(info)
})
.collect();
let missing: Vec<String> = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect();
- let mut fresh = probe_all(s, &missing).into_iter();
+ progress.done.store(todo.len() - missing.len(), Relaxed);
+ let mut fresh = probe_all(input.probe.as_ref(), &missing, progress).into_iter();
let probed: Vec<Result<ProbeInfo, String>> =
cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect();
crate::memory::release();
let mut probes: HashMap<String, Result<ProbeInfo, String>> = todo.into_iter().zip(probed).collect();
- let mut seen_hash: HashMap<String, Option<u64>> = by_hash.into_iter().map(|(h, id)| (h, Some(id.0))).collect();
+ let mut seen_hash: HashMap<String, Option<u64>> = input.by_hash.into_iter().map(|(h, id)| (h, Some(id.0))).collect();
let mut out = Vec::with_capacity(files.len());
+ let mut kept = HashMap::new();
for f in files {
- let name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone());
- let mut c = ImportCandidate { path: f.clone(), name, ..Default::default() };
- if let Some(id) = by_path.get(&f) {
- c.duplicate = Some("path".into());
- c.existing = Some(id.0);
- if let Some(p) = s.catalog.photo(*id) {
- (c.format, c.kind, c.width, c.height, c.file_size, c.captured) =
- (p.format.clone(), p.kind, p.width, p.height, p.file_size, p.captured.clone());
- }
+ if let Some((_, c)) = input.by_path.get(&f) {
+ let mut c = c.clone();
+ c.name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone());
out.push(c);
continue;
}
+ let name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone());
+ let mut c = ImportCandidate { path: f.clone(), name, ..Default::default() };
match probes.remove(&f) {
Some(Ok(info)) => {
(c.format, c.kind, c.width, c.height, c.file_size, c.captured) =
@@ -393,14 +462,14 @@ pub fn scan(s: &mut Session, paths: &[String]) -> Vec<ImportCandidate> {
}
}
}
- s.import_probes.insert(f, info);
+ kept.insert(f, info);
}
Some(Err(e)) => c.error = Some(e),
None => c.error = Some("not probed".into()),
}
out.push(c);
}
- out
+ ScanOutput { candidates: out, probes: kept }
}
/// Import files/folders. See the module docs.
@@ -454,7 +523,7 @@ pub fn import_with(s: &mut Session, paths: &[String], opts: &ImportOptions) -> c
}
}
- let probed = probe_all(s, &todo);
+ let probed = probe_all(s.media.file_probe.as_ref(), &todo, &ScanProgress::default());
crate::memory::release();
let now = (s.clock)();
let mut ops = promote;
diff --git a/crates/ui-egui/src/headless.rs b/crates/ui-egui/src/headless.rs
index c504a46f7..d2669ea6c 100644
--- a/crates/ui-egui/src/headless.rs
+++ b/crates/ui-egui/src/headless.rs
@@ -799,11 +799,36 @@ mod tests {
h.app.ui.dialog = None;
let r = h.request("engine.execute", json!({"command": "file.addFromDevice", "params": {"path": sub.to_string_lossy()}}), t);
assert_eq!(r["ok"], true, "{r}");
+ h.settle(SETTLE);
let Some(crate::state::Dialog::Import { opts }) = &h.app.ui.dialog else { panic!("no import review") };
assert!(opts.copy);
let _ = std::fs::remove_dir_all(&dir);
}
+ /// The folder scan runs in the background: the request returns at once and the review opens
+ /// when the scan finishes (a folder on a network share must not freeze the window).
+ #[test]
+ fn add_folder_scans_in_the_background() {
+ let mut h = demo([1200.0, 900.0]);
+ let t = Duration::from_secs(10);
+ let dir = std::env::temp_dir().join(format!("lc-scanbg-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let img = lightcraft_raster::Rgba8::from_fn(8, 8, |x, y| [(x * 9) as u8, (y * 12) as u8, 80, 255]);
+ let png = lightcraft_engine::export::encode_image(
+ &img,
+ &lightcraft_engine::export::ExportOptions { format: lightcraft_engine::export::ExportFormat::Png, ..Default::default() },
+ )
+ .unwrap();
+ std::fs::write(dir.join("a.png"), &png).unwrap();
+ let r = h.request("engine.execute", json!({"command": "file.addFolder", "params": {"path": dir.to_string_lossy()}}), t);
+ assert_eq!(r["result"]["scanning"], true, "{r}");
+ assert!(h.app.scan.is_some() || h.app.ui.dialog.is_some());
+ h.settle(SETTLE);
+ assert!(h.app.scan.is_none());
+ assert!(matches!(h.app.ui.dialog, Some(crate::state::Dialog::Import { .. })));
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
/// ⇧⌘V opens Paste Selected Settings (prefilled with the copied groups); unchecking a group
/// leaves it alone. ⌘F puts typing into the search field.
#[test]
@@ -914,9 +939,11 @@ mod tests {
let n0 = h.app.session.catalog.len();
let undo0 = h.app.session.undo.len();
let r = h.request("engine.execute", json!({"command": "file.addPhotos", "params": {"paths": [dir.to_string_lossy()]}}), t);
- assert_eq!(r["result"]["candidates"], 6, "{r}");
- assert_eq!(r["result"]["duplicates"], 1, "{r}");
+ assert_eq!(r["result"]["scanning"], true, "{r}");
h.settle(SETTLE);
+ let Some(crate::state::Dialog::Import { opts }) = &h.app.ui.dialog else { panic!("no import review") };
+ assert_eq!(opts.candidates.len(), 6);
+ assert_eq!(opts.candidates.iter().filter(|c| c.duplicate.is_some()).count(), 1);
assert!(h.app.renderer.textures.keys().filter(|s| matches!(s, crate::render::Slot::Import(_))).count() >= 5, "thumbnails");
// uncheck the first photo; name a new album and keywords
let r = h.request("ui.clickWidget", json!({"id": "import:0"}), t);
diff --git a/crates/ui-egui/src/import.rs b/crates/ui-egui/src/import.rs
index d7c5dfd3c..ed0a5bef8 100644
--- a/crates/ui-egui/src/import.rs
+++ b/crates/ui-egui/src/import.rs
@@ -5,9 +5,10 @@
//! frame, with a progress window; the whole import is one undo step.
use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2};
-use lightcraft_engine::import::ImportCandidate;
+use lightcraft_engine::import::{ImportCandidate, ScanInput, ScanOutput, ScanProgress, scan_with};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
+use std::sync::atomic::Ordering;
use crate::LightcraftApp;
use crate::render::Slot;
@@ -79,19 +80,82 @@ pub struct ImportTask {
first: Option<u64>,
}
-/// Scan `paths` and open the review dialog. Returns the candidate counts.
+/// A folder scan running on a worker thread (a network share can take minutes to read; the
+/// window must keep answering meanwhile).
+pub struct ScanTask {
+ progress: std::sync::Arc<ScanProgress>,
+ rx: std::sync::mpsc::Receiver<ScanOutput>,
+ /// Open the review with "copy into the library" checked (a camera / card).
+ pub copy: bool,
+}
+
+/// Scan `paths` in the background, then open the review dialog (see [`poll_scan`]).
pub fn open(app: &mut LightcraftApp, paths: Vec<String>) -> Result<Value, String> {
- let r = app.session.execute("library.importPreview", &json!({"paths": paths})).map_err(|e| e.to_string())?;
- let candidates: Vec<ImportCandidate> = serde_json::from_value(r["candidates"].clone()).unwrap_or_default();
- if candidates.is_empty() {
- app.toast(&egui::Context::default(), "No photos found");
- return Ok(json!({"candidates": 0}));
+ if app.scan.is_some() {
+ return Err("a scan is already running".into());
+ }
+ let (input, paths) = ScanInput::new(&mut app.session, &paths);
+ let progress = std::sync::Arc::new(ScanProgress::default());
+ let (tx, rx) = std::sync::mpsc::channel();
+ let p = progress.clone();
+ let job = move || {
+ let _ = tx.send(scan_with(input, &paths, &p));
+ };
+ #[cfg(not(target_arch = "wasm32"))]
+ std::thread::spawn(job);
+ #[cfg(target_arch = "wasm32")]
+ job();
+ app.scan = Some(ScanTask { progress, rx, copy: false });
+ Ok(json!({"scanning": true}))
+}
+
+/// Collect a finished scan and open the review (called every frame).
+pub fn poll_scan(app: &mut LightcraftApp, ctx: &egui::Context) {
+ let Some(task) = app.scan.as_ref() else { return };
+ ctx.request_repaint_after(std::time::Duration::from_millis(100));
+ let out = match task.rx.try_recv() {
+ Ok(o) => o,
+ Err(std::sync::mpsc::TryRecvError::Empty) => return,
+ Err(_) => {
+ app.scan = None;
+ app.toast(ctx, "Scan failed");
+ return;
+ }
+ };
+ let task = app.scan.take().expect("scan task");
+ if task.progress.cancel.load(Ordering::Relaxed) {
+ return;
+ }
+ app.session.import_probes = out.probes;
+ if out.candidates.is_empty() {
+ app.toast(ctx, "No photos found");
+ return;
}
app.renderer.forget_imports();
- let d = ImportDialog::new(candidates);
- let out = json!({"candidates": d.candidates.len(), "duplicates": r["duplicates"]});
+ let mut d = ImportDialog::new(out.candidates);
+ d.copy = task.copy;
app.ui.dialog = Some(crate::state::Dialog::Import { opts: Box::new(d) });
- Ok(out)
+}
+
+/// The progress window while a folder is being scanned.
+pub fn scan_progress(app: &mut LightcraftApp, ctx: &egui::Context) {
+ let Some(task) = &app.scan else { return };
+ let t = Tokens::get(ctx);
+ let total = task.progress.total.load(Ordering::Relaxed);
+ let done = task.progress.done.load(Ordering::Relaxed);
+ let text = if total == 0 { "Looking for photos…".to_string() } else { format!("Reading photos… {done} of {total}") };
+ let mut cancel = false;
+ egui::Window::new("Scanning").title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([340.0, 80.0]).show(
+ ctx,
+ |ui| {
+ ui.label(egui::RichText::new(text).color(t.text));
+ ui.add(egui::ProgressBar::new(done as f32 / total.max(1) as f32).desired_width(320.0));
+ cancel = ui.button("Cancel").clicked();
+ },
+ );
+ if cancel {
+ task.progress.cancel.store(true, Ordering::Relaxed);
+ }
}
/// Start importing the dialog's checked files (the dialog's OK / `ui.dialog.confirm`).
diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs
index 563417330..6a67bd16d 100644
--- a/crates/ui-egui/src/lib.rs
+++ b/crates/ui-egui/src/lib.rs
@@ -125,6 +125,8 @@ pub struct LightcraftApp {
pub merge: merge::MergeState,
/// An import in progress (the import review dialog's batches).
pub import: Option<import::ImportTask>,
+ /// A folder scan in progress (feeds the import review).
+ pub scan: Option<import::ScanTask>,
/// A background export in progress.
pub export: Option<export_task::ExportTask>,
/// The files of the last finished background export (`ui.inspect` → `export.last`).
@@ -172,6 +174,7 @@ impl LightcraftApp {
loupe_shown: None,
merge: merge::MergeState::default(),
import: None,
+ scan: None,
export: None,
last_export_result: None,
hover_preview: None,
@@ -404,6 +407,7 @@ impl LightcraftApp {
}
self.renderer.poll(ctx, &mut self.session);
merge::poll(self, ctx);
+ import::poll_scan(self, ctx);
import::tick(self, ctx);
self.preview_build_status(ctx);
self.slideshow_tick(ctx);
@@ -565,6 +569,7 @@ impl LightcraftApp {
panels::second::show(self, &ctx);
panels::dialogs::show(self, &ctx);
import::progress(self, &ctx);
+ import::scan_progress(self, &ctx);
export_task::poll(self, &ctx);
panels::grid::drag_feedback(self, &ctx);
panels::toast(self, &ctx);
diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs
index 5112b531e..90b714cd0 100644
--- a/crates/ui-egui/src/menus.rs
+++ b/crates/ui-egui/src/menus.rs
@@ -879,8 +879,8 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option<Re
},
};
let r = crate::import::open(app, vec![path]);
- if let Some(crate::state::Dialog::Import { opts }) = &mut app.ui.dialog {
- opts.copy = true;
+ if let Some(t) = &mut app.scan {
+ t.copy = true;
}
r
}
diff --git a/docs/parity.md b/docs/parity.md
index 0da044e7b..32b48fb85 100644
--- a/docs/parity.md
+++ b/docs/parity.md
@@ -98,7 +98,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins.
| Id | Feature | Tier | Status | Evidence | Missing / notes |
|---|---|---|---|---|---|
-| LR-IMP-ADD-DIALOG | Add photos/folders | P0 | ✅ | `cmd:file.addPhotos`, `cmd:library.importPreview`, `cmd:library.import` (`mode`, `album`, `albumName`, `preset`, `keywords`), `crates/ui-egui/src/import.rs` | review dialog: candidate grid with thumbnails and checkboxes (duplicates by path/content marked and unchecked), destination, album (existing/new), preset, keywords; batched import with a progress window; one undo step. Drag-and-drop still imports directly |
+| LR-IMP-ADD-DIALOG | Add photos/folders | P0 | ✅ | `cmd:file.addPhotos`, `cmd:library.importPreview`, `cmd:library.import` (`mode`, `album`, `albumName`, `preset`, `keywords`), `crates/ui-egui/src/import.rs` | review dialog: candidate grid with thumbnails and checkboxes (duplicates by path/content marked and unchecked), destination, album (existing/new), preset, keywords; the folder scan runs on a background thread with a progress window and Cancel (a NAS folder no longer freezes the window); batched import with a progress window; one undo step. Drag-and-drop still imports directly |
| LR-IMP-DRAGDROP | Drop files/folders to import | P0 | ✅ | `crates/ui-egui/src/lib.rs` (dropped files → `cmd:library.import`) | dropping onto a specific album not supported |
| LR-IMP-DUPES | Skip duplicates by content | P1 | ✅ | `crates/engine/src/import.rs`, `crates/engine/src/tests_import.rs` | |
| LR-IMP-DEVICE | Import from camera/card | P1 | ✅ | `cmd:library.devices`, `cmd:file.addFromDevice`, `crates/engine/src/devices.rs` | mounted volumes with a DCIM folder (macOS /Volumes, Linux /media and /run/media, Windows drive letters); File → Add from Device → the import review, copying into the library by default; no PTP/MTP (cameras that don't mount as a disk) |