Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

LR-IMP-ADD-DIALOG: an import checks the probes a scan left several files at a time

Nathaniel Ledford6c19862876ec
diff --git a/crates/engine/src/import/files.rs b/crates/engine/src/import/files.rs
index 099794597..28f45a73f 100644
--- a/crates/engine/src/import/files.rs
+++ b/crates/engine/src/import/files.rs
@@ -238,6 +238,92 @@ pub(crate) fn probe_all(
.collect()
}
+/// What the disk says about a file ([`stat_all`]).
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum Seen {
+ /// Not asked about: `cancel` was set first.
+ Unchecked,
+ /// Not there, or not readable.
+ Gone,
+ At { is_file: bool, len: u64 },
+}
+
+/// [`Seen`] of `path` from the file system.
+fn stat_file(path: &str) -> Seen {
+ match std::fs::metadata(path) {
+ Ok(m) => Seen::At { is_file: m.is_file(), len: m.len() },
+ Err(_) => Seen::Gone,
+ }
+}
+
+/// Names asked about at once: a stat on a network share is a round trip (50 ms and more under load)
+/// that waits for nothing but the server, so a few more than the readers' threads.
+#[cfg(not(target_arch = "wasm32"))]
+const STAT_THREADS: usize = 16;
+
+/// Ask `stat` about each of `paths`, several at once (one at a time, a folder of thousands of photos
+/// on a network share waited minutes for its stats alone). The answers come in the order of `paths`;
+/// the ones not reached once `cancel` is set are [`Seen::Unchecked`].
+pub(crate) fn stat_all(paths: &[&str], cancel: &std::sync::atomic::AtomicBool, stat: impl Fn(&str) -> Seen + Sync) -> Vec<Seen> {
+ use std::sync::atomic::Ordering::Relaxed;
+ let mut seen = vec![Seen::Unchecked; paths.len()];
+ #[cfg(not(target_arch = "wasm32"))]
+ {
+ let n = STAT_THREADS.min(paths.len());
+ if n > 1 {
+ let next = std::sync::atomic::AtomicUsize::new(0);
+ let answers: Vec<Vec<(usize, Seen)>> = std::thread::scope(|sc| {
+ let readers: Vec<_> = (0..n)
+ .map(|_| {
+ sc.spawn(|| {
+ let mut mine = Vec::new();
+ loop {
+ let i = next.fetch_add(1, Relaxed);
+ let Some(path) = paths.get(i) else { break };
+ if cancel.load(Relaxed) {
+ break;
+ }
+ mine.push((i, stat(path)));
+ }
+ mine
+ })
+ })
+ .collect();
+ // (a reader that died leaves its files unchecked)
+ readers.into_iter().map(|r| r.join().unwrap_or_default()).collect()
+ });
+ for (i, s) in answers.into_iter().flatten() {
+ if let Some(slot) = seen.get_mut(i) {
+ *slot = s;
+ }
+ }
+ return seen;
+ }
+ }
+ for (slot, path) in seen.iter_mut().zip(paths) {
+ if cancel.load(Relaxed) {
+ break;
+ }
+ *slot = stat(path);
+ }
+ seen
+}
+
+/// The probes in `cache` of `files` that are still true: the file is looked at ([`stat_all`]) and
+/// `ok` says it is as it was probed. `None` for a file with no probe in `cache`, one that changed,
+/// and one not looked at because `cancel` was set.
+pub(super) fn reusable_probes(
+ files: &[String],
+ cache: &mut HashMap<String, ProbeInfo>,
+ cancel: &std::sync::atomic::AtomicBool,
+ ok: impl Fn(Seen, &ProbeInfo) -> bool,
+) -> Vec<Option<ProbeInfo>> {
+ let held: Vec<Option<ProbeInfo>> = files.iter().map(|f| cache.remove(f)).collect();
+ let asked: Vec<&str> = files.iter().zip(&held).filter(|(_, h)| h.is_some()).map(|(f, _)| f.as_str()).collect();
+ let mut seen = stat_all(&asked, cancel, stat_file).into_iter();
+ held.into_iter().map(|h| h.filter(|info| seen.next().is_some_and(|s| s != Seen::Unchecked && ok(s, info)))).collect()
+}
+
/// Copy `src` into `dir` as `name` (default: its own name), made unique with -1, -2…; returns the
/// new path. The copy is verified ([`crate::import_move::copy_new`]: a new file, synced and
/// checked against `probe_hash`, the content hash the probe computed — or compared byte for byte
@@ -370,4 +456,87 @@ mod prepared_tests {
assert!(seen.load(Relaxed) <= 8, "read {} files after cancel", seen.load(Relaxed));
assert!(prepared.items.is_empty());
}
+
+ /// Given files on a network share, when an import checks the probes a scan left, then it asks
+ /// the disk about several files at once: one at a time, 5,990 photos spent 300 s on stats alone
+ /// (99 % of the import's time), each a round trip to the server.
+ #[test]
+ fn stats_are_asked_for_several_files_at_once() {
+ use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
+ let (active, together) = (AtomicUsize::new(0), AtomicUsize::new(0));
+ let names: Vec<String> = (0..6).map(|i| format!("file-{i}")).collect();
+ let paths: Vec<&str> = names.iter().map(String::as_str).collect();
+ let seen = stat_all(&paths, &std::sync::atomic::AtomicBool::new(false), |p| {
+ active.fetch_add(1, Relaxed);
+ // waits (at most a second) for another file to be asked about while this one is
+ let end = std::time::Instant::now() + std::time::Duration::from_secs(1);
+ while active.load(Relaxed) < 2 && std::time::Instant::now() < end {
+ std::thread::yield_now();
+ }
+ together.fetch_max(active.load(Relaxed), Relaxed);
+ active.fetch_sub(1, Relaxed);
+ Seen::At { is_file: true, len: p.trim_start_matches("file-").parse().unwrap_or(u64::MAX) }
+ });
+ assert!(together.load(Relaxed) >= 2, "the files were asked about one at a time");
+ let lens: Vec<Seen> = (0..6).map(|len| Seen::At { is_file: true, len }).collect();
+ assert_eq!(seen, lens, "the answers keep the order of the files");
+ }
+
+ /// ✕ while the stats run: the files not asked about yet stay unchecked, and no more are asked.
+ #[test]
+ fn stats_stop_when_cancelled() {
+ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::Relaxed};
+ let (cancel, asked) = (AtomicBool::new(false), AtomicUsize::new(0));
+ let names: Vec<String> = (0..400).map(|i| format!("file-{i}")).collect();
+ let paths: Vec<&str> = names.iter().map(String::as_str).collect();
+ let seen = stat_all(&paths, &cancel, |_| {
+ asked.fetch_add(1, Relaxed);
+ cancel.store(true, Relaxed);
+ Seen::Gone
+ });
+ assert_eq!(seen.len(), 400);
+ // at most the files the (up to 16) readers had already started
+ assert!(asked.load(Relaxed) <= 16, "asked about {} files after cancel", asked.load(Relaxed));
+ assert_eq!(seen.iter().filter(|s| **s == Seen::Gone).count(), asked.load(Relaxed));
+ assert!(seen.iter().all(|s| matches!(s, Seen::Gone | Seen::Unchecked)));
+ }
+
+ /// A probe a scan left is used again for a file that is as it was, and only for it: a file that
+ /// changed size, one that is gone and one with no probe are read.
+ #[test]
+ fn a_scans_probe_is_reused_only_for_a_file_as_it_was() {
+ use std::sync::atomic::Ordering::Relaxed;
+ let dir = std::env::temp_dir().join(format!("lightcraft-reuse-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ let mut files = Vec::new();
+ for (name, bytes) in [("same.jpg", 10usize), ("grown.jpg", 10), ("fresh.jpg", 10)] {
+ let p = dir.join(name);
+ std::fs::write(&p, vec![0u8; bytes]).unwrap();
+ files.push(p.to_string_lossy().to_string());
+ }
+ let gone = dir.join("gone.jpg").to_string_lossy().to_string();
+ files.push(gone.clone());
+ let mut s = Session::new();
+ let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }).unwrap();
+ let probed = |size: u64| ProbeInfo { format: "JPEG".into(), file_size: size, ..ProbeInfo::default() };
+ job.cache.insert(files[0].clone(), probed(10));
+ job.cache.insert(files[1].clone(), probed(5));
+ job.cache.insert(gone.clone(), probed(10));
+ let read = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
+ let read_in_probe = read.clone();
+ job.probe = Some(std::sync::Arc::new(move |p: &str| {
+ read_in_probe.lock().unwrap().push(p.to_string());
+ Ok(probed(10))
+ }));
+ let prepared = job.prepare_files(files.clone(), &std::sync::atomic::AtomicBool::new(false));
+ let mut read = read.lock().unwrap().clone();
+ read.sort();
+ let mut expect = vec![files[1].clone(), files[2].clone(), gone];
+ expect.sort();
+ assert_eq!(read, expect, "only what was not as the scan saw it is read");
+ assert_eq!(job.read_total.load(Relaxed), 3);
+ assert_eq!(prepared.items.len(), 4);
+ let _ = std::fs::remove_dir_all(&dir);
+ }
}
diff --git a/crates/engine/src/import/job.rs b/crates/engine/src/import/job.rs
index 1730518d5..41fc56a94 100644
--- a/crates/engine/src/import/job.rs
+++ b/crates/engine/src/import/job.rs
@@ -1,5 +1,5 @@
use super::commit::{How, Plan, Slot, Transfer, commit_prepared, name_family, run_transfers, system_clock};
-use super::files::{expand, probe_all};
+use super::files::{Seen, expand, probe_all, reusable_probes, sidecars_of};
use super::types::{ImportCandidate, ImportMode, ImportOptions, ImportReport, Kept, OnDeleted};
use super::{HashMap, MediaKind, Path, PathBuf, Photo, PhotoId, ProbeInfo, Session, Source};
@@ -44,7 +44,7 @@ pub struct ImportJob {
pub(super) probe: Option<crate::media::FileProbe>,
file_bytes: Option<crate::merge::ByteReader>,
/// Probes from a preceding scan (not read again).
- cache: HashMap<String, ProbeInfo>,
+ pub(super) cache: HashMap<String, ProbeInfo>,
naming: crate::sidecar::SidecarNaming,
now: String,
seq: usize,
@@ -270,15 +270,10 @@ impl ImportJob {
// files a preceding scan already probed are not read again (a network share is slow to read),
// as long as they are still there, as they were: a file gone or changed since is probed again
- let cached: Vec<Option<ProbeInfo>> = todo
- .iter()
- .map(|f| {
- // (the web's files are not std::fs files: kept as probed)
- self.cache
- .remove(f)
- .filter(|info| cfg!(target_arch = "wasm32") || std::fs::metadata(f).is_ok_and(|m| m.is_file() && m.len() == info.file_size))
- })
- .collect();
+ let cached = reusable_probes(&todo, &mut self.cache, cancel, |seen, info| {
+ // (the web's files are not std::fs files: kept as probed)
+ cfg!(target_arch = "wasm32") || matches!(seen, Seen::At { is_file: true, len } if len == info.file_size)
+ });
let missing: Vec<String> = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect();
// Reading the files is the slow part of adding them where they are: it is counted on its
// own, and `cancel` stops it too (a large library takes long enough to want both).
diff --git a/crates/engine/src/import/scan.rs b/crates/engine/src/import/scan.rs
index ef925fa05..57ee995f2 100644
--- a/crates/engine/src/import/scan.rs
+++ b/crates/engine/src/import/scan.rs
@@ -1,4 +1,4 @@
-use super::files::{expand, probe_all};
+use super::files::{Seen, expand, probe_all, reusable_probes};
use super::types::ImportCandidate;
use super::{HashMap, Path, PathBuf, PhotoId, ProbeInfo, Session, Source};
@@ -123,20 +123,14 @@ pub(crate) fn scan_files(mut input: ScanInput, files: Vec<String>, progress: &Sc
let by_path = library_files(&input.library, &files);
let todo: Vec<String> = files.iter().filter(|f| !by_path.contains_key(Path::new(f))).cloned().collect();
progress.total.store(todo.len(), Relaxed);
- // probes from a preceding `scan` are reused when the file is unchanged (same size)
- let cached: Vec<Option<ProbeInfo>> = todo
- .iter()
- .map(|f| {
- let info = input.cache.remove(f)?;
- // (✕ set while the files' cached probes are checked: no more round trips to the disk; what is
- // not reached counts as not probed, like the files after it)
- if progress.cancel.load(Relaxed) {
- return None;
- }
- let size = std::fs::metadata(f).map(|m| m.len()).ok();
- (size.is_none() || size == Some(info.file_size)).then_some(info)
- })
- .collect();
+ // probes from a preceding `scan` are reused when the file is unchanged (same size); a file that
+ // can't be looked at counts as unchanged (✕ set while they are checked: no more round trips to the
+ // disk; what is not reached counts as not probed, like the files after it)
+ let cached = reusable_probes(&todo, &mut input.cache, &progress.cancel, |seen, info| match seen {
+ Seen::At { len, .. } => len == info.file_size,
+ Seen::Gone => true,
+ Seen::Unchecked => false,
+ });
let missing: Vec<String> = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect();
progress.done.store(todo.len() - missing.len(), Relaxed);
let mut fresh = probe_all(input.probe.as_ref(), &missing, &progress.done, &progress.cancel).into_iter();
diff --git a/docs/parity-checklist.md b/docs/parity-checklist.md
index 90cfc248f..2f659407c 100644
--- a/docs/parity-checklist.md
+++ b/docs/parity-checklist.md
@@ -1,6 +1,6 @@
# Lightroom parity checklist
-> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-25 · **Change:** minor (LR-BEHAV-TITLEBAR: a double-click follows the system setting, and a panel over the bar keeps the window still; KEYC-METADATA, LRC-LIB-KEYWORDS: ⌥0 / ⌥⇧0 go to the next / previous keyword set; LRC-DEV-HISTOGRAM-DRAG: the histogram's five tonal zones — hover for the value, drag to adjust the Basic control the press locked, double-click to reset; LR-VIEW-HISTOGRAM note updated; MENU-BAR: on macOS a menu item that yields its key to a focused text field or to shortcut recording really lets it go; LR-MERGE-HDR / LR-MERGE-PANO: Lanczos-3 warps; LR-IMP-LOCAL: browsing from the sidebar records the folder; LR-VIEW-ZOOM: only a full-size drag within 20 ms clears a slow-drag verdict; LR-IMP-ADD-DIALOG: hierarchical import album destination reconciled with main, #2362; KEY-SLIDER-NUDGE: vertical wheel nudges need a click or drag in the row; leaving it disarms them; LR-EDIT-OPTICS-PROFILE: Fujifilm nine-knot lens tables read at the plain radius; LR-VIEW-ZOOM: only a full-size drag within 20 ms clears a slow-drag verdict; LRC-LIB-LIBFOLDERS, LR-BEHAV-BG-TASKS: Synchronize Folder's scan shows its folder listing and stops it on ✕; LRC-LIB-LIBFOLDERS: Synchronize Folder says a folder is up to date; LR-LIB-STACK: filmstrip stack badge click; LRC-LIB-REFVIEW / CATALOG-SWITCH: Escape preserves edits; catalog-local pins; deleted and malformed references refused; LRC-LIB-COMPARE: the Info panel labels the active photo while culling, #2398; KEYC-RATING: `\` shows / hides the Filter Bar in the grids; LR-VIEW-PHOTOGRID: the grid reads only the photos an import batch or an edit touched, not the whole list; earlier: LRC-PRINT-PAGE: Print Page Options; LR-META-XMP: fewer false unimported-XMP warnings; KEYC-METADATA: ⌘K / Ctrl+K Add Keywords; LRC-DEV-HISTOGRAM-DRAG: the histogram's five tonal zones — hover for the value, drag to adjust the Basic control the press locked, double-click to reset; LR-VIEW-HISTOGRAM note updated; MENU-BAR: on macOS a menu item that yields its key to a focused text field or to shortcut recording really lets it go; LR-MERGE-HDR / LR-MERGE-PANO: Lanczos-3 warps; LR-IMP-LOCAL: browsing from the sidebar records the folder; LR-VIEW-ZOOM: only a full-size drag within 20 ms clears a slow-drag verdict; LR-IMP-ADD-DIALOG: hierarchical import album destination reconciled with main, #2362; KEY-SLIDER-NUDGE: vertical wheel nudges need a click or drag in the row; leaving it disarms them; LR-EDIT-OPTICS-PROFILE: Fujifilm nine-knot lens tables read at the plain radius; LR-VIEW-ZOOM: only a full-size drag within 20 ms clears a slow-drag verdict; LRC-LIB-LIBFOLDERS, LR-BEHAV-BG-TASKS: Synchronize Folder's scan shows its folder listing and stops it on ✕; LRC-LIB-LIBFOLDERS: Synchronize Folder says a folder is up to date; LR-LIB-STACK: filmstrip stack badge click; LRC-LIB-REFVIEW / CATALOG-SWITCH: Escape preserves edits; catalog-local pins; deleted and malformed references refused; LRC-LIB-COMPARE: the Info panel labels the active photo while culling, #2398; KEYC-RATING: `\` shows / hides the Filter Bar in the grids; LR-VIEW-PHOTOGRID: the grid reads only the photos an import batch or an edit touched, not the whole list; earlier: LRC-PRINT-PAGE: Print Page Options; LR-META-XMP: fewer false unimported-XMP warnings) · **Target:** Adobe Lightroom Classic 15.5 / Lightroom 9.6
+> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-25 · **Change:** minor (LR-IMP-ADD-DIALOG: an import asks the disk about the probes a scan left 16 files at a time; LR-BEHAV-TITLEBAR: a double-click follows the system setting, and a panel over the bar keeps the window still; KEYC-METADATA, LRC-LIB-KEYWORDS: ⌥0 / ⌥⇧0 go to the next / previous keyword set; LRC-DEV-HISTOGRAM-DRAG: the histogram's five tonal zones — hover for the value, drag to adjust the Basic control the press locked, double-click to reset; LR-VIEW-HISTOGRAM note updated; MENU-BAR: on macOS a menu item that yields its key to a focused text field or to shortcut recording really lets it go; LR-MERGE-HDR / LR-MERGE-PANO: Lanczos-3 warps; LR-IMP-LOCAL: browsing from the sidebar records the folder; LR-VIEW-ZOOM: only a full-size drag within 20 ms clears a slow-drag verdict; LR-IMP-ADD-DIALOG: hierarchical import album destination reconciled with main, #2362; KEY-SLIDER-NUDGE: vertical wheel nudges need a click or drag in the row; leaving it disarms them; LR-EDIT-OPTICS-PROFILE: Fujifilm nine-knot lens tables read at the plain radius; LR-VIEW-ZOOM: only a full-size drag within 20 ms clears a slow-drag verdict; LRC-LIB-LIBFOLDERS, LR-BEHAV-BG-TASKS: Synchronize Folder's scan shows its folder listing and stops it on ✕; LRC-LIB-LIBFOLDERS: Synchronize Folder says a folder is up to date; LR-LIB-STACK: filmstrip stack badge click; LRC-LIB-REFVIEW / CATALOG-SWITCH: Escape preserves edits; catalog-local pins; deleted and malformed references refused; LRC-LIB-COMPARE: the Info panel labels the active photo while culling, #2398; KEYC-RATING: `\` shows / hides the Filter Bar in the grids; LR-VIEW-PHOTOGRID: the grid reads only the photos an import batch or an edit touched, not the whole list; earlier: LRC-PRINT-PAGE: Print Page Options; LR-META-XMP: fewer false unimported-XMP warnings; KEYC-METADATA: ⌘K / Ctrl+K Add Keywords; LRC-DEV-HISTOGRAM-DRAG: the histogram's five tonal zones — hover for the value, drag to adjust the Basic control the press locked, double-click to reset; LR-VIEW-HISTOGRAM note updated; MENU-BAR: on macOS a menu item that yields its key to a focused text field or to shortcut recording really lets it go; LR-MERGE-HDR / LR-MERGE-PANO: Lanczos-3 warps; LR-IMP-LOCAL: browsing from the sidebar records the folder; LR-VIEW-ZOOM: only a full-size drag within 20 ms clears a slow-drag verdict; LR-IMP-ADD-DIALOG: hierarchical import album destination reconciled with main, #2362; KEY-SLIDER-NUDGE: vertical wheel nudges need a click or drag in the row; leaving it disarms them; LR-EDIT-OPTICS-PROFILE: Fujifilm nine-knot lens tables read at the plain radius; LR-VIEW-ZOOM: only a full-size drag within 20 ms clears a slow-drag verdict; LRC-LIB-LIBFOLDERS, LR-BEHAV-BG-TASKS: Synchronize Folder's scan shows its folder listing and stops it on ✕; LRC-LIB-LIBFOLDERS: Synchronize Folder says a folder is up to date; LR-LIB-STACK: filmstrip stack badge click; LRC-LIB-REFVIEW / CATALOG-SWITCH: Escape preserves edits; catalog-local pins; deleted and malformed references refused; LRC-LIB-COMPARE: the Info panel labels the active photo while culling, #2398; KEYC-RATING: `\` shows / hides the Filter Bar in the grids; LR-VIEW-PHOTOGRID: the grid reads only the photos an import batch or an edit touched, not the whole list; earlier: LRC-PRINT-PAGE: Print Page Options; LR-META-XMP: fewer false unimported-XMP warnings) · **Target:** Adobe Lightroom Classic 15.5 / Lightroom 9.6
The measured, row-by-row checklist: one row per Lightroom feature, menu item and shortcut, what we
have and what is missing. It answers *does the feature exist?* (feature breadth). Whether it is good
@@ -959,3 +959,4 @@ Latin stays upright. Advanced Japanese composition remains open. Regression cove
| 2026-10-23 | patch | LR-BEHAV-ACCESS: the name field of New Album, New Smart Album, Rename and the other one-field dialogs takes the keyboard focus once when the dialog opens instead of every frame, so Tab and Shift+Tab reach the check box, Cancel and OK; status unchanged (🟡) |
| 2026-10-22 | minor | LR-CROP-AUTO: Auto straighten turns by at most 10° and only on at least 3 agreeing lines, and says when it finds nothing to level by; measured on 68 photos straightened by hand and 300 left as shot, unwanted turns of more than 1° fell from 79 to 19 of the 300. Downgraded ✅ → 🟡: it still leaves most tilted photos alone |
| 2026-10-23 | patch | LRC-LIB-FILTERBAR: showing an album (`library.source`) while the filter holds an album drops the filter's album, as showing a library folder already dropped its folder filter; before, the old album stayed in the filter unseen (the filter bar has no album chip) and narrowed the next source such as All Photos; the row's note says so; status unchanged (✅) |
+| 2026-10-24 | patch | LR-IMP-ADD-DIALOG: an import checks that each file a scan already probed is as it was 16 files at a time instead of one after another; on a network share a folder of thousands of photos spent most of its import time waiting for those checks; status unchanged (✅) |
diff --git a/docs/whats-new.md b/docs/whats-new.md
index c6c259200..04daf118e 100644
--- a/docs/whats-new.md
+++ b/docs/whats-new.md
@@ -2,6 +2,12 @@
## 2026-10-24
+### Importing a folder from a network share waits less
+- After the import review, the import used to check that each photo was still as the review had seen it by asking the
+ disk about one photo after another, so on a network share every photo waited for a round trip to the server. It now
+ asks about 16 photos at a time, the same wait for every 16 instead of for each one. Reading the photos in the review
+ is unchanged.
+
### Relative-date filters and smart albums follow the clock
- A current view or source count using “in the last” rules now refreshes when the session clock moves into a new
minute, even when no catalog edit occurred. This fixes stale photos remaining in the grid and stale counts for