Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

PERF-SYNC: a Synchronize Folder scan reads off the folder's listing whether a photo and its sidecar are there

Nathaniel Ledford540a281ccda0
diff --git a/crates/engine/examples/sync_scan_bench.rs b/crates/engine/examples/sync_scan_bench.rs
new file mode 100644
index 000000000..3b9bc51c1
--- /dev/null
+++ b/crates/engine/examples/sync_scan_bench.rs
@@ -0,0 +1,47 @@
+//! What a Synchronize Folder scan costs on a folder whose photos are all in the library and
+//! unchanged: the walk, and the check of every photo and its sidecar. Reads the folder; writes
+//! nothing there, and probes no file (the library is made from the folder's own listing).
+//!
+//! `cargo run --release -p lightcraft-engine --example sync_scan_bench -- <folder>`
+use std::time::Instant;
+
+use lightcraft_catalog::{Op, Photo, Source};
+use lightcraft_engine::Session;
+use lightcraft_engine::sync::{SyncInput, SyncProgress, scan_with};
+
+fn main() {
+ let Some(folder) = std::env::args().nth(1) else {
+ eprintln!("usage: sync_scan_bench <folder>");
+ return;
+ };
+ let mut s = Session::new();
+ let t = Instant::now();
+ let files = lightcraft_engine::import::expand(std::slice::from_ref(&folder), None);
+ println!("{} files listed in {:.1} s (the library is made of them)", files.len(), t.elapsed().as_secs_f64());
+ for f in &files {
+ let id = s.catalog.alloc_photo_id();
+ let name = std::path::Path::new(f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
+ let p = Photo::new(id, Source::File { path: f.clone() }, &name, "JPEG", 6, 4, "2026-01-01T00:00:00");
+ if s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).is_err() {
+ eprintln!("could not add {name}");
+ }
+ }
+ for run in 0..2 {
+ let input = match SyncInput::new(&mut s, &folder, true) {
+ Ok(input) => input,
+ Err(e) => {
+ eprintln!("{e}");
+ return;
+ }
+ };
+ let t = Instant::now();
+ let changes = scan_with(input, &SyncProgress::default());
+ println!(
+ "run {run}: scan {:.1} s ({} new, {} missing, {} with sidecar news)",
+ t.elapsed().as_secs_f64(),
+ changes.new.len(),
+ changes.missing.len(),
+ changes.metadata.len()
+ );
+ }
+}
diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs
index ecb80741e..821454c72 100644
--- a/crates/engine/src/import.rs
+++ b/crates/engine/src/import.rs
@@ -380,6 +380,17 @@ pub fn expand(paths: &[String], skip: Option<&Path>) -> Vec<String> {
/// [`expand`] with the walk bounded by `limits` → (files, whether a walk stopped at a bound).
pub fn expand_within(paths: &[String], skip: Option<&Path>, limits: crate::walk::Limits) -> (Vec<String>, bool) {
+ expand_noting(paths, skip, limits, None)
+}
+
+/// [`expand_within`], handing the listing of every folder that is walked to `note`
+/// ([`crate::walk::files_in_noting`]).
+pub(crate) fn expand_noting(
+ paths: &[String],
+ skip: Option<&Path>,
+ limits: crate::walk::Limits,
+ mut note: Option<crate::walk::NoteFolder>,
+) -> (Vec<String>, bool) {
let mut out = Vec::new();
let mut truncated = false;
for p in paths {
@@ -388,7 +399,10 @@ pub fn expand_within(paths: &[String], skip: Option<&Path>, limits: crate::walk:
continue;
}
if p.is_dir() {
- let w = crate::walk::files_in(p, skip, limits, is_supported);
+ let w = match note.as_mut() {
+ Some(note) => crate::walk::files_in_noting(p, skip, limits, is_supported, &mut **note),
+ None => crate::walk::files_in(p, skip, limits, is_supported),
+ };
if w.truncated {
log::warn!(
"import: stopped looking in {} after {} entries (limits: {} entries, {} folder levels)",
@@ -630,6 +644,11 @@ impl ScanInput {
(input, paths.to_vec())
}
+ /// The folder a scan never enters (the library's own).
+ pub(crate) fn skip(&self) -> Option<&Path> {
+ self.skip.as_deref()
+ }
+
/// The library as the scan sees it (shared, for [`crate::sync`], which looks at the same one).
pub(crate) fn library(&self) -> std::sync::Arc<lightcraft_catalog::Catalog> {
std::sync::Arc::clone(&self.library)
@@ -648,9 +667,14 @@ pub fn scan(s: &mut Session, paths: &[String]) -> Vec<ImportCandidate> {
/// [`scan`] without the session, so it can run on a worker thread. Stops early (returning what it
/// has) when `progress.cancel` is set.
-pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput {
- use std::sync::atomic::Ordering::Relaxed;
+pub fn scan_with(input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput {
let files = expand(paths, input.skip.as_deref());
+ scan_files(input, files, progress)
+}
+
+/// [`scan_with`] for files that were found already (`paths` expanded, [`expand`]).
+pub(crate) fn scan_files(mut input: ScanInput, files: Vec<String>, progress: &ScanProgress) -> ScanOutput {
+ use std::sync::atomic::Ordering::Relaxed;
let by_path = library_files(&input.library, &files);
let todo: Vec<String> = files.iter().filter(|f| !by_path.contains_key(Path::new(f))).cloned().collect();
progress.total.store(todo.len(), Relaxed);
diff --git a/crates/engine/src/sidecar.rs b/crates/engine/src/sidecar.rs
index c67a2c84f..41d1c5f57 100644
--- a/crates/engine/src/sidecar.rs
+++ b/crates/engine/src/sidecar.rs
@@ -65,15 +65,18 @@ pub fn sidecar_path(original: &str, naming: SidecarNaming) -> PathBuf {
/// An existing sidecar for `original`: the preferred naming first, then the other one (also
/// accepting an upper-case `.XMP`).
pub fn find_sidecar(original: &str, naming: SidecarNaming) -> Option<PathBuf> {
+ sidecar_candidates(original, naming).find(|p| p.is_file())
+}
+
+/// The names `original`'s sidecar can have, in the order they are looked for: under `naming`,
+/// then under the other naming, each as `.xmp` and as `.XMP`.
+pub(crate) fn sidecar_candidates(original: &str, naming: SidecarNaming) -> impl Iterator<Item = PathBuf> {
let other = if naming == SidecarNaming::Stem { SidecarNaming::Full } else { SidecarNaming::Stem };
- [naming, other]
- .into_iter()
- .flat_map(|n| {
- let p = sidecar_path(original, n);
- let upper = p.with_extension("XMP");
- [p, upper]
- })
- .find(|p| p.is_file())
+ [naming, other].into_iter().flat_map(move |n| {
+ let p = sidecar_path(original, n);
+ let upper = p.with_extension("XMP");
+ [p, upper]
+ })
}
/// What a sidecar (or embedded XMP packet) says about a photo. `None` = not stated.
diff --git a/crates/engine/src/sync.rs b/crates/engine/src/sync.rs
index 4342ec6ad..6f236796e 100644
--- a/crates/engine/src/sync.rs
+++ b/crates/engine/src/sync.rs
@@ -38,7 +38,7 @@
//! not cancelled midway.
use std::collections::HashMap;
-use std::path::Path;
+use std::path::{Path, PathBuf};
use std::sync::Arc;
use lightcraft_catalog::{Catalog, Filter, Op, Photo, PhotoId, Sort, Source};
@@ -203,18 +203,37 @@ pub fn scan_with(input: SyncInput, progress: &SyncProgress) -> FolderChanges {
let namings = crate::sidecar::namings_in(&labels, naming, &ids);
let photos: Vec<(Arc<Photo>, SidecarNaming)> =
ids.iter().zip(namings).filter_map(|(id, naming)| labels.photo(*id).map(|p| (Arc::clone(p), naming))).collect();
+ // The folder is walked once, and its listing answers for the library's photos in it: on a
+ // network share that is one request for a folder's names where it was several for each
+ // photo. Then both halves wait on the disk at once: the new files are probed (on their own
+ // threads) beside the checks of the photos, which ask the disk only what the listing can't say.
+ // (how many photos there are to check is known before the walk: the count shows while it runs)
+ progress.to_check.store(photos.len(), std::sync::atomic::Ordering::Relaxed);
+ let mut listing = Listing::default();
+ let (files, _) = crate::import::expand_noting(
+ std::slice::from_ref(&folder),
+ scan.skip(),
+ crate::walk::Limits::default(),
+ Some(&mut |dir, names, complete| listing.note(dir, names, complete)),
+ );
+ let listing = &listing;
// (no threads on the web: one after the other there)
#[cfg(target_arch = "wasm32")]
- let (out, checks) = (crate::import::scan_with(scan, std::slice::from_ref(&folder), &progress.files), check_photos(&photos, &labels, progress));
+ let (out, checks) = (crate::import::scan_files(scan, files, &progress.files), check_photos(&photos, &labels, listing, progress));
#[cfg(not(target_arch = "wasm32"))]
let (out, checks) = std::thread::scope(|sc| {
let photos = &photos;
let labels = &labels;
- let checks = sc.spawn(move || check_photos(photos, labels, progress));
- let out = crate::import::scan_with(scan, std::slice::from_ref(&folder), &progress.files);
+ let checks = sc.spawn(move || check_photos(photos, labels, listing, progress));
+ let out = crate::import::scan_files(scan, files, &progress.files);
// (a check that panicked counts as nothing found; the scan still reports the rest)
(out, checks.join().unwrap_or_default())
});
+ log::debug!(
+ "synchronize: {} folders listed; the disk was asked {} times about the library's photos",
+ listing.folders.len(),
+ listing.asked.load(std::sync::atomic::Ordering::Relaxed)
+ );
let mut changes = FolderChanges { path: folder, seen, ..Default::default() };
for ((p, _), check) in photos.iter().zip(checks) {
let Some(file) = crate::cmd::missing::checked_path(p) else { continue };
@@ -335,11 +354,136 @@ enum Check {
News,
}
-fn check_photo(p: &Photo, naming: SidecarNaming, labels: &Catalog) -> Check {
+/// The listings of the folders a scan walked: what is in each, by name. It answers "is this file
+/// there" for the library's photos and their sidecars without asking the disk, which on a
+/// network share is a round trip for every name (and a photo without a sidecar took five).
+///
+/// It answers only what holds on any file system. A name is there when the folder lists exactly
+/// that name ([`asks_the_disk`] aside). It is not there when the folder was listed to its end,
+/// every name in the folder is ASCII, the name asked about is plain ASCII ([`plain_name`]), and
+/// no name in the folder is that name in another case. Anything else is not answered, and the
+/// disk is asked as before: a name that differs in case only, a link, a folder that was not
+/// walked or not to its end, and every name that is not listed in a folder that holds a name
+/// outside ASCII. (File systems hold names outside ASCII equal in more ways than can be listed
+/// here: by case, where the capital sharp s is `SS`; by how accents are written; and some pass
+/// over marks altogether.)
+///
+/// Where the listing and a look-up would differ, the listing is the folder as the scan saw it:
+/// a file it lists is there though it could not be opened by its path (a folder that may be
+/// listed but not searched, a path longer than the system takes), and a share that drops after
+/// the walk does not turn every photo the listing answers for into a missing one.
+#[derive(Default)]
+pub(crate) struct Listing {
+ folders: HashMap<PathBuf, Listed>,
+ /// How often the disk was asked after all (for the tests, and the log).
+ pub(crate) asked: std::sync::atomic::AtomicUsize,
+}
+
+/// One folder's listing.
+struct Listed {
+ /// name → what it is.
+ names: HashMap<String, crate::walk::EntryKind>,
+ /// The names, lower-cased (only kept for a folder whose names are all ASCII): what a plain
+ /// name asked about could be in another case.
+ lower: std::collections::HashSet<String>,
+ /// Every entry was read, and every name is ASCII: a plain name that is not listed, in any
+ /// case, is not there.
+ sure: bool,
+}
+
+/// Whether the disk is asked about `name` whatever the listing holds: a name Windows takes for
+/// a device (`NUL`, `aux.jpg`: there in every folder, or not openable where it is listed).
+fn asks_the_disk(name: &str) -> bool {
+ let stem = name.split('.').next().unwrap_or_default().trim_end_matches(' ').to_ascii_lowercase();
+ // (`COM` and `LPT` with any one digit, the superscript ones Windows also takes among them)
+ let numbered = ["com", "lpt"].iter().any(|port| {
+ let mut rest = stem.strip_prefix(port).unwrap_or_default().chars();
+ matches!((rest.next(), rest.next()), (Some(d), None) if d.is_ascii_digit() || ['\u{b9}', '\u{b2}', '\u{b3}'].contains(&d))
+ });
+ numbered || ["con", "prn", "aux", "nul", "conin$", "conout$", "clock$"].contains(&stem.as_str())
+}
+
+/// Whether `name` can only be found under itself in some case: ASCII, and nothing in it that a
+/// file system resolves another way (a short name with `~`, a stream after `:`, dots or spaces
+/// at its end, which Windows drops).
+fn plain_name(name: &str) -> bool {
+ name.is_ascii() && !name.contains(['~', ':']) && !name.ends_with(['.', ' ']) && !name.is_empty()
+}
+
+impl Listing {
+ /// Take the listing of `folder` from the walk ([`crate::walk::files_in_noting`]).
+ pub(crate) fn note(&mut self, folder: &Path, entries: Vec<(std::ffi::OsString, crate::walk::EntryKind)>, complete: bool) {
+ let mut sure = complete;
+ let mut names = HashMap::with_capacity(entries.len());
+ let mut lower = std::collections::HashSet::new();
+ for (name, kind) in entries {
+ match name.into_string() {
+ Ok(name) => {
+ sure &= name.is_ascii();
+ lower.insert(name.to_ascii_lowercase());
+ names.insert(name, kind);
+ }
+ // (a name that is no text is not one a photo's path can be compared with)
+ Err(_) => sure = false,
+ }
+ }
+ if !sure {
+ lower.clear();
+ }
+ self.folders.insert(folder.to_path_buf(), Listed { names, lower, sure });
+ }
+
+ /// What the listing of `path`'s folder says of it: its kind when it is listed under exactly
+ /// this name, `Some(None)` when it is surely not there, `None` when the listing can't say.
+ fn kind(&self, path: &Path) -> Option<Option<crate::walk::EntryKind>> {
+ let name = path.file_name()?.to_str()?;
+ // (the path ends in that name: no `/.` or separator after it, which the disk would resolve)
+ if !path.to_str()?.ends_with(name) {
+ return None;
+ }
+ if asks_the_disk(name) {
+ return None;
+ }
+ let folder = self.folders.get(path.parent()?)?;
+ if let Some(kind) = folder.names.get(name) {
+ return Some(Some(*kind));
+ }
+ (folder.sure && plain_name(name) && !folder.lower.contains(&name.to_ascii_lowercase())).then_some(None)
+ }
+
+ /// `Path::exists`, where the listing can say.
+ pub(crate) fn exists(&self, path: &Path) -> Option<bool> {
+ match self.kind(path)? {
+ Some(crate::walk::EntryKind::Other) => None,
+ Some(_) => Some(true),
+ None => Some(false),
+ }
+ }
+
+ /// `Path::is_file`, where the listing can say.
+ pub(crate) fn is_file(&self, path: &Path) -> Option<bool> {
+ match self.kind(path)? {
+ Some(crate::walk::EntryKind::Other) => None,
+ Some(kind) => Some(kind == crate::walk::EntryKind::File),
+ None => Some(false),
+ }
+ }
+
+ /// The listing's answer, or the disk's where it has none.
+ fn or_ask(&self, answer: Option<bool>, ask: impl FnOnce() -> bool) -> bool {
+ answer.unwrap_or_else(|| {
+ self.asked.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ ask()
+ })
+ }
+}
+
+fn check_photo(p: &Photo, naming: SidecarNaming, labels: &Catalog, listing: &Listing) -> Check {
let Some(file) = crate::cmd::missing::checked_path(p) else { return Check::Nothing };
- if !Path::new(file).exists() {
+ let file = Path::new(file);
+ if !listing.or_ask(listing.exists(file), || file.exists()) {
Check::Missing
- } else if p.copy_of.is_none() && sidecar_has_news(labels, p, naming) {
+ } else if p.copy_of.is_none() && sidecar_has_news(labels, p, naming, listing) {
Check::News
} else {
Check::Nothing
@@ -348,12 +492,12 @@ fn check_photo(p: &Photo, naming: SidecarNaming, labels: &Catalog) -> Check {
/// [`check_photo`] for each photo, in order, on several threads (each file is a wait on the
/// disk, not work for the CPU). Photos not reached before a cancel count as nothing found.
-fn check_photos(photos: &[(Arc<Photo>, SidecarNaming)], labels: &Catalog, progress: &SyncProgress) -> Vec<Check> {
+fn check_photos(photos: &[(Arc<Photo>, SidecarNaming)], labels: &Catalog, listing: &Listing, progress: &SyncProgress) -> Vec<Check> {
use std::sync::atomic::Ordering::Relaxed;
progress.to_check.store(photos.len(), Relaxed);
let cancel = &progress.files.cancel;
let check = |p: &Photo, naming: SidecarNaming| {
- let c = check_photo(p, naming, labels);
+ let c = check_photo(p, naming, labels, listing);
progress.checked.fetch_add(1, Relaxed);
c
};
@@ -421,10 +565,17 @@ impl Session {
/// Whether `p`'s XMP sidecar file was saved after the library last had its say about the photo
/// (imported, or edited here) and says something the library doesn't.
-fn sidecar_has_news(labels: &Catalog, p: &Photo, naming: SidecarNaming) -> bool {
+fn sidecar_has_news(labels: &Catalog, p: &Photo, naming: SidecarNaming, listing: &Listing) -> bool {
let Source::File { path } = &p.source else { return false };
- let Some(file) = crate::sidecar::find_sidecar(path, naming) else { return false };
- let Ok(meta) = std::fs::metadata(&file) else { return false };
+ // ([`crate::sidecar::find_sidecar`], each name looked up in the listing before the disk)
+ // (a name the listing holds is read for its date next, and one that can't be read is passed
+ // over for the next name, as a name that is no file was)
+ let found = crate::sidecar::sidecar_candidates(path, naming).find_map(|c| {
+ let there = listing.or_ask(listing.is_file(&c), || c.is_file());
+ let meta = there.then(|| std::fs::metadata(&c).ok()).flatten().filter(std::fs::Metadata::is_file)?;
+ Some((c, meta))
+ });
+ let Some((file, meta)) = found else { return false };
if meta.len() > MAX_SIDECAR_BYTES {
return false;
}
@@ -708,3 +859,7 @@ impl SyncCommit {
self.report
}
}
+
+#[cfg(test)]
+#[path = "sync/tests_listing.rs"]
+mod tests_listing;
diff --git a/crates/engine/src/sync/tests_listing.rs b/crates/engine/src/sync/tests_listing.rs
new file mode 100644
index 000000000..e54ed76ef
--- /dev/null
+++ b/crates/engine/src/sync/tests_listing.rs
@@ -0,0 +1,191 @@
+//! Synchronize Folder answers "is the photo's file there" and "has it a sidecar" from the
+//! listing of the folder its scan has just made, instead of asking the disk about every photo.
+//!
+//! On a network share each question is a round trip (about 5 ms a file on a hard disk over SMB,
+//! measured), and a photo without a sidecar took five of them; the listing has every name.
+//! The listing answers only what it can be sure of on any file system: where a name could match
+//! another way (case, letters outside ASCII, a link), the disk is asked as before.
+use super::*;
+use crate::walk::EntryKind::{File, Folder, Other};
+
+fn listing(folders: &[(&str, &[(&str, crate::walk::EntryKind)], bool)]) -> Listing {
+ let mut l = Listing::default();
+ for (dir, names, complete) in folders {
+ l.note(Path::new(dir), names.iter().map(|(n, k)| (std::ffi::OsString::from(n), *k)).collect(), *complete);
+ }
+ l
+}
+
+/// Given the listing of a folder, when it is asked about a name, then it says "there" for a
+/// name it lists, "not there" for one nothing in the folder could be taken for, and nothing
+/// (ask the disk) wherever a file system could see it otherwise.
+#[test]
+fn the_listing_answers_only_what_it_is_sure_of() {
+ let l = listing(&[
+ (
+ "/pics/trip",
+ &[("IMG_1.CR3", File), ("IMG_1.xmp", File), ("img_2.jpg", File), ("sub", Folder), ("link.jpg", Other), (".hidden.xmp", File)],
+ true,
+ ),
+ ("/pics/cut", &[("IMG_1.CR3", File)], false),
+ ("/pics/names", &[("IMG_1.CR3", File), ("Größe.JPG", File)], true),
+ // (the capital sharp s, which a file system that folds case takes for `SS`)
+ ("/pics/sharp", &[("IMG_1.CR3", File), ("STRA\u{1e9e}E.jpg", File)], true),
+ ("/pics/devices", &[("aux.jpg", File), ("COM\u{b9}.jpg", File), ("IMG_1.CR3", File)], true),
+ ]);
+ let (there, gone, ask) = (Some(true), Some(false), None);
+ for (path, exists, is_file) in [
+ ("/pics/trip/IMG_1.CR3", there, there),
+ ("/pics/trip/IMG_1.xmp", there, there),
+ ("/pics/trip/.hidden.xmp", there, there),
+ ("/pics/trip/sub", there, gone),
+ ("/pics/trip/IMG_9.CR3", gone, gone),
+ ("/pics/trip/IMG_1.XMP", ask, ask),
+ ("/pics/trip/IMG_2.JPG", ask, ask),
+ ("/pics/trip/link.jpg", ask, ask),
+ ("/pics/trip/SUB", ask, ask),
+ // a folder the walk did not list to its end, or not at all
+ ("/pics/cut/IMG_1.CR3", there, there),
+ ("/pics/cut/IMG_2.CR3", ask, ask),
+ ("/pics/other/IMG_1.CR3", ask, ask),
+ // a folder that holds a name outside ASCII: what it lists is there, and nothing is surely not there (file
+ // systems hold such names equal in more ways than one)
+ ("/pics/names/IMG_1.CR3", there, there),
+ ("/pics/names/Größe.JPG", there, there),
+ ("/pics/names/IMG_2.CR3", ask, ask),
+ ("/pics/names/größe.JPG", ask, ask),
+ ("/pics/trip/Größe.JPG", ask, ask),
+ ("/pics/sharp/strasse.jpg", ask, ask),
+ ("/pics/sharp/STRASSE.xmp", ask, ask),
+ ("/pics/sharp/IMG_2.CR3", ask, ask),
+ // names Windows resolves another way: a short name, a stream, dots or spaces at the end, a device
+ ("/pics/trip/IMG_10~1.CR3", ask, ask),
+ ("/pics/trip/IMG_1.CR3::$DATA", ask, ask),
+ ("/pics/trip/IMG_1.CR3.", ask, ask),
+ ("/pics/trip/IMG_1.CR3 ", ask, ask),
+ ("/pics/devices/aux.jpg", ask, ask),
+ ("/pics/devices/NUL", ask, ask),
+ ("/pics/devices/nul.xmp", ask, ask),
+ ("/pics/devices/COM1 .jpg", ask, ask),
+ ("/pics/devices/COM\u{b9}.jpg", ask, ask),
+ ("/pics/trip/con", ask, ask),
+ ("/pics/trip/PRN.txt", ask, ask),
+ ("/pics/trip/com2.jpg", ask, ask),
+ ("/pics/trip/LPT9.xmp", ask, ask),
+ ("/pics/trip/lpt0", ask, ask),
+ ("/pics/trip/nul.tar.gz", ask, ask),
+ ("/pics/trip/CONIN$", ask, ask),
+ ("/pics/trip/com10.jpg", gone, gone),
+ ("/pics/trip/computer.jpg", gone, gone),
+ ("/pics/devices/IMG_1.CR3", there, there),
+ ("/pics/devices/auxiliary.jpg", ask, ask),
+ ("/pics/trip/auxiliary.jpg", gone, gone),
+ // a path that does not end in its file's name is the disk's to resolve
+ ("/pics/trip/IMG_1.CR3/", ask, ask),
+ ("/pics/trip/IMG_1.CR3/.", ask, ask),
+ ("/pics/trip/", ask, ask),
+ ("", ask, ask),
+ ] {
+ assert_eq!((l.exists(Path::new(path)), l.is_file(Path::new(path))), (exists, is_file), "{path:?}");
+ }
+ // a name that is no text: nothing in its folder is surely not there
+ #[cfg(unix)]
+ {
+ use std::os::unix::ffi::OsStringExt;
+ let mut l = Listing::default();
+ l.note(
+ Path::new("/pics/raw"),
+ vec![(std::ffi::OsString::from("IMG_1.CR3"), File), (std::ffi::OsString::from_vec(vec![0x66, 0xff, 0x6f]), File)],
+ true,
+ );
+ assert_eq!((l.exists(Path::new("/pics/raw/IMG_1.CR3")), l.exists(Path::new("/pics/raw/IMG_2.CR3"))), (there, ask));
+ }
+}
+
+fn photo(c: &mut Catalog, path: &Path) -> Arc<Photo> {
+ let id = c.alloc_photo_id();
+ let name = path.file_name().unwrap().to_string_lossy().to_string();
+ c.apply(lightcraft_catalog::Op::AddPhoto {
+ photo: Box::new(Photo::new(id, Source::File { path: path.to_string_lossy().to_string() }, &name, "JPEG", 6, 4, "2020-01-01T00:00:00")),
+ })
+ .unwrap();
+ Arc::clone(c.photo(id).unwrap())
+}
+
+/// Given a folder on disk whose photos are there or gone, with sidecars under every name a
+/// sidecar can have, when each photo is checked with the folder's listing, then the answer is
+/// the one the disk gives; and for photos with plain names the disk was asked nothing but the
+/// sidecars' dates.
+#[test]
+fn a_check_from_the_listing_is_the_check_from_the_disk() {
+ let dir = std::env::temp_dir().join(format!("lc-listing-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(dir.join("sub")).unwrap();
+ let news = r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><rdf:Description xmlns:xmp="http://ns.adobe.com/xap/1.0/" xmp:Rating="5"/></rdf:RDF></x:xmpmeta>"#;
+ for name in [
+ "plain.jpg",
+ "stem.jpg",
+ "stem.xmp",
+ "full.jpg",
+ "full.jpg.xmp",
+ "upper.jpg",
+ "upper.XMP",
+ "sub/deep.jpg",
+ "sub/deep.xmp",
+ "Größe.jpg",
+ "Größe.xmp",
+ "dir.jpg.xmp",
+ ] {
+ std::fs::write(dir.join(name), if name.ends_with("mp") || name.ends_with("MP") { news.as_bytes() } else { b"jpeg" }).unwrap();
+ }
+ let mut c = Catalog::new();
+ let photos: Vec<Arc<Photo>> =
+ ["plain.jpg", "stem.jpg", "full.jpg", "upper.jpg", "sub/deep.jpg", "Größe.jpg", "gone.jpg", "sub/gone.jpg", "dir.jpg"]
+ .iter()
+ .map(|n| photo(&mut c, &dir.join(n)))
+ .collect();
+ // (`dir.jpg` is gone, though a sidecar of its name is there)
+ let mut l = Listing::default();
+ let walked =
+ crate::walk::files_in_noting(&dir, None, crate::walk::Limits::default(), |_| true, &mut |d, names, complete| l.note(d, names, complete));
+ assert_eq!(walked.files.len(), 12);
+ let nothing = Listing::default();
+ for naming in [SidecarNaming::Stem, SidecarNaming::Full] {
+ for p in &photos {
+ let from_disk = check_photo(p, naming, &c, &nothing);
+ assert_eq!(check_photo(p, naming, &c, &l), from_disk, "{:?} {naming:?}", p.file_name);
+ }
+ }
+ let expected = [Check::Nothing, Check::News, Check::News, Check::News, Check::News, Check::News, Check::Missing, Check::Missing, Check::Missing];
+ assert_eq!(photos.iter().map(|p| check_photo(p, SidecarNaming::Stem, &c, &l)).collect::<Vec<_>>(), expected);
+
+ // what the disk was asked about the same photos, with the listing and without: fewer questions with it
+ let asked = |l: &Listing| {
+ for p in &photos {
+ check_photo(p, SidecarNaming::Stem, &c, l);
+ }
+ l.asked.load(std::sync::atomic::Ordering::Relaxed)
+ };
+ let mut with = Listing::default();
+ crate::walk::files_in_noting(&dir, None, crate::walk::Limits::default(), |_| true, &mut |d, names, complete| with.note(d, names, complete));
+ let (with, without) = (asked(&with), asked(&Listing::default()));
+ // (without: each photo's file, and the names of its sidecar up to the one that is there: 20 questions where
+ // names are found in any case, 21 where they are not)
+ assert!(without >= 20, "{without}");
+ // (with: this folder holds `Größe.*`, so what it does not list is still asked about: the four names of the
+ // sidecar `plain.jpg` has not, the two `full.jpg` has not, `upper.xmp`, and the two photos that are gone. What
+ // it lists is not asked about, nor anything in `sub`, whose names are all ASCII)
+ assert_eq!(with, 9, "asked {with} times with the listing, {without} without");
+
+ // a folder of plain names only: the disk is asked nothing at all
+ let sub = dir.join("sub");
+ let mut only_plain = Listing::default();
+ crate::walk::files_in_noting(&sub, None, crate::walk::Limits::default(), |_| true, &mut |d, names, complete| only_plain.note(d, names, complete));
+ let mut c2 = Catalog::new();
+ for (name, expected) in [("deep.jpg", Check::News), ("gone.jpg", Check::Missing), ("gone.jpg.xmp", Check::Missing)] {
+ let p = photo(&mut c2, &sub.join(name));
+ assert_eq!(check_photo(&p, SidecarNaming::Stem, &c2, &only_plain), expected, "{name}");
+ }
+ assert_eq!(only_plain.asked.load(std::sync::atomic::Ordering::Relaxed), 0, "every answer came from the listing");
+ let _ = std::fs::remove_dir_all(&dir);
+}
diff --git a/crates/engine/src/walk.rs b/crates/engine/src/walk.rs
index 1c1b53b4a..f8a3ba251 100644
--- a/crates/engine/src/walk.rs
+++ b/crates/engine/src/walk.rs
@@ -47,10 +47,35 @@ fn hidden(p: &Path) -> bool {
p.file_name().is_some_and(|n| n.to_string_lossy().starts_with('.'))
}
+/// What a folder's listing says an entry is.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum EntryKind {
+ File,
+ Folder,
+ /// A link, or an entry whose kind the listing did not say.
+ Other,
+}
+
+/// What [`files_in_noting`] hands over for each folder it lists: the folder, the name and kind
+/// of every entry in it (hidden ones too), and whether that is all of them (not when the walk
+/// stopped inside the folder, or an entry could not be read).
+pub type NoteFolder<'a> = &'a mut dyn FnMut(&Path, Vec<(std::ffi::OsString, EntryKind)>, bool);
+
/// The files below folder `root` that `keep` accepts. Hidden entries (names starting with `.`)
/// are left out, the folder `skip` (e.g. the library's own) is never entered, and the walk stops
/// at `limits` (see [`Walked::truncated`]). Unreadable folders are skipped.
-pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl FnMut(&Path) -> bool) -> Walked {
+pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, keep: impl FnMut(&Path) -> bool) -> Walked {
+ walk(root, skip, limits, keep, None)
+}
+
+/// [`files_in`], handing the listing of every folder it reads to `note`: who walks a folder to
+/// find its files has, with that, the answer to "is this name there" for everything in it, and
+/// need not ask the disk again (a round trip for each name on a network share).
+pub fn files_in_noting(root: &Path, skip: Option<&Path>, limits: Limits, keep: impl FnMut(&Path) -> bool, note: NoteFolder) -> Walked {
+ walk(root, skip, limits, keep, Some(note))
+}
+
+fn walk(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl FnMut(&Path) -> bool, mut note: Option<NoteFolder>) -> Walked {
let mut out = Walked::default();
// canonical folders reached through a link, so a link loop is walked at most once
let mut linked: HashSet<PathBuf> = HashSet::new();
@@ -77,17 +102,41 @@ pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl
}
let Ok(rd) = std::fs::read_dir(&path) else { continue };
let mut children: Vec<(PathBuf, bool)> = Vec::new();
- for e in rd.flatten() {
+ // (the folder's listing, for who asked for it; all of it unless something below says otherwise)
+ let mut listed = note.is_some().then(Vec::new);
+ let mut complete = true;
+ for e in rd {
+ let Ok(e) = e else {
+ complete = false;
+ continue;
+ };
if out.entries >= limits.max_entries {
out.truncated = true;
+ complete = false;
break;
}
out.entries = out.entries.saturating_add(1);
let p = e.path();
- if hidden(&p) {
+ // (a hidden entry's kind is only asked for when the listing is wanted; every other
+ // entry's is asked once)
+ let is_hidden = hidden(&p);
+ if is_hidden && listed.is_none() {
continue;
}
- let Ok(ft) = e.file_type() else { continue };
+ let ft = e.file_type();
+ if let Some(listed) = listed.as_mut() {
+ let kind = match &ft {
+ Ok(t) if t.is_symlink() => EntryKind::Other,
+ Ok(t) if t.is_dir() => EntryKind::Folder,
+ Ok(t) if t.is_file() => EntryKind::File,
+ _ => EntryKind::Other,
+ };
+ listed.push((e.file_name(), kind));
+ }
+ if is_hidden {
+ continue;
+ }
+ let Ok(ft) = ft else { continue };
let dir = if ft.is_symlink() {
// a link (or a Windows junction): a folder is entered once, a file is a file
match std::fs::metadata(&p) {
@@ -105,6 +154,9 @@ pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl
};
children.push((p, dir));
}
+ if let (Some(note), Some(listed)) = (note.as_mut(), listed) {
+ note(&path, listed, complete);
+ }
children.sort();
let below = depth.saturating_add(1);
stack.extend(children.into_iter().rev().map(|(p, d)| (p, below, d)));
@@ -123,6 +175,43 @@ mod tests {
d
}
+ /// Given a folder with files, a folder inside it, a hidden file and a link, when it is walked
+ /// with its listings noted, then the walk finds what it finds without them, and each folder's
+ /// listing holds every entry by name with its kind (hidden ones too, a link as neither file nor
+ /// folder) and says it is all of them; a walk that stops inside a folder says it is not.
+ #[test]
+ fn a_walk_notes_each_folders_listing() {
+ let root = tmp("noting");
+ std::fs::create_dir_all(root.join("sub")).unwrap();
+ for name in ["a.jpg", ".hidden", "sub/b.jpg"] {
+ std::fs::write(root.join(name), b"x").unwrap();
+ }
+ #[cfg(unix)]
+ std::os::unix::fs::symlink(root.join("a.jpg"), root.join("link.jpg")).unwrap();
+ let mut noted: Vec<(PathBuf, Vec<(String, EntryKind)>, bool)> = Vec::new();
+ let walked = files_in_noting(&root, None, Limits::default(), |_| true, &mut |dir, entries, complete| {
+ let mut entries: Vec<(String, EntryKind)> = entries.into_iter().map(|(n, k)| (n.to_string_lossy().to_string(), k)).collect();
+ entries.sort_by(|a, b| a.0.cmp(&b.0));
+ noted.push((dir.to_path_buf(), entries, complete));
+ });
+ assert_eq!(walked, files_in(&root, None, Limits::default(), |_| true), "the same walk, noted or not");
+ let named = |n: &str, k: EntryKind| (n.to_string(), k);
+ let mut top = vec![named(".hidden", EntryKind::File), named("a.jpg", EntryKind::File), named("sub", EntryKind::Folder)];
+ if cfg!(unix) {
+ top.insert(2, named("link.jpg", EntryKind::Other));
+ }
+ assert_eq!(noted, [(root.clone(), top, true), (root.join("sub"), vec![named("b.jpg", EntryKind::File)], true)]);
+
+ // stopped inside the first folder: its listing is not all of it
+ let mut cut = Vec::new();
+ let walked = files_in_noting(&root, None, Limits { max_depth: 8, max_entries: 2 }, |_| true, &mut |dir, entries, complete| {
+ cut.push((dir.to_path_buf(), entries.len(), complete))
+ });
+ assert!(walked.truncated);
+ assert_eq!(cut.first().map(|(dir, n, complete)| (dir.clone(), *n, *complete)), Some((root.clone(), 2, false)));
+ let _ = std::fs::remove_dir_all(&root);
+ }
+
#[test]
fn files_come_in_path_order_without_hidden_ones_or_the_skipped_folder() {
let d = tmp("order");
diff --git a/docs/whats-new.md b/docs/whats-new.md
index 1bf44c3f4..591513d84 100644
--- a/docs/whats-new.md
+++ b/docs/whats-new.md
@@ -53,6 +53,23 @@
else (issue #1504): trimming or clearing the cache can't delete files outside it, and thumbnails are never read
from or written through such a link.
+### Synchronize Folder asks a network share less
+- A Synchronize Folder scan no longer asks the disk about every photo of the library in the folder (is its file
+ there, has it a sidecar): it has just listed the folder, and reads the answers off the listing. On a network share
+ each of those questions is a round trip. Measured on a hard disk over SMB, for folders of 1,600 to 2,400 photos
+ that had not changed: 4.6 to 7.1 s of questions after the listing (about 3 ms a photo), now under 0.05 s. The
+ listing itself took 4 to 5 s and is as it was.
+- The disk is still asked wherever a listing can't be sure what a file system would say: about a name that is
+ there in another case, a link, a name Windows reads another way (`aux.jpg`, a `~`, a `:`, a dot or a space at the
+ end), a folder the scan did not walk to its end, and, in a folder that holds any name with a character outside
+ ASCII (an umlaut, a curly apostrophe, a dash), about every name the folder does not list. Names it does list are
+ answered there too. In such a folder the scan saves less, and its checks now start after the listing where they
+ ran beside it.
+- A photo the folder lists is there for the scan, also where its file could not be opened by its path (a folder
+ that may be listed but not searched, a path longer than the system takes): such photos used to be listed as
+ missing. And a share that drops after the scan has listed a folder no longer turns that folder's photos into
+ missing ones.
+
### Large libraries: Synchronize Folder, import and sidecars start sooner
- Opening Synchronize Folder, an import review or an import no longer makes a table of every photo in the library
before it looks at the first file. In a library of 3.5 million photos that held the window for about 6 s before