LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
LRC-LIB-LIBFOLDERS: catalog queries and streamed file hashing for locating a folder (#1112)
Nathaniel Ledforda9be2e14f31b
diff --git a/crates/catalog/src/folders.rs b/crates/catalog/src/folders.rsindex aab3b867b..e8637ceca 100644--- a/crates/catalog/src/folders.rs+++ b/crates/catalog/src/folders.rs@@ -207,6 +207,18 @@ impl Catalog {/// folder that is no longer there and are dropped (undo restores them). Empty when there is/// nothing to carry or drop.pub fn folder_records_follow(&self, from: &str, to: &str) -> Vec<Op> {+ self.folder_records_carried(from, to, true)+ }++ /// [`Catalog::folder_records_follow`] for a folder located where it already is on disk+ /// (`folder.updateLocation`): nothing arrived at `to`, the library only learned that the+ /// folder is there. So what it knows about folders at or below `to` (other folders of the+ /// library there) stays; a record carried over replaces the one at the same place.+ pub fn folder_records_located(&self, from: &str, to: &str) -> Vec<Op> {+ self.folder_records_carried(from, to, false)+ }++ fn folder_records_carried(&self, from: &str, to: &str, clear_destination: bool) -> Vec<Op> {let (Some(root), Some(dest)) = (record_key(from), record_key(to)) else { return Vec::new() };if root == dest {return Vec::new();@@ -224,7 +236,7 @@ impl Catalog {.collect();// what was known at the old place goes, and so does what is left at the new one (also// when the moved folder carries nothing: the folder now there is not the one it was about)- let stale = self.folder_records.keys().filter(|k| key_within(k, &root) || key_within(k, &dest));+ let stale = self.folder_records.keys().filter(|k| key_within(k, &root) || (clear_destination && key_within(k, &dest)));let mut ops: Vec<Op> = stale.map(|k| Op::SetFolderRecord { folder: k.clone(), record: None }).collect();ops.extend(moved.into_iter().map(|(k, r)| Op::SetFolderRecord { folder: k, record: Some(r) }));opsdiff --git a/crates/catalog/src/lib.rs b/crates/catalog/src/lib.rsindex 1776c895c..09bba131f 100644--- a/crates/catalog/src/lib.rs+++ b/crates/catalog/src/lib.rs@@ -36,7 +36,7 @@ pub use folders::{FolderNode, FolderRecord};pub use journal::{Journal, LoadReport, PersistStats, SnapshotPolicy, SnapshotTiming};pub use keywords::KeywordNode;use lightcraft_develop::DevelopSettings;-pub use local::{DEFAULT_FORGET_DAYS, ForgetPlan, folder_of};+pub use local::{DEFAULT_FORGET_DAYS, ForgetPlan, LocalRecords, folder_of};pub use lock::{LibraryLock, LockError, LockOwner};pub use model::*;pub use query::{DateGroup, Filter, Person, RatingOp, Sort, SortKey, mix64};diff --git a/crates/catalog/src/local.rs b/crates/catalog/src/local.rsindex 093c22427..70d8ea44a 100644--- a/crates/catalog/src/local.rs+++ b/crates/catalog/src/local.rs@@ -34,6 +34,15 @@ use serde::Serialize;use crate::{Catalog, Op, Photo, PhotoId, Source};+/// The Local records of some files ([`Catalog::local_records_of`]).+#[derive(Clone, Debug, Default, PartialEq, Eq)]+pub struct LocalRecords {+ /// Only a record of having looked: nothing was changed on them and nothing refers to them.+ pub untouched: Vec<PhotoId>,+ /// Holding something of the person's, with their files.+ pub kept: Vec<(PhotoId, String)>,+}+/// Default for how long a Local folder may go unbrowsed before its untouched records are/// forgotten (days).pub const DEFAULT_FORGET_DAYS: u32 = 30;@@ -176,6 +185,34 @@ impl Catalog {&self.browsed}+ /// The Local records of `files` (paths as stored), for library photos about to point at+ /// those files (a folder located where it was browsed, `folder.updateLocation`): a file has one+ /// record, so the Local one gives way. Told apart: those that are only a record of having+ /// looked (untouched, see the module docs, and nothing refers to them), which can go, and+ /// those that hold something of the person's (a rating, an edit, a place in an album), with+ /// their files. Records in Recently Deleted are not Local records to this.+ pub fn local_records_of(&self, files: &HashSet<String>) -> LocalRecords {+ let mut out = LocalRecords::default();+ if files.is_empty() {+ return out;+ }+ let referenced = self.referenced();+ for p in self.photos.values().filter(|p| p.local && !p.deleted) {+ let Source::File { path } = &p.source else { continue };+ if !files.contains(path) {+ continue;+ }+ if p.untouched_local() && !referenced.contains(&p.id) {+ out.untouched.push(p.id);+ } else {+ out.kept.push((p.id, path.clone()));+ }+ }+ out.untouched.sort_unstable();+ out.kept.sort_unstable();+ out+ }+/// Plan forgetting untouched Local records whose folder wasn't browsed for `days` days (as of/// `now`, ISO 8601). `in_use` keeps records the caller still needs. `days == 0` (never) or an/// unreadable `now` forgets nothing. See the module docs.diff --git a/crates/catalog/src/tests_folder_records.rs b/crates/catalog/src/tests_folder_records.rsindex 3ae19c33a..eb2215ba6 100644--- a/crates/catalog/src/tests_folder_records.rs+++ b/crates/catalog/src/tests_folder_records.rs@@ -161,6 +161,44 @@ fn an_unlabelled_folder_moved_onto_a_stale_label_does_not_inherit_it() {assert_eq!(c.folder_color_label("/pics/holiday"), Some(ColorLabel::Purple));}+#[test]+fn a_located_folder_keeps_what_the_library_knows_of_the_folders_already_at_the_destination() {+ let mut c = library();+ label(&mut c, "/pics/trip", Some(ColorLabel::Red));+ label(&mut c, "/pics/trip/day1", Some(ColorLabel::Green));+ // other folders of the library at the place it is said to be at: nothing arrived there+ label(&mut c, "/pics/home", Some(ColorLabel::Blue));+ label(&mut c, "/pics/home/old", Some(ColorLabel::Yellow));+ let before = c.to_snapshot();+ let ops = c.folder_records_located("/pics/trip", "/pics/home");+ let undo = c.apply(Op::Batch { ops }).unwrap();+ assert_eq!(c.folder_record("/pics/trip"), None, "what was known at the old place goes");+ assert_eq!(c.folder_record("/pics/trip/day1"), None);+ assert_eq!(c.folder_color_label("/pics/home"), Some(ColorLabel::Red), "a record carried over replaces the one at the same place");+ assert_eq!(c.folder_color_label("/pics/home/old"), Some(ColorLabel::Yellow), "a folder of the library there keeps its own");+ c.apply(undo).unwrap();+ assert_eq!(c.to_snapshot(), before);+}++#[test]+fn a_located_folder_with_labels_inside_lands_them_at_the_same_places_below() {+ let mut c = library();+ label(&mut c, "/pics/trip", Some(ColorLabel::Red));+ label(&mut c, "/pics/trip/day1", Some(ColorLabel::Green));+ let ops = c.folder_records_located("/pics/trip", "/mnt/nas/trip");+ c.apply(Op::Batch { ops }).unwrap();+ assert_eq!(c.folder_color_label("/mnt/nas/trip"), Some(ColorLabel::Red));+ assert_eq!(c.folder_color_label("/mnt/nas/trip/day1"), Some(ColorLabel::Green));+}++#[test]+fn a_folder_located_where_it_is_or_without_labels_gives_no_ops() {+ let mut c = library();+ assert!(c.folder_records_located("/pics/trip", "/elsewhere/trip").is_empty());+ label(&mut c, "/pics/trip", Some(ColorLabel::Red));+ assert!(c.folder_records_located("/pics/trip", "/pics/trip").is_empty());+}+#[test]fn labels_survive_the_journal_without_a_snapshot() {let m = MemStore::new();diff --git a/crates/catalog/src/tests_local.rs b/crates/catalog/src/tests_local.rsindex 4b76dace6..f89d326d0 100644--- a/crates/catalog/src/tests_local.rs+++ b/crates/catalog/src/tests_local.rs@@ -1,5 +1,6 @@//! Forgetting untouched Local browse records (see [`crate::local`]).+use std::collections::HashSet;use std::sync::Arc;use lightcraft_develop::DevelopSettings;@@ -265,3 +266,37 @@ fn op_photo_ids_cover_nested_ops() {op.photo_ids(&mut |id| ids.push(id.0));assert_eq!(ids, vec![1, 2, 3, 4]);}++fn paths(folder: &str, ns: &[u32]) -> HashSet<String> {+ ns.iter().map(|i| format!("/pics/{folder}/IMG_{i}.JPG")).collect()+}++#[test]+fn local_records_of_files_tell_what_can_give_way_from_what_holds_something() {+ let mut c = Catalog::new();+ let untouched = local(&mut c, "trip", 1);+ let rated = local(&mut c, "trip", 2);+ let in_album = local(&mut c, "trip", 3);+ let _elsewhere = local(&mut c, "trip", 4);+ c.apply(Op::SetRating { id: rated, rating: 3 }).unwrap();+ let a = c.alloc_album_id();+ let mut album = Album::new(a, "A");+ album.photos.push(in_album);+ c.apply(Op::AddAlbum { album }).unwrap();++ let found = c.local_records_of(&paths("trip", &[1, 2, 3]));+ assert_eq!(found.untouched, vec![untouched]);+ assert_eq!(found.kept, vec![(rated, "/pics/trip/IMG_2.JPG".to_string()), (in_album, "/pics/trip/IMG_3.JPG".to_string())]);+}++#[test]+fn local_records_of_files_leave_out_library_photos_and_recently_deleted_ones() {+ let mut c = Catalog::new();+ let gone = local(&mut c, "trip", 1);+ c.apply(Op::SetDeleted { id: gone, deleted: true }).unwrap();+ let id = c.alloc_photo_id();+ let p = Photo::new(id, Source::File { path: "/pics/trip/IMG_2.JPG".into() }, "IMG_2.JPG", "JPEG", 60, 40, OLD);+ c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();+ assert_eq!(c.local_records_of(&paths("trip", &[1, 2])), LocalRecords::default());+ assert_eq!(c.local_records_of(&HashSet::new()), LocalRecords::default());+}diff --git a/crates/engine/src/cmd/missing.rs b/crates/engine/src/cmd/missing.rsindex 1bff30d2b..29decc4f3 100644--- a/crates/engine/src/cmd/missing.rs+++ b/crates/engine/src/cmd/missing.rs@@ -90,9 +90,77 @@ fn stored_hash(p: &Photo) -> Option<&str> {p.content_hash.as_deref().map(|h| h.split(':').next().unwrap_or(h)).filter(|h| !h.is_empty())}-/// The content hash of the file at `path` (as import computes it), if it can be read.+/// The content hash of the file at `path` (as import computes it), if it can be read. Read in+/// pieces: the file is whatever is on the disk, of any size.fn file_hash(path: &str) -> Option<String> {- std::fs::read(path).ok().map(|b| lightcraft_preview::hash_bytes(&b).to_string())+ hash_reader(std::fs::File::open(path).ok()?)+}++/// The content hash of what `r` gives, read in pieces (the hash import computes of the whole).+fn hash_reader(mut r: impl std::io::Read) -> Option<String> {+ let mut hasher = lightcraft_preview::Hasher128::new();+ let mut piece = vec![0u8; 1 << 16];+ loop {+ match r.read(&mut piece) {+ Ok(0) => return Some(hasher.finish().to_string()),+ Ok(n) => {+ hasher.update(piece.get(..n)?);+ }+ Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}+ Err(_) => return None,+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ /// A reader that gives its bytes a few at a time, and an interruption in between.+ struct Trickle<'a>(&'a [u8], bool);++ impl std::io::Read for Trickle<'_> {+ fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {+ self.1 = !self.1;+ if self.1 {+ return Err(std::io::ErrorKind::Interrupted.into());+ }+ let n = self.0.len().min(buf.len()).min(7);+ buf[..n].copy_from_slice(&self.0[..n]);+ self.0 = &self.0[n..];+ Ok(n)+ }+ }++ #[test]+ fn a_hash_read_in_pieces_is_the_hash_of_the_whole() {+ // longer than one piece, so the read loop runs more than once+ let bytes: Vec<u8> = (0..200_000u32).map(|i| (i % 251) as u8).collect();+ let whole = lightcraft_preview::hash_bytes(&bytes).to_string();+ assert_eq!(hash_reader(bytes.as_slice()).as_deref(), Some(whole.as_str()));+ assert_eq!(hash_reader(Trickle(&bytes[..1000], false)).as_deref(), Some(lightcraft_preview::hash_bytes(&bytes[..1000]).to_string().as_str()));+ assert_eq!(hash_reader(std::io::empty()).as_deref(), Some(lightcraft_preview::hash_bytes(&[]).to_string().as_str()));+ }++ /// A scratch folder that goes away with the test, however it ends.+ struct Scratch(std::path::PathBuf);++ impl Drop for Scratch {+ fn drop(&mut self) {+ let _ = std::fs::remove_dir_all(&self.0);+ }+ }++ #[test]+ fn a_file_hash_is_the_import_hash_and_a_missing_file_has_none() {+ let dir = Scratch(std::env::temp_dir().join(format!("lc-file-hash-{}", std::process::id())));+ let _ = std::fs::remove_dir_all(&dir.0);+ std::fs::create_dir_all(&dir.0).unwrap();+ let f = dir.0.join("a.bin");+ std::fs::write(&f, b"pixels").unwrap();+ assert_eq!(file_hash(f.to_str().unwrap()), Some(lightcraft_preview::hash_bytes(b"pixels").to_string()));+ assert_eq!(file_hash(dir.0.join("nope").to_str().unwrap()), None);+ }}/// An in-scope photo ([`checked_path`]) as Find Missing Photos sees it: its file, the size and