π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Fix legacy Live Photo adoption and export initialization retry
Manav Rathi04751c645fcb
diff --git a/rust/apps/cli-next/src/args.rs b/rust/apps/cli-next/src/args.rsindex 699bac3f85..3bd5d99450 100644--- a/rust/apps/cli-next/src/args.rs+++ b/rust/apps/cli-next/src/args.rs@@ -515,7 +515,8 @@ Fields: id, name, type, ownerId (strings); albumIds (string array);createdAt, updatedAt (RFC3339 timestamp strings).Nullable fields: modifiedAt (RFC3339 string); location ({latitude, longitude}, numbers);caption, hash, dateTime, offsetTime, visibility (strings); durationSeconds, width, height (integers).-dateTime and offsetTime retain the stored metadata strings.";+dateTime and offsetTime retain the stored metadata strings.+hash comes from Ente's stored metadata. For Live Photos, it can contain the image and video hashes separated by a colon.";#[derive(Subcommand)]pub enum FileCommand {diff --git a/rust/apps/cli-next/tests/cli/export_adoption.rs b/rust/apps/cli-next/tests/cli/export_adoption.rsindex fcfe06657e..2a4c8bc8e1 100644--- a/rust/apps/cli-next/tests/cli/export_adoption.rs+++ b/rust/apps/cli-next/tests/cli/export_adoption.rs@@ -191,15 +191,42 @@ fn snapshot(root: &Path) -> BTreeMap<PathBuf, (Vec<u8>, std::time::SystemTime)>#[test]fn adoption_reuses_captured_producer_shapes_and_converts_current_names() {- for producer in ["go", "desktop"] {+ for producer in ["go", "desktop", "rust"] {let mut server = mockito::Server::new();- let (mut engine, downloads) = new_engine(&server, producer);+ let (mut engine, downloads) =+ new_engine(&server, if producer == "rust" { "go" } else { producer });engine.hardlinks = true;let requests: Vec<_> = downloads.iter().map(|(id, bytes)| download(&mut server, *id, bytes, 0)).collect();let root = engine.root();+ let pair = if producer == "rust" {+ "Pair.heic"+ } else {+ "Pair.jpg"+ };+ if producer == "rust" {+ for folder in ["Family", "Live Pair"] {+ nominal_live_record(&root, folder);+ let folder = root.join(folder);+ fs::rename(folder.join("Pair.jpg"), folder.join(pair)).unwrap();+ fs::rename(folder.join("Pair.mov"), folder.join("Pair.MOV")).unwrap();+ }+ let folder = root.join("Family");+ fs::write(folder.join("Pair.xmp"), b"unrelated metadata").unwrap();+ fs::copy(folder.join(pair), folder.join("Pair.png")).unwrap();+ let before = snapshot(&root);+ let error = engine.prepare(engine.options(true, &[])).err().unwrap();+ assert!(+ error+ .to_string()+ .contains("ambiguous legacy Live Photo component"),+ "{error:#}"+ );+ assert_eq!(snapshot(&root), before);+ fs::remove_file(folder.join("Pair.png")).unwrap();+ }if producer == "desktop" {fs::rename(root.join("Family"), root.join("Family(1)")).unwrap();fs::rename(@@ -266,11 +293,23 @@ fn adoption_reuses_captured_producer_shapes_and_converts_current_names() {assert!(root.join("Renamed/Portrait.jpg").is_file());assert!(same_file::is_same_file(- root.join("Renamed/Pair.jpg"),- root.join("Live Pair/Pair.jpg")+ root.join("Renamed").join(pair),+ root.join("Live Pair").join(pair)).unwrap());+ if producer == "rust" {+ assert_eq!(+ fs::read(root.join("Renamed/Pair.xmp")).unwrap(),+ b"unrelated metadata"+ );+ assert_component(+ &root,+ "Renamed",+ "Pair.MOV",+ format!("export fixture {PAIR} component 1").as_bytes(),+ );+ }assert!(root.join("Renamed/Live-1.jpg").is_file());assert!(root.join("Renamed/Live.mov").is_file());assert_eq!(@@ -575,7 +614,10 @@ fn adoption_requires_initial_presence_and_never_treats_a_failed_refresh_as_histo#[test]fn adoption_hashless_verification_downloads_once_per_source_and_keeps_scope_per_invocation() {let mut server = mockito::Server::new();- let (mut engine, downloads) = new_engine(&server, "desktop");+ let (mut engine, downloads) = new_engine(&server, "go");+ for folder in ["Family", "Live Pair"] {+ nominal_live_record(&engine.root(), folder);+ }for ((_, id), remote) in &mut engine.catalog.files {if ![PHOTO, PAIR].contains(id) {continue;@@ -990,22 +1032,69 @@ fn adoption_defers_wholly_missing_hashless_originals_until_after_acceptance() {requested.assert();}+fn nominal_live_record(root: &Path, folder: &str) {+ let path = root.join(folder).join(".meta/Pair.jpg.json");+ let mut record = read_json(&path);+ record["info"]["fileNames"] = json!(["Pair.jpg"]);+ fs::write(&path, serde_json::to_vec(&record).unwrap()).unwrap();+ fs::rename(path, root.join(folder).join(".meta/Pair_0.jpg.json")).unwrap();+}++#[test]+fn adoption_leaves_unmatched_old_rust_live_files_for_ordinary_conflict_handling() {+ for renamed in [false, true] {+ let server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, "go");+ let root = engine.root();+ nominal_live_record(&root, "Family");+ fs::write(root.join("Family/Pair.mov"), b"edited video").unwrap();+ if renamed {+ rename(&mut engine, FAMILY, PAIR, "Holiday.jpg");+ }+ let export = engine.prepare(engine.options(true, &[])).unwrap();+ assert_eq!(+ fs::read(root.join("Family/Pair.mov")).unwrap(),+ b"edited video"+ );+ assert_eq!(+ fs::read(root.join("Trash/Family/Pair.jpg")).unwrap(),+ format!("export fixture {PAIR} component 0").as_bytes()+ );+ let result = engine.execute(export).unwrap();+ assert_eq!(result.complete, renamed, "{}", result.json);+ assert_eq!(+ fs::read(root.join("Family/Pair.mov")).unwrap(),+ b"edited video"+ );+ if renamed {+ assert_component(+ &root,+ "Family",+ "Holiday.mov",+ format!("export fixture {PAIR} component 1").as_bytes(),+ );+ } else {+ assert_eq!(result.json["conflicts"], 1);+ fs::remove_file(root.join("Family/Pair.mov")).unwrap();+ assert!(engine.run(false, &[]).unwrap().complete);+ assert_component(+ &root,+ "Family",+ "Pair.mov",+ format!("export fixture {PAIR} component 1").as_bytes(),+ );+ }+ }+}+#[test]-fn adoption_verifies_a_single_undeclared_live_survivor_before_preserving_it() {+fn adoption_preserves_a_single_hash_identified_live_survivor() {let mut server = mockito::Server::new();let (mut engine, downloads) = new_engine(&server, "go");let root = engine.root();- let path = root.join("Family/.meta/Pair.jpg.json");- let mut record = read_json(&path);- record["info"]["fileNames"] = json!(["Pair.jpg"]);- fs::write(path, serde_json::to_vec(&record).unwrap()).unwrap();+ nominal_live_record(&root, "Family");fs::remove_file(root.join("Family/Pair.mov")).unwrap();let original = fs::read(root.join("Family/Pair.jpg")).unwrap();- fs::write(root.join("Family/Pair.jpg"), b"edited survivor").unwrap();- let before = snapshot(&root);- assert!(engine.run(true, &[]).is_err());- assert_eq!(snapshot(&root), before);- fs::write(root.join("Family/Pair.jpg"), &original).unwrap();for (path, _) in snapshot(&root) {if !path.to_string_lossy().ends_with(".json")&& path.as_path() != Path::new("Family/Pair.jpg")diff --git a/rust/crates/photos-export/src/legacy/read.rs b/rust/crates/photos-export/src/legacy/read.rsindex cf38daead4..387a24cad4 100644--- a/rust/crates/photos-export/src/legacy/read.rs+++ b/rust/crates/photos-export/src/legacy/read.rs@@ -53,6 +53,7 @@ pub struct Record {pub name: String,pub hash: Option<String>,pub parts: Vec<Part>,+ pub candidates: Vec<String>,pub go_record: Option<String>,pub desktop: bool,}@@ -153,6 +154,7 @@ pub fn scan(name: paths[0].name.clone(),hash: None,parts: paths,+ candidates: Vec::new(),go_record: None,desktop: true,};@@ -221,6 +223,7 @@ pub fn scan(Some(record.owner_id),true,)?;+ let candidates = same_stem_files(&root.join(&path))?;for entry in fs::read_dir(identity.parent().context("missing metadata parent")?)? {let entry = entry?;if entry.file_name() == "album_meta.json"@@ -244,6 +247,14 @@ pub fn scan(.as_str().context("missing legacy file title")?.to_owned();+ let siblings = if info.file_names.len() == 1 {+ candidates+ .get(&names::folded(names::split(&info.file_names[0], false).0))+ .cloned()+ .unwrap_or_default()+ } else {+ Vec::new()+ };let record = Record {album: record.id,file: info.id,@@ -255,6 +266,7 @@ pub fn scan(.into_iter().map(|name| Part { name, role: None }).collect(),+ candidates: siblings,go_record: Some(format!(".meta/{name}")),desktop: false,};@@ -269,6 +281,22 @@ pub fn scan(Ok(())}+fn same_stem_files(directory: &Path) -> Result<BTreeMap<String, Vec<String>>> {+ let mut files: BTreeMap<String, Vec<String>> = BTreeMap::new();+ for entry in fs::read_dir(directory)? {+ let entry = entry?;+ if entry.file_type()?.is_file()+ && let Ok(name) = entry.file_name().into_string()+ {+ files+ .entry(names::folded(names::split(&name, false).0))+ .or_default()+ .push(name);+ }+ }+ Ok(files)+}+fn album(root: &Path,index: &Index,diff --git a/rust/crates/photos-export/src/legacy/verify.rs b/rust/crates/photos-export/src/legacy/verify.rsindex 918d059fae..0f5819adb4 100644--- a/rust/crates/photos-export/src/legacy/verify.rs+++ b/rust/crates/photos-export/src/legacy/verify.rs@@ -23,7 +23,7 @@ pub async fn file(source: &FileSnapshot,expected: &mut Option<Result<Vec<(Role, String)>, String>>,) -> Result<usize> {- let record = item.legacy.as_ref().context("missing legacy record")?;+ let record = item.legacy.as_mut().context("missing legacy record")?;let roles = transfer::roles(source);ensure!(source.kind == "livephoto" || record.parts.len() == 1,@@ -42,6 +42,13 @@ pub async fn file(|| record.parts[0].role != record.parts[1].role,"duplicate legacy component role");+ let discover = !record.desktop && source.kind == "livephoto" && record.parts.len() == 1;+ if discover {+ record.parts = std::mem::take(&mut record.candidates)+ .into_iter()+ .map(|name| super::read::Part { name, role: None })+ .collect();+ }let mut measured = Vec::new();for part in &record.parts {let path = names::check(check.root, &format!("{}/{}", album.path, part.name))?;@@ -79,6 +86,35 @@ pub async fn file(.filter_map(|r| transfer::expected_hash(source, r).map(|h| (r.clone(), h.to_owned()))).collect()};+ if discover {+ let parts = std::mem::take(&mut record.parts);+ let mut matched = Vec::new();+ for (mut part, measurement) in parts.into_iter().zip(measured) {+ let Some((_, hash, _)) = &measurement else {+ continue;+ };+ let mut matches = known.iter().filter(|(_, expected)| expected == hash);+ let Some((role, _)) = matches.next() else {+ continue;+ };+ ensure!(+ matches.next().is_none()+ && !record+ .parts+ .iter()+ .any(|part| part.role.as_ref() == Some(role)),+ "ambiguous legacy Live Photo component at {}/{}",+ album.path,+ part.name+ );+ adopt::claim(check.index, album, &item.placement.id, &part.name, "")?;+ part.role = Some(role.clone());+ record.parts.push(part);+ matched.push(measurement);+ }+ measured = matched;+ }+ let present = measured.iter().filter(|p| p.is_some()).count();let complete = present == roles.len();item.leftover = !complete && (source.kind == "livephoto" || record.desktop);item.placement.kind = source.kind.clone();@@ -140,6 +176,8 @@ pub async fn file(}}if complete {+ item.components+ .sort_by_key(|component| component.role.name());let layout: Vec<_> = item.components.iter().map(Component::portable).collect();let favorited = check.store.db.query_row("SELECT favorited FROM desired_files WHERE album=?1 AND file=?2",diff --git a/rust/crates/photos-export/src/lib.rs b/rust/crates/photos-export/src/lib.rsindex 63ba875557..316bfe8143 100644--- a/rust/crates/photos-export/src/lib.rs+++ b/rust/crates/photos-export/src/lib.rs@@ -424,17 +424,25 @@ fn initialize(destination: &Path, root: &Root, legacy: bool) -> Result<RootLock>.create_new(true).open(&path).context("destination was initialized by another writer; retry")?;- file.lock().context("cannot lock the new export")?;- let mut lock = RootLock(file);- ensure!(- legacy- || std::fs::read_dir(destination)?- .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),- "destination became nonempty during initialization"- );- serde_json::to_writer_pretty(&mut lock.0, root)?;- lock.0.write_all(b"\n")?;- Ok(lock)+ let result = (|| {+ file.lock().context("cannot lock the new export")?;+ let mut lock = RootLock(file);+ ensure!(+ legacy+ || std::fs::read_dir(destination)?+ .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),+ "destination became nonempty during initialization"+ );+ serde_json::to_writer_pretty(&mut lock.0, root)?;+ lock.0.write_all(b"\n")?;+ Ok(lock)+ })();+ if result.is_err()+ && let Err(error) = std::fs::remove_file(&path)+ {+ return result.with_context(|| format!("cannot remove incomplete export.json: {error}"));+ }+ result}fn select(@@ -912,6 +920,21 @@ fn canonical_id(selector: &str) -> rusqlite::types::Value {mod tests {use super::*;+ #[test]+ fn initialization_removes_its_marker_on_failure_and_retries() {+ let root = Root::new(&ente_core::crypto::Key::generate()).unwrap();+ let directory = tempfile::tempdir().unwrap();+ let stray = directory.path().join("photo.jpg");+ std::fs::write(&stray, b"unrelated").unwrap();+ let error = initialize(directory.path(), &root, false).err().unwrap();+ assert!(error.to_string().contains("destination became nonempty"));+ assert!(!directory.path().join("export.json").exists());+ assert_eq!(std::fs::read(&stray).unwrap(), b"unrelated");+ std::fs::remove_file(stray).unwrap();+ drop(initialize(directory.path(), &root, false).unwrap());+ assert!(open_root(directory.path(), &root).unwrap().is_some());+ }+#[test]fn transparent_errors_preserve_fatal_classification() {for status in [401, 404] {