Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Fix legacy Live Photo adoption and export initialization retry

Manav Rathi04751c645fcb
diff --git a/rust/apps/cli-next/src/args.rs b/rust/apps/cli-next/src/args.rs
index 699bac3f85..3bd5d99450 100644
--- a/rust/apps/cli-next/src/args.rs
+++ b/rust/apps/cli-next/src/args.rs
@@ -515,7 +515,8 @@ Fields: id, name, type, ownerId (strings); albumIds (string array);
createdAt, updatedAt (RFC3339 timestamp strings).
Nullable fields: modifiedAt (RFC3339 string); location ({latitude, longitude}, numbers);
caption, hash, dateTime, offsetTime, visibility (strings); durationSeconds, width, height (integers).
-dateTime and offsetTime retain the stored metadata strings.";
+dateTime and offsetTime retain the stored metadata strings.
+hash comes from Ente's stored metadata. For Live Photos, it can contain the image and video hashes separated by a colon.";
#[derive(Subcommand)]
pub enum FileCommand {
diff --git a/rust/apps/cli-next/tests/cli/export_adoption.rs b/rust/apps/cli-next/tests/cli/export_adoption.rs
index fcfe06657e..2a4c8bc8e1 100644
--- a/rust/apps/cli-next/tests/cli/export_adoption.rs
+++ b/rust/apps/cli-next/tests/cli/export_adoption.rs
@@ -191,15 +191,42 @@ fn snapshot(root: &Path) -> BTreeMap<PathBuf, (Vec<u8>, std::time::SystemTime)>
#[test]
fn adoption_reuses_captured_producer_shapes_and_converts_current_names() {
- for producer in ["go", "desktop"] {
+ for producer in ["go", "desktop", "rust"] {
let mut server = mockito::Server::new();
- let (mut engine, downloads) = new_engine(&server, producer);
+ let (mut engine, downloads) =
+ new_engine(&server, if producer == "rust" { "go" } else { producer });
engine.hardlinks = true;
let requests: Vec<_> = downloads
.iter()
.map(|(id, bytes)| download(&mut server, *id, bytes, 0))
.collect();
let root = engine.root();
+ let pair = if producer == "rust" {
+ "Pair.heic"
+ } else {
+ "Pair.jpg"
+ };
+ if producer == "rust" {
+ for folder in ["Family", "Live Pair"] {
+ nominal_live_record(&root, folder);
+ let folder = root.join(folder);
+ fs::rename(folder.join("Pair.jpg"), folder.join(pair)).unwrap();
+ fs::rename(folder.join("Pair.mov"), folder.join("Pair.MOV")).unwrap();
+ }
+ let folder = root.join("Family");
+ fs::write(folder.join("Pair.xmp"), b"unrelated metadata").unwrap();
+ fs::copy(folder.join(pair), folder.join("Pair.png")).unwrap();
+ let before = snapshot(&root);
+ let error = engine.prepare(engine.options(true, &[])).err().unwrap();
+ assert!(
+ error
+ .to_string()
+ .contains("ambiguous legacy Live Photo component"),
+ "{error:#}"
+ );
+ assert_eq!(snapshot(&root), before);
+ fs::remove_file(folder.join("Pair.png")).unwrap();
+ }
if producer == "desktop" {
fs::rename(root.join("Family"), root.join("Family(1)")).unwrap();
fs::rename(
@@ -266,11 +293,23 @@ fn adoption_reuses_captured_producer_shapes_and_converts_current_names() {
assert!(root.join("Renamed/Portrait.jpg").is_file());
assert!(
same_file::is_same_file(
- root.join("Renamed/Pair.jpg"),
- root.join("Live Pair/Pair.jpg")
+ root.join("Renamed").join(pair),
+ root.join("Live Pair").join(pair)
)
.unwrap()
);
+ if producer == "rust" {
+ assert_eq!(
+ fs::read(root.join("Renamed/Pair.xmp")).unwrap(),
+ b"unrelated metadata"
+ );
+ assert_component(
+ &root,
+ "Renamed",
+ "Pair.MOV",
+ format!("export fixture {PAIR} component 1").as_bytes(),
+ );
+ }
assert!(root.join("Renamed/Live-1.jpg").is_file());
assert!(root.join("Renamed/Live.mov").is_file());
assert_eq!(
@@ -575,7 +614,10 @@ fn adoption_requires_initial_presence_and_never_treats_a_failed_refresh_as_histo
#[test]
fn adoption_hashless_verification_downloads_once_per_source_and_keeps_scope_per_invocation() {
let mut server = mockito::Server::new();
- let (mut engine, downloads) = new_engine(&server, "desktop");
+ let (mut engine, downloads) = new_engine(&server, "go");
+ for folder in ["Family", "Live Pair"] {
+ nominal_live_record(&engine.root(), folder);
+ }
for ((_, id), remote) in &mut engine.catalog.files {
if ![PHOTO, PAIR].contains(id) {
continue;
@@ -990,22 +1032,69 @@ fn adoption_defers_wholly_missing_hashless_originals_until_after_acceptance() {
requested.assert();
}
+fn nominal_live_record(root: &Path, folder: &str) {
+ let path = root.join(folder).join(".meta/Pair.jpg.json");
+ let mut record = read_json(&path);
+ record["info"]["fileNames"] = json!(["Pair.jpg"]);
+ fs::write(&path, serde_json::to_vec(&record).unwrap()).unwrap();
+ fs::rename(path, root.join(folder).join(".meta/Pair_0.jpg.json")).unwrap();
+}
+
+#[test]
+fn adoption_leaves_unmatched_old_rust_live_files_for_ordinary_conflict_handling() {
+ for renamed in [false, true] {
+ let server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, "go");
+ let root = engine.root();
+ nominal_live_record(&root, "Family");
+ fs::write(root.join("Family/Pair.mov"), b"edited video").unwrap();
+ if renamed {
+ rename(&mut engine, FAMILY, PAIR, "Holiday.jpg");
+ }
+ let export = engine.prepare(engine.options(true, &[])).unwrap();
+ assert_eq!(
+ fs::read(root.join("Family/Pair.mov")).unwrap(),
+ b"edited video"
+ );
+ assert_eq!(
+ fs::read(root.join("Trash/Family/Pair.jpg")).unwrap(),
+ format!("export fixture {PAIR} component 0").as_bytes()
+ );
+ let result = engine.execute(export).unwrap();
+ assert_eq!(result.complete, renamed, "{}", result.json);
+ assert_eq!(
+ fs::read(root.join("Family/Pair.mov")).unwrap(),
+ b"edited video"
+ );
+ if renamed {
+ assert_component(
+ &root,
+ "Family",
+ "Holiday.mov",
+ format!("export fixture {PAIR} component 1").as_bytes(),
+ );
+ } else {
+ assert_eq!(result.json["conflicts"], 1);
+ fs::remove_file(root.join("Family/Pair.mov")).unwrap();
+ assert!(engine.run(false, &[]).unwrap().complete);
+ assert_component(
+ &root,
+ "Family",
+ "Pair.mov",
+ format!("export fixture {PAIR} component 1").as_bytes(),
+ );
+ }
+ }
+}
+
#[test]
-fn adoption_verifies_a_single_undeclared_live_survivor_before_preserving_it() {
+fn adoption_preserves_a_single_hash_identified_live_survivor() {
let mut server = mockito::Server::new();
let (mut engine, downloads) = new_engine(&server, "go");
let root = engine.root();
- let path = root.join("Family/.meta/Pair.jpg.json");
- let mut record = read_json(&path);
- record["info"]["fileNames"] = json!(["Pair.jpg"]);
- fs::write(path, serde_json::to_vec(&record).unwrap()).unwrap();
+ nominal_live_record(&root, "Family");
fs::remove_file(root.join("Family/Pair.mov")).unwrap();
let original = fs::read(root.join("Family/Pair.jpg")).unwrap();
- fs::write(root.join("Family/Pair.jpg"), b"edited survivor").unwrap();
- let before = snapshot(&root);
- assert!(engine.run(true, &[]).is_err());
- assert_eq!(snapshot(&root), before);
- fs::write(root.join("Family/Pair.jpg"), &original).unwrap();
for (path, _) in snapshot(&root) {
if !path.to_string_lossy().ends_with(".json")
&& path.as_path() != Path::new("Family/Pair.jpg")
diff --git a/rust/crates/photos-export/src/legacy/read.rs b/rust/crates/photos-export/src/legacy/read.rs
index cf38daead4..387a24cad4 100644
--- a/rust/crates/photos-export/src/legacy/read.rs
+++ b/rust/crates/photos-export/src/legacy/read.rs
@@ -53,6 +53,7 @@ pub struct Record {
pub name: String,
pub hash: Option<String>,
pub parts: Vec<Part>,
+ pub candidates: Vec<String>,
pub go_record: Option<String>,
pub desktop: bool,
}
@@ -153,6 +154,7 @@ pub fn scan(
name: paths[0].name.clone(),
hash: None,
parts: paths,
+ candidates: Vec::new(),
go_record: None,
desktop: true,
};
@@ -221,6 +223,7 @@ pub fn scan(
Some(record.owner_id),
true,
)?;
+ let candidates = same_stem_files(&root.join(&path))?;
for entry in fs::read_dir(identity.parent().context("missing metadata parent")?)? {
let entry = entry?;
if entry.file_name() == "album_meta.json"
@@ -244,6 +247,14 @@ pub fn scan(
.as_str()
.context("missing legacy file title")?
.to_owned();
+ let siblings = if info.file_names.len() == 1 {
+ candidates
+ .get(&names::folded(names::split(&info.file_names[0], false).0))
+ .cloned()
+ .unwrap_or_default()
+ } else {
+ Vec::new()
+ };
let record = Record {
album: record.id,
file: info.id,
@@ -255,6 +266,7 @@ pub fn scan(
.into_iter()
.map(|name| Part { name, role: None })
.collect(),
+ candidates: siblings,
go_record: Some(format!(".meta/{name}")),
desktop: false,
};
@@ -269,6 +281,22 @@ pub fn scan(
Ok(())
}
+fn same_stem_files(directory: &Path) -> Result<BTreeMap<String, Vec<String>>> {
+ let mut files: BTreeMap<String, Vec<String>> = BTreeMap::new();
+ for entry in fs::read_dir(directory)? {
+ let entry = entry?;
+ if entry.file_type()?.is_file()
+ && let Ok(name) = entry.file_name().into_string()
+ {
+ files
+ .entry(names::folded(names::split(&name, false).0))
+ .or_default()
+ .push(name);
+ }
+ }
+ Ok(files)
+}
+
fn album(
root: &Path,
index: &Index,
diff --git a/rust/crates/photos-export/src/legacy/verify.rs b/rust/crates/photos-export/src/legacy/verify.rs
index 918d059fae..0f5819adb4 100644
--- a/rust/crates/photos-export/src/legacy/verify.rs
+++ b/rust/crates/photos-export/src/legacy/verify.rs
@@ -23,7 +23,7 @@ pub async fn file(
source: &FileSnapshot,
expected: &mut Option<Result<Vec<(Role, String)>, String>>,
) -> Result<usize> {
- let record = item.legacy.as_ref().context("missing legacy record")?;
+ let record = item.legacy.as_mut().context("missing legacy record")?;
let roles = transfer::roles(source);
ensure!(
source.kind == "livephoto" || record.parts.len() == 1,
@@ -42,6 +42,13 @@ pub async fn file(
|| record.parts[0].role != record.parts[1].role,
"duplicate legacy component role"
);
+ let discover = !record.desktop && source.kind == "livephoto" && record.parts.len() == 1;
+ if discover {
+ record.parts = std::mem::take(&mut record.candidates)
+ .into_iter()
+ .map(|name| super::read::Part { name, role: None })
+ .collect();
+ }
let mut measured = Vec::new();
for part in &record.parts {
let path = names::check(check.root, &format!("{}/{}", album.path, part.name))?;
@@ -79,6 +86,35 @@ pub async fn file(
.filter_map(|r| transfer::expected_hash(source, r).map(|h| (r.clone(), h.to_owned())))
.collect()
};
+ if discover {
+ let parts = std::mem::take(&mut record.parts);
+ let mut matched = Vec::new();
+ for (mut part, measurement) in parts.into_iter().zip(measured) {
+ let Some((_, hash, _)) = &measurement else {
+ continue;
+ };
+ let mut matches = known.iter().filter(|(_, expected)| expected == hash);
+ let Some((role, _)) = matches.next() else {
+ continue;
+ };
+ ensure!(
+ matches.next().is_none()
+ && !record
+ .parts
+ .iter()
+ .any(|part| part.role.as_ref() == Some(role)),
+ "ambiguous legacy Live Photo component at {}/{}",
+ album.path,
+ part.name
+ );
+ adopt::claim(check.index, album, &item.placement.id, &part.name, "")?;
+ part.role = Some(role.clone());
+ record.parts.push(part);
+ matched.push(measurement);
+ }
+ measured = matched;
+ }
+ let present = measured.iter().filter(|p| p.is_some()).count();
let complete = present == roles.len();
item.leftover = !complete && (source.kind == "livephoto" || record.desktop);
item.placement.kind = source.kind.clone();
@@ -140,6 +176,8 @@ pub async fn file(
}
}
if complete {
+ item.components
+ .sort_by_key(|component| component.role.name());
let layout: Vec<_> = item.components.iter().map(Component::portable).collect();
let favorited = check.store.db.query_row(
"SELECT favorited FROM desired_files WHERE album=?1 AND file=?2",
diff --git a/rust/crates/photos-export/src/lib.rs b/rust/crates/photos-export/src/lib.rs
index 63ba875557..316bfe8143 100644
--- a/rust/crates/photos-export/src/lib.rs
+++ b/rust/crates/photos-export/src/lib.rs
@@ -424,17 +424,25 @@ fn initialize(destination: &Path, root: &Root, legacy: bool) -> Result<RootLock>
.create_new(true)
.open(&path)
.context("destination was initialized by another writer; retry")?;
- file.lock().context("cannot lock the new export")?;
- let mut lock = RootLock(file);
- ensure!(
- legacy
- || std::fs::read_dir(destination)?
- .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),
- "destination became nonempty during initialization"
- );
- serde_json::to_writer_pretty(&mut lock.0, root)?;
- lock.0.write_all(b"\n")?;
- Ok(lock)
+ let result = (|| {
+ file.lock().context("cannot lock the new export")?;
+ let mut lock = RootLock(file);
+ ensure!(
+ legacy
+ || std::fs::read_dir(destination)?
+ .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),
+ "destination became nonempty during initialization"
+ );
+ serde_json::to_writer_pretty(&mut lock.0, root)?;
+ lock.0.write_all(b"\n")?;
+ Ok(lock)
+ })();
+ if result.is_err()
+ && let Err(error) = std::fs::remove_file(&path)
+ {
+ return result.with_context(|| format!("cannot remove incomplete export.json: {error}"));
+ }
+ result
}
fn select(
@@ -912,6 +920,21 @@ fn canonical_id(selector: &str) -> rusqlite::types::Value {
mod tests {
use super::*;
+ #[test]
+ fn initialization_removes_its_marker_on_failure_and_retries() {
+ let root = Root::new(&ente_core::crypto::Key::generate()).unwrap();
+ let directory = tempfile::tempdir().unwrap();
+ let stray = directory.path().join("photo.jpg");
+ std::fs::write(&stray, b"unrelated").unwrap();
+ let error = initialize(directory.path(), &root, false).err().unwrap();
+ assert!(error.to_string().contains("destination became nonempty"));
+ assert!(!directory.path().join("export.json").exists());
+ assert_eq!(std::fs::read(&stray).unwrap(), b"unrelated");
+ std::fs::remove_file(stray).unwrap();
+ drop(initialize(directory.path(), &root, false).unwrap());
+ assert!(open_root(directory.path(), &root).unwrap().is_some());
+ }
+
#[test]
fn transparent_errors_preserve_fatal_classification() {
for status in [401, 404] {