π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Adopt old Photos exports and protect local edits
Manav Rathi306a0dc523ce
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex a0530d87d9..fcbbf527c1 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2861,6 +2861,7 @@ dependencies = ["rusqlite","serde","serde_json",+ "tempfile","thiserror 2.0.21","tokio","unicode-normalization",diff --git a/rust/apps/cli-next/src/args.rs b/rust/apps/cli-next/src/args.rsindex 27b8d4cb52..0bc782bafb 100644--- a/rust/apps/cli-next/src/args.rs+++ b/rust/apps/cli-next/src/args.rs@@ -141,7 +141,7 @@ pub struct ExportArgs {pub exclude_album: Vec<String>,#[arg(long,- help = "Reconnect an existing export after moving it or losing local state"+ help = "Take over an old CLI/Desktop export or reconnect an existing one")]pub adopt: bool,#[arg(diff --git a/rust/apps/cli-next/src/db.rs b/rust/apps/cli-next/src/db.rsindex 779c6befff..e754fa3da2 100644--- a/rust/apps/cli-next/src/db.rs+++ b/rust/apps/cli-next/src/db.rs@@ -42,3 +42,31 @@ pub fn connect(path: &Path, key: &DbKey, create: bool) -> Result<Connection> {.context("cannot open the encrypted database")?;Ok(connection)}++pub fn scratch(path: &Path) -> Result<Connection> {+ connect(+ path,+ &DbKey(*ente_core::crypto::Key::generate().as_bytes()),+ true,+ )+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn scratch_pages_require_the_discarded_attempt_key() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("index.db");+ let db = scratch(&path).unwrap();+ db.execute_batch(+ "CREATE TABLE names(name TEXT); INSERT INTO names VALUES('private export album');",+ )+ .unwrap();+ drop(db);+ let bytes = std::fs::read(&path).unwrap();+ assert!(!bytes.starts_with(b"SQLite format 3"));+ assert!(scratch(&path).is_err());+ }+}diff --git a/rust/apps/cli-next/src/export/mod.rs b/rust/apps/cli-next/src/export/mod.rsindex b38dd62f0b..832c6dbc69 100644--- a/rust/apps/cli-next/src/export/mod.rs+++ b/rust/apps/cli-next/src/export/mod.rs@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};use anyhow::{Context, Result, ensure};use ente_core::crypto::hash;-use ente_photos_export::{AdoptionRequired, Export};+use ente_photos_export::Export;use tokio::sync::watch;use crate::{@@ -26,13 +26,6 @@ pub async fn run(args: ExportArgs, selected: Option<&str>, options: &Options) ->runtime.block_on(run_locked(args, selected.as_deref(), as_json))}).await?- .map_err(|error| {- if error.is::<AdoptionRequired>() {- error.context("this export requires --adopt")- } else {- error- }- })}async fn run_locked(args: ExportArgs, selected: Option<&str>, as_json: bool) -> Result<()> {@@ -81,7 +74,14 @@ async fn run_locked(args: ExportArgs, selected: Option<&str>, as_json: bool) ->let mut source_db = db::open(&account_home.path, &account.db_key, true)?;let mut source = source::ReplicaSource::new(&mut source_db, session.user_id);export- .prepare(connection, &mut source, &session, &mut cancel, &report)+ .prepare(+ connection,+ db::scratch,+ &mut source,+ &session,+ &mut cancel,+ &report,+ ).await?;drop(source);drop(source_db);diff --git a/rust/apps/cli-next/tests/fixtures/export-adoption/desktop-file.json b/rust/apps/cli-next/tests/fixtures/export-adoption/desktop-file.jsonnew file mode 100644index 0000000000..3c62654089--- /dev/null+++ b/rust/apps/cli-next/tests/fixtures/export-adoption/desktop-file.json@@ -0,0 +1,15 @@+{+ "title": "Photo.jpg",+ "photoTakenTime": {+ "timestamp": "1791179341",+ "formatted": "Oct 5, 2026, 5:49:01 AM UTC"+ },+ "creationTime": {+ "timestamp": "1791179341",+ "formatted": "Oct 5, 2026, 5:49:01 AM UTC"+ },+ "modificationTime": {+ "timestamp": "1791179341",+ "formatted": "Oct 5, 2026, 5:49:01 AM UTC"+ }+}diff --git a/rust/apps/cli-next/tests/fixtures/export-adoption/desktop-manifest.json b/rust/apps/cli-next/tests/fixtures/export-adoption/desktop-manifest.jsonnew file mode 100644index 0000000000..8a748e29fb--- /dev/null+++ b/rust/apps/cli-next/tests/fixtures/export-adoption/desktop-manifest.json@@ -0,0 +1,18 @@+{+ "version": 5,+ "stage": 7,+ "lastAttemptTimestamp": 1791186928697,+ "collectionExportNames": {+ "1580559962386472": "Travel",+ "1580559962386474": "Live Pair",+ "1580559962386473": "Family"+ },+ "fileExportNames": {+ "10000064_1580559962386472_1791179545995470": "Photo.jpg",+ "10000066_1580559962386474_1791179670357138": "{\"image\":\"Pair.jpg\",\"video\":\"Pair.mov\"}",+ "10000062_1580559962386473_1791179545928006": "Photo.jpg",+ "10000063_1580559962386473_1791179545987020": "Live.jpg",+ "10000065_1580559962386473_1791179546000091": "Live.mov",+ "10000066_1580559962386473_1791179689874921": "{\"image\":\"Pair.jpg\",\"video\":\"Pair.mov\"}"+ }+}diff --git a/rust/apps/cli-next/tests/fixtures/export-adoption/go-album.json b/rust/apps/cli-next/tests/fixtures/export-adoption/go-album.jsonnew file mode 100644index 0000000000..a1fbc9c7a4--- /dev/null+++ b/rust/apps/cli-next/tests/fixtures/export-adoption/go-album.json@@ -0,0 +1,9 @@+{+ "id": 1580559962386473,+ "ownerID": 1580559962386471,+ "albumName": "Family",+ "isDeleted": false,+ "accountOwnerIDs": [+ 1580559962386471+ ]+}diff --git a/rust/apps/cli-next/tests/fixtures/export-adoption/go-file.json b/rust/apps/cli-next/tests/fixtures/export-adoption/go-file.jsonnew file mode 100644index 0000000000..4a1e60d164--- /dev/null+++ b/rust/apps/cli-next/tests/fixtures/export-adoption/go-file.json@@ -0,0 +1,15 @@+{+ "title": "Photo.jpg",+ "description": null,+ "location": null,+ "creationTime": "2026-10-05T11:19:01.464+05:30",+ "modificationTime": "2026-10-05T11:19:01.464+05:30",+ "info": {+ "id": 10000062,+ "hash": null,+ "ownerID": 1580559962386471,+ "fileNames": [+ "Photo.jpg"+ ]+ }+}diff --git a/rust/apps/cli-next/tests/support/export.rs b/rust/apps/cli-next/tests/support/export.rsindex f5264ddb83..513007950c 100644--- a/rust/apps/cli-next/tests/support/export.rs+++ b/rust/apps/cli-next/tests/support/export.rs@@ -162,8 +162,11 @@ fn export_metadata_updates_and_repairs_preserve_source_values() -> TestResult {}export(&home, &root, &["--album", "Travel"]);place(origin,&account,favorites,&favorites_key,file,&key,"add-files").await;+ let mut edited_metadata=json!({"editedName":"Renamed.jpg","editedTime":1_700_000_002_654_321i64,"caption":"","lat":0,"long":0,"dateTime":"2023:11:15 03:43:22","offsetTime":"+05:30","w":4096,"h":3072,"cameraMake":"Example","cameraModel":"Camera","uploaderName":"Guest","mediaType":1,"mvi":123});for (version,lat,long) in [(1,0,77),(2,12,0),(3,0,0)] {- edit_file(origin,&account,file,&key,version,json!({"editedName":"Renamed.jpg","editedTime":1_700_000_002_654_321i64,"caption":"","lat":lat,"long":long,"dateTime":"2023:11:15 03:43:22","offsetTime":"+05:30","w":4096,"h":3072,"cameraMake":"Example","cameraModel":"Camera","uploaderName":"Guest","mediaType":1,"mvi":123})).await;+ edited_metadata["lat"]=json!(lat);+ edited_metadata["long"]=json!(long);+ edit_file(origin,&account,file,&key,version,edited_metadata.clone()).await;export(&home,&root,&["--album","Family"]);let record=read_json(&root.join("Family/metadata/Renamed.jpg.json"));assert_eq!(record["description"],"");@@ -187,7 +190,29 @@ fn export_metadata_updates_and_repairs_preserve_source_values() -> TestResult {export(&home,&root,&["--album","Family"]);assert!(read_json(&root.join("Family/metadata/Renamed.jpg.json")).get("custom").is_none());fs::write(root.join("Family/Renamed.jpg"),vec![0;original.len()])?;- export(&home,&root,&["--album","Family"]);+ let db=export_database(&home);+ let baseline: String=db.query_row("SELECT hash FROM components WHERE folder=?1",[format!("active:{family}")],|r|r.get(0))?;+ let state=home.read_vault();+ let saved=&state["accounts"][0];+ let replica=rusqlite::Connection::open(home.dir.path().join("accounts").join(saved["storage_id"].as_str().unwrap()).join("data.db"))?;+ let db_key: Vec<u8>=serde_json::from_value(saved["db_key"].clone())?;+ let hex: String=db_key.iter().map(|byte|format!("{byte:02x}")).collect();+ replica.pragma_update(None,"key",format!("x'{hex}'"))?;+ let before: i64=replica.query_row("SELECT files_cursor FROM photos_collections WHERE id=?1",[family],|r|r.get(0))?;+ edited_metadata["caption"]=json!("source advanced");+ edit_file(origin,&account,file,&key,4,edited_metadata).await;+ let output=home.run(&["photos","export",root.to_str().unwrap(),"--json"]);+ assert!(failure(&output).contains("locally changed media"));+ let result: Value=serde_json::from_slice(&output.stdout)?;+ assert_eq!(result["complete"],false);+ assert_eq!(result["conflicts"],1);+ assert_eq!(fs::read(root.join("Family/Renamed.jpg"))?,vec![0;original.len()]);+ assert_eq!(read_json(&root.join("Travel/metadata/Renamed.jpg.json"))["description"],"source advanced");+ let after: i64=replica.query_row("SELECT files_cursor FROM photos_collections WHERE id=?1",[family],|r|r.get(0))?;+ assert!(after>before);+ assert_eq!(db.query_row("SELECT hash FROM components WHERE folder=?1",[format!("active:{family}")],|r|r.get::<_,String>(0))?,baseline);+ fs::rename(root.join("Family/Renamed.jpg"),destination.path().join("local-edit.jpg"))?;+ export(&home,&root,&[]);assert_eq!(fs::read(root.join("Family/Renamed.jpg"))?,original);assert_eq!(objects.reads(),initial_reads+1);Ok(())@@ -365,6 +390,22 @@ fn export_adopts_relocated_output_and_verifies_later_selections() -> TestResultassert_eq!(objects.reads(), reads);assert_eq!(fs::read(relocated.join("export.json"))?, root_bytes);fs::write(relocated.join("Work/Work.jpg"), b"xxxx")?;+ let output = fresh.run(&[+ "photos",+ "export",+ relocated.to_str().unwrap(),+ "--album",+ "Work",+ "--json",+ ]);+ assert!(failure(&output).contains("locally changed media"));+ assert_eq!(+ serde_json::from_slice::<Value>(&output.stdout)?["conflicts"],+ 1+ );+ assert_eq!(fs::read(relocated.join("Work/Work.jpg"))?, b"xxxx");+ assert_eq!(objects.reads(), reads);+ fs::remove_file(relocated.join("Work/Work.jpg"))?;export(&fresh, &relocated, &["--album", "Work"]);assert_eq!(objects.reads(), reads + 1);assert_eq!(fs::read(relocated.join("Work/Work.jpg"))?, b"work");@@ -1676,3 +1717,76 @@ async fn export_account(origin: &str) -> (AuthenticatedAccount, TestHome, Stringlogin(&home, "photos", &email, &["--host", origin]);(account, home, email)}++#[test]+fn export_adopts_go_and_desktop_fixtures_with_a_real_source_account() -> TestResult {+ Museum::run(|museum| {+ tokio::runtime::Runtime::new()?.block_on(async {+ let origin = museum.endpoint();+ let objects = museum.object_store();+ let (account, home, _) = export_account(origin).await;+ let (album, key) = create_album(origin, &account, "Family", "album").await;+ let fixtures =+ Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/export-adoption");+ let bytes = b"adoption integration original".to_vec();+ let (file, _) = upload_fixture(+ origin,+ &account,+ album,+ &key,+ &bytes,+ metadata("Photo.jpg", &bytes),+ )+ .await;+ for producer in ["go", "desktop"] {+ let directory = tempfile::tempdir()?;+ let root = directory.path().join("photos");+ fs::create_dir_all(root.join("Family"))?;+ fs::write(root.join("Family/Photo.jpg"), &bytes)?;+ if producer == "go" {+ fs::create_dir_all(root.join("Family/.meta"))?;+ let mut record = read_json(&fixtures.join("go-album.json"));+ record["id"] = json!(album);+ record["ownerID"] = json!(account.user_id);+ record["accountOwnerIDs"] = json!([account.user_id]);+ fs::write(+ root.join("Family/.meta/album_meta.json"),+ serde_json::to_vec(&record)?,+ )?;+ let mut record = read_json(&fixtures.join("go-file.json"));+ record["info"]["id"] = json!(file);+ record["info"]["ownerID"] = json!(account.user_id);+ record["info"]["hash"] = json!(digest(&bytes));+ fs::write(+ root.join("Family/.meta/Photo.jpg.json"),+ serde_json::to_vec(&record)?,+ )?;+ } else {+ let mut record =+ json!({"version":5,"collectionExportNames":{},"fileExportNames":{}});+ record["collectionExportNames"][album.to_string()] = json!("Family");+ record["fileExportNames"][format!("{file}_{album}_100")] = json!("Photo.jpg");+ fs::write(+ root.join("export_status.json"),+ serde_json::to_vec(&record)?,+ )?;+ }+ let reads = objects.reads();+ let output = home.run(&["photos", "export", root.to_str().unwrap()]);+ assert_eq!(+ failure(&output),+ "Error: This folder contains an export from the old CLI or Ente Desktop. Use --adopt if you want to take it over.\n"+ );+ let result = export(&home, &root, &["--adopt", "--album", "Family"]);+ assert_eq!(result["copies"]["completed"], 1);+ assert_eq!(objects.reads(), reads);+ assert_eq!(fs::read(root.join("Family/Photo.jpg"))?, bytes);+ assert_eq!(+ read_json(&root.join("Family/metadata/Photo.jpg.json"))["ente"]["fileID"],+ file.to_string()+ );+ }+ Ok(())+ })+ })+}diff --git a/rust/apps/cli-next/tests/support/export_adoption.rs b/rust/apps/cli-next/tests/support/export_adoption.rsnew file mode 100644index 0000000000..4792a99834--- /dev/null+++ b/rust/apps/cli-next/tests/support/export_adoption.rs@@ -0,0 +1,1029 @@+use super::engine::{Catalog, Engine};+use super::*;+use ente_photos::files;+use std::{collections::BTreeMap, path::PathBuf};++const USER: i64 = 1580559962386471;+const FAMILY: i64 = 1580559962386473;+const PAIR: i64 = 10000066;+const PHOTO: i64 = 10000062;++fn new_engine(server: &mockito::ServerGuard, producer: &str) -> (Engine, BTreeMap<i64, Vec<u8>>) {+ let fixtures = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/export-adoption");+ let manifest = read_json(&fixtures.join("desktop-manifest.json"));+ let albums = [+ (FAMILY, "Family"),+ (1580559962386472, "Travel"),+ (1580559962386474, "Live Pair"),+ ]+ .into_iter()+ .map(|(id, name)| (id, name.into()))+ .collect();+ let mut engine = Engine::new(+ server,+ Catalog {+ user: USER,+ key: Key::generate(),+ albums,+ files: BTreeMap::new(),+ failed_album: None,+ },+ );+ write_fixture(&engine.root(), producer);+ let mut files = BTreeMap::new();+ let mut downloads = BTreeMap::new();+ let mut records = BTreeMap::new();+ for (identity, name) in manifest["fileExportNames"].as_object().unwrap() {+ let ids: Vec<i64> = identity.split('_').map(|s| s.parse().unwrap()).collect();+ let (id, album) = (ids[0], ids[1]);+ let record = records.entry(id).or_insert_with(|| {+ let folder = engine.root().join(&engine.catalog.albums[&album]);+ let exported = name.as_str().unwrap();+ let (original,kind,digest) = if exported.starts_with('{') {+ let pair: Value = serde_json::from_str(exported).unwrap();+ let image = fs::read(folder.join(pair["image"].as_str().unwrap())).unwrap();+ let video = fs::read(folder.join(pair["video"].as_str().unwrap())).unwrap();+ let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));+ for (name,bytes) in [("image.jpg",&image),("video.mov",&video)] {+ zip.start_file(name,zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored)).unwrap();+ zip.write_all(bytes).unwrap();+ }+ (zip.finish().unwrap().into_inner(),2,format!("{}:{}",digest(&image),digest(&video)))+ } else {+ let bytes = fs::read(folder.join(exported)).unwrap();+ (bytes.clone(),if exported.ends_with(".mov") {1} else {0},digest(&bytes))+ };+ let name = match id {+ PHOTO | 10000064 => "Photo.jpg",+ 10000063 => "Live.jpg",+ 10000065 => "Live.mov",+ PAIR => "Pair.jpg",+ _ => unreachable!(),+ };+ let (mut remote,encrypted) = source(id,&engine.catalog.key,&original,name);+ remote["ownerID"] = json!(USER);+ let metadata = blob::encrypt_json(&json!({"title":name,"fileType":kind,"hash":digest,"creationTime":1_700_000_000_000_000i64}),&file_key(&remote,&engine.catalog.key)).unwrap();+ remote["metadata"] = json!({"encryptedData":b64::encode(&metadata.encrypted_data),"decryptionHeader":b64::encode(metadata.decryption_header.as_bytes())});+ downloads.insert(id,encrypted);+ remote+ });+ files.insert((album, id), record.clone());+ }+ engine.catalog.files = files;+ (engine, downloads)+}++fn rename(engine: &mut Engine, album: i64, file: i64, name: &str) {+ let previous = engine.catalog.files[&(album, file)].clone();+ engine.catalog.files.insert(+ (album, file),+ public(+ previous,+ &engine.catalog.key,+ json!({"editedName":name}),+ 200,+ ),+ );+}++fn digest(bytes: &[u8]) -> String {+ b64::encode(&hash::hash(bytes, Some(64), None).unwrap())+}++fn write_fixture(root: &Path, producer: &str) {+ let fixtures = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/export-adoption");+ let manifest = read_json(&fixtures.join("desktop-manifest.json"));+ fs::create_dir_all(root).unwrap();+ if producer == "desktop" {+ fs::write(+ root.join("export_status.json"),+ serde_json::to_vec(&manifest).unwrap(),+ )+ .unwrap();+ }+ for (id, name) in manifest["collectionExportNames"].as_object().unwrap() {+ let folder = root.join(name.as_str().unwrap());+ fs::create_dir_all(&folder).unwrap();+ if producer == "go" {+ fs::create_dir_all(folder.join(".meta")).unwrap();+ let mut album = read_json(&fixtures.join("go-album.json"));+ album["id"] = json!(id.parse::<i64>().unwrap());+ album["albumName"] = name.clone();+ fs::write(+ folder.join(".meta/album_meta.json"),+ serde_json::to_vec(&album).unwrap(),+ )+ .unwrap();+ } else {+ fs::create_dir_all(folder.join("metadata")).unwrap();+ }+ }+ for (key, value) in manifest["fileExportNames"].as_object().unwrap() {+ let ids: Vec<_> = key.split('_').collect();+ let id = ids[0].parse::<i64>().unwrap();+ let folder = root.join(manifest["collectionExportNames"][ids[1]].as_str().unwrap());+ let value = value.as_str().unwrap();+ let parts = if value.starts_with('{') {+ let pair: Value = serde_json::from_str(value).unwrap();+ vec![+ pair["image"].as_str().unwrap().to_owned(),+ pair["video"].as_str().unwrap().to_owned(),+ ]+ } else {+ vec![value.to_owned()]+ };+ let mut hashes = Vec::new();+ for (index, name) in parts.iter().enumerate() {+ let bytes = format!("export fixture {id} component {index}").into_bytes();+ hashes.push(digest(&bytes));+ fs::write(folder.join(name), bytes).unwrap();+ if producer == "desktop" {+ let mut sidecar = read_json(&fixtures.join("desktop-file.json"));+ sidecar["title"] = json!(name);+ fs::write(+ folder.join(format!("metadata/{name}.json")),+ serde_json::to_vec(&sidecar).unwrap(),+ )+ .unwrap();+ }+ }+ if producer == "go" {+ let mut sidecar = read_json(&fixtures.join("go-file.json"));+ sidecar["title"] = json!(parts[0]);+ sidecar["info"]["id"] = json!(id);+ sidecar["info"]["fileNames"] = json!(parts);+ sidecar["info"]["hash"] = json!(hashes.join(":"));+ fs::write(+ folder.join(format!(".meta/{}.json", parts[0])),+ serde_json::to_vec(&sidecar).unwrap(),+ )+ .unwrap();+ }+ }+}++fn snapshot(root: &Path) -> BTreeMap<PathBuf, (Vec<u8>, std::time::SystemTime)> {+ fn visit(+ root: &Path,+ path: &Path,+ result: &mut BTreeMap<PathBuf, (Vec<u8>, std::time::SystemTime)>,+ ) {+ for entry in fs::read_dir(path).unwrap() {+ let entry = entry.unwrap();+ let path = entry.path();+ let metadata = fs::symlink_metadata(&path).unwrap();+ if metadata.is_dir() {+ visit(root, &path, result);+ } else if metadata.is_file() {+ result.insert(+ path.strip_prefix(root).unwrap().to_owned(),+ (fs::read(&path).unwrap(), metadata.modified().unwrap()),+ );+ }+ }+ }+ let mut result = BTreeMap::new();+ visit(root, root, &mut result);+ result+}++#[test]+fn adoption_reuses_captured_producer_shapes_and_converts_current_names() {+ for producer in ["go", "desktop"] {+ let mut server = mockito::Server::new();+ let (mut engine, downloads) = new_engine(&server, producer);+ let requests: Vec<_> = downloads+ .iter()+ .map(|(id, bytes)| download(&mut server, *id, bytes, 0))+ .collect();+ let root = engine.root();+ if producer == "desktop" {+ fs::rename(root.join("Family"), root.join("Family(1)")).unwrap();+ fs::rename(+ root.join("Family(1)/Photo.jpg"),+ root.join("Family(1)/Photo(1).jpg"),+ )+ .unwrap();+ fs::rename(+ root.join("Family(1)/metadata/Photo.jpg.json"),+ root.join("Family(1)/metadata/Photo(1).jpg.json"),+ )+ .unwrap();+ let mut manifest = read_json(&root.join("export_status.json"));+ manifest["collectionExportNames"][FAMILY.to_string()] = json!("Family(1)");+ let key = manifest["fileExportNames"]+ .as_object()+ .unwrap()+ .keys()+ .find(|key| key.starts_with(&format!("{PHOTO}_{FAMILY}_")))+ .unwrap()+ .clone();+ manifest["fileExportNames"][key] = json!("Photo(1).jpg");+ fs::write(+ root.join("export_status.json"),+ serde_json::to_vec(&manifest).unwrap(),+ )+ .unwrap();+ fs::write(+ root.join("Family(1)/metadata/Photo(1).jpg.json"),+ b"broken JSON",+ )+ .unwrap();+ } else {+ engine.catalog.albums.insert(FAMILY, "Renamed".into());+ rename(&mut engine, FAMILY, PHOTO, "Portrait.jpg");+ }+ let result = engine.run(true, &[]).unwrap();+ assert!(result.complete, "{}", result.json);+ assert_eq!(result.expected, 6);+ if producer == "desktop" {+ assert_component(+ &root,+ "Family(1)",+ "Photo(1).jpg",+ format!("export fixture {PHOTO} component 0").as_bytes(),+ );+ assert_eq!(+ read_json(&root.join("Family(1)/metadata.json"))["title"],+ "Family"+ );+ assert_eq!(+ read_json(&root.join("Family(1)/metadata/Photo(1).jpg.json"))["ente"]["name"],+ "Photo.jpg"+ );+ let before = snapshot(&root);+ assert!(engine.run(false, &[]).unwrap().complete);+ assert_eq!(snapshot(&root), before);+ engine.catalog.albums.insert(FAMILY, "Renamed".into());+ rename(&mut engine, FAMILY, PHOTO, "Portrait.jpg");+ assert!(engine.run(false, &[]).unwrap().complete);+ assert!(!root.join("Family(1)").exists());+ }+ assert!(!root.join("Family").exists());+ assert!(root.join("Renamed/Portrait.jpg").is_file());+ assert!(root.join("Renamed/Live-1.jpg").is_file());+ assert!(root.join("Renamed/Live.mov").is_file());+ assert_eq!(+ read_json(&root.join("Renamed/metadata/Portrait.jpg.json"))["ente"]["fileID"],+ PHOTO.to_string()+ );+ assert_eq!(count(&engine.db(), "SELECT count(*) FROM components"), 8);+ assert_eq!(count(&engine.db(), "SELECT bound FROM export"), 1);+ assert!(!root.join("export_status.json").exists());+ assert!(!root.join("Renamed/.meta").exists());+ let before = snapshot(&root);+ assert!(engine.run(false, &[]).unwrap().complete);+ assert_eq!(snapshot(&root), before);+ engine.database = "recovered.db".into();+ assert!(engine.run(true, &[]).unwrap().complete);+ for request in requests {+ request.assert();+ }+ }+}++#[test]+fn adoption_preserves_desktop_paths_for_names_changed_before_takeover() {+ let server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, "desktop");+ let root = engine.root();+ let original = fs::read(root.join("Family/Photo.jpg")).unwrap();+ engine.catalog.albums.insert(FAMILY, "Renamed".into());+ rename(&mut engine, FAMILY, PHOTO, "Portrait.jpg");+ assert!(engine.run(true, &[]).unwrap().complete);+ assert_eq!(fs::read(root.join("Family/Photo.jpg")).unwrap(), original);+ assert_eq!(+ read_json(&root.join("Family/metadata.json"))["title"],+ "Renamed"+ );+ assert_eq!(+ read_json(&root.join("Family/metadata/Photo.jpg.json"))["ente"]["name"],+ "Portrait.jpg"+ );+ let before = snapshot(&root);+ assert!(engine.run(false, &[]).unwrap().complete);+ assert_eq!(snapshot(&root), before);+}++#[test]+fn adoption_preflight_refuses_corrupt_media_scope_and_metadata_blockers_without_writes() {+ for producer in ["go", "desktop"] {+ let server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, producer);+ let root = engine.root();+ let before = snapshot(&root);+ assert!(+ engine+ .run(true, &["Family"])+ .err()+ .unwrap()+ .to_string()+ .contains("exactly")+ );+ assert_eq!(snapshot(&root), before);+ fs::write(root.join("Family/Photo.jpg"), b"local edit").unwrap();+ fs::write(root.join("Travel/Photo.jpg"), b"second edit").unwrap();+ let before = snapshot(&root);+ let error = engine.run(true, &[]).err().unwrap().to_string();+ assert!(error.contains("2 failing items"), "{error}");+ assert_eq!(snapshot(&root), before);+ assert!(!root.join("export.json").exists());+ assert_eq!(count(&engine.db(), "SELECT count(*) FROM albums"), 0);+ write_fixture(&root, producer);+ fs::write(root.join("Family/metadata.json"), b"user media").unwrap();+ let before = snapshot(&root);+ assert!(engine.run(true, &[]).is_err());+ assert_eq!(snapshot(&root), before);+ }+}++#[test]+fn adoption_preserves_partial_pairs_and_restores_missing_originals() {+ for producer in ["go", "desktop"] {+ let mut server = mockito::Server::new();+ let (mut engine, downloads) = new_engine(&server, producer);+ let root = engine.root();+ let survivor = fs::read(root.join("Family/Pair.jpg")).unwrap();+ let original = fs::read(root.join("Family/Photo.jpg")).unwrap();+ let sidecar = if producer == "desktop" {+ Some(fs::read(root.join("Family/metadata/Photo.jpg.json")).unwrap())+ } else {+ None+ };+ fs::remove_file(root.join("Family/Photo.jpg")).unwrap();+ fs::remove_file(root.join("Family/Pair.mov")).unwrap();+ fs::remove_file(root.join("Live Pair/Pair.jpg")).unwrap();+ fs::remove_file(root.join("Live Pair/Pair.mov")).unwrap();+ let ordinary = download(&mut server, PHOTO, &downloads[&PHOTO], 1);+ let requested = download(&mut server, PAIR, &downloads[&PAIR], 1);+ let result = engine.run(true, &[]).unwrap();+ assert!(result.complete, "{producer}: {}", result.json);+ requested.assert();+ ordinary.assert();+ assert_eq!(fs::read(root.join("Family/Photo.jpg")).unwrap(), original);+ if let Some(sidecar) = sidecar {+ assert_eq!(+ fs::read(root.join("Trash/Family/metadata/Photo.jpg.json")).unwrap(),+ sidecar+ );+ }+ assert_eq!(+ fs::read(root.join("Trash/Family/Pair.jpg")).unwrap(),+ survivor+ );+ assert!(+ root.join(if producer == "go" {+ "Trash/Family/.meta/Pair.jpg.json"+ } else {+ "Trash/Family/metadata/Pair.jpg.json"+ })+ .is_file()+ );+ assert_eq!(+ fs::read(root.join("Family/Pair.mov")).unwrap(),+ fs::read(root.join("Live Pair/Pair.mov")).unwrap()+ );+ }+ let server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, "go");+ let root = engine.root();+ fs::remove_file(root.join("Family/Pair.mov")).unwrap();+ let sidecar = root.join("Family/.meta/Pair.jpg.json");+ let mut value = read_json(&sidecar);+ value["info"]["hash"] = json!(format!(+ "{}:{}",+ value["info"]["hash"]+ .as_str()+ .unwrap()+ .split(':')+ .next()+ .unwrap(),+ digest(b"stale video")+ ));+ fs::write(&sidecar, serde_json::to_vec(&value).unwrap()).unwrap();+ let before = snapshot(&root);+ assert!(+ engine+ .run(true, &[])+ .err()+ .unwrap()+ .to_string()+ .contains("video component")+ );+ assert_eq!(snapshot(&root), before);+}++#[test]+fn adoption_history_resumes_after_media_move_and_keeps_the_full_scope() {+ let mut server = mockito::Server::new();+ let (mut engine, downloads) = new_engine(&server, "desktop");+ let root = engine.root();+ engine.catalog.files.remove(&(FAMILY, PAIR));+ fs::create_dir_all(root.join("Trash/Family")).unwrap();+ fs::write(root.join("Trash/Family/Pair.jpg"), b"existing trash").unwrap();+ fs::write(root.join("Family/user.txt"), b"unrelated").unwrap();+ let manifest = fs::read(root.join("export_status.json")).unwrap();+ fs::write(root.join("Trash/Family/metadata"), b"block sidecar move").unwrap();+ assert!(engine.prepare(engine.options(true, &[])).is_err());+ assert_eq!(count(&engine.db(), "SELECT count(*) FROM albums"), 0);+ assert_eq!(fs::read(root.join("export_status.json")).unwrap(), manifest);+ assert!(!root.join("Family/Pair.jpg").exists());+ assert!(root.join("Trash/Family/Pair-1.jpg").is_file());+ assert_eq!(count(&engine.db(), "SELECT bound FROM export"), 0);+ let before = snapshot(&root);+ assert!(+ engine+ .run(false, &[])+ .err()+ .unwrap()+ .is::<ente_photos_export::AdoptionRequired>()+ );+ assert!(engine.run(true, &["Family"]).is_err());+ assert_eq!(snapshot(&root), before);+ fs::remove_file(root.join("Trash/Family/metadata")).unwrap();+ let export = engine.prepare(engine.options(true, &[])).unwrap();+ drop(export);+ fs::write(root.join("Family/Photo.jpg"), b"edit after acceptance").unwrap();+ let result = engine.run(false, &[]).unwrap();+ assert!(!result.complete);+ assert_eq!(+ fs::read(root.join("Family/Photo.jpg")).unwrap(),+ b"edit after acceptance"+ );+ assert_eq!(fs::read(root.join("export_status.json")).unwrap(), manifest);+ assert!(root.join("Trash/Family/Pair.mov").is_file());+ assert!(root.join("Trash/Family/metadata/Pair.jpg.json").is_file());+ assert_eq!(+ fs::read(root.join("Trash/Family/Pair.jpg")).unwrap(),+ b"existing trash"+ );+ assert!(root.join("Live Pair/Pair.jpg").is_file());+ assert_eq!(+ fs::read(root.join("Family/user.txt")).unwrap(),+ b"unrelated"+ );+ fs::remove_file(root.join("Family/Photo.jpg")).unwrap();+ let downloaded = download(&mut server, PHOTO, &downloads[&PHOTO], 1);+ assert!(engine.run(false, &[]).unwrap().complete);+ downloaded.assert();+ assert!(!root.join("export_status.json").exists());+ assert_eq!(count(&engine.db(), "SELECT bound FROM export"), 1);+}++#[test]+fn adoption_replays_recorded_moves_without_rehashing_every_album() {+ for (producer, directory) in [("go", false), ("desktop", true)] {+ let server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, producer);+ let root = engine.root();+ let export = engine.prepare(engine.options(true, &[])).unwrap();+ drop(export);+ rename(&mut engine, FAMILY, PHOTO, "Portrait.jpg");+ if directory {+ engine.catalog.albums.insert(FAMILY, "Renamed".into());+ }+ let export = engine.prepare(engine.options(false, &[])).unwrap();+ let db = engine.db();+ if directory {+ db.execute_batch(&format!("CREATE TRIGGER interrupt_album BEFORE UPDATE ON albums WHEN old.id={FAMILY} AND new.path='Renamed' BEGIN SELECT RAISE(ABORT,'interrupt after directory rename'); END;")).unwrap();+ } else {+ db.execute_batch("CREATE TRIGGER interrupt_component BEFORE UPDATE ON components WHEN old.name='Photo.jpg' AND new.name='Portrait.jpg' BEGIN SELECT RAISE(ABORT,'interrupt after component rename'); END;").unwrap();+ }+ assert!(!engine.execute(export).unwrap().complete);+ let folder = if directory { "Renamed" } else { "Family" };+ assert!(root.join(folder).is_dir());+ db.execute_batch(if directory {+ "DROP TRIGGER interrupt_album"+ } else {+ "DROP TRIGGER interrupt_component"+ })+ .unwrap();+ drop(db);+ fs::write(root.join("Travel/Photo.jpg"), b"late independent edit").unwrap();+ let result = engine.run(false, &[]).unwrap();+ assert!(!result.complete);+ assert!(root.join(format!("{folder}/Portrait.jpg")).is_file());+ assert_eq!(+ fs::read(root.join("Travel/Photo.jpg")).unwrap(),+ b"late independent edit"+ );+ assert_eq!(+ read_json(&root.join(format!("{folder}/metadata/Portrait.jpg.json")))["ente"]["fileID"],+ PHOTO.to_string()+ );+ }+}++#[test]+fn adoption_requires_initial_presence_and_never_treats_a_failed_refresh_as_history() {+ let server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, "go");+ let root = engine.root();+ engine.catalog.failed_album = Some(FAMILY);+ let before = snapshot(&root);+ assert!(engine.run(true, &[]).is_err());+ assert_eq!(snapshot(&root), before);+ assert!(!root.join("Trash").exists());+ engine.catalog.failed_album = None;+ for (name, _) in snapshot(&root) {+ if !name.to_string_lossy().ends_with(".json") {+ fs::remove_file(root.join(name)).unwrap();+ }+ }+ let before = snapshot(&root);+ assert!(+ engine+ .run(true, &[])+ .err()+ .unwrap()+ .to_string()+ .contains("at least one")+ );+ assert_eq!(snapshot(&root), before);+}++#[test]+fn adoption_hashless_verification_downloads_once_per_source_and_keeps_scope_per_invocation() {+ let mut server = mockito::Server::new();+ let (mut engine, downloads) = new_engine(&server, "desktop");+ for ((_, id), remote) in &mut engine.catalog.files {+ if ![PHOTO, PAIR].contains(id) {+ continue;+ }+ let (_, documents) = files::decrypt(+ &serde_json::from_value(remote.clone()).unwrap(),+ &engine.catalog.key,+ )+ .unwrap();+ let mut metadata: Value = serde_json::from_slice(&documents.original).unwrap();+ metadata.as_object_mut().unwrap().remove("hash");+ let metadata =+ blob::encrypt_json(&metadata, &file_key(remote, &engine.catalog.key)).unwrap();+ remote["metadata"] = json!({"encryptedData":b64::encode(&metadata.encrypted_data),"decryptionHeader":b64::encode(metadata.decryption_header.as_bytes())});+ }+ let ordinary = download(&mut server, PHOTO, &downloads[&PHOTO], 2);+ let failed = server+ .mock("GET", format!("/files/download/v3/{PAIR}").as_str())+ .with_status(404)+ .expect(1)+ .create();+ let before = snapshot(&engine.root());+ let error = engine.run(true, &[]).err().unwrap().to_string();+ assert!(error.contains("1 failing items"), "{error}");+ assert_eq!(snapshot(&engine.root()), before);+ assert!(!engine.root().join("export.json").exists());+ failed.assert();+ drop(failed);+ let url = server+ .mock("GET", format!("/files/download/v3/{PAIR}").as_str())+ .with_body(json!({"url":format!("{}/comparison",server.url())}).to_string())+ .expect(1)+ .create();+ let db_path = engine.directory.path().join(&engine.database);+ let bytes = downloads[&PAIR].clone();+ let live = server+ .mock("GET", "/comparison")+ .with_body_from_request(move |_| {+ let db = rusqlite::Connection::open(&db_path).unwrap();+ db.busy_timeout(std::time::Duration::ZERO).unwrap();+ db.execute_batch("BEGIN IMMEDIATE; ROLLBACK;").unwrap();+ assert_eq!(count(&db, "SELECT count(*) FROM albums"), 0);+ bytes.clone()+ })+ .expect(1)+ .create();+ engine.catalog.albums.insert(99, "Other".into());+ assert!(engine.run(true, &[]).unwrap().complete);+ ordinary.assert();+ live.assert();+ url.assert();+ assert!(fs::read_dir(engine.root()).unwrap().all(|entry| {+ let entry = entry.unwrap();+ entry.file_type().unwrap().is_dir() || entry.file_name() == "export.json"+ }));+ assert!(!engine.root().join("Other").exists());+ assert!(engine.run(false, &["Family"]).unwrap().complete);+ assert!(!engine.root().join("Other").exists());+ assert!(engine.run(true, &[]).unwrap().complete);+ assert!(engine.root().join("Other/metadata.json").exists());+}++#[test]+fn adoption_relocates_an_unavailable_go_album_named_trash_without_inventing_native_history() {+ for keep_unknown in [false, true] {+ let server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, "go");+ let root = engine.root();+ fs::rename(root.join("Family"), root.join("Trash")).unwrap();+ let path = root.join("Trash/.meta/album_meta.json");+ let mut album = read_json(&path);+ album["albumName"] = json!("Trash");+ fs::write(path, serde_json::to_vec(&album).unwrap()).unwrap();+ engine.catalog.albums.remove(&FAMILY);+ engine+ .catalog+ .files+ .retain(|(album, _), _| *album != FAMILY);+ if keep_unknown {+ fs::write(root.join("Trash/user.txt"), b"keep").unwrap();+ }+ fs::write(root.join("Trash/Photo.jpg"), b"historical local edit").unwrap();+ assert!(engine.run(true, &[]).unwrap().complete);+ assert_eq!(+ fs::read(root.join("Trash/Trash-1/Photo.jpg")).unwrap(),+ b"historical local edit"+ );+ if keep_unknown {+ assert_eq!(fs::read(root.join("Trash-1/user.txt")).unwrap(), b"keep");+ } else {+ assert!(!root.join("Trash-1").exists());+ }+ assert!(root.join("Trash/Trash-1/.meta/Photo.jpg.json").is_file());+ assert!(!root.join("Trash/Trash-1/metadata.json").exists());+ assert!(!root.join("Trash-1/.meta/album_meta.json").exists());+ assert_eq!(+ count(+ &engine.db(),+ &format!("SELECT count(*) FROM placements WHERE album={FAMILY}")+ ),+ 0+ );+ }+}++#[test]+fn adoption_bounds_failure_examples() {+ let server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, "desktop");+ let root = engine.root();+ let path = root.join("export_status.json");+ let mut manifest = read_json(&path);+ for id in 100..113 {+ let mut remote = engine.catalog.files[&(FAMILY, PHOTO)].clone();+ remote["id"] = json!(id);+ engine.catalog.files.insert((FAMILY, id), remote);+ let name = format!("Bad-{id}.jpg");+ fs::write(root.join("Family").join(&name), b"bad original").unwrap();+ manifest["fileExportNames"][format!("{id}_{FAMILY}_100")] = json!(name);+ }+ fs::write(&path, serde_json::to_vec(&manifest).unwrap()).unwrap();+ let before = snapshot(&root);+ let error = engine.run(true, &[]).err().unwrap().to_string();+ assert!(error.contains("13 failing items"), "{error}");+ assert!(error.contains("3 more omitted"), "{error}");+ assert_eq!(snapshot(&root), before);++ manifest["collectionExportNames"][FAMILY.to_string()] = json!("../outside");+ fs::write(&path, serde_json::to_vec(&manifest).unwrap()).unwrap();+ let before = snapshot(&root);+ let error = engine.run(true, &[]).err().unwrap().to_string();+ assert!(error.contains("1 failing items"), "{error}");+ assert!(+ !error.contains("file has no valid album mapping"),+ "{error}"+ );+ assert_eq!(snapshot(&root), before);++ manifest["fileExportNames"]["900_77_100"] = json!("Missing.jpg");+ fs::write(&path, serde_json::to_vec(&manifest).unwrap()).unwrap();+ let before = snapshot(&root);+ let error = engine.run(true, &[]).err().unwrap().to_string();+ assert!(error.contains("2 failing items"), "{error}");+ assert!(error.contains("file has no valid album mapping"), "{error}");+ assert_eq!(snapshot(&root), before);+}++#[test]+fn adoption_rejects_missing_album_identity_wrong_account_and_escaping_paths() {+ for case in ["orphan", "account", "escape"] {+ let server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, "go");+ let root = engine.root();+ match case {+ "orphan" => {+ fs::remove_file(root.join("Travel/.meta/album_meta.json")).unwrap();+ }+ "account" => {+ let path = root.join("Family/.meta/album_meta.json");+ let mut value = read_json(&path);+ value["accountOwnerIDs"] = json!([42]);+ fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();+ }+ _ => {+ let path = root.join("Family/.meta/Photo.jpg.json");+ let mut value = read_json(&path);+ value["info"]["fileNames"] = json!(["../outside.jpg"]);+ fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();+ }+ }+ let before = snapshot(&root);+ let error = engine.run(true, &[]).err().unwrap().to_string();+ assert_eq!(snapshot(&root), before);+ if case == "orphan" {+ assert!(error.contains("without album_meta.json"), "{error}");+ }+ }+}++#[test]+fn adoption_preserves_unknown_meta_files_and_records_empty_go_albums() {+ let server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, "go");+ let root = engine.root();+ fs::create_dir(root.join("Personal")).unwrap();+ fs::write(root.join("Personal/.meta"), b"keep unrelated file").unwrap();+ fs::create_dir_all(root.join("Empty/.meta")).unwrap();+ let mut album = read_json(&root.join("Family/.meta/album_meta.json"));+ album["id"] = json!(77);+ album["albumName"] = json!("Empty");+ fs::write(+ root.join("Empty/.meta/album_meta.json"),+ serde_json::to_vec(&album).unwrap(),+ )+ .unwrap();+ fs::write(root.join("Empty/.meta/user.txt"), b"keep").unwrap();+ engine.catalog.albums.insert(77, "Empty".into());+ assert!(engine.run(true, &[]).unwrap().complete);+ assert_eq!(+ read_json(&root.join("Empty/metadata.json"))["ente"]["albumID"],+ "77"+ );+ assert_eq!(+ fs::read(root.join("Empty/.meta/user.txt")).unwrap(),+ b"keep"+ );+ assert_eq!(+ fs::read(root.join("Personal/.meta")).unwrap(),+ b"keep unrelated file"+ );+}++#[test]+fn adoption_retains_accepted_bytes_when_the_source_layout_changes_before_conversion() {+ for producer in ["go", "desktop"] {+ let mut server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, producer);+ let root = engine.root();+ let original = fs::read(root.join("Family/Photo.jpg")).unwrap();+ let export = engine.prepare(engine.options(true, &[])).unwrap();+ drop(export);+ let (mut replacement, encrypted) =+ original_fixture(&engine.catalog.key, b"new Live component", true, true, 300);+ replacement["id"] = json!(PHOTO);+ replacement["ownerID"] = json!(USER);+ engine.catalog.files.insert((FAMILY, PHOTO), replacement);+ let requested = download(&mut server, PHOTO, &encrypted, 1);+ assert!(engine.run(true, &[]).unwrap().complete);+ requested.assert();+ assert_eq!(+ fs::read(root.join("Trash/Family/Photo.jpg")).unwrap(),+ original+ );+ assert_component(&root, "Family", "Motion.heic", b"new Live component");+ assert_component(&root, "Family", "Motion.mov", b"new Live component");+ }+}++#[cfg(unix)]+#[test]+fn adoption_rejects_symlinks_while_detecting_orphaned_go_records() {+ for directory in [false, true] {+ let server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, "go");+ let root = engine.root();+ fs::remove_file(root.join("Travel/.meta/album_meta.json")).unwrap();+ let source = root.join(if directory {+ "Travel/.meta"+ } else {+ "Travel/.meta/Photo.jpg.json"+ });+ let outside = engine.directory.path().join("outside");+ fs::rename(&source, &outside).unwrap();+ std::os::unix::fs::symlink(&outside, &source).unwrap();+ let before = snapshot(&root);+ let error = engine.run(true, &[]).err().unwrap().to_string();+ assert!(error.contains("symlink"), "{error}");+ assert_eq!(snapshot(&root), before);+ assert_eq!(fs::read_link(&source).unwrap(), outside);+ }+}++#[test]+fn adoption_recovers_native_paths_and_live_ownership_with_an_unchanged_manifest() {+ let mut server = mockito::Server::new();+ let (mut engine, downloads) = new_engine(&server, "desktop");+ let root = engine.root();+ let manifest = fs::read(root.join("export_status.json")).unwrap();+ engine.catalog.files.remove(&(FAMILY, PHOTO));+ let export = engine.prepare(engine.options(true, &[])).unwrap();+ drop(export);+ engine.catalog.albums.insert(FAMILY, "Renamed".into());+ rename(&mut engine, FAMILY, PAIR, "Photo.jpg");+ let (mut remote, _) = source(90, &engine.catalog.key, b"new original", "Pending.jpg");+ remote["ownerID"] = json!(USER);+ engine.catalog.files.insert((FAMILY, 90), remote);+ let failed = server+ .mock("GET", "/files/download/v3/90")+ .with_status(404)+ .expect(1)+ .create();+ assert!(!engine.run(true, &[]).unwrap().complete);+ failed.assert();+ assert_eq!(fs::read(root.join("export_status.json")).unwrap(), manifest);+ let original = fs::read(root.join("Renamed/Photo.jpg")).unwrap();+ assert_eq!(+ read_json(&root.join("Renamed/metadata/Photo.jpg.json"))["ente"]["fileID"],+ PAIR.to_string()+ );+ fs::remove_file(root.join("Renamed/metadata/Photo.mov.json")).unwrap();+ let path = root.join("Renamed/metadata/Live.mov.json");+ fs::write(path, b"unfinished legacy common JSON").unwrap();+ engine.catalog.files.remove(&(FAMILY, 90));+ engine.database = "recovered.db".into();+ assert!(+ engine+ .run(false, &[])+ .err()+ .unwrap()+ .is::<ente_photos_export::AdoptionRequired>()+ );+ let requests: Vec<_> = downloads+ .iter()+ .map(|(id, bytes)| download(&mut server, *id, bytes, 0))+ .collect();+ assert!(engine.run(true, &[]).unwrap().complete);+ assert_eq!(fs::read(root.join("Renamed/Photo.jpg")).unwrap(), original);+ assert_eq!(+ read_json(&root.join("Renamed/metadata/Photo.mov.json"))["ente"]["fileID"],+ PAIR.to_string()+ );+ assert_eq!(+ read_json(&root.join("Renamed/metadata/Live.mov.json"))["ente"]["fileID"],+ "10000065"+ );+ assert!(!root.join("export_status.json").exists());+ assert_eq!(+ fs::read_dir(root.join("Trash/Family"))+ .unwrap()+ .filter_map(Result::ok)+ .filter(|e| e.path().extension().is_some_and(|x| x == "jpg"))+ .count(),+ 1+ );+ for request in requests {+ request.assert();+ }+}++#[test]+fn adoption_cleanup_waits_for_new_files_and_resumes_after_its_final_database_write() {+ let mut server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, "go");+ let root = engine.root();+ let (mut remote, encrypted) = source(90, &engine.catalog.key, b"new original", "New.jpg");+ remote["ownerID"] = json!(USER);+ engine.catalog.files.insert((FAMILY, 90), remote);+ engine.catalog.albums.insert(99, "Other".into());+ let failed = server+ .mock("GET", "/files/download/v3/90")+ .with_status(404)+ .expect(1)+ .create();+ assert!(!engine.run(true, &[]).unwrap().complete);+ failed.assert();+ assert!(root.join("Family/.meta/album_meta.json").is_file());+ assert!(root.join("Family/.meta/Photo.jpg.json").is_file());+ assert!(root.join("Family/metadata/Photo.jpg.json").is_file());+ drop(failed);+ let requested = download(&mut server, 90, &encrypted, 1);+ engine.db().execute_batch("CREATE TRIGGER stop_completion BEFORE UPDATE OF bound ON export WHEN new.bound=1 BEGIN SELECT RAISE(ABORT,'interrupt after cleanup'); END;").unwrap();+ assert!(!engine.run(false, &[]).unwrap().complete);+ requested.assert();+ assert!(!root.join("Family/.meta").exists());+ assert!(!root.join("Other").exists());+ assert_eq!(count(&engine.db(), "SELECT bound FROM export"), 0);+ assert_eq!(+ count(&engine.db(), "SELECT count(*) FROM albums WHERE retained=0"),+ 3+ );+ engine+ .db()+ .execute_batch("DROP TRIGGER stop_completion")+ .unwrap();+ assert!(+ engine+ .run(false, &["Family"])+ .err()+ .unwrap()+ .to_string()+ .contains("exactly")+ );+ assert!(engine.run(false, &[]).unwrap().complete);+ assert!(!root.join("Other").exists());+ assert!(engine.run(true, &[]).unwrap().complete);+ assert!(root.join("Other/metadata.json").is_file());+}++#[test]+fn adoption_defers_wholly_missing_hashless_originals_until_after_acceptance() {+ let mut server = mockito::Server::new();+ let (mut engine, downloads) = new_engine(&server, "go");+ let previous = engine.catalog.files[&(FAMILY, PHOTO)].clone();+ let (_, documents) = files::decrypt(+ &serde_json::from_value(previous.clone()).unwrap(),+ &engine.catalog.key,+ )+ .unwrap();+ let mut value: Value = serde_json::from_slice(&documents.original).unwrap();+ value.as_object_mut().unwrap().remove("hash");+ let encrypted = blob::encrypt_json(&value, &file_key(&previous, &engine.catalog.key)).unwrap();+ engine.catalog.files.get_mut(&(FAMILY, PHOTO)).unwrap()["metadata"] = json!({"encryptedData":b64::encode(&encrypted.encrypted_data),"decryptionHeader":b64::encode(encrypted.decryption_header.as_bytes())});+ fs::remove_file(engine.root().join("Family/Photo.jpg")).unwrap();+ let absent = download(&mut server, PHOTO, &downloads[&PHOTO], 0);+ let export = engine.prepare(engine.options(true, &[])).unwrap();+ absent.assert();+ assert_eq!(+ count(+ &engine.db(),+ &format!(+ "SELECT count(*) FROM components c JOIN placements p ON p.id=c.placement WHERE p.file={PHOTO}"+ )+ ),+ 0+ );+ drop(absent);+ let requested = download(&mut server, PHOTO, &downloads[&PHOTO], 1);+ assert!(engine.execute(export).unwrap().complete);+ requested.assert();+}++#[test]+fn adoption_verifies_a_single_undeclared_live_survivor_before_preserving_it() {+ let mut server = mockito::Server::new();+ let (mut engine, downloads) = new_engine(&server, "go");+ let root = engine.root();+ let path = root.join("Family/.meta/Pair.jpg.json");+ let mut record = read_json(&path);+ record["info"]["fileNames"] = json!(["Pair.jpg"]);+ fs::write(path, serde_json::to_vec(&record).unwrap()).unwrap();+ fs::remove_file(root.join("Family/Pair.mov")).unwrap();+ let original = fs::read(root.join("Family/Pair.jpg")).unwrap();+ fs::write(root.join("Family/Pair.jpg"), b"edited survivor").unwrap();+ let before = snapshot(&root);+ assert!(engine.run(true, &[]).is_err());+ assert_eq!(snapshot(&root), before);+ fs::write(root.join("Family/Pair.jpg"), &original).unwrap();+ for (path, _) in snapshot(&root) {+ if !path.to_string_lossy().ends_with(".json")+ && path.as_path() != Path::new("Family/Pair.jpg")+ {+ fs::remove_file(root.join(path)).unwrap();+ }+ }+ let export = engine.prepare(engine.options(true, &[])).unwrap();+ assert_eq!(count(&engine.db(), "SELECT count(*) FROM components"), 0);+ let requests: Vec<_> = downloads+ .iter()+ .map(|(id, bytes)| download(&mut server, *id, bytes, 1))+ .collect();+ assert!(engine.execute(export).unwrap().complete);+ for request in requests {+ request.assert();+ }+ assert_eq!(+ fs::read(root.join("Trash/Family/Pair.jpg")).unwrap(),+ original+ );+ assert_eq!(fs::read(root.join("Family/Pair.jpg")).unwrap(), original);+}++#[test]+fn adoption_of_one_recorded_album_includes_new_photos_but_not_unrelated_albums() {+ let mut server = mockito::Server::new();+ let (mut engine, _) = new_engine(&server, "desktop");+ let root = engine.root();+ fs::remove_dir_all(root.join("Travel")).unwrap();+ fs::remove_dir_all(root.join("Live Pair")).unwrap();+ let path = root.join("export_status.json");+ let mut manifest = read_json(&path);+ manifest["collectionExportNames"]+ .as_object_mut()+ .unwrap()+ .retain(|id, _| id == &FAMILY.to_string());+ manifest["fileExportNames"]+ .as_object_mut()+ .unwrap()+ .retain(|id, _| id.split('_').nth(1) == Some(FAMILY.to_string().as_str()));+ fs::write(path, serde_json::to_vec(&manifest).unwrap()).unwrap();+ let (mut remote, encrypted) = source(90, &engine.catalog.key, b"new photo", "New.jpg");+ remote["ownerID"] = json!(USER);+ engine.catalog.files.insert((FAMILY, 90), remote);+ let requested = download(&mut server, 90, &encrypted, 1);+ let export = engine.prepare(engine.options(true, &["Family"])).unwrap();+ drop(export);+ assert!(engine.run(true, &[]).unwrap().complete);+ requested.assert();+ assert_component(&root, "Family", "New.jpg", b"new photo");+ assert!(!root.join("Travel").exists());+ assert!(!root.join("Live Pair").exists());+}diff --git a/rust/apps/cli-next/tests/support/export_engine.rs b/rust/apps/cli-next/tests/support/export_engine.rsnew file mode 100644index 0000000000..80cd583b4a--- /dev/null+++ b/rust/apps/cli-next/tests/support/export_engine.rs@@ -0,0 +1,160 @@+use super::*;+use anyhow::{Result, bail};+use ente_core::{Session, crypto::SecretKey, http::ApiConfig};+use ente_photos::{collections, files, source::Documents};+use ente_photos_export::{CollectionEntry, Export, FileEntry, Options, Source, Summary};+use std::{collections::BTreeMap, path::PathBuf};+use tokio::sync::watch;++pub(super) struct Catalog {+ pub user: i64,+ pub key: Key,+ pub albums: BTreeMap<i64, String>,+ pub files: BTreeMap<(i64, i64), Value>,+ pub failed_album: Option<i64>,+}++impl Catalog {+ fn album(&self, id: i64) -> CollectionEntry {+ let name = self.albums[&id].clone();+ CollectionEntry {+ id,+ name: Some(name.clone()),+ favorites: false,+ value: Ok((+ collections::Collection {+ id,+ key: Key::from_bytes(*self.key.as_bytes()),+ name,+ kind: collections::Kind::Album,+ visibility: collections::Visibility::Visible,+ owner_id: self.user,+ updated_at_micros: 100,+ },+ Documents {+ original: Vec::new(),+ public: None,+ private: None,+ shared: None,+ },+ )),+ }+ }++ fn entry(&self, record: &Value) -> Result<FileEntry> {+ let remote = serde_json::from_value(record.clone())?;+ let (key, documents) = files::decrypt(&remote, &self.key)?;+ let file = files::interpret(&remote, &key, &documents, self.user)?;+ Ok(FileEntry {+ id: file.id,+ value: Ok((file, documents)),+ })+ }+}++impl Source for Catalog {+ async fn refresh_catalog(&mut self, _: &Session) -> Result<()> {+ Ok(())+ }+ fn catalog_page(&self, after: i64) -> Result<Vec<CollectionEntry>> {+ Ok(self+ .albums+ .keys()+ .copied()+ .filter(|id| *id > after)+ .map(|id| self.album(id))+ .collect())+ }+ async fn refresh_album(&mut self, _: &Session, id: i64) -> Result<CollectionEntry> {+ if self.failed_album == Some(id) {+ bail!("fixture source refresh failed");+ }+ Ok(self.album(id))+ }+ fn file_page(&self, album: i64, after: i64) -> Result<Vec<FileEntry>> {+ self.files+ .iter()+ .filter(|((a, f), _)| *a == album && *f > after)+ .map(|(_, record)| self.entry(record))+ .collect()+ }+ fn favorite_page(&self, _: i64) -> Result<Vec<i64>> {+ Ok(Vec::new())+ }+ fn file(&self, album: i64, id: i64) -> Result<FileEntry> {+ self.entry(&self.files[&(album, id)])+ }+}++pub(super) struct Engine {+ pub directory: tempfile::TempDir,+ session: Session,+ pub catalog: Catalog,+ pub database: String,+}++impl Engine {+ pub fn new(server: &mockito::ServerGuard, catalog: Catalog) -> Self {+ Self {+ directory: tempfile::tempdir().unwrap(),+ session: Session::new(+ ApiConfig::new(server.url()),+ catalog.user,+ Key::generate(),+ Key::generate(),+ SecretKey::generate(),+ )+ .unwrap(),+ catalog,+ database: "export.db".into(),+ }+ }++ pub fn root(&self) -> PathBuf {+ self.directory.path().join("photos")+ }++ pub fn db(&self) -> rusqlite::Connection {+ rusqlite::Connection::open(self.directory.path().join(&self.database)).unwrap()+ }++ pub fn options(&self, adopt: bool, albums: &[&str]) -> Options {+ Options {+ albums: albums.iter().map(|s| (*s).into()).collect(),+ exclude_albums: Vec::new(),+ adopt,+ jobs: std::num::NonZeroUsize::new(1).unwrap(),+ }+ }++ pub fn prepare(&mut self, options: Options) -> Result<Export> {+ let mut export = Export::open(&self.root(), &self.session.master_key, options)?;+ let (_send, mut cancel) = watch::channel(false);+ let db = self.db();+ tokio::runtime::Runtime::new()?.block_on(export.prepare(+ db,+ |path| {+ let db = rusqlite::Connection::open(path)?;+ db.pragma_update(None, "key", b64::encode(Key::generate().as_bytes()))?;+ Ok(db)+ },+ &mut self.catalog,+ &self.session,+ &mut cancel,+ &|line| eprintln!("{line}"),+ ))?;+ Ok(export)+ }++ pub fn execute(&self, mut export: Export) -> Result<Summary> {+ let (send, cancel) = watch::channel(false);+ tokio::runtime::Runtime::new()?.block_on(async {+ export.execute(&self.session, &send, &cancel, &|line| eprintln!("{line}"))+ })+ }++ pub fn run(&mut self, adopt: bool, albums: &[&str]) -> Result<Summary> {+ let export = self.prepare(self.options(adopt, albums))?;+ self.execute(export)+ }+}diff --git a/rust/apps/cli-next/tests/support/export_maintenance.rs b/rust/apps/cli-next/tests/support/export_maintenance.rsindex 2a7f25c6ee..1e58775d1b 100644--- a/rust/apps/cli-next/tests/support/export_maintenance.rs+++ b/rust/apps/cli-next/tests/support/export_maintenance.rs@@ -1,6 +1,15 @@use super::*;use crate::export_process::ExportChild;+#[path = "export_engine.rs"]+mod engine;++#[path = "export_media.rs"]+mod media;++#[path = "export_adoption.rs"]+mod adoption;+fn listing(server: &mut mockito::ServerGuard, albums: Value) -> mockito::Mock {server.mock("GET", "/collections/v2")@@ -1909,7 +1918,7 @@ fn export_recovers_cross_album_publication_after_album_rename() {}#[test]-fn export_repairs_missing_or_changed_cross_album_publications() {+fn export_restores_missing_cross_album_publications_and_preserves_edits() {for (owned, missing) in [(false, true), (false, false), (true, true), (true, false)] {let mut fixture = interrupted_cross_album_publication(owned, 2);let root = fixture.root.path();@@ -1925,7 +1934,7 @@ fn export_repairs_missing_or_changed_cross_album_publications() {json!([revision(collection(2, "Renamed", &fixture.key), 80)]),);page(&mut fixture.server, 2, 50, json!([]), false).create();- if !missing && !owned {+ if !missing {let result = run(&fixture.home, root, &["--album", "Renamed"], false);assert!(result["conflicts"].as_i64().unwrap() > 0);assert_eq!(fs::read(root.join("Renamed/Photo.jpg")).unwrap(), b"wrong");@@ -2311,12 +2320,6 @@ fn export_automatically_retries_saved_selected_album_and_file_failures() {1);assert_component(&root, "First", "Photo.jpg", b"original");- assert!(- !home- .run(&["photos", "export", root.to_str().unwrap(), "--retry-failed"])- .status- .success()- );first_page.assert();fetched.assert();}diff --git a/rust/apps/cli-next/tests/support/export_media.rs b/rust/apps/cli-next/tests/support/export_media.rsnew file mode 100644index 0000000000..dd3723567a--- /dev/null+++ b/rust/apps/cli-next/tests/support/export_media.rs@@ -0,0 +1,416 @@+use super::engine::{Catalog, Engine};+use super::*;+use std::collections::BTreeMap;++fn new_engine(server: &mockito::ServerGuard, key: Key, record: Value) -> Engine {+ let id = record["id"].as_i64().unwrap();+ Engine::new(+ server,+ Catalog {+ user: 9007199254740993,+ key,+ albums: BTreeMap::from([(1, "First".into()), (2, "Second".into())]),+ files: BTreeMap::from([((1, id), record.clone()), ((2, id), record)]),+ failed_album: None,+ },+ )+}++fn run(engine: &mut Engine, complete: bool) -> Value {+ let result = engine.run(false, &[]).unwrap();+ assert_eq!(result.complete, complete);+ result.json+}++#[test]+fn engine_preserves_local_media_and_restores_missing_copies() {+ let mut server = mockito::Server::new();+ let key = Key::generate();+ let (file, bytes) = original_fixture(&key, b"original", false, true, 10);+ let fetched = download(&mut server, 10, &bytes, 1);+ let mut engine = new_engine(&server, key, file.clone());+ run(&mut engine, true);+ let root = engine.root();+ let photo = root.join("First/Photo.jpg");+ let sidecar = root.join("First/metadata/Photo.jpg.json");+ let baseline = fs::read(&sidecar).unwrap();+ let hash = read_json(&sidecar)["ente"]["components"][0]["hash"].clone();+ fs::write(&photo, b"embedded EXIF edit").unwrap();+ fs::write(+ root.join("Second/metadata/Photo.jpg.json"),+ b"{\"custom\":true}",+ )+ .unwrap();+ let changed = public(+ file,+ &engine.catalog.key,+ json!({"caption":"current caption"}),+ 30,+ );+ engine.catalog.files = BTreeMap::from([((1, 10), changed.clone()), ((2, 10), changed)]);+ for _ in 0..2 {+ let result = run(&mut engine, false);+ assert_eq!(result["conflicts"], 1);+ assert_eq!(+ result["copies"],+ json!({"expected":2,"completed":1,"pending":1})+ );+ assert_eq!(fs::read(&photo).unwrap(), b"embedded EXIF edit");+ assert_eq!(fs::read(&sidecar).unwrap(), baseline);+ assert_eq!(+ engine+ .db()+ .query_row(+ "SELECT hash FROM components WHERE folder='active:1'",+ [],+ |r| r.get::<_, String>(0),+ )+ .unwrap(),+ hash.as_str().unwrap()+ );+ let metadata = read_json(&root.join("Second/metadata/Photo.jpg.json"));+ assert_eq!(metadata["description"], "current caption");+ assert!(metadata.get("custom").is_none());+ }+ fs::remove_file(&photo).unwrap();+ run(&mut engine, true);+ assert_component(&root, "First", "Photo.jpg", b"original");+ fs::write(&photo, b"local edit before removal").unwrap();+ engine.catalog.files.retain(|(album, _), _| *album != 1);+ run(&mut engine, true);+ assert_eq!(+ fs::read(root.join("Trash/First/Photo.jpg")).unwrap(),+ b"local edit before removal"+ );+ assert_eq!(+ read_json(&root.join("Trash/First/metadata/Photo.jpg.json"))["ente"]["components"][0]["hash"],+ hash+ );+ assert_component(&root, "Second", "Photo.jpg", b"original");+ engine.catalog.files.clear();+ run(&mut engine, true);+ assert_component(&root, "Trash/Second", "Photo.jpg", b"original");+ run(&mut engine, true);+ assert_eq!(+ count(+ &engine.db(),+ "SELECT count(*) FROM placements WHERE retained=1"+ ),+ 2+ );+ fetched.assert();+}++#[test]+fn engine_checks_previous_bytes_before_source_and_layout_replacement() {+ for (before_live, after_live, hashed) in [+ (false, false, true),+ (false, false, false),+ (true, true, true),+ (false, true, true),+ (true, false, true),+ ] {+ let mut server = mockito::Server::new();+ let key = Key::generate();+ let (old, bytes) = original_fixture(&key, b"old", before_live, hashed, 10);+ let fetched = download(&mut server, 10, &bytes, 1);+ let mut engine = new_engine(&server, key, old);+ run(&mut engine, true);+ fetched.assert();+ fetched.remove();+ let root = engine.root();+ let name = if before_live {+ "Motion.mov"+ } else {+ "Photo.jpg"+ };+ let photo = root.join("First").join(name);+ let modified = fs::metadata(&photo).unwrap().modified().unwrap();+ let baseline = fs::read(root.join("First/metadata").join(format!("{name}.json"))).unwrap();+ fs::write(&photo, b"bad").unwrap();+ fs::File::options()+ .write(true)+ .open(&photo)+ .unwrap()+ .set_modified(modified)+ .unwrap();+ run(&mut engine, true);+ let (new, bytes) = original_fixture(&engine.catalog.key, b"new", after_live, hashed, 30);+ engine.catalog.files = BTreeMap::from([((1, 10), new.clone()), ((2, 10), new)]);+ let fetched = download(&mut server, 10, &bytes, 1);+ let result = run(&mut engine, false);+ assert_eq!(result["conflicts"], 1);+ assert_eq!(result["copies"]["completed"], 1);+ assert_eq!(fs::read(&photo).unwrap(), b"bad");+ assert_eq!(fs::metadata(&photo).unwrap().modified().unwrap(), modified);+ assert_eq!(+ fs::read(root.join("First/metadata").join(format!("{name}.json"))).unwrap(),+ baseline+ );+ if before_live {+ assert_component(&root, "First", "Motion.heic", b"old");+ }+ assert!(!root.join("Trash/First").exists());+ let names: &[&str] = if after_live {+ &["Motion.heic", "Motion.mov"]+ } else {+ &["Photo.jpg"]+ };+ for name in names {+ assert_component(&root, "Second", name, b"new");+ }+ if before_live == after_live {+ fs::write(&photo, b"old").unwrap();+ } else {+ fs::remove_file(&photo).unwrap();+ }+ run(&mut engine, true);+ for name in names {+ assert_component(&root, "First", name, b"new");+ }+ if before_live != after_live {+ assert!(!root.join("Trash/First").join(name).exists());+ assert_eq!(+ read_json(+ &root+ .join("Trash/First/metadata")+ .join(format!("{name}.json"))+ )["ente"]["components"]+ .as_array()+ .unwrap()+ .iter()+ .find(|component| component["path"] == name)+ .unwrap()["hash"],+ b64::encode(&hash::hash(b"old", Some(64), None).unwrap())+ );+ if before_live {+ assert_component(&root, "Trash/First", "Motion.heic", b"old");+ }+ }+ fetched.assert();+ }+}++#[test]+fn engine_restores_a_new_layout_after_missing_interrupted_retention() {+ for old_live in [false, true] {+ let mut server = mockito::Server::new();+ let key = Key::generate();+ let (old, bytes) = original_fixture(&key, b"old", old_live, true, 10);+ let fetched = download(&mut server, 10, &bytes, 1);+ let mut engine = new_engine(&server, key, old);+ run(&mut engine, true);+ fetched.assert();+ fetched.remove();+ let db = engine.db();+ db.execute_batch("CREATE TRIGGER stop_retention BEFORE UPDATE ON pending WHEN old.album=1 AND old.retained=1 AND json_type(old.action,'$.Move') IS NOT NULL AND new.action='null' BEGIN SELECT RAISE(ABORT,'stop after retention move'); END;").unwrap();+ let (new, bytes) = original_fixture(&engine.catalog.key, b"new", !old_live, true, 30);+ engine.catalog.files = BTreeMap::from([((1, 10), new.clone()), ((2, 10), new)]);+ let fetched = download(&mut server, 10, &bytes, 2);+ run(&mut engine, false);+ let root = engine.root();+ let old_name = if old_live { "Motion.heic" } else { "Photo.jpg" };+ fs::remove_file(root.join("Trash/First").join(old_name)).unwrap();+ db.execute_batch("DROP TRIGGER stop_retention;").unwrap();+ if old_live {+ fs::write(root.join("First/Motion.mov"), b"local edit").unwrap();+ assert_eq!(run(&mut engine, false)["conflicts"], 1);+ assert_eq!(+ fs::read(root.join("First/Motion.mov")).unwrap(),+ b"local edit"+ );+ assert!(!root.join("First/Photo.jpg").exists());+ fs::remove_file(root.join("First/Motion.mov")).unwrap();+ }+ run(&mut engine, true);+ let new_names: &[&str] = if old_live {+ &["Photo.jpg"]+ } else {+ &["Motion.heic", "Motion.mov"]+ };+ for folder in ["First", "Second"] {+ for name in new_names {+ assert_component(&root, folder, name, b"new");+ }+ }+ assert!(!root.join("Trash/First").join(old_name).exists());+ assert_eq!(+ read_json(+ &root+ .join("Trash/First/metadata")+ .join(format!("{old_name}.json"))+ )["ente"]["components"][0]["hash"],+ b64::encode(&hash::hash(b"old", Some(64), None).unwrap())+ );+ assert_eq!(run(&mut engine, true)["changes"]["retained"], 0);+ assert_eq!(+ count(+ &db,+ "SELECT count(*) FROM placements WHERE album=1 AND retained=1"+ ),+ 1+ );+ fetched.assert();+ }+}++#[test]+fn engine_checks_both_live_components_when_only_one_source_component_changes() {+ let mut server = mockito::Server::new();+ let key = Key::generate();+ let (old, bytes) = original_fixture(&key, b"old", true, true, 10);+ let fetched = download(&mut server, 10, &bytes, 1);+ let mut engine = new_engine(&server, key, old);+ run(&mut engine, true);+ fetched.assert();+ fetched.remove();+ let root = engine.root();+ let photo = root.join("First/Motion.heic");+ let modified = fs::metadata(&photo).unwrap().modified().unwrap();+ fs::write(&photo, b"bad").unwrap();+ fs::File::options()+ .write(true)+ .open(&photo)+ .unwrap()+ .set_modified(modified)+ .unwrap();+ let mut archive = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));+ for (name, bytes) in [("image.heic", b"old"), ("video.mov", b"new")] {+ archive+ .start_file(name, zip::write::SimpleFileOptions::default())+ .unwrap();+ archive.write_all(bytes).unwrap();+ }+ let archive = archive.finish().unwrap().into_inner();+ let (mut new, bytes) = source(10, &engine.catalog.key, &archive, "Motion.heic");+ let image = b64::encode(&hash::hash(b"old", Some(64), None).unwrap());+ let video = b64::encode(&hash::hash(b"new", Some(64), None).unwrap());+ let metadata = blob::encrypt_json(+ &json!({"title":"Motion.heic","fileType":2,"creationTime":1_700_000_000_123_456i64,"hash":format!("{image}:{video}")}),+ &file_key(&new, &engine.catalog.key),+ ).unwrap();+ new["metadata"] = json!({"encryptedData":b64::encode(&metadata.encrypted_data),"decryptionHeader":b64::encode(metadata.decryption_header.as_bytes())});+ let new = revision(new, 30);+ engine.catalog.files = BTreeMap::from([((1, 10), new.clone()), ((2, 10), new)]);+ let fetched = download(&mut server, 10, &bytes, 1);+ assert_eq!(run(&mut engine, false)["conflicts"], 1);+ assert_eq!(fs::read(&photo).unwrap(), b"bad");+ assert_component(&root, "First", "Motion.mov", b"old");+ assert_component(&root, "Second", "Motion.heic", b"old");+ assert_component(&root, "Second", "Motion.mov", b"new");+ fs::write(&photo, b"old").unwrap();+ run(&mut engine, true);+ assert_component(&root, "First", "Motion.heic", b"old");+ assert_component(&root, "First", "Motion.mov", b"new");+ fetched.assert();+}++#[test]+fn engine_recovery_checks_live_siblings_before_acknowledging_publication() {+ let mut server = mockito::Server::new();+ let key = Key::generate();+ let (old, bytes) = original_fixture(&key, b"old", true, true, 10);+ let fetched = download(&mut server, 10, &bytes, 1);+ let mut engine = new_engine(&server, key, old);+ run(&mut engine, true);+ fetched.assert();+ fetched.remove();+ let db = engine.db();+ db.execute_batch("CREATE TRIGGER stop_ack BEFORE UPDATE ON pending WHEN old.album=1 AND json_extract(old.action,'$.Publish.output.Media.role')='image' AND new.action='null' BEGIN SELECT RAISE(ABORT,'stop after image publication'); END;").unwrap();+ let (new, bytes) = original_fixture(&engine.catalog.key, b"new", true, true, 30);+ engine.catalog.files = BTreeMap::from([((1, 10), new.clone()), ((2, 10), new)]);+ let fetched = download(&mut server, 10, &bytes, 2);+ run(&mut engine, false);+ let root = engine.root();+ assert_eq!(fs::read(root.join("First/Motion.heic")).unwrap(), b"new");+ fs::write(root.join("First/Motion.mov"), b"edited video").unwrap();+ let pending: String = db+ .query_row(+ "SELECT action FROM pending WHERE album=1 AND file=10",+ [],+ |r| r.get(0),+ )+ .unwrap();+ db.execute_batch("DROP TRIGGER stop_ack;").unwrap();+ let result = run(&mut engine, false);+ assert_eq!(result["conflicts"], 1);+ assert_eq!(+ fs::read(root.join("First/Motion.mov")).unwrap(),+ b"edited video"+ );+ assert_eq!(+ db.query_row(+ "SELECT action FROM pending WHERE album=1 AND file=10",+ [],+ |r| r.get::<_, String>(0)+ )+ .unwrap(),+ pending+ );+ assert_component(&root, "Second", "Motion.heic", b"new");+ assert_component(&root, "Second", "Motion.mov", b"new");+ fs::remove_file(root.join("First/Motion.mov")).unwrap();+ run(&mut engine, true);+ assert_component(&root, "First", "Motion.heic", b"new");+ assert_component(&root, "First", "Motion.mov", b"new");+ fetched.assert();+}++#[test]+fn engine_recovery_preserves_edits_at_either_move_endpoint() {+ for (moved, removed) in [(false, false), (true, false), (false, true), (true, true)] {+ let mut server = mockito::Server::new();+ let key = Key::generate();+ let (file, bytes) = original_fixture(&key, b"original", false, true, 10);+ let fetched = download(&mut server, 10, &bytes, 1);+ let mut engine = new_engine(&server, key, file.clone());+ run(&mut engine, true);+ let db = engine.db();+ db.execute_batch("CREATE TRIGGER stop_move BEFORE UPDATE ON pending WHEN old.album=1 AND json_type(old.action,'$.Move') IS NOT NULL AND new.action='null' BEGIN SELECT RAISE(ABORT,'stop after media move'); END;").unwrap();+ let renamed = public(+ file,+ &engine.catalog.key,+ json!({"editedName":"Renamed.jpg"}),+ 30,+ );+ engine.catalog.files = BTreeMap::from([((1, 10), renamed.clone()), ((2, 10), renamed)]);+ run(&mut engine, false);+ let root = engine.root();+ let old = root.join("First/Photo.jpg");+ let new = root.join("First/Renamed.jpg");+ if !moved {+ fs::rename(&new, &old).unwrap();+ }+ let changed = if moved { &new } else { &old };+ fs::write(changed, b"local edit").unwrap();+ db.execute_batch("DROP TRIGGER stop_move;").unwrap();+ if removed {+ engine.catalog.files.retain(|(album, _), _| *album != 1);+ run(&mut engine, true);+ assert_eq!(+ fs::read(root.join("Trash/First/Photo.jpg")).unwrap(),+ b"local edit"+ );+ assert_eq!(+ read_json(&root.join("Trash/First/metadata/Photo.jpg.json"))["ente"]["components"]+ [0]["hash"],+ b64::encode(&hash::hash(b"original", Some(64), None).unwrap())+ );+ assert_component(&root, "Second", "Renamed.jpg", b"original");+ fetched.assert();+ continue;+ }+ let result = run(&mut engine, false);+ assert_eq!(result["conflicts"], 1);+ assert_eq!(fs::read(changed).unwrap(), b"local edit");+ assert_component(&root, "Second", "Renamed.jpg", b"original");+ assert_eq!(+ read_json(&root.join("First/metadata/Photo.jpg.json"))["ente"]["components"][0]["hash"],+ b64::encode(&hash::hash(b"original", Some(64), None).unwrap())+ );+ fs::write(changed, b"original").unwrap();+ run(&mut engine, true);+ assert_component(&root, "First", "Renamed.jpg", b"original");+ fetched.assert();+ }+}diff --git a/rust/apps/cli-next/tests/support/museum.rs b/rust/apps/cli-next/tests/support/museum.rsindex 1177c4a7d9..26c54adc60 100644--- a/rust/apps/cli-next/tests/support/museum.rs+++ b/rust/apps/cli-next/tests/support/museum.rs@@ -370,11 +370,17 @@ async fn exercise(origin: String) -> TestResult {assert!(file_record["ente"].get("visibility").is_none());let adopted = TestHome::new();adopted.write_vault(&home.read_vault());- fs::write(- shared_export.join("Monsoon π§/original.jpg"),- vec![0; original.len()],- )- .unwrap();+ let shared_original = shared_export.join("Monsoon π§/original.jpg");+ fs::write(&shared_original, vec![0; original.len()]).unwrap();+ let output = adopted.run(&[+ "photos",+ "export",+ shared_export.to_str().unwrap(),+ "--adopt",+ ]);+ assert!(failure(&output).contains("locally changed media"));+ assert_eq!(fs::read(&shared_original).unwrap(), vec![0; original.len()]);+ fs::remove_file(&shared_original).unwrap();adopted.json(&["photos","export",diff --git a/rust/crates/photos-export/Cargo.toml b/rust/crates/photos-export/Cargo.tomlindex b3d3deb980..f0ea90eefe 100644--- a/rust/crates/photos-export/Cargo.toml+++ b/rust/crates/photos-export/Cargo.toml@@ -13,6 +13,7 @@ icu_casemap.workspace = truerusqlite.workspace = trueserde.workspace = trueserde_json.workspace = true+tempfile.workspace = truethiserror.workspace = truetokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }unicode-normalization.workspace = truediff --git a/rust/crates/photos-export/src/adopt.rs b/rust/crates/photos-export/src/adopt.rsindex a0f7788bff..a43e3912a9 100644--- a/rust/crates/photos-export/src/adopt.rs+++ b/rust/crates/photos-export/src/adopt.rs@@ -1,50 +1,230 @@use std::{fs, path::Path};-use crate::metadata::{Component, Role};use anyhow::{Context, Result, ensure};+use rusqlite::{Connection, params};+use serde::{Deserialize, Serialize};use serde_json::Value;use uuid::Uuid;-use super::{+use crate::{+ legacy::read::Desktop,+ metadata::{Component, Role},names,- store::{Album, JsonRecord, Location, Placement, Properties, Store},+ store::{self, Album, JsonRecord, Location, Placement, Properties, Store},};-pub fn scan(root: &Path, store: &Store) -> Result<()> {- store- .db- .execute_batch(- "DELETE FROM components; DELETE FROM json_records; DELETE FROM placements; DELETE FROM albums; DELETE FROM pending; DELETE FROM temporaries;",+#[derive(Serialize, Deserialize)]+pub struct AlbumRecord {+ pub album: Album,+ pub json: Option<JsonRecord>,+ pub legacy: Option<crate::legacy::read::AlbumInfo>,+}++#[derive(Serialize, Deserialize)]+pub struct Item {+ pub placement: Placement,+ pub components: Vec<store::Component>,+ pub json: Vec<JsonRecord>,+ pub legacy: Option<crate::legacy::read::Record>,+ pub leftover: bool,+}++pub struct Index {+ pub db: Connection,+ _directory: tempfile::TempDir,+}++impl Index {+ pub fn new(open: impl FnOnce(&Path) -> Result<Connection>) -> Result<Self> {+ let directory = tempfile::tempdir()?;+ let db = open(&directory.path().join("index.db"))?;+ db.execute_batch(+ "PRAGMA temp_store=MEMORY; PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF; BEGIN;",)?;+ db.execute_batch("+CREATE TABLE adoption_album(key TEXT PRIMARY KEY,id INTEGER NOT NULL,retained INTEGER NOT NULL,parent TEXT NOT NULL,folded TEXT NOT NULL,record TEXT NOT NULL);+CREATE UNIQUE INDEX adoption_album_id ON adoption_album(id) WHERE retained=0;+CREATE UNIQUE INDEX adoption_album_path ON adoption_album(parent,folded);+CREATE TABLE adoption_item(key TEXT PRIMARY KEY,album_key TEXT NOT NULL,file_id INTEGER NOT NULL,active INTEGER NOT NULL,origin TEXT NOT NULL,anchor TEXT NOT NULL,record TEXT NOT NULL);+CREATE UNIQUE INDEX adoption_membership ON adoption_item(album_key,file_id) WHERE active=1;+CREATE INDEX adoption_retained ON adoption_item(album_key,file_id,anchor);+CREATE INDEX adoption_file ON adoption_item(file_id,key);+CREATE TABLE adoption_path(path TEXT PRIMARY KEY,album_key TEXT NOT NULL,stem TEXT NOT NULL,kind TEXT NOT NULL,item_key TEXT NOT NULL);+CREATE INDEX adoption_stem ON adoption_path(album_key,stem);+")?;+ Ok(Self {+ db,+ _directory: directory,+ })+ }+}++pub fn save_album(index: &Index, record: &AlbumRecord) -> Result<()> {+ let a = &record.album;+ let (parent, name) = a.path.rsplit_once('/').unwrap_or(("", &a.path));+ index.db.execute("INSERT INTO adoption_album VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(key) DO UPDATE SET parent=excluded.parent,folded=excluded.folded,record=excluded.record",params![a.key,a.id,a.retained,parent,names::folded(name),serde_json::to_string(record)?])?;+ Ok(())+}++pub fn save_item(index: &Index, album: &Album, item: &Item) -> Result<()> {+ index.db.execute("INSERT INTO adoption_item VALUES(?1,?2,?3,?4,?5,?6,?7) ON CONFLICT(key) DO UPDATE SET record=excluded.record",params![item.placement.id,album.key,item.placement.file,!item.placement.retained,if item.legacy.is_some(){"legacy"}else{"native"},item.components.first().map(|c|c.location.name.as_str()).unwrap_or(""),serde_json::to_string(item)?])?;+ Ok(())+}++pub fn claim(index: &Index, album: &Album, item: &str, name: &str, kind: &str) -> Result<()> {+ let path = names::folded(&format!("{}/{name}", album.path));+ let stem = names::folded(names::split(name, false).0);+ let overlap: bool=index.db.query_row("SELECT EXISTS(SELECT 1 FROM adoption_path WHERE item_key<>?1 AND (path=?2 OR (album_key=?3 AND stem=?4 AND kind<>'' AND ?5<>'' AND NOT(kind=?5 AND kind IN ('image','video')))))",params![item,path,album.key,stem,kind],|r|r.get(0))?;+ ensure!(+ !overlap,+ crate::Conflict(format!("overlapping media paths at {}/{name}", album.path))+ );+ index.db.execute("INSERT INTO adoption_path VALUES(?1,?2,?3,?4,?5) ON CONFLICT(path) DO UPDATE SET kind=excluded.kind",params![path,album.key,stem,kind,item])?;+ Ok(())+}++pub fn scan(root: &Path, index: &Index, desktop: Option<&Desktop>) -> Result<()> {for entry in fs::read_dir(root)? {let entry = entry?;- if entry.file_name() == "Trash" {+ if entry.file_name() == "Trash"+ && !entry.path().join(".meta/album_meta.json").try_exists()?+ {ensure!(entry.file_type()?.is_dir(),- super::Conflict("at Trash: expected directory".into())+ crate::Conflict("at Trash: expected directory".into()));for entry in fs::read_dir(entry.path())? {let entry = entry?;if entry.file_type()?.is_dir() {- album(root, store, &entry.path(), true)?;+ scan_album(root, index, &entry.path(), true, desktop)?;}}} else if entry.file_type()?.is_dir() {- album(root, store, &entry.path(), false)?;+ scan_album(root, index, &entry.path(), false, desktop)?;}}Ok(())}-fn album(root: &Path, store: &Store, directory: &Path, retained: bool) -> Result<()> {+fn scan_album(+ root: &Path,+ index: &Index,+ directory: &Path,+ retained: bool,+ desktop: Option<&Desktop>,+) -> Result<()> {+ let Some(record) = parse_album(root, directory, retained)? else {+ return Ok(());+ };+ let album = &record.album;+ let (parent, name) = album.path.rsplit_once('/').unwrap_or(("", &album.path));+ let (duplicate_id, duplicate_path): (bool, bool)=index.db.query_row("SELECT EXISTS(SELECT 1 FROM adoption_album WHERE id=?1 AND retained=0 AND ?2=0), EXISTS(SELECT 1 FROM adoption_album WHERE parent=?3 AND folded=?4)",params![album.id,retained,parent,names::folded(name)],|r|Ok((r.get(0)?,r.get(1)?)))?;+ ensure!(+ !duplicate_id,+ crate::Conflict(format!("active album ID {}", album.id))+ );+ ensure!(+ !duplicate_path,+ crate::Conflict(format!("duplicate native album path {}", album.path))+ );+ save_album(index, &record)?;+ let metadata = names::check(root, &format!("{}/metadata", album.path))?;+ if !metadata.try_exists()? {+ return Ok(());+ }+ ensure!(+ metadata.is_dir(),+ crate::Conflict("expected metadata directory".into())+ );+ for entry in fs::read_dir(metadata)? {+ let entry = entry?;+ if entry.path().extension().is_none_or(|e| e != "json") {+ continue;+ }+ let name = entry+ .file_name()+ .into_string()+ .map_err(|_| anyhow::anyhow!("claimed Ente sidecar name is not UTF-8"))?;+ let path = names::check(root, &format!("{}/metadata/{name}", album.path))?;+ let properties = Properties::read(&path)?;+ let bytes = fs::read(path)?;+ let value: Value = match serde_json::from_slice(&bytes) {+ Ok(value) => value,+ Err(_) if desktop.is_some_and(|d| d.common_sidecar(album.id, &name)) => continue,+ Err(error) => return Err(error.into()),+ };+ let Some(mut item) = parse_sidecar(album, &name, value, &bytes, properties)? else {+ continue;+ };+ let previous: Option<Item> = store::json(+ &index.db,+ "SELECT record FROM adoption_item WHERE album_key=?1 AND file_id=?2 AND (active=1 OR anchor=?3)",+ params![+ album.key,+ item.placement.file,+ item.components[0].location.name+ ],+ )?;+ if let Some(mut previous) = previous {+ ensure!(+ previous.placement.kind == item.placement.kind+ && previous.components+ == item+ .components+ .iter()+ .map(|c| {+ let mut c = c.clone();+ c.placement = previous.placement.id.clone();+ c+ })+ .collect::<Vec<_>>(),+ crate::Conflict(format!(+ "contradictory native placement for file {}",+ item.placement.file+ ))+ );+ let actual = item+ .json+ .iter()+ .find(|r| r.location.is_some())+ .context("missing sidecar")?;+ let slot = previous+ .json+ .iter_mut()+ .find(|r| r.role == actual.role)+ .context("missing component")?;+ ensure!(+ slot.location.is_none(),+ "duplicate component identity for file {}",+ item.placement.file+ );+ *slot = actual.clone();+ slot.owner = previous.placement.id.clone();+ item = previous;+ }+ for component in &item.components {+ claim(+ index,+ album,+ &item.placement.id,+ &component.location.name,+ &item.placement.kind,+ )?;+ }+ save_item(index, album, &item)?;+ }+ Ok(())+}++fn parse_album(root: &Path, directory: &Path, retained: bool) -> Result<Option<AlbumRecord>> {let path = directory.join("metadata.json");let Some(properties) = Properties::optional(&path)? else {- return Ok(());+ return Ok(None);};let bytes = fs::read(&path)?;let record: Value = serde_json::from_slice(&bytes)?;let Some(identity) = record.get("ente") else {- return Ok(());+ return Ok(None);};let folder = directory.strip_prefix(root)?@@ -74,8 +254,8 @@ fn album(root: &Path, store: &Store, directory: &Path, retained: bool) -> Result"unsupported album visibility in {folder}");ensure!(- retained || store.album(id, false)?.is_none(),- super::Conflict(format!("active album ID {id}"))+ retained || !names::reserved(folder, true),+ "reserved active album path {folder}");let album = Album {key: if retained {@@ -88,17 +268,7 @@ fn album(root: &Path, store: &Store, directory: &Path, retained: bool) -> Resultname,path: folder.into(),};- let (parent, name) = folder.rsplit_once('/').unwrap_or(("", folder));- ensure!(- retained || !names::reserved(name, true),- "reserved active album path {folder}"- );- ensure!(- names::available(store, parent, &[name.into()], &album.key, "album")?,- super::Conflict(format!("overlapping album paths at {folder}"))- );- store.save_album(&album)?;- store.save_json(&JsonRecord {+ let json = JsonRecord {owner: album.key.clone(),role: None,location: Some(Location {@@ -108,209 +278,179 @@ fn album(root: &Path, store: &Store, directory: &Path, retained: bool) -> Resultvalue: record,hash: Some(digest(&bytes)?),properties: Some(properties),- })?;- let metadata = names::check(root, &format!("{folder}/metadata"))?;- if !metadata.try_exists()? {- return Ok(());- }+ };+ Ok(Some(AlbumRecord {+ album,+ json: Some(json),+ legacy: None,+ }))+}++fn parse_sidecar(+ album: &Album,+ name: &str,+ record: Value,+ bytes: &[u8],+ properties: Properties,+) -> Result<Option<Item>> {+ let Some(identity) = record.get("ente") else {+ return Ok(None);+ };+ let file = parse_id(identity, "fileID")?;+ let display_name = identity["name"]+ .as_str()+ .context("file record has no name")?;+ chrono::DateTime::parse_from_rfc3339(+ identity["creationTime"]+ .as_str()+ .context("file record has no creationTime")?,+ )?;+ let kind = identity["type"]+ .as_str()+ .context("file record has no type")?;ensure!(- metadata.is_dir(),- super::Conflict(format!(- "at {}: expected metadata directory",- metadata.display()- ))+ ["image", "video", "livePhoto", "unknown"].contains(&kind),+ "unsupported file type for {file}");- for entry in fs::read_dir(metadata)? {- let entry = entry?;- if entry- .path()- .extension()- .is_none_or(|extension| extension != "json")- {- continue;- }- let properties = Properties::read(&entry.path())?;- let bytes = fs::read(entry.path())?;- let record: Value = serde_json::from_slice(&bytes)?;- let Some(identity) = record.get("ente") else {- continue;- };- let name = entry- .file_name()- .into_string()- .map_err(|_| anyhow::anyhow!("claimed Ente sidecar name is not UTF-8"))?;- names::check(root, &format!("{folder}/metadata/{name}"))?;- let file = parse_id(identity, "fileID")?;- let display_name = identity["name"]- .as_str()- .context("file record has no name")?;- chrono::DateTime::parse_from_rfc3339(- identity["creationTime"]- .as_str()- .context("file record has no creationTime")?,- )?;- let kind = identity["type"]- .as_str()- .context("file record has no type")?;+ let mut components: Vec<Component> = serde_json::from_value(identity["components"].clone())?;+ ensure!(+ if kind == "livePhoto" {+ components.len() == 2+ && components.iter().any(|c| c.role == Role::Image)+ && components.iter().any(|c| c.role == Role::Video)+ } else {+ components.len() == 1 && components[0].role == Role::Original+ },+ "invalid components for file {file}"+ );+ components.sort_by_key(|component| match component.role {+ Role::Original => 0,+ Role::Image => 1,+ Role::Video => 2,+ });+ for component in &components {+ names::relative(&component.path)?;ensure!(- ["image", "video", "livePhoto", "unknown"].contains(&kind),- "unsupported file type for {file}"+ !component.path.contains('/') && !names::reserved(&component.path, false),+ "invalid component path for file {file}");- let mut components: Vec<Component> =- serde_json::from_value(identity["components"].clone())?;ensure!(- if kind == "livePhoto" {- components.len() == 2- && components.iter().any(|c| c.role == Role::Image)- && components.iter().any(|c| c.role == Role::Video)- } else {- components.len() == 1 && components[0].role == Role::Original- },- "invalid components for file {file}"+ ente_core::b64::decode(&component.hash)?.len() == 64,+ "invalid component hash for file {file}");- components.sort_by_key(|component| match component.role {- Role::Original => 0,- Role::Image => 1,- Role::Video => 2,- });- for component in &components {- names::relative(&component.path)?;- ensure!(- !component.path.contains('/') && !names::reserved(&component.path, false),- "invalid component path for file {file}"- );- ensure!(- ente_core::b64::decode(&component.hash)?.len() == 64,- "invalid component hash for file {file}"- );+ }+ ensure!(+ components.len() == 1+ || names::folded(&components[0].path) != names::folded(&components[1].path),+ "overlapping Live Photo components for file {file}"+ );+ let own: Role = serde_json::from_value(identity["component"].clone())?;+ let index = components+ .iter()+ .position(|c| c.role == own)+ .context("undeclared sidecar component")?;+ ensure!(+ record["title"].as_str() == Some(components[index].path.as_str())+ && name == format!("{}.json", components[index].path),+ "contradictory sidecar location for file {file}"+ );+ let kind = if kind == "livePhoto" {+ "livephoto"+ } else {+ kind+ };+ let placement = Placement {+ id: Uuid::new_v4().to_string(),+ album: album.id,+ file,+ retained: album.retained,+ name: display_name.into(),+ kind: kind.into(),+ };+ let mut json = Vec::new();+ let components = components+ .into_iter()+ .map(|component| {+ let actual = component.role == own;+ let mut value = record.clone();+ value["title"] = Value::String(component.path.clone());+ value["ente"]["component"] = serde_json::to_value(&component.role)?;+ json.push(JsonRecord {+ owner: placement.id.clone(),+ role: Some(component.role.clone()),+ location: actual.then(|| Location {+ folder: album.key.clone(),+ name: format!("metadata/{name}"),+ }),+ value,+ hash: if actual { Some(digest(bytes)?) } else { None },+ properties: actual.then(|| properties.clone()),+ });+ Ok(store::Component {+ placement: placement.id.clone(),+ role: component.role,+ location: Location {+ folder: album.key.clone(),+ name: component.path,+ },+ size: component.size,+ hash: component.hash,+ signature: None,+ properties: None,+ intended_time: None,+ })+ })+ .collect::<Result<Vec<_>>>()?;+ Ok(Some(Item {+ placement,+ components,+ json,+ legacy: None,+ leftover: false,+ }))+}++pub fn accept(index: &Index, store: &Store, bound: bool) -> Result<()> {+ let transaction = store.db.unchecked_transaction()?;+ store.db.execute_batch("DELETE FROM components; DELETE FROM json_records; DELETE FROM placements; DELETE FROM albums; DELETE FROM pending; DELETE FROM temporaries;")?;+ let mut after = String::new();+ while let Some(record) = store::json::<AlbumRecord>(+ &index.db,+ "SELECT record FROM adoption_album WHERE key>?1 ORDER BY key LIMIT 1",+ [&after],+ )? {+ after = record.album.key.clone();+ store.save_album(&record.album)?;+ if let Some(json) = record.json {+ store.save_json(&json)?;}- ensure!(- components.len() == 1- || names::folded(&components[0].path) != names::folded(&components[1].path),- "overlapping Live Photo components for file {file}"- );- let own: Role = serde_json::from_value(identity["component"].clone())?;- let index = components- .iter()- .position(|c| c.role == own)- .context("undeclared sidecar component")?;- ensure!(- record["title"].as_str() == Some(components[index].path.as_str())- && name == format!("{}.json", components[index].path),- "contradictory sidecar location for file {file}"- );- let kind = if kind == "livePhoto" {- "livephoto"- } else {- kind- };- let existing = if retained {- let mut query = store- .db- .prepare(- "SELECT DISTINCT p.id,p.album,p.file,p.retained,p.name,p.kind FROM placements p JOIN components c ON c.placement=p.id WHERE p.album=?1 AND p.file=?2 AND p.retained=1 AND c.folder=?3",- )?;- let mut rows = query.query(rusqlite::params![id, file, album.key])?;- let mut found = None;- while let Some(row) = rows.next()? {- let candidate = Placement::read(row)?;- if store- .components(&candidate.id)?- .iter()- .map(super::store::Component::portable)- .eq(components.iter().cloned())- {- found = Some(candidate);- break;- }- }- found- } else {- store.placement(id, file)?- };- let fresh = existing.is_none();- let placement = existing.unwrap_or_else(|| Placement {- id: Uuid::new_v4().to_string(),- album: id,- file,- retained,- name: display_name.into(),- kind: kind.into(),- });- if !fresh {- ensure!(- placement.kind == kind- && store- .components(&placement.id)?- .iter()- .all(|component| component.location.folder == album.key)- && store- .components(&placement.id)?- .iter()- .map(super::store::Component::portable)- .eq(components.iter().cloned()),- super::Conflict(format!("active placement for file {file} in album {id}"))- );+ }+ after.clear();+ while let Some(item) = store::json::<Item>(+ &index.db,+ "SELECT record FROM adoption_item WHERE key>?1 ORDER BY key LIMIT 1",+ [&after],+ )? {+ after = item.placement.id.clone();+ if item.leftover || item.components.is_empty() {+ continue;}- ensure!(- !store- .json_record(&placement.id, Some(&own))?- .is_some_and(|record| record.location.is_some()),- "duplicate component identity for file {file}"- );- let media_names: Vec<_> = components- .iter()- .map(|component| component.path.clone())- .collect();- ensure!(- names::available(store, &album.key, &media_names, &placement.id, kind)?,- super::Conflict(format!("overlapping media paths in {folder}"))- );- store.save_placement(&placement)?;- if fresh {- for component in &components {- store.save_component(&super::store::Component {- placement: placement.id.clone(),- role: component.role.clone(),- location: Location {- folder: album.key.clone(),- name: component.path.clone(),- },- size: component.size,- hash: component.hash.clone(),- signature: None,- properties: None,- intended_time: None,- })?;- let mut fallback = record.clone();- fallback["title"] = Value::String(component.path.clone());- fallback["ente"]["component"] = serde_json::to_value(&component.role)?;- store.save_json(&JsonRecord {- owner: placement.id.clone(),- role: Some(component.role.clone()),- location: None,- value: fallback,- hash: None,- properties: None,- })?;- }+ store.save_placement(&item.placement)?;+ for component in item.components {+ store.save_component(&component)?;+ }+ for json in item.json {+ store.save_json(&json)?;}- store.save_json(&JsonRecord {- owner: placement.id,- role: Some(own),- location: Some(Location {- folder: album.key.clone(),- name: format!("metadata/{name}"),- }),- value: record,- hash: Some(digest(&bytes)?),- properties: Some(properties),- })?;}+ store+ .db+ .execute("UPDATE export SET bound=?1 WHERE id=1", [bound])?;+ transaction.commit()?;Ok(())}-fn digest(bytes: &[u8]) -> Result<String> {+pub fn digest(bytes: &[u8]) -> Result<String> {Ok(ente_core::b64::encode(&ente_core::crypto::hash::hash(bytes,Some(64),diff --git a/rust/crates/photos-export/src/fs.rs b/rust/crates/photos-export/src/fs.rsindex f12364c424..20e58de8a3 100644--- a/rust/crates/photos-export/src/fs.rs+++ b/rust/crates/photos-export/src/fs.rs@@ -130,6 +130,24 @@ pub fn sync_move(source: &Path, destination: &Path) -> Result<()> {Ok(())}+pub fn free_rename(source: &Path, destination: &Path) -> Result<()> {+ ensure!(+ !destination.try_exists()? || names::same_path(source, destination),+ super::Conflict(format!(+ "at {}: occupied rename target",+ destination.display()+ ))+ );+ Ok(())+}++pub fn move_file(source: &Path, destination: &Path) -> Result<()> {+ free_rename(source, destination)?;+ create_directory(destination.parent().context("move has no parent")?)?;+ fs::rename(source, destination)?;+ sync_move(source, destination)+}+pub struct Json {pub value: Value,pub bytes: Vec<u8>,diff --git a/rust/crates/photos-export/src/legacy/mod.rs b/rust/crates/photos-export/src/legacy/mod.rsnew file mode 100644index 0000000000..f20a7d8a31--- /dev/null+++ b/rust/crates/photos-export/src/legacy/mod.rs@@ -0,0 +1,497 @@+pub mod read;+mod verify;++use crate::{+ Options, Source,+ adopt::{self, AlbumRecord, Index, Item},+ fs, names,+ snapshot::FileSnapshot,+ store::{self, AlbumSource, JsonRecord, Properties, Store},+};+use anyhow::{Context, Result, ensure};+use ente_core::Session;+use rusqlite::{OptionalExtension, params};+use std::{+ collections::{BTreeMap, HashSet, btree_map::Entry},+ fs as disk,+ path::Path,+};+use tokio::sync::watch;++#[derive(Default)]+pub struct Failures {+ count: usize,+ examples: Vec<String>,+}++impl Failures {+ pub fn record(&mut self, result: Result<()>, context: &str) -> Result<()> {+ if let Err(error) = result {+ if crate::fatal(&error) {+ return Err(error.context(format!("adoption preflight incomplete: {context}")));+ }+ self.count += 1;+ if self.examples.len() < 10 {+ self.examples.push(format!("{context}: {error:#}"));+ }+ }+ Ok(())+ }++ fn message(&self) -> String {+ let omitted = self.count.saturating_sub(self.examples.len());+ format!(+ "{} failing items\n{}{}",+ self.count,+ self.examples.join("\n"),+ if omitted == 0 {+ String::new()+ } else {+ format!("\n{omitted} more omitted")+ }+ )+ }++ pub fn finish(&self) -> Result<()> {+ ensure!(+ self.count == 0,+ "adoption preflight failed: {}",+ self.message()+ );+ Ok(())+ }+}++pub fn scope(store: &Store, options: &Options, index: Option<&Index>) -> Result<()> {+ let explicit = !options.albums.is_empty() || !options.exclude_albums.is_empty();+ if explicit {+ crate::select(store, options)?;+ }+ let (db, table) = match index {+ Some(index) => (&index.db, "adoption_album"),+ None => (&store.db, "albums"),+ };+ let mut membership = db.prepare(&format!(+ "SELECT EXISTS(SELECT 1 FROM {table} WHERE id=?1 AND retained=0)"+ ))?;+ let mut desired = store.db.prepare("SELECT id,selected FROM desired_albums")?;+ for row in desired.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, bool>(1)?)))? {+ let (id, selected) = row?;+ let recorded: bool = membership.query_row([id], |r| r.get(0))?;+ if explicit {+ ensure!(+ selected == recorded,+ "adoption filters must select exactly the recorded album IDs"+ );+ } else {+ store.db.execute(+ "UPDATE desired_albums SET selected=?1 WHERE id=?2",+ params![recorded, id],+ )?;+ }+ }+ Ok(())+}++pub struct Preflight<'a> {+ pub root: &'a Path,+ pub store: &'a Store,+ pub session: &'a Session,+ pub index: &'a Index,+ pub cancel: &'a watch::Receiver<bool>,+ pub matching_root: bool,+}++impl Preflight<'_> {+ pub async fn run(&self, source: &impl Source, mut failures: Failures) -> Result<()> {+ let store = self.store;+ let index = self.index;+ let mut after = String::new();+ while let Some(mut record) = store::json::<AlbumRecord>(+ &index.db,+ "SELECT record FROM adoption_album WHERE retained=0 AND key>?1 ORDER BY key LIMIT 1",+ [&after],+ )? {+ after = record.album.key.clone();+ let id = record.album.id;+ let result = (|| {+ let (ready, present, failure): (bool, bool, Option<String>) = store.db.query_row(+ "SELECT ready,present,failure FROM desired_albums WHERE id=?1",+ [id],+ |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),+ )?;+ ensure!(+ ready && failure.is_none(),+ "album source is incomplete: {}",+ failure.as_deref().unwrap_or("refresh failed")+ );+ if let Some(legacy) = &record.legacy+ && present+ && let Some(owner) = legacy.owner+ {+ let entry = source+ .catalog_page(id - 1)?+ .into_iter()+ .find(|e| e.id == id)+ .context("missing current album")?;+ ensure!(+ entry.value?.0.owner_id == owner,+ "album owner disagrees with source"+ );+ }+ if record.json.is_none() && present {+ let source: AlbumSource = store::json(+ &store.db,+ "SELECT record FROM album_sources WHERE id=?1",+ [id],+ )?+ .context("missing album source")?;+ if record.legacy.as_ref().is_some_and(|legacy| !legacy.go) {+ record.album.name = source.name;+ }+ record.json = Some(JsonRecord {+ owner: record.album.key.clone(),+ role: None,+ location: None,+ value: source.metadata,+ hash: None,+ properties: None,+ });+ }+ adopt::save_album(index, &record)+ })();+ if result.is_err() {+ store.db.execute("UPDATE desired_albums SET failure='adoption album preflight failed' WHERE id=?1",[id])?;+ }+ failures.record(result, &format!("album:{id}"))?;+ }+ let mut after = (0, String::new());+ let mut expected = None;+ let mut verified = 0;+ loop {+ let next = store::json::<Item>(+ &index.db,+ "SELECT record FROM adoption_item WHERE origin='legacy' AND (file_id,key)>(?1,?2) ORDER BY file_id,key LIMIT 1",+ params![after.0, after.1],+ )?;+ let Some(mut item) = next else { break };+ if after.0 != item.placement.file {+ expected = None;+ }+ after = (item.placement.file, item.placement.id.clone());+ ensure!(!*self.cancel.borrow(), crate::Cancelled);+ let ready: bool = store.db.query_row(+ "SELECT ready=1 AND failure IS NULL FROM desired_albums WHERE id=?1",+ [item.placement.album],+ |r| r.get(0),+ )?;+ if !ready || expected.as_ref().is_some_and(Result::is_err) {+ continue;+ }+ let result = async {+ let record = item.legacy.as_ref().context("missing legacy identity")?;+ let album: AlbumRecord = store::json(+ &index.db,+ "SELECT record FROM adoption_album WHERE id=?1 AND retained=0",+ [record.album],+ )?+ .context("missing album")?;+ let failure: Option<Option<String>> = store+ .db+ .query_row(+ "SELECT failure FROM desired_files WHERE album=?1 AND file=?2",+ params![record.album, record.file],+ |r| r.get(0),+ )+ .optional()?;+ if let Some(failure) = failure {+ ensure!(+ failure.is_none(),+ "file source is incomplete: {}",+ failure.unwrap_or_default()+ );+ let remote = source.file(record.album, record.file)?.value?.0;+ ensure!(+ record.owner.is_none_or(|owner| owner == remote.owner_id),+ "file owner disagrees with source"+ );+ let file: FileSnapshot = store::json(+ &store.db,+ "SELECT record FROM sources WHERE file=?1",+ [record.file],+ )?+ .context("missing source snapshot")?;+ verified +=+ verify::file(self, &album.album, &mut item, &file, &mut expected).await?;+ } else {+ item.leftover = true;+ }+ adopt::save_item(index, &album.album, &item)+ }+ .await;+ failures.record(+ result,+ &format!("file:{}:{}", item.placement.album, item.placement.file),+ )?;+ }+ failures.record((|| {+ let trash=names::check(self.root,"Trash")?;+ ensure!(!trash.try_exists()? || trash.is_dir(),"Trash is not a directory");+ let reserved:Option<AlbumRecord>=store::json(&index.db,"SELECT record FROM adoption_album WHERE retained=0 AND parent='' AND folded='trash'",[])?;+ ensure!(reserved.is_none_or(|r|r.legacy.is_some_and(|l|l.go)),"reserved legacy album path");+ Ok(())+ })(),"Trash")?;+ if !self.matching_root && verified == 0 && failures.count == 0 {+ failures.record(+ Err(anyhow::anyhow!(+ "at least one existing active original must verify"+ )),+ "source association",+ )?;+ }+ failures.finish()+ }+}++pub fn move_leftovers(root: &Path, index: &Index, cancel: &watch::Receiver<bool>) -> Result<()> {+ if let Some(mut record) = store::json::<AlbumRecord>(+ &index.db,+ "SELECT record FROM adoption_album WHERE retained=0 AND parent='' AND folded='trash'",+ [],+ )? {+ let source = names::check(root, &record.album.path)?;+ if source.try_exists()? {+ let occupied = occupied_names(root, cancel)?;+ for suffix in 0.. {+ ensure!(!*cancel.borrow(), crate::Cancelled);+ let Some(candidate) = names::candidates(&record.album.name, true, None, suffix)?+ else {+ continue;+ };+ let name = &candidate[0];+ let claimed: bool = index.db.query_row(+ "SELECT EXISTS(SELECT 1 FROM adoption_album WHERE parent='' AND folded=?1)",+ [names::folded(name)],+ |r| r.get(0),+ )?;+ let destination = names::check(root, name)?;+ if claimed || occupied.contains(&names::folded(name)) {+ continue;+ }+ fs::free_rename(&source, &destination)?;+ disk::rename(&source, &destination)?;+ fs::sync_move(&source, &destination)?;+ record.album.path = name.clone();+ adopt::save_album(index, &record)?;+ break;+ }+ }+ }+ let mut after = String::new();+ // Legacy keys keep each album's leftovers together.+ let mut album_id = None;+ let mut occupied = BTreeMap::new();+ while let Some(item) = store::json::<Item>(+ &index.db,+ "SELECT record FROM adoption_item WHERE origin='legacy' AND key>?1 ORDER BY key LIMIT 1",+ [&after],+ )? {+ after = item.placement.id.clone();+ if !item.leftover {+ continue;+ }+ let record = item.legacy.context("missing leftover identity")?;+ if album_id != Some(record.album) {+ occupied.clear();+ album_id = Some(record.album);+ }+ let album: AlbumRecord = store::json(+ &index.db,+ "SELECT record FROM adoption_album WHERE id=?1 AND retained=0",+ [record.album],+ )?+ .context("missing album")?;+ let folder = album.album.path;+ let trash = format!("Trash/{folder}");+ for part in &record.parts {+ move_one(+ root,+ index,+ &format!("{folder}/{}", part.name),+ &trash,+ &part.name,+ &mut occupied,+ cancel,+ )?;+ if record.desktop {+ move_one(+ root,+ index,+ &format!("{folder}/metadata/{}.json", part.name),+ &format!("{trash}/metadata"),+ &format!("{}.json", part.name),+ &mut occupied,+ cancel,+ )?;+ }+ }+ if let Some(sidecar) = record.go_record {+ let name = sidecar+ .strip_prefix(".meta/")+ .context("invalid Go record")?;+ move_one(+ root,+ index,+ &format!("{folder}/{sidecar}"),+ &format!("{trash}/.meta"),+ name,+ &mut occupied,+ cancel,+ )?;+ }+ }+ Ok(())+}++fn occupied_names(directory: &Path, cancel: &watch::Receiver<bool>) -> Result<HashSet<String>> {+ let mut occupied = HashSet::new();+ if directory.try_exists()? {+ for entry in disk::read_dir(directory)? {+ ensure!(!*cancel.borrow(), crate::Cancelled);+ occupied.insert(names::folded(&entry?.file_name().to_string_lossy()));+ }+ }+ Ok(occupied)+}++fn move_one(+ root: &Path,+ index: &Index,+ source: &str,+ directory: &str,+ name: &str,+ occupied: &mut BTreeMap<String, HashSet<String>>,+ cancel: &watch::Receiver<bool>,+) -> Result<()> {+ ensure!(!*cancel.borrow(), crate::Cancelled);+ let source = names::check(root, source)?;+ let parent = names::check(root, directory)?;+ if Properties::optional(&source)?.is_none() {+ return fs::sync_move(&source, &parent.join(name));+ }+ for suffix in 0.. {+ ensure!(!*cancel.borrow(), crate::Cancelled);+ let Some(candidate) = names::candidates(name, false, None, suffix)? else {+ continue;+ };+ let relative = format!("{directory}/{}", candidate[0]);+ let destination = names::check(root, &relative)?;+ let occupied = match occupied.entry(directory.to_owned()) {+ Entry::Occupied(entry) => entry.into_mut(),+ Entry::Vacant(entry) => entry.insert(occupied_names(&parent, cancel)?),+ };+ let folded = names::folded(&candidate[0]);+ if occupied.contains(&folded) {+ continue;+ }+ let media = directory+ .strip_suffix("/metadata")+ .map(|p| {+ format!(+ "{p}/{}",+ candidate[0].strip_suffix(".json").unwrap_or(&candidate[0])+ )+ })+ .unwrap_or_else(|| relative.clone());+ let claimed: bool = index.db.query_row(+ "SELECT EXISTS(SELECT 1 FROM adoption_path WHERE path=?1)",+ [names::folded(&media)],+ |r| r.get(0),+ )?;+ if !claimed {+ fs::move_file(&source, &destination)?;+ occupied.insert(folded);+ return Ok(());+ }+ }+ unreachable!()+}++pub fn finish(root: &Path, store: &Store, cancel: &watch::Receiver<bool>) -> Result<()> {+ let mut after = String::new();+ loop {+ let next=store.db.query_row(+ "SELECT key,id,retained,name,path FROM albums WHERE retained=0 AND key>?1 ORDER BY key LIMIT 1",+ [&after],crate::store::Album::read,+ ).optional()?;+ let Some(album) = next else { break };+ ensure!(!*cancel.borrow(), crate::Cancelled);+ after = album.key.clone();+ let meta = names::check(root, &format!("{}/.meta", album.path))?;+ if meta.is_dir() {+ for entry in disk::read_dir(&meta)? {+ ensure!(!*cancel.borrow(), crate::Cancelled);+ let entry = entry?;+ if entry.file_name() == "album_meta.json"+ || entry.path().extension().is_none_or(|e| e != "json")+ || !entry.file_type()?.is_file()+ {+ continue;+ }+ if read::go_record(&entry.path())? {+ fs::remove(&entry.path())?;+ }+ }+ let identity = meta.join("album_meta.json");+ if identity.try_exists()? {+ let value: serde_json::Value = read::json(&identity)?;+ if value["id"].as_i64() == Some(album.id) {+ fs::remove(&identity)?;+ }+ }+ if disk::read_dir(&meta)?.next().is_none() {+ disk::remove_dir(&meta)?;+ fs::sync_parent(&meta)?;+ }+ }+ let absent: bool = store.db.query_row(+ "SELECT present=0 FROM desired_albums WHERE id=?1",+ [album.id],+ |r| r.get(0),+ )?;+ if absent {+ for relative in [format!("{}/metadata", album.path), album.path] {+ let path = names::check(root, &relative)?;+ if path.is_dir() && disk::read_dir(&path)?.next().is_none() {+ disk::remove_dir(&path)?;+ fs::sync_parent(&path)?;+ }+ }+ }+ }+ let manifest = names::check(root, "export_status.json")?;+ if manifest.try_exists()? {+ fs::remove(&manifest)?;+ }+ let transaction = store.db.unchecked_transaction()?;+ store.db.execute("DELETE FROM albums WHERE retained=0 AND id IN (SELECT id FROM desired_albums WHERE selected=1 AND present=0) AND NOT EXISTS(SELECT 1 FROM placements WHERE album=albums.id AND retained=0) AND NOT EXISTS(SELECT 1 FROM temporaries WHERE folder=albums.key)",[])?;+ store+ .db+ .execute("UPDATE export SET bound=1 WHERE id=1", [])?;+ transaction.commit()?;+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn nested_record_preserves_fatal_errors() {+ let mut failures = Failures::default();+ let result = failures.record(Err(rusqlite::Error::InvalidQuery.into()), "file");+ let error = failures.record(result, "album").unwrap_err();+ assert!(error.is::<rusqlite::Error>() && crate::fatal(&error));+ assert_eq!(failures.count, 0);+ }+}diff --git a/rust/crates/photos-export/src/legacy/read.rs b/rust/crates/photos-export/src/legacy/read.rsnew file mode 100644index 0000000000..cf38daead4--- /dev/null+++ b/rust/crates/photos-export/src/legacy/read.rs@@ -0,0 +1,446 @@+use std::{collections::BTreeMap, fs, path::Path};++use anyhow::{Context, Result, ensure};+use rusqlite::params;+use serde::{Deserialize, Serialize};+use serde_json::Value;++use super::Failures;+use crate::adopt::{self, AlbumRecord, Index, Item};+use crate::store::{self, Album, Placement};+use crate::{metadata::Role, names, store::Properties};++#[derive(Deserialize)]+#[serde(rename_all = "camelCase")]+struct GoAlbum {+ id: i64,+ #[serde(rename = "ownerID")]+ owner_id: i64,+ album_name: String,+ #[serde(default, rename = "accountOwnerIDs")]+ account_owner_ids: Vec<i64>,+}++#[derive(Deserialize)]+#[serde(rename_all = "camelCase")]+struct GoInfo {+ id: i64,+ #[serde(rename = "ownerID")]+ owner_id: i64,+ file_names: Vec<String>,+ hash: Option<String>,+}++#[derive(Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct Desktop {+ pub version: u32,+ pub collection_export_names: BTreeMap<String, String>,+ pub file_export_names: BTreeMap<String, String>,+}++#[derive(Clone, Serialize, Deserialize)]+pub struct Part {+ pub name: String,+ pub role: Option<Role>,+}++#[derive(Clone, Serialize, Deserialize)]+pub struct Record {+ pub album: i64,+ pub file: i64,+ pub owner: Option<i64>,+ pub name: String,+ pub hash: Option<String>,+ pub parts: Vec<Part>,+ pub go_record: Option<String>,+ pub desktop: bool,+}++#[derive(Serialize, Deserialize)]+pub struct AlbumInfo {+ pub owner: Option<i64>,+ pub go: bool,+}++pub fn recognized(root: &Path) -> Result<bool> {+ if !root.is_dir() {+ return Ok(false);+ }+ if root.join("export_status.json").try_exists()? {+ return Ok(true);+ }+ for entry in fs::read_dir(root)? {+ let path = entry?.path().join(".meta");+ if path.is_dir() && path.join("album_meta.json").try_exists()? {+ return Ok(true);+ }+ }+ Ok(false)+}++pub fn desktop(root: &Path) -> Result<Option<Desktop>> {+ let manifest = names::check(root, "export_status.json")?;+ if !manifest.try_exists()? {+ return Ok(None);+ }+ Ok(Some(json(&manifest)?))+}++pub fn scan(+ root: &Path,+ index: &Index,+ user: i64,+ desktop: Option<&Desktop>,+ failures: &mut Failures,+) -> Result<()> {+ if let Some(desktop) = desktop {+ let result = (|| {+ ensure!(+ desktop.version == 5,+ "unsupported Desktop export version {}",+ desktop.version+ );+ for (id, path) in &desktop.collection_export_names {+ let result = (|| {+ let id = id_string(id)?;+ album(root, index, id, path, path, None, false)+ })();+ failures.record(result, path)?;+ }+ for (key, name) in &desktop.file_export_names {+ let result = (|| {+ let parts: Vec<_> = key.split('_').collect();+ ensure!(parts.len() == 3, "invalid Desktop file identity");+ let file = id_string(parts[0])?;+ let album = id_string(parts[1])?;+ id_string(parts[2])?;+ let mapped: bool = index.db.query_row(+ "SELECT EXISTS(SELECT 1 FROM adoption_album WHERE id=?1 AND retained=0)",+ [album],+ |r| r.get(0),+ )?;+ if !mapped && desktop.collection_export_names.contains_key(parts[1]) {+ return Ok(());+ }+ let paths = if name.starts_with('{') {+ #[derive(Deserialize)]+ struct Pair {+ image: String,+ video: String,+ }+ let pair: Pair = serde_json::from_str(name)?;+ vec![+ Part {+ name: pair.image,+ role: Some(Role::Image),+ },+ Part {+ name: pair.video,+ role: Some(Role::Video),+ },+ ]+ } else {+ vec![Part {+ name: name.clone(),+ role: Some(Role::Original),+ }]+ };+ let record = Record {+ album,+ file,+ owner: None,+ name: paths[0].name.clone(),+ hash: None,+ parts: paths,+ go_record: None,+ desktop: true,+ };+ insert(root, index, &record)+ })();+ failures.record(result, key)?;+ }+ Ok(())+ })();+ failures.record(result, "export_status.json")?;+ }+ for entry in fs::read_dir(root)? {+ let entry = entry?;+ let meta = entry.path().join(".meta");+ if !meta.is_dir() {+ continue;+ }+ let identity = meta.join("album_meta.json");+ if !identity.try_exists()? {+ let result = (|| {+ let folder = entry+ .file_name()+ .into_string()+ .map_err(|_| anyhow::anyhow!("legacy album path is not UTF-8"))?;+ let meta = names::check(root, &format!("{folder}/.meta"))?;+ for sidecar in fs::read_dir(meta)? {+ let sidecar = sidecar?;+ if !sidecar.path().extension().is_some_and(|s| s == "json") {+ continue;+ }+ let name = sidecar+ .file_name()+ .into_string()+ .map_err(|_| anyhow::anyhow!("legacy sidecar name is not UTF-8"))?;+ let path = names::check(root, &format!("{folder}/.meta/{name}"))?;+ if json::<Value>(&path)+ .ok()+ .is_some_and(|value| value["info"]["id"].as_i64().is_some())+ {+ anyhow::bail!("Go file identities remain without album_meta.json");+ }+ }+ Ok(())+ })();+ failures.record(result, &entry.path().display().to_string())?;+ continue;+ }+ let path = entry+ .file_name()+ .into_string()+ .map_err(|_| anyhow::anyhow!("legacy album path is not UTF-8"))?;+ let result = (|| {+ names::check(root, &format!("{path}/.meta/album_meta.json"))?;+ let record: GoAlbum = json(&identity)?;+ ensure!(+ record.account_owner_ids.is_empty() || record.account_owner_ids.contains(&user),+ "legacy account owners exclude the selected account"+ );+ ensure!(record.owner_id > 0, "invalid album owner");+ album(+ root,+ index,+ record.id,+ &path,+ &record.album_name,+ Some(record.owner_id),+ true,+ )?;+ for entry in fs::read_dir(identity.parent().context("missing metadata parent")?)? {+ let entry = entry?;+ if entry.file_name() == "album_meta.json"+ || entry.path().extension().is_none_or(|s| s != "json")+ {+ continue;+ }+ let name = entry+ .file_name()+ .into_string()+ .map_err(|_| anyhow::anyhow!("legacy sidecar name is not UTF-8"))?;+ let result = (|| {+ names::check(root, &format!("{path}/.meta/{name}"))?;+ let value: Value = json(&entry.path())?;+ let Some(info) = value.get("info") else {+ return Ok(());+ };+ let info: GoInfo = serde_json::from_value(info.clone())?;+ ensure!(info.owner_id > 0, "invalid file owner");+ let title = value["title"]+ .as_str()+ .context("missing legacy file title")?+ .to_owned();+ let record = Record {+ album: record.id,+ file: info.id,+ owner: Some(info.owner_id),+ name: title,+ hash: info.hash,+ parts: info+ .file_names+ .into_iter()+ .map(|name| Part { name, role: None })+ .collect(),+ go_record: Some(format!(".meta/{name}")),+ desktop: false,+ };+ insert(root, index, &record)+ })();+ failures.record(result, &format!("{path}/.meta/{name}"))?;+ }+ Ok(())+ })();+ failures.record(result, &path)?;+ }+ Ok(())+}++fn album(+ root: &Path,+ index: &Index,+ id: i64,+ path: &str,+ name: &str,+ owner: Option<i64>,+ go: bool,+) -> Result<()> {+ ensure!(id > 0, "invalid album ID");+ leaf(path)?;+ let previous: Option<AlbumRecord> = store::json(+ &index.db,+ "SELECT record FROM adoption_album WHERE id=?1 AND retained=0",+ [id],+ )?;+ let mut record = if let Some(record) = previous {+ ensure!(record.legacy.is_none(), "duplicate legacy album identity");+ record+ } else {+ let directory = names::check(root, path)?;+ ensure!(+ !directory.try_exists()? || directory.is_dir(),+ "legacy album is not a directory"+ );+ let occupied: bool = index.db.query_row(+ "SELECT EXISTS(SELECT 1 FROM adoption_album WHERE parent='' AND folded=?1)",+ [names::folded(path)],+ |r| r.get(0),+ )?;+ ensure!(!occupied, "overlapping legacy album path");+ AlbumRecord {+ album: Album {+ key: format!("active:{id}"),+ id,+ retained: false,+ name: name.into(),+ path: path.into(),+ },+ json: None,+ legacy: None,+ }+ };+ let metadata = names::check(root, &format!("{}/metadata", record.album.path))?;+ ensure!(+ !metadata.try_exists()? || metadata.is_dir(),+ "required metadata directory is occupied"+ );+ let json = names::check(root, &format!("{}/metadata.json", record.album.path))?;+ ensure!(+ !json.try_exists()? || record.json.is_some(),+ "required metadata.json is occupied"+ );+ record.legacy = Some(AlbumInfo { owner, go });+ adopt::save_album(index, &record)+}++fn insert(root: &Path, index: &Index, record: &Record) -> Result<()> {+ ensure!(+ record.file > 0 && (1..=2).contains(&record.parts.len()),+ "invalid legacy file identity or components"+ );+ let album: AlbumRecord = store::json(+ &index.db,+ "SELECT record FROM adoption_album WHERE id=?1 AND retained=0",+ [record.album],+ )?+ .context("file has no valid album mapping")?;+ if let Some(existing) = store::json::<Item>(+ &index.db,+ "SELECT record FROM adoption_item WHERE album_key=?1 AND file_id=?2 AND active=1",+ params![album.album.key, record.file],+ )? {+ ensure!(existing.legacy.is_none(), "duplicate legacy membership");+ return Ok(());+ }+ let folder = album.album;+ for part in &record.parts {+ leaf(&part.name)?;+ if store::json::<Item>(&index.db,"SELECT i.record FROM adoption_path p JOIN adoption_item i ON i.key=p.item_key WHERE p.path=?1 AND i.origin='native'",[names::folded(&format!("{}/{}",folder.path,part.name))])?.is_some() { return Ok(()) }+ }+ let item = Item {+ placement: Placement {+ id: format!("legacy:{}:{}", record.album, record.file),+ album: record.album,+ file: record.file,+ retained: false,+ name: record.name.clone(),+ kind: String::new(),+ },+ components: Vec::new(),+ json: Vec::new(),+ legacy: Some(record.clone()),+ leftover: false,+ };+ let mut claimed = Vec::new();+ for part in &record.parts {+ ensure!(+ !["metadata", "metadata.json"].contains(&names::folded(&part.name).as_str()),+ "legacy media occupies a required metadata path"+ );+ ensure!(+ !claimed.contains(&names::folded(&part.name)),+ "duplicate legacy component path"+ );+ claimed.push(names::folded(&part.name));+ Properties::optional(&names::check(+ root,+ &format!("{}/{}", folder.path, part.name),+ )?)?;+ adopt::claim(index, &folder, &item.placement.id, &part.name, "")?;+ let sidecar = Properties::optional(&names::check(+ root,+ &format!("{}/metadata/{}.json", folder.path, part.name),+ )?)?;+ ensure!(+ record.desktop || sidecar.is_none(),+ "required metadata path is occupied"+ );+ }+ adopt::save_item(index, &folder, &item)+}++impl Desktop {+ pub fn common_sidecar(&self, album: i64, name: &str) -> bool {+ self.file_export_names.iter().any(|(key, value)| {+ let Some(id) = key.split('_').nth(1) else {+ return false;+ };+ if id.parse::<i64>().ok() != Some(album) {+ return false;+ }+ if value.starts_with('{') {+ serde_json::from_str::<Value>(value).is_ok_and(|pair| {+ ["image", "video"].iter().any(|role| {+ pair[role]+ .as_str()+ .is_some_and(|p| format!("{p}.json") == name)+ })+ })+ } else {+ format!("{value}.json") == name+ }+ })+ }+}++pub fn go_record(path: &Path) -> Result<bool> {+ let bytes = fs::read(path)?;+ Ok(serde_json::from_slice::<Value>(&bytes)+ .is_ok_and(|value| value["info"]["id"].as_i64().is_some_and(|id| id > 0)))+}++pub fn json<T: serde::de::DeserializeOwned>(path: &Path) -> Result<T> {+ Properties::read(path)?;+ Ok(serde_json::from_reader(fs::File::open(path)?)?)+}++pub fn id_string(text: &str) -> Result<i64> {+ let id: i64 = text.parse()?;+ ensure!(+ id > 0 && id.to_string() == text,+ "invalid identity {text:?}"+ );+ Ok(id)+}++fn leaf(name: &str) -> Result<()> {+ names::relative(name)?;+ ensure!(+ !name.contains('/'),+ "legacy path is not a single name: {name:?}"+ );+ Ok(())+}diff --git a/rust/crates/photos-export/src/legacy/verify.rs b/rust/crates/photos-export/src/legacy/verify.rsnew file mode 100644index 0000000000..48ae8541dd--- /dev/null+++ b/rust/crates/photos-export/src/legacy/verify.rs@@ -0,0 +1,259 @@+use crate::{+ adopt::{self, Item},+ fs,+ metadata::{self, Role},+ names,+ snapshot::FileSnapshot,+ store::{Album, Component, JsonRecord, Location, Properties},+ transfer,+};+use anyhow::{Context, Result, ensure};+use ente_core::{Session, b64, crypto::hash::HashState};+use ente_photos::files;+use std::{+ io::{self, Seek, SeekFrom, Write},+ path::Path,+};+use tokio::sync::watch;++pub async fn file(+ check: &super::Preflight<'_>,+ album: &Album,+ item: &mut Item,+ source: &FileSnapshot,+ expected: &mut Option<Result<Vec<(Role, String)>, String>>,+) -> Result<usize> {+ let record = item.legacy.as_ref().context("missing legacy record")?;+ let roles = transfer::roles(source);+ ensure!(+ source.kind == "livephoto" || record.parts.len() == 1,+ "ordinary file has multiple legacy components"+ );+ ensure!(+ source.hash.is_none()+ || roles+ .iter()+ .all(|role| transfer::expected_hash(source, role).is_some_and(valid_hash)),+ "invalid authenticated source hash"+ );+ ensure!(+ record.parts.len() == 1+ || record.parts[0].role.is_none()+ || record.parts[0].role != record.parts[1].role,+ "duplicate legacy component role"+ );+ let mut measured = Vec::new();+ for part in &record.parts {+ let path = names::check(check.root, &format!("{}/{}", album.path, part.name))?;+ measured.push(+ Properties::optional(&path)?+ .map(|properties| {+ let (size, hash) = fs::hash_cancellable(&path, check.cancel)?;+ Ok::<_, anyhow::Error>((size, hash, properties))+ })+ .transpose()?,+ );+ }+ let present = measured.iter().filter(|p| p.is_some()).count();+ if present > 0 && expected.is_none() {+ match hashes(check.root, source, check.session, check.cancel).await {+ Ok(hashes) => *expected = Some(Ok(hashes)),+ Err(error) => {+ if crate::fatal(&error) {+ return Err(error);+ }+ *expected = Some(Err(format!("{error:#}")));+ }+ }+ }+ let known = if present > 0 {+ expected+ .as_ref()+ .context("missing comparison")?+ .as_ref()+ .map_err(|error| anyhow::anyhow!("{error}"))?+ .clone()+ } else {+ roles+ .iter()+ .filter_map(|r| transfer::expected_hash(source, r).map(|h| (r.clone(), h.to_owned())))+ .collect()+ };+ let complete = present == roles.len();+ item.leftover = !complete && (source.kind == "livephoto" || record.desktop);+ item.placement.kind = source.kind.clone();+ if record.desktop {+ item.placement.name = source.name.clone();+ }+ for (index, part) in record.parts.iter().enumerate() {+ let role = part.role.clone().or_else(|| {+ if roles.len() == 1 {+ Some(Role::Original)+ } else if record.parts.len() == 2 {+ Some(roles[index].clone())+ } else {+ None+ }+ });+ ensure!(+ role.as_ref().is_none_or(|r| roles.contains(r)),+ "legacy component role disagrees with source"+ );+ if let Some((size, hash, properties)) = &measured[index] {+ ensure!(+ known+ .iter()+ .any(|(r, h)| role.as_ref().is_none_or(|own| own == r) && h == hash),+ "changed original at {}/{}",+ album.path,+ part.name+ );+ if complete {+ let role = role.context("missing complete component role")?;+ ensure!(+ !item.components.iter().any(|c| c.role == role),+ "duplicate legacy component role"+ );+ item.components.push(Component {+ placement: item.placement.id.clone(),+ role,+ location: Location {+ folder: album.key.clone(),+ name: part.name.clone(),+ },+ size: *size,+ hash: hash.clone(),+ signature: Some(transfer::signature(source)?),+ properties: Some(properties.clone()),+ intended_time: None,+ });+ }+ }+ }+ for (role, hash) in &known {+ if let Some(legacy) = legacy_hash(record.hash.as_deref(), role) {+ ensure!(+ legacy == hash,+ "legacy hash contradicts the {} component",+ role.name()+ );+ }+ }+ if complete {+ let layout: Vec<_> = item.components.iter().map(Component::portable).collect();+ let favorited = check.store.db.query_row(+ "SELECT favorited FROM desired_files WHERE album=?1 AND file=?2",+ rusqlite::params![record.album, record.file],+ |r| r.get(0),+ )?;+ for (index, component) in item.components.iter().enumerate() {+ let name = format!("metadata/{}.json", component.location.name);+ let path = names::check(check.root, &format!("{}/{name}", album.path))?;+ let properties = Properties::optional(&path)?;+ ensure!(+ record.desktop || properties.is_none(),+ "required metadata path is occupied"+ );+ let hash = properties+ .as_ref()+ .map(|_| adopt::digest(&std::fs::read(&path)?))+ .transpose()?;+ item.json.push(JsonRecord {+ owner: item.placement.id.clone(),+ role: Some(component.role.clone()),+ location: properties.as_ref().map(|_| Location {+ folder: album.key.clone(),+ name,+ }),+ value: metadata::publish(&source.metadata, &layout, index, favorited)?,+ hash,+ properties,+ });+ }+ }+ Ok(present)+}++fn valid_hash(hash: &str) -> bool {+ b64::decode(hash).is_ok_and(|bytes| bytes.len() == 64)+}++fn legacy_hash<'a>(hash: Option<&'a str>, role: &Role) -> Option<&'a str> {+ let hash = hash?;+ let hash = match role {+ Role::Original => hash,+ Role::Image => hash.split_once(':')?.0,+ Role::Video => hash.split_once(':')?.1,+ };+ valid_hash(hash).then_some(hash)+}++async fn hashes(+ root: &Path,+ file: &FileSnapshot,+ session: &Session,+ cancel: &watch::Receiver<bool>,+) -> Result<Vec<(Role, String)>> {+ let roles = transfer::roles(file);+ let known: Option<Vec<_>> = roles+ .iter()+ .map(|role| transfer::expected_hash(file, role).map(|hash| (role.clone(), hash.to_owned())))+ .collect();+ if let Some(known) = known {+ return Ok(known);+ }+ let mut receiver = cancel.clone();+ let hashes = if file.kind == "livephoto" {+ let scratch = tempfile::tempfile_in(root)?;+ let download = files::download(session, file.id, &file.key, &file.header, || {+ let mut file = scratch.try_clone()?;+ file.set_len(0)?;+ file.seek(SeekFrom::Start(0))?;+ Ok(file)+ });+ let archive = tokio::select! { result=download=>result?, _=receiver.changed()=>return Err(crate::Cancelled.into()) };+ crate::live_photo::extract(archive, |_, _| {+ Digest::new(cancel).map_err(io::Error::other)+ })?+ .into_iter()+ .map(|(role, _, writer)| Ok((role, writer.finish()?)))+ .collect::<Result<Vec<_>>>()?+ } else {+ let download = files::download(session, file.id, &file.key, &file.header, || {+ Digest::new(cancel).map_err(io::Error::other)+ });+ let writer = tokio::select! { result=download=>result?, _=receiver.changed()=>return Err(crate::Cancelled.into()) };+ vec![(Role::Original, writer.finish()?)]+ };+ Ok(hashes)+}++struct Digest {+ state: HashState,+ cancel: watch::Receiver<bool>,+}++impl Digest {+ fn new(cancel: &watch::Receiver<bool>) -> Result<Self> {+ Ok(Self {+ state: HashState::new(Some(64), None)?,+ cancel: cancel.clone(),+ })+ }+ fn finish(self) -> Result<String> {+ Ok(b64::encode(&self.state.finalize()?))+ }+}++impl Write for Digest {+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {+ if *self.cancel.borrow() {+ return Err(io::Error::other(crate::Cancelled));+ }+ self.state.update(bytes).map_err(io::Error::other)?;+ Ok(bytes.len())+ }+ fn flush(&mut self) -> io::Result<()> {+ Ok(())+ }+}diff --git a/rust/crates/photos-export/src/lib.rs b/rust/crates/photos-export/src/lib.rsindex 275330c99e..87fc661fdd 100644--- a/rust/crates/photos-export/src/lib.rs+++ b/rust/crates/photos-export/src/lib.rs@@ -1,5 +1,6 @@mod adopt;mod fs;+mod legacy;mod live_photo;mod metadata;mod names;@@ -59,9 +60,10 @@ pub struct Summary {pub struct Export {destination: PathBuf,expected_root: Root,+ prepared: Option<(Store, Result<()>)>,root_lock: Option<File>,options: Options,- prepared: Option<(Store, Result<()>)>,+ adopting: bool,}impl Export {@@ -73,16 +75,31 @@ impl Export {let destination = destination(path)?;let expected_root = Root::new(master_key)?;let root_lock = open_root(&destination, &expected_root)?;+ let adopting = root_lock.is_none() && legacy::read::recognized(&destination)?;+ ensure!(+ !adopting || options.adopt,+ AdoptionRequired(+ "This folder contains an export from the old CLI or Ente Desktop. Use --adopt if you want to take it over."+ )+ );ensure!(- !options.adopt || root_lock.is_some(),+ !options.adopt || root_lock.is_some() || adopting,"there is no export to adopt");+ ensure!(+ root_lock.is_some()+ || adopting+ || !destination.try_exists()?+ || std::fs::read_dir(&destination)?.next().is_none(),+ "nonempty destination has no recognized export.json"+ );Ok(Self {destination,expected_root,root_lock,options,prepared: None,+ adopting,})}@@ -105,6 +122,7 @@ impl Export {pub async fn prepare(&mut self,connection: rusqlite::Connection,+ open_index: impl FnOnce(&Path) -> Result<rusqlite::Connection>,source: &mut impl Source,session: &ente_core::Session,cancel: &mut watch::Receiver<bool>,@@ -114,13 +132,44 @@ impl Export {let bound: bool = store.db.query_row("SELECT bound FROM export WHERE id=1", [], |r| r.get(0))?;+ let accepted = !bound+ && store.db.query_row(+ "SELECT EXISTS(SELECT 1 FROM albums WHERE retained=0)",+ [],+ |r| r.get::<_, bool>(0),+ )?;ensure!(- self.root_lock.is_none() || bound || self.options.adopt,- AdoptionRequired("this export is not associated")+ self.root_lock.is_none() || bound || accepted || self.options.adopt,+ AdoptionRequired("this export is not associated; use --adopt"));+ let initial = !bound && !accepted && self.options.adopt;+ let index = initial.then(|| adopt::Index::new(open_index)).transpose()?;store.reset_desired()?;- if self.root_lock.is_some() && !bound {- adopt::scan(&self.destination, &store)?;+ let mut failures = legacy::Failures::default();+ self.adopting = !bound+ && (accepted+ || (initial && (self.adopting || legacy::read::recognized(&self.destination)?)));+ if let Some(index) = &index {+ let mut desktop = None;+ failures.record(+ legacy::read::desktop(&self.destination).map(|value| desktop = value),+ "export_status.json",+ )?;+ if self.root_lock.is_some() {+ failures.record(+ adopt::scan(&self.destination, index, desktop.as_ref()),+ "native records",+ )?;+ }+ if self.adopting {+ legacy::read::scan(+ &self.destination,+ index,+ session.user_id,+ desktop.as_ref(),+ &mut failures,+ )?;+ }}report("Refreshing album metadata.");tokio::select! {@@ -157,19 +206,66 @@ impl Export {"INSERT OR IGNORE INTO desired_albums(id,name) SELECT album,name FROM pending WHERE file IS NULL AND retained=0",[],)?;- select(&store, &self.options)?;- if self.root_lock.is_none() {- self.root_lock = Some(initialize(&self.destination, &self.expected_root)?);+ if let Some(index) = &index {+ let mut albums = index.db.prepare("SELECT id,json_extract(record,'$.album.name') FROM adoption_album ORDER BY retained")?;+ for row in+ albums.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)))?+ {+ let (id, name) = row?;+ store.db.execute(+ "INSERT OR IGNORE INTO desired_albums(id,name) VALUES(?1,?2)",+ params![id, name],+ )?;+ }+ }+ if self.adopting {+ legacy::scope(&store, &self.options, index.as_ref())?;+ } else {+ failures.finish()?;+ select(&store, &self.options)?;+ }+ if !self.adopting {+ if self.root_lock.is_none() {+ self.root_lock = Some(initialize(&self.destination, &self.expected_root, false)?);+ }+ if !initial {+ store+ .db+ .execute("UPDATE export SET bound=1 WHERE id=1 AND bound=0", [])?;+ }}- store- .db- .execute("UPDATE export SET bound=1 WHERE id=1 AND bound=0", [])?;-let refresh_cancel = cancel.clone();let refreshed = tokio::select! {result=refresh(&store,source,session,&refresh_cancel,report)=>result,_=cancel.changed()=>Err(Cancelled.into()),};+ if self.adopting {+ refreshed?;+ if let Some(index) = &index {+ report("Verifying every existing active legacy original before adoption.");+ legacy::Preflight {+ root: &self.destination,+ store: &store,+ session,+ index,+ cancel,+ matching_root: self.root_lock.is_some(),+ }+ .run(source, failures)+ .await?;+ if self.root_lock.is_none() {+ self.root_lock =+ Some(initialize(&self.destination, &self.expected_root, true)?);+ }+ legacy::move_leftovers(&self.destination, index, cancel)?;+ adopt::accept(index, &store, false)?;+ }+ self.prepared = Some((store, Ok(())));+ return Ok(());+ }+ if let Some(index) = &index {+ adopt::accept(index, &store, true)?;+ }self.prepared = Some((store, refreshed));Ok(())}@@ -215,6 +311,13 @@ impl Export {let store = shared.into_inner().map_err(|_| anyhow::anyhow!("export database mutex poisoned"))?;+ if self.adopting+ && work_complete(&store, prepared)?+ && let Err(error) = legacy::finish(&self.destination, &store, cancel)+ {+ record_outcome(&store, "run", &error)?;+ report(&format!("run: {error:#}"));+ }summary(&store, &self.destination, prepared)}}@@ -260,10 +363,6 @@ fn open_root(destination: &Path, expected: &Root) -> Result<Option<File>> {);let path = names::check(destination, "export.json")?;if !path.try_exists()? {- ensure!(- std::fs::read_dir(destination)?.next().is_none(),- "nonempty destination has no recognized export.json"- );return Ok(None);}let file = OpenOptions::new().read(true).write(true).open(&path)?;@@ -281,7 +380,7 @@ fn open_root(destination: &Path, expected: &Root) -> Result<Option<File>> {Ok(Some(file))}-fn initialize(destination: &Path, root: &Root) -> Result<File> {+fn initialize(destination: &Path, root: &Root, legacy: bool) -> Result<File> {fs::create_directory(destination)?;let path = names::check(destination, "export.json")?;let mut file = OpenOptions::new()@@ -292,8 +391,9 @@ fn initialize(destination: &Path, root: &Root) -> Result<File> {.context("destination was initialized by another writer; retry")?;file.lock().context("cannot lock the new export")?;ensure!(- std::fs::read_dir(destination)?- .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),+ legacy+ || std::fs::read_dir(destination)?+ .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),"destination became nonempty during initialization");serde_json::to_writer_pretty(&mut file, root)?;@@ -581,8 +681,7 @@ fn transfers("UPDATE desired_files SET claimed=1 WHERE file=?1",[id],)?;- let file: FileSnapshot = store- .json("SELECT record FROM sources WHERE file=?1", [id])?+ let file: FileSnapshot = store::json(&store.db,"SELECT record FROM sources WHERE file=?1", [id])?.context("missing file source snapshot")?;transaction.commit()?;file@@ -684,6 +783,10 @@ fn fatal_cause(cause: &(dyn std::error::Error + 'static)) -> bool {cause.source().is_some_and(fatal_cause)}+fn work_complete(store: &Store, prepared: bool) -> Result<bool> {+ Ok(prepared && store.db.query_row("SELECT NOT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND (ready=0 OR failure IS NOT NULL)) AND NOT EXISTS(SELECT 1 FROM desired_files WHERE completed=0 OR failure IS NOT NULL) AND NOT EXISTS(SELECT 1 FROM outcomes) AND NOT EXISTS(SELECT 1 FROM pending p JOIN desired_albums a ON a.id=p.album WHERE a.selected=1)",[],|r|r.get::<_,bool>(0))?)+}+fn summary(store: &Store, destination: &Path, prepared: bool) -> Result<Summary> {let unknown: bool = !prepared|| store.db.query_row(@@ -708,7 +811,10 @@ fn summary(store: &Store, destination: &Path, prepared: bool) -> Result<Summary>[],|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),)?;- let complete = !unknown && failures == 0 && completed == expected;+ let bound: bool = store+ .db+ .query_row("SELECT bound FROM export WHERE id=1", [], |r| r.get(0))?;+ let complete = bound && work_complete(store, prepared)?;let result = json!({"destination":destination.to_string_lossy(),"complete":complete,"files":if unknown {None}else{Some(files)},"copies":if unknown {None}else{Some(json!({"expected":expected,"completed":completed,"pending":expected-completed}))},"changes":{"exported":exported,"metadataUpdated":metadata_updated,"renamed":renamed,"retained":retained},"failures":failures,"conflicts":conflicts});Ok(Summary {json: result,diff --git a/rust/crates/photos-export/src/names.rs b/rust/crates/photos-export/src/names.rsindex dd23cf0a9d..32215593dc 100644--- a/rust/crates/photos-export/src/names.rs+++ b/rust/crates/photos-export/src/names.rs@@ -156,6 +156,13 @@ pub fn available(Ok(true)}+pub fn valid(name: &str, album: bool) -> bool {+ !name.is_empty()+ && portable(name) == name+ && !reserved(name, album)+ && name.len() <= if album { 255 } else { 250 }+}+pub fn allocate(store: &Store,folder: &str,diff --git a/rust/crates/photos-export/src/reconcile.rs b/rust/crates/photos-export/src/reconcile.rsindex aa51f92e52..1e1ea244c6 100644--- a/rust/crates/photos-export/src/reconcile.rs+++ b/rust/crates/photos-export/src/reconcile.rs@@ -278,6 +278,54 @@ impl Run<'_> {Ok(())}+ fn verify_pending_media(&self, target: &Pending, action: &Action) -> Result<()> {+ let components = store::lock(self.store)?.components(&target.owner)?;+ for component in components {+ if target.retained && component.location.folder == target.folder {+ continue;+ }+ let mut path = self.path(&component.location)?;+ match action {+ Action::Move {+ source,+ destination,+ role,+ ..+ } if *role == component.role => {+ if Properties::optional(&self.path(source)?)?.is_none() {+ if target.retained {+ continue;+ }+ path = self.path(destination)?;+ }+ }+ Action::Publish {+ temporary,+ destination,+ output:+ Output::Media {+ role, size, hash, ..+ },+ } if *role == component.role => {+ path = self.path(destination)?;+ if Properties::optional(&self.path(temporary)?)?.is_none()+ && Properties::optional(&path)?.is_some()+ && fs::matches(&path, *size, hash, self.cancel)?+ {+ continue;+ }+ }+ _ => {}+ }+ ensure!(+ Properties::optional(&path)?.is_none()+ || fs::matches(&path, component.size, &component.hash, self.cancel)?,+ super::Conflict(format!("at {}: locally changed media", path.display()))+ );+ }+ Ok(())+ }+fn finish_action(&self,target: &mut Pending,@@ -288,13 +336,30 @@ impl Run<'_> {return Ok(());};self.check_cancel()?;+ let current_media = recovering+ && matches!(+ action,+ Action::Move { .. }+ | Action::Publish {+ output: Output::Media { .. },+ ..+ }+ )+ && store::lock(self.store)?.db.query_row(+ "SELECT EXISTS(SELECT 1 FROM desired_files WHERE album=?1 AND file=?2 AND failure IS NULL)",+ params![target.album, target.file],+ |r| r.get::<_, bool>(0),+ )?;+ if current_media {+ self.verify_pending_media(target, &action)?;+ }match action {Action::Directory { source } => {let destination = names::check(self.root, &target.album_path())?;if let Some(source) = source {let source = names::check(self.root, &source)?;if directory_exists(&source)? {- free_rename(&source, &destination)?;+ fs::free_rename(&source, &destination)?;disk::rename(&source, &destination)?;} else if !directory_exists(&destination)? {let unrepairable: bool = store::lock(self.store)?@@ -389,22 +454,21 @@ impl Run<'_> {let from = self.path(&source)?;let to = self.path(&destination)?;if Properties::optional(&from)?.is_some() {- free_rename(&from, &to)?;- fs::create_directory(to.parent().context("move has no parent")?)?;- disk::rename(&from, &to)?;+ fs::move_file(&from, &to)?;} else if Properties::optional(&to)?.is_some() {ensure!(- fs::matches(&to, size, &hash, self.cancel)?,+ (recovering && (target.retained || !current_media))+ || fs::matches(&to, size, &hash, self.cancel)?,super::Conflict(format!("at {}: moved bytes differ", to.display())));+ fs::sync_move(&from, &to)?;} else {ensure!(- !target.retained,+ !target.retained || current_media,"retained original is missing at both move endpoints");return self.clear_action(target);}- fs::sync_move(&from, &to)?;let store = store::lock(self.store)?;let transaction = store.db.unchecked_transaction()?;let mut component = store@@ -553,7 +617,7 @@ impl Run<'_> {}let from = self.path(&component.location)?;let to = self.path(&destination)?;- free_rename(&from, &to)?;+ fs::free_rename(&from, &to)?;let properties = Properties::optional(&from)?;if properties.is_none() {ensure!(!target.retained, "previously published original is missing");@@ -594,17 +658,6 @@ fn directory_exists(path: &Path) -> Result<bool> {}}-fn free_rename(source: &Path, destination: &Path) -> Result<()> {- ensure!(- !destination.try_exists()? || names::same_path(source, destination),- super::Conflict(format!(- "at {}: occupied rename target",- destination.display()- ))- );- Ok(())-}-impl Run<'_> {pub fn recover(&self) -> Result<()> {for albums in [true, false] {@@ -752,15 +805,11 @@ impl Run<'_> {let parent = if retained { "Trash" } else { "" };let names = if let Some(existing) = &existing&& (retained || existing.name == name)+ && let Some(name) = existing.path.rsplit('/').next()+ && names::valid(name, true)+ && names::available(&store, parent, &[name.to_owned()], &owner, "album")?{- vec![- existing- .path- .rsplit('/')- .next()- .context("empty album path")?- .to_owned(),- ]+ vec![name.to_owned()]} else {names::allocate(&store, parent, name, "album", None, &owner)?};@@ -820,7 +869,7 @@ impl Run<'_> {let source = existing.as_ref().map(|album| album.path.clone());let destination = names::check(self.root, &target_path)?;if let Some(source) = &source {- free_rename(&names::check(self.root, source)?, &destination)?;+ fs::free_rename(&names::check(self.root, source)?, &destination)?;} else {ensure!(!destination.try_exists()?,@@ -925,6 +974,7 @@ impl Run<'_> {.collect::<Result<_>>()?;let compatible = |names: &[String]| {names.len() == roles.len()+ && names.iter().all(|name| names::valid(name, false))&& (roles.len() == 1|| names.iter().zip(&extensions).all(|(name, extension)| {names::split(name, false).1 == names::portable(extension)@@ -940,6 +990,7 @@ impl Run<'_> {&& previous.name == file.name&& previous.kind == file.kind.as_str()&& compatible(&previous.names)+ && names::available(&store, &album.key, &previous.names, &owner, &file.kind)?{return Ok(previous);}@@ -953,7 +1004,8 @@ impl Run<'_> {}) && components.iter().all(|component| component.location.folder == album.key)- && compatible(&existing_names);+ && compatible(&existing_names)+ && names::available(&store, &album.key, &existing_names, &owner, &file.kind)?;let transaction = store.db.unchecked_transaction()?;let names = if unchanged {existing_names@@ -1339,10 +1391,6 @@ impl Run<'_> {}}let components = store::lock(self.store)?.components(&placement.id)?;- for component in &components {- Properties::read(&self.path(&component.location)?)- .context("previously published original is missing")?;- }let active = store::lock(self.store)?.album(placement.album, false)?.context("missing active album")?;@@ -1409,6 +1457,11 @@ impl Run<'_> {}fn finish_retention(&self, target: &mut Pending) -> Result<()> {+ let restoring: bool = store::lock(self.store)?.db.query_row(+ "SELECT EXISTS(SELECT 1 FROM desired_files WHERE album=?1 AND file=?2 AND failure IS NULL)",+ params![target.album, target.file],+ |r| r.get(0),+ )?;let components = store::lock(self.store)?.components(&target.owner)?;for component in &components {let name = target_component(target, &component.role)?;@@ -1416,12 +1469,24 @@ impl Run<'_> {folder: target.folder.clone(),name: name.into(),};- self.move_media(target, component, location, component.properties.as_ref())?;+ let from = self.path(&component.location)?;+ if restoring && Properties::optional(&from)?.is_none() {+ fs::free_rename(&from, &self.path(&location)?)?;+ let mut missing = component.clone();+ missing.location = location;+ missing.properties = None;+ missing.intended_time = None;+ store::lock(self.store)?.save_component(&missing)?;+ } else {+ self.move_media(target, component, location, component.properties.as_ref())?;+ }}let components = store::lock(self.store)?.components(&target.owner)?;for component in &components {- Properties::read(&self.path(&component.location)?)- .context("retained original is missing")?;+ if !restoring {+ Properties::read(&self.path(&component.location)?)+ .context("retained original is missing")?;+ }let previous = store::lock(self.store)?.json_record(&target.owner, Some(&component.role))?.context("missing retained metadata")?;@@ -1536,9 +1601,21 @@ impl Run<'_> {|r| r.get(0),)?;ensure!(!incomplete, "album has unfinished retention");- self.retained_album(&active)?;+ let (bound, record) = {+ let store = store::lock(self.store)?;+ (+ store+ .db+ .query_row("SELECT bound FROM export WHERE id=1", [], |r| {+ r.get::<_, bool>(0)+ })?,+ store.json_record(&active.key, None)?,+ )+ };+ if bound || record.is_some() {+ self.retained_album(&active)?;+ }let mut target = self.album_target(id, &active.name, false)?;- let record = store::lock(self.store)?.json_record(&active.key, None)?;if let Some(record) = record&& let Some(location) = record.location{@@ -1564,7 +1641,7 @@ impl Run<'_> {store.db.execute(- "DELETE FROM albums WHERE key=?1 AND NOT EXISTS(SELECT 1 FROM temporaries WHERE folder=?1)",+ "DELETE FROM albums WHERE key=?1 AND (SELECT bound FROM export WHERE id=1)=1 AND NOT EXISTS(SELECT 1 FROM temporaries WHERE folder=?1)",[&active.key],)?;store.clear_pending(&target.owner)?;diff --git a/rust/crates/photos-export/src/store.rs b/rust/crates/photos-export/src/store.rsindex e05d491efc..0ca3350b41 100644--- a/rust/crates/photos-export/src/store.rs+++ b/rust/crates/photos-export/src/store.rs@@ -23,7 +23,17 @@ pub fn lock(store: &Mutex<Store>) -> Result<MutexGuard<'_, Store>> {.map_err(|_| anyhow::anyhow!("export database mutex poisoned"))}-#[derive(Clone)]+pub fn json<T: DeserializeOwned>(+ db: &Connection,+ sql: &str,+ parameters: impl rusqlite::Params,+) -> Result<Option<T>> {+ let text: Option<String> = db.query_row(sql, parameters, |r| r.get(0)).optional()?;+ text.map(|text| serde_json::from_str(&text).map_err(Into::into))+ .transpose()+}++#[derive(Clone, Serialize, Deserialize)]pub struct Album {pub key: String,pub id: i64,@@ -44,7 +54,7 @@ impl Album {}}-#[derive(Clone)]+#[derive(Clone, Serialize, Deserialize)]pub struct Placement {pub id: String,pub album: i64,@@ -73,7 +83,7 @@ pub struct Location {pub name: String,}-#[derive(Clone, PartialEq, Eq)]+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct Component {pub placement: String,pub role: Role,@@ -158,7 +168,7 @@ pub fn system_time(time: SystemTime) -> i128 {}}-#[derive(Clone)]+#[derive(Clone, Serialize, Deserialize)]pub struct JsonRecord {pub owner: String,pub role: Option<Role>,@@ -334,7 +344,7 @@ impl Store {if let Some(existing) = existing {ensure!(existing == destination.as_os_str().as_encoded_bytes(),- AdoptionRequired("destination association does not match")+ AdoptionRequired("destination association does not match; use --adopt"));} else {connection.execute(@@ -535,19 +545,6 @@ impl Store {Ok(())}- pub fn json<T: DeserializeOwned>(- &self,- sql: &str,- parameters: impl rusqlite::Params,- ) -> Result<Option<T>> {- let text: Option<String> = self- .db- .query_row(sql, parameters, |r| r.get(0))- .optional()?;- text.map(|text| serde_json::from_str(&text).map_err(Into::into))- .transpose()- }-pub fn temporary(&self, temporary: &Temporary) -> Result<()> {self.db.execute("INSERT INTO temporaries VALUES(?1,?2,?3,?4)",diff --git a/rust/crates/photos-export/src/transfer.rs b/rust/crates/photos-export/src/transfer.rsindex b5f970e16c..d633b28bf6 100644--- a/rust/crates/photos-export/src/transfer.rs+++ b/rust/crates/photos-export/src/transfer.rs@@ -39,29 +39,26 @@ pub fn roles(file: &FileSnapshot) -> Vec<Role> {}}-pub fn verify(- run: &Run<'_>,- component: &Component,- file: &FileSnapshot,- signature: &str,-) -> Result<Option<Properties>> {- let path = run.path(&component.location)?;- let Some(properties) = Properties::optional(&path)? else {- return Ok(None);- };- if expected_hash(file, &component.role).is_some_and(|hash| hash != component.hash)+fn source_changed(component: &Component, file: &FileSnapshot, signature: &str) -> bool {+ !roles(file).contains(&component.role)+ || expected_hash(file, &component.role).is_some_and(|hash| hash != component.hash)|| (expected_hash(file, &component.role).is_none()&& component.signature.as_ref().is_some_and(|original| original != signature))- {- return Ok(None);- }- if component.properties.as_ref() != Some(&properties)- && !fs::matches(&path, component.size, &component.hash, run.cancel)?- {+}++fn verify(run: &Run<'_>, component: &Component, replacing: bool) -> Result<Option<Properties>> {+ let path = run.path(&component.location)?;+ let Some(properties) = Properties::optional(&path)? else {return Ok(None);+ };+ if replacing || component.properties.as_ref() != Some(&properties) {+ ensure!(+ fs::matches(&path, component.size, &component.hash, run.cancel)?,+ super::Conflict(format!("at {}: locally changed media", path.display()))+ );}Ok(Some(properties))}@@ -87,23 +84,30 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> R};after = album;let result = (|| {- let components = {+ let (components, retained_folder) = {let store = store::lock(run.store)?;- needs_bytes |= store+ let retained_folder = store.pending_file(album, file.id)?- .is_some_and(|pending| pending.retained);- store+ .filter(|pending| pending.retained)+ .map(|pending| pending.folder);+ let components = store.placement(album, file.id)?.map(|placement| store.components(&placement.id)).transpose()?- .unwrap_or_default()+ .unwrap_or_default();+ (components, retained_folder)};let mut verified = Vec::new();+ let replacing = components+ .iter()+ .any(|component| source_changed(component, file, &signature));for component in &components {- if !roles.contains(&component.role) {+ if retained_folder.as_ref() == Some(&component.location.folder) {continue;}- if let Some(properties) = verify(run, component, file, &signature)? {+ if let Some(properties) = verify(run, component, replacing)?+ && !source_changed(component, file, &signature)+ {verified.push((component.role.clone(), properties));}}