Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Adopt old Photos exports and protect local edits

Manav Rathi306a0dc523ce
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index a0530d87d9..fcbbf527c1 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2861,6 +2861,7 @@ dependencies = [
"rusqlite",
"serde",
"serde_json",
+ "tempfile",
"thiserror 2.0.21",
"tokio",
"unicode-normalization",
diff --git a/rust/apps/cli-next/src/args.rs b/rust/apps/cli-next/src/args.rs
index 27b8d4cb52..0bc782bafb 100644
--- a/rust/apps/cli-next/src/args.rs
+++ b/rust/apps/cli-next/src/args.rs
@@ -141,7 +141,7 @@ pub struct ExportArgs {
pub exclude_album: Vec<String>,
#[arg(
long,
- help = "Reconnect an existing export after moving it or losing local state"
+ help = "Take over an old CLI/Desktop export or reconnect an existing one"
)]
pub adopt: bool,
#[arg(
diff --git a/rust/apps/cli-next/src/db.rs b/rust/apps/cli-next/src/db.rs
index 779c6befff..e754fa3da2 100644
--- a/rust/apps/cli-next/src/db.rs
+++ b/rust/apps/cli-next/src/db.rs
@@ -42,3 +42,31 @@ pub fn connect(path: &Path, key: &DbKey, create: bool) -> Result<Connection> {
.context("cannot open the encrypted database")?;
Ok(connection)
}
+
+pub fn scratch(path: &Path) -> Result<Connection> {
+ connect(
+ path,
+ &DbKey(*ente_core::crypto::Key::generate().as_bytes()),
+ true,
+ )
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn scratch_pages_require_the_discarded_attempt_key() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("index.db");
+ let db = scratch(&path).unwrap();
+ db.execute_batch(
+ "CREATE TABLE names(name TEXT); INSERT INTO names VALUES('private export album');",
+ )
+ .unwrap();
+ drop(db);
+ let bytes = std::fs::read(&path).unwrap();
+ assert!(!bytes.starts_with(b"SQLite format 3"));
+ assert!(scratch(&path).is_err());
+ }
+}
diff --git a/rust/apps/cli-next/src/export/mod.rs b/rust/apps/cli-next/src/export/mod.rs
index b38dd62f0b..832c6dbc69 100644
--- a/rust/apps/cli-next/src/export/mod.rs
+++ b/rust/apps/cli-next/src/export/mod.rs
@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
use anyhow::{Context, Result, ensure};
use ente_core::crypto::hash;
-use ente_photos_export::{AdoptionRequired, Export};
+use ente_photos_export::Export;
use tokio::sync::watch;
use crate::{
@@ -26,13 +26,6 @@ pub async fn run(args: ExportArgs, selected: Option<&str>, options: &Options) ->
runtime.block_on(run_locked(args, selected.as_deref(), as_json))
})
.await?
- .map_err(|error| {
- if error.is::<AdoptionRequired>() {
- error.context("this export requires --adopt")
- } else {
- error
- }
- })
}
async fn run_locked(args: ExportArgs, selected: Option<&str>, as_json: bool) -> Result<()> {
@@ -81,7 +74,14 @@ async fn run_locked(args: ExportArgs, selected: Option<&str>, as_json: bool) ->
let mut source_db = db::open(&account_home.path, &account.db_key, true)?;
let mut source = source::ReplicaSource::new(&mut source_db, session.user_id);
export
- .prepare(connection, &mut source, &session, &mut cancel, &report)
+ .prepare(
+ connection,
+ db::scratch,
+ &mut source,
+ &session,
+ &mut cancel,
+ &report,
+ )
.await?;
drop(source);
drop(source_db);
diff --git a/rust/apps/cli-next/tests/fixtures/export-adoption/desktop-file.json b/rust/apps/cli-next/tests/fixtures/export-adoption/desktop-file.json
new file mode 100644
index 0000000000..3c62654089
--- /dev/null
+++ b/rust/apps/cli-next/tests/fixtures/export-adoption/desktop-file.json
@@ -0,0 +1,15 @@
+{
+ "title": "Photo.jpg",
+ "photoTakenTime": {
+ "timestamp": "1791179341",
+ "formatted": "Oct 5, 2026, 5:49:01 AM UTC"
+ },
+ "creationTime": {
+ "timestamp": "1791179341",
+ "formatted": "Oct 5, 2026, 5:49:01 AM UTC"
+ },
+ "modificationTime": {
+ "timestamp": "1791179341",
+ "formatted": "Oct 5, 2026, 5:49:01 AM UTC"
+ }
+}
diff --git a/rust/apps/cli-next/tests/fixtures/export-adoption/desktop-manifest.json b/rust/apps/cli-next/tests/fixtures/export-adoption/desktop-manifest.json
new file mode 100644
index 0000000000..8a748e29fb
--- /dev/null
+++ b/rust/apps/cli-next/tests/fixtures/export-adoption/desktop-manifest.json
@@ -0,0 +1,18 @@
+{
+ "version": 5,
+ "stage": 7,
+ "lastAttemptTimestamp": 1791186928697,
+ "collectionExportNames": {
+ "1580559962386472": "Travel",
+ "1580559962386474": "Live Pair",
+ "1580559962386473": "Family"
+ },
+ "fileExportNames": {
+ "10000064_1580559962386472_1791179545995470": "Photo.jpg",
+ "10000066_1580559962386474_1791179670357138": "{\"image\":\"Pair.jpg\",\"video\":\"Pair.mov\"}",
+ "10000062_1580559962386473_1791179545928006": "Photo.jpg",
+ "10000063_1580559962386473_1791179545987020": "Live.jpg",
+ "10000065_1580559962386473_1791179546000091": "Live.mov",
+ "10000066_1580559962386473_1791179689874921": "{\"image\":\"Pair.jpg\",\"video\":\"Pair.mov\"}"
+ }
+}
diff --git a/rust/apps/cli-next/tests/fixtures/export-adoption/go-album.json b/rust/apps/cli-next/tests/fixtures/export-adoption/go-album.json
new file mode 100644
index 0000000000..a1fbc9c7a4
--- /dev/null
+++ b/rust/apps/cli-next/tests/fixtures/export-adoption/go-album.json
@@ -0,0 +1,9 @@
+{
+ "id": 1580559962386473,
+ "ownerID": 1580559962386471,
+ "albumName": "Family",
+ "isDeleted": false,
+ "accountOwnerIDs": [
+ 1580559962386471
+ ]
+}
diff --git a/rust/apps/cli-next/tests/fixtures/export-adoption/go-file.json b/rust/apps/cli-next/tests/fixtures/export-adoption/go-file.json
new file mode 100644
index 0000000000..4a1e60d164
--- /dev/null
+++ b/rust/apps/cli-next/tests/fixtures/export-adoption/go-file.json
@@ -0,0 +1,15 @@
+{
+ "title": "Photo.jpg",
+ "description": null,
+ "location": null,
+ "creationTime": "2026-10-05T11:19:01.464+05:30",
+ "modificationTime": "2026-10-05T11:19:01.464+05:30",
+ "info": {
+ "id": 10000062,
+ "hash": null,
+ "ownerID": 1580559962386471,
+ "fileNames": [
+ "Photo.jpg"
+ ]
+ }
+}
diff --git a/rust/apps/cli-next/tests/support/export.rs b/rust/apps/cli-next/tests/support/export.rs
index f5264ddb83..513007950c 100644
--- a/rust/apps/cli-next/tests/support/export.rs
+++ b/rust/apps/cli-next/tests/support/export.rs
@@ -162,8 +162,11 @@ fn export_metadata_updates_and_repairs_preserve_source_values() -> TestResult {
}
export(&home, &root, &["--album", "Travel"]);
place(origin,&account,favorites,&favorites_key,file,&key,"add-files").await;
+ let mut edited_metadata=json!({"editedName":"Renamed.jpg","editedTime":1_700_000_002_654_321i64,"caption":"","lat":0,"long":0,"dateTime":"2023:11:15 03:43:22","offsetTime":"+05:30","w":4096,"h":3072,"cameraMake":"Example","cameraModel":"Camera","uploaderName":"Guest","mediaType":1,"mvi":123});
for (version,lat,long) in [(1,0,77),(2,12,0),(3,0,0)] {
- edit_file(origin,&account,file,&key,version,json!({"editedName":"Renamed.jpg","editedTime":1_700_000_002_654_321i64,"caption":"","lat":lat,"long":long,"dateTime":"2023:11:15 03:43:22","offsetTime":"+05:30","w":4096,"h":3072,"cameraMake":"Example","cameraModel":"Camera","uploaderName":"Guest","mediaType":1,"mvi":123})).await;
+ edited_metadata["lat"]=json!(lat);
+ edited_metadata["long"]=json!(long);
+ edit_file(origin,&account,file,&key,version,edited_metadata.clone()).await;
export(&home,&root,&["--album","Family"]);
let record=read_json(&root.join("Family/metadata/Renamed.jpg.json"));
assert_eq!(record["description"],"");
@@ -187,7 +190,29 @@ fn export_metadata_updates_and_repairs_preserve_source_values() -> TestResult {
export(&home,&root,&["--album","Family"]);
assert!(read_json(&root.join("Family/metadata/Renamed.jpg.json")).get("custom").is_none());
fs::write(root.join("Family/Renamed.jpg"),vec![0;original.len()])?;
- export(&home,&root,&["--album","Family"]);
+ let db=export_database(&home);
+ let baseline: String=db.query_row("SELECT hash FROM components WHERE folder=?1",[format!("active:{family}")],|r|r.get(0))?;
+ let state=home.read_vault();
+ let saved=&state["accounts"][0];
+ let replica=rusqlite::Connection::open(home.dir.path().join("accounts").join(saved["storage_id"].as_str().unwrap()).join("data.db"))?;
+ let db_key: Vec<u8>=serde_json::from_value(saved["db_key"].clone())?;
+ let hex: String=db_key.iter().map(|byte|format!("{byte:02x}")).collect();
+ replica.pragma_update(None,"key",format!("x'{hex}'"))?;
+ let before: i64=replica.query_row("SELECT files_cursor FROM photos_collections WHERE id=?1",[family],|r|r.get(0))?;
+ edited_metadata["caption"]=json!("source advanced");
+ edit_file(origin,&account,file,&key,4,edited_metadata).await;
+ let output=home.run(&["photos","export",root.to_str().unwrap(),"--json"]);
+ assert!(failure(&output).contains("locally changed media"));
+ let result: Value=serde_json::from_slice(&output.stdout)?;
+ assert_eq!(result["complete"],false);
+ assert_eq!(result["conflicts"],1);
+ assert_eq!(fs::read(root.join("Family/Renamed.jpg"))?,vec![0;original.len()]);
+ assert_eq!(read_json(&root.join("Travel/metadata/Renamed.jpg.json"))["description"],"source advanced");
+ let after: i64=replica.query_row("SELECT files_cursor FROM photos_collections WHERE id=?1",[family],|r|r.get(0))?;
+ assert!(after>before);
+ assert_eq!(db.query_row("SELECT hash FROM components WHERE folder=?1",[format!("active:{family}")],|r|r.get::<_,String>(0))?,baseline);
+ fs::rename(root.join("Family/Renamed.jpg"),destination.path().join("local-edit.jpg"))?;
+ export(&home,&root,&[]);
assert_eq!(fs::read(root.join("Family/Renamed.jpg"))?,original);
assert_eq!(objects.reads(),initial_reads+1);
Ok(())
@@ -365,6 +390,22 @@ fn export_adopts_relocated_output_and_verifies_later_selections() -> TestResult
assert_eq!(objects.reads(), reads);
assert_eq!(fs::read(relocated.join("export.json"))?, root_bytes);
fs::write(relocated.join("Work/Work.jpg"), b"xxxx")?;
+ let output = fresh.run(&[
+ "photos",
+ "export",
+ relocated.to_str().unwrap(),
+ "--album",
+ "Work",
+ "--json",
+ ]);
+ assert!(failure(&output).contains("locally changed media"));
+ assert_eq!(
+ serde_json::from_slice::<Value>(&output.stdout)?["conflicts"],
+ 1
+ );
+ assert_eq!(fs::read(relocated.join("Work/Work.jpg"))?, b"xxxx");
+ assert_eq!(objects.reads(), reads);
+ fs::remove_file(relocated.join("Work/Work.jpg"))?;
export(&fresh, &relocated, &["--album", "Work"]);
assert_eq!(objects.reads(), reads + 1);
assert_eq!(fs::read(relocated.join("Work/Work.jpg"))?, b"work");
@@ -1676,3 +1717,76 @@ async fn export_account(origin: &str) -> (AuthenticatedAccount, TestHome, String
login(&home, "photos", &email, &["--host", origin]);
(account, home, email)
}
+
+#[test]
+fn export_adopts_go_and_desktop_fixtures_with_a_real_source_account() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let (account, home, _) = export_account(origin).await;
+ let (album, key) = create_album(origin, &account, "Family", "album").await;
+ let fixtures =
+ Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/export-adoption");
+ let bytes = b"adoption integration original".to_vec();
+ let (file, _) = upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ &bytes,
+ metadata("Photo.jpg", &bytes),
+ )
+ .await;
+ for producer in ["go", "desktop"] {
+ let directory = tempfile::tempdir()?;
+ let root = directory.path().join("photos");
+ fs::create_dir_all(root.join("Family"))?;
+ fs::write(root.join("Family/Photo.jpg"), &bytes)?;
+ if producer == "go" {
+ fs::create_dir_all(root.join("Family/.meta"))?;
+ let mut record = read_json(&fixtures.join("go-album.json"));
+ record["id"] = json!(album);
+ record["ownerID"] = json!(account.user_id);
+ record["accountOwnerIDs"] = json!([account.user_id]);
+ fs::write(
+ root.join("Family/.meta/album_meta.json"),
+ serde_json::to_vec(&record)?,
+ )?;
+ let mut record = read_json(&fixtures.join("go-file.json"));
+ record["info"]["id"] = json!(file);
+ record["info"]["ownerID"] = json!(account.user_id);
+ record["info"]["hash"] = json!(digest(&bytes));
+ fs::write(
+ root.join("Family/.meta/Photo.jpg.json"),
+ serde_json::to_vec(&record)?,
+ )?;
+ } else {
+ let mut record =
+ json!({"version":5,"collectionExportNames":{},"fileExportNames":{}});
+ record["collectionExportNames"][album.to_string()] = json!("Family");
+ record["fileExportNames"][format!("{file}_{album}_100")] = json!("Photo.jpg");
+ fs::write(
+ root.join("export_status.json"),
+ serde_json::to_vec(&record)?,
+ )?;
+ }
+ let reads = objects.reads();
+ let output = home.run(&["photos", "export", root.to_str().unwrap()]);
+ assert_eq!(
+ failure(&output),
+ "Error: This folder contains an export from the old CLI or Ente Desktop. Use --adopt if you want to take it over.\n"
+ );
+ let result = export(&home, &root, &["--adopt", "--album", "Family"]);
+ assert_eq!(result["copies"]["completed"], 1);
+ assert_eq!(objects.reads(), reads);
+ assert_eq!(fs::read(root.join("Family/Photo.jpg"))?, bytes);
+ assert_eq!(
+ read_json(&root.join("Family/metadata/Photo.jpg.json"))["ente"]["fileID"],
+ file.to_string()
+ );
+ }
+ Ok(())
+ })
+ })
+}
diff --git a/rust/apps/cli-next/tests/support/export_adoption.rs b/rust/apps/cli-next/tests/support/export_adoption.rs
new file mode 100644
index 0000000000..4792a99834
--- /dev/null
+++ b/rust/apps/cli-next/tests/support/export_adoption.rs
@@ -0,0 +1,1029 @@
+use super::engine::{Catalog, Engine};
+use super::*;
+use ente_photos::files;
+use std::{collections::BTreeMap, path::PathBuf};
+
+const USER: i64 = 1580559962386471;
+const FAMILY: i64 = 1580559962386473;
+const PAIR: i64 = 10000066;
+const PHOTO: i64 = 10000062;
+
+fn new_engine(server: &mockito::ServerGuard, producer: &str) -> (Engine, BTreeMap<i64, Vec<u8>>) {
+ let fixtures = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/export-adoption");
+ let manifest = read_json(&fixtures.join("desktop-manifest.json"));
+ let albums = [
+ (FAMILY, "Family"),
+ (1580559962386472, "Travel"),
+ (1580559962386474, "Live Pair"),
+ ]
+ .into_iter()
+ .map(|(id, name)| (id, name.into()))
+ .collect();
+ let mut engine = Engine::new(
+ server,
+ Catalog {
+ user: USER,
+ key: Key::generate(),
+ albums,
+ files: BTreeMap::new(),
+ failed_album: None,
+ },
+ );
+ write_fixture(&engine.root(), producer);
+ let mut files = BTreeMap::new();
+ let mut downloads = BTreeMap::new();
+ let mut records = BTreeMap::new();
+ for (identity, name) in manifest["fileExportNames"].as_object().unwrap() {
+ let ids: Vec<i64> = identity.split('_').map(|s| s.parse().unwrap()).collect();
+ let (id, album) = (ids[0], ids[1]);
+ let record = records.entry(id).or_insert_with(|| {
+ let folder = engine.root().join(&engine.catalog.albums[&album]);
+ let exported = name.as_str().unwrap();
+ let (original,kind,digest) = if exported.starts_with('{') {
+ let pair: Value = serde_json::from_str(exported).unwrap();
+ let image = fs::read(folder.join(pair["image"].as_str().unwrap())).unwrap();
+ let video = fs::read(folder.join(pair["video"].as_str().unwrap())).unwrap();
+ let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
+ for (name,bytes) in [("image.jpg",&image),("video.mov",&video)] {
+ zip.start_file(name,zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored)).unwrap();
+ zip.write_all(bytes).unwrap();
+ }
+ (zip.finish().unwrap().into_inner(),2,format!("{}:{}",digest(&image),digest(&video)))
+ } else {
+ let bytes = fs::read(folder.join(exported)).unwrap();
+ (bytes.clone(),if exported.ends_with(".mov") {1} else {0},digest(&bytes))
+ };
+ let name = match id {
+ PHOTO | 10000064 => "Photo.jpg",
+ 10000063 => "Live.jpg",
+ 10000065 => "Live.mov",
+ PAIR => "Pair.jpg",
+ _ => unreachable!(),
+ };
+ let (mut remote,encrypted) = source(id,&engine.catalog.key,&original,name);
+ remote["ownerID"] = json!(USER);
+ let metadata = blob::encrypt_json(&json!({"title":name,"fileType":kind,"hash":digest,"creationTime":1_700_000_000_000_000i64}),&file_key(&remote,&engine.catalog.key)).unwrap();
+ remote["metadata"] = json!({"encryptedData":b64::encode(&metadata.encrypted_data),"decryptionHeader":b64::encode(metadata.decryption_header.as_bytes())});
+ downloads.insert(id,encrypted);
+ remote
+ });
+ files.insert((album, id), record.clone());
+ }
+ engine.catalog.files = files;
+ (engine, downloads)
+}
+
+fn rename(engine: &mut Engine, album: i64, file: i64, name: &str) {
+ let previous = engine.catalog.files[&(album, file)].clone();
+ engine.catalog.files.insert(
+ (album, file),
+ public(
+ previous,
+ &engine.catalog.key,
+ json!({"editedName":name}),
+ 200,
+ ),
+ );
+}
+
+fn digest(bytes: &[u8]) -> String {
+ b64::encode(&hash::hash(bytes, Some(64), None).unwrap())
+}
+
+fn write_fixture(root: &Path, producer: &str) {
+ let fixtures = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/export-adoption");
+ let manifest = read_json(&fixtures.join("desktop-manifest.json"));
+ fs::create_dir_all(root).unwrap();
+ if producer == "desktop" {
+ fs::write(
+ root.join("export_status.json"),
+ serde_json::to_vec(&manifest).unwrap(),
+ )
+ .unwrap();
+ }
+ for (id, name) in manifest["collectionExportNames"].as_object().unwrap() {
+ let folder = root.join(name.as_str().unwrap());
+ fs::create_dir_all(&folder).unwrap();
+ if producer == "go" {
+ fs::create_dir_all(folder.join(".meta")).unwrap();
+ let mut album = read_json(&fixtures.join("go-album.json"));
+ album["id"] = json!(id.parse::<i64>().unwrap());
+ album["albumName"] = name.clone();
+ fs::write(
+ folder.join(".meta/album_meta.json"),
+ serde_json::to_vec(&album).unwrap(),
+ )
+ .unwrap();
+ } else {
+ fs::create_dir_all(folder.join("metadata")).unwrap();
+ }
+ }
+ for (key, value) in manifest["fileExportNames"].as_object().unwrap() {
+ let ids: Vec<_> = key.split('_').collect();
+ let id = ids[0].parse::<i64>().unwrap();
+ let folder = root.join(manifest["collectionExportNames"][ids[1]].as_str().unwrap());
+ let value = value.as_str().unwrap();
+ let parts = if value.starts_with('{') {
+ let pair: Value = serde_json::from_str(value).unwrap();
+ vec![
+ pair["image"].as_str().unwrap().to_owned(),
+ pair["video"].as_str().unwrap().to_owned(),
+ ]
+ } else {
+ vec![value.to_owned()]
+ };
+ let mut hashes = Vec::new();
+ for (index, name) in parts.iter().enumerate() {
+ let bytes = format!("export fixture {id} component {index}").into_bytes();
+ hashes.push(digest(&bytes));
+ fs::write(folder.join(name), bytes).unwrap();
+ if producer == "desktop" {
+ let mut sidecar = read_json(&fixtures.join("desktop-file.json"));
+ sidecar["title"] = json!(name);
+ fs::write(
+ folder.join(format!("metadata/{name}.json")),
+ serde_json::to_vec(&sidecar).unwrap(),
+ )
+ .unwrap();
+ }
+ }
+ if producer == "go" {
+ let mut sidecar = read_json(&fixtures.join("go-file.json"));
+ sidecar["title"] = json!(parts[0]);
+ sidecar["info"]["id"] = json!(id);
+ sidecar["info"]["fileNames"] = json!(parts);
+ sidecar["info"]["hash"] = json!(hashes.join(":"));
+ fs::write(
+ folder.join(format!(".meta/{}.json", parts[0])),
+ serde_json::to_vec(&sidecar).unwrap(),
+ )
+ .unwrap();
+ }
+ }
+}
+
+fn snapshot(root: &Path) -> BTreeMap<PathBuf, (Vec<u8>, std::time::SystemTime)> {
+ fn visit(
+ root: &Path,
+ path: &Path,
+ result: &mut BTreeMap<PathBuf, (Vec<u8>, std::time::SystemTime)>,
+ ) {
+ for entry in fs::read_dir(path).unwrap() {
+ let entry = entry.unwrap();
+ let path = entry.path();
+ let metadata = fs::symlink_metadata(&path).unwrap();
+ if metadata.is_dir() {
+ visit(root, &path, result);
+ } else if metadata.is_file() {
+ result.insert(
+ path.strip_prefix(root).unwrap().to_owned(),
+ (fs::read(&path).unwrap(), metadata.modified().unwrap()),
+ );
+ }
+ }
+ }
+ let mut result = BTreeMap::new();
+ visit(root, root, &mut result);
+ result
+}
+
+#[test]
+fn adoption_reuses_captured_producer_shapes_and_converts_current_names() {
+ for producer in ["go", "desktop"] {
+ let mut server = mockito::Server::new();
+ let (mut engine, downloads) = new_engine(&server, producer);
+ let requests: Vec<_> = downloads
+ .iter()
+ .map(|(id, bytes)| download(&mut server, *id, bytes, 0))
+ .collect();
+ let root = engine.root();
+ if producer == "desktop" {
+ fs::rename(root.join("Family"), root.join("Family(1)")).unwrap();
+ fs::rename(
+ root.join("Family(1)/Photo.jpg"),
+ root.join("Family(1)/Photo(1).jpg"),
+ )
+ .unwrap();
+ fs::rename(
+ root.join("Family(1)/metadata/Photo.jpg.json"),
+ root.join("Family(1)/metadata/Photo(1).jpg.json"),
+ )
+ .unwrap();
+ let mut manifest = read_json(&root.join("export_status.json"));
+ manifest["collectionExportNames"][FAMILY.to_string()] = json!("Family(1)");
+ let key = manifest["fileExportNames"]
+ .as_object()
+ .unwrap()
+ .keys()
+ .find(|key| key.starts_with(&format!("{PHOTO}_{FAMILY}_")))
+ .unwrap()
+ .clone();
+ manifest["fileExportNames"][key] = json!("Photo(1).jpg");
+ fs::write(
+ root.join("export_status.json"),
+ serde_json::to_vec(&manifest).unwrap(),
+ )
+ .unwrap();
+ fs::write(
+ root.join("Family(1)/metadata/Photo(1).jpg.json"),
+ b"broken JSON",
+ )
+ .unwrap();
+ } else {
+ engine.catalog.albums.insert(FAMILY, "Renamed".into());
+ rename(&mut engine, FAMILY, PHOTO, "Portrait.jpg");
+ }
+ let result = engine.run(true, &[]).unwrap();
+ assert!(result.complete, "{}", result.json);
+ assert_eq!(result.expected, 6);
+ if producer == "desktop" {
+ assert_component(
+ &root,
+ "Family(1)",
+ "Photo(1).jpg",
+ format!("export fixture {PHOTO} component 0").as_bytes(),
+ );
+ assert_eq!(
+ read_json(&root.join("Family(1)/metadata.json"))["title"],
+ "Family"
+ );
+ assert_eq!(
+ read_json(&root.join("Family(1)/metadata/Photo(1).jpg.json"))["ente"]["name"],
+ "Photo.jpg"
+ );
+ let before = snapshot(&root);
+ assert!(engine.run(false, &[]).unwrap().complete);
+ assert_eq!(snapshot(&root), before);
+ engine.catalog.albums.insert(FAMILY, "Renamed".into());
+ rename(&mut engine, FAMILY, PHOTO, "Portrait.jpg");
+ assert!(engine.run(false, &[]).unwrap().complete);
+ assert!(!root.join("Family(1)").exists());
+ }
+ assert!(!root.join("Family").exists());
+ assert!(root.join("Renamed/Portrait.jpg").is_file());
+ assert!(root.join("Renamed/Live-1.jpg").is_file());
+ assert!(root.join("Renamed/Live.mov").is_file());
+ assert_eq!(
+ read_json(&root.join("Renamed/metadata/Portrait.jpg.json"))["ente"]["fileID"],
+ PHOTO.to_string()
+ );
+ assert_eq!(count(&engine.db(), "SELECT count(*) FROM components"), 8);
+ assert_eq!(count(&engine.db(), "SELECT bound FROM export"), 1);
+ assert!(!root.join("export_status.json").exists());
+ assert!(!root.join("Renamed/.meta").exists());
+ let before = snapshot(&root);
+ assert!(engine.run(false, &[]).unwrap().complete);
+ assert_eq!(snapshot(&root), before);
+ engine.database = "recovered.db".into();
+ assert!(engine.run(true, &[]).unwrap().complete);
+ for request in requests {
+ request.assert();
+ }
+ }
+}
+
+#[test]
+fn adoption_preserves_desktop_paths_for_names_changed_before_takeover() {
+ let server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, "desktop");
+ let root = engine.root();
+ let original = fs::read(root.join("Family/Photo.jpg")).unwrap();
+ engine.catalog.albums.insert(FAMILY, "Renamed".into());
+ rename(&mut engine, FAMILY, PHOTO, "Portrait.jpg");
+ assert!(engine.run(true, &[]).unwrap().complete);
+ assert_eq!(fs::read(root.join("Family/Photo.jpg")).unwrap(), original);
+ assert_eq!(
+ read_json(&root.join("Family/metadata.json"))["title"],
+ "Renamed"
+ );
+ assert_eq!(
+ read_json(&root.join("Family/metadata/Photo.jpg.json"))["ente"]["name"],
+ "Portrait.jpg"
+ );
+ let before = snapshot(&root);
+ assert!(engine.run(false, &[]).unwrap().complete);
+ assert_eq!(snapshot(&root), before);
+}
+
+#[test]
+fn adoption_preflight_refuses_corrupt_media_scope_and_metadata_blockers_without_writes() {
+ for producer in ["go", "desktop"] {
+ let server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, producer);
+ let root = engine.root();
+ let before = snapshot(&root);
+ assert!(
+ engine
+ .run(true, &["Family"])
+ .err()
+ .unwrap()
+ .to_string()
+ .contains("exactly")
+ );
+ assert_eq!(snapshot(&root), before);
+ fs::write(root.join("Family/Photo.jpg"), b"local edit").unwrap();
+ fs::write(root.join("Travel/Photo.jpg"), b"second edit").unwrap();
+ let before = snapshot(&root);
+ let error = engine.run(true, &[]).err().unwrap().to_string();
+ assert!(error.contains("2 failing items"), "{error}");
+ assert_eq!(snapshot(&root), before);
+ assert!(!root.join("export.json").exists());
+ assert_eq!(count(&engine.db(), "SELECT count(*) FROM albums"), 0);
+ write_fixture(&root, producer);
+ fs::write(root.join("Family/metadata.json"), b"user media").unwrap();
+ let before = snapshot(&root);
+ assert!(engine.run(true, &[]).is_err());
+ assert_eq!(snapshot(&root), before);
+ }
+}
+
+#[test]
+fn adoption_preserves_partial_pairs_and_restores_missing_originals() {
+ for producer in ["go", "desktop"] {
+ let mut server = mockito::Server::new();
+ let (mut engine, downloads) = new_engine(&server, producer);
+ let root = engine.root();
+ let survivor = fs::read(root.join("Family/Pair.jpg")).unwrap();
+ let original = fs::read(root.join("Family/Photo.jpg")).unwrap();
+ let sidecar = if producer == "desktop" {
+ Some(fs::read(root.join("Family/metadata/Photo.jpg.json")).unwrap())
+ } else {
+ None
+ };
+ fs::remove_file(root.join("Family/Photo.jpg")).unwrap();
+ fs::remove_file(root.join("Family/Pair.mov")).unwrap();
+ fs::remove_file(root.join("Live Pair/Pair.jpg")).unwrap();
+ fs::remove_file(root.join("Live Pair/Pair.mov")).unwrap();
+ let ordinary = download(&mut server, PHOTO, &downloads[&PHOTO], 1);
+ let requested = download(&mut server, PAIR, &downloads[&PAIR], 1);
+ let result = engine.run(true, &[]).unwrap();
+ assert!(result.complete, "{producer}: {}", result.json);
+ requested.assert();
+ ordinary.assert();
+ assert_eq!(fs::read(root.join("Family/Photo.jpg")).unwrap(), original);
+ if let Some(sidecar) = sidecar {
+ assert_eq!(
+ fs::read(root.join("Trash/Family/metadata/Photo.jpg.json")).unwrap(),
+ sidecar
+ );
+ }
+ assert_eq!(
+ fs::read(root.join("Trash/Family/Pair.jpg")).unwrap(),
+ survivor
+ );
+ assert!(
+ root.join(if producer == "go" {
+ "Trash/Family/.meta/Pair.jpg.json"
+ } else {
+ "Trash/Family/metadata/Pair.jpg.json"
+ })
+ .is_file()
+ );
+ assert_eq!(
+ fs::read(root.join("Family/Pair.mov")).unwrap(),
+ fs::read(root.join("Live Pair/Pair.mov")).unwrap()
+ );
+ }
+ let server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, "go");
+ let root = engine.root();
+ fs::remove_file(root.join("Family/Pair.mov")).unwrap();
+ let sidecar = root.join("Family/.meta/Pair.jpg.json");
+ let mut value = read_json(&sidecar);
+ value["info"]["hash"] = json!(format!(
+ "{}:{}",
+ value["info"]["hash"]
+ .as_str()
+ .unwrap()
+ .split(':')
+ .next()
+ .unwrap(),
+ digest(b"stale video")
+ ));
+ fs::write(&sidecar, serde_json::to_vec(&value).unwrap()).unwrap();
+ let before = snapshot(&root);
+ assert!(
+ engine
+ .run(true, &[])
+ .err()
+ .unwrap()
+ .to_string()
+ .contains("video component")
+ );
+ assert_eq!(snapshot(&root), before);
+}
+
+#[test]
+fn adoption_history_resumes_after_media_move_and_keeps_the_full_scope() {
+ let mut server = mockito::Server::new();
+ let (mut engine, downloads) = new_engine(&server, "desktop");
+ let root = engine.root();
+ engine.catalog.files.remove(&(FAMILY, PAIR));
+ fs::create_dir_all(root.join("Trash/Family")).unwrap();
+ fs::write(root.join("Trash/Family/Pair.jpg"), b"existing trash").unwrap();
+ fs::write(root.join("Family/user.txt"), b"unrelated").unwrap();
+ let manifest = fs::read(root.join("export_status.json")).unwrap();
+ fs::write(root.join("Trash/Family/metadata"), b"block sidecar move").unwrap();
+ assert!(engine.prepare(engine.options(true, &[])).is_err());
+ assert_eq!(count(&engine.db(), "SELECT count(*) FROM albums"), 0);
+ assert_eq!(fs::read(root.join("export_status.json")).unwrap(), manifest);
+ assert!(!root.join("Family/Pair.jpg").exists());
+ assert!(root.join("Trash/Family/Pair-1.jpg").is_file());
+ assert_eq!(count(&engine.db(), "SELECT bound FROM export"), 0);
+ let before = snapshot(&root);
+ assert!(
+ engine
+ .run(false, &[])
+ .err()
+ .unwrap()
+ .is::<ente_photos_export::AdoptionRequired>()
+ );
+ assert!(engine.run(true, &["Family"]).is_err());
+ assert_eq!(snapshot(&root), before);
+ fs::remove_file(root.join("Trash/Family/metadata")).unwrap();
+ let export = engine.prepare(engine.options(true, &[])).unwrap();
+ drop(export);
+ fs::write(root.join("Family/Photo.jpg"), b"edit after acceptance").unwrap();
+ let result = engine.run(false, &[]).unwrap();
+ assert!(!result.complete);
+ assert_eq!(
+ fs::read(root.join("Family/Photo.jpg")).unwrap(),
+ b"edit after acceptance"
+ );
+ assert_eq!(fs::read(root.join("export_status.json")).unwrap(), manifest);
+ assert!(root.join("Trash/Family/Pair.mov").is_file());
+ assert!(root.join("Trash/Family/metadata/Pair.jpg.json").is_file());
+ assert_eq!(
+ fs::read(root.join("Trash/Family/Pair.jpg")).unwrap(),
+ b"existing trash"
+ );
+ assert!(root.join("Live Pair/Pair.jpg").is_file());
+ assert_eq!(
+ fs::read(root.join("Family/user.txt")).unwrap(),
+ b"unrelated"
+ );
+ fs::remove_file(root.join("Family/Photo.jpg")).unwrap();
+ let downloaded = download(&mut server, PHOTO, &downloads[&PHOTO], 1);
+ assert!(engine.run(false, &[]).unwrap().complete);
+ downloaded.assert();
+ assert!(!root.join("export_status.json").exists());
+ assert_eq!(count(&engine.db(), "SELECT bound FROM export"), 1);
+}
+
+#[test]
+fn adoption_replays_recorded_moves_without_rehashing_every_album() {
+ for (producer, directory) in [("go", false), ("desktop", true)] {
+ let server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, producer);
+ let root = engine.root();
+ let export = engine.prepare(engine.options(true, &[])).unwrap();
+ drop(export);
+ rename(&mut engine, FAMILY, PHOTO, "Portrait.jpg");
+ if directory {
+ engine.catalog.albums.insert(FAMILY, "Renamed".into());
+ }
+ let export = engine.prepare(engine.options(false, &[])).unwrap();
+ let db = engine.db();
+ if directory {
+ db.execute_batch(&format!("CREATE TRIGGER interrupt_album BEFORE UPDATE ON albums WHEN old.id={FAMILY} AND new.path='Renamed' BEGIN SELECT RAISE(ABORT,'interrupt after directory rename'); END;")).unwrap();
+ } else {
+ db.execute_batch("CREATE TRIGGER interrupt_component BEFORE UPDATE ON components WHEN old.name='Photo.jpg' AND new.name='Portrait.jpg' BEGIN SELECT RAISE(ABORT,'interrupt after component rename'); END;").unwrap();
+ }
+ assert!(!engine.execute(export).unwrap().complete);
+ let folder = if directory { "Renamed" } else { "Family" };
+ assert!(root.join(folder).is_dir());
+ db.execute_batch(if directory {
+ "DROP TRIGGER interrupt_album"
+ } else {
+ "DROP TRIGGER interrupt_component"
+ })
+ .unwrap();
+ drop(db);
+ fs::write(root.join("Travel/Photo.jpg"), b"late independent edit").unwrap();
+ let result = engine.run(false, &[]).unwrap();
+ assert!(!result.complete);
+ assert!(root.join(format!("{folder}/Portrait.jpg")).is_file());
+ assert_eq!(
+ fs::read(root.join("Travel/Photo.jpg")).unwrap(),
+ b"late independent edit"
+ );
+ assert_eq!(
+ read_json(&root.join(format!("{folder}/metadata/Portrait.jpg.json")))["ente"]["fileID"],
+ PHOTO.to_string()
+ );
+ }
+}
+
+#[test]
+fn adoption_requires_initial_presence_and_never_treats_a_failed_refresh_as_history() {
+ let server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, "go");
+ let root = engine.root();
+ engine.catalog.failed_album = Some(FAMILY);
+ let before = snapshot(&root);
+ assert!(engine.run(true, &[]).is_err());
+ assert_eq!(snapshot(&root), before);
+ assert!(!root.join("Trash").exists());
+ engine.catalog.failed_album = None;
+ for (name, _) in snapshot(&root) {
+ if !name.to_string_lossy().ends_with(".json") {
+ fs::remove_file(root.join(name)).unwrap();
+ }
+ }
+ let before = snapshot(&root);
+ assert!(
+ engine
+ .run(true, &[])
+ .err()
+ .unwrap()
+ .to_string()
+ .contains("at least one")
+ );
+ assert_eq!(snapshot(&root), before);
+}
+
+#[test]
+fn adoption_hashless_verification_downloads_once_per_source_and_keeps_scope_per_invocation() {
+ let mut server = mockito::Server::new();
+ let (mut engine, downloads) = new_engine(&server, "desktop");
+ for ((_, id), remote) in &mut engine.catalog.files {
+ if ![PHOTO, PAIR].contains(id) {
+ continue;
+ }
+ let (_, documents) = files::decrypt(
+ &serde_json::from_value(remote.clone()).unwrap(),
+ &engine.catalog.key,
+ )
+ .unwrap();
+ let mut metadata: Value = serde_json::from_slice(&documents.original).unwrap();
+ metadata.as_object_mut().unwrap().remove("hash");
+ let metadata =
+ blob::encrypt_json(&metadata, &file_key(remote, &engine.catalog.key)).unwrap();
+ remote["metadata"] = json!({"encryptedData":b64::encode(&metadata.encrypted_data),"decryptionHeader":b64::encode(metadata.decryption_header.as_bytes())});
+ }
+ let ordinary = download(&mut server, PHOTO, &downloads[&PHOTO], 2);
+ let failed = server
+ .mock("GET", format!("/files/download/v3/{PAIR}").as_str())
+ .with_status(404)
+ .expect(1)
+ .create();
+ let before = snapshot(&engine.root());
+ let error = engine.run(true, &[]).err().unwrap().to_string();
+ assert!(error.contains("1 failing items"), "{error}");
+ assert_eq!(snapshot(&engine.root()), before);
+ assert!(!engine.root().join("export.json").exists());
+ failed.assert();
+ drop(failed);
+ let url = server
+ .mock("GET", format!("/files/download/v3/{PAIR}").as_str())
+ .with_body(json!({"url":format!("{}/comparison",server.url())}).to_string())
+ .expect(1)
+ .create();
+ let db_path = engine.directory.path().join(&engine.database);
+ let bytes = downloads[&PAIR].clone();
+ let live = server
+ .mock("GET", "/comparison")
+ .with_body_from_request(move |_| {
+ let db = rusqlite::Connection::open(&db_path).unwrap();
+ db.busy_timeout(std::time::Duration::ZERO).unwrap();
+ db.execute_batch("BEGIN IMMEDIATE; ROLLBACK;").unwrap();
+ assert_eq!(count(&db, "SELECT count(*) FROM albums"), 0);
+ bytes.clone()
+ })
+ .expect(1)
+ .create();
+ engine.catalog.albums.insert(99, "Other".into());
+ assert!(engine.run(true, &[]).unwrap().complete);
+ ordinary.assert();
+ live.assert();
+ url.assert();
+ assert!(fs::read_dir(engine.root()).unwrap().all(|entry| {
+ let entry = entry.unwrap();
+ entry.file_type().unwrap().is_dir() || entry.file_name() == "export.json"
+ }));
+ assert!(!engine.root().join("Other").exists());
+ assert!(engine.run(false, &["Family"]).unwrap().complete);
+ assert!(!engine.root().join("Other").exists());
+ assert!(engine.run(true, &[]).unwrap().complete);
+ assert!(engine.root().join("Other/metadata.json").exists());
+}
+
+#[test]
+fn adoption_relocates_an_unavailable_go_album_named_trash_without_inventing_native_history() {
+ for keep_unknown in [false, true] {
+ let server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, "go");
+ let root = engine.root();
+ fs::rename(root.join("Family"), root.join("Trash")).unwrap();
+ let path = root.join("Trash/.meta/album_meta.json");
+ let mut album = read_json(&path);
+ album["albumName"] = json!("Trash");
+ fs::write(path, serde_json::to_vec(&album).unwrap()).unwrap();
+ engine.catalog.albums.remove(&FAMILY);
+ engine
+ .catalog
+ .files
+ .retain(|(album, _), _| *album != FAMILY);
+ if keep_unknown {
+ fs::write(root.join("Trash/user.txt"), b"keep").unwrap();
+ }
+ fs::write(root.join("Trash/Photo.jpg"), b"historical local edit").unwrap();
+ assert!(engine.run(true, &[]).unwrap().complete);
+ assert_eq!(
+ fs::read(root.join("Trash/Trash-1/Photo.jpg")).unwrap(),
+ b"historical local edit"
+ );
+ if keep_unknown {
+ assert_eq!(fs::read(root.join("Trash-1/user.txt")).unwrap(), b"keep");
+ } else {
+ assert!(!root.join("Trash-1").exists());
+ }
+ assert!(root.join("Trash/Trash-1/.meta/Photo.jpg.json").is_file());
+ assert!(!root.join("Trash/Trash-1/metadata.json").exists());
+ assert!(!root.join("Trash-1/.meta/album_meta.json").exists());
+ assert_eq!(
+ count(
+ &engine.db(),
+ &format!("SELECT count(*) FROM placements WHERE album={FAMILY}")
+ ),
+ 0
+ );
+ }
+}
+
+#[test]
+fn adoption_bounds_failure_examples() {
+ let server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, "desktop");
+ let root = engine.root();
+ let path = root.join("export_status.json");
+ let mut manifest = read_json(&path);
+ for id in 100..113 {
+ let mut remote = engine.catalog.files[&(FAMILY, PHOTO)].clone();
+ remote["id"] = json!(id);
+ engine.catalog.files.insert((FAMILY, id), remote);
+ let name = format!("Bad-{id}.jpg");
+ fs::write(root.join("Family").join(&name), b"bad original").unwrap();
+ manifest["fileExportNames"][format!("{id}_{FAMILY}_100")] = json!(name);
+ }
+ fs::write(&path, serde_json::to_vec(&manifest).unwrap()).unwrap();
+ let before = snapshot(&root);
+ let error = engine.run(true, &[]).err().unwrap().to_string();
+ assert!(error.contains("13 failing items"), "{error}");
+ assert!(error.contains("3 more omitted"), "{error}");
+ assert_eq!(snapshot(&root), before);
+
+ manifest["collectionExportNames"][FAMILY.to_string()] = json!("../outside");
+ fs::write(&path, serde_json::to_vec(&manifest).unwrap()).unwrap();
+ let before = snapshot(&root);
+ let error = engine.run(true, &[]).err().unwrap().to_string();
+ assert!(error.contains("1 failing items"), "{error}");
+ assert!(
+ !error.contains("file has no valid album mapping"),
+ "{error}"
+ );
+ assert_eq!(snapshot(&root), before);
+
+ manifest["fileExportNames"]["900_77_100"] = json!("Missing.jpg");
+ fs::write(&path, serde_json::to_vec(&manifest).unwrap()).unwrap();
+ let before = snapshot(&root);
+ let error = engine.run(true, &[]).err().unwrap().to_string();
+ assert!(error.contains("2 failing items"), "{error}");
+ assert!(error.contains("file has no valid album mapping"), "{error}");
+ assert_eq!(snapshot(&root), before);
+}
+
+#[test]
+fn adoption_rejects_missing_album_identity_wrong_account_and_escaping_paths() {
+ for case in ["orphan", "account", "escape"] {
+ let server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, "go");
+ let root = engine.root();
+ match case {
+ "orphan" => {
+ fs::remove_file(root.join("Travel/.meta/album_meta.json")).unwrap();
+ }
+ "account" => {
+ let path = root.join("Family/.meta/album_meta.json");
+ let mut value = read_json(&path);
+ value["accountOwnerIDs"] = json!([42]);
+ fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
+ }
+ _ => {
+ let path = root.join("Family/.meta/Photo.jpg.json");
+ let mut value = read_json(&path);
+ value["info"]["fileNames"] = json!(["../outside.jpg"]);
+ fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
+ }
+ }
+ let before = snapshot(&root);
+ let error = engine.run(true, &[]).err().unwrap().to_string();
+ assert_eq!(snapshot(&root), before);
+ if case == "orphan" {
+ assert!(error.contains("without album_meta.json"), "{error}");
+ }
+ }
+}
+
+#[test]
+fn adoption_preserves_unknown_meta_files_and_records_empty_go_albums() {
+ let server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, "go");
+ let root = engine.root();
+ fs::create_dir(root.join("Personal")).unwrap();
+ fs::write(root.join("Personal/.meta"), b"keep unrelated file").unwrap();
+ fs::create_dir_all(root.join("Empty/.meta")).unwrap();
+ let mut album = read_json(&root.join("Family/.meta/album_meta.json"));
+ album["id"] = json!(77);
+ album["albumName"] = json!("Empty");
+ fs::write(
+ root.join("Empty/.meta/album_meta.json"),
+ serde_json::to_vec(&album).unwrap(),
+ )
+ .unwrap();
+ fs::write(root.join("Empty/.meta/user.txt"), b"keep").unwrap();
+ engine.catalog.albums.insert(77, "Empty".into());
+ assert!(engine.run(true, &[]).unwrap().complete);
+ assert_eq!(
+ read_json(&root.join("Empty/metadata.json"))["ente"]["albumID"],
+ "77"
+ );
+ assert_eq!(
+ fs::read(root.join("Empty/.meta/user.txt")).unwrap(),
+ b"keep"
+ );
+ assert_eq!(
+ fs::read(root.join("Personal/.meta")).unwrap(),
+ b"keep unrelated file"
+ );
+}
+
+#[test]
+fn adoption_retains_accepted_bytes_when_the_source_layout_changes_before_conversion() {
+ for producer in ["go", "desktop"] {
+ let mut server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, producer);
+ let root = engine.root();
+ let original = fs::read(root.join("Family/Photo.jpg")).unwrap();
+ let export = engine.prepare(engine.options(true, &[])).unwrap();
+ drop(export);
+ let (mut replacement, encrypted) =
+ original_fixture(&engine.catalog.key, b"new Live component", true, true, 300);
+ replacement["id"] = json!(PHOTO);
+ replacement["ownerID"] = json!(USER);
+ engine.catalog.files.insert((FAMILY, PHOTO), replacement);
+ let requested = download(&mut server, PHOTO, &encrypted, 1);
+ assert!(engine.run(true, &[]).unwrap().complete);
+ requested.assert();
+ assert_eq!(
+ fs::read(root.join("Trash/Family/Photo.jpg")).unwrap(),
+ original
+ );
+ assert_component(&root, "Family", "Motion.heic", b"new Live component");
+ assert_component(&root, "Family", "Motion.mov", b"new Live component");
+ }
+}
+
+#[cfg(unix)]
+#[test]
+fn adoption_rejects_symlinks_while_detecting_orphaned_go_records() {
+ for directory in [false, true] {
+ let server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, "go");
+ let root = engine.root();
+ fs::remove_file(root.join("Travel/.meta/album_meta.json")).unwrap();
+ let source = root.join(if directory {
+ "Travel/.meta"
+ } else {
+ "Travel/.meta/Photo.jpg.json"
+ });
+ let outside = engine.directory.path().join("outside");
+ fs::rename(&source, &outside).unwrap();
+ std::os::unix::fs::symlink(&outside, &source).unwrap();
+ let before = snapshot(&root);
+ let error = engine.run(true, &[]).err().unwrap().to_string();
+ assert!(error.contains("symlink"), "{error}");
+ assert_eq!(snapshot(&root), before);
+ assert_eq!(fs::read_link(&source).unwrap(), outside);
+ }
+}
+
+#[test]
+fn adoption_recovers_native_paths_and_live_ownership_with_an_unchanged_manifest() {
+ let mut server = mockito::Server::new();
+ let (mut engine, downloads) = new_engine(&server, "desktop");
+ let root = engine.root();
+ let manifest = fs::read(root.join("export_status.json")).unwrap();
+ engine.catalog.files.remove(&(FAMILY, PHOTO));
+ let export = engine.prepare(engine.options(true, &[])).unwrap();
+ drop(export);
+ engine.catalog.albums.insert(FAMILY, "Renamed".into());
+ rename(&mut engine, FAMILY, PAIR, "Photo.jpg");
+ let (mut remote, _) = source(90, &engine.catalog.key, b"new original", "Pending.jpg");
+ remote["ownerID"] = json!(USER);
+ engine.catalog.files.insert((FAMILY, 90), remote);
+ let failed = server
+ .mock("GET", "/files/download/v3/90")
+ .with_status(404)
+ .expect(1)
+ .create();
+ assert!(!engine.run(true, &[]).unwrap().complete);
+ failed.assert();
+ assert_eq!(fs::read(root.join("export_status.json")).unwrap(), manifest);
+ let original = fs::read(root.join("Renamed/Photo.jpg")).unwrap();
+ assert_eq!(
+ read_json(&root.join("Renamed/metadata/Photo.jpg.json"))["ente"]["fileID"],
+ PAIR.to_string()
+ );
+ fs::remove_file(root.join("Renamed/metadata/Photo.mov.json")).unwrap();
+ let path = root.join("Renamed/metadata/Live.mov.json");
+ fs::write(path, b"unfinished legacy common JSON").unwrap();
+ engine.catalog.files.remove(&(FAMILY, 90));
+ engine.database = "recovered.db".into();
+ assert!(
+ engine
+ .run(false, &[])
+ .err()
+ .unwrap()
+ .is::<ente_photos_export::AdoptionRequired>()
+ );
+ let requests: Vec<_> = downloads
+ .iter()
+ .map(|(id, bytes)| download(&mut server, *id, bytes, 0))
+ .collect();
+ assert!(engine.run(true, &[]).unwrap().complete);
+ assert_eq!(fs::read(root.join("Renamed/Photo.jpg")).unwrap(), original);
+ assert_eq!(
+ read_json(&root.join("Renamed/metadata/Photo.mov.json"))["ente"]["fileID"],
+ PAIR.to_string()
+ );
+ assert_eq!(
+ read_json(&root.join("Renamed/metadata/Live.mov.json"))["ente"]["fileID"],
+ "10000065"
+ );
+ assert!(!root.join("export_status.json").exists());
+ assert_eq!(
+ fs::read_dir(root.join("Trash/Family"))
+ .unwrap()
+ .filter_map(Result::ok)
+ .filter(|e| e.path().extension().is_some_and(|x| x == "jpg"))
+ .count(),
+ 1
+ );
+ for request in requests {
+ request.assert();
+ }
+}
+
+#[test]
+fn adoption_cleanup_waits_for_new_files_and_resumes_after_its_final_database_write() {
+ let mut server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, "go");
+ let root = engine.root();
+ let (mut remote, encrypted) = source(90, &engine.catalog.key, b"new original", "New.jpg");
+ remote["ownerID"] = json!(USER);
+ engine.catalog.files.insert((FAMILY, 90), remote);
+ engine.catalog.albums.insert(99, "Other".into());
+ let failed = server
+ .mock("GET", "/files/download/v3/90")
+ .with_status(404)
+ .expect(1)
+ .create();
+ assert!(!engine.run(true, &[]).unwrap().complete);
+ failed.assert();
+ assert!(root.join("Family/.meta/album_meta.json").is_file());
+ assert!(root.join("Family/.meta/Photo.jpg.json").is_file());
+ assert!(root.join("Family/metadata/Photo.jpg.json").is_file());
+ drop(failed);
+ let requested = download(&mut server, 90, &encrypted, 1);
+ engine.db().execute_batch("CREATE TRIGGER stop_completion BEFORE UPDATE OF bound ON export WHEN new.bound=1 BEGIN SELECT RAISE(ABORT,'interrupt after cleanup'); END;").unwrap();
+ assert!(!engine.run(false, &[]).unwrap().complete);
+ requested.assert();
+ assert!(!root.join("Family/.meta").exists());
+ assert!(!root.join("Other").exists());
+ assert_eq!(count(&engine.db(), "SELECT bound FROM export"), 0);
+ assert_eq!(
+ count(&engine.db(), "SELECT count(*) FROM albums WHERE retained=0"),
+ 3
+ );
+ engine
+ .db()
+ .execute_batch("DROP TRIGGER stop_completion")
+ .unwrap();
+ assert!(
+ engine
+ .run(false, &["Family"])
+ .err()
+ .unwrap()
+ .to_string()
+ .contains("exactly")
+ );
+ assert!(engine.run(false, &[]).unwrap().complete);
+ assert!(!root.join("Other").exists());
+ assert!(engine.run(true, &[]).unwrap().complete);
+ assert!(root.join("Other/metadata.json").is_file());
+}
+
+#[test]
+fn adoption_defers_wholly_missing_hashless_originals_until_after_acceptance() {
+ let mut server = mockito::Server::new();
+ let (mut engine, downloads) = new_engine(&server, "go");
+ let previous = engine.catalog.files[&(FAMILY, PHOTO)].clone();
+ let (_, documents) = files::decrypt(
+ &serde_json::from_value(previous.clone()).unwrap(),
+ &engine.catalog.key,
+ )
+ .unwrap();
+ let mut value: Value = serde_json::from_slice(&documents.original).unwrap();
+ value.as_object_mut().unwrap().remove("hash");
+ let encrypted = blob::encrypt_json(&value, &file_key(&previous, &engine.catalog.key)).unwrap();
+ engine.catalog.files.get_mut(&(FAMILY, PHOTO)).unwrap()["metadata"] = json!({"encryptedData":b64::encode(&encrypted.encrypted_data),"decryptionHeader":b64::encode(encrypted.decryption_header.as_bytes())});
+ fs::remove_file(engine.root().join("Family/Photo.jpg")).unwrap();
+ let absent = download(&mut server, PHOTO, &downloads[&PHOTO], 0);
+ let export = engine.prepare(engine.options(true, &[])).unwrap();
+ absent.assert();
+ assert_eq!(
+ count(
+ &engine.db(),
+ &format!(
+ "SELECT count(*) FROM components c JOIN placements p ON p.id=c.placement WHERE p.file={PHOTO}"
+ )
+ ),
+ 0
+ );
+ drop(absent);
+ let requested = download(&mut server, PHOTO, &downloads[&PHOTO], 1);
+ assert!(engine.execute(export).unwrap().complete);
+ requested.assert();
+}
+
+#[test]
+fn adoption_verifies_a_single_undeclared_live_survivor_before_preserving_it() {
+ let mut server = mockito::Server::new();
+ let (mut engine, downloads) = new_engine(&server, "go");
+ let root = engine.root();
+ let path = root.join("Family/.meta/Pair.jpg.json");
+ let mut record = read_json(&path);
+ record["info"]["fileNames"] = json!(["Pair.jpg"]);
+ fs::write(path, serde_json::to_vec(&record).unwrap()).unwrap();
+ fs::remove_file(root.join("Family/Pair.mov")).unwrap();
+ let original = fs::read(root.join("Family/Pair.jpg")).unwrap();
+ fs::write(root.join("Family/Pair.jpg"), b"edited survivor").unwrap();
+ let before = snapshot(&root);
+ assert!(engine.run(true, &[]).is_err());
+ assert_eq!(snapshot(&root), before);
+ fs::write(root.join("Family/Pair.jpg"), &original).unwrap();
+ for (path, _) in snapshot(&root) {
+ if !path.to_string_lossy().ends_with(".json")
+ && path.as_path() != Path::new("Family/Pair.jpg")
+ {
+ fs::remove_file(root.join(path)).unwrap();
+ }
+ }
+ let export = engine.prepare(engine.options(true, &[])).unwrap();
+ assert_eq!(count(&engine.db(), "SELECT count(*) FROM components"), 0);
+ let requests: Vec<_> = downloads
+ .iter()
+ .map(|(id, bytes)| download(&mut server, *id, bytes, 1))
+ .collect();
+ assert!(engine.execute(export).unwrap().complete);
+ for request in requests {
+ request.assert();
+ }
+ assert_eq!(
+ fs::read(root.join("Trash/Family/Pair.jpg")).unwrap(),
+ original
+ );
+ assert_eq!(fs::read(root.join("Family/Pair.jpg")).unwrap(), original);
+}
+
+#[test]
+fn adoption_of_one_recorded_album_includes_new_photos_but_not_unrelated_albums() {
+ let mut server = mockito::Server::new();
+ let (mut engine, _) = new_engine(&server, "desktop");
+ let root = engine.root();
+ fs::remove_dir_all(root.join("Travel")).unwrap();
+ fs::remove_dir_all(root.join("Live Pair")).unwrap();
+ let path = root.join("export_status.json");
+ let mut manifest = read_json(&path);
+ manifest["collectionExportNames"]
+ .as_object_mut()
+ .unwrap()
+ .retain(|id, _| id == &FAMILY.to_string());
+ manifest["fileExportNames"]
+ .as_object_mut()
+ .unwrap()
+ .retain(|id, _| id.split('_').nth(1) == Some(FAMILY.to_string().as_str()));
+ fs::write(path, serde_json::to_vec(&manifest).unwrap()).unwrap();
+ let (mut remote, encrypted) = source(90, &engine.catalog.key, b"new photo", "New.jpg");
+ remote["ownerID"] = json!(USER);
+ engine.catalog.files.insert((FAMILY, 90), remote);
+ let requested = download(&mut server, 90, &encrypted, 1);
+ let export = engine.prepare(engine.options(true, &["Family"])).unwrap();
+ drop(export);
+ assert!(engine.run(true, &[]).unwrap().complete);
+ requested.assert();
+ assert_component(&root, "Family", "New.jpg", b"new photo");
+ assert!(!root.join("Travel").exists());
+ assert!(!root.join("Live Pair").exists());
+}
diff --git a/rust/apps/cli-next/tests/support/export_engine.rs b/rust/apps/cli-next/tests/support/export_engine.rs
new file mode 100644
index 0000000000..80cd583b4a
--- /dev/null
+++ b/rust/apps/cli-next/tests/support/export_engine.rs
@@ -0,0 +1,160 @@
+use super::*;
+use anyhow::{Result, bail};
+use ente_core::{Session, crypto::SecretKey, http::ApiConfig};
+use ente_photos::{collections, files, source::Documents};
+use ente_photos_export::{CollectionEntry, Export, FileEntry, Options, Source, Summary};
+use std::{collections::BTreeMap, path::PathBuf};
+use tokio::sync::watch;
+
+pub(super) struct Catalog {
+ pub user: i64,
+ pub key: Key,
+ pub albums: BTreeMap<i64, String>,
+ pub files: BTreeMap<(i64, i64), Value>,
+ pub failed_album: Option<i64>,
+}
+
+impl Catalog {
+ fn album(&self, id: i64) -> CollectionEntry {
+ let name = self.albums[&id].clone();
+ CollectionEntry {
+ id,
+ name: Some(name.clone()),
+ favorites: false,
+ value: Ok((
+ collections::Collection {
+ id,
+ key: Key::from_bytes(*self.key.as_bytes()),
+ name,
+ kind: collections::Kind::Album,
+ visibility: collections::Visibility::Visible,
+ owner_id: self.user,
+ updated_at_micros: 100,
+ },
+ Documents {
+ original: Vec::new(),
+ public: None,
+ private: None,
+ shared: None,
+ },
+ )),
+ }
+ }
+
+ fn entry(&self, record: &Value) -> Result<FileEntry> {
+ let remote = serde_json::from_value(record.clone())?;
+ let (key, documents) = files::decrypt(&remote, &self.key)?;
+ let file = files::interpret(&remote, &key, &documents, self.user)?;
+ Ok(FileEntry {
+ id: file.id,
+ value: Ok((file, documents)),
+ })
+ }
+}
+
+impl Source for Catalog {
+ async fn refresh_catalog(&mut self, _: &Session) -> Result<()> {
+ Ok(())
+ }
+ fn catalog_page(&self, after: i64) -> Result<Vec<CollectionEntry>> {
+ Ok(self
+ .albums
+ .keys()
+ .copied()
+ .filter(|id| *id > after)
+ .map(|id| self.album(id))
+ .collect())
+ }
+ async fn refresh_album(&mut self, _: &Session, id: i64) -> Result<CollectionEntry> {
+ if self.failed_album == Some(id) {
+ bail!("fixture source refresh failed");
+ }
+ Ok(self.album(id))
+ }
+ fn file_page(&self, album: i64, after: i64) -> Result<Vec<FileEntry>> {
+ self.files
+ .iter()
+ .filter(|((a, f), _)| *a == album && *f > after)
+ .map(|(_, record)| self.entry(record))
+ .collect()
+ }
+ fn favorite_page(&self, _: i64) -> Result<Vec<i64>> {
+ Ok(Vec::new())
+ }
+ fn file(&self, album: i64, id: i64) -> Result<FileEntry> {
+ self.entry(&self.files[&(album, id)])
+ }
+}
+
+pub(super) struct Engine {
+ pub directory: tempfile::TempDir,
+ session: Session,
+ pub catalog: Catalog,
+ pub database: String,
+}
+
+impl Engine {
+ pub fn new(server: &mockito::ServerGuard, catalog: Catalog) -> Self {
+ Self {
+ directory: tempfile::tempdir().unwrap(),
+ session: Session::new(
+ ApiConfig::new(server.url()),
+ catalog.user,
+ Key::generate(),
+ Key::generate(),
+ SecretKey::generate(),
+ )
+ .unwrap(),
+ catalog,
+ database: "export.db".into(),
+ }
+ }
+
+ pub fn root(&self) -> PathBuf {
+ self.directory.path().join("photos")
+ }
+
+ pub fn db(&self) -> rusqlite::Connection {
+ rusqlite::Connection::open(self.directory.path().join(&self.database)).unwrap()
+ }
+
+ pub fn options(&self, adopt: bool, albums: &[&str]) -> Options {
+ Options {
+ albums: albums.iter().map(|s| (*s).into()).collect(),
+ exclude_albums: Vec::new(),
+ adopt,
+ jobs: std::num::NonZeroUsize::new(1).unwrap(),
+ }
+ }
+
+ pub fn prepare(&mut self, options: Options) -> Result<Export> {
+ let mut export = Export::open(&self.root(), &self.session.master_key, options)?;
+ let (_send, mut cancel) = watch::channel(false);
+ let db = self.db();
+ tokio::runtime::Runtime::new()?.block_on(export.prepare(
+ db,
+ |path| {
+ let db = rusqlite::Connection::open(path)?;
+ db.pragma_update(None, "key", b64::encode(Key::generate().as_bytes()))?;
+ Ok(db)
+ },
+ &mut self.catalog,
+ &self.session,
+ &mut cancel,
+ &|line| eprintln!("{line}"),
+ ))?;
+ Ok(export)
+ }
+
+ pub fn execute(&self, mut export: Export) -> Result<Summary> {
+ let (send, cancel) = watch::channel(false);
+ tokio::runtime::Runtime::new()?.block_on(async {
+ export.execute(&self.session, &send, &cancel, &|line| eprintln!("{line}"))
+ })
+ }
+
+ pub fn run(&mut self, adopt: bool, albums: &[&str]) -> Result<Summary> {
+ let export = self.prepare(self.options(adopt, albums))?;
+ self.execute(export)
+ }
+}
diff --git a/rust/apps/cli-next/tests/support/export_maintenance.rs b/rust/apps/cli-next/tests/support/export_maintenance.rs
index 2a7f25c6ee..1e58775d1b 100644
--- a/rust/apps/cli-next/tests/support/export_maintenance.rs
+++ b/rust/apps/cli-next/tests/support/export_maintenance.rs
@@ -1,6 +1,15 @@
use super::*;
use crate::export_process::ExportChild;
+#[path = "export_engine.rs"]
+mod engine;
+
+#[path = "export_media.rs"]
+mod media;
+
+#[path = "export_adoption.rs"]
+mod adoption;
+
fn listing(server: &mut mockito::ServerGuard, albums: Value) -> mockito::Mock {
server
.mock("GET", "/collections/v2")
@@ -1909,7 +1918,7 @@ fn export_recovers_cross_album_publication_after_album_rename() {
}
#[test]
-fn export_repairs_missing_or_changed_cross_album_publications() {
+fn export_restores_missing_cross_album_publications_and_preserves_edits() {
for (owned, missing) in [(false, true), (false, false), (true, true), (true, false)] {
let mut fixture = interrupted_cross_album_publication(owned, 2);
let root = fixture.root.path();
@@ -1925,7 +1934,7 @@ fn export_repairs_missing_or_changed_cross_album_publications() {
json!([revision(collection(2, "Renamed", &fixture.key), 80)]),
);
page(&mut fixture.server, 2, 50, json!([]), false).create();
- if !missing && !owned {
+ if !missing {
let result = run(&fixture.home, root, &["--album", "Renamed"], false);
assert!(result["conflicts"].as_i64().unwrap() > 0);
assert_eq!(fs::read(root.join("Renamed/Photo.jpg")).unwrap(), b"wrong");
@@ -2311,12 +2320,6 @@ fn export_automatically_retries_saved_selected_album_and_file_failures() {
1
);
assert_component(&root, "First", "Photo.jpg", b"original");
- assert!(
- !home
- .run(&["photos", "export", root.to_str().unwrap(), "--retry-failed"])
- .status
- .success()
- );
first_page.assert();
fetched.assert();
}
diff --git a/rust/apps/cli-next/tests/support/export_media.rs b/rust/apps/cli-next/tests/support/export_media.rs
new file mode 100644
index 0000000000..dd3723567a
--- /dev/null
+++ b/rust/apps/cli-next/tests/support/export_media.rs
@@ -0,0 +1,416 @@
+use super::engine::{Catalog, Engine};
+use super::*;
+use std::collections::BTreeMap;
+
+fn new_engine(server: &mockito::ServerGuard, key: Key, record: Value) -> Engine {
+ let id = record["id"].as_i64().unwrap();
+ Engine::new(
+ server,
+ Catalog {
+ user: 9007199254740993,
+ key,
+ albums: BTreeMap::from([(1, "First".into()), (2, "Second".into())]),
+ files: BTreeMap::from([((1, id), record.clone()), ((2, id), record)]),
+ failed_album: None,
+ },
+ )
+}
+
+fn run(engine: &mut Engine, complete: bool) -> Value {
+ let result = engine.run(false, &[]).unwrap();
+ assert_eq!(result.complete, complete);
+ result.json
+}
+
+#[test]
+fn engine_preserves_local_media_and_restores_missing_copies() {
+ let mut server = mockito::Server::new();
+ let key = Key::generate();
+ let (file, bytes) = original_fixture(&key, b"original", false, true, 10);
+ let fetched = download(&mut server, 10, &bytes, 1);
+ let mut engine = new_engine(&server, key, file.clone());
+ run(&mut engine, true);
+ let root = engine.root();
+ let photo = root.join("First/Photo.jpg");
+ let sidecar = root.join("First/metadata/Photo.jpg.json");
+ let baseline = fs::read(&sidecar).unwrap();
+ let hash = read_json(&sidecar)["ente"]["components"][0]["hash"].clone();
+ fs::write(&photo, b"embedded EXIF edit").unwrap();
+ fs::write(
+ root.join("Second/metadata/Photo.jpg.json"),
+ b"{\"custom\":true}",
+ )
+ .unwrap();
+ let changed = public(
+ file,
+ &engine.catalog.key,
+ json!({"caption":"current caption"}),
+ 30,
+ );
+ engine.catalog.files = BTreeMap::from([((1, 10), changed.clone()), ((2, 10), changed)]);
+ for _ in 0..2 {
+ let result = run(&mut engine, false);
+ assert_eq!(result["conflicts"], 1);
+ assert_eq!(
+ result["copies"],
+ json!({"expected":2,"completed":1,"pending":1})
+ );
+ assert_eq!(fs::read(&photo).unwrap(), b"embedded EXIF edit");
+ assert_eq!(fs::read(&sidecar).unwrap(), baseline);
+ assert_eq!(
+ engine
+ .db()
+ .query_row(
+ "SELECT hash FROM components WHERE folder='active:1'",
+ [],
+ |r| r.get::<_, String>(0),
+ )
+ .unwrap(),
+ hash.as_str().unwrap()
+ );
+ let metadata = read_json(&root.join("Second/metadata/Photo.jpg.json"));
+ assert_eq!(metadata["description"], "current caption");
+ assert!(metadata.get("custom").is_none());
+ }
+ fs::remove_file(&photo).unwrap();
+ run(&mut engine, true);
+ assert_component(&root, "First", "Photo.jpg", b"original");
+ fs::write(&photo, b"local edit before removal").unwrap();
+ engine.catalog.files.retain(|(album, _), _| *album != 1);
+ run(&mut engine, true);
+ assert_eq!(
+ fs::read(root.join("Trash/First/Photo.jpg")).unwrap(),
+ b"local edit before removal"
+ );
+ assert_eq!(
+ read_json(&root.join("Trash/First/metadata/Photo.jpg.json"))["ente"]["components"][0]["hash"],
+ hash
+ );
+ assert_component(&root, "Second", "Photo.jpg", b"original");
+ engine.catalog.files.clear();
+ run(&mut engine, true);
+ assert_component(&root, "Trash/Second", "Photo.jpg", b"original");
+ run(&mut engine, true);
+ assert_eq!(
+ count(
+ &engine.db(),
+ "SELECT count(*) FROM placements WHERE retained=1"
+ ),
+ 2
+ );
+ fetched.assert();
+}
+
+#[test]
+fn engine_checks_previous_bytes_before_source_and_layout_replacement() {
+ for (before_live, after_live, hashed) in [
+ (false, false, true),
+ (false, false, false),
+ (true, true, true),
+ (false, true, true),
+ (true, false, true),
+ ] {
+ let mut server = mockito::Server::new();
+ let key = Key::generate();
+ let (old, bytes) = original_fixture(&key, b"old", before_live, hashed, 10);
+ let fetched = download(&mut server, 10, &bytes, 1);
+ let mut engine = new_engine(&server, key, old);
+ run(&mut engine, true);
+ fetched.assert();
+ fetched.remove();
+ let root = engine.root();
+ let name = if before_live {
+ "Motion.mov"
+ } else {
+ "Photo.jpg"
+ };
+ let photo = root.join("First").join(name);
+ let modified = fs::metadata(&photo).unwrap().modified().unwrap();
+ let baseline = fs::read(root.join("First/metadata").join(format!("{name}.json"))).unwrap();
+ fs::write(&photo, b"bad").unwrap();
+ fs::File::options()
+ .write(true)
+ .open(&photo)
+ .unwrap()
+ .set_modified(modified)
+ .unwrap();
+ run(&mut engine, true);
+ let (new, bytes) = original_fixture(&engine.catalog.key, b"new", after_live, hashed, 30);
+ engine.catalog.files = BTreeMap::from([((1, 10), new.clone()), ((2, 10), new)]);
+ let fetched = download(&mut server, 10, &bytes, 1);
+ let result = run(&mut engine, false);
+ assert_eq!(result["conflicts"], 1);
+ assert_eq!(result["copies"]["completed"], 1);
+ assert_eq!(fs::read(&photo).unwrap(), b"bad");
+ assert_eq!(fs::metadata(&photo).unwrap().modified().unwrap(), modified);
+ assert_eq!(
+ fs::read(root.join("First/metadata").join(format!("{name}.json"))).unwrap(),
+ baseline
+ );
+ if before_live {
+ assert_component(&root, "First", "Motion.heic", b"old");
+ }
+ assert!(!root.join("Trash/First").exists());
+ let names: &[&str] = if after_live {
+ &["Motion.heic", "Motion.mov"]
+ } else {
+ &["Photo.jpg"]
+ };
+ for name in names {
+ assert_component(&root, "Second", name, b"new");
+ }
+ if before_live == after_live {
+ fs::write(&photo, b"old").unwrap();
+ } else {
+ fs::remove_file(&photo).unwrap();
+ }
+ run(&mut engine, true);
+ for name in names {
+ assert_component(&root, "First", name, b"new");
+ }
+ if before_live != after_live {
+ assert!(!root.join("Trash/First").join(name).exists());
+ assert_eq!(
+ read_json(
+ &root
+ .join("Trash/First/metadata")
+ .join(format!("{name}.json"))
+ )["ente"]["components"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .find(|component| component["path"] == name)
+ .unwrap()["hash"],
+ b64::encode(&hash::hash(b"old", Some(64), None).unwrap())
+ );
+ if before_live {
+ assert_component(&root, "Trash/First", "Motion.heic", b"old");
+ }
+ }
+ fetched.assert();
+ }
+}
+
+#[test]
+fn engine_restores_a_new_layout_after_missing_interrupted_retention() {
+ for old_live in [false, true] {
+ let mut server = mockito::Server::new();
+ let key = Key::generate();
+ let (old, bytes) = original_fixture(&key, b"old", old_live, true, 10);
+ let fetched = download(&mut server, 10, &bytes, 1);
+ let mut engine = new_engine(&server, key, old);
+ run(&mut engine, true);
+ fetched.assert();
+ fetched.remove();
+ let db = engine.db();
+ db.execute_batch("CREATE TRIGGER stop_retention BEFORE UPDATE ON pending WHEN old.album=1 AND old.retained=1 AND json_type(old.action,'$.Move') IS NOT NULL AND new.action='null' BEGIN SELECT RAISE(ABORT,'stop after retention move'); END;").unwrap();
+ let (new, bytes) = original_fixture(&engine.catalog.key, b"new", !old_live, true, 30);
+ engine.catalog.files = BTreeMap::from([((1, 10), new.clone()), ((2, 10), new)]);
+ let fetched = download(&mut server, 10, &bytes, 2);
+ run(&mut engine, false);
+ let root = engine.root();
+ let old_name = if old_live { "Motion.heic" } else { "Photo.jpg" };
+ fs::remove_file(root.join("Trash/First").join(old_name)).unwrap();
+ db.execute_batch("DROP TRIGGER stop_retention;").unwrap();
+ if old_live {
+ fs::write(root.join("First/Motion.mov"), b"local edit").unwrap();
+ assert_eq!(run(&mut engine, false)["conflicts"], 1);
+ assert_eq!(
+ fs::read(root.join("First/Motion.mov")).unwrap(),
+ b"local edit"
+ );
+ assert!(!root.join("First/Photo.jpg").exists());
+ fs::remove_file(root.join("First/Motion.mov")).unwrap();
+ }
+ run(&mut engine, true);
+ let new_names: &[&str] = if old_live {
+ &["Photo.jpg"]
+ } else {
+ &["Motion.heic", "Motion.mov"]
+ };
+ for folder in ["First", "Second"] {
+ for name in new_names {
+ assert_component(&root, folder, name, b"new");
+ }
+ }
+ assert!(!root.join("Trash/First").join(old_name).exists());
+ assert_eq!(
+ read_json(
+ &root
+ .join("Trash/First/metadata")
+ .join(format!("{old_name}.json"))
+ )["ente"]["components"][0]["hash"],
+ b64::encode(&hash::hash(b"old", Some(64), None).unwrap())
+ );
+ assert_eq!(run(&mut engine, true)["changes"]["retained"], 0);
+ assert_eq!(
+ count(
+ &db,
+ "SELECT count(*) FROM placements WHERE album=1 AND retained=1"
+ ),
+ 1
+ );
+ fetched.assert();
+ }
+}
+
+#[test]
+fn engine_checks_both_live_components_when_only_one_source_component_changes() {
+ let mut server = mockito::Server::new();
+ let key = Key::generate();
+ let (old, bytes) = original_fixture(&key, b"old", true, true, 10);
+ let fetched = download(&mut server, 10, &bytes, 1);
+ let mut engine = new_engine(&server, key, old);
+ run(&mut engine, true);
+ fetched.assert();
+ fetched.remove();
+ let root = engine.root();
+ let photo = root.join("First/Motion.heic");
+ let modified = fs::metadata(&photo).unwrap().modified().unwrap();
+ fs::write(&photo, b"bad").unwrap();
+ fs::File::options()
+ .write(true)
+ .open(&photo)
+ .unwrap()
+ .set_modified(modified)
+ .unwrap();
+ let mut archive = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
+ for (name, bytes) in [("image.heic", b"old"), ("video.mov", b"new")] {
+ archive
+ .start_file(name, zip::write::SimpleFileOptions::default())
+ .unwrap();
+ archive.write_all(bytes).unwrap();
+ }
+ let archive = archive.finish().unwrap().into_inner();
+ let (mut new, bytes) = source(10, &engine.catalog.key, &archive, "Motion.heic");
+ let image = b64::encode(&hash::hash(b"old", Some(64), None).unwrap());
+ let video = b64::encode(&hash::hash(b"new", Some(64), None).unwrap());
+ let metadata = blob::encrypt_json(
+ &json!({"title":"Motion.heic","fileType":2,"creationTime":1_700_000_000_123_456i64,"hash":format!("{image}:{video}")}),
+ &file_key(&new, &engine.catalog.key),
+ ).unwrap();
+ new["metadata"] = json!({"encryptedData":b64::encode(&metadata.encrypted_data),"decryptionHeader":b64::encode(metadata.decryption_header.as_bytes())});
+ let new = revision(new, 30);
+ engine.catalog.files = BTreeMap::from([((1, 10), new.clone()), ((2, 10), new)]);
+ let fetched = download(&mut server, 10, &bytes, 1);
+ assert_eq!(run(&mut engine, false)["conflicts"], 1);
+ assert_eq!(fs::read(&photo).unwrap(), b"bad");
+ assert_component(&root, "First", "Motion.mov", b"old");
+ assert_component(&root, "Second", "Motion.heic", b"old");
+ assert_component(&root, "Second", "Motion.mov", b"new");
+ fs::write(&photo, b"old").unwrap();
+ run(&mut engine, true);
+ assert_component(&root, "First", "Motion.heic", b"old");
+ assert_component(&root, "First", "Motion.mov", b"new");
+ fetched.assert();
+}
+
+#[test]
+fn engine_recovery_checks_live_siblings_before_acknowledging_publication() {
+ let mut server = mockito::Server::new();
+ let key = Key::generate();
+ let (old, bytes) = original_fixture(&key, b"old", true, true, 10);
+ let fetched = download(&mut server, 10, &bytes, 1);
+ let mut engine = new_engine(&server, key, old);
+ run(&mut engine, true);
+ fetched.assert();
+ fetched.remove();
+ let db = engine.db();
+ db.execute_batch("CREATE TRIGGER stop_ack BEFORE UPDATE ON pending WHEN old.album=1 AND json_extract(old.action,'$.Publish.output.Media.role')='image' AND new.action='null' BEGIN SELECT RAISE(ABORT,'stop after image publication'); END;").unwrap();
+ let (new, bytes) = original_fixture(&engine.catalog.key, b"new", true, true, 30);
+ engine.catalog.files = BTreeMap::from([((1, 10), new.clone()), ((2, 10), new)]);
+ let fetched = download(&mut server, 10, &bytes, 2);
+ run(&mut engine, false);
+ let root = engine.root();
+ assert_eq!(fs::read(root.join("First/Motion.heic")).unwrap(), b"new");
+ fs::write(root.join("First/Motion.mov"), b"edited video").unwrap();
+ let pending: String = db
+ .query_row(
+ "SELECT action FROM pending WHERE album=1 AND file=10",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ db.execute_batch("DROP TRIGGER stop_ack;").unwrap();
+ let result = run(&mut engine, false);
+ assert_eq!(result["conflicts"], 1);
+ assert_eq!(
+ fs::read(root.join("First/Motion.mov")).unwrap(),
+ b"edited video"
+ );
+ assert_eq!(
+ db.query_row(
+ "SELECT action FROM pending WHERE album=1 AND file=10",
+ [],
+ |r| r.get::<_, String>(0)
+ )
+ .unwrap(),
+ pending
+ );
+ assert_component(&root, "Second", "Motion.heic", b"new");
+ assert_component(&root, "Second", "Motion.mov", b"new");
+ fs::remove_file(root.join("First/Motion.mov")).unwrap();
+ run(&mut engine, true);
+ assert_component(&root, "First", "Motion.heic", b"new");
+ assert_component(&root, "First", "Motion.mov", b"new");
+ fetched.assert();
+}
+
+#[test]
+fn engine_recovery_preserves_edits_at_either_move_endpoint() {
+ for (moved, removed) in [(false, false), (true, false), (false, true), (true, true)] {
+ let mut server = mockito::Server::new();
+ let key = Key::generate();
+ let (file, bytes) = original_fixture(&key, b"original", false, true, 10);
+ let fetched = download(&mut server, 10, &bytes, 1);
+ let mut engine = new_engine(&server, key, file.clone());
+ run(&mut engine, true);
+ let db = engine.db();
+ db.execute_batch("CREATE TRIGGER stop_move BEFORE UPDATE ON pending WHEN old.album=1 AND json_type(old.action,'$.Move') IS NOT NULL AND new.action='null' BEGIN SELECT RAISE(ABORT,'stop after media move'); END;").unwrap();
+ let renamed = public(
+ file,
+ &engine.catalog.key,
+ json!({"editedName":"Renamed.jpg"}),
+ 30,
+ );
+ engine.catalog.files = BTreeMap::from([((1, 10), renamed.clone()), ((2, 10), renamed)]);
+ run(&mut engine, false);
+ let root = engine.root();
+ let old = root.join("First/Photo.jpg");
+ let new = root.join("First/Renamed.jpg");
+ if !moved {
+ fs::rename(&new, &old).unwrap();
+ }
+ let changed = if moved { &new } else { &old };
+ fs::write(changed, b"local edit").unwrap();
+ db.execute_batch("DROP TRIGGER stop_move;").unwrap();
+ if removed {
+ engine.catalog.files.retain(|(album, _), _| *album != 1);
+ run(&mut engine, true);
+ assert_eq!(
+ fs::read(root.join("Trash/First/Photo.jpg")).unwrap(),
+ b"local edit"
+ );
+ assert_eq!(
+ read_json(&root.join("Trash/First/metadata/Photo.jpg.json"))["ente"]["components"]
+ [0]["hash"],
+ b64::encode(&hash::hash(b"original", Some(64), None).unwrap())
+ );
+ assert_component(&root, "Second", "Renamed.jpg", b"original");
+ fetched.assert();
+ continue;
+ }
+ let result = run(&mut engine, false);
+ assert_eq!(result["conflicts"], 1);
+ assert_eq!(fs::read(changed).unwrap(), b"local edit");
+ assert_component(&root, "Second", "Renamed.jpg", b"original");
+ assert_eq!(
+ read_json(&root.join("First/metadata/Photo.jpg.json"))["ente"]["components"][0]["hash"],
+ b64::encode(&hash::hash(b"original", Some(64), None).unwrap())
+ );
+ fs::write(changed, b"original").unwrap();
+ run(&mut engine, true);
+ assert_component(&root, "First", "Renamed.jpg", b"original");
+ fetched.assert();
+ }
+}
diff --git a/rust/apps/cli-next/tests/support/museum.rs b/rust/apps/cli-next/tests/support/museum.rs
index 1177c4a7d9..26c54adc60 100644
--- a/rust/apps/cli-next/tests/support/museum.rs
+++ b/rust/apps/cli-next/tests/support/museum.rs
@@ -370,11 +370,17 @@ async fn exercise(origin: String) -> TestResult {
assert!(file_record["ente"].get("visibility").is_none());
let adopted = TestHome::new();
adopted.write_vault(&home.read_vault());
- fs::write(
- shared_export.join("Monsoon 🌧/original.jpg"),
- vec![0; original.len()],
- )
- .unwrap();
+ let shared_original = shared_export.join("Monsoon 🌧/original.jpg");
+ fs::write(&shared_original, vec![0; original.len()]).unwrap();
+ let output = adopted.run(&[
+ "photos",
+ "export",
+ shared_export.to_str().unwrap(),
+ "--adopt",
+ ]);
+ assert!(failure(&output).contains("locally changed media"));
+ assert_eq!(fs::read(&shared_original).unwrap(), vec![0; original.len()]);
+ fs::remove_file(&shared_original).unwrap();
adopted.json(&[
"photos",
"export",
diff --git a/rust/crates/photos-export/Cargo.toml b/rust/crates/photos-export/Cargo.toml
index b3d3deb980..f0ea90eefe 100644
--- a/rust/crates/photos-export/Cargo.toml
+++ b/rust/crates/photos-export/Cargo.toml
@@ -13,6 +13,7 @@ icu_casemap.workspace = true
rusqlite.workspace = true
serde.workspace = true
serde_json.workspace = true
+tempfile.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }
unicode-normalization.workspace = true
diff --git a/rust/crates/photos-export/src/adopt.rs b/rust/crates/photos-export/src/adopt.rs
index a0f7788bff..a43e3912a9 100644
--- a/rust/crates/photos-export/src/adopt.rs
+++ b/rust/crates/photos-export/src/adopt.rs
@@ -1,50 +1,230 @@
use std::{fs, path::Path};
-use crate::metadata::{Component, Role};
use anyhow::{Context, Result, ensure};
+use rusqlite::{Connection, params};
+use serde::{Deserialize, Serialize};
use serde_json::Value;
use uuid::Uuid;
-use super::{
+use crate::{
+ legacy::read::Desktop,
+ metadata::{Component, Role},
names,
- store::{Album, JsonRecord, Location, Placement, Properties, Store},
+ store::{self, Album, JsonRecord, Location, Placement, Properties, Store},
};
-pub fn scan(root: &Path, store: &Store) -> Result<()> {
- store
- .db
- .execute_batch(
- "DELETE FROM components; DELETE FROM json_records; DELETE FROM placements; DELETE FROM albums; DELETE FROM pending; DELETE FROM temporaries;",
+#[derive(Serialize, Deserialize)]
+pub struct AlbumRecord {
+ pub album: Album,
+ pub json: Option<JsonRecord>,
+ pub legacy: Option<crate::legacy::read::AlbumInfo>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Item {
+ pub placement: Placement,
+ pub components: Vec<store::Component>,
+ pub json: Vec<JsonRecord>,
+ pub legacy: Option<crate::legacy::read::Record>,
+ pub leftover: bool,
+}
+
+pub struct Index {
+ pub db: Connection,
+ _directory: tempfile::TempDir,
+}
+
+impl Index {
+ pub fn new(open: impl FnOnce(&Path) -> Result<Connection>) -> Result<Self> {
+ let directory = tempfile::tempdir()?;
+ let db = open(&directory.path().join("index.db"))?;
+ db.execute_batch(
+ "PRAGMA temp_store=MEMORY; PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF; BEGIN;",
)?;
+ db.execute_batch("
+CREATE TABLE adoption_album(key TEXT PRIMARY KEY,id INTEGER NOT NULL,retained INTEGER NOT NULL,parent TEXT NOT NULL,folded TEXT NOT NULL,record TEXT NOT NULL);
+CREATE UNIQUE INDEX adoption_album_id ON adoption_album(id) WHERE retained=0;
+CREATE UNIQUE INDEX adoption_album_path ON adoption_album(parent,folded);
+CREATE TABLE adoption_item(key TEXT PRIMARY KEY,album_key TEXT NOT NULL,file_id INTEGER NOT NULL,active INTEGER NOT NULL,origin TEXT NOT NULL,anchor TEXT NOT NULL,record TEXT NOT NULL);
+CREATE UNIQUE INDEX adoption_membership ON adoption_item(album_key,file_id) WHERE active=1;
+CREATE INDEX adoption_retained ON adoption_item(album_key,file_id,anchor);
+CREATE INDEX adoption_file ON adoption_item(file_id,key);
+CREATE TABLE adoption_path(path TEXT PRIMARY KEY,album_key TEXT NOT NULL,stem TEXT NOT NULL,kind TEXT NOT NULL,item_key TEXT NOT NULL);
+CREATE INDEX adoption_stem ON adoption_path(album_key,stem);
+")?;
+ Ok(Self {
+ db,
+ _directory: directory,
+ })
+ }
+}
+
+pub fn save_album(index: &Index, record: &AlbumRecord) -> Result<()> {
+ let a = &record.album;
+ let (parent, name) = a.path.rsplit_once('/').unwrap_or(("", &a.path));
+ index.db.execute("INSERT INTO adoption_album VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(key) DO UPDATE SET parent=excluded.parent,folded=excluded.folded,record=excluded.record",params![a.key,a.id,a.retained,parent,names::folded(name),serde_json::to_string(record)?])?;
+ Ok(())
+}
+
+pub fn save_item(index: &Index, album: &Album, item: &Item) -> Result<()> {
+ index.db.execute("INSERT INTO adoption_item VALUES(?1,?2,?3,?4,?5,?6,?7) ON CONFLICT(key) DO UPDATE SET record=excluded.record",params![item.placement.id,album.key,item.placement.file,!item.placement.retained,if item.legacy.is_some(){"legacy"}else{"native"},item.components.first().map(|c|c.location.name.as_str()).unwrap_or(""),serde_json::to_string(item)?])?;
+ Ok(())
+}
+
+pub fn claim(index: &Index, album: &Album, item: &str, name: &str, kind: &str) -> Result<()> {
+ let path = names::folded(&format!("{}/{name}", album.path));
+ let stem = names::folded(names::split(name, false).0);
+ let overlap: bool=index.db.query_row("SELECT EXISTS(SELECT 1 FROM adoption_path WHERE item_key<>?1 AND (path=?2 OR (album_key=?3 AND stem=?4 AND kind<>'' AND ?5<>'' AND NOT(kind=?5 AND kind IN ('image','video')))))",params![item,path,album.key,stem,kind],|r|r.get(0))?;
+ ensure!(
+ !overlap,
+ crate::Conflict(format!("overlapping media paths at {}/{name}", album.path))
+ );
+ index.db.execute("INSERT INTO adoption_path VALUES(?1,?2,?3,?4,?5) ON CONFLICT(path) DO UPDATE SET kind=excluded.kind",params![path,album.key,stem,kind,item])?;
+ Ok(())
+}
+
+pub fn scan(root: &Path, index: &Index, desktop: Option<&Desktop>) -> Result<()> {
for entry in fs::read_dir(root)? {
let entry = entry?;
- if entry.file_name() == "Trash" {
+ if entry.file_name() == "Trash"
+ && !entry.path().join(".meta/album_meta.json").try_exists()?
+ {
ensure!(
entry.file_type()?.is_dir(),
- super::Conflict("at Trash: expected directory".into())
+ crate::Conflict("at Trash: expected directory".into())
);
for entry in fs::read_dir(entry.path())? {
let entry = entry?;
if entry.file_type()?.is_dir() {
- album(root, store, &entry.path(), true)?;
+ scan_album(root, index, &entry.path(), true, desktop)?;
}
}
} else if entry.file_type()?.is_dir() {
- album(root, store, &entry.path(), false)?;
+ scan_album(root, index, &entry.path(), false, desktop)?;
}
}
Ok(())
}
-fn album(root: &Path, store: &Store, directory: &Path, retained: bool) -> Result<()> {
+fn scan_album(
+ root: &Path,
+ index: &Index,
+ directory: &Path,
+ retained: bool,
+ desktop: Option<&Desktop>,
+) -> Result<()> {
+ let Some(record) = parse_album(root, directory, retained)? else {
+ return Ok(());
+ };
+ let album = &record.album;
+ let (parent, name) = album.path.rsplit_once('/').unwrap_or(("", &album.path));
+ let (duplicate_id, duplicate_path): (bool, bool)=index.db.query_row("SELECT EXISTS(SELECT 1 FROM adoption_album WHERE id=?1 AND retained=0 AND ?2=0), EXISTS(SELECT 1 FROM adoption_album WHERE parent=?3 AND folded=?4)",params![album.id,retained,parent,names::folded(name)],|r|Ok((r.get(0)?,r.get(1)?)))?;
+ ensure!(
+ !duplicate_id,
+ crate::Conflict(format!("active album ID {}", album.id))
+ );
+ ensure!(
+ !duplicate_path,
+ crate::Conflict(format!("duplicate native album path {}", album.path))
+ );
+ save_album(index, &record)?;
+ let metadata = names::check(root, &format!("{}/metadata", album.path))?;
+ if !metadata.try_exists()? {
+ return Ok(());
+ }
+ ensure!(
+ metadata.is_dir(),
+ crate::Conflict("expected metadata directory".into())
+ );
+ for entry in fs::read_dir(metadata)? {
+ let entry = entry?;
+ if entry.path().extension().is_none_or(|e| e != "json") {
+ continue;
+ }
+ let name = entry
+ .file_name()
+ .into_string()
+ .map_err(|_| anyhow::anyhow!("claimed Ente sidecar name is not UTF-8"))?;
+ let path = names::check(root, &format!("{}/metadata/{name}", album.path))?;
+ let properties = Properties::read(&path)?;
+ let bytes = fs::read(path)?;
+ let value: Value = match serde_json::from_slice(&bytes) {
+ Ok(value) => value,
+ Err(_) if desktop.is_some_and(|d| d.common_sidecar(album.id, &name)) => continue,
+ Err(error) => return Err(error.into()),
+ };
+ let Some(mut item) = parse_sidecar(album, &name, value, &bytes, properties)? else {
+ continue;
+ };
+ let previous: Option<Item> = store::json(
+ &index.db,
+ "SELECT record FROM adoption_item WHERE album_key=?1 AND file_id=?2 AND (active=1 OR anchor=?3)",
+ params![
+ album.key,
+ item.placement.file,
+ item.components[0].location.name
+ ],
+ )?;
+ if let Some(mut previous) = previous {
+ ensure!(
+ previous.placement.kind == item.placement.kind
+ && previous.components
+ == item
+ .components
+ .iter()
+ .map(|c| {
+ let mut c = c.clone();
+ c.placement = previous.placement.id.clone();
+ c
+ })
+ .collect::<Vec<_>>(),
+ crate::Conflict(format!(
+ "contradictory native placement for file {}",
+ item.placement.file
+ ))
+ );
+ let actual = item
+ .json
+ .iter()
+ .find(|r| r.location.is_some())
+ .context("missing sidecar")?;
+ let slot = previous
+ .json
+ .iter_mut()
+ .find(|r| r.role == actual.role)
+ .context("missing component")?;
+ ensure!(
+ slot.location.is_none(),
+ "duplicate component identity for file {}",
+ item.placement.file
+ );
+ *slot = actual.clone();
+ slot.owner = previous.placement.id.clone();
+ item = previous;
+ }
+ for component in &item.components {
+ claim(
+ index,
+ album,
+ &item.placement.id,
+ &component.location.name,
+ &item.placement.kind,
+ )?;
+ }
+ save_item(index, album, &item)?;
+ }
+ Ok(())
+}
+
+fn parse_album(root: &Path, directory: &Path, retained: bool) -> Result<Option<AlbumRecord>> {
let path = directory.join("metadata.json");
let Some(properties) = Properties::optional(&path)? else {
- return Ok(());
+ return Ok(None);
};
let bytes = fs::read(&path)?;
let record: Value = serde_json::from_slice(&bytes)?;
let Some(identity) = record.get("ente") else {
- return Ok(());
+ return Ok(None);
};
let folder = directory
.strip_prefix(root)?
@@ -74,8 +254,8 @@ fn album(root: &Path, store: &Store, directory: &Path, retained: bool) -> Result
"unsupported album visibility in {folder}"
);
ensure!(
- retained || store.album(id, false)?.is_none(),
- super::Conflict(format!("active album ID {id}"))
+ retained || !names::reserved(folder, true),
+ "reserved active album path {folder}"
);
let album = Album {
key: if retained {
@@ -88,17 +268,7 @@ fn album(root: &Path, store: &Store, directory: &Path, retained: bool) -> Result
name,
path: folder.into(),
};
- let (parent, name) = folder.rsplit_once('/').unwrap_or(("", folder));
- ensure!(
- retained || !names::reserved(name, true),
- "reserved active album path {folder}"
- );
- ensure!(
- names::available(store, parent, &[name.into()], &album.key, "album")?,
- super::Conflict(format!("overlapping album paths at {folder}"))
- );
- store.save_album(&album)?;
- store.save_json(&JsonRecord {
+ let json = JsonRecord {
owner: album.key.clone(),
role: None,
location: Some(Location {
@@ -108,209 +278,179 @@ fn album(root: &Path, store: &Store, directory: &Path, retained: bool) -> Result
value: record,
hash: Some(digest(&bytes)?),
properties: Some(properties),
- })?;
- let metadata = names::check(root, &format!("{folder}/metadata"))?;
- if !metadata.try_exists()? {
- return Ok(());
- }
+ };
+ Ok(Some(AlbumRecord {
+ album,
+ json: Some(json),
+ legacy: None,
+ }))
+}
+
+fn parse_sidecar(
+ album: &Album,
+ name: &str,
+ record: Value,
+ bytes: &[u8],
+ properties: Properties,
+) -> Result<Option<Item>> {
+ let Some(identity) = record.get("ente") else {
+ return Ok(None);
+ };
+ let file = parse_id(identity, "fileID")?;
+ let display_name = identity["name"]
+ .as_str()
+ .context("file record has no name")?;
+ chrono::DateTime::parse_from_rfc3339(
+ identity["creationTime"]
+ .as_str()
+ .context("file record has no creationTime")?,
+ )?;
+ let kind = identity["type"]
+ .as_str()
+ .context("file record has no type")?;
ensure!(
- metadata.is_dir(),
- super::Conflict(format!(
- "at {}: expected metadata directory",
- metadata.display()
- ))
+ ["image", "video", "livePhoto", "unknown"].contains(&kind),
+ "unsupported file type for {file}"
);
- for entry in fs::read_dir(metadata)? {
- let entry = entry?;
- if entry
- .path()
- .extension()
- .is_none_or(|extension| extension != "json")
- {
- continue;
- }
- let properties = Properties::read(&entry.path())?;
- let bytes = fs::read(entry.path())?;
- let record: Value = serde_json::from_slice(&bytes)?;
- let Some(identity) = record.get("ente") else {
- continue;
- };
- let name = entry
- .file_name()
- .into_string()
- .map_err(|_| anyhow::anyhow!("claimed Ente sidecar name is not UTF-8"))?;
- names::check(root, &format!("{folder}/metadata/{name}"))?;
- let file = parse_id(identity, "fileID")?;
- let display_name = identity["name"]
- .as_str()
- .context("file record has no name")?;
- chrono::DateTime::parse_from_rfc3339(
- identity["creationTime"]
- .as_str()
- .context("file record has no creationTime")?,
- )?;
- let kind = identity["type"]
- .as_str()
- .context("file record has no type")?;
+ let mut components: Vec<Component> = serde_json::from_value(identity["components"].clone())?;
+ ensure!(
+ if kind == "livePhoto" {
+ components.len() == 2
+ && components.iter().any(|c| c.role == Role::Image)
+ && components.iter().any(|c| c.role == Role::Video)
+ } else {
+ components.len() == 1 && components[0].role == Role::Original
+ },
+ "invalid components for file {file}"
+ );
+ components.sort_by_key(|component| match component.role {
+ Role::Original => 0,
+ Role::Image => 1,
+ Role::Video => 2,
+ });
+ for component in &components {
+ names::relative(&component.path)?;
ensure!(
- ["image", "video", "livePhoto", "unknown"].contains(&kind),
- "unsupported file type for {file}"
+ !component.path.contains('/') && !names::reserved(&component.path, false),
+ "invalid component path for file {file}"
);
- let mut components: Vec<Component> =
- serde_json::from_value(identity["components"].clone())?;
ensure!(
- if kind == "livePhoto" {
- components.len() == 2
- && components.iter().any(|c| c.role == Role::Image)
- && components.iter().any(|c| c.role == Role::Video)
- } else {
- components.len() == 1 && components[0].role == Role::Original
- },
- "invalid components for file {file}"
+ ente_core::b64::decode(&component.hash)?.len() == 64,
+ "invalid component hash for file {file}"
);
- components.sort_by_key(|component| match component.role {
- Role::Original => 0,
- Role::Image => 1,
- Role::Video => 2,
- });
- for component in &components {
- names::relative(&component.path)?;
- ensure!(
- !component.path.contains('/') && !names::reserved(&component.path, false),
- "invalid component path for file {file}"
- );
- ensure!(
- ente_core::b64::decode(&component.hash)?.len() == 64,
- "invalid component hash for file {file}"
- );
+ }
+ ensure!(
+ components.len() == 1
+ || names::folded(&components[0].path) != names::folded(&components[1].path),
+ "overlapping Live Photo components for file {file}"
+ );
+ let own: Role = serde_json::from_value(identity["component"].clone())?;
+ let index = components
+ .iter()
+ .position(|c| c.role == own)
+ .context("undeclared sidecar component")?;
+ ensure!(
+ record["title"].as_str() == Some(components[index].path.as_str())
+ && name == format!("{}.json", components[index].path),
+ "contradictory sidecar location for file {file}"
+ );
+ let kind = if kind == "livePhoto" {
+ "livephoto"
+ } else {
+ kind
+ };
+ let placement = Placement {
+ id: Uuid::new_v4().to_string(),
+ album: album.id,
+ file,
+ retained: album.retained,
+ name: display_name.into(),
+ kind: kind.into(),
+ };
+ let mut json = Vec::new();
+ let components = components
+ .into_iter()
+ .map(|component| {
+ let actual = component.role == own;
+ let mut value = record.clone();
+ value["title"] = Value::String(component.path.clone());
+ value["ente"]["component"] = serde_json::to_value(&component.role)?;
+ json.push(JsonRecord {
+ owner: placement.id.clone(),
+ role: Some(component.role.clone()),
+ location: actual.then(|| Location {
+ folder: album.key.clone(),
+ name: format!("metadata/{name}"),
+ }),
+ value,
+ hash: if actual { Some(digest(bytes)?) } else { None },
+ properties: actual.then(|| properties.clone()),
+ });
+ Ok(store::Component {
+ placement: placement.id.clone(),
+ role: component.role,
+ location: Location {
+ folder: album.key.clone(),
+ name: component.path,
+ },
+ size: component.size,
+ hash: component.hash,
+ signature: None,
+ properties: None,
+ intended_time: None,
+ })
+ })
+ .collect::<Result<Vec<_>>>()?;
+ Ok(Some(Item {
+ placement,
+ components,
+ json,
+ legacy: None,
+ leftover: false,
+ }))
+}
+
+pub fn accept(index: &Index, store: &Store, bound: bool) -> Result<()> {
+ let transaction = store.db.unchecked_transaction()?;
+ store.db.execute_batch("DELETE FROM components; DELETE FROM json_records; DELETE FROM placements; DELETE FROM albums; DELETE FROM pending; DELETE FROM temporaries;")?;
+ let mut after = String::new();
+ while let Some(record) = store::json::<AlbumRecord>(
+ &index.db,
+ "SELECT record FROM adoption_album WHERE key>?1 ORDER BY key LIMIT 1",
+ [&after],
+ )? {
+ after = record.album.key.clone();
+ store.save_album(&record.album)?;
+ if let Some(json) = record.json {
+ store.save_json(&json)?;
}
- ensure!(
- components.len() == 1
- || names::folded(&components[0].path) != names::folded(&components[1].path),
- "overlapping Live Photo components for file {file}"
- );
- let own: Role = serde_json::from_value(identity["component"].clone())?;
- let index = components
- .iter()
- .position(|c| c.role == own)
- .context("undeclared sidecar component")?;
- ensure!(
- record["title"].as_str() == Some(components[index].path.as_str())
- && name == format!("{}.json", components[index].path),
- "contradictory sidecar location for file {file}"
- );
- let kind = if kind == "livePhoto" {
- "livephoto"
- } else {
- kind
- };
- let existing = if retained {
- let mut query = store
- .db
- .prepare(
- "SELECT DISTINCT p.id,p.album,p.file,p.retained,p.name,p.kind FROM placements p JOIN components c ON c.placement=p.id WHERE p.album=?1 AND p.file=?2 AND p.retained=1 AND c.folder=?3",
- )?;
- let mut rows = query.query(rusqlite::params![id, file, album.key])?;
- let mut found = None;
- while let Some(row) = rows.next()? {
- let candidate = Placement::read(row)?;
- if store
- .components(&candidate.id)?
- .iter()
- .map(super::store::Component::portable)
- .eq(components.iter().cloned())
- {
- found = Some(candidate);
- break;
- }
- }
- found
- } else {
- store.placement(id, file)?
- };
- let fresh = existing.is_none();
- let placement = existing.unwrap_or_else(|| Placement {
- id: Uuid::new_v4().to_string(),
- album: id,
- file,
- retained,
- name: display_name.into(),
- kind: kind.into(),
- });
- if !fresh {
- ensure!(
- placement.kind == kind
- && store
- .components(&placement.id)?
- .iter()
- .all(|component| component.location.folder == album.key)
- && store
- .components(&placement.id)?
- .iter()
- .map(super::store::Component::portable)
- .eq(components.iter().cloned()),
- super::Conflict(format!("active placement for file {file} in album {id}"))
- );
+ }
+ after.clear();
+ while let Some(item) = store::json::<Item>(
+ &index.db,
+ "SELECT record FROM adoption_item WHERE key>?1 ORDER BY key LIMIT 1",
+ [&after],
+ )? {
+ after = item.placement.id.clone();
+ if item.leftover || item.components.is_empty() {
+ continue;
}
- ensure!(
- !store
- .json_record(&placement.id, Some(&own))?
- .is_some_and(|record| record.location.is_some()),
- "duplicate component identity for file {file}"
- );
- let media_names: Vec<_> = components
- .iter()
- .map(|component| component.path.clone())
- .collect();
- ensure!(
- names::available(store, &album.key, &media_names, &placement.id, kind)?,
- super::Conflict(format!("overlapping media paths in {folder}"))
- );
- store.save_placement(&placement)?;
- if fresh {
- for component in &components {
- store.save_component(&super::store::Component {
- placement: placement.id.clone(),
- role: component.role.clone(),
- location: Location {
- folder: album.key.clone(),
- name: component.path.clone(),
- },
- size: component.size,
- hash: component.hash.clone(),
- signature: None,
- properties: None,
- intended_time: None,
- })?;
- let mut fallback = record.clone();
- fallback["title"] = Value::String(component.path.clone());
- fallback["ente"]["component"] = serde_json::to_value(&component.role)?;
- store.save_json(&JsonRecord {
- owner: placement.id.clone(),
- role: Some(component.role.clone()),
- location: None,
- value: fallback,
- hash: None,
- properties: None,
- })?;
- }
+ store.save_placement(&item.placement)?;
+ for component in item.components {
+ store.save_component(&component)?;
+ }
+ for json in item.json {
+ store.save_json(&json)?;
}
- store.save_json(&JsonRecord {
- owner: placement.id,
- role: Some(own),
- location: Some(Location {
- folder: album.key.clone(),
- name: format!("metadata/{name}"),
- }),
- value: record,
- hash: Some(digest(&bytes)?),
- properties: Some(properties),
- })?;
}
+ store
+ .db
+ .execute("UPDATE export SET bound=?1 WHERE id=1", [bound])?;
+ transaction.commit()?;
Ok(())
}
-fn digest(bytes: &[u8]) -> Result<String> {
+pub fn digest(bytes: &[u8]) -> Result<String> {
Ok(ente_core::b64::encode(&ente_core::crypto::hash::hash(
bytes,
Some(64),
diff --git a/rust/crates/photos-export/src/fs.rs b/rust/crates/photos-export/src/fs.rs
index f12364c424..20e58de8a3 100644
--- a/rust/crates/photos-export/src/fs.rs
+++ b/rust/crates/photos-export/src/fs.rs
@@ -130,6 +130,24 @@ pub fn sync_move(source: &Path, destination: &Path) -> Result<()> {
Ok(())
}
+pub fn free_rename(source: &Path, destination: &Path) -> Result<()> {
+ ensure!(
+ !destination.try_exists()? || names::same_path(source, destination),
+ super::Conflict(format!(
+ "at {}: occupied rename target",
+ destination.display()
+ ))
+ );
+ Ok(())
+}
+
+pub fn move_file(source: &Path, destination: &Path) -> Result<()> {
+ free_rename(source, destination)?;
+ create_directory(destination.parent().context("move has no parent")?)?;
+ fs::rename(source, destination)?;
+ sync_move(source, destination)
+}
+
pub struct Json {
pub value: Value,
pub bytes: Vec<u8>,
diff --git a/rust/crates/photos-export/src/legacy/mod.rs b/rust/crates/photos-export/src/legacy/mod.rs
new file mode 100644
index 0000000000..f20a7d8a31
--- /dev/null
+++ b/rust/crates/photos-export/src/legacy/mod.rs
@@ -0,0 +1,497 @@
+pub mod read;
+mod verify;
+
+use crate::{
+ Options, Source,
+ adopt::{self, AlbumRecord, Index, Item},
+ fs, names,
+ snapshot::FileSnapshot,
+ store::{self, AlbumSource, JsonRecord, Properties, Store},
+};
+use anyhow::{Context, Result, ensure};
+use ente_core::Session;
+use rusqlite::{OptionalExtension, params};
+use std::{
+ collections::{BTreeMap, HashSet, btree_map::Entry},
+ fs as disk,
+ path::Path,
+};
+use tokio::sync::watch;
+
+#[derive(Default)]
+pub struct Failures {
+ count: usize,
+ examples: Vec<String>,
+}
+
+impl Failures {
+ pub fn record(&mut self, result: Result<()>, context: &str) -> Result<()> {
+ if let Err(error) = result {
+ if crate::fatal(&error) {
+ return Err(error.context(format!("adoption preflight incomplete: {context}")));
+ }
+ self.count += 1;
+ if self.examples.len() < 10 {
+ self.examples.push(format!("{context}: {error:#}"));
+ }
+ }
+ Ok(())
+ }
+
+ fn message(&self) -> String {
+ let omitted = self.count.saturating_sub(self.examples.len());
+ format!(
+ "{} failing items\n{}{}",
+ self.count,
+ self.examples.join("\n"),
+ if omitted == 0 {
+ String::new()
+ } else {
+ format!("\n{omitted} more omitted")
+ }
+ )
+ }
+
+ pub fn finish(&self) -> Result<()> {
+ ensure!(
+ self.count == 0,
+ "adoption preflight failed: {}",
+ self.message()
+ );
+ Ok(())
+ }
+}
+
+pub fn scope(store: &Store, options: &Options, index: Option<&Index>) -> Result<()> {
+ let explicit = !options.albums.is_empty() || !options.exclude_albums.is_empty();
+ if explicit {
+ crate::select(store, options)?;
+ }
+ let (db, table) = match index {
+ Some(index) => (&index.db, "adoption_album"),
+ None => (&store.db, "albums"),
+ };
+ let mut membership = db.prepare(&format!(
+ "SELECT EXISTS(SELECT 1 FROM {table} WHERE id=?1 AND retained=0)"
+ ))?;
+ let mut desired = store.db.prepare("SELECT id,selected FROM desired_albums")?;
+ for row in desired.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, bool>(1)?)))? {
+ let (id, selected) = row?;
+ let recorded: bool = membership.query_row([id], |r| r.get(0))?;
+ if explicit {
+ ensure!(
+ selected == recorded,
+ "adoption filters must select exactly the recorded album IDs"
+ );
+ } else {
+ store.db.execute(
+ "UPDATE desired_albums SET selected=?1 WHERE id=?2",
+ params![recorded, id],
+ )?;
+ }
+ }
+ Ok(())
+}
+
+pub struct Preflight<'a> {
+ pub root: &'a Path,
+ pub store: &'a Store,
+ pub session: &'a Session,
+ pub index: &'a Index,
+ pub cancel: &'a watch::Receiver<bool>,
+ pub matching_root: bool,
+}
+
+impl Preflight<'_> {
+ pub async fn run(&self, source: &impl Source, mut failures: Failures) -> Result<()> {
+ let store = self.store;
+ let index = self.index;
+ let mut after = String::new();
+ while let Some(mut record) = store::json::<AlbumRecord>(
+ &index.db,
+ "SELECT record FROM adoption_album WHERE retained=0 AND key>?1 ORDER BY key LIMIT 1",
+ [&after],
+ )? {
+ after = record.album.key.clone();
+ let id = record.album.id;
+ let result = (|| {
+ let (ready, present, failure): (bool, bool, Option<String>) = store.db.query_row(
+ "SELECT ready,present,failure FROM desired_albums WHERE id=?1",
+ [id],
+ |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
+ )?;
+ ensure!(
+ ready && failure.is_none(),
+ "album source is incomplete: {}",
+ failure.as_deref().unwrap_or("refresh failed")
+ );
+ if let Some(legacy) = &record.legacy
+ && present
+ && let Some(owner) = legacy.owner
+ {
+ let entry = source
+ .catalog_page(id - 1)?
+ .into_iter()
+ .find(|e| e.id == id)
+ .context("missing current album")?;
+ ensure!(
+ entry.value?.0.owner_id == owner,
+ "album owner disagrees with source"
+ );
+ }
+ if record.json.is_none() && present {
+ let source: AlbumSource = store::json(
+ &store.db,
+ "SELECT record FROM album_sources WHERE id=?1",
+ [id],
+ )?
+ .context("missing album source")?;
+ if record.legacy.as_ref().is_some_and(|legacy| !legacy.go) {
+ record.album.name = source.name;
+ }
+ record.json = Some(JsonRecord {
+ owner: record.album.key.clone(),
+ role: None,
+ location: None,
+ value: source.metadata,
+ hash: None,
+ properties: None,
+ });
+ }
+ adopt::save_album(index, &record)
+ })();
+ if result.is_err() {
+ store.db.execute("UPDATE desired_albums SET failure='adoption album preflight failed' WHERE id=?1",[id])?;
+ }
+ failures.record(result, &format!("album:{id}"))?;
+ }
+ let mut after = (0, String::new());
+ let mut expected = None;
+ let mut verified = 0;
+ loop {
+ let next = store::json::<Item>(
+ &index.db,
+ "SELECT record FROM adoption_item WHERE origin='legacy' AND (file_id,key)>(?1,?2) ORDER BY file_id,key LIMIT 1",
+ params![after.0, after.1],
+ )?;
+ let Some(mut item) = next else { break };
+ if after.0 != item.placement.file {
+ expected = None;
+ }
+ after = (item.placement.file, item.placement.id.clone());
+ ensure!(!*self.cancel.borrow(), crate::Cancelled);
+ let ready: bool = store.db.query_row(
+ "SELECT ready=1 AND failure IS NULL FROM desired_albums WHERE id=?1",
+ [item.placement.album],
+ |r| r.get(0),
+ )?;
+ if !ready || expected.as_ref().is_some_and(Result::is_err) {
+ continue;
+ }
+ let result = async {
+ let record = item.legacy.as_ref().context("missing legacy identity")?;
+ let album: AlbumRecord = store::json(
+ &index.db,
+ "SELECT record FROM adoption_album WHERE id=?1 AND retained=0",
+ [record.album],
+ )?
+ .context("missing album")?;
+ let failure: Option<Option<String>> = store
+ .db
+ .query_row(
+ "SELECT failure FROM desired_files WHERE album=?1 AND file=?2",
+ params![record.album, record.file],
+ |r| r.get(0),
+ )
+ .optional()?;
+ if let Some(failure) = failure {
+ ensure!(
+ failure.is_none(),
+ "file source is incomplete: {}",
+ failure.unwrap_or_default()
+ );
+ let remote = source.file(record.album, record.file)?.value?.0;
+ ensure!(
+ record.owner.is_none_or(|owner| owner == remote.owner_id),
+ "file owner disagrees with source"
+ );
+ let file: FileSnapshot = store::json(
+ &store.db,
+ "SELECT record FROM sources WHERE file=?1",
+ [record.file],
+ )?
+ .context("missing source snapshot")?;
+ verified +=
+ verify::file(self, &album.album, &mut item, &file, &mut expected).await?;
+ } else {
+ item.leftover = true;
+ }
+ adopt::save_item(index, &album.album, &item)
+ }
+ .await;
+ failures.record(
+ result,
+ &format!("file:{}:{}", item.placement.album, item.placement.file),
+ )?;
+ }
+ failures.record((|| {
+ let trash=names::check(self.root,"Trash")?;
+ ensure!(!trash.try_exists()? || trash.is_dir(),"Trash is not a directory");
+ let reserved:Option<AlbumRecord>=store::json(&index.db,"SELECT record FROM adoption_album WHERE retained=0 AND parent='' AND folded='trash'",[])?;
+ ensure!(reserved.is_none_or(|r|r.legacy.is_some_and(|l|l.go)),"reserved legacy album path");
+ Ok(())
+ })(),"Trash")?;
+ if !self.matching_root && verified == 0 && failures.count == 0 {
+ failures.record(
+ Err(anyhow::anyhow!(
+ "at least one existing active original must verify"
+ )),
+ "source association",
+ )?;
+ }
+ failures.finish()
+ }
+}
+
+pub fn move_leftovers(root: &Path, index: &Index, cancel: &watch::Receiver<bool>) -> Result<()> {
+ if let Some(mut record) = store::json::<AlbumRecord>(
+ &index.db,
+ "SELECT record FROM adoption_album WHERE retained=0 AND parent='' AND folded='trash'",
+ [],
+ )? {
+ let source = names::check(root, &record.album.path)?;
+ if source.try_exists()? {
+ let occupied = occupied_names(root, cancel)?;
+ for suffix in 0.. {
+ ensure!(!*cancel.borrow(), crate::Cancelled);
+ let Some(candidate) = names::candidates(&record.album.name, true, None, suffix)?
+ else {
+ continue;
+ };
+ let name = &candidate[0];
+ let claimed: bool = index.db.query_row(
+ "SELECT EXISTS(SELECT 1 FROM adoption_album WHERE parent='' AND folded=?1)",
+ [names::folded(name)],
+ |r| r.get(0),
+ )?;
+ let destination = names::check(root, name)?;
+ if claimed || occupied.contains(&names::folded(name)) {
+ continue;
+ }
+ fs::free_rename(&source, &destination)?;
+ disk::rename(&source, &destination)?;
+ fs::sync_move(&source, &destination)?;
+ record.album.path = name.clone();
+ adopt::save_album(index, &record)?;
+ break;
+ }
+ }
+ }
+ let mut after = String::new();
+ // Legacy keys keep each album's leftovers together.
+ let mut album_id = None;
+ let mut occupied = BTreeMap::new();
+ while let Some(item) = store::json::<Item>(
+ &index.db,
+ "SELECT record FROM adoption_item WHERE origin='legacy' AND key>?1 ORDER BY key LIMIT 1",
+ [&after],
+ )? {
+ after = item.placement.id.clone();
+ if !item.leftover {
+ continue;
+ }
+ let record = item.legacy.context("missing leftover identity")?;
+ if album_id != Some(record.album) {
+ occupied.clear();
+ album_id = Some(record.album);
+ }
+ let album: AlbumRecord = store::json(
+ &index.db,
+ "SELECT record FROM adoption_album WHERE id=?1 AND retained=0",
+ [record.album],
+ )?
+ .context("missing album")?;
+ let folder = album.album.path;
+ let trash = format!("Trash/{folder}");
+ for part in &record.parts {
+ move_one(
+ root,
+ index,
+ &format!("{folder}/{}", part.name),
+ &trash,
+ &part.name,
+ &mut occupied,
+ cancel,
+ )?;
+ if record.desktop {
+ move_one(
+ root,
+ index,
+ &format!("{folder}/metadata/{}.json", part.name),
+ &format!("{trash}/metadata"),
+ &format!("{}.json", part.name),
+ &mut occupied,
+ cancel,
+ )?;
+ }
+ }
+ if let Some(sidecar) = record.go_record {
+ let name = sidecar
+ .strip_prefix(".meta/")
+ .context("invalid Go record")?;
+ move_one(
+ root,
+ index,
+ &format!("{folder}/{sidecar}"),
+ &format!("{trash}/.meta"),
+ name,
+ &mut occupied,
+ cancel,
+ )?;
+ }
+ }
+ Ok(())
+}
+
+fn occupied_names(directory: &Path, cancel: &watch::Receiver<bool>) -> Result<HashSet<String>> {
+ let mut occupied = HashSet::new();
+ if directory.try_exists()? {
+ for entry in disk::read_dir(directory)? {
+ ensure!(!*cancel.borrow(), crate::Cancelled);
+ occupied.insert(names::folded(&entry?.file_name().to_string_lossy()));
+ }
+ }
+ Ok(occupied)
+}
+
+fn move_one(
+ root: &Path,
+ index: &Index,
+ source: &str,
+ directory: &str,
+ name: &str,
+ occupied: &mut BTreeMap<String, HashSet<String>>,
+ cancel: &watch::Receiver<bool>,
+) -> Result<()> {
+ ensure!(!*cancel.borrow(), crate::Cancelled);
+ let source = names::check(root, source)?;
+ let parent = names::check(root, directory)?;
+ if Properties::optional(&source)?.is_none() {
+ return fs::sync_move(&source, &parent.join(name));
+ }
+ for suffix in 0.. {
+ ensure!(!*cancel.borrow(), crate::Cancelled);
+ let Some(candidate) = names::candidates(name, false, None, suffix)? else {
+ continue;
+ };
+ let relative = format!("{directory}/{}", candidate[0]);
+ let destination = names::check(root, &relative)?;
+ let occupied = match occupied.entry(directory.to_owned()) {
+ Entry::Occupied(entry) => entry.into_mut(),
+ Entry::Vacant(entry) => entry.insert(occupied_names(&parent, cancel)?),
+ };
+ let folded = names::folded(&candidate[0]);
+ if occupied.contains(&folded) {
+ continue;
+ }
+ let media = directory
+ .strip_suffix("/metadata")
+ .map(|p| {
+ format!(
+ "{p}/{}",
+ candidate[0].strip_suffix(".json").unwrap_or(&candidate[0])
+ )
+ })
+ .unwrap_or_else(|| relative.clone());
+ let claimed: bool = index.db.query_row(
+ "SELECT EXISTS(SELECT 1 FROM adoption_path WHERE path=?1)",
+ [names::folded(&media)],
+ |r| r.get(0),
+ )?;
+ if !claimed {
+ fs::move_file(&source, &destination)?;
+ occupied.insert(folded);
+ return Ok(());
+ }
+ }
+ unreachable!()
+}
+
+pub fn finish(root: &Path, store: &Store, cancel: &watch::Receiver<bool>) -> Result<()> {
+ let mut after = String::new();
+ loop {
+ let next=store.db.query_row(
+ "SELECT key,id,retained,name,path FROM albums WHERE retained=0 AND key>?1 ORDER BY key LIMIT 1",
+ [&after],crate::store::Album::read,
+ ).optional()?;
+ let Some(album) = next else { break };
+ ensure!(!*cancel.borrow(), crate::Cancelled);
+ after = album.key.clone();
+ let meta = names::check(root, &format!("{}/.meta", album.path))?;
+ if meta.is_dir() {
+ for entry in disk::read_dir(&meta)? {
+ ensure!(!*cancel.borrow(), crate::Cancelled);
+ let entry = entry?;
+ if entry.file_name() == "album_meta.json"
+ || entry.path().extension().is_none_or(|e| e != "json")
+ || !entry.file_type()?.is_file()
+ {
+ continue;
+ }
+ if read::go_record(&entry.path())? {
+ fs::remove(&entry.path())?;
+ }
+ }
+ let identity = meta.join("album_meta.json");
+ if identity.try_exists()? {
+ let value: serde_json::Value = read::json(&identity)?;
+ if value["id"].as_i64() == Some(album.id) {
+ fs::remove(&identity)?;
+ }
+ }
+ if disk::read_dir(&meta)?.next().is_none() {
+ disk::remove_dir(&meta)?;
+ fs::sync_parent(&meta)?;
+ }
+ }
+ let absent: bool = store.db.query_row(
+ "SELECT present=0 FROM desired_albums WHERE id=?1",
+ [album.id],
+ |r| r.get(0),
+ )?;
+ if absent {
+ for relative in [format!("{}/metadata", album.path), album.path] {
+ let path = names::check(root, &relative)?;
+ if path.is_dir() && disk::read_dir(&path)?.next().is_none() {
+ disk::remove_dir(&path)?;
+ fs::sync_parent(&path)?;
+ }
+ }
+ }
+ }
+ let manifest = names::check(root, "export_status.json")?;
+ if manifest.try_exists()? {
+ fs::remove(&manifest)?;
+ }
+ let transaction = store.db.unchecked_transaction()?;
+ store.db.execute("DELETE FROM albums WHERE retained=0 AND id IN (SELECT id FROM desired_albums WHERE selected=1 AND present=0) AND NOT EXISTS(SELECT 1 FROM placements WHERE album=albums.id AND retained=0) AND NOT EXISTS(SELECT 1 FROM temporaries WHERE folder=albums.key)",[])?;
+ store
+ .db
+ .execute("UPDATE export SET bound=1 WHERE id=1", [])?;
+ transaction.commit()?;
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn nested_record_preserves_fatal_errors() {
+ let mut failures = Failures::default();
+ let result = failures.record(Err(rusqlite::Error::InvalidQuery.into()), "file");
+ let error = failures.record(result, "album").unwrap_err();
+ assert!(error.is::<rusqlite::Error>() && crate::fatal(&error));
+ assert_eq!(failures.count, 0);
+ }
+}
diff --git a/rust/crates/photos-export/src/legacy/read.rs b/rust/crates/photos-export/src/legacy/read.rs
new file mode 100644
index 0000000000..cf38daead4
--- /dev/null
+++ b/rust/crates/photos-export/src/legacy/read.rs
@@ -0,0 +1,446 @@
+use std::{collections::BTreeMap, fs, path::Path};
+
+use anyhow::{Context, Result, ensure};
+use rusqlite::params;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use super::Failures;
+use crate::adopt::{self, AlbumRecord, Index, Item};
+use crate::store::{self, Album, Placement};
+use crate::{metadata::Role, names, store::Properties};
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct GoAlbum {
+ id: i64,
+ #[serde(rename = "ownerID")]
+ owner_id: i64,
+ album_name: String,
+ #[serde(default, rename = "accountOwnerIDs")]
+ account_owner_ids: Vec<i64>,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct GoInfo {
+ id: i64,
+ #[serde(rename = "ownerID")]
+ owner_id: i64,
+ file_names: Vec<String>,
+ hash: Option<String>,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Desktop {
+ pub version: u32,
+ pub collection_export_names: BTreeMap<String, String>,
+ pub file_export_names: BTreeMap<String, String>,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct Part {
+ pub name: String,
+ pub role: Option<Role>,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct Record {
+ pub album: i64,
+ pub file: i64,
+ pub owner: Option<i64>,
+ pub name: String,
+ pub hash: Option<String>,
+ pub parts: Vec<Part>,
+ pub go_record: Option<String>,
+ pub desktop: bool,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct AlbumInfo {
+ pub owner: Option<i64>,
+ pub go: bool,
+}
+
+pub fn recognized(root: &Path) -> Result<bool> {
+ if !root.is_dir() {
+ return Ok(false);
+ }
+ if root.join("export_status.json").try_exists()? {
+ return Ok(true);
+ }
+ for entry in fs::read_dir(root)? {
+ let path = entry?.path().join(".meta");
+ if path.is_dir() && path.join("album_meta.json").try_exists()? {
+ return Ok(true);
+ }
+ }
+ Ok(false)
+}
+
+pub fn desktop(root: &Path) -> Result<Option<Desktop>> {
+ let manifest = names::check(root, "export_status.json")?;
+ if !manifest.try_exists()? {
+ return Ok(None);
+ }
+ Ok(Some(json(&manifest)?))
+}
+
+pub fn scan(
+ root: &Path,
+ index: &Index,
+ user: i64,
+ desktop: Option<&Desktop>,
+ failures: &mut Failures,
+) -> Result<()> {
+ if let Some(desktop) = desktop {
+ let result = (|| {
+ ensure!(
+ desktop.version == 5,
+ "unsupported Desktop export version {}",
+ desktop.version
+ );
+ for (id, path) in &desktop.collection_export_names {
+ let result = (|| {
+ let id = id_string(id)?;
+ album(root, index, id, path, path, None, false)
+ })();
+ failures.record(result, path)?;
+ }
+ for (key, name) in &desktop.file_export_names {
+ let result = (|| {
+ let parts: Vec<_> = key.split('_').collect();
+ ensure!(parts.len() == 3, "invalid Desktop file identity");
+ let file = id_string(parts[0])?;
+ let album = id_string(parts[1])?;
+ id_string(parts[2])?;
+ let mapped: bool = index.db.query_row(
+ "SELECT EXISTS(SELECT 1 FROM adoption_album WHERE id=?1 AND retained=0)",
+ [album],
+ |r| r.get(0),
+ )?;
+ if !mapped && desktop.collection_export_names.contains_key(parts[1]) {
+ return Ok(());
+ }
+ let paths = if name.starts_with('{') {
+ #[derive(Deserialize)]
+ struct Pair {
+ image: String,
+ video: String,
+ }
+ let pair: Pair = serde_json::from_str(name)?;
+ vec![
+ Part {
+ name: pair.image,
+ role: Some(Role::Image),
+ },
+ Part {
+ name: pair.video,
+ role: Some(Role::Video),
+ },
+ ]
+ } else {
+ vec![Part {
+ name: name.clone(),
+ role: Some(Role::Original),
+ }]
+ };
+ let record = Record {
+ album,
+ file,
+ owner: None,
+ name: paths[0].name.clone(),
+ hash: None,
+ parts: paths,
+ go_record: None,
+ desktop: true,
+ };
+ insert(root, index, &record)
+ })();
+ failures.record(result, key)?;
+ }
+ Ok(())
+ })();
+ failures.record(result, "export_status.json")?;
+ }
+ for entry in fs::read_dir(root)? {
+ let entry = entry?;
+ let meta = entry.path().join(".meta");
+ if !meta.is_dir() {
+ continue;
+ }
+ let identity = meta.join("album_meta.json");
+ if !identity.try_exists()? {
+ let result = (|| {
+ let folder = entry
+ .file_name()
+ .into_string()
+ .map_err(|_| anyhow::anyhow!("legacy album path is not UTF-8"))?;
+ let meta = names::check(root, &format!("{folder}/.meta"))?;
+ for sidecar in fs::read_dir(meta)? {
+ let sidecar = sidecar?;
+ if !sidecar.path().extension().is_some_and(|s| s == "json") {
+ continue;
+ }
+ let name = sidecar
+ .file_name()
+ .into_string()
+ .map_err(|_| anyhow::anyhow!("legacy sidecar name is not UTF-8"))?;
+ let path = names::check(root, &format!("{folder}/.meta/{name}"))?;
+ if json::<Value>(&path)
+ .ok()
+ .is_some_and(|value| value["info"]["id"].as_i64().is_some())
+ {
+ anyhow::bail!("Go file identities remain without album_meta.json");
+ }
+ }
+ Ok(())
+ })();
+ failures.record(result, &entry.path().display().to_string())?;
+ continue;
+ }
+ let path = entry
+ .file_name()
+ .into_string()
+ .map_err(|_| anyhow::anyhow!("legacy album path is not UTF-8"))?;
+ let result = (|| {
+ names::check(root, &format!("{path}/.meta/album_meta.json"))?;
+ let record: GoAlbum = json(&identity)?;
+ ensure!(
+ record.account_owner_ids.is_empty() || record.account_owner_ids.contains(&user),
+ "legacy account owners exclude the selected account"
+ );
+ ensure!(record.owner_id > 0, "invalid album owner");
+ album(
+ root,
+ index,
+ record.id,
+ &path,
+ &record.album_name,
+ Some(record.owner_id),
+ true,
+ )?;
+ for entry in fs::read_dir(identity.parent().context("missing metadata parent")?)? {
+ let entry = entry?;
+ if entry.file_name() == "album_meta.json"
+ || entry.path().extension().is_none_or(|s| s != "json")
+ {
+ continue;
+ }
+ let name = entry
+ .file_name()
+ .into_string()
+ .map_err(|_| anyhow::anyhow!("legacy sidecar name is not UTF-8"))?;
+ let result = (|| {
+ names::check(root, &format!("{path}/.meta/{name}"))?;
+ let value: Value = json(&entry.path())?;
+ let Some(info) = value.get("info") else {
+ return Ok(());
+ };
+ let info: GoInfo = serde_json::from_value(info.clone())?;
+ ensure!(info.owner_id > 0, "invalid file owner");
+ let title = value["title"]
+ .as_str()
+ .context("missing legacy file title")?
+ .to_owned();
+ let record = Record {
+ album: record.id,
+ file: info.id,
+ owner: Some(info.owner_id),
+ name: title,
+ hash: info.hash,
+ parts: info
+ .file_names
+ .into_iter()
+ .map(|name| Part { name, role: None })
+ .collect(),
+ go_record: Some(format!(".meta/{name}")),
+ desktop: false,
+ };
+ insert(root, index, &record)
+ })();
+ failures.record(result, &format!("{path}/.meta/{name}"))?;
+ }
+ Ok(())
+ })();
+ failures.record(result, &path)?;
+ }
+ Ok(())
+}
+
+fn album(
+ root: &Path,
+ index: &Index,
+ id: i64,
+ path: &str,
+ name: &str,
+ owner: Option<i64>,
+ go: bool,
+) -> Result<()> {
+ ensure!(id > 0, "invalid album ID");
+ leaf(path)?;
+ let previous: Option<AlbumRecord> = store::json(
+ &index.db,
+ "SELECT record FROM adoption_album WHERE id=?1 AND retained=0",
+ [id],
+ )?;
+ let mut record = if let Some(record) = previous {
+ ensure!(record.legacy.is_none(), "duplicate legacy album identity");
+ record
+ } else {
+ let directory = names::check(root, path)?;
+ ensure!(
+ !directory.try_exists()? || directory.is_dir(),
+ "legacy album is not a directory"
+ );
+ let occupied: bool = index.db.query_row(
+ "SELECT EXISTS(SELECT 1 FROM adoption_album WHERE parent='' AND folded=?1)",
+ [names::folded(path)],
+ |r| r.get(0),
+ )?;
+ ensure!(!occupied, "overlapping legacy album path");
+ AlbumRecord {
+ album: Album {
+ key: format!("active:{id}"),
+ id,
+ retained: false,
+ name: name.into(),
+ path: path.into(),
+ },
+ json: None,
+ legacy: None,
+ }
+ };
+ let metadata = names::check(root, &format!("{}/metadata", record.album.path))?;
+ ensure!(
+ !metadata.try_exists()? || metadata.is_dir(),
+ "required metadata directory is occupied"
+ );
+ let json = names::check(root, &format!("{}/metadata.json", record.album.path))?;
+ ensure!(
+ !json.try_exists()? || record.json.is_some(),
+ "required metadata.json is occupied"
+ );
+ record.legacy = Some(AlbumInfo { owner, go });
+ adopt::save_album(index, &record)
+}
+
+fn insert(root: &Path, index: &Index, record: &Record) -> Result<()> {
+ ensure!(
+ record.file > 0 && (1..=2).contains(&record.parts.len()),
+ "invalid legacy file identity or components"
+ );
+ let album: AlbumRecord = store::json(
+ &index.db,
+ "SELECT record FROM adoption_album WHERE id=?1 AND retained=0",
+ [record.album],
+ )?
+ .context("file has no valid album mapping")?;
+ if let Some(existing) = store::json::<Item>(
+ &index.db,
+ "SELECT record FROM adoption_item WHERE album_key=?1 AND file_id=?2 AND active=1",
+ params![album.album.key, record.file],
+ )? {
+ ensure!(existing.legacy.is_none(), "duplicate legacy membership");
+ return Ok(());
+ }
+ let folder = album.album;
+ for part in &record.parts {
+ leaf(&part.name)?;
+ if store::json::<Item>(&index.db,"SELECT i.record FROM adoption_path p JOIN adoption_item i ON i.key=p.item_key WHERE p.path=?1 AND i.origin='native'",[names::folded(&format!("{}/{}",folder.path,part.name))])?.is_some() { return Ok(()) }
+ }
+ let item = Item {
+ placement: Placement {
+ id: format!("legacy:{}:{}", record.album, record.file),
+ album: record.album,
+ file: record.file,
+ retained: false,
+ name: record.name.clone(),
+ kind: String::new(),
+ },
+ components: Vec::new(),
+ json: Vec::new(),
+ legacy: Some(record.clone()),
+ leftover: false,
+ };
+ let mut claimed = Vec::new();
+ for part in &record.parts {
+ ensure!(
+ !["metadata", "metadata.json"].contains(&names::folded(&part.name).as_str()),
+ "legacy media occupies a required metadata path"
+ );
+ ensure!(
+ !claimed.contains(&names::folded(&part.name)),
+ "duplicate legacy component path"
+ );
+ claimed.push(names::folded(&part.name));
+ Properties::optional(&names::check(
+ root,
+ &format!("{}/{}", folder.path, part.name),
+ )?)?;
+ adopt::claim(index, &folder, &item.placement.id, &part.name, "")?;
+ let sidecar = Properties::optional(&names::check(
+ root,
+ &format!("{}/metadata/{}.json", folder.path, part.name),
+ )?)?;
+ ensure!(
+ record.desktop || sidecar.is_none(),
+ "required metadata path is occupied"
+ );
+ }
+ adopt::save_item(index, &folder, &item)
+}
+
+impl Desktop {
+ pub fn common_sidecar(&self, album: i64, name: &str) -> bool {
+ self.file_export_names.iter().any(|(key, value)| {
+ let Some(id) = key.split('_').nth(1) else {
+ return false;
+ };
+ if id.parse::<i64>().ok() != Some(album) {
+ return false;
+ }
+ if value.starts_with('{') {
+ serde_json::from_str::<Value>(value).is_ok_and(|pair| {
+ ["image", "video"].iter().any(|role| {
+ pair[role]
+ .as_str()
+ .is_some_and(|p| format!("{p}.json") == name)
+ })
+ })
+ } else {
+ format!("{value}.json") == name
+ }
+ })
+ }
+}
+
+pub fn go_record(path: &Path) -> Result<bool> {
+ let bytes = fs::read(path)?;
+ Ok(serde_json::from_slice::<Value>(&bytes)
+ .is_ok_and(|value| value["info"]["id"].as_i64().is_some_and(|id| id > 0)))
+}
+
+pub fn json<T: serde::de::DeserializeOwned>(path: &Path) -> Result<T> {
+ Properties::read(path)?;
+ Ok(serde_json::from_reader(fs::File::open(path)?)?)
+}
+
+pub fn id_string(text: &str) -> Result<i64> {
+ let id: i64 = text.parse()?;
+ ensure!(
+ id > 0 && id.to_string() == text,
+ "invalid identity {text:?}"
+ );
+ Ok(id)
+}
+
+fn leaf(name: &str) -> Result<()> {
+ names::relative(name)?;
+ ensure!(
+ !name.contains('/'),
+ "legacy path is not a single name: {name:?}"
+ );
+ Ok(())
+}
diff --git a/rust/crates/photos-export/src/legacy/verify.rs b/rust/crates/photos-export/src/legacy/verify.rs
new file mode 100644
index 0000000000..48ae8541dd
--- /dev/null
+++ b/rust/crates/photos-export/src/legacy/verify.rs
@@ -0,0 +1,259 @@
+use crate::{
+ adopt::{self, Item},
+ fs,
+ metadata::{self, Role},
+ names,
+ snapshot::FileSnapshot,
+ store::{Album, Component, JsonRecord, Location, Properties},
+ transfer,
+};
+use anyhow::{Context, Result, ensure};
+use ente_core::{Session, b64, crypto::hash::HashState};
+use ente_photos::files;
+use std::{
+ io::{self, Seek, SeekFrom, Write},
+ path::Path,
+};
+use tokio::sync::watch;
+
+pub async fn file(
+ check: &super::Preflight<'_>,
+ album: &Album,
+ item: &mut Item,
+ source: &FileSnapshot,
+ expected: &mut Option<Result<Vec<(Role, String)>, String>>,
+) -> Result<usize> {
+ let record = item.legacy.as_ref().context("missing legacy record")?;
+ let roles = transfer::roles(source);
+ ensure!(
+ source.kind == "livephoto" || record.parts.len() == 1,
+ "ordinary file has multiple legacy components"
+ );
+ ensure!(
+ source.hash.is_none()
+ || roles
+ .iter()
+ .all(|role| transfer::expected_hash(source, role).is_some_and(valid_hash)),
+ "invalid authenticated source hash"
+ );
+ ensure!(
+ record.parts.len() == 1
+ || record.parts[0].role.is_none()
+ || record.parts[0].role != record.parts[1].role,
+ "duplicate legacy component role"
+ );
+ let mut measured = Vec::new();
+ for part in &record.parts {
+ let path = names::check(check.root, &format!("{}/{}", album.path, part.name))?;
+ measured.push(
+ Properties::optional(&path)?
+ .map(|properties| {
+ let (size, hash) = fs::hash_cancellable(&path, check.cancel)?;
+ Ok::<_, anyhow::Error>((size, hash, properties))
+ })
+ .transpose()?,
+ );
+ }
+ let present = measured.iter().filter(|p| p.is_some()).count();
+ if present > 0 && expected.is_none() {
+ match hashes(check.root, source, check.session, check.cancel).await {
+ Ok(hashes) => *expected = Some(Ok(hashes)),
+ Err(error) => {
+ if crate::fatal(&error) {
+ return Err(error);
+ }
+ *expected = Some(Err(format!("{error:#}")));
+ }
+ }
+ }
+ let known = if present > 0 {
+ expected
+ .as_ref()
+ .context("missing comparison")?
+ .as_ref()
+ .map_err(|error| anyhow::anyhow!("{error}"))?
+ .clone()
+ } else {
+ roles
+ .iter()
+ .filter_map(|r| transfer::expected_hash(source, r).map(|h| (r.clone(), h.to_owned())))
+ .collect()
+ };
+ let complete = present == roles.len();
+ item.leftover = !complete && (source.kind == "livephoto" || record.desktop);
+ item.placement.kind = source.kind.clone();
+ if record.desktop {
+ item.placement.name = source.name.clone();
+ }
+ for (index, part) in record.parts.iter().enumerate() {
+ let role = part.role.clone().or_else(|| {
+ if roles.len() == 1 {
+ Some(Role::Original)
+ } else if record.parts.len() == 2 {
+ Some(roles[index].clone())
+ } else {
+ None
+ }
+ });
+ ensure!(
+ role.as_ref().is_none_or(|r| roles.contains(r)),
+ "legacy component role disagrees with source"
+ );
+ if let Some((size, hash, properties)) = &measured[index] {
+ ensure!(
+ known
+ .iter()
+ .any(|(r, h)| role.as_ref().is_none_or(|own| own == r) && h == hash),
+ "changed original at {}/{}",
+ album.path,
+ part.name
+ );
+ if complete {
+ let role = role.context("missing complete component role")?;
+ ensure!(
+ !item.components.iter().any(|c| c.role == role),
+ "duplicate legacy component role"
+ );
+ item.components.push(Component {
+ placement: item.placement.id.clone(),
+ role,
+ location: Location {
+ folder: album.key.clone(),
+ name: part.name.clone(),
+ },
+ size: *size,
+ hash: hash.clone(),
+ signature: Some(transfer::signature(source)?),
+ properties: Some(properties.clone()),
+ intended_time: None,
+ });
+ }
+ }
+ }
+ for (role, hash) in &known {
+ if let Some(legacy) = legacy_hash(record.hash.as_deref(), role) {
+ ensure!(
+ legacy == hash,
+ "legacy hash contradicts the {} component",
+ role.name()
+ );
+ }
+ }
+ if complete {
+ let layout: Vec<_> = item.components.iter().map(Component::portable).collect();
+ let favorited = check.store.db.query_row(
+ "SELECT favorited FROM desired_files WHERE album=?1 AND file=?2",
+ rusqlite::params![record.album, record.file],
+ |r| r.get(0),
+ )?;
+ for (index, component) in item.components.iter().enumerate() {
+ let name = format!("metadata/{}.json", component.location.name);
+ let path = names::check(check.root, &format!("{}/{name}", album.path))?;
+ let properties = Properties::optional(&path)?;
+ ensure!(
+ record.desktop || properties.is_none(),
+ "required metadata path is occupied"
+ );
+ let hash = properties
+ .as_ref()
+ .map(|_| adopt::digest(&std::fs::read(&path)?))
+ .transpose()?;
+ item.json.push(JsonRecord {
+ owner: item.placement.id.clone(),
+ role: Some(component.role.clone()),
+ location: properties.as_ref().map(|_| Location {
+ folder: album.key.clone(),
+ name,
+ }),
+ value: metadata::publish(&source.metadata, &layout, index, favorited)?,
+ hash,
+ properties,
+ });
+ }
+ }
+ Ok(present)
+}
+
+fn valid_hash(hash: &str) -> bool {
+ b64::decode(hash).is_ok_and(|bytes| bytes.len() == 64)
+}
+
+fn legacy_hash<'a>(hash: Option<&'a str>, role: &Role) -> Option<&'a str> {
+ let hash = hash?;
+ let hash = match role {
+ Role::Original => hash,
+ Role::Image => hash.split_once(':')?.0,
+ Role::Video => hash.split_once(':')?.1,
+ };
+ valid_hash(hash).then_some(hash)
+}
+
+async fn hashes(
+ root: &Path,
+ file: &FileSnapshot,
+ session: &Session,
+ cancel: &watch::Receiver<bool>,
+) -> Result<Vec<(Role, String)>> {
+ let roles = transfer::roles(file);
+ let known: Option<Vec<_>> = roles
+ .iter()
+ .map(|role| transfer::expected_hash(file, role).map(|hash| (role.clone(), hash.to_owned())))
+ .collect();
+ if let Some(known) = known {
+ return Ok(known);
+ }
+ let mut receiver = cancel.clone();
+ let hashes = if file.kind == "livephoto" {
+ let scratch = tempfile::tempfile_in(root)?;
+ let download = files::download(session, file.id, &file.key, &file.header, || {
+ let mut file = scratch.try_clone()?;
+ file.set_len(0)?;
+ file.seek(SeekFrom::Start(0))?;
+ Ok(file)
+ });
+ let archive = tokio::select! { result=download=>result?, _=receiver.changed()=>return Err(crate::Cancelled.into()) };
+ crate::live_photo::extract(archive, |_, _| {
+ Digest::new(cancel).map_err(io::Error::other)
+ })?
+ .into_iter()
+ .map(|(role, _, writer)| Ok((role, writer.finish()?)))
+ .collect::<Result<Vec<_>>>()?
+ } else {
+ let download = files::download(session, file.id, &file.key, &file.header, || {
+ Digest::new(cancel).map_err(io::Error::other)
+ });
+ let writer = tokio::select! { result=download=>result?, _=receiver.changed()=>return Err(crate::Cancelled.into()) };
+ vec![(Role::Original, writer.finish()?)]
+ };
+ Ok(hashes)
+}
+
+struct Digest {
+ state: HashState,
+ cancel: watch::Receiver<bool>,
+}
+
+impl Digest {
+ fn new(cancel: &watch::Receiver<bool>) -> Result<Self> {
+ Ok(Self {
+ state: HashState::new(Some(64), None)?,
+ cancel: cancel.clone(),
+ })
+ }
+ fn finish(self) -> Result<String> {
+ Ok(b64::encode(&self.state.finalize()?))
+ }
+}
+
+impl Write for Digest {
+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+ if *self.cancel.borrow() {
+ return Err(io::Error::other(crate::Cancelled));
+ }
+ self.state.update(bytes).map_err(io::Error::other)?;
+ Ok(bytes.len())
+ }
+ fn flush(&mut self) -> io::Result<()> {
+ Ok(())
+ }
+}
diff --git a/rust/crates/photos-export/src/lib.rs b/rust/crates/photos-export/src/lib.rs
index 275330c99e..87fc661fdd 100644
--- a/rust/crates/photos-export/src/lib.rs
+++ b/rust/crates/photos-export/src/lib.rs
@@ -1,5 +1,6 @@
mod adopt;
mod fs;
+mod legacy;
mod live_photo;
mod metadata;
mod names;
@@ -59,9 +60,10 @@ pub struct Summary {
pub struct Export {
destination: PathBuf,
expected_root: Root,
+ prepared: Option<(Store, Result<()>)>,
root_lock: Option<File>,
options: Options,
- prepared: Option<(Store, Result<()>)>,
+ adopting: bool,
}
impl Export {
@@ -73,16 +75,31 @@ impl Export {
let destination = destination(path)?;
let expected_root = Root::new(master_key)?;
let root_lock = open_root(&destination, &expected_root)?;
+ let adopting = root_lock.is_none() && legacy::read::recognized(&destination)?;
+ ensure!(
+ !adopting || options.adopt,
+ AdoptionRequired(
+ "This folder contains an export from the old CLI or Ente Desktop. Use --adopt if you want to take it over."
+ )
+ );
ensure!(
- !options.adopt || root_lock.is_some(),
+ !options.adopt || root_lock.is_some() || adopting,
"there is no export to adopt"
);
+ ensure!(
+ root_lock.is_some()
+ || adopting
+ || !destination.try_exists()?
+ || std::fs::read_dir(&destination)?.next().is_none(),
+ "nonempty destination has no recognized export.json"
+ );
Ok(Self {
destination,
expected_root,
root_lock,
options,
prepared: None,
+ adopting,
})
}
@@ -105,6 +122,7 @@ impl Export {
pub async fn prepare(
&mut self,
connection: rusqlite::Connection,
+ open_index: impl FnOnce(&Path) -> Result<rusqlite::Connection>,
source: &mut impl Source,
session: &ente_core::Session,
cancel: &mut watch::Receiver<bool>,
@@ -114,13 +132,44 @@ impl Export {
let bound: bool = store
.db
.query_row("SELECT bound FROM export WHERE id=1", [], |r| r.get(0))?;
+ let accepted = !bound
+ && store.db.query_row(
+ "SELECT EXISTS(SELECT 1 FROM albums WHERE retained=0)",
+ [],
+ |r| r.get::<_, bool>(0),
+ )?;
ensure!(
- self.root_lock.is_none() || bound || self.options.adopt,
- AdoptionRequired("this export is not associated")
+ self.root_lock.is_none() || bound || accepted || self.options.adopt,
+ AdoptionRequired("this export is not associated; use --adopt")
);
+ let initial = !bound && !accepted && self.options.adopt;
+ let index = initial.then(|| adopt::Index::new(open_index)).transpose()?;
store.reset_desired()?;
- if self.root_lock.is_some() && !bound {
- adopt::scan(&self.destination, &store)?;
+ let mut failures = legacy::Failures::default();
+ self.adopting = !bound
+ && (accepted
+ || (initial && (self.adopting || legacy::read::recognized(&self.destination)?)));
+ if let Some(index) = &index {
+ let mut desktop = None;
+ failures.record(
+ legacy::read::desktop(&self.destination).map(|value| desktop = value),
+ "export_status.json",
+ )?;
+ if self.root_lock.is_some() {
+ failures.record(
+ adopt::scan(&self.destination, index, desktop.as_ref()),
+ "native records",
+ )?;
+ }
+ if self.adopting {
+ legacy::read::scan(
+ &self.destination,
+ index,
+ session.user_id,
+ desktop.as_ref(),
+ &mut failures,
+ )?;
+ }
}
report("Refreshing album metadata.");
tokio::select! {
@@ -157,19 +206,66 @@ impl Export {
"INSERT OR IGNORE INTO desired_albums(id,name) SELECT album,name FROM pending WHERE file IS NULL AND retained=0",
[],
)?;
- select(&store, &self.options)?;
- if self.root_lock.is_none() {
- self.root_lock = Some(initialize(&self.destination, &self.expected_root)?);
+ if let Some(index) = &index {
+ let mut albums = index.db.prepare("SELECT id,json_extract(record,'$.album.name') FROM adoption_album ORDER BY retained")?;
+ for row in
+ albums.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)))?
+ {
+ let (id, name) = row?;
+ store.db.execute(
+ "INSERT OR IGNORE INTO desired_albums(id,name) VALUES(?1,?2)",
+ params![id, name],
+ )?;
+ }
+ }
+ if self.adopting {
+ legacy::scope(&store, &self.options, index.as_ref())?;
+ } else {
+ failures.finish()?;
+ select(&store, &self.options)?;
+ }
+ if !self.adopting {
+ if self.root_lock.is_none() {
+ self.root_lock = Some(initialize(&self.destination, &self.expected_root, false)?);
+ }
+ if !initial {
+ store
+ .db
+ .execute("UPDATE export SET bound=1 WHERE id=1 AND bound=0", [])?;
+ }
}
- store
- .db
- .execute("UPDATE export SET bound=1 WHERE id=1 AND bound=0", [])?;
-
let refresh_cancel = cancel.clone();
let refreshed = tokio::select! {
result=refresh(&store,source,session,&refresh_cancel,report)=>result,
_=cancel.changed()=>Err(Cancelled.into()),
};
+ if self.adopting {
+ refreshed?;
+ if let Some(index) = &index {
+ report("Verifying every existing active legacy original before adoption.");
+ legacy::Preflight {
+ root: &self.destination,
+ store: &store,
+ session,
+ index,
+ cancel,
+ matching_root: self.root_lock.is_some(),
+ }
+ .run(source, failures)
+ .await?;
+ if self.root_lock.is_none() {
+ self.root_lock =
+ Some(initialize(&self.destination, &self.expected_root, true)?);
+ }
+ legacy::move_leftovers(&self.destination, index, cancel)?;
+ adopt::accept(index, &store, false)?;
+ }
+ self.prepared = Some((store, Ok(())));
+ return Ok(());
+ }
+ if let Some(index) = &index {
+ adopt::accept(index, &store, true)?;
+ }
self.prepared = Some((store, refreshed));
Ok(())
}
@@ -215,6 +311,13 @@ impl Export {
let store = shared
.into_inner()
.map_err(|_| anyhow::anyhow!("export database mutex poisoned"))?;
+ if self.adopting
+ && work_complete(&store, prepared)?
+ && let Err(error) = legacy::finish(&self.destination, &store, cancel)
+ {
+ record_outcome(&store, "run", &error)?;
+ report(&format!("run: {error:#}"));
+ }
summary(&store, &self.destination, prepared)
}
}
@@ -260,10 +363,6 @@ fn open_root(destination: &Path, expected: &Root) -> Result<Option<File>> {
);
let path = names::check(destination, "export.json")?;
if !path.try_exists()? {
- ensure!(
- std::fs::read_dir(destination)?.next().is_none(),
- "nonempty destination has no recognized export.json"
- );
return Ok(None);
}
let file = OpenOptions::new().read(true).write(true).open(&path)?;
@@ -281,7 +380,7 @@ fn open_root(destination: &Path, expected: &Root) -> Result<Option<File>> {
Ok(Some(file))
}
-fn initialize(destination: &Path, root: &Root) -> Result<File> {
+fn initialize(destination: &Path, root: &Root, legacy: bool) -> Result<File> {
fs::create_directory(destination)?;
let path = names::check(destination, "export.json")?;
let mut file = OpenOptions::new()
@@ -292,8 +391,9 @@ fn initialize(destination: &Path, root: &Root) -> Result<File> {
.context("destination was initialized by another writer; retry")?;
file.lock().context("cannot lock the new export")?;
ensure!(
- std::fs::read_dir(destination)?
- .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),
+ legacy
+ || std::fs::read_dir(destination)?
+ .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),
"destination became nonempty during initialization"
);
serde_json::to_writer_pretty(&mut file, root)?;
@@ -581,8 +681,7 @@ fn transfers(
"UPDATE desired_files SET claimed=1 WHERE file=?1",
[id],
)?;
- let file: FileSnapshot = store
- .json("SELECT record FROM sources WHERE file=?1", [id])?
+ let file: FileSnapshot = store::json(&store.db,"SELECT record FROM sources WHERE file=?1", [id])?
.context("missing file source snapshot")?;
transaction.commit()?;
file
@@ -684,6 +783,10 @@ fn fatal_cause(cause: &(dyn std::error::Error + 'static)) -> bool {
cause.source().is_some_and(fatal_cause)
}
+fn work_complete(store: &Store, prepared: bool) -> Result<bool> {
+ Ok(prepared && store.db.query_row("SELECT NOT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND (ready=0 OR failure IS NOT NULL)) AND NOT EXISTS(SELECT 1 FROM desired_files WHERE completed=0 OR failure IS NOT NULL) AND NOT EXISTS(SELECT 1 FROM outcomes) AND NOT EXISTS(SELECT 1 FROM pending p JOIN desired_albums a ON a.id=p.album WHERE a.selected=1)",[],|r|r.get::<_,bool>(0))?)
+}
+
fn summary(store: &Store, destination: &Path, prepared: bool) -> Result<Summary> {
let unknown: bool = !prepared
|| store.db.query_row(
@@ -708,7 +811,10 @@ fn summary(store: &Store, destination: &Path, prepared: bool) -> Result<Summary>
[],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
)?;
- let complete = !unknown && failures == 0 && completed == expected;
+ let bound: bool = store
+ .db
+ .query_row("SELECT bound FROM export WHERE id=1", [], |r| r.get(0))?;
+ let complete = bound && work_complete(store, prepared)?;
let result = json!({"destination":destination.to_string_lossy(),"complete":complete,"files":if unknown {None}else{Some(files)},"copies":if unknown {None}else{Some(json!({"expected":expected,"completed":completed,"pending":expected-completed}))},"changes":{"exported":exported,"metadataUpdated":metadata_updated,"renamed":renamed,"retained":retained},"failures":failures,"conflicts":conflicts});
Ok(Summary {
json: result,
diff --git a/rust/crates/photos-export/src/names.rs b/rust/crates/photos-export/src/names.rs
index dd23cf0a9d..32215593dc 100644
--- a/rust/crates/photos-export/src/names.rs
+++ b/rust/crates/photos-export/src/names.rs
@@ -156,6 +156,13 @@ pub fn available(
Ok(true)
}
+pub fn valid(name: &str, album: bool) -> bool {
+ !name.is_empty()
+ && portable(name) == name
+ && !reserved(name, album)
+ && name.len() <= if album { 255 } else { 250 }
+}
+
pub fn allocate(
store: &Store,
folder: &str,
diff --git a/rust/crates/photos-export/src/reconcile.rs b/rust/crates/photos-export/src/reconcile.rs
index aa51f92e52..1e1ea244c6 100644
--- a/rust/crates/photos-export/src/reconcile.rs
+++ b/rust/crates/photos-export/src/reconcile.rs
@@ -278,6 +278,54 @@ impl Run<'_> {
Ok(())
}
+ fn verify_pending_media(&self, target: &Pending, action: &Action) -> Result<()> {
+ let components = store::lock(self.store)?.components(&target.owner)?;
+ for component in components {
+ if target.retained && component.location.folder == target.folder {
+ continue;
+ }
+ let mut path = self.path(&component.location)?;
+ match action {
+ Action::Move {
+ source,
+ destination,
+ role,
+ ..
+ } if *role == component.role => {
+ if Properties::optional(&self.path(source)?)?.is_none() {
+ if target.retained {
+ continue;
+ }
+ path = self.path(destination)?;
+ }
+ }
+ Action::Publish {
+ temporary,
+ destination,
+ output:
+ Output::Media {
+ role, size, hash, ..
+ },
+ } if *role == component.role => {
+ path = self.path(destination)?;
+ if Properties::optional(&self.path(temporary)?)?.is_none()
+ && Properties::optional(&path)?.is_some()
+ && fs::matches(&path, *size, hash, self.cancel)?
+ {
+ continue;
+ }
+ }
+ _ => {}
+ }
+ ensure!(
+ Properties::optional(&path)?.is_none()
+ || fs::matches(&path, component.size, &component.hash, self.cancel)?,
+ super::Conflict(format!("at {}: locally changed media", path.display()))
+ );
+ }
+ Ok(())
+ }
+
fn finish_action(
&self,
target: &mut Pending,
@@ -288,13 +336,30 @@ impl Run<'_> {
return Ok(());
};
self.check_cancel()?;
+ let current_media = recovering
+ && matches!(
+ action,
+ Action::Move { .. }
+ | Action::Publish {
+ output: Output::Media { .. },
+ ..
+ }
+ )
+ && store::lock(self.store)?.db.query_row(
+ "SELECT EXISTS(SELECT 1 FROM desired_files WHERE album=?1 AND file=?2 AND failure IS NULL)",
+ params![target.album, target.file],
+ |r| r.get::<_, bool>(0),
+ )?;
+ if current_media {
+ self.verify_pending_media(target, &action)?;
+ }
match action {
Action::Directory { source } => {
let destination = names::check(self.root, &target.album_path())?;
if let Some(source) = source {
let source = names::check(self.root, &source)?;
if directory_exists(&source)? {
- free_rename(&source, &destination)?;
+ fs::free_rename(&source, &destination)?;
disk::rename(&source, &destination)?;
} else if !directory_exists(&destination)? {
let unrepairable: bool = store::lock(self.store)?
@@ -389,22 +454,21 @@ impl Run<'_> {
let from = self.path(&source)?;
let to = self.path(&destination)?;
if Properties::optional(&from)?.is_some() {
- free_rename(&from, &to)?;
- fs::create_directory(to.parent().context("move has no parent")?)?;
- disk::rename(&from, &to)?;
+ fs::move_file(&from, &to)?;
} else if Properties::optional(&to)?.is_some() {
ensure!(
- fs::matches(&to, size, &hash, self.cancel)?,
+ (recovering && (target.retained || !current_media))
+ || fs::matches(&to, size, &hash, self.cancel)?,
super::Conflict(format!("at {}: moved bytes differ", to.display()))
);
+ fs::sync_move(&from, &to)?;
} else {
ensure!(
- !target.retained,
+ !target.retained || current_media,
"retained original is missing at both move endpoints"
);
return self.clear_action(target);
}
- fs::sync_move(&from, &to)?;
let store = store::lock(self.store)?;
let transaction = store.db.unchecked_transaction()?;
let mut component = store
@@ -553,7 +617,7 @@ impl Run<'_> {
}
let from = self.path(&component.location)?;
let to = self.path(&destination)?;
- free_rename(&from, &to)?;
+ fs::free_rename(&from, &to)?;
let properties = Properties::optional(&from)?;
if properties.is_none() {
ensure!(!target.retained, "previously published original is missing");
@@ -594,17 +658,6 @@ fn directory_exists(path: &Path) -> Result<bool> {
}
}
-fn free_rename(source: &Path, destination: &Path) -> Result<()> {
- ensure!(
- !destination.try_exists()? || names::same_path(source, destination),
- super::Conflict(format!(
- "at {}: occupied rename target",
- destination.display()
- ))
- );
- Ok(())
-}
-
impl Run<'_> {
pub fn recover(&self) -> Result<()> {
for albums in [true, false] {
@@ -752,15 +805,11 @@ impl Run<'_> {
let parent = if retained { "Trash" } else { "" };
let names = if let Some(existing) = &existing
&& (retained || existing.name == name)
+ && let Some(name) = existing.path.rsplit('/').next()
+ && names::valid(name, true)
+ && names::available(&store, parent, &[name.to_owned()], &owner, "album")?
{
- vec![
- existing
- .path
- .rsplit('/')
- .next()
- .context("empty album path")?
- .to_owned(),
- ]
+ vec![name.to_owned()]
} else {
names::allocate(&store, parent, name, "album", None, &owner)?
};
@@ -820,7 +869,7 @@ impl Run<'_> {
let source = existing.as_ref().map(|album| album.path.clone());
let destination = names::check(self.root, &target_path)?;
if let Some(source) = &source {
- free_rename(&names::check(self.root, source)?, &destination)?;
+ fs::free_rename(&names::check(self.root, source)?, &destination)?;
} else {
ensure!(
!destination.try_exists()?,
@@ -925,6 +974,7 @@ impl Run<'_> {
.collect::<Result<_>>()?;
let compatible = |names: &[String]| {
names.len() == roles.len()
+ && names.iter().all(|name| names::valid(name, false))
&& (roles.len() == 1
|| names.iter().zip(&extensions).all(|(name, extension)| {
names::split(name, false).1 == names::portable(extension)
@@ -940,6 +990,7 @@ impl Run<'_> {
&& previous.name == file.name
&& previous.kind == file.kind.as_str()
&& compatible(&previous.names)
+ && names::available(&store, &album.key, &previous.names, &owner, &file.kind)?
{
return Ok(previous);
}
@@ -953,7 +1004,8 @@ impl Run<'_> {
}) && components
.iter()
.all(|component| component.location.folder == album.key)
- && compatible(&existing_names);
+ && compatible(&existing_names)
+ && names::available(&store, &album.key, &existing_names, &owner, &file.kind)?;
let transaction = store.db.unchecked_transaction()?;
let names = if unchanged {
existing_names
@@ -1339,10 +1391,6 @@ impl Run<'_> {
}
}
let components = store::lock(self.store)?.components(&placement.id)?;
- for component in &components {
- Properties::read(&self.path(&component.location)?)
- .context("previously published original is missing")?;
- }
let active = store::lock(self.store)?
.album(placement.album, false)?
.context("missing active album")?;
@@ -1409,6 +1457,11 @@ impl Run<'_> {
}
fn finish_retention(&self, target: &mut Pending) -> Result<()> {
+ let restoring: bool = store::lock(self.store)?.db.query_row(
+ "SELECT EXISTS(SELECT 1 FROM desired_files WHERE album=?1 AND file=?2 AND failure IS NULL)",
+ params![target.album, target.file],
+ |r| r.get(0),
+ )?;
let components = store::lock(self.store)?.components(&target.owner)?;
for component in &components {
let name = target_component(target, &component.role)?;
@@ -1416,12 +1469,24 @@ impl Run<'_> {
folder: target.folder.clone(),
name: name.into(),
};
- self.move_media(target, component, location, component.properties.as_ref())?;
+ let from = self.path(&component.location)?;
+ if restoring && Properties::optional(&from)?.is_none() {
+ fs::free_rename(&from, &self.path(&location)?)?;
+ let mut missing = component.clone();
+ missing.location = location;
+ missing.properties = None;
+ missing.intended_time = None;
+ store::lock(self.store)?.save_component(&missing)?;
+ } else {
+ self.move_media(target, component, location, component.properties.as_ref())?;
+ }
}
let components = store::lock(self.store)?.components(&target.owner)?;
for component in &components {
- Properties::read(&self.path(&component.location)?)
- .context("retained original is missing")?;
+ if !restoring {
+ Properties::read(&self.path(&component.location)?)
+ .context("retained original is missing")?;
+ }
let previous = store::lock(self.store)?
.json_record(&target.owner, Some(&component.role))?
.context("missing retained metadata")?;
@@ -1536,9 +1601,21 @@ impl Run<'_> {
|r| r.get(0),
)?;
ensure!(!incomplete, "album has unfinished retention");
- self.retained_album(&active)?;
+ let (bound, record) = {
+ let store = store::lock(self.store)?;
+ (
+ store
+ .db
+ .query_row("SELECT bound FROM export WHERE id=1", [], |r| {
+ r.get::<_, bool>(0)
+ })?,
+ store.json_record(&active.key, None)?,
+ )
+ };
+ if bound || record.is_some() {
+ self.retained_album(&active)?;
+ }
let mut target = self.album_target(id, &active.name, false)?;
- let record = store::lock(self.store)?.json_record(&active.key, None)?;
if let Some(record) = record
&& let Some(location) = record.location
{
@@ -1564,7 +1641,7 @@ impl Run<'_> {
store
.db
.execute(
- "DELETE FROM albums WHERE key=?1 AND NOT EXISTS(SELECT 1 FROM temporaries WHERE folder=?1)",
+ "DELETE FROM albums WHERE key=?1 AND (SELECT bound FROM export WHERE id=1)=1 AND NOT EXISTS(SELECT 1 FROM temporaries WHERE folder=?1)",
[&active.key],
)?;
store.clear_pending(&target.owner)?;
diff --git a/rust/crates/photos-export/src/store.rs b/rust/crates/photos-export/src/store.rs
index e05d491efc..0ca3350b41 100644
--- a/rust/crates/photos-export/src/store.rs
+++ b/rust/crates/photos-export/src/store.rs
@@ -23,7 +23,17 @@ pub fn lock(store: &Mutex<Store>) -> Result<MutexGuard<'_, Store>> {
.map_err(|_| anyhow::anyhow!("export database mutex poisoned"))
}
-#[derive(Clone)]
+pub fn json<T: DeserializeOwned>(
+ db: &Connection,
+ sql: &str,
+ parameters: impl rusqlite::Params,
+) -> Result<Option<T>> {
+ let text: Option<String> = db.query_row(sql, parameters, |r| r.get(0)).optional()?;
+ text.map(|text| serde_json::from_str(&text).map_err(Into::into))
+ .transpose()
+}
+
+#[derive(Clone, Serialize, Deserialize)]
pub struct Album {
pub key: String,
pub id: i64,
@@ -44,7 +54,7 @@ impl Album {
}
}
-#[derive(Clone)]
+#[derive(Clone, Serialize, Deserialize)]
pub struct Placement {
pub id: String,
pub album: i64,
@@ -73,7 +83,7 @@ pub struct Location {
pub name: String,
}
-#[derive(Clone, PartialEq, Eq)]
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Component {
pub placement: String,
pub role: Role,
@@ -158,7 +168,7 @@ pub fn system_time(time: SystemTime) -> i128 {
}
}
-#[derive(Clone)]
+#[derive(Clone, Serialize, Deserialize)]
pub struct JsonRecord {
pub owner: String,
pub role: Option<Role>,
@@ -334,7 +344,7 @@ impl Store {
if let Some(existing) = existing {
ensure!(
existing == destination.as_os_str().as_encoded_bytes(),
- AdoptionRequired("destination association does not match")
+ AdoptionRequired("destination association does not match; use --adopt")
);
} else {
connection.execute(
@@ -535,19 +545,6 @@ impl Store {
Ok(())
}
- pub fn json<T: DeserializeOwned>(
- &self,
- sql: &str,
- parameters: impl rusqlite::Params,
- ) -> Result<Option<T>> {
- let text: Option<String> = self
- .db
- .query_row(sql, parameters, |r| r.get(0))
- .optional()?;
- text.map(|text| serde_json::from_str(&text).map_err(Into::into))
- .transpose()
- }
-
pub fn temporary(&self, temporary: &Temporary) -> Result<()> {
self.db.execute(
"INSERT INTO temporaries VALUES(?1,?2,?3,?4)",
diff --git a/rust/crates/photos-export/src/transfer.rs b/rust/crates/photos-export/src/transfer.rs
index b5f970e16c..d633b28bf6 100644
--- a/rust/crates/photos-export/src/transfer.rs
+++ b/rust/crates/photos-export/src/transfer.rs
@@ -39,29 +39,26 @@ pub fn roles(file: &FileSnapshot) -> Vec<Role> {
}
}
-pub fn verify(
- run: &Run<'_>,
- component: &Component,
- file: &FileSnapshot,
- signature: &str,
-) -> Result<Option<Properties>> {
- let path = run.path(&component.location)?;
- let Some(properties) = Properties::optional(&path)? else {
- return Ok(None);
- };
- if expected_hash(file, &component.role).is_some_and(|hash| hash != component.hash)
+fn source_changed(component: &Component, file: &FileSnapshot, signature: &str) -> bool {
+ !roles(file).contains(&component.role)
+ || expected_hash(file, &component.role).is_some_and(|hash| hash != component.hash)
|| (expected_hash(file, &component.role).is_none()
&& component
.signature
.as_ref()
.is_some_and(|original| original != signature))
- {
- return Ok(None);
- }
- if component.properties.as_ref() != Some(&properties)
- && !fs::matches(&path, component.size, &component.hash, run.cancel)?
- {
+}
+
+fn verify(run: &Run<'_>, component: &Component, replacing: bool) -> Result<Option<Properties>> {
+ let path = run.path(&component.location)?;
+ let Some(properties) = Properties::optional(&path)? else {
return Ok(None);
+ };
+ if replacing || component.properties.as_ref() != Some(&properties) {
+ ensure!(
+ fs::matches(&path, component.size, &component.hash, run.cancel)?,
+ super::Conflict(format!("at {}: locally changed media", path.display()))
+ );
}
Ok(Some(properties))
}
@@ -87,23 +84,30 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> R
};
after = album;
let result = (|| {
- let components = {
+ let (components, retained_folder) = {
let store = store::lock(run.store)?;
- needs_bytes |= store
+ let retained_folder = store
.pending_file(album, file.id)?
- .is_some_and(|pending| pending.retained);
- store
+ .filter(|pending| pending.retained)
+ .map(|pending| pending.folder);
+ let components = store
.placement(album, file.id)?
.map(|placement| store.components(&placement.id))
.transpose()?
- .unwrap_or_default()
+ .unwrap_or_default();
+ (components, retained_folder)
};
let mut verified = Vec::new();
+ let replacing = components
+ .iter()
+ .any(|component| source_changed(component, file, &signature));
for component in &components {
- if !roles.contains(&component.role) {
+ if retained_folder.as_ref() == Some(&component.location.folder) {
continue;
}
- if let Some(properties) = verify(run, component, file, &signature)? {
+ if let Some(properties) = verify(run, component, replacing)?
+ && !source_changed(component, file, &signature)
+ {
verified.push((component.role.clone(), properties));
}
}