π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Preserve native album metadata and recovery categories
Manav Rathi4998caa7bd4e
diff --git a/rust/apps/cli-next/tests/support/export.rs b/rust/apps/cli-next/tests/support/export.rsindex dc85514ef7..80d768a14d 100644--- a/rust/apps/cli-next/tests/support/export.rs+++ b/rust/apps/cli-next/tests/support/export.rs@@ -582,18 +582,18 @@ fn export_preserves_album_types_visibility_and_settings() -> TestResult {let home = TestHome::new();login(&home, "photos", &email, &["--host", origin]);let cases = [- ("Archive", "album", "album", 1, 0, "archived"),- ("Private", "album", "album", 2, 0, "hidden"),- ("Hidden", "album", "defaultHidden", 0, 1, "hidden"),- ("Quick link", "album", "quicklink", 0, 2, "visible"),- ("Folder", "folder", "folder", 0, 0, "visible"),- ("Uncategorized", "uncategorized", "uncategorized", 0, 0, "visible"),+ ("Archive", "album", "album", 1, 0),+ ("Private", "album", "album", 2, 0),+ ("Hidden", "album", "defaultHidden", 0, 1),+ ("Quick link", "album", "quicklink", 0, 2),+ ("Folder", "folder", "folder", 0, 0),+ ("Uncategorized", "uncategorized", "uncategorized", 0, 0),];- for (name, kind, _, visibility, subtype, _) in cases {+ for (name, kind, _, visibility, subtype) in cases {let (album, key) = create_album(origin, &account, name, kind).await;let (file, file_key) = upload_fixture(origin, &account, album, &key, name.as_bytes(), metadata("original.jpg", name.as_bytes())).await;set_album_metadata(origin, &account, album, &key, "magic-metadata", json!({"visibility":visibility,"subType":subtype,"order":7})).await;- set_album_metadata(origin, &account, album, &key, "public-magic-metadata", json!({"asc":true,"coverID":file,"layout":"trip","caption":"album description"})).await;+ set_album_metadata(origin, &account, album, &key, "public-magic-metadata", json!({"asc":true,"sortBy":"fileName","coverID":file,"layout":"trip","caption":"album description","extension":{"nested":[true,null,17]}})).await;let file_visibility = if name == "Quick link" { 2 } else { visibility };let private = blob::encrypt_json(&json!({"visibility":file_visibility}), &file_key)?;request(origin, &account, reqwest::Method::PUT, "/files/magic-metadata", json!({"metadataList":[{"id":file,"magicMetadata":{"version":1,"count":1,"data":b64::encode(&private.encrypted_data),"header":b64::encode(private.decryption_header.as_bytes())}}]})).await;@@ -606,19 +606,16 @@ fn export_preserves_album_types_visibility_and_settings() -> TestResult {assert_eq!(read_json(&root.join("Quick link/metadata/original.jpg.json"))["ente"]["magicMetadata"]["visibility"], 2);let result = export(&home, &root, &[]);assert_eq!(result["copies"]["completed"], cases.len());- for (name, _, kind, file_visibility, _, visibility) in cases {+ for (name, _, kind, visibility, subtype) in cases {let folder = root.join(name);let album = read_json(&folder.join("metadata.json"));let file = read_json(&folder.join("metadata/original.jpg.json"));assert_eq!(fs::read(folder.join("original.jpg"))?, name.as_bytes());assert_eq!(album["ente"]["type"], kind);- assert_eq!(album["ente"]["visibility"], visibility);- assert_eq!(album["ente"]["displayOrder"], 7);- assert_eq!(album["ente"]["sortOrder"], "ascending");- assert_eq!(album["ente"]["description"], "album description");- assert_eq!(album["ente"]["layout"], "trip");- assert_eq!(album["ente"]["coverFileID"], file["ente"]["fileID"]);- assert_eq!(file["ente"]["magicMetadata"]["visibility"], if name == "Quick link" { 2 } else { file_visibility });+ assert_eq!(album["title"], name);+ assert_eq!(album["ente"]["magicMetadata"], json!({"visibility":visibility,"subType":subtype,"order":7}));+ assert_eq!(album["ente"]["pubMagicMetadata"], json!({"asc":true,"sortBy":"fileName","coverID":file["ente"]["fileID"].as_str().unwrap().parse::<i64>()?,"layout":"trip","caption":"album description","extension":{"nested":[true,null,17]}}));+ assert_eq!(file["ente"]["magicMetadata"]["visibility"], if name == "Quick link" { 2 } else { visibility });}Ok(())})diff --git a/rust/apps/cli-next/tests/support/export_engine.rs b/rust/apps/cli-next/tests/support/export_engine.rsindex b1b12d4683..21ab64e477 100644--- a/rust/apps/cli-next/tests/support/export_engine.rs+++ b/rust/apps/cli-next/tests/support/export_engine.rs@@ -20,6 +20,9 @@ impl Catalog {fn album(&self, id: i64) -> CollectionEntry {let name = self.albums[&id].clone();let (shared, hidden) = self.categories.get(&id).copied().unwrap_or_default();+ let metadata = self.categories.get(&id).map(|_| {+ serde_json::to_vec(&json!({"visibility": if hidden { 2 } else { 0 }})).unwrap()+ });CollectionEntry {id,owner_id: self.user + i64::from(shared),@@ -42,8 +45,8 @@ impl Catalog {Documents {original: Vec::new(),public: None,- private: None,- shared: None,+ private: if shared { None } else { metadata.clone() },+ shared: if shared { metadata } else { None },},)),}diff --git a/rust/apps/cli-next/tests/support/export_maintenance.rs b/rust/apps/cli-next/tests/support/export_maintenance.rsindex 291949059c..242a392336 100644--- a/rust/apps/cli-next/tests/support/export_maintenance.rs+++ b/rust/apps/cli-next/tests/support/export_maintenance.rs@@ -2655,22 +2655,19 @@ fn export_preserves_native_fields_unused_by_common_commands() {}#[test]-fn export_only_album_metadata_preserves_common_commands_and_favorites() {- for (document, value, cause) in [+fn export_preserves_album_documents_and_favorites() {+ for (document, value) in [("magicMetadata",- json!({"order":"broken"}),- "invalid album private metadata",+ json!({"order":7,"extension":{"pinned":true}}),),("sharedMagicMetadata",- json!({"order":"broken","mute":"broken"}),- "invalid album sharee metadata",+ json!({"order":9,"mute":true,"extension":{"pinned":false}}),),("pubMagicMetadata",- json!({"asc":"broken","coverID":"broken","layout":17,"caption":17}),- "invalid album public metadata",+ json!({"asc":true,"sortBy":"fileName","coverID":10,"layout":"trip","caption":"Album"}),),] {let mut server = mockito::Server::new();@@ -2718,16 +2715,15 @@ fn export_only_album_metadata_preserves_common_commands_and_favorites() {metadata["favorited"] = json!(false);fs::write(&sidecar, serde_json::to_vec(&metadata).unwrap()).unwrap();let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);- let error = failure(&output);- assert!(error.contains(&format!("album:5: {cause}")), "{error}");- assert!(!error.contains("favorite state is incomplete"), "{error}");- let result: Value = serde_json::from_slice(&output.stdout).unwrap();- assert_eq!(result["complete"], false);- assert_eq!(result["failures"], 1);- assert_eq!(result["changes"]["metadataUpdated"], 1);+ let result: Value = serde_json::from_slice(&success(output).stdout).unwrap();+ assert_eq!(result["complete"], true);+ assert_eq!(result["copies"]["completed"], 2);assert_eq!(read_json(&sidecar)["favorited"], true);assert_component(&root, "Healthy", "Photo.jpg", b"original");- assert!(!root.join("Favorites").exists());+ assert_eq!(+ read_json(&root.join("Favorites/metadata.json"))["ente"][document],+ value+ );memberships.assert();downloaded.assert();}diff --git a/rust/apps/cli-next/tests/support/export_media.rs b/rust/apps/cli-next/tests/support/export_media.rsindex 66d5cb4074..2f04dd1c41 100644--- a/rust/apps/cli-next/tests/support/export_media.rs+++ b/rust/apps/cli-next/tests/support/export_media.rs@@ -119,6 +119,11 @@ fn engine_category_recovery_preserves_unknown_albums_after_database_loss() {count(&engine.db(), "SELECT count(*) FROM albums WHERE retained=0"),2);+ let path = root.join("First/metadata.json");+ let mut metadata = read_json(&path);+ metadata["ente"]["type"] = json!("defaultHidden");+ metadata["ente"]["magicMetadata"] = json!({"subType":1});+ fs::write(&path, serde_json::to_vec(&metadata).unwrap()).unwrap();engine.catalog.albums.remove(&2);let mut options = engine.options(false, &[]);options.exclude_shared = true;@@ -134,6 +139,21 @@ fn engine_category_recovery_preserves_unknown_albums_after_database_loss() {options.exclude_hidden = true;let export = engine.prepare(options).unwrap();let result = engine.execute(export).unwrap();+ assert!(result.complete);+ assert_eq!(result.completed, 1);+ let path = root.join("Second/metadata.json");+ let mut metadata = read_json(&path);+ metadata["ente"]+ .as_object_mut()+ .unwrap()+ .remove("sharedMagicMetadata");+ fs::write(&path, serde_json::to_vec(&metadata).unwrap()).unwrap();+ engine.database = "unknown-owner.db".into();+ let mut options = engine.options(true, &[]);+ options.exclude_shared = true;+ options.exclude_hidden = true;+ let export = engine.prepare(options).unwrap();+ let result = engine.execute(export).unwrap();assert!(!result.complete);assert_eq!(result.json["failures"], 1);assert_eq!(result.completed, 1);diff --git a/rust/apps/cli-next/tests/support/museum.rs b/rust/apps/cli-next/tests/support/museum.rsindex 4b1e0b1cf4..463d798a4a 100644--- a/rust/apps/cli-next/tests/support/museum.rs+++ b/rust/apps/cli-next/tests/support/museum.rs@@ -487,7 +487,10 @@ async fn exercise(origin: String) -> TestResult {let album_record: Value =serde_json::from_slice(&fs::read(shared_export.join("Monsoon π§/metadata.json")).unwrap()).unwrap();- assert_eq!(album_record["ente"]["visibility"], "hidden");+ assert_eq!(+ album_record["ente"]["sharedMagicMetadata"],+ json!({"visibility": 2})+ );let shared_sidecar = shared_export.join("Monsoon π§/metadata/original.jpg.json");let file_record: Value = serde_json::from_slice(&fs::read(&shared_sidecar).unwrap()).unwrap();assert!(file_record["ente"].get("visibility").is_none());diff --git a/rust/crates/photos-export/src/adopt.rs b/rust/crates/photos-export/src/adopt.rsindex b0f0901eb9..2822766704 100644--- a/rust/crates/photos-export/src/adopt.rs+++ b/rust/crates/photos-export/src/adopt.rs@@ -249,11 +249,7 @@ fn parse_album(root: &Path, directory: &Path, retained: bool) -> Result<Option<A.contains(&identity["type"].as_str().unwrap_or_default()),"unsupported album type in {folder}");- ensure!(- ["visible", "archived", "hidden"]- .contains(&identity["visibility"].as_str().unwrap_or_default()),- "unsupported album visibility in {folder}"- );+ crate::metadata::album_categories(&record)?;ensure!(retained || !names::reserved(&folder, true),"reserved active album path {folder}"diff --git a/rust/crates/photos-export/src/lib.rs b/rust/crates/photos-export/src/lib.rsindex 56b4d3ab6a..4b79cc744c 100644--- a/rust/crates/photos-export/src/lib.rs+++ b/rust/crates/photos-export/src/lib.rs@@ -182,7 +182,7 @@ impl Export {)?;}}- let mut albums = store.db.prepare("SELECT a.id,a.name,j.value FROM albums a JOIN json_records j ON j.owner=a.key AND j.role='' LEFT JOIN album_sources s ON s.id=a.id WHERE json_extract(s.record,'$.hidden') IS NULL ORDER BY a.retained")?;+ let mut albums = store.db.prepare("SELECT a.id,a.name,j.value FROM albums a JOIN json_records j ON j.owner=a.key AND j.role='' LEFT JOIN album_sources s ON s.id=a.id WHERE json_extract(s.record,'$.shared') IS NULL OR json_extract(s.record,'$.hidden') IS NULL ORDER BY a.retained")?;for row in albums.query_map([], |r| {Ok((r.get::<_, i64>(0)?,diff --git a/rust/crates/photos-export/src/metadata.rs b/rust/crates/photos-export/src/metadata.rsindex 6e7731be61..59db7d577f 100644--- a/rust/crates/photos-export/src/metadata.rs+++ b/rust/crates/photos-export/src/metadata.rs@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};use serde_json::{Map, Value, json};use ente_photos::{- collections::Collection,+ collections::{Collection, Visibility},files::{File, Kind},source::{self, Documents, MetadataError},};@@ -56,64 +56,41 @@ pub struct Component {pub hash: String,}-pub fn album(- album: &Collection,- documents: &Documents,- user_id: i64,-) -> Result<(Value, Vec<String>), MetadataError> {- let private: AlbumPrivate = source::optional(- documents.private.as_deref(),- "invalid album private metadata",- )?;- let shared: AlbumShared =- source::optional(documents.shared.as_deref(), "invalid album sharee metadata")?;- let public: AlbumPublic =- source::optional(documents.public.as_deref(), "invalid album public metadata")?;- let owned = album.owner_id == user_id;- let mut metadata = json!({- "albumID": album.id.to_string(), "type": album.kind.name(), "visibility": album.visibility.name(),- });- optional(- &mut metadata,- "displayOrder",- if owned { private.order } else { shared.order },- );- optional(- &mut metadata,- "muted",- (!owned).then_some(shared.mute).flatten(),- );- optional(- &mut metadata,- "sortOrder",- public- .asc- .map(|asc| if asc { "ascending" } else { "descending" }),- );- optional(&mut metadata, "description", public.caption.as_deref());- optional(- &mut metadata,- "coverFileID",- public.cover_id.map(|id| id.to_string()),- );- optional(&mut metadata, "layout", public.layout.as_deref());- let mut warnings = Vec::new();- unfamiliar(- documents.private.as_deref(),- &["visibility", "subType", "order"],- &mut warnings,- );- unfamiliar(- documents.shared.as_deref(),- &["visibility", "mute", "order"],- &mut warnings,- );- unfamiliar(- documents.public.as_deref(),- &["asc", "coverID", "layout", "caption"],- &mut warnings,- );- Ok((json!({"title": album.name, "ente": metadata}), warnings))+pub fn album(album: &Collection, documents: &Documents) -> Result<Value, MetadataError> {+ let mut metadata = json!({"albumID": album.id.to_string(), "type": album.kind.name()});+ for (name, bytes) in [+ ("magicMetadata", &documents.private),+ ("pubMagicMetadata", &documents.public),+ ("sharedMagicMetadata", &documents.shared),+ ] {+ if let Some(bytes) = bytes {+ metadata[name] = json!(source::parse::<Map<String, Value>>(+ bytes,+ "invalid album metadata document",+ )?);+ }+ }+ Ok(json!({"title": album.name, "ente": metadata}))+}++pub fn album_categories(record: &Value) -> Result<(Option<bool>, bool), MetadataError> {+ let metadata = &record["ente"];+ for name in ["magicMetadata", "pubMagicMetadata", "sharedMagicMetadata"] {+ if let Some(document) = metadata.get(name)+ && !document.is_object()+ {+ return Err(MetadataError("invalid album metadata document"));+ }+ }+ let private = metadata.get("magicMetadata");+ let shared = metadata.get("sharedMagicMetadata");+ let visibility = private.or(shared).map_or(Ok(None), |document| {+ serde_json::from_value::<Option<u8>>(document["visibility"].clone())+ .map_err(|_| MetadataError("invalid album visibility"))+ })?;+ let hidden = metadata["type"] == "defaultHidden"+ || matches!(Visibility::from_value(visibility)?, Visibility::Hidden);+ Ok((private.map(|_| false).or(shared.map(|_| true)), hidden))}pub fn file(file: &File, documents: &Documents) -> Result<Value, MetadataError> {@@ -226,35 +203,3 @@ pub fn publish(record["ente"]["components"] = json!(components);Ok(record)}--#[derive(Default, Deserialize)]-struct AlbumPrivate {- order: Option<i64>,-}--#[derive(Default, Deserialize)]-struct AlbumShared {- order: Option<i64>,- mute: Option<bool>,-}--#[derive(Default, Deserialize)]-struct AlbumPublic {- asc: Option<bool>,- #[serde(rename = "coverID")]- cover_id: Option<i64>,- layout: Option<String>,- caption: Option<String>,-}--fn unfamiliar(bytes: Option<&[u8]>, known: &[&str], warnings: &mut Vec<String>) {- if let Some(bytes) = bytes- && let Ok(Value::Object(fields)) = serde_json::from_slice(bytes)- {- for key in fields.keys() {- if !known.contains(&key.as_str()) {- warnings.push(key.clone());- }- }- }-}diff --git a/rust/crates/photos-export/src/reconcile.rs b/rust/crates/photos-export/src/reconcile.rsindex 51e6e3e2b1..d3ae735467 100644--- a/rust/crates/photos-export/src/reconcile.rs+++ b/rust/crates/photos-export/src/reconcile.rs@@ -929,11 +929,6 @@ impl Run<'_> {};after = id;let source: AlbumSource = serde_json::from_str(&source)?;- for field in &source.warnings {- (self.report)(&format!(- "album {id}: unfamiliar metadata field {field:?}; exporting supported fields"- ));- }if let Err(error) = self.maintain_album(id, &source.name, false, source.metadata) {self.block(id, None, error)?;}diff --git a/rust/crates/photos-export/src/store.rs b/rust/crates/photos-export/src/store.rsindex 975c32a4ec..cab105eff1 100644--- a/rust/crates/photos-export/src/store.rs+++ b/rust/crates/photos-export/src/store.rs@@ -212,7 +212,6 @@ pub struct AlbumSource {pub shared: Option<bool>,pub hidden: Option<bool>,pub metadata: Value,- pub warnings: Vec<String>,}#[derive(Clone)]@@ -588,7 +587,6 @@ impl Store {shared: None,hidden: None,metadata: Value::Null,- warnings: Vec::new(),});source.shared = Some(shared);source.hidden = hidden.or(source.hidden);@@ -612,14 +610,13 @@ impl Store {shared: None,hidden: None,metadata: Value::Null,- warnings: Vec::new(),});source.shared = source.shared.or(shared);- source.hidden = source.hidden.or_else(|| {- metadata["ente"]["visibility"]- .as_str()- .map(|visibility| visibility == "hidden")- });+ if metadata.is_object() {+ let (shared, hidden) = crate::metadata::album_categories(&metadata)?;+ source.shared = source.shared.or(shared);+ source.hidden = source.hidden.or(Some(hidden));+ }self.save_album_source(id, &source)}@@ -629,13 +626,12 @@ impl Store {documents: &Documents,user_id: i64,) -> Result<()> {- let (metadata, warnings) = crate::metadata::album(album, documents, user_id)?;+ let metadata = crate::metadata::album(album, documents)?;let source = AlbumSource {name: album.name.clone(),shared: Some(album.owner_id != user_id),hidden: Some(matches!(album.visibility, Visibility::Hidden)),metadata,- warnings,};self.save_album_source(album.id, &source)}