Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Preserve native album metadata and recovery categories

Manav Rathi4998caa7bd4e
diff --git a/rust/apps/cli-next/tests/support/export.rs b/rust/apps/cli-next/tests/support/export.rs
index dc85514ef7..80d768a14d 100644
--- a/rust/apps/cli-next/tests/support/export.rs
+++ b/rust/apps/cli-next/tests/support/export.rs
@@ -582,18 +582,18 @@ fn export_preserves_album_types_visibility_and_settings() -> TestResult {
let home = TestHome::new();
login(&home, "photos", &email, &["--host", origin]);
let cases = [
- ("Archive", "album", "album", 1, 0, "archived"),
- ("Private", "album", "album", 2, 0, "hidden"),
- ("Hidden", "album", "defaultHidden", 0, 1, "hidden"),
- ("Quick link", "album", "quicklink", 0, 2, "visible"),
- ("Folder", "folder", "folder", 0, 0, "visible"),
- ("Uncategorized", "uncategorized", "uncategorized", 0, 0, "visible"),
+ ("Archive", "album", "album", 1, 0),
+ ("Private", "album", "album", 2, 0),
+ ("Hidden", "album", "defaultHidden", 0, 1),
+ ("Quick link", "album", "quicklink", 0, 2),
+ ("Folder", "folder", "folder", 0, 0),
+ ("Uncategorized", "uncategorized", "uncategorized", 0, 0),
];
- for (name, kind, _, visibility, subtype, _) in cases {
+ for (name, kind, _, visibility, subtype) in cases {
let (album, key) = create_album(origin, &account, name, kind).await;
let (file, file_key) = upload_fixture(origin, &account, album, &key, name.as_bytes(), metadata("original.jpg", name.as_bytes())).await;
set_album_metadata(origin, &account, album, &key, "magic-metadata", json!({"visibility":visibility,"subType":subtype,"order":7})).await;
- set_album_metadata(origin, &account, album, &key, "public-magic-metadata", json!({"asc":true,"coverID":file,"layout":"trip","caption":"album description"})).await;
+ set_album_metadata(origin, &account, album, &key, "public-magic-metadata", json!({"asc":true,"sortBy":"fileName","coverID":file,"layout":"trip","caption":"album description","extension":{"nested":[true,null,17]}})).await;
let file_visibility = if name == "Quick link" { 2 } else { visibility };
let private = blob::encrypt_json(&json!({"visibility":file_visibility}), &file_key)?;
request(origin, &account, reqwest::Method::PUT, "/files/magic-metadata", json!({"metadataList":[{"id":file,"magicMetadata":{"version":1,"count":1,"data":b64::encode(&private.encrypted_data),"header":b64::encode(private.decryption_header.as_bytes())}}]})).await;
@@ -606,19 +606,16 @@ fn export_preserves_album_types_visibility_and_settings() -> TestResult {
assert_eq!(read_json(&root.join("Quick link/metadata/original.jpg.json"))["ente"]["magicMetadata"]["visibility"], 2);
let result = export(&home, &root, &[]);
assert_eq!(result["copies"]["completed"], cases.len());
- for (name, _, kind, file_visibility, _, visibility) in cases {
+ for (name, _, kind, visibility, subtype) in cases {
let folder = root.join(name);
let album = read_json(&folder.join("metadata.json"));
let file = read_json(&folder.join("metadata/original.jpg.json"));
assert_eq!(fs::read(folder.join("original.jpg"))?, name.as_bytes());
assert_eq!(album["ente"]["type"], kind);
- assert_eq!(album["ente"]["visibility"], visibility);
- assert_eq!(album["ente"]["displayOrder"], 7);
- assert_eq!(album["ente"]["sortOrder"], "ascending");
- assert_eq!(album["ente"]["description"], "album description");
- assert_eq!(album["ente"]["layout"], "trip");
- assert_eq!(album["ente"]["coverFileID"], file["ente"]["fileID"]);
- assert_eq!(file["ente"]["magicMetadata"]["visibility"], if name == "Quick link" { 2 } else { file_visibility });
+ assert_eq!(album["title"], name);
+ assert_eq!(album["ente"]["magicMetadata"], json!({"visibility":visibility,"subType":subtype,"order":7}));
+ assert_eq!(album["ente"]["pubMagicMetadata"], json!({"asc":true,"sortBy":"fileName","coverID":file["ente"]["fileID"].as_str().unwrap().parse::<i64>()?,"layout":"trip","caption":"album description","extension":{"nested":[true,null,17]}}));
+ assert_eq!(file["ente"]["magicMetadata"]["visibility"], if name == "Quick link" { 2 } else { visibility });
}
Ok(())
})
diff --git a/rust/apps/cli-next/tests/support/export_engine.rs b/rust/apps/cli-next/tests/support/export_engine.rs
index b1b12d4683..21ab64e477 100644
--- a/rust/apps/cli-next/tests/support/export_engine.rs
+++ b/rust/apps/cli-next/tests/support/export_engine.rs
@@ -20,6 +20,9 @@ impl Catalog {
fn album(&self, id: i64) -> CollectionEntry {
let name = self.albums[&id].clone();
let (shared, hidden) = self.categories.get(&id).copied().unwrap_or_default();
+ let metadata = self.categories.get(&id).map(|_| {
+ serde_json::to_vec(&json!({"visibility": if hidden { 2 } else { 0 }})).unwrap()
+ });
CollectionEntry {
id,
owner_id: self.user + i64::from(shared),
@@ -42,8 +45,8 @@ impl Catalog {
Documents {
original: Vec::new(),
public: None,
- private: None,
- shared: None,
+ private: if shared { None } else { metadata.clone() },
+ shared: if shared { metadata } else { None },
},
)),
}
diff --git a/rust/apps/cli-next/tests/support/export_maintenance.rs b/rust/apps/cli-next/tests/support/export_maintenance.rs
index 291949059c..242a392336 100644
--- a/rust/apps/cli-next/tests/support/export_maintenance.rs
+++ b/rust/apps/cli-next/tests/support/export_maintenance.rs
@@ -2655,22 +2655,19 @@ fn export_preserves_native_fields_unused_by_common_commands() {
}
#[test]
-fn export_only_album_metadata_preserves_common_commands_and_favorites() {
- for (document, value, cause) in [
+fn export_preserves_album_documents_and_favorites() {
+ for (document, value) in [
(
"magicMetadata",
- json!({"order":"broken"}),
- "invalid album private metadata",
+ json!({"order":7,"extension":{"pinned":true}}),
),
(
"sharedMagicMetadata",
- json!({"order":"broken","mute":"broken"}),
- "invalid album sharee metadata",
+ json!({"order":9,"mute":true,"extension":{"pinned":false}}),
),
(
"pubMagicMetadata",
- json!({"asc":"broken","coverID":"broken","layout":17,"caption":17}),
- "invalid album public metadata",
+ json!({"asc":true,"sortBy":"fileName","coverID":10,"layout":"trip","caption":"Album"}),
),
] {
let mut server = mockito::Server::new();
@@ -2718,16 +2715,15 @@ fn export_only_album_metadata_preserves_common_commands_and_favorites() {
metadata["favorited"] = json!(false);
fs::write(&sidecar, serde_json::to_vec(&metadata).unwrap()).unwrap();
let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
- let error = failure(&output);
- assert!(error.contains(&format!("album:5: {cause}")), "{error}");
- assert!(!error.contains("favorite state is incomplete"), "{error}");
- let result: Value = serde_json::from_slice(&output.stdout).unwrap();
- assert_eq!(result["complete"], false);
- assert_eq!(result["failures"], 1);
- assert_eq!(result["changes"]["metadataUpdated"], 1);
+ let result: Value = serde_json::from_slice(&success(output).stdout).unwrap();
+ assert_eq!(result["complete"], true);
+ assert_eq!(result["copies"]["completed"], 2);
assert_eq!(read_json(&sidecar)["favorited"], true);
assert_component(&root, "Healthy", "Photo.jpg", b"original");
- assert!(!root.join("Favorites").exists());
+ assert_eq!(
+ read_json(&root.join("Favorites/metadata.json"))["ente"][document],
+ value
+ );
memberships.assert();
downloaded.assert();
}
diff --git a/rust/apps/cli-next/tests/support/export_media.rs b/rust/apps/cli-next/tests/support/export_media.rs
index 66d5cb4074..2f04dd1c41 100644
--- a/rust/apps/cli-next/tests/support/export_media.rs
+++ b/rust/apps/cli-next/tests/support/export_media.rs
@@ -119,6 +119,11 @@ fn engine_category_recovery_preserves_unknown_albums_after_database_loss() {
count(&engine.db(), "SELECT count(*) FROM albums WHERE retained=0"),
2
);
+ let path = root.join("First/metadata.json");
+ let mut metadata = read_json(&path);
+ metadata["ente"]["type"] = json!("defaultHidden");
+ metadata["ente"]["magicMetadata"] = json!({"subType":1});
+ fs::write(&path, serde_json::to_vec(&metadata).unwrap()).unwrap();
engine.catalog.albums.remove(&2);
let mut options = engine.options(false, &[]);
options.exclude_shared = true;
@@ -134,6 +139,21 @@ fn engine_category_recovery_preserves_unknown_albums_after_database_loss() {
options.exclude_hidden = true;
let export = engine.prepare(options).unwrap();
let result = engine.execute(export).unwrap();
+ assert!(result.complete);
+ assert_eq!(result.completed, 1);
+ let path = root.join("Second/metadata.json");
+ let mut metadata = read_json(&path);
+ metadata["ente"]
+ .as_object_mut()
+ .unwrap()
+ .remove("sharedMagicMetadata");
+ fs::write(&path, serde_json::to_vec(&metadata).unwrap()).unwrap();
+ engine.database = "unknown-owner.db".into();
+ let mut options = engine.options(true, &[]);
+ options.exclude_shared = true;
+ options.exclude_hidden = true;
+ let export = engine.prepare(options).unwrap();
+ let result = engine.execute(export).unwrap();
assert!(!result.complete);
assert_eq!(result.json["failures"], 1);
assert_eq!(result.completed, 1);
diff --git a/rust/apps/cli-next/tests/support/museum.rs b/rust/apps/cli-next/tests/support/museum.rs
index 4b1e0b1cf4..463d798a4a 100644
--- a/rust/apps/cli-next/tests/support/museum.rs
+++ b/rust/apps/cli-next/tests/support/museum.rs
@@ -487,7 +487,10 @@ async fn exercise(origin: String) -> TestResult {
let album_record: Value =
serde_json::from_slice(&fs::read(shared_export.join("Monsoon 🌧/metadata.json")).unwrap())
.unwrap();
- assert_eq!(album_record["ente"]["visibility"], "hidden");
+ assert_eq!(
+ album_record["ente"]["sharedMagicMetadata"],
+ json!({"visibility": 2})
+ );
let shared_sidecar = shared_export.join("Monsoon 🌧/metadata/original.jpg.json");
let file_record: Value = serde_json::from_slice(&fs::read(&shared_sidecar).unwrap()).unwrap();
assert!(file_record["ente"].get("visibility").is_none());
diff --git a/rust/crates/photos-export/src/adopt.rs b/rust/crates/photos-export/src/adopt.rs
index b0f0901eb9..2822766704 100644
--- a/rust/crates/photos-export/src/adopt.rs
+++ b/rust/crates/photos-export/src/adopt.rs
@@ -249,11 +249,7 @@ fn parse_album(root: &Path, directory: &Path, retained: bool) -> Result<Option<A
.contains(&identity["type"].as_str().unwrap_or_default()),
"unsupported album type in {folder}"
);
- ensure!(
- ["visible", "archived", "hidden"]
- .contains(&identity["visibility"].as_str().unwrap_or_default()),
- "unsupported album visibility in {folder}"
- );
+ crate::metadata::album_categories(&record)?;
ensure!(
retained || !names::reserved(&folder, true),
"reserved active album path {folder}"
diff --git a/rust/crates/photos-export/src/lib.rs b/rust/crates/photos-export/src/lib.rs
index 56b4d3ab6a..4b79cc744c 100644
--- a/rust/crates/photos-export/src/lib.rs
+++ b/rust/crates/photos-export/src/lib.rs
@@ -182,7 +182,7 @@ impl Export {
)?;
}
}
- let mut albums = store.db.prepare("SELECT a.id,a.name,j.value FROM albums a JOIN json_records j ON j.owner=a.key AND j.role='' LEFT JOIN album_sources s ON s.id=a.id WHERE json_extract(s.record,'$.hidden') IS NULL ORDER BY a.retained")?;
+ let mut albums = store.db.prepare("SELECT a.id,a.name,j.value FROM albums a JOIN json_records j ON j.owner=a.key AND j.role='' LEFT JOIN album_sources s ON s.id=a.id WHERE json_extract(s.record,'$.shared') IS NULL OR json_extract(s.record,'$.hidden') IS NULL ORDER BY a.retained")?;
for row in albums.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
diff --git a/rust/crates/photos-export/src/metadata.rs b/rust/crates/photos-export/src/metadata.rs
index 6e7731be61..59db7d577f 100644
--- a/rust/crates/photos-export/src/metadata.rs
+++ b/rust/crates/photos-export/src/metadata.rs
@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
use serde_json::{Map, Value, json};
use ente_photos::{
- collections::Collection,
+ collections::{Collection, Visibility},
files::{File, Kind},
source::{self, Documents, MetadataError},
};
@@ -56,64 +56,41 @@ pub struct Component {
pub hash: String,
}
-pub fn album(
- album: &Collection,
- documents: &Documents,
- user_id: i64,
-) -> Result<(Value, Vec<String>), MetadataError> {
- let private: AlbumPrivate = source::optional(
- documents.private.as_deref(),
- "invalid album private metadata",
- )?;
- let shared: AlbumShared =
- source::optional(documents.shared.as_deref(), "invalid album sharee metadata")?;
- let public: AlbumPublic =
- source::optional(documents.public.as_deref(), "invalid album public metadata")?;
- let owned = album.owner_id == user_id;
- let mut metadata = json!({
- "albumID": album.id.to_string(), "type": album.kind.name(), "visibility": album.visibility.name(),
- });
- optional(
- &mut metadata,
- "displayOrder",
- if owned { private.order } else { shared.order },
- );
- optional(
- &mut metadata,
- "muted",
- (!owned).then_some(shared.mute).flatten(),
- );
- optional(
- &mut metadata,
- "sortOrder",
- public
- .asc
- .map(|asc| if asc { "ascending" } else { "descending" }),
- );
- optional(&mut metadata, "description", public.caption.as_deref());
- optional(
- &mut metadata,
- "coverFileID",
- public.cover_id.map(|id| id.to_string()),
- );
- optional(&mut metadata, "layout", public.layout.as_deref());
- let mut warnings = Vec::new();
- unfamiliar(
- documents.private.as_deref(),
- &["visibility", "subType", "order"],
- &mut warnings,
- );
- unfamiliar(
- documents.shared.as_deref(),
- &["visibility", "mute", "order"],
- &mut warnings,
- );
- unfamiliar(
- documents.public.as_deref(),
- &["asc", "coverID", "layout", "caption"],
- &mut warnings,
- );
- Ok((json!({"title": album.name, "ente": metadata}), warnings))
+pub fn album(album: &Collection, documents: &Documents) -> Result<Value, MetadataError> {
+ let mut metadata = json!({"albumID": album.id.to_string(), "type": album.kind.name()});
+ for (name, bytes) in [
+ ("magicMetadata", &documents.private),
+ ("pubMagicMetadata", &documents.public),
+ ("sharedMagicMetadata", &documents.shared),
+ ] {
+ if let Some(bytes) = bytes {
+ metadata[name] = json!(source::parse::<Map<String, Value>>(
+ bytes,
+ "invalid album metadata document",
+ )?);
+ }
+ }
+ Ok(json!({"title": album.name, "ente": metadata}))
+}
+
+pub fn album_categories(record: &Value) -> Result<(Option<bool>, bool), MetadataError> {
+ let metadata = &record["ente"];
+ for name in ["magicMetadata", "pubMagicMetadata", "sharedMagicMetadata"] {
+ if let Some(document) = metadata.get(name)
+ && !document.is_object()
+ {
+ return Err(MetadataError("invalid album metadata document"));
+ }
+ }
+ let private = metadata.get("magicMetadata");
+ let shared = metadata.get("sharedMagicMetadata");
+ let visibility = private.or(shared).map_or(Ok(None), |document| {
+ serde_json::from_value::<Option<u8>>(document["visibility"].clone())
+ .map_err(|_| MetadataError("invalid album visibility"))
+ })?;
+ let hidden = metadata["type"] == "defaultHidden"
+ || matches!(Visibility::from_value(visibility)?, Visibility::Hidden);
+ Ok((private.map(|_| false).or(shared.map(|_| true)), hidden))
}
pub fn file(file: &File, documents: &Documents) -> Result<Value, MetadataError> {
@@ -226,35 +203,3 @@ pub fn publish(
record["ente"]["components"] = json!(components);
Ok(record)
}
-
-#[derive(Default, Deserialize)]
-struct AlbumPrivate {
- order: Option<i64>,
-}
-
-#[derive(Default, Deserialize)]
-struct AlbumShared {
- order: Option<i64>,
- mute: Option<bool>,
-}
-
-#[derive(Default, Deserialize)]
-struct AlbumPublic {
- asc: Option<bool>,
- #[serde(rename = "coverID")]
- cover_id: Option<i64>,
- layout: Option<String>,
- caption: Option<String>,
-}
-
-fn unfamiliar(bytes: Option<&[u8]>, known: &[&str], warnings: &mut Vec<String>) {
- if let Some(bytes) = bytes
- && let Ok(Value::Object(fields)) = serde_json::from_slice(bytes)
- {
- for key in fields.keys() {
- if !known.contains(&key.as_str()) {
- warnings.push(key.clone());
- }
- }
- }
-}
diff --git a/rust/crates/photos-export/src/reconcile.rs b/rust/crates/photos-export/src/reconcile.rs
index 51e6e3e2b1..d3ae735467 100644
--- a/rust/crates/photos-export/src/reconcile.rs
+++ b/rust/crates/photos-export/src/reconcile.rs
@@ -929,11 +929,6 @@ impl Run<'_> {
};
after = id;
let source: AlbumSource = serde_json::from_str(&source)?;
- for field in &source.warnings {
- (self.report)(&format!(
- "album {id}: unfamiliar metadata field {field:?}; exporting supported fields"
- ));
- }
if let Err(error) = self.maintain_album(id, &source.name, false, source.metadata) {
self.block(id, None, error)?;
}
diff --git a/rust/crates/photos-export/src/store.rs b/rust/crates/photos-export/src/store.rs
index 975c32a4ec..cab105eff1 100644
--- a/rust/crates/photos-export/src/store.rs
+++ b/rust/crates/photos-export/src/store.rs
@@ -212,7 +212,6 @@ pub struct AlbumSource {
pub shared: Option<bool>,
pub hidden: Option<bool>,
pub metadata: Value,
- pub warnings: Vec<String>,
}
#[derive(Clone)]
@@ -588,7 +587,6 @@ impl Store {
shared: None,
hidden: None,
metadata: Value::Null,
- warnings: Vec::new(),
});
source.shared = Some(shared);
source.hidden = hidden.or(source.hidden);
@@ -612,14 +610,13 @@ impl Store {
shared: None,
hidden: None,
metadata: Value::Null,
- warnings: Vec::new(),
});
source.shared = source.shared.or(shared);
- source.hidden = source.hidden.or_else(|| {
- metadata["ente"]["visibility"]
- .as_str()
- .map(|visibility| visibility == "hidden")
- });
+ if metadata.is_object() {
+ let (shared, hidden) = crate::metadata::album_categories(&metadata)?;
+ source.shared = source.shared.or(shared);
+ source.hidden = source.hidden.or(Some(hidden));
+ }
self.save_album_source(id, &source)
}
@@ -629,13 +626,12 @@ impl Store {
documents: &Documents,
user_id: i64,
) -> Result<()> {
- let (metadata, warnings) = crate::metadata::album(album, documents, user_id)?;
+ let metadata = crate::metadata::album(album, documents)?;
let source = AlbumSource {
name: album.name.clone(),
shared: Some(album.owner_id != user_id),
hidden: Some(matches!(album.visibility, Visibility::Hidden)),
metadata,
- warnings,
};
self.save_album_source(album.id, &source)
}