π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Add shared and hidden album export filters
Manav Rathi6983fbfee92a
diff --git a/rust/apps/cli-next/src/args.rs b/rust/apps/cli-next/src/args.rsindex b14da1b13a..6e1977a771 100644--- a/rust/apps/cli-next/src/args.rs+++ b/rust/apps/cli-next/src/args.rs@@ -315,7 +315,20 @@ pub struct PhotosExportArgs {pub exclude_album: Vec<String>,#[arg(long,- help = "Take over an old CLI/Desktop export or reconnect an existing one"+ help = "Exclude albums owned by other accounts",+ long_help = "Exclude albums owned by other accounts, including albums explicitly included with --album. Albums you own remain included when shared with others. Excluded output is left alone. Removed albums use their last known category; an unknown category is reported as incomplete. Applies only to this invocation."+ )]+ pub exclude_shared: bool,+ #[arg(+ long,+ help = "Exclude hidden albums, including the Hidden album",+ long_help = "Exclude albums hidden for this account, including the Hidden album and albums explicitly included with --album. Archived albums and individually hidden files in included albums remain included. Excluded output is left alone. Removed albums use their last known category; an unknown category is reported as incomplete. Applies only to this invocation."+ )]+ pub exclude_hidden: bool,+ #[arg(+ long,+ help = "Take over an old CLI/Desktop export or reconnect an existing one",+ long_help = "Take over an old CLI/Desktop export or reconnect an existing one. Taking over an old export requires filters to select exactly its recorded albums. Reconnecting a native export associates the whole export, then applies filters to this invocation.")]pub adopt: bool,#[arg(diff --git a/rust/apps/cli-next/src/photos/export/mod.rs b/rust/apps/cli-next/src/photos/export/mod.rsindex fab819d21f..5dc0a85b5e 100644--- a/rust/apps/cli-next/src/photos/export/mod.rs+++ b/rust/apps/cli-next/src/photos/export/mod.rs@@ -41,6 +41,8 @@ async fn run_locked(args: PhotosExportArgs, selected: Option<&str>, as_json: booente_photos_export::Options {albums: args.album,exclude_albums: args.exclude_album,+ exclude_shared: args.exclude_shared,+ exclude_hidden: args.exclude_hidden,adopt: args.adopt,hardlinks: args.hardlinks,jobs: args.jobs,diff --git a/rust/apps/cli-next/tests/cli.rs b/rust/apps/cli-next/tests/cli.rsindex 5101defc4e..3c7f5e44c8 100644--- a/rust/apps/cli-next/tests/cli.rs+++ b/rust/apps/cli-next/tests/cli.rs@@ -33,6 +33,8 @@ fn completion_prints_scripts_without_storage() {"account","paste","hardlinks",+ "exclude-shared",+ "exclude-hidden","completion","admin","update-subscription",@@ -66,6 +68,8 @@ fn reads_do_not_initialize_storage() {assert!(help.contains(&format!("=== {binary} photos file download ===")));assert!(help.contains(&format!("=== {binary} vault key generate ===")));assert!(help.contains("--hardlinks"));+ assert!(help.contains("--exclude-shared"));+ assert!(help.contains("--exclude-hidden"));assert!(help.contains("Linked files share contents, timestamps and permissions."));assert!(help.contains(&format!("=== {binary} paste create ===")));assert!(help.contains(&format!("=== {binary} paste consume ===")));diff --git a/rust/apps/cli-next/tests/support/export.rs b/rust/apps/cli-next/tests/support/export.rsindex 99aef6f422..2126ab0b9c 100644--- a/rust/apps/cli-next/tests/support/export.rs+++ b/rust/apps/cli-next/tests/support/export.rs@@ -600,11 +600,16 @@ fn export_preserves_album_types_visibility_and_settings() -> TestResult {let (file, file_key) = upload_fixture(origin, &account, album, &key, name.as_bytes(), metadata("original.jpg", name.as_bytes())).await;set_album_metadata(origin, &account, album, &key, "magic-metadata", json!({"visibility":visibility,"subType":subtype,"order":7})).await;set_album_metadata(origin, &account, album, &key, "public-magic-metadata", json!({"asc":true,"coverID":file,"layout":"trip","caption":"album description"})).await;- let private = blob::encrypt_json(&json!({"visibility":visibility}), &file_key)?;+ let file_visibility = if name == "Quick link" { 2 } else { visibility };+ let private = blob::encrypt_json(&json!({"visibility":file_visibility}), &file_key)?;request(origin, &account, reqwest::Method::PUT, "/files/magic-metadata", json!({"metadataList":[{"id":file,"magicMetadata":{"version":1,"count":1,"data":b64::encode(&private.encrypted_data),"header":b64::encode(private.decryption_header.as_bytes())}}]})).await;}let destination = tempfile::tempdir()?;let root = destination.path().join("photos");+ assert_eq!(export(&home, &root, &["--exclude-hidden"])["copies"]["completed"], 4);+ assert!(!root.join("Private").exists());+ assert!(!root.join("Hidden").exists());+ assert_eq!(read_json(&root.join("Quick link/metadata/original.jpg.json"))["ente"]["visibility"], "hidden");let result = export(&home, &root, &[]);assert_eq!(result["copies"]["completed"], cases.len());for (name, _, kind, file_visibility, _, visibility) in cases {@@ -619,7 +624,7 @@ fn export_preserves_album_types_visibility_and_settings() -> TestResult {assert_eq!(album["ente"]["description"], "album description");assert_eq!(album["ente"]["layout"], "trip");assert_eq!(album["ente"]["coverFileID"], file["ente"]["fileID"]);- assert_eq!(file["ente"]["visibility"], ["visible", "archived", "hidden"][file_visibility]);+ assert_eq!(file["ente"]["visibility"], if name == "Quick link" { "hidden" } else { ["visible", "archived", "hidden"][file_visibility] });}Ok(())})diff --git a/rust/apps/cli-next/tests/support/export_adoption.rs b/rust/apps/cli-next/tests/support/export_adoption.rsindex 2dba95f730..592cc8678c 100644--- a/rust/apps/cli-next/tests/support/export_adoption.rs+++ b/rust/apps/cli-next/tests/support/export_adoption.rs@@ -25,6 +25,8 @@ fn new_engine(server: &mockito::ServerGuard, producer: &str) -> (Engine, BTreeMauser: USER,key: Key::generate(),albums,+ categories: BTreeMap::new(),+ unreadable_album: None,files: BTreeMap::new(),failed_album: None,},@@ -329,6 +331,26 @@ fn adoption_preflight_refuses_corrupt_media_scope_and_metadata_blockers_without_.contains("exactly"));assert_eq!(snapshot(&root), before);+ engine.catalog.categories.insert(FAMILY, (true, true));+ for unreadable in [None, Some(FAMILY)] {+ engine.catalog.unreadable_album = unreadable;+ for shared in [false, true] {+ let mut options = engine.options(true, &[]);+ options.exclude_shared = shared;+ options.exclude_hidden = !shared;+ assert!(+ engine+ .prepare(options)+ .err()+ .unwrap()+ .to_string()+ .contains("exactly")+ );+ assert_eq!(snapshot(&root), before);+ }+ }+ engine.catalog.unreadable_album = None;+ engine.catalog.categories.clear();fs::write(root.join("Family/Photo.jpg"), b"local edit").unwrap();fs::write(root.join("Travel/Photo.jpg"), b"second edit").unwrap();let before = snapshot(&root);diff --git a/rust/apps/cli-next/tests/support/export_engine.rs b/rust/apps/cli-next/tests/support/export_engine.rsindex 042dd36f5b..7b72cb6317 100644--- a/rust/apps/cli-next/tests/support/export_engine.rs+++ b/rust/apps/cli-next/tests/support/export_engine.rs@@ -10,6 +10,8 @@ pub(super) struct Catalog {pub user: i64,pub key: Key,pub albums: BTreeMap<i64, String>,+ pub categories: BTreeMap<i64, (bool, bool)>,+ pub unreadable_album: Option<i64>,pub files: BTreeMap<(i64, i64), Value>,pub failed_album: Option<i64>,}@@ -17,6 +19,7 @@ pub(super) struct Catalog {impl Catalog {fn album(&self, id: i64) -> CollectionEntry {let name = self.albums[&id].clone();+ let (shared, hidden) = self.categories.get(&id).copied().unwrap_or_default();CollectionEntry {id,name: Some(name.clone()),@@ -27,8 +30,12 @@ impl Catalog {key: Key::from_bytes(*self.key.as_bytes()),name,kind: collections::Kind::Album,- visibility: collections::Visibility::Visible,- owner_id: self.user,+ visibility: if hidden {+ collections::Visibility::Hidden+ } else {+ collections::Visibility::Visible+ },+ owner_id: self.user + i64::from(shared),updated_at_micros: 100,},Documents {@@ -62,7 +69,13 @@ impl Source for Catalog {.keys().copied().filter(|id| *id > after)- .map(|id| self.album(id))+ .map(|id| {+ let mut entry = self.album(id);+ if self.unreadable_album == Some(id) {+ entry.value = Err(anyhow::anyhow!("fixture unreadable album"));+ }+ entry+ }).collect())}async fn refresh_album(&mut self, _: &Session, id: i64) -> Result<CollectionEntry> {@@ -124,6 +137,8 @@ impl Engine {Options {albums: albums.iter().map(|s| (*s).into()).collect(),exclude_albums: Vec::new(),+ exclude_shared: false,+ exclude_hidden: false,adopt,hardlinks: self.hardlinks,jobs: std::num::NonZeroUsize::new(1).unwrap(),diff --git a/rust/apps/cli-next/tests/support/export_media.rs b/rust/apps/cli-next/tests/support/export_media.rsindex 93b3de49f5..4e3c9bb82d 100644--- a/rust/apps/cli-next/tests/support/export_media.rs+++ b/rust/apps/cli-next/tests/support/export_media.rs@@ -10,6 +10,8 @@ fn new_engine(server: &mockito::ServerGuard, key: Key, record: Value) -> Engineuser: 9007199254740993,key,albums: BTreeMap::from([(1, "First".into()), (2, "Second".into())]),+ categories: BTreeMap::new(),+ unreadable_album: None,files: BTreeMap::from([((1, id), record.clone()), ((2, id), record)]),failed_album: None,},@@ -22,6 +24,183 @@ fn run(engine: &mut Engine, complete: bool) -> Value {result.json}+#[test]+fn engine_category_filters_preserve_history_and_reuse_excluded_originals() {+ let mut server = mockito::Server::new();+ let key = Key::generate();+ let (record, bytes) = original_fixture(&key, b"original", false, true, 10);+ let fetched = download(&mut server, 10, &bytes, 1);+ let mut engine = new_engine(&server, key, record.clone());+ run(&mut engine, true);+ let root = engine.root();+ let first_metadata = fs::read(root.join("First/metadata.json")).unwrap();+ engine.catalog.categories.insert(1, (false, true));+ engine.catalog.failed_album = Some(1);+ fs::remove_file(root.join("Second/Photo.jpg")).unwrap();+ let mut options = engine.options(false, &["First", "Second"]);+ options.exclude_hidden = true;+ options.hardlinks = true;+ let export = engine.prepare(options).unwrap();+ let result = engine.execute(export).unwrap();+ assert!(result.complete);+ assert_eq!(result.expected, 1);+ assert!(+ same_file::is_same_file(root.join("First/Photo.jpg"), root.join("Second/Photo.jpg"))+ .unwrap()+ );+ assert_eq!(+ fs::read(root.join("First/metadata.json")).unwrap(),+ first_metadata+ );+ let mut options = engine.options(false, &["First", "Second"]);+ options.exclude_hidden = true;+ options.exclude_albums.push("Second".into());+ let export = engine.prepare(options).unwrap();+ assert_eq!(engine.execute(export).unwrap().expected, 0);++ for (id, name, shared) in [(3, "Owned", false), (4, "Incoming", true)] {+ engine.catalog.albums.insert(id, name.into());+ engine.catalog.categories.insert(id, (shared, false));+ engine.catalog.files.insert((id, 10), record.clone());+ }+ assert!(engine.run(false, &["Owned", "Incoming"]).unwrap().complete);+ fs::write(root.join("First/.ente-excluded.part"), b"pending").unwrap();+ engine+ .db()+ .execute(+ "INSERT INTO temporaries VALUES('.ente-excluded.part','active:1',1,10)",+ [],+ )+ .unwrap();+ engine.catalog.albums.clear();+ let mut options = engine.options(false, &[]);+ options.exclude_shared = true;+ options.exclude_hidden = true;+ let export = engine.prepare(options).unwrap();+ let result = engine.execute(export).unwrap();+ assert!(result.complete);+ assert_eq!(result.json["changes"]["retained"], 2);+ for album in ["First", "Incoming", "Trash/Second", "Trash/Owned"] {+ assert_eq!(+ fs::read(root.join(album).join("Photo.jpg")).unwrap(),+ b"original"+ );+ }+ assert_eq!(+ fs::read(root.join("First/.ente-excluded.part")).unwrap(),+ b"pending"+ );+ assert_eq!(run(&mut engine, true)["changes"]["retained"], 2);+ assert!(!root.join("First/.ente-excluded.part").exists());+ fetched.assert();+}++#[test]+fn engine_category_recovery_preserves_unknown_albums_after_database_loss() {+ let mut server = mockito::Server::new();+ let key = Key::generate();+ let (record, bytes) = original_fixture(&key, b"original", false, true, 10);+ let fetched = download(&mut server, 10, &bytes, 1);+ let mut engine = new_engine(&server, key, record.clone());+ engine+ .catalog+ .categories+ .extend([(1, (false, true)), (2, (true, false))]);+ run(&mut engine, true);+ let root = engine.root();+ engine.catalog.albums.remove(&1);+ engine.database = "reconnected.db".into();+ let mut options = engine.options(true, &[]);+ options.exclude_shared = true;+ options.exclude_hidden = true;+ let export = engine.prepare(options).unwrap();+ assert!(engine.execute(export).unwrap().complete);+ assert_eq!(+ count(&engine.db(), "SELECT count(*) FROM albums WHERE retained=0"),+ 2+ );+ engine.catalog.albums.remove(&2);+ let mut options = engine.options(false, &[]);+ options.exclude_shared = true;+ options.exclude_hidden = true;+ let export = engine.prepare(options).unwrap();+ assert!(engine.execute(export).unwrap().complete);++ engine.catalog.albums.insert(3, "Current".into());+ engine.catalog.files.insert((3, 10), record);+ engine.database = "lost-again.db".into();+ let mut options = engine.options(true, &[]);+ options.exclude_shared = true;+ options.exclude_hidden = true;+ let export = engine.prepare(options).unwrap();+ let result = engine.execute(export).unwrap();+ assert!(!result.complete);+ assert_eq!(result.json["failures"], 1);+ assert_eq!(result.completed, 1);+ for album in ["First", "Second", "Current"] {+ assert_eq!(+ fs::read(root.join(album).join("Photo.jpg")).unwrap(),+ b"original"+ );+ }+ let mut options = engine.options(false, &[]);+ options.exclude_shared = true;+ options.exclude_hidden = true;+ options.exclude_albums.push("Second".into());+ let export = engine.prepare(options).unwrap();+ assert!(engine.execute(export).unwrap().complete);+ assert_eq!(run(&mut engine, true)["changes"]["retained"], 2);+ engine.database = "history-only.db".into();+ let mut options = engine.options(true, &[]);+ options.exclude_shared = true;+ options.exclude_hidden = true;+ let export = engine.prepare(options).unwrap();+ assert!(engine.execute(export).unwrap().complete);+ fetched.assert();+}++#[test]+fn engine_category_filters_recheck_recovered_metadata() {+ let mut server = mockito::Server::new();+ let key = Key::generate();+ let (record, bytes) = original_fixture(&key, b"original", false, true, 10);+ let fetched = download(&mut server, 10, &bytes, 1);+ let mut engine = new_engine(&server, key, record);+ engine.catalog.categories.insert(1, (false, true));+ engine.catalog.unreadable_album = Some(1);+ let mut options = engine.options(false, &[]);+ options.exclude_hidden = true;+ let export = engine.prepare(options).unwrap();+ let result = engine.execute(export).unwrap();+ assert!(result.complete);+ assert_eq!(result.expected, 1);+ assert!(!engine.root().join("First").exists());+ engine.catalog.albums.insert(2, "First".into());+ let mut options = engine.options(false, &["First"]);+ options.exclude_hidden = true;+ assert!(+ engine+ .prepare(options)+ .err()+ .unwrap()+ .to_string()+ .contains("ambiguous")+ );+ engine.catalog.albums.insert(2, "Second".into());+ engine.catalog.failed_album = Some(2);+ let mut options = engine.options(false, &[]);+ options.exclude_hidden = true;+ let export = engine.prepare(options).unwrap();+ let result = engine.execute(export).unwrap();+ assert!(!result.complete);+ assert_eq!(result.json["failures"], 1);+ assert_eq!(+ fs::read(engine.root().join("Second/Photo.jpg")).unwrap(),+ b"original"+ );+ fetched.assert();+}+#[test]fn engine_preserves_local_media_and_restores_missing_copies() {let mut server = mockito::Server::new();diff --git a/rust/apps/cli-next/tests/support/museum.rs b/rust/apps/cli-next/tests/support/museum.rsindex 67d10592fa..3d7ad36bad 100644--- a/rust/apps/cli-next/tests/support/museum.rs+++ b/rust/apps/cli-next/tests/support/museum.rs@@ -445,6 +445,33 @@ async fn exercise(origin: String) -> TestResult {let shared_file = home.json(&["photos", "file", "view", &file_id]);assert_eq!(shared_file["ownerId"], alice_id);let shared_export = output_dir.path().join("shared-export");+ for flag in ["--exclude-shared", "--exclude-hidden"] {+ let result = home.json(&[+ "photos",+ "export",+ shared_export.to_str().unwrap(),+ "--album",+ "Monsoon π§",+ flag,+ ]);+ assert_eq!(result["copies"]["expected"], 0);+ assert!(!shared_export.join("Monsoon π§").exists());+ }+ let owned_export = output_dir.path().join("owned-export");+ assert_eq!(+ home.json(&[+ "photos",+ "export",+ owned_export.to_str().unwrap(),+ "--account",+ "work",+ "--album",+ "Monsoon π§",+ "--exclude-shared",+ "--exclude-hidden"+ ])["copies"]["completed"],+ 1+ );home.json(&["photos","export",@@ -503,6 +530,14 @@ async fn exercise(origin: String) -> TestResult {.unwrap().error_for_status().unwrap();+ for flag in ["--exclude-shared", "--exclude-hidden"] {+ let result = adopted.json(&["photos", "export", shared_export.to_str().unwrap(), flag]);+ assert_eq!(result["changes"]["retained"], 0);+ assert_eq!(+ fs::read(shared_export.join("Monsoon π§/original.jpg")).unwrap(),+ original+ );+ }adopted.json(&["photos","export",diff --git a/rust/crates/photos-export/src/legacy/mod.rs b/rust/crates/photos-export/src/legacy/mod.rsindex f20a7d8a31..0735a35d54 100644--- a/rust/crates/photos-export/src/legacy/mod.rs+++ b/rust/crates/photos-export/src/legacy/mod.rs@@ -62,10 +62,18 @@ impl Failures {}}-pub fn scope(store: &Store, options: &Options, index: Option<&Index>) -> Result<()> {- let explicit = !options.albums.is_empty() || !options.exclude_albums.is_empty();+pub fn scope(+ store: &Store,+ options: &Options,+ index: Option<&Index>,+ report: &(dyn Fn(&str) + Sync),+) -> Result<()> {+ let explicit = !options.albums.is_empty()+ || !options.exclude_albums.is_empty()+ || options.exclude_shared+ || options.exclude_hidden;if explicit {- crate::select(store, options)?;+ crate::select(store, options, index, report)?;}let (db, table) = match index {Some(index) => (&index.db, "adoption_album"),diff --git a/rust/crates/photos-export/src/lib.rs b/rust/crates/photos-export/src/lib.rsindex c78f8d6223..71fab924bc 100644--- a/rust/crates/photos-export/src/lib.rs+++ b/rust/crates/photos-export/src/lib.rs@@ -42,6 +42,8 @@ struct Cancelled;pub struct Options {pub albums: Vec<String>,pub exclude_albums: Vec<String>,+ pub exclude_shared: bool,+ pub exclude_hidden: bool,pub adopt: bool,pub hardlinks: bool,pub jobs: NonZeroUsize,@@ -180,6 +182,37 @@ impl Export {)?;}}+ let mut albums = store.db.prepare("SELECT a.id,a.name,j.value FROM albums a JOIN json_records j ON j.owner=a.key AND j.role='' LEFT JOIN album_sources s ON s.id=a.id WHERE json_extract(s.record,'$.hidden') IS NULL ORDER BY a.retained")?;+ for row in albums.query_map([], |r| {+ Ok((+ r.get::<_, i64>(0)?,+ r.get::<_, String>(1)?,+ r.get::<_, String>(2)?,+ ))+ })? {+ let (id, name, metadata) = row?;+ store.recover_album(id, &name, serde_json::from_str(&metadata)?)?;+ }+ drop(albums);+ if let Some(index) = &index {+ let mut albums = index.db.prepare("SELECT id,json_extract(record,'$.album.name'),json_extract(record,'$.json.value') FROM adoption_album ORDER BY retained")?;+ for row in albums.query_map([], |r| {+ Ok((+ r.get::<_, i64>(0)?,+ r.get::<_, String>(1)?,+ r.get::<_, Option<String>>(2)?,+ ))+ })? {+ let (id, name, metadata) = row?;+ if let Some(metadata) = metadata {+ store.recover_album(id, &name, serde_json::from_str(&metadata)?)?;+ }+ store.db.execute(+ "INSERT OR IGNORE INTO desired_albums(id,name) VALUES(?1,?2)",+ params![id, name],+ )?;+ }+ }report("Refreshing album metadata.");tokio::select! {result=source.refresh_catalog(session)=>result?,@@ -194,7 +227,7 @@ impl Export {}for entry in entries {after = entry.id;- store.desired_album(&entry)?;+ store.desired_album(&entry, session.user_id)?;}}store@@ -215,23 +248,12 @@ impl Export {"INSERT OR IGNORE INTO desired_albums(id,name) SELECT album,name FROM pending WHERE file IS NULL AND retained=0",[],)?;- if let Some(index) = &index {- let mut albums = index.db.prepare("SELECT id,json_extract(record,'$.album.name') FROM adoption_album ORDER BY retained")?;- for row in- albums.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)))?- {- let (id, name) = row?;- store.db.execute(- "INSERT OR IGNORE INTO desired_albums(id,name) VALUES(?1,?2)",- params![id, name],- )?;- }- }+ store.db.execute("UPDATE desired_albums SET shared=(SELECT json_extract(record,'$.shared') FROM album_sources WHERE id=desired_albums.id),hidden=(SELECT json_extract(record,'$.hidden') FROM album_sources WHERE id=desired_albums.id) WHERE present=0", [])?;if self.adopting {- legacy::scope(&store, &self.options, index.as_ref())?;+ legacy::scope(&store, &self.options, index.as_ref(), report)?;} else {failures.finish()?;- select(&store, &self.options)?;+ select(&store, &self.options, index.as_ref(), report)?;}if !self.adopting {if self.root_lock.is_none() {@@ -244,12 +266,13 @@ impl Export {}}let refresh_cancel = cancel.clone();- let refreshed = tokio::select! {- result=refresh(&store,source,session,&refresh_cancel,report)=>result,+ let mut refreshed = tokio::select! {+ result=refresh(&store,source,session,&self.options,&refresh_cancel,report)=>result,_=cancel.changed()=>Err(Cancelled.into()),};if self.adopting {refreshed?;+ legacy::scope(&store, &self.options, index.as_ref(), report)?;if let Some(index) = &index {report("Verifying every existing active legacy original before adoption.");legacy::Preflight {@@ -269,12 +292,14 @@ impl Export {legacy::move_leftovers(&self.destination, index, cancel)?;adopt::accept(index, &store, false)?;}- self.prepared = Some((store, Ok(())));- return Ok(());- }- if let Some(index) = &index {+ refreshed = Ok(());+ } else if let Some(index) = &index {adopt::accept(index, &store, true)?;}+ let refreshed = refreshed.and_then(|()| {+ store.db.execute("DELETE FROM album_sources WHERE id NOT IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM albums WHERE albums.id=album_sources.id) AND NOT EXISTS(SELECT 1 FROM pending WHERE album=album_sources.id)", [])?;+ Ok(())+ });self.prepared = Some((store, refreshed));Ok(())}@@ -407,7 +432,12 @@ fn initialize(destination: &Path, root: &Root, legacy: bool) -> Result<RootLock>Ok(lock)}-fn select(store: &Store, options: &Options) -> Result<()> {+fn select(+ store: &Store,+ options: &Options,+ index: Option<&adopt::Index>,+ report: &(dyn Fn(&str) + Sync),+) -> Result<()> {if options.albums.is_empty() {store.db@@ -443,6 +473,42 @@ fn select(store: &Store, options: &Options) -> Result<()> {)?;}}+ store.db.execute(+ "UPDATE desired_albums SET selected=0 WHERE (?1 AND shared=1) OR (?2 AND hidden=1)",+ params![options.exclude_shared, options.exclude_hidden],+ )?;+ let mut unknown = store.db.prepare("SELECT id,CASE WHEN ?1 AND shared IS NULL AND ?2 AND hidden IS NULL THEN '--exclude-shared and --exclude-hidden' WHEN ?1 AND shared IS NULL THEN '--exclude-shared' ELSE '--exclude-hidden' END FROM desired_albums WHERE selected=1 AND present=0 AND ((?1 AND shared IS NULL) OR (?2 AND hidden IS NULL))")?;+ for row in unknown.query_map(+ params![options.exclude_shared, options.exclude_hidden],+ |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)),+ )? {+ let (id, flags) = row?;+ store+ .db+ .execute("UPDATE desired_albums SET selected=0 WHERE id=?1", [id])?;+ let work: bool = store.db.query_row("SELECT EXISTS(SELECT 1 FROM albums WHERE id=?1 AND retained=0) OR EXISTS(SELECT 1 FROM pending WHERE album=?1) OR EXISTS(SELECT 1 FROM temporaries WHERE album=?1)", [id], |r| r.get(0))?;+ let importing = index+ .map(|index| {+ index.db.query_row(+ "SELECT EXISTS(SELECT 1 FROM adoption_album WHERE id=?1 AND retained=0)",+ [id],+ |r| r.get::<_, bool>(0),+ )+ })+ .transpose()?+ .unwrap_or(false);+ let unit = format!("album:{id}");+ if (work || importing)+ && store+ .db+ .execute("INSERT OR IGNORE INTO outcomes VALUES(?1,0)", [&unit])?+ != 0+ {+ report(&format!(+ "{unit}: last known category is unavailable; use --exclude-album {id} or remove {flags}"+ ));+ }+ }Ok(())}@@ -450,6 +516,7 @@ async fn refresh(store: &Store,source: &mut impl Source,session: &ente_core::Session,+ options: &Options,cancel: &watch::Receiver<bool>,report: &(dyn Fn(&str) + Sync),) -> Result<()> {@@ -471,7 +538,7 @@ async fn refresh(|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),).optional()?;- let Some((id, selected, favorites)) = next else {+ let Some((id, mut selected, favorites)) = next else {break;};after = id;@@ -480,8 +547,18 @@ async fn refresh(favorites_known = false;}let result = refreshed.and_then(|entry| {- store.desired_album(&entry)?;+ store.desired_album(&entry, session.user_id)?;let (album, documents) = entry.value?;+ selected &= !(options.exclude_shared && album.owner_id != session.user_id+ || options.exclude_hidden+ && matches!(+ album.visibility,+ ente_photos::collections::Visibility::Hidden+ ));+ store.db.execute(+ "UPDATE desired_albums SET selected=?1 WHERE id=?2",+ params![selected, id],+ )?;if selected {store.snapshot_album(&album, &documents, session.user_id)?;}@@ -600,18 +677,10 @@ async fn refresh(}transaction.commit()?;}- let transaction = store.db.unchecked_transaction()?;store.db.execute("DELETE FROM sources WHERE file NOT IN (SELECT file FROM chosen_files)",[],)?;- store- .db- .execute(- "DELETE FROM album_sources WHERE id NOT IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM albums WHERE albums.id=album_sources.id) AND NOT EXISTS(SELECT 1 FROM pending WHERE album=album_sources.id)",- [],- )?;- transaction.commit()?;let mut query = store.db.prepare("SELECT album,file,failure FROM desired_files WHERE failure IS NOT NULL")?;diff --git a/rust/crates/photos-export/src/store.rs b/rust/crates/photos-export/src/store.rsindex 0ca3350b41..fdacc9565b 100644--- a/rust/crates/photos-export/src/store.rs+++ b/rust/crates/photos-export/src/store.rs@@ -5,7 +5,10 @@ use std::{};use anyhow::{Context, Result, ensure};-use ente_photos::{collections::Collection, source::Documents};+use ente_photos::{+ collections::{Collection, Visibility},+ source::Documents,+};use rusqlite::{Connection, OptionalExtension, Row, TransactionBehavior, params};use serde::{Deserialize, Serialize, de::DeserializeOwned};use serde_json::Value;@@ -206,6 +209,8 @@ impl JsonRecord {#[derive(Serialize, Deserialize)]pub struct AlbumSource {pub name: String,+ pub shared: Option<bool>,+ pub hidden: Option<bool>,pub metadata: Value,pub warnings: Vec<String>,}@@ -562,14 +567,62 @@ impl Store {Ok(())}- pub fn desired_album(&self, entry: &CollectionEntry) -> Result<()> {+ pub fn desired_album(&self, entry: &CollectionEntry, user_id: i64) -> Result<()> {+ let shared = entry+ .value+ .as_ref()+ .ok()+ .map(|(album, _)| album.owner_id != user_id);+ let hidden = entry+ .value+ .as_ref()+ .ok()+ .map(|(album, _)| matches!(album.visibility, Visibility::Hidden));self.db.execute(- "INSERT INTO desired_albums(id,name,present,favorites) VALUES(?1,?2,1,?3) ON CONFLICT(id) DO UPDATE SET name=excluded.name,present=1,favorites=excluded.favorites",- params![entry.id, entry.name, entry.favorites],+ "INSERT INTO desired_albums(id,name,present,favorites,shared,hidden) VALUES(?1,?2,1,?3,?4,?5) ON CONFLICT(id) DO UPDATE SET name=excluded.name,present=1,favorites=excluded.favorites,shared=excluded.shared,hidden=excluded.hidden",+ params![entry.id, entry.name, entry.favorites, shared, hidden],)?;+ if let Ok((album, _)) = &entry.value {+ let mut source: AlbumSource = json(+ &self.db,+ "SELECT record FROM album_sources WHERE id=?1",+ [entry.id],+ )?+ .unwrap_or(AlbumSource {+ name: album.name.clone(),+ shared: None,+ hidden: None,+ metadata: Value::Null,+ warnings: Vec::new(),+ });+ source.shared = shared;+ source.hidden = hidden;+ self.save_album_source(entry.id, &source)?;+ }Ok(())}+ pub fn recover_album(&self, id: i64, name: &str, metadata: Value) -> Result<()> {+ let mut source: AlbumSource = json(+ &self.db,+ "SELECT record FROM album_sources WHERE id=?1",+ [id],+ )?+ .unwrap_or(AlbumSource {+ name: name.to_owned(),+ shared: None,+ hidden: None,+ metadata: Value::Null,+ warnings: Vec::new(),+ });+ source.hidden = source.hidden.or_else(|| {+ metadata["ente"]["visibility"]+ .as_str()+ .map(|visibility| visibility == "hidden")+ });+ self.save_album_source(id, &source)+ }+pub fn snapshot_album(&self,album: &Collection,@@ -579,12 +632,18 @@ impl Store {let (metadata, warnings) = crate::metadata::album(album, documents, user_id)?;let source = AlbumSource {name: album.name.clone(),+ shared: Some(album.owner_id != user_id),+ hidden: Some(matches!(album.visibility, Visibility::Hidden)),metadata,warnings,};+ self.save_album_source(album.id, &source)+ }++ fn save_album_source(&self, id: i64, source: &AlbumSource) -> Result<()> {self.db.execute("INSERT INTO album_sources VALUES(?1,?2) ON CONFLICT(id) DO UPDATE SET record=excluded.record WHERE album_sources.record<>excluded.record",- params![album.id, serde_json::to_string(&source)?],+ params![id, serde_json::to_string(source)?],)?;Ok(())}@@ -627,7 +686,7 @@ CREATE TABLE sources (file INTEGER PRIMARY KEY, record TEXT NOT NULL);";const RUN_SCHEMA: &str = "-CREATE TEMP TABLE desired_albums (id INTEGER PRIMARY KEY, name TEXT, present INTEGER NOT NULL DEFAULT 0, favorites INTEGER NOT NULL DEFAULT 0, selected INTEGER NOT NULL DEFAULT 0, ready INTEGER NOT NULL DEFAULT 0, failure TEXT);+CREATE TEMP TABLE desired_albums (id INTEGER PRIMARY KEY, name TEXT, present INTEGER NOT NULL DEFAULT 0, favorites INTEGER NOT NULL DEFAULT 0, shared INTEGER, hidden INTEGER, selected INTEGER NOT NULL DEFAULT 0, ready INTEGER NOT NULL DEFAULT 0, failure TEXT);CREATE TEMP TABLE desired_files (album INTEGER NOT NULL, file INTEGER NOT NULL, favorited INTEGER, completed INTEGER NOT NULL DEFAULT 0, failure TEXT, claimed INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(album,file));CREATE INDEX desired_file_id ON desired_files(file,album);CREATE INDEX eligible_files ON desired_files(file) WHERE failure IS NULL AND claimed=0;