π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[photos][rust] Protect hidden uploads during duplicate cleanup (#14320)
Aswin Asokcd9ac88b2370
diff --git a/docs/docs/cli/photos.md b/docs/docs/cli/photos.mdindex 9576c09d7f..88ebc56a26 100644--- a/docs/docs/cli/photos.md+++ b/docs/docs/cli/photos.md@@ -126,7 +126,7 @@ ente photos duplicate remove --all --dry-run --jsonGroup IDs are the smallest file ID in each group. Removal refreshes metadata first; a group ID that no longer exists stops the command. `--dry-run` previews without changes. `--yes` approves removal; `--no-input` disables prompts and requires IDs or `--all`, plus `--yes` unless previewing.-Duplicates are matched by stored hashes in non-hidden albums you own. The CLI keeps a copy with a caption first, then one with an edited name or date. It preserves album memberships before moving extra copies to Ente's trash. Files are not permanently deleted. If interrupted, list duplicates again before retrying.+Duplicates are matched by stored hashes in non-hidden albums you own. Files with any hidden album membership are excluded, even if they also belong to a visible album. The CLI keeps a copy with a caption first, then one with an edited name or date. It preserves album memberships before moving extra copies to Ente's trash. Files are not permanently deleted. If interrupted, list duplicates again before retrying.With `--json`, `list` returns an array and `remove` returns one object; see [Read as JSON](#read-as-json) for fields. Previews and progress go to stderr.@@ -183,7 +183,7 @@ Lists return arrays and `view` returns one object. IDs are strings; timestamps a## Read offline-`album list`, `album view`, `file list`, `file view`, `duplicate list` and `place list` work with `--offline` using the metadata an earlier online run synced. An online `album list` syncs albums only; run `file list` online for each album whose files you want to read offline, or `duplicate list` to sync albums eligible for duplicate detection. Places also need the location data that an earlier online `place list`, `--place` or `--country` downloaded. Export, download, duplicate removal and usage need the network.+`album list`, `album view`, `file list`, `file view`, `duplicate list` and `place list` work with `--offline` using the metadata an earlier online run synced. An online `album list` syncs albums only; run `file list` online for each album whose files you want to read offline, or `duplicate list` to sync albums needed for duplicate detection, including hidden memberships. Places also need the location data that an earlier online `place list`, `--place` or `--country` downloaded. Export, download, duplicate removal and usage need the network.## Limitsdiff --git a/rust/apps/cli/src/photos/duplicates.rs b/rust/apps/cli/src/photos/duplicates.rsindex 1b8daf7f9f..9363a5ab20 100644--- a/rust/apps/cli/src/photos/duplicates.rs+++ b/rust/apps/cli/src/photos/duplicates.rs@@ -183,8 +183,7 @@ async fn load(session: &Session, replica: &mut Replica<'_>, offline: bool) -> Relet album = entry.value.with_context(|| {format!("cannot scan duplicates: album {} is unavailable", entry.id)})?;- if album.owner_id == session.user_id && !matches!(album.visibility, Visibility::Hidden)- {+ if album.owner_id == session.user_id || matches!(album.visibility, Visibility::Hidden) {albums.push(album);}}@@ -225,6 +224,7 @@ async fn load(session: &Session, replica: &mut Replica<'_>, offline: bool) -> Re.map(|a| dedup::Collection {id: a.id,owner_id: a.owner_id,+ is_hidden: matches!(a.visibility, Visibility::Hidden),}).collect();let mut groups = Vec::new();diff --git a/rust/apps/cli/tests/cli/duplicates.rs b/rust/apps/cli/tests/cli/duplicates.rsindex cb0a887595..a7e3c2d6a7 100644--- a/rust/apps/cli/tests/cli/duplicates.rs+++ b/rust/apps/cli/tests/cli/duplicates.rs@@ -50,14 +50,23 @@ fn duplicates_list_and_preview_use_owned_visible_memberships_and_shared_retentiounknown,unknown_copy,live,- live_copy+ live_copy,+ fixture(60, &key, "mixed", json!({})),+ fixture(61, &key, "mixed", json!({}))]),false,).create();page(&mut server, 2, 0, json!([caption]), false).create();page(&mut server, 4, 0, json!([first]), false).create();- let hidden_request = page(&mut server, 3, 0, json!([]), false).expect(0).create();+ let hidden_request = page(+ &mut server,+ 3,+ 0,+ json!([fixture(60, &key, "mixed", json!({}))]),+ false,+ )+ .create();let add = server.mock("POST", "/collections/add-files").expect(0)diff --git a/rust/apps/cli/tests/cli/duplicates_museum.rs b/rust/apps/cli/tests/cli/duplicates_museum.rsindex 795662dbfd..da3718ec09 100644--- a/rust/apps/cli/tests/cli/duplicates_museum.rs+++ b/rust/apps/cli/tests/cli/duplicates_museum.rs@@ -119,3 +119,93 @@ fn duplicates_select_remove_and_refresh_against_museum() -> TestResult {Ok(())})}++#[test]+fn duplicates_protect_hidden_memberships_when_listing_and_removing() -> TestResult {+ Museum::run_async(|origin| async move {+ let email = format!("hidden-duplicates-{}@example.org", Uuid::new_v4());+ let owner = create_account(&origin, &email).await;+ let (visible, visible_key) = create_album(&origin, &owner, "Visible", "album").await;+ let (hidden, hidden_key) = create_album(&origin, &owner, "Hidden", "album").await;+ set_album_metadata(+ &origin,+ &owner,+ hidden,+ &hidden_key,+ "magic-metadata",+ json!({"visibility": 2}),+ )+ .await;+ let mut files = Vec::new();+ for _ in 0..3 {+ files.push(+ upload_fixture(+ &origin,+ &owner,+ visible,+ &visible_key,+ b"duplicate",+ json!({+ "title":"photo.jpg", "fileType":0, "creationTime":1, "hash":"same"+ }),+ )+ .await,+ );+ }+ let protected = files[0].0.to_string();+ let home = TestHome::new();+ login(&home, "photos", &email, &["--host", &origin, "--no-input"]);+ assert_eq!(+ home.json(&["photos", "duplicate", "list", "--all"])[0]["files"]+ .as_array()+ .unwrap()+ .len(),+ 3+ );+ export::place(+ &origin,+ &owner,+ hidden,+ &hidden_key,+ files[0].0,+ &files[0].1,+ "add-files",+ )+ .await;+ let groups = home.json(&["photos", "duplicate", "list", "--all"]);+ assert_eq!(groups.as_array().unwrap().len(), 1);+ let candidates = groups[0]["files"].as_array().unwrap();+ assert_eq!(candidates.len(), 2);+ assert!(candidates.iter().all(|file| file["id"] != protected));+ assert_eq!(+ home.json(&["photos", "duplicate", "list", "--all", "--offline"]),+ groups+ );+ let preview = home.json(&["photos", "duplicate", "remove", "--all", "--dry-run"]);+ assert_eq!(preview["trashIds"].as_array().unwrap().len(), 1);+ assert!(+ !preview["trashIds"]+ .as_array()+ .unwrap()+ .contains(&json!(protected))+ );+ let removed = home.json(&["photos", "duplicate", "remove", "--all", "--yes"]);+ assert_eq!(removed["trashIds"], preview["trashIds"]);+ for album in [visible, hidden] {+ let file = home.json(&[+ "photos",+ "file",+ "view",+ &protected,+ "--album",+ &album.to_string(),+ ]);+ assert_eq!(file["id"], protected);+ }+ assert_eq!(+ home.json(&["photos", "duplicate", "list", "--all"]),+ json!([])+ );+ Ok(())+ })+}diff --git a/rust/bindings/wasm/lib/src/photos.rs b/rust/bindings/wasm/lib/src/photos.rsindex 0336418628..1911cf6f41 100644--- a/rust/bindings/wasm/lib/src/photos.rs+++ b/rust/bindings/wasm/lib/src/photos.rs@@ -33,6 +33,8 @@ impl From<Error> for JsValue {struct DedupCollection {id: i64,owner_id: i64,+ #[serde(default)]+ is_hidden: bool,}#[derive(Deserialize, Tsify)]@@ -83,6 +85,7 @@ pub fn find_duplicates(.map(|collection| dedup::Collection {id: collection.id,owner_id: collection.owner_id,+ is_hidden: collection.is_hidden,}).collect();let files: Vec<_> = inputdiff --git a/rust/crates/photos/src/dedup.rs b/rust/crates/photos/src/dedup.rsindex ee2b6e09ba..ad0d46b587 100644--- a/rust/crates/photos/src/dedup.rs+++ b/rust/crates/photos/src/dedup.rs@@ -10,6 +10,7 @@ pub use removal::{pub struct Collection {pub id: i64,pub owner_id: i64,+ pub is_hidden: bool,}pub struct File {@@ -35,14 +36,27 @@ pub struct Group {pub fn find_duplicates(user_id: i64, collections: &[Collection], files: &[File]) -> Vec<Group> {let allowed: HashSet<_> = collections.iter()- .filter(|collection| collection.owner_id == user_id)+ .filter(|collection| collection.owner_id == user_id && !collection.is_hidden).map(|collection| collection.id).collect();+ let hidden_collections: HashSet<_> = collections+ .iter()+ .filter(|collection| collection.is_hidden)+ .map(|collection| collection.id)+ .collect();+ let hidden_file_ids: HashSet<_> = files+ .iter()+ .filter(|file| hidden_collections.contains(&file.collection_id))+ .map(|file| file.id)+ .collect();let mut by_id = HashMap::<i64, (usize, usize)>::new();let mut by_hash = HashMap::<&str, usize>::new();let mut groups = Vec::<Group>::new();for (file_index, file) in files.iter().enumerate() {- if file.owner_id != user_id || !allowed.contains(&file.collection_id) {+ if file.owner_id != user_id+ || !allowed.contains(&file.collection_id)+ || hidden_file_ids.contains(&file.id)+ {continue;}let Some(hash) = file.hash.as_deref().filter(|hash| !hash.is_empty()) else {@@ -165,14 +179,17 @@ mod tests {Collection {id: 11,owner_id: 1,+ is_hidden: false,},Collection {id: 12,owner_id: 1,+ is_hidden: false,},Collection {id: 13,owner_id: 2,+ is_hidden: false,},]}@@ -249,6 +266,39 @@ mod tests {assert_eq!(groups[0].items.len(), 3);}+ #[test]+ fn excludes_entire_upload_with_hidden_membership_in_any_input_order() {+ let mut collections = collections();+ collections.push(Collection {+ id: 14,+ owner_id: 1,+ is_hidden: true,+ });+ let mut hidden = file(1, 14, "", None);+ hidden.hash = None;+ let mut files = vec![+ file(1, 11, "hash", Some(100)),+ file(2, 11, "hash", Some(100)),+ file(3, 12, "hash", Some(100)),+ hidden,+ file(4, 14, "hash", Some(100)),+ ];+ for _ in 0..files.len() {+ let groups = find_duplicates(1, &collections, &files);+ assert_eq!(groups.len(), 1);+ let mut ids: Vec<_> = groups[0]+ .items+ .iter()+ .map(|item| files[item.file_index].id)+ .collect();+ ids.sort();+ assert_eq!(ids, vec![2, 3]);+ files.rotate_left(1);+ }+ files.retain(|file| file.id != 3);+ assert!(find_duplicates(1, &collections, &files).is_empty());+ }+fn duplicate(id: i64, collection_ids: &[i64]) -> Duplicate {Duplicate {file_id: id,@@ -329,7 +379,13 @@ mod tests {#[test]fn detection_and_removal_preserve_every_album_membership_combination() {- let collections: Vec<_> = (11..=13).map(|id| Collection { id, owner_id: 1 }).collect();+ let collections: Vec<_> = (11..=13)+ .map(|id| Collection {+ id,+ owner_id: 1,+ is_hidden: false,+ })+ .collect();for first in 1..8 {for second in 1..8 {for third in 1..8 {diff --git a/rust/crates/photos/tests/dedup_museum_test.rs b/rust/crates/photos/tests/dedup_museum_test.rsindex 0ef44f68cf..99d8a4a22c 100644--- a/rust/crates/photos/tests/dedup_museum_test.rs+++ b/rust/crates/photos/tests/dedup_museum_test.rs@@ -379,6 +379,7 @@ async fn load_groups(.map(|a| dedup::Collection {id: a.id,owner_id: session.user_id,+ is_hidden: false,}).collect();let groups = dedup::find_duplicates(session.user_id, &collections, &candidates);diff --git a/web/apps/photos/src/services/dedup.ts b/web/apps/photos/src/services/dedup.tsindex 549999725d..c068669b93 100644--- a/web/apps/photos/src/services/dedup.ts+++ b/web/apps/photos/src/services/dedup.ts@@ -9,6 +9,8 @@ import {import {createCollectionNameByID,forgetPendingFavoriteFileIDs,+ isHiddenCollection,+ savedAllCollections,savedNormalCollections,} from "ente-new/photos/services/collection";import { savedCollectionFiles } from "ente-new/photos/services/photos-fdb";@@ -31,14 +33,18 @@ export interface DuplicateGroup {export const deduceDuplicates = async () => {const userID = ensureLocalUser().id;- const normalCollections = await savedNormalCollections();+ const collections = await savedAllCollections();+ const normalCollections = collections.filter(+ (c) => !isHiddenCollection(c, userID),+ );const collectionNameByID = createCollectionNameByID(normalCollections);const collectionFiles = await savedCollectionFiles();const { groups } = await findDuplicates({userId: userID,- collections: normalCollections.map(({ id, owner }) => ({- id,- ownerId: owner.id,+ collections: collections.map((collection) => ({+ id: collection.id,+ ownerId: collection.owner.id,+ isHidden: isHiddenCollection(collection, userID),})),files: collectionFiles.map((file) => ({id: file.id,diff --git a/web/apps/photos/tests/dedup.test.ts b/web/apps/photos/tests/dedup.test.tsindex d448a42266..ba5e411938 100644--- a/web/apps/photos/tests/dedup.test.ts+++ b/web/apps/photos/tests/dedup.test.ts@@ -8,6 +8,7 @@ import {const {executeDuplicateRemoval,savedNormalCollections,+ savedAllCollections,savedCollectionFiles,forgetPendingFavoriteFileIDs,session,@@ -15,6 +16,7 @@ const {executeDuplicateRemoval:vi.fn<typeof import("ente-photos-wasm").executeDuplicateRemoval>(),savedNormalCollections: vi.fn(),+ savedAllCollections: vi.fn(),savedCollectionFiles: vi.fn(),forgetPendingFavoriteFileIDs: vi.fn(),session: {},@@ -28,6 +30,8 @@ vi.mock("../src/services/authenticated-session", () => ({}));vi.mock("ente-new/photos/services/collection", () => ({savedNormalCollections,+ savedAllCollections,+ isHiddenCollection: (c: { hidden?: boolean }) => !!c.hidden,forgetPendingFavoriteFileIDs,createCollectionNameByID: (collections: { id: number; name: string }[]) =>new Map(collections.map(({ id, name }) => [id, name])),@@ -56,10 +60,15 @@ describe("deduceDuplicates through Rust WASM", () => {beforeEach(() => {vi.resetAllMocks();executeDuplicateRemoval.mockResolvedValue({ trashIds: [1] });- savedNormalCollections.mockResolvedValue([+ const collections = [{ id: 11, owner: { id: 1 }, name: "B" },{ id: 12, owner: { id: 1 }, name: "A" },{ id: 13, owner: { id: 2 }, name: "Shared" },+ ];+ savedNormalCollections.mockResolvedValue(collections);+ savedAllCollections.mockResolvedValue([+ ...collections,+ { id: 14, owner: { id: 1 }, name: "Hidden", hidden: true },]);});@@ -107,6 +116,27 @@ describe("deduceDuplicates through Rust WASM", () => {]);});+ test("excludes mixed hidden and visible memberships from cleanup", async () => {+ savedCollectionFiles.mockResolvedValue([+ file(1, 11, "hash"),+ file(2, 11, "hash"),+ file(3, 12, "hash"),+ file(1, 14, "hash"),+ file(4, 14, "hash"),+ ]);+ const groups = await deduceDuplicates();+ expect(groups).toHaveLength(1);+ expect(groups[0]!.items.map(({ file }) => file.id).sort()).toEqual([+ 2, 3,+ ]);+ await removeSelectedDuplicateGroups(groups, vi.fn(), vi.fn());+ expect(+ executeDuplicateRemoval.mock.calls[0]![1].groups[0]!.map(+ (f) => f.fileId,+ ).sort(),+ ).toEqual([2, 3]);+ });+test("matches legacy Live Photo hashes despite different ZIP sizes and skips groups without sizes", async () => {const legacy = file(1, 11, undefined, 0);legacy.metadata = {