π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[photos][web][rust] Execute duplicate removal through Rust (#14241)
Aswin Asok9b1decf8b5d4
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex d4b43d6919..3cf5364d9d 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2835,15 +2835,18 @@ dependencies = ["ente-core","ente-exif","ente-location",+ "ente-test-support","ente-vecdb","futures-util","log",+ "mockito","quick-xml 0.42.0","rusqlite","serde","serde_json","tempfile","thiserror 2.0.21",+ "tokio",][[package]]diff --git a/rust/bindings/wasm/lib/Cargo.toml b/rust/bindings/wasm/lib/Cargo.tomlindex a9f0c99d01..0bf5324f52 100644--- a/rust/bindings/wasm/lib/Cargo.toml+++ b/rust/bindings/wasm/lib/Cargo.toml@@ -19,7 +19,7 @@ crypto-seal = ["crypto"]legacy = ["session", "dep:ente-legacy", "dep:wasm-bindgen-futures"]location = ["dep:ente-location", "dep:serde", "dep:serde-wasm-bindgen", "dep:tsify"]locker = ["session", "dep:ente-locker"]-photos = ["dep:ente-photos", "dep:serde", "dep:serde-wasm-bindgen", "dep:tsify"]+photos = ["session", "dep:ente-photos", "dep:serde", "dep:serde-wasm-bindgen", "dep:tsify", "dep:wasm-bindgen-futures"]prelogin = ["accounts", "crypto", "dep:wasm-bindgen-futures"]session = ["dep:ente-accounts",diff --git a/rust/bindings/wasm/lib/src/photos.rs b/rust/bindings/wasm/lib/src/photos.rsindex b653213832..0336418628 100644--- a/rust/bindings/wasm/lib/src/photos.rs+++ b/rust/bindings/wasm/lib/src/photos.rs@@ -4,10 +4,22 @@ use serde_wasm_bindgen as swb;use tsify::Tsify;use wasm_bindgen::prelude::*;+use crate::session::Session;+use ente_core::{+ b64,+ crypto::{self, Key},+};+#[derive(Debug, thiserror::Error)]pub enum Error {#[error(transparent)]Serde(#[from] swb::Error),+ #[error(transparent)]+ Removal(#[from] dedup::RemovalError),+ #[error(transparent)]+ Decode(#[from] b64::DecodeError),+ #[error(transparent)]+ Crypto(#[from] crypto::Error),}impl From<Error> for JsValue {@@ -114,30 +126,40 @@ struct Duplicate {}#[derive(Deserialize, Tsify)]-pub struct PlanDuplicateRemovalInput {- groups: Vec<Vec<Duplicate>>,+struct RemovalCollection {+ id: i64,+ key: String,}-#[derive(Serialize, Tsify)]+#[derive(Deserialize, Tsify)]#[serde(rename_all = "camelCase")]-struct DuplicateAddition {+struct RemovalFile {+ id: i64,collection_id: i64,- file_ids: Vec<i64>,+ key: String,+}++#[derive(Deserialize, Tsify)]+pub struct ExecuteDuplicateRemovalInput {+ groups: Vec<Vec<Duplicate>>,+ collections: Vec<RemovalCollection>,+ files: Vec<RemovalFile>,}#[derive(Serialize, Tsify)]#[serde(rename_all = "camelCase")]-pub struct DuplicateRemovalPlan {- additions: Vec<DuplicateAddition>,+pub struct DuplicateRemovalResult {trash_ids: Vec<i64>,}-#[wasm_bindgen(js_name = photosPlanDuplicateRemoval)]-pub fn plan_duplicate_removal(- input: <PlanDuplicateRemovalInput as Tsify>::JsType,-) -> Result<<DuplicateRemovalPlan as Tsify>::JsType, Error> {- let input = PlanDuplicateRemovalInput::from_js(input)?;- let groups: Vec<_> = input+#[wasm_bindgen(js_name = photosExecuteDuplicateRemoval)]+pub async fn execute_duplicate_removal(+ session: &Session,+ input: <ExecuteDuplicateRemovalInput as Tsify>::JsType,+ on_progress: js_sys::Function,+) -> Result<<DuplicateRemovalResult as Tsify>::JsType, Error> {+ let input = ExecuteDuplicateRemovalInput::from_js(input)?;+ let groups = input.groups.into_iter().map(|group| {@@ -152,19 +174,37 @@ pub fn plan_duplicate_removal(}).collect()})- .collect();- let plan = dedup::plan_removal(&groups);- DuplicateRemovalPlan {- additions: plan- .additions- .into_iter()- .map(|addition| DuplicateAddition {- collection_id: addition.collection_id,- file_ids: addition.file_ids,+ .collect::<Vec<_>>();+ let collections = input+ .collections+ .into_iter()+ .map(|c| {+ Ok(dedup::RemovalCollection {+ id: c.id,+ key: Key::try_from_slice(&b64::decode(&c.key)?)?,})- .collect(),- trash_ids: plan.trash_ids,- }- .into_js()- .map_err(Into::into)+ })+ .collect::<Result<Vec<_>, Error>>()?;+ let files = input+ .files+ .into_iter()+ .map(|f| {+ Ok(dedup::RemovalFile {+ id: f.id,+ collection_id: f.collection_id,+ key: Key::try_from_slice(&b64::decode(&f.key)?)?,+ })+ })+ .collect::<Result<Vec<_>, Error>>()?;+ let trash_ids = dedup::remove_duplicates(session.inner(), &groups, &collections, &files, |p| {+ let _ = on_progress.call2(+ &JsValue::UNDEFINED,+ &JsValue::from_f64(p.completed as f64),+ &JsValue::from_f64(p.total as f64),+ );+ })+ .await?;+ DuplicateRemovalResult { trash_ids }+ .into_js()+ .map_err(Into::into)}diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.tomlindex 3330385dc2..aabd56a426 100644--- a/rust/crates/photos/Cargo.toml+++ b/rust/crates/photos/Cargo.toml@@ -2,6 +2,9 @@name = "ente-photos"edition.workspace = true+[features]+museum = []+[dependencies]ente-collections.workspace = trueente-core.workspace = true@@ -20,7 +23,10 @@ futures-util.workspace = truerusqlite = { workspace = true, default-features = true }[dev-dependencies]+ente-test-support.workspace = true+mockito.workspace = truetempfile.workspace = true+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }[lints]workspace = truediff --git a/rust/crates/photos/src/dedup.rs b/rust/crates/photos/src/dedup.rsindex f79ffbd3dd..ee2b6e09ba 100644--- a/rust/crates/photos/src/dedup.rs+++ b/rust/crates/photos/src/dedup.rs@@ -1,5 +1,12 @@use std::collections::{HashMap, HashSet};+mod removal;++pub use removal::{+ RemovalCollection, RemovalError, RemovalFile, RemovalProgress, execute_removal,+ remove_duplicates,+};+pub struct Collection {pub id: i64,pub owner_id: i64,diff --git a/rust/crates/photos/src/dedup/removal.rs b/rust/crates/photos/src/dedup/removal.rsnew file mode 100644index 0000000000..8a6e40d9a6--- /dev/null+++ b/rust/crates/photos/src/dedup/removal.rs@@ -0,0 +1,170 @@+use std::collections::{HashMap, HashSet};++use ente_core::{+ Session, b64,+ crypto::{Key, secretbox},+ http,+};+use serde::Serialize;++use super::{Duplicate, RemovalPlan, plan_removal};++const REQUEST_BATCH_SIZE: usize = 1000;++pub struct RemovalCollection {+ pub id: i64,+ pub key: Key,+}++pub struct RemovalFile {+ pub id: i64,+ pub collection_id: i64,+ pub key: Key,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct RemovalProgress {+ pub completed: usize,+ pub total: usize,+}++#[derive(Debug, thiserror::Error)]+pub enum RemovalError {+ #[error(transparent)]+ Http(#[from] http::Error),+ #[error("Missing collection {0}")]+ MissingCollection(i64),+ #[error("Missing file {0}")]+ MissingFile(i64),+ #[error("Invalid duplicate removal plan: {0}")]+ InvalidPlan(&'static str),+}++#[derive(Serialize)]+#[serde(rename_all = "camelCase")]+struct WrappedFile {+ id: i64,+ encrypted_key: String,+ key_decryption_nonce: String,+}++#[derive(Serialize)]+struct AddFiles<'a> {+ #[serde(rename = "collectionID")]+ collection_id: i64,+ files: &'a [WrappedFile],+}++#[derive(Serialize)]+struct TrashItem {+ #[serde(rename = "fileID")]+ file_id: i64,+ #[serde(rename = "collectionID")]+ collection_id: i64,+}++#[derive(Serialize)]+struct TrashFiles<'a> {+ items: &'a [TrashItem],+}++pub async fn remove_duplicates(+ session: &Session,+ groups: &[Vec<Duplicate>],+ collections: &[RemovalCollection],+ files: &[RemovalFile],+ on_progress: impl FnMut(RemovalProgress),+) -> Result<Vec<i64>, RemovalError> {+ let plan = plan_removal(groups);+ execute_removal(session, &plan, collections, files, on_progress).await?;+ Ok(plan.trash_ids)+}++pub async fn execute_removal(+ session: &Session,+ plan: &RemovalPlan,+ collections: &[RemovalCollection],+ files: &[RemovalFile],+ mut on_progress: impl FnMut(RemovalProgress),+) -> Result<(), RemovalError> {+ let collections: HashMap<_, _> = collections.iter().map(|c| (c.id, c)).collect();+ let files: HashMap<_, _> = files.iter().map(|f| (f.id, f)).collect();+ let trash_ids: HashSet<_> = plan.trash_ids.iter().copied().collect();+ if trash_ids.len() != plan.trash_ids.len() {+ return Err(RemovalError::InvalidPlan("repeated trash ID"));+ }+ let mut additions = Vec::with_capacity(plan.additions.len());+ for addition in &plan.additions {+ let collection = collections+ .get(&addition.collection_id)+ .ok_or(RemovalError::MissingCollection(addition.collection_id))?;+ let mut addition_files = Vec::with_capacity(addition.file_ids.len());+ for id in &addition.file_ids {+ if trash_ids.contains(id) {+ return Err(RemovalError::InvalidPlan(+ "retained file is also scheduled for trash",+ ));+ }+ let file = files.get(id).ok_or(RemovalError::MissingFile(*id))?;+ addition_files.push(*file);+ }+ additions.push((*collection, addition_files));+ }+ let items = plan+ .trash_ids+ .iter()+ .map(|id| {+ let file = files.get(id).ok_or(RemovalError::MissingFile(*id))?;+ Ok(TrashItem {+ file_id: *id,+ collection_id: file.collection_id,+ })+ })+ .collect::<Result<Vec<_>, RemovalError>>()?;++ let total = additions.len() + usize::from(!items.is_empty());+ let mut completed = 0;+ for (collection, files) in additions {+ for batch in files.chunks(REQUEST_BATCH_SIZE) {+ let wrapped: Vec<_> = batch+ .iter()+ .map(|file| {+ let encrypted = secretbox::encrypt(file.key.as_bytes(), &collection.key);+ WrappedFile {+ id: file.id,+ encrypted_key: b64::encode(&encrypted.encrypted_data),+ key_decryption_nonce: b64::encode(encrypted.nonce.as_bytes()),+ }+ })+ .collect();+ session+ .api+ .post("/collections/add-files")+ .json(&AddFiles {+ collection_id: collection.id,+ files: &wrapped,+ })+ .send()+ .await?+ .error_for_code()+ .await?;+ }+ completed += 1;+ on_progress(RemovalProgress { completed, total });+ }+ for batch in items.chunks(REQUEST_BATCH_SIZE) {+ session+ .api+ .post("/files/trash")+ .json(&TrashFiles { items: batch })+ .send()+ .await?+ .error_for_code()+ .await?;+ }+ if !items.is_empty() {+ completed += 1;+ on_progress(RemovalProgress { completed, total });+ }+ Ok(())+}diff --git a/rust/crates/photos/src/face_clustering/mod.rs b/rust/crates/photos/src/face_clustering/mod.rsindex 1ef63e5b1d..7130ec4600 100644--- a/rust/crates/photos/src/face_clustering/mod.rs+++ b/rust/crates/photos/src/face_clustering/mod.rs@@ -104,6 +104,7 @@ pub(crate) fn new_cluster_id() -> String {format!("{CLUSTER_ID_PREFIX}{suffix}")}+#[cfg(any(test, not(target_arch = "wasm32")))]pub(crate) fn normalized_mean(embeddings: &[Vec<f32>]) -> Option<Vec<f32>> {let (first, rest) = embeddings.split_first()?;let mut sum = first.clone();diff --git a/rust/crates/photos/tests/dedup_museum_test.rs b/rust/crates/photos/tests/dedup_museum_test.rsnew file mode 100644index 0000000000..0ef44f68cf--- /dev/null+++ b/rust/crates/photos/tests/dedup_museum_test.rs@@ -0,0 +1,421 @@+#![cfg(feature = "museum")]++use std::{collections::HashSet, io::Write};++use ente_collections::client;+use ente_core::{+ Session, b64,+ crypto::{Header, Key, SecretKey, blob, hash, secretbox},+ http::{ApiConfig, Auth},+ io::Md5Writer,+};+use ente_photos::dedup::{self, Duplicate, RemovalCollection, RemovalFile, RemovalProgress};+use ente_test_support::{Museum, TestResult, account_fixture};+use serde::Deserialize;+use serde_json::{Value, json};++const ORIGINAL: &[u8] = b"Duplicate removal local Museum encrypted file fixture";++#[test]+fn duplicate_removal_against_museum() -> TestResult {+ Museum::run_async(run)+}++async fn run(endpoint: String) -> TestResult {+ let account = account_fixture::create_account(&endpoint, "dedup-owner").await?;+ let session = open_session(&endpoint, &account)?;+ let albums = vec![+ create_album(&session, "A").await?,+ create_album(&session, "B").await?,+ create_album(&session, "C").await?,+ ];+ let cases = [+ (+ "caption",+ [+ json!({"editedName":"renamed"}),+ json!({"caption":"keep"}),+ json!({}),+ ],+ 1,+ ),+ ("fallback", [json!({}), json!({}), json!({})], 0),+ (+ "rename",+ [json!({}), json!({}), json!({"editedName":"renamed"})],+ 2,+ ),+ ("date", [json!({}), json!({"editedTime":123}), json!({})], 1),+ (+ "caption-tie",+ [+ json!({"caption":"first"}),+ json!({"caption":"second"}),+ json!({}),+ ],+ 0,+ ),+ (+ "edit-tie",+ [+ json!({"editedName":"first"}),+ json!({"editedTime":123}),+ json!({}),+ ],+ 0,+ ),+ (+ "empty-name",+ [+ json!({}),+ json!({"editedName":"","editedTime":123}),+ json!({}),+ ],+ 0,+ ),+ (+ "normalized-zero-caption",+ [json!({}), json!({"caption":"0"}), json!({})],+ 1,+ ),+ ];+ for (name, edits, retained_index) in cases {+ let files = upload_group(&session, &albums, name, &edits).await?;+ let groups = load_groups(&session, &albums, &files).await?;+ let retained = &files[retained_index];+ let expected_trash: Vec<_> = files+ .iter()+ .filter(|f| f.id != retained.id)+ .map(|f| f.id)+ .collect();+ let mut progress = Vec::new();+ let trashed =+ dedup::remove_duplicates(&session, &groups, &albums, &files, |p| progress.push(p))+ .await?;+ assert_eq!(trashed, expected_trash, "{name}");+ assert_eq!(+ progress,+ vec![+ RemovalProgress {+ completed: 1,+ total: 3+ },+ RemovalProgress {+ completed: 2,+ total: 3+ },+ RemovalProgress {+ completed: 3,+ total: 3+ }+ ],+ "{name}"+ );+ let ids: HashSet<_> = files.iter().map(|f| f.id).collect();+ for album in &albums {+ let active = active_files(&session, album.id, &ids).await?;+ assert_eq!(active.len(), 1, "{name}");+ let remote = &active[0];+ assert_eq!(remote.id, retained.id, "{name}");+ let key = remote.open_key(&album.key)?;+ assert_eq!(key.as_bytes(), retained.key.as_bytes(), "{name}");+ assert_eq!(+ public_metadata(remote, &key)?,+ edits[retained_index],+ "{name}"+ );+ let header = Header::try_from_slice(&b64::decode(&remote.file.decryption_header)?)?;+ let plaintext =+ ente_photos::files::download(&session, remote.id, &key, &header, || Ok(Vec::new()))+ .await?;+ assert_eq!(plaintext, ORIGINAL, "{name}");+ }+ assert_eq!(+ trash_ids(&session, &ids).await?,+ expected_trash.into_iter().collect(),+ "{name}"+ );+ }++ let outsider = account_fixture::create_account(&endpoint, "dedup-outsider").await?;+ let outsider_session = open_session(&endpoint, &outsider)?;+ let foreign_album = create_album(&outsider_session, "Private").await?;+ for fail_first in [true, false] {+ let files = upload_group(+ &session,+ &albums,+ "permission-failure",+ &[json!({}), json!({"caption":"keep"}), json!({})],+ )+ .await?;+ let mut groups = load_groups(&session, &albums, &files).await?;+ if fail_first {+ groups[0][0].collection_ids.insert(0, foreign_album.id);+ } else {+ groups[0][0].collection_ids.push(foreign_album.id);+ }+ let mut available: Vec<_> = albums+ .iter()+ .map(|a| RemovalCollection {+ id: a.id,+ key: Key::from_bytes(*a.key.as_bytes()),+ })+ .collect();+ available.push(RemovalCollection {+ id: foreign_album.id,+ key: Key::from_bytes(*foreign_album.key.as_bytes()),+ });+ let mut progress = Vec::new();+ let error =+ dedup::remove_duplicates(&session, &groups, &available, &files, |p| progress.push(p))+ .await+ .err()+ .ok_or("Expected Museum to reject an inaccessible album")?;+ match error {+ dedup::RemovalError::Http(error) => assert_eq!(error.status_code(), Some(404)),+ error => panic!("Expected Museum permission failure, got {error}"),+ }+ assert_eq!(progress.len(), usize::from(!fail_first));+ let ids: HashSet<_> = files.iter().map(|f| f.id).collect();+ assert!(trash_ids(&session, &ids).await?.is_empty());+ for (index, album) in albums.iter().enumerate() {+ let active = active_files(&session, album.id, &ids).await?;+ let active_ids: HashSet<_> = active.iter().map(|f| f.id).collect();+ let mut expected = HashSet::from([files[index].id]);+ if !fail_first && index == 0 {+ expected.insert(files[1].id);+ }+ assert_eq!(active_ids, expected);+ }+ }+ Ok(())+}++fn open_session(endpoint: &str, account: &account_fixture::TestAccount) -> TestResult<Session> {+ Ok(Session::new(+ ApiConfig {+ origin: endpoint.into(),+ client_package: Some("io.ente.photos".into()),+ client_version: None,+ user_agent: Some("ente-photos-dedup-e2e".into()),+ auth: Some(Auth::User(account.auth_token.clone())),+ },+ account.user_id,+ Key::try_from_slice(&account.master_key)?,+ Key::try_from_slice(&account.recovery_key)?,+ SecretKey::try_from_slice(&account.secret_key)?,+ )?)+}++async fn create_album(session: &Session, name: &str) -> TestResult<RemovalCollection> {+ #[derive(Deserialize)]+ struct Created {+ collection: client::Collection,+ }+ let key = Key::generate();+ let wrapped = secretbox::encrypt(key.as_bytes(), &session.master_key);+ let name = secretbox::encrypt(name.as_bytes(), &key);+ let created: Created = session.api.post("/collections").json(&json!({+ "encryptedKey":b64::encode(&wrapped.encrypted_data),"keyDecryptionNonce":b64::encode(wrapped.nonce.as_bytes()),+ "encryptedName":b64::encode(&name.encrypted_data),"nameDecryptionNonce":b64::encode(name.nonce.as_bytes()),"type":"album","attributes":{"version":1}+ })).send().await?.error_for_code().await?.json().await?;+ Ok(RemovalCollection {+ id: created.collection.id,+ key,+ })+}++async fn upload_object(session: &Session, bytes: Vec<u8>) -> TestResult<String> {+ #[derive(Deserialize)]+ #[serde(rename_all = "camelCase")]+ struct Upload {+ url: String,+ object_key: String,+ }+ let mut checksum_writer = Md5Writer::new(std::io::sink());+ checksum_writer.write_all(&bytes)?;+ let checksum = b64::encode(&checksum_writer.finalize().1);+ let upload: Upload = session+ .api+ .post("/files/upload-url")+ .json(&json!({"contentLength":bytes.len(),"contentMD5":checksum}))+ .send()+ .await?+ .error_for_code()+ .await?+ .json()+ .await?;+ session+ .api+ .http()+ .put(&upload.url)+ .header("content-md5", &checksum)+ .header("content-length", &bytes.len().to_string())+ .body(bytes)+ .send()+ .await?+ .error_for_status()?;+ Ok(upload.object_key)+}++async fn upload_group(+ session: &Session,+ albums: &[RemovalCollection],+ name: &str,+ edits: &[Value; 3],+) -> TestResult<Vec<RemovalFile>> {+ let mut files = Vec::new();+ for (index, album) in albums.iter().enumerate() {+ let key = Key::generate();+ let wrapped = secretbox::encrypt(key.as_bytes(), &album.key);+ let file = blob::encrypt(ORIGINAL, &key)?;+ let thumbnail = blob::encrypt(b"thumbnail", &key)?;+ let metadata = blob::encrypt_json(+ &json!({"title":format!("{name}-{index}.jpg"),"fileType":0,"hash":b64::encode(&hash::hash(ORIGINAL,None,None)?),"creationTime":1_700_000_000_000_000_i64,"modificationTime":1_700_000_000_000_000_i64}),+ &key,+ )?;+ let file_object = upload_object(session, file.encrypted_data).await?;+ let thumbnail_object = upload_object(session, thumbnail.encrypted_data).await?;+ let mut request = json!({"collectionID":album.id,"encryptedKey":b64::encode(&wrapped.encrypted_data),"keyDecryptionNonce":b64::encode(wrapped.nonce.as_bytes()),+ "file":{"objectKey":file_object,"decryptionHeader":b64::encode(file.decryption_header.as_bytes())},+ "thumbnail":{"objectKey":thumbnail_object,"decryptionHeader":b64::encode(thumbnail.decryption_header.as_bytes())},+ "metadata":{"encryptedData":b64::encode(&metadata.encrypted_data),"decryptionHeader":b64::encode(metadata.decryption_header.as_bytes())}});+ let count = edits[index]+ .as_object()+ .ok_or("Expected metadata object")?+ .len();+ if count > 0 {+ let encrypted = blob::encrypt_json(&edits[index], &key)?;+ request["pubMagicMetadata"] = json!({"version":1,"count":count,"data":b64::encode(&encrypted.encrypted_data),"header":b64::encode(encrypted.decryption_header.as_bytes())});+ }+ #[derive(Deserialize)]+ struct Uploaded {+ id: i64,+ }+ let uploaded: Uploaded = session+ .api+ .post("/files")+ .json(&request)+ .send()+ .await?+ .error_for_code()+ .await?+ .json()+ .await?;+ files.push(RemovalFile {+ id: uploaded.id,+ collection_id: album.id,+ key,+ });+ }+ Ok(files)+}++fn public_metadata(file: &client::File, key: &Key) -> TestResult<Value> {+ Ok(match &file.pub_magic_metadata {+ Some(metadata) => serde_json::from_slice(&metadata.decrypt(key)?)?,+ None => json!({}),+ })+}++async fn active_files(+ session: &Session,+ collection_id: i64,+ ids: &HashSet<i64>,+) -> TestResult<Vec<client::File>> {+ let page = client::files_diff(session, collection_id, 0).await?;+ assert!(!page.has_more);+ Ok(page+ .files+ .into_iter()+ .filter(|f| ids.contains(&f.id) && !f.is_deleted())+ .collect())+}++async fn load_groups(+ session: &Session,+ albums: &[RemovalCollection],+ files: &[RemovalFile],+) -> TestResult<Vec<Vec<Duplicate>>> {+ let mut candidates = Vec::new();+ let mut duplicates = Vec::new();+ let ids: HashSet<_> = files.iter().map(|f| f.id).collect();+ for album in albums {+ for remote in active_files(session, album.id, &ids).await? {+ let key = remote.open_key(&album.key)?;+ let metadata: Value = serde_json::from_slice(&blob::decrypt(+ &b64::decode(+ remote+ .metadata+ .encrypted_data+ .as_deref()+ .ok_or("Missing metadata")?,+ )?,+ &Header::try_from_slice(&b64::decode(&remote.metadata.decryption_header)?)?,+ &key,+ )?)?;+ let edits = public_metadata(&remote, &key)?;+ candidates.push(dedup::File {+ id: remote.id,+ collection_id: album.id,+ owner_id: remote.owner_id,+ hash: metadata["hash"].as_str().map(String::from),+ size: remote+ .extra+ .get("info")+ .and_then(|info| info["fileSize"].as_u64()),+ });+ duplicates.push(Duplicate {+ file_id: remote.id,+ collection_ids: vec![album.id],+ caption: edits["caption"].as_str().map(String::from),+ edited_name: edits["editedName"].as_str().map(String::from),+ edited_time: edits["editedTime"].as_i64(),+ });+ }+ }+ let collections: Vec<_> = albums+ .iter()+ .map(|a| dedup::Collection {+ id: a.id,+ owner_id: session.user_id,+ })+ .collect();+ let groups = dedup::find_duplicates(session.user_id, &collections, &candidates);+ assert_eq!(groups.len(), 1);+ assert_eq!(groups[0].items.len(), 3);+ Ok(vec![duplicates])+}++async fn trash_ids(session: &Session, ids: &HashSet<i64>) -> TestResult<HashSet<i64>> {+ #[derive(Deserialize)]+ #[serde(rename_all = "camelCase")]+ struct Page {+ diff: Vec<Item>,+ has_more: bool,+ }+ #[derive(Deserialize)]+ #[serde(rename_all = "camelCase")]+ struct Item {+ file: client::File,+ is_deleted: bool,+ is_restored: bool,+ }+ let page: Page = session+ .api+ .get("/trash/v2/diff")+ .query(&[("sinceTime", 0)])+ .send()+ .await?+ .error_for_code()+ .await?+ .json()+ .await?;+ assert!(!page.has_more);+ Ok(page+ .diff+ .into_iter()+ .filter(|t| ids.contains(&t.file.id) && !t.is_deleted && !t.is_restored)+ .map(|t| t.file.id)+ .collect())+}diff --git a/rust/crates/photos/tests/dedup_removal_test.rs b/rust/crates/photos/tests/dedup_removal_test.rsnew file mode 100644index 0000000000..87a7078a78--- /dev/null+++ b/rust/crates/photos/tests/dedup_removal_test.rs@@ -0,0 +1,609 @@+use std::sync::{Arc, Mutex};++use ente_core::{+ Session, b64,+ crypto::{Key, Nonce, SecretKey, secretbox},+ http::{ApiConfig, Auth},+};+use ente_photos::dedup::{+ Addition, RemovalCollection, RemovalError, RemovalFile, RemovalPlan, execute_removal,+};+use mockito::{Matcher, Server};++fn session(origin: String) -> Result<Session, ente_core::http::Error> {+ Session::new(+ ApiConfig {+ origin,+ client_package: Some("io.ente.photos".into()),+ client_version: None,+ user_agent: None,+ auth: Some(Auth::User("test-token".into())),+ },+ 1,+ Key::generate(),+ Key::generate(),+ SecretKey::generate(),+ )+}++fn fixture() -> (RemovalPlan, Vec<RemovalCollection>, Vec<RemovalFile>) {+ (+ RemovalPlan {+ additions: vec![+ Addition {+ collection_id: 11,+ file_ids: vec![1],+ },+ Addition {+ collection_id: 12,+ file_ids: vec![1],+ },+ ],+ trash_ids: vec![2],+ },+ vec![+ RemovalCollection {+ id: 11,+ key: Key::generate(),+ },+ RemovalCollection {+ id: 12,+ key: Key::generate(),+ },+ ],+ vec![+ RemovalFile {+ id: 1,+ collection_id: 11,+ key: Key::generate(),+ },+ RemovalFile {+ id: 2,+ collection_id: 12,+ key: Key::generate(),+ },+ ],+ )+}++#[tokio::test]+async fn authenticates_encrypts_keys_and_finishes_all_additions_before_trash() {+ let mut server = Server::new_async().await;+ let (plan, collections, files) = fixture();+ let events = Arc::new(Mutex::new(Vec::new()));+ let mut mocks = Vec::new();+ for c in &collections {+ let key = Key::from_bytes(*c.key.as_bytes());+ let file_key = *files[0].key.as_bytes();+ let id = c.id;+ let events = events.clone();+ mocks.push(+ server+ .mock("POST", "/collections/add-files")+ .match_header("x-auth-token", "test-token")+ .match_header("x-client-package", "io.ente.photos")+ .match_body(Matcher::PartialJson(+ serde_json::json!({"collectionID": id}),+ ))+ .match_request(move |request| {+ let payload: serde_json::Value =+ serde_json::from_slice(request.body().unwrap()).unwrap();+ let f = &payload["files"][0];+ let plaintext = secretbox::decrypt(+ &b64::decode(f["encryptedKey"].as_str().unwrap()).unwrap(),+ &Nonce::try_from_slice(+ &b64::decode(f["keyDecryptionNonce"].as_str().unwrap()).unwrap(),+ )+ .unwrap(),+ &key,+ )+ .unwrap();+ events.lock().unwrap().push(id);+ f["id"] == 1 && plaintext == file_key+ })+ .with_status(200)+ .create_async()+ .await,+ );+ }+ let trash_events = events.clone();+ let trash = server+ .mock("POST", "/files/trash")+ .match_header("x-auth-token", "test-token")+ .match_body(Matcher::Json(+ serde_json::json!({"items": [{"fileID": 2, "collectionID": 12}]}),+ ))+ .match_request(move |_| *trash_events.lock().unwrap() == vec![11, 12])+ .with_status(200)+ .create_async()+ .await;+ let mut progress = Vec::new();+ execute_removal(+ &session(server.url()).unwrap(),+ &plan,+ &collections,+ &files,+ |p| progress.push((p.completed, p.total)),+ )+ .await+ .unwrap();+ assert_eq!(progress, vec![(1, 3), (2, 3), (3, 3)]);+ for mock in mocks {+ mock.assert_async().await;+ }+ trash.assert_async().await;+}++#[tokio::test]+async fn does_not_trash_when_a_later_album_addition_fails() {+ let mut server = Server::new_async().await;+ let (plan, collections, files) = fixture();+ let first = server+ .mock("POST", "/collections/add-files")+ .match_body(Matcher::PartialJson(+ serde_json::json!({"collectionID": 11}),+ ))+ .with_status(200)+ .create_async()+ .await;+ let second = server+ .mock("POST", "/collections/add-files")+ .match_body(Matcher::PartialJson(+ serde_json::json!({"collectionID": 12}),+ ))+ .with_status(503)+ .create_async()+ .await;+ let trash = server+ .mock("POST", "/files/trash")+ .expect(0)+ .create_async()+ .await;+ let mut progress = Vec::new();+ assert!(+ execute_removal(+ &session(server.url()).unwrap(),+ &plan,+ &collections,+ &files,+ |p| { progress.push(p.completed) }+ )+ .await+ .is_err()+ );+ assert_eq!(progress, vec![1]);+ first.assert_async().await;+ second.assert_async().await;+ trash.assert_async().await;+}++#[tokio::test]+async fn validates_every_file_before_the_first_mutation() {+ let mut server = Server::new_async().await;+ let (mut plan, collections, files) = fixture();+ plan.trash_ids.push(999);+ let addition = server+ .mock("POST", "/collections/add-files")+ .expect(0)+ .create_async()+ .await;+ let mut progress = Vec::new();+ assert!(+ execute_removal(+ &session(server.url()).unwrap(),+ &plan,+ &collections,+ &files,+ |p| { progress.push(p) }+ )+ .await+ .is_err()+ );+ assert!(progress.is_empty());+ addition.assert_async().await;+}++#[tokio::test]+async fn validates_later_additions_before_the_first_mutation() {+ for invalid in ["collection", "file", "trashed file"] {+ let mut server = Server::new_async().await;+ let (mut plan, collections, files) = fixture();+ match invalid {+ "collection" => plan.additions[1].collection_id = 999,+ "file" => plan.additions[1].file_ids.push(999),+ "trashed file" => plan.additions[1].file_ids.push(2),+ _ => unreachable!(),+ }+ let addition = server+ .mock("POST", "/collections/add-files")+ .expect(0)+ .create_async()+ .await;+ let trash = server+ .mock("POST", "/files/trash")+ .expect(0)+ .create_async()+ .await;+ let mut progress = Vec::new();+ let error = execute_removal(+ &session(server.url()).unwrap(),+ &plan,+ &collections,+ &files,+ |p| progress.push(p),+ )+ .await+ .unwrap_err();+ match invalid {+ "collection" => assert!(matches!(error, RemovalError::MissingCollection(999))),+ "file" => assert!(matches!(error, RemovalError::MissingFile(999))),+ "trashed file" => assert!(matches!(error, RemovalError::InvalidPlan(_))),+ _ => unreachable!(),+ }+ assert!(progress.is_empty(), "{invalid}");+ addition.assert_async().await;+ trash.assert_async().await;+ }+}++#[tokio::test]+async fn batches_requests_and_propagates_trash_failure_without_completion() {+ let mut server = Server::new_async().await;+ let collections = vec![RemovalCollection {+ id: 11,+ key: Key::generate(),+ }];+ let files: Vec<_> = (1..=1002)+ .map(|id| RemovalFile {+ id,+ collection_id: 11,+ key: Key::generate(),+ })+ .collect();+ let plan = RemovalPlan {+ additions: vec![],+ trash_ids: (1..=1002).collect(),+ };+ let first = server+ .mock("POST", "/files/trash")+ .match_request(|r| {+ serde_json::from_slice::<serde_json::Value>(r.body().unwrap()).unwrap()["items"]+ .as_array()+ .unwrap()+ .len()+ == 1000+ })+ .with_status(200)+ .create_async()+ .await;+ let second = server+ .mock("POST", "/files/trash")+ .match_request(|r| {+ serde_json::from_slice::<serde_json::Value>(r.body().unwrap()).unwrap()["items"]+ .as_array()+ .unwrap()+ .len()+ == 2+ })+ .with_status(503)+ .create_async()+ .await;+ let mut progress = Vec::new();+ assert!(+ execute_removal(+ &session(server.url()).unwrap(),+ &plan,+ &collections,+ &files,+ |p| { progress.push(p) }+ )+ .await+ .is_err()+ );+ assert!(progress.is_empty());+ first.assert_async().await;+ second.assert_async().await;+}++#[tokio::test]+async fn batches_large_album_additions_before_trashing() {+ let mut server = Server::new_async().await;+ let collections = vec![RemovalCollection {+ id: 11,+ key: Key::generate(),+ }];+ let files: Vec<_> = (1..=1002)+ .map(|id| RemovalFile {+ id,+ collection_id: 11,+ key: Key::generate(),+ })+ .collect();+ let plan = RemovalPlan {+ additions: vec![Addition {+ collection_id: 11,+ file_ids: (1..=1001).collect(),+ }],+ trash_ids: vec![1002],+ };+ let batches = Arc::new(Mutex::new(Vec::new()));+ let mut mocks = Vec::new();+ for (start, size) in [(1, 1000), (1001, 1)] {+ let seen = batches.clone();+ let collection_key = Key::from_bytes(*collections[0].key.as_bytes());+ let expected_keys: Vec<_> = files[(start - 1)..(start - 1 + size)]+ .iter()+ .map(|file| *file.key.as_bytes())+ .collect();+ mocks.push(+ server+ .mock("POST", "/collections/add-files")+ .match_request(move |r| {+ let value: serde_json::Value =+ serde_json::from_slice(r.body().unwrap()).unwrap();+ let batch = value["files"].as_array().unwrap();+ if batch.len() != size {+ return false;+ }+ assert_eq!(value["collectionID"], 11);+ for (index, (file, expected_key)) in+ batch.iter().zip(&expected_keys).enumerate()+ {+ assert_eq!(file["id"], start + index);+ let plaintext = secretbox::decrypt(+ &b64::decode(file["encryptedKey"].as_str().unwrap()).unwrap(),+ &Nonce::try_from_slice(+ &b64::decode(file["keyDecryptionNonce"].as_str().unwrap()).unwrap(),+ )+ .unwrap(),+ &collection_key,+ )+ .unwrap();+ assert_eq!(&plaintext, expected_key);+ }+ seen.lock().unwrap().push(size);+ true+ })+ .with_status(200)+ .create_async()+ .await,+ );+ }+ let seen = batches.clone();+ let trash = server+ .mock("POST", "/files/trash")+ .match_request(move |_| *seen.lock().unwrap() == vec![1000, 1])+ .with_status(200)+ .create_async()+ .await;+ let mut progress = Vec::new();+ execute_removal(+ &session(server.url()).unwrap(),+ &plan,+ &collections,+ &files,+ |p| progress.push((p.completed, p.total)),+ )+ .await+ .unwrap();+ assert_eq!(progress, vec![(1, 2), (2, 2)]);+ for mock in mocks {+ mock.assert_async().await;+ }+ trash.assert_async().await;+}++#[tokio::test]+async fn plans_and_executes_retention_rules_without_a_client_supplied_plan() {+ use ente_photos::dedup::{Duplicate, remove_duplicates};++ let cases = [+ (+ "caption before edits",+ Some("caption"),+ Some("renamed"),+ None,+ None,+ 2,+ ),+ ("renamed copy", None, None, Some("renamed"), None, 3),+ ("edited date", None, None, None, Some(123), 3),+ ("caption tie", Some("caption"), None, None, None, 1),+ ("edit tie", None, Some("renamed"), None, Some(123), 1),+ (+ "empty name overrides date",+ None,+ None,+ Some(""),+ Some(123),+ 1,+ ),+ ("fallback", None, None, None, None, 1),+ ];+ for (name, caption, first_name, last_name, last_time, retained) in cases {+ let mut server = Server::new_async().await;+ let mut groups = vec![vec![+ Duplicate {+ file_id: 1,+ collection_ids: vec![11],+ edited_name: first_name.map(String::from),+ ..Default::default()+ },+ Duplicate {+ file_id: 2,+ collection_ids: vec![12],+ caption: caption.map(String::from),+ ..Default::default()+ },+ Duplicate {+ file_id: 3,+ collection_ids: vec![13],+ edited_name: last_name.map(String::from),+ edited_time: last_time,+ ..Default::default()+ },+ ]];+ if name == "caption tie" {+ groups[0][0].caption = Some("first".into());+ }+ let collections: Vec<_> = (11..=13)+ .map(|id| RemovalCollection {+ id,+ key: Key::generate(),+ })+ .collect();+ let files: Vec<_> = (1..=3)+ .map(|id| RemovalFile {+ id,+ collection_id: id + 10,+ key: Key::generate(),+ })+ .collect();+ let mut additions = Vec::new();+ for collection in &collections {+ if collection.id == retained + 10 {+ continue;+ }+ let key = Key::from_bytes(*collection.key.as_bytes());+ let expected_key = *files[(retained - 1) as usize].key.as_bytes();+ additions.push(+ server+ .mock("POST", "/collections/add-files")+ .match_body(Matcher::PartialJson(+ serde_json::json!({"collectionID": collection.id}),+ ))+ .match_request(move |request| {+ let body: serde_json::Value =+ serde_json::from_slice(request.body().unwrap()).unwrap();+ let f = &body["files"][0];+ f["id"] == retained+ && secretbox::decrypt(+ &b64::decode(f["encryptedKey"].as_str().unwrap()).unwrap(),+ &Nonce::try_from_slice(+ &b64::decode(f["keyDecryptionNonce"].as_str().unwrap())+ .unwrap(),+ )+ .unwrap(),+ &key,+ )+ .unwrap()+ == expected_key+ })+ .with_status(200)+ .create_async()+ .await,+ );+ }+ let expected: Vec<_> = (1..=3).filter(|id| *id != retained).collect();+ let items: Vec<_> = expected+ .iter()+ .map(|id| serde_json::json!({"fileID": id, "collectionID": id + 10}))+ .collect();+ let trash = server+ .mock("POST", "/files/trash")+ .match_body(Matcher::Json(serde_json::json!({"items": items})))+ .with_status(200)+ .create_async()+ .await;+ let mut progress = Vec::new();+ let result = remove_duplicates(+ &session(server.url()).unwrap(),+ &groups,+ &collections,+ &files,+ |p| progress.push((p.completed, p.total)),+ )+ .await+ .unwrap();+ assert_eq!(result, expected, "{name}");+ assert_eq!(progress, vec![(1, 3), (2, 3), (3, 3)], "{name}");+ for addition in additions {+ addition.assert_async().await;+ }+ trash.assert_async().await;+ }+}++#[tokio::test]+async fn combined_removal_batches_groups_and_skips_empty_input() {+ use ente_photos::dedup::{Duplicate, remove_duplicates};++ let mut server = Server::new_async().await;+ let collections = vec![RemovalCollection {+ id: 11,+ key: Key::generate(),+ }];+ let files: Vec<_> = (1..=4)+ .map(|id| RemovalFile {+ id,+ collection_id: if id % 2 == 0 { 11 } else { 12 },+ key: Key::generate(),+ })+ .collect();+ let groups = vec![+ vec![+ Duplicate {+ file_id: 1,+ collection_ids: vec![12],+ ..Default::default()+ },+ Duplicate {+ file_id: 2,+ collection_ids: vec![11],+ ..Default::default()+ },+ ],+ vec![+ Duplicate {+ file_id: 3,+ collection_ids: vec![12],+ ..Default::default()+ },+ Duplicate {+ file_id: 4,+ collection_ids: vec![11],+ ..Default::default()+ },+ ],+ ];+ let addition = server+ .mock("POST", "/collections/add-files")+ .match_request(|r| {+ let body: serde_json::Value = serde_json::from_slice(r.body().unwrap()).unwrap();+ body["collectionID"] == 11+ && body["files"]+ .as_array()+ .unwrap()+ .iter()+ .map(|f| f["id"].as_i64().unwrap())+ .collect::<Vec<_>>()+ == vec![1, 3]+ })+ .with_status(200)+ .create_async()+ .await;+ let trash = server.mock("POST", "/files/trash").match_body(Matcher::Json(serde_json::json!({"items": [{"fileID":2,"collectionID":11},{"fileID":4,"collectionID":11}]}))).with_status(200).create_async().await;+ let session = session(server.url()).unwrap();+ let mut progress = Vec::new();+ assert_eq!(+ remove_duplicates(&session, &groups, &collections, &files, |p| progress+ .push((p.completed, p.total)))+ .await+ .unwrap(),+ vec![2, 4]+ );+ assert_eq!(progress, vec![(1, 2), (2, 2)]);+ addition.assert_async().await;+ trash.assert_async().await;+ progress.clear();+ assert!(+ remove_duplicates(&session, &[], &[], &[], |p| progress+ .push((p.completed, p.total)))+ .await+ .unwrap()+ .is_empty()+ );+ assert!(progress.is_empty());+ addition.assert_async().await;+ trash.assert_async().await;+}diff --git a/web/apps/photos/src/services/dedup.ts b/web/apps/photos/src/services/dedup.tsindex 7fac57b866..549999725d 100644--- a/web/apps/photos/src/services/dedup.ts+++ b/web/apps/photos/src/services/dedup.ts@@ -7,13 +7,13 @@ import {metadataHash,} from "ente-media/file-metadata";import {- addToCollection,createCollectionNameByID,- moveToTrash,+ forgetPendingFavoriteFileIDs,savedNormalCollections,} from "ente-new/photos/services/collection";import { savedCollectionFiles } from "ente-new/photos/services/photos-fdb";-import { findDuplicates, planDuplicateRemoval } from "ente-photos-wasm";+import { executeDuplicateRemoval, findDuplicates } from "ente-photos-wasm";+import { ensureAuthenticatedSession } from "./authenticated-session";export interface DuplicateGroup {id: string;@@ -94,47 +94,45 @@ export const removeSelectedDuplicateGroups = async () => {const selectedDuplicateGroups = duplicateGroups.filter((g) => g.isSelected);+ if (!selectedDuplicateGroups.length) {+ await onRemoteFilesPull();+ onProgress(100);+ return new Set<string>();+ }+const filesByID = new Map(selectedDuplicateGroups.flatMap((group) =>group.items.map(({ file }) => [file.id, file] as const),),);- const plan = await planDuplicateRemoval({- groups: selectedDuplicateGroups.map((group) =>- group.items.map(({ file, collectionIDs }) => ({- fileId: file.id,- collectionIds: [...collectionIDs],- caption: FilePublicMagicMetadataData.shape.caption.parse(- file.pubMagicMetadata?.data.caption,- ),- editedName: file.pubMagicMetadata?.data.editedName,- editedTime: file.pubMagicMetadata?.data.editedTime,- })),- ),- });-- let np = 0;- const ntotal =- plan.additions.length + (plan.trashIds.length ? 1 : 0) + /* sync */ 1;- const tickProgress = () => onProgress((++np / ntotal) * 100);-const collections = await savedNormalCollections();- const collectionsByID = new Map(collections.map((c) => [c.id, c]));- for (const { collectionId, fileIds } of plan.additions) {- await addToCollection(- collectionsByID.get(collectionId)!,- fileIds.map((id) => filesByID.get(id)!),- );- tickProgress();- }-- if (plan.trashIds.length) {- await moveToTrash(plan.trashIds.map((id) => filesByID.get(id)!));- tickProgress();- }+ const { trashIds } = await executeDuplicateRemoval(+ await ensureAuthenticatedSession(),+ {+ groups: selectedDuplicateGroups.map((group) =>+ group.items.map(({ file, collectionIDs }) => ({+ fileId: file.id,+ collectionIds: [...collectionIDs],+ caption: FilePublicMagicMetadataData.shape.caption.parse(+ file.pubMagicMetadata?.data.caption,+ ),+ editedName: file.pubMagicMetadata?.data.editedName,+ editedTime: file.pubMagicMetadata?.data.editedTime,+ })),+ ),+ collections: collections.map(({ id, key }) => ({ id, key })),+ files: [...filesByID.values()].map(({ id, collectionID, key }) => ({+ id,+ collectionId: collectionID,+ key,+ })),+ },+ (completed, total) => onProgress((completed / (total + 1)) * 100),+ );+ forgetPendingFavoriteFileIDs(trashIds);await onRemoteFilesPull();- tickProgress();+ onProgress(100);return new Set(selectedDuplicateGroups.map((g) => g.id));};diff --git a/web/apps/photos/tests/dedup.test.ts b/web/apps/photos/tests/dedup.test.tsindex 3d2942394e..d448a42266 100644--- a/web/apps/photos/tests/dedup.test.ts+++ b/web/apps/photos/tests/dedup.test.ts@@ -5,22 +5,30 @@ import {removeSelectedDuplicateGroups,type DuplicateGroup,} from "../src/services/dedup";-const {- addToCollection,- moveToTrash,+ executeDuplicateRemoval,savedNormalCollections,savedCollectionFiles,+ forgetPendingFavoriteFileIDs,+ session,} = vi.hoisted(() => ({- addToCollection: vi.fn(),- moveToTrash: vi.fn(),+ executeDuplicateRemoval:+ vi.fn<typeof import("ente-photos-wasm").executeDuplicateRemoval>(),savedNormalCollections: vi.fn(),savedCollectionFiles: vi.fn(),+ forgetPendingFavoriteFileIDs: vi.fn(),+ session: {},+}));+vi.mock("ente-photos-wasm", async (importOriginal) => ({+ ...(await importOriginal<typeof import("ente-photos-wasm")>()),+ executeDuplicateRemoval,+}));+vi.mock("../src/services/authenticated-session", () => ({+ ensureAuthenticatedSession: () => Promise.resolve(session),}));vi.mock("ente-new/photos/services/collection", () => ({- addToCollection,- moveToTrash,savedNormalCollections,+ forgetPendingFavoriteFileIDs,createCollectionNameByID: (collections: { id: number; name: string }[]) =>new Map(collections.map(({ id, name }) => [id, name])),}));@@ -30,7 +38,6 @@ vi.mock("ente-new/photos/services/photos-fdb", () => ({vi.mock("ente-accounts/services/user", () => ({ensureLocalUser: () => ({ id: 1 }),}));-const file = (id: number,collectionID: number,@@ -48,6 +55,7 @@ const file = (describe("deduceDuplicates through Rust WASM", () => {beforeEach(() => {vi.resetAllMocks();+ executeDuplicateRemoval.mockResolvedValue({ trashIds: [1] });savedNormalCollections.mockResolvedValue([{ id: 11, owner: { id: 1 }, name: "B" },{ id: 12, owner: { id: 1 }, name: "A" },@@ -91,11 +99,12 @@ describe("deduceDuplicates through Rust WASM", () => {},]);await removeSelectedDuplicateGroups(groups, vi.fn(), vi.fn());- expect(addToCollection).toHaveBeenCalledExactlyOnceWith(- expect.objectContaining({ id: 11 }),- [second],- );- expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([first]);+ expect(executeDuplicateRemoval.mock.calls[0]![1].groups).toEqual([+ [+ expect.objectContaining({ fileId: 2, collectionIds: [12] }),+ expect.objectContaining({ fileId: 1, collectionIds: [11, 12] }),+ ],+ ]);});test("matches legacy Live Photo hashes despite different ZIP sizes and skips groups without sizes", async () => {@@ -163,227 +172,181 @@ const duplicateGroup = (): DuplicateGroup => ({isSelected: true,});-const collections = [{ id: 11 }, { id: 12 }, { id: 13 }];+const collections = [+ { id: 11, key: "album-11" },+ { id: 12, key: "album-12" },+ { id: 13, key: "album-13" },+];describe("removeSelectedDuplicateGroups", () => {beforeEach(() => {vi.resetAllMocks();savedNormalCollections.mockResolvedValue(collections);+ executeDuplicateRemoval.mockResolvedValue({ trashIds: [1, 3] });});-test.each([123, 0])(- "normalizes legacy cached numeric caption %s before planning in Rust",+ "normalizes numeric caption %s before Rust planning",async (caption) => {const group = duplicateGroup();- const retained = group.items[1]!.file;- Object.assign(retained.pubMagicMetadata!.data, { caption });+ Object.assign(group.items[1]!.file.pubMagicMetadata!.data, {+ caption,+ });const refresh = vi.fn();-await expect(removeSelectedDuplicateGroups([group], vi.fn(), refresh),).resolves.toEqual(new Set([group.id]));- expect(addToCollection.mock.calls).toEqual([- [collections[0], [retained]],- [collections[2], [retained]],- ]);- expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([- group.items[0]!.file,- group.items[2]!.file,- ]);+ expect(+ executeDuplicateRemoval.mock.calls[0]![1].groups[0]![1],+ ).toMatchObject({ fileId: 2, caption: String(caption) });expect(refresh).toHaveBeenCalledOnce();},);-- test.each([- {- label: "caption before edits",- metadata: [{ editedName: "renamed" }, { caption: "caption" }, {}],- retainedId: 2,- },- {- label: "renamed copy",- metadata: [{}, {}, { editedName: "renamed" }],- retainedId: 3,- },- {- label: "edited timestamp",- metadata: [{}, { editedTime: 123 }, {}],- retainedId: 2,- },- {- label: "first of equal captions",- metadata: [{}, { caption: "one" }, { caption: "two" }],- retainedId: 2,- },- {- label: "first of equal edits",- metadata: [{}, { editedName: "one" }, { editedTime: 123 }],- retainedId: 2,- },- {- label: "empty caption and null caption",- metadata: [{}, { caption: "" }, { caption: null }],- retainedId: 1,- },- {- label: "empty name overrides edited timestamp",- metadata: [{}, { editedName: "", editedTime: 123 }, {}],- retainedId: 1,- },- ])(- "retains the $label through Rust WASM",- async ({ metadata, retainedId }) => {- const group = duplicateGroup();- group.items.forEach(({ file }, index) => {- Object.assign(file, {- pubMagicMetadata: { data: metadata[index] },- });+ test("passes only selected groups and their files, saved album keys, and the shared session to Rust", async () => {+ const group = duplicateGroup();+ for (const item of group.items)+ Object.assign(item.file, {+ collectionID: [...item.collectionIDs][0],+ key: `file-${item.file.id}`,});- await removeSelectedDuplicateGroups([group], vi.fn(), vi.fn());- expect(moveToTrash).toHaveBeenCalledExactlyOnceWith(- group.items- .filter(({ file }) => file.id !== retainedId)- .map(({ file }) => file),- );- },- );-- test("batches retained copies from different groups into each missing album", async () => {- const first = duplicateGroup();- const second = duplicateGroup();- second.id = "group-2";- for (const { file } of second.items) file.id += 3;const refresh = vi.fn();- await expect(- removeSelectedDuplicateGroups([first, second], vi.fn(), refresh),- ).resolves.toEqual(new Set([first.id, second.id]));- const retained = [first.items[1]!.file, second.items[1]!.file];- expect(addToCollection.mock.calls).toEqual([- [collections[0], retained],- [collections[2], retained],- ]);- expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([- first.items[0]!.file,- first.items[2]!.file,- second.items[0]!.file,- second.items[2]!.file,+ await removeSelectedDuplicateGroups(+ [+ group,+ { ...duplicateGroup(), id: "untouched", isSelected: false },+ ],+ vi.fn(),+ refresh,+ );+ expect(executeDuplicateRemoval).toHaveBeenCalledExactlyOnceWith(+ session,+ {+ groups: [+ [+ {+ fileId: 1,+ collectionIds: [11, 12],+ caption: undefined,+ editedName: undefined,+ editedTime: undefined,+ },+ {+ fileId: 2,+ collectionIds: [12],+ caption: "Keep this copy",+ editedName: undefined,+ editedTime: undefined,+ },+ {+ fileId: 3,+ collectionIds: [13],+ caption: undefined,+ editedName: undefined,+ editedTime: undefined,+ },+ ],+ ],+ collections,+ files: [+ { id: 1, collectionId: 11, key: "file-1" },+ { id: 2, collectionId: 12, key: "file-2" },+ { id: 3, collectionId: 13, key: "file-3" },+ ],+ },+ expect.any(Function),+ );+ expect(forgetPendingFavoriteFileIDs).toHaveBeenCalledExactlyOnceWith([+ 1, 3,]);expect(refresh).toHaveBeenCalledOnce();});-- test("rejects invalid planner input before modifying albums or trash", async () => {- const group = duplicateGroup();- group.items[1]!.file.id = Number.MAX_SAFE_INTEGER + 1;- const refresh = vi.fn();- await expect(- removeSelectedDuplicateGroups([group], vi.fn(), refresh),- ).rejects.toBeInstanceOf(Error);- expect(addToCollection).not.toHaveBeenCalled();- expect(moveToTrash).not.toHaveBeenCalled();- expect(refresh).not.toHaveBeenCalled();- });-- test("preserves album membership before trashing selected duplicates and awaits gallery refresh", async () => {- const group = duplicateGroup();- const untouched = {- ...duplicateGroup(),- id: "unselected",- isSelected: false,- };+ test("reports Rust execution progress and awaits refresh before completion", async () => {const progress = vi.fn();- let finishRefresh!: () => void;+ executeDuplicateRemoval.mockImplementation(+ (_session, _input, onProgress) => {+ onProgress(1, 3);+ onProgress(2, 3);+ onProgress(3, 3);+ return Promise.resolve({ trashIds: [1, 3] });+ },+ );+ let finish!: () => void;const refresh = vi.fn(() =>new Promise<void>((resolve) => {- finishRefresh = resolve;+ finish = resolve;}),);const removal = removeSelectedDuplicateGroups(- [group, untouched],+ [duplicateGroup()],progress,refresh,);await vi.waitFor(() => expect(refresh).toHaveBeenCalledOnce());- expect(addToCollection.mock.calls).toEqual([- [collections[0], [group.items[1]!.file]],- [collections[2], [group.items[1]!.file]],- ]);- expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([- group.items[0]!.file,- group.items[2]!.file,- ]);- expect(addToCollection.mock.invocationCallOrder[1]).toBeLessThan(- moveToTrash.mock.invocationCallOrder[0]!,- );- expect(moveToTrash.mock.invocationCallOrder[0]).toBeLessThan(- refresh.mock.invocationCallOrder[0]!,- );expect(progress.mock.calls).toEqual([[25], [50], [75]]);- finishRefresh();- await expect(removal).resolves.toEqual(new Set([group.id]));+ expect(+ forgetPendingFavoriteFileIDs.mock.invocationCallOrder[0],+ ).toBeLessThan(refresh.mock.invocationCallOrder[0]!);+ finish();+ await expect(removal).resolves.toEqual(new Set(["group-1"]));expect(progress).toHaveBeenLastCalledWith(100);});-- test("does not trash files when preserving album membership fails", async () => {- const failure = new Error("Add failed");- addToCollection.mockRejectedValue(failure);+ test("does not clear local state or refresh when Rust execution fails", async () => {+ const failure = new Error("Rust request failed");+ executeDuplicateRemoval.mockRejectedValue(failure);const refresh = vi.fn();await expect(removeSelectedDuplicateGroups([duplicateGroup()], vi.fn(), refresh),).rejects.toBe(failure);- expect(moveToTrash).not.toHaveBeenCalled();+ expect(forgetPendingFavoriteFileIDs).not.toHaveBeenCalled();expect(refresh).not.toHaveBeenCalled();});-- test("does not trash any files if a later album addition fails", async () => {- const failure = new Error("Second album failed");- addToCollection- .mockResolvedValueOnce(undefined)- .mockRejectedValueOnce(failure);- const refresh = vi.fn();- await expect(- removeSelectedDuplicateGroups([duplicateGroup()], vi.fn(), refresh),- ).rejects.toBe(failure);- expect(addToCollection).toHaveBeenCalledTimes(2);- expect(moveToTrash).not.toHaveBeenCalled();- expect(refresh).not.toHaveBeenCalled();- });-- test("does not refresh when trashing fails", async () => {- const failure = new Error("Trash failed");- moveToTrash.mockRejectedValue(failure);- const refresh = vi.fn();- await expect(- removeSelectedDuplicateGroups([duplicateGroup()], vi.fn(), refresh),- ).rejects.toBe(failure);- expect(refresh).not.toHaveBeenCalled();- });-- test("does not report completion when gallery refresh fails", async () => {+ test("propagates refresh failures without reporting completion", async () => {const failure = new Error("Refresh failed");- const refresh = vi.fn().mockRejectedValue(failure);const progress = vi.fn();await expect(- removeSelectedDuplicateGroups(- [duplicateGroup()],- progress,- refresh,+ removeSelectedDuplicateGroups([duplicateGroup()], progress, () =>+ Promise.reject(failure),),).rejects.toBe(failure);expect(progress).not.toHaveBeenCalledWith(100);});-- test("only refreshes when no groups are selected", async () => {+ test("passes multiple selected groups in one call and uses Rust's returned trash IDs", async () => {+ const first = duplicateGroup();+ const second = duplicateGroup();+ second.id = "group-2";+ for (const { file } of second.items) file.id += 3;+ executeDuplicateRemoval.mockResolvedValue({ trashIds: [1, 3, 4, 6] });+ await expect(+ removeSelectedDuplicateGroups([first, second], vi.fn(), vi.fn()),+ ).resolves.toEqual(new Set([first.id, second.id]));+ expect(executeDuplicateRemoval).toHaveBeenCalledOnce();+ expect(+ executeDuplicateRemoval.mock.calls[0]![1].groups.map((g) =>+ g.map((i) => i.fileId),+ ),+ ).toEqual([+ [1, 2, 3],+ [4, 5, 6],+ ]);+ expect(+ executeDuplicateRemoval.mock.calls[0]![1].files.map((f) => f.id),+ ).toEqual([1, 2, 3, 4, 5, 6]);+ expect(forgetPendingFavoriteFileIDs).toHaveBeenCalledExactlyOnceWith([+ 1, 3, 4, 6,+ ]);+ });+ test("refreshes and reports completion when no groups are selected", async () => {const refresh = vi.fn();+ const progress = vi.fn();await expect(removeSelectedDuplicateGroups([{ ...duplicateGroup(), isSelected: false }],- vi.fn(),+ progress,refresh,),).resolves.toEqual(new Set());- expect(addToCollection).not.toHaveBeenCalled();- expect(moveToTrash).not.toHaveBeenCalled();+ expect(forgetPendingFavoriteFileIDs).not.toHaveBeenCalled();expect(refresh).toHaveBeenCalledOnce();+ expect(progress).toHaveBeenCalledExactlyOnceWith(100);});});diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.tsindex 23d48e3ebe..6aa8a67021 100644--- a/web/packages/new/photos/services/collection.ts+++ b/web/packages/new/photos/services/collection.ts@@ -563,7 +563,7 @@ const splitPendingFavoriteFiles = (files: EnteFile[], userID: number) => {return { pendingFiles, remainingFiles };};-const forgetPendingFavoriteFileIDs = (fileIDs: number[]) => {+export const forgetPendingFavoriteFileIDs = (fileIDs: number[]) => {if (!fileIDs.length) return;const deletedFileIDs = new Set(fileIDs);diff --git a/web/packages/wasm/photos/index.ts b/web/packages/wasm/photos/index.tsindex 51fa827c94..ab6ff61281 100644--- a/web/packages/wasm/photos/index.ts+++ b/web/packages/wasm/photos/index.ts@@ -1,9 +1,9 @@import type {EncryptedBox,+ ExecuteDuplicateRemovalInput,FindDuplicatesInput,LocationIndex,OpenSessionInput,- PlanDuplicateRemovalInput,Session,WrappedRootContactKey,} from "./pkg/ente_photos_wasm";@@ -20,8 +20,11 @@ const wasm = () => import("./pkg/ente_photos_wasm");export const findDuplicates = async (input: FindDuplicatesInput) =>(await wasm()).photosFindDuplicates(input);-export const planDuplicateRemoval = async (input: PlanDuplicateRemovalInput) =>- (await wasm()).photosPlanDuplicateRemoval(input);+export const executeDuplicateRemoval = async (+ session: Session,+ input: ExecuteDuplicateRemovalInput,+ onProgress: (completed: number, total: number) => void,+) => (await wasm()).photosExecuteDuplicateRemoval(session, input, onProgress);export const createLocationIndex = async (cities: Uint8Array,diff --git a/web/packages/wasm/tests/bindings.test.ts b/web/packages/wasm/tests/bindings.test.tsindex ed0f3f886b..00d37e9069 100644--- a/web/packages/wasm/tests/bindings.test.ts+++ b/web/packages/wasm/tests/bindings.test.ts@@ -93,6 +93,115 @@ test("Photos deletion keeps summary fallback and decrypts the challenge in Rust"}});+test("Photos plans and executes duplicate removal in one WASM call", async () => {+ const masterKey = await generateKey();+ const input = await sessionKeyAttributes(masterKey);+ const albumKey = await generateKey();+ const fileKey = await generateKey();+ const requests: string[] = [];+ const progress: number[][] = [];+ mockFetch(async (request) => {+ expect(request.headers.get("X-Auth-Token")).toBe("test-token");+ const path = new URL(request.url).pathname;+ requests.push(path);+ const body = (await request.json()) as {+ collectionID: number;+ files: {+ id: number;+ encryptedKey: string;+ keyDecryptionNonce: string;+ }[];+ };+ if (path === "/collections/add-files") {+ expect(body.collectionID).toBe(11);+ expect(body.files).toHaveLength(1);+ expect(body.files[0]!.id).toBe(2);+ expect(+ await photos.decryptBox(+ {+ encryptedData: body.files[0]!.encryptedKey,+ nonce: body.files[0]!.keyDecryptionNonce,+ },+ albumKey,+ ),+ ).toBe(fileKey);+ } else {+ expect(path).toBe("/files/trash");+ expect(requests).toEqual([+ "/collections/add-files",+ "/files/trash",+ ]);+ expect(body).toEqual({ items: [{ fileID: 1, collectionID: 11 }] });+ }+ return Response.json({});+ });+ const session = await photos.openSession({+ baseUrl: "http://localhost",+ authToken: "test-token",+ masterKeyB64: masterKey,+ ...input,+ });+ const removal = {+ groups: [+ [+ {+ fileId: 1,+ collectionIds: [11],+ caption: undefined,+ editedName: undefined,+ editedTime: undefined,+ },+ {+ fileId: 2,+ collectionIds: [12],+ caption: "keep",+ editedName: undefined,+ editedTime: undefined,+ },+ ],+ ],+ collections: [+ { id: 11, key: albumKey },+ { id: 99, key: await generateKey() },+ ],+ files: [+ { id: 1, collectionId: 11, key: await generateKey() },+ { id: 2, collectionId: 12, key: fileKey },+ ],+ };+ try {+ expect(+ await photos.executeDuplicateRemoval(+ session,+ removal,+ (completed, total) => progress.push([completed, total]),+ ),+ ).toEqual({ trashIds: [1] });+ expect(requests).toEqual(["/collections/add-files", "/files/trash"]);+ expect(progress).toEqual([+ [1, 2],+ [2, 2],+ ]);+ requests.length = 0;+ const invalid = structuredClone(removal);+ invalid.groups[0]![0]!.fileId = Number.MAX_SAFE_INTEGER + 1;+ await expect(+ photos.executeDuplicateRemoval(session, invalid, vi.fn()),+ ).rejects.toBeInstanceOf(Error);+ expect(requests).toEqual([]);+ expect(+ await photos.executeDuplicateRemoval(+ session,+ { groups: [], collections: [], files: [] },+ vi.fn(),+ ),+ ).toEqual({ trashIds: [] });+ expect(requests).toEqual([]);+ } finally {+ session.free();+ }+});+test.each(["classical", "post-quantum"])("Photos %s Cast payloads round-trip and enforce safe collection IDs",async (encryption) => {