Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[photos][web][rust] Execute duplicate removal through Rust (#14241)

Aswin Asok9b1decf8b5d4
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index d4b43d6919..3cf5364d9d 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2835,15 +2835,18 @@ dependencies = [
"ente-core",
"ente-exif",
"ente-location",
+ "ente-test-support",
"ente-vecdb",
"futures-util",
"log",
+ "mockito",
"quick-xml 0.42.0",
"rusqlite",
"serde",
"serde_json",
"tempfile",
"thiserror 2.0.21",
+ "tokio",
]
[[package]]
diff --git a/rust/bindings/wasm/lib/Cargo.toml b/rust/bindings/wasm/lib/Cargo.toml
index a9f0c99d01..0bf5324f52 100644
--- a/rust/bindings/wasm/lib/Cargo.toml
+++ b/rust/bindings/wasm/lib/Cargo.toml
@@ -19,7 +19,7 @@ crypto-seal = ["crypto"]
legacy = ["session", "dep:ente-legacy", "dep:wasm-bindgen-futures"]
location = ["dep:ente-location", "dep:serde", "dep:serde-wasm-bindgen", "dep:tsify"]
locker = ["session", "dep:ente-locker"]
-photos = ["dep:ente-photos", "dep:serde", "dep:serde-wasm-bindgen", "dep:tsify"]
+photos = ["session", "dep:ente-photos", "dep:serde", "dep:serde-wasm-bindgen", "dep:tsify", "dep:wasm-bindgen-futures"]
prelogin = ["accounts", "crypto", "dep:wasm-bindgen-futures"]
session = [
"dep:ente-accounts",
diff --git a/rust/bindings/wasm/lib/src/photos.rs b/rust/bindings/wasm/lib/src/photos.rs
index b653213832..0336418628 100644
--- a/rust/bindings/wasm/lib/src/photos.rs
+++ b/rust/bindings/wasm/lib/src/photos.rs
@@ -4,10 +4,22 @@ use serde_wasm_bindgen as swb;
use tsify::Tsify;
use wasm_bindgen::prelude::*;
+use crate::session::Session;
+use ente_core::{
+ b64,
+ crypto::{self, Key},
+};
+
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error(transparent)]
Serde(#[from] swb::Error),
+ #[error(transparent)]
+ Removal(#[from] dedup::RemovalError),
+ #[error(transparent)]
+ Decode(#[from] b64::DecodeError),
+ #[error(transparent)]
+ Crypto(#[from] crypto::Error),
}
impl From<Error> for JsValue {
@@ -114,30 +126,40 @@ struct Duplicate {
}
#[derive(Deserialize, Tsify)]
-pub struct PlanDuplicateRemovalInput {
- groups: Vec<Vec<Duplicate>>,
+struct RemovalCollection {
+ id: i64,
+ key: String,
}
-#[derive(Serialize, Tsify)]
+#[derive(Deserialize, Tsify)]
#[serde(rename_all = "camelCase")]
-struct DuplicateAddition {
+struct RemovalFile {
+ id: i64,
collection_id: i64,
- file_ids: Vec<i64>,
+ key: String,
+}
+
+#[derive(Deserialize, Tsify)]
+pub struct ExecuteDuplicateRemovalInput {
+ groups: Vec<Vec<Duplicate>>,
+ collections: Vec<RemovalCollection>,
+ files: Vec<RemovalFile>,
}
#[derive(Serialize, Tsify)]
#[serde(rename_all = "camelCase")]
-pub struct DuplicateRemovalPlan {
- additions: Vec<DuplicateAddition>,
+pub struct DuplicateRemovalResult {
trash_ids: Vec<i64>,
}
-#[wasm_bindgen(js_name = photosPlanDuplicateRemoval)]
-pub fn plan_duplicate_removal(
- input: <PlanDuplicateRemovalInput as Tsify>::JsType,
-) -> Result<<DuplicateRemovalPlan as Tsify>::JsType, Error> {
- let input = PlanDuplicateRemovalInput::from_js(input)?;
- let groups: Vec<_> = input
+#[wasm_bindgen(js_name = photosExecuteDuplicateRemoval)]
+pub async fn execute_duplicate_removal(
+ session: &Session,
+ input: <ExecuteDuplicateRemovalInput as Tsify>::JsType,
+ on_progress: js_sys::Function,
+) -> Result<<DuplicateRemovalResult as Tsify>::JsType, Error> {
+ let input = ExecuteDuplicateRemovalInput::from_js(input)?;
+ let groups = input
.groups
.into_iter()
.map(|group| {
@@ -152,19 +174,37 @@ pub fn plan_duplicate_removal(
})
.collect()
})
- .collect();
- let plan = dedup::plan_removal(&groups);
- DuplicateRemovalPlan {
- additions: plan
- .additions
- .into_iter()
- .map(|addition| DuplicateAddition {
- collection_id: addition.collection_id,
- file_ids: addition.file_ids,
+ .collect::<Vec<_>>();
+ let collections = input
+ .collections
+ .into_iter()
+ .map(|c| {
+ Ok(dedup::RemovalCollection {
+ id: c.id,
+ key: Key::try_from_slice(&b64::decode(&c.key)?)?,
})
- .collect(),
- trash_ids: plan.trash_ids,
- }
- .into_js()
- .map_err(Into::into)
+ })
+ .collect::<Result<Vec<_>, Error>>()?;
+ let files = input
+ .files
+ .into_iter()
+ .map(|f| {
+ Ok(dedup::RemovalFile {
+ id: f.id,
+ collection_id: f.collection_id,
+ key: Key::try_from_slice(&b64::decode(&f.key)?)?,
+ })
+ })
+ .collect::<Result<Vec<_>, Error>>()?;
+ let trash_ids = dedup::remove_duplicates(session.inner(), &groups, &collections, &files, |p| {
+ let _ = on_progress.call2(
+ &JsValue::UNDEFINED,
+ &JsValue::from_f64(p.completed as f64),
+ &JsValue::from_f64(p.total as f64),
+ );
+ })
+ .await?;
+ DuplicateRemovalResult { trash_ids }
+ .into_js()
+ .map_err(Into::into)
}
diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.toml
index 3330385dc2..aabd56a426 100644
--- a/rust/crates/photos/Cargo.toml
+++ b/rust/crates/photos/Cargo.toml
@@ -2,6 +2,9 @@
name = "ente-photos"
edition.workspace = true
+[features]
+museum = []
+
[dependencies]
ente-collections.workspace = true
ente-core.workspace = true
@@ -20,7 +23,10 @@ futures-util.workspace = true
rusqlite = { workspace = true, default-features = true }
[dev-dependencies]
+ente-test-support.workspace = true
+mockito.workspace = true
tempfile.workspace = true
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
[lints]
workspace = true
diff --git a/rust/crates/photos/src/dedup.rs b/rust/crates/photos/src/dedup.rs
index f79ffbd3dd..ee2b6e09ba 100644
--- a/rust/crates/photos/src/dedup.rs
+++ b/rust/crates/photos/src/dedup.rs
@@ -1,5 +1,12 @@
use std::collections::{HashMap, HashSet};
+mod removal;
+
+pub use removal::{
+ RemovalCollection, RemovalError, RemovalFile, RemovalProgress, execute_removal,
+ remove_duplicates,
+};
+
pub struct Collection {
pub id: i64,
pub owner_id: i64,
diff --git a/rust/crates/photos/src/dedup/removal.rs b/rust/crates/photos/src/dedup/removal.rs
new file mode 100644
index 0000000000..8a6e40d9a6
--- /dev/null
+++ b/rust/crates/photos/src/dedup/removal.rs
@@ -0,0 +1,170 @@
+use std::collections::{HashMap, HashSet};
+
+use ente_core::{
+ Session, b64,
+ crypto::{Key, secretbox},
+ http,
+};
+use serde::Serialize;
+
+use super::{Duplicate, RemovalPlan, plan_removal};
+
+const REQUEST_BATCH_SIZE: usize = 1000;
+
+pub struct RemovalCollection {
+ pub id: i64,
+ pub key: Key,
+}
+
+pub struct RemovalFile {
+ pub id: i64,
+ pub collection_id: i64,
+ pub key: Key,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct RemovalProgress {
+ pub completed: usize,
+ pub total: usize,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum RemovalError {
+ #[error(transparent)]
+ Http(#[from] http::Error),
+ #[error("Missing collection {0}")]
+ MissingCollection(i64),
+ #[error("Missing file {0}")]
+ MissingFile(i64),
+ #[error("Invalid duplicate removal plan: {0}")]
+ InvalidPlan(&'static str),
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+struct WrappedFile {
+ id: i64,
+ encrypted_key: String,
+ key_decryption_nonce: String,
+}
+
+#[derive(Serialize)]
+struct AddFiles<'a> {
+ #[serde(rename = "collectionID")]
+ collection_id: i64,
+ files: &'a [WrappedFile],
+}
+
+#[derive(Serialize)]
+struct TrashItem {
+ #[serde(rename = "fileID")]
+ file_id: i64,
+ #[serde(rename = "collectionID")]
+ collection_id: i64,
+}
+
+#[derive(Serialize)]
+struct TrashFiles<'a> {
+ items: &'a [TrashItem],
+}
+
+pub async fn remove_duplicates(
+ session: &Session,
+ groups: &[Vec<Duplicate>],
+ collections: &[RemovalCollection],
+ files: &[RemovalFile],
+ on_progress: impl FnMut(RemovalProgress),
+) -> Result<Vec<i64>, RemovalError> {
+ let plan = plan_removal(groups);
+ execute_removal(session, &plan, collections, files, on_progress).await?;
+ Ok(plan.trash_ids)
+}
+
+pub async fn execute_removal(
+ session: &Session,
+ plan: &RemovalPlan,
+ collections: &[RemovalCollection],
+ files: &[RemovalFile],
+ mut on_progress: impl FnMut(RemovalProgress),
+) -> Result<(), RemovalError> {
+ let collections: HashMap<_, _> = collections.iter().map(|c| (c.id, c)).collect();
+ let files: HashMap<_, _> = files.iter().map(|f| (f.id, f)).collect();
+ let trash_ids: HashSet<_> = plan.trash_ids.iter().copied().collect();
+ if trash_ids.len() != plan.trash_ids.len() {
+ return Err(RemovalError::InvalidPlan("repeated trash ID"));
+ }
+ let mut additions = Vec::with_capacity(plan.additions.len());
+ for addition in &plan.additions {
+ let collection = collections
+ .get(&addition.collection_id)
+ .ok_or(RemovalError::MissingCollection(addition.collection_id))?;
+ let mut addition_files = Vec::with_capacity(addition.file_ids.len());
+ for id in &addition.file_ids {
+ if trash_ids.contains(id) {
+ return Err(RemovalError::InvalidPlan(
+ "retained file is also scheduled for trash",
+ ));
+ }
+ let file = files.get(id).ok_or(RemovalError::MissingFile(*id))?;
+ addition_files.push(*file);
+ }
+ additions.push((*collection, addition_files));
+ }
+ let items = plan
+ .trash_ids
+ .iter()
+ .map(|id| {
+ let file = files.get(id).ok_or(RemovalError::MissingFile(*id))?;
+ Ok(TrashItem {
+ file_id: *id,
+ collection_id: file.collection_id,
+ })
+ })
+ .collect::<Result<Vec<_>, RemovalError>>()?;
+
+ let total = additions.len() + usize::from(!items.is_empty());
+ let mut completed = 0;
+ for (collection, files) in additions {
+ for batch in files.chunks(REQUEST_BATCH_SIZE) {
+ let wrapped: Vec<_> = batch
+ .iter()
+ .map(|file| {
+ let encrypted = secretbox::encrypt(file.key.as_bytes(), &collection.key);
+ WrappedFile {
+ id: file.id,
+ encrypted_key: b64::encode(&encrypted.encrypted_data),
+ key_decryption_nonce: b64::encode(encrypted.nonce.as_bytes()),
+ }
+ })
+ .collect();
+ session
+ .api
+ .post("/collections/add-files")
+ .json(&AddFiles {
+ collection_id: collection.id,
+ files: &wrapped,
+ })
+ .send()
+ .await?
+ .error_for_code()
+ .await?;
+ }
+ completed += 1;
+ on_progress(RemovalProgress { completed, total });
+ }
+ for batch in items.chunks(REQUEST_BATCH_SIZE) {
+ session
+ .api
+ .post("/files/trash")
+ .json(&TrashFiles { items: batch })
+ .send()
+ .await?
+ .error_for_code()
+ .await?;
+ }
+ if !items.is_empty() {
+ completed += 1;
+ on_progress(RemovalProgress { completed, total });
+ }
+ Ok(())
+}
diff --git a/rust/crates/photos/src/face_clustering/mod.rs b/rust/crates/photos/src/face_clustering/mod.rs
index 1ef63e5b1d..7130ec4600 100644
--- a/rust/crates/photos/src/face_clustering/mod.rs
+++ b/rust/crates/photos/src/face_clustering/mod.rs
@@ -104,6 +104,7 @@ pub(crate) fn new_cluster_id() -> String {
format!("{CLUSTER_ID_PREFIX}{suffix}")
}
+#[cfg(any(test, not(target_arch = "wasm32")))]
pub(crate) fn normalized_mean(embeddings: &[Vec<f32>]) -> Option<Vec<f32>> {
let (first, rest) = embeddings.split_first()?;
let mut sum = first.clone();
diff --git a/rust/crates/photos/tests/dedup_museum_test.rs b/rust/crates/photos/tests/dedup_museum_test.rs
new file mode 100644
index 0000000000..0ef44f68cf
--- /dev/null
+++ b/rust/crates/photos/tests/dedup_museum_test.rs
@@ -0,0 +1,421 @@
+#![cfg(feature = "museum")]
+
+use std::{collections::HashSet, io::Write};
+
+use ente_collections::client;
+use ente_core::{
+ Session, b64,
+ crypto::{Header, Key, SecretKey, blob, hash, secretbox},
+ http::{ApiConfig, Auth},
+ io::Md5Writer,
+};
+use ente_photos::dedup::{self, Duplicate, RemovalCollection, RemovalFile, RemovalProgress};
+use ente_test_support::{Museum, TestResult, account_fixture};
+use serde::Deserialize;
+use serde_json::{Value, json};
+
+const ORIGINAL: &[u8] = b"Duplicate removal local Museum encrypted file fixture";
+
+#[test]
+fn duplicate_removal_against_museum() -> TestResult {
+ Museum::run_async(run)
+}
+
+async fn run(endpoint: String) -> TestResult {
+ let account = account_fixture::create_account(&endpoint, "dedup-owner").await?;
+ let session = open_session(&endpoint, &account)?;
+ let albums = vec![
+ create_album(&session, "A").await?,
+ create_album(&session, "B").await?,
+ create_album(&session, "C").await?,
+ ];
+ let cases = [
+ (
+ "caption",
+ [
+ json!({"editedName":"renamed"}),
+ json!({"caption":"keep"}),
+ json!({}),
+ ],
+ 1,
+ ),
+ ("fallback", [json!({}), json!({}), json!({})], 0),
+ (
+ "rename",
+ [json!({}), json!({}), json!({"editedName":"renamed"})],
+ 2,
+ ),
+ ("date", [json!({}), json!({"editedTime":123}), json!({})], 1),
+ (
+ "caption-tie",
+ [
+ json!({"caption":"first"}),
+ json!({"caption":"second"}),
+ json!({}),
+ ],
+ 0,
+ ),
+ (
+ "edit-tie",
+ [
+ json!({"editedName":"first"}),
+ json!({"editedTime":123}),
+ json!({}),
+ ],
+ 0,
+ ),
+ (
+ "empty-name",
+ [
+ json!({}),
+ json!({"editedName":"","editedTime":123}),
+ json!({}),
+ ],
+ 0,
+ ),
+ (
+ "normalized-zero-caption",
+ [json!({}), json!({"caption":"0"}), json!({})],
+ 1,
+ ),
+ ];
+ for (name, edits, retained_index) in cases {
+ let files = upload_group(&session, &albums, name, &edits).await?;
+ let groups = load_groups(&session, &albums, &files).await?;
+ let retained = &files[retained_index];
+ let expected_trash: Vec<_> = files
+ .iter()
+ .filter(|f| f.id != retained.id)
+ .map(|f| f.id)
+ .collect();
+ let mut progress = Vec::new();
+ let trashed =
+ dedup::remove_duplicates(&session, &groups, &albums, &files, |p| progress.push(p))
+ .await?;
+ assert_eq!(trashed, expected_trash, "{name}");
+ assert_eq!(
+ progress,
+ vec![
+ RemovalProgress {
+ completed: 1,
+ total: 3
+ },
+ RemovalProgress {
+ completed: 2,
+ total: 3
+ },
+ RemovalProgress {
+ completed: 3,
+ total: 3
+ }
+ ],
+ "{name}"
+ );
+ let ids: HashSet<_> = files.iter().map(|f| f.id).collect();
+ for album in &albums {
+ let active = active_files(&session, album.id, &ids).await?;
+ assert_eq!(active.len(), 1, "{name}");
+ let remote = &active[0];
+ assert_eq!(remote.id, retained.id, "{name}");
+ let key = remote.open_key(&album.key)?;
+ assert_eq!(key.as_bytes(), retained.key.as_bytes(), "{name}");
+ assert_eq!(
+ public_metadata(remote, &key)?,
+ edits[retained_index],
+ "{name}"
+ );
+ let header = Header::try_from_slice(&b64::decode(&remote.file.decryption_header)?)?;
+ let plaintext =
+ ente_photos::files::download(&session, remote.id, &key, &header, || Ok(Vec::new()))
+ .await?;
+ assert_eq!(plaintext, ORIGINAL, "{name}");
+ }
+ assert_eq!(
+ trash_ids(&session, &ids).await?,
+ expected_trash.into_iter().collect(),
+ "{name}"
+ );
+ }
+
+ let outsider = account_fixture::create_account(&endpoint, "dedup-outsider").await?;
+ let outsider_session = open_session(&endpoint, &outsider)?;
+ let foreign_album = create_album(&outsider_session, "Private").await?;
+ for fail_first in [true, false] {
+ let files = upload_group(
+ &session,
+ &albums,
+ "permission-failure",
+ &[json!({}), json!({"caption":"keep"}), json!({})],
+ )
+ .await?;
+ let mut groups = load_groups(&session, &albums, &files).await?;
+ if fail_first {
+ groups[0][0].collection_ids.insert(0, foreign_album.id);
+ } else {
+ groups[0][0].collection_ids.push(foreign_album.id);
+ }
+ let mut available: Vec<_> = albums
+ .iter()
+ .map(|a| RemovalCollection {
+ id: a.id,
+ key: Key::from_bytes(*a.key.as_bytes()),
+ })
+ .collect();
+ available.push(RemovalCollection {
+ id: foreign_album.id,
+ key: Key::from_bytes(*foreign_album.key.as_bytes()),
+ });
+ let mut progress = Vec::new();
+ let error =
+ dedup::remove_duplicates(&session, &groups, &available, &files, |p| progress.push(p))
+ .await
+ .err()
+ .ok_or("Expected Museum to reject an inaccessible album")?;
+ match error {
+ dedup::RemovalError::Http(error) => assert_eq!(error.status_code(), Some(404)),
+ error => panic!("Expected Museum permission failure, got {error}"),
+ }
+ assert_eq!(progress.len(), usize::from(!fail_first));
+ let ids: HashSet<_> = files.iter().map(|f| f.id).collect();
+ assert!(trash_ids(&session, &ids).await?.is_empty());
+ for (index, album) in albums.iter().enumerate() {
+ let active = active_files(&session, album.id, &ids).await?;
+ let active_ids: HashSet<_> = active.iter().map(|f| f.id).collect();
+ let mut expected = HashSet::from([files[index].id]);
+ if !fail_first && index == 0 {
+ expected.insert(files[1].id);
+ }
+ assert_eq!(active_ids, expected);
+ }
+ }
+ Ok(())
+}
+
+fn open_session(endpoint: &str, account: &account_fixture::TestAccount) -> TestResult<Session> {
+ Ok(Session::new(
+ ApiConfig {
+ origin: endpoint.into(),
+ client_package: Some("io.ente.photos".into()),
+ client_version: None,
+ user_agent: Some("ente-photos-dedup-e2e".into()),
+ auth: Some(Auth::User(account.auth_token.clone())),
+ },
+ account.user_id,
+ Key::try_from_slice(&account.master_key)?,
+ Key::try_from_slice(&account.recovery_key)?,
+ SecretKey::try_from_slice(&account.secret_key)?,
+ )?)
+}
+
+async fn create_album(session: &Session, name: &str) -> TestResult<RemovalCollection> {
+ #[derive(Deserialize)]
+ struct Created {
+ collection: client::Collection,
+ }
+ let key = Key::generate();
+ let wrapped = secretbox::encrypt(key.as_bytes(), &session.master_key);
+ let name = secretbox::encrypt(name.as_bytes(), &key);
+ let created: Created = session.api.post("/collections").json(&json!({
+ "encryptedKey":b64::encode(&wrapped.encrypted_data),"keyDecryptionNonce":b64::encode(wrapped.nonce.as_bytes()),
+ "encryptedName":b64::encode(&name.encrypted_data),"nameDecryptionNonce":b64::encode(name.nonce.as_bytes()),"type":"album","attributes":{"version":1}
+ })).send().await?.error_for_code().await?.json().await?;
+ Ok(RemovalCollection {
+ id: created.collection.id,
+ key,
+ })
+}
+
+async fn upload_object(session: &Session, bytes: Vec<u8>) -> TestResult<String> {
+ #[derive(Deserialize)]
+ #[serde(rename_all = "camelCase")]
+ struct Upload {
+ url: String,
+ object_key: String,
+ }
+ let mut checksum_writer = Md5Writer::new(std::io::sink());
+ checksum_writer.write_all(&bytes)?;
+ let checksum = b64::encode(&checksum_writer.finalize().1);
+ let upload: Upload = session
+ .api
+ .post("/files/upload-url")
+ .json(&json!({"contentLength":bytes.len(),"contentMD5":checksum}))
+ .send()
+ .await?
+ .error_for_code()
+ .await?
+ .json()
+ .await?;
+ session
+ .api
+ .http()
+ .put(&upload.url)
+ .header("content-md5", &checksum)
+ .header("content-length", &bytes.len().to_string())
+ .body(bytes)
+ .send()
+ .await?
+ .error_for_status()?;
+ Ok(upload.object_key)
+}
+
+async fn upload_group(
+ session: &Session,
+ albums: &[RemovalCollection],
+ name: &str,
+ edits: &[Value; 3],
+) -> TestResult<Vec<RemovalFile>> {
+ let mut files = Vec::new();
+ for (index, album) in albums.iter().enumerate() {
+ let key = Key::generate();
+ let wrapped = secretbox::encrypt(key.as_bytes(), &album.key);
+ let file = blob::encrypt(ORIGINAL, &key)?;
+ let thumbnail = blob::encrypt(b"thumbnail", &key)?;
+ let metadata = blob::encrypt_json(
+ &json!({"title":format!("{name}-{index}.jpg"),"fileType":0,"hash":b64::encode(&hash::hash(ORIGINAL,None,None)?),"creationTime":1_700_000_000_000_000_i64,"modificationTime":1_700_000_000_000_000_i64}),
+ &key,
+ )?;
+ let file_object = upload_object(session, file.encrypted_data).await?;
+ let thumbnail_object = upload_object(session, thumbnail.encrypted_data).await?;
+ let mut request = json!({"collectionID":album.id,"encryptedKey":b64::encode(&wrapped.encrypted_data),"keyDecryptionNonce":b64::encode(wrapped.nonce.as_bytes()),
+ "file":{"objectKey":file_object,"decryptionHeader":b64::encode(file.decryption_header.as_bytes())},
+ "thumbnail":{"objectKey":thumbnail_object,"decryptionHeader":b64::encode(thumbnail.decryption_header.as_bytes())},
+ "metadata":{"encryptedData":b64::encode(&metadata.encrypted_data),"decryptionHeader":b64::encode(metadata.decryption_header.as_bytes())}});
+ let count = edits[index]
+ .as_object()
+ .ok_or("Expected metadata object")?
+ .len();
+ if count > 0 {
+ let encrypted = blob::encrypt_json(&edits[index], &key)?;
+ request["pubMagicMetadata"] = json!({"version":1,"count":count,"data":b64::encode(&encrypted.encrypted_data),"header":b64::encode(encrypted.decryption_header.as_bytes())});
+ }
+ #[derive(Deserialize)]
+ struct Uploaded {
+ id: i64,
+ }
+ let uploaded: Uploaded = session
+ .api
+ .post("/files")
+ .json(&request)
+ .send()
+ .await?
+ .error_for_code()
+ .await?
+ .json()
+ .await?;
+ files.push(RemovalFile {
+ id: uploaded.id,
+ collection_id: album.id,
+ key,
+ });
+ }
+ Ok(files)
+}
+
+fn public_metadata(file: &client::File, key: &Key) -> TestResult<Value> {
+ Ok(match &file.pub_magic_metadata {
+ Some(metadata) => serde_json::from_slice(&metadata.decrypt(key)?)?,
+ None => json!({}),
+ })
+}
+
+async fn active_files(
+ session: &Session,
+ collection_id: i64,
+ ids: &HashSet<i64>,
+) -> TestResult<Vec<client::File>> {
+ let page = client::files_diff(session, collection_id, 0).await?;
+ assert!(!page.has_more);
+ Ok(page
+ .files
+ .into_iter()
+ .filter(|f| ids.contains(&f.id) && !f.is_deleted())
+ .collect())
+}
+
+async fn load_groups(
+ session: &Session,
+ albums: &[RemovalCollection],
+ files: &[RemovalFile],
+) -> TestResult<Vec<Vec<Duplicate>>> {
+ let mut candidates = Vec::new();
+ let mut duplicates = Vec::new();
+ let ids: HashSet<_> = files.iter().map(|f| f.id).collect();
+ for album in albums {
+ for remote in active_files(session, album.id, &ids).await? {
+ let key = remote.open_key(&album.key)?;
+ let metadata: Value = serde_json::from_slice(&blob::decrypt(
+ &b64::decode(
+ remote
+ .metadata
+ .encrypted_data
+ .as_deref()
+ .ok_or("Missing metadata")?,
+ )?,
+ &Header::try_from_slice(&b64::decode(&remote.metadata.decryption_header)?)?,
+ &key,
+ )?)?;
+ let edits = public_metadata(&remote, &key)?;
+ candidates.push(dedup::File {
+ id: remote.id,
+ collection_id: album.id,
+ owner_id: remote.owner_id,
+ hash: metadata["hash"].as_str().map(String::from),
+ size: remote
+ .extra
+ .get("info")
+ .and_then(|info| info["fileSize"].as_u64()),
+ });
+ duplicates.push(Duplicate {
+ file_id: remote.id,
+ collection_ids: vec![album.id],
+ caption: edits["caption"].as_str().map(String::from),
+ edited_name: edits["editedName"].as_str().map(String::from),
+ edited_time: edits["editedTime"].as_i64(),
+ });
+ }
+ }
+ let collections: Vec<_> = albums
+ .iter()
+ .map(|a| dedup::Collection {
+ id: a.id,
+ owner_id: session.user_id,
+ })
+ .collect();
+ let groups = dedup::find_duplicates(session.user_id, &collections, &candidates);
+ assert_eq!(groups.len(), 1);
+ assert_eq!(groups[0].items.len(), 3);
+ Ok(vec![duplicates])
+}
+
+async fn trash_ids(session: &Session, ids: &HashSet<i64>) -> TestResult<HashSet<i64>> {
+ #[derive(Deserialize)]
+ #[serde(rename_all = "camelCase")]
+ struct Page {
+ diff: Vec<Item>,
+ has_more: bool,
+ }
+ #[derive(Deserialize)]
+ #[serde(rename_all = "camelCase")]
+ struct Item {
+ file: client::File,
+ is_deleted: bool,
+ is_restored: bool,
+ }
+ let page: Page = session
+ .api
+ .get("/trash/v2/diff")
+ .query(&[("sinceTime", 0)])
+ .send()
+ .await?
+ .error_for_code()
+ .await?
+ .json()
+ .await?;
+ assert!(!page.has_more);
+ Ok(page
+ .diff
+ .into_iter()
+ .filter(|t| ids.contains(&t.file.id) && !t.is_deleted && !t.is_restored)
+ .map(|t| t.file.id)
+ .collect())
+}
diff --git a/rust/crates/photos/tests/dedup_removal_test.rs b/rust/crates/photos/tests/dedup_removal_test.rs
new file mode 100644
index 0000000000..87a7078a78
--- /dev/null
+++ b/rust/crates/photos/tests/dedup_removal_test.rs
@@ -0,0 +1,609 @@
+use std::sync::{Arc, Mutex};
+
+use ente_core::{
+ Session, b64,
+ crypto::{Key, Nonce, SecretKey, secretbox},
+ http::{ApiConfig, Auth},
+};
+use ente_photos::dedup::{
+ Addition, RemovalCollection, RemovalError, RemovalFile, RemovalPlan, execute_removal,
+};
+use mockito::{Matcher, Server};
+
+fn session(origin: String) -> Result<Session, ente_core::http::Error> {
+ Session::new(
+ ApiConfig {
+ origin,
+ client_package: Some("io.ente.photos".into()),
+ client_version: None,
+ user_agent: None,
+ auth: Some(Auth::User("test-token".into())),
+ },
+ 1,
+ Key::generate(),
+ Key::generate(),
+ SecretKey::generate(),
+ )
+}
+
+fn fixture() -> (RemovalPlan, Vec<RemovalCollection>, Vec<RemovalFile>) {
+ (
+ RemovalPlan {
+ additions: vec![
+ Addition {
+ collection_id: 11,
+ file_ids: vec![1],
+ },
+ Addition {
+ collection_id: 12,
+ file_ids: vec![1],
+ },
+ ],
+ trash_ids: vec![2],
+ },
+ vec![
+ RemovalCollection {
+ id: 11,
+ key: Key::generate(),
+ },
+ RemovalCollection {
+ id: 12,
+ key: Key::generate(),
+ },
+ ],
+ vec![
+ RemovalFile {
+ id: 1,
+ collection_id: 11,
+ key: Key::generate(),
+ },
+ RemovalFile {
+ id: 2,
+ collection_id: 12,
+ key: Key::generate(),
+ },
+ ],
+ )
+}
+
+#[tokio::test]
+async fn authenticates_encrypts_keys_and_finishes_all_additions_before_trash() {
+ let mut server = Server::new_async().await;
+ let (plan, collections, files) = fixture();
+ let events = Arc::new(Mutex::new(Vec::new()));
+ let mut mocks = Vec::new();
+ for c in &collections {
+ let key = Key::from_bytes(*c.key.as_bytes());
+ let file_key = *files[0].key.as_bytes();
+ let id = c.id;
+ let events = events.clone();
+ mocks.push(
+ server
+ .mock("POST", "/collections/add-files")
+ .match_header("x-auth-token", "test-token")
+ .match_header("x-client-package", "io.ente.photos")
+ .match_body(Matcher::PartialJson(
+ serde_json::json!({"collectionID": id}),
+ ))
+ .match_request(move |request| {
+ let payload: serde_json::Value =
+ serde_json::from_slice(request.body().unwrap()).unwrap();
+ let f = &payload["files"][0];
+ let plaintext = secretbox::decrypt(
+ &b64::decode(f["encryptedKey"].as_str().unwrap()).unwrap(),
+ &Nonce::try_from_slice(
+ &b64::decode(f["keyDecryptionNonce"].as_str().unwrap()).unwrap(),
+ )
+ .unwrap(),
+ &key,
+ )
+ .unwrap();
+ events.lock().unwrap().push(id);
+ f["id"] == 1 && plaintext == file_key
+ })
+ .with_status(200)
+ .create_async()
+ .await,
+ );
+ }
+ let trash_events = events.clone();
+ let trash = server
+ .mock("POST", "/files/trash")
+ .match_header("x-auth-token", "test-token")
+ .match_body(Matcher::Json(
+ serde_json::json!({"items": [{"fileID": 2, "collectionID": 12}]}),
+ ))
+ .match_request(move |_| *trash_events.lock().unwrap() == vec![11, 12])
+ .with_status(200)
+ .create_async()
+ .await;
+ let mut progress = Vec::new();
+ execute_removal(
+ &session(server.url()).unwrap(),
+ &plan,
+ &collections,
+ &files,
+ |p| progress.push((p.completed, p.total)),
+ )
+ .await
+ .unwrap();
+ assert_eq!(progress, vec![(1, 3), (2, 3), (3, 3)]);
+ for mock in mocks {
+ mock.assert_async().await;
+ }
+ trash.assert_async().await;
+}
+
+#[tokio::test]
+async fn does_not_trash_when_a_later_album_addition_fails() {
+ let mut server = Server::new_async().await;
+ let (plan, collections, files) = fixture();
+ let first = server
+ .mock("POST", "/collections/add-files")
+ .match_body(Matcher::PartialJson(
+ serde_json::json!({"collectionID": 11}),
+ ))
+ .with_status(200)
+ .create_async()
+ .await;
+ let second = server
+ .mock("POST", "/collections/add-files")
+ .match_body(Matcher::PartialJson(
+ serde_json::json!({"collectionID": 12}),
+ ))
+ .with_status(503)
+ .create_async()
+ .await;
+ let trash = server
+ .mock("POST", "/files/trash")
+ .expect(0)
+ .create_async()
+ .await;
+ let mut progress = Vec::new();
+ assert!(
+ execute_removal(
+ &session(server.url()).unwrap(),
+ &plan,
+ &collections,
+ &files,
+ |p| { progress.push(p.completed) }
+ )
+ .await
+ .is_err()
+ );
+ assert_eq!(progress, vec![1]);
+ first.assert_async().await;
+ second.assert_async().await;
+ trash.assert_async().await;
+}
+
+#[tokio::test]
+async fn validates_every_file_before_the_first_mutation() {
+ let mut server = Server::new_async().await;
+ let (mut plan, collections, files) = fixture();
+ plan.trash_ids.push(999);
+ let addition = server
+ .mock("POST", "/collections/add-files")
+ .expect(0)
+ .create_async()
+ .await;
+ let mut progress = Vec::new();
+ assert!(
+ execute_removal(
+ &session(server.url()).unwrap(),
+ &plan,
+ &collections,
+ &files,
+ |p| { progress.push(p) }
+ )
+ .await
+ .is_err()
+ );
+ assert!(progress.is_empty());
+ addition.assert_async().await;
+}
+
+#[tokio::test]
+async fn validates_later_additions_before_the_first_mutation() {
+ for invalid in ["collection", "file", "trashed file"] {
+ let mut server = Server::new_async().await;
+ let (mut plan, collections, files) = fixture();
+ match invalid {
+ "collection" => plan.additions[1].collection_id = 999,
+ "file" => plan.additions[1].file_ids.push(999),
+ "trashed file" => plan.additions[1].file_ids.push(2),
+ _ => unreachable!(),
+ }
+ let addition = server
+ .mock("POST", "/collections/add-files")
+ .expect(0)
+ .create_async()
+ .await;
+ let trash = server
+ .mock("POST", "/files/trash")
+ .expect(0)
+ .create_async()
+ .await;
+ let mut progress = Vec::new();
+ let error = execute_removal(
+ &session(server.url()).unwrap(),
+ &plan,
+ &collections,
+ &files,
+ |p| progress.push(p),
+ )
+ .await
+ .unwrap_err();
+ match invalid {
+ "collection" => assert!(matches!(error, RemovalError::MissingCollection(999))),
+ "file" => assert!(matches!(error, RemovalError::MissingFile(999))),
+ "trashed file" => assert!(matches!(error, RemovalError::InvalidPlan(_))),
+ _ => unreachable!(),
+ }
+ assert!(progress.is_empty(), "{invalid}");
+ addition.assert_async().await;
+ trash.assert_async().await;
+ }
+}
+
+#[tokio::test]
+async fn batches_requests_and_propagates_trash_failure_without_completion() {
+ let mut server = Server::new_async().await;
+ let collections = vec![RemovalCollection {
+ id: 11,
+ key: Key::generate(),
+ }];
+ let files: Vec<_> = (1..=1002)
+ .map(|id| RemovalFile {
+ id,
+ collection_id: 11,
+ key: Key::generate(),
+ })
+ .collect();
+ let plan = RemovalPlan {
+ additions: vec![],
+ trash_ids: (1..=1002).collect(),
+ };
+ let first = server
+ .mock("POST", "/files/trash")
+ .match_request(|r| {
+ serde_json::from_slice::<serde_json::Value>(r.body().unwrap()).unwrap()["items"]
+ .as_array()
+ .unwrap()
+ .len()
+ == 1000
+ })
+ .with_status(200)
+ .create_async()
+ .await;
+ let second = server
+ .mock("POST", "/files/trash")
+ .match_request(|r| {
+ serde_json::from_slice::<serde_json::Value>(r.body().unwrap()).unwrap()["items"]
+ .as_array()
+ .unwrap()
+ .len()
+ == 2
+ })
+ .with_status(503)
+ .create_async()
+ .await;
+ let mut progress = Vec::new();
+ assert!(
+ execute_removal(
+ &session(server.url()).unwrap(),
+ &plan,
+ &collections,
+ &files,
+ |p| { progress.push(p) }
+ )
+ .await
+ .is_err()
+ );
+ assert!(progress.is_empty());
+ first.assert_async().await;
+ second.assert_async().await;
+}
+
+#[tokio::test]
+async fn batches_large_album_additions_before_trashing() {
+ let mut server = Server::new_async().await;
+ let collections = vec![RemovalCollection {
+ id: 11,
+ key: Key::generate(),
+ }];
+ let files: Vec<_> = (1..=1002)
+ .map(|id| RemovalFile {
+ id,
+ collection_id: 11,
+ key: Key::generate(),
+ })
+ .collect();
+ let plan = RemovalPlan {
+ additions: vec![Addition {
+ collection_id: 11,
+ file_ids: (1..=1001).collect(),
+ }],
+ trash_ids: vec![1002],
+ };
+ let batches = Arc::new(Mutex::new(Vec::new()));
+ let mut mocks = Vec::new();
+ for (start, size) in [(1, 1000), (1001, 1)] {
+ let seen = batches.clone();
+ let collection_key = Key::from_bytes(*collections[0].key.as_bytes());
+ let expected_keys: Vec<_> = files[(start - 1)..(start - 1 + size)]
+ .iter()
+ .map(|file| *file.key.as_bytes())
+ .collect();
+ mocks.push(
+ server
+ .mock("POST", "/collections/add-files")
+ .match_request(move |r| {
+ let value: serde_json::Value =
+ serde_json::from_slice(r.body().unwrap()).unwrap();
+ let batch = value["files"].as_array().unwrap();
+ if batch.len() != size {
+ return false;
+ }
+ assert_eq!(value["collectionID"], 11);
+ for (index, (file, expected_key)) in
+ batch.iter().zip(&expected_keys).enumerate()
+ {
+ assert_eq!(file["id"], start + index);
+ let plaintext = secretbox::decrypt(
+ &b64::decode(file["encryptedKey"].as_str().unwrap()).unwrap(),
+ &Nonce::try_from_slice(
+ &b64::decode(file["keyDecryptionNonce"].as_str().unwrap()).unwrap(),
+ )
+ .unwrap(),
+ &collection_key,
+ )
+ .unwrap();
+ assert_eq!(&plaintext, expected_key);
+ }
+ seen.lock().unwrap().push(size);
+ true
+ })
+ .with_status(200)
+ .create_async()
+ .await,
+ );
+ }
+ let seen = batches.clone();
+ let trash = server
+ .mock("POST", "/files/trash")
+ .match_request(move |_| *seen.lock().unwrap() == vec![1000, 1])
+ .with_status(200)
+ .create_async()
+ .await;
+ let mut progress = Vec::new();
+ execute_removal(
+ &session(server.url()).unwrap(),
+ &plan,
+ &collections,
+ &files,
+ |p| progress.push((p.completed, p.total)),
+ )
+ .await
+ .unwrap();
+ assert_eq!(progress, vec![(1, 2), (2, 2)]);
+ for mock in mocks {
+ mock.assert_async().await;
+ }
+ trash.assert_async().await;
+}
+
+#[tokio::test]
+async fn plans_and_executes_retention_rules_without_a_client_supplied_plan() {
+ use ente_photos::dedup::{Duplicate, remove_duplicates};
+
+ let cases = [
+ (
+ "caption before edits",
+ Some("caption"),
+ Some("renamed"),
+ None,
+ None,
+ 2,
+ ),
+ ("renamed copy", None, None, Some("renamed"), None, 3),
+ ("edited date", None, None, None, Some(123), 3),
+ ("caption tie", Some("caption"), None, None, None, 1),
+ ("edit tie", None, Some("renamed"), None, Some(123), 1),
+ (
+ "empty name overrides date",
+ None,
+ None,
+ Some(""),
+ Some(123),
+ 1,
+ ),
+ ("fallback", None, None, None, None, 1),
+ ];
+ for (name, caption, first_name, last_name, last_time, retained) in cases {
+ let mut server = Server::new_async().await;
+ let mut groups = vec![vec![
+ Duplicate {
+ file_id: 1,
+ collection_ids: vec![11],
+ edited_name: first_name.map(String::from),
+ ..Default::default()
+ },
+ Duplicate {
+ file_id: 2,
+ collection_ids: vec![12],
+ caption: caption.map(String::from),
+ ..Default::default()
+ },
+ Duplicate {
+ file_id: 3,
+ collection_ids: vec![13],
+ edited_name: last_name.map(String::from),
+ edited_time: last_time,
+ ..Default::default()
+ },
+ ]];
+ if name == "caption tie" {
+ groups[0][0].caption = Some("first".into());
+ }
+ let collections: Vec<_> = (11..=13)
+ .map(|id| RemovalCollection {
+ id,
+ key: Key::generate(),
+ })
+ .collect();
+ let files: Vec<_> = (1..=3)
+ .map(|id| RemovalFile {
+ id,
+ collection_id: id + 10,
+ key: Key::generate(),
+ })
+ .collect();
+ let mut additions = Vec::new();
+ for collection in &collections {
+ if collection.id == retained + 10 {
+ continue;
+ }
+ let key = Key::from_bytes(*collection.key.as_bytes());
+ let expected_key = *files[(retained - 1) as usize].key.as_bytes();
+ additions.push(
+ server
+ .mock("POST", "/collections/add-files")
+ .match_body(Matcher::PartialJson(
+ serde_json::json!({"collectionID": collection.id}),
+ ))
+ .match_request(move |request| {
+ let body: serde_json::Value =
+ serde_json::from_slice(request.body().unwrap()).unwrap();
+ let f = &body["files"][0];
+ f["id"] == retained
+ && secretbox::decrypt(
+ &b64::decode(f["encryptedKey"].as_str().unwrap()).unwrap(),
+ &Nonce::try_from_slice(
+ &b64::decode(f["keyDecryptionNonce"].as_str().unwrap())
+ .unwrap(),
+ )
+ .unwrap(),
+ &key,
+ )
+ .unwrap()
+ == expected_key
+ })
+ .with_status(200)
+ .create_async()
+ .await,
+ );
+ }
+ let expected: Vec<_> = (1..=3).filter(|id| *id != retained).collect();
+ let items: Vec<_> = expected
+ .iter()
+ .map(|id| serde_json::json!({"fileID": id, "collectionID": id + 10}))
+ .collect();
+ let trash = server
+ .mock("POST", "/files/trash")
+ .match_body(Matcher::Json(serde_json::json!({"items": items})))
+ .with_status(200)
+ .create_async()
+ .await;
+ let mut progress = Vec::new();
+ let result = remove_duplicates(
+ &session(server.url()).unwrap(),
+ &groups,
+ &collections,
+ &files,
+ |p| progress.push((p.completed, p.total)),
+ )
+ .await
+ .unwrap();
+ assert_eq!(result, expected, "{name}");
+ assert_eq!(progress, vec![(1, 3), (2, 3), (3, 3)], "{name}");
+ for addition in additions {
+ addition.assert_async().await;
+ }
+ trash.assert_async().await;
+ }
+}
+
+#[tokio::test]
+async fn combined_removal_batches_groups_and_skips_empty_input() {
+ use ente_photos::dedup::{Duplicate, remove_duplicates};
+
+ let mut server = Server::new_async().await;
+ let collections = vec![RemovalCollection {
+ id: 11,
+ key: Key::generate(),
+ }];
+ let files: Vec<_> = (1..=4)
+ .map(|id| RemovalFile {
+ id,
+ collection_id: if id % 2 == 0 { 11 } else { 12 },
+ key: Key::generate(),
+ })
+ .collect();
+ let groups = vec![
+ vec![
+ Duplicate {
+ file_id: 1,
+ collection_ids: vec![12],
+ ..Default::default()
+ },
+ Duplicate {
+ file_id: 2,
+ collection_ids: vec![11],
+ ..Default::default()
+ },
+ ],
+ vec![
+ Duplicate {
+ file_id: 3,
+ collection_ids: vec![12],
+ ..Default::default()
+ },
+ Duplicate {
+ file_id: 4,
+ collection_ids: vec![11],
+ ..Default::default()
+ },
+ ],
+ ];
+ let addition = server
+ .mock("POST", "/collections/add-files")
+ .match_request(|r| {
+ let body: serde_json::Value = serde_json::from_slice(r.body().unwrap()).unwrap();
+ body["collectionID"] == 11
+ && body["files"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|f| f["id"].as_i64().unwrap())
+ .collect::<Vec<_>>()
+ == vec![1, 3]
+ })
+ .with_status(200)
+ .create_async()
+ .await;
+ let trash = server.mock("POST", "/files/trash").match_body(Matcher::Json(serde_json::json!({"items": [{"fileID":2,"collectionID":11},{"fileID":4,"collectionID":11}]}))).with_status(200).create_async().await;
+ let session = session(server.url()).unwrap();
+ let mut progress = Vec::new();
+ assert_eq!(
+ remove_duplicates(&session, &groups, &collections, &files, |p| progress
+ .push((p.completed, p.total)))
+ .await
+ .unwrap(),
+ vec![2, 4]
+ );
+ assert_eq!(progress, vec![(1, 2), (2, 2)]);
+ addition.assert_async().await;
+ trash.assert_async().await;
+ progress.clear();
+ assert!(
+ remove_duplicates(&session, &[], &[], &[], |p| progress
+ .push((p.completed, p.total)))
+ .await
+ .unwrap()
+ .is_empty()
+ );
+ assert!(progress.is_empty());
+ addition.assert_async().await;
+ trash.assert_async().await;
+}
diff --git a/web/apps/photos/src/services/dedup.ts b/web/apps/photos/src/services/dedup.ts
index 7fac57b866..549999725d 100644
--- a/web/apps/photos/src/services/dedup.ts
+++ b/web/apps/photos/src/services/dedup.ts
@@ -7,13 +7,13 @@ import {
metadataHash,
} from "ente-media/file-metadata";
import {
- addToCollection,
createCollectionNameByID,
- moveToTrash,
+ forgetPendingFavoriteFileIDs,
savedNormalCollections,
} from "ente-new/photos/services/collection";
import { savedCollectionFiles } from "ente-new/photos/services/photos-fdb";
-import { findDuplicates, planDuplicateRemoval } from "ente-photos-wasm";
+import { executeDuplicateRemoval, findDuplicates } from "ente-photos-wasm";
+import { ensureAuthenticatedSession } from "./authenticated-session";
export interface DuplicateGroup {
id: string;
@@ -94,47 +94,45 @@ export const removeSelectedDuplicateGroups = async (
) => {
const selectedDuplicateGroups = duplicateGroups.filter((g) => g.isSelected);
+ if (!selectedDuplicateGroups.length) {
+ await onRemoteFilesPull();
+ onProgress(100);
+ return new Set<string>();
+ }
+
const filesByID = new Map(
selectedDuplicateGroups.flatMap((group) =>
group.items.map(({ file }) => [file.id, file] as const),
),
);
- const plan = await planDuplicateRemoval({
- groups: selectedDuplicateGroups.map((group) =>
- group.items.map(({ file, collectionIDs }) => ({
- fileId: file.id,
- collectionIds: [...collectionIDs],
- caption: FilePublicMagicMetadataData.shape.caption.parse(
- file.pubMagicMetadata?.data.caption,
- ),
- editedName: file.pubMagicMetadata?.data.editedName,
- editedTime: file.pubMagicMetadata?.data.editedTime,
- })),
- ),
- });
-
- let np = 0;
- const ntotal =
- plan.additions.length + (plan.trashIds.length ? 1 : 0) + /* sync */ 1;
- const tickProgress = () => onProgress((++np / ntotal) * 100);
-
const collections = await savedNormalCollections();
- const collectionsByID = new Map(collections.map((c) => [c.id, c]));
- for (const { collectionId, fileIds } of plan.additions) {
- await addToCollection(
- collectionsByID.get(collectionId)!,
- fileIds.map((id) => filesByID.get(id)!),
- );
- tickProgress();
- }
-
- if (plan.trashIds.length) {
- await moveToTrash(plan.trashIds.map((id) => filesByID.get(id)!));
- tickProgress();
- }
+ const { trashIds } = await executeDuplicateRemoval(
+ await ensureAuthenticatedSession(),
+ {
+ groups: selectedDuplicateGroups.map((group) =>
+ group.items.map(({ file, collectionIDs }) => ({
+ fileId: file.id,
+ collectionIds: [...collectionIDs],
+ caption: FilePublicMagicMetadataData.shape.caption.parse(
+ file.pubMagicMetadata?.data.caption,
+ ),
+ editedName: file.pubMagicMetadata?.data.editedName,
+ editedTime: file.pubMagicMetadata?.data.editedTime,
+ })),
+ ),
+ collections: collections.map(({ id, key }) => ({ id, key })),
+ files: [...filesByID.values()].map(({ id, collectionID, key }) => ({
+ id,
+ collectionId: collectionID,
+ key,
+ })),
+ },
+ (completed, total) => onProgress((completed / (total + 1)) * 100),
+ );
+ forgetPendingFavoriteFileIDs(trashIds);
await onRemoteFilesPull();
- tickProgress();
+ onProgress(100);
return new Set(selectedDuplicateGroups.map((g) => g.id));
};
diff --git a/web/apps/photos/tests/dedup.test.ts b/web/apps/photos/tests/dedup.test.ts
index 3d2942394e..d448a42266 100644
--- a/web/apps/photos/tests/dedup.test.ts
+++ b/web/apps/photos/tests/dedup.test.ts
@@ -5,22 +5,30 @@ import {
removeSelectedDuplicateGroups,
type DuplicateGroup,
} from "../src/services/dedup";
-
const {
- addToCollection,
- moveToTrash,
+ executeDuplicateRemoval,
savedNormalCollections,
savedCollectionFiles,
+ forgetPendingFavoriteFileIDs,
+ session,
} = vi.hoisted(() => ({
- addToCollection: vi.fn(),
- moveToTrash: vi.fn(),
+ executeDuplicateRemoval:
+ vi.fn<typeof import("ente-photos-wasm").executeDuplicateRemoval>(),
savedNormalCollections: vi.fn(),
savedCollectionFiles: vi.fn(),
+ forgetPendingFavoriteFileIDs: vi.fn(),
+ session: {},
+}));
+vi.mock("ente-photos-wasm", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("ente-photos-wasm")>()),
+ executeDuplicateRemoval,
+}));
+vi.mock("../src/services/authenticated-session", () => ({
+ ensureAuthenticatedSession: () => Promise.resolve(session),
}));
vi.mock("ente-new/photos/services/collection", () => ({
- addToCollection,
- moveToTrash,
savedNormalCollections,
+ forgetPendingFavoriteFileIDs,
createCollectionNameByID: (collections: { id: number; name: string }[]) =>
new Map(collections.map(({ id, name }) => [id, name])),
}));
@@ -30,7 +38,6 @@ vi.mock("ente-new/photos/services/photos-fdb", () => ({
vi.mock("ente-accounts/services/user", () => ({
ensureLocalUser: () => ({ id: 1 }),
}));
-
const file = (
id: number,
collectionID: number,
@@ -48,6 +55,7 @@ const file = (
describe("deduceDuplicates through Rust WASM", () => {
beforeEach(() => {
vi.resetAllMocks();
+ executeDuplicateRemoval.mockResolvedValue({ trashIds: [1] });
savedNormalCollections.mockResolvedValue([
{ id: 11, owner: { id: 1 }, name: "B" },
{ id: 12, owner: { id: 1 }, name: "A" },
@@ -91,11 +99,12 @@ describe("deduceDuplicates through Rust WASM", () => {
},
]);
await removeSelectedDuplicateGroups(groups, vi.fn(), vi.fn());
- expect(addToCollection).toHaveBeenCalledExactlyOnceWith(
- expect.objectContaining({ id: 11 }),
- [second],
- );
- expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([first]);
+ expect(executeDuplicateRemoval.mock.calls[0]![1].groups).toEqual([
+ [
+ expect.objectContaining({ fileId: 2, collectionIds: [12] }),
+ expect.objectContaining({ fileId: 1, collectionIds: [11, 12] }),
+ ],
+ ]);
});
test("matches legacy Live Photo hashes despite different ZIP sizes and skips groups without sizes", async () => {
@@ -163,227 +172,181 @@ const duplicateGroup = (): DuplicateGroup => ({
isSelected: true,
});
-const collections = [{ id: 11 }, { id: 12 }, { id: 13 }];
+const collections = [
+ { id: 11, key: "album-11" },
+ { id: 12, key: "album-12" },
+ { id: 13, key: "album-13" },
+];
describe("removeSelectedDuplicateGroups", () => {
beforeEach(() => {
vi.resetAllMocks();
savedNormalCollections.mockResolvedValue(collections);
+ executeDuplicateRemoval.mockResolvedValue({ trashIds: [1, 3] });
});
-
test.each([123, 0])(
- "normalizes legacy cached numeric caption %s before planning in Rust",
+ "normalizes numeric caption %s before Rust planning",
async (caption) => {
const group = duplicateGroup();
- const retained = group.items[1]!.file;
- Object.assign(retained.pubMagicMetadata!.data, { caption });
+ Object.assign(group.items[1]!.file.pubMagicMetadata!.data, {
+ caption,
+ });
const refresh = vi.fn();
-
await expect(
removeSelectedDuplicateGroups([group], vi.fn(), refresh),
).resolves.toEqual(new Set([group.id]));
- expect(addToCollection.mock.calls).toEqual([
- [collections[0], [retained]],
- [collections[2], [retained]],
- ]);
- expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([
- group.items[0]!.file,
- group.items[2]!.file,
- ]);
+ expect(
+ executeDuplicateRemoval.mock.calls[0]![1].groups[0]![1],
+ ).toMatchObject({ fileId: 2, caption: String(caption) });
expect(refresh).toHaveBeenCalledOnce();
},
);
-
- test.each([
- {
- label: "caption before edits",
- metadata: [{ editedName: "renamed" }, { caption: "caption" }, {}],
- retainedId: 2,
- },
- {
- label: "renamed copy",
- metadata: [{}, {}, { editedName: "renamed" }],
- retainedId: 3,
- },
- {
- label: "edited timestamp",
- metadata: [{}, { editedTime: 123 }, {}],
- retainedId: 2,
- },
- {
- label: "first of equal captions",
- metadata: [{}, { caption: "one" }, { caption: "two" }],
- retainedId: 2,
- },
- {
- label: "first of equal edits",
- metadata: [{}, { editedName: "one" }, { editedTime: 123 }],
- retainedId: 2,
- },
- {
- label: "empty caption and null caption",
- metadata: [{}, { caption: "" }, { caption: null }],
- retainedId: 1,
- },
- {
- label: "empty name overrides edited timestamp",
- metadata: [{}, { editedName: "", editedTime: 123 }, {}],
- retainedId: 1,
- },
- ])(
- "retains the $label through Rust WASM",
- async ({ metadata, retainedId }) => {
- const group = duplicateGroup();
- group.items.forEach(({ file }, index) => {
- Object.assign(file, {
- pubMagicMetadata: { data: metadata[index] },
- });
+ test("passes only selected groups and their files, saved album keys, and the shared session to Rust", async () => {
+ const group = duplicateGroup();
+ for (const item of group.items)
+ Object.assign(item.file, {
+ collectionID: [...item.collectionIDs][0],
+ key: `file-${item.file.id}`,
});
- await removeSelectedDuplicateGroups([group], vi.fn(), vi.fn());
- expect(moveToTrash).toHaveBeenCalledExactlyOnceWith(
- group.items
- .filter(({ file }) => file.id !== retainedId)
- .map(({ file }) => file),
- );
- },
- );
-
- test("batches retained copies from different groups into each missing album", async () => {
- const first = duplicateGroup();
- const second = duplicateGroup();
- second.id = "group-2";
- for (const { file } of second.items) file.id += 3;
const refresh = vi.fn();
- await expect(
- removeSelectedDuplicateGroups([first, second], vi.fn(), refresh),
- ).resolves.toEqual(new Set([first.id, second.id]));
- const retained = [first.items[1]!.file, second.items[1]!.file];
- expect(addToCollection.mock.calls).toEqual([
- [collections[0], retained],
- [collections[2], retained],
- ]);
- expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([
- first.items[0]!.file,
- first.items[2]!.file,
- second.items[0]!.file,
- second.items[2]!.file,
+ await removeSelectedDuplicateGroups(
+ [
+ group,
+ { ...duplicateGroup(), id: "untouched", isSelected: false },
+ ],
+ vi.fn(),
+ refresh,
+ );
+ expect(executeDuplicateRemoval).toHaveBeenCalledExactlyOnceWith(
+ session,
+ {
+ groups: [
+ [
+ {
+ fileId: 1,
+ collectionIds: [11, 12],
+ caption: undefined,
+ editedName: undefined,
+ editedTime: undefined,
+ },
+ {
+ fileId: 2,
+ collectionIds: [12],
+ caption: "Keep this copy",
+ editedName: undefined,
+ editedTime: undefined,
+ },
+ {
+ fileId: 3,
+ collectionIds: [13],
+ caption: undefined,
+ editedName: undefined,
+ editedTime: undefined,
+ },
+ ],
+ ],
+ collections,
+ files: [
+ { id: 1, collectionId: 11, key: "file-1" },
+ { id: 2, collectionId: 12, key: "file-2" },
+ { id: 3, collectionId: 13, key: "file-3" },
+ ],
+ },
+ expect.any(Function),
+ );
+ expect(forgetPendingFavoriteFileIDs).toHaveBeenCalledExactlyOnceWith([
+ 1, 3,
]);
expect(refresh).toHaveBeenCalledOnce();
});
-
- test("rejects invalid planner input before modifying albums or trash", async () => {
- const group = duplicateGroup();
- group.items[1]!.file.id = Number.MAX_SAFE_INTEGER + 1;
- const refresh = vi.fn();
- await expect(
- removeSelectedDuplicateGroups([group], vi.fn(), refresh),
- ).rejects.toBeInstanceOf(Error);
- expect(addToCollection).not.toHaveBeenCalled();
- expect(moveToTrash).not.toHaveBeenCalled();
- expect(refresh).not.toHaveBeenCalled();
- });
-
- test("preserves album membership before trashing selected duplicates and awaits gallery refresh", async () => {
- const group = duplicateGroup();
- const untouched = {
- ...duplicateGroup(),
- id: "unselected",
- isSelected: false,
- };
+ test("reports Rust execution progress and awaits refresh before completion", async () => {
const progress = vi.fn();
- let finishRefresh!: () => void;
+ executeDuplicateRemoval.mockImplementation(
+ (_session, _input, onProgress) => {
+ onProgress(1, 3);
+ onProgress(2, 3);
+ onProgress(3, 3);
+ return Promise.resolve({ trashIds: [1, 3] });
+ },
+ );
+ let finish!: () => void;
const refresh = vi.fn(
() =>
new Promise<void>((resolve) => {
- finishRefresh = resolve;
+ finish = resolve;
}),
);
const removal = removeSelectedDuplicateGroups(
- [group, untouched],
+ [duplicateGroup()],
progress,
refresh,
);
await vi.waitFor(() => expect(refresh).toHaveBeenCalledOnce());
- expect(addToCollection.mock.calls).toEqual([
- [collections[0], [group.items[1]!.file]],
- [collections[2], [group.items[1]!.file]],
- ]);
- expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([
- group.items[0]!.file,
- group.items[2]!.file,
- ]);
- expect(addToCollection.mock.invocationCallOrder[1]).toBeLessThan(
- moveToTrash.mock.invocationCallOrder[0]!,
- );
- expect(moveToTrash.mock.invocationCallOrder[0]).toBeLessThan(
- refresh.mock.invocationCallOrder[0]!,
- );
expect(progress.mock.calls).toEqual([[25], [50], [75]]);
- finishRefresh();
- await expect(removal).resolves.toEqual(new Set([group.id]));
+ expect(
+ forgetPendingFavoriteFileIDs.mock.invocationCallOrder[0],
+ ).toBeLessThan(refresh.mock.invocationCallOrder[0]!);
+ finish();
+ await expect(removal).resolves.toEqual(new Set(["group-1"]));
expect(progress).toHaveBeenLastCalledWith(100);
});
-
- test("does not trash files when preserving album membership fails", async () => {
- const failure = new Error("Add failed");
- addToCollection.mockRejectedValue(failure);
+ test("does not clear local state or refresh when Rust execution fails", async () => {
+ const failure = new Error("Rust request failed");
+ executeDuplicateRemoval.mockRejectedValue(failure);
const refresh = vi.fn();
await expect(
removeSelectedDuplicateGroups([duplicateGroup()], vi.fn(), refresh),
).rejects.toBe(failure);
- expect(moveToTrash).not.toHaveBeenCalled();
+ expect(forgetPendingFavoriteFileIDs).not.toHaveBeenCalled();
expect(refresh).not.toHaveBeenCalled();
});
-
- test("does not trash any files if a later album addition fails", async () => {
- const failure = new Error("Second album failed");
- addToCollection
- .mockResolvedValueOnce(undefined)
- .mockRejectedValueOnce(failure);
- const refresh = vi.fn();
- await expect(
- removeSelectedDuplicateGroups([duplicateGroup()], vi.fn(), refresh),
- ).rejects.toBe(failure);
- expect(addToCollection).toHaveBeenCalledTimes(2);
- expect(moveToTrash).not.toHaveBeenCalled();
- expect(refresh).not.toHaveBeenCalled();
- });
-
- test("does not refresh when trashing fails", async () => {
- const failure = new Error("Trash failed");
- moveToTrash.mockRejectedValue(failure);
- const refresh = vi.fn();
- await expect(
- removeSelectedDuplicateGroups([duplicateGroup()], vi.fn(), refresh),
- ).rejects.toBe(failure);
- expect(refresh).not.toHaveBeenCalled();
- });
-
- test("does not report completion when gallery refresh fails", async () => {
+ test("propagates refresh failures without reporting completion", async () => {
const failure = new Error("Refresh failed");
- const refresh = vi.fn().mockRejectedValue(failure);
const progress = vi.fn();
await expect(
- removeSelectedDuplicateGroups(
- [duplicateGroup()],
- progress,
- refresh,
+ removeSelectedDuplicateGroups([duplicateGroup()], progress, () =>
+ Promise.reject(failure),
),
).rejects.toBe(failure);
expect(progress).not.toHaveBeenCalledWith(100);
});
-
- test("only refreshes when no groups are selected", async () => {
+ test("passes multiple selected groups in one call and uses Rust's returned trash IDs", async () => {
+ const first = duplicateGroup();
+ const second = duplicateGroup();
+ second.id = "group-2";
+ for (const { file } of second.items) file.id += 3;
+ executeDuplicateRemoval.mockResolvedValue({ trashIds: [1, 3, 4, 6] });
+ await expect(
+ removeSelectedDuplicateGroups([first, second], vi.fn(), vi.fn()),
+ ).resolves.toEqual(new Set([first.id, second.id]));
+ expect(executeDuplicateRemoval).toHaveBeenCalledOnce();
+ expect(
+ executeDuplicateRemoval.mock.calls[0]![1].groups.map((g) =>
+ g.map((i) => i.fileId),
+ ),
+ ).toEqual([
+ [1, 2, 3],
+ [4, 5, 6],
+ ]);
+ expect(
+ executeDuplicateRemoval.mock.calls[0]![1].files.map((f) => f.id),
+ ).toEqual([1, 2, 3, 4, 5, 6]);
+ expect(forgetPendingFavoriteFileIDs).toHaveBeenCalledExactlyOnceWith([
+ 1, 3, 4, 6,
+ ]);
+ });
+ test("refreshes and reports completion when no groups are selected", async () => {
const refresh = vi.fn();
+ const progress = vi.fn();
await expect(
removeSelectedDuplicateGroups(
[{ ...duplicateGroup(), isSelected: false }],
- vi.fn(),
+ progress,
refresh,
),
).resolves.toEqual(new Set());
- expect(addToCollection).not.toHaveBeenCalled();
- expect(moveToTrash).not.toHaveBeenCalled();
+ expect(forgetPendingFavoriteFileIDs).not.toHaveBeenCalled();
expect(refresh).toHaveBeenCalledOnce();
+ expect(progress).toHaveBeenCalledExactlyOnceWith(100);
});
});
diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.ts
index 23d48e3ebe..6aa8a67021 100644
--- a/web/packages/new/photos/services/collection.ts
+++ b/web/packages/new/photos/services/collection.ts
@@ -563,7 +563,7 @@ const splitPendingFavoriteFiles = (files: EnteFile[], userID: number) => {
return { pendingFiles, remainingFiles };
};
-const forgetPendingFavoriteFileIDs = (fileIDs: number[]) => {
+export const forgetPendingFavoriteFileIDs = (fileIDs: number[]) => {
if (!fileIDs.length) return;
const deletedFileIDs = new Set(fileIDs);
diff --git a/web/packages/wasm/photos/index.ts b/web/packages/wasm/photos/index.ts
index 51fa827c94..ab6ff61281 100644
--- a/web/packages/wasm/photos/index.ts
+++ b/web/packages/wasm/photos/index.ts
@@ -1,9 +1,9 @@
import type {
EncryptedBox,
+ ExecuteDuplicateRemovalInput,
FindDuplicatesInput,
LocationIndex,
OpenSessionInput,
- PlanDuplicateRemovalInput,
Session,
WrappedRootContactKey,
} from "./pkg/ente_photos_wasm";
@@ -20,8 +20,11 @@ const wasm = () => import("./pkg/ente_photos_wasm");
export const findDuplicates = async (input: FindDuplicatesInput) =>
(await wasm()).photosFindDuplicates(input);
-export const planDuplicateRemoval = async (input: PlanDuplicateRemovalInput) =>
- (await wasm()).photosPlanDuplicateRemoval(input);
+export const executeDuplicateRemoval = async (
+ session: Session,
+ input: ExecuteDuplicateRemovalInput,
+ onProgress: (completed: number, total: number) => void,
+) => (await wasm()).photosExecuteDuplicateRemoval(session, input, onProgress);
export const createLocationIndex = async (
cities: Uint8Array,
diff --git a/web/packages/wasm/tests/bindings.test.ts b/web/packages/wasm/tests/bindings.test.ts
index ed0f3f886b..00d37e9069 100644
--- a/web/packages/wasm/tests/bindings.test.ts
+++ b/web/packages/wasm/tests/bindings.test.ts
@@ -93,6 +93,115 @@ test("Photos deletion keeps summary fallback and decrypts the challenge in Rust"
}
});
+test("Photos plans and executes duplicate removal in one WASM call", async () => {
+ const masterKey = await generateKey();
+ const input = await sessionKeyAttributes(masterKey);
+ const albumKey = await generateKey();
+ const fileKey = await generateKey();
+ const requests: string[] = [];
+ const progress: number[][] = [];
+ mockFetch(async (request) => {
+ expect(request.headers.get("X-Auth-Token")).toBe("test-token");
+ const path = new URL(request.url).pathname;
+ requests.push(path);
+ const body = (await request.json()) as {
+ collectionID: number;
+ files: {
+ id: number;
+ encryptedKey: string;
+ keyDecryptionNonce: string;
+ }[];
+ };
+ if (path === "/collections/add-files") {
+ expect(body.collectionID).toBe(11);
+ expect(body.files).toHaveLength(1);
+ expect(body.files[0]!.id).toBe(2);
+ expect(
+ await photos.decryptBox(
+ {
+ encryptedData: body.files[0]!.encryptedKey,
+ nonce: body.files[0]!.keyDecryptionNonce,
+ },
+ albumKey,
+ ),
+ ).toBe(fileKey);
+ } else {
+ expect(path).toBe("/files/trash");
+ expect(requests).toEqual([
+ "/collections/add-files",
+ "/files/trash",
+ ]);
+ expect(body).toEqual({ items: [{ fileID: 1, collectionID: 11 }] });
+ }
+ return Response.json({});
+ });
+ const session = await photos.openSession({
+ baseUrl: "http://localhost",
+ authToken: "test-token",
+ masterKeyB64: masterKey,
+ ...input,
+ });
+ const removal = {
+ groups: [
+ [
+ {
+ fileId: 1,
+ collectionIds: [11],
+ caption: undefined,
+ editedName: undefined,
+ editedTime: undefined,
+ },
+ {
+ fileId: 2,
+ collectionIds: [12],
+ caption: "keep",
+ editedName: undefined,
+ editedTime: undefined,
+ },
+ ],
+ ],
+ collections: [
+ { id: 11, key: albumKey },
+ { id: 99, key: await generateKey() },
+ ],
+ files: [
+ { id: 1, collectionId: 11, key: await generateKey() },
+ { id: 2, collectionId: 12, key: fileKey },
+ ],
+ };
+ try {
+ expect(
+ await photos.executeDuplicateRemoval(
+ session,
+ removal,
+ (completed, total) => progress.push([completed, total]),
+ ),
+ ).toEqual({ trashIds: [1] });
+ expect(requests).toEqual(["/collections/add-files", "/files/trash"]);
+ expect(progress).toEqual([
+ [1, 2],
+ [2, 2],
+ ]);
+ requests.length = 0;
+ const invalid = structuredClone(removal);
+ invalid.groups[0]![0]!.fileId = Number.MAX_SAFE_INTEGER + 1;
+ await expect(
+ photos.executeDuplicateRemoval(session, invalid, vi.fn()),
+ ).rejects.toBeInstanceOf(Error);
+ expect(requests).toEqual([]);
+ expect(
+ await photos.executeDuplicateRemoval(
+ session,
+ { groups: [], collections: [], files: [] },
+ vi.fn(),
+ ),
+ ).toEqual({ trashIds: [] });
+ expect(requests).toEqual([]);
+ } finally {
+ session.free();
+ }
+});
+
test.each(["classical", "post-quantum"])(
"Photos %s Cast payloads round-trip and enforce safe collection IDs",
async (encryption) => {