π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[photos][web] Share deduplication logic through Rust
AswinAsoka0cf0dde79e7
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex a10103296f..533097d96a 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -3068,6 +3068,7 @@ dependencies = ["ente-legacy","ente-location","ente-locker",+ "ente-photos","js-sys","log","md-5 0.11.0",diff --git a/rust/bindings/wasm/lib/Cargo.toml b/rust/bindings/wasm/lib/Cargo.tomlindex b50be7d25b..a9f0c99d01 100644--- a/rust/bindings/wasm/lib/Cargo.toml+++ b/rust/bindings/wasm/lib/Cargo.toml@@ -19,6 +19,7 @@ crypto-seal = ["crypto"]legacy = ["session", "dep:ente-legacy", "dep:wasm-bindgen-futures"]location = ["dep:ente-location", "dep:serde", "dep:serde-wasm-bindgen", "dep:tsify"]locker = ["session", "dep:ente-locker"]+photos = ["dep:ente-photos", "dep:serde", "dep:serde-wasm-bindgen", "dep:tsify"]prelogin = ["accounts", "crypto", "dep:wasm-bindgen-futures"]session = ["dep:ente-accounts",@@ -36,6 +37,7 @@ ente-core.workspace = trueente-legacy = { workspace = true, optional = true }ente-location = { workspace = true, optional = true }ente-locker = { workspace = true, optional = true }+ente-photos = { workspace = true, optional = true }js-sys.workspace = truelog.workspace = truemd-5 = { workspace = true, optional = true }diff --git a/rust/bindings/wasm/lib/src/lib.rs b/rust/bindings/wasm/lib/src/lib.rsindex 0844392602..16b37216a4 100644--- a/rust/bindings/wasm/lib/src/lib.rs+++ b/rust/bindings/wasm/lib/src/lib.rs@@ -17,6 +17,8 @@ pub mod location;#[cfg(feature = "locker")]pub mod locker;mod logging;+#[cfg(feature = "photos")]+pub mod photos;#[cfg(feature = "prelogin")]mod prelogin;#[cfg(feature = "session")]diff --git a/rust/bindings/wasm/lib/src/photos.rs b/rust/bindings/wasm/lib/src/photos.rsnew file mode 100644index 0000000000..b653213832--- /dev/null+++ b/rust/bindings/wasm/lib/src/photos.rs@@ -0,0 +1,170 @@+use ente_photos::dedup;+use serde::{Deserialize, Serialize};+use serde_wasm_bindgen as swb;+use tsify::Tsify;+use wasm_bindgen::prelude::*;++#[derive(Debug, thiserror::Error)]+pub enum Error {+ #[error(transparent)]+ Serde(#[from] swb::Error),+}++impl From<Error> for JsValue {+ fn from(error: Error) -> Self {+ crate::js_error(&error, None)+ }+}++#[derive(Deserialize, Tsify)]+#[serde(rename_all = "camelCase")]+struct DedupCollection {+ id: i64,+ owner_id: i64,+}++#[derive(Deserialize, Tsify)]+#[serde(rename_all = "camelCase")]+struct DedupFile {+ id: i64,+ collection_id: i64,+ owner_id: i64,+ hash: Option<String>,+ size: Option<u64>,+}++#[derive(Deserialize, Tsify)]+#[serde(rename_all = "camelCase")]+pub struct FindDuplicatesInput {+ user_id: i64,+ collections: Vec<DedupCollection>,+ files: Vec<DedupFile>,+}++#[derive(Serialize, Tsify)]+#[serde(rename_all = "camelCase")]+struct DuplicateItem {+ file_index: usize,+ collection_ids: Vec<i64>,+}++#[derive(Serialize, Tsify)]+#[serde(rename_all = "camelCase")]+struct DuplicateGroup {+ items: Vec<DuplicateItem>,+ item_size: u64,+}++#[derive(Serialize, Tsify)]+pub struct FindDuplicatesOutput {+ groups: Vec<DuplicateGroup>,+}++#[wasm_bindgen(js_name = photosFindDuplicates)]+pub fn find_duplicates(+ input: <FindDuplicatesInput as Tsify>::JsType,+) -> Result<<FindDuplicatesOutput as Tsify>::JsType, Error> {+ let input = FindDuplicatesInput::from_js(input)?;+ let collections: Vec<_> = input+ .collections+ .into_iter()+ .map(|collection| dedup::Collection {+ id: collection.id,+ owner_id: collection.owner_id,+ })+ .collect();+ let files: Vec<_> = input+ .files+ .into_iter()+ .map(|file| dedup::File {+ id: file.id,+ collection_id: file.collection_id,+ owner_id: file.owner_id,+ hash: file.hash,+ size: file.size,+ })+ .collect();+ let groups = dedup::find_duplicates(input.user_id, &collections, &files)+ .into_iter()+ .map(|group| DuplicateGroup {+ items: group+ .items+ .into_iter()+ .map(|item| DuplicateItem {+ file_index: item.file_index,+ collection_ids: item.collection_ids,+ })+ .collect(),+ item_size: group.item_size,+ })+ .collect();+ FindDuplicatesOutput { groups }+ .into_js()+ .map_err(Into::into)+}++#[derive(Deserialize, Tsify)]+#[serde(rename_all = "camelCase")]+struct Duplicate {+ file_id: i64,+ collection_ids: Vec<i64>,+ caption: Option<String>,+ edited_name: Option<String>,+ edited_time: Option<i64>,+}++#[derive(Deserialize, Tsify)]+pub struct PlanDuplicateRemovalInput {+ groups: Vec<Vec<Duplicate>>,+}++#[derive(Serialize, Tsify)]+#[serde(rename_all = "camelCase")]+struct DuplicateAddition {+ collection_id: i64,+ file_ids: Vec<i64>,+}++#[derive(Serialize, Tsify)]+#[serde(rename_all = "camelCase")]+pub struct DuplicateRemovalPlan {+ additions: Vec<DuplicateAddition>,+ trash_ids: Vec<i64>,+}++#[wasm_bindgen(js_name = photosPlanDuplicateRemoval)]+pub fn plan_duplicate_removal(+ input: <PlanDuplicateRemovalInput as Tsify>::JsType,+) -> Result<<DuplicateRemovalPlan as Tsify>::JsType, Error> {+ let input = PlanDuplicateRemovalInput::from_js(input)?;+ let groups: Vec<_> = input+ .groups+ .into_iter()+ .map(|group| {+ group+ .into_iter()+ .map(|item| dedup::Duplicate {+ file_id: item.file_id,+ collection_ids: item.collection_ids,+ caption: item.caption,+ edited_name: item.edited_name,+ edited_time: item.edited_time,+ })+ .collect()+ })+ .collect();+ let plan = dedup::plan_removal(&groups);+ DuplicateRemovalPlan {+ additions: plan+ .additions+ .into_iter()+ .map(|addition| DuplicateAddition {+ collection_id: addition.collection_id,+ file_ids: addition.file_ids,+ })+ .collect(),+ trash_ids: plan.trash_ids,+ }+ .into_js()+ .map_err(Into::into)+}diff --git a/rust/bindings/wasm/photos/Cargo.toml b/rust/bindings/wasm/photos/Cargo.tomlindex 00faeb76ba..b54eef2f7b 100644--- a/rust/bindings/wasm/photos/Cargo.toml+++ b/rust/bindings/wasm/photos/Cargo.toml@@ -6,7 +6,7 @@ edition.workspace = truecrate-type = ["cdylib"][dependencies]-ente-wasm-lib = { workspace = true, features = ["account-deletion", "cast", "collections", "contacts", "crypto", "location"] }+ente-wasm-lib = { workspace = true, features = ["account-deletion", "cast", "collections", "contacts", "crypto", "location", "photos"] }[lints]workspace = truediff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.tomlindex 25ca94302a..3330385dc2 100644--- a/rust/crates/photos/Cargo.toml+++ b/rust/crates/photos/Cargo.toml@@ -3,21 +3,21 @@ name = "ente-photos"edition.workspace = true[dependencies]-ente-assets.workspace = trueente-collections.workspace = trueente-core.workspace = trueente-exif.workspace = true-ente-location.workspace = true-ente-vecdb.workspace = truelog.workspace = truequick-xml.workspace = true-rusqlite = { workspace = true, default-features = true }serde.workspace = trueserde_json.workspace = truethiserror.workspace = true[target.'cfg(not(target_arch = "wasm32"))'.dependencies]+ente-assets.workspace = true+ente-location.workspace = true+ente-vecdb.workspace = truefutures-util.workspace = true+rusqlite = { workspace = true, default-features = true }[dev-dependencies]tempfile.workspace = truediff --git a/rust/crates/photos/src/dedup.rs b/rust/crates/photos/src/dedup.rsnew file mode 100644index 0000000000..425582448c--- /dev/null+++ b/rust/crates/photos/src/dedup.rs@@ -0,0 +1,399 @@+//! Duplicate detection and removal planning, independent of storage and UI.++use std::collections::{HashMap, HashSet};++pub struct Collection {+ pub id: i64,+ pub owner_id: i64,+}++/// A file's membership in one collection. Hashes must be normalized, including+/// the `image:video` hash for Live Photos uploaded by legacy clients.+pub struct File {+ pub id: i64,+ pub collection_id: i64,+ pub owner_id: i64,+ pub hash: Option<String>,+ pub size: Option<u64>,+}++#[derive(Debug, PartialEq, Eq)]+pub struct Item {+ /// Index of the first eligible occurrence in the input files.+ pub file_index: usize,+ pub collection_ids: Vec<i64>,+}++#[derive(Debug, PartialEq, Eq)]+pub struct Group {+ pub items: Vec<Item>,+ /// Estimate based on the first known nonzero size in the group. Live Photo+ /// ZIP sizes can vary between clients even when their content hashes match.+ pub item_size: u64,+}++/// Find distinct owned files with matching hashes in owned, non-hidden albums.+/// The caller supplies only non-hidden collections. Groups, files and album+/// memberships preserve input order; clients may sort items for display.+pub fn find_duplicates(user_id: i64, collections: &[Collection], files: &[File]) -> Vec<Group> {+ let allowed: HashSet<_> = collections+ .iter()+ .filter(|collection| collection.owner_id == user_id)+ .map(|collection| collection.id)+ .collect();+ let mut by_id = HashMap::<i64, (usize, usize)>::new();+ let mut by_hash = HashMap::<&str, usize>::new();+ let mut groups = Vec::<Group>::new();+ for (file_index, file) in files.iter().enumerate() {+ if file.owner_id != user_id || !allowed.contains(&file.collection_id) {+ continue;+ }+ let Some(hash) = file.hash.as_deref().filter(|hash| !hash.is_empty()) else {+ continue;+ };+ if let Some(&(group_index, item_index)) = by_id.get(&file.id) {+ let ids = &mut groups[group_index].items[item_index].collection_ids;+ if !ids.contains(&file.collection_id) {+ ids.push(file.collection_id);+ }+ continue;+ }+ let group_index = *by_hash.entry(hash).or_insert_with(|| {+ groups.push(Group {+ items: Vec::new(),+ item_size: 0,+ });+ groups.len() - 1+ });+ let group = &mut groups[group_index];+ by_id.insert(file.id, (group_index, group.items.len()));+ group.items.push(Item {+ file_index,+ collection_ids: vec![file.collection_id],+ });+ if group.item_size == 0 {+ group.item_size = file.size.unwrap_or(0);+ }+ }+ groups+ .into_iter()+ .filter(|group| group.items.len() > 1 && group.item_size != 0)+ .collect()+}++#[derive(Default)]+pub struct Duplicate {+ pub file_id: i64,+ pub collection_ids: Vec<i64>,+ pub caption: Option<String>,+ pub edited_name: Option<String>,+ pub edited_time: Option<i64>,+}++#[derive(Debug, PartialEq, Eq)]+pub struct Addition {+ pub collection_id: i64,+ pub file_ids: Vec<i64>,+}++#[derive(Debug, Default, PartialEq, Eq)]+pub struct RemovalPlan {+ /// Complete every album addition successfully before trashing any files.+ pub additions: Vec<Addition>,+ pub trash_ids: Vec<i64>,+}++/// Plan removal of selected duplicate groups. Retain the first item with a+/// caption, then the first with edits, otherwise the first item. Input order+/// breaks ties (the web client uses its locale's album-name ordering).+pub fn plan_removal(groups: &[Vec<Duplicate>]) -> RemovalPlan {+ let mut plan = RemovalPlan::default();+ let mut by_collection = HashMap::<i64, usize>::new();+ for items in groups {+ if items.len() < 2 {+ continue;+ }+ let retained = items+ .iter()+ .find(|item| {+ item.caption+ .as_ref()+ .is_some_and(|caption| !caption.is_empty())+ })+ .or_else(|| {+ items.iter().find(|item| {+ // Match editedName ?? editedTime: an explicitly empty name+ // takes precedence over an edited timestamp.+ item.edited_name.as_ref().map_or_else(+ || item.edited_time.is_some_and(|time| time != 0),+ |name| !name.is_empty(),+ )+ })+ })+ .unwrap_or(&items[0]);+ let mut memberships: HashSet<_> = retained.collection_ids.iter().copied().collect();+ for item in items {+ if item.file_id == retained.file_id {+ continue;+ }+ plan.trash_ids.push(item.file_id);+ for &collection_id in &item.collection_ids {+ if !memberships.insert(collection_id) {+ continue;+ }+ let index = *by_collection.entry(collection_id).or_insert_with(|| {+ plan.additions.push(Addition {+ collection_id,+ file_ids: Vec::new(),+ });+ plan.additions.len() - 1+ });+ plan.additions[index].file_ids.push(retained.file_id);+ }+ }+ }+ plan+}++#[cfg(test)]+mod tests {+ use super::*;++ fn file(id: i64, collection_id: i64, hash: &str, size: Option<u64>) -> File {+ File {+ id,+ collection_id,+ owner_id: 1,+ hash: Some(hash.into()),+ size,+ }+ }++ fn collections() -> Vec<Collection> {+ vec![+ Collection {+ id: 11,+ owner_id: 1,+ },+ Collection {+ id: 12,+ owner_id: 1,+ },+ Collection {+ id: 13,+ owner_id: 2,+ },+ ]+ }++ #[test]+ fn groups_distinct_owned_files_and_preserves_all_eligible_memberships() {+ let mut foreign = file(3, 11, "hash", Some(100));+ foreign.owner_id = 2;+ let files = vec![+ file(1, 11, "hash", Some(100)),+ file(1, 12, "hash", Some(100)),+ file(1, 12, "hash", Some(100)),+ file(2, 12, "hash", Some(100)),+ foreign,+ file(4, 13, "hash", Some(100)),+ file(5, 99, "hash", Some(100)),+ file(6, 11, "", Some(100)),+ ];+ assert_eq!(+ find_duplicates(1, &collections(), &files),+ vec![Group {+ items: vec![+ Item {+ file_index: 0,+ collection_ids: vec![11, 12]+ },+ Item {+ file_index: 3,+ collection_ids: vec![12]+ }+ ],+ item_size: 100,+ }]+ );+ }++ #[test]+ fn ignores_singletons_symlinks_missing_hashes_and_unknown_sizes() {+ let files = vec![+ file(1, 11, "single", Some(100)),+ file(1, 12, "single", Some(100)),+ file(2, 11, "no-size", None),+ file(3, 11, "no-size", Some(0)),+ File {+ hash: None,+ ..file(4, 11, "", Some(100))+ },+ File {+ hash: None,+ ..file(5, 11, "", Some(100))+ },+ ];+ assert!(find_duplicates(1, &collections(), &files).is_empty());+ assert!(find_duplicates(1, &[], &files).is_empty());+ }++ #[test]+ fn uses_first_nonzero_size_without_requiring_matching_live_photo_zip_sizes() {+ let files = vec![+ file(1, 11, "image:video", None),+ file(2, 11, "image:video", Some(150)),+ file(3, 12, "image:video", Some(200)),+ file(4, 11, "other", Some(90)),+ file(5, 11, "other", Some(100)),+ ];+ let groups = find_duplicates(1, &collections(), &files);+ assert_eq!(+ groups+ .iter()+ .map(|group| group.item_size)+ .collect::<Vec<_>>(),+ vec![150, 90]+ );+ assert_eq!(groups[0].items.len(), 3);+ }++ fn duplicate(id: i64, collection_ids: &[i64]) -> Duplicate {+ Duplicate {+ file_id: id,+ collection_ids: collection_ids.into(),+ ..Default::default()+ }+ }++ #[test]+ fn retains_captions_before_edits_and_batches_missing_album_memberships() {+ let groups = vec![+ vec![+ Duplicate {+ edited_name: Some("renamed".into()),+ ..duplicate(1, &[11, 12])+ },+ Duplicate {+ caption: Some("caption".into()),+ ..duplicate(2, &[12])+ },+ duplicate(3, &[11, 13]),+ ],+ vec![duplicate(4, &[12]), duplicate(5, &[11])],+ ];+ assert_eq!(+ plan_removal(&groups),+ RemovalPlan {+ additions: vec![+ Addition {+ collection_id: 11,+ file_ids: vec![2, 4]+ },+ Addition {+ collection_id: 13,+ file_ids: vec![2]+ }+ ],+ trash_ids: vec![1, 3, 5],+ }+ );+ }++ #[test]+ fn retention_ties_follow_input_order_and_empty_edits_do_not_win() {+ let mut items = vec![+ duplicate(1, &[11]),+ duplicate(2, &[11]),+ duplicate(3, &[11]),+ ];+ assert_eq!(plan_removal(&[vec![]]), RemovalPlan::default());+ assert_eq!(+ plan_removal(&[vec![duplicate(1, &[11])]]),+ RemovalPlan::default()+ );+ items[1].edited_time = Some(123);+ assert_eq!(+ plan_removal(std::slice::from_ref(&items)).trash_ids,+ vec![1, 3]+ );+ items[1].edited_name = Some(String::new());+ assert_eq!(+ plan_removal(std::slice::from_ref(&items)).trash_ids,+ vec![2, 3]+ );+ items[1].caption = Some("first caption".into());+ items[2].caption = Some("second caption".into());+ assert_eq!(+ plan_removal(std::slice::from_ref(&items)).trash_ids,+ vec![1, 3]+ );+ items[1].caption = Some(String::new());+ assert_eq!(+ plan_removal(std::slice::from_ref(&items)).trash_ids,+ vec![1, 2]+ );+ assert!(plan_removal(&[items]).additions.is_empty());+ }++ #[test]+ fn detection_and_removal_preserve_every_album_membership_combination() {+ let collections: Vec<_> = (11..=13).map(|id| Collection { id, owner_id: 1 }).collect();+ // Each of three distinct copies can belong to any nonempty subset of+ // three albums. Applying the plan must leave one copy in every album.+ for first in 1..8 {+ for second in 1..8 {+ for third in 1..8 {+ let mut files = Vec::new();+ let mut memberships = HashMap::<i64, HashSet<i64>>::new();+ for (index, mask) in [first, second, third].into_iter().enumerate() {+ let id = index as i64 + 1;+ for bit in 0..3 {+ if mask & (1 << bit) != 0 {+ let collection_id = 11 + bit;+ files.push(file(id, collection_id, "hash", Some(100)));+ memberships.entry(collection_id).or_default().insert(id);+ }+ }+ }+ let expected_albums: HashSet<_> = memberships.keys().copied().collect();+ let groups = find_duplicates(1, &collections, &files);+ assert_eq!(groups.len(), 1);+ assert_eq!(groups[0].items.len(), 3);+ let selected: Vec<_> = groups+ .into_iter()+ .map(|group| {+ group+ .items+ .into_iter()+ .map(|item| {+ let id = files[item.file_index].id;+ Duplicate {+ caption: (id == 2).then(|| "keep".into()),+ ..duplicate(id, &item.collection_ids)+ }+ })+ .collect()+ })+ .collect();+ let plan = plan_removal(&selected);+ assert_eq!(plan.trash_ids, vec![1, 3]);+ for addition in plan.additions {+ memberships+ .entry(addition.collection_id)+ .or_default()+ .extend(addition.file_ids);+ }+ for ids in memberships.values_mut() {+ ids.retain(|id| !plan.trash_ids.contains(id));+ assert_eq!(*ids, HashSet::from([2]));+ }+ assert_eq!(+ memberships.keys().copied().collect::<HashSet<_>>(),+ expected_albums+ );+ }+ }+ }+ }+}diff --git a/rust/crates/photos/src/files.rs b/rust/crates/photos/src/files.rsindex 163b17d32e..66cfca872a 100644--- a/rust/crates/photos/src/files.rs+++ b/rust/crates/photos/src/files.rs@@ -1,5 +1,7 @@+#[cfg(not(target_arch = "wasm32"))]+use ente_core::Session;use ente_core::{- Session, b64,+ b64,crypto::{self, Header, Key, blob},http,};diff --git a/rust/crates/photos/src/lib.rs b/rust/crates/photos/src/lib.rsindex 377197a4ae..1d9cea7883 100644--- a/rust/crates/photos/src/lib.rs+++ b/rust/crates/photos/src/lib.rs@@ -1,11 +1,16 @@pub mod collections;+#[cfg(not(target_arch = "wasm32"))]pub mod db;+pub mod dedup;pub mod face_clustering;pub mod files;+#[cfg(not(target_arch = "wasm32"))]pub mod location;pub mod memories;pub mod metadata;+#[cfg(not(target_arch = "wasm32"))]pub mod ml_db;+#[cfg(not(target_arch = "wasm32"))]pub mod ml_store;pub mod motion_photo;pub mod source;diff --git a/web/apps/photos/package.json b/web/apps/photos/package.jsonindex 7319d624de..34b89104b8 100644--- a/web/apps/photos/package.json+++ b/web/apps/photos/package.json@@ -39,6 +39,7 @@"@types/react-dom": "19.3.0","@types/react-window": "1.8.8","ente-build-config": "*",+ "vite-plugin-wasm": "3.6.0","vitest": "5.0.4"}}diff --git a/web/apps/photos/src/services/dedup.ts b/web/apps/photos/src/services/dedup.tsindex 2bc56ec9cd..de8d245605 100644--- a/web/apps/photos/src/services/dedup.ts+++ b/web/apps/photos/src/services/dedup.ts@@ -2,7 +2,10 @@ import { ensureLocalUser } from "ente-accounts/services/user";import { assertionFailed } from "ente-base/assert";import { newID } from "ente-base/id";import type { EnteFile } from "ente-media/file";-import { metadataHash } from "ente-media/file-metadata";+import {+ FilePublicMagicMetadataData,+ metadataHash,+} from "ente-media/file-metadata";import {addToCollection,createCollectionNameByID,@@ -10,6 +13,7 @@ import {savedNormalCollections,} from "ente-new/photos/services/collection";import { savedCollectionFiles } from "ente-new/photos/services/photos-fdb";+import { findDuplicates, planDuplicateRemoval } from "ente-photos-wasm";export interface DuplicateGroup {id: string;@@ -28,67 +32,42 @@ export const deduceDuplicates = async () => {const userID = ensureLocalUser().id;const normalCollections = await savedNormalCollections();- const normalOwnedCollections = normalCollections.filter(- ({ owner }) => owner.id == userID,- );- const allowedCollectionIDs = new Set(- normalOwnedCollections.map(({ id }) => id),- );- const collectionNameByID = createCollectionNameByID(normalOwnedCollections);-+ const collectionNameByID = createCollectionNameByID(normalCollections);const collectionFiles = await savedCollectionFiles();- const filteredCollectionFiles = collectionFiles.filter((f) =>- allowedCollectionIDs.has(f.collectionID),- );-- // One file ID can appear once per collection; hash it only once.- const collectionIDsByFileID = new Map<number, Set<number>>();- const filesByHash = new Map<string, EnteFile[]>();- for (const file of filteredCollectionFiles) {- // An owned album can contain files the user does not own.- if (file.ownerID != userID) continue;-- const hash = metadataHash(file.metadata);- if (!hash) {- continue;- }-- let collectionIDs = collectionIDsByFileID.get(file.id);- if (!collectionIDs) {- collectionIDsByFileID.set(file.id, (collectionIDs = new Set()));- filesByHash.set(hash, [...(filesByHash.get(hash) ?? []), file]);- }- collectionIDs.add(file.collectionID);- }+ const { groups } = await findDuplicates({+ userId: userID,+ collections: normalCollections.map(({ id, owner }) => ({+ id,+ ownerId: owner.id,+ })),+ files: collectionFiles.map((file) => ({+ id: file.id,+ collectionId: file.collectionID,+ ownerId: file.ownerID,+ hash: metadataHash(file.metadata),+ size: file.info?.fileSize,+ })),+ });const duplicateGroups: DuplicateGroup[] = [];- for (const duplicates of filesByHash.values()) {- if (duplicates.length < 2) continue;-- // Live-photo ZIP sizes can vary by client despite matching content hashes.- let size = 0;- for (const file of duplicates) {- if (file.info?.fileSize) {- size = file.info.fileSize;- break;- }- }-- if (!size) continue;-- const items = duplicates- .map((file) => {+ for (const group of groups) {+ const items = group.items+ .map(({ fileIndex, collectionIds }) => {+ const file = collectionFiles[fileIndex]!;const collectionName = collectionNameByID.get(file.collectionID,);- const collectionIDs = collectionIDsByFileID.get(file.id);- if (!collectionName || !collectionIDs) {+ if (!collectionName) {assertionFailed();return undefined;}- return { file, collectionIDs, collectionName };+ return {+ file,+ collectionIDs: new Set(collectionIds),+ collectionName,+ };}).filter((item) => !!item);if (items.length < 2) continue;@@ -98,9 +77,9 @@ export const deduceDuplicates = async () => {duplicateGroups.push({id: newID("dg_"),items,- itemSize: size,+ itemSize: group.itemSize,prunableCount: items.length - 1,- prunableSize: size * (items.length - 1),+ prunableSize: group.itemSize * (items.length - 1),isSelected: true,});}@@ -115,44 +94,44 @@ export const removeSelectedDuplicateGroups = async () => {const selectedDuplicateGroups = duplicateGroups.filter((g) => g.isSelected);- // Symlink the retained file into every affected album before trashing duplicates.- const filesToAdd = new Map<number, EnteFile[]>();- const filesToTrash: EnteFile[] = [];-- for (const duplicateGroup of selectedDuplicateGroups) {- const retainedItem = duplicateGroupItemToRetain(duplicateGroup);-- let collectionIDs = new Set<number>();- for (const item of duplicateGroup.items) {- if (item.file.id == retainedItem.file.id) continue;- collectionIDs = collectionIDs.union(item.collectionIDs);- filesToTrash.push(item.file);- }-- collectionIDs = collectionIDs.difference(retainedItem.collectionIDs);-- for (const collectionID of collectionIDs) {- filesToAdd.set(collectionID, [- ...(filesToAdd.get(collectionID) ?? []),- retainedItem.file,- ]);- }- }+ const filesByID = new Map(+ selectedDuplicateGroups.flatMap((group) =>+ group.items.map(({ file }) => [file.id, file] as const),+ ),+ );+ const plan = await planDuplicateRemoval({+ groups: selectedDuplicateGroups.map((group) =>+ group.items.map(({ file, collectionIDs }) => ({+ fileId: file.id,+ collectionIds: [...collectionIDs],+ // Cached files can predate numeric-caption normalization.+ caption: FilePublicMagicMetadataData.shape.caption.parse(+ file.pubMagicMetadata?.data.caption,+ ),+ editedName: file.pubMagicMetadata?.data.editedName,+ editedTime: file.pubMagicMetadata?.data.editedTime,+ })),+ ),+ });let np = 0;const ntotal =- filesToAdd.size + (filesToTrash.length ? 1 : 0) + /* sync */ 1;+ plan.additions.length + (plan.trashIds.length ? 1 : 0) + /* sync */ 1;const tickProgress = () => onProgress((++np / ntotal) * 100);const collections = await savedNormalCollections();const collectionsByID = new Map(collections.map((c) => [c.id, c]));- for (const [collectionID, files] of filesToAdd.entries()) {- await addToCollection(collectionsByID.get(collectionID)!, files);+ // Complete every album addition before trashing any duplicates.+ for (const { collectionId, fileIds } of plan.additions) {+ await addToCollection(+ collectionsByID.get(collectionId)!,+ fileIds.map((id) => filesByID.get(id)!),+ );tickProgress();}- if (filesToTrash.length) {- await moveToTrash(filesToTrash);+ if (plan.trashIds.length) {+ await moveToTrash(plan.trashIds.map((id) => filesByID.get(id)!));tickProgress();}@@ -161,21 +140,3 @@ export const removeSelectedDuplicateGroups = async (return new Set(selectedDuplicateGroups.map((g) => g.id));};--const duplicateGroupItemToRetain = (duplicateGroup: DuplicateGroup) => {- const itemsWithCaption: DuplicateGroup["items"] = [];- const itemsWithOtherEdits: DuplicateGroup["items"] = [];- for (const item of duplicateGroup.items) {- const pubMM = item.file.pubMagicMetadata?.data;- if (!pubMM) continue;- if (pubMM.caption) itemsWithCaption.push(item);- if (pubMM.editedName ?? pubMM.editedTime)- itemsWithOtherEdits.push(item);- }-- return (- itemsWithCaption[0] ??- itemsWithOtherEdits[0] ??- duplicateGroup.items[0]!- );-};diff --git a/web/apps/photos/tests/dedup.test.ts b/web/apps/photos/tests/dedup.test.tsindex fac7c2e3c5..da47d8555c 100644--- a/web/apps/photos/tests/dedup.test.ts+++ b/web/apps/photos/tests/dedup.test.ts@@ -1,24 +1,140 @@import type { EnteFile } from "ente-media/file";import { beforeEach, describe, expect, test, vi } from "vitest";import {+ deduceDuplicates,removeSelectedDuplicateGroups,type DuplicateGroup,} from "../src/services/dedup";-const { addToCollection, moveToTrash, savedNormalCollections } = vi.hoisted(- () => ({- addToCollection: vi.fn(),- moveToTrash: vi.fn(),- savedNormalCollections: vi.fn(),- }),-);+const {+ addToCollection,+ moveToTrash,+ savedNormalCollections,+ savedCollectionFiles,+} = vi.hoisted(() => ({+ addToCollection: vi.fn(),+ moveToTrash: vi.fn(),+ savedNormalCollections: vi.fn(),+ savedCollectionFiles: vi.fn(),+}));vi.mock("ente-new/photos/services/collection", () => ({addToCollection,moveToTrash,savedNormalCollections,+ createCollectionNameByID: (collections: { id: number; name: string }[]) =>+ new Map(collections.map(({ id, name }) => [id, name])),+}));+vi.mock("ente-new/photos/services/photos-fdb", () => ({+ savedCollectionFiles,}));-vi.mock("ente-new/photos/services/photos-fdb", () => ({}));-vi.mock("ente-accounts/services/user", () => ({}));+vi.mock("ente-accounts/services/user", () => ({+ ensureLocalUser: () => ({ id: 1 }),+}));++const file = (+ id: number,+ collectionID: number,+ hash: string | undefined,+ size: number | undefined = 100,+) =>+ ({+ id,+ collectionID,+ ownerID: 1,+ metadata: { fileType: 0, hash },+ info: { fileSize: size },+ }) as EnteFile;++describe("deduceDuplicates through Rust WASM", () => {+ beforeEach(() => {+ vi.resetAllMocks();+ savedNormalCollections.mockResolvedValue([+ { id: 11, owner: { id: 1 }, name: "B" },+ { id: 12, owner: { id: 1 }, name: "A" },+ { id: 13, owner: { id: 2 }, name: "Shared" },+ ]);+ });++ test("groups owned files once across albums, excluding shared files and hidden albums", async () => {+ const first = file(1, 11, "hash");+ const second = file(2, 12, "hash");+ savedCollectionFiles.mockResolvedValue([+ first,+ { ...first, collectionID: 12 },+ second,+ { ...file(3, 11, "hash"), ownerID: 2 },+ file(4, 13, "hash"),+ file(5, 99, "hash"),+ file(6, 11, undefined),+ file(7, 11, "unique"),+ ]);+ const groups = await deduceDuplicates();+ expect(groups).toEqual([+ {+ id: expect.any(String) as string,+ items: [+ {+ file: second,+ collectionIDs: new Set([12]),+ collectionName: "A",+ },+ {+ file: first,+ collectionIDs: new Set([11, 12]),+ collectionName: "B",+ },+ ],+ itemSize: 100,+ prunableCount: 1,+ prunableSize: 100,+ isSelected: true,+ },+ ]);+ // The locale-sorted first item is the fallback retained copy in Rust.+ await removeSelectedDuplicateGroups(groups, vi.fn(), vi.fn());+ expect(addToCollection).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({ id: 11 }),+ [second],+ );+ expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([first]);+ });++ test("matches legacy Live Photo hashes despite different ZIP sizes and skips groups without sizes", async () => {+ const legacy = file(1, 11, undefined, 0);+ legacy.metadata = {+ ...legacy.metadata,+ fileType: 2,+ imageHash: "image",+ videoHash: "video",+ };+ savedCollectionFiles.mockResolvedValue([+ legacy,+ file(2, 12, "image:video", 150),+ file(3, 11, "image:video", 200),+ file(4, 11, "unknown-size", 0),+ { ...file(5, 11, "unknown-size"), info: undefined },+ ]);+ const groups = await deduceDuplicates();+ expect(groups).toHaveLength(1);+ expect(groups[0]).toMatchObject({+ itemSize: 150,+ prunableCount: 2,+ prunableSize: 300,+ });+ expect(groups[0]!.items.map(({ file }) => file.id)).toEqual([2, 1, 3]);+ });++ test("ignores a single file linked to multiple albums and empty libraries", async () => {+ savedCollectionFiles.mockResolvedValue([+ file(1, 11, "hash"),+ file(1, 12, "hash"),+ file(1, 12, "hash"),+ ]);+ expect(await deduceDuplicates()).toEqual([]);+ savedCollectionFiles.mockResolvedValue([]);+ expect(await deduceDuplicates()).toEqual([]);+ });+});const duplicateGroup = (): DuplicateGroup => ({id: "group-1",@@ -56,6 +172,119 @@ describe("removeSelectedDuplicateGroups", () => {savedNormalCollections.mockResolvedValue(collections);});+ test.each([123, 0])(+ "normalizes legacy cached numeric caption %s before planning in Rust",+ async (caption) => {+ const group = duplicateGroup();+ const retained = group.items[1]!.file;+ // Cached records bypass the metadata schema and can predate normalization.+ Object.assign(retained.pubMagicMetadata!.data, { caption });+ const refresh = vi.fn();++ await expect(+ removeSelectedDuplicateGroups([group], vi.fn(), refresh),+ ).resolves.toEqual(new Set([group.id]));+ expect(addToCollection.mock.calls).toEqual([+ [collections[0], [retained]],+ [collections[2], [retained]],+ ]);+ expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([+ group.items[0]!.file,+ group.items[2]!.file,+ ]);+ expect(refresh).toHaveBeenCalledOnce();+ },+ );++ test.each([+ {+ label: "caption before edits",+ metadata: [{ editedName: "renamed" }, { caption: "caption" }, {}],+ retainedId: 2,+ },+ {+ label: "renamed copy",+ metadata: [{}, {}, { editedName: "renamed" }],+ retainedId: 3,+ },+ {+ label: "edited timestamp",+ metadata: [{}, { editedTime: 123 }, {}],+ retainedId: 2,+ },+ {+ label: "first of equal captions",+ metadata: [{}, { caption: "one" }, { caption: "two" }],+ retainedId: 2,+ },+ {+ label: "first of equal edits",+ metadata: [{}, { editedName: "one" }, { editedTime: 123 }],+ retainedId: 2,+ },+ {+ label: "empty caption and null caption",+ metadata: [{}, { caption: "" }, { caption: null }],+ retainedId: 1,+ },+ {+ label: "empty name overrides edited timestamp",+ metadata: [{}, { editedName: "", editedTime: 123 }, {}],+ retainedId: 1,+ },+ ])(+ "retains the $label through Rust WASM",+ async ({ metadata, retainedId }) => {+ const group = duplicateGroup();+ group.items.forEach(({ file }, index) => {+ Object.assign(file, {+ pubMagicMetadata: { data: metadata[index] },+ });+ });+ await removeSelectedDuplicateGroups([group], vi.fn(), vi.fn());+ expect(moveToTrash).toHaveBeenCalledExactlyOnceWith(+ group.items+ .filter(({ file }) => file.id !== retainedId)+ .map(({ file }) => file),+ );+ },+ );++ test("batches retained copies from different groups into each missing album", async () => {+ const first = duplicateGroup();+ const second = duplicateGroup();+ second.id = "group-2";+ for (const { file } of second.items) file.id += 3;+ const refresh = vi.fn();+ await expect(+ removeSelectedDuplicateGroups([first, second], vi.fn(), refresh),+ ).resolves.toEqual(new Set([first.id, second.id]));+ const retained = [first.items[1]!.file, second.items[1]!.file];+ expect(addToCollection.mock.calls).toEqual([+ [collections[0], retained],+ [collections[2], retained],+ ]);+ expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([+ first.items[0]!.file,+ first.items[2]!.file,+ second.items[0]!.file,+ second.items[2]!.file,+ ]);+ expect(refresh).toHaveBeenCalledOnce();+ });++ test("rejects invalid planner input before modifying albums or trash", async () => {+ const group = duplicateGroup();+ group.items[1]!.file.id = Number.MAX_SAFE_INTEGER + 1;+ const refresh = vi.fn();+ await expect(+ removeSelectedDuplicateGroups([group], vi.fn(), refresh),+ ).rejects.toBeInstanceOf(Error);+ expect(addToCollection).not.toHaveBeenCalled();+ expect(moveToTrash).not.toHaveBeenCalled();+ expect(refresh).not.toHaveBeenCalled();+ });+test("preserves album membership before trashing selected duplicates and awaits gallery refresh", async () => {const group = duplicateGroup();const untouched = {@@ -108,6 +337,20 @@ describe("removeSelectedDuplicateGroups", () => {expect(refresh).not.toHaveBeenCalled();});+ test("does not trash any files if a later album addition fails", async () => {+ const failure = new Error("Second album failed");+ addToCollection+ .mockResolvedValueOnce(undefined)+ .mockRejectedValueOnce(failure);+ const refresh = vi.fn();+ await expect(+ removeSelectedDuplicateGroups([duplicateGroup()], vi.fn(), refresh),+ ).rejects.toBe(failure);+ expect(addToCollection).toHaveBeenCalledTimes(2);+ expect(moveToTrash).not.toHaveBeenCalled();+ expect(refresh).not.toHaveBeenCalled();+ });+test("does not refresh when trashing fails", async () => {const failure = new Error("Trash failed");moveToTrash.mockRejectedValue(failure);diff --git a/web/apps/photos/vitest.config.ts b/web/apps/photos/vitest.config.tsindex 51e689a419..9559186f8c 100644--- a/web/apps/photos/vitest.config.ts+++ b/web/apps/photos/vitest.config.ts@@ -1,6 +1,9 @@+import type { PluginOption } from "vite";+import wasm from "vite-plugin-wasm";import { defineConfig } from "vitest/config";export default defineConfig({+ plugins: [wasm() as PluginOption],test: { include: ["tests/**/*.test.{ts,tsx}"] },// Next preserves JSX for its own compiler; component tests need it compiled.oxc: { jsx: { runtime: "automatic" } },diff --git a/web/package-lock.json b/web/package-lock.jsonindex af90d17724..cef2db017e 100644--- a/web/package-lock.json+++ b/web/package-lock.json@@ -352,6 +352,7 @@"@types/react-dom": "19.3.0","@types/react-window": "1.8.8","ente-build-config": "*",+ "vite-plugin-wasm": "3.6.0","vitest": "5.0.4"}},diff --git a/web/packages/wasm/photos/index.ts b/web/packages/wasm/photos/index.tsindex e691f74a57..f23806457e 100644--- a/web/packages/wasm/photos/index.ts+++ b/web/packages/wasm/photos/index.ts@@ -1,7 +1,9 @@import type {EncryptedBox,+ FindDuplicatesInput,LocationIndex,OpenSessionInput,+ PlanDuplicateRemovalInput,Session,WrappedRootContactKey,} from "./pkg/ente_photos_wasm";@@ -15,6 +17,12 @@ export type {const wasm = () => import("./pkg/ente_photos_wasm");+export const findDuplicates = async (input: FindDuplicatesInput) =>+ (await wasm()).photosFindDuplicates(input);++export const planDuplicateRemoval = async (input: PlanDuplicateRemovalInput) =>+ (await wasm()).photosPlanDuplicateRemoval(input);+export const createLocationIndex = async (cities: Uint8Array,urbanCenters: Uint8Array,