Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[photos][web] Share deduplication logic through Rust

AswinAsoka0cf0dde79e7
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index a10103296f..533097d96a 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -3068,6 +3068,7 @@ dependencies = [
"ente-legacy",
"ente-location",
"ente-locker",
+ "ente-photos",
"js-sys",
"log",
"md-5 0.11.0",
diff --git a/rust/bindings/wasm/lib/Cargo.toml b/rust/bindings/wasm/lib/Cargo.toml
index b50be7d25b..a9f0c99d01 100644
--- a/rust/bindings/wasm/lib/Cargo.toml
+++ b/rust/bindings/wasm/lib/Cargo.toml
@@ -19,6 +19,7 @@ crypto-seal = ["crypto"]
legacy = ["session", "dep:ente-legacy", "dep:wasm-bindgen-futures"]
location = ["dep:ente-location", "dep:serde", "dep:serde-wasm-bindgen", "dep:tsify"]
locker = ["session", "dep:ente-locker"]
+photos = ["dep:ente-photos", "dep:serde", "dep:serde-wasm-bindgen", "dep:tsify"]
prelogin = ["accounts", "crypto", "dep:wasm-bindgen-futures"]
session = [
"dep:ente-accounts",
@@ -36,6 +37,7 @@ ente-core.workspace = true
ente-legacy = { workspace = true, optional = true }
ente-location = { workspace = true, optional = true }
ente-locker = { workspace = true, optional = true }
+ente-photos = { workspace = true, optional = true }
js-sys.workspace = true
log.workspace = true
md-5 = { workspace = true, optional = true }
diff --git a/rust/bindings/wasm/lib/src/lib.rs b/rust/bindings/wasm/lib/src/lib.rs
index 0844392602..16b37216a4 100644
--- a/rust/bindings/wasm/lib/src/lib.rs
+++ b/rust/bindings/wasm/lib/src/lib.rs
@@ -17,6 +17,8 @@ pub mod location;
#[cfg(feature = "locker")]
pub mod locker;
mod logging;
+#[cfg(feature = "photos")]
+pub mod photos;
#[cfg(feature = "prelogin")]
mod prelogin;
#[cfg(feature = "session")]
diff --git a/rust/bindings/wasm/lib/src/photos.rs b/rust/bindings/wasm/lib/src/photos.rs
new file mode 100644
index 0000000000..b653213832
--- /dev/null
+++ b/rust/bindings/wasm/lib/src/photos.rs
@@ -0,0 +1,170 @@
+use ente_photos::dedup;
+use serde::{Deserialize, Serialize};
+use serde_wasm_bindgen as swb;
+use tsify::Tsify;
+use wasm_bindgen::prelude::*;
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error(transparent)]
+ Serde(#[from] swb::Error),
+}
+
+impl From<Error> for JsValue {
+ fn from(error: Error) -> Self {
+ crate::js_error(&error, None)
+ }
+}
+
+#[derive(Deserialize, Tsify)]
+#[serde(rename_all = "camelCase")]
+struct DedupCollection {
+ id: i64,
+ owner_id: i64,
+}
+
+#[derive(Deserialize, Tsify)]
+#[serde(rename_all = "camelCase")]
+struct DedupFile {
+ id: i64,
+ collection_id: i64,
+ owner_id: i64,
+ hash: Option<String>,
+ size: Option<u64>,
+}
+
+#[derive(Deserialize, Tsify)]
+#[serde(rename_all = "camelCase")]
+pub struct FindDuplicatesInput {
+ user_id: i64,
+ collections: Vec<DedupCollection>,
+ files: Vec<DedupFile>,
+}
+
+#[derive(Serialize, Tsify)]
+#[serde(rename_all = "camelCase")]
+struct DuplicateItem {
+ file_index: usize,
+ collection_ids: Vec<i64>,
+}
+
+#[derive(Serialize, Tsify)]
+#[serde(rename_all = "camelCase")]
+struct DuplicateGroup {
+ items: Vec<DuplicateItem>,
+ item_size: u64,
+}
+
+#[derive(Serialize, Tsify)]
+pub struct FindDuplicatesOutput {
+ groups: Vec<DuplicateGroup>,
+}
+
+#[wasm_bindgen(js_name = photosFindDuplicates)]
+pub fn find_duplicates(
+ input: <FindDuplicatesInput as Tsify>::JsType,
+) -> Result<<FindDuplicatesOutput as Tsify>::JsType, Error> {
+ let input = FindDuplicatesInput::from_js(input)?;
+ let collections: Vec<_> = input
+ .collections
+ .into_iter()
+ .map(|collection| dedup::Collection {
+ id: collection.id,
+ owner_id: collection.owner_id,
+ })
+ .collect();
+ let files: Vec<_> = input
+ .files
+ .into_iter()
+ .map(|file| dedup::File {
+ id: file.id,
+ collection_id: file.collection_id,
+ owner_id: file.owner_id,
+ hash: file.hash,
+ size: file.size,
+ })
+ .collect();
+ let groups = dedup::find_duplicates(input.user_id, &collections, &files)
+ .into_iter()
+ .map(|group| DuplicateGroup {
+ items: group
+ .items
+ .into_iter()
+ .map(|item| DuplicateItem {
+ file_index: item.file_index,
+ collection_ids: item.collection_ids,
+ })
+ .collect(),
+ item_size: group.item_size,
+ })
+ .collect();
+ FindDuplicatesOutput { groups }
+ .into_js()
+ .map_err(Into::into)
+}
+
+#[derive(Deserialize, Tsify)]
+#[serde(rename_all = "camelCase")]
+struct Duplicate {
+ file_id: i64,
+ collection_ids: Vec<i64>,
+ caption: Option<String>,
+ edited_name: Option<String>,
+ edited_time: Option<i64>,
+}
+
+#[derive(Deserialize, Tsify)]
+pub struct PlanDuplicateRemovalInput {
+ groups: Vec<Vec<Duplicate>>,
+}
+
+#[derive(Serialize, Tsify)]
+#[serde(rename_all = "camelCase")]
+struct DuplicateAddition {
+ collection_id: i64,
+ file_ids: Vec<i64>,
+}
+
+#[derive(Serialize, Tsify)]
+#[serde(rename_all = "camelCase")]
+pub struct DuplicateRemovalPlan {
+ additions: Vec<DuplicateAddition>,
+ trash_ids: Vec<i64>,
+}
+
+#[wasm_bindgen(js_name = photosPlanDuplicateRemoval)]
+pub fn plan_duplicate_removal(
+ input: <PlanDuplicateRemovalInput as Tsify>::JsType,
+) -> Result<<DuplicateRemovalPlan as Tsify>::JsType, Error> {
+ let input = PlanDuplicateRemovalInput::from_js(input)?;
+ let groups: Vec<_> = input
+ .groups
+ .into_iter()
+ .map(|group| {
+ group
+ .into_iter()
+ .map(|item| dedup::Duplicate {
+ file_id: item.file_id,
+ collection_ids: item.collection_ids,
+ caption: item.caption,
+ edited_name: item.edited_name,
+ edited_time: item.edited_time,
+ })
+ .collect()
+ })
+ .collect();
+ let plan = dedup::plan_removal(&groups);
+ DuplicateRemovalPlan {
+ additions: plan
+ .additions
+ .into_iter()
+ .map(|addition| DuplicateAddition {
+ collection_id: addition.collection_id,
+ file_ids: addition.file_ids,
+ })
+ .collect(),
+ trash_ids: plan.trash_ids,
+ }
+ .into_js()
+ .map_err(Into::into)
+}
diff --git a/rust/bindings/wasm/photos/Cargo.toml b/rust/bindings/wasm/photos/Cargo.toml
index 00faeb76ba..b54eef2f7b 100644
--- a/rust/bindings/wasm/photos/Cargo.toml
+++ b/rust/bindings/wasm/photos/Cargo.toml
@@ -6,7 +6,7 @@ edition.workspace = true
crate-type = ["cdylib"]
[dependencies]
-ente-wasm-lib = { workspace = true, features = ["account-deletion", "cast", "collections", "contacts", "crypto", "location"] }
+ente-wasm-lib = { workspace = true, features = ["account-deletion", "cast", "collections", "contacts", "crypto", "location", "photos"] }
[lints]
workspace = true
diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.toml
index 25ca94302a..3330385dc2 100644
--- a/rust/crates/photos/Cargo.toml
+++ b/rust/crates/photos/Cargo.toml
@@ -3,21 +3,21 @@ name = "ente-photos"
edition.workspace = true
[dependencies]
-ente-assets.workspace = true
ente-collections.workspace = true
ente-core.workspace = true
ente-exif.workspace = true
-ente-location.workspace = true
-ente-vecdb.workspace = true
log.workspace = true
quick-xml.workspace = true
-rusqlite = { workspace = true, default-features = true }
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+ente-assets.workspace = true
+ente-location.workspace = true
+ente-vecdb.workspace = true
futures-util.workspace = true
+rusqlite = { workspace = true, default-features = true }
[dev-dependencies]
tempfile.workspace = true
diff --git a/rust/crates/photos/src/dedup.rs b/rust/crates/photos/src/dedup.rs
new file mode 100644
index 0000000000..425582448c
--- /dev/null
+++ b/rust/crates/photos/src/dedup.rs
@@ -0,0 +1,399 @@
+//! Duplicate detection and removal planning, independent of storage and UI.
+
+use std::collections::{HashMap, HashSet};
+
+pub struct Collection {
+ pub id: i64,
+ pub owner_id: i64,
+}
+
+/// A file's membership in one collection. Hashes must be normalized, including
+/// the `image:video` hash for Live Photos uploaded by legacy clients.
+pub struct File {
+ pub id: i64,
+ pub collection_id: i64,
+ pub owner_id: i64,
+ pub hash: Option<String>,
+ pub size: Option<u64>,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct Item {
+ /// Index of the first eligible occurrence in the input files.
+ pub file_index: usize,
+ pub collection_ids: Vec<i64>,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct Group {
+ pub items: Vec<Item>,
+ /// Estimate based on the first known nonzero size in the group. Live Photo
+ /// ZIP sizes can vary between clients even when their content hashes match.
+ pub item_size: u64,
+}
+
+/// Find distinct owned files with matching hashes in owned, non-hidden albums.
+/// The caller supplies only non-hidden collections. Groups, files and album
+/// memberships preserve input order; clients may sort items for display.
+pub fn find_duplicates(user_id: i64, collections: &[Collection], files: &[File]) -> Vec<Group> {
+ let allowed: HashSet<_> = collections
+ .iter()
+ .filter(|collection| collection.owner_id == user_id)
+ .map(|collection| collection.id)
+ .collect();
+ let mut by_id = HashMap::<i64, (usize, usize)>::new();
+ let mut by_hash = HashMap::<&str, usize>::new();
+ let mut groups = Vec::<Group>::new();
+ for (file_index, file) in files.iter().enumerate() {
+ if file.owner_id != user_id || !allowed.contains(&file.collection_id) {
+ continue;
+ }
+ let Some(hash) = file.hash.as_deref().filter(|hash| !hash.is_empty()) else {
+ continue;
+ };
+ if let Some(&(group_index, item_index)) = by_id.get(&file.id) {
+ let ids = &mut groups[group_index].items[item_index].collection_ids;
+ if !ids.contains(&file.collection_id) {
+ ids.push(file.collection_id);
+ }
+ continue;
+ }
+ let group_index = *by_hash.entry(hash).or_insert_with(|| {
+ groups.push(Group {
+ items: Vec::new(),
+ item_size: 0,
+ });
+ groups.len() - 1
+ });
+ let group = &mut groups[group_index];
+ by_id.insert(file.id, (group_index, group.items.len()));
+ group.items.push(Item {
+ file_index,
+ collection_ids: vec![file.collection_id],
+ });
+ if group.item_size == 0 {
+ group.item_size = file.size.unwrap_or(0);
+ }
+ }
+ groups
+ .into_iter()
+ .filter(|group| group.items.len() > 1 && group.item_size != 0)
+ .collect()
+}
+
+#[derive(Default)]
+pub struct Duplicate {
+ pub file_id: i64,
+ pub collection_ids: Vec<i64>,
+ pub caption: Option<String>,
+ pub edited_name: Option<String>,
+ pub edited_time: Option<i64>,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct Addition {
+ pub collection_id: i64,
+ pub file_ids: Vec<i64>,
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct RemovalPlan {
+ /// Complete every album addition successfully before trashing any files.
+ pub additions: Vec<Addition>,
+ pub trash_ids: Vec<i64>,
+}
+
+/// Plan removal of selected duplicate groups. Retain the first item with a
+/// caption, then the first with edits, otherwise the first item. Input order
+/// breaks ties (the web client uses its locale's album-name ordering).
+pub fn plan_removal(groups: &[Vec<Duplicate>]) -> RemovalPlan {
+ let mut plan = RemovalPlan::default();
+ let mut by_collection = HashMap::<i64, usize>::new();
+ for items in groups {
+ if items.len() < 2 {
+ continue;
+ }
+ let retained = items
+ .iter()
+ .find(|item| {
+ item.caption
+ .as_ref()
+ .is_some_and(|caption| !caption.is_empty())
+ })
+ .or_else(|| {
+ items.iter().find(|item| {
+ // Match editedName ?? editedTime: an explicitly empty name
+ // takes precedence over an edited timestamp.
+ item.edited_name.as_ref().map_or_else(
+ || item.edited_time.is_some_and(|time| time != 0),
+ |name| !name.is_empty(),
+ )
+ })
+ })
+ .unwrap_or(&items[0]);
+ let mut memberships: HashSet<_> = retained.collection_ids.iter().copied().collect();
+ for item in items {
+ if item.file_id == retained.file_id {
+ continue;
+ }
+ plan.trash_ids.push(item.file_id);
+ for &collection_id in &item.collection_ids {
+ if !memberships.insert(collection_id) {
+ continue;
+ }
+ let index = *by_collection.entry(collection_id).or_insert_with(|| {
+ plan.additions.push(Addition {
+ collection_id,
+ file_ids: Vec::new(),
+ });
+ plan.additions.len() - 1
+ });
+ plan.additions[index].file_ids.push(retained.file_id);
+ }
+ }
+ }
+ plan
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn file(id: i64, collection_id: i64, hash: &str, size: Option<u64>) -> File {
+ File {
+ id,
+ collection_id,
+ owner_id: 1,
+ hash: Some(hash.into()),
+ size,
+ }
+ }
+
+ fn collections() -> Vec<Collection> {
+ vec![
+ Collection {
+ id: 11,
+ owner_id: 1,
+ },
+ Collection {
+ id: 12,
+ owner_id: 1,
+ },
+ Collection {
+ id: 13,
+ owner_id: 2,
+ },
+ ]
+ }
+
+ #[test]
+ fn groups_distinct_owned_files_and_preserves_all_eligible_memberships() {
+ let mut foreign = file(3, 11, "hash", Some(100));
+ foreign.owner_id = 2;
+ let files = vec![
+ file(1, 11, "hash", Some(100)),
+ file(1, 12, "hash", Some(100)),
+ file(1, 12, "hash", Some(100)),
+ file(2, 12, "hash", Some(100)),
+ foreign,
+ file(4, 13, "hash", Some(100)),
+ file(5, 99, "hash", Some(100)),
+ file(6, 11, "", Some(100)),
+ ];
+ assert_eq!(
+ find_duplicates(1, &collections(), &files),
+ vec![Group {
+ items: vec![
+ Item {
+ file_index: 0,
+ collection_ids: vec![11, 12]
+ },
+ Item {
+ file_index: 3,
+ collection_ids: vec![12]
+ }
+ ],
+ item_size: 100,
+ }]
+ );
+ }
+
+ #[test]
+ fn ignores_singletons_symlinks_missing_hashes_and_unknown_sizes() {
+ let files = vec![
+ file(1, 11, "single", Some(100)),
+ file(1, 12, "single", Some(100)),
+ file(2, 11, "no-size", None),
+ file(3, 11, "no-size", Some(0)),
+ File {
+ hash: None,
+ ..file(4, 11, "", Some(100))
+ },
+ File {
+ hash: None,
+ ..file(5, 11, "", Some(100))
+ },
+ ];
+ assert!(find_duplicates(1, &collections(), &files).is_empty());
+ assert!(find_duplicates(1, &[], &files).is_empty());
+ }
+
+ #[test]
+ fn uses_first_nonzero_size_without_requiring_matching_live_photo_zip_sizes() {
+ let files = vec![
+ file(1, 11, "image:video", None),
+ file(2, 11, "image:video", Some(150)),
+ file(3, 12, "image:video", Some(200)),
+ file(4, 11, "other", Some(90)),
+ file(5, 11, "other", Some(100)),
+ ];
+ let groups = find_duplicates(1, &collections(), &files);
+ assert_eq!(
+ groups
+ .iter()
+ .map(|group| group.item_size)
+ .collect::<Vec<_>>(),
+ vec![150, 90]
+ );
+ assert_eq!(groups[0].items.len(), 3);
+ }
+
+ fn duplicate(id: i64, collection_ids: &[i64]) -> Duplicate {
+ Duplicate {
+ file_id: id,
+ collection_ids: collection_ids.into(),
+ ..Default::default()
+ }
+ }
+
+ #[test]
+ fn retains_captions_before_edits_and_batches_missing_album_memberships() {
+ let groups = vec![
+ vec![
+ Duplicate {
+ edited_name: Some("renamed".into()),
+ ..duplicate(1, &[11, 12])
+ },
+ Duplicate {
+ caption: Some("caption".into()),
+ ..duplicate(2, &[12])
+ },
+ duplicate(3, &[11, 13]),
+ ],
+ vec![duplicate(4, &[12]), duplicate(5, &[11])],
+ ];
+ assert_eq!(
+ plan_removal(&groups),
+ RemovalPlan {
+ additions: vec![
+ Addition {
+ collection_id: 11,
+ file_ids: vec![2, 4]
+ },
+ Addition {
+ collection_id: 13,
+ file_ids: vec![2]
+ }
+ ],
+ trash_ids: vec![1, 3, 5],
+ }
+ );
+ }
+
+ #[test]
+ fn retention_ties_follow_input_order_and_empty_edits_do_not_win() {
+ let mut items = vec![
+ duplicate(1, &[11]),
+ duplicate(2, &[11]),
+ duplicate(3, &[11]),
+ ];
+ assert_eq!(plan_removal(&[vec![]]), RemovalPlan::default());
+ assert_eq!(
+ plan_removal(&[vec![duplicate(1, &[11])]]),
+ RemovalPlan::default()
+ );
+ items[1].edited_time = Some(123);
+ assert_eq!(
+ plan_removal(std::slice::from_ref(&items)).trash_ids,
+ vec![1, 3]
+ );
+ items[1].edited_name = Some(String::new());
+ assert_eq!(
+ plan_removal(std::slice::from_ref(&items)).trash_ids,
+ vec![2, 3]
+ );
+ items[1].caption = Some("first caption".into());
+ items[2].caption = Some("second caption".into());
+ assert_eq!(
+ plan_removal(std::slice::from_ref(&items)).trash_ids,
+ vec![1, 3]
+ );
+ items[1].caption = Some(String::new());
+ assert_eq!(
+ plan_removal(std::slice::from_ref(&items)).trash_ids,
+ vec![1, 2]
+ );
+ assert!(plan_removal(&[items]).additions.is_empty());
+ }
+
+ #[test]
+ fn detection_and_removal_preserve_every_album_membership_combination() {
+ let collections: Vec<_> = (11..=13).map(|id| Collection { id, owner_id: 1 }).collect();
+ // Each of three distinct copies can belong to any nonempty subset of
+ // three albums. Applying the plan must leave one copy in every album.
+ for first in 1..8 {
+ for second in 1..8 {
+ for third in 1..8 {
+ let mut files = Vec::new();
+ let mut memberships = HashMap::<i64, HashSet<i64>>::new();
+ for (index, mask) in [first, second, third].into_iter().enumerate() {
+ let id = index as i64 + 1;
+ for bit in 0..3 {
+ if mask & (1 << bit) != 0 {
+ let collection_id = 11 + bit;
+ files.push(file(id, collection_id, "hash", Some(100)));
+ memberships.entry(collection_id).or_default().insert(id);
+ }
+ }
+ }
+ let expected_albums: HashSet<_> = memberships.keys().copied().collect();
+ let groups = find_duplicates(1, &collections, &files);
+ assert_eq!(groups.len(), 1);
+ assert_eq!(groups[0].items.len(), 3);
+ let selected: Vec<_> = groups
+ .into_iter()
+ .map(|group| {
+ group
+ .items
+ .into_iter()
+ .map(|item| {
+ let id = files[item.file_index].id;
+ Duplicate {
+ caption: (id == 2).then(|| "keep".into()),
+ ..duplicate(id, &item.collection_ids)
+ }
+ })
+ .collect()
+ })
+ .collect();
+ let plan = plan_removal(&selected);
+ assert_eq!(plan.trash_ids, vec![1, 3]);
+ for addition in plan.additions {
+ memberships
+ .entry(addition.collection_id)
+ .or_default()
+ .extend(addition.file_ids);
+ }
+ for ids in memberships.values_mut() {
+ ids.retain(|id| !plan.trash_ids.contains(id));
+ assert_eq!(*ids, HashSet::from([2]));
+ }
+ assert_eq!(
+ memberships.keys().copied().collect::<HashSet<_>>(),
+ expected_albums
+ );
+ }
+ }
+ }
+ }
+}
diff --git a/rust/crates/photos/src/files.rs b/rust/crates/photos/src/files.rs
index 163b17d32e..66cfca872a 100644
--- a/rust/crates/photos/src/files.rs
+++ b/rust/crates/photos/src/files.rs
@@ -1,5 +1,7 @@
+#[cfg(not(target_arch = "wasm32"))]
+use ente_core::Session;
use ente_core::{
- Session, b64,
+ b64,
crypto::{self, Header, Key, blob},
http,
};
diff --git a/rust/crates/photos/src/lib.rs b/rust/crates/photos/src/lib.rs
index 377197a4ae..1d9cea7883 100644
--- a/rust/crates/photos/src/lib.rs
+++ b/rust/crates/photos/src/lib.rs
@@ -1,11 +1,16 @@
pub mod collections;
+#[cfg(not(target_arch = "wasm32"))]
pub mod db;
+pub mod dedup;
pub mod face_clustering;
pub mod files;
+#[cfg(not(target_arch = "wasm32"))]
pub mod location;
pub mod memories;
pub mod metadata;
+#[cfg(not(target_arch = "wasm32"))]
pub mod ml_db;
+#[cfg(not(target_arch = "wasm32"))]
pub mod ml_store;
pub mod motion_photo;
pub mod source;
diff --git a/web/apps/photos/package.json b/web/apps/photos/package.json
index 7319d624de..34b89104b8 100644
--- a/web/apps/photos/package.json
+++ b/web/apps/photos/package.json
@@ -39,6 +39,7 @@
"@types/react-dom": "19.3.0",
"@types/react-window": "1.8.8",
"ente-build-config": "*",
+ "vite-plugin-wasm": "3.6.0",
"vitest": "5.0.4"
}
}
diff --git a/web/apps/photos/src/services/dedup.ts b/web/apps/photos/src/services/dedup.ts
index 2bc56ec9cd..de8d245605 100644
--- a/web/apps/photos/src/services/dedup.ts
+++ b/web/apps/photos/src/services/dedup.ts
@@ -2,7 +2,10 @@ import { ensureLocalUser } from "ente-accounts/services/user";
import { assertionFailed } from "ente-base/assert";
import { newID } from "ente-base/id";
import type { EnteFile } from "ente-media/file";
-import { metadataHash } from "ente-media/file-metadata";
+import {
+ FilePublicMagicMetadataData,
+ metadataHash,
+} from "ente-media/file-metadata";
import {
addToCollection,
createCollectionNameByID,
@@ -10,6 +13,7 @@ import {
savedNormalCollections,
} from "ente-new/photos/services/collection";
import { savedCollectionFiles } from "ente-new/photos/services/photos-fdb";
+import { findDuplicates, planDuplicateRemoval } from "ente-photos-wasm";
export interface DuplicateGroup {
id: string;
@@ -28,67 +32,42 @@ export const deduceDuplicates = async () => {
const userID = ensureLocalUser().id;
const normalCollections = await savedNormalCollections();
- const normalOwnedCollections = normalCollections.filter(
- ({ owner }) => owner.id == userID,
- );
- const allowedCollectionIDs = new Set(
- normalOwnedCollections.map(({ id }) => id),
- );
- const collectionNameByID = createCollectionNameByID(normalOwnedCollections);
-
+ const collectionNameByID = createCollectionNameByID(normalCollections);
const collectionFiles = await savedCollectionFiles();
- const filteredCollectionFiles = collectionFiles.filter((f) =>
- allowedCollectionIDs.has(f.collectionID),
- );
-
- // One file ID can appear once per collection; hash it only once.
- const collectionIDsByFileID = new Map<number, Set<number>>();
- const filesByHash = new Map<string, EnteFile[]>();
- for (const file of filteredCollectionFiles) {
- // An owned album can contain files the user does not own.
- if (file.ownerID != userID) continue;
-
- const hash = metadataHash(file.metadata);
- if (!hash) {
- continue;
- }
-
- let collectionIDs = collectionIDsByFileID.get(file.id);
- if (!collectionIDs) {
- collectionIDsByFileID.set(file.id, (collectionIDs = new Set()));
- filesByHash.set(hash, [...(filesByHash.get(hash) ?? []), file]);
- }
- collectionIDs.add(file.collectionID);
- }
+ const { groups } = await findDuplicates({
+ userId: userID,
+ collections: normalCollections.map(({ id, owner }) => ({
+ id,
+ ownerId: owner.id,
+ })),
+ files: collectionFiles.map((file) => ({
+ id: file.id,
+ collectionId: file.collectionID,
+ ownerId: file.ownerID,
+ hash: metadataHash(file.metadata),
+ size: file.info?.fileSize,
+ })),
+ });
const duplicateGroups: DuplicateGroup[] = [];
- for (const duplicates of filesByHash.values()) {
- if (duplicates.length < 2) continue;
-
- // Live-photo ZIP sizes can vary by client despite matching content hashes.
- let size = 0;
- for (const file of duplicates) {
- if (file.info?.fileSize) {
- size = file.info.fileSize;
- break;
- }
- }
-
- if (!size) continue;
-
- const items = duplicates
- .map((file) => {
+ for (const group of groups) {
+ const items = group.items
+ .map(({ fileIndex, collectionIds }) => {
+ const file = collectionFiles[fileIndex]!;
const collectionName = collectionNameByID.get(
file.collectionID,
);
- const collectionIDs = collectionIDsByFileID.get(file.id);
- if (!collectionName || !collectionIDs) {
+ if (!collectionName) {
assertionFailed();
return undefined;
}
- return { file, collectionIDs, collectionName };
+ return {
+ file,
+ collectionIDs: new Set(collectionIds),
+ collectionName,
+ };
})
.filter((item) => !!item);
if (items.length < 2) continue;
@@ -98,9 +77,9 @@ export const deduceDuplicates = async () => {
duplicateGroups.push({
id: newID("dg_"),
items,
- itemSize: size,
+ itemSize: group.itemSize,
prunableCount: items.length - 1,
- prunableSize: size * (items.length - 1),
+ prunableSize: group.itemSize * (items.length - 1),
isSelected: true,
});
}
@@ -115,44 +94,44 @@ export const removeSelectedDuplicateGroups = async (
) => {
const selectedDuplicateGroups = duplicateGroups.filter((g) => g.isSelected);
- // Symlink the retained file into every affected album before trashing duplicates.
- const filesToAdd = new Map<number, EnteFile[]>();
- const filesToTrash: EnteFile[] = [];
-
- for (const duplicateGroup of selectedDuplicateGroups) {
- const retainedItem = duplicateGroupItemToRetain(duplicateGroup);
-
- let collectionIDs = new Set<number>();
- for (const item of duplicateGroup.items) {
- if (item.file.id == retainedItem.file.id) continue;
- collectionIDs = collectionIDs.union(item.collectionIDs);
- filesToTrash.push(item.file);
- }
-
- collectionIDs = collectionIDs.difference(retainedItem.collectionIDs);
-
- for (const collectionID of collectionIDs) {
- filesToAdd.set(collectionID, [
- ...(filesToAdd.get(collectionID) ?? []),
- retainedItem.file,
- ]);
- }
- }
+ const filesByID = new Map(
+ selectedDuplicateGroups.flatMap((group) =>
+ group.items.map(({ file }) => [file.id, file] as const),
+ ),
+ );
+ const plan = await planDuplicateRemoval({
+ groups: selectedDuplicateGroups.map((group) =>
+ group.items.map(({ file, collectionIDs }) => ({
+ fileId: file.id,
+ collectionIds: [...collectionIDs],
+ // Cached files can predate numeric-caption normalization.
+ caption: FilePublicMagicMetadataData.shape.caption.parse(
+ file.pubMagicMetadata?.data.caption,
+ ),
+ editedName: file.pubMagicMetadata?.data.editedName,
+ editedTime: file.pubMagicMetadata?.data.editedTime,
+ })),
+ ),
+ });
let np = 0;
const ntotal =
- filesToAdd.size + (filesToTrash.length ? 1 : 0) + /* sync */ 1;
+ plan.additions.length + (plan.trashIds.length ? 1 : 0) + /* sync */ 1;
const tickProgress = () => onProgress((++np / ntotal) * 100);
const collections = await savedNormalCollections();
const collectionsByID = new Map(collections.map((c) => [c.id, c]));
- for (const [collectionID, files] of filesToAdd.entries()) {
- await addToCollection(collectionsByID.get(collectionID)!, files);
+ // Complete every album addition before trashing any duplicates.
+ for (const { collectionId, fileIds } of plan.additions) {
+ await addToCollection(
+ collectionsByID.get(collectionId)!,
+ fileIds.map((id) => filesByID.get(id)!),
+ );
tickProgress();
}
- if (filesToTrash.length) {
- await moveToTrash(filesToTrash);
+ if (plan.trashIds.length) {
+ await moveToTrash(plan.trashIds.map((id) => filesByID.get(id)!));
tickProgress();
}
@@ -161,21 +140,3 @@ export const removeSelectedDuplicateGroups = async (
return new Set(selectedDuplicateGroups.map((g) => g.id));
};
-
-const duplicateGroupItemToRetain = (duplicateGroup: DuplicateGroup) => {
- const itemsWithCaption: DuplicateGroup["items"] = [];
- const itemsWithOtherEdits: DuplicateGroup["items"] = [];
- for (const item of duplicateGroup.items) {
- const pubMM = item.file.pubMagicMetadata?.data;
- if (!pubMM) continue;
- if (pubMM.caption) itemsWithCaption.push(item);
- if (pubMM.editedName ?? pubMM.editedTime)
- itemsWithOtherEdits.push(item);
- }
-
- return (
- itemsWithCaption[0] ??
- itemsWithOtherEdits[0] ??
- duplicateGroup.items[0]!
- );
-};
diff --git a/web/apps/photos/tests/dedup.test.ts b/web/apps/photos/tests/dedup.test.ts
index fac7c2e3c5..da47d8555c 100644
--- a/web/apps/photos/tests/dedup.test.ts
+++ b/web/apps/photos/tests/dedup.test.ts
@@ -1,24 +1,140 @@
import type { EnteFile } from "ente-media/file";
import { beforeEach, describe, expect, test, vi } from "vitest";
import {
+ deduceDuplicates,
removeSelectedDuplicateGroups,
type DuplicateGroup,
} from "../src/services/dedup";
-const { addToCollection, moveToTrash, savedNormalCollections } = vi.hoisted(
- () => ({
- addToCollection: vi.fn(),
- moveToTrash: vi.fn(),
- savedNormalCollections: vi.fn(),
- }),
-);
+const {
+ addToCollection,
+ moveToTrash,
+ savedNormalCollections,
+ savedCollectionFiles,
+} = vi.hoisted(() => ({
+ addToCollection: vi.fn(),
+ moveToTrash: vi.fn(),
+ savedNormalCollections: vi.fn(),
+ savedCollectionFiles: vi.fn(),
+}));
vi.mock("ente-new/photos/services/collection", () => ({
addToCollection,
moveToTrash,
savedNormalCollections,
+ createCollectionNameByID: (collections: { id: number; name: string }[]) =>
+ new Map(collections.map(({ id, name }) => [id, name])),
+}));
+vi.mock("ente-new/photos/services/photos-fdb", () => ({
+ savedCollectionFiles,
}));
-vi.mock("ente-new/photos/services/photos-fdb", () => ({}));
-vi.mock("ente-accounts/services/user", () => ({}));
+vi.mock("ente-accounts/services/user", () => ({
+ ensureLocalUser: () => ({ id: 1 }),
+}));
+
+const file = (
+ id: number,
+ collectionID: number,
+ hash: string | undefined,
+ size: number | undefined = 100,
+) =>
+ ({
+ id,
+ collectionID,
+ ownerID: 1,
+ metadata: { fileType: 0, hash },
+ info: { fileSize: size },
+ }) as EnteFile;
+
+describe("deduceDuplicates through Rust WASM", () => {
+ beforeEach(() => {
+ vi.resetAllMocks();
+ savedNormalCollections.mockResolvedValue([
+ { id: 11, owner: { id: 1 }, name: "B" },
+ { id: 12, owner: { id: 1 }, name: "A" },
+ { id: 13, owner: { id: 2 }, name: "Shared" },
+ ]);
+ });
+
+ test("groups owned files once across albums, excluding shared files and hidden albums", async () => {
+ const first = file(1, 11, "hash");
+ const second = file(2, 12, "hash");
+ savedCollectionFiles.mockResolvedValue([
+ first,
+ { ...first, collectionID: 12 },
+ second,
+ { ...file(3, 11, "hash"), ownerID: 2 },
+ file(4, 13, "hash"),
+ file(5, 99, "hash"),
+ file(6, 11, undefined),
+ file(7, 11, "unique"),
+ ]);
+ const groups = await deduceDuplicates();
+ expect(groups).toEqual([
+ {
+ id: expect.any(String) as string,
+ items: [
+ {
+ file: second,
+ collectionIDs: new Set([12]),
+ collectionName: "A",
+ },
+ {
+ file: first,
+ collectionIDs: new Set([11, 12]),
+ collectionName: "B",
+ },
+ ],
+ itemSize: 100,
+ prunableCount: 1,
+ prunableSize: 100,
+ isSelected: true,
+ },
+ ]);
+ // The locale-sorted first item is the fallback retained copy in Rust.
+ await removeSelectedDuplicateGroups(groups, vi.fn(), vi.fn());
+ expect(addToCollection).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({ id: 11 }),
+ [second],
+ );
+ expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([first]);
+ });
+
+ test("matches legacy Live Photo hashes despite different ZIP sizes and skips groups without sizes", async () => {
+ const legacy = file(1, 11, undefined, 0);
+ legacy.metadata = {
+ ...legacy.metadata,
+ fileType: 2,
+ imageHash: "image",
+ videoHash: "video",
+ };
+ savedCollectionFiles.mockResolvedValue([
+ legacy,
+ file(2, 12, "image:video", 150),
+ file(3, 11, "image:video", 200),
+ file(4, 11, "unknown-size", 0),
+ { ...file(5, 11, "unknown-size"), info: undefined },
+ ]);
+ const groups = await deduceDuplicates();
+ expect(groups).toHaveLength(1);
+ expect(groups[0]).toMatchObject({
+ itemSize: 150,
+ prunableCount: 2,
+ prunableSize: 300,
+ });
+ expect(groups[0]!.items.map(({ file }) => file.id)).toEqual([2, 1, 3]);
+ });
+
+ test("ignores a single file linked to multiple albums and empty libraries", async () => {
+ savedCollectionFiles.mockResolvedValue([
+ file(1, 11, "hash"),
+ file(1, 12, "hash"),
+ file(1, 12, "hash"),
+ ]);
+ expect(await deduceDuplicates()).toEqual([]);
+ savedCollectionFiles.mockResolvedValue([]);
+ expect(await deduceDuplicates()).toEqual([]);
+ });
+});
const duplicateGroup = (): DuplicateGroup => ({
id: "group-1",
@@ -56,6 +172,119 @@ describe("removeSelectedDuplicateGroups", () => {
savedNormalCollections.mockResolvedValue(collections);
});
+ test.each([123, 0])(
+ "normalizes legacy cached numeric caption %s before planning in Rust",
+ async (caption) => {
+ const group = duplicateGroup();
+ const retained = group.items[1]!.file;
+ // Cached records bypass the metadata schema and can predate normalization.
+ Object.assign(retained.pubMagicMetadata!.data, { caption });
+ const refresh = vi.fn();
+
+ await expect(
+ removeSelectedDuplicateGroups([group], vi.fn(), refresh),
+ ).resolves.toEqual(new Set([group.id]));
+ expect(addToCollection.mock.calls).toEqual([
+ [collections[0], [retained]],
+ [collections[2], [retained]],
+ ]);
+ expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([
+ group.items[0]!.file,
+ group.items[2]!.file,
+ ]);
+ expect(refresh).toHaveBeenCalledOnce();
+ },
+ );
+
+ test.each([
+ {
+ label: "caption before edits",
+ metadata: [{ editedName: "renamed" }, { caption: "caption" }, {}],
+ retainedId: 2,
+ },
+ {
+ label: "renamed copy",
+ metadata: [{}, {}, { editedName: "renamed" }],
+ retainedId: 3,
+ },
+ {
+ label: "edited timestamp",
+ metadata: [{}, { editedTime: 123 }, {}],
+ retainedId: 2,
+ },
+ {
+ label: "first of equal captions",
+ metadata: [{}, { caption: "one" }, { caption: "two" }],
+ retainedId: 2,
+ },
+ {
+ label: "first of equal edits",
+ metadata: [{}, { editedName: "one" }, { editedTime: 123 }],
+ retainedId: 2,
+ },
+ {
+ label: "empty caption and null caption",
+ metadata: [{}, { caption: "" }, { caption: null }],
+ retainedId: 1,
+ },
+ {
+ label: "empty name overrides edited timestamp",
+ metadata: [{}, { editedName: "", editedTime: 123 }, {}],
+ retainedId: 1,
+ },
+ ])(
+ "retains the $label through Rust WASM",
+ async ({ metadata, retainedId }) => {
+ const group = duplicateGroup();
+ group.items.forEach(({ file }, index) => {
+ Object.assign(file, {
+ pubMagicMetadata: { data: metadata[index] },
+ });
+ });
+ await removeSelectedDuplicateGroups([group], vi.fn(), vi.fn());
+ expect(moveToTrash).toHaveBeenCalledExactlyOnceWith(
+ group.items
+ .filter(({ file }) => file.id !== retainedId)
+ .map(({ file }) => file),
+ );
+ },
+ );
+
+ test("batches retained copies from different groups into each missing album", async () => {
+ const first = duplicateGroup();
+ const second = duplicateGroup();
+ second.id = "group-2";
+ for (const { file } of second.items) file.id += 3;
+ const refresh = vi.fn();
+ await expect(
+ removeSelectedDuplicateGroups([first, second], vi.fn(), refresh),
+ ).resolves.toEqual(new Set([first.id, second.id]));
+ const retained = [first.items[1]!.file, second.items[1]!.file];
+ expect(addToCollection.mock.calls).toEqual([
+ [collections[0], retained],
+ [collections[2], retained],
+ ]);
+ expect(moveToTrash).toHaveBeenCalledExactlyOnceWith([
+ first.items[0]!.file,
+ first.items[2]!.file,
+ second.items[0]!.file,
+ second.items[2]!.file,
+ ]);
+ expect(refresh).toHaveBeenCalledOnce();
+ });
+
+ test("rejects invalid planner input before modifying albums or trash", async () => {
+ const group = duplicateGroup();
+ group.items[1]!.file.id = Number.MAX_SAFE_INTEGER + 1;
+ const refresh = vi.fn();
+ await expect(
+ removeSelectedDuplicateGroups([group], vi.fn(), refresh),
+ ).rejects.toBeInstanceOf(Error);
+ expect(addToCollection).not.toHaveBeenCalled();
+ expect(moveToTrash).not.toHaveBeenCalled();
+ expect(refresh).not.toHaveBeenCalled();
+ });
+
test("preserves album membership before trashing selected duplicates and awaits gallery refresh", async () => {
const group = duplicateGroup();
const untouched = {
@@ -108,6 +337,20 @@ describe("removeSelectedDuplicateGroups", () => {
expect(refresh).not.toHaveBeenCalled();
});
+ test("does not trash any files if a later album addition fails", async () => {
+ const failure = new Error("Second album failed");
+ addToCollection
+ .mockResolvedValueOnce(undefined)
+ .mockRejectedValueOnce(failure);
+ const refresh = vi.fn();
+ await expect(
+ removeSelectedDuplicateGroups([duplicateGroup()], vi.fn(), refresh),
+ ).rejects.toBe(failure);
+ expect(addToCollection).toHaveBeenCalledTimes(2);
+ expect(moveToTrash).not.toHaveBeenCalled();
+ expect(refresh).not.toHaveBeenCalled();
+ });
+
test("does not refresh when trashing fails", async () => {
const failure = new Error("Trash failed");
moveToTrash.mockRejectedValue(failure);
diff --git a/web/apps/photos/vitest.config.ts b/web/apps/photos/vitest.config.ts
index 51e689a419..9559186f8c 100644
--- a/web/apps/photos/vitest.config.ts
+++ b/web/apps/photos/vitest.config.ts
@@ -1,6 +1,9 @@
+import type { PluginOption } from "vite";
+import wasm from "vite-plugin-wasm";
import { defineConfig } from "vitest/config";
export default defineConfig({
+ plugins: [wasm() as PluginOption],
test: { include: ["tests/**/*.test.{ts,tsx}"] },
// Next preserves JSX for its own compiler; component tests need it compiled.
oxc: { jsx: { runtime: "automatic" } },
diff --git a/web/package-lock.json b/web/package-lock.json
index af90d17724..cef2db017e 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -352,6 +352,7 @@
"@types/react-dom": "19.3.0",
"@types/react-window": "1.8.8",
"ente-build-config": "*",
+ "vite-plugin-wasm": "3.6.0",
"vitest": "5.0.4"
}
},
diff --git a/web/packages/wasm/photos/index.ts b/web/packages/wasm/photos/index.ts
index e691f74a57..f23806457e 100644
--- a/web/packages/wasm/photos/index.ts
+++ b/web/packages/wasm/photos/index.ts
@@ -1,7 +1,9 @@
import type {
EncryptedBox,
+ FindDuplicatesInput,
LocationIndex,
OpenSessionInput,
+ PlanDuplicateRemovalInput,
Session,
WrappedRootContactKey,
} from "./pkg/ente_photos_wasm";
@@ -15,6 +17,12 @@ export type {
const wasm = () => import("./pkg/ente_photos_wasm");
+export const findDuplicates = async (input: FindDuplicatesInput) =>
+ (await wasm()).photosFindDuplicates(input);
+
+export const planDuplicateRemoval = async (input: PlanDuplicateRemovalInput) =>
+ (await wasm()).photosPlanDuplicateRemoval(input);
+
export const createLocationIndex = async (
cities: Uint8Array,
urbanCenters: Uint8Array,