π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[photos][rust] Expose deduplication through FRB (#14328)
Aswin Asok20720fd7c644
diff --git a/.github/workflows/mobile-lint.yml b/.github/workflows/mobile-lint.ymlindex a1672d4d27..6a006e16a5 100644--- a/.github/workflows/mobile-lint.yml+++ b/.github/workflows/mobile-lint.yml@@ -101,9 +101,16 @@ jobs:- run: flutter analyze --no-pubworking-directory: mobile+ - name: Build Photos Rust library for tests+ run: cargo build -p ente-photos-frb --features flutter --locked+ working-directory: rust+- name: Testworking-directory: mobile+ env:+ ENTE_PHOTOS_FRB_LIBRARY: ${{ github.workspace }}/rust/target/debug/libente_photos_frb.sorun: |+ test -f "$ENTE_PHOTOS_FRB_LIBRARY"test_package() (member=${1#"$PWD/"}echo "Testing $member"diff --git a/mobile/apps/photos/test/rust/dedup_bridge_test.dart b/mobile/apps/photos/test/rust/dedup_bridge_test.dartnew file mode 100644index 0000000000..53b5cb2255--- /dev/null+++ b/mobile/apps/photos/test/rust/dedup_bridge_test.dart@@ -0,0 +1,141 @@+import 'dart:convert';+import 'dart:io';++import 'package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart';+import 'package:flutter_test/flutter_test.dart';+import 'package:photos/src/rust/api/dedup_api.dart';+import 'package:photos/src/rust/frb_generated.dart';+import 'package:photos/src/rust/third_party/ente_frb_lib/session.dart';++void main() {+ final library = Platform.environment['ENTE_PHOTOS_FRB_LIBRARY'] ?? '';++ test(+ 'finds and removes duplicates through the generated Dart bindings',+ () async {+ await EntePhotosRust.init(externalLibrary: ExternalLibrary.open(library));+ final requests = <(String, Map<String, dynamic>)>[];+ final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);+ addTearDown(() => server.close(force: true));+ server.listen((request) async {+ final body = await utf8.decoder.bind(request).join();+ requests.add((+ request.uri.path,+ jsonDecode(body) as Map<String, dynamic>,+ ));+ request.response+ ..headers.contentType = ContentType.json+ ..write('{}');+ await request.response.close();+ });++ final files = [+ for (final (id, album) in [(21, 11), (21, 12), (22, 12)])+ DedupFile(+ id: id,+ collectionId: album,+ ownerId: 1,+ hash: 'test hash',+ size: BigInt.from(100),+ ),+ ];+ final groups = await findDuplicates(+ userId: 1,+ collections: [+ for (final id in [11, 12])+ DedupCollection(id: id, ownerId: 1, isHidden: false),+ ],+ files: files,+ );+ final group = groups.single;+ expect(group.itemSize, BigInt.from(100));+ expect(group.items.map((item) => item.fileIndex.toInt()), [0, 2]);+ expect(group.items.first.collectionIds.map((id) => id.toInt()), [11, 12]);++ final session = _openTestSession(server.port);+ addTearDown(session.dispose);+ final events = await executeDuplicateRemoval(+ session: session,+ input: ExecuteDuplicateRemovalInput(+ groups: [+ [+ for (final item in group.items)+ Duplicate(+ fileId: files[item.fileIndex.toInt()].id,+ collectionIds: item.collectionIds,+ caption: item.fileIndex == BigInt.two+ ? 'Keep this copy'+ : null,+ ),+ ],+ ],+ collections: [RemovalCollection(id: 11, key: Uint8List(32))],+ files: [+ for (final index in [0, 2])+ RemovalFile(+ id: files[index].id,+ collectionId: files[index].collectionId,+ key: Uint8List(32),+ ),+ ],+ ),+ ).toList();++ expect(events, [+ DuplicateRemovalEvent.progress(+ completed: BigInt.one,+ total: BigInt.two,+ ),+ DuplicateRemovalEvent.progress(+ completed: BigInt.two,+ total: BigInt.two,+ ),+ isA<DuplicateRemovalEvent_Completed>().having(+ (event) => event.trashIds.map((id) => id.toInt()),+ 'trashed IDs',+ [21],+ ),+ ]);+ expect(requests.map((request) => request.$1), [+ '/collections/add-files',+ '/files/trash',+ ]);+ expect(requests.first.$2, {+ 'collectionID': 11,+ 'files': [+ {+ 'id': 22,+ 'encryptedKey': isA<String>(),+ 'keyDecryptionNonce': isA<String>(),+ },+ ],+ });+ expect(requests.last.$2, {+ 'items': [+ {'fileID': 21, 'collectionID': 11},+ ],+ });+ },+ skip: library.isEmpty+ ? 'Set ENTE_PHOTOS_FRB_LIBRARY to run against Rust'+ : false,+ );+}++Session _openTestSession(int port) => openSession(+ input: OpenSessionInput(+ baseUrl: 'http://127.0.0.1:$port',+ authToken: 'test-token',+ userId: 1,+ masterKey: Uint8List(32),+ keyAttributes: const SessionKeyAttributes(+ publicKey: 'pOCSkrZRwni5dyxWn1+puxPZBrRqtoyd+dwrRAn4ogk=',+ encryptedSecretKey:+ '+X2V/HMj8w3AXsDR/40cCYKhGG+xIjoYEY1fhrVki/8lsbbYnoppV9livaNsC7WX',+ secretKeyDecryptionNonce: 'AgICAgICAgICAgICAgICAgICAgICAgIC',+ recoveryKeyEncryptedWithMasterKey:+ 'jok1oUB0jgsas9Ske6QuwF0e3qieBz+U7/ls4H+8IvoAiqgtr2AGgrqU4SSFwV4F',+ recoveryKeyDecryptionNonce: 'BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE',+ ),+ ),+);diff --git a/rust/bindings/frb/photos/src/api/dedup_api.rs b/rust/bindings/frb/photos/src/api/dedup_api.rsnew file mode 100644index 0000000000..e0655aefa9--- /dev/null+++ b/rust/bindings/frb/photos/src/api/dedup_api.rs@@ -0,0 +1,213 @@+#[cfg(any(feature = "flutter", frb_expand))]+use ente_core::crypto::Key;+#[cfg(any(feature = "flutter", frb_expand))]+use ente_frb_lib::session::{Session, SessionError};+use ente_photos::dedup;++#[cfg(any(feature = "flutter", frb_expand))]+use crate::frb_generated::StreamSink;++pub struct DedupCollection {+ pub id: i64,+ pub owner_id: i64,+ pub is_hidden: bool,+}++pub struct DedupFile {+ pub id: i64,+ pub collection_id: i64,+ pub owner_id: i64,+ pub hash: Option<String>,+ pub size: Option<u64>,+}++pub struct DuplicateItem {+ pub file_index: usize,+ pub collection_ids: Vec<i64>,+}++pub struct DuplicateGroup {+ pub items: Vec<DuplicateItem>,+ pub item_size: u64,+}++pub fn find_duplicates(+ user_id: i64,+ collections: Vec<DedupCollection>,+ files: Vec<DedupFile>,+) -> Vec<DuplicateGroup> {+ let collections: Vec<_> = collections+ .into_iter()+ .map(|collection| dedup::Collection {+ id: collection.id,+ owner_id: collection.owner_id,+ is_hidden: collection.is_hidden,+ })+ .collect();+ let files: Vec<_> = files+ .into_iter()+ .map(|file| dedup::File {+ id: file.id,+ collection_id: file.collection_id,+ owner_id: file.owner_id,+ hash: file.hash,+ size: file.size,+ })+ .collect();+ dedup::find_duplicates(user_id, &collections, &files)+ .into_iter()+ .map(|group| DuplicateGroup {+ items: group+ .items+ .into_iter()+ .map(|item| DuplicateItem {+ file_index: item.file_index,+ collection_ids: item.collection_ids,+ })+ .collect(),+ item_size: group.item_size,+ })+ .collect()+}++pub struct Duplicate {+ pub file_id: i64,+ pub collection_ids: Vec<i64>,+ pub caption: Option<String>,+ pub edited_name: Option<String>,+ pub edited_time: Option<i64>,+}++pub struct RemovalCollection {+ pub id: i64,+ pub key: Vec<u8>,+}++pub struct RemovalFile {+ pub id: i64,+ pub collection_id: i64,+ pub key: Vec<u8>,+}++pub struct ExecuteDuplicateRemovalInput {+ pub groups: Vec<Vec<Duplicate>>,+ pub collections: Vec<RemovalCollection>,+ pub files: Vec<RemovalFile>,+}++#[cfg(any(feature = "flutter", frb_expand))]+pub enum DuplicateRemovalEvent {+ Progress { completed: usize, total: usize },+ Completed { trash_ids: Vec<i64> },+ Failed { error: SessionError },+}++#[cfg(any(feature = "flutter", frb_expand))]+pub async fn execute_duplicate_removal(+ session: &Session,+ input: ExecuteDuplicateRemovalInput,+ events: StreamSink<DuplicateRemovalEvent>,+) {+ let result = remove_duplicates(session, input, &events).await;+ let _ = events.add(match result {+ Ok(trash_ids) => DuplicateRemovalEvent::Completed { trash_ids },+ Err(error) => DuplicateRemovalEvent::Failed { error },+ });+}++#[cfg(any(feature = "flutter", frb_expand))]+async fn remove_duplicates(+ session: &Session,+ input: ExecuteDuplicateRemovalInput,+ events: &StreamSink<DuplicateRemovalEvent>,+) -> Result<Vec<i64>, SessionError> {+ let groups: Vec<_> = input+ .groups+ .into_iter()+ .map(|group| {+ group+ .into_iter()+ .map(|item| dedup::Duplicate {+ file_id: item.file_id,+ collection_ids: item.collection_ids,+ caption: item.caption,+ edited_name: item.edited_name,+ edited_time: item.edited_time,+ })+ .collect()+ })+ .collect();+ let collections = input+ .collections+ .into_iter()+ .map(|collection| {+ Ok(dedup::RemovalCollection {+ id: collection.id,+ key: Key::try_from_slice(&collection.key)?,+ })+ })+ .collect::<Result<Vec<_>, SessionError>>()?;+ let files = input+ .files+ .into_iter()+ .map(|file| {+ Ok(dedup::RemovalFile {+ id: file.id,+ collection_id: file.collection_id,+ key: Key::try_from_slice(&file.key)?,+ })+ })+ .collect::<Result<Vec<_>, SessionError>>()?;+ dedup::remove_duplicates(+ session.as_ref(),+ &groups,+ &collections,+ &files,+ |progress| {+ let _ = events.add(DuplicateRemovalEvent::Progress {+ completed: progress.completed,+ total: progress.total,+ });+ },+ )+ .await+ .map_err(SessionError::from)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn detection_preserves_memberships() {+ let collections = vec![+ DedupCollection {+ id: 11,+ owner_id: 1,+ is_hidden: false,+ },+ DedupCollection {+ id: 12,+ owner_id: 1,+ is_hidden: false,+ },+ ];+ let files: Vec<_> = [(1, 11), (1, 12), (2, 12)]+ .into_iter()+ .map(|(id, collection_id)| DedupFile {+ id,+ collection_id,+ owner_id: 1,+ hash: Some("hash".into()),+ size: Some(100),+ })+ .collect();+ let groups = find_duplicates(1, collections, files);+ assert_eq!(groups.len(), 1);+ assert_eq!(groups[0].item_size, 100);+ assert_eq!(groups[0].items.len(), 2);+ assert_eq!(groups[0].items[0].file_index, 0);+ assert_eq!(groups[0].items[0].collection_ids, vec![11, 12]);+ assert_eq!(groups[0].items[1].file_index, 2);+ }+}diff --git a/rust/bindings/frb/photos/src/api/mod.rs b/rust/bindings/frb/photos/src/api/mod.rsindex 6f650a56ec..406438cc28 100644--- a/rust/bindings/frb/photos/src/api/mod.rs+++ b/rust/bindings/frb/photos/src/api/mod.rs@@ -1,4 +1,5 @@pub mod cast_api;+pub mod dedup_api;pub mod image_processing_api;pub mod init;pub mod location_api;