Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[photos][rust] Expose deduplication through FRB (#14328)

Aswin Asok20720fd7c644
diff --git a/.github/workflows/mobile-lint.yml b/.github/workflows/mobile-lint.yml
index a1672d4d27..6a006e16a5 100644
--- a/.github/workflows/mobile-lint.yml
+++ b/.github/workflows/mobile-lint.yml
@@ -101,9 +101,16 @@ jobs:
- run: flutter analyze --no-pub
working-directory: mobile
+ - name: Build Photos Rust library for tests
+ run: cargo build -p ente-photos-frb --features flutter --locked
+ working-directory: rust
+
- name: Test
working-directory: mobile
+ env:
+ ENTE_PHOTOS_FRB_LIBRARY: ${{ github.workspace }}/rust/target/debug/libente_photos_frb.so
run: |
+ test -f "$ENTE_PHOTOS_FRB_LIBRARY"
test_package() (
member=${1#"$PWD/"}
echo "Testing $member"
diff --git a/mobile/apps/photos/test/rust/dedup_bridge_test.dart b/mobile/apps/photos/test/rust/dedup_bridge_test.dart
new file mode 100644
index 0000000000..53b5cb2255
--- /dev/null
+++ b/mobile/apps/photos/test/rust/dedup_bridge_test.dart
@@ -0,0 +1,141 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:photos/src/rust/api/dedup_api.dart';
+import 'package:photos/src/rust/frb_generated.dart';
+import 'package:photos/src/rust/third_party/ente_frb_lib/session.dart';
+
+void main() {
+ final library = Platform.environment['ENTE_PHOTOS_FRB_LIBRARY'] ?? '';
+
+ test(
+ 'finds and removes duplicates through the generated Dart bindings',
+ () async {
+ await EntePhotosRust.init(externalLibrary: ExternalLibrary.open(library));
+ final requests = <(String, Map<String, dynamic>)>[];
+ final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
+ addTearDown(() => server.close(force: true));
+ server.listen((request) async {
+ final body = await utf8.decoder.bind(request).join();
+ requests.add((
+ request.uri.path,
+ jsonDecode(body) as Map<String, dynamic>,
+ ));
+ request.response
+ ..headers.contentType = ContentType.json
+ ..write('{}');
+ await request.response.close();
+ });
+
+ final files = [
+ for (final (id, album) in [(21, 11), (21, 12), (22, 12)])
+ DedupFile(
+ id: id,
+ collectionId: album,
+ ownerId: 1,
+ hash: 'test hash',
+ size: BigInt.from(100),
+ ),
+ ];
+ final groups = await findDuplicates(
+ userId: 1,
+ collections: [
+ for (final id in [11, 12])
+ DedupCollection(id: id, ownerId: 1, isHidden: false),
+ ],
+ files: files,
+ );
+ final group = groups.single;
+ expect(group.itemSize, BigInt.from(100));
+ expect(group.items.map((item) => item.fileIndex.toInt()), [0, 2]);
+ expect(group.items.first.collectionIds.map((id) => id.toInt()), [11, 12]);
+
+ final session = _openTestSession(server.port);
+ addTearDown(session.dispose);
+ final events = await executeDuplicateRemoval(
+ session: session,
+ input: ExecuteDuplicateRemovalInput(
+ groups: [
+ [
+ for (final item in group.items)
+ Duplicate(
+ fileId: files[item.fileIndex.toInt()].id,
+ collectionIds: item.collectionIds,
+ caption: item.fileIndex == BigInt.two
+ ? 'Keep this copy'
+ : null,
+ ),
+ ],
+ ],
+ collections: [RemovalCollection(id: 11, key: Uint8List(32))],
+ files: [
+ for (final index in [0, 2])
+ RemovalFile(
+ id: files[index].id,
+ collectionId: files[index].collectionId,
+ key: Uint8List(32),
+ ),
+ ],
+ ),
+ ).toList();
+
+ expect(events, [
+ DuplicateRemovalEvent.progress(
+ completed: BigInt.one,
+ total: BigInt.two,
+ ),
+ DuplicateRemovalEvent.progress(
+ completed: BigInt.two,
+ total: BigInt.two,
+ ),
+ isA<DuplicateRemovalEvent_Completed>().having(
+ (event) => event.trashIds.map((id) => id.toInt()),
+ 'trashed IDs',
+ [21],
+ ),
+ ]);
+ expect(requests.map((request) => request.$1), [
+ '/collections/add-files',
+ '/files/trash',
+ ]);
+ expect(requests.first.$2, {
+ 'collectionID': 11,
+ 'files': [
+ {
+ 'id': 22,
+ 'encryptedKey': isA<String>(),
+ 'keyDecryptionNonce': isA<String>(),
+ },
+ ],
+ });
+ expect(requests.last.$2, {
+ 'items': [
+ {'fileID': 21, 'collectionID': 11},
+ ],
+ });
+ },
+ skip: library.isEmpty
+ ? 'Set ENTE_PHOTOS_FRB_LIBRARY to run against Rust'
+ : false,
+ );
+}
+
+Session _openTestSession(int port) => openSession(
+ input: OpenSessionInput(
+ baseUrl: 'http://127.0.0.1:$port',
+ authToken: 'test-token',
+ userId: 1,
+ masterKey: Uint8List(32),
+ keyAttributes: const SessionKeyAttributes(
+ publicKey: 'pOCSkrZRwni5dyxWn1+puxPZBrRqtoyd+dwrRAn4ogk=',
+ encryptedSecretKey:
+ '+X2V/HMj8w3AXsDR/40cCYKhGG+xIjoYEY1fhrVki/8lsbbYnoppV9livaNsC7WX',
+ secretKeyDecryptionNonce: 'AgICAgICAgICAgICAgICAgICAgICAgIC',
+ recoveryKeyEncryptedWithMasterKey:
+ 'jok1oUB0jgsas9Ske6QuwF0e3qieBz+U7/ls4H+8IvoAiqgtr2AGgrqU4SSFwV4F',
+ recoveryKeyDecryptionNonce: 'BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE',
+ ),
+ ),
+);
diff --git a/rust/bindings/frb/photos/src/api/dedup_api.rs b/rust/bindings/frb/photos/src/api/dedup_api.rs
new file mode 100644
index 0000000000..e0655aefa9
--- /dev/null
+++ b/rust/bindings/frb/photos/src/api/dedup_api.rs
@@ -0,0 +1,213 @@
+#[cfg(any(feature = "flutter", frb_expand))]
+use ente_core::crypto::Key;
+#[cfg(any(feature = "flutter", frb_expand))]
+use ente_frb_lib::session::{Session, SessionError};
+use ente_photos::dedup;
+
+#[cfg(any(feature = "flutter", frb_expand))]
+use crate::frb_generated::StreamSink;
+
+pub struct DedupCollection {
+ pub id: i64,
+ pub owner_id: i64,
+ pub is_hidden: bool,
+}
+
+pub struct DedupFile {
+ pub id: i64,
+ pub collection_id: i64,
+ pub owner_id: i64,
+ pub hash: Option<String>,
+ pub size: Option<u64>,
+}
+
+pub struct DuplicateItem {
+ pub file_index: usize,
+ pub collection_ids: Vec<i64>,
+}
+
+pub struct DuplicateGroup {
+ pub items: Vec<DuplicateItem>,
+ pub item_size: u64,
+}
+
+pub fn find_duplicates(
+ user_id: i64,
+ collections: Vec<DedupCollection>,
+ files: Vec<DedupFile>,
+) -> Vec<DuplicateGroup> {
+ let collections: Vec<_> = collections
+ .into_iter()
+ .map(|collection| dedup::Collection {
+ id: collection.id,
+ owner_id: collection.owner_id,
+ is_hidden: collection.is_hidden,
+ })
+ .collect();
+ let files: Vec<_> = files
+ .into_iter()
+ .map(|file| dedup::File {
+ id: file.id,
+ collection_id: file.collection_id,
+ owner_id: file.owner_id,
+ hash: file.hash,
+ size: file.size,
+ })
+ .collect();
+ dedup::find_duplicates(user_id, &collections, &files)
+ .into_iter()
+ .map(|group| DuplicateGroup {
+ items: group
+ .items
+ .into_iter()
+ .map(|item| DuplicateItem {
+ file_index: item.file_index,
+ collection_ids: item.collection_ids,
+ })
+ .collect(),
+ item_size: group.item_size,
+ })
+ .collect()
+}
+
+pub struct Duplicate {
+ pub file_id: i64,
+ pub collection_ids: Vec<i64>,
+ pub caption: Option<String>,
+ pub edited_name: Option<String>,
+ pub edited_time: Option<i64>,
+}
+
+pub struct RemovalCollection {
+ pub id: i64,
+ pub key: Vec<u8>,
+}
+
+pub struct RemovalFile {
+ pub id: i64,
+ pub collection_id: i64,
+ pub key: Vec<u8>,
+}
+
+pub struct ExecuteDuplicateRemovalInput {
+ pub groups: Vec<Vec<Duplicate>>,
+ pub collections: Vec<RemovalCollection>,
+ pub files: Vec<RemovalFile>,
+}
+
+#[cfg(any(feature = "flutter", frb_expand))]
+pub enum DuplicateRemovalEvent {
+ Progress { completed: usize, total: usize },
+ Completed { trash_ids: Vec<i64> },
+ Failed { error: SessionError },
+}
+
+#[cfg(any(feature = "flutter", frb_expand))]
+pub async fn execute_duplicate_removal(
+ session: &Session,
+ input: ExecuteDuplicateRemovalInput,
+ events: StreamSink<DuplicateRemovalEvent>,
+) {
+ let result = remove_duplicates(session, input, &events).await;
+ let _ = events.add(match result {
+ Ok(trash_ids) => DuplicateRemovalEvent::Completed { trash_ids },
+ Err(error) => DuplicateRemovalEvent::Failed { error },
+ });
+}
+
+#[cfg(any(feature = "flutter", frb_expand))]
+async fn remove_duplicates(
+ session: &Session,
+ input: ExecuteDuplicateRemovalInput,
+ events: &StreamSink<DuplicateRemovalEvent>,
+) -> Result<Vec<i64>, SessionError> {
+ let groups: Vec<_> = input
+ .groups
+ .into_iter()
+ .map(|group| {
+ group
+ .into_iter()
+ .map(|item| dedup::Duplicate {
+ file_id: item.file_id,
+ collection_ids: item.collection_ids,
+ caption: item.caption,
+ edited_name: item.edited_name,
+ edited_time: item.edited_time,
+ })
+ .collect()
+ })
+ .collect();
+ let collections = input
+ .collections
+ .into_iter()
+ .map(|collection| {
+ Ok(dedup::RemovalCollection {
+ id: collection.id,
+ key: Key::try_from_slice(&collection.key)?,
+ })
+ })
+ .collect::<Result<Vec<_>, SessionError>>()?;
+ let files = input
+ .files
+ .into_iter()
+ .map(|file| {
+ Ok(dedup::RemovalFile {
+ id: file.id,
+ collection_id: file.collection_id,
+ key: Key::try_from_slice(&file.key)?,
+ })
+ })
+ .collect::<Result<Vec<_>, SessionError>>()?;
+ dedup::remove_duplicates(
+ session.as_ref(),
+ &groups,
+ &collections,
+ &files,
+ |progress| {
+ let _ = events.add(DuplicateRemovalEvent::Progress {
+ completed: progress.completed,
+ total: progress.total,
+ });
+ },
+ )
+ .await
+ .map_err(SessionError::from)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn detection_preserves_memberships() {
+ let collections = vec![
+ DedupCollection {
+ id: 11,
+ owner_id: 1,
+ is_hidden: false,
+ },
+ DedupCollection {
+ id: 12,
+ owner_id: 1,
+ is_hidden: false,
+ },
+ ];
+ let files: Vec<_> = [(1, 11), (1, 12), (2, 12)]
+ .into_iter()
+ .map(|(id, collection_id)| DedupFile {
+ id,
+ collection_id,
+ owner_id: 1,
+ hash: Some("hash".into()),
+ size: Some(100),
+ })
+ .collect();
+ let groups = find_duplicates(1, collections, files);
+ assert_eq!(groups.len(), 1);
+ assert_eq!(groups[0].item_size, 100);
+ assert_eq!(groups[0].items.len(), 2);
+ assert_eq!(groups[0].items[0].file_index, 0);
+ assert_eq!(groups[0].items[0].collection_ids, vec![11, 12]);
+ assert_eq!(groups[0].items[1].file_index, 2);
+ }
+}
diff --git a/rust/bindings/frb/photos/src/api/mod.rs b/rust/bindings/frb/photos/src/api/mod.rs
index 6f650a56ec..406438cc28 100644
--- a/rust/bindings/frb/photos/src/api/mod.rs
+++ b/rust/bindings/frb/photos/src/api/mod.rs
@@ -1,4 +1,5 @@
pub mod cast_api;
+pub mod dedup_api;
pub mod image_processing_api;
pub mod init;
pub mod location_api;