π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mobile][photos] Reduce CLIP memory selection overhead
laurenspriem35972b7d7def
diff --git a/mobile/apps/photos/lib/services/smart_memories_clip_calculator.dart b/mobile/apps/photos/lib/services/smart_memories_clip_calculator.dartindex d1dacca491..94d691d4c4 100644--- a/mobile/apps/photos/lib/services/smart_memories_clip_calculator.dart+++ b/mobile/apps/photos/lib/services/smart_memories_clip_calculator.dart@@ -26,51 +26,33 @@ class ClipMemoriesCalculator {dev.log("No vector for clipMemoryType $clipMemoryType");return null;}- final Map<int, double> similarities = {};- for (final entry in fileIDToImageEmbedding.entries) {- similarities[entry.key] = entry.value.vector.dot(activityVector);- }- w?.log('comparing embeddings for clipMemoryType $clipMemoryType');- final List<EnteFile> clipFiles = [];+ final scoredFiles = <({EnteFile file, int fileID, double similarity})>[];for (final file in allFiles) {- final memoryFileID = SmartMemoriesService._memoryFileId(+ final fileID = SmartMemoriesService._memoryFileId(file,isLocalGalleryMode: isLocalGalleryMode,);- if (memoryFileID == null) continue;- final similarity = similarities[memoryFileID];- if (similarity == null) continue;+ if (fileID == null) continue;+ final embedding = fileIDToImageEmbedding[fileID];+ if (embedding == null) continue;+ final similarity = embedding.vector.dot(activityVector);if (similarity > SmartMemoriesService._clipMemoryTypeQueryThreshold) {- clipFiles.add(file);+ scoredFiles.add((file: file, fileID: fileID, similarity: similarity));}}- if (clipFiles.length < 10) return null;- clipFiles.sort((a, b) {- final int bFileID = SmartMemoriesService._memoryFileId(- b,- isLocalGalleryMode: isLocalGalleryMode,- )!;- final int aFileID = SmartMemoriesService._memoryFileId(- a,- isLocalGalleryMode: isLocalGalleryMode,- )!;- return similarities[bFileID]!.compareTo(similarities[aFileID]!);- });- final int limit = min(clipFiles.length, 50);- final List<EnteFile> topCandidates = clipFiles.take(limit).toList();+ w?.log('comparing embeddings for clipMemoryType $clipMemoryType');+ if (scoredFiles.length < 10) return null;+ scoredFiles.sort((a, b) => b.similarity.compareTo(a.similarity));+ final int limit = min(scoredFiles.length, 50);+ final topCandidates = scoredFiles.take(limit).toList();topCandidates.shuffle(Random());- final List<EnteFile> selected = [];+ final selected = <({EnteFile file, int fileID, double similarity})>[];final selectedFileIDs = <int>[];final selectedCreationTimes = <int>[];int skipped = 0;- for (final file in topCandidates) {+ for (final candidate in topCandidates) {if (selected.length >= 10) break;- final fileID = SmartMemoriesService._memoryFileId(- file,- isLocalGalleryMode: isLocalGalleryMode,- );- if (fileID == null) continue;- final creationTime = file.creationTime;+ final creationTime = candidate.file.creationTime;if (SmartMemoriesService._isTooCloseInTime(creationTime,selectedCreationTimes,@@ -79,7 +61,7 @@ class ClipMemoriesCalculator {continue;}if (SmartMemoriesService._isNearDuplicate(- fileID,+ candidate.fileID,selectedFileIDs,fileIDToImageEmbedding,) &&@@ -87,25 +69,17 @@ class ClipMemoriesCalculator {skipped++;continue;}- selected.add(file);- selectedFileIDs.add(fileID);+ selected.add(candidate);+ selectedFileIDs.add(candidate.fileID);if (creationTime != null) {selectedCreationTimes.add(creationTime);}}- selected.sort((a, b) {- final int bFileID = SmartMemoriesService._memoryFileId(- b,- isLocalGalleryMode: isLocalGalleryMode,- )!;- final int aFileID = SmartMemoriesService._memoryFileId(- a,- isLocalGalleryMode: isLocalGalleryMode,- )!;- return similarities[bFileID]!.compareTo(similarities[aFileID]!);- });+ selected.sort((a, b) => b.similarity.compareTo(a.similarity));return ClipMemory(- selected.map((f) => Memory.fromFile(f, seenTimes)).toList(),+ selected+ .map((candidate) => Memory.fromFile(candidate.file, seenTimes))+ .toList(),nowInMicroseconds,windowEnd,clipMemoryType,diff --git a/mobile/apps/photos/test/services/memories/smart_memories_source_selection_test.dart b/mobile/apps/photos/test/services/memories/smart_memories_source_selection_test.dartindex 26f55a45bf..0a853baacc 100644--- a/mobile/apps/photos/test/services/memories/smart_memories_source_selection_test.dart+++ b/mobile/apps/photos/test/services/memories/smart_memories_source_selection_test.dart@@ -196,5 +196,49 @@ void main() {expect(depletedClipMemories, isEmpty);},);++ test(+ "ClipMemoriesCalculator selects only from the top 50 matches",+ () async {+ final currentTime = DateTime.utc(2026, 4, 10);+ final files = <EnteFile>[+ for (int i = 0; i < 60; i++)+ _file(+ id: i + 1,+ createdAt: DateTime.utc(2024, 1, 1).add(Duration(hours: i)),+ ),+ ];+ final selectedClipType = ClipMemoryType.values.first;+ final embeddings = <int, EmbeddingVector>{+ for (final file in files)+ file.uploadedFileID!: EmbeddingVector(+ fileID: file.uploadedFileID!,+ embedding: <double>[file.uploadedFileID! / 60],+ ),+ for (int id = 1000; id < 1100; id++)+ id: EmbeddingVector(fileID: id, embedding: const <double>[1]),+ };++ final memories = await ClipMemoriesCalculator.compute(+ files,+ currentTime,+ <ClipShownLog>[],+ surfaceAll: true,+ isLocalGalleryMode: false,+ seenTimes: const <int, int>{},+ fileIDToImageEmbedding: embeddings,+ clipMemoryTypeVectors: <ClipMemoryType, Vector>{+ selectedClipType: Vector.fromList(const <double>[1]),+ },+ );++ expect(memories, hasLength(1));+ expect(memories.first.memories, hasLength(10));+ expect(+ memories.first.memories.map((memory) => memory.file.uploadedFileID),+ everyElement(greaterThan(10)),+ );+ },+ );});}