Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mobile][photos] Reduce CLIP memory selection overhead

laurenspriem35972b7d7def
diff --git a/mobile/apps/photos/lib/services/smart_memories_clip_calculator.dart b/mobile/apps/photos/lib/services/smart_memories_clip_calculator.dart
index d1dacca491..94d691d4c4 100644
--- a/mobile/apps/photos/lib/services/smart_memories_clip_calculator.dart
+++ b/mobile/apps/photos/lib/services/smart_memories_clip_calculator.dart
@@ -26,51 +26,33 @@ class ClipMemoriesCalculator {
dev.log("No vector for clipMemoryType $clipMemoryType");
return null;
}
- final Map<int, double> similarities = {};
- for (final entry in fileIDToImageEmbedding.entries) {
- similarities[entry.key] = entry.value.vector.dot(activityVector);
- }
- w?.log('comparing embeddings for clipMemoryType $clipMemoryType');
- final List<EnteFile> clipFiles = [];
+ final scoredFiles = <({EnteFile file, int fileID, double similarity})>[];
for (final file in allFiles) {
- final memoryFileID = SmartMemoriesService._memoryFileId(
+ final fileID = SmartMemoriesService._memoryFileId(
file,
isLocalGalleryMode: isLocalGalleryMode,
);
- if (memoryFileID == null) continue;
- final similarity = similarities[memoryFileID];
- if (similarity == null) continue;
+ if (fileID == null) continue;
+ final embedding = fileIDToImageEmbedding[fileID];
+ if (embedding == null) continue;
+ final similarity = embedding.vector.dot(activityVector);
if (similarity > SmartMemoriesService._clipMemoryTypeQueryThreshold) {
- clipFiles.add(file);
+ scoredFiles.add((file: file, fileID: fileID, similarity: similarity));
}
}
- if (clipFiles.length < 10) return null;
- clipFiles.sort((a, b) {
- final int bFileID = SmartMemoriesService._memoryFileId(
- b,
- isLocalGalleryMode: isLocalGalleryMode,
- )!;
- final int aFileID = SmartMemoriesService._memoryFileId(
- a,
- isLocalGalleryMode: isLocalGalleryMode,
- )!;
- return similarities[bFileID]!.compareTo(similarities[aFileID]!);
- });
- final int limit = min(clipFiles.length, 50);
- final List<EnteFile> topCandidates = clipFiles.take(limit).toList();
+ w?.log('comparing embeddings for clipMemoryType $clipMemoryType');
+ if (scoredFiles.length < 10) return null;
+ scoredFiles.sort((a, b) => b.similarity.compareTo(a.similarity));
+ final int limit = min(scoredFiles.length, 50);
+ final topCandidates = scoredFiles.take(limit).toList();
topCandidates.shuffle(Random());
- final List<EnteFile> selected = [];
+ final selected = <({EnteFile file, int fileID, double similarity})>[];
final selectedFileIDs = <int>[];
final selectedCreationTimes = <int>[];
int skipped = 0;
- for (final file in topCandidates) {
+ for (final candidate in topCandidates) {
if (selected.length >= 10) break;
- final fileID = SmartMemoriesService._memoryFileId(
- file,
- isLocalGalleryMode: isLocalGalleryMode,
- );
- if (fileID == null) continue;
- final creationTime = file.creationTime;
+ final creationTime = candidate.file.creationTime;
if (SmartMemoriesService._isTooCloseInTime(
creationTime,
selectedCreationTimes,
@@ -79,7 +61,7 @@ class ClipMemoriesCalculator {
continue;
}
if (SmartMemoriesService._isNearDuplicate(
- fileID,
+ candidate.fileID,
selectedFileIDs,
fileIDToImageEmbedding,
) &&
@@ -87,25 +69,17 @@ class ClipMemoriesCalculator {
skipped++;
continue;
}
- selected.add(file);
- selectedFileIDs.add(fileID);
+ selected.add(candidate);
+ selectedFileIDs.add(candidate.fileID);
if (creationTime != null) {
selectedCreationTimes.add(creationTime);
}
}
- selected.sort((a, b) {
- final int bFileID = SmartMemoriesService._memoryFileId(
- b,
- isLocalGalleryMode: isLocalGalleryMode,
- )!;
- final int aFileID = SmartMemoriesService._memoryFileId(
- a,
- isLocalGalleryMode: isLocalGalleryMode,
- )!;
- return similarities[bFileID]!.compareTo(similarities[aFileID]!);
- });
+ selected.sort((a, b) => b.similarity.compareTo(a.similarity));
return ClipMemory(
- selected.map((f) => Memory.fromFile(f, seenTimes)).toList(),
+ selected
+ .map((candidate) => Memory.fromFile(candidate.file, seenTimes))
+ .toList(),
nowInMicroseconds,
windowEnd,
clipMemoryType,
diff --git a/mobile/apps/photos/test/services/memories/smart_memories_source_selection_test.dart b/mobile/apps/photos/test/services/memories/smart_memories_source_selection_test.dart
index 26f55a45bf..0a853baacc 100644
--- a/mobile/apps/photos/test/services/memories/smart_memories_source_selection_test.dart
+++ b/mobile/apps/photos/test/services/memories/smart_memories_source_selection_test.dart
@@ -196,5 +196,49 @@ void main() {
expect(depletedClipMemories, isEmpty);
},
);
+
+ test(
+ "ClipMemoriesCalculator selects only from the top 50 matches",
+ () async {
+ final currentTime = DateTime.utc(2026, 4, 10);
+ final files = <EnteFile>[
+ for (int i = 0; i < 60; i++)
+ _file(
+ id: i + 1,
+ createdAt: DateTime.utc(2024, 1, 1).add(Duration(hours: i)),
+ ),
+ ];
+ final selectedClipType = ClipMemoryType.values.first;
+ final embeddings = <int, EmbeddingVector>{
+ for (final file in files)
+ file.uploadedFileID!: EmbeddingVector(
+ fileID: file.uploadedFileID!,
+ embedding: <double>[file.uploadedFileID! / 60],
+ ),
+ for (int id = 1000; id < 1100; id++)
+ id: EmbeddingVector(fileID: id, embedding: const <double>[1]),
+ };
+
+ final memories = await ClipMemoriesCalculator.compute(
+ files,
+ currentTime,
+ <ClipShownLog>[],
+ surfaceAll: true,
+ isLocalGalleryMode: false,
+ seenTimes: const <int, int>{},
+ fileIDToImageEmbedding: embeddings,
+ clipMemoryTypeVectors: <ClipMemoryType, Vector>{
+ selectedClipType: Vector.fromList(const <double>[1]),
+ },
+ );
+
+ expect(memories, hasLength(1));
+ expect(memories.first.memories, hasLength(10));
+ expect(
+ memories.first.memories.map((memory) => memory.file.uploadedFileID),
+ everyElement(greaterThan(10)),
+ );
+ },
+ );
});
}