Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mob][photos][rust] Compute memories in Rust behind a flag (#14339)

laurens-pilotfb9d53d82ae1
diff --git a/mobile/apps/photos/lib/service_locator.dart b/mobile/apps/photos/lib/service_locator.dart
index f8146f6d9d..e147e9b0c8 100644
--- a/mobile/apps/photos/lib/service_locator.dart
+++ b/mobile/apps/photos/lib/service_locator.dart
@@ -6,6 +6,7 @@ import "package:package_info_plus/package_info_plus.dart";
import "package:photos/core/configuration.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/core/network/endpoint_config.dart";
+import "package:photos/db/ml/db.dart";
import "package:photos/events/ml_consent_changed_event.dart";
import "package:photos/gateways/billing/billing_gateway.dart";
import "package:photos/gateways/cast/cast_gateway.dart";
@@ -37,6 +38,7 @@ import "package:photos/services/machine_learning/ml_decryption_record_store.dart
import "package:photos/services/magic_cache_service.dart";
import "package:photos/services/memories/dart_memories_computer.dart";
import "package:photos/services/memories/memories_computer.dart";
+import "package:photos/services/memories/rust_memories_computer.dart";
import "package:photos/services/memories/smart_memories_service.dart";
import "package:photos/services/memories_cache_service.dart";
import "package:photos/services/permission/service.dart";
@@ -235,10 +237,13 @@ SmartMemoriesService get smartMemoriesService {
return _smartMemoriesService!;
}
-MemoriesComputer? _memoriesComputer;
+DartMemoriesComputer? _dartMemoriesComputer;
+RustMemoriesComputer? _rustMemoriesComputer;
MemoriesComputer get memoriesComputer {
- _memoriesComputer ??= DartMemoriesComputer();
- return _memoriesComputer!;
+ if (flagService.rustMemories && MLDataDB.isRustBackend) {
+ return _rustMemoriesComputer ??= RustMemoriesComputer();
+ }
+ return _dartMemoriesComputer ??= DartMemoriesComputer();
}
TextEmbeddingsCacheService? _textEmbeddingsCacheService;
diff --git a/mobile/apps/photos/lib/services/location_service.dart b/mobile/apps/photos/lib/services/location_service.dart
index 007ac6c537..79167746fe 100644
--- a/mobile/apps/photos/lib/services/location_service.dart
+++ b/mobile/apps/photos/lib/services/location_service.dart
@@ -141,6 +141,8 @@ class LocationService {
};
}
+ Future<rust.LocationIndex?> getLocationIndex() => _loadIndex();
+
Future<List<City>> getCities() async {
final index = await _loadIndex();
if (index == null) return [];
diff --git a/mobile/apps/photos/lib/services/memories/dart_memories_computer.dart b/mobile/apps/photos/lib/services/memories/dart_memories_computer.dart
index 64b7a7a2a8..751e5f5114 100644
--- a/mobile/apps/photos/lib/services/memories/dart_memories_computer.dart
+++ b/mobile/apps/photos/lib/services/memories/dart_memories_computer.dart
@@ -9,8 +9,6 @@ import "package:logging/logging.dart";
import "package:ml_linalg/vector.dart";
import "package:photos/core/configuration.dart";
import "package:photos/core/constants.dart";
-import "package:photos/db/files_db.dart";
-import "package:photos/db/memories_db.dart";
import "package:photos/db/ml/base.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/db/offline_files_db.dart";
@@ -27,20 +25,18 @@ import "package:photos/models/memories/smart_memory.dart";
import "package:photos/models/memories/smart_memory_constants.dart";
import "package:photos/models/memories/time_memory.dart";
import "package:photos/models/memories/trip_memory.dart";
-import "package:photos/models/metadata/common_keys.dart";
import "package:photos/models/ml/face/face_with_embedding.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/models/ml/vector.dart";
import "package:photos/service_locator.dart";
-import "package:photos/services/collections_service.dart";
import "package:photos/services/location_service.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import "package:photos/services/machine_learning/ml_model_download_service.dart";
import "package:photos/services/machine_learning/ml_result.dart";
import "package:photos/services/memories/memories_computation_context.dart";
import "package:photos/services/memories/memories_computer.dart";
+import "package:photos/services/memories/memory_sources.dart";
import "package:photos/services/memories/photo_selector.dart";
-import "package:photos/services/search_service.dart";
part "../smart_memories_clip_calculator.dart";
part "memory_file_index.dart";
@@ -58,9 +54,6 @@ typedef _PreparedMemoriesData = ({
class DartMemoriesComputer implements MemoriesComputer {
final _logger = Logger("DartMemoriesComputer");
- MemoriesDB get _memoriesDB => isLocalGalleryMode
- ? MemoriesDB.localGalleryInstance
- : MemoriesDB.instance;
static const _clipSimilarImageThreshold =
PhotoSelector.clipSimilarImageThreshold;
@@ -223,7 +216,7 @@ class DartMemoriesComputer implements MemoriesComputer {
required bool mlEnabled,
required TimeLogger timeLogger,
}) async {
- final allFileIdsToFile = await _getFilesAndMapForMemories(
+ final allFileIdsToFile = await getFilesAndMapForMemories(
useLocalIntIds: isLocalGalleryMode,
requireLocalId: isLocalGalleryMode,
);
@@ -235,7 +228,7 @@ class DartMemoriesComputer implements MemoriesComputer {
'$timeLogger',
);
- final seenTimes = await _memoriesDB.getSeenTimes();
+ final seenTimes = await memoriesDB.getSeenTimes();
_logger.info('seenTimes has ${seenTimes.length} entries $timeLogger');
final mlDataDB = isLocalGalleryMode
@@ -637,81 +630,6 @@ class DartMemoriesComputer implements MemoriesComputer {
return false;
}
- Future<Map<int, EnteFile>> _getFilesAndMapForMemories({
- bool useGeneratedIds = false,
- bool requireLocalId = false,
- bool useLocalIntIds = false,
- }) async {
- final allFilesFromSearchService = await SearchService.instance
- .getAllFilesForSearch();
- final archivedOrHiddenCollectionIDs = CollectionsService.instance
- .archivedOrHiddenCollectionIds();
- final excludedUploadFileIDs = <int>{};
- if (archivedOrHiddenCollectionIDs.isNotEmpty) {
- final filesInArchivedCollections = await FilesDB.instance
- .getAllFilesFromCollections(archivedOrHiddenCollectionIDs);
- for (final archivedFile in filesInArchivedCollections) {
- final archivedUploadID = archivedFile.uploadedFileID;
- if (archivedUploadID != null && archivedUploadID != -1) {
- excludedUploadFileIDs.add(archivedUploadID);
- }
- }
- }
- final candidateFiles = <EnteFile>[];
- for (final file in allFilesFromSearchService) {
- final localId = file.localID;
- final hasLocalId = localId != null && localId.isNotEmpty;
- if (requireLocalId && !hasLocalId) {
- continue;
- }
- final hasId = useLocalIntIds
- ? hasLocalId
- : useGeneratedIds
- ? file.generatedID != null &&
- (file.uploadedFileID != null || hasLocalId)
- : file.uploadedFileID != null;
- if (hasId && file.creationTime != null) {
- if (excludedUploadFileIDs.contains(file.uploadedFileID)) {
- continue;
- }
- if (file.magicMetadata.visibility == archiveVisibility ||
- file.magicMetadata.visibility == hiddenVisibility) {
- continue;
- }
- final collectionID = file.collectionID;
- if (collectionID != null &&
- archivedOrHiddenCollectionIDs.contains(collectionID)) {
- continue;
- }
- candidateFiles.add(file);
- }
- }
- final Map<String, int> localIdToIntId = useLocalIntIds
- ? await OfflineFilesDB.instance.ensureLocalIntIds(
- candidateFiles
- .map((file) => file.localID)
- .whereType<String>()
- .where((id) => id.isNotEmpty),
- )
- : <String, int>{};
- final allFileIdsToFile = <int, EnteFile>{};
- for (final file in candidateFiles) {
- final localIntId = useLocalIntIds ? localIdToIntId[file.localID] : null;
- final mappedFile = localIntId != null
- ? file.copyWith(generatedID: localIntId)
- : file;
- final key = useLocalIntIds
- ? localIntId
- : useGeneratedIds
- ? mappedFile.generatedID
- : mappedFile.uploadedFileID;
- if (key != null) {
- allFileIdsToFile[key] = mappedFile;
- }
- }
- return allFileIdsToFile;
- }
-
static Future<MemoriesResult> _allMemoriesCalculations(
Map<String, dynamic> args,
) async {
@@ -1042,13 +960,13 @@ class DartMemoriesComputer implements MemoriesComputer {
@override
Future<List<SmartMemory>> computeSimple() async {
final now = DateTime.now();
- final allFileIdsToFile = await _getFilesAndMapForMemories(
+ final allFileIdsToFile = await getFilesAndMapForMemories(
useLocalIntIds: isLocalGalleryMode,
requireLocalId: isLocalGalleryMode,
);
final fileIndex = MemoryFileIndex(allFileIdsToFile.values);
final usedMemoryFileIds = <int>{};
- final seenTimes = await _memoriesDB.getSeenTimes();
+ final seenTimes = await memoriesDB.getSeenTimes();
final collectionIDsToExclude = await getCollectionIDsToExclude();
final localIdToIntId = isLocalGalleryMode
? await OfflineFilesDB.instance.ensureLocalIntIds(
@@ -1271,50 +1189,6 @@ class DartMemoriesComputer implements MemoriesComputer {
);
}
- Future<Set<int>> getCollectionIDsToExclude() async {
- if (isLocalGalleryMode) {
- return <int>{};
- }
- final collections = CollectionsService.instance.getCollectionsForUI();
-
- const excludedNames = {
- 'screenshot',
- 'whatsapp',
- 'telegram',
- 'download',
- 'facebook',
- 'instagram',
- 'messenger',
- 'twitter',
- 'reddit',
- 'discord',
- 'signal',
- 'viber',
- 'wechat',
- 'line',
- 'meme',
- 'internet',
- 'saved images',
- 'document',
- };
-
- final excludedCollectionIDs = Set<int>.from(
- CollectionsService.instance.archivedOrHiddenCollectionIds(),
- );
- collectionLoop:
- for (final collection in collections) {
- final collectionName = collection.displayName.toLowerCase();
- for (final excludedName in excludedNames) {
- if (collectionName.contains(excludedName)) {
- excludedCollectionIDs.add(collection.id);
- continue collectionLoop;
- }
- }
- }
-
- return excludedCollectionIDs;
- }
-
static Future<List<OnThisDayMemory>> _getOnThisDayResults(
Iterable<EnteFile> allFiles,
DateTime currentTime, {
diff --git a/mobile/apps/photos/lib/services/memories/memory_sources.dart b/mobile/apps/photos/lib/services/memories/memory_sources.dart
new file mode 100644
index 0000000000..362427e68a
--- /dev/null
+++ b/mobile/apps/photos/lib/services/memories/memory_sources.dart
@@ -0,0 +1,130 @@
+import "package:photos/db/files_db.dart";
+import "package:photos/db/memories_db.dart";
+import "package:photos/db/offline_files_db.dart";
+import "package:photos/models/file/file.dart";
+import "package:photos/models/metadata/common_keys.dart";
+import "package:photos/service_locator.dart";
+import "package:photos/services/collections_service.dart";
+import "package:photos/services/search_service.dart";
+
+MemoriesDB get memoriesDB =>
+ isLocalGalleryMode ? MemoriesDB.localGalleryInstance : MemoriesDB.instance;
+
+Future<Map<int, EnteFile>> getFilesAndMapForMemories({
+ bool useGeneratedIds = false,
+ bool requireLocalId = false,
+ bool useLocalIntIds = false,
+}) async {
+ final allFilesFromSearchService = await SearchService.instance
+ .getAllFilesForSearch();
+ final archivedOrHiddenCollectionIDs = CollectionsService.instance
+ .archivedOrHiddenCollectionIds();
+ final excludedUploadFileIDs = <int>{};
+ if (archivedOrHiddenCollectionIDs.isNotEmpty) {
+ final filesInArchivedCollections = await FilesDB.instance
+ .getAllFilesFromCollections(archivedOrHiddenCollectionIDs);
+ for (final archivedFile in filesInArchivedCollections) {
+ final archivedUploadID = archivedFile.uploadedFileID;
+ if (archivedUploadID != null && archivedUploadID != -1) {
+ excludedUploadFileIDs.add(archivedUploadID);
+ }
+ }
+ }
+ final candidateFiles = <EnteFile>[];
+ for (final file in allFilesFromSearchService) {
+ final localId = file.localID;
+ final hasLocalId = localId != null && localId.isNotEmpty;
+ if (requireLocalId && !hasLocalId) {
+ continue;
+ }
+ final hasId = useLocalIntIds
+ ? hasLocalId
+ : useGeneratedIds
+ ? file.generatedID != null &&
+ (file.uploadedFileID != null || hasLocalId)
+ : file.uploadedFileID != null;
+ if (hasId && file.creationTime != null) {
+ if (excludedUploadFileIDs.contains(file.uploadedFileID)) {
+ continue;
+ }
+ if (file.magicMetadata.visibility == archiveVisibility ||
+ file.magicMetadata.visibility == hiddenVisibility) {
+ continue;
+ }
+ final collectionID = file.collectionID;
+ if (collectionID != null &&
+ archivedOrHiddenCollectionIDs.contains(collectionID)) {
+ continue;
+ }
+ candidateFiles.add(file);
+ }
+ }
+ final Map<String, int> localIdToIntId = useLocalIntIds
+ ? await OfflineFilesDB.instance.ensureLocalIntIds(
+ candidateFiles
+ .map((file) => file.localID)
+ .whereType<String>()
+ .where((id) => id.isNotEmpty),
+ )
+ : <String, int>{};
+ final allFileIdsToFile = <int, EnteFile>{};
+ for (final file in candidateFiles) {
+ final localIntId = useLocalIntIds ? localIdToIntId[file.localID] : null;
+ final mappedFile = localIntId != null
+ ? file.copyWith(generatedID: localIntId)
+ : file;
+ final key = useLocalIntIds
+ ? localIntId
+ : useGeneratedIds
+ ? mappedFile.generatedID
+ : mappedFile.uploadedFileID;
+ if (key != null) {
+ allFileIdsToFile[key] = mappedFile;
+ }
+ }
+ return allFileIdsToFile;
+}
+
+Future<Set<int>> getCollectionIDsToExclude() async {
+ if (isLocalGalleryMode) {
+ return <int>{};
+ }
+ final collections = CollectionsService.instance.getCollectionsForUI();
+
+ const excludedNames = {
+ 'screenshot',
+ 'whatsapp',
+ 'telegram',
+ 'download',
+ 'facebook',
+ 'instagram',
+ 'messenger',
+ 'twitter',
+ 'reddit',
+ 'discord',
+ 'signal',
+ 'viber',
+ 'wechat',
+ 'line',
+ 'meme',
+ 'internet',
+ 'saved images',
+ 'document',
+ };
+
+ final excludedCollectionIDs = Set<int>.from(
+ CollectionsService.instance.archivedOrHiddenCollectionIds(),
+ );
+ collectionLoop:
+ for (final collection in collections) {
+ final collectionName = collection.displayName.toLowerCase();
+ for (final excludedName in excludedNames) {
+ if (collectionName.contains(excludedName)) {
+ excludedCollectionIDs.add(collection.id);
+ continue collectionLoop;
+ }
+ }
+ }
+
+ return excludedCollectionIDs;
+}
diff --git a/mobile/apps/photos/lib/services/memories/rust_memories_computer.dart b/mobile/apps/photos/lib/services/memories/rust_memories_computer.dart
new file mode 100644
index 0000000000..de21382b38
--- /dev/null
+++ b/mobile/apps/photos/lib/services/memories/rust_memories_computer.dart
@@ -0,0 +1,419 @@
+import "dart:math" show Random;
+
+import "package:collection/collection.dart";
+import "package:flutter_rust_bridge/flutter_rust_bridge.dart" show Int64List;
+import "package:logging/logging.dart";
+import "package:photos/core/configuration.dart";
+import "package:photos/db/ml/rust_db.dart";
+import "package:photos/models/base_location.dart";
+import "package:photos/models/file/file.dart";
+import "package:photos/models/location/location.dart";
+import "package:photos/models/memories/clip_memory.dart";
+import "package:photos/models/memories/filler_memory.dart";
+import "package:photos/models/memories/memories_cache.dart";
+import "package:photos/models/memories/memory.dart";
+import "package:photos/models/memories/on_this_day_memory.dart";
+import "package:photos/models/memories/people_memory.dart";
+import "package:photos/models/memories/smart_memory.dart";
+import "package:photos/models/memories/time_memory.dart";
+import "package:photos/models/memories/trip_memory.dart";
+import "package:photos/models/ml/face/person.dart";
+import "package:photos/service_locator.dart";
+import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
+import "package:photos/services/memories/memories_computer.dart";
+import "package:photos/services/memories/memory_sources.dart";
+import "package:photos/src/rust/api/memories_api.dart" as rust;
+
+class RustMemoriesComputer implements MemoriesComputer {
+ final _logger = Logger("RustMemoriesComputer");
+
+ static const _minimumClipIndexedFiles = 5;
+
+ RustMLDataDB get _mlDataDB => isLocalGalleryMode
+ ? RustMLDataDB.localGalleryInstance
+ : RustMLDataDB.instance;
+
+ @override
+ Future<bool> isMlReady() async {
+ if (!hasGrantedMLConsent || !localSettings.isMLLocalIndexingEnabled) {
+ return false;
+ }
+ try {
+ return await _mlDataDB.getClipIndexedFileCount() >=
+ _minimumClipIndexedFiles;
+ } catch (e, s) {
+ _logger.warning("Failed to determine ML readiness", e, s);
+ return false;
+ }
+ }
+
+ @override
+ Future<MemoriesResult> computeAll(
+ DateTime now,
+ MemoriesCache oldCache, {
+ required bool mlEnabled,
+ }) async {
+ final results = await _compute(
+ [now],
+ oldCache,
+ mlEnabled: mlEnabled,
+ surfaceAll: true,
+ );
+ return results?.single ?? MemoriesResult.failed();
+ }
+
+ @override
+ Future<({MemoriesResult current, MemoriesResult next})> computeCurrentAndNext(
+ DateTime current,
+ DateTime next,
+ MemoriesCache oldCache, {
+ required bool mlEnabled,
+ }) async {
+ final results = await _compute(
+ [current, next],
+ oldCache,
+ mlEnabled: mlEnabled,
+ surfaceAll: false,
+ );
+ if (results == null) {
+ return (current: MemoriesResult.failed(), next: MemoriesResult.failed());
+ }
+ return (current: results[0], next: results[1]);
+ }
+
+ @override
+ Future<List<SmartMemory>> computeSimple() async {
+ final inputs = await _loadInputs(mlEnabled: false, surfaceAll: false);
+ _logger.info(
+ "Computing simple memories over ${inputs.files.length} files (seed: "
+ "${inputs.request.seed})",
+ );
+ final memories = await rust.computeSimpleMemories(
+ request: inputs.request,
+ period: _period(DateTime.now()),
+ );
+ return memories.map(inputs.smartMemory).toList();
+ }
+
+ Future<List<MemoriesResult>?> _compute(
+ List<DateTime> periods,
+ MemoriesCache oldCache, {
+ required bool mlEnabled,
+ required bool surfaceAll,
+ }) async {
+ final inputs = await _loadInputs(
+ oldCache: oldCache,
+ mlEnabled: mlEnabled,
+ surfaceAll: surfaceAll,
+ );
+ _logger.info(
+ "Computing memories for ${periods.join(", ")} over "
+ "${inputs.files.length} files (mlEnabled: $mlEnabled, seed: "
+ "${inputs.request.seed})",
+ );
+ final rust.SmartMemories result;
+ try {
+ result = await rust.computeSmartMemories(
+ request: inputs.request,
+ periods: periods.map(_period).toList(),
+ store: mlEnabled ? await _mlDataDB.store : null,
+ cities: await locationService.getLocationIndex(),
+ );
+ } catch (e, s) {
+ _logger.severe("Rust memories computation failed", e, s);
+ return null;
+ }
+ final baseLocations = result.baseLocations.map(_baseLocation).toList();
+ return [
+ for (final memories in result.periods)
+ MemoriesResult(
+ memories.map(inputs.smartMemory).toList(),
+ baseLocations,
+ ),
+ ];
+ }
+
+ Future<_Inputs> _loadInputs({
+ MemoriesCache? oldCache,
+ required bool mlEnabled,
+ required bool surfaceAll,
+ }) async {
+ final files = await getFilesAndMapForMemories(
+ useLocalIntIds: isLocalGalleryMode,
+ requireLocalId: isLocalGalleryMode,
+ );
+ final collectionIDsToExclude = await getCollectionIDsToExclude();
+ final seenTimes = await memoriesDB.getSeenTimes();
+ final allPersons = (!mlEnabled || isLocalGalleryMode)
+ ? const <PersonEntity>[]
+ : await PersonService.instance.getPersons();
+ final persons = allPersons
+ .where((person) => !person.data.hideFromMemories)
+ .toList();
+ final currentUserEmail = isLocalGalleryMode
+ ? null
+ : Configuration.instance.getEmail();
+ final me = currentUserEmail == null
+ ? null
+ : persons.firstWhereOrNull(
+ (person) => person.data.email == currentUserEmail,
+ );
+ final request = rust.MemoriesRequest(
+ files: [
+ for (final entry in files.entries) _memoryFile(entry.key, entry.value),
+ ],
+ excludedCollectionIds: Int64List.fromList(
+ collectionIDsToExclude.toList(),
+ ),
+ people: persons.map(_memoryPerson).toList(),
+ peopleHiddenFromMemories: [
+ for (final person in allPersons)
+ if (person.data.hideFromMemories) person.remoteID,
+ ],
+ mePersonId: me?.remoteID,
+ showsUnnamedPeople: mlEnabled && localSettings.showLocalGalleryModeOption,
+ shownPeople: [
+ for (final log in oldCache?.peopleShownLogs ?? <PeopleShownLog>[])
+ rust.ShownPerson(
+ personId: log.personID,
+ memoryType: rust.PeopleMemoryType.values.byName(
+ log.peopleMemoryType.name,
+ ),
+ lastTimeShown: log.lastTimeShown,
+ ),
+ ],
+ shownTrips: [
+ for (final log in oldCache?.tripsShownLogs ?? <TripsShownLog>[])
+ rust.ShownTrip(
+ key: log.tripKey,
+ location: _coordinate(log.location),
+ lastTimeShown: log.lastTimeShown,
+ ),
+ ],
+ cachedTrips: [
+ for (final memory in oldCache?.toShowMemories ?? <ToShowMemory>[])
+ if (memory.type == MemoryType.trips)
+ rust.CachedTrip(
+ identityKey: memory.tripIdentityKey,
+ firstDateToShow: memory.firstTimeToShow,
+ lastDateToShow: memory.lastTimeToShow,
+ ),
+ ],
+ shownClipMemories: [
+ for (final log in oldCache?.clipShownLogs ?? <ClipShownLog>[])
+ rust.ShownClipMemory(
+ kind: rust.ClipMemoryType.values.byName(log.clipMemoryType.name),
+ lastTimeShown: log.lastTimeShown,
+ ),
+ ],
+ surfaceAll: surfaceAll,
+ localGalleryMode: isLocalGalleryMode,
+ seed: _seed(),
+ );
+ return _Inputs(
+ request: request,
+ files: files,
+ seenTimes: seenTimes,
+ persons: {for (final person in persons) person.remoteID: person},
+ );
+ }
+
+ static rust.MemoryFile _memoryFile(int id, EnteFile file) {
+ final creationTime = file.creationTime!;
+ return rust.MemoryFile(
+ id: id,
+ creationTime: creationTime,
+ localDateTime: file.pubMagicMetadata?.dateTime,
+ utcOffsetSeconds: DateTime.fromMicrosecondsSinceEpoch(
+ creationTime,
+ ).timeZoneOffset.inSeconds,
+ collectionId: file.collectionID,
+ location: file.hasLocation ? _coordinate(file.location!) : null,
+ );
+ }
+
+ static rust.MemoryPerson _memoryPerson(PersonEntity person) {
+ final birthDate = DateTime.tryParse(person.data.birthDate ?? "");
+ return rust.MemoryPerson(
+ id: person.remoteID,
+ isIgnored: person.data.isIgnored,
+ birthDate: birthDate == null
+ ? null
+ : rust.LocalDate(
+ year: birthDate.year,
+ month: birthDate.month,
+ day: birthDate.day,
+ ),
+ );
+ }
+
+ static rust.Period _period(DateTime time) => rust.Period(
+ now: time.microsecondsSinceEpoch,
+ utcOffsetSeconds: time.timeZoneOffset.inSeconds,
+ );
+
+ static rust.Coordinate _coordinate(Location location) => rust.Coordinate(
+ latitude: location.latitude!,
+ longitude: location.longitude!,
+ );
+
+ static BigInt _seed() {
+ final random = Random.secure();
+ return (BigInt.from(random.nextInt(1 << 32)) << 32) |
+ BigInt.from(random.nextInt(1 << 32));
+ }
+
+ static BaseLocation _baseLocation(rust.BaseLocation base) => BaseLocation(
+ base.fileIds.inner.toList(),
+ _location(base.location),
+ base.isCurrent,
+ );
+
+ static Location _location(rust.Coordinate coordinate) =>
+ Location(latitude: coordinate.latitude, longitude: coordinate.longitude);
+}
+
+class _Inputs {
+ final rust.MemoriesRequest request;
+ final Map<int, EnteFile> files;
+ final Map<int, int> seenTimes;
+ final Map<String, PersonEntity> persons;
+
+ _Inputs({
+ required this.request,
+ required this.files,
+ required this.seenTimes,
+ required this.persons,
+ });
+
+ SmartMemory smartMemory(rust.Memory memory) {
+ final memories = [
+ for (final fileID in memory.fileIds.inner)
+ if (files[fileID] case final file?) Memory.fromFile(file, seenTimes),
+ ];
+ final first = memory.firstDateToShow;
+ final last = memory.lastDateToShow;
+ return switch (memory.kind) {
+ rust.MemoryKind_OnThisDay() => OnThisDayMemory(memories, first, last),
+ rust.MemoryKind_People(field0: final people) => _peopleMemory(
+ memories,
+ first,
+ last,
+ people,
+ ),
+ rust.MemoryKind_Trip(field0: final trip) => TripMemory(
+ memories,
+ first,
+ last,
+ RustMemoriesComputer._location(trip.location),
+ locationName: trip.locationName,
+ tripYear: trip.year,
+ tripKey: trip.key,
+ ),
+ rust.MemoryKind_Base(field0: final base) => TripMemory(
+ memories,
+ first,
+ last,
+ RustMemoriesComputer._location(base.location),
+ locationName: _baseName(base),
+ ),
+ rust.MemoryKind_Clip(field0: final clipType) => ClipMemory(
+ memories,
+ first,
+ last,
+ ClipMemoryType.values.byName(clipType.name),
+ ),
+ rust.MemoryKind_Time(field0: final time) => _timeMemory(
+ memories,
+ first,
+ last,
+ time,
+ ),
+ rust.MemoryKind_Filler(:final yearsAgo) => FillerMemory(
+ memories,
+ yearsAgo,
+ first,
+ last,
+ ),
+ };
+ }
+
+ PeopleMemory _peopleMemory(
+ List<Memory> memories,
+ int first,
+ int last,
+ rust.PeopleMemory people,
+ ) {
+ final memoryType = PeopleMemoryType.values.byName(people.memoryType.name);
+ final isMySpotlight =
+ people.personId == request.mePersonId &&
+ memoryType == PeopleMemoryType.spotlight;
+ final activity = people.activity;
+ final birthday = people.birthday;
+ return PeopleMemory(
+ memories,
+ first,
+ last,
+ memoryType,
+ people.personId,
+ people.isUnnamedCluster || isMySpotlight
+ ? null
+ : persons[people.personId]?.data.name,
+ activity: activity == null
+ ? null
+ : PeopleActivity.values.byName(activity.name),
+ isUnnamedCluster: people.isUnnamedCluster,
+ isBirthday: birthday == null ? null : !birthday.isUpcoming,
+ newAge: birthday?.newAge,
+ );
+ }
+
+ static TimeMemory _timeMemory(
+ List<Memory> memories,
+ int first,
+ int last,
+ rust.TimeMemory time,
+ ) => switch (time) {
+ rust.TimeMemory_Day(:final date, :final yearsAgo) => TimeMemory(
+ memories,
+ first,
+ last,
+ day: DateTime(date.year, date.month, date.day),
+ yearsAgo: yearsAgo,
+ ),
+ rust.TimeMemory_Week(:final yearsAgo) => TimeMemory(
+ memories,
+ first,
+ last,
+ yearsAgo: yearsAgo,
+ ),
+ rust.TimeMemory_Month(:final year, :final month, :final yearsAgo) =>
+ TimeMemory(
+ memories,
+ first,
+ last,
+ month: DateTime(year, month),
+ yearsAgo: yearsAgo,
+ ),
+ rust.TimeMemory_LastWeek() => TimeMemory(
+ memories,
+ first,
+ last,
+ kind: TimeMemoryKind.lastWeek,
+ ),
+ rust.TimeMemory_LastMonth() => TimeMemory(
+ memories,
+ first,
+ last,
+ kind: TimeMemoryKind.lastMonth,
+ ),
+ };
+
+ static String _baseName(rust.BaseMemory base) {
+ final status = base.isCurrent ? "current" : "old";
+ final locationName = base.locationName;
+ return locationName == null
+ ? "Base ($status)"
+ : "$locationName (Base, $status)";
+ }
+}
diff --git a/mobile/packages/feature_flag/lib/src/flag_service.dart b/mobile/packages/feature_flag/lib/src/flag_service.dart
index d91f846de7..76285638f8 100644
--- a/mobile/packages/feature_flag/lib/src/flag_service.dart
+++ b/mobile/packages/feature_flag/lib/src/flag_service.dart
@@ -89,6 +89,8 @@ class FlagService {
// Dart anymore and internal users have soaked it for a while.
bool get rustMlDb => internalUser;
+ bool get rustMemories => internalUser;
+
String get castUrl => flags.castUrl;
String get customDomain => flags.customDomain;
diff --git a/rust/bindings/frb/photos/src/api/location_api.rs b/rust/bindings/frb/photos/src/api/location_api.rs
index 8fe0b61eca..7c64c4932e 100644
--- a/rust/bindings/frb/photos/src/api/location_api.rs
+++ b/rust/bindings/frb/photos/src/api/location_api.rs
@@ -23,7 +23,7 @@ impl From<ente_photos::location::Error> for LocationError {
}
#[frb(opaque)]
-pub struct LocationIndex(ente_photos::location::LocationIndex);
+pub struct LocationIndex(pub(crate) ente_photos::location::LocationIndex);
#[derive(Clone, Debug)]
pub struct LocationCoordinate {
diff --git a/rust/bindings/frb/photos/src/api/memories_api.rs b/rust/bindings/frb/photos/src/api/memories_api.rs
new file mode 100644
index 0000000000..3df920ef8f
--- /dev/null
+++ b/rust/bindings/frb/photos/src/api/memories_api.rs
@@ -0,0 +1,267 @@
+pub use ente_location::Coordinate;
+use ente_photos::memories;
+use ente_photos::memories::CityLookup;
+pub use ente_photos::memories::{
+ BaseLocation, BaseMemory, Birthday, CachedTrip, ClipMemoryType, LocalDate, MemoriesRequest,
+ Memory, MemoryFile, MemoryKind, MemoryPerson, PeopleActivity, PeopleMemory, PeopleMemoryType,
+ Period, ShownClipMemory, ShownPerson, ShownTrip, SmartMemories, TimeMemory, TripMemory,
+};
+use flutter_rust_bridge::frb;
+
+use super::location_api::LocationIndex;
+use super::ml_db_api::{MlDbError, MlStore};
+use crate::frb_generated::RustAutoOpaque;
+
+#[frb(mirror(Coordinate))]
+pub struct _Coordinate {
+ pub latitude: f64,
+ pub longitude: f64,
+}
+
+#[frb(mirror(LocalDate))]
+pub struct _LocalDate {
+ pub year: i32,
+ pub month: i32,
+ pub day: i32,
+}
+
+#[frb(mirror(MemoryFile))]
+pub struct _MemoryFile {
+ pub id: i64,
+ pub creation_time: i64,
+ pub local_date_time: Option<String>,
+ pub utc_offset_seconds: i32,
+ pub collection_id: Option<i64>,
+ pub location: Option<Coordinate>,
+}
+
+#[frb(mirror(MemoriesRequest))]
+pub struct _MemoriesRequest {
+ pub files: Vec<MemoryFile>,
+ pub excluded_collection_ids: Vec<i64>,
+ pub people: Vec<MemoryPerson>,
+ pub people_hidden_from_memories: Vec<String>,
+ pub me_person_id: Option<String>,
+ pub shows_unnamed_people: bool,
+ pub shown_people: Vec<ShownPerson>,
+ pub shown_trips: Vec<ShownTrip>,
+ pub cached_trips: Vec<CachedTrip>,
+ pub shown_clip_memories: Vec<ShownClipMemory>,
+ pub surface_all: bool,
+ pub local_gallery_mode: bool,
+ pub seed: u64,
+}
+
+#[frb(mirror(MemoryPerson))]
+pub struct _MemoryPerson {
+ pub id: String,
+ pub is_ignored: bool,
+ pub birth_date: Option<LocalDate>,
+}
+
+#[frb(mirror(ShownPerson))]
+pub struct _ShownPerson {
+ pub person_id: String,
+ pub memory_type: PeopleMemoryType,
+ pub last_time_shown: i64,
+}
+
+#[frb(mirror(ShownTrip))]
+pub struct _ShownTrip {
+ pub key: Option<String>,
+ pub location: Coordinate,
+ pub last_time_shown: i64,
+}
+
+#[frb(mirror(ShownClipMemory))]
+pub struct _ShownClipMemory {
+ pub kind: ClipMemoryType,
+ pub last_time_shown: i64,
+}
+
+#[frb(mirror(CachedTrip))]
+pub struct _CachedTrip {
+ pub identity_key: String,
+ pub first_date_to_show: i64,
+ pub last_date_to_show: i64,
+}
+
+#[frb(mirror(Period))]
+pub struct _Period {
+ pub now: i64,
+ pub utc_offset_seconds: i32,
+}
+
+#[frb(mirror(SmartMemories))]
+pub struct _SmartMemories {
+ pub periods: Vec<Vec<Memory>>,
+ pub base_locations: Vec<BaseLocation>,
+}
+
+#[frb(mirror(BaseLocation))]
+pub struct _BaseLocation {
+ pub file_ids: Vec<i64>,
+ pub location: Coordinate,
+ pub is_current: bool,
+}
+
+#[frb(mirror(Memory))]
+pub struct _Memory {
+ pub kind: MemoryKind,
+ pub file_ids: Vec<i64>,
+ pub first_date_to_show: i64,
+ pub last_date_to_show: i64,
+}
+
+#[frb(mirror(MemoryKind))]
+pub enum _MemoryKind {
+ OnThisDay,
+ People(PeopleMemory),
+ Trip(TripMemory),
+ Base(BaseMemory),
+ Clip(ClipMemoryType),
+ Time(TimeMemory),
+ Filler { years_ago: i32 },
+}
+
+#[frb(mirror(PeopleMemory))]
+pub struct _PeopleMemory {
+ pub person_id: String,
+ pub is_unnamed_cluster: bool,
+ pub memory_type: PeopleMemoryType,
+ pub activity: Option<PeopleActivity>,
+ pub birthday: Option<Birthday>,
+}
+
+#[frb(mirror(PeopleMemoryType))]
+pub enum _PeopleMemoryType {
+ YouAndThem,
+ DoingSomethingTogether,
+ Spotlight,
+ LastTimeYouSawThem,
+}
+
+#[frb(mirror(PeopleActivity))]
+pub enum _PeopleActivity {
+ Admiring,
+ Embracing,
+ Party,
+ Hiking,
+ Feast,
+ Selfies,
+ Posing,
+ Background,
+ Sports,
+ Roadtrip,
+}
+
+#[frb(mirror(Birthday))]
+pub struct _Birthday {
+ pub is_upcoming: bool,
+ pub new_age: i32,
+}
+
+#[frb(mirror(TripMemory))]
+pub struct _TripMemory {
+ pub location: Coordinate,
+ pub location_name: Option<String>,
+ pub year: i32,
+ pub key: String,
+}
+
+#[frb(mirror(BaseMemory))]
+pub struct _BaseMemory {
+ pub location: Coordinate,
+ pub location_name: Option<String>,
+ pub is_current: bool,
+}
+
+#[frb(mirror(ClipMemoryType))]
+pub enum _ClipMemoryType {
+ Sunrise,
+ Mountains,
+ Greenery,
+ Beach,
+ City,
+ Moon,
+ OnTheRoad,
+ Food,
+ Pets,
+ Festivities,
+ SnowAdventures,
+ Waterfalls,
+ Wildlife,
+ Flowers,
+ NightLights,
+ Architecture,
+ AutumnColors,
+ DesertDreams,
+ Stargazing,
+ Lakeside,
+ RainyDays,
+ SportsAction,
+ StreetArt,
+ FamilyMoments,
+ Fireworks,
+ HistoricSites,
+ TropicalParadise,
+ ForestTrails,
+ CitySunsets,
+ ColorfulMarkets,
+ CozyCafes,
+ VintageVibes,
+ AerialViews,
+ ArtisticPortraits,
+ StreetFood,
+ RiverCruises,
+ PlayfulKids,
+ CoastalCliffs,
+ CandidLaughter,
+ GroupSelfies,
+ BirthdayJoy,
+ WeddingMoments,
+ GoldenHourPortraits,
+ ConcertNights,
+ PoolDays,
+ PicnicDays,
+ CampingMoments,
+ BabySmiles,
+}
+
+#[frb(mirror(TimeMemory))]
+pub enum _TimeMemory {
+ Day {
+ date: LocalDate,
+ years_ago: Option<i32>,
+ },
+ Week {
+ years_ago: Option<i32>,
+ },
+ Month {
+ year: i32,
+ month: i32,
+ years_ago: Option<i32>,
+ },
+ LastWeek,
+ LastMonth,
+}
+
+pub fn compute_smart_memories(
+ request: MemoriesRequest,
+ periods: Vec<Period>,
+ store: Option<RustAutoOpaque<MlStore>>,
+ cities: Option<RustAutoOpaque<LocationIndex>>,
+) -> Result<SmartMemories, MlDbError> {
+ let store = store.as_ref().map(RustAutoOpaque::blocking_read);
+ let cities = cities.as_ref().map(RustAutoOpaque::blocking_read);
+ Ok(memories::compute_smart_memories_from_store(
+ &request,
+ &periods,
+ store.as_deref().map(|store| &store.inner),
+ cities.as_deref().map(|cities| &cities.0 as &dyn CityLookup),
+ )?)
+}
+
+pub fn compute_simple_memories(request: MemoriesRequest, period: Period) -> Vec<Memory> {
+ memories::compute_simple_memories(&request, period)
+}
diff --git a/rust/bindings/frb/photos/src/api/ml_db_api.rs b/rust/bindings/frb/photos/src/api/ml_db_api.rs
index 5339723593..425d03d0e6 100644
--- a/rust/bindings/frb/photos/src/api/ml_db_api.rs
+++ b/rust/bindings/frb/photos/src/api/ml_db_api.rs
@@ -267,7 +267,7 @@ fn search_params(
#[frb(opaque)]
pub struct MlStore {
- inner: ml_store::MlStore,
+ pub(crate) inner: ml_store::MlStore,
}
impl MlStore {
diff --git a/rust/bindings/frb/photos/src/api/mod.rs b/rust/bindings/frb/photos/src/api/mod.rs
index 406438cc28..f1164f95fb 100644
--- a/rust/bindings/frb/photos/src/api/mod.rs
+++ b/rust/bindings/frb/photos/src/api/mod.rs
@@ -6,6 +6,8 @@ pub mod location_api;
#[cfg(any(feature = "flutter", frb_expand))]
pub mod log;
pub mod map_cluster_api;
+#[cfg(any(feature = "flutter", frb_expand))]
+pub mod memories_api;
pub mod metadata_api;
pub mod ml_db_api;
pub mod ml_indexing_api;
diff --git a/rust/crates/photos/src/memories/input.rs b/rust/crates/photos/src/memories/input.rs
index fef186682f..10e2502516 100644
--- a/rust/crates/photos/src/memories/input.rs
+++ b/rust/crates/photos/src/memories/input.rs
@@ -22,6 +22,7 @@ pub struct MemoriesRequest {
pub files: Vec<MemoryFile>,
pub excluded_collection_ids: Vec<i64>,
pub people: Vec<MemoryPerson>,
+ pub people_hidden_from_memories: Vec<String>,
pub me_person_id: Option<String>,
pub shows_unnamed_people: bool,
pub shown_people: Vec<ShownPerson>,
diff --git a/rust/crates/photos/src/memories/mod.rs b/rust/crates/photos/src/memories/mod.rs
index 6ca1f09b4f..8624fa0821 100644
--- a/rust/crates/photos/src/memories/mod.rs
+++ b/rust/crates/photos/src/memories/mod.rs
@@ -11,6 +11,7 @@ mod places;
mod rng;
mod selector;
mod signals;
+mod store;
mod text_embeddings;
mod time;
mod trips;
@@ -27,6 +28,7 @@ pub use output::{
BaseLocation, BaseMemory, Birthday, ClipMemoryType, Memory, MemoryKind, PeopleActivity,
PeopleMemory, PeopleMemoryType, SmartMemories, TimeMemory, TripMemory,
};
+pub use store::compute_smart_memories_from_store;
use input::PhotoSignals;
use people::People;
@@ -120,6 +122,11 @@ pub fn compute_smart_memories(
}
pub fn compute_simple_memories(request: &MemoriesRequest, period: Period) -> Vec<Memory> {
+ log::info!(
+ "computing simple memories for {} files with seed {}",
+ request.files.len(),
+ request.seed
+ );
let photos = photo::prepare(&request.files);
let excluded_collection_ids: HashSet<i64> =
request.excluded_collection_ids.iter().copied().collect();
diff --git a/rust/crates/photos/src/memories/people.rs b/rust/crates/photos/src/memories/people.rs
index c32d9702f7..e1e3292bc4 100644
--- a/rust/crates/photos/src/memories/people.rs
+++ b/rust/crates/photos/src/memories/people.rs
@@ -82,11 +82,7 @@ impl<'a> People<'a> {
id,
file_ids: inputs.person_file_ids.get(id).unwrap_or(&no_file_ids),
});
- let mut important: Vec<&MemoryPerson> = request
- .people
- .iter()
- .filter(|person| !request.local_gallery_mode && !person.is_ignored)
- .collect();
+ let mut important: Vec<&MemoryPerson> = important_people(request).collect();
important.sort_by_key(|&person| &person.id);
let activity_file_ids = match me {
Some(me) if important.iter().any(|person| person.id != me.id) => {
@@ -94,15 +90,11 @@ impl<'a> People<'a> {
}
_ => Vec::new(),
};
- let uses_unnamed_fallback = request.shows_unnamed_people
- && (request.local_gallery_mode
- || important.len() < NAMED_PEOPLE_DISABLING_UNNAMED_FALLBACK);
- let unnamed_people =
- if uses_unnamed_fallback || (request.shows_unnamed_people && request.surface_all) {
- unnamed_people(inputs, &photos_by_id, me)
- } else {
- Vec::new()
- };
+ let unnamed_people = if shows_unnamed_clusters(request) {
+ unnamed_people(inputs, &photos_by_id, me)
+ } else {
+ Vec::new()
+ };
Ok(Self {
show_timeout_days: PERSON_SHOW_TIMEOUT_DAYS
.min(important.len().max(1) as i64 * UPDATE_FREQUENCY_DAYS),
@@ -111,7 +103,7 @@ impl<'a> People<'a> {
.map(|person| Person::new(person, inputs, &photos_by_id, me, &activity_file_ids))
.collect(),
unnamed_people,
- uses_unnamed_fallback,
+ uses_unnamed_fallback: uses_unnamed_fallback(request),
surface_all: request.surface_all,
shown: &request.shown_people,
face_counts: &inputs.face_counts,
@@ -479,6 +471,23 @@ fn activity_file_ids(signals: &Signals) -> Result<Vec<(PeopleActivity, HashSet<i
.collect()
}
+pub(super) fn shows_unnamed_clusters(request: &MemoriesRequest) -> bool {
+ uses_unnamed_fallback(request) || (request.shows_unnamed_people && request.surface_all)
+}
+
+fn uses_unnamed_fallback(request: &MemoriesRequest) -> bool {
+ request.shows_unnamed_people
+ && (request.local_gallery_mode
+ || important_people(request).count() < NAMED_PEOPLE_DISABLING_UNNAMED_FALLBACK)
+}
+
+fn important_people(request: &MemoriesRequest) -> impl Iterator<Item = &MemoryPerson> {
+ request
+ .people
+ .iter()
+ .filter(|person| !request.local_gallery_mode && !person.is_ignored)
+}
+
fn is_last_met_anniversary_near(last_seen: Candidate, today: LocalDate) -> bool {
last_seen.photos.last().is_some_and(|photo| {
let anniversary = LocalDate::new(today.year, photo.date.month, photo.date.day);
diff --git a/rust/crates/photos/src/memories/store.rs b/rust/crates/photos/src/memories/store.rs
new file mode 100644
index 0000000000..cd2b837ebc
--- /dev/null
+++ b/rust/crates/photos/src/memories/store.rs
@@ -0,0 +1,250 @@
+use std::collections::{HashMap, HashSet};
+use std::time::Instant;
+
+use super::compute_smart_memories;
+use super::input::{CityLookup, MemoriesRequest, MlInputs, PeopleInputs, Period, UnnamedCluster};
+use super::output::SmartMemories;
+use super::people::shows_unnamed_clusters;
+use crate::ml_db::{MlDb, try_file_id_from_face_id};
+use crate::ml_store::{Error, FillState, Index, MlStore, Result};
+
+const NAMED_FACE_WEIGHT: u32 = 10;
+const OTHER_FACE_WEIGHT: u32 = 1;
+
+pub fn compute_smart_memories_from_store(
+ request: &MemoriesRequest,
+ periods: &[Period],
+ store: Option<&MlStore>,
+ cities: Option<&dyn CityLookup>,
+) -> Result<SmartMemories> {
+ let started = Instant::now();
+ let ml = store
+ .map(|store| load_ml_inputs(store, request))
+ .transpose()?;
+ log::info!(
+ "loaded memories ML inputs in {} ms",
+ started.elapsed().as_millis()
+ );
+ let started = Instant::now();
+ let memories = compute_smart_memories(request, periods, ml.as_ref(), cities)?;
+ log::info!(
+ "computed smart memories for {} files with seed {} in {} ms",
+ request.files.len(),
+ request.seed,
+ started.elapsed().as_millis()
+ );
+ Ok(memories)
+}
+
+fn load_ml_inputs<'s>(store: &'s MlStore, request: &MemoriesRequest) -> Result<MlInputs<'s>> {
+ store.fill_clip_index(false)?;
+ if store.fill_state(Index::Clip)? != FillState::Filled {
+ return Err(Error::NotFilled(Index::Clip));
+ }
+ let db = store.db();
+ let person_face_ids = person_face_ids(db, request)?;
+ let named_face_ids: HashSet<&String> = person_face_ids.values().flatten().collect();
+ let mut face_counts = HashMap::new();
+ let mut face_weights = HashMap::new();
+ for (file_id, face_ids) in db.get_file_id_to_face_ids()? {
+ let weight = face_ids
+ .iter()
+ .map(|face_id| {
+ if named_face_ids.contains(face_id) {
+ NAMED_FACE_WEIGHT
+ } else {
+ OTHER_FACE_WEIGHT
+ }
+ })
+ .sum();
+ face_counts.insert(file_id, face_ids.len() as u32);
+ face_weights.insert(file_id, weight);
+ }
+ let unnamed_clusters = if shows_unnamed_clusters(request) {
+ unnamed_clusters(db, request)?
+ } else {
+ Vec::new()
+ };
+ Ok(MlInputs {
+ clip: store,
+ face_weights,
+ people: PeopleInputs {
+ face_counts,
+ person_file_ids: person_face_ids
+ .iter()
+ .map(|(person_id, face_ids)| (person_id.clone(), file_ids(face_ids)))
+ .collect(),
+ unnamed_clusters,
+ },
+ })
+}
+
+fn person_face_ids(db: &MlDb, request: &MemoriesRequest) -> Result<HashMap<String, Vec<String>>> {
+ let person_ids: HashSet<&str> = request
+ .people
+ .iter()
+ .map(|person| person.id.as_str())
+ .collect();
+ Ok(db
+ .get_person_to_cluster_id_to_face_ids()?
+ .into_iter()
+ .filter(|(person_id, _)| person_ids.contains(person_id.as_str()))
+ .map(|(person_id, clusters)| (person_id, clusters.into_values().flatten().collect()))
+ .collect())
+}
+
+fn unnamed_clusters(db: &MlDb, request: &MemoriesRequest) -> Result<Vec<UnnamedCluster>> {
+ let person_ids: Vec<String> = request
+ .people
+ .iter()
+ .map(|person| person.id.clone())
+ .chain(request.people_hidden_from_memories.iter().cloned())
+ .collect();
+ let assigned = db.get_persons_cluster_ids(&person_ids)?;
+ Ok(db
+ .get_all_cluster_id_to_face_ids()?
+ .into_iter()
+ .filter(|(cluster_id, _)| !assigned.contains(cluster_id))
+ .map(|(id, face_ids)| UnnamedCluster {
+ id,
+ face_count: face_ids.len() as u32,
+ file_ids: file_ids(&face_ids),
+ })
+ .collect())
+}
+
+fn file_ids(face_ids: &[String]) -> HashSet<i64> {
+ face_ids
+ .iter()
+ .filter_map(|face_id| try_file_id_from_face_id(face_id))
+ .collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::{HashMap, HashSet};
+
+ use super::load_ml_inputs;
+ use crate::memories::{MemoriesRequest, MemoryPerson, UnnamedCluster};
+ use crate::ml_db::{CLIP_EMBEDDING_DIMENSIONS, ClipEmbedding, FaceRow};
+ use crate::ml_store::{CLUSTER_CENTROID_DIMENSIONS, Error, FillState, Index, MlStore};
+
+ fn face(face_id: &str) -> FaceRow {
+ FaceRow {
+ file_id: face_id.split_once('_').unwrap().0.parse().unwrap(),
+ face_id: face_id.to_string(),
+ detection_json: "{}".to_string(),
+ embedding: vec![0.1; CLUSTER_CENTROID_DIMENSIONS],
+ score: 0.9,
+ blur: 300.0,
+ is_sideways: false,
+ image_height: 100,
+ image_width: 100,
+ ml_version: 1,
+ }
+ }
+
+ fn person(id: &str) -> MemoryPerson {
+ MemoryPerson {
+ id: id.to_string(),
+ is_ignored: false,
+ birth_date: None,
+ }
+ }
+
+ fn library() -> (tempfile::TempDir, MlStore) {
+ let directory = tempfile::tempdir().unwrap();
+ let store = MlStore::open(directory.path().join("ente.ml.db")).unwrap();
+ let db = store.db();
+ db.bulk_insert_faces(&["1_a", "1_c", "2_a", "3_b", "4_c"].map(face))
+ .unwrap();
+ db.update_face_id_to_cluster_id(&HashMap::from(
+ [
+ ("1_a", "named"),
+ ("2_a", "named"),
+ ("3_b", "hidden"),
+ ("1_c", "unnamed"),
+ ("4_c", "unnamed"),
+ ]
+ .map(|(face_id, cluster_id)| (face_id.to_string(), cluster_id.to_string())),
+ ))
+ .unwrap();
+ db.assign_cluster_to_person("alice", "named").unwrap();
+ db.assign_cluster_to_person("bob", "hidden").unwrap();
+ store
+ .put_clip(&[ClipEmbedding {
+ file_id: 1,
+ embedding: vec![0.5; CLIP_EMBEDDING_DIMENSIONS],
+ version: 1,
+ }])
+ .unwrap();
+ (directory, store)
+ }
+
+ #[test]
+ fn inputs_count_faces_weight_named_people_and_skip_assigned_clusters() {
+ let (_directory, store) = library();
+ let request = MemoriesRequest {
+ people: vec![person("alice")],
+ people_hidden_from_memories: vec!["bob".to_string()],
+ shows_unnamed_people: true,
+ ..MemoriesRequest::default()
+ };
+
+ let inputs = load_ml_inputs(&store, &request).unwrap();
+
+ assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Filled);
+ assert_eq!(
+ inputs.face_weights,
+ HashMap::from([(1, 11), (2, 10), (3, 1), (4, 1)])
+ );
+ assert_eq!(
+ inputs.people.face_counts,
+ HashMap::from([(1, 2), (2, 1), (3, 1), (4, 1)])
+ );
+ assert_eq!(
+ inputs.people.person_file_ids,
+ HashMap::from([("alice".to_string(), HashSet::from([1, 2]))])
+ );
+ assert_eq!(
+ inputs.people.unnamed_clusters,
+ [UnnamedCluster {
+ id: "unnamed".to_string(),
+ face_count: 2,
+ file_ids: HashSet::from([1, 4]),
+ }]
+ );
+ }
+
+ #[test]
+ fn unnamed_clusters_stay_unloaded_once_enough_people_are_named() {
+ let (_directory, store) = library();
+ let request = MemoriesRequest {
+ people: ["alice", "bob", "carol", "dave", "erin"]
+ .map(person)
+ .to_vec(),
+ shows_unnamed_people: true,
+ ..MemoriesRequest::default()
+ };
+
+ let inputs = load_ml_inputs(&store, &request).unwrap();
+
+ assert!(inputs.people.unnamed_clusters.is_empty());
+ assert_eq!(inputs.face_weights[&3], 10);
+ }
+
+ #[test]
+ fn unusable_clip_index_fails_the_load() {
+ let (directory, store) = library();
+ std::fs::create_dir(
+ directory
+ .path()
+ .join(format!("ente.ml.vecdb.{}", Index::Clip.name())),
+ )
+ .unwrap();
+
+ let result = load_ml_inputs(&store, &MemoriesRequest::default());
+
+ assert!(matches!(result, Err(Error::Index(_))));
+ }
+}
diff --git a/rust/crates/photos/src/ml_db/queries/faces.rs b/rust/crates/photos/src/ml_db/queries/faces.rs
index a07e342e0e..864f10423b 100644
--- a/rust/crates/photos/src/ml_db/queries/faces.rs
+++ b/rust/crates/photos/src/ml_db/queries/faces.rs
@@ -308,6 +308,10 @@ impl MlDb {
))
}
+ pub fn get_file_id_to_face_ids(&self) -> Result<HashMap<i64, Vec<String>>> {
+ self.read_grouped("SELECT file_id, face_id FROM faces", ())
+ }
+
pub fn get_face_info_for_clustering(
&self,
max_faces: i64,
diff --git a/rust/crates/photos/src/ml_store/mod.rs b/rust/crates/photos/src/ml_store/mod.rs
index f2e3f1d2dd..62a506e221 100644
--- a/rust/crates/photos/src/ml_store/mod.rs
+++ b/rust/crates/photos/src/ml_store/mod.rs
@@ -34,6 +34,8 @@ pub enum Error {
Index(#[from] VecDbError),
#[error("{0}")]
InvalidArgument(String),
+ #[error("{0:?} index is not filled")]
+ NotFilled(Index),
}
pub type Result<T> = std::result::Result<T, Error>;