π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Reuse trip analysis across memory periods
Neeraj Gupta0f7f910148e2
diff --git a/mobile/apps/photos/lib/services/location_service.dart b/mobile/apps/photos/lib/services/location_service.dartindex c27eb3d246..776414a647 100644--- a/mobile/apps/photos/lib/services/location_service.dart+++ b/mobile/apps/photos/lib/services/location_service.dart@@ -21,6 +21,46 @@ import "package:shared_preferences/shared_preferences.dart";const double earthRadius = 6371; // Earth's radius in kilometers+class CitySearchIndex {+ static const empty = CitySearchIndex(+ cities: <City>[],+ nodes: <List<int>>[],+ root: -1,+ maxLatDelta: 0,+ maxLngDelta: 0,+ );++ final List<City> cities;+ final List<List<int>> nodes;+ final int root;+ final double maxLatDelta;+ final double maxLngDelta;++ const CitySearchIndex({+ required this.cities,+ required this.nodes,+ required this.root,+ required this.maxLatDelta,+ required this.maxLngDelta,+ });++ bool get isEmpty => cities.isEmpty;++ Map<String, dynamic> searchArgs(List<EnteFile> files, {String query = ''}) {+ return <String, dynamic>{+ "query": query,+ "cities": cities,+ "files": files,+ if (nodes.isNotEmpty && root >= 0) ...{+ "kdTreeNodes": nodes,+ "kdTreeRoot": root,+ "kdTreeMaxLatDelta": maxLatDelta,+ "kdTreeMaxLngDelta": maxLngDelta,+ },+ };+ }+}+class LocationService {final SharedPreferences prefs;final Logger _logger = Logger((LocationService).toString());@@ -32,11 +72,7 @@ class LocationService {static const _kCitiesKdTreeRemotePath ="https://assets.ente.com/world_cities.kdtree.bin";- List<City> _cities = [];- List<List<int>> _kdTreeNodes = [];- int _kdTreeRoot = -1;- double _kdTreeMaxLatDelta = 0;- double _kdTreeMaxLngDelta = 0;+ CitySearchIndex _citySearchIndex = CitySearchIndex.empty;// TODO: lau: consider actually using this in location sectionList<BaseLocation> baseLocations = [];@@ -72,22 +108,12 @@ class LocationService {List<EnteFile> allFiles,String query,) async {- if (allFiles.isNotEmpty && _cities.isEmpty && query.isEmpty) {+ if (allFiles.isNotEmpty && _citySearchIndex.isEmpty && query.isEmpty) {reloadLocationDiscoverySection = true;}final EnteWatch w = EnteWatch("cities_search")..start();w.log('start for files ${allFiles.length} and query $query');- final args = <String, dynamic>{- "query": query,- "cities": _cities,- "files": allFiles,- };- if (_kdTreeNodes.isNotEmpty && _kdTreeRoot >= 0) {- args["kdTreeNodes"] = _kdTreeNodes;- args["kdTreeRoot"] = _kdTreeRoot;- args["kdTreeMaxLatDelta"] = _kdTreeMaxLatDelta;- args["kdTreeMaxLngDelta"] = _kdTreeMaxLngDelta;- }+ final args = _citySearchIndex.searchArgs(allFiles, query: query);final result = await _computer.compute(getCityResults, param: args);w.log('end for query: $query on ${allFiles.length} files, found '@@ -97,10 +123,14 @@ class LocationService {}Future<List<City>> getCities() async {- if (_cities.isEmpty) {+ return (await getCitySearchIndex()).cities;+ }++ Future<CitySearchIndex> getCitySearchIndex() async {+ if (_citySearchIndex.isEmpty) {await _loadCities();}- return _cities;+ return _citySearchIndex;}Future<Iterable<LocalEntity<LocationTag>>> getLocationTags() {@@ -241,11 +271,13 @@ class LocationService {}void _applyKdTreeLoadResult(Map<String, dynamic> result) {- _cities = result["cities"] as List<City>;- _kdTreeNodes = (result["nodes"] as List).cast<List<int>>();- _kdTreeRoot = result["root"] as int;- _kdTreeMaxLatDelta = (result["maxLatDelta"] as num).toDouble();- _kdTreeMaxLngDelta = (result["maxLngDelta"] as num).toDouble();+ _citySearchIndex = CitySearchIndex(+ cities: result["cities"] as List<City>,+ nodes: (result["nodes"] as List).cast<List<int>>(),+ root: result["root"] as int,+ maxLatDelta: (result["maxLatDelta"] as num).toDouble(),+ maxLngDelta: (result["maxLngDelta"] as num).toDouble(),+ );}Future<void> _loadCities() async {diff --git a/mobile/apps/photos/lib/services/memories/memories_computation_context.dart b/mobile/apps/photos/lib/services/memories/memories_computation_context.dartindex 9fb9db3153..0b3ed8bf30 100644--- a/mobile/apps/photos/lib/services/memories/memories_computation_context.dart+++ b/mobile/apps/photos/lib/services/memories/memories_computation_context.dart@@ -20,7 +20,7 @@ class MemoriesComputationContext {final Map<int, int> seenTimes;final List<PersonEntity> persons;final String? currentUserEmail;- final List<City> cities;+ final CitySearchIndex citySearchIndex;final Map<int, List<FaceWithoutEmbedding>> fileIdToFaces;final Map<String, int> clusterIdToFaceCount;final Map<String, Iterable<String>> clusterIdToFaceIDs;@@ -42,7 +42,7 @@ class MemoriesComputationContext {required this.seenTimes,required this.persons,required this.currentUserEmail,- required this.cities,+ required this.citySearchIndex,required this.fileIdToFaces,required this.clusterIdToFaceCount,required this.clusterIdToFaceIDs,@@ -71,7 +71,7 @@ class MemoriesComputationContext {seenTimes: Map<int, int>.from(args["seenTimes"] as Map),persons: (args["persons"] as List).cast<PersonEntity>(),currentUserEmail: args["currentUserEmail"] as String?,- cities: (args["cities"] as List).cast<City>(),+ citySearchIndex: args["citySearchIndex"] as CitySearchIndex,fileIdToFaces: Map<int, List<FaceWithoutEmbedding>>.from(args["fileIdToFaces"] as Map,),@@ -107,7 +107,7 @@ class MemoriesComputationContext {"seenTimes": seenTimes,"persons": persons,"currentUserEmail": currentUserEmail,- "cities": cities,+ "citySearchIndex": citySearchIndex,"fileIdToFaces": fileIdToFaces,"clusterIdToFaceCount": clusterIdToFaceCount,"clusterIdToFaceIDs": clusterIdToFaceIDs,diff --git a/mobile/apps/photos/lib/services/smart_memories_service.dart b/mobile/apps/photos/lib/services/smart_memories_service.dartindex e62620de45..9897ea460a 100644--- a/mobile/apps/photos/lib/services/smart_memories_service.dart+++ b/mobile/apps/photos/lib/services/smart_memories_service.dart@@ -68,6 +68,7 @@ typedef _PreparedMemoriesData = ({List<PersonEntity> persons,Map<String, String> faceIDsToPersonID,Map<int, EmbeddingVector> fileIDToImageEmbedding,+ TripMemoriesAnalysis tripAnalysis,});class SmartMemoriesService {@@ -353,8 +354,10 @@ class SmartMemoriesService {: Configuration.instance.getEmail();_logger.info('currentUserEmail: $currentUserEmail $timeLogger');- final cities = await locationService.getCities();- _logger.info('cities has ${cities.length} entries $timeLogger');+ final citySearchIndex = await locationService.getCitySearchIndex();+ _logger.info(+ 'cities has ${citySearchIndex.cities.length} entries $timeLogger',+ );final Map<int, List<FaceWithoutEmbedding>> fileIdToFaces = mlEnabled? await mlDataDB.getFileIDsToFacesWithoutEmbedding()@@ -420,7 +423,7 @@ class SmartMemoriesService {seenTimes: seenTimes,persons: persons,currentUserEmail: currentUserEmail,- cities: cities,+ citySearchIndex: citySearchIndex,fileIdToFaces: fileIdToFaces,clusterIdToFaceCount: unnamedClusterData.clusterIdToFaceCount,clusterIdToFaceIDs: unnamedClusterData.clusterIdToFaceIDs,@@ -848,10 +851,18 @@ class SmartMemoriesService {for (final embedding in computationContext.allImageEmbeddings) {fileIDToImageEmbedding[embedding.fileID] = embedding;}+ final tripAnalysis = TripMemoriesCalculatorV2.analyze(+ computationContext.allFileIdsToFile.values,+ computationContext.allFileIdsToFile,+ isLocalGalleryMode: computationContext.isLocalGalleryMode,+ seenTimes: computationContext.seenTimes,+ citySearchIndex: computationContext.citySearchIndex,+ );return (persons: persons,faceIDsToPersonID: faceIDsToPersonID,fileIDToImageEmbedding: fileIDToImageEmbedding,+ tripAnalysis: tripAnalysis,);}@@ -896,7 +907,8 @@ class SmartMemoriesService {final Map<int, int> seenTimes = computationContext.seenTimes;final List<PersonEntity> persons = preparedData.persons;final String? currentUserEmail = computationContext.currentUserEmail;- final List<City> cities = computationContext.cities;+ final CitySearchIndex citySearchIndex =+ computationContext.citySearchIndex;final Map<int, List<FaceWithoutEmbedding>> fileIdToFaces =computationContext.fileIdToFaces;final Map<String, int> clusterIdToFaceCount =@@ -978,8 +990,8 @@ class SmartMemoriesService {dev.log('ML disabled, skipping people memories $t');}- final (tripMemories, bases) = await _getTripsResults(- tripSourceFiles: fullSourceFiles,+ final (tripMemories, bases) = await _surfaceTrips(+ tripAnalysis: preparedData.tripAnalysis,allFileIdsToFile: allFileIdsToFile,currentTime: now,shownTrips: oldCache.tripsShownLogs,@@ -992,7 +1004,7 @@ class SmartMemoriesService {faceIDsToPersonID: faceIDsToPersonID,fileIDToImageEmbedding: fileIDToImageEmbedding,clipPositiveTextVector: clipPositiveTextVector,- cities: cities,+ citySearchIndex: citySearchIndex,);_markUsedMemories(usedMemoryFileIds,@@ -1245,8 +1257,8 @@ class SmartMemoriesService {);}- static Future<(List<TripMemory>, List<BaseLocation>)> _getTripsResults({- required Iterable<EnteFile> tripSourceFiles,+ static Future<(List<TripMemory>, List<BaseLocation>)> _surfaceTrips({+ required TripMemoriesAnalysis tripAnalysis,required Map<int, EnteFile> allFileIdsToFile,required DateTime currentTime,required List<TripsShownLog> shownTrips,@@ -1259,10 +1271,10 @@ class SmartMemoriesService {required Map<String, String> faceIDsToPersonID,required Map<int, EmbeddingVector> fileIDToImageEmbedding,required Vector clipPositiveTextVector,- required List<City> cities,+ required CitySearchIndex citySearchIndex,}) async {- return TripMemoriesCalculatorV2.compute(- tripSourceFiles,+ return TripMemoriesCalculatorV2.surface(+ tripAnalysis,allFileIdsToFile,currentTime,shownTrips,@@ -1275,7 +1287,7 @@ class SmartMemoriesService {faceIDsToPersonID: faceIDsToPersonID,fileIDToImageEmbedding: fileIDToImageEmbedding,clipPositiveTextVector: clipPositiveTextVector,- cities: cities,+ citySearchIndex: citySearchIndex,);}@@ -1414,11 +1426,11 @@ class SmartMemoriesService {static String? _tryFindLocationName(List<Memory> memories,- List<City> cities, {+ CitySearchIndex citySearchIndex, {bool base = false,Set<String> excludedCountryNames = const {},}) {- final locationContext = _getLocationNameContext(memories, cities);+ final locationContext = _getLocationNameContext(memories, citySearchIndex);if (locationContext == null) return null;final files = locationContext.files;@@ -1437,8 +1449,11 @@ class SmartMemoriesService {return null;}- static String? _tryFindCountryName(List<Memory> memories, List<City> cities) {- final locationContext = _getLocationNameContext(memories, cities);+ static String? _tryFindCountryName(+ List<Memory> memories,+ CitySearchIndex citySearchIndex,+ ) {+ final locationContext = _getLocationNameContext(memories, citySearchIndex);return locationContext?.biggestPlace.country;}@@ -1447,13 +1462,12 @@ class SmartMemoriesService {Map<City, List<EnteFile>> results,City biggestPlace,})?- _getLocationNameContext(List<Memory> memories, List<City> cities) {+ _getLocationNameContext(+ List<Memory> memories,+ CitySearchIndex citySearchIndex,+ ) {final files = Memory.filesFromMemories(memories);- final results = getCityResults({- "query": '',- "cities": cities,- "files": files,- });+ final results = getCityResults(citySearchIndex.searchArgs(files));final List<City> sortedByResultCount = results.keys.toList()..sort((a, b) => results[b]!.length.compareTo(results[a]!.length));if (sortedByResultCount.isEmpty) return null;diff --git a/mobile/apps/photos/lib/services/smart_memories_trip_calculator_v2.dart b/mobile/apps/photos/lib/services/smart_memories_trip_calculator_v2.dartindex 92b76a8e22..0940f3fb31 100644--- a/mobile/apps/photos/lib/services/smart_memories_trip_calculator_v2.dart+++ b/mobile/apps/photos/lib/services/smart_memories_trip_calculator_v2.dart@@ -84,6 +84,18 @@ class _TripSurfaceCandidate {});}+class TripMemoriesAnalysis {+ final List<BaseLocation> baseLocations;+ final Set<String> baseCountriesToExclude;+ final List<TripMemory> tripCandidates;++ const TripMemoriesAnalysis({+ required this.baseLocations,+ required this.baseCountriesToExclude,+ required this.tripCandidates,+ });+}+class TripMemoriesCalculatorV2 {static const _maxTemporalGapDays = 15;@@ -133,15 +145,47 @@ class TripMemoriesCalculatorV2 {required Map<String, String> faceIDsToPersonID,required Map<int, EmbeddingVector> fileIDToImageEmbedding,required Vector clipPositiveTextVector,- required List<City> cities,+ required CitySearchIndex citySearchIndex,}) async {- if (allFiles.isEmpty) return (<TripMemory>[], <BaseLocation>[]);+ final analysis = analyze(+ allFiles,+ allFileIdsToFile,+ isLocalGalleryMode: isLocalGalleryMode,+ seenTimes: seenTimes,+ citySearchIndex: citySearchIndex,+ );+ return surface(+ analysis,+ allFileIdsToFile,+ currentTime,+ shownTrips,+ surfaceAll: surfaceAll,+ cachedTripMemories: cachedTripMemories,+ isLocalGalleryMode: isLocalGalleryMode,+ mlEnabled: mlEnabled,+ seenTimes: seenTimes,+ fileIdToFaces: fileIdToFaces,+ faceIDsToPersonID: faceIDsToPersonID,+ fileIDToImageEmbedding: fileIDToImageEmbedding,+ clipPositiveTextVector: clipPositiveTextVector,+ citySearchIndex: citySearchIndex,+ );+ }- final nowInMicroseconds = currentTime.microsecondsSinceEpoch;- final windowEnd = currentTime- .add(kMemoriesUpdateFrequency)- .microsecondsSinceEpoch;- final cutOffTime = currentTime.subtract(const Duration(days: 365));+ static TripMemoriesAnalysis analyze(+ Iterable<EnteFile> allFiles,+ Map<int, EnteFile> allFileIdsToFile, {+ required bool isLocalGalleryMode,+ required Map<int, int> seenTimes,+ required CitySearchIndex citySearchIndex,+ }) {+ if (allFiles.isEmpty) {+ return const TripMemoriesAnalysis(+ baseLocations: <BaseLocation>[],+ baseCountriesToExclude: <String>{},+ tripCandidates: <TripMemory>[],+ );+ }final baseLocations = _detectBaseLocations(allFiles,@@ -150,7 +194,7 @@ class TripMemoriesCalculatorV2 {final baseCountriesToExclude = _baseCountriesToExcludeFromTripTitles(baseLocations,allFileIdsToFile,- cities,+ citySearchIndex,seenTimes,);@@ -158,7 +202,11 @@ class TripMemoriesCalculatorV2 {..sort((a, b) => a.creationTime!.compareTo(b.creationTime!));if (filesWithLocation.isEmpty) {- return (<TripMemory>[], baseLocations);+ return TripMemoriesAnalysis(+ baseLocations: baseLocations,+ baseCountriesToExclude: baseCountriesToExclude,+ tripCandidates: const <TripMemory>[],+ );}// Returning to a base separates otherwise nearby trips.@@ -167,7 +215,7 @@ class TripMemoriesCalculatorV2 {baseLocations,);- final tripCandidates = <TripMemory>[];+ final rawTripCandidates = <TripMemory>[];for (final block in temporalBlocks) {var spatialClusters = _clusterByLocation(block,@@ -182,7 +230,7 @@ class TripMemoriesCalculatorV2 {if (!_isValidTrip(files, location, baseLocations)) continue;- tripCandidates.add(+ rawTripCandidates.add(TripMemory(Memory.fromFiles(files, seenTimes),0,@@ -200,15 +248,48 @@ class TripMemoriesCalculatorV2 {}}- final mergedTrips = _mergeNearbyTrips(tripCandidates);+ final mergedTrips = _mergeNearbyTrips(rawTripCandidates);++ final tripCandidates = mergedTrips+ .where((trip) => trip.memories.length >= _minTripPhotos)+ .map(_ensureTripKey)+ .toList();- final validTrips = mergedTrips+ return TripMemoriesAnalysis(+ baseLocations: baseLocations,+ baseCountriesToExclude: baseCountriesToExclude,+ tripCandidates: tripCandidates,+ );+ }++ static Future<(List<TripMemory>, List<BaseLocation>)> surface(+ TripMemoriesAnalysis analysis,+ Map<int, EnteFile> allFileIdsToFile,+ DateTime currentTime,+ List<TripsShownLog> shownTrips, {+ bool surfaceAll = false,+ required Iterable<ToShowMemory> cachedTripMemories,+ required bool isLocalGalleryMode,+ required bool mlEnabled,+ required Map<int, int> seenTimes,+ required Map<int, List<FaceWithoutEmbedding>> fileIdToFaces,+ required Map<String, String> faceIDsToPersonID,+ required Map<int, EmbeddingVector> fileIDToImageEmbedding,+ required Vector clipPositiveTextVector,+ required CitySearchIndex citySearchIndex,+ }) async {+ final nowInMicroseconds = currentTime.microsecondsSinceEpoch;+ final windowEnd = currentTime+ .add(kMemoriesUpdateFrequency)+ .microsecondsSinceEpoch;+ final cutOffTime = currentTime.subtract(const Duration(days: 365));+ final baseLocations = analysis.baseLocations;+ final baseCountriesToExclude = analysis.baseCountriesToExclude;+ final validTrips = analysis.tripCandidates.where(- (t) =>- t.memories.length >= _minTripPhotos &&- t.averageCreationTime() < cutOffTime.microsecondsSinceEpoch,+ (trip) =>+ trip.averageCreationTime() < cutOffTime.microsecondsSinceEpoch,)- .map(_ensureTripKey).toList();final List<TripMemory> memoryResults = [];@@ -229,7 +310,7 @@ class TripMemoriesCalculatorV2 {faceIDsToPersonID: faceIDsToPersonID,fileIDToImageEmbedding: fileIDToImageEmbedding,clipPositiveTextVector: clipPositiveTextVector,- cities: cities,+ citySearchIndex: citySearchIndex,);}@@ -249,7 +330,7 @@ class TripMemoriesCalculatorV2 {faceIDsToPersonID: faceIDsToPersonID,fileIDToImageEmbedding: fileIDToImageEmbedding,clipPositiveTextVector: clipPositiveTextVector,- cities: cities,+ citySearchIndex: citySearchIndex,);}@@ -848,7 +929,7 @@ class TripMemoriesCalculatorV2 {required Map<String, String> faceIDsToPersonID,required Map<int, EmbeddingVector> fileIDToImageEmbedding,required Vector clipPositiveTextVector,- required List<City> cities,+ required CitySearchIndex citySearchIndex,}) async {for (final baseLocation in baseLocations) {String name = "Base (${baseLocation.isCurrentBase ? 'current' : 'old'})";@@ -857,7 +938,7 @@ class TripMemoriesCalculatorV2 {.toList();final String? locationName = SmartMemoriesService._tryFindLocationName(Memory.fromFiles(files, seenTimes),- cities,+ citySearchIndex,base: true,);if (locationName != null) {@@ -880,7 +961,7 @@ class TripMemoriesCalculatorV2 {).year;final String? locationName = SmartMemoriesService._tryFindLocationName(trip.memories,- cities,+ citySearchIndex,excludedCountryNames: baseCountriesToExclude,);final photoSelection = await SmartMemoriesService._bestSelection(@@ -921,7 +1002,7 @@ class TripMemoriesCalculatorV2 {required Map<String, String> faceIDsToPersonID,required Map<int, EmbeddingVector> fileIDToImageEmbedding,required Vector clipPositiveTextVector,- required List<City> cities,+ required CitySearchIndex citySearchIndex,}) async {final activeTripIdentityKeys = _activeCachedTripIdentityKeys(cachedTripMemories,@@ -1029,7 +1110,7 @@ class TripMemoriesCalculatorV2 {).year;final String? locationName = SmartMemoriesService._tryFindLocationName(trip.memories,- cities,+ citySearchIndex,excludedCountryNames: baseCountriesToExclude,);final photoSelection = await SmartMemoriesService._bestSelection(@@ -1057,7 +1138,7 @@ class TripMemoriesCalculatorV2 {static Set<String> _baseCountriesToExcludeFromTripTitles(List<BaseLocation> baseLocations,Map<int, EnteFile> allFileIdsToFile,- List<City> cities,+ CitySearchIndex citySearchIndex,Map<int, int> seenTimes,) {final excludedCountries = <String>{};@@ -1072,7 +1153,7 @@ class TripMemoriesCalculatorV2 {final countryName = SmartMemoriesService._tryFindCountryName(Memory.fromFiles(files, seenTimes),- cities,+ citySearchIndex,);if (countryName == null) {continue;diff --git a/mobile/apps/photos/test/services/memories/smart_memories_source_selection_test.dart b/mobile/apps/photos/test/services/memories/smart_memories_source_selection_test.dartindex 26f55a45bf..7baf323fd9 100644--- a/mobile/apps/photos/test/services/memories/smart_memories_source_selection_test.dart+++ b/mobile/apps/photos/test/services/memories/smart_memories_source_selection_test.dart@@ -54,6 +54,22 @@ void main() {final allFileIdsToFile = {for (final file in fullSourceFiles) file.uploadedFileID!: file,};+ final citySearchIndex = CitySearchIndex(+ cities: [+ City.fromMap({+ "city": "Paris",+ "country": "France",+ "lat": tripLocation.latitude,+ "lng": tripLocation.longitude,+ }),+ ],+ nodes: const [+ [0, -1, -1, 0],+ ],+ root: 0,+ maxLatDelta: 1,+ maxLngDelta: 1,+ );final (fullTrips, _) = await TripMemoriesCalculatorV2.compute(fullSourceFiles,@@ -69,7 +85,7 @@ void main() {faceIDsToPersonID: const <String, String>{},fileIDToImageEmbedding: const <int, EmbeddingVector>{},clipPositiveTextVector: _positiveTextVector,- cities: const <City>[],+ citySearchIndex: citySearchIndex,);final (remainingTrips, _) = await TripMemoriesCalculatorV2.compute(@@ -86,11 +102,12 @@ void main() {faceIDsToPersonID: const <String, String>{},fileIDToImageEmbedding: const <int, EmbeddingVector>{},clipPositiveTextVector: _positiveTextVector,- cities: const <City>[],+ citySearchIndex: citySearchIndex,);expect(fullTrips, hasLength(1));expect(fullTrips.first.memories, hasLength(10));+ expect(fullTrips.first.locationName, "Paris");expect(remainingTrips, isEmpty);},);