π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Move photo map clustering to Rust
Neeraj Gupta265cfcf081a0
diff --git a/mobile/apps/photos/lib/ui/map/map_isolate.dart b/mobile/apps/photos/lib/ui/map/map_isolate.dartdeleted file mode 100644index 21b36139aa..0000000000--- a/mobile/apps/photos/lib/ui/map/map_isolate.dart+++ /dev/null@@ -1,253 +0,0 @@-import "dart:isolate";-import "dart:math";--import "package:flutter_map/flutter_map.dart";--class MapPoint {- final int imageIndex;- final double latitude;- final double longitude;-- const MapPoint({- required this.imageIndex,- required this.latitude,- required this.longitude,- });-}--class MapMarkerGroup {- final int imageIndex;- final double latitude;- final double longitude;- final int imageCount;- final double north;- final double south;- final double east;- final double west;-- const MapMarkerGroup({- required this.imageIndex,- required this.latitude,- required this.longitude,- required this.imageCount,- required this.north,- required this.south,- required this.east,- required this.west,- });-- LatLngBounds get bounds =>- LatLngBounds.unsafe(north: north, south: south, east: east, west: west);-}--class _ProjectedMarkerGroup {- final int imageIndex;- final double latitude;- final double longitude;- final double x;- final double y;- int imageCount = 1;- double north;- double south;- double east;- double west;-- _ProjectedMarkerGroup({- required this.imageIndex,- required this.latitude,- required this.longitude,- required this.x,- required this.y,- }) : north = latitude,- south = latitude,- east = longitude,- west = longitude;-- MapMarkerGroup toMarkerGroup() => MapMarkerGroup(- imageIndex: imageIndex,- latitude: latitude,- longitude: longitude,- imageCount: imageCount,- north: north,- south: south,- east: east,- west: west,- );-- void add(MapPoint point) {- imageCount++;- north = max(north, point.latitude);- south = min(south, point.latitude);- east = max(east, point.longitude);- west = min(west, point.longitude);- }-}--class _ProjectedMapPoint {- final MapPoint point;- final double x;- final double y;-- const _ProjectedMapPoint({- required this.point,- required this.x,- required this.y,- });-}--class MapWorkerInit {- final List<MapPoint> points;- final SendPort sendPort;-- const MapWorkerInit({required this.points, required this.sendPort});-}--class MapViewportRequest {- final int id;- final LatLngBounds bounds;- final double zoom;- final double markerMinX;- final double markerMinY;- final double markerMaxX;- final double markerMaxY;-- const MapViewportRequest({- required this.id,- required this.bounds,- required this.zoom,- required this.markerMinX,- required this.markerMinY,- required this.markerMaxX,- required this.markerMaxY,- });-}--class MapViewportResult {- final int id;- final List<int> visibleImageIndexes;- final List<MapMarkerGroup> markerGroups;-- const MapViewportResult({- required this.id,- required this.visibleImageIndexes,- required this.markerGroups,- });-}--const double _minimumMarkerDistancePixels = 100;-const double _webMercatorTileSize = 256;-const double _mercatorLatitudeLimit = 85.05112878;--void mapWorker(MapWorkerInit init) {- final receivePort = ReceivePort();- init.sendPort.send(receivePort.sendPort);- receivePort.listen((message) {- if (message is! MapViewportRequest) {- return;- }-- final visibleIndexes = <int>[];- final visiblePoints = <_ProjectedMapPoint>[];- final bufferedPoints = <_ProjectedMapPoint>[];- final groupsByCell = <(int, int), List<_ProjectedMarkerGroup>>{};- final groups = <_ProjectedMarkerGroup>[];- final scale = _webMercatorTileSize * pow(2, message.zoom).toDouble();-- for (final point in init.points) {- final isVisible =- point.longitude >= message.bounds.west &&- point.longitude <= message.bounds.east &&- point.latitude >= message.bounds.south &&- point.latitude <= message.bounds.north;- if (isVisible) {- visibleIndexes.add(point.imageIndex);- }-- final latitude = point.latitude.clamp(- -_mercatorLatitudeLimit,- _mercatorLatitudeLimit,- );- final sinLatitude = sin(latitude * pi / 180);- final x = (point.longitude + 180) / 360 * scale;- final y =- (0.5 - log((1 + sinLatitude) / (1 - sinLatitude)) / (4 * pi)) * scale;- if (x < message.markerMinX ||- x > message.markerMaxX ||- y < message.markerMinY ||- y > message.markerMaxY) {- continue;- }-- final projectedPoint = _ProjectedMapPoint(point: point, x: x, y: y);- (isVisible ? visiblePoints : bufferedPoints).add(projectedPoint);- }-- void addToMarkerGroup(_ProjectedMapPoint projectedPoint) {- final point = projectedPoint.point;- final x = projectedPoint.x;- final y = projectedPoint.y;- final cell = (- (x / _minimumMarkerDistancePixels).floor(),- (y / _minimumMarkerDistancePixels).floor(),- );- _ProjectedMarkerGroup? closestGroup;- var closestDistanceSquared = double.infinity;-- for (var xOffset = -1; xOffset <= 1; xOffset++) {- for (var yOffset = -1; yOffset <= 1; yOffset++) {- final nearbyGroups =- groupsByCell[(cell.$1 + xOffset, cell.$2 + yOffset)];- if (nearbyGroups == null) {- continue;- }-- for (final group in nearbyGroups) {- final xDistance = x - group.x;- final yDistance = y - group.y;- final distanceSquared =- xDistance * xDistance + yDistance * yDistance;- if (distanceSquared <=- _minimumMarkerDistancePixels *- _minimumMarkerDistancePixels &&- distanceSquared < closestDistanceSquared) {- closestGroup = group;- closestDistanceSquared = distanceSquared;- }- }- }- }-- if (closestGroup != null) {- closestGroup.add(point);- return;- }-- final group = _ProjectedMarkerGroup(- imageIndex: point.imageIndex,- latitude: point.latitude,- longitude: point.longitude,- x: x,- y: y,- );- groups.add(group);- groupsByCell.putIfAbsent(cell, () => []).add(group);- }-- for (final point in visiblePoints) {- addToMarkerGroup(point);- }- for (final point in bufferedPoints) {- addToMarkerGroup(point);- }-- init.sendPort.send(- MapViewportResult(- id: message.id,- visibleImageIndexes: visibleIndexes,- markerGroups: groups- .map((group) => group.toMarkerGroup())- .toList(growable: false),- ),- );- });-}diff --git a/mobile/apps/photos/lib/ui/map/map_screen.dart b/mobile/apps/photos/lib/ui/map/map_screen.dartindex 99f0c21a54..841d7ff71d 100644--- a/mobile/apps/photos/lib/ui/map/map_screen.dart+++ b/mobile/apps/photos/lib/ui/map/map_screen.dart@@ -1,5 +1,4 @@import "dart:async";-import "dart:isolate";import "dart:math";import "package:computer/computer.dart";@@ -13,9 +12,9 @@ import "package:latlong2/latlong.dart";import "package:logging/logging.dart";import 'package:photos/models/file/file.dart';import "package:photos/models/location/location.dart";+import "package:photos/src/rust/api/map_cluster_api.dart";import "package:photos/theme/ente_theme.dart";import "package:photos/ui/map/image_marker.dart";-import "package:photos/ui/map/map_isolate.dart";import "package:photos/ui/map/map_pull_up_gallery.dart";import "package:photos/ui/map/map_view.dart";import "package:photos/ui/notification/toast.dart";@@ -50,10 +49,7 @@ class _MapScreenState extends State<MapScreen> {double minZoom = 2.8;LatLng? center;final Logger _logger = Logger("_MapScreenState");- ReceivePort? _mapWorkerReceivePort;- StreamSubscription? _mapWorkerSubscription;- SendPort? _mapWorkerSendPort;- Isolate? _mapWorkerIsolate;+ MapClusterer? _mapClusterer;static const bottomSheetDraggableAreaHeight = 32.0;List<int>? _previousVisibleIndexes;int _viewportRequestID = 0;@@ -72,9 +68,6 @@ class _MapScreenState extends State<MapScreen> {@overridevoid dispose() {_viewportDebouncer.cancelDebounceTimer();- unawaited(_mapWorkerSubscription?.cancel());- _mapWorkerReceivePort?.close();- _mapWorkerIsolate?.kill();unawaited(visibleImages.close());super.dispose();}@@ -128,7 +121,7 @@ class _MapScreenState extends State<MapScreen> {);}- await _startMapWorker(mapPoints);+ _mapClusterer = await createMapClusterer(points: mapPoints);if (!mounted) return;setState(() {hasLocationData = true;@@ -140,37 +133,6 @@ class _MapScreenState extends State<MapScreen> {});}- Future<void> _startMapWorker(List<MapPoint> mapPoints) async {- final receivePort = ReceivePort();- final sendPortCompleter = Completer<SendPort>();- final subscription = receivePort.listen((message) {- if (message is SendPort) {- if (!sendPortCompleter.isCompleted) {- sendPortCompleter.complete(message);- }- } else if (message is MapViewportResult) {- _applyViewportResult(message);- }- });- final isolate = await Isolate.spawn(- mapWorker,- MapWorkerInit(points: mapPoints, sendPort: receivePort.sendPort),- );- final sendPort = await sendPortCompleter.future;-- if (!mounted) {- await subscription.cancel();- receivePort.close();- isolate.kill();- return;- }-- _mapWorkerReceivePort = receivePort;- _mapWorkerSubscription = subscription;- _mapWorkerSendPort = sendPort;- _mapWorkerIsolate = isolate;- }-void _updateViewport(MapCamera camera) {final requestID = ++_viewportRequestID;_viewportDebouncer.run(() async {@@ -179,24 +141,44 @@ class _MapScreenState extends State<MapScreen> {}void _sendViewportRequest(MapCamera camera, int requestID) {- final sendPort = _mapWorkerSendPort;- if (sendPort == null) return;+ final clusterer = _mapClusterer;+ if (clusterer == null) return;final pixelBounds = camera.pixelBounds;- sendPort.send(- MapViewportRequest(- id: requestID,- bounds: camera.visibleBounds,- zoom: camera.zoom,- markerMinX: pixelBounds.left - _markerViewportPaddingPixels,- markerMinY: pixelBounds.top - _markerViewportPaddingPixels,- markerMaxX: pixelBounds.right + _markerViewportPaddingPixels,- markerMaxY: pixelBounds.bottom + _markerViewportPaddingPixels,+ final bounds = camera.visibleBounds;+ unawaited(+ _clusterViewport(+ clusterer,+ MapViewport(+ north: bounds.north,+ south: bounds.south,+ east: bounds.east,+ west: bounds.west,+ zoom: camera.zoom,+ markerMinX: pixelBounds.left - _markerViewportPaddingPixels,+ markerMinY: pixelBounds.top - _markerViewportPaddingPixels,+ markerMaxX: pixelBounds.right + _markerViewportPaddingPixels,+ markerMaxY: pixelBounds.bottom + _markerViewportPaddingPixels,+ ),+ requestID,),);}- void _applyViewportResult(MapViewportResult result) {- if (!mounted || result.id != _viewportRequestID) return;+ Future<void> _clusterViewport(+ MapClusterer clusterer,+ MapViewport viewport,+ int requestID,+ ) async {+ try {+ final result = await clusterer.cluster(viewport: viewport);+ _applyViewportResult(result, requestID);+ } catch (e, s) {+ _logger.severe("Rust map clustering failed", e, s);+ }+ }++ void _applyViewportResult(MapViewportResult result, int requestID) {+ if (!mounted || requestID != _viewportRequestID) return;final visibleIndexes = result.visibleImageIndexes;if (!_sameIndexes(visibleIndexes, _previousVisibleIndexes)) {@@ -218,7 +200,14 @@ class _MapScreenState extends State<MapScreen> {latitude: group.latitude,longitude: group.longitude,imageCount: group.imageCount,- clusterBounds: group.imageCount > 1 ? group.bounds : null,+ clusterBounds: group.imageCount > 1+ ? LatLngBounds.unsafe(+ north: group.north,+ south: group.south,+ east: group.east,+ west: group.west,+ )+ : null,),).toList(growable: false);diff --git a/rust/bindings/frb/photos/src/api/map_cluster_api.rs b/rust/bindings/frb/photos/src/api/map_cluster_api.rsnew file mode 100644index 0000000000..e3272cb09d--- /dev/null+++ b/rust/bindings/frb/photos/src/api/map_cluster_api.rs@@ -0,0 +1,99 @@+use ente_photos::map_cluster as core;+use flutter_rust_bridge::frb;++pub struct MapPoint {+ pub image_index: u32,+ pub latitude: f64,+ pub longitude: f64,+}++impl From<MapPoint> for core::MapPoint {+ fn from(point: MapPoint) -> Self {+ Self {+ image_index: point.image_index,+ latitude: point.latitude,+ longitude: point.longitude,+ }+ }+}++pub struct MapViewport {+ pub north: f64,+ pub south: f64,+ pub east: f64,+ pub west: f64,+ pub zoom: f64,+ pub marker_min_x: f64,+ pub marker_min_y: f64,+ pub marker_max_x: f64,+ pub marker_max_y: f64,+}++impl From<MapViewport> for core::MapViewport {+ fn from(viewport: MapViewport) -> Self {+ Self {+ north: viewport.north,+ south: viewport.south,+ east: viewport.east,+ west: viewport.west,+ zoom: viewport.zoom,+ marker_min_x: viewport.marker_min_x,+ marker_min_y: viewport.marker_min_y,+ marker_max_x: viewport.marker_max_x,+ marker_max_y: viewport.marker_max_y,+ }+ }+}++pub struct MapMarkerGroup {+ pub image_index: u32,+ pub latitude: f64,+ pub longitude: f64,+ pub image_count: u32,+ pub north: f64,+ pub south: f64,+ pub east: f64,+ pub west: f64,+}++impl From<core::MapMarkerGroup> for MapMarkerGroup {+ fn from(group: core::MapMarkerGroup) -> Self {+ Self {+ image_index: group.image_index,+ latitude: group.latitude,+ longitude: group.longitude,+ image_count: group.image_count,+ north: group.north,+ south: group.south,+ east: group.east,+ west: group.west,+ }+ }+}++pub struct MapViewportResult {+ pub visible_image_indexes: Vec<u32>,+ pub marker_groups: Vec<MapMarkerGroup>,+}++#[frb(opaque)]+pub struct MapClusterer {+ inner: core::MapClusterer,+}++pub fn create_map_clusterer(points: Vec<MapPoint>) -> MapClusterer {+ MapClusterer {+ inner: core::MapClusterer::new(points.into_iter().map(Into::into).collect()),+ }+}++impl MapClusterer {+ pub fn cluster(&self, viewport: MapViewport) -> MapViewportResult {+ let result = self.inner.cluster(viewport.into());++ MapViewportResult {+ visible_image_indexes: result.visible_image_indexes,+ marker_groups: result.marker_groups.into_iter().map(Into::into).collect(),+ }+ }+}diff --git a/rust/bindings/frb/photos/src/api/mod.rs b/rust/bindings/frb/photos/src/api/mod.rsindex 69c9ba0d9c..f5886570e2 100644--- a/rust/bindings/frb/photos/src/api/mod.rs+++ b/rust/bindings/frb/photos/src/api/mod.rs@@ -4,6 +4,7 @@ pub mod image_processing_api;pub mod init;#[cfg(any(feature = "flutter", frb_expand))]pub mod log;+pub mod map_cluster_api;pub mod ml_indexing_api;pub mod motion_photo_api;pub mod urls;diff --git a/rust/crates/photos/src/lib.rs b/rust/crates/photos/src/lib.rsindex 156da6c8ed..c36484458a 100644--- a/rust/crates/photos/src/lib.rs+++ b/rust/crates/photos/src/lib.rs@@ -1,3 +1,4 @@+pub mod map_cluster;pub mod motion_photo;pub use motion_photo::{diff --git a/rust/crates/photos/src/map_cluster.rs b/rust/crates/photos/src/map_cluster.rsnew file mode 100644index 0000000000..706e33c9f1--- /dev/null+++ b/rust/crates/photos/src/map_cluster.rs@@ -0,0 +1,211 @@+use std::collections::HashMap;+use std::f64::consts::PI;++const MINIMUM_MARKER_DISTANCE_PIXELS: f64 = 100.0;+const WEB_MERCATOR_TILE_SIZE: f64 = 256.0;+const MERCATOR_LATITUDE_LIMIT: f64 = 85.051_128_78;++#[derive(Clone, Copy)]+pub struct MapPoint {+ pub image_index: u32,+ pub latitude: f64,+ pub longitude: f64,+}++pub struct MapViewport {+ pub north: f64,+ pub south: f64,+ pub east: f64,+ pub west: f64,+ pub zoom: f64,+ pub marker_min_x: f64,+ pub marker_min_y: f64,+ pub marker_max_x: f64,+ pub marker_max_y: f64,+}++pub struct MapMarkerGroup {+ pub image_index: u32,+ pub latitude: f64,+ pub longitude: f64,+ pub image_count: u32,+ pub north: f64,+ pub south: f64,+ pub east: f64,+ pub west: f64,+}++pub struct MapViewportResult {+ pub visible_image_indexes: Vec<u32>,+ pub marker_groups: Vec<MapMarkerGroup>,+}++pub struct MapClusterer {+ points: Vec<MapPoint>,+}++impl MapClusterer {+ pub fn new(points: Vec<MapPoint>) -> Self {+ Self { points }+ }++ pub fn cluster(&self, viewport: MapViewport) -> MapViewportResult {+ let mut visible_image_indexes = Vec::new();+ let mut visible_points = Vec::new();+ let mut buffered_points = Vec::new();+ let scale = WEB_MERCATOR_TILE_SIZE * 2.0_f64.powf(viewport.zoom);++ for point in &self.points {+ let is_visible = point.longitude >= viewport.west+ && point.longitude <= viewport.east+ && point.latitude >= viewport.south+ && point.latitude <= viewport.north;+ if is_visible {+ visible_image_indexes.push(point.image_index);+ }++ let latitude = point+ .latitude+ .clamp(-MERCATOR_LATITUDE_LIMIT, MERCATOR_LATITUDE_LIMIT);+ let sin_latitude = (latitude * PI / 180.0).sin();+ let x = (point.longitude + 180.0) / 360.0 * scale;+ let y = (0.5 - ((1.0 + sin_latitude) / (1.0 - sin_latitude)).ln() / (4.0 * PI)) * scale;+ if x < viewport.marker_min_x+ || x > viewport.marker_max_x+ || y < viewport.marker_min_y+ || y > viewport.marker_max_y+ {+ continue;+ }++ let projected_point = ProjectedMapPoint {+ point: *point,+ x,+ y,+ };+ if is_visible {+ visible_points.push(projected_point);+ } else {+ buffered_points.push(projected_point);+ }+ }++ let mut groups_by_cell = HashMap::<(i64, i64), Vec<usize>>::new();+ let mut groups = Vec::<ProjectedMarkerGroup>::new();+ for point in visible_points.iter().chain(&buffered_points) {+ add_to_marker_group(*point, &mut groups_by_cell, &mut groups);+ }++ MapViewportResult {+ visible_image_indexes,+ marker_groups: groups+ .into_iter()+ .map(ProjectedMarkerGroup::into_marker_group)+ .collect(),+ }+ }+}++#[derive(Clone, Copy)]+struct ProjectedMapPoint {+ point: MapPoint,+ x: f64,+ y: f64,+}++struct ProjectedMarkerGroup {+ image_index: u32,+ latitude: f64,+ longitude: f64,+ x: f64,+ y: f64,+ image_count: u32,+ north: f64,+ south: f64,+ east: f64,+ west: f64,+}++impl ProjectedMarkerGroup {+ fn new(point: ProjectedMapPoint) -> Self {+ Self {+ image_index: point.point.image_index,+ latitude: point.point.latitude,+ longitude: point.point.longitude,+ x: point.x,+ y: point.y,+ image_count: 1,+ north: point.point.latitude,+ south: point.point.latitude,+ east: point.point.longitude,+ west: point.point.longitude,+ }+ }++ fn add(&mut self, point: MapPoint) {+ self.image_count += 1;+ self.north = self.north.max(point.latitude);+ self.south = self.south.min(point.latitude);+ self.east = self.east.max(point.longitude);+ self.west = self.west.min(point.longitude);+ }++ fn into_marker_group(self) -> MapMarkerGroup {+ MapMarkerGroup {+ image_index: self.image_index,+ latitude: self.latitude,+ longitude: self.longitude,+ image_count: self.image_count,+ north: self.north,+ south: self.south,+ east: self.east,+ west: self.west,+ }+ }+}++fn add_to_marker_group(+ point: ProjectedMapPoint,+ groups_by_cell: &mut HashMap<(i64, i64), Vec<usize>>,+ groups: &mut Vec<ProjectedMarkerGroup>,+) {+ let cell = (+ (point.x / MINIMUM_MARKER_DISTANCE_PIXELS).floor() as i64,+ (point.y / MINIMUM_MARKER_DISTANCE_PIXELS).floor() as i64,+ );+ let mut closest_group_index = None;+ let mut closest_distance_squared = f64::INFINITY;++ for x_offset in -1..=1 {+ for y_offset in -1..=1 {+ let Some(nearby_group_indexes) =+ groups_by_cell.get(&(cell.0 + x_offset, cell.1 + y_offset))+ else {+ continue;+ };++ for group_index in nearby_group_indexes {+ let group = &groups[*group_index];+ let x_distance = point.x - group.x;+ let y_distance = point.y - group.y;+ let distance_squared = x_distance * x_distance + y_distance * y_distance;+ if distance_squared+ <= MINIMUM_MARKER_DISTANCE_PIXELS * MINIMUM_MARKER_DISTANCE_PIXELS+ && distance_squared < closest_distance_squared+ {+ closest_group_index = Some(*group_index);+ closest_distance_squared = distance_squared;+ }+ }+ }+ }++ if let Some(group_index) = closest_group_index {+ groups[group_index].add(point.point);+ return;+ }++ let group_index = groups.len();+ groups.push(ProjectedMarkerGroup::new(point));+ groups_by_cell.entry(cell).or_default().push(group_index);+}