Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Move photo map clustering to Rust

Neeraj Gupta265cfcf081a0
diff --git a/mobile/apps/photos/lib/ui/map/map_isolate.dart b/mobile/apps/photos/lib/ui/map/map_isolate.dart
deleted file mode 100644
index 21b36139aa..0000000000
--- a/mobile/apps/photos/lib/ui/map/map_isolate.dart
+++ /dev/null
@@ -1,253 +0,0 @@
-import "dart:isolate";
-import "dart:math";
-
-import "package:flutter_map/flutter_map.dart";
-
-class MapPoint {
- final int imageIndex;
- final double latitude;
- final double longitude;
-
- const MapPoint({
- required this.imageIndex,
- required this.latitude,
- required this.longitude,
- });
-}
-
-class MapMarkerGroup {
- final int imageIndex;
- final double latitude;
- final double longitude;
- final int imageCount;
- final double north;
- final double south;
- final double east;
- final double west;
-
- const MapMarkerGroup({
- required this.imageIndex,
- required this.latitude,
- required this.longitude,
- required this.imageCount,
- required this.north,
- required this.south,
- required this.east,
- required this.west,
- });
-
- LatLngBounds get bounds =>
- LatLngBounds.unsafe(north: north, south: south, east: east, west: west);
-}
-
-class _ProjectedMarkerGroup {
- final int imageIndex;
- final double latitude;
- final double longitude;
- final double x;
- final double y;
- int imageCount = 1;
- double north;
- double south;
- double east;
- double west;
-
- _ProjectedMarkerGroup({
- required this.imageIndex,
- required this.latitude,
- required this.longitude,
- required this.x,
- required this.y,
- }) : north = latitude,
- south = latitude,
- east = longitude,
- west = longitude;
-
- MapMarkerGroup toMarkerGroup() => MapMarkerGroup(
- imageIndex: imageIndex,
- latitude: latitude,
- longitude: longitude,
- imageCount: imageCount,
- north: north,
- south: south,
- east: east,
- west: west,
- );
-
- void add(MapPoint point) {
- imageCount++;
- north = max(north, point.latitude);
- south = min(south, point.latitude);
- east = max(east, point.longitude);
- west = min(west, point.longitude);
- }
-}
-
-class _ProjectedMapPoint {
- final MapPoint point;
- final double x;
- final double y;
-
- const _ProjectedMapPoint({
- required this.point,
- required this.x,
- required this.y,
- });
-}
-
-class MapWorkerInit {
- final List<MapPoint> points;
- final SendPort sendPort;
-
- const MapWorkerInit({required this.points, required this.sendPort});
-}
-
-class MapViewportRequest {
- final int id;
- final LatLngBounds bounds;
- final double zoom;
- final double markerMinX;
- final double markerMinY;
- final double markerMaxX;
- final double markerMaxY;
-
- const MapViewportRequest({
- required this.id,
- required this.bounds,
- required this.zoom,
- required this.markerMinX,
- required this.markerMinY,
- required this.markerMaxX,
- required this.markerMaxY,
- });
-}
-
-class MapViewportResult {
- final int id;
- final List<int> visibleImageIndexes;
- final List<MapMarkerGroup> markerGroups;
-
- const MapViewportResult({
- required this.id,
- required this.visibleImageIndexes,
- required this.markerGroups,
- });
-}
-
-const double _minimumMarkerDistancePixels = 100;
-const double _webMercatorTileSize = 256;
-const double _mercatorLatitudeLimit = 85.05112878;
-
-void mapWorker(MapWorkerInit init) {
- final receivePort = ReceivePort();
- init.sendPort.send(receivePort.sendPort);
- receivePort.listen((message) {
- if (message is! MapViewportRequest) {
- return;
- }
-
- final visibleIndexes = <int>[];
- final visiblePoints = <_ProjectedMapPoint>[];
- final bufferedPoints = <_ProjectedMapPoint>[];
- final groupsByCell = <(int, int), List<_ProjectedMarkerGroup>>{};
- final groups = <_ProjectedMarkerGroup>[];
- final scale = _webMercatorTileSize * pow(2, message.zoom).toDouble();
-
- for (final point in init.points) {
- final isVisible =
- point.longitude >= message.bounds.west &&
- point.longitude <= message.bounds.east &&
- point.latitude >= message.bounds.south &&
- point.latitude <= message.bounds.north;
- if (isVisible) {
- visibleIndexes.add(point.imageIndex);
- }
-
- final latitude = point.latitude.clamp(
- -_mercatorLatitudeLimit,
- _mercatorLatitudeLimit,
- );
- final sinLatitude = sin(latitude * pi / 180);
- final x = (point.longitude + 180) / 360 * scale;
- final y =
- (0.5 - log((1 + sinLatitude) / (1 - sinLatitude)) / (4 * pi)) * scale;
- if (x < message.markerMinX ||
- x > message.markerMaxX ||
- y < message.markerMinY ||
- y > message.markerMaxY) {
- continue;
- }
-
- final projectedPoint = _ProjectedMapPoint(point: point, x: x, y: y);
- (isVisible ? visiblePoints : bufferedPoints).add(projectedPoint);
- }
-
- void addToMarkerGroup(_ProjectedMapPoint projectedPoint) {
- final point = projectedPoint.point;
- final x = projectedPoint.x;
- final y = projectedPoint.y;
- final cell = (
- (x / _minimumMarkerDistancePixels).floor(),
- (y / _minimumMarkerDistancePixels).floor(),
- );
- _ProjectedMarkerGroup? closestGroup;
- var closestDistanceSquared = double.infinity;
-
- for (var xOffset = -1; xOffset <= 1; xOffset++) {
- for (var yOffset = -1; yOffset <= 1; yOffset++) {
- final nearbyGroups =
- groupsByCell[(cell.$1 + xOffset, cell.$2 + yOffset)];
- if (nearbyGroups == null) {
- continue;
- }
-
- for (final group in nearbyGroups) {
- final xDistance = x - group.x;
- final yDistance = y - group.y;
- final distanceSquared =
- xDistance * xDistance + yDistance * yDistance;
- if (distanceSquared <=
- _minimumMarkerDistancePixels *
- _minimumMarkerDistancePixels &&
- distanceSquared < closestDistanceSquared) {
- closestGroup = group;
- closestDistanceSquared = distanceSquared;
- }
- }
- }
- }
-
- if (closestGroup != null) {
- closestGroup.add(point);
- return;
- }
-
- final group = _ProjectedMarkerGroup(
- imageIndex: point.imageIndex,
- latitude: point.latitude,
- longitude: point.longitude,
- x: x,
- y: y,
- );
- groups.add(group);
- groupsByCell.putIfAbsent(cell, () => []).add(group);
- }
-
- for (final point in visiblePoints) {
- addToMarkerGroup(point);
- }
- for (final point in bufferedPoints) {
- addToMarkerGroup(point);
- }
-
- init.sendPort.send(
- MapViewportResult(
- id: message.id,
- visibleImageIndexes: visibleIndexes,
- markerGroups: groups
- .map((group) => group.toMarkerGroup())
- .toList(growable: false),
- ),
- );
- });
-}
diff --git a/mobile/apps/photos/lib/ui/map/map_screen.dart b/mobile/apps/photos/lib/ui/map/map_screen.dart
index 99f0c21a54..841d7ff71d 100644
--- a/mobile/apps/photos/lib/ui/map/map_screen.dart
+++ b/mobile/apps/photos/lib/ui/map/map_screen.dart
@@ -1,5 +1,4 @@
import "dart:async";
-import "dart:isolate";
import "dart:math";
import "package:computer/computer.dart";
@@ -13,9 +12,9 @@ import "package:latlong2/latlong.dart";
import "package:logging/logging.dart";
import 'package:photos/models/file/file.dart';
import "package:photos/models/location/location.dart";
+import "package:photos/src/rust/api/map_cluster_api.dart";
import "package:photos/theme/ente_theme.dart";
import "package:photos/ui/map/image_marker.dart";
-import "package:photos/ui/map/map_isolate.dart";
import "package:photos/ui/map/map_pull_up_gallery.dart";
import "package:photos/ui/map/map_view.dart";
import "package:photos/ui/notification/toast.dart";
@@ -50,10 +49,7 @@ class _MapScreenState extends State<MapScreen> {
double minZoom = 2.8;
LatLng? center;
final Logger _logger = Logger("_MapScreenState");
- ReceivePort? _mapWorkerReceivePort;
- StreamSubscription? _mapWorkerSubscription;
- SendPort? _mapWorkerSendPort;
- Isolate? _mapWorkerIsolate;
+ MapClusterer? _mapClusterer;
static const bottomSheetDraggableAreaHeight = 32.0;
List<int>? _previousVisibleIndexes;
int _viewportRequestID = 0;
@@ -72,9 +68,6 @@ class _MapScreenState extends State<MapScreen> {
@override
void dispose() {
_viewportDebouncer.cancelDebounceTimer();
- unawaited(_mapWorkerSubscription?.cancel());
- _mapWorkerReceivePort?.close();
- _mapWorkerIsolate?.kill();
unawaited(visibleImages.close());
super.dispose();
}
@@ -128,7 +121,7 @@ class _MapScreenState extends State<MapScreen> {
);
}
- await _startMapWorker(mapPoints);
+ _mapClusterer = await createMapClusterer(points: mapPoints);
if (!mounted) return;
setState(() {
hasLocationData = true;
@@ -140,37 +133,6 @@ class _MapScreenState extends State<MapScreen> {
});
}
- Future<void> _startMapWorker(List<MapPoint> mapPoints) async {
- final receivePort = ReceivePort();
- final sendPortCompleter = Completer<SendPort>();
- final subscription = receivePort.listen((message) {
- if (message is SendPort) {
- if (!sendPortCompleter.isCompleted) {
- sendPortCompleter.complete(message);
- }
- } else if (message is MapViewportResult) {
- _applyViewportResult(message);
- }
- });
- final isolate = await Isolate.spawn(
- mapWorker,
- MapWorkerInit(points: mapPoints, sendPort: receivePort.sendPort),
- );
- final sendPort = await sendPortCompleter.future;
-
- if (!mounted) {
- await subscription.cancel();
- receivePort.close();
- isolate.kill();
- return;
- }
-
- _mapWorkerReceivePort = receivePort;
- _mapWorkerSubscription = subscription;
- _mapWorkerSendPort = sendPort;
- _mapWorkerIsolate = isolate;
- }
-
void _updateViewport(MapCamera camera) {
final requestID = ++_viewportRequestID;
_viewportDebouncer.run(() async {
@@ -179,24 +141,44 @@ class _MapScreenState extends State<MapScreen> {
}
void _sendViewportRequest(MapCamera camera, int requestID) {
- final sendPort = _mapWorkerSendPort;
- if (sendPort == null) return;
+ final clusterer = _mapClusterer;
+ if (clusterer == null) return;
final pixelBounds = camera.pixelBounds;
- sendPort.send(
- MapViewportRequest(
- id: requestID,
- bounds: camera.visibleBounds,
- zoom: camera.zoom,
- markerMinX: pixelBounds.left - _markerViewportPaddingPixels,
- markerMinY: pixelBounds.top - _markerViewportPaddingPixels,
- markerMaxX: pixelBounds.right + _markerViewportPaddingPixels,
- markerMaxY: pixelBounds.bottom + _markerViewportPaddingPixels,
+ final bounds = camera.visibleBounds;
+ unawaited(
+ _clusterViewport(
+ clusterer,
+ MapViewport(
+ north: bounds.north,
+ south: bounds.south,
+ east: bounds.east,
+ west: bounds.west,
+ zoom: camera.zoom,
+ markerMinX: pixelBounds.left - _markerViewportPaddingPixels,
+ markerMinY: pixelBounds.top - _markerViewportPaddingPixels,
+ markerMaxX: pixelBounds.right + _markerViewportPaddingPixels,
+ markerMaxY: pixelBounds.bottom + _markerViewportPaddingPixels,
+ ),
+ requestID,
),
);
}
- void _applyViewportResult(MapViewportResult result) {
- if (!mounted || result.id != _viewportRequestID) return;
+ Future<void> _clusterViewport(
+ MapClusterer clusterer,
+ MapViewport viewport,
+ int requestID,
+ ) async {
+ try {
+ final result = await clusterer.cluster(viewport: viewport);
+ _applyViewportResult(result, requestID);
+ } catch (e, s) {
+ _logger.severe("Rust map clustering failed", e, s);
+ }
+ }
+
+ void _applyViewportResult(MapViewportResult result, int requestID) {
+ if (!mounted || requestID != _viewportRequestID) return;
final visibleIndexes = result.visibleImageIndexes;
if (!_sameIndexes(visibleIndexes, _previousVisibleIndexes)) {
@@ -218,7 +200,14 @@ class _MapScreenState extends State<MapScreen> {
latitude: group.latitude,
longitude: group.longitude,
imageCount: group.imageCount,
- clusterBounds: group.imageCount > 1 ? group.bounds : null,
+ clusterBounds: group.imageCount > 1
+ ? LatLngBounds.unsafe(
+ north: group.north,
+ south: group.south,
+ east: group.east,
+ west: group.west,
+ )
+ : null,
),
)
.toList(growable: false);
diff --git a/rust/bindings/frb/photos/src/api/map_cluster_api.rs b/rust/bindings/frb/photos/src/api/map_cluster_api.rs
new file mode 100644
index 0000000000..e3272cb09d
--- /dev/null
+++ b/rust/bindings/frb/photos/src/api/map_cluster_api.rs
@@ -0,0 +1,99 @@
+use ente_photos::map_cluster as core;
+use flutter_rust_bridge::frb;
+
+pub struct MapPoint {
+ pub image_index: u32,
+ pub latitude: f64,
+ pub longitude: f64,
+}
+
+impl From<MapPoint> for core::MapPoint {
+ fn from(point: MapPoint) -> Self {
+ Self {
+ image_index: point.image_index,
+ latitude: point.latitude,
+ longitude: point.longitude,
+ }
+ }
+}
+
+pub struct MapViewport {
+ pub north: f64,
+ pub south: f64,
+ pub east: f64,
+ pub west: f64,
+ pub zoom: f64,
+ pub marker_min_x: f64,
+ pub marker_min_y: f64,
+ pub marker_max_x: f64,
+ pub marker_max_y: f64,
+}
+
+impl From<MapViewport> for core::MapViewport {
+ fn from(viewport: MapViewport) -> Self {
+ Self {
+ north: viewport.north,
+ south: viewport.south,
+ east: viewport.east,
+ west: viewport.west,
+ zoom: viewport.zoom,
+ marker_min_x: viewport.marker_min_x,
+ marker_min_y: viewport.marker_min_y,
+ marker_max_x: viewport.marker_max_x,
+ marker_max_y: viewport.marker_max_y,
+ }
+ }
+}
+
+pub struct MapMarkerGroup {
+ pub image_index: u32,
+ pub latitude: f64,
+ pub longitude: f64,
+ pub image_count: u32,
+ pub north: f64,
+ pub south: f64,
+ pub east: f64,
+ pub west: f64,
+}
+
+impl From<core::MapMarkerGroup> for MapMarkerGroup {
+ fn from(group: core::MapMarkerGroup) -> Self {
+ Self {
+ image_index: group.image_index,
+ latitude: group.latitude,
+ longitude: group.longitude,
+ image_count: group.image_count,
+ north: group.north,
+ south: group.south,
+ east: group.east,
+ west: group.west,
+ }
+ }
+}
+
+pub struct MapViewportResult {
+ pub visible_image_indexes: Vec<u32>,
+ pub marker_groups: Vec<MapMarkerGroup>,
+}
+
+#[frb(opaque)]
+pub struct MapClusterer {
+ inner: core::MapClusterer,
+}
+
+pub fn create_map_clusterer(points: Vec<MapPoint>) -> MapClusterer {
+ MapClusterer {
+ inner: core::MapClusterer::new(points.into_iter().map(Into::into).collect()),
+ }
+}
+
+impl MapClusterer {
+ pub fn cluster(&self, viewport: MapViewport) -> MapViewportResult {
+ let result = self.inner.cluster(viewport.into());
+
+ MapViewportResult {
+ visible_image_indexes: result.visible_image_indexes,
+ marker_groups: result.marker_groups.into_iter().map(Into::into).collect(),
+ }
+ }
+}
diff --git a/rust/bindings/frb/photos/src/api/mod.rs b/rust/bindings/frb/photos/src/api/mod.rs
index 69c9ba0d9c..f5886570e2 100644
--- a/rust/bindings/frb/photos/src/api/mod.rs
+++ b/rust/bindings/frb/photos/src/api/mod.rs
@@ -4,6 +4,7 @@ pub mod image_processing_api;
pub mod init;
#[cfg(any(feature = "flutter", frb_expand))]
pub mod log;
+pub mod map_cluster_api;
pub mod ml_indexing_api;
pub mod motion_photo_api;
pub mod urls;
diff --git a/rust/crates/photos/src/lib.rs b/rust/crates/photos/src/lib.rs
index 156da6c8ed..c36484458a 100644
--- a/rust/crates/photos/src/lib.rs
+++ b/rust/crates/photos/src/lib.rs
@@ -1,3 +1,4 @@
+pub mod map_cluster;
pub mod motion_photo;
pub use motion_photo::{
diff --git a/rust/crates/photos/src/map_cluster.rs b/rust/crates/photos/src/map_cluster.rs
new file mode 100644
index 0000000000..706e33c9f1
--- /dev/null
+++ b/rust/crates/photos/src/map_cluster.rs
@@ -0,0 +1,211 @@
+use std::collections::HashMap;
+use std::f64::consts::PI;
+
+const MINIMUM_MARKER_DISTANCE_PIXELS: f64 = 100.0;
+const WEB_MERCATOR_TILE_SIZE: f64 = 256.0;
+const MERCATOR_LATITUDE_LIMIT: f64 = 85.051_128_78;
+
+#[derive(Clone, Copy)]
+pub struct MapPoint {
+ pub image_index: u32,
+ pub latitude: f64,
+ pub longitude: f64,
+}
+
+pub struct MapViewport {
+ pub north: f64,
+ pub south: f64,
+ pub east: f64,
+ pub west: f64,
+ pub zoom: f64,
+ pub marker_min_x: f64,
+ pub marker_min_y: f64,
+ pub marker_max_x: f64,
+ pub marker_max_y: f64,
+}
+
+pub struct MapMarkerGroup {
+ pub image_index: u32,
+ pub latitude: f64,
+ pub longitude: f64,
+ pub image_count: u32,
+ pub north: f64,
+ pub south: f64,
+ pub east: f64,
+ pub west: f64,
+}
+
+pub struct MapViewportResult {
+ pub visible_image_indexes: Vec<u32>,
+ pub marker_groups: Vec<MapMarkerGroup>,
+}
+
+pub struct MapClusterer {
+ points: Vec<MapPoint>,
+}
+
+impl MapClusterer {
+ pub fn new(points: Vec<MapPoint>) -> Self {
+ Self { points }
+ }
+
+ pub fn cluster(&self, viewport: MapViewport) -> MapViewportResult {
+ let mut visible_image_indexes = Vec::new();
+ let mut visible_points = Vec::new();
+ let mut buffered_points = Vec::new();
+ let scale = WEB_MERCATOR_TILE_SIZE * 2.0_f64.powf(viewport.zoom);
+
+ for point in &self.points {
+ let is_visible = point.longitude >= viewport.west
+ && point.longitude <= viewport.east
+ && point.latitude >= viewport.south
+ && point.latitude <= viewport.north;
+ if is_visible {
+ visible_image_indexes.push(point.image_index);
+ }
+
+ let latitude = point
+ .latitude
+ .clamp(-MERCATOR_LATITUDE_LIMIT, MERCATOR_LATITUDE_LIMIT);
+ let sin_latitude = (latitude * PI / 180.0).sin();
+ let x = (point.longitude + 180.0) / 360.0 * scale;
+ let y = (0.5 - ((1.0 + sin_latitude) / (1.0 - sin_latitude)).ln() / (4.0 * PI)) * scale;
+ if x < viewport.marker_min_x
+ || x > viewport.marker_max_x
+ || y < viewport.marker_min_y
+ || y > viewport.marker_max_y
+ {
+ continue;
+ }
+
+ let projected_point = ProjectedMapPoint {
+ point: *point,
+ x,
+ y,
+ };
+ if is_visible {
+ visible_points.push(projected_point);
+ } else {
+ buffered_points.push(projected_point);
+ }
+ }
+
+ let mut groups_by_cell = HashMap::<(i64, i64), Vec<usize>>::new();
+ let mut groups = Vec::<ProjectedMarkerGroup>::new();
+ for point in visible_points.iter().chain(&buffered_points) {
+ add_to_marker_group(*point, &mut groups_by_cell, &mut groups);
+ }
+
+ MapViewportResult {
+ visible_image_indexes,
+ marker_groups: groups
+ .into_iter()
+ .map(ProjectedMarkerGroup::into_marker_group)
+ .collect(),
+ }
+ }
+}
+
+#[derive(Clone, Copy)]
+struct ProjectedMapPoint {
+ point: MapPoint,
+ x: f64,
+ y: f64,
+}
+
+struct ProjectedMarkerGroup {
+ image_index: u32,
+ latitude: f64,
+ longitude: f64,
+ x: f64,
+ y: f64,
+ image_count: u32,
+ north: f64,
+ south: f64,
+ east: f64,
+ west: f64,
+}
+
+impl ProjectedMarkerGroup {
+ fn new(point: ProjectedMapPoint) -> Self {
+ Self {
+ image_index: point.point.image_index,
+ latitude: point.point.latitude,
+ longitude: point.point.longitude,
+ x: point.x,
+ y: point.y,
+ image_count: 1,
+ north: point.point.latitude,
+ south: point.point.latitude,
+ east: point.point.longitude,
+ west: point.point.longitude,
+ }
+ }
+
+ fn add(&mut self, point: MapPoint) {
+ self.image_count += 1;
+ self.north = self.north.max(point.latitude);
+ self.south = self.south.min(point.latitude);
+ self.east = self.east.max(point.longitude);
+ self.west = self.west.min(point.longitude);
+ }
+
+ fn into_marker_group(self) -> MapMarkerGroup {
+ MapMarkerGroup {
+ image_index: self.image_index,
+ latitude: self.latitude,
+ longitude: self.longitude,
+ image_count: self.image_count,
+ north: self.north,
+ south: self.south,
+ east: self.east,
+ west: self.west,
+ }
+ }
+}
+
+fn add_to_marker_group(
+ point: ProjectedMapPoint,
+ groups_by_cell: &mut HashMap<(i64, i64), Vec<usize>>,
+ groups: &mut Vec<ProjectedMarkerGroup>,
+) {
+ let cell = (
+ (point.x / MINIMUM_MARKER_DISTANCE_PIXELS).floor() as i64,
+ (point.y / MINIMUM_MARKER_DISTANCE_PIXELS).floor() as i64,
+ );
+ let mut closest_group_index = None;
+ let mut closest_distance_squared = f64::INFINITY;
+
+ for x_offset in -1..=1 {
+ for y_offset in -1..=1 {
+ let Some(nearby_group_indexes) =
+ groups_by_cell.get(&(cell.0 + x_offset, cell.1 + y_offset))
+ else {
+ continue;
+ };
+
+ for group_index in nearby_group_indexes {
+ let group = &groups[*group_index];
+ let x_distance = point.x - group.x;
+ let y_distance = point.y - group.y;
+ let distance_squared = x_distance * x_distance + y_distance * y_distance;
+ if distance_squared
+ <= MINIMUM_MARKER_DISTANCE_PIXELS * MINIMUM_MARKER_DISTANCE_PIXELS
+ && distance_squared < closest_distance_squared
+ {
+ closest_group_index = Some(*group_index);
+ closest_distance_squared = distance_squared;
+ }
+ }
+ }
+ }
+
+ if let Some(group_index) = closest_group_index {
+ groups[group_index].add(point.point);
+ return;
+ }
+
+ let group_index = groups.len();
+ groups.push(ProjectedMarkerGroup::new(point));
+ groups_by_cell.entry(cell).or_default().push(group_index);
+}