π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mobile][photos] Fix blank panorama viewer for cropped panoramas (#11435)
Rahul Agarwalb11769d55601
diff --git a/mobile/apps/photos/lib/ui/viewer/file/panorama_view_data.dart b/mobile/apps/photos/lib/ui/viewer/file/panorama_view_data.dartnew file mode 100644index 0000000000..6e77b79d33--- /dev/null+++ b/mobile/apps/photos/lib/ui/viewer/file/panorama_view_data.dart@@ -0,0 +1,125 @@+import "package:flutter/material.dart";++/// View parameters for rendering a (possibly partial) panorama on a sphere,+/// derived from GPano XMP metadata.+///+/// See https://developers.google.com/streetview/spherical-metadata+class PanoramaViewData {+ const PanoramaViewData({+ required this.fullWidth,+ required this.fullHeight,+ required this.croppedArea,+ });++ /// Width in pixels of the full equirectangular canvas.+ final double fullWidth;++ /// Height in pixels of the full equirectangular canvas.+ final double fullHeight;++ /// Area of the full canvas that the image actually covers, in pixels.+ final Rect croppedArea;++ /// Initial camera longitude, in degrees within [-180, 180], that points the+ /// view at the horizontal center of the cropped area.+ ///+ /// The panorama widget maps the cropped area to its absolute position on the+ /// full equirectangular canvas, and its longitude 0 faces the horizontal+ /// edge (u = 0) of that canvas. Partial panoramas (e.g. Pixel sweep+ /// panoramas) keep the compass heading of the shot, so their cropped area+ /// can sit anywhere on the canvas; without this correction the initial view+ /// often faces an empty part of the sphere.+ double get initialLongitude {+ final double uCenter =+ (croppedArea.left + croppedArea.width / 2) / fullWidth;+ double longitude = (1 - uCenter) * 360;+ if (longitude > 180) longitude -= 360;+ return longitude;+ }++ /// Parses GPano attributes extracted from XMP. Returns null when the+ /// metadata is insufficient to place the image on the panorama sphere.+ static PanoramaViewData? fromXmp(Map<String, String> data) {+ double? cWidth = double.tryParse(+ data["GPano:CroppedAreaImageWidthPixels"] ?? "",+ );+ double? cHeight = double.tryParse(+ data["GPano:CroppedAreaImageHeightPixels"] ?? "",+ );+ double? fWidth = double.tryParse(data["GPano:FullPanoWidthPixels"] ?? "");+ double? fHeight = double.tryParse(data["GPano:FullPanoHeightPixels"] ?? "");+ double? cLeft = double.tryParse(data["GPano:CroppedAreaLeftPixels"] ?? "");+ double? cTop = double.tryParse(data["GPano:CroppedAreaTopPixels"] ?? "");++ // handle missing `fullPanoHeight` (e.g. Samsung camera app panorama mode)+ if (fHeight == null && fWidth != null && cHeight != null) {+ fHeight = (fWidth / 2).round().toDouble();+ cTop = ((fHeight - cHeight) / 2).round().toDouble();+ }++ // handle inconsistent sizing (e.g. rotated image taken with OnePlus EB2103)+ if (cWidth != null &&+ cHeight != null &&+ fWidth != null &&+ fHeight != null) {+ double cw = cWidth;+ double ch = cHeight;+ double fw = fWidth;+ double fh = fHeight;+ final croppedOrientation = cw > ch+ ? Orientation.landscape+ : Orientation.portrait;+ final fullOrientation = fw > fh+ ? Orientation.landscape+ : Orientation.portrait;+ var inconsistent = false;+ if (croppedOrientation != fullOrientation) {+ // inconsistent orientation+ inconsistent = true;+ final tmp = ch;+ ch = cw;+ cw = tmp;+ }++ if (cw > fw) {+ // inconsistent full/cropped width+ inconsistent = true;+ final tmp = fw;+ fw = cw;+ cw = tmp;+ }++ if (ch > fh) {+ // inconsistent full/cropped height+ inconsistent = true;+ final tmp = ch;+ ch = fh;+ fh = tmp;+ }++ if (inconsistent) {+ cLeft = ((fw - cw) ~/ 2).toDouble();+ cTop = ((fh - ch) ~/ 2).toDouble();+ }+ cWidth = cw;+ cHeight = ch;+ fWidth = fw;+ fHeight = fh;+ }++ if (cLeft == null ||+ cTop == null ||+ cWidth == null ||+ cHeight == null ||+ fWidth == null ||+ fHeight == null) {+ return null;+ }++ return PanoramaViewData(+ fullWidth: fWidth,+ fullHeight: fHeight,+ croppedArea: Rect.fromLTWH(cLeft, cTop, cWidth, cHeight),+ );+ }+}diff --git a/mobile/apps/photos/lib/ui/viewer/file/panorama_viewer_screen.dart b/mobile/apps/photos/lib/ui/viewer/file/panorama_viewer_screen.dartindex 3938897b12..43b18b8309 100644--- a/mobile/apps/photos/lib/ui/viewer/file/panorama_viewer_screen.dart+++ b/mobile/apps/photos/lib/ui/viewer/file/panorama_viewer_screen.dart@@ -7,27 +7,34 @@ import "package:logging/logging.dart";import "package:panorama/panorama.dart";import "package:photos/generated/l10n.dart";import "package:photos/src/rust/api/motion_photo_api.dart";+import "package:photos/ui/viewer/file/panorama_view_data.dart";final _logger = Logger("PanoramaViewerScreen");+/// Extracts XMP attributes from the file at [filePath].+typedef XmpExtractor = Future<Map<String, String>> Function(String filePath);+class PanoramaViewerScreen extends StatefulWidget {const PanoramaViewerScreen({super.key,required this.file,required this.thumbnail,+ this.xmpExtractor,});final File file;final Uint8List? thumbnail;+ /// Overridable for tests. Defaults to the Rust XMP extractor.+ final XmpExtractor? xmpExtractor;+@overrideState<PanoramaViewerScreen> createState() => _PanoramaViewerScreenState();}class _PanoramaViewerScreenState extends State<PanoramaViewerScreen> {- double width = 1.0;- double height = 1.0;- Rect croppedRect = const Rect.fromLTWH(0.0, 0.0, 1.0, 1.0);+ PanoramaViewData? viewData;+ bool isReady = false;SensorControl control = SensorControl.none;Timer? timer;bool isVisible = true;@@ -60,159 +67,106 @@ class _PanoramaViewerScreenState extends State<PanoramaViewerScreen> {}Future<void> init() async {- final Map<String, String> data;+ Map<String, String>? data;try {- data = await extractXmp(filePath: widget.file.path);+ data = await (widget.xmpExtractor ?? _extractXmp)(widget.file.path);} catch (e, s) {_logger.warning("Failed to extract panorama XMP", e, s);- return;}if (!mounted) {return;}- double? cWidth = double.tryParse(- data["GPano:CroppedAreaImageWidthPixels"] ?? "",- );- double? cHeight = double.tryParse(- data["GPano:CroppedAreaImageHeightPixels"] ?? "",- );- double? fWidth = double.tryParse(data["GPano:FullPanoWidthPixels"] ?? "");- double? fHeight = double.tryParse(data["GPano:FullPanoHeightPixels"] ?? "");- double? cLeft = double.tryParse(data["GPano:CroppedAreaLeftPixels"] ?? "");- double? cTop = double.tryParse(data["GPano:CroppedAreaTopPixels"] ?? "");-- // handle missing `fullPanoHeight` (e.g. Samsung camera app panorama mode)- if (fHeight == null && fWidth != null && cHeight != null) {- fHeight = (fWidth / 2).round().toDouble();- cTop = ((fHeight - cHeight) / 2).round().toDouble();- }-- // handle inconsistent sizing (e.g. rotated image taken with OnePlus EB2103)- if (cHeight != null && fWidth != null && fHeight != null) {- final croppedOrientation = cWidth! > cHeight- ? Orientation.landscape- : Orientation.portrait;- final fullOrientation = fWidth > fHeight- ? Orientation.landscape- : Orientation.portrait;- var inconsistent = false;- if (croppedOrientation != fullOrientation) {- // inconsistent orientation- inconsistent = true;- final tmp = cHeight;- cHeight = cWidth;- cWidth = tmp;- }-- if (cWidth > fWidth) {- // inconsistent full/cropped width- inconsistent = true;- final tmp = fWidth;- fWidth = cWidth;- cWidth = tmp;- }-- if (cHeight > fHeight) {- // inconsistent full/cropped height- inconsistent = true;- final tmp = cHeight;- cHeight = fHeight;- fHeight = tmp;- }-- if (inconsistent) {- cLeft = ((fWidth - cWidth) ~/ 2).toDouble();- cTop = ((fHeight - cHeight) ~/ 2).toDouble();- }- }-- Rect? croppedAreaRect;- if (cLeft != null && cTop != null && cWidth != null && cHeight != null) {- croppedAreaRect = Rect.fromLTWH(- cLeft.toDouble(),- cTop.toDouble(),- cWidth.toDouble(),- cHeight.toDouble(),- );- }-- if (croppedAreaRect == null || fWidth == null || fHeight == null) return;- width = fWidth.toDouble();- height = fHeight.toDouble();- croppedRect = croppedAreaRect;-- setState(() {});+ setState(() {+ viewData = data == null ? null : PanoramaViewData.fromXmp(data);+ isReady = true;+ });}+ static Future<Map<String, String>> _extractXmp(String filePath) =>+ extractXmp(filePath: filePath);+@overrideWidget build(BuildContext context) {return Scaffold(appBar: isVisible ? AppBar() : null,- body: Stack(- children: [- Panorama(- onTap: (_, _, _) {- setState(() {- if (isVisible) {- timer?.cancel();- SystemChrome.setEnabledSystemUIMode(- SystemUiMode.immersiveSticky,- );- } else {- initTimer();- }- isVisible = !isVisible;- });- },- croppedArea: croppedRect,- croppedFullWidth: width,- croppedFullHeight: height,- sensorControl: control,- background: widget.thumbnail != null- ? Image.memory(widget.thumbnail!)- : null,- child: Image.file(widget.file),- ),- Visibility(- visible: isVisible,- child: Align(- alignment: Alignment.bottomRight,- child: Tooltip(- message: AppLocalizations.of(context).panorama,- child: Padding(- padding: const EdgeInsets.only(- top: 12,- bottom: 32,- right: 20,+ // The Panorama widget must only be created once the view parameters are+ // final: changing its croppedArea afterwards makes it regenerate the+ // sphere mesh, which drops an already-loaded texture and leaves the+ // panorama blank (https://github.com/ente-io/ente/issues/11435).+ body: isReady ? _buildPanorama() : _buildPlaceholder(),+ );+ }++ Widget _buildPlaceholder() {+ if (widget.thumbnail == null) {+ return const SizedBox.shrink();+ }+ return Center(child: Image.memory(widget.thumbnail!));+ }++ Widget _buildPanorama() {+ return Stack(+ children: [+ Panorama(+ onTap: (_, _, _) {+ setState(() {+ if (isVisible) {+ timer?.cancel();+ SystemChrome.setEnabledSystemUIMode(+ SystemUiMode.immersiveSticky,+ );+ } else {+ initTimer();+ }+ isVisible = !isVisible;+ });+ },+ longitude: viewData?.initialLongitude ?? 0.0,+ croppedArea:+ viewData?.croppedArea ?? const Rect.fromLTWH(0.0, 0.0, 1.0, 1.0),+ croppedFullWidth: viewData?.fullWidth ?? 1.0,+ croppedFullHeight: viewData?.fullHeight ?? 1.0,+ sensorControl: control,+ background: widget.thumbnail != null+ ? Image.memory(widget.thumbnail!)+ : null,+ child: Image.file(widget.file),+ ),+ Visibility(+ visible: isVisible,+ child: Align(+ alignment: Alignment.bottomRight,+ child: Tooltip(+ message: AppLocalizations.of(context).panorama,+ child: Padding(+ padding: const EdgeInsets.only(top: 12, bottom: 32, right: 20),+ child: IconButton(+ style: IconButton.styleFrom(+ backgroundColor: const Color(0xFF252525),+ fixedSize: const Size(44, 44),),- child: IconButton(- style: IconButton.styleFrom(- backgroundColor: const Color(0xFF252525),- fixedSize: const Size(44, 44),- ),- icon: Icon(- control == SensorControl.none- ? Icons.explore_outlined- : Icons.explore_off_outlined,- color: Colors.white,- size: 26,- ),- onPressed: () async {- if (control != SensorControl.none) {- control = SensorControl.none;- } else {- control = SensorControl.orientation;- }-- setState(() {});- },+ icon: Icon(+ control == SensorControl.none+ ? Icons.explore_outlined+ : Icons.explore_off_outlined,+ color: Colors.white,+ size: 26,),+ onPressed: () async {+ if (control != SensorControl.none) {+ control = SensorControl.none;+ } else {+ control = SensorControl.orientation;+ }++ setState(() {});+ },),),),),- ],- ),+ ),+ ],);}}diff --git a/mobile/apps/photos/test/ui/viewer/file/panorama_view_data_test.dart b/mobile/apps/photos/test/ui/viewer/file/panorama_view_data_test.dartnew file mode 100644index 0000000000..57443ce42a--- /dev/null+++ b/mobile/apps/photos/test/ui/viewer/file/panorama_view_data_test.dart@@ -0,0 +1,94 @@+import "dart:ui";++import "package:flutter_test/flutter_test.dart";+import "package:photos/ui/viewer/file/panorama_view_data.dart";++void main() {+ group("PanoramaViewData.fromXmp", () {+ test("parses Pixel GPano metadata (issue #11435 sample)", () {+ // GPano values from the Pixel 8a sample panorama attached to+ // https://github.com/ente-io/ente/issues/11435+ final data = PanoramaViewData.fromXmp(const {+ "GPano:CroppedAreaLeftPixels": "6183",+ "GPano:CroppedAreaTopPixels": "1040",+ "GPano:CroppedAreaImageWidthPixels": "2520",+ "GPano:CroppedAreaImageHeightPixels": "1664",+ "GPano:FullPanoWidthPixels": "8762",+ "GPano:FullPanoHeightPixels": "4381",+ "GPano:InitialViewHeadingDegrees": "305",+ "GPano:ProjectionType": "equirectangular",+ });++ expect(data, isNotNull);+ expect(data!.croppedArea, const Rect.fromLTWH(6183, 1040, 2520, 1664));+ expect(data.fullWidth, 8762);+ expect(data.fullHeight, 4381);+ });++ test("initial longitude points at the cropped area center", () {+ const data = PanoramaViewData(+ fullWidth: 8762,+ fullHeight: 4381,+ croppedArea: Rect.fromLTWH(6183, 1040, 2520, 1664),+ );+ // Crop center sits at u = (6183 + 1260) / 8762 = 0.8495 of the full+ // canvas; the widget's longitude 0 faces u = 0 and longitude L faces+ // u = 1 - L / 360, hence L = (1 - 0.8495) * 360.+ expect(data.initialLongitude, closeTo(54.19, 0.01));+ // Matches the recorded compass heading of the sample within rounding:+ // GPano:InitialViewHeadingDegrees = 305 = 0.8495 * 360.+ final uCenter = 1 - data.initialLongitude / 360;+ expect(uCenter * 360, closeTo(305.8, 0.1));+ });++ test("initial longitude is normalized to [-180, 180]", () {+ const data = PanoramaViewData(+ fullWidth: 1000,+ fullHeight: 500,+ croppedArea: Rect.fromLTWH(100, 100, 200, 300),+ );+ // uCenter = 0.2 -> raw longitude 288 -> normalized -72.+ expect(data.initialLongitude, closeTo(-72, 0.001));+ });++ test("derives full height and top for Samsung-style metadata", () {+ final data = PanoramaViewData.fromXmp(const {+ "GPano:CroppedAreaLeftPixels": "1024",+ "GPano:CroppedAreaImageWidthPixels": "6144",+ "GPano:CroppedAreaImageHeightPixels": "1536",+ "GPano:FullPanoWidthPixels": "8192",+ "GPano:ProjectionType": "cylindrical",+ });++ expect(data, isNotNull);+ expect(data!.fullHeight, 4096);+ expect(data.croppedArea, const Rect.fromLTWH(1024, 1280, 6144, 1536));+ });++ test("re-centers inconsistently rotated metadata", () {+ final data = PanoramaViewData.fromXmp(const {+ "GPano:CroppedAreaLeftPixels": "100",+ "GPano:CroppedAreaTopPixels": "200",+ "GPano:CroppedAreaImageWidthPixels": "1000",+ "GPano:CroppedAreaImageHeightPixels": "3000",+ "GPano:FullPanoWidthPixels": "8000",+ "GPano:FullPanoHeightPixels": "4000",+ });++ expect(data, isNotNull);+ expect(data!.croppedArea, const Rect.fromLTWH(2500, 1500, 3000, 1000));+ expect(data.fullWidth, 8000);+ expect(data.fullHeight, 4000);+ });++ test("returns null when GPano crop metadata is missing", () {+ expect(+ PanoramaViewData.fromXmp(const {+ "GPano:ProjectionType": "equirectangular",+ }),+ isNull,+ );+ expect(PanoramaViewData.fromXmp(const {}), isNull);+ });+ });+}diff --git a/mobile/apps/photos/test/ui/viewer/file/panorama_viewer_screen_test.dart b/mobile/apps/photos/test/ui/viewer/file/panorama_viewer_screen_test.dartnew file mode 100644index 0000000000..54e2d82874--- /dev/null+++ b/mobile/apps/photos/test/ui/viewer/file/panorama_viewer_screen_test.dart@@ -0,0 +1,205 @@+import "dart:async";+import "dart:io";+import "dart:typed_data";++import "package:flutter/material.dart";+import "package:flutter_test/flutter_test.dart";+import "package:panorama/panorama.dart";+import "package:photos/generated/l10n.dart";+import "package:photos/ui/viewer/file/panorama_viewer_screen.dart";++// A minimal valid 1x1 RGBA PNG, so Image.file/Image.memory can decode it.+const List<int> _tinyImage = [+ 0x89,+ 0x50,+ 0x4e,+ 0x47,+ 0x0d,+ 0x0a,+ 0x1a,+ 0x0a,+ 0x00,+ 0x00,+ 0x00,+ 0x0d,+ 0x49,+ 0x48,+ 0x44,+ 0x52,+ 0x00,+ 0x00,+ 0x00,+ 0x01,+ 0x00,+ 0x00,+ 0x00,+ 0x01,+ 0x08,+ 0x06,+ 0x00,+ 0x00,+ 0x00,+ 0x1f,+ 0x15,+ 0xc4,+ 0x89,+ 0x00,+ 0x00,+ 0x00,+ 0x0d,+ 0x49,+ 0x44,+ 0x41,+ 0x54,+ 0x08,+ 0xd7,+ 0x63,+ 0xf8,+ 0xcf,+ 0xc0,+ 0xf0,+ 0x1f,+ 0x00,+ 0x05,+ 0x00,+ 0x01,+ 0xff,+ 0x89,+ 0x99,+ 0xd1,+ 0x8d,+ 0x00,+ 0x00,+ 0x00,+ 0x00,+ 0x49,+ 0x45,+ 0x4e,+ 0x44,+ 0xae,+ 0x42,+ 0x60,+ 0x82,+];++// GPano values of the Pixel 8a sample panorama from issue #11435.+const Map<String, String> _pixelXmp = {+ "GPano:CroppedAreaLeftPixels": "6183",+ "GPano:CroppedAreaTopPixels": "1040",+ "GPano:CroppedAreaImageWidthPixels": "2520",+ "GPano:CroppedAreaImageHeightPixels": "1664",+ "GPano:FullPanoWidthPixels": "8762",+ "GPano:FullPanoHeightPixels": "4381",+ "GPano:InitialViewHeadingDegrees": "305",+ "GPano:ProjectionType": "equirectangular",+};++void main() {+ late Directory tempDir;+ late File imageFile;++ setUpAll(() {+ tempDir = Directory.systemTemp.createTempSync("pano_viewer_test");+ imageFile = File("${tempDir.path}/pano.png")+ ..writeAsBytesSync(_tinyImage, flush: true);+ });++ tearDownAll(() {+ tempDir.deleteSync(recursive: true);+ });++ Widget wrap(Widget child) {+ return MaterialApp(+ localizationsDelegates: AppLocalizations.localizationsDelegates,+ supportedLocales: AppLocalizations.supportedLocales,+ home: child,+ );+ }++ testWidgets(+ "builds Panorama only after XMP is parsed, with the final crop values",+ (tester) async {+ final completer = Completer<Map<String, String>>();++ await tester.pumpWidget(+ wrap(+ PanoramaViewerScreen(+ file: imageFile,+ thumbnail: null,+ xmpExtractor: (_) => completer.future,+ ),+ ),+ );++ // Regression check for #11435: the Panorama widget must not exist yet.+ // Building it with placeholder crop values and updating them afterwards+ // makes the panorama package regenerate its sphere mesh, dropping the+ // already-loaded texture and rendering a blank (untextured) panorama.+ expect(find.byType(Panorama), findsNothing);++ completer.complete(_pixelXmp);+ await tester.pump();++ final pano = tester.widget<Panorama>(find.byType(Panorama));+ expect(pano.croppedArea, const Rect.fromLTWH(6183, 1040, 2520, 1664));+ expect(pano.croppedFullWidth, 8762);+ expect(pano.croppedFullHeight, 4381);+ // The initial view must face the cropped area (compass heading 305.8+ // degrees maps to longitude 54.19), not the canvas edge.+ expect(pano.longitude, closeTo(54.19, 0.01));++ // Let the auto-hide timer fire and dispose the screen.+ await tester.pump(const Duration(seconds: 6));+ await tester.pumpWidget(const SizedBox());+ },+ );++ testWidgets("falls back to a full-sphere mapping when XMP is unavailable", (+ tester,+ ) async {+ await tester.pumpWidget(+ wrap(+ PanoramaViewerScreen(+ file: imageFile,+ thumbnail: null,+ xmpExtractor: (_) async => throw StateError("no xmp"),+ ),+ ),+ );+ await tester.pump();++ final pano = tester.widget<Panorama>(find.byType(Panorama));+ expect(pano.croppedArea, const Rect.fromLTWH(0, 0, 1, 1));+ expect(pano.croppedFullWidth, 1.0);+ expect(pano.croppedFullHeight, 1.0);+ expect(pano.longitude, 0.0);++ await tester.pump(const Duration(seconds: 6));+ await tester.pumpWidget(const SizedBox());+ });++ testWidgets("shows the thumbnail while XMP is being parsed", (tester) async {+ final completer = Completer<Map<String, String>>();+ final thumbnail = Uint8List.fromList(_tinyImage);++ await tester.pumpWidget(+ wrap(+ PanoramaViewerScreen(+ file: imageFile,+ thumbnail: thumbnail,+ xmpExtractor: (_) => completer.future,+ ),+ ),+ );++ expect(find.byType(Panorama), findsNothing);+ expect(find.byType(Image), findsOneWidget);++ completer.complete(const {});+ await tester.pump();+ expect(find.byType(Panorama), findsOneWidget);++ await tester.pump(const Duration(seconds: 6));+ await tester.pumpWidget(const SizedBox());+ });+}