Skip to content

High performance self-hosted photo and video management solution.

feat(web): show detected faces in spherical photos (#23974)

Mees Frenseld952b620531a
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 4a00b6755a..1228fcd55f 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -726,19 +726,22 @@ importers:
specifier: ^7.4.47
version: 7.4.47
'@photo-sphere-viewer/core':
- specifier: ^5.11.5
+ specifier: ^5.14.0
version: 5.14.0
'@photo-sphere-viewer/equirectangular-video-adapter':
- specifier: ^5.11.5
+ specifier: ^5.14.0
version: 5.14.0(@photo-sphere-viewer/[email protected])(@photo-sphere-viewer/[email protected](@photo-sphere-viewer/[email protected]))
+ '@photo-sphere-viewer/markers-plugin':
+ specifier: ^5.14.0
+ version: 5.14.0(@photo-sphere-viewer/[email protected])
'@photo-sphere-viewer/resolution-plugin':
- specifier: ^5.11.5
+ specifier: ^5.14.0
version: 5.14.0(@photo-sphere-viewer/[email protected])(@photo-sphere-viewer/[email protected](@photo-sphere-viewer/[email protected]))
'@photo-sphere-viewer/settings-plugin':
- specifier: ^5.11.5
+ specifier: ^5.14.0
version: 5.14.0(@photo-sphere-viewer/[email protected])
'@photo-sphere-viewer/video-plugin':
- specifier: ^5.11.5
+ specifier: ^5.14.0
version: 5.14.0(@photo-sphere-viewer/[email protected])
'@types/geojson':
specifier: ^7946.0.16
@@ -3573,6 +3576,11 @@ packages:
'@photo-sphere-viewer/core': 5.14.0
'@photo-sphere-viewer/video-plugin': 5.14.0
+ '@photo-sphere-viewer/[email protected]':
+ resolution: {integrity: sha512-w7txVHtLxXMS61m0EbNjgvdNXQYRh6Aa0oatft5oruKgoXLg/UlCu1mG6Btg+zrNsG05W2zl4gRM3fcWoVdneA==}
+ peerDependencies:
+ '@photo-sphere-viewer/core': 5.14.0
+
'@photo-sphere-viewer/[email protected]':
resolution: {integrity: sha512-PvDMX1h+8FzWdySxiorQ2bSmyBGTPsZjNNFRBqIfmb5C+01aWCIE7kuXodXGHwpXQNcOojsVX9IiX0Vz4CiW4A==}
peerDependencies:
@@ -15374,6 +15382,10 @@ snapshots:
'@photo-sphere-viewer/video-plugin': 5.14.0(@photo-sphere-viewer/[email protected])
three: 0.180.0
+ '@photo-sphere-viewer/[email protected](@photo-sphere-viewer/[email protected])':
+ dependencies:
+ '@photo-sphere-viewer/core': 5.14.0
+
'@photo-sphere-viewer/[email protected](@photo-sphere-viewer/[email protected])(@photo-sphere-viewer/[email protected](@photo-sphere-viewer/[email protected]))':
dependencies:
'@photo-sphere-viewer/core': 5.14.0
diff --git a/web/package.json b/web/package.json
index 27454f7e88..1afe94ac07 100644
--- a/web/package.json
+++ b/web/package.json
@@ -31,11 +31,12 @@
"@immich/ui": "^0.43.0",
"@mapbox/mapbox-gl-rtl-text": "0.2.3",
"@mdi/js": "^7.4.47",
- "@photo-sphere-viewer/core": "^5.11.5",
- "@photo-sphere-viewer/equirectangular-video-adapter": "^5.11.5",
- "@photo-sphere-viewer/resolution-plugin": "^5.11.5",
- "@photo-sphere-viewer/settings-plugin": "^5.11.5",
- "@photo-sphere-viewer/video-plugin": "^5.11.5",
+ "@photo-sphere-viewer/core": "^5.14.0",
+ "@photo-sphere-viewer/equirectangular-video-adapter": "^5.14.0",
+ "@photo-sphere-viewer/markers-plugin": "^5.14.0",
+ "@photo-sphere-viewer/resolution-plugin": "^5.14.0",
+ "@photo-sphere-viewer/settings-plugin": "^5.14.0",
+ "@photo-sphere-viewer/video-plugin": "^5.14.0",
"@types/geojson": "^7946.0.16",
"@zoom-image/core": "^0.41.0",
"@zoom-image/svelte": "^0.3.0",
diff --git a/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte b/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte
index cd9a010f78..00f3a6795f 100644
--- a/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte
+++ b/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte
@@ -1,4 +1,5 @@
<script lang="ts">
+ import { boundingBoxesArray, type Faces } from '$lib/stores/people.store';
import { alwaysLoadOriginalFile } from '$lib/stores/preferences.store';
import {
EquirectangularAdapter,
@@ -8,11 +9,21 @@
type PluginConstructor,
} from '@photo-sphere-viewer/core';
import '@photo-sphere-viewer/core/index.css';
+ import { MarkersPlugin } from '@photo-sphere-viewer/markers-plugin';
+ import '@photo-sphere-viewer/markers-plugin/index.css';
import { ResolutionPlugin } from '@photo-sphere-viewer/resolution-plugin';
import { SettingsPlugin } from '@photo-sphere-viewer/settings-plugin';
import '@photo-sphere-viewer/settings-plugin/index.css';
import { onDestroy, onMount } from 'svelte';
+ // Adapted as well as possible from classlist 'border-solid border-white border-3 rounded-lg'
+ const FACE_BOX_SVG_STYLE = {
+ fill: 'rgba(0, 0, 0, 0)',
+ stroke: '#ffffff',
+ strokeWidth: '3px',
+ strokeLinejoin: 'round',
+ };
+
interface Props {
panorama: string | { source: string };
originalPanorama?: string | { source: string };
@@ -26,6 +37,62 @@
let container: HTMLDivElement | undefined = $state();
let viewer: Viewer;
+ let animationInProgress: { cancel: () => void } | undefined;
+ let previousFaces: Faces[] = [];
+
+ const boundingBoxesUnsubscribe = boundingBoxesArray.subscribe((faces: Faces[]) => {
+ // Debounce; don't do anything when the data didn't actually change.
+ if (faces === previousFaces) {
+ return;
+ }
+ previousFaces = faces;
+
+ if (animationInProgress) {
+ animationInProgress.cancel();
+ animationInProgress = undefined;
+ }
+ if (!viewer || !viewer.state.textureData || !viewer.getPlugin(MarkersPlugin)) {
+ return;
+ }
+ const markersPlugin = viewer.getPlugin<MarkersPlugin>(MarkersPlugin);
+
+ // croppedWidth is the size of the texture, which might be cropped to be less than 360/180 degrees.
+ // This is what we want because the facial recognition is done on the image, not the sphere.
+ const currentTextureWidth = viewer.state.textureData.panoData.croppedWidth;
+
+ markersPlugin.clearMarkers();
+ for (const [index, face] of faces.entries()) {
+ const { boundingBoxX1: x1, boundingBoxY1: y1, boundingBoxX2: x2, boundingBoxY2: y2 } = face;
+ const ratio = currentTextureWidth / face.imageWidth;
+ // Pixel values are translated to spherical coordinates and only then added to the panorama;
+ // no need to recalculate when the texture image changes to the original size.
+ markersPlugin.addMarker({
+ id: `face_${index}`,
+ polygonPixels: [
+ [x1 * ratio, y1 * ratio],
+ [x2 * ratio, y1 * ratio],
+ [x2 * ratio, y2 * ratio],
+ [x1 * ratio, y2 * ratio],
+ ],
+ svgStyle: FACE_BOX_SVG_STYLE,
+ });
+ }
+
+ // Smoothly pan to the highlighted (hovered-over) face.
+ if (faces.length === 1) {
+ const { boundingBoxX1: x1, boundingBoxY1: y1, boundingBoxX2: x2, boundingBoxY2: y2, imageWidth: w } = faces[0];
+ const ratio = currentTextureWidth / w;
+ const x = ((x1 + x2) * ratio) / 2;
+ const y = ((y1 + y2) * ratio) / 2;
+ animationInProgress = viewer.animate({
+ textureX: x,
+ textureY: y,
+ zoom: Math.min(viewer.getZoomLevel(), 75),
+ speed: 500, // duration in ms
+ });
+ }
+ });
+
onMount(() => {
if (!container) {
return;
@@ -34,6 +101,7 @@
viewer = new Viewer({
adapter,
plugins: [
+ MarkersPlugin,
SettingsPlugin,
[
ResolutionPlugin,
@@ -68,7 +136,7 @@
zoomSpeed: 0.5,
fisheye: false,
});
- const resolutionPlugin = viewer.getPlugin(ResolutionPlugin) as ResolutionPlugin;
+ const resolutionPlugin = viewer.getPlugin<ResolutionPlugin>(ResolutionPlugin);
const zoomHandler = ({ zoomLevel }: events.ZoomUpdatedEvent) => {
// zoomLevel range: [0, 100]
if (Math.round(zoomLevel) >= 75) {
@@ -89,6 +157,7 @@
if (viewer) {
viewer.destroy();
}
+ boundingBoxesUnsubscribe();
});
</script>