High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat(web): OCR overlay interactivity during zoom (#27039)
Min Idzelis18201a26d969
diff --git a/e2e/src/ui/mock-network/ocr-network.ts b/e2e/src/ui/mock-network/ocr-network.tsnew file mode 100644index 0000000000..3b1a2fe62e--- /dev/null+++ b/e2e/src/ui/mock-network/ocr-network.ts@@ -0,0 +1,55 @@+import { faker } from '@faker-js/faker';+import type { AssetOcrResponseDto } from '@immich/sdk';+import { BrowserContext } from '@playwright/test';++export type MockOcrBox = {+ text: string;+ x1: number;+ y1: number;+ x2: number;+ y2: number;+ x3: number;+ y3: number;+ x4: number;+ y4: number;+};++export const createMockOcrData = (assetId: string, boxes: MockOcrBox[]): AssetOcrResponseDto[] => {+ return boxes.map((box) => ({+ id: faker.string.uuid(),+ assetId,+ x1: box.x1,+ y1: box.y1,+ x2: box.x2,+ y2: box.y2,+ x3: box.x3,+ y3: box.y3,+ x4: box.x4,+ y4: box.y4,+ boxScore: 0.95,+ textScore: 0.9,+ text: box.text,+ }));+};++export const setupOcrMockApiRoutes = async (+ context: BrowserContext,+ ocrDataByAssetId: Map<string, AssetOcrResponseDto[]>,+) => {+ await context.route('**/assets/*/ocr', async (route, request) => {+ if (request.method() !== 'GET') {+ return route.fallback();+ }+ const url = new URL(request.url());+ const segments = url.pathname.split('/');+ const assetIdIndex = segments.indexOf('assets') + 1;+ const assetId = segments[assetIdIndex];++ const ocrData = ocrDataByAssetId.get(assetId) ?? [];+ return route.fulfill({+ status: 200,+ contentType: 'application/json',+ json: ocrData,+ });+ });+};diff --git a/e2e/src/ui/specs/asset-viewer/ocr.e2e-spec.ts b/e2e/src/ui/specs/asset-viewer/ocr.e2e-spec.tsnew file mode 100644index 0000000000..5a442a6081--- /dev/null+++ b/e2e/src/ui/specs/asset-viewer/ocr.e2e-spec.ts@@ -0,0 +1,300 @@+import type { AssetOcrResponseDto, AssetResponseDto } from '@immich/sdk';+import { expect, test } from '@playwright/test';+import { toAssetResponseDto } from 'src/ui/generators/timeline';+import {+ createMockStack,+ createMockStackAsset,+ MockStack,+ setupBrokenAssetMockApiRoutes,+} from 'src/ui/mock-network/broken-asset-network';+import { createMockOcrData, setupOcrMockApiRoutes } from 'src/ui/mock-network/ocr-network';+import { assetViewerUtils } from '../timeline/utils';+import { setupAssetViewerFixture } from './utils';++test.describe.configure({ mode: 'parallel' });++const PRIMARY_OCR_BOXES = [+ { text: 'Hello World', x1: 0.1, y1: 0.1, x2: 0.4, y2: 0.1, x3: 0.4, y3: 0.15, x4: 0.1, y4: 0.15 },+ { text: 'Immich Photo', x1: 0.2, y1: 0.3, x2: 0.6, y2: 0.3, x3: 0.6, y3: 0.36, x4: 0.2, y4: 0.36 },+];++const SECONDARY_OCR_BOXES = [+ { text: 'Second Asset Text', x1: 0.15, y1: 0.2, x2: 0.55, y2: 0.2, x3: 0.55, y3: 0.26, x4: 0.15, y4: 0.26 },+];++test.describe('OCR bounding boxes', () => {+ const fixture = setupAssetViewerFixture(920);++ test.beforeEach(async ({ context }) => {+ const primaryAssetDto = toAssetResponseDto(fixture.primaryAsset);+ const ocrDataByAssetId = new Map<string, AssetOcrResponseDto[]>([+ [primaryAssetDto.id, createMockOcrData(primaryAssetDto.id, PRIMARY_OCR_BOXES)],+ ]);++ await setupOcrMockApiRoutes(context, ocrDataByAssetId);+ });++ test('OCR bounding boxes appear when clicking OCR button', async ({ page }) => {+ await page.goto(`/photos/${fixture.primaryAsset.id}`);+ await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);++ const ocrButton = page.getByLabel('Text recognition');+ await expect(ocrButton).toBeVisible();+ await ocrButton.click();++ const ocrBoxes = page.locator('[data-viewer-content] [data-testid="ocr-box"]');+ await expect(ocrBoxes).toHaveCount(2);++ await expect(ocrBoxes.nth(0)).toContainText('Hello World');+ await expect(ocrBoxes.nth(1)).toContainText('Immich Photo');+ });++ test('OCR bounding boxes toggle off on second click', async ({ page }) => {+ await page.goto(`/photos/${fixture.primaryAsset.id}`);+ await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);++ const ocrButton = page.getByLabel('Text recognition');+ await ocrButton.click();+ await expect(page.locator('[data-viewer-content] [data-testid="ocr-box"]').first()).toBeVisible();++ await ocrButton.click();+ await expect(page.locator('[data-viewer-content] [data-testid="ocr-box"]')).toHaveCount(0);+ });+});++test.describe('OCR with stacked assets', () => {+ const fixture = setupAssetViewerFixture(921);+ let mockStack: MockStack;+ let primaryAssetDto: AssetResponseDto;+ let secondAssetDto: AssetResponseDto;++ test.beforeAll(async () => {+ primaryAssetDto = toAssetResponseDto(fixture.primaryAsset);+ secondAssetDto = createMockStackAsset(fixture.adminUserId);+ secondAssetDto.originalFileName = 'second-ocr-asset.jpg';+ mockStack = createMockStack(primaryAssetDto, [secondAssetDto], new Set());+ });++ test.beforeEach(async ({ context }) => {+ await setupBrokenAssetMockApiRoutes(context, mockStack);++ const ocrDataByAssetId = new Map<string, AssetOcrResponseDto[]>([+ [primaryAssetDto.id, createMockOcrData(primaryAssetDto.id, PRIMARY_OCR_BOXES)],+ [secondAssetDto.id, createMockOcrData(secondAssetDto.id, SECONDARY_OCR_BOXES)],+ ]);++ await setupOcrMockApiRoutes(context, ocrDataByAssetId);+ });++ test('different OCR boxes shown for different stacked assets', async ({ page }) => {+ await page.goto(`/photos/${fixture.primaryAsset.id}`);+ await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);++ const ocrButton = page.getByLabel('Text recognition');+ await expect(ocrButton).toBeVisible();+ await ocrButton.click();++ const ocrBoxes = page.locator('[data-viewer-content] [data-testid="ocr-box"]');+ await expect(ocrBoxes).toHaveCount(2);+ await expect(ocrBoxes.nth(0)).toContainText('Hello World');++ const stackThumbnails = page.locator('#stack-slideshow [data-asset]');+ await expect(stackThumbnails).toHaveCount(2);+ await stackThumbnails.nth(1).click();++ // refreshOcr() clears showOverlay when switching assets, so re-enable it+ await expect(ocrBoxes).toHaveCount(0);+ await expect(ocrButton).toBeVisible();+ await ocrButton.click();++ await expect(ocrBoxes).toHaveCount(1);+ await expect(ocrBoxes.first()).toContainText('Second Asset Text');+ });+});++test.describe('OCR boxes and zoom', () => {+ const fixture = setupAssetViewerFixture(922);++ test.beforeEach(async ({ context }) => {+ const primaryAssetDto = toAssetResponseDto(fixture.primaryAsset);+ const ocrDataByAssetId = new Map<string, AssetOcrResponseDto[]>([+ [primaryAssetDto.id, createMockOcrData(primaryAssetDto.id, PRIMARY_OCR_BOXES)],+ ]);++ await setupOcrMockApiRoutes(context, ocrDataByAssetId);+ });++ test('OCR boxes scale with zoom', async ({ page }) => {+ await page.goto(`/photos/${fixture.primaryAsset.id}`);+ await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);++ const ocrButton = page.getByLabel('Text recognition');+ await expect(ocrButton).toBeVisible();+ await ocrButton.click();++ const ocrBox = page.locator('[data-viewer-content] [data-testid="ocr-box"]').first();+ await expect(ocrBox).toBeVisible();++ const initialBox = await ocrBox.boundingBox();+ expect(initialBox).toBeTruthy();++ const { width, height } = page.viewportSize()!;+ await page.mouse.move(width / 2, height / 2);+ await page.mouse.wheel(0, -3);++ await expect(async () => {+ const zoomedBox = await ocrBox.boundingBox();+ expect(zoomedBox).toBeTruthy();+ expect(zoomedBox!.width).toBeGreaterThan(initialBox!.width);+ expect(zoomedBox!.height).toBeGreaterThan(initialBox!.height);+ }).toPass({ timeout: 2000 });+ });+});++test.describe('OCR text interaction', () => {+ const fixture = setupAssetViewerFixture(923);++ test.beforeEach(async ({ context }) => {+ const primaryAssetDto = toAssetResponseDto(fixture.primaryAsset);+ const ocrDataByAssetId = new Map<string, AssetOcrResponseDto[]>([+ [primaryAssetDto.id, createMockOcrData(primaryAssetDto.id, PRIMARY_OCR_BOXES)],+ ]);++ await setupOcrMockApiRoutes(context, ocrDataByAssetId);+ });++ test('OCR text box has data-overlay-interactive attribute', async ({ page }) => {+ await page.goto(`/photos/${fixture.primaryAsset.id}`);+ await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);++ await page.getByLabel('Text recognition').click();++ const ocrBox = page.locator('[data-viewer-content] [data-testid="ocr-box"]').first();+ await expect(ocrBox).toBeVisible();+ await expect(ocrBox).toHaveAttribute('data-overlay-interactive');+ });++ test('OCR text box receives focus on click', async ({ page }) => {+ await page.goto(`/photos/${fixture.primaryAsset.id}`);+ await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);++ await page.getByLabel('Text recognition').click();++ const ocrBox = page.locator('[data-viewer-content] [data-testid="ocr-box"]').first();+ await expect(ocrBox).toBeVisible();++ await ocrBox.click();+ await expect(ocrBox).toBeFocused();+ });++ test('dragging on OCR text box does not trigger image pan', async ({ page }) => {+ await page.goto(`/photos/${fixture.primaryAsset.id}`);+ await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);++ await page.getByLabel('Text recognition').click();++ const ocrBox = page.locator('[data-viewer-content] [data-testid="ocr-box"]').first();+ await expect(ocrBox).toBeVisible();++ const imgLocator = page.locator('[data-viewer-content] img[draggable="false"]');+ const initialTransform = await imgLocator.evaluate((element) => {+ return getComputedStyle(element.closest('[style*="transform"]') ?? element).transform;+ });++ const box = await ocrBox.boundingBox();+ expect(box).toBeTruthy();+ const centerX = box!.x + box!.width / 2;+ const centerY = box!.y + box!.height / 2;++ await page.mouse.move(centerX, centerY);+ await page.mouse.down();+ await page.mouse.move(centerX + 50, centerY + 30, { steps: 5 });+ await page.mouse.up();++ const afterTransform = await imgLocator.evaluate((element) => {+ return getComputedStyle(element.closest('[style*="transform"]') ?? element).transform;+ });+ expect(afterTransform).toBe(initialTransform);+ });++ test('split touch gesture across zoom container does not trigger zoom', async ({ page }) => {+ await page.goto(`/photos/${fixture.primaryAsset.id}`);+ await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);++ await page.getByLabel('Text recognition').click();+ const ocrBox = page.locator('[data-viewer-content] [data-testid="ocr-box"]').first();+ await expect(ocrBox).toBeVisible();++ const imgLocator = page.locator('[data-viewer-content] img[draggable="false"]');+ const initialTransform = await imgLocator.evaluate((element) => {+ return getComputedStyle(element.closest('[style*="transform"]') ?? element).transform;+ });++ const viewerContent = page.locator('[data-viewer-content]');+ const viewerBox = await viewerContent.boundingBox();+ expect(viewerBox).toBeTruthy();++ // Dispatch a synthetic split gesture: one touch inside the viewer, one outside+ await page.evaluate(+ ({ viewerCenterX, viewerCenterY, outsideY }) => {+ const viewer = document.querySelector('[data-viewer-content]');+ if (!viewer) {+ return;+ }++ const createTouch = (id: number, x: number, y: number) => {+ return new Touch({+ identifier: id,+ target: viewer,+ clientX: x,+ clientY: y,+ });+ };++ const insideTouch = createTouch(0, viewerCenterX, viewerCenterY);+ const outsideTouch = createTouch(1, viewerCenterX, outsideY);++ const touchStartEvent = new TouchEvent('touchstart', {+ touches: [insideTouch, outsideTouch],+ targetTouches: [insideTouch],+ changedTouches: [insideTouch, outsideTouch],+ bubbles: true,+ cancelable: true,+ });++ const touchMoveEvent = new TouchEvent('touchmove', {+ touches: [createTouch(0, viewerCenterX, viewerCenterY - 30), createTouch(1, viewerCenterX, outsideY + 30)],+ targetTouches: [createTouch(0, viewerCenterX, viewerCenterY - 30)],+ changedTouches: [+ createTouch(0, viewerCenterX, viewerCenterY - 30),+ createTouch(1, viewerCenterX, outsideY + 30),+ ],+ bubbles: true,+ cancelable: true,+ });++ const touchEndEvent = new TouchEvent('touchend', {+ touches: [],+ targetTouches: [],+ changedTouches: [insideTouch, outsideTouch],+ bubbles: true,+ cancelable: true,+ });++ viewer.dispatchEvent(touchStartEvent);+ viewer.dispatchEvent(touchMoveEvent);+ viewer.dispatchEvent(touchEndEvent);+ },+ {+ viewerCenterX: viewerBox!.x + viewerBox!.width / 2,+ viewerCenterY: viewerBox!.y + viewerBox!.height / 2,+ outsideY: 10, // near the top of the page, outside the viewer+ },+ );++ const afterTransform = await imgLocator.evaluate((element) => {+ return getComputedStyle(element.closest('[style*="transform"]') ?? element).transform;+ });+ expect(afterTransform).toBe(initialTransform);+ });+});diff --git a/web/src/lib/actions/zoom-image.ts b/web/src/lib/actions/zoom-image.tsindex 35c3d3a106..07c44569cd 100644--- a/web/src/lib/actions/zoom-image.ts+++ b/web/src/lib/actions/zoom-image.ts@@ -1,11 +1,18 @@import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';import { createZoomImageWheel } from '@zoom-image/core';-export const zoomImageAction = (node: HTMLElement, options?: { disabled?: boolean }) => {+// Minimal touch shape — avoids importing DOM TouchEvent which isn't available in all TS targets.+type TouchEventLike = {+ touches: Iterable<{ clientX: number; clientY: number }> & { length: number };+ targetTouches: ArrayLike<unknown>;+};+const asTouchEvent = (event: Event) => event as unknown as TouchEventLike;++export const zoomImageAction = (node: HTMLElement, options?: { zoomTarget?: HTMLElement }) => {const zoomInstance = createZoomImageWheel(node, {maxZoom: 10,initialState: assetViewerManager.zoomState,- zoomTarget: null,+ zoomTarget: options?.zoomTarget,});const unsubscribes = [@@ -13,47 +20,130 @@ export const zoomImageAction = (node: HTMLElement, options?: { disabled?: booleazoomInstance.subscribe(({ state }) => assetViewerManager.onZoomChange(state)),];- const onInteractionStart = (event: Event) => {- if (options?.disabled) {- event.stopImmediatePropagation();+ const controller = new AbortController();+ const { signal } = controller;++ node.addEventListener('pointerdown', () => assetViewerManager.cancelZoomAnimation(), { capture: true, signal });++ // Intercept events in capture phase to prevent zoom-image from seeing interactions on+ // overlay elements (e.g. OCR text boxes), preserving browser defaults like text selection.+ const isOverlayEvent = (event: Event) => !!(event.target as HTMLElement).closest('[data-overlay-interactive]');+ const isOverlayAtPoint = (x: number, y: number) =>+ !!document.elementFromPoint(x, y)?.closest('[data-overlay-interactive]');++ // Pointer event interception: track pointers that start on overlays and intercept the entire gesture.+ const overlayPointers = new Set<number>();+ const interceptedPointers = new Set<number>();+ const interceptOverlayPointerDown = (event: PointerEvent) => {+ if (isOverlayEvent(event) || isOverlayAtPoint(event.clientX, event.clientY)) {+ overlayPointers.add(event.pointerId);+ interceptedPointers.add(event.pointerId);+ event.stopPropagation();+ } else if (overlayPointers.size > 0) {+ // Split gesture (e.g. pinch with one finger on overlay) — intercept entirely.+ interceptedPointers.add(event.pointerId);+ event.stopPropagation();}- assetViewerManager.cancelZoomAnimation();};+ const interceptOverlayPointerEvent = (event: PointerEvent) => {+ if (interceptedPointers.has(event.pointerId)) {+ event.stopPropagation();+ }+ };+ const interceptOverlayPointerEnd = (event: PointerEvent) => {+ overlayPointers.delete(event.pointerId);+ if (interceptedPointers.delete(event.pointerId)) {+ event.stopPropagation();+ }+ };+ node.addEventListener('pointerdown', interceptOverlayPointerDown, { capture: true, signal });+ node.addEventListener('pointermove', interceptOverlayPointerEvent, { capture: true, signal });+ node.addEventListener('pointerup', interceptOverlayPointerEnd, { capture: true, signal });+ node.addEventListener('pointerleave', interceptOverlayPointerEnd, { capture: true, signal });- node.addEventListener('wheel', onInteractionStart, { capture: true });- node.addEventListener('pointerdown', onInteractionStart, { capture: true });-- // Suppress Safari's synthetic dblclick on double-tap. Without this, zoom-image's touchstart- // handler zooms to maxZoom (10x), then Safari's synthetic dblclick triggers photo-viewer's- // handler which conflicts. Chrome does not fire synthetic dblclick on touch.- let lastPointerWasTouch = false;- const trackPointerType = (event: PointerEvent) => {- lastPointerWasTouch = event.pointerType === 'touch';+ // Touch event interception for overlay touches or split gestures (pinch across container boundary).+ // Once intercepted, stays intercepted until all fingers are lifted.+ let touchGestureIntercepted = false;+ const interceptOverlayTouchEvent = (event: Event) => {+ if (touchGestureIntercepted) {+ event.stopPropagation();+ return;+ }+ const { touches, targetTouches } = asTouchEvent(event);+ if (touches && targetTouches) {+ if (touches.length > targetTouches.length) {+ touchGestureIntercepted = true;+ event.stopPropagation();+ return;+ }+ for (const touch of touches) {+ if (isOverlayAtPoint(touch.clientX, touch.clientY)) {+ touchGestureIntercepted = true;+ event.stopPropagation();+ return;+ }+ }+ } else if (isOverlayEvent(event)) {+ event.stopPropagation();+ }};- const suppressTouchDblClick = (event: MouseEvent) => {- if (lastPointerWasTouch) {- event.stopImmediatePropagation();+ const resetTouchGesture = (event: Event) => {+ const { touches } = asTouchEvent(event);+ if (touches.length === 0) {+ touchGestureIntercepted = false;}};- node.addEventListener('pointerdown', trackPointerType, { capture: true });- node.addEventListener('dblclick', suppressTouchDblClick, { capture: true });+ node.addEventListener('touchstart', interceptOverlayTouchEvent, { capture: true, signal });+ node.addEventListener('touchmove', interceptOverlayTouchEvent, { capture: true, signal });+ node.addEventListener('touchend', resetTouchGesture, { capture: true, signal });++ // Wheel and dblclick interception on overlay elements.+ // Dblclick also intercepted for all touch double-taps (Safari fires synthetic dblclick+ // on double-tap, which conflicts with zoom-image's touch zoom handler).+ let lastPointerWasTouch = false;+ node.addEventListener('pointerdown', (event) => (lastPointerWasTouch = event.pointerType === 'touch'), {+ capture: true,+ signal,+ });+ node.addEventListener(+ 'wheel',+ (event) => {+ if (isOverlayEvent(event)) {+ event.stopPropagation();+ }+ },+ { capture: true, signal },+ );+ node.addEventListener(+ 'dblclick',+ (event) => {+ if (lastPointerWasTouch || isOverlayEvent(event)) {+ event.stopImmediatePropagation();+ }+ },+ { capture: true, signal },+ );- // Allow zoomed content to render outside the container bounds+ if (options?.zoomTarget) {+ options.zoomTarget.style.willChange = 'transform';+ }node.style.overflow = 'visible';- // Prevent browser handling of touch gestures so zoom-image can manage themnode.style.touchAction = 'none';return {- update(newOptions?: { disabled?: boolean }) {+ update(newOptions?: { zoomTarget?: HTMLElement }) {options = newOptions;+ if (newOptions?.zoomTarget !== undefined) {+ zoomInstance.setState({ zoomTarget: newOptions.zoomTarget });+ }},destroy() {+ controller.abort();+ if (options?.zoomTarget) {+ options.zoomTarget.style.willChange = '';+ }for (const unsubscribe of unsubscribes) {unsubscribe();}- node.removeEventListener('wheel', onInteractionStart, { capture: true });- node.removeEventListener('pointerdown', onInteractionStart, { capture: true });- node.removeEventListener('pointerdown', trackPointerType, { capture: true });- node.removeEventListener('dblclick', suppressTouchDblClick, { capture: true });zoomInstance.cleanup();},};diff --git a/web/src/lib/components/AdaptiveImage.svelte b/web/src/lib/components/AdaptiveImage.svelteindex fad4d49d1b..90c9328cf8 100644--- a/web/src/lib/components/AdaptiveImage.svelte+++ b/web/src/lib/components/AdaptiveImage.svelte@@ -7,7 +7,7 @@import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';import { getAssetUrls } from '$lib/utils';import { AdaptiveImageLoader, type QualityList } from '$lib/utils/adaptive-image-loader.svelte';- import { scaleToCover, scaleToFit } from '$lib/utils/container-utils';+ import { scaleToCover, scaleToFit, type Size } from '$lib/utils/container-utils';import { getAltText } from '$lib/utils/thumbnail-util';import { toTimelineAsset } from '$lib/utils/timeline-util';import type { AssetResponseDto, SharedLinkResponseDto } from '@immich/sdk';@@ -17,10 +17,7 @@asset: AssetResponseDto;sharedLink?: SharedLinkResponseDto;objectFit?: 'contain' | 'cover';- container: {- width: number;- height: number;- };+ container: Size;onUrlChange?: (url: string) => void;onImageReady?: () => void;onError?: () => void;@@ -149,81 +146,66 @@(quality.preview === 'success' ? previewElement : undefined) ??(quality.thumbnail === 'success' ? thumbnailElement : undefined);});-- const zoomTransform = $derived.by(() => {- const { currentZoom, currentPositionX, currentPositionY } = assetViewerManager.zoomState;- if (currentZoom === 1 && currentPositionX === 0 && currentPositionY === 0) {- return undefined;- }- return `translate(${currentPositionX}px, ${currentPositionY}px) scale(${currentZoom})`;- });</script><div class="relative h-full w-full overflow-hidden will-change-transform" bind:this={ref}>{@render backdrop?.()}- <!-- pointer-events-none so events pass through to the container where zoom-image listens -->- <div- class="absolute inset-0 pointer-events-none"- style:transform={zoomTransform}- style:transform-origin={zoomTransform ? '0 0' : undefined}- >- <div class="absolute" style:left style:top style:width style:height>- {#if show.alphaBackground}- <AlphaBackground />- {/if}-- {#if show.thumbhash}- {#if asset.thumbhash}- <!-- Thumbhash / spinner layer -->- <canvas use:thumbhash={{ base64ThumbHash: asset.thumbhash }} class="h-full w-full absolute"></canvas>- {:else if show.spinner}- <DelayedLoadingSpinner />- {/if}- {/if}-- {#if show.thumbnail}- <ImageLayer- {adaptiveImageLoader}- {width}- {height}- quality="thumbnail"- src={status.urls.thumbnail}- alt=""- role="presentation"- bind:ref={thumbnailElement}- />- {/if}-- {#if show.brokenAsset}- <BrokenAsset class="text-xl h-full w-full absolute" />- {/if}-- {#if show.preview}- <ImageLayer- {adaptiveImageLoader}- {alt}- {width}- {height}- {overlays}- quality="preview"- src={status.urls.preview}- bind:ref={previewElement}- />- {/if}-- {#if show.original}- <ImageLayer- {adaptiveImageLoader}- {alt}- {width}- {height}- {overlays}- quality="original"- src={status.urls.original}- bind:ref={originalElement}- />+ <div class="absolute inset-0 pointer-events-none" style:left style:top style:width style:height>+ {#if show.alphaBackground}+ <AlphaBackground />+ {/if}++ {#if show.thumbhash}+ {#if asset.thumbhash}+ <!-- Thumbhash / spinner layer -->+ <canvas use:thumbhash={{ base64ThumbHash: asset.thumbhash }} class="h-full w-full absolute"></canvas>+ {:else if show.spinner}+ <DelayedLoadingSpinner />{/if}- </div>+ {/if}++ {#if show.thumbnail}+ <ImageLayer+ {adaptiveImageLoader}+ {width}+ {height}+ quality="thumbnail"+ src={status.urls.thumbnail}+ alt=""+ role="presentation"+ bind:ref={thumbnailElement}+ />+ {/if}++ {#if show.brokenAsset}+ <BrokenAsset class="text-xl h-full w-full absolute" />+ {/if}++ {#if show.preview}+ <ImageLayer+ {adaptiveImageLoader}+ {alt}+ {width}+ {height}+ {overlays}+ quality="preview"+ src={status.urls.preview}+ bind:ref={previewElement}+ />+ {/if}++ {#if show.original}+ <ImageLayer+ {adaptiveImageLoader}+ {alt}+ {width}+ {height}+ {overlays}+ quality="original"+ src={status.urls.original}+ bind:ref={originalElement}+ />+ {/if}</div></div>diff --git a/web/src/lib/components/asset-viewer/face-editor/face-editor.svelte b/web/src/lib/components/asset-viewer/face-editor/face-editor.svelteindex ea2babfc11..8f7103c2a3 100644--- a/web/src/lib/components/asset-viewer/face-editor/face-editor.svelte+++ b/web/src/lib/components/asset-viewer/face-editor/face-editor.svelte@@ -12,12 +12,12 @@import { onDestroy, onMount, tick } from 'svelte';import { t } from 'svelte-i18n';- interface Props {+ type Props = {htmlElement: HTMLImageElement | HTMLVideoElement;containerWidth: number;containerHeight: number;assetId: string;- }+ };let { htmlElement, containerWidth, containerHeight, assetId }: Props = $props();@@ -304,6 +304,7 @@<divid="face-editor-data"class="absolute start-0 top-0 z-5 h-full w-full overflow-hidden"+ data-overlay-interactivedata-face-left={faceBoxPosition.left}data-face-top={faceBoxPosition.top}data-face-width={faceBoxPosition.width}diff --git a/web/src/lib/components/asset-viewer/ocr-bounding-box.svelte b/web/src/lib/components/asset-viewer/ocr-bounding-box.svelteindex d5551b9cc5..c2ee1ba193 100644--- a/web/src/lib/components/asset-viewer/ocr-bounding-box.svelte+++ b/web/src/lib/components/asset-viewer/ocr-bounding-box.svelte@@ -1,4 +1,5 @@<script lang="ts">+ import { mediaQueryManager } from '$lib/stores/media-query-manager.svelte';import type { OcrBox } from '$lib/utils/ocr-utils';import { calculateBoundingBoxMatrix, calculateFittedFontSize } from '$lib/utils/ocr-utils';@@ -8,6 +9,7 @@let { ocrBox }: Props = $props();+ const isTouch = $derived(mediaQueryManager.pointerCoarse);const dimensions = $derived(calculateBoundingBoxMatrix(ocrBox.points));const transform = $derived(`matrix3d(${dimensions.matrix.join(',')})`);@@ -15,13 +17,23 @@calculateFittedFontSize(ocrBox.text, dimensions.width, dimensions.height, ocrBox.verticalMode) + 'px',);+ const handleSelectStart = (event: Event) => {+ const target = event.currentTarget as HTMLElement;+ requestAnimationFrame(() => {+ const selection = globalThis.getSelection();+ if (selection) {+ selection.selectAllChildren(target);+ }+ });+ };+const verticalStyle = $derived.by(() => {switch (ocrBox.verticalMode) {case 'cjk': {- return ' writing-mode: vertical-rl;';+ return 'writing-mode: vertical-rl;';}case 'rotated': {- return ' writing-mode: vertical-rl; text-orientation: sideways;';+ return 'writing-mode: vertical-rl; text-orientation: sideways;';}default: {return '';@@ -30,17 +42,23 @@});</script>-<div class="absolute left-0 top-0">- <div- class="absolute flex items-center justify-center text-transparent border-2 border-blue-500 bg-blue-500/10 pointer-events-auto cursor-text select-text transition-colors hover:z-1 hover:text-white hover:bg-black/60 hover:border-blue-600 hover:border-3 focus:z-1 focus:text-white focus:bg-black/60 focus:border-blue-600 focus:border-3 focus:outline-none {ocrBox.verticalMode ===- 'none'- ? 'px-2 py-1 whitespace-nowrap'- : 'px-1 py-2'}"- style="font-size: {fontSize}; width: {dimensions.width}px; height: {dimensions.height}px; transform: {transform}; transform-origin: 0 0;{verticalStyle}"- tabindex="0"- role="button"- aria-label={ocrBox.text}- >- {ocrBox.text}- </div>+<div+ class={[+ 'absolute left-0 top-0 flex items-center justify-center',+ 'border-2 border-blue-500 pointer-events-auto cursor-text',+ 'focus:z-1 focus:border-blue-600 focus:border-3 focus:outline-none',+ isTouch+ ? 'text-white bg-black/60 select-all'+ : 'select-text text-transparent bg-blue-500/10 transition-colors hover:z-1 hover:text-white hover:bg-black/60 hover:border-blue-600 hover:border-3',+ ocrBox.verticalMode === 'none' ? 'px-2 py-1 whitespace-nowrap' : 'px-1 py-2',+ ]}+ style="font-size: {fontSize}; width: {dimensions.width}px; height: {dimensions.height}px; transform: {transform}; transform-origin: 0 0; touch-action: none; {verticalStyle}"+ data-testid="ocr-box"+ data-overlay-interactive+ tabindex="0"+ role="button"+ aria-label={ocrBox.text}+ onselectstart={isTouch ? handleSelectStart : undefined}+>+ {ocrBox.text}</div>diff --git a/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte b/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelteindex 926383d9c2..d46b5e0dc1 100644--- a/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte+++ b/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte@@ -128,10 +128,8 @@}const boxes = getOcrBoundingBoxes(ocrData, {- contentWidth: viewer.state.textureData.panoData.croppedWidth,- contentHeight: viewer.state.textureData.panoData.croppedHeight,- offsetX: 0,- offsetY: 0,+ width: viewer.state.textureData.panoData.croppedWidth,+ height: viewer.state.textureData.panoData.croppedHeight,});for (const [index, box] of boxes.entries()) {diff --git a/web/src/lib/components/asset-viewer/photo-viewer.svelte b/web/src/lib/components/asset-viewer/photo-viewer.svelteindex a42eab6c4f..a3c560ef8d 100644--- a/web/src/lib/components/asset-viewer/photo-viewer.svelte+++ b/web/src/lib/components/asset-viewer/photo-viewer.svelte@@ -13,7 +13,7 @@import { SlideshowLook, SlideshowState, slideshowStore } from '$lib/stores/slideshow.store';import { handlePromiseError } from '$lib/utils';import { canCopyImageToClipboard, copyImageToClipboard } from '$lib/utils/asset-utils';- import { getNaturalSize, scaleToFit, type ContentMetrics } from '$lib/utils/container-utils';+ import { getNaturalSize, scaleToFit, type Size } from '$lib/utils/container-utils';import { handleError } from '$lib/utils/handle-error';import { getOcrBoundingBoxes } from '$lib/utils/ocr-utils';import { getBoundingBox, type BoundingBox } from '$lib/utils/people-utils';@@ -24,14 +24,14 @@import { t } from 'svelte-i18n';import type { AssetCursor } from './asset-viewer.svelte';- interface Props {+ type Props = {cursor: AssetCursor;element?: HTMLDivElement;sharedLink?: SharedLinkResponseDto;onReady?: () => void;onError?: () => void;onSwipe?: (event: SwipeCustomEvent) => void;- }+ };let { cursor, element = $bindable(), sharedLink, onReady, onError, onSwipe }: Props = $props();@@ -66,23 +66,15 @@height: containerHeight,});- const overlayMetrics = $derived.by((): ContentMetrics => {+ const overlaySize = $derived.by((): Size => {if (!assetViewerManager.imgRef || !visibleImageReady) {- return { contentWidth: 0, contentHeight: 0, offsetX: 0, offsetY: 0 };+ return { width: 0, height: 0 };}- const natural = getNaturalSize(assetViewerManager.imgRef);- const scaled = scaleToFit(natural, { width: containerWidth, height: containerHeight });-- return {- contentWidth: scaled.width,- contentHeight: scaled.height,- offsetX: 0,- offsetY: 0,- };+ return scaleToFit(getNaturalSize(assetViewerManager.imgRef), { width: containerWidth, height: containerHeight });});- const highlightedBoxes = $derived(getBoundingBox($boundingBoxesArray, overlayMetrics));+ const highlightedBoxes = $derived(getBoundingBox($boundingBoxesArray, overlaySize));const isHighlighting = $derived(highlightedBoxes.length > 0);let visibleBoxes = $state<BoundingBox[]>([]);@@ -94,7 +86,7 @@}});- const ocrBoxes = $derived(ocrManager.showOverlay ? getOcrBoundingBoxes(ocrManager.data, overlayMetrics) : []);+ const ocrBoxes = $derived(ocrManager.showOverlay ? getOcrBoundingBoxes(ocrManager.data, overlaySize) : []);const onCopy = async () => {if (!canCopyImageToClipboard() || !assetViewerManager.imgRef) {@@ -162,6 +154,8 @@$slideshowState !== SlideshowState.None && $slideshowLook === SlideshowLook.BlurredBackground && !!asset.thumbhash,);+ let adaptiveImage = $state<HTMLDivElement | undefined>();+const faceToNameMap = $derived.by(() => {// eslint-disable-next-line svelte/prefer-svelte-reactivityconst map = new Map<Faces, string>();@@ -192,7 +186,7 @@const mouseX = (event.clientX - containerRect.left - contentOffsetX * currentZoom - currentPositionX) / currentZoom;const mouseY = (event.clientY - containerRect.top - contentOffsetY * currentZoom - currentPositionY) / currentZoom;- const faceBoxes = getBoundingBox(faces, overlayMetrics);+ const faceBoxes = getBoundingBox(faces, overlaySize);for (const [index, box] of faceBoxes.entries()) {if (mouseX >= box.left && mouseX <= box.left + box.width && mouseY >= box.top && mouseY <= box.top + box.height) {@@ -226,7 +220,7 @@ondblclick={onZoom}onmousemove={handleImageMouseMove}onmouseleave={handleImageMouseLeave}- use:zoomImageAction={{ disabled: assetViewerManager.isFaceEditMode || ocrManager.showOverlay }}+ use:zoomImageAction={{ zoomTarget: adaptiveImage }}{...useSwipe((event) => onSwipe?.(event))}><AdaptiveImage@@ -244,6 +238,7 @@onReady?.();}}bind:imgRef={assetViewerManager.imgRef}+ bind:ref={adaptiveImage}>{#snippet backdrop()}{#if blurredSlideshow}diff --git a/web/src/lib/utils/container-utils.spec.ts b/web/src/lib/utils/container-utils.spec.tsindex 802ed24e40..d6a1efbe6a 100644--- a/web/src/lib/utils/container-utils.spec.ts+++ b/web/src/lib/utils/container-utils.spec.ts@@ -1,4 +1,11 @@-import { getContentMetrics, getNaturalSize, scaleToFit } from '$lib/utils/container-utils';+import {+ getContentMetrics,+ getNaturalSize,+ mapNormalizedRectToContent,+ mapNormalizedToContent,+ scaleToCover,+ scaleToFit,+} from '$lib/utils/container-utils';const mockImage = (props: {naturalWidth: number;@@ -92,3 +99,81 @@ describe('getNaturalSize', () => {expect(getNaturalSize(video)).toEqual({ width: 1920, height: 1080 });});});++describe('scaleToCover', () => {+ it('should scale up to cover container when image is smaller', () => {+ expect(scaleToCover({ width: 400, height: 300 }, { width: 800, height: 600 })).toEqual({+ width: 800,+ height: 600,+ });+ });++ it('should use height scale when image is wider than container', () => {+ expect(scaleToCover({ width: 2000, height: 1000 }, { width: 800, height: 600 })).toEqual({+ width: 1200,+ height: 600,+ });+ });++ it('should use width scale when image is taller than container', () => {+ expect(scaleToCover({ width: 1000, height: 2000 }, { width: 800, height: 600 })).toEqual({+ width: 800,+ height: 1600,+ });+ });+});++describe('mapNormalizedToContent', () => {+ const metrics = { contentWidth: 800, contentHeight: 400, offsetX: 0, offsetY: 100 };++ it('should map top-left corner', () => {+ expect(mapNormalizedToContent({ x: 0, y: 0 }, metrics)).toEqual({ x: 0, y: 100 });+ });++ it('should map bottom-right corner', () => {+ expect(mapNormalizedToContent({ x: 1, y: 1 }, metrics)).toEqual({ x: 800, y: 500 });+ });++ it('should map center point', () => {+ expect(mapNormalizedToContent({ x: 0.5, y: 0.5 }, metrics)).toEqual({ x: 400, y: 300 });+ });++ it('should apply offsets correctly for letterboxed content', () => {+ const letterboxed = { contentWidth: 300, contentHeight: 600, offsetX: 250, offsetY: 0 };+ expect(mapNormalizedToContent({ x: 0, y: 0 }, letterboxed)).toEqual({ x: 250, y: 0 });+ expect(mapNormalizedToContent({ x: 1, y: 1 }, letterboxed)).toEqual({ x: 550, y: 600 });+ });++ it('should accept Size (zero offsets)', () => {+ const size = { width: 800, height: 400 };+ expect(mapNormalizedToContent({ x: 0, y: 0 }, size)).toEqual({ x: 0, y: 0 });+ expect(mapNormalizedToContent({ x: 1, y: 1 }, size)).toEqual({ x: 800, y: 400 });+ expect(mapNormalizedToContent({ x: 0.5, y: 0.5 }, size)).toEqual({ x: 400, y: 200 });+ });+});++describe('mapNormalizedRectToContent', () => {+ const metrics = { contentWidth: 800, contentHeight: 400, offsetX: 0, offsetY: 100 };++ it('should map a normalized rect to content pixel coordinates', () => {+ const rect = mapNormalizedRectToContent({ x: 0.25, y: 0.25 }, { x: 0.75, y: 0.75 }, metrics);+ expect(rect).toEqual({ left: 200, top: 200, width: 400, height: 200 });+ });++ it('should map full image rect', () => {+ const rect = mapNormalizedRectToContent({ x: 0, y: 0 }, { x: 1, y: 1 }, metrics);+ expect(rect).toEqual({ left: 0, top: 100, width: 800, height: 400 });+ });++ it('should handle letterboxed content with horizontal offsets', () => {+ const letterboxed = { contentWidth: 300, contentHeight: 600, offsetX: 250, offsetY: 0 };+ const rect = mapNormalizedRectToContent({ x: 0, y: 0 }, { x: 1, y: 1 }, letterboxed);+ expect(rect).toEqual({ left: 250, top: 0, width: 300, height: 600 });+ });++ it('should accept Size (zero offsets)', () => {+ const size = { width: 800, height: 400 };+ const rect = mapNormalizedRectToContent({ x: 0.25, y: 0.25 }, { x: 0.75, y: 0.75 }, size);+ expect(rect).toEqual({ left: 200, top: 100, width: 400, height: 200 });+ });+});diff --git a/web/src/lib/utils/container-utils.ts b/web/src/lib/utils/container-utils.tsindex ffa2fae769..36e260fcc7 100644--- a/web/src/lib/utils/container-utils.ts+++ b/web/src/lib/utils/container-utils.ts@@ -1,14 +1,35 @@-export interface ContentMetrics {+// Coordinate spaces used throughout the viewer:+//+// "Normalized": 0–1 range, (0,0) = top-left, (1,1) = bottom-right. Resolution-independent.+// Example: OCR coordinates, or face coords after dividing by metadata dimensions.+//+// "Content": pixel position within the container after scaling (scaleToFit/scaleToCover)+// and centering. Used for DOM overlay positioning (face boxes, OCR text).+//+// "Natural": pixel position in the original full-resolution image file (e.g. 4000×3000).+// Used when cropping or drawing on the source image.+//+// "Metadata pixel space": coordinates from face detection / OCR models, in pixels relative+// to face.imageWidth/imageHeight. Divide by those dimensions to get normalized coords.++export type Point = {+ x: number;+ y: number;+};++export type Size = {+ width: number;+ height: number;+};++export type ContentMetrics = {contentWidth: number;contentHeight: number;offsetX: number;offsetY: number;-}+};-export const scaleToCover = (- dimensions: { width: number; height: number },- container: { width: number; height: number },-): { width: number; height: number } => {+export const scaleToCover = (dimensions: Size, container: Size): Size => {const scaleX = container.width / dimensions.width;const scaleY = container.height / dimensions.height;const scale = Math.max(scaleX, scaleY);@@ -18,10 +39,7 @@ export const scaleToCover = (};};-export const scaleToFit = (- dimensions: { width: number; height: number },- container: { width: number; height: number },-): { width: number; height: number } => {+export const scaleToFit = (dimensions: Size, container: Size): Size => {const scaleX = container.width / dimensions.width;const scaleY = container.height / dimensions.height;const scale = Math.min(scaleX, scaleY);@@ -31,14 +49,14 @@ export const scaleToFit = (};};-const getElementSize = (element: HTMLImageElement | HTMLVideoElement): { width: number; height: number } => {+const getElementSize = (element: HTMLImageElement | HTMLVideoElement): Size => {if (element instanceof HTMLVideoElement) {return { width: element.clientWidth, height: element.clientHeight };}return { width: element.width, height: element.height };};-export const getNaturalSize = (element: HTMLImageElement | HTMLVideoElement): { width: number; height: number } => {+export const getNaturalSize = (element: HTMLImageElement | HTMLVideoElement): Size => {if (element instanceof HTMLVideoElement) {return { width: element.videoWidth, height: element.videoHeight };}@@ -56,3 +74,38 @@ export const getContentMetrics = (element: HTMLImageElement | HTMLVideoElement):offsetY: (client.height - contentHeight) / 2,};};++export function mapNormalizedToContent(point: Point, sizeOrMetrics: Size | ContentMetrics): Point {+ if ('contentWidth' in sizeOrMetrics) {+ return {+ x: point.x * sizeOrMetrics.contentWidth + sizeOrMetrics.offsetX,+ y: point.y * sizeOrMetrics.contentHeight + sizeOrMetrics.offsetY,+ };+ }+ return {+ x: point.x * sizeOrMetrics.width,+ y: point.y * sizeOrMetrics.height,+ };+}++export type Rect = {+ top: number;+ left: number;+ width: number;+ height: number;+};++export function mapNormalizedRectToContent(+ topLeft: Point,+ bottomRight: Point,+ sizeOrMetrics: Size | ContentMetrics,+): Rect {+ const tl = mapNormalizedToContent(topLeft, sizeOrMetrics);+ const br = mapNormalizedToContent(bottomRight, sizeOrMetrics);+ return {+ top: tl.y,+ left: tl.x,+ width: br.x - tl.x,+ height: br.y - tl.y,+ };+}diff --git a/web/src/lib/utils/ocr-utils.spec.ts b/web/src/lib/utils/ocr-utils.spec.tsindex c3ce70394d..c88936fddf 100644--- a/web/src/lib/utils/ocr-utils.spec.ts+++ b/web/src/lib/utils/ocr-utils.spec.ts@@ -1,5 +1,5 @@import type { OcrBoundingBox } from '$lib/stores/ocr.svelte';-import type { ContentMetrics } from '$lib/utils/container-utils';+import type { Size } from '$lib/utils/container-utils';import { getOcrBoundingBoxes } from '$lib/utils/ocr-utils';describe('getOcrBoundingBoxes', () => {@@ -21,9 +21,9 @@ describe('getOcrBoundingBoxes', () => {text: 'hello',},];- const metrics: ContentMetrics = { contentWidth: 1000, contentHeight: 500, offsetX: 0, offsetY: 0 };+ const imageSize: Size = { width: 1000, height: 500 };- const boxes = getOcrBoundingBoxes(ocrData, metrics);+ const boxes = getOcrBoundingBoxes(ocrData, imageSize);expect(boxes).toHaveLength(1);expect(boxes[0].id).toBe('box1');@@ -37,7 +37,7 @@ describe('getOcrBoundingBoxes', () => {]);});- it('should apply offsets for letterboxed images', () => {+ it('should map full-image box to full display area', () => {const ocrData: OcrBoundingBox[] = [{id: 'box1',@@ -55,21 +55,20 @@ describe('getOcrBoundingBoxes', () => {text: 'test',},];- const metrics: ContentMetrics = { contentWidth: 600, contentHeight: 400, offsetX: 100, offsetY: 50 };+ const imageSize: Size = { width: 600, height: 400 };- const boxes = getOcrBoundingBoxes(ocrData, metrics);+ const boxes = getOcrBoundingBoxes(ocrData, imageSize);expect(boxes[0].points).toEqual([- { x: 100, y: 50 },- { x: 700, y: 50 },- { x: 700, y: 450 },- { x: 100, y: 450 },+ { x: 0, y: 0 },+ { x: 600, y: 0 },+ { x: 600, y: 400 },+ { x: 0, y: 400 },]);});it('should return empty array for empty input', () => {- const metrics: ContentMetrics = { contentWidth: 800, contentHeight: 600, offsetX: 0, offsetY: 0 };- expect(getOcrBoundingBoxes([], metrics)).toEqual([]);+ expect(getOcrBoundingBoxes([], { width: 800, height: 600 })).toEqual([]);});it('should handle multiple boxes', () => {@@ -105,9 +104,9 @@ describe('getOcrBoundingBoxes', () => {text: 'second',},];- const metrics: ContentMetrics = { contentWidth: 200, contentHeight: 200, offsetX: 0, offsetY: 0 };+ const imageSize: Size = { width: 200, height: 200 };- const boxes = getOcrBoundingBoxes(ocrData, metrics);+ const boxes = getOcrBoundingBoxes(ocrData, imageSize);expect(boxes).toHaveLength(2);expect(boxes[0].text).toBe('first');diff --git a/web/src/lib/utils/ocr-utils.ts b/web/src/lib/utils/ocr-utils.tsindex c483eb9551..2bf23bff2d 100644--- a/web/src/lib/utils/ocr-utils.ts+++ b/web/src/lib/utils/ocr-utils.ts@@ -1,23 +1,19 @@import type { OcrBoundingBox } from '$lib/stores/ocr.svelte';-import type { ContentMetrics } from '$lib/utils/container-utils';+import { mapNormalizedToContent, type Point, type Size } from '$lib/utils/container-utils';import { clamp } from 'lodash-es';--export type Point = {- x: number;- y: number;-};+export type { Point } from '$lib/utils/container-utils';const distance = (p1: Point, p2: Point) => Math.hypot(p2.x - p1.x, p2.y - p1.y);export type VerticalMode = 'none' | 'cjk' | 'rotated';-export interface OcrBox {+export type OcrBox = {id: string;points: Point[];text: string;confidence: number;verticalMode: VerticalMode;-}+};const CJK_PATTERN =/[\u3000-\u303F\u3040-\u309F\u30A0-\u30FF\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\uAC00-\uD7AF\uFF00-\uFFEF]/;@@ -38,7 +34,7 @@ const getVerticalMode = (width: number, height: number, text: string): VerticalM* @param points - Array of 4 corner points of the bounding box* @returns 4x4 matrix to transform the div with text onto the polygon defined by the corner points, and size to set on the source div.*/-export const calculateBoundingBoxMatrix = (points: Point[]): { matrix: number[]; width: number; height: number } => {+export const calculateBoundingBoxMatrix = (points: Point[]): Size & { matrix: number[] } => {const [topLeft, topRight, bottomRight, bottomLeft] = points;const width = Math.max(distance(topLeft, topRight), distance(bottomLeft, bottomRight));@@ -163,7 +159,7 @@ export const calculateFittedFontSize = (return clamp(Math.min(scaleFromWidth, scaleFromHeight), MIN_FONT_SIZE, MAX_FONT_SIZE);};-export const getOcrBoundingBoxes = (ocrData: OcrBoundingBox[], metrics: ContentMetrics): OcrBox[] => {+export const getOcrBoundingBoxes = (ocrData: OcrBoundingBox[], imageSize: Size): OcrBox[] => {const boxes: OcrBox[] = [];for (const ocr of ocrData) {const points = [@@ -171,10 +167,7 @@ export const getOcrBoundingBoxes = (ocrData: OcrBoundingBox[], metrics: ContentM{ x: ocr.x2, y: ocr.y2 },{ x: ocr.x3, y: ocr.y3 },{ x: ocr.x4, y: ocr.y4 },- ].map((point) => ({- x: point.x * metrics.contentWidth + metrics.offsetX,- y: point.y * metrics.contentHeight + metrics.offsetY,- }));+ ].map((point) => mapNormalizedToContent(point, imageSize));const boxWidth = Math.max(distance(points[0], points[1]), distance(points[3], points[2]));const boxHeight = Math.max(distance(points[0], points[3]), distance(points[1], points[2]));@@ -188,7 +181,7 @@ export const getOcrBoundingBoxes = (ocrData: OcrBoundingBox[], metrics: ContentM});}- const rowThreshold = metrics.contentHeight * 0.02;+ const rowThreshold = imageSize.height * 0.02;boxes.sort((a, b) => {const yDifference = a.points[0].y - b.points[0].y;if (Math.abs(yDifference) < rowThreshold) {diff --git a/web/src/lib/utils/people-utils.spec.ts b/web/src/lib/utils/people-utils.spec.tsindex 80371bd9c4..f27a1855b5 100644--- a/web/src/lib/utils/people-utils.spec.ts+++ b/web/src/lib/utils/people-utils.spec.ts@@ -1,5 +1,5 @@import type { Faces } from '$lib/stores/people.store';-import type { ContentMetrics } from '$lib/utils/container-utils';+import type { Size } from '$lib/utils/container-utils';import { getBoundingBox } from '$lib/utils/people-utils';const makeFace = (overrides: Partial<Faces> = {}): Faces => ({@@ -16,21 +16,21 @@ const makeFace = (overrides: Partial<Faces> = {}): Faces => ({describe('getBoundingBox', () => {it('should scale face coordinates to display dimensions', () => {const face = makeFace();- const metrics: ContentMetrics = { contentWidth: 800, contentHeight: 600, offsetX: 0, offsetY: 0 };+ const imageSize: Size = { width: 800, height: 600 };- const boxes = getBoundingBox([face], metrics);+ const boxes = getBoundingBox([face], imageSize);expect(boxes).toHaveLength(1);expect(boxes[0]).toEqual({id: 'face-1',- top: Math.round(600 * (750 / 3000)),- left: Math.round(800 * (1000 / 4000)),- width: Math.round(800 * (2000 / 4000) - 800 * (1000 / 4000)),- height: Math.round(600 * (1500 / 3000) - 600 * (750 / 3000)),+ top: 600 * (750 / 3000),+ left: 800 * (1000 / 4000),+ width: 800 * (2000 / 4000) - 800 * (1000 / 4000),+ height: 600 * (1500 / 3000) - 600 * (750 / 3000),});});- it('should apply offsets for letterboxed display', () => {+ it('should map full-image face to full display area', () => {const face = makeFace({imageWidth: 1000,imageHeight: 1000,@@ -39,49 +39,21 @@ describe('getBoundingBox', () => {boundingBoxX2: 1000,boundingBoxY2: 1000,});- const metrics: ContentMetrics = { contentWidth: 600, contentHeight: 600, offsetX: 100, offsetY: 0 };+ const imageSize: Size = { width: 600, height: 600 };- const boxes = getBoundingBox([face], metrics);+ const boxes = getBoundingBox([face], imageSize);expect(boxes[0]).toEqual({id: 'face-1',top: 0,- left: 100,+ left: 0,width: 600,height: 600,});});- it('should handle zoom by pre-scaled metrics', () => {- const face = makeFace({- imageWidth: 1000,- imageHeight: 1000,- boundingBoxX1: 0,- boundingBoxY1: 0,- boundingBoxX2: 500,- boundingBoxY2: 500,- });- const metrics: ContentMetrics = {- contentWidth: 1600,- contentHeight: 1200,- offsetX: -200,- offsetY: -100,- };-- const boxes = getBoundingBox([face], metrics);-- expect(boxes[0]).toEqual({- id: 'face-1',- top: -100,- left: -200,- width: 800,- height: 600,- });- });-it('should return empty array for empty faces', () => {- const metrics: ContentMetrics = { contentWidth: 800, contentHeight: 600, offsetX: 0, offsetY: 0 };- expect(getBoundingBox([], metrics)).toEqual([]);+ expect(getBoundingBox([], { width: 800, height: 600 })).toEqual([]);});it('should handle multiple faces', () => {@@ -89,9 +61,8 @@ describe('getBoundingBox', () => {makeFace({ id: 'face-1', boundingBoxX1: 0, boundingBoxY1: 0, boundingBoxX2: 1000, boundingBoxY2: 1000 }),makeFace({ id: 'face-2', boundingBoxX1: 2000, boundingBoxY1: 1500, boundingBoxX2: 3000, boundingBoxY2: 2500 }),];- const metrics: ContentMetrics = { contentWidth: 800, contentHeight: 600, offsetX: 0, offsetY: 0 };- const boxes = getBoundingBox(faces, metrics);+ const boxes = getBoundingBox(faces, { width: 800, height: 600 });expect(boxes).toHaveLength(2);expect(boxes[0].left).toBeLessThan(boxes[1].left);diff --git a/web/src/lib/utils/people-utils.ts b/web/src/lib/utils/people-utils.tsindex b8fb8973e6..f7f9f4ee42 100644--- a/web/src/lib/utils/people-utils.ts+++ b/web/src/lib/utils/people-utils.ts@@ -1,37 +1,21 @@import type { Faces } from '$lib/stores/people.store';import { getAssetMediaUrl } from '$lib/utils';-import type { ContentMetrics } from '$lib/utils/container-utils';+import { mapNormalizedRectToContent, type Rect, type Size } from '$lib/utils/container-utils';import { AssetTypeEnum, type AssetFaceResponseDto } from '@immich/sdk';-export interface BoundingBox {- id: string;- top: number;- left: number;- width: number;- height: number;-}+export type BoundingBox = Rect & { id: string };-export const getBoundingBox = (faces: Faces[], metrics: ContentMetrics): BoundingBox[] => {+export const getBoundingBox = (faces: Faces[], imageSize: Size): BoundingBox[] => {const boxes: BoundingBox[] = [];for (const face of faces) {- const scaleX = metrics.contentWidth / face.imageWidth;- const scaleY = metrics.contentHeight / face.imageHeight;+ const rect = mapNormalizedRectToContent(+ { x: face.boundingBoxX1 / face.imageWidth, y: face.boundingBoxY1 / face.imageHeight },+ { x: face.boundingBoxX2 / face.imageWidth, y: face.boundingBoxY2 / face.imageHeight },+ imageSize,+ );- const coordinates = {- x1: scaleX * face.boundingBoxX1 + metrics.offsetX,- x2: scaleX * face.boundingBoxX2 + metrics.offsetX,- y1: scaleY * face.boundingBoxY1 + metrics.offsetY,- y2: scaleY * face.boundingBoxY2 + metrics.offsetY,- };-- boxes.push({- id: face.id,- top: Math.round(coordinates.y1),- left: Math.round(coordinates.x1),- width: Math.round(coordinates.x2 - coordinates.x1),- height: Math.round(coordinates.y2 - coordinates.y1),- });+ boxes.push({ id: face.id, ...rect });}return boxes;