High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat(web): wasm justified layout (#19150)
Mertbc062da11b18
diff --git a/web/eslint.config.js b/web/eslint.config.jsindex 6b7b343ad1..78b87d24ef 100644--- a/web/eslint.config.js+++ b/web/eslint.config.js@@ -118,6 +118,7 @@ export default typescriptEslint.config('unicorn/filename-case': 'off','unicorn/prefer-top-level-await': 'off','unicorn/import-style': 'off',+ 'unicorn/no-for-loop': 'off','svelte/button-has-type': 'error','@typescript-eslint/await-thenable': 'error','@typescript-eslint/no-floating-promises': 'error',diff --git a/web/package-lock.json b/web/package-lock.jsonindex 72b9eb6a7c..2888dfb71b 100644--- a/web/package-lock.json+++ b/web/package-lock.json@@ -10,6 +10,7 @@"license": "GNU Affero General Public License version 3","dependencies": {"@formatjs/icu-messageformat-parser": "^2.9.8",+ "@immich/justified-layout-wasm": "^0.3.0","@immich/sdk": "file:../open-api/typescript-sdk","@immich/ui": "^0.22.7","@mapbox/mapbox-gl-rtl-text": "0.2.3",@@ -1328,6 +1329,12 @@"url": "https://opencollective.com/libvips"}},+ "node_modules/@immich/justified-layout-wasm": {+ "version": "0.3.0",+ "resolved": "https://registry.npmjs.org/@immich/justified-layout-wasm/-/justified-layout-wasm-0.3.0.tgz",+ "integrity": "sha512-eiKaPHHVsm0YL8SZVUuEs8miTT2uF3b9Tggve7QvIh+KF1Vq41EZFUeDI0RzLvoXAUnoAh37iFvYxaA9iXMHZA==",+ "license": "AGPL-3"+ },"node_modules/@immich/sdk": {"resolved": "../open-api/typescript-sdk","link": truediff --git a/web/package.json b/web/package.jsonindex a06b12f826..2665904c62 100644--- a/web/package.json+++ b/web/package.json@@ -27,6 +27,7 @@},"dependencies": {"@formatjs/icu-messageformat-parser": "^2.9.8",+ "@immich/justified-layout-wasm": "^0.3.0","@immich/sdk": "file:../open-api/typescript-sdk","@immich/ui": "^0.22.7","@mapbox/mapbox-gl-rtl-text": "0.2.3",diff --git a/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte b/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelteindex 09998ed060..eb1869cfbb 100644--- a/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte+++ b/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte@@ -16,7 +16,7 @@import { archiveAssets, cancelMultiselect } from '$lib/utils/asset-utils';import { moveFocus } from '$lib/utils/focus-util';import { handleError } from '$lib/utils/handle-error';- import { getJustifiedLayoutFromAssets, type CommonJustifiedLayout } from '$lib/utils/layout-utils';+ import { getJustifiedLayoutFromAssets } from '$lib/utils/layout-utils';import { navigate } from '$lib/utils/navigation';import { isTimelineAsset, toTimelineAsset } from '$lib/utils/timeline-util';import { AssetVisibility, type AssetResponseDto } from '@immich/sdk';@@ -27,7 +27,7 @@import Portal from '../portal/portal.svelte';interface Props {- assets: (TimelineAsset | AssetResponseDto)[];+ assets: TimelineAsset[] | AssetResponseDto[];assetInteraction: AssetInteraction;disableAssetSelect?: boolean;showArchiveIcon?: boolean;@@ -62,91 +62,39 @@let { isViewing: isViewerOpen, asset: viewingAsset, setAssetId } = assetViewingStore;- let geometry: CommonJustifiedLayout | undefined = $state();-- $effect(() => {- const _assets = assets;- updateSlidingWindow();-- const rowWidth = Math.floor(viewport.width);- const rowHeight = rowWidth < 850 ? 100 : 235;-- geometry = getJustifiedLayoutFromAssets(_assets, {+ const geometry = $derived(+ getJustifiedLayoutFromAssets(assets, {spacing: 2,- heightTolerance: 0.15,- rowHeight,- rowWidth,- });- });-- let assetLayouts = $derived.by(() => {- const assetLayout = [];- let containerHeight = 0;- let containerWidth = 0;- if (geometry) {- containerHeight = geometry.containerHeight;- containerWidth = geometry.containerWidth;- for (const [index, asset] of assets.entries()) {- const top = geometry.getTop(index);- const left = geometry.getLeft(index);- const width = geometry.getWidth(index);- const height = geometry.getHeight(index);-- const layoutTopWithOffset = top + pageHeaderOffset;- const layoutBottom = layoutTopWithOffset + height;-- const display = layoutTopWithOffset < slidingWindow.bottom && layoutBottom > slidingWindow.top;-- const layout = {- asset,- top,- left,- width,- height,- display,- };-- assetLayout.push(layout);- }- }-- return {- assetLayout,- containerHeight,- containerWidth,- };- });+ heightTolerance: 0.3,+ rowHeight: Math.floor(viewport.width) < 850 ? 100 : 235,+ rowWidth: Math.floor(viewport.width),+ }),+ );let currentViewAssetIndex = 0;let shiftKeyIsDown = $state(false);let lastAssetMouseEvent: TimelineAsset | null = $state(null);- let slidingWindow = $state({ top: 0, bottom: 0 });+ let slidingTop = $state(0);+ let slidingBottom = $state(0);const updateSlidingWindow = () => {const v = $state.snapshot(viewport);const top = (document.scrollingElement?.scrollTop || 0) - slidingWindowOffset;- const bottom = top + v.height;- const w = {- top,- bottom,- };- slidingWindow = w;+ slidingTop = top;+ slidingBottom = top + v.height;};+ $effect(updateSlidingWindow);const debouncedOnIntersected = debounce(() => onIntersected?.(), 750, { maxWait: 100, leading: true });let lastIntersectedHeight = 0;$effect(() => {// notify we got to (near) the end of scrollconst scrollPercentage =- ((slidingWindow.bottom - viewport.height) / (viewport.height - (document.scrollingElement?.clientHeight || 0))) *- 100;-- if (scrollPercentage > 90) {- const intersectedHeight = geometry?.containerHeight || 0;- if (lastIntersectedHeight !== intersectedHeight) {- debouncedOnIntersected();- lastIntersectedHeight = intersectedHeight;- }+ (slidingBottom - viewport.height) / (viewport.height - (document.scrollingElement?.clientHeight || 0));++ if (scrollPercentage > 0.9 && lastIntersectedHeight !== geometry.containerHeight) {+ debouncedOnIntersected();+ lastIntersectedHeight = geometry.containerHeight;}});const viewAssetHandler = async (asset: TimelineAsset) => {@@ -256,7 +204,7 @@isShowDeleteConfirmation = false;await deleteAssets(!(isTrashEnabled && !force),- (assetIds) => (assets = assets.filter((asset) => !assetIds.includes(asset.id))),+ (assetIds) => (assets = assets.filter((asset) => !assetIds.includes(asset.id)) as TimelineAsset[]),assetInteraction.selectedAssets,onReload,);@@ -269,7 +217,7 @@assetInteraction.isAllArchived ? AssetVisibility.Timeline : AssetVisibility.Archive,);if (ids) {- assets = assets.filter((asset) => !ids.includes(asset.id));+ assets = assets.filter((asset) => !ids.includes(asset.id)) as TimelineAsset[];deselectAllAssets();}};@@ -454,7 +402,7 @@onkeyup={onKeyUp}onselectstart={onSelectStart}use:shortcuts={shortcutList}- onscroll={() => updateSlidingWindow()}+ onscroll={updateSlidingWindow}/>{#if isShowDeleteConfirmation}@@ -468,13 +416,12 @@{#if assets.length > 0}<divstyle:position="relative"- style:height={assetLayouts.containerHeight + 'px'}- style:width={assetLayouts.containerWidth - 1 + 'px'}+ style:height={geometry.containerHeight + 'px'}+ style:width={geometry.containerWidth + 'px'}>- {#each assetLayouts.assetLayout as layout, layoutIndex (layout.asset.id + '-' + layoutIndex)}- {@const currentAsset = layout.asset}-- {#if layout.display}+ {#each assets as asset, i (asset.id + i)}+ {#if geometry.getTop(i) + pageHeaderOffset < slidingBottom && geometry.getTop(i) + pageHeaderOffset + geometry.getHeight(i) > slidingTop}+ {@const layout = geometry.getPosition(i)}<divclass="absolute"style:overflow="clip"@@ -484,25 +431,25 @@readonly={disableAssetSelect}onClick={() => {if (assetInteraction.selectionActive) {- handleSelectAssets(toTimelineAsset(currentAsset));+ handleSelectAssets(toTimelineAsset(asset));return;}- void viewAssetHandler(toTimelineAsset(currentAsset));+ void viewAssetHandler(toTimelineAsset(asset));}}- onSelect={() => handleSelectAssets(toTimelineAsset(currentAsset))}- onMouseEvent={() => assetMouseEventHandler(toTimelineAsset(currentAsset))}+ onSelect={() => handleSelectAssets(toTimelineAsset(asset))}+ onMouseEvent={() => assetMouseEventHandler(toTimelineAsset(asset))}{showArchiveIcon}- asset={toTimelineAsset(currentAsset)}- selected={assetInteraction.hasSelectedAsset(currentAsset.id)}- selectionCandidate={assetInteraction.hasSelectionCandidate(currentAsset.id)}+ asset={toTimelineAsset(asset)}+ selected={assetInteraction.hasSelectedAsset(asset.id)}+ selectionCandidate={assetInteraction.hasSelectionCandidate(asset.id)}thumbnailWidth={layout.width}thumbnailHeight={layout.height}/>- {#if showAssetName && !isTimelineAsset(currentAsset)}+ {#if showAssetName && !isTimelineAsset(asset)}<divclass="absolute text-center p-1 text-xs font-mono font-semibold w-full bottom-0 bg-linear-to-t bg-slate-50/75 dark:bg-slate-800/75 overflow-clip text-ellipsis whitespace-pre-wrap">- {currentAsset.originalFileName}+ {asset.originalFileName}</div>{/if}</div>diff --git a/web/src/lib/managers/timeline-manager/day-group.svelte.ts b/web/src/lib/managers/timeline-manager/day-group.svelte.tsindex 2a949499ec..a43a71c511 100644--- a/web/src/lib/managers/timeline-manager/day-group.svelte.ts+++ b/web/src/lib/managers/timeline-manager/day-group.svelte.ts@@ -1,7 +1,7 @@import { AssetOrder } from '@immich/sdk';import type { CommonLayoutOptions } from '$lib/utils/layout-utils';-import { getJustifiedLayoutFromAssets, getPosition } from '$lib/utils/layout-utils';+import { getJustifiedLayoutFromAssets } from '$lib/utils/layout-utils';import { plainDateTimeCompare } from '$lib/utils/timeline-util';import type { MonthGroup } from './month-group.svelte';@@ -153,8 +153,7 @@ export class DayGroup {this.width = geometry.containerWidth;this.height = assets.length === 0 ? 0 : geometry.containerHeight;for (let i = 0; i < this.viewerAssets.length; i++) {- const position = getPosition(geometry, i);- this.viewerAssets[i].position = position;+ this.viewerAssets[i].position = geometry.getPosition(i);}}diff --git a/web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts b/web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.tsindex 05f8b7c7f7..9aa06d940d 100644--- a/web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts+++ b/web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts@@ -2,8 +2,10 @@ import { sdkMock } from '$lib/__mocks__/sdk.mock';import { getMonthGroupByDate } from '$lib/managers/timeline-manager/internal/search-support.svelte';import { AbortError } from '$lib/utils';import { fromISODateTimeUTCToObject } from '$lib/utils/timeline-util';+import { initSync } from '@immich/justified-layout-wasm';import { type AssetResponseDto, type TimeBucketAssetResponseDto } from '@immich/sdk';import { timelineAssetFactory, toResponseDto } from '@test-data/factories/asset-factory';+import { readFile } from 'node:fs/promises';import { TimelineManager } from './timeline-manager.svelte';import type { TimelineAsset } from './types';@@ -23,6 +25,12 @@ function deriveLocalDateTimeFromFileCreatedAt(arg: TimelineAsset): TimelineAsset}describe('TimelineManager', () => {+ beforeAll(async () => {+ // needed for Node.js+ const file = await readFile('node_modules/@immich/justified-layout-wasm/pkg/justified-layout-wasm_bg.wasm');+ initSync({ module: file });+ });+beforeEach(() => {vi.resetAllMocks();});@@ -80,15 +88,15 @@ describe('TimelineManager', () => {expect(plainMonths).toEqual(expect.arrayContaining([- expect.objectContaining({ year: 2024, month: 3, height: 185.5 }),- expect.objectContaining({ year: 2024, month: 2, height: 12_016 }),+ expect.objectContaining({ year: 2024, month: 3, height: 353.5 }),+ expect.objectContaining({ year: 2024, month: 2, height: 7786.452_636_718_75 }),expect.objectContaining({ year: 2024, month: 1, height: 286 }),]),);});it('calculates timeline height', () => {- expect(timelineManager.timelineHeight).toBe(12_487.5);+ expect(timelineManager.timelineHeight).toBe(8425.952_636_718_75);});});diff --git a/web/src/lib/managers/timeline-manager/timeline-manager.svelte.ts b/web/src/lib/managers/timeline-manager/timeline-manager.svelte.tsindex 8aacd0a90a..e17082c816 100644--- a/web/src/lib/managers/timeline-manager/timeline-manager.svelte.ts+++ b/web/src/lib/managers/timeline-manager/timeline-manager.svelte.ts@@ -377,13 +377,11 @@ export class TimelineManager {}createLayoutOptions() {- const viewportWidth = this.viewportWidth;-return {spacing: 2,- heightTolerance: 0.15,+ heightTolerance: 0.3,rowHeight: this.#rowHeight,- rowWidth: Math.floor(viewportWidth),+ rowWidth: Math.floor(this.viewportWidth),};}diff --git a/web/src/lib/managers/timeline-manager/viewer-asset.svelte.ts b/web/src/lib/managers/timeline-manager/viewer-asset.svelte.tsindex b6e28df576..161cc049f1 100644--- a/web/src/lib/managers/timeline-manager/viewer-asset.svelte.ts+++ b/web/src/lib/managers/timeline-manager/viewer-asset.svelte.ts@@ -18,7 +18,7 @@ export class ViewerAsset {return calculateViewerAssetIntersecting(store, positionTop, this.position.height);});- position: CommonPosition | undefined = $state();+ position: CommonPosition | undefined = $state.raw();asset: TimelineAsset = <TimelineAsset>$state();id: string = $derived(this.asset.id);diff --git a/web/src/lib/utils/layout-utils.ts b/web/src/lib/utils/layout-utils.tsindex e60fa3b9e1..090a7168d5 100644--- a/web/src/lib/utils/layout-utils.ts+++ b/web/src/lib/utils/layout-utils.ts@@ -1,16 +1,13 @@-// import { TUNABLES } from '$lib/utils/tunables';-// note: it's important that this is not imported in more than one file due to https://github.com/sveltejs/kit/issues/7805-// import { JustifiedLayout, type LayoutOptions } from '@immich/justified-layout-wasm';+import { TUNABLES } from '$lib/utils/tunables';import type { TimelineAsset } from '$lib/managers/timeline-manager/types';import { getAssetRatio } from '$lib/utils/asset-utils';-import { isTimelineAsset } from '$lib/utils/timeline-util';+import { isTimelineAsset, isTimelineAssets } from '$lib/utils/timeline-util';+import { JustifiedLayout, type LayoutOptions } from '@immich/justified-layout-wasm';import type { AssetResponseDto } from '@immich/sdk';import createJustifiedLayout from 'justified-layout';-export type getJustifiedLayoutFromAssetsFunction = typeof getJustifiedLayoutFromAssets;--// let useWasm = TUNABLES.LAYOUT.WASM;+const useWasm = TUNABLES.LAYOUT.WASM;export type CommonJustifiedLayout = {containerWidth: number;@@ -19,6 +16,12 @@ export type CommonJustifiedLayout = {getLeft(boxIdx: number): number;getWidth(boxIdx: number): number;getHeight(boxIdx: number): number;+ getPosition(boxIdx: number): {+ top: number;+ left: number;+ width: number;+ height: number;+ };};export type CommonLayoutOptions = {@@ -29,25 +32,32 @@ export type CommonLayoutOptions = {};export function getJustifiedLayoutFromAssets(- assets: (TimelineAsset | AssetResponseDto)[],+ assets: AssetResponseDto[] | TimelineAsset[],options: CommonLayoutOptions,-): CommonJustifiedLayout {- // if (useWasm) {- // return wasmJustifiedLayout(assets, options);- // }+) {+ if (useWasm) {+ return isTimelineAssets(assets) ? wasmLayoutFromTimeline(assets, options) : wasmLayoutFromDto(assets, options);+ }+return justifiedLayout(assets, options);}-// commented out until a solution for top level awaits on safari is fixed-// function wasmJustifiedLayout(assets: AssetResponseDto[], options: LayoutOptions) {-// const aspectRatios = new Float32Array(assets.length);-// // eslint-disable-next-line unicorn/no-for-loop-// for (let i = 0; i < assets.length; i++) {-// const { width, height } = getAssetRatio(assets[i]);-// aspectRatios[i] = width / height;-// }-// return new JustifiedLayout(aspectRatios, options);-// }+function wasmLayoutFromTimeline(assets: TimelineAsset[], options: LayoutOptions) {+ const aspectRatios = new Float32Array(assets.length);+ for (let i = 0; i < assets.length; i++) {+ aspectRatios[i] = assets[i].ratio;+ }+ return new JustifiedLayout(aspectRatios, options);+}++function wasmLayoutFromDto(assets: AssetResponseDto[], options: LayoutOptions) {+ const aspectRatios = new Float32Array(assets.length);+ for (let i = 0; i < assets.length; i++) {+ const { width, height } = getAssetRatio(assets[i]);+ aspectRatios[i] = width / height;+ }+ return new JustifiedLayout(aspectRatios, options);+}type Geometry = ReturnType<typeof createJustifiedLayout>;class Adapter {@@ -88,9 +98,13 @@ class Adapter {getHeight(boxIdx: number) {return this.result.boxes[boxIdx]?.height;}++ getPosition(boxIdx: number): CommonPosition {+ return this.result.boxes[boxIdx];+ }}-export function justifiedLayout(assets: (TimelineAsset | AssetResponseDto)[], options: CommonLayoutOptions) {+export function justifiedLayout(assets: TimelineAsset[] | AssetResponseDto[], options: CommonLayoutOptions) {const adapter = {targetRowHeight: options.rowHeight,containerWidth: options.rowWidth,@@ -105,25 +119,9 @@ export function justifiedLayout(assets: (TimelineAsset | AssetResponseDto)[], opreturn new Adapter(result);}-export const emptyGeometry = () =>- new Adapter({- containerHeight: 0,- widowCount: 0,- boxes: [],- });-export type CommonPosition = {top: number;left: number;width: number;height: number;};--export function getPosition(geometry: CommonJustifiedLayout, boxIdx: number): CommonPosition {- const top = geometry.getTop(boxIdx);- const left = geometry.getLeft(boxIdx);- const width = geometry.getWidth(boxIdx);- const height = geometry.getHeight(boxIdx);-- return { top, left, width, height };-}diff --git a/web/src/lib/utils/server.ts b/web/src/lib/utils/server.tsindex 1c52274d23..b99ab80ec5 100644--- a/web/src/lib/utils/server.ts+++ b/web/src/lib/utils/server.ts@@ -1,17 +1,17 @@import { retrieveServerConfig } from '$lib/stores/server-config.store';import { initLanguage } from '$lib/utils';+import { init as initLayout } from '@immich/justified-layout-wasm';import { defaults } from '@immich/sdk';import { memoize } from 'lodash-es';type Fetch = typeof fetch;-async function _init(fetch: Fetch) {+function _init(fetch: Fetch) {// set event.fetch on the fetch-client used by @immich/sdk// https://kit.svelte.dev/docs/load#making-fetch-requests// https://github.com/oazapfts/oazapfts/blob/main/README.md#fetch-optionsdefaults.fetch = fetch;- await initLanguage();- await retrieveServerConfig();+ return Promise.all([initLayout(), initLanguage(), retrieveServerConfig()]);}export const init = memoize(_init, () => 'singlevalue');diff --git a/web/src/lib/utils/timeline-util.ts b/web/src/lib/utils/timeline-util.tsindex c3e41c01be..ca9dece6b2 100644--- a/web/src/lib/utils/timeline-util.ts+++ b/web/src/lib/utils/timeline-util.ts@@ -190,8 +190,10 @@ export const toTimelineAsset = (unknownAsset: AssetResponseDto | TimelineAsset):};};-export const isTimelineAsset = (unknownAsset: AssetResponseDto | TimelineAsset): unknownAsset is TimelineAsset =>- (unknownAsset as TimelineAsset).ratio !== undefined;+export const isTimelineAsset = (asset: AssetResponseDto | TimelineAsset): asset is TimelineAsset => 'ratio' in asset;++export const isTimelineAssets = (assets: AssetResponseDto[] | TimelineAsset[]): assets is TimelineAsset[] =>+ assets.length === 0 || 'ratio' in assets[0];export const plainDateTimeCompare = (ascending: boolean, a: TimelinePlainDateTime, b: TimelinePlainDateTime) => {const [aDateTime, bDateTime] = ascending ? [a, b] : [b, a];diff --git a/web/src/lib/utils/tunables.ts b/web/src/lib/utils/tunables.tsindex 6ce64ed041..c586e11957 100644--- a/web/src/lib/utils/tunables.ts+++ b/web/src/lib/utils/tunables.ts@@ -19,7 +19,7 @@ const storage = browser};export const TUNABLES = {LAYOUT: {- WASM: getBoolean(storage.getItem('LAYOUT.WASM'), false),+ WASM: getBoolean(storage.getItem('LAYOUT.WASM'), true),},TIMELINE: {INTERSECTION_EXPAND_TOP: getNumber(storage.getItem('TIMELINE_INTERSECTION_EXPAND_TOP'), 500),diff --git a/web/src/test-data/factories/asset-factory.ts b/web/src/test-data/factories/asset-factory.tsindex c2f03f9c6a..273bb6f97b 100644--- a/web/src/test-data/factories/asset-factory.ts+++ b/web/src/test-data/factories/asset-factory.ts@@ -31,7 +31,7 @@ export const assetFactory = Sync.makeFactory<AssetResponseDto>({export const timelineAssetFactory = Sync.makeFactory<TimelineAsset>({id: Sync.each(() => faker.string.uuid()),- ratio: Sync.each(() => faker.number.int()),+ ratio: Sync.each((i) => 0.2 + ((i * 0.618_034) % 3.8)), // deterministic random float between 0.2 and 4.0ownerId: Sync.each(() => faker.string.uuid()),thumbhash: Sync.each(() => faker.string.alphanumeric(28)),localDateTime: Sync.each(() => fromISODateTimeUTCToObject(faker.date.past().toISOString())),