High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat(web): Improve duplicate suggestion (#14947)
Sebastian Schneiderb4c1304b4603
diff --git a/i18n/de.json b/i18n/de.jsonindex d8d3f87134..89eea9fd49 100644--- a/i18n/de.json+++ b/i18n/de.json@@ -523,6 +523,10 @@"date_range": "Datumsbereich","day": "Tag","deduplicate_all": "Alle Duplikate entfernen",+ "deduplication_info": "Deduplizierungsinformationen",+ "deduplication_info_description": "Für die automatische Datei-Vorauswahl und das Deduplizieren aller Dateien berücksichtigen wir:",+ "deduplication_criteria_1": "Bildgröße in Bytes",+ "deduplication_criteria_2": "Anzahl der EXIF-Daten","default_locale": "Standard-Sprache","default_locale_description": "Datumsangaben und Zahlen basierend auf dem Gebietsschema des Browsers formatieren","delete": "Löschen",diff --git a/i18n/en.json b/i18n/en.jsonindex 2abc586c23..737ec2704d 100644--- a/i18n/en.json+++ b/i18n/en.json@@ -523,6 +523,10 @@"date_range": "Date range","day": "Day","deduplicate_all": "Deduplicate All",+ "deduplication_info": "Deduplication Info",+ "deduplication_info_description": "To automatically preselect assets and remove duplicates in bulk, we look at:",+ "deduplication_criteria_1": "Image size in bytes",+ "deduplication_criteria_2": "Count of EXIF data","default_locale": "Default Locale","default_locale_description": "Format dates and numbers based on your browser locale","delete": "Delete",diff --git a/web/src/lib/components/shared-components/duplicates-modal.svelte b/web/src/lib/components/shared-components/duplicates-modal.sveltenew file mode 100644index 0000000000..96f563989b--- /dev/null+++ b/web/src/lib/components/shared-components/duplicates-modal.svelte@@ -0,0 +1,20 @@+<script lang="ts">+ import { t } from 'svelte-i18n';+ import FullScreenModal from './full-screen-modal.svelte';++ interface Props {+ onClose: () => void;+ }++ let { onClose }: Props = $props();+</script>++<FullScreenModal title={$t('deduplication_info')} width="auto" {onClose}>+ <div class="text-sm dark:text-white">+ <p>{$t('deduplication_info_description')}</p>+ <ol class="ml-8 mt-2" style="list-style: decimal">+ <li>{$t('deduplication_criteria_1')}</li>+ <li>{$t('deduplication_criteria_2')}</li>+ </ol>+ </div>+</FullScreenModal>diff --git a/web/src/lib/components/utilities-page/duplicates/duplicate-asset.svelte b/web/src/lib/components/utilities-page/duplicates/duplicate-asset.svelteindex 19190745d1..97f44e3ec4 100644--- a/web/src/lib/components/utilities-page/duplicates/duplicate-asset.svelte+++ b/web/src/lib/components/utilities-page/duplicates/duplicate-asset.svelte@@ -3,8 +3,8 @@import { getAssetThumbnailUrl } from '$lib/utils';import { getAssetResolution, getFileSize } from '$lib/utils/asset-utils';import { getAltText } from '$lib/utils/thumbnail-util';- import { getAllAlbums, type AssetResponseDto } from '@immich/sdk';- import { mdiHeart, mdiMagnifyPlus, mdiImageMultipleOutline } from '@mdi/js';+ import { type AssetResponseDto, getAllAlbums } from '@immich/sdk';+ import { mdiHeart, mdiImageMultipleOutline, mdiMagnifyPlus } from '@mdi/js';import { t } from 'svelte-i18n';interface Props {diff --git a/web/src/lib/components/utilities-page/duplicates/duplicates-compare-control.svelte b/web/src/lib/components/utilities-page/duplicates/duplicates-compare-control.svelteindex 2afeffb6e4..11a5c67fcf 100644--- a/web/src/lib/components/utilities-page/duplicates/duplicates-compare-control.svelte+++ b/web/src/lib/components/utilities-page/duplicates/duplicates-compare-control.svelte@@ -4,7 +4,8 @@import Portal from '$lib/components/shared-components/portal/portal.svelte';import DuplicateAsset from '$lib/components/utilities-page/duplicates/duplicate-asset.svelte';import { assetViewingStore } from '$lib/stores/asset-viewing.store';- import { handlePromiseError, suggestDuplicateByFileSize } from '$lib/utils';+ import { handlePromiseError } from '$lib/utils';+ import { suggestDuplicate } from '$lib/utils/duplicate-utils';import { navigate } from '$lib/utils/navigation';import { shortcuts } from '$lib/actions/shortcut';import { type AssetResponseDto } from '@immich/sdk';@@ -27,7 +28,7 @@let trashCount = $derived(assets.length - selectedAssetIds.size);onMount(() => {- const suggestedAsset = suggestDuplicateByFileSize(assets);+ const suggestedAsset = suggestDuplicate(assets);if (!suggestedAsset) {selectedAssetIds = new SvelteSet(assets[0].id);diff --git a/web/src/lib/utils.ts b/web/src/lib/utils.tsindex 349ae1c479..77e40226af 100644--- a/web/src/lib/utils.ts+++ b/web/src/lib/utils.ts@@ -16,13 +16,11 @@ import {linkOAuthAccount,startOAuth,unlinkOAuthAccount,- type AssetResponseDto,type PersonResponseDto,type SharedLinkResponseDto,type UserResponseDto,} from '@immich/sdk';import { mdiCogRefreshOutline, mdiDatabaseRefreshOutline, mdiHeadSyncOutline, mdiImageRefreshOutline } from '@mdi/js';-import { sortBy } from 'lodash-es';import { init, register, t } from 'svelte-i18n';import { derived, get } from 'svelte/store';@@ -332,9 +330,5 @@ export const withError = async <T>(fn: () => Promise<T>): Promise<[undefined, T]}};-export const suggestDuplicateByFileSize = (assets: AssetResponseDto[]): AssetResponseDto | undefined => {- return sortBy(assets, (asset) => asset.exifInfo?.fileSizeInByte).pop();-};-// eslint-disable-next-line unicorn/prefer-code-pointexport const decodeBase64 = (data: string) => Uint8Array.from(atob(data), (c) => c.charCodeAt(0));diff --git a/web/src/lib/utils/duplicate-utils.spec.ts b/web/src/lib/utils/duplicate-utils.spec.tsnew file mode 100644index 0000000000..4fa427989a--- /dev/null+++ b/web/src/lib/utils/duplicate-utils.spec.ts@@ -0,0 +1,37 @@+import { suggestDuplicate } from '$lib/utils/duplicate-utils';+import type { AssetResponseDto } from '@immich/sdk';++describe('choosing a duplicate', () => {+ it('picks the asset with the largest file size', () => {+ const assets = [+ { exifInfo: { fileSizeInByte: 300 } },+ { exifInfo: { fileSizeInByte: 200 } },+ { exifInfo: { fileSizeInByte: 100 } },+ ];+ expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);+ });++ it('picks the asset with the most exif data if multiple assets have the same file size', () => {+ const assets = [+ { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: 1 } },+ { exifInfo: { fileSizeInByte: 200, rating: 5 } },+ { exifInfo: { fileSizeInByte: 100, rating: 5 } },+ ];+ expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);+ });++ it('returns undefined for an empty array', () => {+ const assets: AssetResponseDto[] = [];+ expect(suggestDuplicate(assets)).toBeUndefined();+ });++ it('handles assets with no exifInfo', () => {+ const assets = [{ exifInfo: { fileSizeInByte: 200 } }, {}];+ expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);+ });++ it('handles assets with exifInfo but no fileSizeInByte', () => {+ const assets = [{ exifInfo: { rating: 5, fNumber: 1 } }, { exifInfo: { rating: 5 } }];+ expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);+ });+});diff --git a/web/src/lib/utils/duplicate-utils.ts b/web/src/lib/utils/duplicate-utils.tsnew file mode 100644index 0000000000..1c783a3667--- /dev/null+++ b/web/src/lib/utils/duplicate-utils.ts@@ -0,0 +1,30 @@+import { getExifCount } from '$lib/utils/exif-utils';+import type { AssetResponseDto } from '@immich/sdk';+import { sortBy } from 'lodash-es';++/**+ * Suggests the best duplicate asset to keep from a list of duplicates.+ *+ * The best asset is determined by the following criteria:+ * - Largest image file size in bytes+ * - Largest count of exif data+ *+ * @param assets List of duplicate assets+ * @returns The best asset to keep+ */+export const suggestDuplicate = (assets: AssetResponseDto[]): AssetResponseDto | undefined => {+ let duplicateAssets = sortBy(assets, (asset) => asset.exifInfo?.fileSizeInByte ?? 0);++ // Update the list to only include assets with the largest file size+ duplicateAssets = duplicateAssets.filter(+ (asset) => asset.exifInfo?.fileSizeInByte === duplicateAssets.at(-1)?.exifInfo?.fileSizeInByte,+ );++ // If there are multiple assets with the same file size, sort the list by the count of exif data+ if (duplicateAssets.length >= 2) {+ duplicateAssets = sortBy(duplicateAssets, getExifCount);+ }++ // Return the last asset in the list+ return duplicateAssets.pop();+};diff --git a/web/src/lib/utils/exif-utils.spec.ts b/web/src/lib/utils/exif-utils.spec.tsnew file mode 100644index 0000000000..7ce2e88d6f--- /dev/null+++ b/web/src/lib/utils/exif-utils.spec.ts@@ -0,0 +1,29 @@+import { getExifCount } from '$lib/utils/exif-utils';+import type { AssetResponseDto } from '@immich/sdk';++describe('getting the exif count', () => {+ it('returns 0 when exifInfo is undefined', () => {+ const asset = {};+ expect(getExifCount(asset as AssetResponseDto)).toBe(0);+ });++ it('returns 0 when exifInfo is empty', () => {+ const asset = { exifInfo: {} };+ expect(getExifCount(asset as AssetResponseDto)).toBe(0);+ });++ it('returns the correct count of non-null exifInfo properties', () => {+ const asset = { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: null } };+ expect(getExifCount(asset as AssetResponseDto)).toBe(2);+ });++ it('ignores null, undefined and empty properties in exifInfo', () => {+ const asset = { exifInfo: { fileSizeInByte: 200, rating: null, fNumber: undefined, description: '' } };+ expect(getExifCount(asset as AssetResponseDto)).toBe(1);+ });++ it('returns the correct count when all exifInfo properties are non-null', () => {+ const asset = { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: 1, description: 'test' } };+ expect(getExifCount(asset as AssetResponseDto)).toBe(4);+ });+});diff --git a/web/src/lib/utils/exif-utils.ts b/web/src/lib/utils/exif-utils.tsnew file mode 100644index 0000000000..75a6bbd055--- /dev/null+++ b/web/src/lib/utils/exif-utils.ts@@ -0,0 +1,5 @@+import type { AssetResponseDto } from '@immich/sdk';++export const getExifCount = (asset: AssetResponseDto) => {+ return Object.values(asset.exifInfo ?? {}).filter(Boolean).length;+};diff --git a/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte b/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelteindex fd2bcb438c..22e4f86c74 100644--- a/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte+++ b/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte@@ -12,11 +12,12 @@import { deleteAssets, updateAssets } from '@immich/sdk';import { t } from 'svelte-i18n';import type { PageData } from './$types';- import { suggestDuplicateByFileSize } from '$lib/utils';+ import { suggestDuplicate } from '$lib/utils/duplicate-utils';import LinkButton from '$lib/components/elements/buttons/link-button.svelte';- import { mdiCheckOutline, mdiTrashCanOutline } from '@mdi/js';+ import { mdiCheckOutline, mdiInformationOutline, mdiTrashCanOutline } from '@mdi/js';import { stackAssets } from '$lib/utils/asset-utils';import ShowShortcuts from '$lib/components/shared-components/show-shortcuts.svelte';+ import DuplicatesModal from '$lib/components/shared-components/duplicates-modal.svelte';import CircleIconButton from '$lib/components/elements/buttons/circle-icon-button.svelte';import { mdiKeyboard } from '@mdi/js';import Icon from '$lib/components/elements/icon.svelte';@@ -25,9 +26,14 @@interface Props {data: PageData;isShowKeyboardShortcut?: boolean;+ isShowDuplicateInfo?: boolean;}- let { data = $bindable(), isShowKeyboardShortcut = $bindable(false) }: Props = $props();+ let {+ data = $bindable(),+ isShowKeyboardShortcut = $bindable(false),+ isShowDuplicateInfo = $bindable(false),+ }: Props = $props();interface Shortcuts {general: ExplainedShortcut[];@@ -103,7 +109,7 @@};const handleDeduplicateAll = async () => {- const idsToKeep = duplicates.map((group) => suggestDuplicateByFileSize(group.assets)).map((asset) => asset?.id);+ const idsToKeep = duplicates.map((group) => suggestDuplicate(group.assets)).map((asset) => asset?.id);const idsToDelete = duplicates.flatMap((group, i) =>group.assets.map((asset) => asset.id).filter((asset) => asset !== idsToKeep[i]),);@@ -178,11 +184,21 @@</div>{/snippet}- <div class="mt-4">+ <div class="">{#if duplicates && duplicates.length > 0}- <div class="mb-4 text-sm dark:text-white">- <p>{$t('duplicates_description')}</p>+ <div class="flex items-center mb-2">+ <div class="text-sm dark:text-white">+ <p>{$t('duplicates_description')}</p>+ </div>+ <CircleIconButton+ icon={mdiInformationOutline}+ title={$t('deduplication_info')}+ size="16"+ padding="2"+ onclick={() => (isShowDuplicateInfo = true)}+ /></div>+{#key duplicates[0].duplicateId}<DuplicatesCompareControlassets={duplicates[0].assets}@@ -202,3 +218,6 @@{#if isShowKeyboardShortcut}<ShowShortcuts shortcuts={duplicateShortcuts} onClose={() => (isShowKeyboardShortcut = false)} />{/if}+{#if isShowDuplicateInfo}+ <DuplicatesModal onClose={() => (isShowDuplicateInfo = false)} />+{/if}