High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat: create new person in face editor (#27364)
Alex37823bcd519f
diff --git a/i18n/en.json b/i18n/en.jsonindex 42a89586f9..29af88b5b0 100644--- a/i18n/en.json+++ b/i18n/en.json@@ -849,9 +849,12 @@"create_link_to_share": "Create link to share","create_link_to_share_description": "Let anyone with the link see the selected photo(s)","create_new": "CREATE NEW",+ "create_new_face": "Create new face","create_new_person": "Create new person","create_new_person_hint": "Assign selected assets to a new person","create_new_user": "Create new user",+ "create_person": "Create person",+ "create_person_subtitle": "Add a name to the selected face to create and tag the new person","create_shared_album_page_share_add_assets": "ADD ASSETS","create_shared_album_page_share_select_photos": "Select Photos","create_shared_link": "Create shared link",@@ -2214,6 +2217,7 @@"tag": "Tag","tag_assets": "Tag assets","tag_created": "Created tag: {tag}",+ "tag_face": "Tag face","tag_feature_description": "Browsing photos and videos grouped by logical tag topics","tag_not_found_question": "Cannot find a tag? <link>Create a new tag.</link>","tag_people": "Tag People",diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.tsindex 5c262d892d..8b303d04f6 100644--- a/server/src/services/person.service.spec.ts+++ b/server/src/services/person.service.spec.ts@@ -12,7 +12,13 @@ import { PersonFactory } from 'test/factories/person.factory';import { UserFactory } from 'test/factories/user.factory';import { authStub } from 'test/fixtures/auth.stub';import { systemConfigStub } from 'test/fixtures/system-config.stub';-import { getAsDetectedFace, getForAssetFace, getForDetectedFaces, getForFacialRecognitionJob } from 'test/mappers';+import {+ getAsDetectedFace,+ getForAsset,+ getForAssetFace,+ getForDetectedFaces,+ getForFacialRecognitionJob,+} from 'test/mappers';import { newDate, newUuid } from 'test/small.factory';import { makeStream, newTestService, ServiceMocks } from 'test/utils';@@ -370,6 +376,86 @@ describe(PersonService.name, () => {});});+ describe('createFace', () => {+ it('should create a manual face and initialize the person feature photo creation', async () => {+ const auth = AuthFactory.create();+ const asset = AssetFactory.create();+ const person = PersonFactory.create({ faceAssetId: null });+ const featureFace = AssetFaceFactory.create({+ assetId: asset.id,+ personId: person.id,+ sourceType: SourceType.Manual,+ });++ mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([asset.id]));+ mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id]));+ mocks.asset.getById.mockResolvedValue(getForAsset(asset));+ mocks.person.getById.mockResolvedValue(person);+ mocks.person.getRandomFace.mockResolvedValue(featureFace);+ mocks.person.update.mockResolvedValue({ ...person, faceAssetId: featureFace.id });++ await expect(+ sut.createFace(auth, {+ assetId: asset.id,+ personId: person.id,+ imageHeight: 500,+ imageWidth: 400,+ x: 10,+ y: 20,+ width: 100,+ height: 110,+ }),+ ).resolves.toBeUndefined();++ expect(mocks.asset.getById).toHaveBeenCalledWith(asset.id, { edits: true, exifInfo: true });+ expect(mocks.person.createAssetFace).toHaveBeenCalledWith({+ assetId: asset.id,+ personId: person.id,+ imageHeight: 500,+ imageWidth: 400,+ boundingBoxX1: 10,+ boundingBoxX2: 110,+ boundingBoxY1: 20,+ boundingBoxY2: 130,+ sourceType: SourceType.Manual,+ });+ expect(mocks.person.getRandomFace).toHaveBeenCalledWith(person.id);+ expect(mocks.person.update).toHaveBeenCalledWith({ id: person.id, faceAssetId: featureFace.id });+ expect(mocks.job.queueAll).toHaveBeenCalledWith([+ { name: JobName.PersonGenerateThumbnail, data: { id: person.id } },+ ]);+ });++ it('should not update the person feature photo if one already exists', async () => {+ const auth = AuthFactory.create();+ const asset = AssetFactory.create();+ const person = PersonFactory.create({ faceAssetId: newUuid() });++ mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([asset.id]));+ mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id]));+ mocks.asset.getById.mockResolvedValue(getForAsset(asset));+ mocks.person.getById.mockResolvedValue(person);++ await expect(+ sut.createFace(auth, {+ assetId: asset.id,+ personId: person.id,+ imageHeight: 500,+ imageWidth: 400,+ x: 10,+ y: 20,+ width: 100,+ height: 110,+ }),+ ).resolves.toBeUndefined();++ expect(mocks.person.createAssetFace).toHaveBeenCalledOnce();+ expect(mocks.person.getRandomFace).not.toHaveBeenCalled();+ expect(mocks.person.update).not.toHaveBeenCalled();+ expect(mocks.job.queueAll).not.toHaveBeenCalled();+ });+ });+describe('createNewFeaturePhoto', () => {it('should change person feature photo', async () => {const person = PersonFactory.create();diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.tsindex fb04ace4f2..fde5313f4d 100644--- a/server/src/services/person.service.ts+++ b/server/src/services/person.service.ts@@ -631,7 +631,11 @@ export class PersonService extends BaseService {this.requireAccess({ auth, permission: Permission.PersonRead, ids: [dto.personId] }),]);- const asset = await this.assetRepository.getById(dto.assetId, { edits: true, exifInfo: true });+ const [asset, person] = await Promise.all([+ this.assetRepository.getById(dto.assetId, { edits: true, exifInfo: true }),+ this.findOrFail(dto.personId),+ ]);+if (!asset) {throw new NotFoundException('Asset not found');}@@ -689,6 +693,10 @@ export class PersonService extends BaseService {boundingBoxY2: Math.round(bottomRight.y),sourceType: SourceType.Manual,});++ if (!person.faceAssetId) {+ await this.createNewFeaturePhoto([person.id]);+ }}async deleteFace(auth: AuthDto, id: string, dto: AssetFaceDeleteDto): Promise<void> {diff --git a/server/test/medium/specs/services/person.service.spec.ts b/server/test/medium/specs/services/person.service.spec.tsindex 4cd705b5bd..39805580f6 100644--- a/server/test/medium/specs/services/person.service.spec.ts+++ b/server/test/medium/specs/services/person.service.spec.ts@@ -5,6 +5,7 @@ import { AccessRepository } from 'src/repositories/access.repository';import { AssetEditRepository } from 'src/repositories/asset-edit.repository';import { AssetRepository } from 'src/repositories/asset.repository';import { DatabaseRepository } from 'src/repositories/database.repository';+import { JobRepository } from 'src/repositories/job.repository';import { LoggingRepository } from 'src/repositories/logging.repository';import { PersonRepository } from 'src/repositories/person.repository';import { StorageRepository } from 'src/repositories/storage.repository';@@ -20,7 +21,7 @@ const setup = (db?: Kysely<DB>) => {return newMediumService(PersonService, {database: db || defaultDatabase,real: [AccessRepository, DatabaseRepository, PersonRepository, AssetRepository, AssetEditRepository],- mock: [LoggingRepository, StorageRepository],+ mock: [JobRepository, LoggingRepository, StorageRepository],});};@@ -89,6 +90,7 @@ describe(PersonService.name, () => {const { person } = await ctx.newPerson({ ownerId: user.id });const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 200, height: 200 });await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 200 });+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();const auth = factory.auth({ user });@@ -128,6 +130,7 @@ describe(PersonService.name, () => {const { person } = await ctx.newPerson({ ownerId: user.id });const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 150, height: 200 });await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 200 });+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();await ctx.newEdits(asset.id, {edits: [@@ -199,6 +202,7 @@ describe(PersonService.name, () => {const { person } = await ctx.newPerson({ ownerId: user.id });const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 100, height: 200 });await ctx.newExif({ assetId: asset.id, exifImageWidth: 200, exifImageHeight: 100 });+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();await ctx.newEdits(asset.id, {edits: [@@ -263,6 +267,7 @@ describe(PersonService.name, () => {const { person } = await ctx.newPerson({ ownerId: user.id });const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 200, height: 100 });await ctx.newExif({ assetId: asset.id, exifImageHeight: 100, exifImageWidth: 200 });+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();await ctx.newEdits(asset.id, {edits: [@@ -327,6 +332,7 @@ describe(PersonService.name, () => {const { person } = await ctx.newPerson({ ownerId: user.id });const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 200, height: 150 });await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 200 });+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();await ctx.newEdits(asset.id, {edits: [@@ -400,6 +406,7 @@ describe(PersonService.name, () => {const { person } = await ctx.newPerson({ ownerId: user.id });const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 150, height: 100 });await ctx.newExif({ assetId: asset.id, exifImageHeight: 100, exifImageWidth: 200 });+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();await ctx.newEdits(asset.id, {edits: [@@ -473,6 +480,7 @@ describe(PersonService.name, () => {const { person } = await ctx.newPerson({ ownerId: user.id });const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 200, height: 150 });await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 150 });+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();await ctx.newEdits(asset.id, {edits: [@@ -543,6 +551,7 @@ describe(PersonService.name, () => {const { person } = await ctx.newPerson({ ownerId: user.id });const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 150, height: 100 });await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 200 });+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();await ctx.newEdits(asset.id, {edits: [@@ -622,6 +631,7 @@ describe(PersonService.name, () => {const { person } = await ctx.newPerson({ ownerId: user.id });const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 100, height: 100 });await ctx.newExif({ assetId: asset.id, exifImageHeight: 100, exifImageWidth: 100 });+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();await ctx.newEdits(asset.id, {edits: [@@ -692,6 +702,7 @@ describe(PersonService.name, () => {const { person } = await ctx.newPerson({ ownerId: user.id });const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 100, height: 100 });await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 100, orientation: '6' });+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();await ctx.newEdits(asset.id, {edits: [diff --git a/web/src/lib/components/asset-viewer/face-editor/face-editor.svelte b/web/src/lib/components/asset-viewer/face-editor/face-editor.svelteindex 8f7103c2a3..2418e5bf85 100644--- a/web/src/lib/components/asset-viewer/face-editor/face-editor.svelte+++ b/web/src/lib/components/asset-viewer/face-editor/face-editor.svelte@@ -2,6 +2,7 @@import { shortcut } from '$lib/actions/shortcut';import ImageThumbnail from '$lib/components/assets/thumbnail/image-thumbnail.svelte';import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';+ import FaceCreateTagModal from '$lib/modals/CreateFaceModal.svelte';import { getPeopleThumbnailUrl } from '$lib/utils';import { getNaturalSize, scaleToFit } from '$lib/utils/container-utils';import { handleError } from '$lib/utils/handle-error';@@ -260,6 +261,44 @@};};+ type FaceCoordinates = NonNullable<ReturnType<typeof getFaceCroppedCoordinates>>;++ const getFacePreviewUrl = (data: FaceCoordinates) => {+ if (!htmlElement) {+ return;+ }++ const natural = getNaturalSize(htmlElement);+ if (natural.width <= 0 || natural.height <= 0) {+ return;+ }++ const x = clamp(data.x, 0, natural.width - 1);+ const y = clamp(data.y, 0, natural.height - 1);+ const width = clamp(data.width, 1, natural.width - x);+ const height = clamp(data.height, 1, natural.height - y);++ if (width <= 0 || height <= 0) {+ return;+ }++ const canvas = document.createElement('canvas');+ canvas.width = width;+ canvas.height = height;++ const context = canvas.getContext('2d');+ if (!context) {+ return;+ }++ try {+ context.drawImage(htmlElement, x, y, width, height, 0, 0, width, height);+ return canvas.toDataURL('image/png');+ } catch {+ return;+ }+ };+const tagFace = async (person: PersonResponseDto) => {try {const data = getFaceCroppedCoordinates();@@ -294,6 +333,28 @@}};+ const showCreateFaceModal = async () => {+ try {+ const data = getFaceCroppedCoordinates();+ if (!data) {+ return;+ }++ const created = await modalManager.show(FaceCreateTagModal, {+ assetId,+ ...data,+ previewUrl: getFacePreviewUrl(data),+ });+ if (!created) {+ return;+ }++ onClose();+ } catch (error) {+ handleError(error, 'Error creating and tagging face');+ }+ };+onDestroy(() => {onClose();});@@ -303,19 +364,19 @@<divid="face-editor-data"- class="absolute start-0 top-0 z-5 h-full w-full overflow-hidden"+ class="absolute inset-s-0 top-0 z-5 h-full w-full overflow-hidden"data-overlay-interactivedata-face-left={faceBoxPosition.left}data-face-top={faceBoxPosition.top}data-face-width={faceBoxPosition.width}data-face-height={faceBoxPosition.height}>- <canvas bind:this={canvasEl} id="face-editor" class="absolute top-0 start-0"></canvas>+ <canvas bind:this={canvasEl} id="face-editor" class="absolute top-0 inset-s-0"></canvas><divid="face-selector"bind:this={faceSelectorEl}- class="absolute top-[calc(50%-250px)] start-[calc(50%-125px)] max-w-[250px] w-[250px] bg-white dark:bg-immich-dark-gray dark:text-immich-dark-fg backdrop-blur-sm px-2 py-4 rounded-xl border border-gray-200 dark:border-gray-800 transition-[top,left] duration-200 ease-out"+ class="absolute top-[calc(50%-250px)] inset-s-[calc(50%-125px)] max-w-62.5 w-62.5 bg-white dark:bg-immich-dark-gray dark:text-immich-dark-fg backdrop-blur-sm px-2 py-4 rounded-xl border border-gray-200 dark:border-gray-800 transition-[top,left] duration-200 ease-out"><p class="text-center text-sm">{$t('select_person_to_tag')}</p>@@ -354,6 +415,12 @@{/if}</div>- <Button size="small" fullWidth onclick={onClose} color="danger" class="mt-2">{$t('cancel')}</Button>+ <Button size="small" fullWidth onclick={showCreateFaceModal} variant="outline" class="mt-2">+ {$t('create_person')}+ </Button>++ <Button size="small" fullWidth onclick={onClose} color="danger" class="mt-2">+ {$t('cancel')}+ </Button></div></div>diff --git a/web/src/lib/modals/CreateFaceModal.svelte b/web/src/lib/modals/CreateFaceModal.sveltenew file mode 100644index 0000000000..fa845cba2d--- /dev/null+++ b/web/src/lib/modals/CreateFaceModal.svelte@@ -0,0 +1,91 @@+<script lang="ts">+ import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';+ import { delay } from '$lib/utils/asset-utils';+ import { handleError } from '$lib/utils/handle-error';+ import { createFace, createPerson } from '@immich/sdk';+ import { Field, FormModal, Input, LoadingSpinner, Text } from '@immich/ui';+ import { t } from 'svelte-i18n';++ type Props = {+ assetId: string;+ imageWidth: number;+ imageHeight: number;+ x: number;+ y: number;+ width: number;+ height: number;+ onClose: (created?: boolean) => void;+ previewUrl?: string;+ };++ let { assetId, imageWidth, imageHeight, x, y, width, height, onClose, previewUrl }: Props = $props();+ let personName = $state('');+ let isSubmitting = $state(false);++ const getTrimmedName = () => personName.trim();++ const onSubmit = async () => {+ const name = getTrimmedName();+ if (!name) {+ return;+ }++ try {+ isSubmitting = true;++ const person = await createPerson({+ personCreateDto: {+ name,+ },+ });++ await createFace({+ assetFaceCreateDto: {+ assetId,+ imageHeight,+ imageWidth,+ personId: person.id,+ x,+ y,+ width,+ height,+ },+ });++ await delay(1500);+ await assetViewerManager.setAssetId(assetId);+ onClose(true);+ } catch (error) {+ handleError(error, 'Error creating and tagging face');+ } finally {+ isSubmitting = false;+ }+ };+</script>++<FormModal+ size="small"+ title={$t('create_person')}+ submitText={$t('tag_face')}+ disabled={!getTrimmedName() || isSubmitting}+ {onClose}+ {onSubmit}+>+ <Text size="tiny" class="mb-4" color="muted">{$t('create_person_subtitle')}</Text>+ {#if previewUrl}+ <Field label={$t('preview')}>+ <div class="flex justify-center rounded-xl bg-gray-50 p-3 dark:border-gray-700 dark:bg-black/20 relative">+ <img src={previewUrl} alt={$t('preview')} class="max-h-48 rounded-lg object-contain shadow-sm" />+ {#if isSubmitting}+ <div class="flex place-items-center place-content-center absolute inset-0 bg-black/20 rounded-lg">+ <LoadingSpinner />+ </div>+ {/if}+ </div>+ </Field>+ {/if}++ <Field label={$t('name')} required class="mt-3">+ <Input autofocus bind:value={personName} disabled={isSubmitting} />+ </Field>+</FormModal>