Skip to content

High performance self-hosted photo and video management solution.

feat: create new person in face editor (#27364)

Alex37823bcd519f
diff --git a/i18n/en.json b/i18n/en.json
index 42a89586f9..29af88b5b0 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -849,9 +849,12 @@
"create_link_to_share": "Create link to share",
"create_link_to_share_description": "Let anyone with the link see the selected photo(s)",
"create_new": "CREATE NEW",
+ "create_new_face": "Create new face",
"create_new_person": "Create new person",
"create_new_person_hint": "Assign selected assets to a new person",
"create_new_user": "Create new user",
+ "create_person": "Create person",
+ "create_person_subtitle": "Add a name to the selected face to create and tag the new person",
"create_shared_album_page_share_add_assets": "ADD ASSETS",
"create_shared_album_page_share_select_photos": "Select Photos",
"create_shared_link": "Create shared link",
@@ -2214,6 +2217,7 @@
"tag": "Tag",
"tag_assets": "Tag assets",
"tag_created": "Created tag: {tag}",
+ "tag_face": "Tag face",
"tag_feature_description": "Browsing photos and videos grouped by logical tag topics",
"tag_not_found_question": "Cannot find a tag? <link>Create a new tag.</link>",
"tag_people": "Tag People",
diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.ts
index 5c262d892d..8b303d04f6 100644
--- a/server/src/services/person.service.spec.ts
+++ b/server/src/services/person.service.spec.ts
@@ -12,7 +12,13 @@ import { PersonFactory } from 'test/factories/person.factory';
import { UserFactory } from 'test/factories/user.factory';
import { authStub } from 'test/fixtures/auth.stub';
import { systemConfigStub } from 'test/fixtures/system-config.stub';
-import { getAsDetectedFace, getForAssetFace, getForDetectedFaces, getForFacialRecognitionJob } from 'test/mappers';
+import {
+ getAsDetectedFace,
+ getForAsset,
+ getForAssetFace,
+ getForDetectedFaces,
+ getForFacialRecognitionJob,
+} from 'test/mappers';
import { newDate, newUuid } from 'test/small.factory';
import { makeStream, newTestService, ServiceMocks } from 'test/utils';
@@ -370,6 +376,86 @@ describe(PersonService.name, () => {
});
});
+ describe('createFace', () => {
+ it('should create a manual face and initialize the person feature photo creation', async () => {
+ const auth = AuthFactory.create();
+ const asset = AssetFactory.create();
+ const person = PersonFactory.create({ faceAssetId: null });
+ const featureFace = AssetFaceFactory.create({
+ assetId: asset.id,
+ personId: person.id,
+ sourceType: SourceType.Manual,
+ });
+
+ mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([asset.id]));
+ mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id]));
+ mocks.asset.getById.mockResolvedValue(getForAsset(asset));
+ mocks.person.getById.mockResolvedValue(person);
+ mocks.person.getRandomFace.mockResolvedValue(featureFace);
+ mocks.person.update.mockResolvedValue({ ...person, faceAssetId: featureFace.id });
+
+ await expect(
+ sut.createFace(auth, {
+ assetId: asset.id,
+ personId: person.id,
+ imageHeight: 500,
+ imageWidth: 400,
+ x: 10,
+ y: 20,
+ width: 100,
+ height: 110,
+ }),
+ ).resolves.toBeUndefined();
+
+ expect(mocks.asset.getById).toHaveBeenCalledWith(asset.id, { edits: true, exifInfo: true });
+ expect(mocks.person.createAssetFace).toHaveBeenCalledWith({
+ assetId: asset.id,
+ personId: person.id,
+ imageHeight: 500,
+ imageWidth: 400,
+ boundingBoxX1: 10,
+ boundingBoxX2: 110,
+ boundingBoxY1: 20,
+ boundingBoxY2: 130,
+ sourceType: SourceType.Manual,
+ });
+ expect(mocks.person.getRandomFace).toHaveBeenCalledWith(person.id);
+ expect(mocks.person.update).toHaveBeenCalledWith({ id: person.id, faceAssetId: featureFace.id });
+ expect(mocks.job.queueAll).toHaveBeenCalledWith([
+ { name: JobName.PersonGenerateThumbnail, data: { id: person.id } },
+ ]);
+ });
+
+ it('should not update the person feature photo if one already exists', async () => {
+ const auth = AuthFactory.create();
+ const asset = AssetFactory.create();
+ const person = PersonFactory.create({ faceAssetId: newUuid() });
+
+ mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([asset.id]));
+ mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id]));
+ mocks.asset.getById.mockResolvedValue(getForAsset(asset));
+ mocks.person.getById.mockResolvedValue(person);
+
+ await expect(
+ sut.createFace(auth, {
+ assetId: asset.id,
+ personId: person.id,
+ imageHeight: 500,
+ imageWidth: 400,
+ x: 10,
+ y: 20,
+ width: 100,
+ height: 110,
+ }),
+ ).resolves.toBeUndefined();
+
+ expect(mocks.person.createAssetFace).toHaveBeenCalledOnce();
+ expect(mocks.person.getRandomFace).not.toHaveBeenCalled();
+ expect(mocks.person.update).not.toHaveBeenCalled();
+ expect(mocks.job.queueAll).not.toHaveBeenCalled();
+ });
+ });
+
describe('createNewFeaturePhoto', () => {
it('should change person feature photo', async () => {
const person = PersonFactory.create();
diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts
index fb04ace4f2..fde5313f4d 100644
--- a/server/src/services/person.service.ts
+++ b/server/src/services/person.service.ts
@@ -631,7 +631,11 @@ export class PersonService extends BaseService {
this.requireAccess({ auth, permission: Permission.PersonRead, ids: [dto.personId] }),
]);
- const asset = await this.assetRepository.getById(dto.assetId, { edits: true, exifInfo: true });
+ const [asset, person] = await Promise.all([
+ this.assetRepository.getById(dto.assetId, { edits: true, exifInfo: true }),
+ this.findOrFail(dto.personId),
+ ]);
+
if (!asset) {
throw new NotFoundException('Asset not found');
}
@@ -689,6 +693,10 @@ export class PersonService extends BaseService {
boundingBoxY2: Math.round(bottomRight.y),
sourceType: SourceType.Manual,
});
+
+ if (!person.faceAssetId) {
+ await this.createNewFeaturePhoto([person.id]);
+ }
}
async deleteFace(auth: AuthDto, id: string, dto: AssetFaceDeleteDto): Promise<void> {
diff --git a/server/test/medium/specs/services/person.service.spec.ts b/server/test/medium/specs/services/person.service.spec.ts
index 4cd705b5bd..39805580f6 100644
--- a/server/test/medium/specs/services/person.service.spec.ts
+++ b/server/test/medium/specs/services/person.service.spec.ts
@@ -5,6 +5,7 @@ import { AccessRepository } from 'src/repositories/access.repository';
import { AssetEditRepository } from 'src/repositories/asset-edit.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { DatabaseRepository } from 'src/repositories/database.repository';
+import { JobRepository } from 'src/repositories/job.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { PersonRepository } from 'src/repositories/person.repository';
import { StorageRepository } from 'src/repositories/storage.repository';
@@ -20,7 +21,7 @@ const setup = (db?: Kysely<DB>) => {
return newMediumService(PersonService, {
database: db || defaultDatabase,
real: [AccessRepository, DatabaseRepository, PersonRepository, AssetRepository, AssetEditRepository],
- mock: [LoggingRepository, StorageRepository],
+ mock: [JobRepository, LoggingRepository, StorageRepository],
});
};
@@ -89,6 +90,7 @@ describe(PersonService.name, () => {
const { person } = await ctx.newPerson({ ownerId: user.id });
const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 200, height: 200 });
await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 200 });
+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();
const auth = factory.auth({ user });
@@ -128,6 +130,7 @@ describe(PersonService.name, () => {
const { person } = await ctx.newPerson({ ownerId: user.id });
const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 150, height: 200 });
await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 200 });
+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();
await ctx.newEdits(asset.id, {
edits: [
@@ -199,6 +202,7 @@ describe(PersonService.name, () => {
const { person } = await ctx.newPerson({ ownerId: user.id });
const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 100, height: 200 });
await ctx.newExif({ assetId: asset.id, exifImageWidth: 200, exifImageHeight: 100 });
+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();
await ctx.newEdits(asset.id, {
edits: [
@@ -263,6 +267,7 @@ describe(PersonService.name, () => {
const { person } = await ctx.newPerson({ ownerId: user.id });
const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 200, height: 100 });
await ctx.newExif({ assetId: asset.id, exifImageHeight: 100, exifImageWidth: 200 });
+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();
await ctx.newEdits(asset.id, {
edits: [
@@ -327,6 +332,7 @@ describe(PersonService.name, () => {
const { person } = await ctx.newPerson({ ownerId: user.id });
const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 200, height: 150 });
await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 200 });
+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();
await ctx.newEdits(asset.id, {
edits: [
@@ -400,6 +406,7 @@ describe(PersonService.name, () => {
const { person } = await ctx.newPerson({ ownerId: user.id });
const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 150, height: 100 });
await ctx.newExif({ assetId: asset.id, exifImageHeight: 100, exifImageWidth: 200 });
+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();
await ctx.newEdits(asset.id, {
edits: [
@@ -473,6 +480,7 @@ describe(PersonService.name, () => {
const { person } = await ctx.newPerson({ ownerId: user.id });
const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 200, height: 150 });
await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 150 });
+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();
await ctx.newEdits(asset.id, {
edits: [
@@ -543,6 +551,7 @@ describe(PersonService.name, () => {
const { person } = await ctx.newPerson({ ownerId: user.id });
const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 150, height: 100 });
await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 200 });
+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();
await ctx.newEdits(asset.id, {
edits: [
@@ -622,6 +631,7 @@ describe(PersonService.name, () => {
const { person } = await ctx.newPerson({ ownerId: user.id });
const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 100, height: 100 });
await ctx.newExif({ assetId: asset.id, exifImageHeight: 100, exifImageWidth: 100 });
+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();
await ctx.newEdits(asset.id, {
edits: [
@@ -692,6 +702,7 @@ describe(PersonService.name, () => {
const { person } = await ctx.newPerson({ ownerId: user.id });
const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 100, height: 100 });
await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 100, orientation: '6' });
+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();
await ctx.newEdits(asset.id, {
edits: [
diff --git a/web/src/lib/components/asset-viewer/face-editor/face-editor.svelte b/web/src/lib/components/asset-viewer/face-editor/face-editor.svelte
index 8f7103c2a3..2418e5bf85 100644
--- a/web/src/lib/components/asset-viewer/face-editor/face-editor.svelte
+++ b/web/src/lib/components/asset-viewer/face-editor/face-editor.svelte
@@ -2,6 +2,7 @@
import { shortcut } from '$lib/actions/shortcut';
import ImageThumbnail from '$lib/components/assets/thumbnail/image-thumbnail.svelte';
import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';
+ import FaceCreateTagModal from '$lib/modals/CreateFaceModal.svelte';
import { getPeopleThumbnailUrl } from '$lib/utils';
import { getNaturalSize, scaleToFit } from '$lib/utils/container-utils';
import { handleError } from '$lib/utils/handle-error';
@@ -260,6 +261,44 @@
};
};
+ type FaceCoordinates = NonNullable<ReturnType<typeof getFaceCroppedCoordinates>>;
+
+ const getFacePreviewUrl = (data: FaceCoordinates) => {
+ if (!htmlElement) {
+ return;
+ }
+
+ const natural = getNaturalSize(htmlElement);
+ if (natural.width <= 0 || natural.height <= 0) {
+ return;
+ }
+
+ const x = clamp(data.x, 0, natural.width - 1);
+ const y = clamp(data.y, 0, natural.height - 1);
+ const width = clamp(data.width, 1, natural.width - x);
+ const height = clamp(data.height, 1, natural.height - y);
+
+ if (width <= 0 || height <= 0) {
+ return;
+ }
+
+ const canvas = document.createElement('canvas');
+ canvas.width = width;
+ canvas.height = height;
+
+ const context = canvas.getContext('2d');
+ if (!context) {
+ return;
+ }
+
+ try {
+ context.drawImage(htmlElement, x, y, width, height, 0, 0, width, height);
+ return canvas.toDataURL('image/png');
+ } catch {
+ return;
+ }
+ };
+
const tagFace = async (person: PersonResponseDto) => {
try {
const data = getFaceCroppedCoordinates();
@@ -294,6 +333,28 @@
}
};
+ const showCreateFaceModal = async () => {
+ try {
+ const data = getFaceCroppedCoordinates();
+ if (!data) {
+ return;
+ }
+
+ const created = await modalManager.show(FaceCreateTagModal, {
+ assetId,
+ ...data,
+ previewUrl: getFacePreviewUrl(data),
+ });
+ if (!created) {
+ return;
+ }
+
+ onClose();
+ } catch (error) {
+ handleError(error, 'Error creating and tagging face');
+ }
+ };
+
onDestroy(() => {
onClose();
});
@@ -303,19 +364,19 @@
<div
id="face-editor-data"
- class="absolute start-0 top-0 z-5 h-full w-full overflow-hidden"
+ class="absolute inset-s-0 top-0 z-5 h-full w-full overflow-hidden"
data-overlay-interactive
data-face-left={faceBoxPosition.left}
data-face-top={faceBoxPosition.top}
data-face-width={faceBoxPosition.width}
data-face-height={faceBoxPosition.height}
>
- <canvas bind:this={canvasEl} id="face-editor" class="absolute top-0 start-0"></canvas>
+ <canvas bind:this={canvasEl} id="face-editor" class="absolute top-0 inset-s-0"></canvas>
<div
id="face-selector"
bind:this={faceSelectorEl}
- class="absolute top-[calc(50%-250px)] start-[calc(50%-125px)] max-w-[250px] w-[250px] bg-white dark:bg-immich-dark-gray dark:text-immich-dark-fg backdrop-blur-sm px-2 py-4 rounded-xl border border-gray-200 dark:border-gray-800 transition-[top,left] duration-200 ease-out"
+ class="absolute top-[calc(50%-250px)] inset-s-[calc(50%-125px)] max-w-62.5 w-62.5 bg-white dark:bg-immich-dark-gray dark:text-immich-dark-fg backdrop-blur-sm px-2 py-4 rounded-xl border border-gray-200 dark:border-gray-800 transition-[top,left] duration-200 ease-out"
>
<p class="text-center text-sm">{$t('select_person_to_tag')}</p>
@@ -354,6 +415,12 @@
{/if}
</div>
- <Button size="small" fullWidth onclick={onClose} color="danger" class="mt-2">{$t('cancel')}</Button>
+ <Button size="small" fullWidth onclick={showCreateFaceModal} variant="outline" class="mt-2">
+ {$t('create_person')}
+ </Button>
+
+ <Button size="small" fullWidth onclick={onClose} color="danger" class="mt-2">
+ {$t('cancel')}
+ </Button>
</div>
</div>
diff --git a/web/src/lib/modals/CreateFaceModal.svelte b/web/src/lib/modals/CreateFaceModal.svelte
new file mode 100644
index 0000000000..fa845cba2d
--- /dev/null
+++ b/web/src/lib/modals/CreateFaceModal.svelte
@@ -0,0 +1,91 @@
+<script lang="ts">
+ import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';
+ import { delay } from '$lib/utils/asset-utils';
+ import { handleError } from '$lib/utils/handle-error';
+ import { createFace, createPerson } from '@immich/sdk';
+ import { Field, FormModal, Input, LoadingSpinner, Text } from '@immich/ui';
+ import { t } from 'svelte-i18n';
+
+ type Props = {
+ assetId: string;
+ imageWidth: number;
+ imageHeight: number;
+ x: number;
+ y: number;
+ width: number;
+ height: number;
+ onClose: (created?: boolean) => void;
+ previewUrl?: string;
+ };
+
+ let { assetId, imageWidth, imageHeight, x, y, width, height, onClose, previewUrl }: Props = $props();
+ let personName = $state('');
+ let isSubmitting = $state(false);
+
+ const getTrimmedName = () => personName.trim();
+
+ const onSubmit = async () => {
+ const name = getTrimmedName();
+ if (!name) {
+ return;
+ }
+
+ try {
+ isSubmitting = true;
+
+ const person = await createPerson({
+ personCreateDto: {
+ name,
+ },
+ });
+
+ await createFace({
+ assetFaceCreateDto: {
+ assetId,
+ imageHeight,
+ imageWidth,
+ personId: person.id,
+ x,
+ y,
+ width,
+ height,
+ },
+ });
+
+ await delay(1500);
+ await assetViewerManager.setAssetId(assetId);
+ onClose(true);
+ } catch (error) {
+ handleError(error, 'Error creating and tagging face');
+ } finally {
+ isSubmitting = false;
+ }
+ };
+</script>
+
+<FormModal
+ size="small"
+ title={$t('create_person')}
+ submitText={$t('tag_face')}
+ disabled={!getTrimmedName() || isSubmitting}
+ {onClose}
+ {onSubmit}
+>
+ <Text size="tiny" class="mb-4" color="muted">{$t('create_person_subtitle')}</Text>
+ {#if previewUrl}
+ <Field label={$t('preview')}>
+ <div class="flex justify-center rounded-xl bg-gray-50 p-3 dark:border-gray-700 dark:bg-black/20 relative">
+ <img src={previewUrl} alt={$t('preview')} class="max-h-48 rounded-lg object-contain shadow-sm" />
+ {#if isSubmitting}
+ <div class="flex place-items-center place-content-center absolute inset-0 bg-black/20 rounded-lg">
+ <LoadingSpinner />
+ </div>
+ {/if}
+ </div>
+ </Field>
+ {/if}
+
+ <Field label={$t('name')} required class="mt-3">
+ <Input autofocus bind:value={personName} disabled={isSubmitting} />
+ </Field>
+</FormModal>