Skip to content

High performance self-hosted photo and video management solution.

fix: face detection of edited assets (#31240)

Daniel Dietzler399217bf5fbd
diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql
index 96af89edb5..0611cd5973 100644
--- a/server/src/queries/asset.job.repository.sql
+++ b/server/src/queries/asset.job.repository.sql
@@ -465,7 +465,7 @@ select
) as "faces",
(
select
- coalesce(json_agg(agg), '[]')
+ to_json(obj)
from
(
select
@@ -477,14 +477,18 @@ select
"asset_file"
where
"asset_file"."assetId" = "asset"."id"
- and "asset_file"."type" = $1
- ) as agg
- ) as "files"
+ and "asset_file"."type" = 'preview'
+ order by
+ "asset_file"."isEdited" desc
+ limit
+ 1
+ ) as obj
+ ) as "previewFile"
from
"asset"
inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId"
where
- "asset"."id" = $2
+ "asset"."id" = $1
-- AssetJobRepository.getForOcr
select
diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts
index 5c7924a208..78e1b0b3ad 100644
--- a/server/src/repositories/asset-job.repository.ts
+++ b/server/src/repositories/asset-job.repository.ts
@@ -1,6 +1,6 @@
import { Injectable } from '@nestjs/common';
import { Kysely, sql } from 'kysely';
-import { jsonArrayFrom } from 'kysely/helpers/postgres';
+import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
import { InjectKysely } from 'nestjs-kysely';
import { columns } from 'src/database';
import { DummyValue, GenerateSql } from 'src/decorators';
@@ -236,7 +236,17 @@ export class AssetJobRepository {
.select(['asset.id', 'asset.visibility'])
.$call(withExifInner)
.select((eb) => withFaces(eb, true, true))
- .select((eb) => withFiles(eb, AssetFileType.Preview))
+ .select((eb) =>
+ jsonObjectFrom(
+ eb
+ .selectFrom('asset_file')
+ .select(columns.assetFiles)
+ .whereRef('asset_file.assetId', '=', 'asset.id')
+ .where('asset_file.type', '=', sql.lit(AssetFileType.Preview))
+ .orderBy('asset_file.isEdited', 'desc')
+ .limit(sql.lit(1)),
+ ).as('previewFile'),
+ )
.where('asset.id', '=', id)
.executeTakeFirst();
}
diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts
index dcb0c73bea..0606eb70c8 100644
--- a/server/src/services/person.service.ts
+++ b/server/src/services/person.service.ts
@@ -320,8 +320,8 @@ export class PersonService extends BaseService {
}
const asset = await this.assetJobRepository.getForDetectFacesJob(id);
- const previewFile = asset?.files[0];
- if (!asset || asset.files.length !== 1 || !previewFile) {
+ const previewFile = asset?.previewFile;
+ if (!asset || !previewFile) {
return JobStatus.Failed;
}
diff --git a/server/test/mappers.ts b/server/test/mappers.ts
index f2694f184f..6937c55456 100644
--- a/server/test/mappers.ts
+++ b/server/test/mappers.ts
@@ -1,6 +1,7 @@
import { Selectable, ShallowDehydrateObject } from 'kysely';
import { MapAsset } from 'src/dtos/asset-response.dto';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
+import { AssetFileType } from 'src/enum';
import { FaceSearchResult } from 'src/repositories/search.repository';
import { ActivityTable } from 'src/schema/tables/activity.table';
import { AssetTable } from 'src/schema/tables/asset.table';
@@ -188,7 +189,10 @@ export const getForDetectedFaces = (asset: ReturnType<AssetFactory['build']>) =>
visibility: asset.visibility,
exifInfo: getDehydrated(asset.exifInfo),
faces: asset.faces.map((face) => getDehydrated(face)),
- files: asset.files.map((file) => getDehydrated(file)),
+ previewFile: asset.files
+ .filter((file) => file.type === AssetFileType.Preview)
+ .toSorted((a) => (a.isEdited ? -1 : 1))
+ .map((file) => getDehydrated(file))[0],
});
export const getForSidecarWrite = (asset: ReturnType<AssetFactory['build']>) => ({
diff --git a/server/test/medium/specs/services/person.service.spec.ts b/server/test/medium/specs/services/person.service.spec.ts
index 2adefe7e7a..377d8c5eca 100644
--- a/server/test/medium/specs/services/person.service.spec.ts
+++ b/server/test/medium/specs/services/person.service.spec.ts
@@ -2,14 +2,16 @@ import { Kysely } from 'kysely';
import { DateTime } from 'luxon';
import { AssetEditAction, MirrorAxis } from 'src/dtos/editing.dto';
import { AssetFaceCreateDto } from 'src/dtos/person.dto';
-import { JobName } from 'src/enum';
+import { AssetFileType, JobName } from 'src/enum';
import { AccessRepository } from 'src/repositories/access.repository';
import { AssetEditRepository } from 'src/repositories/asset-edit.repository';
+import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { ConfigRepository } from 'src/repositories/config.repository';
import { DatabaseRepository } from 'src/repositories/database.repository';
import { JobRepository } from 'src/repositories/job.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
+import { MachineLearningRepository } from 'src/repositories/machine-learning.repository';
import { PersonRepository } from 'src/repositories/person.repository';
import { StorageRepository } from 'src/repositories/storage.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
@@ -26,6 +28,7 @@ const setup = (db?: Kysely<DB>) => {
database: db || defaultDatabase,
real: [
AccessRepository,
+ AssetJobRepository,
ConfigRepository,
DatabaseRepository,
PersonRepository,
@@ -33,7 +36,7 @@ const setup = (db?: Kysely<DB>) => {
AssetEditRepository,
SystemMetadataRepository,
],
- mock: [JobRepository, LoggingRepository, StorageRepository],
+ mock: [JobRepository, LoggingRepository, StorageRepository, MachineLearningRepository],
});
};
@@ -99,6 +102,38 @@ describe(PersonService.name, () => {
});
});
+ describe('handleDetectFaces', () => {
+ it('should prefer an edited preview file', async () => {
+ const { sut, ctx } = setup();
+ const config = await ctx.getConfig();
+ const { user } = await ctx.newUser();
+ const { asset } = await ctx.newAsset({ ownerId: user.id });
+ await ctx.newExif({ assetId: asset.id, description: '' });
+ await ctx.newAssetFile({
+ assetId: asset.id,
+ type: AssetFileType.Preview,
+ isEdited: true,
+ path: 'edited_file.jpg',
+ });
+ await ctx.newAssetFile({
+ assetId: asset.id,
+ type: AssetFileType.Preview,
+ isEdited: false,
+ path: 'unedited_file.jpg',
+ });
+ ctx
+ .getMock(MachineLearningRepository)
+ .detectFaces.mockResolvedValue({ imageHeight: 42, imageWidth: 69, faces: [] });
+
+ await sut.handleDetectFaces({ id: asset.id });
+
+ expect(ctx.getMock(MachineLearningRepository).detectFaces).toHaveBeenCalledWith(
+ 'edited_file.jpg',
+ config.machineLearning.facialRecognition,
+ );
+ });
+ });
+
describe('handleQueueRecognizeFaces', () => {
it('should delete all people and queue faces for recognition', async () => {
const { sut, ctx } = setup();