High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix(server): normalize bounding boxes when matching detected faces to existing faces (#32486)
Baneina5583cd7857a4
diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.tsindex 29946004c0..2f75230a96 100644--- a/server/src/services/person.service.spec.ts+++ b/server/src/services/person.service.spec.ts@@ -1213,6 +1213,66 @@ describe(PersonService.name, () => {expect(mocks.person.reassignFaces).not.toHaveBeenCalled();});+ it('should match metadata face stored at a different resolution', async () => {+ const face = AssetFaceFactory.create({+ sourceType: SourceType.Exif,+ imageWidth: 2736,+ imageHeight: 3648,+ boundingBoxX1: 1225,+ boundingBoxY1: 1453,+ boundingBoxX2: 1436,+ boundingBoxY2: 1732,+ });+ const asset = AssetFactory.from().face(face).file({ type: AssetFileType.Preview }).exif().build();+ mocks.machineLearning.detectFaces.mockResolvedValue({+ imageWidth: 1440,+ imageHeight: 1920,+ faces: [{ boundingBox: { x1: 645, y1: 765, x2: 756, y2: 912 }, embedding: '[1, 2, 3, 4]', score: 0.9 }],+ });+ mocks.assetJob.getForDetectFacesJob.mockResolvedValue(getForDetectedFaces(asset));++ await sut.handleDetectFaces({ id: asset.id });++ expect(mocks.person.refreshFaces).toHaveBeenCalledWith([], [], [{ faceId: face.id, embedding: '[1, 2, 3, 4]' }]);+ expect(mocks.job.queueAll).not.toHaveBeenCalled();+ });++ it('should prefer metadata face over overlapping machine learning face', async () => {+ const mlFace = AssetFaceFactory.create({+ imageWidth: 1440,+ imageHeight: 1920,+ boundingBoxX1: 645,+ boundingBoxY1: 765,+ boundingBoxX2: 756,+ boundingBoxY2: 912,+ });+ const exifFace = AssetFaceFactory.create({+ sourceType: SourceType.Exif,+ imageWidth: 2736,+ imageHeight: 3648,+ boundingBoxX1: 1225,+ boundingBoxY1: 1453,+ boundingBoxX2: 1436,+ boundingBoxY2: 1732,+ });+ const asset = AssetFactory.from()+ .face(mlFace)+ .face(exifFace)+ .file({ type: AssetFileType.Preview })+ .exif()+ .build();+ mocks.machineLearning.detectFaces.mockResolvedValue(getAsDetectedFace(mlFace));+ mocks.assetJob.getForDetectFacesJob.mockResolvedValue(getForDetectedFaces(asset));++ await sut.handleDetectFaces({ id: asset.id });++ expect(mocks.person.refreshFaces).toHaveBeenCalledWith(+ [],+ [mlFace.id],+ [{ faceId: exifFace.id, embedding: '[1, 2, 3, 4]' }],+ );+ });+it('should not add embedding to non-matching metadata face', async () => {const assetId = newUuid();const face = AssetFaceFactory.create({ assetId, sourceType: SourceType.Exif });diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.tsindex 3aaa34dead..1fcf6eb164 100644--- a/server/src/services/person.service.ts+++ b/server/src/services/person.service.ts@@ -414,16 +414,10 @@ export class PersonService extends BaseService {}}- const heightScale = imageHeight / (asset.faces[0]?.imageHeight || 1);- const widthScale = imageWidth / (asset.faces[0]?.imageWidth || 1);for (const { boundingBox, embedding } of faces) {- const scaledBox = {- x1: boundingBox.x1 * widthScale,- y1: boundingBox.y1 * heightScale,- x2: boundingBox.x2 * widthScale,- y2: boundingBox.y2 * heightScale,- };- const match = asset.faces.find((face) => this.iou(face, scaledBox) > 0.5);+ const matches = asset.faces.filter((face) => this.iou(face, boundingBox, imageWidth, imageHeight) > 0.5);+ // prefer metadata faces over machine learning duplicates+ const match = matches.find((face) => face.sourceType !== SourceType.MachineLearning) ?? matches[0];if (match && !mlFaceIds.delete(match.id)) {embeddings.push({ faceId: match.id, embedding });@@ -466,17 +460,39 @@ export class PersonService extends BaseService {}private iou(- face: { boundingBoxX1: number; boundingBoxY1: number; boundingBoxX2: number; boundingBoxY2: number },+ face: {+ boundingBoxX1: number;+ boundingBoxY1: number;+ boundingBoxX2: number;+ boundingBoxY2: number;+ imageWidth: number;+ imageHeight: number;+ },newBox: BoundingBox,+ imageWidth: number,+ imageHeight: number,): number {- const x1 = Math.max(face.boundingBoxX1, newBox.x1);- const y1 = Math.max(face.boundingBoxY1, newBox.y1);- const x2 = Math.min(face.boundingBoxX2, newBox.x2);- const y2 = Math.min(face.boundingBoxY2, newBox.y2);+ const box1 = {+ x1: face.boundingBoxX1 / face.imageWidth,+ y1: face.boundingBoxY1 / face.imageHeight,+ x2: face.boundingBoxX2 / face.imageWidth,+ y2: face.boundingBoxY2 / face.imageHeight,+ };+ const box2 = {+ x1: newBox.x1 / imageWidth,+ y1: newBox.y1 / imageHeight,+ x2: newBox.x2 / imageWidth,+ y2: newBox.y2 / imageHeight,+ };++ const x1 = Math.max(box1.x1, box2.x1);+ const y1 = Math.max(box1.y1, box2.y1);+ const x2 = Math.min(box1.x2, box2.x2);+ const y2 = Math.min(box1.y2, box2.y2);const intersection = Math.max(0, x2 - x1) * Math.max(0, y2 - y1);- const area1 = (face.boundingBoxX2 - face.boundingBoxX1) * (face.boundingBoxY2 - face.boundingBoxY1);- const area2 = (newBox.x2 - newBox.x1) * (newBox.y2 - newBox.y1);+ const area1 = (box1.x2 - box1.x1) * (box1.y2 - box1.y1);+ const area2 = (box2.x2 - box2.x1) * (box2.y2 - box2.y1);const union = area1 + area2 - intersection;return intersection / union;