High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix: locking around concurrency-1 jobs (#28410)
bo0tzz477452b5c51c
diff --git a/server/src/enum.ts b/server/src/enum.tsindex 4151c88d75..843339d221 100644--- a/server/src/enum.ts+++ b/server/src/enum.ts@@ -1018,6 +1018,8 @@ export enum DatabaseLock {IntegrityCheck = 67,VersionCheck = 800,HlsSessionCleanup = 850,+ FacialRecognition = 900,+ DuplicateDetection = 1000,}export enum MaintenanceAction {diff --git a/server/src/services/duplicate.service.ts b/server/src/services/duplicate.service.tsindex 7bcb371dd8..1d01e195a4 100644--- a/server/src/services/duplicate.service.ts+++ b/server/src/services/duplicate.service.ts@@ -5,7 +5,7 @@ import { BulkIdErrorReason, BulkIdResponseDto, BulkIdsDto } from 'src/dtos/assetimport { MapAsset, mapAsset } from 'src/dtos/asset-response.dto.js';import { AuthDto } from 'src/dtos/auth.dto.js';import { DuplicateResolveDto, DuplicateResolveGroupDto, DuplicateResponseDto } from 'src/dtos/duplicate.dto.js';-import { AssetStatus, AssetVisibility, JobName, JobStatus, Permission, QueueName } from 'src/enum.js';+import { AssetStatus, AssetVisibility, DatabaseLock, JobName, JobStatus, Permission, QueueName } from 'src/enum.js';import { AssetDuplicateResult } from 'src/repositories/search.repository.js';import { BaseService } from 'src/services/base.service.js';import { suggestDuplicateKeepAssetIds } from 'src/utils/duplicate.js';@@ -317,60 +317,62 @@ export class DuplicateService extends BaseService {@OnJob({ name: JobName.AssetDetectDuplicates, queue: QueueName.DuplicateDetection })async handleSearchDuplicates({ id }: JobOf<JobName.AssetDetectDuplicates>): Promise<JobStatus> {- const { machineLearning } = await this.getConfig({ withCache: true });- if (!isDuplicateDetectionEnabled(machineLearning)) {- return JobStatus.Skipped;- }+ return this.databaseRepository.withLock(DatabaseLock.DuplicateDetection, async () => {+ const { machineLearning } = await this.getConfig({ withCache: true });+ if (!isDuplicateDetectionEnabled(machineLearning)) {+ return JobStatus.Skipped;+ }- const asset = await this.assetJobRepository.getForSearchDuplicatesJob(id);- if (!asset) {- this.logger.error(`Asset ${id} not found`);- return JobStatus.Failed;- }+ const asset = await this.assetJobRepository.getForSearchDuplicatesJob(id);+ if (!asset) {+ this.logger.error(`Asset ${id} not found`);+ return JobStatus.Failed;+ }- if (asset.stackId) {- this.logger.debug(`Asset ${id} is part of a stack, skipping`);- return JobStatus.Skipped;- }+ if (asset.stackId) {+ this.logger.debug(`Asset ${id} is part of a stack, skipping`);+ return JobStatus.Skipped;+ }- if (asset.visibility === AssetVisibility.Hidden) {- this.logger.debug(`Asset ${id} is not visible, skipping`);- return JobStatus.Skipped;- }+ if (asset.visibility === AssetVisibility.Hidden) {+ this.logger.debug(`Asset ${id} is not visible, skipping`);+ return JobStatus.Skipped;+ }- if (asset.visibility === AssetVisibility.Locked) {- this.logger.debug(`Asset ${id} is locked, skipping`);- return JobStatus.Skipped;- }+ if (asset.visibility === AssetVisibility.Locked) {+ this.logger.debug(`Asset ${id} is locked, skipping`);+ return JobStatus.Skipped;+ }- if (!asset.embedding) {- this.logger.debug(`Asset ${id} is missing embedding`);- return JobStatus.Failed;- }+ if (!asset.embedding) {+ this.logger.debug(`Asset ${id} is missing embedding`);+ return JobStatus.Failed;+ }- const duplicateAssets = await this.duplicateRepository.search({- assetId: asset.id,- embedding: asset.embedding,- maxDistance: machineLearning.duplicateDetection.maxDistance,- type: asset.type,- userIds: [asset.ownerId],- });+ const duplicateAssets = await this.duplicateRepository.search({+ assetId: asset.id,+ embedding: asset.embedding,+ maxDistance: machineLearning.duplicateDetection.maxDistance,+ type: asset.type,+ userIds: [asset.ownerId],+ });- let assetIds = [asset.id];- if (duplicateAssets.length > 0) {- this.logger.debug(- `Found ${duplicateAssets.length} duplicate${duplicateAssets.length === 1 ? '' : 's'} for asset ${asset.id}`,- );- assetIds = await this.updateDuplicates(asset, duplicateAssets);- } else if (asset.duplicateId) {- this.logger.debug(`No duplicates found for asset ${asset.id}, removing duplicateId`);- await this.assetRepository.update({ id: asset.id, duplicateId: null });- }+ let assetIds = [asset.id];+ if (duplicateAssets.length > 0) {+ this.logger.debug(+ `Found ${duplicateAssets.length} duplicate${duplicateAssets.length === 1 ? '' : 's'} for asset ${asset.id}`,+ );+ assetIds = await this.updateDuplicates(asset, duplicateAssets);+ } else if (asset.duplicateId) {+ this.logger.debug(`No duplicates found for asset ${asset.id}, removing duplicateId`);+ await this.assetRepository.update({ id: asset.id, duplicateId: null });+ }- const duplicatesDetectedAt = new Date();- await this.assetRepository.upsertJobStatus(...assetIds.map((assetId) => ({ assetId, duplicatesDetectedAt })));+ const duplicatesDetectedAt = new Date();+ await this.assetRepository.upsertJobStatus(...assetIds.map((assetId) => ({ assetId, duplicatesDetectedAt })));- return JobStatus.Success;+ return JobStatus.Success;+ });}private async updateDuplicates(diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.tsindex 4109b019b3..2e20cd01cd 100644--- a/server/src/services/person.service.ts+++ b/server/src/services/person.service.ts@@ -32,6 +32,7 @@ import {import {AssetVisibility,CacheControl,+ DatabaseLock,JobName,JobStatus,Permission,@@ -499,150 +500,154 @@ export class PersonService extends BaseService {nightly,clusterGroupId,}: JobOf<JobName.FacialRecognitionQueueAll>): Promise<JobStatus> {- const { machineLearning } = await this.getConfig({ withCache: false });- if (!isFacialRecognitionEnabled(machineLearning)) {- return JobStatus.Skipped;- }+ return this.databaseRepository.withLock(DatabaseLock.FacialRecognition, async () => {+ const { machineLearning } = await this.getConfig({ withCache: false });+ if (!isFacialRecognitionEnabled(machineLearning)) {+ return JobStatus.Skipped;+ }- await this.jobRepository.waitForQueueCompletion(QueueName.ThumbnailGeneration, QueueName.FaceDetection);+ await this.jobRepository.waitForQueueCompletion(QueueName.ThumbnailGeneration, QueueName.FaceDetection);- if (nightly) {- const [state, latestFaceDate] = await Promise.all([- this.systemMetadataRepository.get(SystemMetadataKey.FacialRecognitionState),- this.personRepository.getLatestFaceDate(),- ]);+ if (nightly) {+ const [state, latestFaceDate] = await Promise.all([+ this.systemMetadataRepository.get(SystemMetadataKey.FacialRecognitionState),+ this.personRepository.getLatestFaceDate(),+ ]);- if (state?.lastRun && latestFaceDate && state.lastRun > latestFaceDate) {- this.logger.debug('Skipping facial recognition nightly since no face has been added since the last run');- return JobStatus.Skipped;+ if (state?.lastRun && latestFaceDate && state.lastRun > latestFaceDate) {+ this.logger.debug('Skipping facial recognition nightly since no face has been added since the last run');+ return JobStatus.Skipped;+ }}- }- const { waiting } = await this.jobRepository.getJobCounts(QueueName.FacialRecognition);+ const { waiting } = await this.jobRepository.getJobCounts(QueueName.FacialRecognition);- if (force) {- await this.personRepository.unassignFaces({ clusterGroupId, sourceType: SourceType.MachineLearning });- await this.handlePersonCleanup();- await this.vacuum('asset_face', 'person');- } else if (waiting) {- this.logger.debug(- `Skipping facial recognition queueing because ${waiting} job${waiting > 1 ? 's are' : ' is'} already queued`,- );- return JobStatus.Skipped;- }+ if (force) {+ await this.personRepository.unassignFaces({ clusterGroupId, sourceType: SourceType.MachineLearning });+ await this.handlePersonCleanup();+ await this.vacuum('asset_face', 'person');+ } else if (waiting) {+ this.logger.debug(+ `Skipping facial recognition queueing because ${waiting} job${waiting > 1 ? 's are' : ' is'} already queued`,+ );+ return JobStatus.Skipped;+ }- await this.databaseRepository.prewarm(VectorIndex.Face);+ await this.databaseRepository.prewarm(VectorIndex.Face);- const lastRun = new Date().toISOString();+ const lastRun = new Date().toISOString();- const faces = this.personRepository.getAllFaces(- force- ? { clusterGroupId, sourceType: clusterGroupId ? SourceType.MachineLearning : undefined }- : { personGroupId: null, clusterGroupId, sourceType: SourceType.MachineLearning },- );- for await (const batch of batched(faces)) {- await this.jobRepository.queueAll(- batch.map((face) => ({ name: JobName.FacialRecognition, data: { id: face.id, deferred: false } })),+ const faces = this.personRepository.getAllFaces(+ force+ ? { clusterGroupId, sourceType: clusterGroupId ? SourceType.MachineLearning : undefined }+ : { personGroupId: null, clusterGroupId, sourceType: SourceType.MachineLearning },);- }+ for await (const batch of batched(faces)) {+ await this.jobRepository.queueAll(+ batch.map((face) => ({ name: JobName.FacialRecognition, data: { id: face.id, deferred: false } })),+ );+ }- await this.systemMetadataRepository.set(SystemMetadataKey.FacialRecognitionState, { lastRun });+ await this.systemMetadataRepository.set(SystemMetadataKey.FacialRecognitionState, { lastRun });- return JobStatus.Success;+ return JobStatus.Success;+ });}@OnJob({ name: JobName.FacialRecognition, queue: QueueName.FacialRecognition })async handleRecognizeFaces({ id, deferred }: JobOf<JobName.FacialRecognition>): Promise<JobStatus> {- const { machineLearning } = await this.getConfig({ withCache: true });- if (!isFacialRecognitionEnabled(machineLearning)) {- return JobStatus.Skipped;- }-- const face = await this.personRepository.getFaceForFacialRecognitionJob(id);- if (!face || !face.asset) {- this.logger.warn(`Face ${id} not found`);- return JobStatus.Failed;- }-- if (face.sourceType !== SourceType.MachineLearning) {- this.logger.warn(`Skipping face ${id} due to source ${face.sourceType}`);- return JobStatus.Skipped;- }-- if (!face.faceSearch?.embedding) {- this.logger.warn(`Face ${id} does not have an embedding`);- return JobStatus.Failed;- }-- if (face.personGroupId) {- this.logger.debug(`Face ${id} already has a person assigned`);- return JobStatus.Skipped;- }+ return this.databaseRepository.withLock(DatabaseLock.FacialRecognition, async () => {+ const { machineLearning } = await this.getConfig({ withCache: true });+ if (!isFacialRecognitionEnabled(machineLearning)) {+ return JobStatus.Skipped;+ }- const { ownerId, clusterGroupId } = face.asset;- const matches = await this.searchRepository.searchFaces({- clusterGroupId,- embedding: face.faceSearch.embedding,- maxDistance: machineLearning.facialRecognition.maxDistance,- numResults: machineLearning.facialRecognition.minFaces,- minBirthDate: new Date(face.asset.fileCreatedAt),- });+ const face = await this.personRepository.getFaceForFacialRecognitionJob(id);+ if (!face || !face.asset) {+ this.logger.warn(`Face ${id} not found`);+ return JobStatus.Failed;+ }- // `matches` also includes the face itself- if (machineLearning.facialRecognition.minFaces > 1 && matches.length <= 1) {- this.logger.debug(`Face ${id} only matched the face itself, skipping`);- return JobStatus.Skipped;- }+ if (face.sourceType !== SourceType.MachineLearning) {+ this.logger.warn(`Skipping face ${id} due to source ${face.sourceType}`);+ return JobStatus.Skipped;+ }- this.logger.debug(`Face ${id} has ${matches.length} matches`);+ if (!face.faceSearch?.embedding) {+ this.logger.warn(`Face ${id} does not have an embedding`);+ return JobStatus.Failed;+ }- const isCore =- matches.length >= machineLearning.facialRecognition.minFaces &&- face.asset.visibility === AssetVisibility.Timeline;- if (!isCore && !deferred) {- this.logger.debug(`Deferring non-core face ${id} for later processing`);- await this.jobRepository.queue({ name: JobName.FacialRecognition, data: { id, deferred: true } });- return JobStatus.Skipped;- }+ if (face.personGroupId) {+ this.logger.debug(`Face ${id} already has a person assigned`);+ return JobStatus.Skipped;+ }- let personGroupId = matches.find((match) => match.personGroupId)?.personGroupId;- if (!personGroupId) {- const [matchWithPerson] = await this.searchRepository.searchFaces({+ const { ownerId, clusterGroupId } = face.asset;+ const matches = await this.searchRepository.searchFaces({clusterGroupId,embedding: face.faceSearch.embedding,maxDistance: machineLearning.facialRecognition.maxDistance,- numResults: 1,- hasPerson: true,+ numResults: machineLearning.facialRecognition.minFaces,minBirthDate: new Date(face.asset.fileCreatedAt),});- personGroupId = matchWithPerson?.personGroupId ?? undefined;- }+ // `matches` also includes the face itself+ if (machineLearning.facialRecognition.minFaces > 1 && matches.length <= 1) {+ this.logger.debug(`Face ${id} only matched the face itself, skipping`);+ return JobStatus.Skipped;+ }- if (!personGroupId && isCore) {- const group = await this.personRepository.createGroup(ownerId);- personGroupId = group.id;- this.logger.log(`Created person group ${personGroupId} for face ${id}`);- }+ this.logger.debug(`Face ${id} has ${matches.length} matches`);- if (personGroupId) {- const person = await this.personRepository.getByGroupId({ ownerId, personGroupId });- if (person) {- this.logger.debug(`Face ${id} matched person ${person.personGroupId}`);- } else {- await this.personRepository.create({ ownerId, faceAssetId: face.id, personGroupId });- this.logger.log(`Created person for face ${id} in group ${personGroupId}`);- await this.jobRepository.queue({- name: JobName.PersonGenerateThumbnail,- data: { ownerId, personGroupId },+ const isCore =+ matches.length >= machineLearning.facialRecognition.minFaces &&+ face.asset.visibility === AssetVisibility.Timeline;+ if (!isCore && !deferred) {+ this.logger.debug(`Deferring non-core face ${id} for later processing`);+ await this.jobRepository.queue({ name: JobName.FacialRecognition, data: { id, deferred: true } });+ return JobStatus.Skipped;+ }++ let personGroupId = matches.find((match) => match.personGroupId)?.personGroupId;+ if (!personGroupId) {+ const [matchWithPerson] = await this.searchRepository.searchFaces({+ clusterGroupId,+ embedding: face.faceSearch.embedding,+ maxDistance: machineLearning.facialRecognition.maxDistance,+ numResults: 1,+ hasPerson: true,+ minBirthDate: new Date(face.asset.fileCreatedAt),});++ personGroupId = matchWithPerson?.personGroupId ?? undefined;}- this.logger.debug(`Assigning face ${id} to person group ${personGroupId}`);- await this.personRepository.reassignFaces({ faceIds: [id], newPersonGroupId: personGroupId });- }+ if (!personGroupId && isCore) {+ const group = await this.personRepository.createGroup(ownerId);+ personGroupId = group.id;+ this.logger.log(`Created person group ${personGroupId} for face ${id}`);+ }- return JobStatus.Success;+ if (personGroupId) {+ const person = await this.personRepository.getByGroupId({ ownerId, personGroupId });+ if (person) {+ this.logger.debug(`Face ${id} matched person ${person.personGroupId}`);+ } else {+ await this.personRepository.create({ ownerId, faceAssetId: face.id, personGroupId });+ this.logger.log(`Created person for face ${id} in group ${personGroupId}`);+ await this.jobRepository.queue({+ name: JobName.PersonGenerateThumbnail,+ data: { ownerId, personGroupId },+ });+ }++ this.logger.debug(`Assigning face ${id} to person group ${personGroupId}`);+ await this.personRepository.reassignFaces({ faceIds: [id], newPersonGroupId: personGroupId });+ }++ return JobStatus.Success;+ });}@OnJob({ name: JobName.PersonFileMigration, queue: QueueName.Migration })