High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
refactor: duplicate queries (#19136)
Jason Rasmussen5cd186d3d435
diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sqlindex b6c5d4bea8..9792257652 100644--- a/server/src/queries/asset.repository.sql+++ b/server/src/queries/asset.repository.sql@@ -185,16 +185,6 @@ setwhere"id" = any ($2::uuid[])--- AssetRepository.updateDuplicates-update "assets"-set- "duplicateId" = $1-where- (- "duplicateId" = any ($2::uuid[])- or "id" = any ($3::uuid[])- )--- AssetRepository.getByChecksumselect"assets".*@@ -349,66 +339,6 @@ selectfrom"agg"--- AssetRepository.getDuplicates-with- "duplicates" as (- select- "assets"."duplicateId",- json_agg(- "asset"- order by- "assets"."localDateTime" asc- ) as "assets"- from- "assets"- left join lateral (- select- "assets".*,- "exif" as "exifInfo"- from- "exif"- where- "exif"."assetId" = "assets"."id"- ) as "asset" on true- where- "assets"."visibility" in ('archive', 'timeline')- and "assets"."ownerId" = $1::uuid- and "assets"."duplicateId" is not null- and "assets"."deletedAt" is null- and "assets"."stackId" is null- group by- "assets"."duplicateId"- ),- "unique" as (- select- "duplicateId"- from- "duplicates"- where- json_array_length("assets") = $2- ),- "removed_unique" as (- update "assets"- set- "duplicateId" = $3- from- "unique"- where- "assets"."duplicateId" = "unique"."duplicateId"- )-select- *-from- "duplicates"-where- not exists (- select- from- "unique"- where- "unique"."duplicateId" = "duplicates"."duplicateId"- )--- AssetRepository.getAssetIdByCitywith"cities" as (diff --git a/server/src/queries/duplicate.repository.sql b/server/src/queries/duplicate.repository.sqlnew file mode 100644index 0000000000..f008e57963--- /dev/null+++ b/server/src/queries/duplicate.repository.sql@@ -0,0 +1,103 @@+-- NOTE: This file is auto generated by ./sql-generator++-- DuplicateRepository.getAll+with+ "duplicates" as (+ select+ "assets"."duplicateId",+ json_agg(+ "asset"+ order by+ "assets"."localDateTime" asc+ ) as "assets"+ from+ "assets"+ left join lateral (+ select+ "assets".*,+ "exif" as "exifInfo"+ from+ "exif"+ where+ "exif"."assetId" = "assets"."id"+ ) as "asset" on true+ where+ "assets"."visibility" in ('archive', 'timeline')+ and "assets"."ownerId" = $1::uuid+ and "assets"."duplicateId" is not null+ and "assets"."deletedAt" is null+ and "assets"."stackId" is null+ group by+ "assets"."duplicateId"+ ),+ "unique" as (+ select+ "duplicateId"+ from+ "duplicates"+ where+ json_array_length("assets") = $2+ ),+ "removed_unique" as (+ update "assets"+ set+ "duplicateId" = $3+ from+ "unique"+ where+ "assets"."duplicateId" = "unique"."duplicateId"+ )+select+ *+from+ "duplicates"+where+ not exists (+ select+ from+ "unique"+ where+ "unique"."duplicateId" = "duplicates"."duplicateId"+ )++-- DuplicateRepository.search+begin+set+ local vchordrq.probes = 1+with+ "cte" as (+ select+ "assets"."id" as "assetId",+ "assets"."duplicateId",+ smart_search.embedding <=> $1 as "distance"+ from+ "assets"+ inner join "smart_search" on "assets"."id" = "smart_search"."assetId"+ where+ "assets"."visibility" in ('archive', 'timeline')+ and "assets"."ownerId" = any ($2::uuid[])+ and "assets"."deletedAt" is null+ and "assets"."type" = $3+ and "assets"."id" != $4::uuid+ and "assets"."stackId" is null+ order by+ "distance"+ limit+ $5+ )+select+ *+from+ "cte"+where+ "cte"."distance" <= $6+commit++-- DuplicateRepository.merge+update "assets"+set+where+ (+ "duplicateId" = any ($1::uuid[])+ or "id" = any ($2::uuid[])+ )diff --git a/server/src/queries/search.repository.sql b/server/src/queries/search.repository.sqlindex b68e0e177e..abbc063e8f 100644--- a/server/src/queries/search.repository.sql+++ b/server/src/queries/search.repository.sql@@ -102,39 +102,6 @@ offset$8commit--- SearchRepository.searchDuplicates-begin-set- local vchordrq.probes = 1-with- "cte" as (- select- "assets"."id" as "assetId",- "assets"."duplicateId",- smart_search.embedding <=> $1 as "distance"- from- "assets"- inner join "smart_search" on "assets"."id" = "smart_search"."assetId"- where- "assets"."visibility" in ('archive', 'timeline')- and "assets"."ownerId" = any ($2::uuid[])- and "assets"."deletedAt" is null- and "assets"."type" = $3- and "assets"."id" != $4::uuid- and "assets"."stackId" is null- order by- "distance"- limit- $5- )-select- *-from- "cte"-where- "cte"."distance" <= $6-commit--- SearchRepository.searchFacesbeginsetdiff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.tsindex af5239ed70..8cb971775c 100644--- a/server/src/repositories/asset.repository.ts+++ b/server/src/repositories/asset.repository.ts@@ -5,7 +5,6 @@ import { InjectKysely } from 'nestjs-kysely';import { Stack } from 'src/database';import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db';import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';-import { MapAsset } from 'src/dtos/asset-response.dto';import { AssetFileType, AssetOrder, AssetStatus, AssetType, AssetVisibility } from 'src/enum';import {anyUuid,@@ -29,13 +28,13 @@ import { globToSqlPattern } from 'src/utils/misc';export type AssetStats = Record<AssetType, number>;-export interface AssetStatsOptions {+interface AssetStatsOptions {isFavorite?: boolean;isTrashed?: boolean;visibility?: AssetVisibility;}-export interface LivePhotoSearchOptions {+interface LivePhotoSearchOptions {ownerId: string;libraryId?: string | null;livePhotoCID: string;@@ -43,16 +42,12 @@ export interface LivePhotoSearchOptions {type: AssetType;}-export enum WithProperty {- SIDECAR = 'sidecar',-}-export enum TimeBucketSize {DAY = 'DAY',MONTH = 'MONTH',}-export interface AssetBuilderOptions {+interface AssetBuilderOptions {isFavorite?: boolean;isTrashed?: boolean;isDuplicate?: boolean;@@ -81,43 +76,31 @@ export interface MonthDay {month: number;}-export interface AssetExploreFieldOptions {+interface AssetExploreFieldOptions {maxFields: number;minAssetsPerField: number;}-export interface AssetFullSyncOptions {+interface AssetFullSyncOptions {ownerId: string;lastId?: string;updatedUntil: Date;limit: number;}-export interface AssetDeltaSyncOptions {+interface AssetDeltaSyncOptions {userIds: string[];updatedAfter: Date;limit: number;}-export interface AssetUpdateDuplicateOptions {- targetDuplicateId: string | null;- assetIds: string[];- duplicateIds: string[];-}--export interface UpsertFileOptions {- assetId: string;- type: AssetFileType;- path: string;-}--export interface AssetGetByChecksumOptions {+interface AssetGetByChecksumOptions {ownerId: string;checksum: Buffer;libraryId?: string;}-export interface GetByIdsRelations {+interface GetByIdsRelations {exifInfo?: boolean;faces?: { person?: boolean; withDeleted?: boolean };files?: boolean;@@ -128,16 +111,6 @@ export interface GetByIdsRelations {tags?: boolean;}-export interface DuplicateGroup {- duplicateId: string;- assets: MapAsset[];-}--export interface DayOfYearAssets {- yearsAgo: number;- assets: MapAsset[];-}-@Injectable()export class AssetRepository {constructor(@InjectKysely() private db: Kysely<DB>) {}@@ -418,19 +391,6 @@ export class AssetRepository {await this.db.updateTable('assets').set(options).where('libraryId', '=', asUuid(libraryId)).execute();}- @GenerateSql({- params: [{ targetDuplicateId: DummyValue.UUID, duplicateIds: [DummyValue.UUID], assetIds: [DummyValue.UUID] }],- })- async updateDuplicates(options: AssetUpdateDuplicateOptions): Promise<void> {- await this.db- .updateTable('assets')- .set({ duplicateId: options.targetDuplicateId })- .where((eb) =>- eb.or([eb('duplicateId', '=', anyUuid(options.duplicateIds)), eb('id', '=', anyUuid(options.assetIds))]),- )- .execute();- }-async update(asset: Updateable<Assets> & { id: string }) {const value = omitBy(asset, isUndefined);delete value.id;@@ -696,58 +656,6 @@ export class AssetRepository {return query.executeTakeFirstOrThrow();}- @GenerateSql({ params: [DummyValue.UUID] })- getDuplicates(userId: string) {- return (- this.db- .with('duplicates', (qb) =>- qb- .selectFrom('assets')- .$call(withDefaultVisibility)- .leftJoinLateral(- (qb) =>- qb- .selectFrom('exif')- .selectAll('assets')- .select((eb) => eb.table('exif').as('exifInfo'))- .whereRef('exif.assetId', '=', 'assets.id')- .as('asset'),- (join) => join.onTrue(),- )- .select('assets.duplicateId')- .select((eb) =>- eb.fn.jsonAgg('asset').orderBy('assets.localDateTime', 'asc').$castTo<MapAsset[]>().as('assets'),- )- .where('assets.ownerId', '=', asUuid(userId))- .where('assets.duplicateId', 'is not', null)- .$narrowType<{ duplicateId: NotNull }>()- .where('assets.deletedAt', 'is', null)- .where('assets.stackId', 'is', null)- .groupBy('assets.duplicateId'),- )- .with('unique', (qb) =>- qb- .selectFrom('duplicates')- .select('duplicateId')- .where((eb) => eb(eb.fn('json_array_length', ['assets']), '=', 1)),- )- .with('removed_unique', (qb) =>- qb- .updateTable('assets')- .set({ duplicateId: null })- .from('unique')- .whereRef('assets.duplicateId', '=', 'unique.duplicateId'),- )- .selectFrom('duplicates')- .selectAll()- // TODO: compare with filtering by json_array_length > 1- .where(({ not, exists }) =>- not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))),- )- .execute()- );- }-@GenerateSql({ params: [DummyValue.UUID, { minAssetsPerField: 5, maxFields: 12 }] })async getAssetIdByCity(ownerId: string, { minAssetsPerField, maxFields }: AssetExploreFieldOptions) {const items = await this.dbdiff --git a/server/src/repositories/duplicate.repository.ts b/server/src/repositories/duplicate.repository.tsnew file mode 100644index 0000000000..b3329e4ca7--- /dev/null+++ b/server/src/repositories/duplicate.repository.ts@@ -0,0 +1,133 @@+import { Injectable } from '@nestjs/common';+import { Kysely, NotNull, sql } from 'kysely';+import { InjectKysely } from 'nestjs-kysely';+import { DB } from 'src/db';+import { DummyValue, GenerateSql } from 'src/decorators';+import { MapAsset } from 'src/dtos/asset-response.dto';+import { AssetType, VectorIndex } from 'src/enum';+import { probes } from 'src/repositories/database.repository';+import { anyUuid, asUuid, withDefaultVisibility } from 'src/utils/database';++interface DuplicateSearch {+ assetId: string;+ embedding: string;+ maxDistance: number;+ type: AssetType;+ userIds: string[];+}++interface DuplicateMergeOptions {+ targetId: string | null;+ assetIds: string[];+ sourceIds: string[];+}++@Injectable()+export class DuplicateRepository {+ constructor(@InjectKysely() private db: Kysely<DB>) {}++ @GenerateSql({ params: [DummyValue.UUID] })+ getAll(userId: string) {+ return (+ this.db+ .with('duplicates', (qb) =>+ qb+ .selectFrom('assets')+ .$call(withDefaultVisibility)+ .leftJoinLateral(+ (qb) =>+ qb+ .selectFrom('exif')+ .selectAll('assets')+ .select((eb) => eb.table('exif').as('exifInfo'))+ .whereRef('exif.assetId', '=', 'assets.id')+ .as('asset'),+ (join) => join.onTrue(),+ )+ .select('assets.duplicateId')+ .select((eb) =>+ eb.fn.jsonAgg('asset').orderBy('assets.localDateTime', 'asc').$castTo<MapAsset[]>().as('assets'),+ )+ .where('assets.ownerId', '=', asUuid(userId))+ .where('assets.duplicateId', 'is not', null)+ .$narrowType<{ duplicateId: NotNull }>()+ .where('assets.deletedAt', 'is', null)+ .where('assets.stackId', 'is', null)+ .groupBy('assets.duplicateId'),+ )+ .with('unique', (qb) =>+ qb+ .selectFrom('duplicates')+ .select('duplicateId')+ .where((eb) => eb(eb.fn('json_array_length', ['assets']), '=', 1)),+ )+ .with('removed_unique', (qb) =>+ qb+ .updateTable('assets')+ .set({ duplicateId: null })+ .from('unique')+ .whereRef('assets.duplicateId', '=', 'unique.duplicateId'),+ )+ .selectFrom('duplicates')+ .selectAll()+ // TODO: compare with filtering by json_array_length > 1+ .where(({ not, exists }) =>+ not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))),+ )+ .execute()+ );+ }++ @GenerateSql({+ params: [+ {+ assetId: DummyValue.UUID,+ embedding: DummyValue.VECTOR,+ maxDistance: 0.6,+ type: AssetType.IMAGE,+ userIds: [DummyValue.UUID],+ },+ ],+ })+ search({ assetId, embedding, maxDistance, type, userIds }: DuplicateSearch) {+ return this.db.transaction().execute(async (trx) => {+ await sql`set local vchordrq.probes = ${sql.lit(probes[VectorIndex.CLIP])}`.execute(trx);+ return await trx+ .with('cte', (qb) =>+ qb+ .selectFrom('assets')+ .$call(withDefaultVisibility)+ .select([+ 'assets.id as assetId',+ 'assets.duplicateId',+ sql<number>`smart_search.embedding <=> ${embedding}`.as('distance'),+ ])+ .innerJoin('smart_search', 'assets.id', 'smart_search.assetId')+ .where('assets.ownerId', '=', anyUuid(userIds))+ .where('assets.deletedAt', 'is', null)+ .where('assets.type', '=', type)+ .where('assets.id', '!=', asUuid(assetId))+ .where('assets.stackId', 'is', null)+ .orderBy('distance')+ .limit(64),+ )+ .selectFrom('cte')+ .selectAll()+ .where('cte.distance', '<=', maxDistance as number)+ .execute();+ });+ }++ @GenerateSql({+ params: [{ targetDuplicateId: DummyValue.UUID, duplicateIds: [DummyValue.UUID], assetIds: [DummyValue.UUID] }],+ })+ async merge(options: DuplicateMergeOptions): Promise<void> {+ await this.db+ .updateTable('assets')+ .set({ duplicateId: options.targetId })+ .where((eb) =>+ eb.or([eb('duplicateId', '=', anyUuid(options.sourceIds)), eb('id', '=', anyUuid(options.assetIds))]),+ )+ .execute();+ }+}diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.tsindex 453e515fe0..879385f9fd 100644--- a/server/src/repositories/index.ts+++ b/server/src/repositories/index.ts@@ -11,6 +11,7 @@ import { CronRepository } from 'src/repositories/cron.repository';import { CryptoRepository } from 'src/repositories/crypto.repository';import { DatabaseRepository } from 'src/repositories/database.repository';import { DownloadRepository } from 'src/repositories/download.repository';+import { DuplicateRepository } from 'src/repositories/duplicate.repository';import { EmailRepository } from 'src/repositories/email.repository';import { EventRepository } from 'src/repositories/event.repository';import { JobRepository } from 'src/repositories/job.repository';@@ -56,6 +57,7 @@ export const repositories = [CryptoRepository,DatabaseRepository,DownloadRepository,+ DuplicateRepository,EmailRepository,EventRepository,JobRepository,diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.tsindex 068de1eb4d..2455b65b20 100644--- a/server/src/repositories/search.repository.ts+++ b/server/src/repositories/search.repository.ts@@ -7,7 +7,7 @@ import { DummyValue, GenerateSql } from 'src/decorators';import { MapAsset } from 'src/dtos/asset-response.dto';import { AssetStatus, AssetType, AssetVisibility, VectorIndex } from 'src/enum';import { probes } from 'src/repositories/database.repository';-import { anyUuid, asUuid, searchAssetBuilder, withDefaultVisibility } from 'src/utils/database';+import { anyUuid, searchAssetBuilder } from 'src/utils/database';import { paginationHelper } from 'src/utils/pagination';import { isValidInteger } from 'src/validation';@@ -135,14 +135,6 @@ export interface FaceEmbeddingSearch extends SearchEmbeddingOptions {minBirthDate?: Date | null;}-export interface AssetDuplicateSearch {- assetId: string;- embedding: string;- maxDistance: number;- type: AssetType;- userIds: string[];-}-export interface FaceSearchResult {distance: number;id: string;@@ -275,46 +267,6 @@ export class SearchRepository {});}- @GenerateSql({- params: [- {- assetId: DummyValue.UUID,- embedding: DummyValue.VECTOR,- maxDistance: 0.6,- type: AssetType.IMAGE,- userIds: [DummyValue.UUID],- },- ],- })- searchDuplicates({ assetId, embedding, maxDistance, type, userIds }: AssetDuplicateSearch) {- return this.db.transaction().execute(async (trx) => {- await sql`set local vchordrq.probes = ${sql.lit(probes[VectorIndex.CLIP])}`.execute(trx);- return await trx- .with('cte', (qb) =>- qb- .selectFrom('assets')- .$call(withDefaultVisibility)- .select([- 'assets.id as assetId',- 'assets.duplicateId',- sql<number>`smart_search.embedding <=> ${embedding}`.as('distance'),- ])- .innerJoin('smart_search', 'assets.id', 'smart_search.assetId')- .where('assets.ownerId', '=', anyUuid(userIds))- .where('assets.deletedAt', 'is', null)- .where('assets.type', '=', type)- .where('assets.id', '!=', asUuid(assetId))- .where('assets.stackId', 'is', null)- .orderBy('distance')- .limit(64),- )- .selectFrom('cte')- .selectAll()- .where('cte.distance', '<=', maxDistance as number)- .execute();- });- }-@GenerateSql({params: [{diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.tsindex 3381ad7222..1c44abfd92 100644--- a/server/src/services/base.service.ts+++ b/server/src/services/base.service.ts@@ -18,6 +18,7 @@ import { CronRepository } from 'src/repositories/cron.repository';import { CryptoRepository } from 'src/repositories/crypto.repository';import { DatabaseRepository } from 'src/repositories/database.repository';import { DownloadRepository } from 'src/repositories/download.repository';+import { DuplicateRepository } from 'src/repositories/duplicate.repository';import { EmailRepository } from 'src/repositories/email.repository';import { EventRepository } from 'src/repositories/event.repository';import { JobRepository } from 'src/repositories/job.repository';@@ -71,6 +72,7 @@ export class BaseService {protected cryptoRepository: CryptoRepository,protected databaseRepository: DatabaseRepository,protected downloadRepository: DownloadRepository,+ protected duplicateRepository: DuplicateRepository,protected emailRepository: EmailRepository,protected eventRepository: EventRepository,protected jobRepository: JobRepository,diff --git a/server/src/services/duplicate.service.spec.ts b/server/src/services/duplicate.service.spec.tsindex d23144babe..4524a7cb68 100644--- a/server/src/services/duplicate.service.spec.ts+++ b/server/src/services/duplicate.service.spec.ts@@ -38,7 +38,7 @@ describe(SearchService.name, () => {describe('getDuplicates', () => {it('should get duplicates', async () => {- mocks.asset.getDuplicates.mockResolvedValue([+ mocks.duplicateRepository.getAll.mockResolvedValue([{duplicateId: 'duplicate-id',assets: [assetStub.image, assetStub.image],@@ -218,25 +218,26 @@ describe(SearchService.name, () => {it('should search for duplicates and update asset with duplicateId', async () => {mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);- mocks.search.searchDuplicates.mockResolvedValue([+ mocks.duplicateRepository.search.mockResolvedValue([{ assetId: assetStub.image.id, distance: 0.01, duplicateId: null },]);+ mocks.duplicateRepository.merge.mockResolvedValue();const expectedAssetIds = [assetStub.image.id, hasEmbedding.id];const result = await sut.handleSearchDuplicates({ id: hasEmbedding.id });expect(result).toBe(JobStatus.SUCCESS);- expect(mocks.search.searchDuplicates).toHaveBeenCalledWith({+ expect(mocks.duplicateRepository.search).toHaveBeenCalledWith({assetId: hasEmbedding.id,embedding: hasEmbedding.embedding,maxDistance: 0.01,type: hasEmbedding.type,userIds: [hasEmbedding.ownerId],});- expect(mocks.asset.updateDuplicates).toHaveBeenCalledWith({+ expect(mocks.duplicateRepository.merge).toHaveBeenCalledWith({assetIds: expectedAssetIds,- targetDuplicateId: expect.any(String),- duplicateIds: [],+ targetId: expect.any(String),+ sourceIds: [],});expect(mocks.asset.upsertJobStatus).toHaveBeenCalledWith(...expectedAssetIds.map((assetId) => ({ assetId, duplicatesDetectedAt: expect.any(Date) })),@@ -246,23 +247,24 @@ describe(SearchService.name, () => {it('should use existing duplicate ID among matched duplicates', async () => {const duplicateId = hasDupe.duplicateId;mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);- mocks.search.searchDuplicates.mockResolvedValue([{ assetId: hasDupe.id, distance: 0.01, duplicateId }]);+ mocks.duplicateRepository.search.mockResolvedValue([{ assetId: hasDupe.id, distance: 0.01, duplicateId }]);+ mocks.duplicateRepository.merge.mockResolvedValue();const expectedAssetIds = [hasEmbedding.id];const result = await sut.handleSearchDuplicates({ id: hasEmbedding.id });expect(result).toBe(JobStatus.SUCCESS);- expect(mocks.search.searchDuplicates).toHaveBeenCalledWith({+ expect(mocks.duplicateRepository.search).toHaveBeenCalledWith({assetId: hasEmbedding.id,embedding: hasEmbedding.embedding,maxDistance: 0.01,type: hasEmbedding.type,userIds: [hasEmbedding.ownerId],});- expect(mocks.asset.updateDuplicates).toHaveBeenCalledWith({+ expect(mocks.duplicateRepository.merge).toHaveBeenCalledWith({assetIds: expectedAssetIds,- targetDuplicateId: duplicateId,- duplicateIds: [],+ targetId: duplicateId,+ sourceIds: [],});expect(mocks.asset.upsertJobStatus).toHaveBeenCalledWith(...expectedAssetIds.map((assetId) => ({ assetId, duplicatesDetectedAt: expect.any(Date) })),@@ -271,7 +273,7 @@ describe(SearchService.name, () => {it('should remove duplicateId if no duplicates found and asset has duplicateId', async () => {mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasDupe);- mocks.search.searchDuplicates.mockResolvedValue([]);+ mocks.duplicateRepository.search.mockResolvedValue([]);const result = await sut.handleSearchDuplicates({ id: hasDupe.id });diff --git a/server/src/services/duplicate.service.ts b/server/src/services/duplicate.service.tsindex 617f5c5d0d..ed6e5f16e2 100644--- a/server/src/services/duplicate.service.ts+++ b/server/src/services/duplicate.service.ts@@ -13,7 +13,7 @@ import { isDuplicateDetectionEnabled } from 'src/utils/misc';@Injectable()export class DuplicateService extends BaseService {async getDuplicates(auth: AuthDto): Promise<DuplicateResponseDto[]> {- const duplicates = await this.assetRepository.getDuplicates(auth.user.id);+ const duplicates = await this.duplicateRepository.getAll(auth.user.id);return duplicates.map(({ duplicateId, assets }) => ({duplicateId,assets: assets.map((asset) => mapAsset(asset, { auth })),@@ -74,7 +74,7 @@ export class DuplicateService extends BaseService {return JobStatus.FAILED;}- const duplicateAssets = await this.searchRepository.searchDuplicates({+ const duplicateAssets = await this.duplicateRepository.search({assetId: asset.id,embedding: asset.embedding,maxDistance: machineLearning.duplicateDetection.maxDistance,@@ -117,7 +117,11 @@ export class DuplicateService extends BaseService {.map((duplicate) => duplicate.assetId);assetIdsToUpdate.push(asset.id);- await this.assetRepository.updateDuplicates({ targetDuplicateId, assetIds: assetIdsToUpdate, duplicateIds });+ await this.duplicateRepository.merge({+ targetId: targetDuplicateId,+ assetIds: assetIdsToUpdate,+ sourceIds: duplicateIds,+ });return assetIdsToUpdate;}}diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.tsindex bd419f0b34..d11ba6d60a 100644--- a/server/src/services/media.service.ts+++ b/server/src/services/media.service.ts@@ -24,7 +24,6 @@ import {VideoCodec,VideoContainer,} from 'src/enum';-import { UpsertFileOptions } from 'src/repositories/asset.repository';import { BoundingBox } from 'src/repositories/machine-learning.repository';import { BaseService } from 'src/services/base.service';import {@@ -42,6 +41,11 @@ import { getAssetFiles } from 'src/utils/asset.util';import { BaseConfig, ThumbnailConfig } from 'src/utils/media';import { mimeTypes } from 'src/utils/mime-types';import { clamp, isFaceImportEnabled, isFacialRecognitionEnabled } from 'src/utils/misc';+interface UpsertFileOptions {+ assetId: string;+ type: AssetFileType;+ path: string;+}@Injectable()export class MediaService extends BaseService {diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.tsindex cab74f70fb..9a4065aa9d 100644--- a/server/test/medium.factory.ts+++ b/server/test/medium.factory.ts@@ -390,6 +390,7 @@ export const asDeps = (repositories: ServiceOverrides) => {repositories.crypto || getRepositoryMock('crypto'),repositories.database || getRepositoryMock('database'),repositories.downloadRepository,+ repositories.duplicateRepository,repositories.email || getRepositoryMock('email'),repositories.event,repositories.job || getRepositoryMock('job'),diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.tsindex d8230a23f3..a29babbf54 100644--- a/server/test/repositories/asset.repository.mock.ts+++ b/server/test/repositories/asset.repository.mock.ts@@ -22,7 +22,6 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposigetLivePhotoCount: vitest.fn(),getLibraryAssetCount: vitest.fn(),updateAll: vitest.fn(),- updateDuplicates: vitest.fn(),getByLibraryIdAndOriginalPath: vitest.fn(),deleteAll: vitest.fn(),update: vitest.fn(),@@ -34,7 +33,6 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposigetAssetIdByCity: vitest.fn(),getAllForUserFullSync: vitest.fn(),getChangedDeltaSync: vitest.fn(),- getDuplicates: vitest.fn(),upsertFile: vitest.fn(),upsertFiles: vitest.fn(),deleteFiles: vitest.fn(),diff --git a/server/test/utils.ts b/server/test/utils.tsindex c1459bac84..5738ae88dc 100644--- a/server/test/utils.ts+++ b/server/test/utils.ts@@ -24,6 +24,7 @@ import { CronRepository } from 'src/repositories/cron.repository';import { CryptoRepository } from 'src/repositories/crypto.repository';import { DatabaseRepository } from 'src/repositories/database.repository';import { DownloadRepository } from 'src/repositories/download.repository';+import { DuplicateRepository } from 'src/repositories/duplicate.repository';import { EmailRepository } from 'src/repositories/email.repository';import { EventRepository } from 'src/repositories/event.repository';import { JobRepository } from 'src/repositories/job.repository';@@ -193,6 +194,7 @@ export type ServiceOverrides = {crypto: CryptoRepository;database: DatabaseRepository;downloadRepository: DownloadRepository;+ duplicateRepository: DuplicateRepository;email: EmailRepository;event: EventRepository;job: JobRepository;@@ -260,6 +262,7 @@ export const newTestService = <T extends BaseService>(config: newConfigRepositoryMock(),database: newDatabaseRepositoryMock(),downloadRepository: automock(DownloadRepository, { strict: false }),+ duplicateRepository: automock(DuplicateRepository),email: automock(EmailRepository, { args: [loggerMock] }),// eslint-disable-next-line no-sparse-arraysevent: automock(EventRepository, { args: [, , loggerMock], strict: false }),@@ -311,6 +314,7 @@ export const newTestService = <T extends BaseService>(overrides.crypto || (mocks.crypto as As<CryptoRepository>),overrides.database || (mocks.database as As<DatabaseRepository>),overrides.downloadRepository || (mocks.downloadRepository as As<DownloadRepository>),+ overrides.duplicateRepository || (mocks.duplicateRepository as As<DuplicateRepository>),overrides.email || (mocks.email as As<EmailRepository>),overrides.event || (mocks.event as As<EventRepository>),overrides.job || (mocks.job as As<JobRepository>),