High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
refactor: new asset-job repository (#17622)
Jason Rasmussen17e720440d99
diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sqlnew file mode 100644index 0000000000..0e98416117--- /dev/null+++ b/server/src/queries/asset.job.repository.sql@@ -0,0 +1,60 @@+-- NOTE: This file is auto generated by ./sql-generator++-- AssetJobRepository.getForSearchDuplicatesJob+select+ "id",+ "type",+ "ownerId",+ "duplicateId",+ "stackId",+ "isVisible",+ "smart_search"."embedding",+ (+ select+ coalesce(json_agg(agg), '[]')+ from+ (+ select+ "asset_files"."id",+ "asset_files"."path",+ "asset_files"."type"+ from+ "asset_files"+ where+ "asset_files"."assetId" = "assets"."id"+ and "asset_files"."type" = $1+ ) as agg+ ) as "files"+from+ "assets"+ left join "smart_search" on "assets"."id" = "smart_search"."assetId"+where+ "assets"."id" = $2::uuid+limit+ $3++-- AssetJobRepository.getForSidecarWriteJob+select+ "id",+ "sidecarPath",+ "originalPath",+ (+ select+ coalesce(json_agg(agg), '[]')+ from+ (+ select+ "tags"."value"+ from+ "tags"+ inner join "tag_asset" on "tags"."id" = "tag_asset"."tagsId"+ where+ "assets"."id" = "tag_asset"."assetsId"+ ) as agg+ ) as "tags"+from+ "assets"+where+ "assets"."id" = $1::uuid+limit+ $2diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sqlindex b59cee5864..fa89814b8b 100644--- a/server/src/queries/asset.repository.sql+++ b/server/src/queries/asset.repository.sql@@ -181,65 +181,6 @@ fromwhere"livePhotoVideoId" = $1::uuid--- AssetRepository.getAssetForSearchDuplicatesJob-select- "id",- "type",- "ownerId",- "duplicateId",- "stackId",- "isVisible",- "smart_search"."embedding",- (- select- coalesce(json_agg(agg), '[]')- from- (- select- "asset_files"."id",- "asset_files"."path",- "asset_files"."type"- from- "asset_files"- where- "asset_files"."assetId" = "assets"."id"- and "asset_files"."type" = $1- ) as agg- ) as "files"-from- "assets"- left join "smart_search" on "assets"."id" = "smart_search"."assetId"-where- "assets"."id" = $2::uuid-limit- $3---- AssetRepository.getAssetForSidecarWriteJob-select- "id",- "sidecarPath",- "originalPath",- (- select- coalesce(json_agg(agg), '[]')- from- (- select- "tags"."value"- from- "tags"- inner join "tag_asset" on "tags"."id" = "tag_asset"."tagsId"- where- "assets"."id" = "tag_asset"."assetsId"- ) as agg- ) as "tags"-from- "assets"-where- "assets"."id" = $1::uuid-limit- $2--- AssetRepository.getByIdselect"assets".*diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.tsnew file mode 100644index 0000000000..cc5e15d421--- /dev/null+++ b/server/src/repositories/asset-job.repository.ts@@ -0,0 +1,95 @@+import { Injectable } from '@nestjs/common';+import { Kysely } from 'kysely';+import { jsonArrayFrom } from 'kysely/helpers/postgres';+import { InjectKysely } from 'nestjs-kysely';+import { DB } from 'src/db';+import { DummyValue, GenerateSql } from 'src/decorators';+import { withFiles } from 'src/entities/asset.entity';+import { AssetFileType } from 'src/enum';+import { StorageAsset } from 'src/types';+import { asUuid } from 'src/utils/database';++@Injectable()+export class AssetJobRepository {+ constructor(@InjectKysely() private db: Kysely<DB>) {}++ @GenerateSql({ params: [DummyValue.UUID] })+ getForSearchDuplicatesJob(id: string) {+ return this.db+ .selectFrom('assets')+ .where('assets.id', '=', asUuid(id))+ .leftJoin('smart_search', 'assets.id', 'smart_search.assetId')+ .select((eb) => [+ 'id',+ 'type',+ 'ownerId',+ 'duplicateId',+ 'stackId',+ 'isVisible',+ 'smart_search.embedding',+ withFiles(eb, AssetFileType.PREVIEW),+ ])+ .limit(1)+ .executeTakeFirst();+ }++ @GenerateSql({ params: [DummyValue.UUID] })+ getForSidecarWriteJob(id: string) {+ return this.db+ .selectFrom('assets')+ .where('assets.id', '=', asUuid(id))+ .select((eb) => [+ 'id',+ 'sidecarPath',+ 'originalPath',+ jsonArrayFrom(+ eb+ .selectFrom('tags')+ .select(['tags.value'])+ .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId')+ .whereRef('assets.id', '=', 'tag_asset.assetsId'),+ ).as('tags'),+ ])+ .limit(1)+ .executeTakeFirst();+ }++ private storageTemplateAssetQuery() {+ return this.db+ .selectFrom('assets')+ .innerJoin('exif', 'assets.id', 'exif.assetId')+ .select([+ 'assets.id',+ 'assets.ownerId',+ 'assets.type',+ 'assets.checksum',+ 'assets.originalPath',+ 'assets.isExternal',+ 'assets.sidecarPath',+ 'assets.originalFileName',+ 'assets.livePhotoVideoId',+ 'assets.fileCreatedAt',+ 'exif.timeZone',+ 'exif.fileSizeInByte',+ ])+ .where('assets.deletedAt', 'is', null);+ }++ getForStorageTemplateJob(id: string): Promise<StorageAsset | undefined> {+ return this.storageTemplateAssetQuery().where('assets.id', '=', id).executeTakeFirst() as Promise<+ StorageAsset | undefined+ >;+ }++ streamForStorageTemplateJob() {+ return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator<StorageAsset>;+ }++ streamForDeletedJob(trashedBefore: Date) {+ return this.db+ .selectFrom('assets')+ .select(['id', 'isOffline'])+ .where('assets.deletedAt', '<=', trashedBefore)+ .stream();+ }+}diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.tsindex 8b31f47fd1..9871fa711c 100644--- a/server/src/repositories/asset.repository.ts+++ b/server/src/repositories/asset.repository.ts@@ -1,6 +1,5 @@import { Injectable } from '@nestjs/common';import { Insertable, Kysely, Selectable, UpdateResult, Updateable, sql } from 'kysely';-import { jsonArrayFrom } from 'kysely/helpers/postgres';import { isEmpty, isUndefined, omitBy } from 'lodash';import { InjectKysely } from 'nestjs-kysely';import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db';@@ -24,7 +23,6 @@ import {} from 'src/entities/asset.entity';import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum';import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository';-import { StorageAsset } from 'src/types';import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database';import { globToSqlPattern } from 'src/utils/misc';import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination';@@ -473,47 +471,6 @@ export class AssetRepository {return count;}- @GenerateSql({ params: [DummyValue.UUID] })- getAssetForSearchDuplicatesJob(id: string) {- return this.db- .selectFrom('assets')- .where('assets.id', '=', asUuid(id))- .leftJoin('smart_search', 'assets.id', 'smart_search.assetId')- .select((eb) => [- 'id',- 'type',- 'ownerId',- 'duplicateId',- 'stackId',- 'isVisible',- 'smart_search.embedding',- withFiles(eb, AssetFileType.PREVIEW),- ])- .limit(1)- .executeTakeFirst();- }-- @GenerateSql({ params: [DummyValue.UUID] })- getAssetForSidecarWriteJob(id: string) {- return this.db- .selectFrom('assets')- .where('assets.id', '=', asUuid(id))- .select((eb) => [- 'id',- 'sidecarPath',- 'originalPath',- jsonArrayFrom(- eb- .selectFrom('tags')- .select(['tags.value'])- .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId')- .whereRef('assets.id', '=', 'tag_asset.assetsId'),- ).as('tags'),- ])- .limit(1)- .executeTakeFirst();- }-@GenerateSql({ params: [DummyValue.UUID] })getById(id: string,@@ -653,45 +610,6 @@ export class AssetRepository {.executeTakeFirst() as Promise<AssetEntity | undefined>;}- private storageTemplateAssetQuery() {- return this.db- .selectFrom('assets')- .innerJoin('exif', 'assets.id', 'exif.assetId')- .select([- 'assets.id',- 'assets.ownerId',- 'assets.type',- 'assets.checksum',- 'assets.originalPath',- 'assets.isExternal',- 'assets.sidecarPath',- 'assets.originalFileName',- 'assets.livePhotoVideoId',- 'assets.fileCreatedAt',- 'exif.timeZone',- 'exif.fileSizeInByte',- ])- .where('assets.deletedAt', 'is', null);- }-- getStorageTemplateAsset(id: string): Promise<StorageAsset | undefined> {- return this.storageTemplateAssetQuery().where('assets.id', '=', id).executeTakeFirst() as Promise<- StorageAsset | undefined- >;- }-- streamStorageTemplateAssets() {- return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator<StorageAsset>;- }-- streamDeletedAssets(trashedBefore: Date) {- return this.db- .selectFrom('assets')- .select(['id', 'isOffline'])- .where('assets.deletedAt', '<=', trashedBefore)- .stream();- }-@GenerateSql(...Object.values(WithProperty).map((property) => ({name: property,diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.tsindex 8c262edcde..ef36a2b3f8 100644--- a/server/src/repositories/index.ts+++ b/server/src/repositories/index.ts@@ -3,6 +3,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';import { AlbumUserRepository } from 'src/repositories/album-user.repository';import { AlbumRepository } from 'src/repositories/album.repository';import { ApiKeyRepository } from 'src/repositories/api-key.repository';+import { AssetJobRepository } from 'src/repositories/asset-job.repository';import { AssetRepository } from 'src/repositories/asset.repository';import { AuditRepository } from 'src/repositories/audit.repository';import { ConfigRepository } from 'src/repositories/config.repository';@@ -48,6 +49,7 @@ export const repositories = [AuditRepository,ApiKeyRepository,AssetRepository,+ AssetJobRepository,ConfigRepository,CronRepository,CryptoRepository,diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.tsindex a02324a3ae..5fc4984b62 100755--- a/server/src/services/asset.service.spec.ts+++ b/server/src/services/asset.service.spec.ts@@ -536,12 +536,12 @@ describe(AssetService.name, () => {it('should immediately queue assets for deletion if trash is disabled', async () => {const asset = factory.asset({ isOffline: false });- mocks.asset.streamDeletedAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForDeletedJob.mockReturnValue(makeStream([asset]));mocks.systemMetadata.get.mockResolvedValue({ trash: { enabled: false } });await expect(sut.handleAssetDeletionCheck()).resolves.toBe(JobStatus.SUCCESS);- expect(mocks.asset.streamDeletedAssets).toHaveBeenCalledWith(new Date());+ expect(mocks.assetJob.streamForDeletedJob).toHaveBeenCalledWith(new Date());expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.ASSET_DELETION, data: { id: asset.id, deleteOnDisk: true } },]);@@ -550,12 +550,12 @@ describe(AssetService.name, () => {it('should queue assets for deletion after trash duration', async () => {const asset = factory.asset({ isOffline: false });- mocks.asset.streamDeletedAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForDeletedJob.mockReturnValue(makeStream([asset]));mocks.systemMetadata.get.mockResolvedValue({ trash: { enabled: true, days: 7 } });await expect(sut.handleAssetDeletionCheck()).resolves.toBe(JobStatus.SUCCESS);- expect(mocks.asset.streamDeletedAssets).toHaveBeenCalledWith(DateTime.now().minus({ days: 7 }).toJSDate());+ expect(mocks.assetJob.streamForDeletedJob).toHaveBeenCalledWith(DateTime.now().minus({ days: 7 }).toJSDate());expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.ASSET_DELETION, data: { id: asset.id, deleteOnDisk: true } },]);diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.tsindex 6ab3a76f28..1ded79680b 100644--- a/server/src/services/asset.service.ts+++ b/server/src/services/asset.service.ts@@ -172,7 +172,7 @@ export class AssetService extends BaseService {}};- const assets = this.assetRepository.streamDeletedAssets(trashedBefore);+ const assets = this.assetJobRepository.streamForDeletedJob(trashedBefore);for await (const asset of assets) {chunk.push(asset);if (chunk.length >= JOBS_ASSET_PAGINATION_SIZE) {diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.tsindex b985ef8352..2fbdd6e4c0 100644--- a/server/src/services/base.service.ts+++ b/server/src/services/base.service.ts@@ -10,6 +10,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';import { AlbumUserRepository } from 'src/repositories/album-user.repository';import { AlbumRepository } from 'src/repositories/album.repository';import { ApiKeyRepository } from 'src/repositories/api-key.repository';+import { AssetJobRepository } from 'src/repositories/asset-job.repository';import { AssetRepository } from 'src/repositories/asset.repository';import { AuditRepository } from 'src/repositories/audit.repository';import { ConfigRepository } from 'src/repositories/config.repository';@@ -62,6 +63,7 @@ export class BaseService {protected albumUserRepository: AlbumUserRepository,protected apiKeyRepository: ApiKeyRepository,protected assetRepository: AssetRepository,+ protected assetJobRepository: AssetJobRepository,protected auditRepository: AuditRepository,protected configRepository: ConfigRepository,protected cronRepository: CronRepository,diff --git a/server/src/services/duplicate.service.spec.ts b/server/src/services/duplicate.service.spec.tsindex adb884c24f..57ab955514 100644--- a/server/src/services/duplicate.service.spec.ts+++ b/server/src/services/duplicate.service.spec.ts@@ -194,6 +194,8 @@ describe(SearchService.name, () => {});it('should fail if asset is not found', async () => {+ mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(void 0);+const result = await sut.handleSearchDuplicates({ id: assetStub.image.id });expect(result).toBe(JobStatus.FAILED);@@ -202,7 +204,7 @@ describe(SearchService.name, () => {it('should skip if asset is part of stack', async () => {const id = assetStub.primaryImage.id;- mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, stackId: 'stack-id' });+ mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, stackId: 'stack-id' });const result = await sut.handleSearchDuplicates({ id });@@ -212,7 +214,7 @@ describe(SearchService.name, () => {it('should skip if asset is not visible', async () => {const id = assetStub.livePhotoMotionAsset.id;- mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, isVisible: false });+ mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, isVisible: false });const result = await sut.handleSearchDuplicates({ id });@@ -221,7 +223,7 @@ describe(SearchService.name, () => {});it('should fail if asset is missing preview image', async () => {- mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, files: [] });+ mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, files: [] });const result = await sut.handleSearchDuplicates({ id: assetStub.noResizePath.id });@@ -230,7 +232,7 @@ describe(SearchService.name, () => {});it('should fail if asset is missing embedding', async () => {- mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, embedding: null });+ mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, embedding: null });const result = await sut.handleSearchDuplicates({ id: assetStub.image.id });@@ -239,7 +241,7 @@ describe(SearchService.name, () => {});it('should search for duplicates and update asset with duplicateId', async () => {- mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);+ mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);mocks.search.searchDuplicates.mockResolvedValue([{ assetId: assetStub.image.id, distance: 0.01, duplicateId: null },]);@@ -267,7 +269,7 @@ describe(SearchService.name, () => {it('should use existing duplicate ID among matched duplicates', async () => {const duplicateId = hasDupe.duplicateId;- mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);+ mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);mocks.search.searchDuplicates.mockResolvedValue([{ assetId: hasDupe.id, distance: 0.01, duplicateId }]);const expectedAssetIds = [hasEmbedding.id];@@ -292,7 +294,7 @@ describe(SearchService.name, () => {});it('should remove duplicateId if no duplicates found and asset has duplicateId', async () => {- mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue(hasDupe);+ mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasDupe);mocks.search.searchDuplicates.mockResolvedValue([]);const result = await sut.handleSearchDuplicates({ id: hasDupe.id });diff --git a/server/src/services/duplicate.service.ts b/server/src/services/duplicate.service.tsindex 10adb645d3..9f1ac3d4ce 100644--- a/server/src/services/duplicate.service.ts+++ b/server/src/services/duplicate.service.ts@@ -52,7 +52,7 @@ export class DuplicateService extends BaseService {return JobStatus.SKIPPED;}- const asset = await this.assetRepository.getAssetForSearchDuplicatesJob(id);+ const asset = await this.assetJobRepository.getForSearchDuplicatesJob(id);if (!asset) {this.logger.error(`Asset ${id} not found`);return JobStatus.FAILED;diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.tsindex f275477d93..358c0eea5e 100644--- a/server/src/services/metadata.service.spec.ts+++ b/server/src/services/metadata.service.spec.ts@@ -1486,14 +1486,14 @@ describe(MetadataService.name, () => {describe('handleSidecarWrite', () => {it('should skip assets that do not exist anymore', async () => {- mocks.asset.getAssetForSidecarWriteJob.mockResolvedValue(void 0);+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(void 0);await expect(sut.handleSidecarWrite({ id: 'asset-123' })).resolves.toBe(JobStatus.FAILED);expect(mocks.metadata.writeTags).not.toHaveBeenCalled();});it('should skip jobs with no metadata', async () => {const asset = factory.jobAssets.sidecarWrite();- mocks.asset.getAssetForSidecarWriteJob.mockResolvedValue(asset);+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset);await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.SKIPPED);expect(mocks.metadata.writeTags).not.toHaveBeenCalled();});@@ -1504,7 +1504,7 @@ describe(MetadataService.name, () => {const gps = 12;const date = '2023-11-22T04:56:12.196Z';- mocks.asset.getAssetForSidecarWriteJob.mockResolvedValue(asset);+ mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset);await expect(sut.handleSidecarWrite({id: asset.id,diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.tsindex 5277ea2d68..e460170968 100644--- a/server/src/services/metadata.service.ts+++ b/server/src/services/metadata.service.ts@@ -329,7 +329,7 @@ export class MetadataService extends BaseService {@OnJob({ name: JobName.SIDECAR_WRITE, queue: QueueName.SIDECAR })async handleSidecarWrite(job: JobOf<JobName.SIDECAR_WRITE>): Promise<JobStatus> {const { id, description, dateTimeOriginal, latitude, longitude, rating, tags } = job;- const asset = await this.assetRepository.getAssetForSidecarWriteJob(id);+ const asset = await this.assetJobRepository.getForSidecarWriteJob(id);if (!asset) {return JobStatus.FAILED;}diff --git a/server/src/services/storage-template.service.spec.ts b/server/src/services/storage-template.service.spec.tsindex 46603cdbce..971a9e8302 100644--- a/server/src/services/storage-template.service.spec.ts+++ b/server/src/services/storage-template.service.spec.ts@@ -112,8 +112,8 @@ describe(StorageTemplateService.name, () => {const newMotionPicturePath = `upload/library/${motionAsset.ownerId}/2022/2022-06-19/${motionAsset.originalFileName}`;const newStillPicturePath = `upload/library/${stillAsset.ownerId}/2022/2022-06-19/${stillAsset.originalFileName}`;- mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(stillAsset);- mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(motionAsset);+ mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(stillAsset);+ mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(motionAsset);mocks.move.create.mockResolvedValueOnce({id: '123',@@ -148,7 +148,7 @@ describe(StorageTemplateService.name, () => {sut.onConfigInit({ newConfig: config });mocks.user.get.mockResolvedValue(user);- mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset);+ mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(asset);mocks.album.getByAssetId.mockResolvedValueOnce([album]);expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS);@@ -169,7 +169,7 @@ describe(StorageTemplateService.name, () => {sut.onConfigInit({ newConfig: config });mocks.user.get.mockResolvedValue(user);- mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset);+ mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(asset);expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS);@@ -197,7 +197,7 @@ describe(StorageTemplateService.name, () => {oldPath: asset.originalPath,newPath: previousFailedNewPath,});- mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset);+ mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(asset);mocks.move.update.mockResolvedValue({id: '123',entityId: asset.id,@@ -208,7 +208,7 @@ describe(StorageTemplateService.name, () => {await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS);- expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id);+ expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(asset.id);expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);expect(mocks.storage.rename).toHaveBeenCalledWith(asset.originalPath, newPath);expect(mocks.move.update).toHaveBeenCalledWith('123', {@@ -239,7 +239,7 @@ describe(StorageTemplateService.name, () => {oldPath: asset.originalPath,newPath: previousFailedNewPath,});- mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset);+ mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(asset);mocks.move.update.mockResolvedValue({id: '123',entityId: asset.id,@@ -250,7 +250,7 @@ describe(StorageTemplateService.name, () => {await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS);- expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id);+ expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(asset.id);expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath);expect(mocks.storage.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath);@@ -266,7 +266,7 @@ describe(StorageTemplateService.name, () => {mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats);mocks.crypto.hashFile.mockResolvedValue(Buffer.from('different-hash', 'utf8'));- mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset);+ mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(testAsset);mocks.move.create.mockResolvedValue({id: '123',entityId: testAsset.id,@@ -277,7 +277,7 @@ describe(StorageTemplateService.name, () => {await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS);- expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id);+ expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(testAsset.id);expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(1);expect(mocks.storage.stat).toHaveBeenCalledWith(newPath);expect(mocks.move.create).toHaveBeenCalledWith({@@ -316,7 +316,7 @@ describe(StorageTemplateService.name, () => {oldPath: testAsset.originalPath,newPath: previousFailedNewPath,});- mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset);+ mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(testAsset);mocks.move.update.mockResolvedValue({id: '123',entityId: testAsset.id,@@ -327,7 +327,7 @@ describe(StorageTemplateService.name, () => {await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS);- expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id);+ expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(testAsset.id);expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath);expect(mocks.storage.rename).not.toHaveBeenCalled();@@ -340,12 +340,12 @@ describe(StorageTemplateService.name, () => {describe('handle template migration', () => {it('should handle no assets', async () => {- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([]));mocks.user.getList.mockResolvedValue([]);await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();});it('should handle an asset with a duplicate destination', async () => {@@ -354,7 +354,7 @@ describe(StorageTemplateService.name, () => {const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;const newPath2 = newPath.replace('.jpg', '+1.jpg');- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([userStub.user1]);mocks.move.create.mockResolvedValue({id: '123',@@ -369,7 +369,7 @@ describe(StorageTemplateService.name, () => {await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2);expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath2 });expect(mocks.user.getList).toHaveBeenCalled();@@ -378,12 +378,12 @@ describe(StorageTemplateService.name, () => {it('should skip when an asset already matches the template', async () => {const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg' });- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([userStub.user1]);await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();expect(mocks.storage.rename).not.toHaveBeenCalled();expect(mocks.storage.copyFile).not.toHaveBeenCalled();expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2);@@ -393,12 +393,12 @@ describe(StorageTemplateService.name, () => {it('should skip when an asset is probably a duplicate', async () => {const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg' });- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([userStub.user1]);await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();expect(mocks.storage.rename).not.toHaveBeenCalled();expect(mocks.storage.copyFile).not.toHaveBeenCalled();expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2);@@ -409,7 +409,7 @@ describe(StorageTemplateService.name, () => {const asset = assetStub.storageAsset();const oldPath = asset.originalPath;const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([userStub.user1]);mocks.move.create.mockResolvedValue({id: '123',@@ -421,7 +421,7 @@ describe(StorageTemplateService.name, () => {await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath);expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath });});@@ -429,7 +429,7 @@ describe(StorageTemplateService.name, () => {it('should use the user storage label', async () => {const user = factory.userAdmin({ storageLabel: 'label-1' });const asset = assetStub.storageAsset({ ownerId: user.id });- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([user]);mocks.move.create.mockResolvedValue({id: '123',@@ -441,7 +441,7 @@ describe(StorageTemplateService.name, () => {await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();expect(mocks.storage.rename).toHaveBeenCalledWith('/original/path.jpg',`upload/library/${user.storageLabel}/2022/2022-06-19/${asset.originalFileName}`,@@ -456,7 +456,7 @@ describe(StorageTemplateService.name, () => {const asset = assetStub.storageAsset({ originalPath: '/path/to/original.jpg', fileSizeInByte: 5000 });const oldPath = asset.originalPath;const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });mocks.user.getList.mockResolvedValue([userStub.user1]);mocks.move.create.mockResolvedValue({@@ -482,7 +482,7 @@ describe(StorageTemplateService.name, () => {await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath);expect(mocks.storage.copyFile).toHaveBeenCalledWith(oldPath, newPath);expect(mocks.storage.stat).toHaveBeenCalledWith(oldPath);@@ -495,7 +495,7 @@ describe(StorageTemplateService.name, () => {it('should not update the database if the move fails due to incorrect newPath filesize', async () => {const asset = assetStub.storageAsset();- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });mocks.user.getList.mockResolvedValue([userStub.user1]);mocks.move.create.mockResolvedValue({@@ -511,7 +511,7 @@ describe(StorageTemplateService.name, () => {await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();expect(mocks.storage.rename).toHaveBeenCalledWith('/original/path.jpg',`upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,@@ -528,7 +528,7 @@ describe(StorageTemplateService.name, () => {it('should not update the database if the move fails', async () => {const asset = assetStub.storageAsset();- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));mocks.storage.rename.mockRejectedValue(new Error('Read only system'));mocks.storage.copyFile.mockRejectedValue(new Error('Read only system'));mocks.move.create.mockResolvedValue({@@ -542,7 +542,7 @@ describe(StorageTemplateService.name, () => {await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();expect(mocks.storage.rename).toHaveBeenCalledWith('/original/path.jpg',`upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,@@ -559,7 +559,7 @@ describe(StorageTemplateService.name, () => {originalPath: `upload/library/${user.id}/2022/2022-06-19/IMG_7065.heic`,originalFileName: 'IMG_7065.HEIC',});- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([user]);mocks.move.create.mockResolvedValue({id: '123',@@ -571,7 +571,7 @@ describe(StorageTemplateService.name, () => {await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();expect(mocks.storage.rename).toHaveBeenCalledWith(`upload/library/${user.id}/2022/2022-06-19/IMG_7065.heic`,`upload/library/${user.storageLabel}/2022/2022-06-19/IMG_7065.heic`,@@ -585,7 +585,7 @@ describe(StorageTemplateService.name, () => {originalPath: `upload/library/${user.id}/2022/2022-06-19/IMG_7065.HEIC`,originalFileName: 'IMG_7065.HEIC',});- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([user]);mocks.move.create.mockResolvedValue({id: '123',@@ -597,7 +597,7 @@ describe(StorageTemplateService.name, () => {await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();expect(mocks.storage.rename).toHaveBeenCalledWith(`upload/library/${user.id}/2022/2022-06-19/IMG_7065.HEIC`,`upload/library/${user.id}/2022/2022-06-19/IMG_7065.heic`,@@ -611,7 +611,7 @@ describe(StorageTemplateService.name, () => {originalPath: `upload/library/${user.id}/2022/2022-06-19/IMG_7065.JPEG`,originalFileName: 'IMG_7065.JPEG',});- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([user]);mocks.move.create.mockResolvedValue({id: '123',@@ -623,7 +623,7 @@ describe(StorageTemplateService.name, () => {await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();expect(mocks.storage.rename).toHaveBeenCalledWith(`upload/library/${user.id}/2022/2022-06-19/IMG_7065.JPEG`,`upload/library/${user.id}/2022/2022-06-19/IMG_7065.jpg`,@@ -637,7 +637,7 @@ describe(StorageTemplateService.name, () => {originalPath: 'upload/library/user-id/2022/2022-06-19/IMG_7065.JPG',originalFileName: 'IMG_7065.JPG',});- mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));+ mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([user]);mocks.move.create.mockResolvedValue({id: '123',@@ -649,7 +649,7 @@ describe(StorageTemplateService.name, () => {await sut.handleMigration();- expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();expect(mocks.storage.rename).toHaveBeenCalledWith(`upload/library/${user.id}/2022/2022-06-19/IMG_7065.JPG`,`upload/library/${user.id}/2022/2022-06-19/IMG_7065.jpg`,diff --git a/server/src/services/storage-template.service.ts b/server/src/services/storage-template.service.tsindex 07bac9839a..71a0160ee2 100644--- a/server/src/services/storage-template.service.ts+++ b/server/src/services/storage-template.service.ts@@ -118,7 +118,7 @@ export class StorageTemplateService extends BaseService {return JobStatus.SKIPPED;}- const asset = await this.assetRepository.getStorageTemplateAsset(id);+ const asset = await this.assetJobRepository.getForStorageTemplateJob(id);if (!asset) {return JobStatus.FAILED;}@@ -130,7 +130,7 @@ export class StorageTemplateService extends BaseService {// move motion part of live photoif (asset.livePhotoVideoId) {- const livePhotoVideo = await this.assetRepository.getStorageTemplateAsset(asset.livePhotoVideoId);+ const livePhotoVideo = await this.assetJobRepository.getForStorageTemplateJob(asset.livePhotoVideoId);if (!livePhotoVideo) {return JobStatus.FAILED;}@@ -152,7 +152,7 @@ export class StorageTemplateService extends BaseService {await this.moveRepository.cleanMoveHistory();- const assets = this.assetRepository.streamStorageTemplateAssets();+ const assets = this.assetJobRepository.streamForStorageTemplateJob();const users = await this.userRepository.getList();for await (const asset of assets) {diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.tsindex a4674dafc9..a1d05ccd1c 100644--- a/server/test/medium.factory.ts+++ b/server/test/medium.factory.ts@@ -6,6 +6,7 @@ import { AssetJobStatus, Assets, DB } from 'src/db';import { AssetType } from 'src/enum';import { ActivityRepository } from 'src/repositories/activity.repository';import { AlbumRepository } from 'src/repositories/album.repository';+import { AssetJobRepository } from 'src/repositories/asset-job.repository';import { AssetRepository } from 'src/repositories/asset.repository';import { ConfigRepository } from 'src/repositories/config.repository';import { CryptoRepository } from 'src/repositories/crypto.repository';@@ -21,7 +22,7 @@ import { VersionHistoryRepository } from 'src/repositories/version-history.reposimport { UserTable } from 'src/schema/tables/user.table';import { BaseService } from 'src/services/base.service';import { RepositoryInterface } from 'src/types';-import { newUuid } from 'test/small.factory';+import { newDate, newUuid } from 'test/small.factory';import { automock, ServiceOverrides } from 'test/utils';import { Mocked } from 'vitest';@@ -30,6 +31,7 @@ type Repositories = {activity: ActivityRepository;album: AlbumRepository;asset: AssetRepository;+ assetJob: AssetJobRepository;config: ConfigRepository;crypto: CryptoRepository;database: DatabaseRepository;@@ -113,6 +115,10 @@ export const getRepository = <K extends keyof Repositories>(key: K, db: Kysely<Dreturn new AssetRepository(db);}+ case 'assetJob': {+ return new AssetJobRepository(db);+ }+case 'config': {return new ConfigRepository();}@@ -175,6 +181,10 @@ const getRepositoryMock = <K extends keyof Repositories>(key: K) => {return automock(AssetRepository);}+ case 'assetJob': {+ return automock(AssetJobRepository);+ }+case 'config': {return automock(ConfigRepository);}@@ -237,6 +247,7 @@ export const asDeps = (repositories: ServiceOverrides) => {repositories.albumUser,repositories.apiKey,repositories.asset || getRepositoryMock('asset'),+ repositories.assetJob || getRepositoryMock('assetJob'),repositories.audit,repositories.config || getRepositoryMock('config'),repositories.cron,@@ -276,6 +287,7 @@ export const asDeps = (repositories: ServiceOverrides) => {const assetInsert = (asset: Partial<Insertable<Assets>> = {}) => {const id = asset.id || newUuid();+ const now = newDate();const defaults: Insertable<Assets> = {deviceAssetId: '',deviceId: '',@@ -285,6 +297,9 @@ const assetInsert = (asset: Partial<Insertable<Assets>> = {}) => {originalPath: '/path/to/something.jpg',ownerId: '@immich.cloud',isVisible: true,+ fileCreatedAt: now,+ fileModifiedAt: now,+ localDateTime: now,};return {diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.tsindex 36fb298f7f..f33b487888 100644--- a/server/test/repositories/asset.repository.mock.ts+++ b/server/test/repositories/asset.repository.mock.ts@@ -11,8 +11,6 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposiupsertJobStatus: vitest.fn(),getByDayOfYear: vitest.fn(),getByIds: vitest.fn().mockResolvedValue([]),- getAssetForSearchDuplicatesJob: vitest.fn(),- getAssetForSidecarWriteJob: vitest.fn(),getByIdsWithAllRelations: vitest.fn().mockResolvedValue([]),getByAlbumId: vitest.fn(),getByDeviceIds: vitest.fn(),@@ -47,8 +45,5 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposidetectOfflineExternalAssets: vitest.fn(),filterNewExternalAssetPaths: vitest.fn(),updateByLibraryId: vitest.fn(),- streamStorageTemplateAssets: vitest.fn(),- getStorageTemplateAsset: vitest.fn(),- streamDeletedAssets: vitest.fn(),};};diff --git a/server/test/utils.ts b/server/test/utils.tsindex 7ad4c093d3..86d7830f7a 100644--- a/server/test/utils.ts+++ b/server/test/utils.ts@@ -12,6 +12,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';import { AlbumUserRepository } from 'src/repositories/album-user.repository';import { AlbumRepository } from 'src/repositories/album.repository';import { ApiKeyRepository } from 'src/repositories/api-key.repository';+import { AssetJobRepository } from 'src/repositories/asset-job.repository';import { AssetRepository } from 'src/repositories/asset.repository';import { AuditRepository } from 'src/repositories/audit.repository';import { ConfigRepository } from 'src/repositories/config.repository';@@ -114,6 +115,7 @@ export type ServiceOverrides = {apiKey: ApiKeyRepository;audit: AuditRepository;asset: AssetRepository;+ assetJob: AssetJobRepository;config: ConfigRepository;cron: CronRepository;crypto: CryptoRepository;@@ -181,6 +183,7 @@ export const newTestService = <T extends BaseService>(album: automock(AlbumRepository, { strict: false }),albumUser: automock(AlbumUserRepository),asset: newAssetRepositoryMock(),+ assetJob: automock(AssetJobRepository),config: newConfigRepositoryMock(),database: newDatabaseRepositoryMock(),downloadRepository: automock(DownloadRepository, { strict: false }),@@ -227,6 +230,7 @@ export const newTestService = <T extends BaseService>(overrides.albumUser || (mocks.albumUser as As<AlbumUserRepository>),overrides.apiKey || (mocks.apiKey as As<ApiKeyRepository>),overrides.asset || (mocks.asset as As<AssetRepository>),+ overrides.assetJob || (mocks.assetJob as As<AssetJobRepository>),overrides.audit || (mocks.audit as As<AuditRepository>),overrides.config || (mocks.config as As<ConfigRepository> as ConfigRepository),overrides.cron || (mocks.cron as As<CronRepository>),