High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat: use stream for template migrations (#16700)
Jason Rasmussenf82786a297f3
diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.tsindex 6274c88dfe..1efcb3875b 100644--- a/server/src/repositories/asset.repository.ts+++ b/server/src/repositories/asset.repository.ts@@ -23,6 +23,7 @@ import {} from 'src/entities/asset.entity';import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum';import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository';+import { StorageAsset } from 'src/types';import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database';import { globToSqlPattern } from 'src/utils/misc';import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination';@@ -604,6 +605,38 @@ export class AssetRepository {.executeTakeFirst() as Promise<AssetEntity | undefined>;}+ private storageTemplateAssetQuery() {+ return this.db+ .selectFrom('assets')+ .innerJoin('exif', 'assets.id', 'exif.assetId')+ .select([+ 'assets.id',+ 'assets.ownerId',+ 'assets.type',+ 'assets.checksum',+ 'assets.originalPath',+ 'assets.isExternal',+ 'assets.sidecarPath',+ 'assets.originalFileName',+ 'assets.livePhotoVideoId',+ 'assets.fileCreatedAt',+ 'exif.timeZone',+ 'exif.fileSizeInByte',+ ])+ .where('assets.deletedAt', 'is', null)+ .where('assets.fileCreatedAt', 'is not', null);+ }++ getStorageTemplateAsset(id: string): Promise<StorageAsset | undefined> {+ return this.storageTemplateAssetQuery().where('assets.id', '=', id).executeTakeFirst() as Promise<+ StorageAsset | undefined+ >;+ }++ streamStorageTemplateAssets() {+ return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator<StorageAsset>;+ }+@GenerateSql(...Object.values(WithProperty).map((property) => ({name: property,diff --git a/server/src/services/storage-template.service.spec.ts b/server/src/services/storage-template.service.spec.tsindex 9b5f5a483d..1133d25db4 100644--- a/server/src/services/storage-template.service.spec.ts+++ b/server/src/services/storage-template.service.spec.ts@@ -1,12 +1,14 @@import { Stats } from 'node:fs';import { defaults, SystemConfig } from 'src/config';-import { AssetEntity } from 'src/entities/asset.entity';import { AssetPathType, JobStatus } from 'src/enum';import { StorageTemplateService } from 'src/services/storage-template.service';import { albumStub } from 'test/fixtures/album.stub';import { assetStub } from 'test/fixtures/asset.stub';import { userStub } from 'test/fixtures/user.stub';-import { newTestService, ServiceMocks } from 'test/utils';+import { makeStream, newTestService, ServiceMocks } from 'test/utils';++const motionAsset = assetStub.storageAsset({});+const stillAsset = assetStub.storageAsset({ livePhotoVideoId: motionAsset.id });describe(StorageTemplateService.name, () => {let sut: StorageTemplateService;@@ -91,7 +93,7 @@ describe(StorageTemplateService.name, () => {describe('handleMigrationSingle', () => {it('should skip when storage template is disabled', async () => {mocks.systemMetadata.get.mockResolvedValue({ storageTemplate: { enabled: false } });- await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SKIPPED);+ await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SKIPPED);expect(mocks.asset.getByIds).not.toHaveBeenCalled();expect(mocks.storage.checkFileExists).not.toHaveBeenCalled();expect(mocks.storage.rename).not.toHaveBeenCalled();@@ -104,51 +106,37 @@ describe(StorageTemplateService.name, () => {it('should migrate single moving picture', async () => {mocks.user.get.mockResolvedValue(userStub.user1);- const newMotionPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.mp4`;- const newStillPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.jpg`;-- mocks.asset.getByIds.mockImplementation((ids) => {- const assets = [assetStub.livePhotoStillAsset, assetStub.livePhotoMotionAsset];- return Promise.resolve(- ids.map((id) => assets.find((asset) => asset.id === id)).filter((asset) => !!asset),- ) as Promise<AssetEntity[]>;- });+ const newMotionPicturePath = `upload/library/${motionAsset.ownerId}/2022/2022-06-19/${motionAsset.originalFileName}`;+ const newStillPicturePath = `upload/library/${stillAsset.ownerId}/2022/2022-06-19/${stillAsset.originalFileName}`;++ mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(stillAsset);+ mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(motionAsset);mocks.move.create.mockResolvedValueOnce({id: '123',- entityId: assetStub.livePhotoStillAsset.id,+ entityId: stillAsset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.livePhotoStillAsset.originalPath,+ oldPath: stillAsset.originalPath,newPath: newStillPicturePath,});mocks.move.create.mockResolvedValueOnce({id: '124',- entityId: assetStub.livePhotoMotionAsset.id,+ entityId: motionAsset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.livePhotoMotionAsset.originalPath,+ oldPath: motionAsset.originalPath,newPath: newMotionPicturePath,});- await expect(sut.handleMigrationSingle({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe(- JobStatus.SUCCESS,- );+ await expect(sut.handleMigrationSingle({ id: stillAsset.id })).resolves.toBe(JobStatus.SUCCESS);- expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id], { exifInfo: true });- expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id], { exifInfo: true });expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2);- expect(mocks.asset.update).toHaveBeenCalledWith({- id: assetStub.livePhotoStillAsset.id,- originalPath: newStillPicturePath,- });- expect(mocks.asset.update).toHaveBeenCalledWith({- id: assetStub.livePhotoMotionAsset.id,- originalPath: newMotionPicturePath,- });+ expect(mocks.asset.update).toHaveBeenCalledWith({ id: stillAsset.id, originalPath: newStillPicturePath });+ expect(mocks.asset.update).toHaveBeenCalledWith({ id: motionAsset.id, originalPath: newMotionPicturePath });});it('should use handlebar if condition for album', async () => {- const asset = assetStub.image;+ const asset = assetStub.storageAsset();const user = userStub.user1;const album = albumStub.oneAsset;const config = structuredClone(defaults);@@ -157,7 +145,7 @@ describe(StorageTemplateService.name, () => {sut.onConfigInit({ newConfig: config });mocks.user.get.mockResolvedValue(user);- mocks.asset.getByIds.mockResolvedValueOnce([asset]);+ mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset);mocks.album.getByAssetId.mockResolvedValueOnce([album]);expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS);@@ -171,14 +159,14 @@ describe(StorageTemplateService.name, () => {});it('should use handlebar else condition for album', async () => {- const asset = assetStub.image;+ const asset = assetStub.storageAsset();const user = userStub.user1;const config = structuredClone(defaults);config.storageTemplate.template = '{{y}}/{{#if album}}{{album}}{{else}}other//{{MM}}{{/if}}/{{filename}}';sut.onConfigInit({ newConfig: config });mocks.user.get.mockResolvedValue(user);- mocks.asset.getByIds.mockResolvedValueOnce([asset]);+ mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset);expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS);@@ -193,153 +181,150 @@ describe(StorageTemplateService.name, () => {it('should migrate previously failed move from original path when it still exists', async () => {mocks.user.get.mockResolvedValue(userStub.user1);- const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;- const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;- mocks.storage.checkFileExists.mockImplementation((path) =>- Promise.resolve(path === assetStub.image.originalPath),- );+ const asset = assetStub.storageAsset();+ const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${asset.originalFileName}`;+ const newPath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${asset.originalFileName}`;++ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === asset.originalPath));mocks.move.getByEntity.mockResolvedValue({id: '123',- entityId: assetStub.image.id,+ entityId: asset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.image.originalPath,+ oldPath: asset.originalPath,newPath: previousFailedNewPath,});- mocks.asset.getByIds.mockResolvedValue([assetStub.image]);+ mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset);mocks.move.update.mockResolvedValue({id: '123',- entityId: assetStub.image.id,+ entityId: asset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.image.originalPath,+ oldPath: asset.originalPath,newPath,});- await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS);+ await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS);- expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });+ expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id);expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);- expect(mocks.storage.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);+ expect(mocks.storage.rename).toHaveBeenCalledWith(asset.originalPath, newPath);expect(mocks.move.update).toHaveBeenCalledWith('123', {id: '123',- oldPath: assetStub.image.originalPath,+ oldPath: asset.originalPath,newPath,});expect(mocks.asset.update).toHaveBeenCalledWith({- id: assetStub.image.id,+ id: asset.id,originalPath: newPath,});});it('should migrate previously failed move from previous new path when old path no longer exists, should validate file size still matches before moving', async () => {mocks.user.get.mockResolvedValue(userStub.user1);- const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;- const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;++ const asset = assetStub.storageAsset({ fileSizeInByte: 5000 });+ const previousFailedNewPath = `upload/library/${asset.ownerId}/2022/June/${asset.originalFileName}`;+ const newPath = `upload/library/${asset.ownerId}/2022/2022-06-19/${asset.originalFileName}`;mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === previousFailedNewPath));mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats);- mocks.crypto.hashFile.mockResolvedValue(assetStub.image.checksum);+ mocks.crypto.hashFile.mockResolvedValue(asset.checksum);mocks.move.getByEntity.mockResolvedValue({id: '123',- entityId: assetStub.image.id,+ entityId: asset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.image.originalPath,+ oldPath: asset.originalPath,newPath: previousFailedNewPath,});- mocks.asset.getByIds.mockResolvedValue([assetStub.image]);+ mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset);mocks.move.update.mockResolvedValue({id: '123',- entityId: assetStub.image.id,+ entityId: asset.id,pathType: AssetPathType.ORIGINAL,oldPath: previousFailedNewPath,newPath,});- await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS);+ await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS);- expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });+ expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id);expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath);expect(mocks.storage.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath);expect(mocks.storage.copyFile).not.toHaveBeenCalled();- expect(mocks.move.update).toHaveBeenCalledWith('123', {- id: '123',- oldPath: previousFailedNewPath,- newPath,- });- expect(mocks.asset.update).toHaveBeenCalledWith({- id: assetStub.image.id,- originalPath: newPath,- });+ expect(mocks.move.update).toHaveBeenCalledWith('123', { id: '123', oldPath: previousFailedNewPath, newPath });+ expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath });});it('should fail move if copying and hash of asset and the new file do not match', async () => {mocks.user.get.mockResolvedValue(userStub.user1);- const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;+ const newPath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${testAsset.originalFileName}`;mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats);mocks.crypto.hashFile.mockResolvedValue(Buffer.from('different-hash', 'utf8'));- mocks.asset.getByIds.mockResolvedValue([assetStub.image]);+ mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset);mocks.move.create.mockResolvedValue({id: '123',- entityId: assetStub.image.id,+ entityId: testAsset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.image.originalPath,+ oldPath: testAsset.originalPath,newPath,});- await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS);+ await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS);- expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });+ expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id);expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(1);expect(mocks.storage.stat).toHaveBeenCalledWith(newPath);expect(mocks.move.create).toHaveBeenCalledWith({- entityId: assetStub.image.id,+ entityId: testAsset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.image.originalPath,+ oldPath: testAsset.originalPath,newPath,});- expect(mocks.storage.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);- expect(mocks.storage.copyFile).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);+ expect(mocks.storage.rename).toHaveBeenCalledWith(testAsset.originalPath, newPath);+ expect(mocks.storage.copyFile).toHaveBeenCalledWith(testAsset.originalPath, newPath);expect(mocks.storage.unlink).toHaveBeenCalledWith(newPath);expect(mocks.storage.unlink).toHaveBeenCalledTimes(1);expect(mocks.asset.update).not.toHaveBeenCalled();});+ const testAsset = assetStub.storageAsset();+it.each`- failedPathChecksum | failedPathSize | reason- ${assetStub.image.checksum} | ${500} | ${'file size'}- ${Buffer.from('bad checksum', 'utf8')} | ${assetStub.image.exifInfo?.fileSizeInByte} | ${'checksum'}+ failedPathChecksum | failedPathSize | reason+ ${testAsset.checksum} | ${500} | ${'file size'}+ ${Buffer.from('bad checksum', 'utf8')} | ${testAsset.fileSizeInByte} | ${'checksum'}`('should fail to migrate previously failed move from previous new path when old path no longer exists if $reason validation fails',async ({ failedPathChecksum, failedPathSize }) => {mocks.user.get.mockResolvedValue(userStub.user1);- const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;- const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;+ const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${testAsset.originalFileName}`;+ const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${testAsset.originalFileName}`;mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(previousFailedNewPath === path));mocks.storage.stat.mockResolvedValue({ size: failedPathSize } as Stats);mocks.crypto.hashFile.mockResolvedValue(failedPathChecksum);mocks.move.getByEntity.mockResolvedValue({id: '123',- entityId: assetStub.image.id,+ entityId: testAsset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.image.originalPath,+ oldPath: testAsset.originalPath,newPath: previousFailedNewPath,});- mocks.asset.getByIds.mockResolvedValue([assetStub.image]);+ mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset);mocks.move.update.mockResolvedValue({id: '123',- entityId: assetStub.image.id,+ entityId: testAsset.id,pathType: AssetPathType.ORIGINAL,oldPath: previousFailedNewPath,newPath,});- await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS);+ await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS);- expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });+ expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id);expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath);expect(mocks.storage.rename).not.toHaveBeenCalled();@@ -352,29 +337,28 @@ describe(StorageTemplateService.name, () => {describe('handle template migration', () => {it('should handle no assets', async () => {- mocks.asset.getAll.mockResolvedValue({- items: [],- hasNextPage: false,- });+ mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([]));mocks.user.getList.mockResolvedValue([]);await sut.handleMigration();- expect(mocks.asset.getAll).toHaveBeenCalled();+ expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();});it('should handle an asset with a duplicate destination', async () => {- mocks.asset.getAll.mockResolvedValue({- items: [assetStub.image],- hasNextPage: false,- });+ const asset = assetStub.storageAsset();+ const oldPath = asset.originalPath;+ const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;+ const newPath2 = newPath.replace('.jpg', '+1.jpg');++ mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([userStub.user1]);mocks.move.create.mockResolvedValue({id: '123',- entityId: assetStub.image.id,+ entityId: asset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.image.originalPath,- newPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg',+ oldPath,+ newPath,});mocks.storage.checkFileExists.mockResolvedValueOnce(true);@@ -382,30 +366,21 @@ describe(StorageTemplateService.name, () => {await sut.handleMigration();- expect(mocks.asset.getAll).toHaveBeenCalled();+ expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2);- expect(mocks.asset.update).toHaveBeenCalledWith({- id: assetStub.image.id,- originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg',- });+ expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath2 });expect(mocks.user.getList).toHaveBeenCalled();});it('should skip when an asset already matches the template', async () => {- mocks.asset.getAll.mockResolvedValue({- items: [- {- ...assetStub.image,- originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',- },- ],- hasNextPage: false,- });+ const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg' });++ mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([userStub.user1]);await sut.handleMigration();- expect(mocks.asset.getAll).toHaveBeenCalled();+ expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();expect(mocks.storage.rename).not.toHaveBeenCalled();expect(mocks.storage.copyFile).not.toHaveBeenCalled();expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2);@@ -413,20 +388,14 @@ describe(StorageTemplateService.name, () => {});it('should skip when an asset is probably a duplicate', async () => {- mocks.asset.getAll.mockResolvedValue({- items: [- {- ...assetStub.image,- originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg',- },- ],- hasNextPage: false,- });+ const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg' });++ mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([userStub.user1]);await sut.handleMigration();- expect(mocks.asset.getAll).toHaveBeenCalled();+ expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();expect(mocks.storage.rename).not.toHaveBeenCalled();expect(mocks.storage.copyFile).not.toHaveBeenCalled();expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2);@@ -434,72 +403,63 @@ describe(StorageTemplateService.name, () => {});it('should move an asset', async () => {- mocks.asset.getAll.mockResolvedValue({- items: [assetStub.image],- hasNextPage: false,- });+ const asset = assetStub.storageAsset();+ const oldPath = asset.originalPath;+ const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;+ mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([userStub.user1]);mocks.move.create.mockResolvedValue({id: '123',entityId: assetStub.image.id,pathType: AssetPathType.ORIGINAL,oldPath: assetStub.image.originalPath,- newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',+ newPath,});await sut.handleMigration();- expect(mocks.asset.getAll).toHaveBeenCalled();- expect(mocks.storage.rename).toHaveBeenCalledWith(- '/original/path.jpg',- 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',- );- expect(mocks.asset.update).toHaveBeenCalledWith({- id: assetStub.image.id,- originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',- });+ expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath);+ expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath });});it('should use the user storage label', async () => {- mocks.asset.getAll.mockResolvedValue({- items: [assetStub.image],- hasNextPage: false,- });+ const asset = assetStub.storageAsset();+ mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));mocks.user.getList.mockResolvedValue([userStub.storageLabel]);mocks.move.create.mockResolvedValue({id: '123',- entityId: assetStub.image.id,+ entityId: asset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.image.originalPath,- newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',+ oldPath: asset.originalPath,+ newPath: `upload/library/user-id/2023/2023-02-23/${asset.originalFileName}`,});await sut.handleMigration();- expect(mocks.asset.getAll).toHaveBeenCalled();+ expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();expect(mocks.storage.rename).toHaveBeenCalledWith('/original/path.jpg',- 'upload/library/label-1/2023/2023-02-23/asset-id.jpg',+ `upload/library/label-1/2022/2022-06-19/${asset.originalFileName}`,);expect(mocks.asset.update).toHaveBeenCalledWith({- id: assetStub.image.id,- originalPath: 'upload/library/label-1/2023/2023-02-23/asset-id.jpg',+ id: asset.id,+ originalPath: `upload/library/label-1/2022/2022-06-19/${asset.originalFileName}`,});});it('should copy the file if rename fails due to EXDEV (rename across filesystems)', async () => {- const newPath = 'upload/library/user-id/2023/2023-02-23/asset-id.jpg';- mocks.asset.getAll.mockResolvedValue({- items: [assetStub.image],- hasNextPage: false,- });+ const asset = assetStub.storageAsset({ originalPath: '/path/to/original.jpg', fileSizeInByte: 5000 });+ const oldPath = asset.originalPath;+ const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;+ mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });mocks.user.getList.mockResolvedValue([userStub.user1]);mocks.move.create.mockResolvedValue({id: '123',- entityId: assetStub.image.id,+ entityId: asset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.image.originalPath,+ oldPath,newPath,});mocks.storage.stat.mockResolvedValueOnce({@@ -510,40 +470,36 @@ describe(StorageTemplateService.name, () => {size: 5000,} as Stats);mocks.storage.stat.mockResolvedValueOnce({+ size: 5000,atime: new Date(),mtime: new Date(),} as Stats);- mocks.crypto.hashFile.mockResolvedValue(assetStub.image.checksum);+ mocks.crypto.hashFile.mockResolvedValue(asset.checksum);await sut.handleMigration();- expect(mocks.asset.getAll).toHaveBeenCalled();- expect(mocks.storage.rename).toHaveBeenCalledWith('/original/path.jpg', newPath);- expect(mocks.storage.copyFile).toHaveBeenCalledWith('/original/path.jpg', newPath);+ expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();+ expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath);+ expect(mocks.storage.copyFile).toHaveBeenCalledWith(oldPath, newPath);+ expect(mocks.storage.stat).toHaveBeenCalledWith(oldPath);expect(mocks.storage.stat).toHaveBeenCalledWith(newPath);- expect(mocks.storage.stat).toHaveBeenCalledWith(assetStub.image.originalPath);expect(mocks.storage.utimes).toHaveBeenCalledWith(newPath, expect.any(Date), expect.any(Date));- expect(mocks.storage.unlink).toHaveBeenCalledWith(assetStub.image.originalPath);+ expect(mocks.storage.unlink).toHaveBeenCalledWith(oldPath);expect(mocks.storage.unlink).toHaveBeenCalledTimes(1);- expect(mocks.asset.update).toHaveBeenCalledWith({- id: assetStub.image.id,- originalPath: newPath,- });+ expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath });});it('should not update the database if the move fails due to incorrect newPath filesize', async () => {- mocks.asset.getAll.mockResolvedValue({- items: [assetStub.image],- hasNextPage: false,- });+ const asset = assetStub.storageAsset();+ mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });mocks.user.getList.mockResolvedValue([userStub.user1]);mocks.move.create.mockResolvedValue({id: '123',- entityId: assetStub.image.id,+ entityId: asset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.image.originalPath,- newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',+ oldPath: asset.originalPath,+ newPath: `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,});mocks.storage.stat.mockResolvedValue({size: 100,@@ -551,41 +507,41 @@ describe(StorageTemplateService.name, () => {await sut.handleMigration();- expect(mocks.asset.getAll).toHaveBeenCalled();+ expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();expect(mocks.storage.rename).toHaveBeenCalledWith('/original/path.jpg',- 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',+ `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,);expect(mocks.storage.copyFile).toHaveBeenCalledWith('/original/path.jpg',- 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',+ `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,+ );+ expect(mocks.storage.stat).toHaveBeenCalledWith(+ `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,);- expect(mocks.storage.stat).toHaveBeenCalledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg');expect(mocks.asset.update).not.toHaveBeenCalled();});it('should not update the database if the move fails', async () => {- mocks.asset.getAll.mockResolvedValue({- items: [assetStub.image],- hasNextPage: false,- });+ const asset = assetStub.storageAsset();+ mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));mocks.storage.rename.mockRejectedValue(new Error('Read only system'));mocks.storage.copyFile.mockRejectedValue(new Error('Read only system'));mocks.move.create.mockResolvedValue({id: 'move-123',- entityId: '123',+ entityId: asset.id,pathType: AssetPathType.ORIGINAL,- oldPath: assetStub.image.originalPath,+ oldPath: asset.originalPath,newPath: '',});mocks.user.getList.mockResolvedValue([userStub.user1]);await sut.handleMigration();- expect(mocks.asset.getAll).toHaveBeenCalled();+ expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();expect(mocks.storage.rename).toHaveBeenCalledWith('/original/path.jpg',- 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',+ `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,);expect(mocks.asset.update).not.toHaveBeenCalled();});diff --git a/server/src/services/storage-template.service.ts b/server/src/services/storage-template.service.tsindex 53d93b709b..1a0d4f4644 100644--- a/server/src/services/storage-template.service.ts+++ b/server/src/services/storage-template.service.ts@@ -3,17 +3,14 @@ import handlebar from 'handlebars';import { DateTime } from 'luxon';import path from 'node:path';import sanitize from 'sanitize-filename';-import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';import { StorageCore } from 'src/cores/storage.core';import { OnEvent, OnJob } from 'src/decorators';import { SystemConfigTemplateStorageOptionDto } from 'src/dtos/system-config.dto';-import { AssetEntity } from 'src/entities/asset.entity';import { AssetPathType, AssetType, DatabaseLock, JobName, JobStatus, QueueName, StorageFolder } from 'src/enum';import { ArgOf } from 'src/repositories/event.repository';import { BaseService } from 'src/services/base.service';-import { JobOf } from 'src/types';+import { JobOf, StorageAsset } from 'src/types';import { getLivePhotoMotionFilename } from 'src/utils/file';-import { usePagination } from 'src/utils/pagination';const storageTokens = {secondOptions: ['s', 'ss', 'SSS'],@@ -53,7 +50,7 @@ export interface MoveAssetMetadata {}interface RenderMetadata {- asset: AssetEntity;+ asset: StorageAsset;filename: string;extension: string;albumName: string | null;@@ -98,7 +95,7 @@ export class StorageTemplateService extends BaseService {originalPath: '/upload/test/IMG_123.jpg',type: AssetType.IMAGE,id: 'd587e44b-f8c0-4832-9ba3-43268bbf5d4e',- } as AssetEntity,+ } as StorageAsset,filename: 'IMG_123',extension: 'jpg',albumName: 'album',@@ -121,7 +118,7 @@ export class StorageTemplateService extends BaseService {return JobStatus.SKIPPED;}- const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true });+ const asset = await this.assetRepository.getStorageTemplateAsset(id);if (!asset) {return JobStatus.FAILED;}@@ -133,7 +130,7 @@ export class StorageTemplateService extends BaseService {// move motion part of live photoif (asset.livePhotoVideoId) {- const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId], { exifInfo: true });+ const livePhotoVideo = await this.assetRepository.getStorageTemplateAsset(asset.livePhotoVideoId);if (!livePhotoVideo) {return JobStatus.FAILED;}@@ -152,19 +149,17 @@ export class StorageTemplateService extends BaseService {this.logger.log('Storage template migration disabled, skipping');return JobStatus.SKIPPED;}+await this.moveRepository.cleanMoveHistory();- const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>- this.assetRepository.getAll(pagination, { withExif: true, withArchived: true }),- );++ const assets = this.assetRepository.streamStorageTemplateAssets();const users = await this.userRepository.getList();- for await (const assets of assetPagination) {- for (const asset of assets) {- const user = users.find((user) => user.id === asset.ownerId);- const storageLabel = user?.storageLabel || null;- const filename = asset.originalFileName || asset.id;- await this.moveAsset(asset, { storageLabel, filename });- }+ for await (const asset of assets) {+ const user = users.find((user) => user.id === asset.ownerId);+ const storageLabel = user?.storageLabel || null;+ const filename = asset.originalFileName || asset.id;+ await this.moveAsset(asset, { storageLabel, filename });}this.logger.debug('Cleaning up empty directories...');@@ -182,7 +177,7 @@ export class StorageTemplateService extends BaseService {await this.moveRepository.cleanMoveHistorySingle(assetId);}- async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) {+ async moveAsset(asset: StorageAsset, metadata: MoveAssetMetadata) {if (asset.isExternal || StorageCore.isAndroidMotionPath(asset.originalPath)) {// External assets are not affected by storage template// TODO: shouldn't this only apply to external assets?@@ -190,11 +185,11 @@ export class StorageTemplateService extends BaseService {}return this.databaseRepository.withLock(DatabaseLock.StorageTemplateMigration, async () => {- const { id, sidecarPath, originalPath, exifInfo, checksum } = asset;+ const { id, sidecarPath, originalPath, checksum, fileSizeInByte } = asset;const oldPath = originalPath;const newPath = await this.getTemplatePath(asset, metadata);- if (!exifInfo || !exifInfo.fileSizeInByte) {+ if (!fileSizeInByte) {this.logger.error(`Asset ${id} missing exif info, skipping storage template migration`);return;}@@ -205,7 +200,7 @@ export class StorageTemplateService extends BaseService {pathType: AssetPathType.ORIGINAL,oldPath,newPath,- assetInfo: { sizeInBytes: exifInfo.fileSizeInByte, checksum },+ assetInfo: { sizeInBytes: fileSizeInByte, checksum },});if (sidecarPath) {await this.storageCore.moveFile({@@ -221,7 +216,7 @@ export class StorageTemplateService extends BaseService {});}- private async getTemplatePath(asset: AssetEntity, metadata: MoveAssetMetadata): Promise<string> {+ private async getTemplatePath(asset: StorageAsset, metadata: MoveAssetMetadata): Promise<string> {const { storageLabel, filename } = metadata;try {@@ -344,7 +339,7 @@ export class StorageTemplateService extends BaseService {};const systemTimeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;- const zone = asset.exifInfo?.timeZone || systemTimeZone;+ const zone = asset.timeZone || systemTimeZone;const dt = DateTime.fromJSDate(asset.fileCreatedAt, { zone });for (const token of Object.values(storageTokens).flat()) {diff --git a/server/src/types.ts b/server/src/types.tsindex ece5704a7a..1c0a61b259 100644--- a/server/src/types.ts+++ b/server/src/types.ts@@ -1,4 +1,5 @@import {+ AssetType,DatabaseExtension,ExifOrientation,ImageFormat,@@ -438,3 +439,18 @@ export type SyncAck = {type: SyncEntityType;updateId: string;};++export type StorageAsset = {+ id: string;+ ownerId: string;+ livePhotoVideoId: string | null;+ type: AssetType;+ isExternal: boolean;+ checksum: Buffer;+ timeZone: string | null;+ fileCreatedAt: Date;+ originalPath: string;+ originalFileName: string;+ sidecarPath: string | null;+ fileSizeInByte: number | null;+};diff --git a/server/test/fixtures/asset.stub.ts b/server/test/fixtures/asset.stub.tsindex 4230c9350e..b7a8686b1f 100644--- a/server/test/fixtures/asset.stub.ts+++ b/server/test/fixtures/asset.stub.ts@@ -3,6 +3,7 @@ import { AssetEntity } from 'src/entities/asset.entity';import { ExifEntity } from 'src/entities/exif.entity';import { StackEntity } from 'src/entities/stack.entity';import { AssetFileType, AssetStatus, AssetType } from 'src/enum';+import { StorageAsset } from 'src/types';import { authStub } from 'test/fixtures/auth.stub';import { fileStub } from 'test/fixtures/file.stub';import { libraryStub } from 'test/fixtures/library.stub';@@ -40,6 +41,21 @@ export const stackStub = (stackId: string, assets: AssetEntity[]): StackEntity =};export const assetStub = {+ storageAsset: (asset: Partial<StorageAsset> = {}) => ({+ id: 'asset-id',+ ownerId: 'user-id',+ livePhotoVideoId: null,+ type: AssetType.IMAGE,+ isExternal: false,+ checksum: Buffer.from('file hash'),+ timeZone: null,+ fileCreatedAt: new Date('2022-06-19T23:41:36.910Z'),+ originalPath: '/original/path.jpg',+ originalFileName: 'IMG_123.jpg',+ sidecarPath: null,+ fileSizeInByte: 12_345,+ ...asset,+ }),noResizePath: Object.freeze<AssetEntity>({id: 'asset-id',status: AssetStatus.ACTIVE,diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.tsindex bc6a172e05..66b56b0ecc 100644--- a/server/test/repositories/asset.repository.mock.ts+++ b/server/test/repositories/asset.repository.mock.ts@@ -44,5 +44,7 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposidetectOfflineExternalAssets: vitest.fn(),filterNewExternalAssetPaths: vitest.fn(),updateByLibraryId: vitest.fn(),+ streamStorageTemplateAssets: vitest.fn(),+ getStorageTemplateAsset: vitest.fn(),};};