High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat(server): e2e for missing jobs (#15910)
Jonathan Jogenfors22d348beca0d
diff --git a/e2e/src/api/specs/asset.e2e-spec.ts b/e2e/src/api/specs/asset.e2e-spec.tsindex 1b644454aa..8700356256 100644--- a/e2e/src/api/specs/asset.e2e-spec.ts+++ b/e2e/src/api/specs/asset.e2e-spec.ts@@ -4,7 +4,6 @@ import {AssetResponseDto,AssetTypeEnum,getAssetInfo,- getConfig,getMyUser,LoginResponseDto,SharedLinkType,@@ -45,8 +44,6 @@ const locationAssetFilepath = `${testAssetDir}/metadata/gps-position/thompson-spconst ratingAssetFilepath = `${testAssetDir}/metadata/rating/mongolels.jpg`;const facesAssetFilepath = `${testAssetDir}/metadata/faces/portrait.jpg`;-const getSystemConfig = (accessToken: string) => getConfig({ headers: asBearerAuth(accessToken) });-const readTags = async (bytes: Buffer, filename: string) => {const filepath = join(tempDir, filename);await writeFile(filepath, bytes);@@ -228,7 +225,7 @@ describe('/asset', () => {});it('should get the asset faces', async () => {- const config = await getSystemConfig(admin.accessToken);+ const config = await utils.getSystemConfig(admin.accessToken);config.metadata.faces.import = true;await updateConfig({ systemConfigDto: config }, { headers: asBearerAuth(admin.accessToken) });diff --git a/e2e/src/api/specs/jobs.e2e-spec.ts b/e2e/src/api/specs/jobs.e2e-spec.tsindex 4b8126c941..a9afd8475f 100644--- a/e2e/src/api/specs/jobs.e2e-spec.ts+++ b/e2e/src/api/specs/jobs.e2e-spec.ts@@ -1,8 +1,9 @@-import { JobCommand, JobName, LoginResponseDto } from '@immich/sdk';+import { JobCommand, JobName, LoginResponseDto, updateConfig } from '@immich/sdk';+import { cpSync, rmSync } from 'node:fs';import { readFile } from 'node:fs/promises';import { basename } from 'node:path';import { errorDto } from 'src/responses';-import { app, testAssetDir, utils } from 'src/utils';+import { app, asBearerAuth, testAssetDir, utils } from 'src/utils';import request from 'supertest';import { afterEach, beforeAll, describe, expect, it } from 'vitest';@@ -20,6 +21,33 @@ describe('/jobs', () => {command: JobCommand.Resume,force: false,});++ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {+ command: JobCommand.Resume,+ force: false,+ });++ await utils.jobCommand(admin.accessToken, JobName.FaceDetection, {+ command: JobCommand.Resume,+ force: false,+ });++ await utils.jobCommand(admin.accessToken, JobName.SmartSearch, {+ command: JobCommand.Resume,+ force: false,+ });++ await utils.jobCommand(admin.accessToken, JobName.DuplicateDetection, {+ command: JobCommand.Resume,+ force: false,+ });++ const config = await utils.getSystemConfig(admin.accessToken);+ config.machineLearning.duplicateDetection.enabled = false;+ config.machineLearning.enabled = false;+ config.metadata.faces.import = false;+ config.machineLearning.clip.enabled = false;+ await updateConfig({ systemConfigDto: config }, { headers: asBearerAuth(admin.accessToken) });});it('should require authentication', async () => {@@ -29,14 +57,7 @@ describe('/jobs', () => {});it('should queue metadata extraction for missing assets', async () => {- const path1 = `${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`;- const path2 = `${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`;-- await utils.createAsset(admin.accessToken, {- assetData: { bytes: await readFile(path1), filename: basename(path1) },- });-- await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');+ const path = `${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`;await utils.jobCommand(admin.accessToken, JobName.MetadataExtraction, {command: JobCommand.Pause,@@ -44,7 +65,7 @@ describe('/jobs', () => {});const { id } = await utils.createAsset(admin.accessToken, {- assetData: { bytes: await readFile(path2), filename: basename(path2) },+ assetData: { bytes: await readFile(path), filename: basename(path) },});await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');@@ -82,5 +103,123 @@ describe('/jobs', () => {expect(asset.exifInfo?.make).toBe('NIKON CORPORATION');}});++ it('should not re-extract metadata for existing assets', async () => {+ const path = `${testAssetDir}/temp/metadata/asset.jpg`;++ cpSync(`${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`, path);++ const { id } = await utils.createAsset(admin.accessToken, {+ assetData: { bytes: await readFile(path), filename: basename(path) },+ });++ await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');++ {+ const asset = await utils.getAssetInfo(admin.accessToken, id);++ expect(asset.exifInfo).toBeDefined();+ expect(asset.exifInfo?.model).toBe('NIKON D700');+ }++ cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, path);++ await utils.jobCommand(admin.accessToken, JobName.MetadataExtraction, {+ command: JobCommand.Start,+ force: false,+ });++ await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');++ {+ const asset = await utils.getAssetInfo(admin.accessToken, id);++ expect(asset.exifInfo).toBeDefined();+ expect(asset.exifInfo?.model).toBe('NIKON D700');+ }++ rmSync(path);+ });++ it('should queue thumbnail extraction for assets missing thumbs', async () => {+ const path = `${testAssetDir}/albums/nature/tanners_ridge.jpg`;++ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {+ command: JobCommand.Pause,+ force: false,+ });++ const { id } = await utils.createAsset(admin.accessToken, {+ assetData: { bytes: await readFile(path), filename: basename(path) },+ });++ await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);+ await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);++ const assetBefore = await utils.getAssetInfo(admin.accessToken, id);+ expect(assetBefore.thumbhash).toBeNull();++ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {+ command: JobCommand.Empty,+ force: false,+ });++ await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);+ await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);++ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {+ command: JobCommand.Resume,+ force: false,+ });++ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {+ command: JobCommand.Start,+ force: false,+ });++ await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);+ await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);++ const assetAfter = await utils.getAssetInfo(admin.accessToken, id);+ expect(assetAfter.thumbhash).not.toBeNull();+ });++ it('should not reload existing thumbnail when running thumb job for missing assets', async () => {+ const path = `${testAssetDir}/temp/thumbs/asset1.jpg`;++ cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, path);++ const { id } = await utils.createAsset(admin.accessToken, {+ assetData: { bytes: await readFile(path), filename: basename(path) },+ });++ await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);+ await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);++ const assetBefore = await utils.getAssetInfo(admin.accessToken, id);++ cpSync(`${testAssetDir}/albums/nature/notocactus_minimus.jpg`, path);++ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {+ command: JobCommand.Resume,+ force: false,+ });++ // This runs the missing thumbnail job+ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {+ command: JobCommand.Start,+ force: false,+ });++ await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);+ await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);++ const assetAfter = await utils.getAssetInfo(admin.accessToken, id);++ // Asset 1 thumbnail should be untouched since its thumb should not have been reloaded, even though the file was changed+ expect(assetAfter.thumbhash).toEqual(assetBefore.thumbhash);++ rmSync(path);+ });});});diff --git a/e2e/src/utils.ts b/e2e/src/utils.tsindex ecd002a8b9..7446bb708f 100644--- a/e2e/src/utils.ts+++ b/e2e/src/utils.ts@@ -28,6 +28,7 @@ import {deleteAssets,getAllJobsStatus,getAssetInfo,+ getConfig,getConfigDefaults,login,scanLibrary,@@ -121,6 +122,7 @@ const execPromise = promisify(exec);const onEvent = ({ event, id }: { event: EventType; id: string }) => {// console.log(`Received event: ${event} [id=${id}]`);const set = events[event];+set.add(id);const idCallback = idCallbacks[id];@@ -415,6 +417,8 @@ export const utils = {rmSync(path, { recursive: true });},+ getSystemConfig: (accessToken: string) => getConfig({ headers: asBearerAuth(accessToken) }),+getAssetInfo: (accessToken: string, id: string) => getAssetInfo({ id }, { headers: asBearerAuth(accessToken) }),checkExistingAssets: (accessToken: string, checkExistingAssetsDto: CheckExistingAssetsDto) =>diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.tsindex 595be21829..342aec7a7a 100644--- a/server/src/services/job.service.ts+++ b/server/src/services/job.service.ts@@ -70,7 +70,7 @@ export class JobService extends BaseService {}async handleCommand(queueName: QueueName, dto: JobCommandDto): Promise<JobStatusDto> {- this.logger.debug(`Handling command: queue=${queueName},force=${dto.force}`);+ this.logger.debug(`Handling command: queue=${queueName},command=${dto.command},force=${dto.force}`);switch (dto.command) {case JobCommand.START: {