Skip to content

High performance self-hosted photo and video management solution.

feat(server): e2e for missing jobs (#15910)

Jonathan Jogenfors22d348beca0d
diff --git a/e2e/src/api/specs/asset.e2e-spec.ts b/e2e/src/api/specs/asset.e2e-spec.ts
index 1b644454aa..8700356256 100644
--- a/e2e/src/api/specs/asset.e2e-spec.ts
+++ b/e2e/src/api/specs/asset.e2e-spec.ts
@@ -4,7 +4,6 @@ import {
AssetResponseDto,
AssetTypeEnum,
getAssetInfo,
- getConfig,
getMyUser,
LoginResponseDto,
SharedLinkType,
@@ -45,8 +44,6 @@ const locationAssetFilepath = `${testAssetDir}/metadata/gps-position/thompson-sp
const ratingAssetFilepath = `${testAssetDir}/metadata/rating/mongolels.jpg`;
const facesAssetFilepath = `${testAssetDir}/metadata/faces/portrait.jpg`;
-const getSystemConfig = (accessToken: string) => getConfig({ headers: asBearerAuth(accessToken) });
-
const readTags = async (bytes: Buffer, filename: string) => {
const filepath = join(tempDir, filename);
await writeFile(filepath, bytes);
@@ -228,7 +225,7 @@ describe('/asset', () => {
});
it('should get the asset faces', async () => {
- const config = await getSystemConfig(admin.accessToken);
+ const config = await utils.getSystemConfig(admin.accessToken);
config.metadata.faces.import = true;
await updateConfig({ systemConfigDto: config }, { headers: asBearerAuth(admin.accessToken) });
diff --git a/e2e/src/api/specs/jobs.e2e-spec.ts b/e2e/src/api/specs/jobs.e2e-spec.ts
index 4b8126c941..a9afd8475f 100644
--- a/e2e/src/api/specs/jobs.e2e-spec.ts
+++ b/e2e/src/api/specs/jobs.e2e-spec.ts
@@ -1,8 +1,9 @@
-import { JobCommand, JobName, LoginResponseDto } from '@immich/sdk';
+import { JobCommand, JobName, LoginResponseDto, updateConfig } from '@immich/sdk';
+import { cpSync, rmSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import { basename } from 'node:path';
import { errorDto } from 'src/responses';
-import { app, testAssetDir, utils } from 'src/utils';
+import { app, asBearerAuth, testAssetDir, utils } from 'src/utils';
import request from 'supertest';
import { afterEach, beforeAll, describe, expect, it } from 'vitest';
@@ -20,6 +21,33 @@ describe('/jobs', () => {
command: JobCommand.Resume,
force: false,
});
+
+ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
+ command: JobCommand.Resume,
+ force: false,
+ });
+
+ await utils.jobCommand(admin.accessToken, JobName.FaceDetection, {
+ command: JobCommand.Resume,
+ force: false,
+ });
+
+ await utils.jobCommand(admin.accessToken, JobName.SmartSearch, {
+ command: JobCommand.Resume,
+ force: false,
+ });
+
+ await utils.jobCommand(admin.accessToken, JobName.DuplicateDetection, {
+ command: JobCommand.Resume,
+ force: false,
+ });
+
+ const config = await utils.getSystemConfig(admin.accessToken);
+ config.machineLearning.duplicateDetection.enabled = false;
+ config.machineLearning.enabled = false;
+ config.metadata.faces.import = false;
+ config.machineLearning.clip.enabled = false;
+ await updateConfig({ systemConfigDto: config }, { headers: asBearerAuth(admin.accessToken) });
});
it('should require authentication', async () => {
@@ -29,14 +57,7 @@ describe('/jobs', () => {
});
it('should queue metadata extraction for missing assets', async () => {
- const path1 = `${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`;
- const path2 = `${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`;
-
- await utils.createAsset(admin.accessToken, {
- assetData: { bytes: await readFile(path1), filename: basename(path1) },
- });
-
- await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
+ const path = `${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`;
await utils.jobCommand(admin.accessToken, JobName.MetadataExtraction, {
command: JobCommand.Pause,
@@ -44,7 +65,7 @@ describe('/jobs', () => {
});
const { id } = await utils.createAsset(admin.accessToken, {
- assetData: { bytes: await readFile(path2), filename: basename(path2) },
+ assetData: { bytes: await readFile(path), filename: basename(path) },
});
await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
@@ -82,5 +103,123 @@ describe('/jobs', () => {
expect(asset.exifInfo?.make).toBe('NIKON CORPORATION');
}
});
+
+ it('should not re-extract metadata for existing assets', async () => {
+ const path = `${testAssetDir}/temp/metadata/asset.jpg`;
+
+ cpSync(`${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`, path);
+
+ const { id } = await utils.createAsset(admin.accessToken, {
+ assetData: { bytes: await readFile(path), filename: basename(path) },
+ });
+
+ await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
+
+ {
+ const asset = await utils.getAssetInfo(admin.accessToken, id);
+
+ expect(asset.exifInfo).toBeDefined();
+ expect(asset.exifInfo?.model).toBe('NIKON D700');
+ }
+
+ cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, path);
+
+ await utils.jobCommand(admin.accessToken, JobName.MetadataExtraction, {
+ command: JobCommand.Start,
+ force: false,
+ });
+
+ await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
+
+ {
+ const asset = await utils.getAssetInfo(admin.accessToken, id);
+
+ expect(asset.exifInfo).toBeDefined();
+ expect(asset.exifInfo?.model).toBe('NIKON D700');
+ }
+
+ rmSync(path);
+ });
+
+ it('should queue thumbnail extraction for assets missing thumbs', async () => {
+ const path = `${testAssetDir}/albums/nature/tanners_ridge.jpg`;
+
+ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
+ command: JobCommand.Pause,
+ force: false,
+ });
+
+ const { id } = await utils.createAsset(admin.accessToken, {
+ assetData: { bytes: await readFile(path), filename: basename(path) },
+ });
+
+ await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
+ await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
+
+ const assetBefore = await utils.getAssetInfo(admin.accessToken, id);
+ expect(assetBefore.thumbhash).toBeNull();
+
+ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
+ command: JobCommand.Empty,
+ force: false,
+ });
+
+ await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
+ await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
+
+ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
+ command: JobCommand.Resume,
+ force: false,
+ });
+
+ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
+ command: JobCommand.Start,
+ force: false,
+ });
+
+ await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
+ await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
+
+ const assetAfter = await utils.getAssetInfo(admin.accessToken, id);
+ expect(assetAfter.thumbhash).not.toBeNull();
+ });
+
+ it('should not reload existing thumbnail when running thumb job for missing assets', async () => {
+ const path = `${testAssetDir}/temp/thumbs/asset1.jpg`;
+
+ cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, path);
+
+ const { id } = await utils.createAsset(admin.accessToken, {
+ assetData: { bytes: await readFile(path), filename: basename(path) },
+ });
+
+ await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
+ await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
+
+ const assetBefore = await utils.getAssetInfo(admin.accessToken, id);
+
+ cpSync(`${testAssetDir}/albums/nature/notocactus_minimus.jpg`, path);
+
+ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
+ command: JobCommand.Resume,
+ force: false,
+ });
+
+ // This runs the missing thumbnail job
+ await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
+ command: JobCommand.Start,
+ force: false,
+ });
+
+ await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
+ await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
+
+ const assetAfter = await utils.getAssetInfo(admin.accessToken, id);
+
+ // Asset 1 thumbnail should be untouched since its thumb should not have been reloaded, even though the file was changed
+ expect(assetAfter.thumbhash).toEqual(assetBefore.thumbhash);
+
+ rmSync(path);
+ });
});
});
diff --git a/e2e/src/utils.ts b/e2e/src/utils.ts
index ecd002a8b9..7446bb708f 100644
--- a/e2e/src/utils.ts
+++ b/e2e/src/utils.ts
@@ -28,6 +28,7 @@ import {
deleteAssets,
getAllJobsStatus,
getAssetInfo,
+ getConfig,
getConfigDefaults,
login,
scanLibrary,
@@ -121,6 +122,7 @@ const execPromise = promisify(exec);
const onEvent = ({ event, id }: { event: EventType; id: string }) => {
// console.log(`Received event: ${event} [id=${id}]`);
const set = events[event];
+
set.add(id);
const idCallback = idCallbacks[id];
@@ -415,6 +417,8 @@ export const utils = {
rmSync(path, { recursive: true });
},
+ getSystemConfig: (accessToken: string) => getConfig({ headers: asBearerAuth(accessToken) }),
+
getAssetInfo: (accessToken: string, id: string) => getAssetInfo({ id }, { headers: asBearerAuth(accessToken) }),
checkExistingAssets: (accessToken: string, checkExistingAssetsDto: CheckExistingAssetsDto) =>
diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts
index 595be21829..342aec7a7a 100644
--- a/server/src/services/job.service.ts
+++ b/server/src/services/job.service.ts
@@ -70,7 +70,7 @@ export class JobService extends BaseService {
}
async handleCommand(queueName: QueueName, dto: JobCommandDto): Promise<JobStatusDto> {
- this.logger.debug(`Handling command: queue=${queueName},force=${dto.force}`);
+ this.logger.debug(`Handling command: queue=${queueName},command=${dto.command},force=${dto.force}`);
switch (dto.command) {
case JobCommand.START: {