Skip to content

High performance self-hosted photo and video management solution.

feat: nightly tasks (#19879)

Jason Rasmussen47c0dc0d7ee5
diff --git a/e2e/src/api/specs/system-config.e2e-spec.ts b/e2e/src/api/specs/system-config.e2e-spec.ts
index 060163d7c9..1bd7bdc489 100644
--- a/e2e/src/api/specs/system-config.e2e-spec.ts
+++ b/e2e/src/api/specs/system-config.e2e-spec.ts
@@ -15,12 +15,6 @@ describe('/system-config', () => {
});
describe('PUT /system-config', () => {
- it('should require authentication', async () => {
- const { status, body } = await request(app).put('/system-config');
- expect(status).toBe(401);
- expect(body).toEqual(errorDto.unauthorized);
- });
-
it('should always return the new config', async () => {
const config = await getSystemConfig(admin.accessToken);
diff --git a/i18n/en.json b/i18n/en.json
index 06b3a5d599..f547a4e48d 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -166,6 +166,20 @@
"metadata_settings_description": "Manage metadata settings",
"migration_job": "Migration",
"migration_job_description": "Migrate thumbnails for assets and faces to the latest folder structure",
+ "nightly_tasks_cluster_faces_setting_description": "Run facial recognition on newly detected faces",
+ "nightly_tasks_cluster_new_faces_setting": "Cluster new faces",
+ "nightly_tasks_database_cleanup_setting": "Database cleanup tasks",
+ "nightly_tasks_database_cleanup_setting_description": "Clean up old, expired data from the database",
+ "nightly_tasks_generate_memories_setting": "Generate memories",
+ "nightly_tasks_generate_memories_setting_description": "Create new memories from assets",
+ "nightly_tasks_missing_thumbnails_setting": "Generate missing thumbnails",
+ "nightly_tasks_missing_thumbnails_setting_description": "Queue assets without thumbnails for thumbnail generation",
+ "nightly_tasks_settings": "Nightly Tasks Settings",
+ "nightly_tasks_settings_description": "Manage nightly tasks",
+ "nightly_tasks_start_time_setting": "Start time",
+ "nightly_tasks_start_time_setting_description": "The time at which the server starts running the nightly tasks",
+ "nightly_tasks_sync_quota_usage_setting": "Sync quota usage",
+ "nightly_tasks_sync_quota_usage_setting_description": "Update user storage quota, based on current usage",
"no_paths_added": "No paths added",
"no_pattern_added": "No pattern added",
"note_apply_storage_label_previous_assets": "Note: To apply the Storage Label to previously uploaded assets, run the",
diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md
index 5085052c05..28fa63ba84 100644
--- a/mobile/openapi/README.md
+++ b/mobile/openapi/README.md
@@ -509,6 +509,7 @@ Class | Method | HTTP request | Description
- [SystemConfigMapDto](doc//SystemConfigMapDto.md)
- [SystemConfigMetadataDto](doc//SystemConfigMetadataDto.md)
- [SystemConfigNewVersionCheckDto](doc//SystemConfigNewVersionCheckDto.md)
+ - [SystemConfigNightlyTasksDto](doc//SystemConfigNightlyTasksDto.md)
- [SystemConfigNotificationsDto](doc//SystemConfigNotificationsDto.md)
- [SystemConfigOAuthDto](doc//SystemConfigOAuthDto.md)
- [SystemConfigPasswordLoginDto](doc//SystemConfigPasswordLoginDto.md)
diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart
index f30481ecce..becafa06bf 100644
--- a/mobile/openapi/lib/api.dart
+++ b/mobile/openapi/lib/api.dart
@@ -291,6 +291,7 @@ part 'model/system_config_machine_learning_dto.dart';
part 'model/system_config_map_dto.dart';
part 'model/system_config_metadata_dto.dart';
part 'model/system_config_new_version_check_dto.dart';
+part 'model/system_config_nightly_tasks_dto.dart';
part 'model/system_config_notifications_dto.dart';
part 'model/system_config_o_auth_dto.dart';
part 'model/system_config_password_login_dto.dart';
diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart
index f1cf05f110..603163f00e 100644
--- a/mobile/openapi/lib/api_client.dart
+++ b/mobile/openapi/lib/api_client.dart
@@ -638,6 +638,8 @@ class ApiClient {
return SystemConfigMetadataDto.fromJson(value);
case 'SystemConfigNewVersionCheckDto':
return SystemConfigNewVersionCheckDto.fromJson(value);
+ case 'SystemConfigNightlyTasksDto':
+ return SystemConfigNightlyTasksDto.fromJson(value);
case 'SystemConfigNotificationsDto':
return SystemConfigNotificationsDto.fromJson(value);
case 'SystemConfigOAuthDto':
diff --git a/mobile/openapi/lib/model/system_config_dto.dart b/mobile/openapi/lib/model/system_config_dto.dart
index 59d5f09fc9..38dbb30f0c 100644
--- a/mobile/openapi/lib/model/system_config_dto.dart
+++ b/mobile/openapi/lib/model/system_config_dto.dart
@@ -23,6 +23,7 @@ class SystemConfigDto {
required this.map,
required this.metadata,
required this.newVersionCheck,
+ required this.nightlyTasks,
required this.notifications,
required this.oauth,
required this.passwordLogin,
@@ -55,6 +56,8 @@ class SystemConfigDto {
SystemConfigNewVersionCheckDto newVersionCheck;
+ SystemConfigNightlyTasksDto nightlyTasks;
+
SystemConfigNotificationsDto notifications;
SystemConfigOAuthDto oauth;
@@ -87,6 +90,7 @@ class SystemConfigDto {
other.map == map &&
other.metadata == metadata &&
other.newVersionCheck == newVersionCheck &&
+ other.nightlyTasks == nightlyTasks &&
other.notifications == notifications &&
other.oauth == oauth &&
other.passwordLogin == passwordLogin &&
@@ -111,6 +115,7 @@ class SystemConfigDto {
(map.hashCode) +
(metadata.hashCode) +
(newVersionCheck.hashCode) +
+ (nightlyTasks.hashCode) +
(notifications.hashCode) +
(oauth.hashCode) +
(passwordLogin.hashCode) +
@@ -123,7 +128,7 @@ class SystemConfigDto {
(user.hashCode);
@override
- String toString() => 'SystemConfigDto[backup=$backup, ffmpeg=$ffmpeg, image=$image, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, templates=$templates, theme=$theme, trash=$trash, user=$user]';
+ String toString() => 'SystemConfigDto[backup=$backup, ffmpeg=$ffmpeg, image=$image, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, nightlyTasks=$nightlyTasks, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, templates=$templates, theme=$theme, trash=$trash, user=$user]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
@@ -137,6 +142,7 @@ class SystemConfigDto {
json[r'map'] = this.map;
json[r'metadata'] = this.metadata;
json[r'newVersionCheck'] = this.newVersionCheck;
+ json[r'nightlyTasks'] = this.nightlyTasks;
json[r'notifications'] = this.notifications;
json[r'oauth'] = this.oauth;
json[r'passwordLogin'] = this.passwordLogin;
@@ -169,6 +175,7 @@ class SystemConfigDto {
map: SystemConfigMapDto.fromJson(json[r'map'])!,
metadata: SystemConfigMetadataDto.fromJson(json[r'metadata'])!,
newVersionCheck: SystemConfigNewVersionCheckDto.fromJson(json[r'newVersionCheck'])!,
+ nightlyTasks: SystemConfigNightlyTasksDto.fromJson(json[r'nightlyTasks'])!,
notifications: SystemConfigNotificationsDto.fromJson(json[r'notifications'])!,
oauth: SystemConfigOAuthDto.fromJson(json[r'oauth'])!,
passwordLogin: SystemConfigPasswordLoginDto.fromJson(json[r'passwordLogin'])!,
@@ -236,6 +243,7 @@ class SystemConfigDto {
'map',
'metadata',
'newVersionCheck',
+ 'nightlyTasks',
'notifications',
'oauth',
'passwordLogin',
diff --git a/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart b/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart
new file mode 100644
index 0000000000..ab7b4b37c2
--- /dev/null
+++ b/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart
@@ -0,0 +1,139 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.18
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+class SystemConfigNightlyTasksDto {
+ /// Returns a new [SystemConfigNightlyTasksDto] instance.
+ SystemConfigNightlyTasksDto({
+ required this.clusterNewFaces,
+ required this.databaseCleanup,
+ required this.generateMemories,
+ required this.missingThumbnails,
+ required this.startTime,
+ required this.syncQuotaUsage,
+ });
+
+ bool clusterNewFaces;
+
+ bool databaseCleanup;
+
+ bool generateMemories;
+
+ bool missingThumbnails;
+
+ String startTime;
+
+ bool syncQuotaUsage;
+
+ @override
+ bool operator ==(Object other) => identical(this, other) || other is SystemConfigNightlyTasksDto &&
+ other.clusterNewFaces == clusterNewFaces &&
+ other.databaseCleanup == databaseCleanup &&
+ other.generateMemories == generateMemories &&
+ other.missingThumbnails == missingThumbnails &&
+ other.startTime == startTime &&
+ other.syncQuotaUsage == syncQuotaUsage;
+
+ @override
+ int get hashCode =>
+ // ignore: unnecessary_parenthesis
+ (clusterNewFaces.hashCode) +
+ (databaseCleanup.hashCode) +
+ (generateMemories.hashCode) +
+ (missingThumbnails.hashCode) +
+ (startTime.hashCode) +
+ (syncQuotaUsage.hashCode);
+
+ @override
+ String toString() => 'SystemConfigNightlyTasksDto[clusterNewFaces=$clusterNewFaces, databaseCleanup=$databaseCleanup, generateMemories=$generateMemories, missingThumbnails=$missingThumbnails, startTime=$startTime, syncQuotaUsage=$syncQuotaUsage]';
+
+ Map<String, dynamic> toJson() {
+ final json = <String, dynamic>{};
+ json[r'clusterNewFaces'] = this.clusterNewFaces;
+ json[r'databaseCleanup'] = this.databaseCleanup;
+ json[r'generateMemories'] = this.generateMemories;
+ json[r'missingThumbnails'] = this.missingThumbnails;
+ json[r'startTime'] = this.startTime;
+ json[r'syncQuotaUsage'] = this.syncQuotaUsage;
+ return json;
+ }
+
+ /// Returns a new [SystemConfigNightlyTasksDto] instance and imports its values from
+ /// [value] if it's a [Map], null otherwise.
+ // ignore: prefer_constructors_over_static_methods
+ static SystemConfigNightlyTasksDto? fromJson(dynamic value) {
+ upgradeDto(value, "SystemConfigNightlyTasksDto");
+ if (value is Map) {
+ final json = value.cast<String, dynamic>();
+
+ return SystemConfigNightlyTasksDto(
+ clusterNewFaces: mapValueOfType<bool>(json, r'clusterNewFaces')!,
+ databaseCleanup: mapValueOfType<bool>(json, r'databaseCleanup')!,
+ generateMemories: mapValueOfType<bool>(json, r'generateMemories')!,
+ missingThumbnails: mapValueOfType<bool>(json, r'missingThumbnails')!,
+ startTime: mapValueOfType<String>(json, r'startTime')!,
+ syncQuotaUsage: mapValueOfType<bool>(json, r'syncQuotaUsage')!,
+ );
+ }
+ return null;
+ }
+
+ static List<SystemConfigNightlyTasksDto> listFromJson(dynamic json, {bool growable = false,}) {
+ final result = <SystemConfigNightlyTasksDto>[];
+ if (json is List && json.isNotEmpty) {
+ for (final row in json) {
+ final value = SystemConfigNightlyTasksDto.fromJson(row);
+ if (value != null) {
+ result.add(value);
+ }
+ }
+ }
+ return result.toList(growable: growable);
+ }
+
+ static Map<String, SystemConfigNightlyTasksDto> mapFromJson(dynamic json) {
+ final map = <String, SystemConfigNightlyTasksDto>{};
+ if (json is Map && json.isNotEmpty) {
+ json = json.cast<String, dynamic>(); // ignore: parameter_assignments
+ for (final entry in json.entries) {
+ final value = SystemConfigNightlyTasksDto.fromJson(entry.value);
+ if (value != null) {
+ map[entry.key] = value;
+ }
+ }
+ }
+ return map;
+ }
+
+ // maps a json object with a list of SystemConfigNightlyTasksDto-objects as value to a dart map
+ static Map<String, List<SystemConfigNightlyTasksDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
+ final map = <String, List<SystemConfigNightlyTasksDto>>{};
+ if (json is Map && json.isNotEmpty) {
+ // ignore: parameter_assignments
+ json = json.cast<String, dynamic>();
+ for (final entry in json.entries) {
+ map[entry.key] = SystemConfigNightlyTasksDto.listFromJson(entry.value, growable: growable,);
+ }
+ }
+ return map;
+ }
+
+ /// The list of required keys that must be present in a JSON.
+ static const requiredKeys = <String>{
+ 'clusterNewFaces',
+ 'databaseCleanup',
+ 'generateMemories',
+ 'missingThumbnails',
+ 'startTime',
+ 'syncQuotaUsage',
+ };
+}
+
diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json
index 18204c21f3..492d3cfec1 100644
--- a/open-api/immich-openapi-specs.json
+++ b/open-api/immich-openapi-specs.json
@@ -14318,6 +14318,9 @@
"newVersionCheck": {
"$ref": "#/components/schemas/SystemConfigNewVersionCheckDto"
},
+ "nightlyTasks": {
+ "$ref": "#/components/schemas/SystemConfigNightlyTasksDto"
+ },
"notifications": {
"$ref": "#/components/schemas/SystemConfigNotificationsDto"
},
@@ -14360,6 +14363,7 @@
"map",
"metadata",
"newVersionCheck",
+ "nightlyTasks",
"notifications",
"oauth",
"passwordLogin",
@@ -14790,6 +14794,37 @@
],
"type": "object"
},
+ "SystemConfigNightlyTasksDto": {
+ "properties": {
+ "clusterNewFaces": {
+ "type": "boolean"
+ },
+ "databaseCleanup": {
+ "type": "boolean"
+ },
+ "generateMemories": {
+ "type": "boolean"
+ },
+ "missingThumbnails": {
+ "type": "boolean"
+ },
+ "startTime": {
+ "type": "string"
+ },
+ "syncQuotaUsage": {
+ "type": "boolean"
+ }
+ },
+ "required": [
+ "clusterNewFaces",
+ "databaseCleanup",
+ "generateMemories",
+ "missingThumbnails",
+ "startTime",
+ "syncQuotaUsage"
+ ],
+ "type": "object"
+ },
"SystemConfigNotificationsDto": {
"properties": {
"smtp": {
diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts
index 88dee9bf0b..f60fa6dfe8 100644
--- a/open-api/typescript-sdk/src/fetch-client.ts
+++ b/open-api/typescript-sdk/src/fetch-client.ts
@@ -1389,6 +1389,14 @@ export type SystemConfigMetadataDto = {
export type SystemConfigNewVersionCheckDto = {
enabled: boolean;
};
+export type SystemConfigNightlyTasksDto = {
+ clusterNewFaces: boolean;
+ databaseCleanup: boolean;
+ generateMemories: boolean;
+ missingThumbnails: boolean;
+ startTime: string;
+ syncQuotaUsage: boolean;
+};
export type SystemConfigNotificationsDto = {
smtp: SystemConfigSmtpDto;
};
@@ -1457,6 +1465,7 @@ export type SystemConfigDto = {
map: SystemConfigMapDto;
metadata: SystemConfigMetadataDto;
newVersionCheck: SystemConfigNewVersionCheckDto;
+ nightlyTasks: SystemConfigNightlyTasksDto;
notifications: SystemConfigNotificationsDto;
oauth: SystemConfigOAuthDto;
passwordLogin: SystemConfigPasswordLoginDto;
diff --git a/server/src/config.ts b/server/src/config.ts
index 1fcc2e9782..90ca2c1529 100644
--- a/server/src/config.ts
+++ b/server/src/config.ts
@@ -121,6 +121,14 @@ export interface SystemConfig {
newVersionCheck: {
enabled: boolean;
};
+ nightlyTasks: {
+ startTime: string;
+ databaseCleanup: boolean;
+ missingThumbnails: boolean;
+ clusterNewFaces: boolean;
+ generateMemories: boolean;
+ syncQuotaUsage: boolean;
+ };
trash: {
enabled: boolean;
days: number;
@@ -298,6 +306,14 @@ export const defaults = Object.freeze<SystemConfig>({
newVersionCheck: {
enabled: true,
},
+ nightlyTasks: {
+ startTime: '00:00',
+ databaseCleanup: true,
+ generateMemories: true,
+ syncQuotaUsage: true,
+ missingThumbnails: true,
+ clusterNewFaces: true,
+ },
trash: {
enabled: true,
days: 30,
diff --git a/server/src/controllers/system-config.controller.spec.ts b/server/src/controllers/system-config.controller.spec.ts
new file mode 100644
index 0000000000..48b8c1bcf0
--- /dev/null
+++ b/server/src/controllers/system-config.controller.spec.ts
@@ -0,0 +1,74 @@
+import _ from 'lodash';
+import { defaults } from 'src/config';
+import { SystemConfigController } from 'src/controllers/system-config.controller';
+import { StorageTemplateService } from 'src/services/storage-template.service';
+import { SystemConfigService } from 'src/services/system-config.service';
+import request from 'supertest';
+import { errorDto } from 'test/medium/responses';
+import { ControllerContext, controllerSetup, mockBaseService } from 'test/utils';
+
+describe(SystemConfigController.name, () => {
+ let ctx: ControllerContext;
+ const systemConfigService = mockBaseService(SystemConfigService);
+ const templateService = mockBaseService(StorageTemplateService);
+
+ beforeAll(async () => {
+ ctx = await controllerSetup(SystemConfigController, [
+ { provide: SystemConfigService, useValue: systemConfigService },
+ { provide: StorageTemplateService, useValue: templateService },
+ ]);
+ return () => ctx.close();
+ });
+
+ beforeEach(() => {
+ systemConfigService.resetAllMocks();
+ templateService.resetAllMocks();
+ ctx.reset();
+ });
+
+ describe('GET /system-config', () => {
+ it('should be an authenticated route', async () => {
+ await request(ctx.getHttpServer()).get('/system-config');
+ expect(ctx.authenticate).toHaveBeenCalled();
+ });
+ });
+
+ describe('GET /system-config/defaults', () => {
+ it('should be an authenticated route', async () => {
+ await request(ctx.getHttpServer()).get('/system-config/defaults');
+ expect(ctx.authenticate).toHaveBeenCalled();
+ });
+ });
+
+ describe('PUT /system-config', () => {
+ it('should be an authenticated route', async () => {
+ await request(ctx.getHttpServer()).put('/system-config');
+ expect(ctx.authenticate).toHaveBeenCalled();
+ });
+
+ describe('nightlyTasks', () => {
+ it('should validate nightly jobs start time', async () => {
+ const config = _.cloneDeep(defaults);
+ config.nightlyTasks.startTime = 'invalid';
+ const { status, body } = await request(ctx.getHttpServer()).put('/system-config').send(config);
+ expect(status).toBe(400);
+ expect(body).toEqual(errorDto.badRequest(['nightlyTasks.startTime must be in HH:mm format']));
+ });
+
+ it('should accept a valid time', async () => {
+ const config = _.cloneDeep(defaults);
+ config.nightlyTasks.startTime = '05:05';
+ const { status } = await request(ctx.getHttpServer()).put('/system-config').send(config);
+ expect(status).toBe(200);
+ });
+
+ it('should validate a boolean field', async () => {
+ const config = _.cloneDeep(defaults);
+ (config.nightlyTasks.databaseCleanup as any) = 'invalid';
+ const { status, body } = await request(ctx.getHttpServer()).put('/system-config').send(config);
+ expect(status).toBe(400);
+ expect(body).toEqual(errorDto.badRequest(['nightlyTasks.databaseCleanup must be a boolean value']));
+ });
+ });
+ });
+});
diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts
index b0385984b4..49c5e5b4e7 100644
--- a/server/src/dtos/system-config.dto.ts
+++ b/server/src/dtos/system-config.dto.ts
@@ -34,7 +34,7 @@ import {
VideoContainer,
} from 'src/enum';
import { ConcurrentQueueName } from 'src/types';
-import { IsCronExpression, Optional, ValidateBoolean } from 'src/validation';
+import { IsCronExpression, IsDateStringFormat, Optional, ValidateBoolean } from 'src/validation';
const isLibraryScanEnabled = (config: SystemConfigLibraryScanDto) => config.enabled;
const isOAuthEnabled = (config: SystemConfigOAuthDto) => config.enabled;
@@ -329,6 +329,26 @@ class SystemConfigNewVersionCheckDto {
enabled!: boolean;
}
+class SystemConfigNightlyTasksDto {
+ @IsDateStringFormat('HH:mm', { message: 'startTime must be in HH:mm format' })
+ startTime!: string;
+
+ @ValidateBoolean()
+ databaseCleanup!: boolean;
+
+ @ValidateBoolean()
+ missingThumbnails!: boolean;
+
+ @ValidateBoolean()
+ clusterNewFaces!: boolean;
+
+ @ValidateBoolean()
+ generateMemories!: boolean;
+
+ @ValidateBoolean()
+ syncQuotaUsage!: boolean;
+}
+
class SystemConfigOAuthDto {
@ValidateBoolean()
autoLaunch!: boolean;
@@ -638,6 +658,11 @@ export class SystemConfigDto implements SystemConfig {
@IsObject()
newVersionCheck!: SystemConfigNewVersionCheckDto;
+ @Type(() => SystemConfigNightlyTasksDto)
+ @ValidateNested()
+ @IsObject()
+ nightlyTasks!: SystemConfigNightlyTasksDto;
+
@Type(() => SystemConfigOAuthDto)
@ValidateNested()
@IsObject()
diff --git a/server/src/enum.ts b/server/src/enum.ts
index dca0f09555..d7c74a71c6 100644
--- a/server/src/enum.ts
+++ b/server/src/enum.ts
@@ -567,6 +567,7 @@ export enum DatabaseLock {
VersionHistory = 500,
CLIPDimSize = 512,
Library = 1337,
+ NightlyJobs = 600,
GetSystemConfig = 69,
BackupDatabase = 42,
MemoryCreation = 777,
@@ -684,3 +685,8 @@ export enum AssetVisibility {
HIDDEN = 'hidden',
LOCKED = 'locked',
}
+
+export enum CronJob {
+ LibraryScan = 'LibraryScan',
+ NightlyJobs = 'NightlyJobs',
+}
diff --git a/server/src/services/api.service.ts b/server/src/services/api.service.ts
index 064b2e268d..27a776e867 100644
--- a/server/src/services/api.service.ts
+++ b/server/src/services/api.service.ts
@@ -1,5 +1,5 @@
import { Injectable } from '@nestjs/common';
-import { Cron, CronExpression, Interval } from '@nestjs/schedule';
+import { Interval } from '@nestjs/schedule';
import { NextFunction, Request, Response } from 'express';
import { readFileSync } from 'node:fs';
import sanitizeHtml from 'sanitize-html';
@@ -54,11 +54,6 @@ export class ApiService {
await this.versionService.handleQueueVersionCheck();
}
- @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
- async onNightlyJob() {
- await this.jobService.handleNightlyJobs();
- }
-
ssr(excludePaths: string[]) {
const { resourcePaths } = this.configRepository.getEnv();
diff --git a/server/src/services/job.service.spec.ts b/server/src/services/job.service.spec.ts
index c9020ed96a..a18eccdd8b 100644
--- a/server/src/services/job.service.spec.ts
+++ b/server/src/services/job.service.spec.ts
@@ -41,12 +41,12 @@ describe(JobService.name, () => {
{ name: JobName.USER_DELETE_CHECK },
{ name: JobName.PERSON_CLEANUP },
{ name: JobName.MEMORIES_CLEANUP },
- { name: JobName.MEMORIES_CREATE },
- { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
+ { name: JobName.CLEAN_OLD_SESSION_TOKENS },
{ name: JobName.CLEAN_OLD_AUDIT_LOGS },
+ { name: JobName.MEMORIES_CREATE },
{ name: JobName.USER_SYNC_USAGE },
+ { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
{ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },
- { name: JobName.CLEAN_OLD_SESSION_TOKENS },
]);
});
});
diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts
index e4b20ba37f..645b22249f 100644
--- a/server/src/services/job.service.ts
+++ b/server/src/services/job.service.ts
@@ -1,6 +1,7 @@
import { BadRequestException, Injectable } from '@nestjs/common';
import { ClassConstructor } from 'class-transformer';
import { snakeCase } from 'lodash';
+import { SystemConfig } from 'src/config';
import { OnEvent } from 'src/decorators';
import { mapAsset } from 'src/dtos/asset-response.dto';
import { AllJobStatusResponseDto, JobCommandDto, JobCreateDto, JobStatusDto } from 'src/dtos/job.dto';
@@ -8,6 +9,8 @@ import {
AssetType,
AssetVisibility,
BootstrapEventPriority,
+ CronJob,
+ DatabaseLock,
ImmichWorker,
JobCommand,
JobName,
@@ -20,6 +23,7 @@ import { ArgOf, ArgsOf } from 'src/repositories/event.repository';
import { BaseService } from 'src/services/base.service';
import { ConcurrentQueueName, JobItem } from 'src/types';
import { hexOrBufferToBase64 } from 'src/utils/bytes';
+import { handlePromiseError } from 'src/utils/misc';
const asJobItem = (dto: JobCreateDto): JobItem => {
switch (dto.name) {
@@ -53,26 +57,48 @@ const asJobItem = (dto: JobCreateDto): JobItem => {
}
};
+const asNightlyTasksCron = (config: SystemConfig) => {
+ const [hours, minutes] = config.nightlyTasks.startTime.split(':').map(Number);
+ return `${minutes} ${hours} * * *`;
+};
+
@Injectable()
export class JobService extends BaseService {
private services: ClassConstructor<unknown>[] = [];
+ private nightlyJobsLock = false;
- @OnEvent({ name: 'config.init', workers: [ImmichWorker.MICROSERVICES] })
- onConfigInit({ newConfig: config }: ArgOf<'config.init'>) {
- this.logger.debug(`Updating queue concurrency settings`);
- for (const queueName of Object.values(QueueName)) {
- let concurrency = 1;
- if (this.isConcurrentQueue(queueName)) {
- concurrency = config.job[queueName].concurrency;
- }
- this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);
- this.jobRepository.setConcurrency(queueName, concurrency);
+ @OnEvent({ name: 'config.init' })
+ async onConfigInit({ newConfig: config }: ArgOf<'config.init'>) {
+ if (this.worker === ImmichWorker.MICROSERVICES) {
+ this.updateQueueConcurrency(config);
+ return;
+ }
+
+ this.nightlyJobsLock = await this.databaseRepository.tryLock(DatabaseLock.NightlyJobs);
+ if (this.nightlyJobsLock) {
+ const cronExpression = asNightlyTasksCron(config);
+ this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`);
+ this.cronRepository.create({
+ name: CronJob.NightlyJobs,
+ expression: cronExpression,
+ start: true,
+ onTick: () => handlePromiseError(this.handleNightlyJobs(), this.logger),
+ });
}
}
- @OnEvent({ name: 'config.update', server: true, workers: [ImmichWorker.MICROSERVICES] })
+ @OnEvent({ name: 'config.update', server: true })
onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) {
- this.onConfigInit({ newConfig: config });
+ if (this.worker === ImmichWorker.MICROSERVICES) {
+ this.updateQueueConcurrency(config);
+ return;
+ }
+
+ if (this.nightlyJobsLock) {
+ const cronExpression = asNightlyTasksCron(config);
+ this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`);
+ this.cronRepository.update({ name: CronJob.NightlyJobs, expression: cronExpression, start: true });
+ }
}
@OnEvent({ name: 'app.bootstrap', priority: BootstrapEventPriority.JobService })
@@ -83,6 +109,18 @@ export class JobService extends BaseService {
}
}
+ private updateQueueConcurrency(config: SystemConfig) {
+ this.logger.debug(`Updating queue concurrency settings`);
+ for (const queueName of Object.values(QueueName)) {
+ let concurrency = 1;
+ if (this.isConcurrentQueue(queueName)) {
+ concurrency = config.job[queueName].concurrency;
+ }
+ this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);
+ this.jobRepository.setConcurrency(queueName, concurrency);
+ }
+ }
+
setServices(services: ClassConstructor<unknown>[]) {
this.services = services;
}
@@ -233,18 +271,37 @@ export class JobService extends BaseService {
}
async handleNightlyJobs() {
- await this.jobRepository.queueAll([
- { name: JobName.ASSET_DELETION_CHECK },
- { name: JobName.USER_DELETE_CHECK },
- { name: JobName.PERSON_CLEANUP },
- { name: JobName.MEMORIES_CLEANUP },
- { name: JobName.MEMORIES_CREATE },
- { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
- { name: JobName.CLEAN_OLD_AUDIT_LOGS },
- { name: JobName.USER_SYNC_USAGE },
- { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },
- { name: JobName.CLEAN_OLD_SESSION_TOKENS },
- ]);
+ const config = await this.getConfig({ withCache: false });
+ const jobs: JobItem[] = [];
+
+ if (config.nightlyTasks.databaseCleanup) {
+ jobs.push(
+ { name: JobName.ASSET_DELETION_CHECK },
+ { name: JobName.USER_DELETE_CHECK },
+ { name: JobName.PERSON_CLEANUP },
+ { name: JobName.MEMORIES_CLEANUP },
+ { name: JobName.CLEAN_OLD_SESSION_TOKENS },
+ { name: JobName.CLEAN_OLD_AUDIT_LOGS },
+ );
+ }
+
+ if (config.nightlyTasks.generateMemories) {
+ jobs.push({ name: JobName.MEMORIES_CREATE });
+ }
+
+ if (config.nightlyTasks.syncQuotaUsage) {
+ jobs.push({ name: JobName.USER_SYNC_USAGE });
+ }
+
+ if (config.nightlyTasks.missingThumbnails) {
+ jobs.push({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
+ }
+
+ if (config.nightlyTasks.clusterNewFaces) {
+ jobs.push({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } });
+ }
+
+ await this.jobRepository.queueAll(jobs);
}
/**
diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts
index e92cdcf200..ab69e22b99 100644
--- a/server/src/services/library.service.spec.ts
+++ b/server/src/services/library.service.spec.ts
@@ -3,7 +3,7 @@ import { Stats } from 'node:fs';
import { defaults, SystemConfig } from 'src/config';
import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants';
import { mapLibrary } from 'src/dtos/library.dto';
-import { AssetType, ImmichWorker, JobName, JobStatus } from 'src/enum';
+import { AssetType, CronJob, ImmichWorker, JobName, JobStatus } from 'src/enum';
import { LibraryService } from 'src/services/library.service';
import { ILibraryBulkIdsJob, ILibraryFileJob } from 'src/types';
import { assetStub } from 'test/fixtures/asset.stub';
@@ -56,7 +56,11 @@ describe(LibraryService.name, () => {
} as SystemConfig,
});
- expect(mocks.cron.update).toHaveBeenCalledWith({ name: 'libraryScan', expression: '0 1 * * *', start: true });
+ expect(mocks.cron.update).toHaveBeenCalledWith({
+ name: CronJob.LibraryScan,
+ expression: '0 1 * * *',
+ start: true,
+ });
});
it('should initialize watcher for all external libraries', async () => {
@@ -128,7 +132,7 @@ describe(LibraryService.name, () => {
});
expect(mocks.cron.update).toHaveBeenCalledWith({
- name: 'libraryScan',
+ name: CronJob.LibraryScan,
expression: systemConfigStub.libraryScan.library.scan.cronExpression,
start: systemConfigStub.libraryScan.library.scan.enabled,
});
@@ -149,7 +153,7 @@ describe(LibraryService.name, () => {
});
expect(mocks.cron.update).toHaveBeenCalledWith({
- name: 'libraryScan',
+ name: CronJob.LibraryScan,
expression: systemConfigStub.libraryScan.library.scan.cronExpression,
start: systemConfigStub.libraryScan.library.scan.enabled,
});
diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts
index 8c7f79ee5c..f6bc5b2ebb 100644
--- a/server/src/services/library.service.ts
+++ b/server/src/services/library.service.ts
@@ -17,7 +17,7 @@ import {
ValidateLibraryImportPathResponseDto,
ValidateLibraryResponseDto,
} from 'src/dtos/library.dto';
-import { AssetStatus, AssetType, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum';
+import { AssetStatus, AssetType, CronJob, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum';
import { ArgOf } from 'src/repositories/event.repository';
import { AssetSyncResult } from 'src/repositories/library.repository';
import { AssetTable } from 'src/schema/tables/asset.table';
@@ -45,7 +45,7 @@ export class LibraryService extends BaseService {
if (this.lock) {
this.cronRepository.create({
- name: 'libraryScan',
+ name: CronJob.LibraryScan,
expression: scan.cronExpression,
onTick: () =>
handlePromiseError(this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL }), this.logger),
@@ -65,7 +65,7 @@ export class LibraryService extends BaseService {
}
this.cronRepository.update({
- name: 'libraryScan',
+ name: CronJob.LibraryScan,
expression: library.scan.cronExpression,
start: library.scan.enabled,
});
diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts
index c7b98cc990..43be323459 100644
--- a/server/src/services/system-config.service.spec.ts
+++ b/server/src/services/system-config.service.spec.ts
@@ -103,6 +103,14 @@ const updatedConfig = Object.freeze<SystemConfig>({
lightStyle: 'https://tiles.immich.cloud/v1/style/light.json',
darkStyle: 'https://tiles.immich.cloud/v1/style/dark.json',
},
+ nightlyTasks: {
+ startTime: '00:00',
+ databaseCleanup: true,
+ clusterNewFaces: true,
+ missingThumbnails: true,
+ generateMemories: true,
+ syncQuotaUsage: true,
+ },
reverseGeocoding: {
enabled: true,
},
diff --git a/web/src/lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.svelte b/web/src/lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.svelte
new file mode 100644
index 0000000000..af653f2473
--- /dev/null
+++ b/web/src/lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.svelte
@@ -0,0 +1,81 @@
+<script lang="ts">
+ import SettingButtonsRow from '$lib/components/shared-components/settings/setting-buttons-row.svelte';
+ import SettingInputField from '$lib/components/shared-components/settings/setting-input-field.svelte';
+ import SettingSwitch from '$lib/components/shared-components/settings/setting-switch.svelte';
+ import { SettingInputFieldType } from '$lib/constants';
+ import type { SystemConfigDto } from '@immich/sdk';
+ import { isEqual } from 'lodash-es';
+ import { t } from 'svelte-i18n';
+ import { fade } from 'svelte/transition';
+ import type { SettingsResetEvent, SettingsSaveEvent } from '../admin-settings';
+
+ interface Props {
+ savedConfig: SystemConfigDto;
+ defaultConfig: SystemConfigDto;
+ config: SystemConfigDto;
+ disabled?: boolean;
+ onReset: SettingsResetEvent;
+ onSave: SettingsSaveEvent;
+ }
+
+ let { savedConfig, defaultConfig, config = $bindable(), disabled = false, onReset, onSave }: Props = $props();
+
+ const onsubmit = (event: Event) => {
+ event.preventDefault();
+ };
+</script>
+
+<div class="mt-2">
+ <div in:fade={{ duration: 500 }}>
+ <form autocomplete="off" {onsubmit} class="mx-4 mt-4">
+ <div class="ms-4 mt-4 flex flex-col gap-4">
+ <SettingInputField
+ inputType={SettingInputFieldType.TEXT}
+ label={$t('admin.nightly_tasks_start_time_setting')}
+ description={$t('admin.nightly_tasks_start_time_setting_description')}
+ bind:value={config.nightlyTasks.startTime}
+ required={true}
+ {disabled}
+ isEdited={!(config.nightlyTasks.startTime === savedConfig.nightlyTasks.startTime)}
+ />
+ <SettingSwitch
+ title={$t('admin.nightly_tasks_database_cleanup_setting')}
+ subtitle={$t('admin.nightly_tasks_database_cleanup_setting_description')}
+ bind:checked={config.nightlyTasks.databaseCleanup}
+ {disabled}
+ />
+ <SettingSwitch
+ title={$t('admin.nightly_tasks_missing_thumbnails_setting')}
+ subtitle={$t('admin.nightly_tasks_missing_thumbnails_setting_description')}
+ bind:checked={config.nightlyTasks.missingThumbnails}
+ {disabled}
+ />
+ <SettingSwitch
+ title={$t('admin.nightly_tasks_cluster_new_faces_setting')}
+ subtitle={$t('admin.nightly_tasks_cluster_faces_setting_description')}
+ bind:checked={config.nightlyTasks.clusterNewFaces}
+ {disabled}
+ />
+ <SettingSwitch
+ title={$t('admin.nightly_tasks_generate_memories_setting')}
+ subtitle={$t('admin.nightly_tasks_generate_memories_setting_description')}
+ bind:checked={config.nightlyTasks.generateMemories}
+ {disabled}
+ />
+ <SettingSwitch
+ title={$t('admin.nightly_tasks_sync_quota_usage_setting')}
+ subtitle={$t('admin.nightly_tasks_sync_quota_usage_setting_description')}
+ bind:checked={config.nightlyTasks.syncQuotaUsage}
+ {disabled}
+ />
+ </div>
+
+ <SettingButtonsRow
+ onReset={(options) => onReset({ ...options, configKeys: ['nightlyTasks'] })}
+ onSave={() => onSave({ nightlyTasks: config.nightlyTasks })}
+ showResetToDefault={!isEqual(savedConfig.nightlyTasks, defaultConfig.nightlyTasks)}
+ {disabled}
+ />
+ </form>
+ </div>
+</div>
diff --git a/web/src/routes/admin/system-settings/+page.svelte b/web/src/routes/admin/system-settings/+page.svelte
index e053eea179..3b1d68a49e 100644
--- a/web/src/routes/admin/system-settings/+page.svelte
+++ b/web/src/routes/admin/system-settings/+page.svelte
@@ -12,6 +12,7 @@
import MapSettings from '$lib/components/admin-page/settings/map-settings/map-settings.svelte';
import MetadataSettings from '$lib/components/admin-page/settings/metadata-settings/metadata-settings.svelte';
import NewVersionCheckSettings from '$lib/components/admin-page/settings/new-version-check-settings/new-version-check-settings.svelte';
+ import NightlyTasksSettings from '$lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.svelte';
import NotificationSettings from '$lib/components/admin-page/settings/notification-settings/notification-settings.svelte';
import ServerSettings from '$lib/components/admin-page/settings/server/server-settings.svelte';
import StorageTemplateSettings from '$lib/components/admin-page/settings/storage-template/storage-template-settings.svelte';
@@ -33,6 +34,7 @@
mdiBackupRestore,
mdiBellOutline,
mdiBookshelf,
+ mdiClockOutline,
mdiContentCopy,
mdiDatabaseOutline,
mdiDownload,
@@ -136,13 +138,6 @@
key: 'job',
icon: mdiSync,
},
- {
- component: MetadataSettings,
- title: $t('admin.metadata_settings'),
- subtitle: $t('admin.metadata_settings_description'),
- key: 'metadata',
- icon: mdiDatabaseOutline,
- },
{
component: LibrarySettings,
title: $t('admin.library_settings'),
@@ -171,6 +166,20 @@
key: 'location',
icon: mdiMapMarkerOutline,
},
+ {
+ component: MetadataSettings,
+ title: $t('admin.metadata_settings'),
+ subtitle: $t('admin.metadata_settings_description'),
+ key: 'metadata',
+ icon: mdiDatabaseOutline,
+ },
+ {
+ component: NightlyTasksSettings,
+ title: $t('admin.nightly_tasks_settings'),
+ subtitle: $t('admin.nightly_tasks_settings_description'),
+ key: 'nightly-tasks',
+ icon: mdiClockOutline,
+ },
{
component: NotificationSettings,
title: $t('admin.notification_settings'),