High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat: nightly tasks (#19879)
Jason Rasmussen47c0dc0d7ee5
diff --git a/e2e/src/api/specs/system-config.e2e-spec.ts b/e2e/src/api/specs/system-config.e2e-spec.tsindex 060163d7c9..1bd7bdc489 100644--- a/e2e/src/api/specs/system-config.e2e-spec.ts+++ b/e2e/src/api/specs/system-config.e2e-spec.ts@@ -15,12 +15,6 @@ describe('/system-config', () => {});describe('PUT /system-config', () => {- it('should require authentication', async () => {- const { status, body } = await request(app).put('/system-config');- expect(status).toBe(401);- expect(body).toEqual(errorDto.unauthorized);- });-it('should always return the new config', async () => {const config = await getSystemConfig(admin.accessToken);diff --git a/i18n/en.json b/i18n/en.jsonindex 06b3a5d599..f547a4e48d 100644--- a/i18n/en.json+++ b/i18n/en.json@@ -166,6 +166,20 @@"metadata_settings_description": "Manage metadata settings","migration_job": "Migration","migration_job_description": "Migrate thumbnails for assets and faces to the latest folder structure",+ "nightly_tasks_cluster_faces_setting_description": "Run facial recognition on newly detected faces",+ "nightly_tasks_cluster_new_faces_setting": "Cluster new faces",+ "nightly_tasks_database_cleanup_setting": "Database cleanup tasks",+ "nightly_tasks_database_cleanup_setting_description": "Clean up old, expired data from the database",+ "nightly_tasks_generate_memories_setting": "Generate memories",+ "nightly_tasks_generate_memories_setting_description": "Create new memories from assets",+ "nightly_tasks_missing_thumbnails_setting": "Generate missing thumbnails",+ "nightly_tasks_missing_thumbnails_setting_description": "Queue assets without thumbnails for thumbnail generation",+ "nightly_tasks_settings": "Nightly Tasks Settings",+ "nightly_tasks_settings_description": "Manage nightly tasks",+ "nightly_tasks_start_time_setting": "Start time",+ "nightly_tasks_start_time_setting_description": "The time at which the server starts running the nightly tasks",+ "nightly_tasks_sync_quota_usage_setting": "Sync quota usage",+ "nightly_tasks_sync_quota_usage_setting_description": "Update user storage quota, based on current usage","no_paths_added": "No paths added","no_pattern_added": "No pattern added","note_apply_storage_label_previous_assets": "Note: To apply the Storage Label to previously uploaded assets, run the",diff --git a/mobile/openapi/README.md b/mobile/openapi/README.mdindex 5085052c05..28fa63ba84 100644--- a/mobile/openapi/README.md+++ b/mobile/openapi/README.md@@ -509,6 +509,7 @@ Class | Method | HTTP request | Description- [SystemConfigMapDto](doc//SystemConfigMapDto.md)- [SystemConfigMetadataDto](doc//SystemConfigMetadataDto.md)- [SystemConfigNewVersionCheckDto](doc//SystemConfigNewVersionCheckDto.md)+ - [SystemConfigNightlyTasksDto](doc//SystemConfigNightlyTasksDto.md)- [SystemConfigNotificationsDto](doc//SystemConfigNotificationsDto.md)- [SystemConfigOAuthDto](doc//SystemConfigOAuthDto.md)- [SystemConfigPasswordLoginDto](doc//SystemConfigPasswordLoginDto.md)diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dartindex f30481ecce..becafa06bf 100644--- a/mobile/openapi/lib/api.dart+++ b/mobile/openapi/lib/api.dart@@ -291,6 +291,7 @@ part 'model/system_config_machine_learning_dto.dart';part 'model/system_config_map_dto.dart';part 'model/system_config_metadata_dto.dart';part 'model/system_config_new_version_check_dto.dart';+part 'model/system_config_nightly_tasks_dto.dart';part 'model/system_config_notifications_dto.dart';part 'model/system_config_o_auth_dto.dart';part 'model/system_config_password_login_dto.dart';diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dartindex f1cf05f110..603163f00e 100644--- a/mobile/openapi/lib/api_client.dart+++ b/mobile/openapi/lib/api_client.dart@@ -638,6 +638,8 @@ class ApiClient {return SystemConfigMetadataDto.fromJson(value);case 'SystemConfigNewVersionCheckDto':return SystemConfigNewVersionCheckDto.fromJson(value);+ case 'SystemConfigNightlyTasksDto':+ return SystemConfigNightlyTasksDto.fromJson(value);case 'SystemConfigNotificationsDto':return SystemConfigNotificationsDto.fromJson(value);case 'SystemConfigOAuthDto':diff --git a/mobile/openapi/lib/model/system_config_dto.dart b/mobile/openapi/lib/model/system_config_dto.dartindex 59d5f09fc9..38dbb30f0c 100644--- a/mobile/openapi/lib/model/system_config_dto.dart+++ b/mobile/openapi/lib/model/system_config_dto.dart@@ -23,6 +23,7 @@ class SystemConfigDto {required this.map,required this.metadata,required this.newVersionCheck,+ required this.nightlyTasks,required this.notifications,required this.oauth,required this.passwordLogin,@@ -55,6 +56,8 @@ class SystemConfigDto {SystemConfigNewVersionCheckDto newVersionCheck;+ SystemConfigNightlyTasksDto nightlyTasks;+SystemConfigNotificationsDto notifications;SystemConfigOAuthDto oauth;@@ -87,6 +90,7 @@ class SystemConfigDto {other.map == map &&other.metadata == metadata &&other.newVersionCheck == newVersionCheck &&+ other.nightlyTasks == nightlyTasks &&other.notifications == notifications &&other.oauth == oauth &&other.passwordLogin == passwordLogin &&@@ -111,6 +115,7 @@ class SystemConfigDto {(map.hashCode) +(metadata.hashCode) +(newVersionCheck.hashCode) ++ (nightlyTasks.hashCode) +(notifications.hashCode) +(oauth.hashCode) +(passwordLogin.hashCode) +@@ -123,7 +128,7 @@ class SystemConfigDto {(user.hashCode);@override- String toString() => 'SystemConfigDto[backup=$backup, ffmpeg=$ffmpeg, image=$image, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, templates=$templates, theme=$theme, trash=$trash, user=$user]';+ String toString() => 'SystemConfigDto[backup=$backup, ffmpeg=$ffmpeg, image=$image, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, nightlyTasks=$nightlyTasks, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, templates=$templates, theme=$theme, trash=$trash, user=$user]';Map<String, dynamic> toJson() {final json = <String, dynamic>{};@@ -137,6 +142,7 @@ class SystemConfigDto {json[r'map'] = this.map;json[r'metadata'] = this.metadata;json[r'newVersionCheck'] = this.newVersionCheck;+ json[r'nightlyTasks'] = this.nightlyTasks;json[r'notifications'] = this.notifications;json[r'oauth'] = this.oauth;json[r'passwordLogin'] = this.passwordLogin;@@ -169,6 +175,7 @@ class SystemConfigDto {map: SystemConfigMapDto.fromJson(json[r'map'])!,metadata: SystemConfigMetadataDto.fromJson(json[r'metadata'])!,newVersionCheck: SystemConfigNewVersionCheckDto.fromJson(json[r'newVersionCheck'])!,+ nightlyTasks: SystemConfigNightlyTasksDto.fromJson(json[r'nightlyTasks'])!,notifications: SystemConfigNotificationsDto.fromJson(json[r'notifications'])!,oauth: SystemConfigOAuthDto.fromJson(json[r'oauth'])!,passwordLogin: SystemConfigPasswordLoginDto.fromJson(json[r'passwordLogin'])!,@@ -236,6 +243,7 @@ class SystemConfigDto {'map','metadata','newVersionCheck',+ 'nightlyTasks','notifications','oauth','passwordLogin',diff --git a/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart b/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dartnew file mode 100644index 0000000000..ab7b4b37c2--- /dev/null+++ b/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart@@ -0,0 +1,139 @@+//+// AUTO-GENERATED FILE, DO NOT MODIFY!+//+// @dart=2.18++// ignore_for_file: unused_element, unused_import+// ignore_for_file: always_put_required_named_parameters_first+// ignore_for_file: constant_identifier_names+// ignore_for_file: lines_longer_than_80_chars++part of openapi.api;++class SystemConfigNightlyTasksDto {+ /// Returns a new [SystemConfigNightlyTasksDto] instance.+ SystemConfigNightlyTasksDto({+ required this.clusterNewFaces,+ required this.databaseCleanup,+ required this.generateMemories,+ required this.missingThumbnails,+ required this.startTime,+ required this.syncQuotaUsage,+ });++ bool clusterNewFaces;++ bool databaseCleanup;++ bool generateMemories;++ bool missingThumbnails;++ String startTime;++ bool syncQuotaUsage;++ @override+ bool operator ==(Object other) => identical(this, other) || other is SystemConfigNightlyTasksDto &&+ other.clusterNewFaces == clusterNewFaces &&+ other.databaseCleanup == databaseCleanup &&+ other.generateMemories == generateMemories &&+ other.missingThumbnails == missingThumbnails &&+ other.startTime == startTime &&+ other.syncQuotaUsage == syncQuotaUsage;++ @override+ int get hashCode =>+ // ignore: unnecessary_parenthesis+ (clusterNewFaces.hashCode) ++ (databaseCleanup.hashCode) ++ (generateMemories.hashCode) ++ (missingThumbnails.hashCode) ++ (startTime.hashCode) ++ (syncQuotaUsage.hashCode);++ @override+ String toString() => 'SystemConfigNightlyTasksDto[clusterNewFaces=$clusterNewFaces, databaseCleanup=$databaseCleanup, generateMemories=$generateMemories, missingThumbnails=$missingThumbnails, startTime=$startTime, syncQuotaUsage=$syncQuotaUsage]';++ Map<String, dynamic> toJson() {+ final json = <String, dynamic>{};+ json[r'clusterNewFaces'] = this.clusterNewFaces;+ json[r'databaseCleanup'] = this.databaseCleanup;+ json[r'generateMemories'] = this.generateMemories;+ json[r'missingThumbnails'] = this.missingThumbnails;+ json[r'startTime'] = this.startTime;+ json[r'syncQuotaUsage'] = this.syncQuotaUsage;+ return json;+ }++ /// Returns a new [SystemConfigNightlyTasksDto] instance and imports its values from+ /// [value] if it's a [Map], null otherwise.+ // ignore: prefer_constructors_over_static_methods+ static SystemConfigNightlyTasksDto? fromJson(dynamic value) {+ upgradeDto(value, "SystemConfigNightlyTasksDto");+ if (value is Map) {+ final json = value.cast<String, dynamic>();++ return SystemConfigNightlyTasksDto(+ clusterNewFaces: mapValueOfType<bool>(json, r'clusterNewFaces')!,+ databaseCleanup: mapValueOfType<bool>(json, r'databaseCleanup')!,+ generateMemories: mapValueOfType<bool>(json, r'generateMemories')!,+ missingThumbnails: mapValueOfType<bool>(json, r'missingThumbnails')!,+ startTime: mapValueOfType<String>(json, r'startTime')!,+ syncQuotaUsage: mapValueOfType<bool>(json, r'syncQuotaUsage')!,+ );+ }+ return null;+ }++ static List<SystemConfigNightlyTasksDto> listFromJson(dynamic json, {bool growable = false,}) {+ final result = <SystemConfigNightlyTasksDto>[];+ if (json is List && json.isNotEmpty) {+ for (final row in json) {+ final value = SystemConfigNightlyTasksDto.fromJson(row);+ if (value != null) {+ result.add(value);+ }+ }+ }+ return result.toList(growable: growable);+ }++ static Map<String, SystemConfigNightlyTasksDto> mapFromJson(dynamic json) {+ final map = <String, SystemConfigNightlyTasksDto>{};+ if (json is Map && json.isNotEmpty) {+ json = json.cast<String, dynamic>(); // ignore: parameter_assignments+ for (final entry in json.entries) {+ final value = SystemConfigNightlyTasksDto.fromJson(entry.value);+ if (value != null) {+ map[entry.key] = value;+ }+ }+ }+ return map;+ }++ // maps a json object with a list of SystemConfigNightlyTasksDto-objects as value to a dart map+ static Map<String, List<SystemConfigNightlyTasksDto>> mapListFromJson(dynamic json, {bool growable = false,}) {+ final map = <String, List<SystemConfigNightlyTasksDto>>{};+ if (json is Map && json.isNotEmpty) {+ // ignore: parameter_assignments+ json = json.cast<String, dynamic>();+ for (final entry in json.entries) {+ map[entry.key] = SystemConfigNightlyTasksDto.listFromJson(entry.value, growable: growable,);+ }+ }+ return map;+ }++ /// The list of required keys that must be present in a JSON.+ static const requiredKeys = <String>{+ 'clusterNewFaces',+ 'databaseCleanup',+ 'generateMemories',+ 'missingThumbnails',+ 'startTime',+ 'syncQuotaUsage',+ };+}+diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.jsonindex 18204c21f3..492d3cfec1 100644--- a/open-api/immich-openapi-specs.json+++ b/open-api/immich-openapi-specs.json@@ -14318,6 +14318,9 @@"newVersionCheck": {"$ref": "#/components/schemas/SystemConfigNewVersionCheckDto"},+ "nightlyTasks": {+ "$ref": "#/components/schemas/SystemConfigNightlyTasksDto"+ },"notifications": {"$ref": "#/components/schemas/SystemConfigNotificationsDto"},@@ -14360,6 +14363,7 @@"map","metadata","newVersionCheck",+ "nightlyTasks","notifications","oauth","passwordLogin",@@ -14790,6 +14794,37 @@],"type": "object"},+ "SystemConfigNightlyTasksDto": {+ "properties": {+ "clusterNewFaces": {+ "type": "boolean"+ },+ "databaseCleanup": {+ "type": "boolean"+ },+ "generateMemories": {+ "type": "boolean"+ },+ "missingThumbnails": {+ "type": "boolean"+ },+ "startTime": {+ "type": "string"+ },+ "syncQuotaUsage": {+ "type": "boolean"+ }+ },+ "required": [+ "clusterNewFaces",+ "databaseCleanup",+ "generateMemories",+ "missingThumbnails",+ "startTime",+ "syncQuotaUsage"+ ],+ "type": "object"+ },"SystemConfigNotificationsDto": {"properties": {"smtp": {diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.tsindex 88dee9bf0b..f60fa6dfe8 100644--- a/open-api/typescript-sdk/src/fetch-client.ts+++ b/open-api/typescript-sdk/src/fetch-client.ts@@ -1389,6 +1389,14 @@ export type SystemConfigMetadataDto = {export type SystemConfigNewVersionCheckDto = {enabled: boolean;};+export type SystemConfigNightlyTasksDto = {+ clusterNewFaces: boolean;+ databaseCleanup: boolean;+ generateMemories: boolean;+ missingThumbnails: boolean;+ startTime: string;+ syncQuotaUsage: boolean;+};export type SystemConfigNotificationsDto = {smtp: SystemConfigSmtpDto;};@@ -1457,6 +1465,7 @@ export type SystemConfigDto = {map: SystemConfigMapDto;metadata: SystemConfigMetadataDto;newVersionCheck: SystemConfigNewVersionCheckDto;+ nightlyTasks: SystemConfigNightlyTasksDto;notifications: SystemConfigNotificationsDto;oauth: SystemConfigOAuthDto;passwordLogin: SystemConfigPasswordLoginDto;diff --git a/server/src/config.ts b/server/src/config.tsindex 1fcc2e9782..90ca2c1529 100644--- a/server/src/config.ts+++ b/server/src/config.ts@@ -121,6 +121,14 @@ export interface SystemConfig {newVersionCheck: {enabled: boolean;};+ nightlyTasks: {+ startTime: string;+ databaseCleanup: boolean;+ missingThumbnails: boolean;+ clusterNewFaces: boolean;+ generateMemories: boolean;+ syncQuotaUsage: boolean;+ };trash: {enabled: boolean;days: number;@@ -298,6 +306,14 @@ export const defaults = Object.freeze<SystemConfig>({newVersionCheck: {enabled: true,},+ nightlyTasks: {+ startTime: '00:00',+ databaseCleanup: true,+ generateMemories: true,+ syncQuotaUsage: true,+ missingThumbnails: true,+ clusterNewFaces: true,+ },trash: {enabled: true,days: 30,diff --git a/server/src/controllers/system-config.controller.spec.ts b/server/src/controllers/system-config.controller.spec.tsnew file mode 100644index 0000000000..48b8c1bcf0--- /dev/null+++ b/server/src/controllers/system-config.controller.spec.ts@@ -0,0 +1,74 @@+import _ from 'lodash';+import { defaults } from 'src/config';+import { SystemConfigController } from 'src/controllers/system-config.controller';+import { StorageTemplateService } from 'src/services/storage-template.service';+import { SystemConfigService } from 'src/services/system-config.service';+import request from 'supertest';+import { errorDto } from 'test/medium/responses';+import { ControllerContext, controllerSetup, mockBaseService } from 'test/utils';++describe(SystemConfigController.name, () => {+ let ctx: ControllerContext;+ const systemConfigService = mockBaseService(SystemConfigService);+ const templateService = mockBaseService(StorageTemplateService);++ beforeAll(async () => {+ ctx = await controllerSetup(SystemConfigController, [+ { provide: SystemConfigService, useValue: systemConfigService },+ { provide: StorageTemplateService, useValue: templateService },+ ]);+ return () => ctx.close();+ });++ beforeEach(() => {+ systemConfigService.resetAllMocks();+ templateService.resetAllMocks();+ ctx.reset();+ });++ describe('GET /system-config', () => {+ it('should be an authenticated route', async () => {+ await request(ctx.getHttpServer()).get('/system-config');+ expect(ctx.authenticate).toHaveBeenCalled();+ });+ });++ describe('GET /system-config/defaults', () => {+ it('should be an authenticated route', async () => {+ await request(ctx.getHttpServer()).get('/system-config/defaults');+ expect(ctx.authenticate).toHaveBeenCalled();+ });+ });++ describe('PUT /system-config', () => {+ it('should be an authenticated route', async () => {+ await request(ctx.getHttpServer()).put('/system-config');+ expect(ctx.authenticate).toHaveBeenCalled();+ });++ describe('nightlyTasks', () => {+ it('should validate nightly jobs start time', async () => {+ const config = _.cloneDeep(defaults);+ config.nightlyTasks.startTime = 'invalid';+ const { status, body } = await request(ctx.getHttpServer()).put('/system-config').send(config);+ expect(status).toBe(400);+ expect(body).toEqual(errorDto.badRequest(['nightlyTasks.startTime must be in HH:mm format']));+ });++ it('should accept a valid time', async () => {+ const config = _.cloneDeep(defaults);+ config.nightlyTasks.startTime = '05:05';+ const { status } = await request(ctx.getHttpServer()).put('/system-config').send(config);+ expect(status).toBe(200);+ });++ it('should validate a boolean field', async () => {+ const config = _.cloneDeep(defaults);+ (config.nightlyTasks.databaseCleanup as any) = 'invalid';+ const { status, body } = await request(ctx.getHttpServer()).put('/system-config').send(config);+ expect(status).toBe(400);+ expect(body).toEqual(errorDto.badRequest(['nightlyTasks.databaseCleanup must be a boolean value']));+ });+ });+ });+});diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.tsindex b0385984b4..49c5e5b4e7 100644--- a/server/src/dtos/system-config.dto.ts+++ b/server/src/dtos/system-config.dto.ts@@ -34,7 +34,7 @@ import {VideoContainer,} from 'src/enum';import { ConcurrentQueueName } from 'src/types';-import { IsCronExpression, Optional, ValidateBoolean } from 'src/validation';+import { IsCronExpression, IsDateStringFormat, Optional, ValidateBoolean } from 'src/validation';const isLibraryScanEnabled = (config: SystemConfigLibraryScanDto) => config.enabled;const isOAuthEnabled = (config: SystemConfigOAuthDto) => config.enabled;@@ -329,6 +329,26 @@ class SystemConfigNewVersionCheckDto {enabled!: boolean;}+class SystemConfigNightlyTasksDto {+ @IsDateStringFormat('HH:mm', { message: 'startTime must be in HH:mm format' })+ startTime!: string;++ @ValidateBoolean()+ databaseCleanup!: boolean;++ @ValidateBoolean()+ missingThumbnails!: boolean;++ @ValidateBoolean()+ clusterNewFaces!: boolean;++ @ValidateBoolean()+ generateMemories!: boolean;++ @ValidateBoolean()+ syncQuotaUsage!: boolean;+}+class SystemConfigOAuthDto {@ValidateBoolean()autoLaunch!: boolean;@@ -638,6 +658,11 @@ export class SystemConfigDto implements SystemConfig {@IsObject()newVersionCheck!: SystemConfigNewVersionCheckDto;+ @Type(() => SystemConfigNightlyTasksDto)+ @ValidateNested()+ @IsObject()+ nightlyTasks!: SystemConfigNightlyTasksDto;+@Type(() => SystemConfigOAuthDto)@ValidateNested()@IsObject()diff --git a/server/src/enum.ts b/server/src/enum.tsindex dca0f09555..d7c74a71c6 100644--- a/server/src/enum.ts+++ b/server/src/enum.ts@@ -567,6 +567,7 @@ export enum DatabaseLock {VersionHistory = 500,CLIPDimSize = 512,Library = 1337,+ NightlyJobs = 600,GetSystemConfig = 69,BackupDatabase = 42,MemoryCreation = 777,@@ -684,3 +685,8 @@ export enum AssetVisibility {HIDDEN = 'hidden',LOCKED = 'locked',}++export enum CronJob {+ LibraryScan = 'LibraryScan',+ NightlyJobs = 'NightlyJobs',+}diff --git a/server/src/services/api.service.ts b/server/src/services/api.service.tsindex 064b2e268d..27a776e867 100644--- a/server/src/services/api.service.ts+++ b/server/src/services/api.service.ts@@ -1,5 +1,5 @@import { Injectable } from '@nestjs/common';-import { Cron, CronExpression, Interval } from '@nestjs/schedule';+import { Interval } from '@nestjs/schedule';import { NextFunction, Request, Response } from 'express';import { readFileSync } from 'node:fs';import sanitizeHtml from 'sanitize-html';@@ -54,11 +54,6 @@ export class ApiService {await this.versionService.handleQueueVersionCheck();}- @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)- async onNightlyJob() {- await this.jobService.handleNightlyJobs();- }-ssr(excludePaths: string[]) {const { resourcePaths } = this.configRepository.getEnv();diff --git a/server/src/services/job.service.spec.ts b/server/src/services/job.service.spec.tsindex c9020ed96a..a18eccdd8b 100644--- a/server/src/services/job.service.spec.ts+++ b/server/src/services/job.service.spec.ts@@ -41,12 +41,12 @@ describe(JobService.name, () => {{ name: JobName.USER_DELETE_CHECK },{ name: JobName.PERSON_CLEANUP },{ name: JobName.MEMORIES_CLEANUP },- { name: JobName.MEMORIES_CREATE },- { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },+ { name: JobName.CLEAN_OLD_SESSION_TOKENS },{ name: JobName.CLEAN_OLD_AUDIT_LOGS },+ { name: JobName.MEMORIES_CREATE },{ name: JobName.USER_SYNC_USAGE },+ { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },{ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },- { name: JobName.CLEAN_OLD_SESSION_TOKENS },]);});});diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.tsindex e4b20ba37f..645b22249f 100644--- a/server/src/services/job.service.ts+++ b/server/src/services/job.service.ts@@ -1,6 +1,7 @@import { BadRequestException, Injectable } from '@nestjs/common';import { ClassConstructor } from 'class-transformer';import { snakeCase } from 'lodash';+import { SystemConfig } from 'src/config';import { OnEvent } from 'src/decorators';import { mapAsset } from 'src/dtos/asset-response.dto';import { AllJobStatusResponseDto, JobCommandDto, JobCreateDto, JobStatusDto } from 'src/dtos/job.dto';@@ -8,6 +9,8 @@ import {AssetType,AssetVisibility,BootstrapEventPriority,+ CronJob,+ DatabaseLock,ImmichWorker,JobCommand,JobName,@@ -20,6 +23,7 @@ import { ArgOf, ArgsOf } from 'src/repositories/event.repository';import { BaseService } from 'src/services/base.service';import { ConcurrentQueueName, JobItem } from 'src/types';import { hexOrBufferToBase64 } from 'src/utils/bytes';+import { handlePromiseError } from 'src/utils/misc';const asJobItem = (dto: JobCreateDto): JobItem => {switch (dto.name) {@@ -53,26 +57,48 @@ const asJobItem = (dto: JobCreateDto): JobItem => {}};+const asNightlyTasksCron = (config: SystemConfig) => {+ const [hours, minutes] = config.nightlyTasks.startTime.split(':').map(Number);+ return `${minutes} ${hours} * * *`;+};+@Injectable()export class JobService extends BaseService {private services: ClassConstructor<unknown>[] = [];+ private nightlyJobsLock = false;- @OnEvent({ name: 'config.init', workers: [ImmichWorker.MICROSERVICES] })- onConfigInit({ newConfig: config }: ArgOf<'config.init'>) {- this.logger.debug(`Updating queue concurrency settings`);- for (const queueName of Object.values(QueueName)) {- let concurrency = 1;- if (this.isConcurrentQueue(queueName)) {- concurrency = config.job[queueName].concurrency;- }- this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);- this.jobRepository.setConcurrency(queueName, concurrency);+ @OnEvent({ name: 'config.init' })+ async onConfigInit({ newConfig: config }: ArgOf<'config.init'>) {+ if (this.worker === ImmichWorker.MICROSERVICES) {+ this.updateQueueConcurrency(config);+ return;+ }++ this.nightlyJobsLock = await this.databaseRepository.tryLock(DatabaseLock.NightlyJobs);+ if (this.nightlyJobsLock) {+ const cronExpression = asNightlyTasksCron(config);+ this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`);+ this.cronRepository.create({+ name: CronJob.NightlyJobs,+ expression: cronExpression,+ start: true,+ onTick: () => handlePromiseError(this.handleNightlyJobs(), this.logger),+ });}}- @OnEvent({ name: 'config.update', server: true, workers: [ImmichWorker.MICROSERVICES] })+ @OnEvent({ name: 'config.update', server: true })onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) {- this.onConfigInit({ newConfig: config });+ if (this.worker === ImmichWorker.MICROSERVICES) {+ this.updateQueueConcurrency(config);+ return;+ }++ if (this.nightlyJobsLock) {+ const cronExpression = asNightlyTasksCron(config);+ this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`);+ this.cronRepository.update({ name: CronJob.NightlyJobs, expression: cronExpression, start: true });+ }}@OnEvent({ name: 'app.bootstrap', priority: BootstrapEventPriority.JobService })@@ -83,6 +109,18 @@ export class JobService extends BaseService {}}+ private updateQueueConcurrency(config: SystemConfig) {+ this.logger.debug(`Updating queue concurrency settings`);+ for (const queueName of Object.values(QueueName)) {+ let concurrency = 1;+ if (this.isConcurrentQueue(queueName)) {+ concurrency = config.job[queueName].concurrency;+ }+ this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);+ this.jobRepository.setConcurrency(queueName, concurrency);+ }+ }+setServices(services: ClassConstructor<unknown>[]) {this.services = services;}@@ -233,18 +271,37 @@ export class JobService extends BaseService {}async handleNightlyJobs() {- await this.jobRepository.queueAll([- { name: JobName.ASSET_DELETION_CHECK },- { name: JobName.USER_DELETE_CHECK },- { name: JobName.PERSON_CLEANUP },- { name: JobName.MEMORIES_CLEANUP },- { name: JobName.MEMORIES_CREATE },- { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },- { name: JobName.CLEAN_OLD_AUDIT_LOGS },- { name: JobName.USER_SYNC_USAGE },- { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },- { name: JobName.CLEAN_OLD_SESSION_TOKENS },- ]);+ const config = await this.getConfig({ withCache: false });+ const jobs: JobItem[] = [];++ if (config.nightlyTasks.databaseCleanup) {+ jobs.push(+ { name: JobName.ASSET_DELETION_CHECK },+ { name: JobName.USER_DELETE_CHECK },+ { name: JobName.PERSON_CLEANUP },+ { name: JobName.MEMORIES_CLEANUP },+ { name: JobName.CLEAN_OLD_SESSION_TOKENS },+ { name: JobName.CLEAN_OLD_AUDIT_LOGS },+ );+ }++ if (config.nightlyTasks.generateMemories) {+ jobs.push({ name: JobName.MEMORIES_CREATE });+ }++ if (config.nightlyTasks.syncQuotaUsage) {+ jobs.push({ name: JobName.USER_SYNC_USAGE });+ }++ if (config.nightlyTasks.missingThumbnails) {+ jobs.push({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });+ }++ if (config.nightlyTasks.clusterNewFaces) {+ jobs.push({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } });+ }++ await this.jobRepository.queueAll(jobs);}/**diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.tsindex e92cdcf200..ab69e22b99 100644--- a/server/src/services/library.service.spec.ts+++ b/server/src/services/library.service.spec.ts@@ -3,7 +3,7 @@ import { Stats } from 'node:fs';import { defaults, SystemConfig } from 'src/config';import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants';import { mapLibrary } from 'src/dtos/library.dto';-import { AssetType, ImmichWorker, JobName, JobStatus } from 'src/enum';+import { AssetType, CronJob, ImmichWorker, JobName, JobStatus } from 'src/enum';import { LibraryService } from 'src/services/library.service';import { ILibraryBulkIdsJob, ILibraryFileJob } from 'src/types';import { assetStub } from 'test/fixtures/asset.stub';@@ -56,7 +56,11 @@ describe(LibraryService.name, () => {} as SystemConfig,});- expect(mocks.cron.update).toHaveBeenCalledWith({ name: 'libraryScan', expression: '0 1 * * *', start: true });+ expect(mocks.cron.update).toHaveBeenCalledWith({+ name: CronJob.LibraryScan,+ expression: '0 1 * * *',+ start: true,+ });});it('should initialize watcher for all external libraries', async () => {@@ -128,7 +132,7 @@ describe(LibraryService.name, () => {});expect(mocks.cron.update).toHaveBeenCalledWith({- name: 'libraryScan',+ name: CronJob.LibraryScan,expression: systemConfigStub.libraryScan.library.scan.cronExpression,start: systemConfigStub.libraryScan.library.scan.enabled,});@@ -149,7 +153,7 @@ describe(LibraryService.name, () => {});expect(mocks.cron.update).toHaveBeenCalledWith({- name: 'libraryScan',+ name: CronJob.LibraryScan,expression: systemConfigStub.libraryScan.library.scan.cronExpression,start: systemConfigStub.libraryScan.library.scan.enabled,});diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.tsindex 8c7f79ee5c..f6bc5b2ebb 100644--- a/server/src/services/library.service.ts+++ b/server/src/services/library.service.ts@@ -17,7 +17,7 @@ import {ValidateLibraryImportPathResponseDto,ValidateLibraryResponseDto,} from 'src/dtos/library.dto';-import { AssetStatus, AssetType, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum';+import { AssetStatus, AssetType, CronJob, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum';import { ArgOf } from 'src/repositories/event.repository';import { AssetSyncResult } from 'src/repositories/library.repository';import { AssetTable } from 'src/schema/tables/asset.table';@@ -45,7 +45,7 @@ export class LibraryService extends BaseService {if (this.lock) {this.cronRepository.create({- name: 'libraryScan',+ name: CronJob.LibraryScan,expression: scan.cronExpression,onTick: () =>handlePromiseError(this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL }), this.logger),@@ -65,7 +65,7 @@ export class LibraryService extends BaseService {}this.cronRepository.update({- name: 'libraryScan',+ name: CronJob.LibraryScan,expression: library.scan.cronExpression,start: library.scan.enabled,});diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.tsindex c7b98cc990..43be323459 100644--- a/server/src/services/system-config.service.spec.ts+++ b/server/src/services/system-config.service.spec.ts@@ -103,6 +103,14 @@ const updatedConfig = Object.freeze<SystemConfig>({lightStyle: 'https://tiles.immich.cloud/v1/style/light.json',darkStyle: 'https://tiles.immich.cloud/v1/style/dark.json',},+ nightlyTasks: {+ startTime: '00:00',+ databaseCleanup: true,+ clusterNewFaces: true,+ missingThumbnails: true,+ generateMemories: true,+ syncQuotaUsage: true,+ },reverseGeocoding: {enabled: true,},diff --git a/web/src/lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.svelte b/web/src/lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.sveltenew file mode 100644index 0000000000..af653f2473--- /dev/null+++ b/web/src/lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.svelte@@ -0,0 +1,81 @@+<script lang="ts">+ import SettingButtonsRow from '$lib/components/shared-components/settings/setting-buttons-row.svelte';+ import SettingInputField from '$lib/components/shared-components/settings/setting-input-field.svelte';+ import SettingSwitch from '$lib/components/shared-components/settings/setting-switch.svelte';+ import { SettingInputFieldType } from '$lib/constants';+ import type { SystemConfigDto } from '@immich/sdk';+ import { isEqual } from 'lodash-es';+ import { t } from 'svelte-i18n';+ import { fade } from 'svelte/transition';+ import type { SettingsResetEvent, SettingsSaveEvent } from '../admin-settings';++ interface Props {+ savedConfig: SystemConfigDto;+ defaultConfig: SystemConfigDto;+ config: SystemConfigDto;+ disabled?: boolean;+ onReset: SettingsResetEvent;+ onSave: SettingsSaveEvent;+ }++ let { savedConfig, defaultConfig, config = $bindable(), disabled = false, onReset, onSave }: Props = $props();++ const onsubmit = (event: Event) => {+ event.preventDefault();+ };+</script>++<div class="mt-2">+ <div in:fade={{ duration: 500 }}>+ <form autocomplete="off" {onsubmit} class="mx-4 mt-4">+ <div class="ms-4 mt-4 flex flex-col gap-4">+ <SettingInputField+ inputType={SettingInputFieldType.TEXT}+ label={$t('admin.nightly_tasks_start_time_setting')}+ description={$t('admin.nightly_tasks_start_time_setting_description')}+ bind:value={config.nightlyTasks.startTime}+ required={true}+ {disabled}+ isEdited={!(config.nightlyTasks.startTime === savedConfig.nightlyTasks.startTime)}+ />+ <SettingSwitch+ title={$t('admin.nightly_tasks_database_cleanup_setting')}+ subtitle={$t('admin.nightly_tasks_database_cleanup_setting_description')}+ bind:checked={config.nightlyTasks.databaseCleanup}+ {disabled}+ />+ <SettingSwitch+ title={$t('admin.nightly_tasks_missing_thumbnails_setting')}+ subtitle={$t('admin.nightly_tasks_missing_thumbnails_setting_description')}+ bind:checked={config.nightlyTasks.missingThumbnails}+ {disabled}+ />+ <SettingSwitch+ title={$t('admin.nightly_tasks_cluster_new_faces_setting')}+ subtitle={$t('admin.nightly_tasks_cluster_faces_setting_description')}+ bind:checked={config.nightlyTasks.clusterNewFaces}+ {disabled}+ />+ <SettingSwitch+ title={$t('admin.nightly_tasks_generate_memories_setting')}+ subtitle={$t('admin.nightly_tasks_generate_memories_setting_description')}+ bind:checked={config.nightlyTasks.generateMemories}+ {disabled}+ />+ <SettingSwitch+ title={$t('admin.nightly_tasks_sync_quota_usage_setting')}+ subtitle={$t('admin.nightly_tasks_sync_quota_usage_setting_description')}+ bind:checked={config.nightlyTasks.syncQuotaUsage}+ {disabled}+ />+ </div>++ <SettingButtonsRow+ onReset={(options) => onReset({ ...options, configKeys: ['nightlyTasks'] })}+ onSave={() => onSave({ nightlyTasks: config.nightlyTasks })}+ showResetToDefault={!isEqual(savedConfig.nightlyTasks, defaultConfig.nightlyTasks)}+ {disabled}+ />+ </form>+ </div>+</div>diff --git a/web/src/routes/admin/system-settings/+page.svelte b/web/src/routes/admin/system-settings/+page.svelteindex e053eea179..3b1d68a49e 100644--- a/web/src/routes/admin/system-settings/+page.svelte+++ b/web/src/routes/admin/system-settings/+page.svelte@@ -12,6 +12,7 @@import MapSettings from '$lib/components/admin-page/settings/map-settings/map-settings.svelte';import MetadataSettings from '$lib/components/admin-page/settings/metadata-settings/metadata-settings.svelte';import NewVersionCheckSettings from '$lib/components/admin-page/settings/new-version-check-settings/new-version-check-settings.svelte';+ import NightlyTasksSettings from '$lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.svelte';import NotificationSettings from '$lib/components/admin-page/settings/notification-settings/notification-settings.svelte';import ServerSettings from '$lib/components/admin-page/settings/server/server-settings.svelte';import StorageTemplateSettings from '$lib/components/admin-page/settings/storage-template/storage-template-settings.svelte';@@ -33,6 +34,7 @@mdiBackupRestore,mdiBellOutline,mdiBookshelf,+ mdiClockOutline,mdiContentCopy,mdiDatabaseOutline,mdiDownload,@@ -136,13 +138,6 @@key: 'job',icon: mdiSync,},- {- component: MetadataSettings,- title: $t('admin.metadata_settings'),- subtitle: $t('admin.metadata_settings_description'),- key: 'metadata',- icon: mdiDatabaseOutline,- },{component: LibrarySettings,title: $t('admin.library_settings'),@@ -171,6 +166,20 @@key: 'location',icon: mdiMapMarkerOutline,},+ {+ component: MetadataSettings,+ title: $t('admin.metadata_settings'),+ subtitle: $t('admin.metadata_settings_description'),+ key: 'metadata',+ icon: mdiDatabaseOutline,+ },+ {+ component: NightlyTasksSettings,+ title: $t('admin.nightly_tasks_settings'),+ subtitle: $t('admin.nightly_tasks_settings_description'),+ key: 'nightly-tasks',+ icon: mdiClockOutline,+ },{component: NotificationSettings,title: $t('admin.notification_settings'),