High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat: availability checks (#22185)
Jason Rasmussen3f2e0780d545
diff --git a/docs/docs/install/environment-variables.md b/docs/docs/install/environment-variables.mdindex 928e0b26e5..4e081c8966 100644--- a/docs/docs/install/environment-variables.md+++ b/docs/docs/install/environment-variables.md@@ -169,8 +169,6 @@ Redis (Sentinel) URL example JSON before encoding:| `MACHINE_LEARNING_ANN_TUNING_LEVEL` | ARM-NN GPU tuning level (1: rapid, 2: normal, 3: exhaustive) | `2` | machine learning || `MACHINE_LEARNING_DEVICE_IDS`<sup>\*4</sup> | Device IDs to use in multi-GPU environments | `0` | machine learning || `MACHINE_LEARNING_MAX_BATCH_SIZE__FACIAL_RECOGNITION` | Set the maximum number of faces that will be processed at once by the facial recognition model | None (`1` if using OpenVINO) | machine learning |-| `MACHINE_LEARNING_PING_TIMEOUT` | How long (ms) to wait for a PING response when checking if an ML server is available | `2000` | server |-| `MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME` | How long to ignore ML servers that are offline before trying again | `30000` | server || `MACHINE_LEARNING_RKNN` | Enable RKNN hardware acceleration if supported | `True` | machine learning || `MACHINE_LEARNING_RKNN_THREADS` | How many threads of RKNN runtime should be spinned up while inferencing. | `1` | machine learning |diff --git a/i18n/en.json b/i18n/en.jsonindex aa1999adcb..af52770c64 100644--- a/i18n/en.json+++ b/i18n/en.json@@ -123,6 +123,13 @@"logging_enable_description": "Enable logging","logging_level_description": "When enabled, what log level to use.","logging_settings": "Logging",+ "machine_learning_availability_checks": "Availability checks",+ "machine_learning_availability_checks_description": "Automatically detect and prefer available machine learning servers",+ "machine_learning_availability_checks_enabled": "Enable availability checks",+ "machine_learning_availability_checks_interval": "Check interval",+ "machine_learning_availability_checks_interval_description": "Interval in milliseconds between availability checks",+ "machine_learning_availability_checks_timeout": "Request timeout",+ "machine_learning_availability_checks_timeout_description": "Timeout in milliseconds for availability checks","machine_learning_clip_model": "CLIP model","machine_learning_clip_model_description": "The name of a CLIP model listed <link>here</link>. Note that you must re-run the 'Smart Search' job for all images upon changing a model.","machine_learning_duplicate_detection": "Duplicate Detection",diff --git a/mobile/openapi/README.md b/mobile/openapi/README.mdindex 239c62449f..7a426b74fc 100644--- a/mobile/openapi/README.md+++ b/mobile/openapi/README.md@@ -393,6 +393,7 @@ Class | Method | HTTP request | Description- [LoginCredentialDto](doc//LoginCredentialDto.md)- [LoginResponseDto](doc//LoginResponseDto.md)- [LogoutResponseDto](doc//LogoutResponseDto.md)+ - [MachineLearningAvailabilityChecksDto](doc//MachineLearningAvailabilityChecksDto.md)- [ManualJobName](doc//ManualJobName.md)- [MapMarkerResponseDto](doc//MapMarkerResponseDto.md)- [MapReverseGeocodeResponseDto](doc//MapReverseGeocodeResponseDto.md)diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dartindex e87c160d96..df2c2226b1 100644--- a/mobile/openapi/lib/api.dart+++ b/mobile/openapi/lib/api.dart@@ -164,6 +164,7 @@ part 'model/log_level.dart';part 'model/login_credential_dto.dart';part 'model/login_response_dto.dart';part 'model/logout_response_dto.dart';+part 'model/machine_learning_availability_checks_dto.dart';part 'model/manual_job_name.dart';part 'model/map_marker_response_dto.dart';part 'model/map_reverse_geocode_response_dto.dart';diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dartindex ae5fd9227b..06d27593c9 100644--- a/mobile/openapi/lib/api_client.dart+++ b/mobile/openapi/lib/api_client.dart@@ -382,6 +382,8 @@ class ApiClient {return LoginResponseDto.fromJson(value);case 'LogoutResponseDto':return LogoutResponseDto.fromJson(value);+ case 'MachineLearningAvailabilityChecksDto':+ return MachineLearningAvailabilityChecksDto.fromJson(value);case 'ManualJobName':return ManualJobNameTypeTransformer().decode(value);case 'MapMarkerResponseDto':diff --git a/mobile/openapi/lib/model/machine_learning_availability_checks_dto.dart b/mobile/openapi/lib/model/machine_learning_availability_checks_dto.dartnew file mode 100644index 0000000000..84b3181426--- /dev/null+++ b/mobile/openapi/lib/model/machine_learning_availability_checks_dto.dart@@ -0,0 +1,115 @@+//+// AUTO-GENERATED FILE, DO NOT MODIFY!+//+// @dart=2.18++// ignore_for_file: unused_element, unused_import+// ignore_for_file: always_put_required_named_parameters_first+// ignore_for_file: constant_identifier_names+// ignore_for_file: lines_longer_than_80_chars++part of openapi.api;++class MachineLearningAvailabilityChecksDto {+ /// Returns a new [MachineLearningAvailabilityChecksDto] instance.+ MachineLearningAvailabilityChecksDto({+ required this.enabled,+ required this.interval,+ required this.timeout,+ });++ bool enabled;++ num interval;++ num timeout;++ @override+ bool operator ==(Object other) => identical(this, other) || other is MachineLearningAvailabilityChecksDto &&+ other.enabled == enabled &&+ other.interval == interval &&+ other.timeout == timeout;++ @override+ int get hashCode =>+ // ignore: unnecessary_parenthesis+ (enabled.hashCode) ++ (interval.hashCode) ++ (timeout.hashCode);++ @override+ String toString() => 'MachineLearningAvailabilityChecksDto[enabled=$enabled, interval=$interval, timeout=$timeout]';++ Map<String, dynamic> toJson() {+ final json = <String, dynamic>{};+ json[r'enabled'] = this.enabled;+ json[r'interval'] = this.interval;+ json[r'timeout'] = this.timeout;+ return json;+ }++ /// Returns a new [MachineLearningAvailabilityChecksDto] instance and imports its values from+ /// [value] if it's a [Map], null otherwise.+ // ignore: prefer_constructors_over_static_methods+ static MachineLearningAvailabilityChecksDto? fromJson(dynamic value) {+ upgradeDto(value, "MachineLearningAvailabilityChecksDto");+ if (value is Map) {+ final json = value.cast<String, dynamic>();++ return MachineLearningAvailabilityChecksDto(+ enabled: mapValueOfType<bool>(json, r'enabled')!,+ interval: num.parse('${json[r'interval']}'),+ timeout: num.parse('${json[r'timeout']}'),+ );+ }+ return null;+ }++ static List<MachineLearningAvailabilityChecksDto> listFromJson(dynamic json, {bool growable = false,}) {+ final result = <MachineLearningAvailabilityChecksDto>[];+ if (json is List && json.isNotEmpty) {+ for (final row in json) {+ final value = MachineLearningAvailabilityChecksDto.fromJson(row);+ if (value != null) {+ result.add(value);+ }+ }+ }+ return result.toList(growable: growable);+ }++ static Map<String, MachineLearningAvailabilityChecksDto> mapFromJson(dynamic json) {+ final map = <String, MachineLearningAvailabilityChecksDto>{};+ if (json is Map && json.isNotEmpty) {+ json = json.cast<String, dynamic>(); // ignore: parameter_assignments+ for (final entry in json.entries) {+ final value = MachineLearningAvailabilityChecksDto.fromJson(entry.value);+ if (value != null) {+ map[entry.key] = value;+ }+ }+ }+ return map;+ }++ // maps a json object with a list of MachineLearningAvailabilityChecksDto-objects as value to a dart map+ static Map<String, List<MachineLearningAvailabilityChecksDto>> mapListFromJson(dynamic json, {bool growable = false,}) {+ final map = <String, List<MachineLearningAvailabilityChecksDto>>{};+ if (json is Map && json.isNotEmpty) {+ // ignore: parameter_assignments+ json = json.cast<String, dynamic>();+ for (final entry in json.entries) {+ map[entry.key] = MachineLearningAvailabilityChecksDto.listFromJson(entry.value, growable: growable,);+ }+ }+ return map;+ }++ /// The list of required keys that must be present in a JSON.+ static const requiredKeys = <String>{+ 'enabled',+ 'interval',+ 'timeout',+ };+}+diff --git a/mobile/openapi/lib/model/system_config_machine_learning_dto.dart b/mobile/openapi/lib/model/system_config_machine_learning_dto.dartindex a4a9ca7d82..d7b2566d59 100644--- a/mobile/openapi/lib/model/system_config_machine_learning_dto.dart+++ b/mobile/openapi/lib/model/system_config_machine_learning_dto.dart@@ -13,14 +13,16 @@ part of openapi.api;class SystemConfigMachineLearningDto {/// Returns a new [SystemConfigMachineLearningDto] instance.SystemConfigMachineLearningDto({+ required this.availabilityChecks,required this.clip,required this.duplicateDetection,required this.enabled,required this.facialRecognition,- this.url,this.urls = const [],});+ MachineLearningAvailabilityChecksDto availabilityChecks;+CLIPConfig clip;DuplicateDetectionConfig duplicateDetection;@@ -29,50 +31,37 @@ class SystemConfigMachineLearningDto {FacialRecognitionConfig facialRecognition;- /// This property was deprecated in v1.122.0- ///- /// Please note: This property should have been non-nullable! Since the specification file- /// does not include a default value (using the "default:" property), however, the generated- /// source code must fall back to having a nullable type.- /// Consider adding a "default:" property in the specification file to hide this note.- ///- String? url;-List<String> urls;@overridebool operator ==(Object other) => identical(this, other) || other is SystemConfigMachineLearningDto &&+ other.availabilityChecks == availabilityChecks &&other.clip == clip &&other.duplicateDetection == duplicateDetection &&other.enabled == enabled &&other.facialRecognition == facialRecognition &&- other.url == url &&_deepEquality.equals(other.urls, urls);@overrideint get hashCode =>// ignore: unnecessary_parenthesis+ (availabilityChecks.hashCode) +(clip.hashCode) +(duplicateDetection.hashCode) +(enabled.hashCode) +(facialRecognition.hashCode) +- (url == null ? 0 : url!.hashCode) +(urls.hashCode);@override- String toString() => 'SystemConfigMachineLearningDto[clip=$clip, duplicateDetection=$duplicateDetection, enabled=$enabled, facialRecognition=$facialRecognition, url=$url, urls=$urls]';+ String toString() => 'SystemConfigMachineLearningDto[availabilityChecks=$availabilityChecks, clip=$clip, duplicateDetection=$duplicateDetection, enabled=$enabled, facialRecognition=$facialRecognition, urls=$urls]';Map<String, dynamic> toJson() {final json = <String, dynamic>{};+ json[r'availabilityChecks'] = this.availabilityChecks;json[r'clip'] = this.clip;json[r'duplicateDetection'] = this.duplicateDetection;json[r'enabled'] = this.enabled;json[r'facialRecognition'] = this.facialRecognition;- if (this.url != null) {- json[r'url'] = this.url;- } else {- // json[r'url'] = null;- }json[r'urls'] = this.urls;return json;}@@ -86,11 +75,11 @@ class SystemConfigMachineLearningDto {final json = value.cast<String, dynamic>();return SystemConfigMachineLearningDto(+ availabilityChecks: MachineLearningAvailabilityChecksDto.fromJson(json[r'availabilityChecks'])!,clip: CLIPConfig.fromJson(json[r'clip'])!,duplicateDetection: DuplicateDetectionConfig.fromJson(json[r'duplicateDetection'])!,enabled: mapValueOfType<bool>(json, r'enabled')!,facialRecognition: FacialRecognitionConfig.fromJson(json[r'facialRecognition'])!,- url: mapValueOfType<String>(json, r'url'),urls: json[r'urls'] is Iterable? (json[r'urls'] as Iterable).cast<String>().toList(growable: false): const [],@@ -141,6 +130,7 @@ class SystemConfigMachineLearningDto {/// The list of required keys that must be present in a JSON.static const requiredKeys = <String>{+ 'availabilityChecks','clip','duplicateDetection','enabled',diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.jsonindex 0b0391326c..d3fe155a80 100644--- a/open-api/immich-openapi-specs.json+++ b/open-api/immich-openapi-specs.json@@ -12259,6 +12259,25 @@],"type": "object"},+ "MachineLearningAvailabilityChecksDto": {+ "properties": {+ "enabled": {+ "type": "boolean"+ },+ "interval": {+ "type": "number"+ },+ "timeout": {+ "type": "number"+ }+ },+ "required": [+ "enabled",+ "interval",+ "timeout"+ ],+ "type": "object"+ },"ManualJobName": {"enum": ["person-cleanup",@@ -16395,6 +16414,9 @@},"SystemConfigMachineLearningDto": {"properties": {+ "availabilityChecks": {+ "$ref": "#/components/schemas/MachineLearningAvailabilityChecksDto"+ },"clip": {"$ref": "#/components/schemas/CLIPConfig"},@@ -16407,11 +16429,6 @@"facialRecognition": {"$ref": "#/components/schemas/FacialRecognitionConfig"},- "url": {- "deprecated": true,- "description": "This property was deprecated in v1.122.0",- "type": "string"- },"urls": {"format": "uri","items": {@@ -16423,6 +16440,7 @@}},"required": [+ "availabilityChecks","clip","duplicateDetection","enabled",diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.tsindex 079dbda63b..a8b1b0f596 100644--- a/open-api/typescript-sdk/src/fetch-client.ts+++ b/open-api/typescript-sdk/src/fetch-client.ts@@ -1383,6 +1383,11 @@ export type SystemConfigLoggingDto = {enabled: boolean;level: LogLevel;};+export type MachineLearningAvailabilityChecksDto = {+ enabled: boolean;+ interval: number;+ timeout: number;+};export type ClipConfig = {enabled: boolean;modelName: string;@@ -1399,12 +1404,11 @@ export type FacialRecognitionConfig = {modelName: string;};export type SystemConfigMachineLearningDto = {+ availabilityChecks: MachineLearningAvailabilityChecksDto;clip: ClipConfig;duplicateDetection: DuplicateDetectionConfig;enabled: boolean;facialRecognition: FacialRecognitionConfig;- /** This property was deprecated in v1.122.0 */- url?: string;urls: string[];};export type SystemConfigMapDto = {diff --git a/server/src/bin/sync-sql.ts b/server/src/bin/sync-sql.tsindex 6d3cb42fae..b632332069 100644--- a/server/src/bin/sync-sql.ts+++ b/server/src/bin/sync-sql.ts@@ -15,6 +15,7 @@ import { repositories } from 'src/repositories';import { AccessRepository } from 'src/repositories/access.repository';import { ConfigRepository } from 'src/repositories/config.repository';import { LoggingRepository } from 'src/repositories/logging.repository';+import { MachineLearningRepository } from 'src/repositories/machine-learning.repository';import { SyncRepository } from 'src/repositories/sync.repository';import { AuthService } from 'src/services/auth.service';import { getKyselyConfig } from 'src/utils/database';@@ -57,7 +58,7 @@ class SqlGenerator {try {await this.setup();for (const Repository of repositories) {- if (Repository === LoggingRepository) {+ if (Repository === LoggingRepository || Repository === MachineLearningRepository) {continue;}await this.process(Repository);diff --git a/server/src/config.ts b/server/src/config.tsindex 0d1e293be8..66c03450fa 100644--- a/server/src/config.ts+++ b/server/src/config.ts@@ -54,6 +54,11 @@ export interface SystemConfig {machineLearning: {enabled: boolean;urls: string[];+ availabilityChecks: {+ enabled: boolean;+ timeout: number;+ interval: number;+ };clip: {enabled: boolean;modelName: string;@@ -176,6 +181,8 @@ export interface SystemConfig {};}+export type MachineLearningConfig = SystemConfig['machineLearning'];+export const defaults = Object.freeze<SystemConfig>({backup: {database: {@@ -227,6 +234,11 @@ export const defaults = Object.freeze<SystemConfig>({machineLearning: {enabled: process.env.IMMICH_MACHINE_LEARNING_ENABLED !== 'false',urls: [process.env.IMMICH_MACHINE_LEARNING_URL || 'http://immich-machine-learning:3003'],+ availabilityChecks: {+ enabled: true,+ timeout: Number(process.env.IMMICH_MACHINE_LEARNING_PING_TIMEOUT) || 2000,+ interval: 30_000,+ },clip: {enabled: true,modelName: 'ViT-B-32__openai',diff --git a/server/src/constants.ts b/server/src/constants.tsindex b47640c4ae..1bae521a9f 100644--- a/server/src/constants.ts+++ b/server/src/constants.ts@@ -51,11 +51,6 @@ export const serverVersion = new SemVer(version);export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 });export const ONE_HOUR = Duration.fromObject({ hours: 1 });-export const MACHINE_LEARNING_PING_TIMEOUT = Number(process.env.MACHINE_LEARNING_PING_TIMEOUT || 2000);-export const MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME = Number(- process.env.MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME || 30_000,-);-export const citiesFile = 'cities500.txt';export const reverseGeocodeMaxDistance = 25_000;diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.tsindex 8a58995de7..1facc6c331 100644--- a/server/src/dtos/system-config.dto.ts+++ b/server/src/dtos/system-config.dto.ts@@ -1,5 +1,5 @@import { ApiProperty } from '@nestjs/swagger';-import { Exclude, Transform, Type } from 'class-transformer';+import { Type } from 'class-transformer';import {ArrayMinSize,IsInt,@@ -15,7 +15,6 @@ import {ValidateNested,} from 'class-validator';import { SystemConfig } from 'src/config';-import { PropertyLifecycle } from 'src/decorators';import { CLIPConfig, DuplicateDetectionConfig, FacialRecognitionConfig } from 'src/dtos/model-config.dto';import {AudioCodec,@@ -257,21 +256,32 @@ class SystemConfigLoggingDto {level!: LogLevel;}-class SystemConfigMachineLearningDto {+class MachineLearningAvailabilityChecksDto {@ValidateBoolean()enabled!: boolean;- @PropertyLifecycle({ deprecatedAt: 'v1.122.0' })- @Exclude()- url?: string;+ @IsInt()+ timeout!: number;++ @IsInt()+ interval!: number;+}++class SystemConfigMachineLearningDto {+ @ValidateBoolean()+ enabled!: boolean;@IsUrl({ require_tld: false, allow_underscores: true }, { each: true })@ArrayMinSize(1)- @Transform(({ obj, value }) => (obj.url ? [obj.url] : value))@ValidateIf((dto) => dto.enabled)@ApiProperty({ type: 'array', items: { type: 'string', format: 'uri' }, minItems: 1 })urls!: string[];+ @Type(() => MachineLearningAvailabilityChecksDto)+ @ValidateNested()+ @IsObject()+ availabilityChecks!: MachineLearningAvailabilityChecksDto;+@Type(() => CLIPConfig)@ValidateNested()@IsObject()diff --git a/server/src/repositories/logging.repository.ts b/server/src/repositories/logging.repository.tsindex 939ecb718f..576ee6c810 100644--- a/server/src/repositories/logging.repository.ts+++ b/server/src/repositories/logging.repository.ts@@ -142,6 +142,10 @@ export class LoggingRepository {this.handleMessage(LogLevel.Fatal, message, details);}+ deprecate(message: string) {+ this.warn(`[Deprecated] ${message}`);+ }+private handleFunction(level: LogLevel, message: LogFunction, details: LogDetails[]) {if (this.logger.isLevelEnabled(level)) {this.handleMessage(level, message(), details);diff --git a/server/src/repositories/machine-learning.repository.ts b/server/src/repositories/machine-learning.repository.tsindex f880ed1298..d148dc782b 100644--- a/server/src/repositories/machine-learning.repository.ts+++ b/server/src/repositories/machine-learning.repository.ts@@ -1,6 +1,7 @@import { Injectable } from '@nestjs/common';+import { Duration } from 'luxon';import { readFile } from 'node:fs/promises';-import { MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME, MACHINE_LEARNING_PING_TIMEOUT } from 'src/constants';+import { MachineLearningConfig } from 'src/config';import { CLIPConfig } from 'src/dtos/model-config.dto';import { LoggingRepository } from 'src/repositories/logging.repository';@@ -57,82 +58,100 @@ export type TextEncodingOptions = ModelOptions & { language?: string };@Injectable()export class MachineLearningRepository {- // Note that deleted URL's are not removed from this map (ie: they're leaked)- // Cleaning them up is low priority since there should be very few over a- // typical server uptime cycle- private urlAvailability: {- [url: string]:- | {- active: boolean;- lastChecked: number;- }- | undefined;- };+ private healthyMap: Record<string, boolean> = {};+ private interval?: ReturnType<typeof setInterval>;+ private _config?: MachineLearningConfig;++ private get config(): MachineLearningConfig {+ if (!this._config) {+ throw new Error('Machine learning repository not been setup');+ }++ return this._config;+ }constructor(private logger: LoggingRepository) {this.logger.setContext(MachineLearningRepository.name);- this.urlAvailability = {};}- private setUrlAvailability(url: string, active: boolean) {- const current = this.urlAvailability[url];- if (current?.active !== active) {- this.logger.verbose(`Setting ${url} ML server to ${active ? 'active' : 'inactive'}.`);+ setup(config: MachineLearningConfig) {+ this._config = config;+ this.teardown();++ // delete old servers+ for (const url of Object.keys(this.healthyMap)) {+ if (!config.urls.includes(url)) {+ delete this.healthyMap[url];+ }}- this.urlAvailability[url] = {- active,- lastChecked: Date.now(),- };++ if (!config.availabilityChecks.enabled) {+ return;+ }++ this.tick();+ this.interval = setInterval(+ () => this.tick(),+ Duration.fromObject({ milliseconds: config.availabilityChecks.interval }).as('milliseconds'),+ );}- private async checkAvailability(url: string) {- let active = false;+ teardown() {+ if (this.interval) {+ clearInterval(this.interval);+ }+ }++ private tick() {+ for (const url of this.config.urls) {+ void this.check(url);+ }+ }++ private async check(url: string) {+ let healthy = false;try {const response = await fetch(new URL('/ping', url), {- signal: AbortSignal.timeout(MACHINE_LEARNING_PING_TIMEOUT),+ signal: AbortSignal.timeout(this.config.availabilityChecks.timeout),});- active = response.ok;+ if (response.ok) {+ healthy = true;+ }} catch {// nothing to do here}- this.setUrlAvailability(url, active);- return active;++ this.setHealthy(url, healthy);}- private async shouldSkipUrl(url: string) {- const availability = this.urlAvailability[url];- if (availability === undefined) {- // If this is a new endpoint, then check inline and skip if it fails- if (!(await this.checkAvailability(url))) {- return true;- }- return false;+ private setHealthy(url: string, healthy: boolean) {+ if (this.healthyMap[url] !== healthy) {+ this.logger.log(`Machine learning server became ${healthy ? 'healthy' : 'unhealthy'} (${url}).`);}- if (!availability.active && Date.now() - availability.lastChecked < MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME) {- // If this is an old inactive endpoint that hasn't been checked in a- // while then check but don't wait for the result, just skip it- // This avoids delays on every search whilst allowing higher priority- // ML servers to recover over time.- void this.checkAvailability(url);++ this.healthyMap[url] = healthy;+ }++ private isHealthy(url: string) {+ if (!this.config.availabilityChecks.enabled) {return true;}- return false;++ return this.healthyMap[url];}- private async predict<T>(urls: string[], payload: ModelPayload, config: MachineLearningRequest): Promise<T> {+ private async predict<T>(payload: ModelPayload, config: MachineLearningRequest): Promise<T> {const formData = await this.getFormData(payload, config);- let urlCounter = 0;- for (const url of urls) {- urlCounter++;- const isLast = urlCounter >= urls.length;- if (!isLast && (await this.shouldSkipUrl(url))) {- continue;- }+ for (const url of [+ // try healthy servers first+ ...this.config.urls.filter((url) => this.isHealthy(url)),+ ...this.config.urls.filter((url) => !this.isHealthy(url)),+ ]) {try {const response = await fetch(new URL('/predict', url), { method: 'POST', body: formData });if (response.ok) {- this.setUrlAvailability(url, true);+ this.setHealthy(url, true);return response.json();}@@ -144,20 +163,21 @@ export class MachineLearningRepository {`Machine learning request to "${url}" failed: ${error instanceof Error ? error.message : error}`,);}- this.setUrlAvailability(url, false);++ this.setHealthy(url, false);}throw new Error(`Machine learning request '${JSON.stringify(config)}' failed for all URLs`);}- async detectFaces(urls: string[], imagePath: string, { modelName, minScore }: FaceDetectionOptions) {+ async detectFaces(imagePath: string, { modelName, minScore }: FaceDetectionOptions) {const request = {[ModelTask.FACIAL_RECOGNITION]: {[ModelType.DETECTION]: { modelName, options: { minScore } },[ModelType.RECOGNITION]: { modelName },},};- const response = await this.predict<FacialRecognitionResponse>(urls, { imagePath }, request);+ const response = await this.predict<FacialRecognitionResponse>({ imagePath }, request);return {imageHeight: response.imageHeight,imageWidth: response.imageWidth,@@ -165,15 +185,15 @@ export class MachineLearningRepository {};}- async encodeImage(urls: string[], imagePath: string, { modelName }: CLIPConfig) {+ async encodeImage(imagePath: string, { modelName }: CLIPConfig) {const request = { [ModelTask.SEARCH]: { [ModelType.VISUAL]: { modelName } } };- const response = await this.predict<ClipVisualResponse>(urls, { imagePath }, request);+ const response = await this.predict<ClipVisualResponse>({ imagePath }, request);return response[ModelTask.SEARCH];}- async encodeText(urls: string[], text: string, { language, modelName }: TextEncodingOptions) {+ async encodeText(text: string, { language, modelName }: TextEncodingOptions) {const request = { [ModelTask.SEARCH]: { [ModelType.TEXTUAL]: { modelName, options: { language } } } };- const response = await this.predict<ClipTextualResponse>(urls, { text }, request);+ const response = await this.predict<ClipTextualResponse>({ text }, request);return response[ModelTask.SEARCH];}diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.tsindex 13c3128317..41c44ea476 100644--- a/server/src/services/person.service.spec.ts+++ b/server/src/services/person.service.spec.ts@@ -729,7 +729,6 @@ describe(PersonService.name, () => {mocks.assetJob.getForDetectFacesJob.mockResolvedValue({ ...assetStub.image, files: [assetStub.image.files[1]] });await sut.handleDetectFaces({ id: assetStub.image.id });expect(mocks.machineLearning.detectFaces).toHaveBeenCalledWith(- ['http://immich-machine-learning:3003'],'/uploads/user-id/thumbs/path.jpg',expect.objectContaining({ minScore: 0.7, modelName: 'buffalo_l' }),);diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.tsindex 344b69efde..6fa9b3fdd2 100644--- a/server/src/services/person.service.ts+++ b/server/src/services/person.service.ts@@ -316,7 +316,6 @@ export class PersonService extends BaseService {}const { imageHeight, imageWidth, faces } = await this.machineLearningRepository.detectFaces(- machineLearning.urls,previewFile.path,machineLearning.facialRecognition,);diff --git a/server/src/services/search.service.spec.ts b/server/src/services/search.service.spec.tsindex d87ccbde1d..b6e09add19 100644--- a/server/src/services/search.service.spec.ts+++ b/server/src/services/search.service.spec.ts@@ -211,7 +211,6 @@ describe(SearchService.name, () => {await sut.searchSmart(authStub.user1, { query: 'test' });expect(mocks.machineLearning.encodeText).toHaveBeenCalledWith(- [expect.any(String)],'test',expect.objectContaining({ modelName: expect.any(String) }),);@@ -225,7 +224,6 @@ describe(SearchService.name, () => {await sut.searchSmart(authStub.user1, { query: 'test', page: 2, size: 50 });expect(mocks.machineLearning.encodeText).toHaveBeenCalledWith(- [expect.any(String)],'test',expect.objectContaining({ modelName: expect.any(String) }),);@@ -243,7 +241,6 @@ describe(SearchService.name, () => {await sut.searchSmart(authStub.user1, { query: 'test' });expect(mocks.machineLearning.encodeText).toHaveBeenCalledWith(- [expect.any(String)],'test',expect.objectContaining({ modelName: 'ViT-B-16-SigLIP__webli' }),);@@ -253,7 +250,6 @@ describe(SearchService.name, () => {await sut.searchSmart(authStub.user1, { query: 'test', language: 'de' });expect(mocks.machineLearning.encodeText).toHaveBeenCalledWith(- [expect.any(String)],'test',expect.objectContaining({ language: 'de' }),);diff --git a/server/src/services/search.service.ts b/server/src/services/search.service.tsindex 51a2c94338..fea1670e27 100644--- a/server/src/services/search.service.ts+++ b/server/src/services/search.service.ts@@ -118,7 +118,7 @@ export class SearchService extends BaseService {const key = machineLearning.clip.modelName + dto.query + dto.language;embedding = this.embeddingCache.get(key);if (!embedding) {- embedding = await this.machineLearningRepository.encodeText(machineLearning.urls, dto.query, {+ embedding = await this.machineLearningRepository.encodeText(dto.query, {modelName: machineLearning.clip.modelName,language: dto.language,});diff --git a/server/src/services/smart-info.service.spec.ts b/server/src/services/smart-info.service.spec.tsindex edd9f4663a..b3af5cd15f 100644--- a/server/src/services/smart-info.service.spec.ts+++ b/server/src/services/smart-info.service.spec.ts@@ -205,7 +205,6 @@ describe(SmartInfoService.name, () => {expect(await sut.handleEncodeClip({ id: assetStub.image.id })).toEqual(JobStatus.Success);expect(mocks.machineLearning.encodeImage).toHaveBeenCalledWith(- ['http://immich-machine-learning:3003'],'/uploads/user-id/thumbs/path.jpg',expect.objectContaining({ modelName: 'ViT-B-32__openai' }),);@@ -242,7 +241,6 @@ describe(SmartInfoService.name, () => {expect(mocks.database.wait).toHaveBeenCalledWith(512);expect(mocks.machineLearning.encodeImage).toHaveBeenCalledWith(- ['http://immich-machine-learning:3003'],'/uploads/user-id/thumbs/path.jpg',expect.objectContaining({ modelName: 'ViT-B-32__openai' }),);diff --git a/server/src/services/smart-info.service.ts b/server/src/services/smart-info.service.tsindex 3b8e2d1fc3..eff16fea45 100644--- a/server/src/services/smart-info.service.ts+++ b/server/src/services/smart-info.service.ts@@ -108,11 +108,7 @@ export class SmartInfoService extends BaseService {return JobStatus.Skipped;}- const embedding = await this.machineLearningRepository.encodeImage(- machineLearning.urls,- asset.files[0].path,- machineLearning.clip,- );+ const embedding = await this.machineLearningRepository.encodeImage(asset.files[0].path, machineLearning.clip);if (this.databaseRepository.isBusy(DatabaseLock.CLIPDimSize)) {this.logger.verbose(`Waiting for CLIP dimension size to be updated`);diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.tsindex 486945546f..5a9c7f4df3 100644--- a/server/src/services/system-config.service.spec.ts+++ b/server/src/services/system-config.service.spec.ts@@ -82,6 +82,11 @@ const updatedConfig = Object.freeze<SystemConfig>({machineLearning: {enabled: true,urls: ['http://immich-machine-learning:3003'],+ availabilityChecks: {+ enabled: true,+ interval: 30_000,+ timeout: 2000,+ },clip: {enabled: true,modelName: 'ViT-B-32__openai',diff --git a/server/src/services/system-config.service.ts b/server/src/services/system-config.service.tsindex d046b0317a..ea95b4df24 100644--- a/server/src/services/system-config.service.ts+++ b/server/src/services/system-config.service.ts@@ -16,6 +16,20 @@ export class SystemConfigService extends BaseService {async onBootstrap() {const config = await this.getConfig({ withCache: false });await this.eventRepository.emit('ConfigInit', { newConfig: config });++ if (+ process.env.IMMICH_MACHINE_LEARNING_PING_TIMEOUT ||+ process.env.IMMICH_MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME+ ) {+ this.logger.deprecate(+ 'IMMICH_MACHINE_LEARNING_PING_TIMEOUT and MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME have been moved to system config(`machineLearning.availabilityChecks`) and will be removed in a future release.',+ );+ }+ }++ @OnEvent({ name: 'AppShutdown' })+ onShutdown() {+ this.machineLearningRepository.teardown();}async getSystemConfig(): Promise<SystemConfigDto> {@@ -28,12 +42,14 @@ export class SystemConfigService extends BaseService {}@OnEvent({ name: 'ConfigInit', priority: -100 })- onConfigInit({ newConfig: { logging } }: ArgOf<'ConfigInit'>) {+ onConfigInit({ newConfig: { logging, machineLearning } }: ArgOf<'ConfigInit'>) {const { logLevel: envLevel } = this.configRepository.getEnv();const configLevel = logging.enabled ? logging.level : false;const level = envLevel ?? configLevel;this.logger.setLogLevel(level);this.logger.log(`LogLevel=${level} ${envLevel ? '(set via IMMICH_LOG_LEVEL)' : '(set via system config)'}`);++ this.machineLearningRepository.setup(machineLearning);}@OnEvent({ name: 'ConfigUpdate', server: true })diff --git a/web/src/lib/components/admin-settings/MachineLearningSettings.svelte b/web/src/lib/components/admin-settings/MachineLearningSettings.svelteindex 59bd058606..bb1aa04ecd 100644--- a/web/src/lib/components/admin-settings/MachineLearningSettings.svelte+++ b/web/src/lib/components/admin-settings/MachineLearningSettings.svelte@@ -9,7 +9,7 @@import { featureFlags } from '$lib/stores/server-config.store';import type { SystemConfigDto } from '@immich/sdk';import { Button, IconButton } from '@immich/ui';- import { mdiMinusCircle } from '@mdi/js';+ import { mdiPlus, mdiTrashCanOutline } from '@mdi/js';import { isEqual } from 'lodash-es';import { t } from 'svelte-i18n';import { fade } from 'svelte/transition';@@ -46,19 +46,6 @@<div>{#each config.machineLearning.urls as _, i (i)}- {#snippet removeButton()}- {#if config.machineLearning.urls.length > 1}- <IconButton- size="large"- shape="round"- color="danger"- aria-label=""- onclick={() => config.machineLearning.urls.splice(i, 1)}- icon={mdiMinusCircle}- />- {/if}- {/snippet}-<SettingInputFieldinputType={SettingInputFieldType.TEXT}label={i === 0 ? $t('url') : undefined}@@ -67,20 +54,69 @@required={i === 0}disabled={disabled || !config.machineLearning.enabled}isEdited={i === 0 && !isEqual(config.machineLearning.urls, savedConfig.machineLearning.urls)}- trailingSnippet={removeButton}- />+ >+ {#snippet trailingSnippet()}+ {#if config.machineLearning.urls.length > 1}+ <IconButton+ aria-label=""+ onclick={() => config.machineLearning.urls.splice(i, 1)}+ icon={mdiTrashCanOutline}+ color="danger"+ />+ {/if}+ {/snippet}+ </SettingInputField>{/each}</div>- <Button- class="mb-2"- size="small"- shape="round"- onclick={() => config.machineLearning.urls.splice(0, 0, '')}- disabled={disabled || !config.machineLearning.enabled}>{$t('add_url')}</Button- >+ <div class="flex justify-end">+ <Button+ class="mb-2"+ size="small"+ shape="round"+ leadingIcon={mdiPlus}+ onclick={() => config.machineLearning.urls.push('')}+ disabled={disabled || !config.machineLearning.enabled}>{$t('add_url')}</Button+ >+ </div></div>+ <SettingAccordion+ key="availability-checks"+ title={$t('admin.machine_learning_availability_checks')}+ subtitle={$t('admin.machine_learning_availability_checks_description')}+ >+ <div class="ms-4 mt-4 flex flex-col gap-4">+ <SettingSwitch+ title={$t('admin.machine_learning_availability_checks_enabled')}+ bind:checked={config.machineLearning.availabilityChecks.enabled}+ disabled={disabled || !config.machineLearning.enabled}+ />++ <hr />++ <SettingInputField+ inputType={SettingInputFieldType.NUMBER}+ label={$t('admin.machine_learning_availability_checks_interval')}+ bind:value={config.machineLearning.availabilityChecks.interval}+ description={$t('admin.machine_learning_availability_checks_interval_description')}+ disabled={disabled || !config.machineLearning.enabled || !config.machineLearning.availabilityChecks.enabled}+ isEdited={config.machineLearning.availabilityChecks.interval !==+ savedConfig.machineLearning.availabilityChecks.interval}+ />++ <SettingInputField+ inputType={SettingInputFieldType.NUMBER}+ label={$t('admin.machine_learning_availability_checks_timeout')}+ bind:value={config.machineLearning.availabilityChecks.timeout}+ description={$t('admin.machine_learning_availability_checks_timeout_description')}+ disabled={disabled || !config.machineLearning.enabled || !config.machineLearning.availabilityChecks.enabled}+ isEdited={config.machineLearning.availabilityChecks.timeout !==+ savedConfig.machineLearning.availabilityChecks.timeout}+ />+ </div>+ </SettingAccordion>+<SettingAccordionkey="smart-search"title={$t('admin.machine_learning_smart_search')}diff --git a/web/src/lib/components/shared-components/settings/setting-input-field.svelte b/web/src/lib/components/shared-components/settings/setting-input-field.svelteindex 23b95a24a8..db70ba59d7 100644--- a/web/src/lib/components/shared-components/settings/setting-input-field.svelte+++ b/web/src/lib/components/shared-components/settings/setting-input-field.svelte@@ -105,7 +105,7 @@{/if}{#if inputType !== SettingInputFieldType.PASSWORD}- <div class="flex place-items-center place-content-center">+ <div class="flex place-items-center place-content-center gap-2">{#if inputType === SettingInputFieldType.COLOR}<inputbind:this={input}