Skip to content

High performance self-hosted photo and video management solution.

feat: availability checks (#22185)

Jason Rasmussen3f2e0780d545
diff --git a/docs/docs/install/environment-variables.md b/docs/docs/install/environment-variables.md
index 928e0b26e5..4e081c8966 100644
--- a/docs/docs/install/environment-variables.md
+++ b/docs/docs/install/environment-variables.md
@@ -169,8 +169,6 @@ Redis (Sentinel) URL example JSON before encoding:
| `MACHINE_LEARNING_ANN_TUNING_LEVEL` | ARM-NN GPU tuning level (1: rapid, 2: normal, 3: exhaustive) | `2` | machine learning |
| `MACHINE_LEARNING_DEVICE_IDS`<sup>\*4</sup> | Device IDs to use in multi-GPU environments | `0` | machine learning |
| `MACHINE_LEARNING_MAX_BATCH_SIZE__FACIAL_RECOGNITION` | Set the maximum number of faces that will be processed at once by the facial recognition model | None (`1` if using OpenVINO) | machine learning |
-| `MACHINE_LEARNING_PING_TIMEOUT` | How long (ms) to wait for a PING response when checking if an ML server is available | `2000` | server |
-| `MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME` | How long to ignore ML servers that are offline before trying again | `30000` | server |
| `MACHINE_LEARNING_RKNN` | Enable RKNN hardware acceleration if supported | `True` | machine learning |
| `MACHINE_LEARNING_RKNN_THREADS` | How many threads of RKNN runtime should be spinned up while inferencing. | `1` | machine learning |
diff --git a/i18n/en.json b/i18n/en.json
index aa1999adcb..af52770c64 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -123,6 +123,13 @@
"logging_enable_description": "Enable logging",
"logging_level_description": "When enabled, what log level to use.",
"logging_settings": "Logging",
+ "machine_learning_availability_checks": "Availability checks",
+ "machine_learning_availability_checks_description": "Automatically detect and prefer available machine learning servers",
+ "machine_learning_availability_checks_enabled": "Enable availability checks",
+ "machine_learning_availability_checks_interval": "Check interval",
+ "machine_learning_availability_checks_interval_description": "Interval in milliseconds between availability checks",
+ "machine_learning_availability_checks_timeout": "Request timeout",
+ "machine_learning_availability_checks_timeout_description": "Timeout in milliseconds for availability checks",
"machine_learning_clip_model": "CLIP model",
"machine_learning_clip_model_description": "The name of a CLIP model listed <link>here</link>. Note that you must re-run the 'Smart Search' job for all images upon changing a model.",
"machine_learning_duplicate_detection": "Duplicate Detection",
diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md
index 239c62449f..7a426b74fc 100644
--- a/mobile/openapi/README.md
+++ b/mobile/openapi/README.md
@@ -393,6 +393,7 @@ Class | Method | HTTP request | Description
- [LoginCredentialDto](doc//LoginCredentialDto.md)
- [LoginResponseDto](doc//LoginResponseDto.md)
- [LogoutResponseDto](doc//LogoutResponseDto.md)
+ - [MachineLearningAvailabilityChecksDto](doc//MachineLearningAvailabilityChecksDto.md)
- [ManualJobName](doc//ManualJobName.md)
- [MapMarkerResponseDto](doc//MapMarkerResponseDto.md)
- [MapReverseGeocodeResponseDto](doc//MapReverseGeocodeResponseDto.md)
diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart
index e87c160d96..df2c2226b1 100644
--- a/mobile/openapi/lib/api.dart
+++ b/mobile/openapi/lib/api.dart
@@ -164,6 +164,7 @@ part 'model/log_level.dart';
part 'model/login_credential_dto.dart';
part 'model/login_response_dto.dart';
part 'model/logout_response_dto.dart';
+part 'model/machine_learning_availability_checks_dto.dart';
part 'model/manual_job_name.dart';
part 'model/map_marker_response_dto.dart';
part 'model/map_reverse_geocode_response_dto.dart';
diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart
index ae5fd9227b..06d27593c9 100644
--- a/mobile/openapi/lib/api_client.dart
+++ b/mobile/openapi/lib/api_client.dart
@@ -382,6 +382,8 @@ class ApiClient {
return LoginResponseDto.fromJson(value);
case 'LogoutResponseDto':
return LogoutResponseDto.fromJson(value);
+ case 'MachineLearningAvailabilityChecksDto':
+ return MachineLearningAvailabilityChecksDto.fromJson(value);
case 'ManualJobName':
return ManualJobNameTypeTransformer().decode(value);
case 'MapMarkerResponseDto':
diff --git a/mobile/openapi/lib/model/machine_learning_availability_checks_dto.dart b/mobile/openapi/lib/model/machine_learning_availability_checks_dto.dart
new file mode 100644
index 0000000000..84b3181426
--- /dev/null
+++ b/mobile/openapi/lib/model/machine_learning_availability_checks_dto.dart
@@ -0,0 +1,115 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.18
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+class MachineLearningAvailabilityChecksDto {
+ /// Returns a new [MachineLearningAvailabilityChecksDto] instance.
+ MachineLearningAvailabilityChecksDto({
+ required this.enabled,
+ required this.interval,
+ required this.timeout,
+ });
+
+ bool enabled;
+
+ num interval;
+
+ num timeout;
+
+ @override
+ bool operator ==(Object other) => identical(this, other) || other is MachineLearningAvailabilityChecksDto &&
+ other.enabled == enabled &&
+ other.interval == interval &&
+ other.timeout == timeout;
+
+ @override
+ int get hashCode =>
+ // ignore: unnecessary_parenthesis
+ (enabled.hashCode) +
+ (interval.hashCode) +
+ (timeout.hashCode);
+
+ @override
+ String toString() => 'MachineLearningAvailabilityChecksDto[enabled=$enabled, interval=$interval, timeout=$timeout]';
+
+ Map<String, dynamic> toJson() {
+ final json = <String, dynamic>{};
+ json[r'enabled'] = this.enabled;
+ json[r'interval'] = this.interval;
+ json[r'timeout'] = this.timeout;
+ return json;
+ }
+
+ /// Returns a new [MachineLearningAvailabilityChecksDto] instance and imports its values from
+ /// [value] if it's a [Map], null otherwise.
+ // ignore: prefer_constructors_over_static_methods
+ static MachineLearningAvailabilityChecksDto? fromJson(dynamic value) {
+ upgradeDto(value, "MachineLearningAvailabilityChecksDto");
+ if (value is Map) {
+ final json = value.cast<String, dynamic>();
+
+ return MachineLearningAvailabilityChecksDto(
+ enabled: mapValueOfType<bool>(json, r'enabled')!,
+ interval: num.parse('${json[r'interval']}'),
+ timeout: num.parse('${json[r'timeout']}'),
+ );
+ }
+ return null;
+ }
+
+ static List<MachineLearningAvailabilityChecksDto> listFromJson(dynamic json, {bool growable = false,}) {
+ final result = <MachineLearningAvailabilityChecksDto>[];
+ if (json is List && json.isNotEmpty) {
+ for (final row in json) {
+ final value = MachineLearningAvailabilityChecksDto.fromJson(row);
+ if (value != null) {
+ result.add(value);
+ }
+ }
+ }
+ return result.toList(growable: growable);
+ }
+
+ static Map<String, MachineLearningAvailabilityChecksDto> mapFromJson(dynamic json) {
+ final map = <String, MachineLearningAvailabilityChecksDto>{};
+ if (json is Map && json.isNotEmpty) {
+ json = json.cast<String, dynamic>(); // ignore: parameter_assignments
+ for (final entry in json.entries) {
+ final value = MachineLearningAvailabilityChecksDto.fromJson(entry.value);
+ if (value != null) {
+ map[entry.key] = value;
+ }
+ }
+ }
+ return map;
+ }
+
+ // maps a json object with a list of MachineLearningAvailabilityChecksDto-objects as value to a dart map
+ static Map<String, List<MachineLearningAvailabilityChecksDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
+ final map = <String, List<MachineLearningAvailabilityChecksDto>>{};
+ if (json is Map && json.isNotEmpty) {
+ // ignore: parameter_assignments
+ json = json.cast<String, dynamic>();
+ for (final entry in json.entries) {
+ map[entry.key] = MachineLearningAvailabilityChecksDto.listFromJson(entry.value, growable: growable,);
+ }
+ }
+ return map;
+ }
+
+ /// The list of required keys that must be present in a JSON.
+ static const requiredKeys = <String>{
+ 'enabled',
+ 'interval',
+ 'timeout',
+ };
+}
+
diff --git a/mobile/openapi/lib/model/system_config_machine_learning_dto.dart b/mobile/openapi/lib/model/system_config_machine_learning_dto.dart
index a4a9ca7d82..d7b2566d59 100644
--- a/mobile/openapi/lib/model/system_config_machine_learning_dto.dart
+++ b/mobile/openapi/lib/model/system_config_machine_learning_dto.dart
@@ -13,14 +13,16 @@ part of openapi.api;
class SystemConfigMachineLearningDto {
/// Returns a new [SystemConfigMachineLearningDto] instance.
SystemConfigMachineLearningDto({
+ required this.availabilityChecks,
required this.clip,
required this.duplicateDetection,
required this.enabled,
required this.facialRecognition,
- this.url,
this.urls = const [],
});
+ MachineLearningAvailabilityChecksDto availabilityChecks;
+
CLIPConfig clip;
DuplicateDetectionConfig duplicateDetection;
@@ -29,50 +31,37 @@ class SystemConfigMachineLearningDto {
FacialRecognitionConfig facialRecognition;
- /// This property was deprecated in v1.122.0
- ///
- /// Please note: This property should have been non-nullable! Since the specification file
- /// does not include a default value (using the "default:" property), however, the generated
- /// source code must fall back to having a nullable type.
- /// Consider adding a "default:" property in the specification file to hide this note.
- ///
- String? url;
-
List<String> urls;
@override
bool operator ==(Object other) => identical(this, other) || other is SystemConfigMachineLearningDto &&
+ other.availabilityChecks == availabilityChecks &&
other.clip == clip &&
other.duplicateDetection == duplicateDetection &&
other.enabled == enabled &&
other.facialRecognition == facialRecognition &&
- other.url == url &&
_deepEquality.equals(other.urls, urls);
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
+ (availabilityChecks.hashCode) +
(clip.hashCode) +
(duplicateDetection.hashCode) +
(enabled.hashCode) +
(facialRecognition.hashCode) +
- (url == null ? 0 : url!.hashCode) +
(urls.hashCode);
@override
- String toString() => 'SystemConfigMachineLearningDto[clip=$clip, duplicateDetection=$duplicateDetection, enabled=$enabled, facialRecognition=$facialRecognition, url=$url, urls=$urls]';
+ String toString() => 'SystemConfigMachineLearningDto[availabilityChecks=$availabilityChecks, clip=$clip, duplicateDetection=$duplicateDetection, enabled=$enabled, facialRecognition=$facialRecognition, urls=$urls]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
+ json[r'availabilityChecks'] = this.availabilityChecks;
json[r'clip'] = this.clip;
json[r'duplicateDetection'] = this.duplicateDetection;
json[r'enabled'] = this.enabled;
json[r'facialRecognition'] = this.facialRecognition;
- if (this.url != null) {
- json[r'url'] = this.url;
- } else {
- // json[r'url'] = null;
- }
json[r'urls'] = this.urls;
return json;
}
@@ -86,11 +75,11 @@ class SystemConfigMachineLearningDto {
final json = value.cast<String, dynamic>();
return SystemConfigMachineLearningDto(
+ availabilityChecks: MachineLearningAvailabilityChecksDto.fromJson(json[r'availabilityChecks'])!,
clip: CLIPConfig.fromJson(json[r'clip'])!,
duplicateDetection: DuplicateDetectionConfig.fromJson(json[r'duplicateDetection'])!,
enabled: mapValueOfType<bool>(json, r'enabled')!,
facialRecognition: FacialRecognitionConfig.fromJson(json[r'facialRecognition'])!,
- url: mapValueOfType<String>(json, r'url'),
urls: json[r'urls'] is Iterable
? (json[r'urls'] as Iterable).cast<String>().toList(growable: false)
: const [],
@@ -141,6 +130,7 @@ class SystemConfigMachineLearningDto {
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
+ 'availabilityChecks',
'clip',
'duplicateDetection',
'enabled',
diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json
index 0b0391326c..d3fe155a80 100644
--- a/open-api/immich-openapi-specs.json
+++ b/open-api/immich-openapi-specs.json
@@ -12259,6 +12259,25 @@
],
"type": "object"
},
+ "MachineLearningAvailabilityChecksDto": {
+ "properties": {
+ "enabled": {
+ "type": "boolean"
+ },
+ "interval": {
+ "type": "number"
+ },
+ "timeout": {
+ "type": "number"
+ }
+ },
+ "required": [
+ "enabled",
+ "interval",
+ "timeout"
+ ],
+ "type": "object"
+ },
"ManualJobName": {
"enum": [
"person-cleanup",
@@ -16395,6 +16414,9 @@
},
"SystemConfigMachineLearningDto": {
"properties": {
+ "availabilityChecks": {
+ "$ref": "#/components/schemas/MachineLearningAvailabilityChecksDto"
+ },
"clip": {
"$ref": "#/components/schemas/CLIPConfig"
},
@@ -16407,11 +16429,6 @@
"facialRecognition": {
"$ref": "#/components/schemas/FacialRecognitionConfig"
},
- "url": {
- "deprecated": true,
- "description": "This property was deprecated in v1.122.0",
- "type": "string"
- },
"urls": {
"format": "uri",
"items": {
@@ -16423,6 +16440,7 @@
}
},
"required": [
+ "availabilityChecks",
"clip",
"duplicateDetection",
"enabled",
diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts
index 079dbda63b..a8b1b0f596 100644
--- a/open-api/typescript-sdk/src/fetch-client.ts
+++ b/open-api/typescript-sdk/src/fetch-client.ts
@@ -1383,6 +1383,11 @@ export type SystemConfigLoggingDto = {
enabled: boolean;
level: LogLevel;
};
+export type MachineLearningAvailabilityChecksDto = {
+ enabled: boolean;
+ interval: number;
+ timeout: number;
+};
export type ClipConfig = {
enabled: boolean;
modelName: string;
@@ -1399,12 +1404,11 @@ export type FacialRecognitionConfig = {
modelName: string;
};
export type SystemConfigMachineLearningDto = {
+ availabilityChecks: MachineLearningAvailabilityChecksDto;
clip: ClipConfig;
duplicateDetection: DuplicateDetectionConfig;
enabled: boolean;
facialRecognition: FacialRecognitionConfig;
- /** This property was deprecated in v1.122.0 */
- url?: string;
urls: string[];
};
export type SystemConfigMapDto = {
diff --git a/server/src/bin/sync-sql.ts b/server/src/bin/sync-sql.ts
index 6d3cb42fae..b632332069 100644
--- a/server/src/bin/sync-sql.ts
+++ b/server/src/bin/sync-sql.ts
@@ -15,6 +15,7 @@ import { repositories } from 'src/repositories';
import { AccessRepository } from 'src/repositories/access.repository';
import { ConfigRepository } from 'src/repositories/config.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
+import { MachineLearningRepository } from 'src/repositories/machine-learning.repository';
import { SyncRepository } from 'src/repositories/sync.repository';
import { AuthService } from 'src/services/auth.service';
import { getKyselyConfig } from 'src/utils/database';
@@ -57,7 +58,7 @@ class SqlGenerator {
try {
await this.setup();
for (const Repository of repositories) {
- if (Repository === LoggingRepository) {
+ if (Repository === LoggingRepository || Repository === MachineLearningRepository) {
continue;
}
await this.process(Repository);
diff --git a/server/src/config.ts b/server/src/config.ts
index 0d1e293be8..66c03450fa 100644
--- a/server/src/config.ts
+++ b/server/src/config.ts
@@ -54,6 +54,11 @@ export interface SystemConfig {
machineLearning: {
enabled: boolean;
urls: string[];
+ availabilityChecks: {
+ enabled: boolean;
+ timeout: number;
+ interval: number;
+ };
clip: {
enabled: boolean;
modelName: string;
@@ -176,6 +181,8 @@ export interface SystemConfig {
};
}
+export type MachineLearningConfig = SystemConfig['machineLearning'];
+
export const defaults = Object.freeze<SystemConfig>({
backup: {
database: {
@@ -227,6 +234,11 @@ export const defaults = Object.freeze<SystemConfig>({
machineLearning: {
enabled: process.env.IMMICH_MACHINE_LEARNING_ENABLED !== 'false',
urls: [process.env.IMMICH_MACHINE_LEARNING_URL || 'http://immich-machine-learning:3003'],
+ availabilityChecks: {
+ enabled: true,
+ timeout: Number(process.env.IMMICH_MACHINE_LEARNING_PING_TIMEOUT) || 2000,
+ interval: 30_000,
+ },
clip: {
enabled: true,
modelName: 'ViT-B-32__openai',
diff --git a/server/src/constants.ts b/server/src/constants.ts
index b47640c4ae..1bae521a9f 100644
--- a/server/src/constants.ts
+++ b/server/src/constants.ts
@@ -51,11 +51,6 @@ export const serverVersion = new SemVer(version);
export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 });
export const ONE_HOUR = Duration.fromObject({ hours: 1 });
-export const MACHINE_LEARNING_PING_TIMEOUT = Number(process.env.MACHINE_LEARNING_PING_TIMEOUT || 2000);
-export const MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME = Number(
- process.env.MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME || 30_000,
-);
-
export const citiesFile = 'cities500.txt';
export const reverseGeocodeMaxDistance = 25_000;
diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts
index 8a58995de7..1facc6c331 100644
--- a/server/src/dtos/system-config.dto.ts
+++ b/server/src/dtos/system-config.dto.ts
@@ -1,5 +1,5 @@
import { ApiProperty } from '@nestjs/swagger';
-import { Exclude, Transform, Type } from 'class-transformer';
+import { Type } from 'class-transformer';
import {
ArrayMinSize,
IsInt,
@@ -15,7 +15,6 @@ import {
ValidateNested,
} from 'class-validator';
import { SystemConfig } from 'src/config';
-import { PropertyLifecycle } from 'src/decorators';
import { CLIPConfig, DuplicateDetectionConfig, FacialRecognitionConfig } from 'src/dtos/model-config.dto';
import {
AudioCodec,
@@ -257,21 +256,32 @@ class SystemConfigLoggingDto {
level!: LogLevel;
}
-class SystemConfigMachineLearningDto {
+class MachineLearningAvailabilityChecksDto {
@ValidateBoolean()
enabled!: boolean;
- @PropertyLifecycle({ deprecatedAt: 'v1.122.0' })
- @Exclude()
- url?: string;
+ @IsInt()
+ timeout!: number;
+
+ @IsInt()
+ interval!: number;
+}
+
+class SystemConfigMachineLearningDto {
+ @ValidateBoolean()
+ enabled!: boolean;
@IsUrl({ require_tld: false, allow_underscores: true }, { each: true })
@ArrayMinSize(1)
- @Transform(({ obj, value }) => (obj.url ? [obj.url] : value))
@ValidateIf((dto) => dto.enabled)
@ApiProperty({ type: 'array', items: { type: 'string', format: 'uri' }, minItems: 1 })
urls!: string[];
+ @Type(() => MachineLearningAvailabilityChecksDto)
+ @ValidateNested()
+ @IsObject()
+ availabilityChecks!: MachineLearningAvailabilityChecksDto;
+
@Type(() => CLIPConfig)
@ValidateNested()
@IsObject()
diff --git a/server/src/repositories/logging.repository.ts b/server/src/repositories/logging.repository.ts
index 939ecb718f..576ee6c810 100644
--- a/server/src/repositories/logging.repository.ts
+++ b/server/src/repositories/logging.repository.ts
@@ -142,6 +142,10 @@ export class LoggingRepository {
this.handleMessage(LogLevel.Fatal, message, details);
}
+ deprecate(message: string) {
+ this.warn(`[Deprecated] ${message}`);
+ }
+
private handleFunction(level: LogLevel, message: LogFunction, details: LogDetails[]) {
if (this.logger.isLevelEnabled(level)) {
this.handleMessage(level, message(), details);
diff --git a/server/src/repositories/machine-learning.repository.ts b/server/src/repositories/machine-learning.repository.ts
index f880ed1298..d148dc782b 100644
--- a/server/src/repositories/machine-learning.repository.ts
+++ b/server/src/repositories/machine-learning.repository.ts
@@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
+import { Duration } from 'luxon';
import { readFile } from 'node:fs/promises';
-import { MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME, MACHINE_LEARNING_PING_TIMEOUT } from 'src/constants';
+import { MachineLearningConfig } from 'src/config';
import { CLIPConfig } from 'src/dtos/model-config.dto';
import { LoggingRepository } from 'src/repositories/logging.repository';
@@ -57,82 +58,100 @@ export type TextEncodingOptions = ModelOptions & { language?: string };
@Injectable()
export class MachineLearningRepository {
- // Note that deleted URL's are not removed from this map (ie: they're leaked)
- // Cleaning them up is low priority since there should be very few over a
- // typical server uptime cycle
- private urlAvailability: {
- [url: string]:
- | {
- active: boolean;
- lastChecked: number;
- }
- | undefined;
- };
+ private healthyMap: Record<string, boolean> = {};
+ private interval?: ReturnType<typeof setInterval>;
+ private _config?: MachineLearningConfig;
+
+ private get config(): MachineLearningConfig {
+ if (!this._config) {
+ throw new Error('Machine learning repository not been setup');
+ }
+
+ return this._config;
+ }
constructor(private logger: LoggingRepository) {
this.logger.setContext(MachineLearningRepository.name);
- this.urlAvailability = {};
}
- private setUrlAvailability(url: string, active: boolean) {
- const current = this.urlAvailability[url];
- if (current?.active !== active) {
- this.logger.verbose(`Setting ${url} ML server to ${active ? 'active' : 'inactive'}.`);
+ setup(config: MachineLearningConfig) {
+ this._config = config;
+ this.teardown();
+
+ // delete old servers
+ for (const url of Object.keys(this.healthyMap)) {
+ if (!config.urls.includes(url)) {
+ delete this.healthyMap[url];
+ }
}
- this.urlAvailability[url] = {
- active,
- lastChecked: Date.now(),
- };
+
+ if (!config.availabilityChecks.enabled) {
+ return;
+ }
+
+ this.tick();
+ this.interval = setInterval(
+ () => this.tick(),
+ Duration.fromObject({ milliseconds: config.availabilityChecks.interval }).as('milliseconds'),
+ );
}
- private async checkAvailability(url: string) {
- let active = false;
+ teardown() {
+ if (this.interval) {
+ clearInterval(this.interval);
+ }
+ }
+
+ private tick() {
+ for (const url of this.config.urls) {
+ void this.check(url);
+ }
+ }
+
+ private async check(url: string) {
+ let healthy = false;
try {
const response = await fetch(new URL('/ping', url), {
- signal: AbortSignal.timeout(MACHINE_LEARNING_PING_TIMEOUT),
+ signal: AbortSignal.timeout(this.config.availabilityChecks.timeout),
});
- active = response.ok;
+ if (response.ok) {
+ healthy = true;
+ }
} catch {
// nothing to do here
}
- this.setUrlAvailability(url, active);
- return active;
+
+ this.setHealthy(url, healthy);
}
- private async shouldSkipUrl(url: string) {
- const availability = this.urlAvailability[url];
- if (availability === undefined) {
- // If this is a new endpoint, then check inline and skip if it fails
- if (!(await this.checkAvailability(url))) {
- return true;
- }
- return false;
+ private setHealthy(url: string, healthy: boolean) {
+ if (this.healthyMap[url] !== healthy) {
+ this.logger.log(`Machine learning server became ${healthy ? 'healthy' : 'unhealthy'} (${url}).`);
}
- if (!availability.active && Date.now() - availability.lastChecked < MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME) {
- // If this is an old inactive endpoint that hasn't been checked in a
- // while then check but don't wait for the result, just skip it
- // This avoids delays on every search whilst allowing higher priority
- // ML servers to recover over time.
- void this.checkAvailability(url);
+
+ this.healthyMap[url] = healthy;
+ }
+
+ private isHealthy(url: string) {
+ if (!this.config.availabilityChecks.enabled) {
return true;
}
- return false;
+
+ return this.healthyMap[url];
}
- private async predict<T>(urls: string[], payload: ModelPayload, config: MachineLearningRequest): Promise<T> {
+ private async predict<T>(payload: ModelPayload, config: MachineLearningRequest): Promise<T> {
const formData = await this.getFormData(payload, config);
- let urlCounter = 0;
- for (const url of urls) {
- urlCounter++;
- const isLast = urlCounter >= urls.length;
- if (!isLast && (await this.shouldSkipUrl(url))) {
- continue;
- }
+ for (const url of [
+ // try healthy servers first
+ ...this.config.urls.filter((url) => this.isHealthy(url)),
+ ...this.config.urls.filter((url) => !this.isHealthy(url)),
+ ]) {
try {
const response = await fetch(new URL('/predict', url), { method: 'POST', body: formData });
if (response.ok) {
- this.setUrlAvailability(url, true);
+ this.setHealthy(url, true);
return response.json();
}
@@ -144,20 +163,21 @@ export class MachineLearningRepository {
`Machine learning request to "${url}" failed: ${error instanceof Error ? error.message : error}`,
);
}
- this.setUrlAvailability(url, false);
+
+ this.setHealthy(url, false);
}
throw new Error(`Machine learning request '${JSON.stringify(config)}' failed for all URLs`);
}
- async detectFaces(urls: string[], imagePath: string, { modelName, minScore }: FaceDetectionOptions) {
+ async detectFaces(imagePath: string, { modelName, minScore }: FaceDetectionOptions) {
const request = {
[ModelTask.FACIAL_RECOGNITION]: {
[ModelType.DETECTION]: { modelName, options: { minScore } },
[ModelType.RECOGNITION]: { modelName },
},
};
- const response = await this.predict<FacialRecognitionResponse>(urls, { imagePath }, request);
+ const response = await this.predict<FacialRecognitionResponse>({ imagePath }, request);
return {
imageHeight: response.imageHeight,
imageWidth: response.imageWidth,
@@ -165,15 +185,15 @@ export class MachineLearningRepository {
};
}
- async encodeImage(urls: string[], imagePath: string, { modelName }: CLIPConfig) {
+ async encodeImage(imagePath: string, { modelName }: CLIPConfig) {
const request = { [ModelTask.SEARCH]: { [ModelType.VISUAL]: { modelName } } };
- const response = await this.predict<ClipVisualResponse>(urls, { imagePath }, request);
+ const response = await this.predict<ClipVisualResponse>({ imagePath }, request);
return response[ModelTask.SEARCH];
}
- async encodeText(urls: string[], text: string, { language, modelName }: TextEncodingOptions) {
+ async encodeText(text: string, { language, modelName }: TextEncodingOptions) {
const request = { [ModelTask.SEARCH]: { [ModelType.TEXTUAL]: { modelName, options: { language } } } };
- const response = await this.predict<ClipTextualResponse>(urls, { text }, request);
+ const response = await this.predict<ClipTextualResponse>({ text }, request);
return response[ModelTask.SEARCH];
}
diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.ts
index 13c3128317..41c44ea476 100644
--- a/server/src/services/person.service.spec.ts
+++ b/server/src/services/person.service.spec.ts
@@ -729,7 +729,6 @@ describe(PersonService.name, () => {
mocks.assetJob.getForDetectFacesJob.mockResolvedValue({ ...assetStub.image, files: [assetStub.image.files[1]] });
await sut.handleDetectFaces({ id: assetStub.image.id });
expect(mocks.machineLearning.detectFaces).toHaveBeenCalledWith(
- ['http://immich-machine-learning:3003'],
'/uploads/user-id/thumbs/path.jpg',
expect.objectContaining({ minScore: 0.7, modelName: 'buffalo_l' }),
);
diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts
index 344b69efde..6fa9b3fdd2 100644
--- a/server/src/services/person.service.ts
+++ b/server/src/services/person.service.ts
@@ -316,7 +316,6 @@ export class PersonService extends BaseService {
}
const { imageHeight, imageWidth, faces } = await this.machineLearningRepository.detectFaces(
- machineLearning.urls,
previewFile.path,
machineLearning.facialRecognition,
);
diff --git a/server/src/services/search.service.spec.ts b/server/src/services/search.service.spec.ts
index d87ccbde1d..b6e09add19 100644
--- a/server/src/services/search.service.spec.ts
+++ b/server/src/services/search.service.spec.ts
@@ -211,7 +211,6 @@ describe(SearchService.name, () => {
await sut.searchSmart(authStub.user1, { query: 'test' });
expect(mocks.machineLearning.encodeText).toHaveBeenCalledWith(
- [expect.any(String)],
'test',
expect.objectContaining({ modelName: expect.any(String) }),
);
@@ -225,7 +224,6 @@ describe(SearchService.name, () => {
await sut.searchSmart(authStub.user1, { query: 'test', page: 2, size: 50 });
expect(mocks.machineLearning.encodeText).toHaveBeenCalledWith(
- [expect.any(String)],
'test',
expect.objectContaining({ modelName: expect.any(String) }),
);
@@ -243,7 +241,6 @@ describe(SearchService.name, () => {
await sut.searchSmart(authStub.user1, { query: 'test' });
expect(mocks.machineLearning.encodeText).toHaveBeenCalledWith(
- [expect.any(String)],
'test',
expect.objectContaining({ modelName: 'ViT-B-16-SigLIP__webli' }),
);
@@ -253,7 +250,6 @@ describe(SearchService.name, () => {
await sut.searchSmart(authStub.user1, { query: 'test', language: 'de' });
expect(mocks.machineLearning.encodeText).toHaveBeenCalledWith(
- [expect.any(String)],
'test',
expect.objectContaining({ language: 'de' }),
);
diff --git a/server/src/services/search.service.ts b/server/src/services/search.service.ts
index 51a2c94338..fea1670e27 100644
--- a/server/src/services/search.service.ts
+++ b/server/src/services/search.service.ts
@@ -118,7 +118,7 @@ export class SearchService extends BaseService {
const key = machineLearning.clip.modelName + dto.query + dto.language;
embedding = this.embeddingCache.get(key);
if (!embedding) {
- embedding = await this.machineLearningRepository.encodeText(machineLearning.urls, dto.query, {
+ embedding = await this.machineLearningRepository.encodeText(dto.query, {
modelName: machineLearning.clip.modelName,
language: dto.language,
});
diff --git a/server/src/services/smart-info.service.spec.ts b/server/src/services/smart-info.service.spec.ts
index edd9f4663a..b3af5cd15f 100644
--- a/server/src/services/smart-info.service.spec.ts
+++ b/server/src/services/smart-info.service.spec.ts
@@ -205,7 +205,6 @@ describe(SmartInfoService.name, () => {
expect(await sut.handleEncodeClip({ id: assetStub.image.id })).toEqual(JobStatus.Success);
expect(mocks.machineLearning.encodeImage).toHaveBeenCalledWith(
- ['http://immich-machine-learning:3003'],
'/uploads/user-id/thumbs/path.jpg',
expect.objectContaining({ modelName: 'ViT-B-32__openai' }),
);
@@ -242,7 +241,6 @@ describe(SmartInfoService.name, () => {
expect(mocks.database.wait).toHaveBeenCalledWith(512);
expect(mocks.machineLearning.encodeImage).toHaveBeenCalledWith(
- ['http://immich-machine-learning:3003'],
'/uploads/user-id/thumbs/path.jpg',
expect.objectContaining({ modelName: 'ViT-B-32__openai' }),
);
diff --git a/server/src/services/smart-info.service.ts b/server/src/services/smart-info.service.ts
index 3b8e2d1fc3..eff16fea45 100644
--- a/server/src/services/smart-info.service.ts
+++ b/server/src/services/smart-info.service.ts
@@ -108,11 +108,7 @@ export class SmartInfoService extends BaseService {
return JobStatus.Skipped;
}
- const embedding = await this.machineLearningRepository.encodeImage(
- machineLearning.urls,
- asset.files[0].path,
- machineLearning.clip,
- );
+ const embedding = await this.machineLearningRepository.encodeImage(asset.files[0].path, machineLearning.clip);
if (this.databaseRepository.isBusy(DatabaseLock.CLIPDimSize)) {
this.logger.verbose(`Waiting for CLIP dimension size to be updated`);
diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts
index 486945546f..5a9c7f4df3 100644
--- a/server/src/services/system-config.service.spec.ts
+++ b/server/src/services/system-config.service.spec.ts
@@ -82,6 +82,11 @@ const updatedConfig = Object.freeze<SystemConfig>({
machineLearning: {
enabled: true,
urls: ['http://immich-machine-learning:3003'],
+ availabilityChecks: {
+ enabled: true,
+ interval: 30_000,
+ timeout: 2000,
+ },
clip: {
enabled: true,
modelName: 'ViT-B-32__openai',
diff --git a/server/src/services/system-config.service.ts b/server/src/services/system-config.service.ts
index d046b0317a..ea95b4df24 100644
--- a/server/src/services/system-config.service.ts
+++ b/server/src/services/system-config.service.ts
@@ -16,6 +16,20 @@ export class SystemConfigService extends BaseService {
async onBootstrap() {
const config = await this.getConfig({ withCache: false });
await this.eventRepository.emit('ConfigInit', { newConfig: config });
+
+ if (
+ process.env.IMMICH_MACHINE_LEARNING_PING_TIMEOUT ||
+ process.env.IMMICH_MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME
+ ) {
+ this.logger.deprecate(
+ 'IMMICH_MACHINE_LEARNING_PING_TIMEOUT and MACHINE_LEARNING_AVAILABILITY_BACKOFF_TIME have been moved to system config(`machineLearning.availabilityChecks`) and will be removed in a future release.',
+ );
+ }
+ }
+
+ @OnEvent({ name: 'AppShutdown' })
+ onShutdown() {
+ this.machineLearningRepository.teardown();
}
async getSystemConfig(): Promise<SystemConfigDto> {
@@ -28,12 +42,14 @@ export class SystemConfigService extends BaseService {
}
@OnEvent({ name: 'ConfigInit', priority: -100 })
- onConfigInit({ newConfig: { logging } }: ArgOf<'ConfigInit'>) {
+ onConfigInit({ newConfig: { logging, machineLearning } }: ArgOf<'ConfigInit'>) {
const { logLevel: envLevel } = this.configRepository.getEnv();
const configLevel = logging.enabled ? logging.level : false;
const level = envLevel ?? configLevel;
this.logger.setLogLevel(level);
this.logger.log(`LogLevel=${level} ${envLevel ? '(set via IMMICH_LOG_LEVEL)' : '(set via system config)'}`);
+
+ this.machineLearningRepository.setup(machineLearning);
}
@OnEvent({ name: 'ConfigUpdate', server: true })
diff --git a/web/src/lib/components/admin-settings/MachineLearningSettings.svelte b/web/src/lib/components/admin-settings/MachineLearningSettings.svelte
index 59bd058606..bb1aa04ecd 100644
--- a/web/src/lib/components/admin-settings/MachineLearningSettings.svelte
+++ b/web/src/lib/components/admin-settings/MachineLearningSettings.svelte
@@ -9,7 +9,7 @@
import { featureFlags } from '$lib/stores/server-config.store';
import type { SystemConfigDto } from '@immich/sdk';
import { Button, IconButton } from '@immich/ui';
- import { mdiMinusCircle } from '@mdi/js';
+ import { mdiPlus, mdiTrashCanOutline } from '@mdi/js';
import { isEqual } from 'lodash-es';
import { t } from 'svelte-i18n';
import { fade } from 'svelte/transition';
@@ -46,19 +46,6 @@
<div>
{#each config.machineLearning.urls as _, i (i)}
- {#snippet removeButton()}
- {#if config.machineLearning.urls.length > 1}
- <IconButton
- size="large"
- shape="round"
- color="danger"
- aria-label=""
- onclick={() => config.machineLearning.urls.splice(i, 1)}
- icon={mdiMinusCircle}
- />
- {/if}
- {/snippet}
-
<SettingInputField
inputType={SettingInputFieldType.TEXT}
label={i === 0 ? $t('url') : undefined}
@@ -67,20 +54,69 @@
required={i === 0}
disabled={disabled || !config.machineLearning.enabled}
isEdited={i === 0 && !isEqual(config.machineLearning.urls, savedConfig.machineLearning.urls)}
- trailingSnippet={removeButton}
- />
+ >
+ {#snippet trailingSnippet()}
+ {#if config.machineLearning.urls.length > 1}
+ <IconButton
+ aria-label=""
+ onclick={() => config.machineLearning.urls.splice(i, 1)}
+ icon={mdiTrashCanOutline}
+ color="danger"
+ />
+ {/if}
+ {/snippet}
+ </SettingInputField>
{/each}
</div>
- <Button
- class="mb-2"
- size="small"
- shape="round"
- onclick={() => config.machineLearning.urls.splice(0, 0, '')}
- disabled={disabled || !config.machineLearning.enabled}>{$t('add_url')}</Button
- >
+ <div class="flex justify-end">
+ <Button
+ class="mb-2"
+ size="small"
+ shape="round"
+ leadingIcon={mdiPlus}
+ onclick={() => config.machineLearning.urls.push('')}
+ disabled={disabled || !config.machineLearning.enabled}>{$t('add_url')}</Button
+ >
+ </div>
</div>
+ <SettingAccordion
+ key="availability-checks"
+ title={$t('admin.machine_learning_availability_checks')}
+ subtitle={$t('admin.machine_learning_availability_checks_description')}
+ >
+ <div class="ms-4 mt-4 flex flex-col gap-4">
+ <SettingSwitch
+ title={$t('admin.machine_learning_availability_checks_enabled')}
+ bind:checked={config.machineLearning.availabilityChecks.enabled}
+ disabled={disabled || !config.machineLearning.enabled}
+ />
+
+ <hr />
+
+ <SettingInputField
+ inputType={SettingInputFieldType.NUMBER}
+ label={$t('admin.machine_learning_availability_checks_interval')}
+ bind:value={config.machineLearning.availabilityChecks.interval}
+ description={$t('admin.machine_learning_availability_checks_interval_description')}
+ disabled={disabled || !config.machineLearning.enabled || !config.machineLearning.availabilityChecks.enabled}
+ isEdited={config.machineLearning.availabilityChecks.interval !==
+ savedConfig.machineLearning.availabilityChecks.interval}
+ />
+
+ <SettingInputField
+ inputType={SettingInputFieldType.NUMBER}
+ label={$t('admin.machine_learning_availability_checks_timeout')}
+ bind:value={config.machineLearning.availabilityChecks.timeout}
+ description={$t('admin.machine_learning_availability_checks_timeout_description')}
+ disabled={disabled || !config.machineLearning.enabled || !config.machineLearning.availabilityChecks.enabled}
+ isEdited={config.machineLearning.availabilityChecks.timeout !==
+ savedConfig.machineLearning.availabilityChecks.timeout}
+ />
+ </div>
+ </SettingAccordion>
+
<SettingAccordion
key="smart-search"
title={$t('admin.machine_learning_smart_search')}
diff --git a/web/src/lib/components/shared-components/settings/setting-input-field.svelte b/web/src/lib/components/shared-components/settings/setting-input-field.svelte
index 23b95a24a8..db70ba59d7 100644
--- a/web/src/lib/components/shared-components/settings/setting-input-field.svelte
+++ b/web/src/lib/components/shared-components/settings/setting-input-field.svelte
@@ -105,7 +105,7 @@
{/if}
{#if inputType !== SettingInputFieldType.PASSWORD}
- <div class="flex place-items-center place-content-center">
+ <div class="flex place-items-center place-content-center gap-2">
{#if inputType === SettingInputFieldType.COLOR}
<input
bind:this={input}