High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
refactor(server): telemetry (#13588)
Jason Rasmussene6a666f1d307
diff --git a/server/src/app.module.ts b/server/src/app.module.tsindex fd921150fd..3c26faaca3 100644--- a/server/src/app.module.ts+++ b/server/src/app.module.ts@@ -14,6 +14,7 @@ import { entities } from 'src/entities';import { ImmichWorker } from 'src/enum';import { IEventRepository } from 'src/interfaces/event.interface';import { ILoggerRepository } from 'src/interfaces/logger.interface';+import { ITelemetryRepository } from 'src/interfaces/telemetry.interface';import { AuthGuard } from 'src/middleware/auth.guard';import { ErrorInterceptor } from 'src/middleware/error.interceptor';import { FileUploadInterceptor } from 'src/middleware/file-upload.interceptor';@@ -21,6 +22,7 @@ import { GlobalExceptionFilter } from 'src/middleware/global-exception.filter';import { LoggingInterceptor } from 'src/middleware/logging.interceptor';import { repositories } from 'src/repositories';import { ConfigRepository } from 'src/repositories/config.repository';+import { teardownTelemetry } from 'src/repositories/telemetry.repository';import { services } from 'src/services';import { DatabaseService } from 'src/services/database.service';@@ -66,6 +68,7 @@ abstract class BaseModule implements OnModuleInit, OnModuleDestroy {constructor(@Inject(ILoggerRepository) logger: ILoggerRepository,@Inject(IEventRepository) private eventRepository: IEventRepository,+ @Inject(ITelemetryRepository) private telemetryRepository: ITelemetryRepository,) {logger.setAppName(this.worker);}@@ -73,12 +76,14 @@ abstract class BaseModule implements OnModuleInit, OnModuleDestroy {abstract getWorker(): ImmichWorker;async onModuleInit() {+ this.telemetryRepository.setup({ repositories: repositories.map(({ useClass }) => useClass) });this.eventRepository.setup({ services });await this.eventRepository.emit('app.bootstrap', this.worker);}async onModuleDestroy() {await this.eventRepository.emit('app.shutdown', this.worker);+ await teardownTelemetry();}}diff --git a/server/src/config.ts b/server/src/config.tsindex e386c134b4..fca6719bc0 100644--- a/server/src/config.ts+++ b/server/src/config.ts@@ -354,9 +354,9 @@ export const immichAppConfig: ConfigModuleOptions = {),IMMICH_METRICS: Joi.boolean().optional().default(false),- IMMICH_HOST_METRICS: Joi.boolean().optional().default(false),- IMMICH_API_METRICS: Joi.boolean().optional().default(false),- IMMICH_IO_METRICS: Joi.boolean().optional().default(false),+ IMMICH_HOST_METRICS: Joi.boolean().optional(),+ IMMICH_API_METRICS: Joi.boolean().optional(),+ IMMICH_IO_METRICS: Joi.boolean().optional(),}),};diff --git a/server/src/decorators.ts b/server/src/decorators.tsindex 2782368239..db755c5ff9 100644--- a/server/src/decorators.ts+++ b/server/src/decorators.ts@@ -86,27 +86,6 @@ export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {return Chunked({ ...options, mergeFn: setUnion });}-// https://stackoverflow.com/a/74898678-export function DecorateAll(- decorator: <T>(- target: any,- propertyKey: string,- descriptor: TypedPropertyDescriptor<T>,- ) => TypedPropertyDescriptor<T> | void,-) {- return (target: any) => {- const descriptors = Object.getOwnPropertyDescriptors(target.prototype);- for (const [propName, descriptor] of Object.entries(descriptors)) {- const isMethod = typeof descriptor.value == 'function' && propName !== 'constructor';- if (!isMethod) {- continue;- }- decorator({ ...target, constructor: { ...target.constructor, name: target.name } as any }, propName, descriptor);- Object.defineProperty(target.prototype, propName, descriptor);- }- };-}-const UUID = '00000000-0000-4000-a000-000000000000';export const DummyValue = {@@ -128,6 +107,9 @@ export interface GenerateSqlQueries {params: unknown[];}+export const Telemetry = (options: { enabled?: boolean }) =>+ SetMetadata(MetadataKey.TELEMETRY_ENABLED, options?.enabled ?? true);+/** Decorator to enable versioning/tracking of generated Sql */export const GenerateSql = (...options: GenerateSqlQueries[]) => SetMetadata(GENERATE_SQL_KEY, options);diff --git a/server/src/enum.ts b/server/src/enum.tsindex 8c11834dac..902d6635e7 100644--- a/server/src/enum.ts+++ b/server/src/enum.ts@@ -334,6 +334,7 @@ export enum MetadataKey {SHARED_ROUTE = 'shared_route',API_KEY_SECURITY = 'api_key',EVENT_CONFIG = 'event_config',+ TELEMETRY_ENABLED = 'telemetry_enabled',}export enum RouteKey {diff --git a/server/src/interfaces/telemetry.interface.ts b/server/src/interfaces/telemetry.interface.tsindex 070014f2e0..688e52c21e 100644--- a/server/src/interfaces/telemetry.interface.ts+++ b/server/src/interfaces/telemetry.interface.ts@@ -1,4 +1,5 @@import { MetricOptions } from '@opentelemetry/api';+import { ClassConstructor } from 'class-transformer';export const ITelemetryRepository = 'ITelemetryRepository';@@ -14,6 +15,7 @@ export interface IMetricGroupRepository {}export interface ITelemetryRepository {+ setup(options: { repositories: ClassConstructor<unknown>[] }): void;api: IMetricGroupRepository;host: IMetricGroupRepository;jobs: IMetricGroupRepository;diff --git a/server/src/repositories/access.repository.ts b/server/src/repositories/access.repository.tsindex f6921ffe27..f3cbf392db 100644--- a/server/src/repositories/access.repository.ts+++ b/server/src/repositories/access.repository.ts@@ -15,7 +15,6 @@ import { StackEntity } from 'src/entities/stack.entity';import { TagEntity } from 'src/entities/tag.entity';import { AlbumUserRole } from 'src/enum';import { IAccessRepository } from 'src/interfaces/access.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { Brackets, In, Repository } from 'typeorm';type IActivityAccess = IAccessRepository['activity'];@@ -29,7 +28,6 @@ type IStackAccess = IAccessRepository['stack'];type ITagAccess = IAccessRepository['tag'];type ITimelineAccess = IAccessRepository['timeline'];-@Instrumentation()@Injectable()class ActivityAccess implements IActivityAccess {constructor(diff --git a/server/src/repositories/activity.repository.ts b/server/src/repositories/activity.repository.tsindex e21f746483..0f0a0cb60e 100644--- a/server/src/repositories/activity.repository.ts+++ b/server/src/repositories/activity.repository.ts@@ -3,7 +3,6 @@ import { InjectRepository } from '@nestjs/typeorm';import { DummyValue, GenerateSql } from 'src/decorators';import { ActivityEntity } from 'src/entities/activity.entity';import { IActivityRepository } from 'src/interfaces/activity.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { IsNull, Repository } from 'typeorm';export interface ActivitySearch {@@ -13,7 +12,6 @@ export interface ActivitySearch {isLiked?: boolean;}-@Instrumentation()@Injectable()export class ActivityRepository implements IActivityRepository {constructor(@InjectRepository(ActivityEntity) private repository: Repository<ActivityEntity>) {}diff --git a/server/src/repositories/album-user.repository.ts b/server/src/repositories/album-user.repository.tsindex 7fd18711aa..9328ea8cfc 100644--- a/server/src/repositories/album-user.repository.ts+++ b/server/src/repositories/album-user.repository.ts@@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { AlbumUserEntity } from 'src/entities/album-user.entity';import { AlbumPermissionId, IAlbumUserRepository } from 'src/interfaces/album-user.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { Repository } from 'typeorm';-@Instrumentation()@Injectable()export class AlbumUserRepository implements IAlbumUserRepository {constructor(@InjectRepository(AlbumUserEntity) private repository: Repository<AlbumUserEntity>) {}diff --git a/server/src/repositories/album.repository.ts b/server/src/repositories/album.repository.tsindex f7b4cb44aa..8b7565e318 100644--- a/server/src/repositories/album.repository.ts+++ b/server/src/repositories/album.repository.ts@@ -4,7 +4,6 @@ import { Chunked, ChunkedArray, ChunkedSet, DummyValue, GenerateSql } from 'src/import { AlbumEntity } from 'src/entities/album.entity';import { AssetEntity } from 'src/entities/asset.entity';import { AlbumAssetCount, AlbumInfoOptions, IAlbumRepository } from 'src/interfaces/album.interface';-import { Instrumentation } from 'src/utils/instrumentation';import {DataSource,EntityManager,@@ -23,7 +22,6 @@ const withoutDeletedUsers = <T extends AlbumEntity | null>(album: T) => {return album;};-@Instrumentation()@Injectable()export class AlbumRepository implements IAlbumRepository {constructor(diff --git a/server/src/repositories/api-key.repository.ts b/server/src/repositories/api-key.repository.tsindex 5178039177..bb37390de1 100644--- a/server/src/repositories/api-key.repository.ts+++ b/server/src/repositories/api-key.repository.ts@@ -3,10 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm';import { DummyValue, GenerateSql } from 'src/decorators';import { APIKeyEntity } from 'src/entities/api-key.entity';import { IKeyRepository } from 'src/interfaces/api-key.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { Repository } from 'typeorm';-@Instrumentation()@Injectable()export class ApiKeyRepository implements IKeyRepository {constructor(@InjectRepository(APIKeyEntity) private repository: Repository<APIKeyEntity>) {}diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.tsindex fd47a976a5..6080e943e4 100644--- a/server/src/repositories/asset.repository.ts+++ b/server/src/repositories/asset.repository.ts@@ -29,7 +29,6 @@ import {} from 'src/interfaces/asset.interface';import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface';import { searchAssetBuilder } from 'src/utils/database';-import { Instrumentation } from 'src/utils/instrumentation';import { Paginated, PaginationOptions, paginate, paginatedBuilder } from 'src/utils/pagination';import {Brackets,@@ -54,7 +53,6 @@ const dateTrunc = (options: TimeBucketOptions) =>truncateMap[options.size]}', (asset."localDateTime" at time zone 'UTC')) at time zone 'UTC')::timestamptz`;-@Instrumentation()@Injectable()export class AssetRepository implements IAssetRepository {constructor(diff --git a/server/src/repositories/audit.repository.ts b/server/src/repositories/audit.repository.tsindex deb0d0f6f1..ac73c3a8b9 100644--- a/server/src/repositories/audit.repository.ts+++ b/server/src/repositories/audit.repository.ts@@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { AuditEntity } from 'src/entities/audit.entity';import { AuditSearch, IAuditRepository } from 'src/interfaces/audit.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { In, LessThan, MoreThan, Repository } from 'typeorm';-@Instrumentation()@Injectable()export class AuditRepository implements IAuditRepository {constructor(@InjectRepository(AuditEntity) private repository: Repository<AuditEntity>) {}diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.tsindex 44b8c7b605..fabccd7846 100644--- a/server/src/repositories/config.repository.ts+++ b/server/src/repositories/config.repository.ts@@ -1,6 +1,7 @@import { Injectable } from '@nestjs/common';import { join } from 'node:path';import { citiesFile, excludePaths } from 'src/constants';+import { Telemetry } from 'src/decorators';import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';import { EnvData, IConfigRepository } from 'src/interfaces/config.interface';import { DatabaseExtension } from 'src/interfaces/database.interface';@@ -74,9 +75,6 @@ const getEnv = (): EnvData => {const repoMetrics = parseBoolean(process.env.IMMICH_IO_METRICS, globalEnabled);const jobMetrics = parseBoolean(process.env.IMMICH_JOB_METRICS, globalEnabled);const telemetryEnabled = globalEnabled || hostMetrics || apiMetrics || repoMetrics || jobMetrics;- if (!telemetryEnabled && process.env.OTEL_SDK_DISABLED === undefined) {- process.env.OTEL_SDK_DISABLED = 'true';- }return {host: process.env.IMMICH_HOST,@@ -186,6 +184,7 @@ const getEnv = (): EnvData => {let cached: EnvData | undefined;@Injectable()+@Telemetry({ enabled: false })export class ConfigRepository implements IConfigRepository {getEnv(): EnvData {if (!cached) {diff --git a/server/src/repositories/crypto.repository.ts b/server/src/repositories/crypto.repository.tsindex 72e75ef174..ee25609fec 100644--- a/server/src/repositories/crypto.repository.ts+++ b/server/src/repositories/crypto.repository.ts@@ -3,9 +3,7 @@ import { compareSync, hash } from 'bcrypt';import { createHash, createPublicKey, createVerify, randomBytes, randomUUID } from 'node:crypto';import { createReadStream } from 'node:fs';import { ICryptoRepository } from 'src/interfaces/crypto.interface';-import { Instrumentation } from 'src/utils/instrumentation';-@Instrumentation()@Injectable()export class CryptoRepository implements ICryptoRepository {randomUUID() {diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.tsindex 547f03fc20..b5e2edfdea 100644--- a/server/src/repositories/database.repository.ts+++ b/server/src/repositories/database.repository.ts@@ -15,11 +15,9 @@ import {VectorUpdateResult,} from 'src/interfaces/database.interface';import { ILoggerRepository } from 'src/interfaces/logger.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { isValidInteger } from 'src/validation';import { DataSource, EntityManager, QueryRunner } from 'typeorm';-@Instrumentation()@Injectable()export class DatabaseRepository implements IDatabaseRepository {private vectorExtension: VectorExtension;diff --git a/server/src/repositories/event.repository.ts b/server/src/repositories/event.repository.tsindex cb58d56b2a..bb265196f9 100644--- a/server/src/repositories/event.repository.ts+++ b/server/src/repositories/event.repository.ts@@ -24,7 +24,6 @@ import {} from 'src/interfaces/event.interface';import { ILoggerRepository } from 'src/interfaces/logger.interface';import { AuthService } from 'src/services/auth.service';-import { Instrumentation } from 'src/utils/instrumentation';import { handlePromiseError } from 'src/utils/misc';type EmitHandlers = Partial<{ [T in EmitEvent]: Array<EventItem<T>> }>;@@ -37,7 +36,6 @@ type Item<T extends EmitEvent> = {label: string;};-@Instrumentation()@WebSocketGateway({cors: true,path: '/api/socket.io',diff --git a/server/src/repositories/job.repository.ts b/server/src/repositories/job.repository.tsindex 846b6dc9cd..2b783e7d2f 100644--- a/server/src/repositories/job.repository.ts+++ b/server/src/repositories/job.repository.ts@@ -17,7 +17,6 @@ import {QueueStatus,} from 'src/interfaces/job.interface';import { ILoggerRepository } from 'src/interfaces/logger.interface';-import { Instrumentation } from 'src/utils/instrumentation';export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {// misc@@ -99,7 +98,6 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {[JobName.QUEUE_TRASH_EMPTY]: QueueName.BACKGROUND_TASK,};-@Instrumentation()@Injectable()export class JobRepository implements IJobRepository {private workers: Partial<Record<QueueName, Worker>> = {};diff --git a/server/src/repositories/library.repository.ts b/server/src/repositories/library.repository.tsindex 36fb4b9217..1446395854 100644--- a/server/src/repositories/library.repository.ts+++ b/server/src/repositories/library.repository.ts@@ -4,11 +4,9 @@ import { DummyValue, GenerateSql } from 'src/decorators';import { LibraryStatsResponseDto } from 'src/dtos/library.dto';import { LibraryEntity } from 'src/entities/library.entity';import { ILibraryRepository } from 'src/interfaces/library.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { IsNull, Not } from 'typeorm';import { Repository } from 'typeorm/repository/Repository.js';-@Instrumentation()@Injectable()export class LibraryRepository implements ILibraryRepository {constructor(@InjectRepository(LibraryEntity) private repository: Repository<LibraryEntity>) {}diff --git a/server/src/repositories/logger.repository.ts b/server/src/repositories/logger.repository.tsindex 2023cd6c43..4f1d3cac22 100644--- a/server/src/repositories/logger.repository.ts+++ b/server/src/repositories/logger.repository.ts@@ -1,6 +1,7 @@import { ConsoleLogger, Inject, Injectable, Scope } from '@nestjs/common';import { isLogLevelEnabled } from '@nestjs/common/services/utils/is-log-level-enabled.util';import { ClsService } from 'nestjs-cls';+import { Telemetry } from 'src/decorators';import { LogLevel } from 'src/enum';import { IConfigRepository } from 'src/interfaces/config.interface';import { ILoggerRepository } from 'src/interfaces/logger.interface';@@ -17,6 +18,7 @@ enum LogColor {}@Injectable({ scope: Scope.TRANSIENT })+@Telemetry({ enabled: false })export class LoggerRepository extends ConsoleLogger implements ILoggerRepository {private static logLevels: LogLevel[] = [LogLevel.LOG, LogLevel.WARN, LogLevel.ERROR, LogLevel.FATAL];private noColor: boolean;diff --git a/server/src/repositories/machine-learning.repository.ts b/server/src/repositories/machine-learning.repository.tsindex b9404022ef..74b17ca6a7 100644--- a/server/src/repositories/machine-learning.repository.ts+++ b/server/src/repositories/machine-learning.repository.ts@@ -12,11 +12,9 @@ import {ModelTask,ModelType,} from 'src/interfaces/machine-learning.interface';-import { Instrumentation } from 'src/utils/instrumentation';const errorPrefix = 'Machine learning request';-@Instrumentation()@Injectable()export class MachineLearningRepository implements IMachineLearningRepository {private async predict<T>(url: string, payload: ModelPayload, config: MachineLearningRequest): Promise<T> {diff --git a/server/src/repositories/map.repository.ts b/server/src/repositories/map.repository.tsindex 3e5c499f41..7ad94016e8 100644--- a/server/src/repositories/map.repository.ts+++ b/server/src/repositories/map.repository.ts@@ -20,11 +20,9 @@ import {} from 'src/interfaces/map.interface';import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';import { OptionalBetween } from 'src/utils/database';-import { Instrumentation } from 'src/utils/instrumentation';import { DataSource, In, IsNull, Not, QueryRunner, Repository } from 'typeorm';import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';-@Instrumentation()@Injectable()export class MapRepository implements IMapRepository {constructor(diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.tsindex 0777ca3479..64cfc540e5 100644--- a/server/src/repositories/media.repository.ts+++ b/server/src/repositories/media.repository.ts@@ -17,7 +17,6 @@ import {TranscodeCommand,VideoInfo,} from 'src/interfaces/media.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { handlePromiseError } from 'src/utils/misc';const probe = (input: string, options: string[]): Promise<FfprobeData> =>@@ -36,7 +35,6 @@ type ProgressEvent = {percent?: number;};-@Instrumentation()@Injectable()export class MediaRepository implements IMediaRepository {constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) {diff --git a/server/src/repositories/memory.repository.ts b/server/src/repositories/memory.repository.tsindex e9b4532fe9..3c2a1ae191 100644--- a/server/src/repositories/memory.repository.ts+++ b/server/src/repositories/memory.repository.ts@@ -3,10 +3,8 @@ import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';import { Chunked, ChunkedSet, DummyValue, GenerateSql } from 'src/decorators';import { MemoryEntity } from 'src/entities/memory.entity';import { IMemoryRepository } from 'src/interfaces/memory.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { DataSource, In, Repository } from 'typeorm';-@Instrumentation()@Injectable()export class MemoryRepository implements IMemoryRepository {constructor(diff --git a/server/src/repositories/metadata.repository.ts b/server/src/repositories/metadata.repository.tsindex dc2a4cdf9b..81c1b35e15 100644--- a/server/src/repositories/metadata.repository.ts+++ b/server/src/repositories/metadata.repository.ts@@ -3,9 +3,7 @@ import { DefaultReadTaskOptions, ExifTool, Tags } from 'exiftool-vendored';import geotz from 'geo-tz';import { ILoggerRepository } from 'src/interfaces/logger.interface';import { IMetadataRepository, ImmichTags } from 'src/interfaces/metadata.interface';-import { Instrumentation } from 'src/utils/instrumentation';-@Instrumentation()@Injectable()export class MetadataRepository implements IMetadataRepository {private exiftool = new ExifTool({diff --git a/server/src/repositories/move.repository.ts b/server/src/repositories/move.repository.tsindex 45fd446526..16d9004014 100644--- a/server/src/repositories/move.repository.ts+++ b/server/src/repositories/move.repository.ts@@ -4,10 +4,8 @@ import { DummyValue, GenerateSql } from 'src/decorators';import { MoveEntity } from 'src/entities/move.entity';import { PathType } from 'src/enum';import { IMoveRepository, MoveCreate } from 'src/interfaces/move.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { Repository } from 'typeorm';-@Instrumentation()@Injectable()export class MoveRepository implements IMoveRepository {constructor(@InjectRepository(MoveEntity) private repository: Repository<MoveEntity>) {}diff --git a/server/src/repositories/notification.repository.ts b/server/src/repositories/notification.repository.tsindex 9814a7bd5e..293a80576f 100644--- a/server/src/repositories/notification.repository.ts+++ b/server/src/repositories/notification.repository.ts@@ -15,9 +15,7 @@ import {SendEmailResponse,SmtpOptions,} from 'src/interfaces/notification.interface';-import { Instrumentation } from 'src/utils/instrumentation';-@Instrumentation()@Injectable()export class NotificationRepository implements INotificationRepository {constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) {diff --git a/server/src/repositories/oauth.repository.ts b/server/src/repositories/oauth.repository.tsindex adde7099d0..ed038f0137 100644--- a/server/src/repositories/oauth.repository.ts+++ b/server/src/repositories/oauth.repository.ts@@ -2,9 +2,7 @@ import { Inject, Injectable, InternalServerErrorException } from '@nestjs/commonimport { custom, generators, Issuer } from 'openid-client';import { ILoggerRepository } from 'src/interfaces/logger.interface';import { IOAuthRepository, OAuthConfig, OAuthProfile } from 'src/interfaces/oauth.interface';-import { Instrumentation } from 'src/utils/instrumentation';-@Instrumentation()@Injectable()export class OAuthRepository implements IOAuthRepository {constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) {diff --git a/server/src/repositories/partner.repository.ts b/server/src/repositories/partner.repository.tsindex e0c8998dbf..6b11a4e31e 100644--- a/server/src/repositories/partner.repository.ts+++ b/server/src/repositories/partner.repository.ts@@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { PartnerEntity } from 'src/entities/partner.entity';import { IPartnerRepository, PartnerIds } from 'src/interfaces/partner.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { DeepPartial, Repository } from 'typeorm';-@Instrumentation()@Injectable()export class PartnerRepository implements IPartnerRepository {constructor(@InjectRepository(PartnerEntity) private repository: Repository<PartnerEntity>) {}diff --git a/server/src/repositories/person.repository.ts b/server/src/repositories/person.repository.tsindex c62c4b8739..56116d7b3b 100644--- a/server/src/repositories/person.repository.ts+++ b/server/src/repositories/person.repository.ts@@ -20,11 +20,9 @@ import {UnassignFacesOptions,UpdateFacesData,} from 'src/interfaces/person.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { Paginated, PaginationOptions, paginate, paginatedBuilder } from 'src/utils/pagination';import { DataSource, FindManyOptions, FindOptionsRelations, FindOptionsSelect, In, Repository } from 'typeorm';-@Instrumentation()@Injectable()export class PersonRepository implements IPersonRepository {constructor(diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.tsindex 882a2634bd..b5969beecb 100644--- a/server/src/repositories/search.repository.ts+++ b/server/src/repositories/search.repository.ts@@ -23,12 +23,10 @@ import {SmartSearchOptions,} from 'src/interfaces/search.interface';import { asVector, searchAssetBuilder } from 'src/utils/database';-import { Instrumentation } from 'src/utils/instrumentation';import { Paginated, PaginationResult, paginatedBuilder } from 'src/utils/pagination';import { isValidInteger } from 'src/validation';import { Repository } from 'typeorm';-@Instrumentation()@Injectable()export class SearchRepository implements ISearchRepository {private vectorExtension: VectorExtension;diff --git a/server/src/repositories/server-info.repository.ts b/server/src/repositories/server-info.repository.tsindex 1936ecdb61..b4a4652871 100644--- a/server/src/repositories/server-info.repository.ts+++ b/server/src/repositories/server-info.repository.ts@@ -7,7 +7,6 @@ import sharp from 'sharp';import { IConfigRepository } from 'src/interfaces/config.interface';import { ILoggerRepository } from 'src/interfaces/logger.interface';import { GitHubRelease, IServerInfoRepository, ServerBuildVersions } from 'src/interfaces/server-info.interface';-import { Instrumentation } from 'src/utils/instrumentation';const exec = promisify(execCallback);const maybeFirstLine = async (command: string): Promise<string> => {@@ -34,7 +33,6 @@ const getLockfileVersion = (name: string, lockfile?: BuildLockfile) => {return item?.version;};-@Instrumentation()@Injectable()export class ServerInfoRepository implements IServerInfoRepository {constructor(diff --git a/server/src/repositories/session.repository.ts b/server/src/repositories/session.repository.tsindex a4b55a19d7..3a0af1ef69 100644--- a/server/src/repositories/session.repository.ts+++ b/server/src/repositories/session.repository.ts@@ -3,10 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm';import { DummyValue, GenerateSql } from 'src/decorators';import { SessionEntity } from 'src/entities/session.entity';import { ISessionRepository, SessionSearchOptions } from 'src/interfaces/session.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { LessThanOrEqual, Repository } from 'typeorm';-@Instrumentation()@Injectable()export class SessionRepository implements ISessionRepository {constructor(@InjectRepository(SessionEntity) private repository: Repository<SessionEntity>) {}diff --git a/server/src/repositories/shared-link.repository.ts b/server/src/repositories/shared-link.repository.tsindex 48dbb3ab90..1dfde99a75 100644--- a/server/src/repositories/shared-link.repository.ts+++ b/server/src/repositories/shared-link.repository.ts@@ -3,10 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm';import { DummyValue, GenerateSql } from 'src/decorators';import { SharedLinkEntity } from 'src/entities/shared-link.entity';import { ISharedLinkRepository } from 'src/interfaces/shared-link.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { Repository } from 'typeorm';-@Instrumentation()@Injectable()export class SharedLinkRepository implements ISharedLinkRepository {constructor(@InjectRepository(SharedLinkEntity) private repository: Repository<SharedLinkEntity>) {}diff --git a/server/src/repositories/stack.repository.ts b/server/src/repositories/stack.repository.tsindex f23a1c9a9c..ae1a7f70d4 100644--- a/server/src/repositories/stack.repository.ts+++ b/server/src/repositories/stack.repository.ts@@ -3,10 +3,8 @@ import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';import { AssetEntity } from 'src/entities/asset.entity';import { StackEntity } from 'src/entities/stack.entity';import { IStackRepository, StackSearch } from 'src/interfaces/stack.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { DataSource, In, Repository } from 'typeorm';-@Instrumentation()@Injectable()export class StackRepository implements IStackRepository {constructor(diff --git a/server/src/repositories/storage.repository.ts b/server/src/repositories/storage.repository.tsindex b957449984..1ef0e9d6bf 100644--- a/server/src/repositories/storage.repository.ts+++ b/server/src/repositories/storage.repository.ts@@ -14,10 +14,8 @@ import {ImmichZipStream,WatchEvents,} from 'src/interfaces/storage.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { mimeTypes } from 'src/utils/mime-types';-@Instrumentation()@Injectable()export class StorageRepository implements IStorageRepository {constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) {diff --git a/server/src/repositories/system-metadata.repository.ts b/server/src/repositories/system-metadata.repository.tsindex d4e58bf74a..1c6aaf0517 100644--- a/server/src/repositories/system-metadata.repository.ts+++ b/server/src/repositories/system-metadata.repository.ts@@ -3,10 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm';import { readFile } from 'node:fs/promises';import { SystemMetadata, SystemMetadataEntity } from 'src/entities/system-metadata.entity';import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { Repository } from 'typeorm';-@Instrumentation()@Injectable()export class SystemMetadataRepository implements ISystemMetadataRepository {constructor(diff --git a/server/src/repositories/tag.repository.ts b/server/src/repositories/tag.repository.tsindex 1a5415b8db..df5f7e6e42 100644--- a/server/src/repositories/tag.repository.ts+++ b/server/src/repositories/tag.repository.ts@@ -4,10 +4,8 @@ import { Chunked, ChunkedSet, DummyValue, GenerateSql } from 'src/decorators';import { TagEntity } from 'src/entities/tag.entity';import { ILoggerRepository } from 'src/interfaces/logger.interface';import { AssetTagItem, ITagRepository } from 'src/interfaces/tag.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { DataSource, In, Repository, TreeRepository } from 'typeorm';-@Instrumentation()@Injectable()export class TagRepository implements ITagRepository {constructor(diff --git a/server/src/repositories/telemetry.repository.ts b/server/src/repositories/telemetry.repository.tsindex d1dc66ae85..f450c162dc 100644--- a/server/src/repositories/telemetry.repository.ts+++ b/server/src/repositories/telemetry.repository.ts@@ -1,7 +1,22 @@import { Inject, Injectable } from '@nestjs/common';+import { Reflector } from '@nestjs/core';import { MetricOptions } from '@opentelemetry/api';+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';+import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';+import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';+import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';+import { NodeSDK, contextBase, metrics, resources } from '@opentelemetry/sdk-node';+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';+import { ClassConstructor } from 'class-transformer';+import { snakeCase, startCase } from 'lodash';import { MetricService } from 'nestjs-otel';+import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils';+import { serverVersion } from 'src/constants';+import { MetadataKey } from 'src/enum';import { IConfigRepository } from 'src/interfaces/config.interface';+import { ILoggerRepository } from 'src/interfaces/logger.interface';import { IMetricGroupRepository, ITelemetryRepository, MetricGroupOptions } from 'src/interfaces/telemetry.interface';class MetricGroupRepository implements IMetricGroupRepository {@@ -33,6 +48,43 @@ class MetricGroupRepository implements IMetricGroupRepository {}}+const aggregation = new metrics.ExplicitBucketHistogramAggregation(+ [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000],+ true,+);++let instance: NodeSDK | undefined;++export const bootstrapTelemetry = (port: number) => {+ if (instance) {+ throw new Error('OpenTelemetry SDK already started');+ }+ instance = new NodeSDK({+ resource: new resources.Resource({+ [SemanticResourceAttributes.SERVICE_NAME]: `immich`,+ [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),+ }),+ metricReader: new PrometheusExporter({ port }),+ contextManager: new AsyncLocalStorageContextManager(),+ instrumentations: [+ new HttpInstrumentation(),+ new IORedisInstrumentation(),+ new NestInstrumentation(),+ new PgInstrumentation(),+ ],+ views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],+ });++ instance.start();+};++export const teardownTelemetry = async () => {+ if (instance) {+ await instance.shutdown();+ instance = undefined;+ }+};+@Injectable()export class TelemetryRepository implements ITelemetryRepository {api: MetricGroupRepository;@@ -40,8 +92,13 @@ export class TelemetryRepository implements ITelemetryRepository {jobs: MetricGroupRepository;repo: MetricGroupRepository;- constructor(metricService: MetricService, @Inject(IConfigRepository) configRepository: IConfigRepository) {- const { telemetry } = configRepository.getEnv();+ constructor(+ private metricService: MetricService,+ private reflect: Reflector,+ @Inject(IConfigRepository) private configRepository: IConfigRepository,+ @Inject(ILoggerRepository) private logger: ILoggerRepository,+ ) {+ const { telemetry } = this.configRepository.getEnv();const { apiMetrics, hostMetrics, jobMetrics, repoMetrics } = telemetry;this.api = new MetricGroupRepository(metricService).configure({ enabled: apiMetrics });@@ -49,4 +106,61 @@ export class TelemetryRepository implements ITelemetryRepository {this.jobs = new MetricGroupRepository(metricService).configure({ enabled: jobMetrics });this.repo = new MetricGroupRepository(metricService).configure({ enabled: repoMetrics });}++ setup({ repositories }: { repositories: ClassConstructor<unknown>[] }) {+ const { telemetry } = this.configRepository.getEnv();+ if (!telemetry.enabled || !telemetry.repoMetrics) {+ return;+ }++ for (const Repository of repositories) {+ const isEnabled = this.reflect.get(MetadataKey.TELEMETRY_ENABLED, Repository) ?? true;+ if (!isEnabled) {+ this.logger.debug(`Telemetry disabled for ${Repository.name}`);+ continue;+ }++ this.wrap(Repository);+ }+ }++ private wrap(Repository: ClassConstructor<unknown>) {+ const className = Repository.name;+ const descriptors = Object.getOwnPropertyDescriptors(Repository.prototype);+ const unit = 'ms';++ for (const [propName, descriptor] of Object.entries(descriptors)) {+ const isMethod = typeof descriptor.value == 'function' && propName !== 'constructor';+ if (!isMethod) {+ continue;+ }++ const method = descriptor.value;+ const propertyName = snakeCase(String(propName));+ const metricName = `${snakeCase(className).replaceAll(/_(?=(repository)|(controller)|(provider)|(service)|(module))/g, '.')}.${propertyName}.duration`;++ const histogram = this.metricService.getHistogram(metricName, {+ prefix: 'immich',+ description: `The elapsed time in ${unit} for the ${startCase(className)} to ${propertyName.toLowerCase()}`,+ unit,+ valueType: contextBase.ValueType.DOUBLE,+ });++ descriptor.value = function (...args: any[]) {+ const start = performance.now();+ const result = method.apply(this, args);++ void Promise.resolve(result)+ .then(() => histogram.record(performance.now() - start, {}))+ .catch(() => {+ // noop+ });++ return result;+ };++ copyMetadataFromFunctionToFunction(method, descriptor.value);+ Object.defineProperty(Repository.prototype, propName, descriptor);+ }+ }}diff --git a/server/src/repositories/user.repository.ts b/server/src/repositories/user.repository.tsindex c64d5a3655..6ac8536ef8 100644--- a/server/src/repositories/user.repository.ts+++ b/server/src/repositories/user.repository.ts@@ -10,10 +10,8 @@ import {UserListFilter,UserStatsQueryResponse,} from 'src/interfaces/user.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { IsNull, Not, Repository } from 'typeorm';-@Instrumentation()@Injectable()export class UserRepository implements IUserRepository {constructor(diff --git a/server/src/repositories/version-history.repository.ts b/server/src/repositories/version-history.repository.tsindex 26c638bd76..e32ceaf4e9 100644--- a/server/src/repositories/version-history.repository.ts+++ b/server/src/repositories/version-history.repository.ts@@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { VersionHistoryEntity } from 'src/entities/version-history.entity';import { IVersionHistoryRepository } from 'src/interfaces/version-history.interface';-import { Instrumentation } from 'src/utils/instrumentation';import { Repository } from 'typeorm';-@Instrumentation()@Injectable()export class VersionHistoryRepository implements IVersionHistoryRepository {constructor(@InjectRepository(VersionHistoryEntity) private repository: Repository<VersionHistoryEntity>) {}diff --git a/server/src/services/microservices.service.ts b/server/src/services/microservices.service.tsindex d1d2bb8f20..275103d05c 100644--- a/server/src/services/microservices.service.ts+++ b/server/src/services/microservices.service.ts@@ -20,7 +20,6 @@ import { TagService } from 'src/services/tag.service';import { TrashService } from 'src/services/trash.service';import { UserService } from 'src/services/user.service';import { VersionService } from 'src/services/version.service';-import { otelShutdown } from 'src/utils/instrumentation';@Injectable()export class MicroservicesService {@@ -101,8 +100,4 @@ export class MicroservicesService {[JobName.QUEUE_TRASH_EMPTY]: () => this.trashService.handleQueueEmptyTrash(),});}-- async onShutdown() {- await otelShutdown();- }}diff --git a/server/src/utils/instrumentation.ts b/server/src/utils/instrumentation.tsdeleted file mode 100644index bd522f27b2..0000000000--- a/server/src/utils/instrumentation.ts+++ /dev/null@@ -1,100 +0,0 @@-import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';-import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';-import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';-import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';-import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';-import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';-import { NodeSDK, contextBase, metrics, resources } from '@opentelemetry/sdk-node';-import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';-import { snakeCase, startCase } from 'lodash';-import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils';-import { performance } from 'node:perf_hooks';-import { serverVersion } from 'src/constants';-import { DecorateAll } from 'src/decorators';-import { ConfigRepository } from 'src/repositories/config.repository';--const aggregation = new metrics.ExplicitBucketHistogramAggregation(- [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000],- true,-);--const { telemetry } = new ConfigRepository().getEnv();--let otelSingleton: NodeSDK | undefined;--export const otelStart = (port: number) => {- if (otelSingleton) {- throw new Error('OpenTelemetry SDK already started');- }- otelSingleton = new NodeSDK({- resource: new resources.Resource({- [SemanticResourceAttributes.SERVICE_NAME]: `immich`,- [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),- }),- metricReader: new PrometheusExporter({ port }),- contextManager: new AsyncLocalStorageContextManager(),- instrumentations: [- new HttpInstrumentation(),- new IORedisInstrumentation(),- new NestInstrumentation(),- new PgInstrumentation(),- ],- views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],- });- otelSingleton.start();-};--export const otelShutdown = async () => {- if (otelSingleton) {- await otelSingleton.shutdown();- otelSingleton = undefined;- }-};--function ExecutionTimeHistogram({- description,- unit = 'ms',- valueType = contextBase.ValueType.DOUBLE,-}: contextBase.MetricOptions = {}) {- return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {- if (!telemetry.repoMetrics || process.env.OTEL_SDK_DISABLED) {- return;- }-- const method = descriptor.value;- const className = target.constructor.name as string;- const propertyName = String(propertyKey);- const metricName = `${snakeCase(className).replaceAll(/_(?=(repository)|(controller)|(provider)|(service)|(module))/g, '.')}.${snakeCase(propertyName)}.duration`;-- const metricDescription =- description ??- `The elapsed time in ${unit} for the ${startCase(className)} to ${startCase(propertyName).toLowerCase()}`;-- let histogram: contextBase.Histogram | undefined;-- descriptor.value = function (...args: any[]) {- const start = performance.now();- const result = method.apply(this, args);-- void Promise.resolve(result)- .then(() => {- const end = performance.now();- if (!histogram) {- histogram = contextBase.metrics- .getMeter('immich')- .createHistogram(metricName, { description: metricDescription, unit, valueType });- }- histogram.record(end - start, {});- })- .catch(() => {- // noop- });-- return result;- };-- copyMetadataFromFunctionToFunction(method, descriptor.value);- };-}--export const Instrumentation = () => DecorateAll(ExecutionTimeHistogram());diff --git a/server/src/workers/api.ts b/server/src/workers/api.tsindex b369b56953..6451f1b792 100644--- a/server/src/workers/api.ts+++ b/server/src/workers/api.ts@@ -11,16 +11,18 @@ import { IConfigRepository } from 'src/interfaces/config.interface';import { ILoggerRepository } from 'src/interfaces/logger.interface';import { WebSocketAdapter } from 'src/middleware/websocket.adapter';import { ConfigRepository } from 'src/repositories/config.repository';+import { bootstrapTelemetry } from 'src/repositories/telemetry.repository';import { ApiService } from 'src/services/api.service';import { isStartUpError } from 'src/services/storage.service';-import { otelStart } from 'src/utils/instrumentation';import { useSwagger } from 'src/utils/misc';async function bootstrap() {process.title = 'immich-api';const { telemetry, network } = new ConfigRepository().getEnv();- otelStart(telemetry.apiPort);+ if (telemetry.enabled) {+ bootstrapTelemetry(telemetry.apiPort);+ }const app = await NestFactory.create<NestExpressApplication>(ApiModule, { bufferLogs: true });const logger = await app.resolve<ILoggerRepository>(ILoggerRepository);diff --git a/server/src/workers/microservices.ts b/server/src/workers/microservices.tsindex 7b60fb8db6..df4abb01da 100644--- a/server/src/workers/microservices.ts+++ b/server/src/workers/microservices.ts@@ -6,12 +6,14 @@ import { IConfigRepository } from 'src/interfaces/config.interface';import { ILoggerRepository } from 'src/interfaces/logger.interface';import { WebSocketAdapter } from 'src/middleware/websocket.adapter';import { ConfigRepository } from 'src/repositories/config.repository';+import { bootstrapTelemetry } from 'src/repositories/telemetry.repository';import { isStartUpError } from 'src/services/storage.service';-import { otelStart } from 'src/utils/instrumentation';export async function bootstrap() {const { telemetry } = new ConfigRepository().getEnv();- otelStart(telemetry.microservicesPort);+ if (telemetry.enabled) {+ bootstrapTelemetry(telemetry.microservicesPort);+ }const app = await NestFactory.create(MicroservicesModule, { bufferLogs: true });const logger = await app.resolve(ILoggerRepository);diff --git a/server/test/repositories/telemetry.repository.mock.ts b/server/test/repositories/telemetry.repository.mock.tsindex 737463065c..2d537e888a 100644--- a/server/test/repositories/telemetry.repository.mock.ts+++ b/server/test/repositories/telemetry.repository.mock.ts@@ -12,6 +12,7 @@ const newMetricGroupMock = () => {export const newTelemetryRepositoryMock = (): Mocked<ITelemetryRepository> => {return {+ setup: vitest.fn(),api: newMetricGroupMock(),host: newMetricGroupMock(),jobs: newMetricGroupMock(),