Skip to content

High performance self-hosted photo and video management solution.

refactor(server): telemetry (#13588)

Jason Rasmussene6a666f1d307
diff --git a/server/src/app.module.ts b/server/src/app.module.ts
index fd921150fd..3c26faaca3 100644
--- a/server/src/app.module.ts
+++ b/server/src/app.module.ts
@@ -14,6 +14,7 @@ import { entities } from 'src/entities';
import { ImmichWorker } from 'src/enum';
import { IEventRepository } from 'src/interfaces/event.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
+import { ITelemetryRepository } from 'src/interfaces/telemetry.interface';
import { AuthGuard } from 'src/middleware/auth.guard';
import { ErrorInterceptor } from 'src/middleware/error.interceptor';
import { FileUploadInterceptor } from 'src/middleware/file-upload.interceptor';
@@ -21,6 +22,7 @@ import { GlobalExceptionFilter } from 'src/middleware/global-exception.filter';
import { LoggingInterceptor } from 'src/middleware/logging.interceptor';
import { repositories } from 'src/repositories';
import { ConfigRepository } from 'src/repositories/config.repository';
+import { teardownTelemetry } from 'src/repositories/telemetry.repository';
import { services } from 'src/services';
import { DatabaseService } from 'src/services/database.service';
@@ -66,6 +68,7 @@ abstract class BaseModule implements OnModuleInit, OnModuleDestroy {
constructor(
@Inject(ILoggerRepository) logger: ILoggerRepository,
@Inject(IEventRepository) private eventRepository: IEventRepository,
+ @Inject(ITelemetryRepository) private telemetryRepository: ITelemetryRepository,
) {
logger.setAppName(this.worker);
}
@@ -73,12 +76,14 @@ abstract class BaseModule implements OnModuleInit, OnModuleDestroy {
abstract getWorker(): ImmichWorker;
async onModuleInit() {
+ this.telemetryRepository.setup({ repositories: repositories.map(({ useClass }) => useClass) });
this.eventRepository.setup({ services });
await this.eventRepository.emit('app.bootstrap', this.worker);
}
async onModuleDestroy() {
await this.eventRepository.emit('app.shutdown', this.worker);
+ await teardownTelemetry();
}
}
diff --git a/server/src/config.ts b/server/src/config.ts
index e386c134b4..fca6719bc0 100644
--- a/server/src/config.ts
+++ b/server/src/config.ts
@@ -354,9 +354,9 @@ export const immichAppConfig: ConfigModuleOptions = {
),
IMMICH_METRICS: Joi.boolean().optional().default(false),
- IMMICH_HOST_METRICS: Joi.boolean().optional().default(false),
- IMMICH_API_METRICS: Joi.boolean().optional().default(false),
- IMMICH_IO_METRICS: Joi.boolean().optional().default(false),
+ IMMICH_HOST_METRICS: Joi.boolean().optional(),
+ IMMICH_API_METRICS: Joi.boolean().optional(),
+ IMMICH_IO_METRICS: Joi.boolean().optional(),
}),
};
diff --git a/server/src/decorators.ts b/server/src/decorators.ts
index 2782368239..db755c5ff9 100644
--- a/server/src/decorators.ts
+++ b/server/src/decorators.ts
@@ -86,27 +86,6 @@ export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {
return Chunked({ ...options, mergeFn: setUnion });
}
-// https://stackoverflow.com/a/74898678
-export function DecorateAll(
- decorator: <T>(
- target: any,
- propertyKey: string,
- descriptor: TypedPropertyDescriptor<T>,
- ) => TypedPropertyDescriptor<T> | void,
-) {
- return (target: any) => {
- const descriptors = Object.getOwnPropertyDescriptors(target.prototype);
- for (const [propName, descriptor] of Object.entries(descriptors)) {
- const isMethod = typeof descriptor.value == 'function' && propName !== 'constructor';
- if (!isMethod) {
- continue;
- }
- decorator({ ...target, constructor: { ...target.constructor, name: target.name } as any }, propName, descriptor);
- Object.defineProperty(target.prototype, propName, descriptor);
- }
- };
-}
-
const UUID = '00000000-0000-4000-a000-000000000000';
export const DummyValue = {
@@ -128,6 +107,9 @@ export interface GenerateSqlQueries {
params: unknown[];
}
+export const Telemetry = (options: { enabled?: boolean }) =>
+ SetMetadata(MetadataKey.TELEMETRY_ENABLED, options?.enabled ?? true);
+
/** Decorator to enable versioning/tracking of generated Sql */
export const GenerateSql = (...options: GenerateSqlQueries[]) => SetMetadata(GENERATE_SQL_KEY, options);
diff --git a/server/src/enum.ts b/server/src/enum.ts
index 8c11834dac..902d6635e7 100644
--- a/server/src/enum.ts
+++ b/server/src/enum.ts
@@ -334,6 +334,7 @@ export enum MetadataKey {
SHARED_ROUTE = 'shared_route',
API_KEY_SECURITY = 'api_key',
EVENT_CONFIG = 'event_config',
+ TELEMETRY_ENABLED = 'telemetry_enabled',
}
export enum RouteKey {
diff --git a/server/src/interfaces/telemetry.interface.ts b/server/src/interfaces/telemetry.interface.ts
index 070014f2e0..688e52c21e 100644
--- a/server/src/interfaces/telemetry.interface.ts
+++ b/server/src/interfaces/telemetry.interface.ts
@@ -1,4 +1,5 @@
import { MetricOptions } from '@opentelemetry/api';
+import { ClassConstructor } from 'class-transformer';
export const ITelemetryRepository = 'ITelemetryRepository';
@@ -14,6 +15,7 @@ export interface IMetricGroupRepository {
}
export interface ITelemetryRepository {
+ setup(options: { repositories: ClassConstructor<unknown>[] }): void;
api: IMetricGroupRepository;
host: IMetricGroupRepository;
jobs: IMetricGroupRepository;
diff --git a/server/src/repositories/access.repository.ts b/server/src/repositories/access.repository.ts
index f6921ffe27..f3cbf392db 100644
--- a/server/src/repositories/access.repository.ts
+++ b/server/src/repositories/access.repository.ts
@@ -15,7 +15,6 @@ import { StackEntity } from 'src/entities/stack.entity';
import { TagEntity } from 'src/entities/tag.entity';
import { AlbumUserRole } from 'src/enum';
import { IAccessRepository } from 'src/interfaces/access.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { Brackets, In, Repository } from 'typeorm';
type IActivityAccess = IAccessRepository['activity'];
@@ -29,7 +28,6 @@ type IStackAccess = IAccessRepository['stack'];
type ITagAccess = IAccessRepository['tag'];
type ITimelineAccess = IAccessRepository['timeline'];
-@Instrumentation()
@Injectable()
class ActivityAccess implements IActivityAccess {
constructor(
diff --git a/server/src/repositories/activity.repository.ts b/server/src/repositories/activity.repository.ts
index e21f746483..0f0a0cb60e 100644
--- a/server/src/repositories/activity.repository.ts
+++ b/server/src/repositories/activity.repository.ts
@@ -3,7 +3,6 @@ import { InjectRepository } from '@nestjs/typeorm';
import { DummyValue, GenerateSql } from 'src/decorators';
import { ActivityEntity } from 'src/entities/activity.entity';
import { IActivityRepository } from 'src/interfaces/activity.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { IsNull, Repository } from 'typeorm';
export interface ActivitySearch {
@@ -13,7 +12,6 @@ export interface ActivitySearch {
isLiked?: boolean;
}
-@Instrumentation()
@Injectable()
export class ActivityRepository implements IActivityRepository {
constructor(@InjectRepository(ActivityEntity) private repository: Repository<ActivityEntity>) {}
diff --git a/server/src/repositories/album-user.repository.ts b/server/src/repositories/album-user.repository.ts
index 7fd18711aa..9328ea8cfc 100644
--- a/server/src/repositories/album-user.repository.ts
+++ b/server/src/repositories/album-user.repository.ts
@@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { AlbumUserEntity } from 'src/entities/album-user.entity';
import { AlbumPermissionId, IAlbumUserRepository } from 'src/interfaces/album-user.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class AlbumUserRepository implements IAlbumUserRepository {
constructor(@InjectRepository(AlbumUserEntity) private repository: Repository<AlbumUserEntity>) {}
diff --git a/server/src/repositories/album.repository.ts b/server/src/repositories/album.repository.ts
index f7b4cb44aa..8b7565e318 100644
--- a/server/src/repositories/album.repository.ts
+++ b/server/src/repositories/album.repository.ts
@@ -4,7 +4,6 @@ import { Chunked, ChunkedArray, ChunkedSet, DummyValue, GenerateSql } from 'src/
import { AlbumEntity } from 'src/entities/album.entity';
import { AssetEntity } from 'src/entities/asset.entity';
import { AlbumAssetCount, AlbumInfoOptions, IAlbumRepository } from 'src/interfaces/album.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import {
DataSource,
EntityManager,
@@ -23,7 +22,6 @@ const withoutDeletedUsers = <T extends AlbumEntity | null>(album: T) => {
return album;
};
-@Instrumentation()
@Injectable()
export class AlbumRepository implements IAlbumRepository {
constructor(
diff --git a/server/src/repositories/api-key.repository.ts b/server/src/repositories/api-key.repository.ts
index 5178039177..bb37390de1 100644
--- a/server/src/repositories/api-key.repository.ts
+++ b/server/src/repositories/api-key.repository.ts
@@ -3,10 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm';
import { DummyValue, GenerateSql } from 'src/decorators';
import { APIKeyEntity } from 'src/entities/api-key.entity';
import { IKeyRepository } from 'src/interfaces/api-key.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class ApiKeyRepository implements IKeyRepository {
constructor(@InjectRepository(APIKeyEntity) private repository: Repository<APIKeyEntity>) {}
diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts
index fd47a976a5..6080e943e4 100644
--- a/server/src/repositories/asset.repository.ts
+++ b/server/src/repositories/asset.repository.ts
@@ -29,7 +29,6 @@ import {
} from 'src/interfaces/asset.interface';
import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface';
import { searchAssetBuilder } from 'src/utils/database';
-import { Instrumentation } from 'src/utils/instrumentation';
import { Paginated, PaginationOptions, paginate, paginatedBuilder } from 'src/utils/pagination';
import {
Brackets,
@@ -54,7 +53,6 @@ const dateTrunc = (options: TimeBucketOptions) =>
truncateMap[options.size]
}', (asset."localDateTime" at time zone 'UTC')) at time zone 'UTC')::timestamptz`;
-@Instrumentation()
@Injectable()
export class AssetRepository implements IAssetRepository {
constructor(
diff --git a/server/src/repositories/audit.repository.ts b/server/src/repositories/audit.repository.ts
index deb0d0f6f1..ac73c3a8b9 100644
--- a/server/src/repositories/audit.repository.ts
+++ b/server/src/repositories/audit.repository.ts
@@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { AuditEntity } from 'src/entities/audit.entity';
import { AuditSearch, IAuditRepository } from 'src/interfaces/audit.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { In, LessThan, MoreThan, Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class AuditRepository implements IAuditRepository {
constructor(@InjectRepository(AuditEntity) private repository: Repository<AuditEntity>) {}
diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts
index 44b8c7b605..fabccd7846 100644
--- a/server/src/repositories/config.repository.ts
+++ b/server/src/repositories/config.repository.ts
@@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import { join } from 'node:path';
import { citiesFile, excludePaths } from 'src/constants';
+import { Telemetry } from 'src/decorators';
import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';
import { EnvData, IConfigRepository } from 'src/interfaces/config.interface';
import { DatabaseExtension } from 'src/interfaces/database.interface';
@@ -74,9 +75,6 @@ const getEnv = (): EnvData => {
const repoMetrics = parseBoolean(process.env.IMMICH_IO_METRICS, globalEnabled);
const jobMetrics = parseBoolean(process.env.IMMICH_JOB_METRICS, globalEnabled);
const telemetryEnabled = globalEnabled || hostMetrics || apiMetrics || repoMetrics || jobMetrics;
- if (!telemetryEnabled && process.env.OTEL_SDK_DISABLED === undefined) {
- process.env.OTEL_SDK_DISABLED = 'true';
- }
return {
host: process.env.IMMICH_HOST,
@@ -186,6 +184,7 @@ const getEnv = (): EnvData => {
let cached: EnvData | undefined;
@Injectable()
+@Telemetry({ enabled: false })
export class ConfigRepository implements IConfigRepository {
getEnv(): EnvData {
if (!cached) {
diff --git a/server/src/repositories/crypto.repository.ts b/server/src/repositories/crypto.repository.ts
index 72e75ef174..ee25609fec 100644
--- a/server/src/repositories/crypto.repository.ts
+++ b/server/src/repositories/crypto.repository.ts
@@ -3,9 +3,7 @@ import { compareSync, hash } from 'bcrypt';
import { createHash, createPublicKey, createVerify, randomBytes, randomUUID } from 'node:crypto';
import { createReadStream } from 'node:fs';
import { ICryptoRepository } from 'src/interfaces/crypto.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
-@Instrumentation()
@Injectable()
export class CryptoRepository implements ICryptoRepository {
randomUUID() {
diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.ts
index 547f03fc20..b5e2edfdea 100644
--- a/server/src/repositories/database.repository.ts
+++ b/server/src/repositories/database.repository.ts
@@ -15,11 +15,9 @@ import {
VectorUpdateResult,
} from 'src/interfaces/database.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { isValidInteger } from 'src/validation';
import { DataSource, EntityManager, QueryRunner } from 'typeorm';
-@Instrumentation()
@Injectable()
export class DatabaseRepository implements IDatabaseRepository {
private vectorExtension: VectorExtension;
diff --git a/server/src/repositories/event.repository.ts b/server/src/repositories/event.repository.ts
index cb58d56b2a..bb265196f9 100644
--- a/server/src/repositories/event.repository.ts
+++ b/server/src/repositories/event.repository.ts
@@ -24,7 +24,6 @@ import {
} from 'src/interfaces/event.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { AuthService } from 'src/services/auth.service';
-import { Instrumentation } from 'src/utils/instrumentation';
import { handlePromiseError } from 'src/utils/misc';
type EmitHandlers = Partial<{ [T in EmitEvent]: Array<EventItem<T>> }>;
@@ -37,7 +36,6 @@ type Item<T extends EmitEvent> = {
label: string;
};
-@Instrumentation()
@WebSocketGateway({
cors: true,
path: '/api/socket.io',
diff --git a/server/src/repositories/job.repository.ts b/server/src/repositories/job.repository.ts
index 846b6dc9cd..2b783e7d2f 100644
--- a/server/src/repositories/job.repository.ts
+++ b/server/src/repositories/job.repository.ts
@@ -17,7 +17,6 @@ import {
QueueStatus,
} from 'src/interfaces/job.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
// misc
@@ -99,7 +98,6 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
[JobName.QUEUE_TRASH_EMPTY]: QueueName.BACKGROUND_TASK,
};
-@Instrumentation()
@Injectable()
export class JobRepository implements IJobRepository {
private workers: Partial<Record<QueueName, Worker>> = {};
diff --git a/server/src/repositories/library.repository.ts b/server/src/repositories/library.repository.ts
index 36fb4b9217..1446395854 100644
--- a/server/src/repositories/library.repository.ts
+++ b/server/src/repositories/library.repository.ts
@@ -4,11 +4,9 @@ import { DummyValue, GenerateSql } from 'src/decorators';
import { LibraryStatsResponseDto } from 'src/dtos/library.dto';
import { LibraryEntity } from 'src/entities/library.entity';
import { ILibraryRepository } from 'src/interfaces/library.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { IsNull, Not } from 'typeorm';
import { Repository } from 'typeorm/repository/Repository.js';
-@Instrumentation()
@Injectable()
export class LibraryRepository implements ILibraryRepository {
constructor(@InjectRepository(LibraryEntity) private repository: Repository<LibraryEntity>) {}
diff --git a/server/src/repositories/logger.repository.ts b/server/src/repositories/logger.repository.ts
index 2023cd6c43..4f1d3cac22 100644
--- a/server/src/repositories/logger.repository.ts
+++ b/server/src/repositories/logger.repository.ts
@@ -1,6 +1,7 @@
import { ConsoleLogger, Inject, Injectable, Scope } from '@nestjs/common';
import { isLogLevelEnabled } from '@nestjs/common/services/utils/is-log-level-enabled.util';
import { ClsService } from 'nestjs-cls';
+import { Telemetry } from 'src/decorators';
import { LogLevel } from 'src/enum';
import { IConfigRepository } from 'src/interfaces/config.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
@@ -17,6 +18,7 @@ enum LogColor {
}
@Injectable({ scope: Scope.TRANSIENT })
+@Telemetry({ enabled: false })
export class LoggerRepository extends ConsoleLogger implements ILoggerRepository {
private static logLevels: LogLevel[] = [LogLevel.LOG, LogLevel.WARN, LogLevel.ERROR, LogLevel.FATAL];
private noColor: boolean;
diff --git a/server/src/repositories/machine-learning.repository.ts b/server/src/repositories/machine-learning.repository.ts
index b9404022ef..74b17ca6a7 100644
--- a/server/src/repositories/machine-learning.repository.ts
+++ b/server/src/repositories/machine-learning.repository.ts
@@ -12,11 +12,9 @@ import {
ModelTask,
ModelType,
} from 'src/interfaces/machine-learning.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
const errorPrefix = 'Machine learning request';
-@Instrumentation()
@Injectable()
export class MachineLearningRepository implements IMachineLearningRepository {
private async predict<T>(url: string, payload: ModelPayload, config: MachineLearningRequest): Promise<T> {
diff --git a/server/src/repositories/map.repository.ts b/server/src/repositories/map.repository.ts
index 3e5c499f41..7ad94016e8 100644
--- a/server/src/repositories/map.repository.ts
+++ b/server/src/repositories/map.repository.ts
@@ -20,11 +20,9 @@ import {
} from 'src/interfaces/map.interface';
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
import { OptionalBetween } from 'src/utils/database';
-import { Instrumentation } from 'src/utils/instrumentation';
import { DataSource, In, IsNull, Not, QueryRunner, Repository } from 'typeorm';
import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
-@Instrumentation()
@Injectable()
export class MapRepository implements IMapRepository {
constructor(
diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts
index 0777ca3479..64cfc540e5 100644
--- a/server/src/repositories/media.repository.ts
+++ b/server/src/repositories/media.repository.ts
@@ -17,7 +17,6 @@ import {
TranscodeCommand,
VideoInfo,
} from 'src/interfaces/media.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { handlePromiseError } from 'src/utils/misc';
const probe = (input: string, options: string[]): Promise<FfprobeData> =>
@@ -36,7 +35,6 @@ type ProgressEvent = {
percent?: number;
};
-@Instrumentation()
@Injectable()
export class MediaRepository implements IMediaRepository {
constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) {
diff --git a/server/src/repositories/memory.repository.ts b/server/src/repositories/memory.repository.ts
index e9b4532fe9..3c2a1ae191 100644
--- a/server/src/repositories/memory.repository.ts
+++ b/server/src/repositories/memory.repository.ts
@@ -3,10 +3,8 @@ import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { Chunked, ChunkedSet, DummyValue, GenerateSql } from 'src/decorators';
import { MemoryEntity } from 'src/entities/memory.entity';
import { IMemoryRepository } from 'src/interfaces/memory.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { DataSource, In, Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class MemoryRepository implements IMemoryRepository {
constructor(
diff --git a/server/src/repositories/metadata.repository.ts b/server/src/repositories/metadata.repository.ts
index dc2a4cdf9b..81c1b35e15 100644
--- a/server/src/repositories/metadata.repository.ts
+++ b/server/src/repositories/metadata.repository.ts
@@ -3,9 +3,7 @@ import { DefaultReadTaskOptions, ExifTool, Tags } from 'exiftool-vendored';
import geotz from 'geo-tz';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IMetadataRepository, ImmichTags } from 'src/interfaces/metadata.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
-@Instrumentation()
@Injectable()
export class MetadataRepository implements IMetadataRepository {
private exiftool = new ExifTool({
diff --git a/server/src/repositories/move.repository.ts b/server/src/repositories/move.repository.ts
index 45fd446526..16d9004014 100644
--- a/server/src/repositories/move.repository.ts
+++ b/server/src/repositories/move.repository.ts
@@ -4,10 +4,8 @@ import { DummyValue, GenerateSql } from 'src/decorators';
import { MoveEntity } from 'src/entities/move.entity';
import { PathType } from 'src/enum';
import { IMoveRepository, MoveCreate } from 'src/interfaces/move.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class MoveRepository implements IMoveRepository {
constructor(@InjectRepository(MoveEntity) private repository: Repository<MoveEntity>) {}
diff --git a/server/src/repositories/notification.repository.ts b/server/src/repositories/notification.repository.ts
index 9814a7bd5e..293a80576f 100644
--- a/server/src/repositories/notification.repository.ts
+++ b/server/src/repositories/notification.repository.ts
@@ -15,9 +15,7 @@ import {
SendEmailResponse,
SmtpOptions,
} from 'src/interfaces/notification.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
-@Instrumentation()
@Injectable()
export class NotificationRepository implements INotificationRepository {
constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) {
diff --git a/server/src/repositories/oauth.repository.ts b/server/src/repositories/oauth.repository.ts
index adde7099d0..ed038f0137 100644
--- a/server/src/repositories/oauth.repository.ts
+++ b/server/src/repositories/oauth.repository.ts
@@ -2,9 +2,7 @@ import { Inject, Injectable, InternalServerErrorException } from '@nestjs/common
import { custom, generators, Issuer } from 'openid-client';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IOAuthRepository, OAuthConfig, OAuthProfile } from 'src/interfaces/oauth.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
-@Instrumentation()
@Injectable()
export class OAuthRepository implements IOAuthRepository {
constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) {
diff --git a/server/src/repositories/partner.repository.ts b/server/src/repositories/partner.repository.ts
index e0c8998dbf..6b11a4e31e 100644
--- a/server/src/repositories/partner.repository.ts
+++ b/server/src/repositories/partner.repository.ts
@@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { PartnerEntity } from 'src/entities/partner.entity';
import { IPartnerRepository, PartnerIds } from 'src/interfaces/partner.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { DeepPartial, Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class PartnerRepository implements IPartnerRepository {
constructor(@InjectRepository(PartnerEntity) private repository: Repository<PartnerEntity>) {}
diff --git a/server/src/repositories/person.repository.ts b/server/src/repositories/person.repository.ts
index c62c4b8739..56116d7b3b 100644
--- a/server/src/repositories/person.repository.ts
+++ b/server/src/repositories/person.repository.ts
@@ -20,11 +20,9 @@ import {
UnassignFacesOptions,
UpdateFacesData,
} from 'src/interfaces/person.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { Paginated, PaginationOptions, paginate, paginatedBuilder } from 'src/utils/pagination';
import { DataSource, FindManyOptions, FindOptionsRelations, FindOptionsSelect, In, Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class PersonRepository implements IPersonRepository {
constructor(
diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts
index 882a2634bd..b5969beecb 100644
--- a/server/src/repositories/search.repository.ts
+++ b/server/src/repositories/search.repository.ts
@@ -23,12 +23,10 @@ import {
SmartSearchOptions,
} from 'src/interfaces/search.interface';
import { asVector, searchAssetBuilder } from 'src/utils/database';
-import { Instrumentation } from 'src/utils/instrumentation';
import { Paginated, PaginationResult, paginatedBuilder } from 'src/utils/pagination';
import { isValidInteger } from 'src/validation';
import { Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class SearchRepository implements ISearchRepository {
private vectorExtension: VectorExtension;
diff --git a/server/src/repositories/server-info.repository.ts b/server/src/repositories/server-info.repository.ts
index 1936ecdb61..b4a4652871 100644
--- a/server/src/repositories/server-info.repository.ts
+++ b/server/src/repositories/server-info.repository.ts
@@ -7,7 +7,6 @@ import sharp from 'sharp';
import { IConfigRepository } from 'src/interfaces/config.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { GitHubRelease, IServerInfoRepository, ServerBuildVersions } from 'src/interfaces/server-info.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
const exec = promisify(execCallback);
const maybeFirstLine = async (command: string): Promise<string> => {
@@ -34,7 +33,6 @@ const getLockfileVersion = (name: string, lockfile?: BuildLockfile) => {
return item?.version;
};
-@Instrumentation()
@Injectable()
export class ServerInfoRepository implements IServerInfoRepository {
constructor(
diff --git a/server/src/repositories/session.repository.ts b/server/src/repositories/session.repository.ts
index a4b55a19d7..3a0af1ef69 100644
--- a/server/src/repositories/session.repository.ts
+++ b/server/src/repositories/session.repository.ts
@@ -3,10 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm';
import { DummyValue, GenerateSql } from 'src/decorators';
import { SessionEntity } from 'src/entities/session.entity';
import { ISessionRepository, SessionSearchOptions } from 'src/interfaces/session.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { LessThanOrEqual, Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class SessionRepository implements ISessionRepository {
constructor(@InjectRepository(SessionEntity) private repository: Repository<SessionEntity>) {}
diff --git a/server/src/repositories/shared-link.repository.ts b/server/src/repositories/shared-link.repository.ts
index 48dbb3ab90..1dfde99a75 100644
--- a/server/src/repositories/shared-link.repository.ts
+++ b/server/src/repositories/shared-link.repository.ts
@@ -3,10 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm';
import { DummyValue, GenerateSql } from 'src/decorators';
import { SharedLinkEntity } from 'src/entities/shared-link.entity';
import { ISharedLinkRepository } from 'src/interfaces/shared-link.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class SharedLinkRepository implements ISharedLinkRepository {
constructor(@InjectRepository(SharedLinkEntity) private repository: Repository<SharedLinkEntity>) {}
diff --git a/server/src/repositories/stack.repository.ts b/server/src/repositories/stack.repository.ts
index f23a1c9a9c..ae1a7f70d4 100644
--- a/server/src/repositories/stack.repository.ts
+++ b/server/src/repositories/stack.repository.ts
@@ -3,10 +3,8 @@ import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { AssetEntity } from 'src/entities/asset.entity';
import { StackEntity } from 'src/entities/stack.entity';
import { IStackRepository, StackSearch } from 'src/interfaces/stack.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { DataSource, In, Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class StackRepository implements IStackRepository {
constructor(
diff --git a/server/src/repositories/storage.repository.ts b/server/src/repositories/storage.repository.ts
index b957449984..1ef0e9d6bf 100644
--- a/server/src/repositories/storage.repository.ts
+++ b/server/src/repositories/storage.repository.ts
@@ -14,10 +14,8 @@ import {
ImmichZipStream,
WatchEvents,
} from 'src/interfaces/storage.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { mimeTypes } from 'src/utils/mime-types';
-@Instrumentation()
@Injectable()
export class StorageRepository implements IStorageRepository {
constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) {
diff --git a/server/src/repositories/system-metadata.repository.ts b/server/src/repositories/system-metadata.repository.ts
index d4e58bf74a..1c6aaf0517 100644
--- a/server/src/repositories/system-metadata.repository.ts
+++ b/server/src/repositories/system-metadata.repository.ts
@@ -3,10 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm';
import { readFile } from 'node:fs/promises';
import { SystemMetadata, SystemMetadataEntity } from 'src/entities/system-metadata.entity';
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class SystemMetadataRepository implements ISystemMetadataRepository {
constructor(
diff --git a/server/src/repositories/tag.repository.ts b/server/src/repositories/tag.repository.ts
index 1a5415b8db..df5f7e6e42 100644
--- a/server/src/repositories/tag.repository.ts
+++ b/server/src/repositories/tag.repository.ts
@@ -4,10 +4,8 @@ import { Chunked, ChunkedSet, DummyValue, GenerateSql } from 'src/decorators';
import { TagEntity } from 'src/entities/tag.entity';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { AssetTagItem, ITagRepository } from 'src/interfaces/tag.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { DataSource, In, Repository, TreeRepository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class TagRepository implements ITagRepository {
constructor(
diff --git a/server/src/repositories/telemetry.repository.ts b/server/src/repositories/telemetry.repository.ts
index d1dc66ae85..f450c162dc 100644
--- a/server/src/repositories/telemetry.repository.ts
+++ b/server/src/repositories/telemetry.repository.ts
@@ -1,7 +1,22 @@
import { Inject, Injectable } from '@nestjs/common';
+import { Reflector } from '@nestjs/core';
import { MetricOptions } from '@opentelemetry/api';
+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
+import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
+import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';
+import { NodeSDK, contextBase, metrics, resources } from '@opentelemetry/sdk-node';
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+import { ClassConstructor } from 'class-transformer';
+import { snakeCase, startCase } from 'lodash';
import { MetricService } from 'nestjs-otel';
+import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils';
+import { serverVersion } from 'src/constants';
+import { MetadataKey } from 'src/enum';
import { IConfigRepository } from 'src/interfaces/config.interface';
+import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IMetricGroupRepository, ITelemetryRepository, MetricGroupOptions } from 'src/interfaces/telemetry.interface';
class MetricGroupRepository implements IMetricGroupRepository {
@@ -33,6 +48,43 @@ class MetricGroupRepository implements IMetricGroupRepository {
}
}
+const aggregation = new metrics.ExplicitBucketHistogramAggregation(
+ [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000],
+ true,
+);
+
+let instance: NodeSDK | undefined;
+
+export const bootstrapTelemetry = (port: number) => {
+ if (instance) {
+ throw new Error('OpenTelemetry SDK already started');
+ }
+ instance = new NodeSDK({
+ resource: new resources.Resource({
+ [SemanticResourceAttributes.SERVICE_NAME]: `immich`,
+ [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),
+ }),
+ metricReader: new PrometheusExporter({ port }),
+ contextManager: new AsyncLocalStorageContextManager(),
+ instrumentations: [
+ new HttpInstrumentation(),
+ new IORedisInstrumentation(),
+ new NestInstrumentation(),
+ new PgInstrumentation(),
+ ],
+ views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],
+ });
+
+ instance.start();
+};
+
+export const teardownTelemetry = async () => {
+ if (instance) {
+ await instance.shutdown();
+ instance = undefined;
+ }
+};
+
@Injectable()
export class TelemetryRepository implements ITelemetryRepository {
api: MetricGroupRepository;
@@ -40,8 +92,13 @@ export class TelemetryRepository implements ITelemetryRepository {
jobs: MetricGroupRepository;
repo: MetricGroupRepository;
- constructor(metricService: MetricService, @Inject(IConfigRepository) configRepository: IConfigRepository) {
- const { telemetry } = configRepository.getEnv();
+ constructor(
+ private metricService: MetricService,
+ private reflect: Reflector,
+ @Inject(IConfigRepository) private configRepository: IConfigRepository,
+ @Inject(ILoggerRepository) private logger: ILoggerRepository,
+ ) {
+ const { telemetry } = this.configRepository.getEnv();
const { apiMetrics, hostMetrics, jobMetrics, repoMetrics } = telemetry;
this.api = new MetricGroupRepository(metricService).configure({ enabled: apiMetrics });
@@ -49,4 +106,61 @@ export class TelemetryRepository implements ITelemetryRepository {
this.jobs = new MetricGroupRepository(metricService).configure({ enabled: jobMetrics });
this.repo = new MetricGroupRepository(metricService).configure({ enabled: repoMetrics });
}
+
+ setup({ repositories }: { repositories: ClassConstructor<unknown>[] }) {
+ const { telemetry } = this.configRepository.getEnv();
+ if (!telemetry.enabled || !telemetry.repoMetrics) {
+ return;
+ }
+
+ for (const Repository of repositories) {
+ const isEnabled = this.reflect.get(MetadataKey.TELEMETRY_ENABLED, Repository) ?? true;
+ if (!isEnabled) {
+ this.logger.debug(`Telemetry disabled for ${Repository.name}`);
+ continue;
+ }
+
+ this.wrap(Repository);
+ }
+ }
+
+ private wrap(Repository: ClassConstructor<unknown>) {
+ const className = Repository.name;
+ const descriptors = Object.getOwnPropertyDescriptors(Repository.prototype);
+ const unit = 'ms';
+
+ for (const [propName, descriptor] of Object.entries(descriptors)) {
+ const isMethod = typeof descriptor.value == 'function' && propName !== 'constructor';
+ if (!isMethod) {
+ continue;
+ }
+
+ const method = descriptor.value;
+ const propertyName = snakeCase(String(propName));
+ const metricName = `${snakeCase(className).replaceAll(/_(?=(repository)|(controller)|(provider)|(service)|(module))/g, '.')}.${propertyName}.duration`;
+
+ const histogram = this.metricService.getHistogram(metricName, {
+ prefix: 'immich',
+ description: `The elapsed time in ${unit} for the ${startCase(className)} to ${propertyName.toLowerCase()}`,
+ unit,
+ valueType: contextBase.ValueType.DOUBLE,
+ });
+
+ descriptor.value = function (...args: any[]) {
+ const start = performance.now();
+ const result = method.apply(this, args);
+
+ void Promise.resolve(result)
+ .then(() => histogram.record(performance.now() - start, {}))
+ .catch(() => {
+ // noop
+ });
+
+ return result;
+ };
+
+ copyMetadataFromFunctionToFunction(method, descriptor.value);
+ Object.defineProperty(Repository.prototype, propName, descriptor);
+ }
+ }
}
diff --git a/server/src/repositories/user.repository.ts b/server/src/repositories/user.repository.ts
index c64d5a3655..6ac8536ef8 100644
--- a/server/src/repositories/user.repository.ts
+++ b/server/src/repositories/user.repository.ts
@@ -10,10 +10,8 @@ import {
UserListFilter,
UserStatsQueryResponse,
} from 'src/interfaces/user.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { IsNull, Not, Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class UserRepository implements IUserRepository {
constructor(
diff --git a/server/src/repositories/version-history.repository.ts b/server/src/repositories/version-history.repository.ts
index 26c638bd76..e32ceaf4e9 100644
--- a/server/src/repositories/version-history.repository.ts
+++ b/server/src/repositories/version-history.repository.ts
@@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { VersionHistoryEntity } from 'src/entities/version-history.entity';
import { IVersionHistoryRepository } from 'src/interfaces/version-history.interface';
-import { Instrumentation } from 'src/utils/instrumentation';
import { Repository } from 'typeorm';
-@Instrumentation()
@Injectable()
export class VersionHistoryRepository implements IVersionHistoryRepository {
constructor(@InjectRepository(VersionHistoryEntity) private repository: Repository<VersionHistoryEntity>) {}
diff --git a/server/src/services/microservices.service.ts b/server/src/services/microservices.service.ts
index d1d2bb8f20..275103d05c 100644
--- a/server/src/services/microservices.service.ts
+++ b/server/src/services/microservices.service.ts
@@ -20,7 +20,6 @@ import { TagService } from 'src/services/tag.service';
import { TrashService } from 'src/services/trash.service';
import { UserService } from 'src/services/user.service';
import { VersionService } from 'src/services/version.service';
-import { otelShutdown } from 'src/utils/instrumentation';
@Injectable()
export class MicroservicesService {
@@ -101,8 +100,4 @@ export class MicroservicesService {
[JobName.QUEUE_TRASH_EMPTY]: () => this.trashService.handleQueueEmptyTrash(),
});
}
-
- async onShutdown() {
- await otelShutdown();
- }
}
diff --git a/server/src/utils/instrumentation.ts b/server/src/utils/instrumentation.ts
deleted file mode 100644
index bd522f27b2..0000000000
--- a/server/src/utils/instrumentation.ts
+++ /dev/null
@@ -1,100 +0,0 @@
-import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
-import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
-import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
-import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
-import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
-import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';
-import { NodeSDK, contextBase, metrics, resources } from '@opentelemetry/sdk-node';
-import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
-import { snakeCase, startCase } from 'lodash';
-import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils';
-import { performance } from 'node:perf_hooks';
-import { serverVersion } from 'src/constants';
-import { DecorateAll } from 'src/decorators';
-import { ConfigRepository } from 'src/repositories/config.repository';
-
-const aggregation = new metrics.ExplicitBucketHistogramAggregation(
- [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000],
- true,
-);
-
-const { telemetry } = new ConfigRepository().getEnv();
-
-let otelSingleton: NodeSDK | undefined;
-
-export const otelStart = (port: number) => {
- if (otelSingleton) {
- throw new Error('OpenTelemetry SDK already started');
- }
- otelSingleton = new NodeSDK({
- resource: new resources.Resource({
- [SemanticResourceAttributes.SERVICE_NAME]: `immich`,
- [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),
- }),
- metricReader: new PrometheusExporter({ port }),
- contextManager: new AsyncLocalStorageContextManager(),
- instrumentations: [
- new HttpInstrumentation(),
- new IORedisInstrumentation(),
- new NestInstrumentation(),
- new PgInstrumentation(),
- ],
- views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],
- });
- otelSingleton.start();
-};
-
-export const otelShutdown = async () => {
- if (otelSingleton) {
- await otelSingleton.shutdown();
- otelSingleton = undefined;
- }
-};
-
-function ExecutionTimeHistogram({
- description,
- unit = 'ms',
- valueType = contextBase.ValueType.DOUBLE,
-}: contextBase.MetricOptions = {}) {
- return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
- if (!telemetry.repoMetrics || process.env.OTEL_SDK_DISABLED) {
- return;
- }
-
- const method = descriptor.value;
- const className = target.constructor.name as string;
- const propertyName = String(propertyKey);
- const metricName = `${snakeCase(className).replaceAll(/_(?=(repository)|(controller)|(provider)|(service)|(module))/g, '.')}.${snakeCase(propertyName)}.duration`;
-
- const metricDescription =
- description ??
- `The elapsed time in ${unit} for the ${startCase(className)} to ${startCase(propertyName).toLowerCase()}`;
-
- let histogram: contextBase.Histogram | undefined;
-
- descriptor.value = function (...args: any[]) {
- const start = performance.now();
- const result = method.apply(this, args);
-
- void Promise.resolve(result)
- .then(() => {
- const end = performance.now();
- if (!histogram) {
- histogram = contextBase.metrics
- .getMeter('immich')
- .createHistogram(metricName, { description: metricDescription, unit, valueType });
- }
- histogram.record(end - start, {});
- })
- .catch(() => {
- // noop
- });
-
- return result;
- };
-
- copyMetadataFromFunctionToFunction(method, descriptor.value);
- };
-}
-
-export const Instrumentation = () => DecorateAll(ExecutionTimeHistogram());
diff --git a/server/src/workers/api.ts b/server/src/workers/api.ts
index b369b56953..6451f1b792 100644
--- a/server/src/workers/api.ts
+++ b/server/src/workers/api.ts
@@ -11,16 +11,18 @@ import { IConfigRepository } from 'src/interfaces/config.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
import { ConfigRepository } from 'src/repositories/config.repository';
+import { bootstrapTelemetry } from 'src/repositories/telemetry.repository';
import { ApiService } from 'src/services/api.service';
import { isStartUpError } from 'src/services/storage.service';
-import { otelStart } from 'src/utils/instrumentation';
import { useSwagger } from 'src/utils/misc';
async function bootstrap() {
process.title = 'immich-api';
const { telemetry, network } = new ConfigRepository().getEnv();
- otelStart(telemetry.apiPort);
+ if (telemetry.enabled) {
+ bootstrapTelemetry(telemetry.apiPort);
+ }
const app = await NestFactory.create<NestExpressApplication>(ApiModule, { bufferLogs: true });
const logger = await app.resolve<ILoggerRepository>(ILoggerRepository);
diff --git a/server/src/workers/microservices.ts b/server/src/workers/microservices.ts
index 7b60fb8db6..df4abb01da 100644
--- a/server/src/workers/microservices.ts
+++ b/server/src/workers/microservices.ts
@@ -6,12 +6,14 @@ import { IConfigRepository } from 'src/interfaces/config.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
import { ConfigRepository } from 'src/repositories/config.repository';
+import { bootstrapTelemetry } from 'src/repositories/telemetry.repository';
import { isStartUpError } from 'src/services/storage.service';
-import { otelStart } from 'src/utils/instrumentation';
export async function bootstrap() {
const { telemetry } = new ConfigRepository().getEnv();
- otelStart(telemetry.microservicesPort);
+ if (telemetry.enabled) {
+ bootstrapTelemetry(telemetry.microservicesPort);
+ }
const app = await NestFactory.create(MicroservicesModule, { bufferLogs: true });
const logger = await app.resolve(ILoggerRepository);
diff --git a/server/test/repositories/telemetry.repository.mock.ts b/server/test/repositories/telemetry.repository.mock.ts
index 737463065c..2d537e888a 100644
--- a/server/test/repositories/telemetry.repository.mock.ts
+++ b/server/test/repositories/telemetry.repository.mock.ts
@@ -12,6 +12,7 @@ const newMetricGroupMock = () => {
export const newTelemetryRepositoryMock = (): Mocked<ITelemetryRepository> => {
return {
+ setup: vitest.fn(),
api: newMetricGroupMock(),
host: newMetricGroupMock(),
jobs: newMetricGroupMock(),