High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
refactor(server): telemetry env (#13564)
Jason Rasmussen12628b80bce8
diff --git a/server/src/app.module.ts b/server/src/app.module.tsindex 43aefbd0f0..fd921150fd 100644--- a/server/src/app.module.ts+++ b/server/src/app.module.ts@@ -23,7 +23,6 @@ import { repositories } from 'src/repositories';import { ConfigRepository } from 'src/repositories/config.repository';import { services } from 'src/services';import { DatabaseService } from 'src/services/database.service';-import { otelConfig } from 'src/utils/instrumentation';const common = [...services, ...repositories];@@ -37,14 +36,14 @@ const middleware = [];const configRepository = new ConfigRepository();-const { bull } = configRepository.getEnv();+const { bull, otel } = configRepository.getEnv();const imports = [BullModule.forRoot(bull.config),BullModule.registerQueue(...bull.queues),ClsModule.forRoot(clsConfig),ConfigModule.forRoot(immichAppConfig),- OpenTelemetryModule.forRoot(otelConfig),+ OpenTelemetryModule.forRoot(otel),TypeOrmModule.forRootAsync({inject: [ModuleRef],useFactory: (moduleRef: ModuleRef) => {diff --git a/server/src/bin/sync-sql.ts b/server/src/bin/sync-sql.tsindex 92c3cc1103..e4f11cc692 100644--- a/server/src/bin/sync-sql.ts+++ b/server/src/bin/sync-sql.ts@@ -14,8 +14,8 @@ import { entities } from 'src/entities';import { ILoggerRepository } from 'src/interfaces/logger.interface';import { repositories } from 'src/repositories';import { AccessRepository } from 'src/repositories/access.repository';+import { ConfigRepository } from 'src/repositories/config.repository';import { AuthService } from 'src/services/auth.service';-import { otelConfig } from 'src/utils/instrumentation';import { Logger } from 'typeorm';export class SqlLogger implements Logger {@@ -74,6 +74,8 @@ class SqlGenerator {await rm(this.options.targetDir, { force: true, recursive: true });await mkdir(this.options.targetDir);+ const { otel } = new ConfigRepository().getEnv();+const moduleFixture = await Test.createTestingModule({imports: [TypeOrmModule.forRoot({@@ -84,7 +86,7 @@ class SqlGenerator {logger: this.sqlLogger,}),TypeOrmModule.forFeature(entities),- OpenTelemetryModule.forRoot(otelConfig),+ OpenTelemetryModule.forRoot(otel),],providers: [...repositories, AuthService, SchedulerRegistry],}).compile();diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.tsindex 9870b86d10..4391909df7 100644--- a/server/src/interfaces/config.interface.ts+++ b/server/src/interfaces/config.interface.ts@@ -1,6 +1,7 @@import { RegisterQueueOptions } from '@nestjs/bullmq';import { QueueOptions } from 'bullmq';import { RedisOptions } from 'ioredis';+import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';import { VectorExtension } from 'src/interfaces/database.interface';@@ -54,6 +55,8 @@ export interface EnvData {trustedProxies: string[];};+ otel: OpenTelemetryModuleOptions;+resourcePaths: {lockFile: string;geodata: {@@ -74,6 +77,11 @@ export interface EnvData {telemetry: {apiPort: number;microservicesPort: number;+ enabled: boolean;+ apiMetrics: boolean;+ hostMetrics: boolean;+ repoMetrics: boolean;+ jobMetrics: boolean;};storage: {diff --git a/server/src/repositories/config.repository.spec.ts b/server/src/repositories/config.repository.spec.tsindex 36b7b48062..84da211182 100644--- a/server/src/repositories/config.repository.spec.ts+++ b/server/src/repositories/config.repository.spec.ts@@ -12,6 +12,11 @@ const resetEnv = () => {'IMMICH_TRUSTED_PROXIES','IMMICH_API_METRICS_PORT','IMMICH_MICROSERVICES_METRICS_PORT',+ 'IMMICH_METRICS',+ 'IMMICH_API_METRICS',+ 'IMMICH_HOST_METRICS',+ 'IMMICH_IO_METRICS',+ 'IMMICH_JOB_METRICS','DB_URL','DB_HOSTNAME',@@ -200,11 +205,16 @@ describe('getEnv', () => {});describe('telemetry', () => {- it('should return default ports', () => {+ it('should have default values', () => {const { telemetry } = getEnv();expect(telemetry).toEqual({apiPort: 8081,microservicesPort: 8082,+ enabled: false,+ apiMetrics: false,+ hostMetrics: false,+ jobMetrics: false,+ repoMetrics: false,});});@@ -212,10 +222,35 @@ describe('getEnv', () => {process.env.IMMICH_API_METRICS_PORT = '2001';process.env.IMMICH_MICROSERVICES_METRICS_PORT = '2002';const { telemetry } = getEnv();- expect(telemetry).toEqual({+ expect(telemetry).toMatchObject({apiPort: 2001,microservicesPort: 2002,});});++ it('should run with telemetry enabled', () => {+ process.env.IMMICH_METRICS = 'true';+ const { telemetry } = getEnv();+ expect(telemetry).toMatchObject({+ enabled: true,+ apiMetrics: true,+ hostMetrics: true,+ jobMetrics: true,+ repoMetrics: true,+ });+ });++ it('should run with telemetry enabled and jobs disabled', () => {+ process.env.IMMICH_METRICS = 'true';+ process.env.IMMICH_JOB_METRICS = 'false';+ const { telemetry } = getEnv();+ expect(telemetry).toMatchObject({+ enabled: true,+ apiMetrics: true,+ hostMetrics: true,+ jobMetrics: false,+ repoMetrics: true,+ });+ });});});diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.tsindex 8b511fba5a..44b8c7b605 100644--- a/server/src/repositories/config.repository.ts+++ b/server/src/repositories/config.repository.ts@@ -1,6 +1,6 @@import { Injectable } from '@nestjs/common';import { join } from 'node:path';-import { citiesFile } from 'src/constants';+import { citiesFile, excludePaths } from 'src/constants';import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';import { EnvData, IConfigRepository } from 'src/interfaces/config.interface';import { DatabaseExtension } from 'src/interfaces/database.interface';@@ -30,6 +30,8 @@ const asSet = (value: string | undefined, defaults: ImmichWorker[]) => {return new Set(values.length === 0 ? defaults : (values as ImmichWorker[]));};+const parseBoolean = (value: string | undefined, defaultValue: boolean) => (value ? value === 'true' : defaultValue);+const getEnv = (): EnvData => {const included = asSet(process.env.IMMICH_WORKERS_INCLUDE, [ImmichWorker.API, ImmichWorker.MICROSERVICES]);const excluded = asSet(process.env.IMMICH_WORKERS_EXCLUDE, []);@@ -66,6 +68,16 @@ const getEnv = (): EnvData => {}}+ const globalEnabled = parseBoolean(process.env.IMMICH_METRICS, false);+ const hostMetrics = parseBoolean(process.env.IMMICH_HOST_METRICS, globalEnabled);+ const apiMetrics = parseBoolean(process.env.IMMICH_API_METRICS, globalEnabled);+ const repoMetrics = parseBoolean(process.env.IMMICH_IO_METRICS, globalEnabled);+ const jobMetrics = parseBoolean(process.env.IMMICH_JOB_METRICS, globalEnabled);+ const telemetryEnabled = globalEnabled || hostMetrics || apiMetrics || repoMetrics || jobMetrics;+ if (!telemetryEnabled && process.env.OTEL_SDK_DISABLED === undefined) {+ process.env.OTEL_SDK_DISABLED = 'true';+ }+return {host: process.env.IMMICH_HOST,port: Number(process.env.IMMICH_PORT) || 2283,@@ -124,6 +136,16 @@ const getEnv = (): EnvData => {.filter(Boolean),},+ otel: {+ metrics: {+ hostMetrics,+ apiMetrics: {+ enable: apiMetrics,+ ignoreRoutes: excludePaths,+ },+ },+ },+redis: redisConfig,resourcePaths: {@@ -148,6 +170,11 @@ const getEnv = (): EnvData => {telemetry: {apiPort: Number(process.env.IMMICH_API_METRICS_PORT || '') || 8081,microservicesPort: Number(process.env.IMMICH_MICROSERVICES_METRICS_PORT || '') || 8082,+ enabled: telemetryEnabled,+ hostMetrics,+ apiMetrics,+ repoMetrics,+ jobMetrics,},workers,diff --git a/server/src/repositories/metric.repository.ts b/server/src/repositories/metric.repository.tsindex 5948e92fa6..b59bcf9ed1 100644--- a/server/src/repositories/metric.repository.ts+++ b/server/src/repositories/metric.repository.ts@@ -1,11 +1,12 @@-import { Injectable } from '@nestjs/common';+import { Inject, Injectable } from '@nestjs/common';import { MetricOptions } from '@opentelemetry/api';import { MetricService } from 'nestjs-otel';+import { IConfigRepository } from 'src/interfaces/config.interface';import { IMetricGroupRepository, IMetricRepository, MetricGroupOptions } from 'src/interfaces/metric.interface';-import { apiMetrics, hostMetrics, jobMetrics, repoMetrics } from 'src/utils/instrumentation';class MetricGroupRepository implements IMetricGroupRepository {private enabled = false;+constructor(private metricService: MetricService) {}addToCounter(name: string, value: number, options?: MetricOptions): void {@@ -39,10 +40,11 @@ export class MetricRepository implements IMetricRepository {jobs: MetricGroupRepository;repo: MetricGroupRepository;- constructor(metricService: MetricService) {- this.api = new MetricGroupRepository(metricService).configure({ enabled: apiMetrics });- this.host = new MetricGroupRepository(metricService).configure({ enabled: hostMetrics });- this.jobs = new MetricGroupRepository(metricService).configure({ enabled: jobMetrics });- this.repo = new MetricGroupRepository(metricService).configure({ enabled: repoMetrics });+ constructor(metricService: MetricService, @Inject(IConfigRepository) configRepository: IConfigRepository) {+ const { telemetry } = configRepository.getEnv();+ this.api = new MetricGroupRepository(metricService).configure({ enabled: telemetry.apiMetrics });+ this.host = new MetricGroupRepository(metricService).configure({ enabled: telemetry.hostMetrics });+ this.jobs = new MetricGroupRepository(metricService).configure({ enabled: telemetry.jobMetrics });+ this.repo = new MetricGroupRepository(metricService).configure({ enabled: telemetry.repoMetrics });}}diff --git a/server/src/utils/instrumentation.ts b/server/src/utils/instrumentation.tsindex 484ba5901c..bd522f27b2 100644--- a/server/src/utils/instrumentation.ts+++ b/server/src/utils/instrumentation.ts@@ -7,32 +7,19 @@ import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';import { NodeSDK, contextBase, metrics, resources } from '@opentelemetry/sdk-node';import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';import { snakeCase, startCase } from 'lodash';-import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils';import { performance } from 'node:perf_hooks';-import { excludePaths, serverVersion } from 'src/constants';+import { serverVersion } from 'src/constants';import { DecorateAll } from 'src/decorators';--let metricsEnabled = process.env.IMMICH_METRICS === 'true';-export const hostMetrics =- process.env.IMMICH_HOST_METRICS == null ? metricsEnabled : process.env.IMMICH_HOST_METRICS === 'true';-export const apiMetrics =- process.env.IMMICH_API_METRICS == null ? metricsEnabled : process.env.IMMICH_API_METRICS === 'true';-export const repoMetrics =- process.env.IMMICH_IO_METRICS == null ? metricsEnabled : process.env.IMMICH_IO_METRICS === 'true';-export const jobMetrics =- process.env.IMMICH_JOB_METRICS == null ? metricsEnabled : process.env.IMMICH_JOB_METRICS === 'true';--metricsEnabled ||= hostMetrics || apiMetrics || repoMetrics || jobMetrics;-if (!metricsEnabled && process.env.OTEL_SDK_DISABLED === undefined) {- process.env.OTEL_SDK_DISABLED = 'true';-}+import { ConfigRepository } from 'src/repositories/config.repository';const aggregation = new metrics.ExplicitBucketHistogramAggregation([0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000],true,);+const { telemetry } = new ConfigRepository().getEnv();+let otelSingleton: NodeSDK | undefined;export const otelStart = (port: number) => {@@ -64,23 +51,13 @@ export const otelShutdown = async () => {}};-export const otelConfig: OpenTelemetryModuleOptions = {- metrics: {- hostMetrics,- apiMetrics: {- enable: apiMetrics,- ignoreRoutes: excludePaths,- },- },-};-function ExecutionTimeHistogram({description,unit = 'ms',valueType = contextBase.ValueType.DOUBLE,}: contextBase.MetricOptions = {}) {return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {- if (!repoMetrics || process.env.OTEL_SDK_DISABLED) {+ if (!telemetry.repoMetrics || process.env.OTEL_SDK_DISABLED) {return;}diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.tsindex c7da917cc4..bb3cfcebb9 100644--- a/server/test/repositories/config.repository.mock.ts+++ b/server/test/repositories/config.repository.mock.ts@@ -35,6 +35,16 @@ const envData: EnvData = {trustedProxies: [],},+ otel: {+ metrics: {+ hostMetrics: false,+ apiMetrics: {+ enable: false,+ ignoreRoutes: [],+ },+ },+ },+redis: {host: 'redis',port: 6379,@@ -63,6 +73,11 @@ const envData: EnvData = {telemetry: {apiPort: 8081,microservicesPort: 8082,+ enabled: false,+ hostMetrics: false,+ apiMetrics: false,+ jobMetrics: false,+ repoMetrics: false,},workers: [ImmichWorker.API, ImmichWorker.MICROSERVICES],