Skip to content

High performance self-hosted photo and video management solution.

refactor(server): telemetry env (#13564)

Jason Rasmussen12628b80bce8
diff --git a/server/src/app.module.ts b/server/src/app.module.ts
index 43aefbd0f0..fd921150fd 100644
--- a/server/src/app.module.ts
+++ b/server/src/app.module.ts
@@ -23,7 +23,6 @@ import { repositories } from 'src/repositories';
import { ConfigRepository } from 'src/repositories/config.repository';
import { services } from 'src/services';
import { DatabaseService } from 'src/services/database.service';
-import { otelConfig } from 'src/utils/instrumentation';
const common = [...services, ...repositories];
@@ -37,14 +36,14 @@ const middleware = [
];
const configRepository = new ConfigRepository();
-const { bull } = configRepository.getEnv();
+const { bull, otel } = configRepository.getEnv();
const imports = [
BullModule.forRoot(bull.config),
BullModule.registerQueue(...bull.queues),
ClsModule.forRoot(clsConfig),
ConfigModule.forRoot(immichAppConfig),
- OpenTelemetryModule.forRoot(otelConfig),
+ OpenTelemetryModule.forRoot(otel),
TypeOrmModule.forRootAsync({
inject: [ModuleRef],
useFactory: (moduleRef: ModuleRef) => {
diff --git a/server/src/bin/sync-sql.ts b/server/src/bin/sync-sql.ts
index 92c3cc1103..e4f11cc692 100644
--- a/server/src/bin/sync-sql.ts
+++ b/server/src/bin/sync-sql.ts
@@ -14,8 +14,8 @@ import { entities } from 'src/entities';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { repositories } from 'src/repositories';
import { AccessRepository } from 'src/repositories/access.repository';
+import { ConfigRepository } from 'src/repositories/config.repository';
import { AuthService } from 'src/services/auth.service';
-import { otelConfig } from 'src/utils/instrumentation';
import { Logger } from 'typeorm';
export class SqlLogger implements Logger {
@@ -74,6 +74,8 @@ class SqlGenerator {
await rm(this.options.targetDir, { force: true, recursive: true });
await mkdir(this.options.targetDir);
+ const { otel } = new ConfigRepository().getEnv();
+
const moduleFixture = await Test.createTestingModule({
imports: [
TypeOrmModule.forRoot({
@@ -84,7 +86,7 @@ class SqlGenerator {
logger: this.sqlLogger,
}),
TypeOrmModule.forFeature(entities),
- OpenTelemetryModule.forRoot(otelConfig),
+ OpenTelemetryModule.forRoot(otel),
],
providers: [...repositories, AuthService, SchedulerRegistry],
}).compile();
diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.ts
index 9870b86d10..4391909df7 100644
--- a/server/src/interfaces/config.interface.ts
+++ b/server/src/interfaces/config.interface.ts
@@ -1,6 +1,7 @@
import { RegisterQueueOptions } from '@nestjs/bullmq';
import { QueueOptions } from 'bullmq';
import { RedisOptions } from 'ioredis';
+import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';
import { VectorExtension } from 'src/interfaces/database.interface';
@@ -54,6 +55,8 @@ export interface EnvData {
trustedProxies: string[];
};
+ otel: OpenTelemetryModuleOptions;
+
resourcePaths: {
lockFile: string;
geodata: {
@@ -74,6 +77,11 @@ export interface EnvData {
telemetry: {
apiPort: number;
microservicesPort: number;
+ enabled: boolean;
+ apiMetrics: boolean;
+ hostMetrics: boolean;
+ repoMetrics: boolean;
+ jobMetrics: boolean;
};
storage: {
diff --git a/server/src/repositories/config.repository.spec.ts b/server/src/repositories/config.repository.spec.ts
index 36b7b48062..84da211182 100644
--- a/server/src/repositories/config.repository.spec.ts
+++ b/server/src/repositories/config.repository.spec.ts
@@ -12,6 +12,11 @@ const resetEnv = () => {
'IMMICH_TRUSTED_PROXIES',
'IMMICH_API_METRICS_PORT',
'IMMICH_MICROSERVICES_METRICS_PORT',
+ 'IMMICH_METRICS',
+ 'IMMICH_API_METRICS',
+ 'IMMICH_HOST_METRICS',
+ 'IMMICH_IO_METRICS',
+ 'IMMICH_JOB_METRICS',
'DB_URL',
'DB_HOSTNAME',
@@ -200,11 +205,16 @@ describe('getEnv', () => {
});
describe('telemetry', () => {
- it('should return default ports', () => {
+ it('should have default values', () => {
const { telemetry } = getEnv();
expect(telemetry).toEqual({
apiPort: 8081,
microservicesPort: 8082,
+ enabled: false,
+ apiMetrics: false,
+ hostMetrics: false,
+ jobMetrics: false,
+ repoMetrics: false,
});
});
@@ -212,10 +222,35 @@ describe('getEnv', () => {
process.env.IMMICH_API_METRICS_PORT = '2001';
process.env.IMMICH_MICROSERVICES_METRICS_PORT = '2002';
const { telemetry } = getEnv();
- expect(telemetry).toEqual({
+ expect(telemetry).toMatchObject({
apiPort: 2001,
microservicesPort: 2002,
});
});
+
+ it('should run with telemetry enabled', () => {
+ process.env.IMMICH_METRICS = 'true';
+ const { telemetry } = getEnv();
+ expect(telemetry).toMatchObject({
+ enabled: true,
+ apiMetrics: true,
+ hostMetrics: true,
+ jobMetrics: true,
+ repoMetrics: true,
+ });
+ });
+
+ it('should run with telemetry enabled and jobs disabled', () => {
+ process.env.IMMICH_METRICS = 'true';
+ process.env.IMMICH_JOB_METRICS = 'false';
+ const { telemetry } = getEnv();
+ expect(telemetry).toMatchObject({
+ enabled: true,
+ apiMetrics: true,
+ hostMetrics: true,
+ jobMetrics: false,
+ repoMetrics: true,
+ });
+ });
});
});
diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts
index 8b511fba5a..44b8c7b605 100644
--- a/server/src/repositories/config.repository.ts
+++ b/server/src/repositories/config.repository.ts
@@ -1,6 +1,6 @@
import { Injectable } from '@nestjs/common';
import { join } from 'node:path';
-import { citiesFile } from 'src/constants';
+import { citiesFile, excludePaths } from 'src/constants';
import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';
import { EnvData, IConfigRepository } from 'src/interfaces/config.interface';
import { DatabaseExtension } from 'src/interfaces/database.interface';
@@ -30,6 +30,8 @@ const asSet = (value: string | undefined, defaults: ImmichWorker[]) => {
return new Set(values.length === 0 ? defaults : (values as ImmichWorker[]));
};
+const parseBoolean = (value: string | undefined, defaultValue: boolean) => (value ? value === 'true' : defaultValue);
+
const getEnv = (): EnvData => {
const included = asSet(process.env.IMMICH_WORKERS_INCLUDE, [ImmichWorker.API, ImmichWorker.MICROSERVICES]);
const excluded = asSet(process.env.IMMICH_WORKERS_EXCLUDE, []);
@@ -66,6 +68,16 @@ const getEnv = (): EnvData => {
}
}
+ const globalEnabled = parseBoolean(process.env.IMMICH_METRICS, false);
+ const hostMetrics = parseBoolean(process.env.IMMICH_HOST_METRICS, globalEnabled);
+ const apiMetrics = parseBoolean(process.env.IMMICH_API_METRICS, globalEnabled);
+ const repoMetrics = parseBoolean(process.env.IMMICH_IO_METRICS, globalEnabled);
+ const jobMetrics = parseBoolean(process.env.IMMICH_JOB_METRICS, globalEnabled);
+ const telemetryEnabled = globalEnabled || hostMetrics || apiMetrics || repoMetrics || jobMetrics;
+ if (!telemetryEnabled && process.env.OTEL_SDK_DISABLED === undefined) {
+ process.env.OTEL_SDK_DISABLED = 'true';
+ }
+
return {
host: process.env.IMMICH_HOST,
port: Number(process.env.IMMICH_PORT) || 2283,
@@ -124,6 +136,16 @@ const getEnv = (): EnvData => {
.filter(Boolean),
},
+ otel: {
+ metrics: {
+ hostMetrics,
+ apiMetrics: {
+ enable: apiMetrics,
+ ignoreRoutes: excludePaths,
+ },
+ },
+ },
+
redis: redisConfig,
resourcePaths: {
@@ -148,6 +170,11 @@ const getEnv = (): EnvData => {
telemetry: {
apiPort: Number(process.env.IMMICH_API_METRICS_PORT || '') || 8081,
microservicesPort: Number(process.env.IMMICH_MICROSERVICES_METRICS_PORT || '') || 8082,
+ enabled: telemetryEnabled,
+ hostMetrics,
+ apiMetrics,
+ repoMetrics,
+ jobMetrics,
},
workers,
diff --git a/server/src/repositories/metric.repository.ts b/server/src/repositories/metric.repository.ts
index 5948e92fa6..b59bcf9ed1 100644
--- a/server/src/repositories/metric.repository.ts
+++ b/server/src/repositories/metric.repository.ts
@@ -1,11 +1,12 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
import { MetricOptions } from '@opentelemetry/api';
import { MetricService } from 'nestjs-otel';
+import { IConfigRepository } from 'src/interfaces/config.interface';
import { IMetricGroupRepository, IMetricRepository, MetricGroupOptions } from 'src/interfaces/metric.interface';
-import { apiMetrics, hostMetrics, jobMetrics, repoMetrics } from 'src/utils/instrumentation';
class MetricGroupRepository implements IMetricGroupRepository {
private enabled = false;
+
constructor(private metricService: MetricService) {}
addToCounter(name: string, value: number, options?: MetricOptions): void {
@@ -39,10 +40,11 @@ export class MetricRepository implements IMetricRepository {
jobs: MetricGroupRepository;
repo: MetricGroupRepository;
- constructor(metricService: MetricService) {
- this.api = new MetricGroupRepository(metricService).configure({ enabled: apiMetrics });
- this.host = new MetricGroupRepository(metricService).configure({ enabled: hostMetrics });
- this.jobs = new MetricGroupRepository(metricService).configure({ enabled: jobMetrics });
- this.repo = new MetricGroupRepository(metricService).configure({ enabled: repoMetrics });
+ constructor(metricService: MetricService, @Inject(IConfigRepository) configRepository: IConfigRepository) {
+ const { telemetry } = configRepository.getEnv();
+ this.api = new MetricGroupRepository(metricService).configure({ enabled: telemetry.apiMetrics });
+ this.host = new MetricGroupRepository(metricService).configure({ enabled: telemetry.hostMetrics });
+ this.jobs = new MetricGroupRepository(metricService).configure({ enabled: telemetry.jobMetrics });
+ this.repo = new MetricGroupRepository(metricService).configure({ enabled: telemetry.repoMetrics });
}
}
diff --git a/server/src/utils/instrumentation.ts b/server/src/utils/instrumentation.ts
index 484ba5901c..bd522f27b2 100644
--- a/server/src/utils/instrumentation.ts
+++ b/server/src/utils/instrumentation.ts
@@ -7,32 +7,19 @@ import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';
import { NodeSDK, contextBase, metrics, resources } from '@opentelemetry/sdk-node';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
import { snakeCase, startCase } from 'lodash';
-import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils';
import { performance } from 'node:perf_hooks';
-import { excludePaths, serverVersion } from 'src/constants';
+import { serverVersion } from 'src/constants';
import { DecorateAll } from 'src/decorators';
-
-let metricsEnabled = process.env.IMMICH_METRICS === 'true';
-export const hostMetrics =
- process.env.IMMICH_HOST_METRICS == null ? metricsEnabled : process.env.IMMICH_HOST_METRICS === 'true';
-export const apiMetrics =
- process.env.IMMICH_API_METRICS == null ? metricsEnabled : process.env.IMMICH_API_METRICS === 'true';
-export const repoMetrics =
- process.env.IMMICH_IO_METRICS == null ? metricsEnabled : process.env.IMMICH_IO_METRICS === 'true';
-export const jobMetrics =
- process.env.IMMICH_JOB_METRICS == null ? metricsEnabled : process.env.IMMICH_JOB_METRICS === 'true';
-
-metricsEnabled ||= hostMetrics || apiMetrics || repoMetrics || jobMetrics;
-if (!metricsEnabled && process.env.OTEL_SDK_DISABLED === undefined) {
- process.env.OTEL_SDK_DISABLED = 'true';
-}
+import { ConfigRepository } from 'src/repositories/config.repository';
const aggregation = new metrics.ExplicitBucketHistogramAggregation(
[0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000],
true,
);
+const { telemetry } = new ConfigRepository().getEnv();
+
let otelSingleton: NodeSDK | undefined;
export const otelStart = (port: number) => {
@@ -64,23 +51,13 @@ export const otelShutdown = async () => {
}
};
-export const otelConfig: OpenTelemetryModuleOptions = {
- metrics: {
- hostMetrics,
- apiMetrics: {
- enable: apiMetrics,
- ignoreRoutes: excludePaths,
- },
- },
-};
-
function ExecutionTimeHistogram({
description,
unit = 'ms',
valueType = contextBase.ValueType.DOUBLE,
}: contextBase.MetricOptions = {}) {
return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
- if (!repoMetrics || process.env.OTEL_SDK_DISABLED) {
+ if (!telemetry.repoMetrics || process.env.OTEL_SDK_DISABLED) {
return;
}
diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts
index c7da917cc4..bb3cfcebb9 100644
--- a/server/test/repositories/config.repository.mock.ts
+++ b/server/test/repositories/config.repository.mock.ts
@@ -35,6 +35,16 @@ const envData: EnvData = {
trustedProxies: [],
},
+ otel: {
+ metrics: {
+ hostMetrics: false,
+ apiMetrics: {
+ enable: false,
+ ignoreRoutes: [],
+ },
+ },
+ },
+
redis: {
host: 'redis',
port: 6379,
@@ -63,6 +73,11 @@ const envData: EnvData = {
telemetry: {
apiPort: 8081,
microservicesPort: 8082,
+ enabled: false,
+ hostMetrics: false,
+ apiMetrics: false,
+ jobMetrics: false,
+ repoMetrics: false,
},
workers: [ImmichWorker.API, ImmichWorker.MICROSERVICES],