High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix(server): otel not working due to port conflicts after combining containers (#10078)
Zack Pollard2f2aecfb47e8
diff --git a/docker/prometheus.yml b/docker/prometheus.ymlindex e25bb7db6e..3b18e53450 100644--- a/docker/prometheus.yml+++ b/docker/prometheus.yml@@ -3,10 +3,10 @@ global:evaluation_interval: 15sscrape_configs:- - job_name: immich_server+ - job_name: immich_apistatic_configs:- targets: ['immich-server:8081']-+- job_name: immich_microservicesstatic_configs:- - targets: ['immich-microservices:8081']+ - targets: ['immich-server:8082']diff --git a/server/src/config.ts b/server/src/config.tsindex f1f89f3e32..624dd385ad 100644--- a/server/src/config.ts+++ b/server/src/config.ts@@ -374,7 +374,8 @@ export const immichAppConfig: ConfigModuleOptions = {DB_SKIP_MIGRATIONS: Joi.boolean().optional().default(false),IMMICH_PORT: Joi.number().optional(),- IMMICH_METRICS_PORT: Joi.number().optional(),+ IMMICH_API_METRICS_PORT: Joi.number().optional(),+ IMMICH_MICROSERVICES_METRICS_PORT: Joi.number().optional(),IMMICH_METRICS: Joi.boolean().optional().default(false),IMMICH_HOST_METRICS: Joi.boolean().optional().default(false),diff --git a/server/src/services/microservices.service.ts b/server/src/services/microservices.service.tsindex f175ed0459..8e9f6ca082 100644--- a/server/src/services/microservices.service.ts+++ b/server/src/services/microservices.service.ts@@ -17,7 +17,7 @@ import { StorageService } from 'src/services/storage.service';import { SystemConfigService } from 'src/services/system-config.service';import { UserService } from 'src/services/user.service';import { VersionService } from 'src/services/version.service';-import { otelSDK } from 'src/utils/instrumentation';+import { otelShutdown } from 'src/utils/instrumentation';@Injectable()export class MicroservicesService {@@ -102,6 +102,6 @@ export class MicroservicesService {async teardown() {await this.libraryService.teardown();await this.metadataService.teardown();- await otelSDK.shutdown();+ await otelShutdown();}}diff --git a/server/src/utils/instrumentation.ts b/server/src/utils/instrumentation.tsindex 2164fae426..484ba5901c 100644--- a/server/src/utils/instrumentation.ts+++ b/server/src/utils/instrumentation.ts@@ -33,23 +33,36 @@ const aggregation = new metrics.ExplicitBucketHistogramAggregation(true,);-const metricsPort = Number.parseInt(process.env.IMMICH_METRICS_PORT ?? '8081');--export const otelSDK = new NodeSDK({- resource: new resources.Resource({- [SemanticResourceAttributes.SERVICE_NAME]: `immich`,- [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),- }),- metricReader: new PrometheusExporter({ port: metricsPort }),- contextManager: new AsyncLocalStorageContextManager(),- instrumentations: [- new HttpInstrumentation(),- new IORedisInstrumentation(),- new NestInstrumentation(),- new PgInstrumentation(),- ],- views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],-});+let otelSingleton: NodeSDK | undefined;++export const otelStart = (port: number) => {+ if (otelSingleton) {+ throw new Error('OpenTelemetry SDK already started');+ }+ otelSingleton = new NodeSDK({+ resource: new resources.Resource({+ [SemanticResourceAttributes.SERVICE_NAME]: `immich`,+ [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),+ }),+ metricReader: new PrometheusExporter({ port }),+ contextManager: new AsyncLocalStorageContextManager(),+ instrumentations: [+ new HttpInstrumentation(),+ new IORedisInstrumentation(),+ new NestInstrumentation(),+ new PgInstrumentation(),+ ],+ views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],+ });+ otelSingleton.start();+};++export const otelShutdown = async () => {+ if (otelSingleton) {+ await otelSingleton.shutdown();+ otelSingleton = undefined;+ }+};export const otelConfig: OpenTelemetryModuleOptions = {metrics: {diff --git a/server/src/workers/api.ts b/server/src/workers/api.tsindex 6c0bb5e789..dae0f83e86 100644--- a/server/src/workers/api.ts+++ b/server/src/workers/api.ts@@ -9,14 +9,16 @@ import { envName, excludePaths, isDev, serverVersion, WEB_ROOT } from 'src/constimport { ILoggerRepository } from 'src/interfaces/logger.interface';import { WebSocketAdapter } from 'src/middleware/websocket.adapter';import { ApiService } from 'src/services/api.service';-import { otelSDK } from 'src/utils/instrumentation';+import { otelStart } from 'src/utils/instrumentation';import { useSwagger } from 'src/utils/misc';const host = process.env.HOST;async function bootstrap() {process.title = 'immich-api';- otelSDK.start();+ const otelPort = Number.parseInt(process.env.IMMICH_API_METRICS_PORT ?? '8081');++ otelStart(otelPort);const port = Number(process.env.IMMICH_PORT) || 3001;const app = await NestFactory.create<NestExpressApplication>(ApiModule, { bufferLogs: true });diff --git a/server/src/workers/microservices.ts b/server/src/workers/microservices.tsindex 50c82ae7d9..f920e8c947 100644--- a/server/src/workers/microservices.ts+++ b/server/src/workers/microservices.ts@@ -4,10 +4,12 @@ import { MicroservicesModule } from 'src/app.module';import { envName, serverVersion } from 'src/constants';import { ILoggerRepository } from 'src/interfaces/logger.interface';import { WebSocketAdapter } from 'src/middleware/websocket.adapter';-import { otelSDK } from 'src/utils/instrumentation';+import { otelStart } from 'src/utils/instrumentation';export async function bootstrap() {- otelSDK.start();+ const otelPort = Number.parseInt(process.env.IMMICH_MICROSERVICES_METRICS_PORT ?? '8082');++ otelStart(otelPort);const app = await NestFactory.create(MicroservicesModule, { bufferLogs: true });const logger = await app.resolve(ILoggerRepository);