The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
perf(server): mark per-pod metrics with a pod attribute (#26542)
Etienne14cce5c85ac6
diff --git a/packages/twenty-server/src/database/typeorm/database-pool-metrics.service.ts b/packages/twenty-server/src/database/typeorm/database-pool-metrics.service.tsindex 63f42afdec4..58250ecb15d 100644--- a/packages/twenty-server/src/database/typeorm/database-pool-metrics.service.ts+++ b/packages/twenty-server/src/database/typeorm/database-pool-metrics.service.ts@@ -6,6 +6,7 @@ import { type DataSource } from 'typeorm';import { type PostgresDriver } from 'typeorm/driver/postgres/PostgresDriver';import { isDefined } from 'twenty-shared/utils';+import { POD_NAME } from 'src/engine/core-modules/metrics/constants/pod-name.constant';import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';export enum DatabasePoolName {@@ -92,6 +93,7 @@ export class DatabasePoolMetricsService {pool: poolName,},})),+ perPod: true,});}}@@ -116,12 +118,15 @@ export class DatabasePoolMetricsService {const recordAcquisition = (startedAt: number, error?: unknown) => {if (isDefined(error)) {- this.acquisitionFailureCounter.add(1, { pool: poolName });+ this.acquisitionFailureCounter.add(1, {+ pod: POD_NAME,+ pool: poolName,+ });}this.acquisitionDurationHistogram.record((performance.now() - startedAt) / 1000,- { pool: poolName },+ { pod: POD_NAME, pool: poolName },);};@@ -181,6 +186,7 @@ export class DatabasePoolMetricsService {return await obtainMasterConnection();} catch (error) {this.acquisitionFailureCounter.add(1, {+ pod: POD_NAME,pool: poolName,});@@ -189,6 +195,7 @@ export class DatabasePoolMetricsService {this.acquisitionDurationHistogram.record((performance.now() - start) / 1000,{+ pod: POD_NAME,pool: poolName,},);diff --git a/packages/twenty-server/src/engine/core-modules/metrics/constants/pod-name.constant.ts b/packages/twenty-server/src/engine/core-modules/metrics/constants/pod-name.constant.tsnew file mode 100644index 00000000000..1c43c06a072--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/metrics/constants/pod-name.constant.ts@@ -0,0 +1,3 @@+import os from 'os';++export const POD_NAME = process.env.HOSTNAME ?? os.hostname();diff --git a/packages/twenty-server/src/engine/core-modules/metrics/event-loop-metrics.service.ts b/packages/twenty-server/src/engine/core-modules/metrics/event-loop-metrics.service.tsindex 6eaa1d4435b..a358c36b7dc 100644--- a/packages/twenty-server/src/engine/core-modules/metrics/event-loop-metrics.service.ts+++ b/packages/twenty-server/src/engine/core-modules/metrics/event-loop-metrics.service.ts@@ -3,6 +3,7 @@ import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';import { type Histogram } from '@opentelemetry/api';import { performance, type EventLoopUtilization } from 'perf_hooks';+import { POD_NAME } from 'src/engine/core-modules/metrics/constants/pod-name.constant';import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';const SAMPLE_INTERVAL_MS = 1_000;@@ -39,7 +40,9 @@ export class EventLoopMetricsService implements OnModuleInit, OnModuleDestroy {const lagMs = Math.max(0, now - this.lastSampleAt - SAMPLE_INTERVAL_MS);this.lastSampleAt = now;- this.delayHistogram.record(lagMs / MILLISECONDS_PER_SECOND);+ this.delayHistogram.record(lagMs / MILLISECONDS_PER_SECOND, {+ pod: POD_NAME,+ });}, SAMPLE_INTERVAL_MS);this.sampler.unref();@@ -61,6 +64,7 @@ export class EventLoopMetricsService implements OnModuleInit, OnModuleDestroy {return delta.utilization;},+ perPod: true,});}diff --git a/packages/twenty-server/src/engine/core-modules/metrics/gc-metrics.service.ts b/packages/twenty-server/src/engine/core-modules/metrics/gc-metrics.service.tsindex c5aae67b5a6..77a48e1eee7 100644--- a/packages/twenty-server/src/engine/core-modules/metrics/gc-metrics.service.ts+++ b/packages/twenty-server/src/engine/core-modules/metrics/gc-metrics.service.ts@@ -6,6 +6,7 @@ import { getHeapStatistics } from 'v8';import { isDefined } from 'twenty-shared/utils';+import { POD_NAME } from 'src/engine/core-modules/metrics/constants/pod-name.constant';import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';const MILLISECONDS_PER_SECOND = 1_000;@@ -56,6 +57,7 @@ export class GcMetricsService implements OnModuleInit, OnModuleDestroy {this.observer = new PerformanceObserver((list) => {for (const entry of list.getEntries()) {this.pauseHistogram.record(entry.duration / MILLISECONDS_PER_SECOND, {+ pod: POD_NAME,kind: gcKindOf(entry.detail),});}@@ -102,6 +104,7 @@ export class GcMetricsService implements OnModuleInit, OnModuleDestroy {metricName: gauge.metricName,options: { description: gauge.description, unit: 'By' },callback: async () => gauge.read(this.getHeapStatisticsSnapshot()),+ perPod: true,});}}diff --git a/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts b/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.tsindex d265a412c11..0ae7b999b9d 100644--- a/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts+++ b/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts@@ -13,6 +13,7 @@ import { isDefined } from 'twenty-shared/utils';import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';+import { POD_NAME } from 'src/engine/core-modules/metrics/constants/pod-name.constant';import { MetricsCacheService } from 'src/engine/core-modules/metrics/metrics-cache.service';import { type MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';@@ -38,11 +39,13 @@ export class MetricsService {options,callback,cacheValue = false,+ perPod = false,}: {metricName: string;options: MetricOptions;callback: () => number | Promise<number>;cacheValue?: boolean;+ perPod?: boolean;}): ObservableGauge {return this.createObservableGaugeInternal({metricName,@@ -50,7 +53,10 @@ export class MetricsService {callback,cacheValue,observeResult: (observableResult, result) => {- observableResult.observe(result);+ observableResult.observe(+ result,+ perPod ? { pod: POD_NAME } : undefined,+ );},});}@@ -60,11 +66,13 @@ export class MetricsService {options,callback,cacheValue = false,+ perPod = false,}: {metricName: string;options: MetricOptions;callback: () => Promise<Array<{ value: number; attributes: Attributes }>>;cacheValue?: boolean;+ perPod?: boolean;}): ObservableGauge {return this.createObservableGaugeInternal({metricName,@@ -73,7 +81,12 @@ export class MetricsService {cacheValue,observeResult: (observableResult, observations) => {for (const observation of observations) {- observableResult.observe(observation.value, observation.attributes);+ observableResult.observe(+ observation.value,+ perPod+ ? { ...observation.attributes, pod: POD_NAME }+ : observation.attributes,+ );}},});diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.tsindex 71e29e3e340..901ed1891fc 100644--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts@@ -250,6 +250,7 @@ export class WorkspaceCacheMetricsService {description: 'Entries in the per-pod local workspace metadata cache',},callback: async () => this.getStats().entries,+ perPod: true,});this.metricsService.createObservableGauge({metricName: 'twenty_workspace_cache_local_workspaces',@@ -258,6 +259,7 @@ export class WorkspaceCacheMetricsService {'Distinct workspaces held in the per-pod local workspace metadata cache',},callback: async () => this.getStats().workspaces,+ perPod: true,});this.metricsService.createObservableGauge({metricName: 'twenty_workspace_cache_local_versions_total',@@ -266,6 +268,7 @@ export class WorkspaceCacheMetricsService {'Total versions across local workspace metadata cache entries',},callback: async () => this.getStats().versionsTotal,+ perPod: true,});this.metricsService.createObservableGauge({metricName: 'twenty_workspace_cache_local_bytes_estimate',@@ -275,6 +278,7 @@ export class WorkspaceCacheMetricsService {unit: 'By',},callback: async () => this.cacheSizeTotalBytes,+ perPod: true,});this.metricsService.createMultiObservableGauge({metricName: 'twenty_workspace_cache_local_entries_by_version_count',@@ -286,6 +290,7 @@ export class WorkspaceCacheMetricsService {Object.entries(this.getStats().versionsByCount).map(([versions, value]) => ({ value, attributes: { versions } }),),+ perPod: true,});this.metricsService.createMultiObservableGauge({metricName: 'twenty_workspace_cache_local_bytes_by_provider',@@ -299,6 +304,7 @@ export class WorkspaceCacheMetricsService {value,attributes: { provider: keyName },})),+ perPod: true,});this.metricsService.createMultiObservableGauge({metricName: 'twenty_workspace_cache_local_entries_by_provider',@@ -313,6 +319,7 @@ export class WorkspaceCacheMetricsService {attributes: { provider: keyName },}),),+ perPod: true,});this.metricsService.createMultiObservableGauge({metricName: 'twenty_workspace_cache_local_versions_by_state',@@ -338,6 +345,7 @@ export class WorkspaceCacheMetricsService {),];},+ perPod: true,});this.metricsService.createObservableGauge({metricName: 'twenty_workspace_cache_local_live_versions',@@ -346,6 +354,7 @@ export class WorkspaceCacheMetricsService {'Total live (object graph) versions in the local workspace metadata cache',},callback: async () => this.getStats().liveVersionsTotal,+ perPod: true,});this.metricsService.createObservableGauge({metricName: 'twenty_workspace_cache_local_packed_versions',@@ -354,6 +363,7 @@ export class WorkspaceCacheMetricsService {'Total packed (serialized buffer) versions in the local workspace metadata cache',},callback: async () => this.getStats().packedVersionsTotal,+ perPod: true,});this.metricsService.createObservableGauge({metricName: 'twenty_workspace_cache_local_packed_bytes',@@ -363,6 +373,7 @@ export class WorkspaceCacheMetricsService {unit: 'By',},callback: async () => this.getStats().packedBytesTotal,+ perPod: true,});this.metricsService.createObservableGauge({metricName: 'twenty_workspace_cache_packing_backlog',@@ -371,6 +382,7 @@ export class WorkspaceCacheMetricsService {'Versions currently eligible for packing, whether or not the last run reached them',},callback: async () => this.packingBacklog,+ perPod: true,});}}diff --git a/packages/twenty-server/src/instrument.ts b/packages/twenty-server/src/instrument.tsindex bdb21d87570..5498e60f41c 100644--- a/packages/twenty-server/src/instrument.ts+++ b/packages/twenty-server/src/instrument.ts@@ -1,4 +1,3 @@-import os from 'os';import process from 'process';import { metrics as otelMetrics } from '@opentelemetry/api';@@ -18,6 +17,7 @@ import { nodeProfilingIntegration } from '@sentry/profiling-node';import { NodeEnvironment } from 'src/engine/core-modules/twenty-config/interfaces/node-environment.interface';import { ExceptionHandlerDriver } from 'src/engine/core-modules/exception-handler/interfaces';+import { POD_NAME } from 'src/engine/core-modules/metrics/constants/pod-name.constant';import { MeterDriver } from 'src/engine/core-modules/metrics/types/meter-driver.type';import { parseArrayEnvVar } from 'src/utils/parse-array-env-var';@@ -135,7 +135,7 @@ const prometheusExporter = meterDrivers.includes(MeterDriver.Prometheus)const meterProvider = new MeterProvider({resource: resourceFromAttributes({'service.name': process.env.OTEL_SERVICE_NAME ?? 'twenty-server',- 'k8s.pod.name': process.env.HOSTNAME ?? os.hostname(),+ 'k8s.pod.name': POD_NAME,}),readers: [...(meterDrivers.includes(MeterDriver.Console)