Skip to content

The open alternative to Salesforce, designed for AI.

perf(server): mark per-pod metrics with a pod attribute (#26542)

Etienne14cce5c85ac6
diff --git a/packages/twenty-server/src/database/typeorm/database-pool-metrics.service.ts b/packages/twenty-server/src/database/typeorm/database-pool-metrics.service.ts
index 63f42afdec4..58250ecb15d 100644
--- a/packages/twenty-server/src/database/typeorm/database-pool-metrics.service.ts
+++ b/packages/twenty-server/src/database/typeorm/database-pool-metrics.service.ts
@@ -6,6 +6,7 @@ import { type DataSource } from 'typeorm';
import { type PostgresDriver } from 'typeorm/driver/postgres/PostgresDriver';
import { isDefined } from 'twenty-shared/utils';
+import { POD_NAME } from 'src/engine/core-modules/metrics/constants/pod-name.constant';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
export enum DatabasePoolName {
@@ -92,6 +93,7 @@ export class DatabasePoolMetricsService {
pool: poolName,
},
})),
+ perPod: true,
});
}
}
@@ -116,12 +118,15 @@ export class DatabasePoolMetricsService {
const recordAcquisition = (startedAt: number, error?: unknown) => {
if (isDefined(error)) {
- this.acquisitionFailureCounter.add(1, { pool: poolName });
+ this.acquisitionFailureCounter.add(1, {
+ pod: POD_NAME,
+ pool: poolName,
+ });
}
this.acquisitionDurationHistogram.record(
(performance.now() - startedAt) / 1000,
- { pool: poolName },
+ { pod: POD_NAME, pool: poolName },
);
};
@@ -181,6 +186,7 @@ export class DatabasePoolMetricsService {
return await obtainMasterConnection();
} catch (error) {
this.acquisitionFailureCounter.add(1, {
+ pod: POD_NAME,
pool: poolName,
});
@@ -189,6 +195,7 @@ export class DatabasePoolMetricsService {
this.acquisitionDurationHistogram.record(
(performance.now() - start) / 1000,
{
+ pod: POD_NAME,
pool: poolName,
},
);
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/constants/pod-name.constant.ts b/packages/twenty-server/src/engine/core-modules/metrics/constants/pod-name.constant.ts
new file mode 100644
index 00000000000..1c43c06a072
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/metrics/constants/pod-name.constant.ts
@@ -0,0 +1,3 @@
+import os from 'os';
+
+export const POD_NAME = process.env.HOSTNAME ?? os.hostname();
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/event-loop-metrics.service.ts b/packages/twenty-server/src/engine/core-modules/metrics/event-loop-metrics.service.ts
index 6eaa1d4435b..a358c36b7dc 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/event-loop-metrics.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/event-loop-metrics.service.ts
@@ -3,6 +3,7 @@ import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { type Histogram } from '@opentelemetry/api';
import { performance, type EventLoopUtilization } from 'perf_hooks';
+import { POD_NAME } from 'src/engine/core-modules/metrics/constants/pod-name.constant';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
const SAMPLE_INTERVAL_MS = 1_000;
@@ -39,7 +40,9 @@ export class EventLoopMetricsService implements OnModuleInit, OnModuleDestroy {
const lagMs = Math.max(0, now - this.lastSampleAt - SAMPLE_INTERVAL_MS);
this.lastSampleAt = now;
- this.delayHistogram.record(lagMs / MILLISECONDS_PER_SECOND);
+ this.delayHistogram.record(lagMs / MILLISECONDS_PER_SECOND, {
+ pod: POD_NAME,
+ });
}, SAMPLE_INTERVAL_MS);
this.sampler.unref();
@@ -61,6 +64,7 @@ export class EventLoopMetricsService implements OnModuleInit, OnModuleDestroy {
return delta.utilization;
},
+ perPod: true,
});
}
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/gc-metrics.service.ts b/packages/twenty-server/src/engine/core-modules/metrics/gc-metrics.service.ts
index c5aae67b5a6..77a48e1eee7 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/gc-metrics.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/gc-metrics.service.ts
@@ -6,6 +6,7 @@ import { getHeapStatistics } from 'v8';
import { isDefined } from 'twenty-shared/utils';
+import { POD_NAME } from 'src/engine/core-modules/metrics/constants/pod-name.constant';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
const MILLISECONDS_PER_SECOND = 1_000;
@@ -56,6 +57,7 @@ export class GcMetricsService implements OnModuleInit, OnModuleDestroy {
this.observer = new PerformanceObserver((list) => {
for (const entry of list.getEntries()) {
this.pauseHistogram.record(entry.duration / MILLISECONDS_PER_SECOND, {
+ pod: POD_NAME,
kind: gcKindOf(entry.detail),
});
}
@@ -102,6 +104,7 @@ export class GcMetricsService implements OnModuleInit, OnModuleDestroy {
metricName: gauge.metricName,
options: { description: gauge.description, unit: 'By' },
callback: async () => gauge.read(this.getHeapStatisticsSnapshot()),
+ perPod: true,
});
}
}
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts b/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts
index d265a412c11..0ae7b999b9d 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts
@@ -13,6 +13,7 @@ import { isDefined } from 'twenty-shared/utils';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
+import { POD_NAME } from 'src/engine/core-modules/metrics/constants/pod-name.constant';
import { MetricsCacheService } from 'src/engine/core-modules/metrics/metrics-cache.service';
import { type MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
@@ -38,11 +39,13 @@ export class MetricsService {
options,
callback,
cacheValue = false,
+ perPod = false,
}: {
metricName: string;
options: MetricOptions;
callback: () => number | Promise<number>;
cacheValue?: boolean;
+ perPod?: boolean;
}): ObservableGauge {
return this.createObservableGaugeInternal({
metricName,
@@ -50,7 +53,10 @@ export class MetricsService {
callback,
cacheValue,
observeResult: (observableResult, result) => {
- observableResult.observe(result);
+ observableResult.observe(
+ result,
+ perPod ? { pod: POD_NAME } : undefined,
+ );
},
});
}
@@ -60,11 +66,13 @@ export class MetricsService {
options,
callback,
cacheValue = false,
+ perPod = false,
}: {
metricName: string;
options: MetricOptions;
callback: () => Promise<Array<{ value: number; attributes: Attributes }>>;
cacheValue?: boolean;
+ perPod?: boolean;
}): ObservableGauge {
return this.createObservableGaugeInternal({
metricName,
@@ -73,7 +81,12 @@ export class MetricsService {
cacheValue,
observeResult: (observableResult, observations) => {
for (const observation of observations) {
- observableResult.observe(observation.value, observation.attributes);
+ observableResult.observe(
+ observation.value,
+ perPod
+ ? { ...observation.attributes, pod: POD_NAME }
+ : observation.attributes,
+ );
}
},
});
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
index 71e29e3e340..901ed1891fc 100644
--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
@@ -250,6 +250,7 @@ export class WorkspaceCacheMetricsService {
description: 'Entries in the per-pod local workspace metadata cache',
},
callback: async () => this.getStats().entries,
+ perPod: true,
});
this.metricsService.createObservableGauge({
metricName: 'twenty_workspace_cache_local_workspaces',
@@ -258,6 +259,7 @@ export class WorkspaceCacheMetricsService {
'Distinct workspaces held in the per-pod local workspace metadata cache',
},
callback: async () => this.getStats().workspaces,
+ perPod: true,
});
this.metricsService.createObservableGauge({
metricName: 'twenty_workspace_cache_local_versions_total',
@@ -266,6 +268,7 @@ export class WorkspaceCacheMetricsService {
'Total versions across local workspace metadata cache entries',
},
callback: async () => this.getStats().versionsTotal,
+ perPod: true,
});
this.metricsService.createObservableGauge({
metricName: 'twenty_workspace_cache_local_bytes_estimate',
@@ -275,6 +278,7 @@ export class WorkspaceCacheMetricsService {
unit: 'By',
},
callback: async () => this.cacheSizeTotalBytes,
+ perPod: true,
});
this.metricsService.createMultiObservableGauge({
metricName: 'twenty_workspace_cache_local_entries_by_version_count',
@@ -286,6 +290,7 @@ export class WorkspaceCacheMetricsService {
Object.entries(this.getStats().versionsByCount).map(
([versions, value]) => ({ value, attributes: { versions } }),
),
+ perPod: true,
});
this.metricsService.createMultiObservableGauge({
metricName: 'twenty_workspace_cache_local_bytes_by_provider',
@@ -299,6 +304,7 @@ export class WorkspaceCacheMetricsService {
value,
attributes: { provider: keyName },
})),
+ perPod: true,
});
this.metricsService.createMultiObservableGauge({
metricName: 'twenty_workspace_cache_local_entries_by_provider',
@@ -313,6 +319,7 @@ export class WorkspaceCacheMetricsService {
attributes: { provider: keyName },
}),
),
+ perPod: true,
});
this.metricsService.createMultiObservableGauge({
metricName: 'twenty_workspace_cache_local_versions_by_state',
@@ -338,6 +345,7 @@ export class WorkspaceCacheMetricsService {
),
];
},
+ perPod: true,
});
this.metricsService.createObservableGauge({
metricName: 'twenty_workspace_cache_local_live_versions',
@@ -346,6 +354,7 @@ export class WorkspaceCacheMetricsService {
'Total live (object graph) versions in the local workspace metadata cache',
},
callback: async () => this.getStats().liveVersionsTotal,
+ perPod: true,
});
this.metricsService.createObservableGauge({
metricName: 'twenty_workspace_cache_local_packed_versions',
@@ -354,6 +363,7 @@ export class WorkspaceCacheMetricsService {
'Total packed (serialized buffer) versions in the local workspace metadata cache',
},
callback: async () => this.getStats().packedVersionsTotal,
+ perPod: true,
});
this.metricsService.createObservableGauge({
metricName: 'twenty_workspace_cache_local_packed_bytes',
@@ -363,6 +373,7 @@ export class WorkspaceCacheMetricsService {
unit: 'By',
},
callback: async () => this.getStats().packedBytesTotal,
+ perPod: true,
});
this.metricsService.createObservableGauge({
metricName: 'twenty_workspace_cache_packing_backlog',
@@ -371,6 +382,7 @@ export class WorkspaceCacheMetricsService {
'Versions currently eligible for packing, whether or not the last run reached them',
},
callback: async () => this.packingBacklog,
+ perPod: true,
});
}
}
diff --git a/packages/twenty-server/src/instrument.ts b/packages/twenty-server/src/instrument.ts
index bdb21d87570..5498e60f41c 100644
--- a/packages/twenty-server/src/instrument.ts
+++ b/packages/twenty-server/src/instrument.ts
@@ -1,4 +1,3 @@
-import os from 'os';
import process from 'process';
import { metrics as otelMetrics } from '@opentelemetry/api';
@@ -18,6 +17,7 @@ import { nodeProfilingIntegration } from '@sentry/profiling-node';
import { NodeEnvironment } from 'src/engine/core-modules/twenty-config/interfaces/node-environment.interface';
import { ExceptionHandlerDriver } from 'src/engine/core-modules/exception-handler/interfaces';
+import { POD_NAME } from 'src/engine/core-modules/metrics/constants/pod-name.constant';
import { MeterDriver } from 'src/engine/core-modules/metrics/types/meter-driver.type';
import { parseArrayEnvVar } from 'src/utils/parse-array-env-var';
@@ -135,7 +135,7 @@ const prometheusExporter = meterDrivers.includes(MeterDriver.Prometheus)
const meterProvider = new MeterProvider({
resource: resourceFromAttributes({
'service.name': process.env.OTEL_SERVICE_NAME ?? 'twenty-server',
- 'k8s.pod.name': process.env.HOSTNAME ?? os.hostname(),
+ 'k8s.pod.name': POD_NAME,
}),
readers: [
...(meterDrivers.includes(MeterDriver.Console)