Skip to content

The open alternative to Salesforce, designed for AI.

feat: expose upgrade status as Prometheus gauge metrics (#20262)

Charles Bochetfda2295beb84
diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-gauge.service.ts b/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-gauge.service.ts
new file mode 100644
index 00000000000..78f8da74f0e
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-gauge.service.ts
@@ -0,0 +1,112 @@
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+
+import { UpgradeHealthEnum } from 'twenty-shared/types';
+
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import {
+ type InstanceAndAllWorkspacesUpgradeStatus,
+ UpgradeStatusService,
+} from 'src/engine/core-modules/upgrade/services/upgrade-status.service';
+
+const HEALTH_TO_GAUGE_VALUE: Record<UpgradeHealthEnum, number> = {
+ [UpgradeHealthEnum.UP_TO_DATE]: 1,
+ [UpgradeHealthEnum.BEHIND]: 0,
+ [UpgradeHealthEnum.FAILED]: -1,
+};
+
+const HEALTH_UNKNOWN = -2;
+const UPGRADE_STATUS_TTL_MS = 60_000;
+
+@Injectable()
+export class UpgradeGaugeService implements OnModuleInit {
+ private readonly logger = new Logger(UpgradeGaugeService.name);
+
+ private cachedUpgradeStatus: InstanceAndAllWorkspacesUpgradeStatus | null =
+ null;
+ private cachedUpgradeStatusExpiresAt = 0;
+ private inflightUpgradeStatusPromise: Promise<InstanceAndAllWorkspacesUpgradeStatus> | null =
+ null;
+
+ constructor(
+ private readonly metricsService: MetricsService,
+ private readonly upgradeStatusService: UpgradeStatusService,
+ ) {}
+
+ onModuleInit() {
+ this.metricsService.createObservableGauge({
+ metricName: 'twenty_upgrade_instance_health',
+ options: {
+ description:
+ 'Instance upgrade health (1 = up-to-date, 0 = behind, -1 = failed, -2 = unknown)',
+ },
+ callback: async () => {
+ const upgradeStatus = await this.getCachedUpgradeStatus();
+
+ if (!upgradeStatus) {
+ return HEALTH_UNKNOWN;
+ }
+
+ return (
+ HEALTH_TO_GAUGE_VALUE[upgradeStatus.instanceUpgradeStatus.health] ??
+ HEALTH_UNKNOWN
+ );
+ },
+ cacheValue: true,
+ });
+
+ this.metricsService.createObservableGauge({
+ metricName: 'twenty_upgrade_workspaces_behind_total',
+ options: {
+ description: 'Number of workspaces behind on upgrade commands',
+ },
+ callback: async () => {
+ const upgradeStatus = await this.getCachedUpgradeStatus();
+
+ return upgradeStatus?.workspacesBehind.length ?? 0;
+ },
+ cacheValue: true,
+ });
+
+ this.metricsService.createObservableGauge({
+ metricName: 'twenty_upgrade_workspaces_failed_total',
+ options: {
+ description: 'Number of workspaces with a failed upgrade command',
+ },
+ callback: async () => {
+ const upgradeStatus = await this.getCachedUpgradeStatus();
+
+ return upgradeStatus?.workspacesFailed.length ?? 0;
+ },
+ cacheValue: true,
+ });
+ }
+
+ private async getCachedUpgradeStatus(): Promise<InstanceAndAllWorkspacesUpgradeStatus | null> {
+ if (
+ this.cachedUpgradeStatus &&
+ Date.now() < this.cachedUpgradeStatusExpiresAt
+ ) {
+ return this.cachedUpgradeStatus;
+ }
+
+ if (this.inflightUpgradeStatusPromise) {
+ return this.inflightUpgradeStatusPromise.catch(() => null);
+ }
+
+ this.inflightUpgradeStatusPromise =
+ this.upgradeStatusService.getInstanceAndAllWorkspacesStatus();
+
+ try {
+ this.cachedUpgradeStatus = await this.inflightUpgradeStatusPromise;
+ this.cachedUpgradeStatusExpiresAt = Date.now() + UPGRADE_STATUS_TTL_MS;
+
+ return this.cachedUpgradeStatus;
+ } catch (error) {
+ this.logger.error('Failed to fetch upgrade status for gauges', error);
+
+ return null;
+ } finally {
+ this.inflightUpgradeStatusPromise = null;
+ }
+ }
+}
diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/upgrade.module.ts b/packages/twenty-server/src/engine/core-modules/upgrade/upgrade.module.ts
index b6415c16d75..a9bad4f5bcf 100644
--- a/packages/twenty-server/src/engine/core-modules/upgrade/upgrade.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/upgrade/upgrade.module.ts
@@ -6,6 +6,7 @@ import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/w
import { InstanceCommandProviderModule } from 'src/database/commands/upgrade-version-command/instance-command-provider.module';
import { WorkspaceCommandProviderModule } from 'src/database/commands/upgrade-version-command/workspace-command-provider.module';
import { CoreEntityCacheModule } from 'src/engine/core-entity-cache/core-entity-cache.module';
+import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
import { InstanceCommandRunnerService } from 'src/engine/core-modules/upgrade/services/instance-command-runner.service';
import { UpgradeCommandRegistryService } from 'src/engine/core-modules/upgrade/services/upgrade-command-registry.service';
import { UpgradeMigrationService } from 'src/engine/core-modules/upgrade/services/upgrade-migration.service';
@@ -14,6 +15,7 @@ import { UpgradeSequenceRunnerService } from 'src/engine/core-modules/upgrade/se
import { UpgradeStatusCacheService } from 'src/engine/core-modules/upgrade/services/upgrade-status-cache.service';
import { UpgradeStatusService } from 'src/engine/core-modules/upgrade/services/upgrade-status.service';
import { WorkspaceCommandRunnerService } from 'src/engine/core-modules/upgrade/services/workspace-command-runner.service';
+import { UpgradeGaugeService } from 'src/engine/core-modules/upgrade/upgrade-gauge.service';
import { UpgradeMigrationEntity } from 'src/engine/core-modules/upgrade/upgrade-migration.entity';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-version/workspace-version.module';
@@ -23,6 +25,7 @@ import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-v
CoreEntityCacheModule,
DiscoveryModule,
InstanceCommandProviderModule,
+ MetricsModule,
WorkspaceCommandProviderModule,
WorkspaceIteratorModule,
WorkspaceVersionModule,
@@ -37,6 +40,7 @@ import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-v
UpgradeSequenceRunnerService,
UpgradeStatusService,
UpgradeStatusCacheService,
+ UpgradeGaugeService,
],
exports: [
UpgradeMigrationService,