Skip to content

The open alternative to Salesforce, designed for AI.

Stop upgrade status gauges from stampeding the database (#27293)

Charles Bochetc3ee08df58bd
diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/2-46/2-46-instance-command-fast-1791215192958-add-upgrade-migration-workspace-id-created-at-index.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/2-46/2-46-instance-command-fast-1791215192958-add-upgrade-migration-workspace-id-created-at-index.ts
new file mode 100644
index 00000000000..d1e621553b2
--- /dev/null
+++ b/packages/twenty-server/src/database/commands/upgrade-version-command/2-46/2-46-instance-command-fast-1791215192958-add-upgrade-migration-workspace-id-created-at-index.ts
@@ -0,0 +1,19 @@
+import { type QueryRunner } from 'typeorm';
+
+import { RegisteredInstanceCommand } from 'src/engine/core-modules/upgrade/decorators/registered-instance-command.decorator';
+import { type FastInstanceCommand } from 'src/engine/core-modules/upgrade/interfaces/fast-instance-command.interface';
+
+@RegisteredInstanceCommand('2.46.0', 1791215192958)
+export class AddUpgradeMigrationWorkspaceIdCreatedAtIndexFastInstanceCommand implements FastInstanceCommand {
+ public async up(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.query(
+ 'CREATE INDEX IF NOT EXISTS "IDX_UPGRADE_MIGRATION_WORKSPACE_ID_CREATED_AT" ON "core"."upgradeMigration" ("workspaceId", "createdAt") WHERE "workspaceId" IS NOT NULL',
+ );
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.query(
+ 'DROP INDEX IF EXISTS "core"."IDX_UPGRADE_MIGRATION_WORKSPACE_ID_CREATED_AT"',
+ );
+ }
+}
diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/instance-commands.constant.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/instance-commands.constant.ts
index 46e784ff69f..e3f9b03de8f 100644
--- a/packages/twenty-server/src/database/commands/upgrade-version-command/instance-commands.constant.ts
+++ b/packages/twenty-server/src/database/commands/upgrade-version-command/instance-commands.constant.ts
@@ -223,6 +223,7 @@ import { AddIsSystemToAgentAndWorkflowFastInstanceCommand } from 'src/database/c
import { CreateWorkflowStepWaitTableFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-46/2-46-instance-command-fast-1791148642493-create-workflow-step-wait-table';
import { DropCoreAgentHistoryTablesFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-47/2-47-instance-command-fast-1791094130961-drop-core-agent-history-tables';
import { RenameUsageLimitMeterToUnitFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-46/2-46-instance-command-fast-1791186790123-rename-usage-limit-meter-to-unit';
+import { AddUpgradeMigrationWorkspaceIdCreatedAtIndexFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-46/2-46-instance-command-fast-1791215192958-add-upgrade-migration-workspace-id-created-at-index';
export const INSTANCE_COMMANDS = [
AddViewFieldGroupIdIndexOnViewFieldFastInstanceCommand,
@@ -448,4 +449,5 @@ export const INSTANCE_COMMANDS = [
CreateWorkflowStepWaitTableFastInstanceCommand,
DropCoreAgentHistoryTablesFastInstanceCommand,
RenameUsageLimitMeterToUnitFastInstanceCommand,
+ AddUpgradeMigrationWorkspaceIdCreatedAtIndexFastInstanceCommand,
];
diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-migration.service.ts b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-migration.service.ts
index 80a2d4bfe9a..d0d98fa0a8c 100644
--- a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-migration.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-migration.service.ts
@@ -238,17 +238,17 @@ export class UpgradeMigrationService {
}>
>(
`
- SELECT DISTINCT ON (latest_per_name."workspaceId")
- latest_per_name."workspaceId",
- latest_per_name.name,
- latest_per_name.status,
- latest_per_name."executedByVersion",
- latest_per_name."errorMessage",
- latest_per_name."createdAt",
- latest_per_name."isInitial"
- FROM (
- SELECT DISTINCT ON ("workspaceId", name)
- "workspaceId",
+ SELECT
+ workspace_ids."workspaceId",
+ latest.name,
+ latest.status,
+ latest."executedByVersion",
+ latest."errorMessage",
+ latest."createdAt",
+ latest."isInitial"
+ FROM unnest($1::uuid[]) AS workspace_ids("workspaceId")
+ CROSS JOIN LATERAL (
+ SELECT
name,
status,
"executedByVersion",
@@ -256,10 +256,10 @@ export class UpgradeMigrationService {
"createdAt",
"isInitial"
FROM core."upgradeMigration"
- WHERE "workspaceId" = ANY($1)
- ORDER BY "workspaceId", name, attempt DESC
- ) latest_per_name
- ORDER BY latest_per_name."workspaceId", latest_per_name."createdAt" DESC
+ WHERE "workspaceId" = workspace_ids."workspaceId"
+ ORDER BY "createdAt" DESC
+ LIMIT 1
+ ) latest
`,
[workspaceIds],
);
diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status-cache.service.ts b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status-cache.service.ts
index ade3db3c2bf..6c7600adac1 100644
--- a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status-cache.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status-cache.service.ts
@@ -10,8 +10,10 @@ const BEHIND_IDS_KEY = 'upgrade-status:behind-workspace-ids';
const FAILED_IDS_KEY = 'upgrade-status:failed-workspace-ids';
const UP_TO_DATE_COUNT_KEY = 'upgrade-status:up-to-date-workspace-count';
const COMPUTED_AT_KEY = 'upgrade-status:computed-at';
+const REFRESH_LOCK_KEY = 'upgrade-status:refresh-lock';
const CACHE_TTL_MS = 60 * 60 * 1000;
+const REFRESH_LOCK_TTL_MS = CACHE_TTL_MS - 10 * 1000;
@Injectable()
export class UpgradeStatusCacheService {
constructor(
@@ -79,12 +81,21 @@ export class UpgradeStatusCacheService {
]);
}
+ async tryAcquireRefreshLock(): Promise<boolean> {
+ return this.cacheStorage.acquireLock(REFRESH_LOCK_KEY, REFRESH_LOCK_TTL_MS);
+ }
+
+ async releaseRefreshLock(): Promise<void> {
+ await this.cacheStorage.releaseLock(REFRESH_LOCK_KEY);
+ }
+
async invalidate(): Promise<void> {
await Promise.all([
this.cacheStorage.del(BEHIND_IDS_KEY),
this.cacheStorage.del(FAILED_IDS_KEY),
this.cacheStorage.del(UP_TO_DATE_COUNT_KEY),
this.cacheStorage.del(COMPUTED_AT_KEY),
+ this.cacheStorage.releaseLock(REFRESH_LOCK_KEY),
]);
}
}
diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status.service.ts b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status.service.ts
index 70c200a245f..f3a2f469e83 100644
--- a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status.service.ts
@@ -177,20 +177,33 @@ export class UpgradeStatusService {
});
}
- async getInstanceAndWorkspaceCountsStatus(): Promise<InstanceAndWorkspaceCountsUpgradeStatus> {
+ async getInstanceAndWorkspaceCountsStatus(): Promise<InstanceAndWorkspaceCountsUpgradeStatus | null> {
+ const hasRefreshLock =
+ await this.upgradeStatusCacheService.tryAcquireRefreshLock();
+
+ if (hasRefreshLock) {
+ try {
+ const refreshedStatus =
+ await this.refreshInstanceAndAllWorkspacesStatus();
+
+ return {
+ instanceUpgradeStatus: refreshedStatus.instanceUpgradeStatus,
+ behindWorkspaceCount: refreshedStatus.workspacesBehind.length,
+ failedWorkspaceCount: refreshedStatus.workspacesFailed.length,
+ upToDateWorkspaceCount: refreshedStatus.upToDateWorkspaceCount,
+ computedAt: refreshedStatus.computedAt,
+ };
+ } catch (error) {
+ this.logger.error('Failed to refresh upgrade status', error);
+
+ await this.upgradeStatusCacheService.releaseRefreshLock();
+ }
+ }
+
const cachedStatus = await this.getCachedInstanceAndWorkspaceStatus();
if (!isDefined(cachedStatus)) {
- const refreshedStatus =
- await this.refreshInstanceAndAllWorkspacesStatus();
-
- return {
- instanceUpgradeStatus: refreshedStatus.instanceUpgradeStatus,
- behindWorkspaceCount: refreshedStatus.workspacesBehind.length,
- failedWorkspaceCount: refreshedStatus.workspacesFailed.length,
- upToDateWorkspaceCount: refreshedStatus.upToDateWorkspaceCount,
- computedAt: refreshedStatus.computedAt,
- };
+ return null;
}
return {
diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-gauge.service.ts b/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-gauge.service.ts
index 459c28abd67..02e7c3eaddb 100644
--- a/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-gauge.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-gauge.service.ts
@@ -1,6 +1,7 @@
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { UpgradeHealthEnum } from 'twenty-shared/types';
+import { isDefined } from 'twenty-shared/utils';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import {
@@ -24,7 +25,7 @@ export class UpgradeGaugeService implements OnModuleInit {
private cachedUpgradeStatus: InstanceAndWorkspaceCountsUpgradeStatus | null =
null;
private cachedUpgradeStatusExpiresAt = 0;
- private inflightUpgradeStatusPromise: Promise<InstanceAndWorkspaceCountsUpgradeStatus> | null =
+ private inflightUpgradeStatusPromise: Promise<InstanceAndWorkspaceCountsUpgradeStatus | null> | null =
null;
constructor(
@@ -119,14 +120,24 @@ export class UpgradeGaugeService implements OnModuleInit {
}
if (this.inflightUpgradeStatusPromise) {
- return this.inflightUpgradeStatusPromise.catch(() => null);
+ return this.inflightUpgradeStatusPromise.then(
+ (upgradeStatus) =>
+ isDefined(upgradeStatus) ? upgradeStatus : this.cachedUpgradeStatus,
+ () => null,
+ );
}
this.inflightUpgradeStatusPromise =
this.upgradeStatusService.getInstanceAndWorkspaceCountsStatus();
try {
- this.cachedUpgradeStatus = await this.inflightUpgradeStatusPromise;
+ const upgradeStatus = await this.inflightUpgradeStatusPromise;
+
+ if (!isDefined(upgradeStatus)) {
+ return this.cachedUpgradeStatus;
+ }
+
+ this.cachedUpgradeStatus = upgradeStatus;
this.cachedUpgradeStatusExpiresAt = Date.now() + UPGRADE_STATUS_TTL_MS;
return this.cachedUpgradeStatus;
diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-migration.entity.ts b/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-migration.entity.ts
index 198ed7d0429..88bbcc3df54 100644
--- a/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-migration.entity.ts
+++ b/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-migration.entity.ts
@@ -29,6 +29,13 @@ export type UpgradeMigrationStatus = 'completed' | 'failed';
where: '"workspaceId" IS NOT NULL',
},
)
+@Index(
+ 'IDX_UPGRADE_MIGRATION_WORKSPACE_ID_CREATED_AT',
+ ['workspaceId', 'createdAt'],
+ {
+ where: '"workspaceId" IS NOT NULL',
+ },
+)
export class UpgradeMigrationEntity {
@PrimaryGeneratedColumn('uuid')
id: string;