The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix(cache-lock): renew the lock lease while withLock runs and fence every release (#26709)
Aditya Mishra11fdffe9202c
diff --git a/packages/twenty-server/src/engine/core-modules/cache-lock/cache-lock.service.ts b/packages/twenty-server/src/engine/core-modules/cache-lock/cache-lock.service.tsindex e7c20f5916a..0bad47642df 100644--- a/packages/twenty-server/src/engine/core-modules/cache-lock/cache-lock.service.ts+++ b/packages/twenty-server/src/engine/core-modules/cache-lock/cache-lock.service.ts@@ -1,5 +1,7 @@import { Injectable, Logger } from '@nestjs/common';+import { isDefined } from 'twenty-shared/utils';+import {CacheLockException,CacheLockExceptionCode,@@ -35,14 +37,36 @@ export class CacheLockService {const { ms = 100, maxRetries = 50, ttl = 5_500 } = options || {};for (let attempt = 0; attempt < maxRetries; attempt++) {- const acquired = await this.cacheStorageService.acquireLock(key, ttl);+ const ownerToken = await this.cacheStorageService.acquireLock({+ key,+ ttl,+ });++ if (isDefined(ownerToken)) {+ const lockExtensionInterval = setInterval(() => {+ void this.cacheStorageService+ .extendLock({ key, ownerToken, ttl })+ .then((isExtended) => {+ if (!isExtended) {+ this.logger.warn(+ `Lost lock for key "${key}" before it was released`,+ );+ }+ })+ .catch((extendError) => {+ this.logger.warn(+ `Failed to extend lock for key "${key}": ${extendError}`,+ );+ });+ }, ttl / 3);- if (acquired) {try {return await fn();} finally {+ clearInterval(lockExtensionInterval);+try {- await this.cacheStorageService.releaseLock(key);+ await this.cacheStorageService.releaseLock({ key, ownerToken });} catch (releaseError) {this.logger.warn(`Failed to release lock for key "${key}": ${releaseError}`,diff --git a/packages/twenty-server/src/engine/core-modules/record-export/constants/update-record-export-lease-script.constant.ts b/packages/twenty-server/src/engine/core-modules/cache-storage/constants/update-owned-key-lease-script.constant.tssimilarity index 74%rename from packages/twenty-server/src/engine/core-modules/record-export/constants/update-record-export-lease-script.constant.tsrename to packages/twenty-server/src/engine/core-modules/cache-storage/constants/update-owned-key-lease-script.constant.tsindex df4c289563c..58a9e844bc9 100644--- a/packages/twenty-server/src/engine/core-modules/record-export/constants/update-record-export-lease-script.constant.ts+++ b/packages/twenty-server/src/engine/core-modules/cache-storage/constants/update-owned-key-lease-script.constant.ts@@ -1,7 +1,7 @@import { type CacheScript } from 'src/engine/core-modules/cache-storage/types/cache-script.type';-export const UPDATE_RECORD_EXPORT_LEASE_SCRIPT: CacheScript = {- name: 'record-export:update-lease',+export const UPDATE_OWNED_KEY_LEASE_SCRIPT: CacheScript = {+ name: 'update-owned-key-lease',source: `if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 endif tonumber(ARGV[2]) > 0 thendiff --git a/packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.ts b/packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.tsindex 3285f3a31e9..4b1a61b4694 100644--- a/packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.ts+++ b/packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.ts@@ -3,9 +3,11 @@ import { Inject, Injectable } from '@nestjs/common';import { type Milliseconds } from 'cache-manager';import { type RedisCache } from 'cache-manager-redis-yet';+import { v4 } from 'uuid';import { isDefined } from 'twenty-shared/utils';+import { UPDATE_OWNED_KEY_LEASE_SCRIPT } from 'src/engine/core-modules/cache-storage/constants/update-owned-key-lease-script.constant';import {CacheStorageException,CacheStorageExceptionCode,@@ -330,27 +332,58 @@ export class CacheStorageService {return count as number;}- async acquireLock(key: string, ttl = 1000): Promise<boolean> {+ async acquireLock({+ key,+ ttl = 1000,+ }: {+ key: string;+ ttl?: number;+ }): Promise<string | null> {if (!this.isRedisCache(this.cache)) {throw new Error('acquireLock is only supported with Redis cache');}const redisClient = this.cache.store.client;+ const ownerToken = v4();- const result = await redisClient.set(this.getKey(key), 'lock', {+ const result = await redisClient.set(this.getKey(key), ownerToken, {NX: true,PX: ttl,});- return result === 'OK';+ return result === 'OK' ? ownerToken : null;}- async releaseLock(key: string): Promise<void> {- if (!this.isRedisCache(this.cache)) {- throw new Error('releaseLock is only supported with Redis cache');- }+ async extendLock({+ key,+ ownerToken,+ ttl,+ }: {+ key: string;+ ownerToken: string;+ ttl: number;+ }): Promise<boolean> {+ const extended = await this.runScript<number>({+ script: UPDATE_OWNED_KEY_LEASE_SCRIPT,+ keys: [key],+ args: [ownerToken, String(ttl)],+ });++ return extended === 1;+ }- await this.del(key);+ async releaseLock({+ key,+ ownerToken,+ }: {+ key: string;+ ownerToken: string;+ }): Promise<void> {+ await this.runScript<number>({+ script: UPDATE_OWNED_KEY_LEASE_SCRIPT,+ keys: [key],+ args: [ownerToken, '0'],+ });}async incrBy(key: string, increment: number): Promise<number> {diff --git a/packages/twenty-server/src/engine/core-modules/cron/services/cron-trigger-deduplication.service.ts b/packages/twenty-server/src/engine/core-modules/cron/services/cron-trigger-deduplication.service.tsindex be78949b6cd..fea9a27747a 100644--- a/packages/twenty-server/src/engine/core-modules/cron/services/cron-trigger-deduplication.service.ts+++ b/packages/twenty-server/src/engine/core-modules/cron/services/cron-trigger-deduplication.service.ts@@ -1,6 +1,7 @@import { Injectable } from '@nestjs/common';import { CronExpressionParser } from 'cron-parser';+import { isDefined } from 'twenty-shared/utils';import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';@@ -42,9 +43,11 @@ export class CronTriggerDeduplicationService {const dedupKey = `${keyPrefix}:${lastTriggerTimestamp}`;- return this.cacheStorageService.acquireLock(- dedupKey,- CRON_DISPATCH_DEDUP_TTL_MS,- );+ const lockOwnerToken = await this.cacheStorageService.acquireLock({+ key: dedupKey,+ ttl: CRON_DISPATCH_DEDUP_TTL_MS,+ });++ return isDefined(lockOwnerToken);}}diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.tsindex b7dbcf90c27..93ae0c0e9b1 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts@@ -30,6 +30,10 @@ export const SDK_LAYER_PREFIX_IN_ZIP = 'nodejs/node_modules/twenty-client-sdk';export const LAMBDA_PREBUILT_BUNDLE_CHECKSUM_TAG = 'twenty:bundle-checksum';export const PREBUILT_BUNDLE_FILE_NAME = 'prebuilt-logic-function.mjs';+export const BUILD_LOCK_TTL_MS = 120_000;+export const BUILD_LOCK_RETRY_MS = 1_000;+export const BUILD_LOCK_MAX_RETRIES = 480;+export const PREBUILT_INSTALL_LOCK_TTL_MS = 180_000;export const PREBUILT_INSTALL_LOCK_RETRY_MS = 1_000;export const PREBUILT_INSTALL_LOCK_MAX_RETRIES = 180;diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.tsindex d88cffad511..0904fa63887 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts@@ -23,6 +23,9 @@ import {CacheLockExceptionCode,} from 'src/engine/core-modules/cache-lock/exceptions/cache-lock.exception';import {+ BUILD_LOCK_MAX_RETRIES,+ BUILD_LOCK_RETRY_MS,+ BUILD_LOCK_TTL_MS,EXECUTOR_LAMBDA_MEMORY_MB,EXECUTOR_LAMBDA_TIMEOUT_SECONDS,LAMBDA_EPHEMERAL_STORAGE_MB,@@ -103,9 +106,6 @@ export class LambdaExecutorManagerService {return;}- const buildLockTtlMs = 120_000;- const buildLockRetryMs = 500;- const buildLockMaxRetries = 240;const lockKey = `lambda-build:${context.flatLogicFunction.id}`;try {@@ -128,9 +128,9 @@ export class LambdaExecutorManagerService {},lockKey,{- ttl: buildLockTtlMs,- ms: buildLockRetryMs,- maxRetries: buildLockMaxRetries,+ ttl: BUILD_LOCK_TTL_MS,+ ms: BUILD_LOCK_RETRY_MS,+ maxRetries: BUILD_LOCK_MAX_RETRIES,},);} catch (error) {diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-layer-manager.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-layer-manager.service.tsindex 160e4f7a515..5c4c69fcacf 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-layer-manager.service.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-layer-manager.service.ts@@ -13,7 +13,12 @@ import { isDefined } from 'twenty-shared/utils';import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';import { type CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';-import { SDK_LAYER_PREFIX_IN_ZIP } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant';+import {+ BUILD_LOCK_RETRY_MS,+ BUILD_LOCK_TTL_MS,+ BUILD_LOCK_MAX_RETRIES,+ SDK_LAYER_PREFIX_IN_ZIP,+} from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant';import { type LambdaAwsClientService } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service';import { type LambdaToolFunctionsService } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-tool-functions.service';import { type LambdaDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/types/lambda-driver.type';@@ -36,9 +41,9 @@ type LayerAppContext = {};const LAYER_LOCK_OPTIONS = {- ttl: 120_000,- ms: 500,- maxRetries: 240,+ ttl: BUILD_LOCK_TTL_MS,+ ms: BUILD_LOCK_RETRY_MS,+ maxRetries: BUILD_LOCK_MAX_RETRIES,};export class LambdaLayerManagerService {diff --git a/packages/twenty-server/src/engine/core-modules/record-export/services/record-export.workspace-service.ts b/packages/twenty-server/src/engine/core-modules/record-export/services/record-export.workspace-service.tsindex 4344af16d46..a845b6b9674 100644--- a/packages/twenty-server/src/engine/core-modules/record-export/services/record-export.workspace-service.ts+++ b/packages/twenty-server/src/engine/core-modules/record-export/services/record-export.workspace-service.ts@@ -46,7 +46,7 @@ import {RECORD_EXPORT_PROGRESS_INTERVAL_MS,RECORD_EXPORT_PAGE_SIZE,} from 'src/engine/core-modules/record-export/constants/record-export.constants';-import { UPDATE_RECORD_EXPORT_LEASE_SCRIPT } from 'src/engine/core-modules/record-export/constants/update-record-export-lease-script.constant';+import { UPDATE_OWNED_KEY_LEASE_SCRIPT } from 'src/engine/core-modules/cache-storage/constants/update-owned-key-lease-script.constant';import { type RecordExportDTO } from 'src/engine/core-modules/record-export/dtos/record-export.dto';import { RecordExportException } from 'src/engine/core-modules/record-export/record-export.exception';import { type RecordExportColumn } from 'src/engine/core-modules/record-export/types/record-export-column.type';@@ -455,7 +455,7 @@ export class RecordExportWorkspaceService {): Promise<boolean> {return ((await this.cacheStorageService.runScript<number>({- script: UPDATE_RECORD_EXPORT_LEASE_SCRIPT,+ script: UPDATE_OWNED_KEY_LEASE_SCRIPT,keys: [`{${workspaceId}}:active`],args: [JSON.stringify(id), String(ttl)],})) === 1diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status-cache.service.ts b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status-cache.service.tsindex 6c7600adac1..08930c97462 100644--- a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status-cache.service.ts+++ b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status-cache.service.ts@@ -81,12 +81,18 @@ export class UpgradeStatusCacheService {]);}- async tryAcquireRefreshLock(): Promise<boolean> {- return this.cacheStorage.acquireLock(REFRESH_LOCK_KEY, REFRESH_LOCK_TTL_MS);+ async tryAcquireRefreshLock(): Promise<string | null> {+ return this.cacheStorage.acquireLock({+ key: REFRESH_LOCK_KEY,+ ttl: REFRESH_LOCK_TTL_MS,+ });}- async releaseRefreshLock(): Promise<void> {- await this.cacheStorage.releaseLock(REFRESH_LOCK_KEY);+ async releaseRefreshLock(refreshLockOwnerToken: string): Promise<void> {+ await this.cacheStorage.releaseLock({+ key: REFRESH_LOCK_KEY,+ ownerToken: refreshLockOwnerToken,+ });}async invalidate(): Promise<void> {@@ -95,7 +101,7 @@ export class UpgradeStatusCacheService {this.cacheStorage.del(FAILED_IDS_KEY),this.cacheStorage.del(UP_TO_DATE_COUNT_KEY),this.cacheStorage.del(COMPUTED_AT_KEY),- this.cacheStorage.releaseLock(REFRESH_LOCK_KEY),+ this.cacheStorage.del(REFRESH_LOCK_KEY),]);}}diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status.service.ts b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status.service.tsindex f3a2f469e83..c3f89c40d33 100644--- a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status.service.ts+++ b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status.service.ts@@ -178,10 +178,10 @@ export class UpgradeStatusService {}async getInstanceAndWorkspaceCountsStatus(): Promise<InstanceAndWorkspaceCountsUpgradeStatus | null> {- const hasRefreshLock =+ const refreshLockOwnerToken =await this.upgradeStatusCacheService.tryAcquireRefreshLock();- if (hasRefreshLock) {+ if (isDefined(refreshLockOwnerToken)) {try {const refreshedStatus =await this.refreshInstanceAndAllWorkspacesStatus();@@ -196,7 +196,9 @@ export class UpgradeStatusService {} catch (error) {this.logger.error('Failed to refresh upgrade status', error);- await this.upgradeStatusCacheService.releaseRefreshLock();+ await this.upgradeStatusCacheService.releaseRefreshLock(+ refreshLockOwnerToken,+ );}}diff --git a/packages/twenty-server/src/modules/emailing/services/message-campaign-statistics.service.ts b/packages/twenty-server/src/modules/emailing/services/message-campaign-statistics.service.tsindex a9d9db89a93..ffa91262a5c 100644--- a/packages/twenty-server/src/modules/emailing/services/message-campaign-statistics.service.ts+++ b/packages/twenty-server/src/modules/emailing/services/message-campaign-statistics.service.ts@@ -48,12 +48,12 @@ export class MessageCampaignStatisticsService {}): Promise<void> {const lockKey = `campaign-stats-refresh:${workspaceId}:${campaignId}`;- const acquired = await this.cacheStorageService.acquireLock(- lockKey,- REFRESH_LOCK_TTL_MS,- );+ const lockOwnerToken = await this.cacheStorageService.acquireLock({+ key: lockKey,+ ttl: REFRESH_LOCK_TTL_MS,+ });- if (!acquired) {+ if (!isDefined(lockOwnerToken)) {return;}@@ -67,7 +67,10 @@ export class MessageCampaignStatisticsService {},).catch(async (error) => {- await this.cacheStorageService.releaseLock(lockKey);+ await this.cacheStorageService.releaseLock({+ key: lockKey,+ ownerToken: lockOwnerToken,+ });throw error;});diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.tsindex b6e8dc7a665..374c5c7e003 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts@@ -1,6 +1,7 @@import { Injectable, Logger } from '@nestjs/common';import { QUERY_MAX_RECORDS } from 'twenty-shared/constants';+import { isDefined } from 'twenty-shared/utils';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';@@ -37,12 +38,12 @@ export class WorkflowRunEnqueueWorkspaceService {workspaceId: string;isCacheMode: boolean;}) {- const lockAcquired =- await this.workflowThrottlingWorkspaceService.acquireWorkflowEnqueueLock(+ const lockOwnerToken =+ await this.workflowThrottlingWorkspaceService.acquireWorkflowEnqueueLock({workspaceId,- );+ });- if (!lockAcquired) {+ if (!isDefined(lockOwnerToken)) {return;}@@ -164,7 +165,10 @@ export class WorkflowRunEnqueueWorkspaceService {} finally {try {await this.workflowThrottlingWorkspaceService.releaseWorkflowEnqueueLock(- workspaceId,+ {+ workspaceId,+ lockOwnerToken,+ },);} catch (releaseError) {this.logger.warn(diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.tsindex 4d6f0e46a4c..357e40b7d59 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts@@ -105,19 +105,28 @@ export class WorkflowThrottlingWorkspaceService {}, authContext);}- async acquireWorkflowEnqueueLock(- workspaceId: string,+ async acquireWorkflowEnqueueLock({+ workspaceId,ttlMs = 60_000,- ): Promise<boolean> {+ }: {+ workspaceId: string;+ ttlMs?: number;+ }): Promise<string | null> {const key = this.getWorkflowEnqueueRunningCacheKey(workspaceId);- return this.cacheStorage.acquireLock(key, ttlMs);+ return this.cacheStorage.acquireLock({ key, ttl: ttlMs });}- async releaseWorkflowEnqueueLock(workspaceId: string): Promise<void> {+ async releaseWorkflowEnqueueLock({+ workspaceId,+ lockOwnerToken,+ }: {+ workspaceId: string;+ lockOwnerToken: string;+ }): Promise<void> {const key = this.getWorkflowEnqueueRunningCacheKey(workspaceId);- await this.cacheStorage.releaseLock(key);+ await this.cacheStorage.releaseLock({ key, ownerToken: lockOwnerToken });}private async setWorkflowRunNotStartedCount(