Skip to content

The open alternative to Salesforce, designed for AI.

fix(cache-lock): renew the lock lease while withLock runs and fence every release (#26709)

Aditya Mishra11fdffe9202c
diff --git a/packages/twenty-server/src/engine/core-modules/cache-lock/cache-lock.service.ts b/packages/twenty-server/src/engine/core-modules/cache-lock/cache-lock.service.ts
index e7c20f5916a..0bad47642df 100644
--- a/packages/twenty-server/src/engine/core-modules/cache-lock/cache-lock.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/cache-lock/cache-lock.service.ts
@@ -1,5 +1,7 @@
import { Injectable, Logger } from '@nestjs/common';
+import { isDefined } from 'twenty-shared/utils';
+
import {
CacheLockException,
CacheLockExceptionCode,
@@ -35,14 +37,36 @@ export class CacheLockService {
const { ms = 100, maxRetries = 50, ttl = 5_500 } = options || {};
for (let attempt = 0; attempt < maxRetries; attempt++) {
- const acquired = await this.cacheStorageService.acquireLock(key, ttl);
+ const ownerToken = await this.cacheStorageService.acquireLock({
+ key,
+ ttl,
+ });
+
+ if (isDefined(ownerToken)) {
+ const lockExtensionInterval = setInterval(() => {
+ void this.cacheStorageService
+ .extendLock({ key, ownerToken, ttl })
+ .then((isExtended) => {
+ if (!isExtended) {
+ this.logger.warn(
+ `Lost lock for key "${key}" before it was released`,
+ );
+ }
+ })
+ .catch((extendError) => {
+ this.logger.warn(
+ `Failed to extend lock for key "${key}": ${extendError}`,
+ );
+ });
+ }, ttl / 3);
- if (acquired) {
try {
return await fn();
} finally {
+ clearInterval(lockExtensionInterval);
+
try {
- await this.cacheStorageService.releaseLock(key);
+ await this.cacheStorageService.releaseLock({ key, ownerToken });
} catch (releaseError) {
this.logger.warn(
`Failed to release lock for key "${key}": ${releaseError}`,
diff --git a/packages/twenty-server/src/engine/core-modules/record-export/constants/update-record-export-lease-script.constant.ts b/packages/twenty-server/src/engine/core-modules/cache-storage/constants/update-owned-key-lease-script.constant.ts
similarity index 74%
rename from packages/twenty-server/src/engine/core-modules/record-export/constants/update-record-export-lease-script.constant.ts
rename to packages/twenty-server/src/engine/core-modules/cache-storage/constants/update-owned-key-lease-script.constant.ts
index df4c289563c..58a9e844bc9 100644
--- a/packages/twenty-server/src/engine/core-modules/record-export/constants/update-record-export-lease-script.constant.ts
+++ b/packages/twenty-server/src/engine/core-modules/cache-storage/constants/update-owned-key-lease-script.constant.ts
@@ -1,7 +1,7 @@
import { type CacheScript } from 'src/engine/core-modules/cache-storage/types/cache-script.type';
-export const UPDATE_RECORD_EXPORT_LEASE_SCRIPT: CacheScript = {
- name: 'record-export:update-lease',
+export const UPDATE_OWNED_KEY_LEASE_SCRIPT: CacheScript = {
+ name: 'update-owned-key-lease',
source: `
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
if tonumber(ARGV[2]) > 0 then
diff --git a/packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.ts b/packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.ts
index 3285f3a31e9..4b1a61b4694 100644
--- a/packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.ts
@@ -3,9 +3,11 @@ import { Inject, Injectable } from '@nestjs/common';
import { type Milliseconds } from 'cache-manager';
import { type RedisCache } from 'cache-manager-redis-yet';
+import { v4 } from 'uuid';
import { isDefined } from 'twenty-shared/utils';
+import { UPDATE_OWNED_KEY_LEASE_SCRIPT } from 'src/engine/core-modules/cache-storage/constants/update-owned-key-lease-script.constant';
import {
CacheStorageException,
CacheStorageExceptionCode,
@@ -330,27 +332,58 @@ export class CacheStorageService {
return count as number;
}
- async acquireLock(key: string, ttl = 1000): Promise<boolean> {
+ async acquireLock({
+ key,
+ ttl = 1000,
+ }: {
+ key: string;
+ ttl?: number;
+ }): Promise<string | null> {
if (!this.isRedisCache(this.cache)) {
throw new Error('acquireLock is only supported with Redis cache');
}
const redisClient = this.cache.store.client;
+ const ownerToken = v4();
- const result = await redisClient.set(this.getKey(key), 'lock', {
+ const result = await redisClient.set(this.getKey(key), ownerToken, {
NX: true,
PX: ttl,
});
- return result === 'OK';
+ return result === 'OK' ? ownerToken : null;
}
- async releaseLock(key: string): Promise<void> {
- if (!this.isRedisCache(this.cache)) {
- throw new Error('releaseLock is only supported with Redis cache');
- }
+ async extendLock({
+ key,
+ ownerToken,
+ ttl,
+ }: {
+ key: string;
+ ownerToken: string;
+ ttl: number;
+ }): Promise<boolean> {
+ const extended = await this.runScript<number>({
+ script: UPDATE_OWNED_KEY_LEASE_SCRIPT,
+ keys: [key],
+ args: [ownerToken, String(ttl)],
+ });
+
+ return extended === 1;
+ }
- await this.del(key);
+ async releaseLock({
+ key,
+ ownerToken,
+ }: {
+ key: string;
+ ownerToken: string;
+ }): Promise<void> {
+ await this.runScript<number>({
+ script: UPDATE_OWNED_KEY_LEASE_SCRIPT,
+ keys: [key],
+ args: [ownerToken, '0'],
+ });
}
async incrBy(key: string, increment: number): Promise<number> {
diff --git a/packages/twenty-server/src/engine/core-modules/cron/services/cron-trigger-deduplication.service.ts b/packages/twenty-server/src/engine/core-modules/cron/services/cron-trigger-deduplication.service.ts
index be78949b6cd..fea9a27747a 100644
--- a/packages/twenty-server/src/engine/core-modules/cron/services/cron-trigger-deduplication.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/cron/services/cron-trigger-deduplication.service.ts
@@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import { CronExpressionParser } from 'cron-parser';
+import { isDefined } from 'twenty-shared/utils';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
@@ -42,9 +43,11 @@ export class CronTriggerDeduplicationService {
const dedupKey = `${keyPrefix}:${lastTriggerTimestamp}`;
- return this.cacheStorageService.acquireLock(
- dedupKey,
- CRON_DISPATCH_DEDUP_TTL_MS,
- );
+ const lockOwnerToken = await this.cacheStorageService.acquireLock({
+ key: dedupKey,
+ ttl: CRON_DISPATCH_DEDUP_TTL_MS,
+ });
+
+ return isDefined(lockOwnerToken);
}
}
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts
index b7dbcf90c27..93ae0c0e9b1 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts
@@ -30,6 +30,10 @@ export const SDK_LAYER_PREFIX_IN_ZIP = 'nodejs/node_modules/twenty-client-sdk';
export const LAMBDA_PREBUILT_BUNDLE_CHECKSUM_TAG = 'twenty:bundle-checksum';
export const PREBUILT_BUNDLE_FILE_NAME = 'prebuilt-logic-function.mjs';
+export const BUILD_LOCK_TTL_MS = 120_000;
+export const BUILD_LOCK_RETRY_MS = 1_000;
+export const BUILD_LOCK_MAX_RETRIES = 480;
+
export const PREBUILT_INSTALL_LOCK_TTL_MS = 180_000;
export const PREBUILT_INSTALL_LOCK_RETRY_MS = 1_000;
export const PREBUILT_INSTALL_LOCK_MAX_RETRIES = 180;
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts
index d88cffad511..0904fa63887 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts
@@ -23,6 +23,9 @@ import {
CacheLockExceptionCode,
} from 'src/engine/core-modules/cache-lock/exceptions/cache-lock.exception';
import {
+ BUILD_LOCK_MAX_RETRIES,
+ BUILD_LOCK_RETRY_MS,
+ BUILD_LOCK_TTL_MS,
EXECUTOR_LAMBDA_MEMORY_MB,
EXECUTOR_LAMBDA_TIMEOUT_SECONDS,
LAMBDA_EPHEMERAL_STORAGE_MB,
@@ -103,9 +106,6 @@ export class LambdaExecutorManagerService {
return;
}
- const buildLockTtlMs = 120_000;
- const buildLockRetryMs = 500;
- const buildLockMaxRetries = 240;
const lockKey = `lambda-build:${context.flatLogicFunction.id}`;
try {
@@ -128,9 +128,9 @@ export class LambdaExecutorManagerService {
},
lockKey,
{
- ttl: buildLockTtlMs,
- ms: buildLockRetryMs,
- maxRetries: buildLockMaxRetries,
+ ttl: BUILD_LOCK_TTL_MS,
+ ms: BUILD_LOCK_RETRY_MS,
+ maxRetries: BUILD_LOCK_MAX_RETRIES,
},
);
} catch (error) {
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-layer-manager.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-layer-manager.service.ts
index 160e4f7a515..5c4c69fcacf 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-layer-manager.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-layer-manager.service.ts
@@ -13,7 +13,12 @@ import { isDefined } from 'twenty-shared/utils';
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import { type CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';
-import { SDK_LAYER_PREFIX_IN_ZIP } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant';
+import {
+ BUILD_LOCK_RETRY_MS,
+ BUILD_LOCK_TTL_MS,
+ BUILD_LOCK_MAX_RETRIES,
+ SDK_LAYER_PREFIX_IN_ZIP,
+} from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant';
import { type LambdaAwsClientService } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service';
import { type LambdaToolFunctionsService } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-tool-functions.service';
import { type LambdaDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/types/lambda-driver.type';
@@ -36,9 +41,9 @@ type LayerAppContext = {
};
const LAYER_LOCK_OPTIONS = {
- ttl: 120_000,
- ms: 500,
- maxRetries: 240,
+ ttl: BUILD_LOCK_TTL_MS,
+ ms: BUILD_LOCK_RETRY_MS,
+ maxRetries: BUILD_LOCK_MAX_RETRIES,
};
export class LambdaLayerManagerService {
diff --git a/packages/twenty-server/src/engine/core-modules/record-export/services/record-export.workspace-service.ts b/packages/twenty-server/src/engine/core-modules/record-export/services/record-export.workspace-service.ts
index 4344af16d46..a845b6b9674 100644
--- a/packages/twenty-server/src/engine/core-modules/record-export/services/record-export.workspace-service.ts
+++ b/packages/twenty-server/src/engine/core-modules/record-export/services/record-export.workspace-service.ts
@@ -46,7 +46,7 @@ import {
RECORD_EXPORT_PROGRESS_INTERVAL_MS,
RECORD_EXPORT_PAGE_SIZE,
} from 'src/engine/core-modules/record-export/constants/record-export.constants';
-import { UPDATE_RECORD_EXPORT_LEASE_SCRIPT } from 'src/engine/core-modules/record-export/constants/update-record-export-lease-script.constant';
+import { UPDATE_OWNED_KEY_LEASE_SCRIPT } from 'src/engine/core-modules/cache-storage/constants/update-owned-key-lease-script.constant';
import { type RecordExportDTO } from 'src/engine/core-modules/record-export/dtos/record-export.dto';
import { RecordExportException } from 'src/engine/core-modules/record-export/record-export.exception';
import { type RecordExportColumn } from 'src/engine/core-modules/record-export/types/record-export-column.type';
@@ -455,7 +455,7 @@ export class RecordExportWorkspaceService {
): Promise<boolean> {
return (
(await this.cacheStorageService.runScript<number>({
- script: UPDATE_RECORD_EXPORT_LEASE_SCRIPT,
+ script: UPDATE_OWNED_KEY_LEASE_SCRIPT,
keys: [`{${workspaceId}}:active`],
args: [JSON.stringify(id), String(ttl)],
})) === 1
diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status-cache.service.ts b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status-cache.service.ts
index 6c7600adac1..08930c97462 100644
--- a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status-cache.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status-cache.service.ts
@@ -81,12 +81,18 @@ export class UpgradeStatusCacheService {
]);
}
- async tryAcquireRefreshLock(): Promise<boolean> {
- return this.cacheStorage.acquireLock(REFRESH_LOCK_KEY, REFRESH_LOCK_TTL_MS);
+ async tryAcquireRefreshLock(): Promise<string | null> {
+ return this.cacheStorage.acquireLock({
+ key: REFRESH_LOCK_KEY,
+ ttl: REFRESH_LOCK_TTL_MS,
+ });
}
- async releaseRefreshLock(): Promise<void> {
- await this.cacheStorage.releaseLock(REFRESH_LOCK_KEY);
+ async releaseRefreshLock(refreshLockOwnerToken: string): Promise<void> {
+ await this.cacheStorage.releaseLock({
+ key: REFRESH_LOCK_KEY,
+ ownerToken: refreshLockOwnerToken,
+ });
}
async invalidate(): Promise<void> {
@@ -95,7 +101,7 @@ export class UpgradeStatusCacheService {
this.cacheStorage.del(FAILED_IDS_KEY),
this.cacheStorage.del(UP_TO_DATE_COUNT_KEY),
this.cacheStorage.del(COMPUTED_AT_KEY),
- this.cacheStorage.releaseLock(REFRESH_LOCK_KEY),
+ this.cacheStorage.del(REFRESH_LOCK_KEY),
]);
}
}
diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status.service.ts b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status.service.ts
index f3a2f469e83..c3f89c40d33 100644
--- a/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/upgrade/services/upgrade-status.service.ts
@@ -178,10 +178,10 @@ export class UpgradeStatusService {
}
async getInstanceAndWorkspaceCountsStatus(): Promise<InstanceAndWorkspaceCountsUpgradeStatus | null> {
- const hasRefreshLock =
+ const refreshLockOwnerToken =
await this.upgradeStatusCacheService.tryAcquireRefreshLock();
- if (hasRefreshLock) {
+ if (isDefined(refreshLockOwnerToken)) {
try {
const refreshedStatus =
await this.refreshInstanceAndAllWorkspacesStatus();
@@ -196,7 +196,9 @@ export class UpgradeStatusService {
} catch (error) {
this.logger.error('Failed to refresh upgrade status', error);
- await this.upgradeStatusCacheService.releaseRefreshLock();
+ await this.upgradeStatusCacheService.releaseRefreshLock(
+ refreshLockOwnerToken,
+ );
}
}
diff --git a/packages/twenty-server/src/modules/emailing/services/message-campaign-statistics.service.ts b/packages/twenty-server/src/modules/emailing/services/message-campaign-statistics.service.ts
index a9d9db89a93..ffa91262a5c 100644
--- a/packages/twenty-server/src/modules/emailing/services/message-campaign-statistics.service.ts
+++ b/packages/twenty-server/src/modules/emailing/services/message-campaign-statistics.service.ts
@@ -48,12 +48,12 @@ export class MessageCampaignStatisticsService {
}): Promise<void> {
const lockKey = `campaign-stats-refresh:${workspaceId}:${campaignId}`;
- const acquired = await this.cacheStorageService.acquireLock(
- lockKey,
- REFRESH_LOCK_TTL_MS,
- );
+ const lockOwnerToken = await this.cacheStorageService.acquireLock({
+ key: lockKey,
+ ttl: REFRESH_LOCK_TTL_MS,
+ });
- if (!acquired) {
+ if (!isDefined(lockOwnerToken)) {
return;
}
@@ -67,7 +67,10 @@ export class MessageCampaignStatisticsService {
},
)
.catch(async (error) => {
- await this.cacheStorageService.releaseLock(lockKey);
+ await this.cacheStorageService.releaseLock({
+ key: lockKey,
+ ownerToken: lockOwnerToken,
+ });
throw error;
});
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts
index b6e8dc7a665..374c5c7e003 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts
@@ -1,6 +1,7 @@
import { Injectable, Logger } from '@nestjs/common';
import { QUERY_MAX_RECORDS } from 'twenty-shared/constants';
+import { isDefined } from 'twenty-shared/utils';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
@@ -37,12 +38,12 @@ export class WorkflowRunEnqueueWorkspaceService {
workspaceId: string;
isCacheMode: boolean;
}) {
- const lockAcquired =
- await this.workflowThrottlingWorkspaceService.acquireWorkflowEnqueueLock(
+ const lockOwnerToken =
+ await this.workflowThrottlingWorkspaceService.acquireWorkflowEnqueueLock({
workspaceId,
- );
+ });
- if (!lockAcquired) {
+ if (!isDefined(lockOwnerToken)) {
return;
}
@@ -164,7 +165,10 @@ export class WorkflowRunEnqueueWorkspaceService {
} finally {
try {
await this.workflowThrottlingWorkspaceService.releaseWorkflowEnqueueLock(
- workspaceId,
+ {
+ workspaceId,
+ lockOwnerToken,
+ },
);
} catch (releaseError) {
this.logger.warn(
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts
index 4d6f0e46a4c..357e40b7d59 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts
@@ -105,19 +105,28 @@ export class WorkflowThrottlingWorkspaceService {
}, authContext);
}
- async acquireWorkflowEnqueueLock(
- workspaceId: string,
+ async acquireWorkflowEnqueueLock({
+ workspaceId,
ttlMs = 60_000,
- ): Promise<boolean> {
+ }: {
+ workspaceId: string;
+ ttlMs?: number;
+ }): Promise<string | null> {
const key = this.getWorkflowEnqueueRunningCacheKey(workspaceId);
- return this.cacheStorage.acquireLock(key, ttlMs);
+ return this.cacheStorage.acquireLock({ key, ttl: ttlMs });
}
- async releaseWorkflowEnqueueLock(workspaceId: string): Promise<void> {
+ async releaseWorkflowEnqueueLock({
+ workspaceId,
+ lockOwnerToken,
+ }: {
+ workspaceId: string;
+ lockOwnerToken: string;
+ }): Promise<void> {
const key = this.getWorkflowEnqueueRunningCacheKey(workspaceId);
- await this.cacheStorage.releaseLock(key);
+ await this.cacheStorage.releaseLock({ key, ownerToken: lockOwnerToken });
}
private async setWorkflowRunNotStartedCount(