The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix(server): stop redundant lambda rebuilds causing build-lock acquisition failures (#21442)
Thomas Trompette0ac4f237c03e
diff --git a/packages/twenty-server/src/engine/core-modules/cache-lock/cache-lock.service.ts b/packages/twenty-server/src/engine/core-modules/cache-lock/cache-lock.service.tsindex 8e2123481f7..e7c20f5916a 100644--- a/packages/twenty-server/src/engine/core-modules/cache-lock/cache-lock.service.ts+++ b/packages/twenty-server/src/engine/core-modules/cache-lock/cache-lock.service.ts@@ -1,5 +1,9 @@import { Injectable, Logger } from '@nestjs/common';+import {+ CacheLockException,+ CacheLockExceptionCode,+} from 'src/engine/core-modules/cache-lock/exceptions/cache-lock.exception';import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';@@ -50,6 +54,9 @@ export class CacheLockService {await this.delay(ms);}- throw new Error(`Failed to acquire lock for key: ${key}`);+ throw new CacheLockException(+ `Failed to acquire lock for key: ${key}`,+ CacheLockExceptionCode.LOCK_ACQUISITION_TIMEOUT,+ );}}diff --git a/packages/twenty-server/src/engine/core-modules/cache-lock/exceptions/cache-lock.exception.ts b/packages/twenty-server/src/engine/core-modules/cache-lock/exceptions/cache-lock.exception.tsnew file mode 100644index 00000000000..d00417b4ef3--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/cache-lock/exceptions/cache-lock.exception.ts@@ -0,0 +1,26 @@+import { type MessageDescriptor } from '@lingui/core';+import { msg } from '@lingui/core/macro';++import {+ appendCommonExceptionCode,+ CustomException,+} from 'src/utils/custom-exception';++export class CacheLockException extends CustomException<+ keyof typeof CacheLockExceptionCode+> {+ constructor(+ message: string,+ code: keyof typeof CacheLockExceptionCode,+ { userFriendlyMessage }: { userFriendlyMessage?: MessageDescriptor } = {},+ ) {+ super(message, code, {+ userFriendlyMessage:+ userFriendlyMessage ?? msg`A cache lock error occurred.`,+ });+ }+}++export const CacheLockExceptionCode = appendCommonExceptionCode({+ LOCK_ACQUISITION_TIMEOUT: 'LOCK_ACQUISITION_TIMEOUT',+} as const);diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.tsindex 2151fc370cb..67976ca2112 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts@@ -72,6 +72,7 @@ export class LambdaDriver implements LogicFunctionDriver {this.layerManager,options.cacheLockService,options.logicFunctionResourceService,+ options.workspaceCacheService,);}diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.tsindex 834d77792d5..be7a5f9d8cf 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts@@ -18,6 +18,10 @@ import { isDefined } from 'twenty-shared/utils';import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';import { type CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';+import {+ CacheLockException,+ CacheLockExceptionCode,+} from 'src/engine/core-modules/cache-lock/exceptions/cache-lock.exception';import {EXECUTOR_LAMBDA_MEMORY_MB,EXECUTOR_LAMBDA_TIMEOUT_SECONDS,@@ -40,6 +44,7 @@ import {LogicFunctionExceptionCode,} from 'src/engine/metadata-modules/logic-function/logic-function.exception';import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';+import { type WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';type ExecutorBuildContext = {flatLogicFunction: FlatLogicFunction;@@ -56,6 +61,7 @@ export class LambdaExecutorManagerService {private readonly layerManager: LambdaLayerManagerService,private readonly cacheLockService: CacheLockService,private readonly logicFunctionResourceService: LogicFunctionResourceService,+ private readonly workspaceCacheService: WorkspaceCacheService,) {}async getLambdaExecutor(@@ -100,26 +106,74 @@ export class LambdaExecutorManagerService {const buildLockTtlMs = 120_000;const buildLockRetryMs = 500;const buildLockMaxRetries = 240;+ const lockKey = `lambda-build:${context.flatLogicFunction.id}`;- await this.cacheLockService.withLock(- async () => {- // Need to check again inside the lock in case lock was not acquired immediately.- const { canSkip: canSkipAfterLock, lambdaExecutor } =- await this.checkBuildStatus(context);+ try {+ await this.cacheLockService.withLock(+ async () => {+ // Refresh the application before re-checking: another process may+ // have rebuilt the SDK layer (and cleared isSdkLayerStale) while we+ // were waiting for the lock, and our request-time snapshot cannot+ // see it. Without this, every queued waiter rebuilds again.+ const refreshedContext = await this.refreshBuildContext(context);++ const { canSkip: canSkipAfterLock, lambdaExecutor } =+ await this.checkBuildStatus(refreshedContext);++ if (canSkipAfterLock) {+ return;+ }- if (canSkipAfterLock) {- return;- }+ await this.ensureExecutor({ ...refreshedContext, lambdaExecutor });+ },+ lockKey,+ {+ ttl: buildLockTtlMs,+ ms: buildLockRetryMs,+ maxRetries: buildLockMaxRetries,+ },+ );+ } catch (error) {+ const isLockAcquisitionTimeout =+ error instanceof CacheLockException &&+ error.code === CacheLockExceptionCode.LOCK_ACQUISITION_TIMEOUT;- await this.ensureExecutor({ ...context, lambdaExecutor });- },- `lambda-build:${context.flatLogicFunction.id}`,- {- ttl: buildLockTtlMs,- ms: buildLockRetryMs,- maxRetries: buildLockMaxRetries,- },- );+ if (!isLockAcquisitionTimeout) {+ throw error;+ }++ // Lock wait budget exhausted. If concurrent builds left the executor in+ // a usable state, proceed with the invocation instead of failing it.+ const { canSkip: isExecutorUsable } = await this.checkBuildStatus(+ await this.refreshBuildContext(context),+ );++ if (!isExecutorUsable) {+ throw error;+ }++ this.logger.warn(+ `Lock acquisition timed out for ${lockKey} but executor is up to date, proceeding`,+ );+ }+ }++ private async refreshBuildContext(+ context: ExecutorBuildContext,+ ): Promise<ExecutorBuildContext> {+ const { flatApplicationMaps } =+ await this.workspaceCacheService.getOrRecompute(+ context.flatLogicFunction.workspaceId,+ ['flatApplicationMaps'],+ );++ const refreshedFlatApplication =+ flatApplicationMaps.byId[context.flatApplication.id];++ return {+ ...context,+ flatApplication: refreshedFlatApplication ?? context.flatApplication,+ };}async installPrebuiltBundle(context: ExecutorBuildContext): Promise<void> {diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/types/lambda-driver.type.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/types/lambda-driver.type.tsindex 6f23c8d5380..57c6c13a8da 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/types/lambda-driver.type.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/types/lambda-driver.type.ts@@ -3,6 +3,7 @@ import { type LambdaClientConfig } from '@aws-sdk/client-lambda';import { type CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';import { type LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service';import { type SdkClientArchiveService } from 'src/engine/core-modules/sdk-client/sdk-client-archive.service';+import { type WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';export type LambdaDriverExecutorPayload = {code?: string;@@ -37,6 +38,7 @@ export interface LambdaDriverOptions extends LambdaClientConfig {logicFunctionResourceService: LogicFunctionResourceService;sdkClientArchiveService: SdkClientArchiveService;cacheLockService: CacheLockService;+ workspaceCacheService: WorkspaceCacheService;region: string;lambdaRole: string;subhostingRole?: string;diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory.tsindex 53f7cd0a022..befc0ee0fa9 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory.ts@@ -86,6 +86,7 @@ export class LogicFunctionDriverFactory extends DriverFactoryBase<LogicFunctionDreturn new LambdaDriver({logicFunctionResourceService: this.logicFunctionResourceService,cacheLockService: this.cacheLockService,+ workspaceCacheService: this.workspaceCacheService,credentials: accessKeyId? { accessKeyId, secretAccessKey }: fromNodeProviderChain({ clientConfig: { region } }),