The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
[3/3] Cache encrypted user application variable values (#26941)
nitin6467f2073fc5
diff --git a/packages/twenty-server/src/database/commands/secret-encryption-rotation/constants/secret-encryption-rotation-site-entries.constant.ts b/packages/twenty-server/src/database/commands/secret-encryption-rotation/constants/secret-encryption-rotation-site-entries.constant.tsindex 22beb36638a..aacb8e09cae 100644--- a/packages/twenty-server/src/database/commands/secret-encryption-rotation/constants/secret-encryption-rotation-site-entries.constant.ts+++ b/packages/twenty-server/src/database/commands/secret-encryption-rotation/constants/secret-encryption-rotation-site-entries.constant.ts@@ -10,6 +10,7 @@ import { SigningKeyEntity } from 'src/engine/core-modules/jwt/entities/signing-kimport { type ExtractEncryptedColumns } from 'src/engine/core-modules/secret-encryption/branded-strings/extract-encrypted-columns.type';import { TwoFactorAuthenticationMethodEntity } from 'src/engine/core-modules/two-factor-authentication/entities/two-factor-authentication-method.entity';import { ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity';+import { type WorkspaceCacheKeyName } from 'src/engine/workspace-cache/types/workspace-cache-key.type';type DedicatedRotationHandlerClass = Type<SecretEncryptionRotationHandler>;@@ -18,6 +19,7 @@ type ColumnRotationSiteMetadata<E extends Type<unknown>> = {customHandler: DedicatedRotationHandlerClass | undefined;isWorkspaceScoped: boolean;extraWhere: Readonly<Partial<InstanceType<E>>> | undefined;+ workspaceCacheKeyNames: readonly WorkspaceCacheKeyName[];};type SecretEncryptionRotationRegistryShape<R> = {@@ -48,6 +50,7 @@ export const SECRET_ENCRYPTION_ROTATION_SITE_ENTRIES = defineRotationRegistry({customHandler: undefined,isWorkspaceScoped: false,extraWhere: undefined,+ workspaceCacheKeyNames: [],},},},@@ -59,6 +62,7 @@ export const SECRET_ENCRYPTION_ROTATION_SITE_ENTRIES = defineRotationRegistry({customHandler: undefined,isWorkspaceScoped: true,extraWhere: undefined,+ workspaceCacheKeyNames: ['flatApplicationVariableMaps'],},},},@@ -70,18 +74,21 @@ export const SECRET_ENCRYPTION_ROTATION_SITE_ENTRIES = defineRotationRegistry({customHandler: undefined,isWorkspaceScoped: true,extraWhere: undefined,+ workspaceCacheKeyNames: [],},refreshToken: {siteName: 'connected-account-refresh-token',customHandler: undefined,isWorkspaceScoped: true,extraWhere: undefined,+ workspaceCacheKeyNames: [],},connectionParameters: {siteName: 'connected-account-connection-parameters',customHandler: ConnectionParametersRotationHandler,isWorkspaceScoped: false,extraWhere: undefined,+ workspaceCacheKeyNames: [],},},},@@ -93,6 +100,7 @@ export const SECRET_ENCRYPTION_ROTATION_SITE_ENTRIES = defineRotationRegistry({customHandler: undefined,isWorkspaceScoped: false,extraWhere: undefined,+ workspaceCacheKeyNames: [],},},},@@ -104,6 +112,7 @@ export const SECRET_ENCRYPTION_ROTATION_SITE_ENTRIES = defineRotationRegistry({customHandler: undefined,isWorkspaceScoped: true,extraWhere: undefined,+ workspaceCacheKeyNames: [],},},},@@ -115,6 +124,7 @@ export const SECRET_ENCRYPTION_ROTATION_SITE_ENTRIES = defineRotationRegistry({customHandler: undefined,isWorkspaceScoped: true,extraWhere: undefined,+ workspaceCacheKeyNames: ['userApplicationVariableValueMaps'],},},},diff --git a/packages/twenty-server/src/database/commands/secret-encryption-rotation/secret-encryption-rotation.module.ts b/packages/twenty-server/src/database/commands/secret-encryption-rotation/secret-encryption-rotation.module.tsindex b45b8c78ab0..9a693cf8b1a 100644--- a/packages/twenty-server/src/database/commands/secret-encryption-rotation/secret-encryption-rotation.module.ts+++ b/packages/twenty-server/src/database/commands/secret-encryption-rotation/secret-encryption-rotation.module.ts@@ -12,6 +12,7 @@ import { SecretEncryptionRotationRunnerService } from 'src/database/commands/secimport { KeyValuePairEntity } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity';import { SecretEncryptionModule } from 'src/engine/core-modules/secret-encryption/secret-encryption.module';import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';+import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';const ROTATION_ENTITIES = [...Object.values(SECRET_ENCRYPTION_ROTATION_SITE_ENTRIES).map(@@ -35,6 +36,7 @@ const DEDICATED_ROTATION_HANDLERS = [SecretEncryptionModule,TwentyConfigModule,TypeOrmModule.forFeature(ROTATION_ENTITIES),+ WorkspaceCacheModule,],providers: [...DEDICATED_ROTATION_HANDLERS,diff --git a/packages/twenty-server/src/database/commands/secret-encryption-rotation/services/secret-encryption-rotation-runner.service.ts b/packages/twenty-server/src/database/commands/secret-encryption-rotation/services/secret-encryption-rotation-runner.service.tsindex 700dfdde369..3213d987534 100644--- a/packages/twenty-server/src/database/commands/secret-encryption-rotation/services/secret-encryption-rotation-runner.service.ts+++ b/packages/twenty-server/src/database/commands/secret-encryption-rotation/services/secret-encryption-rotation-runner.service.ts@@ -19,6 +19,7 @@ import { SecretEncryptionService } from 'src/engine/core-modules/secret-encryptiimport { computeEncryptionKeyId } from 'src/engine/core-modules/secret-encryption/utils/compute-encryption-key-id.util';import { resolveEncryptionKeysOrThrow } from 'src/engine/core-modules/secret-encryption/utils/resolve-encryption-keys-or-throw.util';import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver';+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';import { isDefined, typedObjectEntries } from 'twenty-shared/utils';export type RotationRunOptions = {@@ -51,6 +52,7 @@ export class SecretEncryptionRotationRunnerService implements OnModuleInit {@InjectDataSource()private readonly coreDataSource: DataSource,private readonly moduleRef: ModuleRef,+ private readonly workspaceCacheService: WorkspaceCacheService,) {}onModuleInit(): void {@@ -161,6 +163,12 @@ export class SecretEncryptionRotationRunnerService implements OnModuleInit {);}+ if (!options.dryRun) {+ await this.flushWorkspaceCaches(+ handlersToRun.map(([siteName]) => siteName),+ );+ }+const totalDurationMs = Math.round(performance.now() - startedAt);this.logSummary({@@ -201,6 +209,38 @@ export class SecretEncryptionRotationRunnerService implements OnModuleInit {return [[siteName, handler]];}+ private async flushWorkspaceCaches(+ siteNames: SecretEncryptionRotationSiteName[],+ ): Promise<void> {+ const workspaceCacheKeyNamesToFlush = Object.values(+ SECRET_ENCRYPTION_ROTATION_SITE_ENTRIES,+ )+ .flatMap(({ columnSiteNames }) => Object.values(columnSiteNames))+ .filter(({ siteName }) => siteNames.includes(siteName))+ .flatMap(({ workspaceCacheKeyNames }) => workspaceCacheKeyNames);++ if (workspaceCacheKeyNamesToFlush.length === 0) {+ return;+ }++ const workspaces = await this.coreDataSource.query<{ id: string }[]>(+ `SELECT "id" FROM "core"."workspace"`,+ );++ for (const { id: workspaceId } of workspaces) {+ await this.workspaceCacheService.flush(+ workspaceId,+ workspaceCacheKeyNamesToFlush,+ );+ }++ this.logger.log(+ `[secret-encryption:rotate] flushed workspace cache keys ${workspaceCacheKeyNamesToFlush.join(+ ', ',+ )} for ${workspaces.length} workspace(s)`,+ );+ }+private logSummary(summary: RotationRunSummary): void {const totalRotated = summary.results.reduce((sum, result) => sum + result.rotated,diff --git a/packages/twenty-server/src/engine/core-modules/application/application-variable/__tests__/user-application-variable-value.service.spec.ts b/packages/twenty-server/src/engine/core-modules/application/application-variable/__tests__/user-application-variable-value.service.spec.tsindex 667f0213960..435a38d7bfa 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-variable/__tests__/user-application-variable-value.service.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-variable/__tests__/user-application-variable-value.service.spec.ts@@ -63,18 +63,7 @@ describe('UserApplicationVariableValueService', () => {provide: getWorkspaceScopedRepositoryToken(UserApplicationVariableValueEntity,),- useValue: {- find: jest.fn().mockResolvedValue([- {- applicationVariableId: 'record-my-meetings-id',- value: 'enc:on',- },- {- applicationVariableId: 'personal-api-key-id',- value: 'enc:key',- },- ]),- },+ useValue: {},},{provide: getWorkspaceScopedRepositoryToken(UserWorkspaceEntity),@@ -95,7 +84,20 @@ describe('UserApplicationVariableValueService', () => {}) => (isSecret ? '********' : `plaintext of ${value}`),},},- { provide: WorkspaceCacheService, useValue: {} },+ {+ provide: WorkspaceCacheService,+ useValue: {+ getOrRecompute: jest.fn().mockResolvedValue({+ userApplicationVariableValueMaps: {+ byApplicationVariableId: {+ 'record-my-meetings-id': { [USER_WORKSPACE_ID]: 'enc:on' },+ 'personal-api-key-id': { [USER_WORKSPACE_ID]: 'enc:key' },+ 'language-id': { 'other-user-workspace-id': 'enc:fr' },+ },+ },+ }),+ },+ },{ provide: SecretEncryptionService, useValue: {} },],}).compile();diff --git a/packages/twenty-server/src/engine/core-modules/application/application-variable/application-variable.module.ts b/packages/twenty-server/src/engine/core-modules/application/application-variable/application-variable.module.tsindex 475bcfefcf5..d7b1c7e15d6 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-variable/application-variable.module.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-variable/application-variable.module.ts@@ -9,6 +9,7 @@ import { UserApplicationVariableValueService } from 'src/engine/core-modules/appimport { ApplicationVariableEntity } from 'src/engine/core-modules/application/application-variable/application-variable.entity';import { ApplicationVariableEntityResolver } from 'src/engine/core-modules/application/application-variable/application-variable.resolver';import { ApplicationVariableEntityService } from 'src/engine/core-modules/application/application-variable/application-variable.service';+import { WorkspaceUserApplicationVariableValueMapCacheService } from 'src/engine/core-modules/application/application-variable/workspace-user-application-variable-value-map-cache.service';import { SecretEncryptionModule } from 'src/engine/core-modules/secret-encryption/secret-encryption.module';import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity';import { FlatApplicationVariableModule } from 'src/engine/metadata-modules/flat-application-variable/flat-application-variable.module';@@ -40,6 +41,7 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cacheApplicationVariableEntityResolver,UserApplicationVariableValueService,UserApplicationVariableValueResolver,+ WorkspaceUserApplicationVariableValueMapCacheService,],exports: [ApplicationVariableEntityService,diff --git a/packages/twenty-server/src/engine/core-modules/application/application-variable/types/user-application-variable-value-maps.type.ts b/packages/twenty-server/src/engine/core-modules/application/application-variable/types/user-application-variable-value-maps.type.tsnew file mode 100644index 00000000000..5a67abc5a8e--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/application/application-variable/types/user-application-variable-value-maps.type.ts@@ -0,0 +1,7 @@+import { type EncryptedString } from 'src/engine/core-modules/secret-encryption/branded-strings/encrypted-string.type';++export type UserApplicationVariableValueMaps = {+ byApplicationVariableId: Partial<+ Record<string, Partial<Record<string, EncryptedString>>>+ >;+};diff --git a/packages/twenty-server/src/engine/core-modules/application/application-variable/user-application-variable-value.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-variable/user-application-variable-value.service.tsindex 7f00acb0788..fd3617094dd 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-variable/user-application-variable-value.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-variable/user-application-variable-value.service.ts@@ -1,8 +1,6 @@import { Injectable } from '@nestjs/common';-import groupBy from 'lodash.groupby';import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';-import { In } from 'typeorm';import {ApplicationException,@@ -107,35 +105,21 @@ export class UserApplicationVariableValueService {return [];}- const [userWorkspaces, userValues, { flatWorkspaceMemberMaps }] =- await Promise.all([- this.userWorkspaceRepository.find(workspaceId, {- select: { id: true, userId: true },- where: isDefined(requestUserWorkspaceId)- ? { id: requestUserWorkspaceId }- : undefined,- }),- this.userApplicationVariableValueRepository.find(workspaceId, {- select: {- userWorkspaceId: true,- applicationVariableId: true,- value: true,- },- where: {- applicationVariableId: In(- flatUserApplicationVariables.map(({ id }) => id),- ),- ...(isDefined(requestUserWorkspaceId)- ? { userWorkspaceId: requestUserWorkspaceId }- : {}),- },- }),- this.workspaceCacheService.getOrRecompute(workspaceId, [- 'flatWorkspaceMemberMaps',- ]),- ]);-- const userValuesByUserWorkspaceId = groupBy(userValues, 'userWorkspaceId');+ const [+ userWorkspaces,+ { userApplicationVariableValueMaps, flatWorkspaceMemberMaps },+ ] = await Promise.all([+ this.userWorkspaceRepository.find(workspaceId, {+ select: { id: true, userId: true },+ where: isDefined(requestUserWorkspaceId)+ ? { id: requestUserWorkspaceId }+ : undefined,+ }),+ this.workspaceCacheService.getOrRecompute(workspaceId, [+ 'userApplicationVariableValueMaps',+ 'flatWorkspaceMemberMaps',+ ]),+ ]);return userWorkspaces.flatMap(({ id: userWorkspaceId, userId }) => {const workspaceMemberId = resolveWorkspaceMemberIdForUser({@@ -153,7 +137,8 @@ export class UserApplicationVariableValueService {workspaceMemberId,variables: toUserApplicationVariableValues({flatUserApplicationVariables,- userValues: userValuesByUserWorkspaceId[userWorkspaceId] ?? [],+ userApplicationVariableValueMaps,+ userWorkspaceId,shouldMaskSecret: isDefined(requestUserWorkspaceId),getDisplayValue: ({ value, isSecret }) =>this.applicationVariableService.getDisplayValue({@@ -203,6 +188,10 @@ export class UserApplicationVariableValueService {},['applicationVariableId', 'userWorkspaceId'],);++ await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [+ 'userApplicationVariableValueMaps',+ ]);}private async toEnvVariables({@@ -221,16 +210,16 @@ export class UserApplicationVariableValueService {return {};}- const userValues = await this.findUserValues({- workspaceId,- userWorkspaceId,- flatUserApplicationVariables,- });+ const { userApplicationVariableValueMaps } =+ await this.workspaceCacheService.getOrRecompute(workspaceId, [+ 'userApplicationVariableValueMaps',+ ]);return Object.fromEntries(toUserApplicationVariableValues({flatUserApplicationVariables,- userValues,+ userApplicationVariableValueMaps,+ userWorkspaceId,shouldMaskSecret: false,getDisplayValue: ({ value, isSecret }) =>this.applicationVariableService.getDisplayValue({@@ -242,31 +231,6 @@ export class UserApplicationVariableValueService {);}- private async findUserValues({- workspaceId,- userWorkspaceId,- flatUserApplicationVariables,- }: {- workspaceId: string;- userWorkspaceId: string;- flatUserApplicationVariables: FlatApplicationVariable[];- }): Promise<- Pick<- UserApplicationVariableValueEntity,- 'applicationVariableId' | 'value'- >[]- > {- return this.userApplicationVariableValueRepository.find(workspaceId, {- select: { applicationVariableId: true, value: true },- where: {- userWorkspaceId,- applicationVariableId: In(- flatUserApplicationVariables.map(({ id }) => id),- ),- },- });- }-private async findApplicationIdOrThrow({workspaceId,applicationUniversalIdentifier,diff --git a/packages/twenty-server/src/engine/core-modules/application/application-variable/utils/__tests__/to-user-application-variable-values.util.spec.ts b/packages/twenty-server/src/engine/core-modules/application/application-variable/utils/__tests__/to-user-application-variable-values.util.spec.tsindex 06d1b89d4c9..482d0a7d5fe 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-variable/utils/__tests__/to-user-application-variable-values.util.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-variable/utils/__tests__/to-user-application-variable-values.util.spec.ts@@ -47,16 +47,19 @@ const FLAT_USER_APPLICATION_VARIABLES = [}),];-const USER_VALUES = [- {- applicationVariableId: 'record-my-meetings-id',- value: 'enc:on' as EncryptedString,- },- {- applicationVariableId: 'personal-api-key-id',- value: 'enc:key' as EncryptedString,+const USER_WORKSPACE_ID = 'user-workspace-id';++const USER_APPLICATION_VARIABLE_VALUE_MAPS = {+ byApplicationVariableId: {+ 'record-my-meetings-id': {+ [USER_WORKSPACE_ID]: 'enc:on' as EncryptedString,+ 'other-user-workspace-id': 'enc:off' as EncryptedString,+ },+ 'personal-api-key-id': {+ [USER_WORKSPACE_ID]: 'enc:key' as EncryptedString,+ },},-];+};const getDisplayValue = ({value,@@ -70,7 +73,8 @@ describe('toUserApplicationVariableValues', () => {it('should use the member value, else the default, else an empty string', () => {const values = toUserApplicationVariableValues({flatUserApplicationVariables: FLAT_USER_APPLICATION_VARIABLES,- userValues: USER_VALUES,+ userApplicationVariableValueMaps: USER_APPLICATION_VARIABLE_VALUE_MAPS,+ userWorkspaceId: USER_WORKSPACE_ID,shouldMaskSecret: false,getDisplayValue,});@@ -86,7 +90,8 @@ describe('toUserApplicationVariableValues', () => {it('should mask secrets only when asked to', () => {const values = toUserApplicationVariableValues({flatUserApplicationVariables: FLAT_USER_APPLICATION_VARIABLES,- userValues: USER_VALUES,+ userApplicationVariableValueMaps: USER_APPLICATION_VARIABLE_VALUE_MAPS,+ userWorkspaceId: USER_WORKSPACE_ID,shouldMaskSecret: true,getDisplayValue,});diff --git a/packages/twenty-server/src/engine/core-modules/application/application-variable/utils/to-user-application-variable-values.util.ts b/packages/twenty-server/src/engine/core-modules/application/application-variable/utils/to-user-application-variable-values.util.tsindex 2f3eb2ebcac..20711faf5cf 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-variable/utils/to-user-application-variable-values.util.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-variable/utils/to-user-application-variable-values.util.ts@@ -1,12 +1,14 @@import { isDefined } from 'twenty-shared/utils';import { type UserApplicationVariableValueDTO } from 'src/engine/core-modules/application/application-variable/dtos/user-application-variable-value.dto';+import { type UserApplicationVariableValueMaps } from 'src/engine/core-modules/application/application-variable/types/user-application-variable-value-maps.type';import { type UserApplicationVariableValueEntity } from 'src/engine/core-modules/application/application-variable/user-application-variable-value.entity';import { type FlatApplicationVariable } from 'src/engine/metadata-modules/flat-application-variable/types/flat-application-variable.type';export const toUserApplicationVariableValues = ({flatUserApplicationVariables,- userValues,+ userApplicationVariableValueMaps,+ userWorkspaceId,shouldMaskSecret,getDisplayValue,}: {@@ -23,28 +25,20 @@ export const toUserApplicationVariableValues = ({| 'isDeprecated'| 'defaultValue'>[];- userValues: Pick<- UserApplicationVariableValueEntity,- 'applicationVariableId' | 'value'- >[];+ userApplicationVariableValueMaps: UserApplicationVariableValueMaps;+ userWorkspaceId: string;shouldMaskSecret: boolean;getDisplayValue: (userValue: Pick<UserApplicationVariableValueEntity, 'value'> & {isSecret: boolean;},) => string;-}): UserApplicationVariableValueDTO[] => {- const userValueByApplicationVariableId = new Map(- userValues.map(({ applicationVariableId, value }) => [- applicationVariableId,- value,- ]),- );-- return flatUserApplicationVariables.map((flatApplicationVariable) => {- const userValue = userValueByApplicationVariableId.get(- flatApplicationVariable.id,- );+}): UserApplicationVariableValueDTO[] =>+ flatUserApplicationVariables.map((flatApplicationVariable) => {+ const userValue =+ userApplicationVariableValueMaps.byApplicationVariableId[+ flatApplicationVariable.id+ ]?.[userWorkspaceId];return {key: flatApplicationVariable.key,@@ -64,4 +58,3 @@ export const toUserApplicationVariableValues = ({: (flatApplicationVariable.defaultValue ?? ''),};});-};diff --git a/packages/twenty-server/src/engine/core-modules/application/application-variable/workspace-user-application-variable-value-map-cache.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-variable/workspace-user-application-variable-value-map-cache.service.tsnew file mode 100644index 00000000000..846f2177434--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/application/application-variable/workspace-user-application-variable-value-map-cache.service.ts@@ -0,0 +1,45 @@+import { Injectable } from '@nestjs/common';++import { type UserApplicationVariableValueMaps } from 'src/engine/core-modules/application/application-variable/types/user-application-variable-value-maps.type';+import { WorkspaceCache } from 'src/engine/workspace-cache/decorators/workspace-cache.decorator';+import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';+import { type WorkspaceCacheProviderContext } from 'src/engine/workspace-cache/types/workspace-cache-provider-context.type';+import { type WorkspaceCacheRowsRequirement } from 'src/engine/workspace-cache/types/workspace-cache-rows-requirement.type';++const USER_APPLICATION_VARIABLE_VALUE_ROWS_REQUIREMENT = {+ userApplicationVariableValue: [+ 'applicationVariableId',+ 'userWorkspaceId',+ 'value',+ ],+} as const satisfies WorkspaceCacheRowsRequirement;++@Injectable()+@WorkspaceCache('userApplicationVariableValueMaps', { packingPonderation: 1 })+export class WorkspaceUserApplicationVariableValueMapCacheService extends WorkspaceCacheProvider<UserApplicationVariableValueMaps> {+ override readonly rowsRequirement =+ USER_APPLICATION_VARIABLE_VALUE_ROWS_REQUIREMENT;++ computeForCache({+ rows,+ }: WorkspaceCacheProviderContext<+ typeof USER_APPLICATION_VARIABLE_VALUE_ROWS_REQUIREMENT+ >): UserApplicationVariableValueMaps {+ const byApplicationVariableId: UserApplicationVariableValueMaps['byApplicationVariableId'] =+ {};++ for (const {+ applicationVariableId,+ userWorkspaceId,+ value,+ } of rows.userApplicationVariableValue) {+ const valuesByUserWorkspaceId = (byApplicationVariableId[+ applicationVariableId+ ] ??= {});++ valuesByUserWorkspaceId[userWorkspaceId] = value;+ }++ return { byApplicationVariableId };+ }+}diff --git a/packages/twenty-server/src/engine/core-modules/application/application.service.ts b/packages/twenty-server/src/engine/core-modules/application/application.service.tsindex cf92ba3aa92..5d5f5d15818 100644--- a/packages/twenty-server/src/engine/core-modules/application/application.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application.service.ts@@ -748,10 +748,10 @@ export class ApplicationService {'flatApplicationMaps',]);- await this.workspaceCacheService.invalidateAndRecompute(- workspaceId,- ALL_FLAT_ENTITY_MAPS_PROPERTIES,- );+ await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [+ ...ALL_FLAT_ENTITY_MAPS_PROPERTIES,+ 'userApplicationVariableValueMaps',+ ]);await this.broadcastApplicationEvent({type: 'deleted',diff --git a/packages/twenty-server/src/engine/core-modules/user-workspace/user-workspace.service.ts b/packages/twenty-server/src/engine/core-modules/user-workspace/user-workspace.service.tsindex 210f0c21252..761624a8fb3 100644--- a/packages/twenty-server/src/engine/core-modules/user-workspace/user-workspace.service.ts+++ b/packages/twenty-server/src/engine/core-modules/user-workspace/user-workspace.service.ts@@ -379,6 +379,7 @@ export class UserWorkspaceService {await this.userWorkspaceRepository.delete({ id: userWorkspaceId });await this.workspaceCacheService.invalidateAndRecompute(workspaceId, ['flatRoleTargetMaps',+ 'userApplicationVariableValueMaps',]);await this.workflowRunRecordShareService.syncRunsOfCoreWorkflows({diff --git a/packages/twenty-server/src/engine/workspace-cache/constants/all-workspace-cache-entity-by-name.constant.ts b/packages/twenty-server/src/engine/workspace-cache/constants/all-workspace-cache-entity-by-name.constant.tsindex 521a29e6048..b634c501cbd 100644--- a/packages/twenty-server/src/engine/workspace-cache/constants/all-workspace-cache-entity-by-name.constant.ts+++ b/packages/twenty-server/src/engine/workspace-cache/constants/all-workspace-cache-entity-by-name.constant.ts@@ -1,5 +1,6 @@import { ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';+import { UserApplicationVariableValueEntity } from 'src/engine/core-modules/application/application-variable/user-application-variable-value.entity';import { BillingEntitlementEntity } from 'src/engine/core-modules/billing/entities/billing-entitlement.entity';import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';import { UsageLimitEntity } from 'src/engine/core-modules/usage-limit/usage-limit.entity';@@ -10,6 +11,7 @@ import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-metaexport const ALL_WORKSPACE_CACHE_ENTITY_BY_NAME = {...ALL_METADATA_ENTITY_BY_METADATA_NAME,application: ApplicationEntity,+ userApplicationVariableValue: UserApplicationVariableValueEntity,indexFieldMetadata: IndexFieldMetadataEntity,apiKey: ApiKeyEntity,featureFlag: FeatureFlagEntity,diff --git a/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts b/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.tsindex 7aa4ec256f5..66322930de1 100644--- a/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts+++ b/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts@@ -8,6 +8,7 @@ import { type OrmFlatFieldMetadata } from 'src/engine/metadata-modules/flat-fielimport { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';import { type ResolverNameMapEntry } from 'src/engine/api/graphql/direct-execution/utils/build-resolver-name-map.util';import { type FlatApiKey } from 'src/engine/core-modules/api-key/types/flat-api-key.type';+import { type UserApplicationVariableValueMaps } from 'src/engine/core-modules/application/application-variable/types/user-application-variable-value-maps.type';import { type FlatApplicationCacheMaps } from 'src/engine/core-modules/application/types/flat-application-cache-maps.type';import { type CurrentBillingSubscription } from 'src/engine/core-modules/billing/types/flat-billing-subscription.type';import { type BillingEntitlements } from 'src/engine/core-modules/billing/types/billing-entitlements.type';@@ -30,6 +31,7 @@ export type AdditionalCacheDataMaps = {flatRowLevelPermissionPredicateMaps: FlatRowLevelPermissionPredicateMaps;flatRowLevelPermissionPredicateGroupMaps: FlatRowLevelPermissionPredicateGroupMaps;flatWorkspaceMemberMaps: FlatWorkspaceMemberMaps;+ userApplicationVariableValueMaps: UserApplicationVariableValueMaps;currentBillingSubscription: CurrentBillingSubscription;billingEntitlements: BillingEntitlements;workflowAutomatedTriggerMaps: WorkflowAutomatedTriggerMaps;diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.tsindex ad58f977c03..b33352cba8a 100644--- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts@@ -85,6 +85,10 @@ export class WorkspaceMigrationRunnerService {'flatWorkflowVersionMaps',);+ const hasApplicationVariablesChanged = allFlatEntityMapsKeys.includes(+ 'flatApplicationVariableMaps',+ );+const cacheKeyNamesToInvalidate: WorkspaceCacheKeyName[] = [...new Set([...withDerivedFieldMetadataMaps(allFlatEntityMapsKeys),@@ -92,6 +96,9 @@ export class WorkspaceMigrationRunnerService {...(hasWorkflowVersionsChanged? ['workflowAutomatedTriggerMaps' as const]: []),+ ...(hasApplicationVariablesChanged+ ? ['userApplicationVariableValueMaps' as const]+ : []),]),];diff --git a/packages/twenty-server/test/integration/secret-encryption/secret-encryption-rotation.integration-spec.ts b/packages/twenty-server/test/integration/secret-encryption/secret-encryption-rotation.integration-spec.tsindex 8d4ac71901c..e7e81f9d15b 100644--- a/packages/twenty-server/test/integration/secret-encryption/secret-encryption-rotation.integration-spec.ts+++ b/packages/twenty-server/test/integration/secret-encryption/secret-encryption-rotation.integration-spec.ts@@ -10,10 +10,21 @@ import {} from 'test/integration/secret-encryption/utils/find-one-application.util';import { runSecretEncryptionRotationCommand } from 'test/integration/secret-encryption/utils/run-secret-encryption-rotation-command.util';import { updateOneApplicationVariable } from 'test/integration/secret-encryption/utils/update-one-application-variable.util';+import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';+import { type ApplicationVariableEntityService } from 'src/engine/core-modules/application/application-variable/application-variable.service';+import { type UserApplicationVariableValueService } from 'src/engine/core-modules/application/application-variable/user-application-variable-value.service';import { SECRET_APPLICATION_VARIABLE_MASK } from 'src/engine/core-modules/application/application-variable/constants/secret-application-variable-mask.constant';+import { computeEncryptionKeyId } from 'src/engine/core-modules/secret-encryption/utils/compute-encryption-key-id.util';+import { encryptAesGcmV2 } from 'src/engine/core-modules/secret-encryption/utils/encrypt-aes-gcm-v2.util';+import { formatSecretEncryptionEnvelopeV2 } from 'src/engine/core-modules/secret-encryption/utils/format-secret-encryption-envelope-v2.util';+import { type WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';+import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';+import { USER_WORKSPACE_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/core/utils/seed-user-workspaces.util';const ROTATION_VARIABLE_KEY = 'TEST_ROTATION_SECRET';+const USER_ROTATION_VARIABLE_KEY = 'TEST_USER_ROTATION_SECRET';+const PREVIOUS_ENCRYPTION_KEY = 'previous-encryption-key-retired-by-rotation';const buildExpectedMask = (plaintext: string): string => {const visibleCharsCount = Math.min(5, Math.floor(plaintext.length / 10));@@ -21,6 +32,16 @@ const buildExpectedMask = (plaintext: string): string => {return `${plaintext.slice(0, visibleCharsCount)}${SECRET_APPLICATION_VARIABLE_MASK}`;};+const encryptWithPreviousEncryptionKey = (plaintext: string): string =>+ formatSecretEncryptionEnvelopeV2({+ keyId: computeEncryptionKeyId({ rawKey: PREVIOUS_ENCRYPTION_KEY }),+ payloadBase64: encryptAesGcmV2({+ plaintext,+ rawKey: PREVIOUS_ENCRYPTION_KEY,+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ }),+ });+describe('secret-encryption:rotate command (integration)', () => {let applicationUniversalIdentifier: string;let applicationId: string;@@ -54,6 +75,11 @@ describe('secret-encryption:rotate command (integration)', () => {universalIdentifier: crypto.randomUUID(),isSecret: true,},+ [USER_ROTATION_VARIABLE_KEY]: {+ universalIdentifier: crypto.randomUUID(),+ scope: 'USER',+ isSecret: true,+ },},packageJsonChecksum: null,yarnLockChecksum: null,@@ -117,4 +143,85 @@ describe('secret-encryption:rotate command (integration)', () => {expect(variable?.value).toBe(buildExpectedMask(plaintext));}, 90000);++ it('refreshes the workspace cache so cached applicationVariables decrypt without the fallback key', async () => {+ await global.testDataSource.query(+ `UPDATE core."applicationVariable"+ SET "value" = $1+ WHERE "applicationId" = $2 AND "key" = $3`,+ [+ encryptWithPreviousEncryptionKey(plaintext),+ applicationId,+ ROTATION_VARIABLE_KEY,+ ],+ );+ await getAppProviderByClassName<WorkspaceCacheService>(+ 'WorkspaceCacheService',+ ).invalidateAndRecompute(SEED_APPLE_WORKSPACE_ID, [+ 'flatApplicationVariableMaps',+ ]);++ await runSecretEncryptionRotationCommand(+ {},+ { FALLBACK_ENCRYPTION_KEY: PREVIOUS_ENCRYPTION_KEY },+ );++ await expect(+ getAppProviderByClassName<ApplicationVariableEntityService>(+ 'ApplicationVariableEntityService',+ ).getServerEnvVariables({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ applicationId,+ }),+ ).resolves.toEqual({ [ROTATION_VARIABLE_KEY]: plaintext });+ }, 90000);++ it('refreshes cached member values so they decrypt without the fallback key', async () => {+ await global.testDataSource.query(+ `INSERT INTO core."userApplicationVariableValue"+ ("workspaceId", "applicationVariableId", "userWorkspaceId", "value")+ SELECT "workspaceId", "id", $1, $2+ FROM core."applicationVariable"+ WHERE "applicationId" = $3 AND "key" = $4`,+ [+ USER_WORKSPACE_DATA_SEED_IDS.JANE,+ encryptWithPreviousEncryptionKey(plaintext),+ applicationId,+ USER_ROTATION_VARIABLE_KEY,+ ],+ );+ await getAppProviderByClassName<WorkspaceCacheService>(+ 'WorkspaceCacheService',+ ).invalidateAndRecompute(SEED_APPLE_WORKSPACE_ID, [+ 'userApplicationVariableValueMaps',+ ]);++ await runSecretEncryptionRotationCommand(+ { site: 'user-application-variable-value' },+ { FALLBACK_ENCRYPTION_KEY: PREVIOUS_ENCRYPTION_KEY },+ );++ const memberValues =+ await getAppProviderByClassName<UserApplicationVariableValueService>(+ 'UserApplicationVariableValueService',+ ).findUserApplicationVariableValues({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ applicationId,+ requestUserWorkspaceId: undefined,+ });++ expect(memberValues).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ userWorkspaceId: USER_WORKSPACE_DATA_SEED_IDS.JANE,+ variables: [+ expect.objectContaining({+ key: USER_ROTATION_VARIABLE_KEY,+ value: plaintext,+ }),+ ],+ }),+ ]),+ );+ }, 90000);});diff --git a/packages/twenty-server/test/integration/secret-encryption/utils/run-secret-encryption-rotation-command.util.ts b/packages/twenty-server/test/integration/secret-encryption/utils/run-secret-encryption-rotation-command.util.tsindex 11b87807a84..d7f6cd68ac1 100644--- a/packages/twenty-server/test/integration/secret-encryption/utils/run-secret-encryption-rotation-command.util.ts+++ b/packages/twenty-server/test/integration/secret-encryption/utils/run-secret-encryption-rotation-command.util.ts@@ -33,11 +33,12 @@ const buildArgs = ({ site, batchSize, dryRun }: RotateArguments): string[] => {export const runSecretEncryptionRotationCommand = async (args: RotateArguments = {},+ environmentOverrides: NodeJS.ProcessEnv = {},): Promise<void> => {await new Promise<void>((resolve, reject) => {const child = spawn('node', [COMMAND_JS_PATH, ...buildArgs(args)], {cwd: TWENTY_SERVER_ROOT,- env: { ...process.env, NODE_ENV: 'test' },+ env: { ...process.env, ...environmentOverrides, NODE_ENV: 'test' },stdio: ['ignore', 'pipe', 'pipe'],});diff --git a/packages/twenty-server/test/integration/workspace-cache/user-application-variable-value-cache.integration-spec.ts b/packages/twenty-server/test/integration/workspace-cache/user-application-variable-value-cache.integration-spec.tsnew file mode 100644index 00000000000..1a095a2ff1c--- /dev/null+++ b/packages/twenty-server/test/integration/workspace-cache/user-application-variable-value-cache.integration-spec.ts@@ -0,0 +1,240 @@+import { v4 as uuidv4 } from 'uuid';++import { myUserApplicationVariables } from 'test/integration/metadata/suites/application/utils/my-user-application-variables.util';+import { buildBaseManifest } from 'test/integration/metadata/suites/application/utils/build-base-manifest.util';+import { cleanupApplicationAndAppRegistration } from 'test/integration/metadata/suites/application/utils/cleanup-application-and-app-registration.util';+import {+ type ApplicationWithVariable,+ setupApplicationWithVariable,+} from 'test/integration/metadata/suites/application/utils/setup-application-with-variable.util';+import { syncApplication } from 'test/integration/metadata/suites/application/utils/sync-application.util';+import { uninstallApplication } from 'test/integration/metadata/suites/application/utils/uninstall-application.util';+import { updateMyUserApplicationVariable } from 'test/integration/metadata/suites/application/utils/update-my-user-application-variable.util';+import { generateAppleAdminApplicationTokenPair } from 'test/integration/utils/generate-apple-admin-application-token-pair.util';+import { generateApplicationTokenPair } from 'test/integration/utils/generate-application-token-pair.util';+import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';++import { type UserApplicationVariableValueService } from 'src/engine/core-modules/application/application-variable/user-application-variable-value.service';+import { SECRET_APPLICATION_VARIABLE_MASK } from 'src/engine/core-modules/application/application-variable/constants/secret-application-variable-mask.constant';+import { plaintextStringSchema } from 'src/engine/core-modules/secret-encryption/branded-strings/plaintext-string.type';+import { SECRET_ENCRYPTION_ENVELOPE_V2_PREFIX } from 'src/engine/core-modules/secret-encryption/constants/secret-encryption.constant';+import { type UserWorkspaceService } from 'src/engine/core-modules/user-workspace/user-workspace.service';+import { type WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';+import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';+import { USER_WORKSPACE_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/core/utils/seed-user-workspaces.util';++describe('User application variable value cache', () => {+ let application: ApplicationWithVariable;+ let applicationVariableId: string;+ let applicationToken: string;+ let janeApplicationToken: string;+ let workspaceCacheService: WorkspaceCacheService;++ const readCachedValues = async () => {+ const { userApplicationVariableValueMaps } =+ await workspaceCacheService.getOrRecompute(SEED_APPLE_WORKSPACE_ID, [+ 'userApplicationVariableValueMaps',+ ]);++ return userApplicationVariableValueMaps.byApplicationVariableId[+ applicationVariableId+ ];+ };++ const updateValue = (value: string) =>+ updateMyUserApplicationVariable({+ input: {+ applicationUniversalIdentifier: application.universalIdentifier,+ key: application.variableKey,+ value,+ },+ token: APPLE_JANE_ADMIN_ACCESS_TOKEN,+ expectToFail: false,+ });++ const expectVisibleValues = async (value: string) => {+ const [{ data: memberData }, { data: applicationData }] = await Promise.all(+ [+ myUserApplicationVariables({+ input: {},+ token: janeApplicationToken,+ expectToFail: false,+ }),+ myUserApplicationVariables({+ input: {},+ token: applicationToken,+ expectToFail: false,+ }),+ ],+ );++ expect(memberData.myUserApplicationVariables).toEqual([+ expect.objectContaining({+ userWorkspaceId: USER_WORKSPACE_DATA_SEED_IDS.JANE,+ variables: [+ {+ key: application.variableKey,+ value: value === '' ? '' : SECRET_APPLICATION_VARIABLE_MASK,+ },+ ],+ }),+ ]);+ expect(+ applicationData.myUserApplicationVariables.find(+ ({ userWorkspaceId }) =>+ userWorkspaceId === USER_WORKSPACE_DATA_SEED_IDS.JANE,+ )?.variables,+ ).toEqual([{ key: application.variableKey, value }]);+ };++ beforeEach(async () => {+ workspaceCacheService = getAppProviderByClassName<WorkspaceCacheService>(+ 'WorkspaceCacheService',+ );+ application = await setupApplicationWithVariable({+ name: 'Cached User Variable Application',+ variableKey: 'API_KEY',+ variableScope: 'USER',+ isSecret: true,+ });++ const [applicationVariable]: { id: string }[] =+ await global.testDataSource.query(+ 'SELECT id FROM core."applicationVariable" WHERE "applicationId" = $1 AND key = $2',+ [application.id, application.variableKey],+ );++ applicationVariableId = applicationVariable.id;++ const [tokenPair, janeTokenPair] = await Promise.all([+ generateApplicationTokenPair({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ applicationId: application.id,+ }),+ generateAppleAdminApplicationTokenPair({+ applicationId: application.id,+ }),+ ]);++ applicationToken = tokenPair.applicationAccessToken.token;+ janeApplicationToken = janeTokenPair.applicationAccessToken.token;+ }, 120000);++ afterEach(async () => {+ await cleanupApplicationAndAppRegistration({+ applicationUniversalIdentifier: application.universalIdentifier,+ });+ });++ it('refreshes encrypted cached values after writes and preserves API masking', async () => {+ expect(await readCachedValues()).toBeUndefined();+ await expectVisibleValues('');+ await updateValue('first');++ expect(+ (await readCachedValues())?.[USER_WORKSPACE_DATA_SEED_IDS.JANE],+ ).toEqual(expect.stringContaining(SECRET_ENCRYPTION_ENVELOPE_V2_PREFIX));+ await expectVisibleValues('first');++ // Bypass invalidation to prove that both APIs read the cached value.+ await global.testDataSource.query(+ 'DELETE FROM core."userApplicationVariableValue" WHERE "applicationVariableId" = $1',+ [applicationVariableId],+ );+ await expectVisibleValues('first');++ await updateValue('second');+ await expectVisibleValues('second');+ await updateValue('');+ await expectVisibleValues('');+ });++ it('removes cached values when a manifest deletes their declaration', async () => {+ await updateValue('first');+ expect(await readCachedValues()).toBeDefined();++ const [role]: { universalIdentifier: string }[] =+ await global.testDataSource.query(+ 'SELECT "universalIdentifier" FROM core.role WHERE id = $1',+ [application.defaultRoleId],+ );++ await syncApplication({+ manifest: buildBaseManifest({+ appId: application.universalIdentifier,+ roleId: role.universalIdentifier,+ }),+ inferDeletionFromMissingEntities: true,+ expectToFail: false,+ });++ expect(await readCachedValues()).toBeUndefined();++ const { data } = await myUserApplicationVariables({+ input: {},+ token: applicationToken,+ expectToFail: false,+ });++ expect(data.myUserApplicationVariables).toEqual([]);+ });++ it('removes cached values when their application is uninstalled', async () => {+ await updateValue('first');+ expect(await readCachedValues()).toBeDefined();++ await uninstallApplication({+ universalIdentifier: application.universalIdentifier,+ expectToFail: false,+ });++ expect(await readCachedValues()).toBeUndefined();+ });++ it('removes a deleted membership from the cache and keeps other members values', async () => {+ const userId = uuidv4();+ const userWorkspaceId = uuidv4();++ await global.testDataSource.query(+ 'INSERT INTO core."user" (id, email) VALUES ($1, $2)',+ [userId, `cache-test-${userId}@example.com`],+ );++ try {+ await global.testDataSource.query(+ 'INSERT INTO core."userWorkspace" (id, "userId", "workspaceId") VALUES ($1, $2, $3)',+ [userWorkspaceId, userId, SEED_APPLE_WORKSPACE_ID],+ );+ await getAppProviderByClassName<UserApplicationVariableValueService>(+ 'UserApplicationVariableValueService',+ ).updateMyUserApplicationVariable({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ applicationUniversalIdentifier: application.universalIdentifier,+ userWorkspaceId,+ key: application.variableKey,+ plainTextValue: plaintextStringSchema.parse('temporary'),+ });+ await updateValue('first');+ expect((await readCachedValues())?.[userWorkspaceId]).toBeDefined();++ await getAppProviderByClassName<UserWorkspaceService>(+ 'UserWorkspaceService',+ ).deleteUserWorkspace({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ userWorkspaceId,+ });++ expect((await readCachedValues())?.[userWorkspaceId]).toBeUndefined();+ await expectVisibleValues('first');+ } finally {+ await global.testDataSource.query(+ 'DELETE FROM core."userWorkspace" WHERE id = $1',+ [userWorkspaceId],+ );+ await global.testDataSource.query(+ 'DELETE FROM core."user" WHERE id = $1',+ [userId],+ );+ }+ });+});