The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
perf(server): compact and cold-store workspace cache payloads behind a feature flag (#23880)
Charles Bochetc6200c7a143b
diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql b/packages/twenty-client-sdk/src/metadata/generated/schema.graphqlindex 31423102e55..e598895d21c 100644--- a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.graphql@@ -1739,6 +1739,7 @@ enum FeatureFlagKey {IS_SETTINGS_DISCOVERY_HERO_ENABLEDIS_WORKFLOW_VERSION_IN_CORE_ENABLEDIS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED+ IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED}type WorkspaceUrls {diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.ts b/packages/twenty-client-sdk/src/metadata/generated/schema.tsindex dcab22580af..6edd8d4bcdb 100644--- a/packages/twenty-client-sdk/src/metadata/generated/schema.ts+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.ts@@ -1395,7 +1395,7 @@ export interface FeatureFlag {__typename: 'FeatureFlag'}-export type FeatureFlagKey = 'IS_AI_CHAT_PAGE_ENABLED' | 'IS_APP_CLAIMING_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_CALENDAR_WEEK_VIEW_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_LIST_VIEW_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' | 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED' | 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED'+export type FeatureFlagKey = 'IS_AI_CHAT_PAGE_ENABLED' | 'IS_APP_CLAIMING_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_CALENDAR_WEEK_VIEW_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_LIST_VIEW_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' | 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED' | 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED' | 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED'export interface WorkspaceUrls {customUrl?: Scalars['String']@@ -9653,7 +9653,8 @@ export const enumFeatureFlagKey = {IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED: 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' as const,IS_SETTINGS_DISCOVERY_HERO_ENABLED: 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' as const,IS_WORKFLOW_VERSION_IN_CORE_ENABLED: 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED' as const,- IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED: 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED' as const+ IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED: 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED' as const,+ IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED: 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED' as const}export const enumIdentityProviderType = {diff --git a/packages/twenty-front/src/generated-admin/graphql.ts b/packages/twenty-front/src/generated-admin/graphql.tsindex 1ce0033c05f..781160320ee 100644--- a/packages/twenty-front/src/generated-admin/graphql.ts+++ b/packages/twenty-front/src/generated-admin/graphql.ts@@ -380,7 +380,8 @@ export enum FeatureFlagKey {IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',IS_UNIQUE_INDEXES_ENABLED = 'IS_UNIQUE_INDEXES_ENABLED',IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED = 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED',- IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED'+ IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED',+ IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED = 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED'}export type FindApplicationRegistrationInstalledWorkspacesInput = {diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.tsindex f911b15cba1..d4f5ec7b18a 100644--- a/packages/twenty-front/src/generated-metadata/graphql.ts+++ b/packages/twenty-front/src/generated-metadata/graphql.ts@@ -1824,7 +1824,8 @@ export enum FeatureFlagKey {IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',IS_UNIQUE_INDEXES_ENABLED = 'IS_UNIQUE_INDEXES_ENABLED',IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED = 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED',- IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED'+ IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED',+ IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED = 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED'}export type Field = {diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/constants/flat-field-metadata-cache-codec.constant.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/constants/flat-field-metadata-cache-codec.constant.tsnew file mode 100644index 00000000000..3415f40282e--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/constants/flat-field-metadata-cache-codec.constant.ts@@ -0,0 +1,86 @@+export const FLAT_FIELD_METADATA_SHORT_CODE_BY_KEY = {+ applicationId: '$0',+ applicationUniversalIdentifier: '$1',+ calendarEndViewIds: '$2',+ calendarEndViewUniversalIdentifiers: '$3',+ calendarViewIds: '$4',+ calendarViewUniversalIdentifiers: '$5',+ createdAt: '$6',+ defaultValue: '$7',+ description: '$8',+ fieldPermissionIds: '$9',+ fieldPermissionUniversalIdentifiers: '$a',+ icon: '$b',+ id: '$c',+ isActive: '$d',+ isLabelSyncedWithName: '$e',+ isNullable: '$f',+ isSystem: '$g',+ isSystemSideEffect: '$h',+ isUIEditable: '$i',+ isUIReadOnly: '$j',+ isUnique: '$k',+ kanbanAggregateOperationViewIds: '$l',+ kanbanAggregateOperationViewUniversalIdentifiers: '$m',+ label: '$n',+ mainGroupByFieldMetadataViewIds: '$o',+ mainGroupByFieldMetadataViewUniversalIdentifiers: '$p',+ morphId: '$q',+ name: '$r',+ objectMetadataId: '$s',+ objectMetadataUniversalIdentifier: '$t',+ options: '$u',+ overrides: '$v',+ relationTargetFieldMetadataId: '$w',+ relationTargetFieldMetadataUniversalIdentifier: '$x',+ relationTargetObjectMetadataId: '$y',+ relationTargetObjectMetadataUniversalIdentifier: '$z',+ searchFieldMetadataIds: '$A',+ searchFieldMetadataUniversalIdentifiers: '$B',+ settings: '$C',+ type: '$D',+ universalIdentifier: '$E',+ universalSettings: '$F',+ updatedAt: '$G',+ viewFieldIds: '$H',+ viewFieldUniversalIdentifiers: '$I',+ viewFilterIds: '$J',+ viewFilterUniversalIdentifiers: '$K',+ viewSortIds: '$L',+ viewSortUniversalIdentifiers: '$M',+ workspaceId: '$N',+} as const satisfies Record<string, string>;++export const FLAT_FIELD_METADATA_SHORT_CODE_LOOKUP = new Map<string, string>(+ Object.entries(FLAT_FIELD_METADATA_SHORT_CODE_BY_KEY),+);++export const FLAT_FIELD_METADATA_KEY_LOOKUP = new Map<string, string>(+ Object.entries(FLAT_FIELD_METADATA_SHORT_CODE_BY_KEY).map(+ ([key, shortCode]) => [shortCode, key],+ ),+);++export const FLAT_FIELD_METADATA_EMPTY_ARRAY_KEYS = [+ 'kanbanAggregateOperationViewIds',+ 'calendarViewIds',+ 'calendarEndViewIds',+ 'mainGroupByFieldMetadataViewIds',+ 'viewFieldIds',+ 'viewFilterIds',+ 'fieldPermissionIds',+ 'viewFieldUniversalIdentifiers',+ 'viewFilterUniversalIdentifiers',+ 'kanbanAggregateOperationViewUniversalIdentifiers',+ 'calendarViewUniversalIdentifiers',+ 'calendarEndViewUniversalIdentifiers',+ 'mainGroupByFieldMetadataViewUniversalIdentifiers',+ 'viewSortIds',+ 'viewSortUniversalIdentifiers',+ 'searchFieldMetadataIds',+ 'searchFieldMetadataUniversalIdentifiers',+ 'fieldPermissionUniversalIdentifiers',+] as const satisfies readonly (keyof typeof FLAT_FIELD_METADATA_SHORT_CODE_BY_KEY)[];++export const FLAT_FIELD_METADATA_EMPTY_ARRAY_KEY_SET: ReadonlySet<string> =+ new Set(FLAT_FIELD_METADATA_EMPTY_ARRAY_KEYS);diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/services/workspace-flat-field-metadata-map-cache.service.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/services/workspace-flat-field-metadata-map-cache.service.tsindex 704357cbdb6..d948fb1d020 100644--- a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/services/workspace-flat-field-metadata-map-cache.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/services/workspace-flat-field-metadata-map-cache.service.ts@@ -10,6 +10,9 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/import { createEmptyFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/constant/create-empty-flat-entity-maps.constant';import { FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';import { FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';+import { type EncodedFlatFieldMetadataMaps } from 'src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type';+import { decodeFlatFieldMetadataMapsFromCache } from 'src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util';+import { encodeFlatFieldMetadataMapsForCache } from 'src/engine/metadata-modules/flat-field-metadata/utils/encode-flat-field-metadata-maps-for-cache.util';import { fromFieldMetadataEntityToFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/from-field-metadata-entity-to-flat-field-metadata.util';import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';import { computeUniqueFieldMetadataIdsFromIndexEntities } from 'src/engine/metadata-modules/index-metadata/utils/compute-unique-field-metadata-ids-from-index-entities.util';@@ -30,7 +33,8 @@ import { ViewSortEntity } from 'src/engine/metadata-modules/view-sort/entities/v@Injectable()@WorkspaceCache('flatFieldMetadataMaps')export class WorkspaceFlatFieldMetadataMapCacheService extends WorkspaceCacheProvider<- FlatEntityMaps<FlatFieldMetadata>+ FlatEntityMaps<FlatFieldMetadata>,+ EncodedFlatFieldMetadataMaps> {constructor(@InjectRepository(FieldMetadataEntity)@@ -57,6 +61,18 @@ export class WorkspaceFlatFieldMetadataMapCacheService extends WorkspaceCacheProsuper();}+ override encodeForCacheStorage(+ data: FlatEntityMaps<FlatFieldMetadata>,+ ): EncodedFlatFieldMetadataMaps {+ return encodeFlatFieldMetadataMapsForCache(data);+ }++ override decodeFromCacheStorage(+ rawData: EncodedFlatFieldMetadataMaps,+ ): FlatEntityMaps<FlatFieldMetadata> {+ return decodeFlatFieldMetadataMapsFromCache(rawData);+ }+async computeForCache(workspaceId: string,): Promise<FlatEntityMaps<FlatFieldMetadata>> {diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type.tsnew file mode 100644index 00000000000..7994eefa9d5--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type.ts@@ -0,0 +1,23 @@+import { type FLAT_FIELD_METADATA_SHORT_CODE_BY_KEY } from 'src/engine/metadata-modules/flat-field-metadata/constants/flat-field-metadata-cache-codec.constant';+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';++type ShortCodeByKey = typeof FLAT_FIELD_METADATA_SHORT_CODE_BY_KEY;++export type EncodedFlatFieldMetadata = {+ [Key in keyof FlatFieldMetadata as Key extends keyof ShortCodeByKey+ ? ShortCodeByKey[Key]+ : Key]?: FlatFieldMetadata[Key];+};++export type EncodedFlatFieldMetadataMaps = {+ byUniversalIdentifier: Record<string, EncodedFlatFieldMetadata>;+ universalIdentifierById: FlatEntityMaps<FlatFieldMetadata>['universalIdentifierById'];+ universalIdentifiersByApplicationId: FlatEntityMaps<FlatFieldMetadata>['universalIdentifiersByApplicationId'];+};++export type EncodableFlatFieldMetadataMaps = {+ byUniversalIdentifier: Partial<Record<string, Partial<FlatFieldMetadata>>>;+ universalIdentifierById: FlatEntityMaps<FlatFieldMetadata>['universalIdentifierById'];+ universalIdentifiersByApplicationId: FlatEntityMaps<FlatFieldMetadata>['universalIdentifiersByApplicationId'];+};diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/__tests__/decode-flat-field-metadata-maps-from-cache.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/__tests__/decode-flat-field-metadata-maps-from-cache.util.spec.tsnew file mode 100644index 00000000000..447fe05eb4a--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/__tests__/decode-flat-field-metadata-maps-from-cache.util.spec.ts@@ -0,0 +1,60 @@+import { type EncodedFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type';+import { decodeFlatFieldMetadataMapsFromCache } from 'src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util';+import { encodeFlatFieldMetadataMapsForCache } from 'src/engine/metadata-modules/flat-field-metadata/utils/encode-flat-field-metadata-maps-for-cache.util';+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';++const decodeOne = (+ encodedFlatFieldMetadata: EncodedFlatFieldMetadata,+): Partial<FlatFieldMetadata> | undefined =>+ decodeFlatFieldMetadataMapsFromCache({+ byUniversalIdentifier: { 'field-uid': encodedFlatFieldMetadata },+ universalIdentifierById: {},+ universalIdentifiersByApplicationId: {},+ }).byUniversalIdentifier['field-uid'];++const encodeOne = (flatFieldMetadata: Partial<FlatFieldMetadata>) =>+ encodeFlatFieldMetadataMapsForCache({+ byUniversalIdentifier: { 'field-uid': flatFieldMetadata },+ universalIdentifierById: {},+ universalIdentifiersByApplicationId: {},+ });++describe('decodeFlatFieldMetadataMapsFromCache', () => {+ it('should restore a short code to its full key', () => {+ expect(decodeOne({ $r: 'firstName' })).toMatchObject({+ name: 'firstName',+ });+ });++ it('should restore an omitted relation array as empty', () => {+ expect(decodeOne({})).toMatchObject({ viewFieldIds: [] });+ });++ it('should keep a decoded relation array over the empty default', () => {+ expect(decodeOne({ $H: ['view-field-1'] })).toMatchObject({+ viewFieldIds: ['view-field-1'],+ });+ });++ it('should round trip a field through JSON, as the cache storage layer does', () => {+ const encoded = JSON.parse(+ JSON.stringify(+ encodeOne({+ name: 'firstName',+ viewFieldIds: ['view-field-1'],+ viewFilterIds: [],+ }),+ ),+ );++ expect(+ decodeFlatFieldMetadataMapsFromCache(encoded).byUniversalIdentifier,+ ).toMatchObject({+ 'field-uid': {+ name: 'firstName',+ viewFieldIds: ['view-field-1'],+ viewFilterIds: [],+ },+ });+ });+});diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/__tests__/encode-flat-field-metadata-maps-for-cache.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/__tests__/encode-flat-field-metadata-maps-for-cache.util.spec.tsnew file mode 100644index 00000000000..7ca91ebc2c0--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/__tests__/encode-flat-field-metadata-maps-for-cache.util.spec.ts@@ -0,0 +1,28 @@+import { type EncodedFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type';+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';+import { encodeFlatFieldMetadataMapsForCache } from 'src/engine/metadata-modules/flat-field-metadata/utils/encode-flat-field-metadata-maps-for-cache.util';++const encodeOne = (+ flatFieldMetadata: Partial<FlatFieldMetadata>,+): EncodedFlatFieldMetadata =>+ encodeFlatFieldMetadataMapsForCache({+ byUniversalIdentifier: { 'field-uid': flatFieldMetadata },+ universalIdentifierById: {},+ universalIdentifiersByApplicationId: {},+ }).byUniversalIdentifier['field-uid'];++describe('encodeFlatFieldMetadataMapsForCache', () => {+ it('should replace a mapped key with its short code', () => {+ expect(encodeOne({ name: 'firstName' })).toEqual({ $r: 'firstName' });+ });++ it('should drop a relation array that is empty', () => {+ expect(encodeOne({ viewFieldIds: [] })).toEqual({});+ });++ it('should keep a relation array that is populated', () => {+ expect(encodeOne({ viewFieldIds: ['view-field-1'] })).toEqual({+ $H: ['view-field-1'],+ });+ });+});diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.tsnew file mode 100644index 00000000000..049aac12d7b--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.ts@@ -0,0 +1,52 @@+import {+ FLAT_FIELD_METADATA_EMPTY_ARRAY_KEYS,+ FLAT_FIELD_METADATA_KEY_LOOKUP,+} from 'src/engine/metadata-modules/flat-field-metadata/constants/flat-field-metadata-cache-codec.constant';+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';+import {+ type EncodedFlatFieldMetadata,+ type EncodedFlatFieldMetadataMaps,+} from 'src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type';+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';++const decodeFlatFieldMetadata = (+ encoded: EncodedFlatFieldMetadata | Partial<FlatFieldMetadata>,+): FlatFieldMetadata => {+ const decoded: Partial<+ Record<string, FlatFieldMetadata[keyof FlatFieldMetadata]>+ > = {};++ for (const [shortCode, value] of Object.entries(encoded)) {+ decoded[FLAT_FIELD_METADATA_KEY_LOOKUP.get(shortCode) ?? shortCode] = value;+ }++ for (const key of FLAT_FIELD_METADATA_EMPTY_ARRAY_KEYS) {+ if (!(key in decoded)) {+ decoded[key] = [];+ }+ }++ return decoded as FlatFieldMetadata;+};++export const decodeFlatFieldMetadataMapsFromCache = (+ encoded: EncodedFlatFieldMetadataMaps | FlatEntityMaps<FlatFieldMetadata>,+): FlatEntityMaps<FlatFieldMetadata> => {+ const byUniversalIdentifier: FlatEntityMaps<FlatFieldMetadata>['byUniversalIdentifier'] =+ {};++ for (const [universalIdentifier, encodedFlatFieldMetadata] of Object.entries(+ encoded.byUniversalIdentifier,+ )) {+ byUniversalIdentifier[universalIdentifier] = decodeFlatFieldMetadata(+ encodedFlatFieldMetadata,+ );+ }++ return {+ byUniversalIdentifier,+ universalIdentifierById: encoded.universalIdentifierById,+ universalIdentifiersByApplicationId:+ encoded.universalIdentifiersByApplicationId,+ };+};diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/encode-flat-field-metadata-maps-for-cache.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/encode-flat-field-metadata-maps-for-cache.util.tsnew file mode 100644index 00000000000..b6a88024f37--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/encode-flat-field-metadata-maps-for-cache.util.ts@@ -0,0 +1,58 @@+import { isDefined } from 'twenty-shared/utils';++import {+ FLAT_FIELD_METADATA_EMPTY_ARRAY_KEY_SET,+ FLAT_FIELD_METADATA_SHORT_CODE_LOOKUP,+} from 'src/engine/metadata-modules/flat-field-metadata/constants/flat-field-metadata-cache-codec.constant';+import {+ type EncodableFlatFieldMetadataMaps,+ type EncodedFlatFieldMetadata,+ type EncodedFlatFieldMetadataMaps,+} from 'src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type';+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';++const encodeFlatFieldMetadata = (+ flatFieldMetadata: Partial<FlatFieldMetadata>,+): EncodedFlatFieldMetadata => {+ const encoded: Partial<+ Record<string, FlatFieldMetadata[keyof FlatFieldMetadata]>+ > = {};++ for (const [key, value] of Object.entries(flatFieldMetadata)) {+ if (+ FLAT_FIELD_METADATA_EMPTY_ARRAY_KEY_SET.has(key) &&+ Array.isArray(value) &&+ value.length === 0+ ) {+ continue;+ }++ encoded[FLAT_FIELD_METADATA_SHORT_CODE_LOOKUP.get(key) ?? key] = value;+ }++ return encoded;+};++export const encodeFlatFieldMetadataMapsForCache = (+ flatEntityMaps: EncodableFlatFieldMetadataMaps,+): EncodedFlatFieldMetadataMaps => {+ const byUniversalIdentifier: Record<string, EncodedFlatFieldMetadata> = {};++ for (const [universalIdentifier, flatFieldMetadata] of Object.entries(+ flatEntityMaps.byUniversalIdentifier,+ )) {+ if (!isDefined(flatFieldMetadata)) {+ continue;+ }++ byUniversalIdentifier[universalIdentifier] =+ encodeFlatFieldMetadata(flatFieldMetadata);+ }++ return {+ byUniversalIdentifier,+ universalIdentifierById: flatEntityMaps.universalIdentifierById,+ universalIdentifiersByApplicationId:+ flatEntityMaps.universalIdentifiersByApplicationId,+ };+};diff --git a/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts b/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.tsindex 3f6975d4b2a..b66e181afc3 100644--- a/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts+++ b/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts@@ -252,6 +252,7 @@ describe('WorkspaceEntityManager', () => {IS_SETTINGS_DISCOVERY_HERO_ENABLED: false,IS_WORKFLOW_VERSION_IN_CORE_ENABLED: false,IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED: false,+ IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED: false,},userWorkspaceRoleMap: {},apiKeyRoleMap: {},diff --git a/packages/twenty-server/src/engine/workspace-cache/interfaces/workspace-cache-provider.service.ts b/packages/twenty-server/src/engine/workspace-cache/interfaces/workspace-cache-provider.service.tsindex 99884c1eced..38b6d4d6a92 100644--- a/packages/twenty-server/src/engine/workspace-cache/interfaces/workspace-cache-provider.service.ts+++ b/packages/twenty-server/src/engine/workspace-cache/interfaces/workspace-cache-provider.service.ts@@ -10,6 +10,15 @@ type WorkspaceCacheDataType = WorkspaceCacheDataMap[WorkspaceCacheKeyName];@Injectable()export abstract class WorkspaceCacheProvider<T extends WorkspaceCacheDataType = WorkspaceCacheDataType,+ TEncoded = T,> {abstract computeForCache(workspaceId: string): Promise<T>;++ encodeForCacheStorage(data: T): T | TEncoded {+ return data;+ }++ decodeFromCacheStorage(rawData: T | TEncoded): T {+ return rawData as T;+ }}diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.tsindex ec689162ab6..340af1a684a 100644--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts@@ -16,7 +16,7 @@ import {computeLocalCacheStats,} from 'src/engine/workspace-cache/utils/compute-local-cache-stats.util';import { deepSizeBytes } from 'src/engine/workspace-cache/utils/deep-size-bytes.util';-import { getProviderFromCacheKey } from 'src/engine/workspace-cache/utils/get-provider-from-cache-key.util';+import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';type LocalCache = ReadonlyMap<string,@@ -40,7 +40,7 @@ export class WorkspaceCacheMetricsService {private readonly redisWriteDurationHistogram: Histogram;private localCache?: LocalCache;- private cacheSizeByProvider: Record<string, number> = {};+ private cacheSizeByKeyName: Record<string, number> = {};private cacheSizeTotalBytes = 0;private sizeSampler?: ReturnType<typeof setInterval>;private sizeStartupTimer?: ReturnType<typeof setTimeout>;@@ -148,14 +148,14 @@ export class WorkspaceCacheMetricsService {return;}- const perProvider: Record<+ const perKeyName: Record<string,{ count: number; sampledBytes: number; sampled: number }> = {};for (const [key, entry] of localCache) {- const provider = getProviderFromCacheKey(key);- const stats = (perProvider[provider] ??= {+ const keyName = getKeyNameFromLocalCacheKey(key);+ const stats = (perKeyName[keyName] ??= {count: 0,sampledBytes: 0,sampled: 0,@@ -168,7 +168,10 @@ export class WorkspaceCacheMetricsService {let entryBytes = 0;for (const version of entry.versions.values()) {- entryBytes += deepSizeBytes(version.data, SIZE_WALK_NODE_CAP);+ entryBytes +=+ version.state === 'cold'+ ? version.blob.byteLength+ : deepSizeBytes(version.data, SIZE_WALK_NODE_CAP);}stats.sampledBytes += entryBytes;@@ -177,20 +180,20 @@ export class WorkspaceCacheMetricsService {}}- const byProvider: Record<string, number> = {};+ const byKeyName: Record<string, number> = {};let total = 0;- for (const [provider, stats] of Object.entries(perProvider)) {+ for (const [keyName, stats] of Object.entries(perKeyName)) {const estimate =stats.sampled === 0? 0: Math.round((stats.sampledBytes / stats.sampled) * stats.count);- byProvider[provider] = estimate;+ byKeyName[keyName] = estimate;total += estimate;}- this.cacheSizeByProvider = byProvider;+ this.cacheSizeByKeyName = byKeyName;this.cacheSizeTotalBytes = total;}@@ -246,9 +249,9 @@ export class WorkspaceCacheMetricsService {unit: 'By',},callback: async () =>- Object.entries(this.cacheSizeByProvider).map(([provider, value]) => ({+ Object.entries(this.cacheSizeByKeyName).map(([keyName, value]) => ({value,- attributes: { provider },+ attributes: { provider: keyName },})),});this.metricsService.createMultiObservableGauge({@@ -258,10 +261,10 @@ export class WorkspaceCacheMetricsService {'Entries in the local workspace metadata cache per provider',},callback: async () =>- Object.entries(this.getStats().entriesByProvider).map(- ([provider, value]) => ({+ Object.entries(this.getStats().entriesByKeyName).map(+ ([keyName, value]) => ({value,- attributes: { provider },+ attributes: { provider: keyName },}),),});diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.tsindex 2c25a8e71b6..2436ed6b05d 100644--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts@@ -9,6 +9,7 @@ import { DiscoveryService, Reflector } from '@nestjs/core';import * as Sentry from '@sentry/node';import crypto from 'crypto';+import { FeatureFlagKey } from 'twenty-shared/types';import { isDefined, isValidUuid } from 'twenty-shared/utils';import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';@@ -29,14 +30,18 @@ import {WorkspaceCacheExceptionCode,} from 'src/engine/workspace-cache/exceptions/workspace-cache.exception';import {- WORKSPACE_CACHE_KEYS_V2,WorkspaceCacheKeyName,type WorkspaceCacheDataMap,type WorkspaceCacheResult,type WorkspaceCacheResultWithHashes,+ type WorkspaceCacheStoredDataMap,} from 'src/engine/workspace-cache/types/workspace-cache-key.type';-import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';+import {+ type VersionEntry,+ type WorkspaceLocalCacheEntry,+} from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';import { combineCacheHashes } from 'src/engine/workspace-cache/utils/combine-cache-hashes.util';+import { demoteColdStorageEntries } from 'src/engine/workspace-cache/utils/demote-cold-storage-entries.util';import { sweepLocalCache } from 'src/engine/workspace-cache/utils/sweep-local-cache.util';const LOCAL_TTL_MS = 100; // 100ms@@ -48,11 +53,14 @@ const MAX_LOCAL_CACHE_ENTRIES = 6_000;const MIN_EVICT_KEYS = 100;const LOCAL_ENTRY_TTL_MS = 30 * 60 * 1000; // 30 minutes idleconst LOCAL_CACHE_SWEEP_INTERVAL_MS = 60 * 1000;+const HOT_ENTRIES_PER_PROVIDER = 64;// Per-provider entry caps, keyed by local cache key prefix (ORM graphs are ~5 MB each).-const MAX_LOCAL_ENTRIES_BY_PREFIX = new Map<string, number>([- [WORKSPACE_CACHE_KEYS_V2.ORMEntityMetadatas, 128],+const MAX_LOCAL_ENTRIES_BY_KEY_NAME = new Map<string, number>([+ ['ORMEntityMetadatas', 128],+ ['flatFieldMetadataMaps', 256],]);type CacheDataType = WorkspaceCacheDataMap[WorkspaceCacheKeyName];+type StoredCacheDataType = WorkspaceCacheStoredDataMap[WorkspaceCacheKeyName];type CacheEntriesResult = {data: Partial<WorkspaceCacheDataMap>;@@ -75,7 +83,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {private lastLocalCacheSweepAt: number | undefined;private readonly workspaceCacheProviders = new Map<WorkspaceCacheKeyName,- WorkspaceCacheProvider<CacheDataType>+ WorkspaceCacheProvider<CacheDataType, StoredCacheDataType>>();private readonly localDataOnlyKeys = new Set<WorkspaceCacheKeyName>();private readonly memoizer = new PromiseMemoizer<CacheEntriesResult>(@@ -426,15 +434,31 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {return [`${baseKey}:data`, `${baseKey}:hash`];});- const allValues = await this.cacheStorage.mget<CacheDataType | string>(- allKeys,- );+ const [allValues, isCompactStorageEnabled] = await Promise.all([+ this.cacheStorage.mget<CacheDataType | string>(allKeys),+ this.resolveCompactStorageEnabled(workspaceId),+ ]);for (const [index, keyName] of cacheKeyNames.entries()) {- const data = allValues[index * 2] as CacheDataType | undefined;+ const rawData = allValues[index * 2] as CacheDataType | undefined;const hash = allValues[index * 2 + 1] as string | undefined;- if (isDefined(data) && isDefined(hash)) {+ if (isDefined(rawData) && isDefined(hash)) {+ let data: CacheDataType;++ try {+ data = isCompactStorageEnabled+ ? this.getProviderOrThrow(keyName).decodeFromCacheStorage(rawData)+ : rawData;+ } catch (error) {+ this.logger.warn(+ `Failed to decode cached ${keyName} for workspace ${workspaceId}, recomputing`,+ error,+ );+ missingInRedis.push(keyName);+ continue;+ }+Object.assign(redisEntries.data, { [keyName]: data });redisEntries.hashes[keyName] = hash;this.setInLocalCache(workspaceId, keyName, data, hash);@@ -499,7 +523,20 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {const computed = await Promise.all(computePromises);- const redisEntries: Array<{ key: string; value: unknown }> = [];+ const recomputedFeatureFlagsMap = computed.find(+ ({ keyName }) => keyName === 'featureFlagsMap',+ )?.data as WorkspaceCacheDataMap['featureFlagsMap'] | undefined;++ const isCompactStorageEnabled = isDefined(recomputedFeatureFlagsMap)+ ? recomputedFeatureFlagsMap[+ FeatureFlagKey.IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED+ ]+ : await this.resolveCompactStorageEnabled(workspaceId);++ const redisEntries: Array<{+ key: string;+ value: StoredCacheDataType | string;+ }> = [];const bootstrapHashEntries: Array<{ key: string; value: string }> = [];for (const { keyName, data, hash, isAdopted } of computed) {@@ -518,7 +555,12 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {}if (!isLocalDataOnly) {- redisEntries.push({ key: `${baseKey}:data`, value: data });+ redisEntries.push({+ key: `${baseKey}:data`,+ value: isCompactStorageEnabled+ ? this.getProviderOrThrow(keyName).encodeForCacheStorage(data)+ : data,+ });}this.setInLocalCache(workspaceId, keyName, data, hash);@@ -562,8 +604,9 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {const version = entry?.versions.get(entry.latestHash);if (isDefined(entry) && isDefined(version)) {- version.lastReadAt = Date.now();- Object.assign(result.data, { [keyName]: version.data });+ const data = this.readVersion({ keyName, entry, version });++ Object.assign(result.data, { [keyName]: data });result.hashes[keyName] = entry.latestHash;this.cleanupStaleVersions(entry);}@@ -613,7 +656,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {this.localCache.set(localKey, entry);}- entry.versions.set(hash, { data, lastReadAt: Date.now() });+ entry.versions.set(hash, { state: 'hot', data, lastReadAt: Date.now() });entry.latestHash = hash;entry.lastHashCheckedAt = Date.now();@@ -633,7 +676,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {const evicted = sweepLocalCache(this.localCache, now, {ttlMs: LOCAL_ENTRY_TTL_MS,- maxEntriesByPrefix: MAX_LOCAL_ENTRIES_BY_PREFIX,+ maxEntriesByKeyName: MAX_LOCAL_ENTRIES_BY_KEY_NAME,globalMaxEntries: MAX_LOCAL_CACHE_ENTRIES,minEvict: MIN_EVICT_KEYS,});@@ -641,6 +684,105 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {if (evicted > 0) {this.cacheMetricsService.recordEviction(evicted);}++ this.demoteColdEntries();+ }++ private demoteColdEntries(): void {+ demoteColdStorageEntries({+ localCache: this.localCache,+ hotEntriesPerProvider: HOT_ENTRIES_PER_PROVIDER,+ serialize: ({ localKey, data }) => {+ const separatorIndex = localKey.lastIndexOf(':');+ const workspaceId = localKey.slice(separatorIndex + 1);+ const keyName = localKey.slice(+ 0,+ separatorIndex,+ ) as WorkspaceCacheKeyName;++ if (+ this.localDataOnlyKeys.has(keyName) ||+ keyName === 'featureFlagsMap' ||+ !this.isCompactStorageEnabledInLocalCache(workspaceId)+ ) {+ return undefined;+ }++ return Buffer.from(+ JSON.stringify(+ this.getProviderOrThrow(keyName).encodeForCacheStorage(data),+ ),+ 'utf8',+ );+ },+ });+ }++ private isCompactStorageEnabledInLocalCache(+ workspaceId: string,+ ): boolean | undefined {+ const entry = this.localCache.get(+ this.buildCacheKey(workspaceId, 'featureFlagsMap'),+ );+ const version = entry?.versions.get(entry.latestHash);++ if (version?.state !== 'hot') {+ return undefined;+ }++ return (version.data as WorkspaceCacheDataMap['featureFlagsMap'])[+ FeatureFlagKey.IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED+ ];+ }++ // A restarted pod reads an already-encoded Redis payload with an empty local+ // cache, so the flag has to resolve from Redis before any decode happens.+ private async resolveCompactStorageEnabled(+ workspaceId: string,+ ): Promise<boolean> {+ const localValue = this.isCompactStorageEnabledInLocalCache(workspaceId);++ if (isDefined(localValue)) {+ return localValue;+ }++ const featureFlagsMap = await this.cacheStorage.get<+ WorkspaceCacheDataMap['featureFlagsMap']+ >(`${this.buildCacheKey(workspaceId, 'featureFlagsMap')}:data`);++ return (+ featureFlagsMap?.[+ FeatureFlagKey.IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED+ ] ?? false+ );+ }++ private readVersion({+ keyName,+ entry,+ version,+ }: {+ keyName: WorkspaceCacheKeyName;+ entry: WorkspaceLocalCacheEntry<CacheDataType>;+ version: VersionEntry<CacheDataType>;+ }): CacheDataType {+ if (version.state === 'hot') {+ version.lastReadAt = Date.now();++ return version.data;+ }++ const data = this.getProviderOrThrow(keyName).decodeFromCacheStorage(+ JSON.parse(version.blob.toString('utf8')),+ );++ entry.versions.set(entry.latestHash, {+ state: 'hot',+ data,+ lastReadAt: Date.now(),+ });++ return data;}private cleanupStaleVersions(@@ -677,7 +819,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {private getProviderOrThrow(keyName: WorkspaceCacheKeyName,- ): WorkspaceCacheProvider<CacheDataType> {+ ): WorkspaceCacheProvider<CacheDataType, StoredCacheDataType> {const provider = this.workspaceCacheProviders.get(keyName);if (!isDefined(provider)) {@@ -691,6 +833,6 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {workspaceId: string,keyName: WorkspaceCacheKeyName,): string {- return `${WORKSPACE_CACHE_KEYS_V2[keyName]}:${workspaceId}`;+ return `${keyName}:${workspaceId}`;}}diff --git a/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts b/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.tsindex d2499955014..60f48214029 100644--- a/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts+++ b/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts@@ -4,6 +4,7 @@ import {} from 'twenty-shared/types';import { type EntityMetadata } from 'typeorm';+import { type EncodedFlatFieldMetadataMaps } from 'src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type';import { type ResolverNameMapEntry } from 'src/engine/api/graphql/direct-execution/utils/build-resolver-name-map.util';import { type FlatApiKey } from 'src/engine/core-modules/api-key/types/flat-api-key.type';import { type ApplicationVariableCacheMaps } from 'src/engine/core-modules/application/application-variable/types/application-variable-cache-maps.type';@@ -17,55 +18,6 @@ import { type UserWorkspaceRoleMap } from 'src/engine/metadata-modules/role-targimport { type FlatRowLevelPermissionPredicateGroupMaps } from 'src/engine/metadata-modules/row-level-permission-predicate/types/flat-row-level-permission-predicate-group-maps.type';import { type FlatRowLevelPermissionPredicateMaps } from 'src/engine/metadata-modules/row-level-permission-predicate/types/flat-row-level-permission-predicate-maps.type';-export const WORKSPACE_CACHE_KEYS_V2 = {- flatObjectMetadataMaps: 'flat-maps:object-metadata',- flatFieldMetadataMaps: 'flat-maps:field-metadata',- flatIndexMaps: 'flat-maps:index',- flatViewMaps: 'flat-maps:view',- flatViewFieldMaps: 'flat-maps:view-field',- flatViewFieldGroupMaps: 'flat-maps:view-field-group',- flatViewGroupMaps: 'flat-maps:view-group',- flatViewFilterMaps: 'flat-maps:view-filter',- flatViewFilterGroupMaps: 'flat-maps:view-filter-group',- flatViewSortMaps: 'flat-maps:view-sort',- flatSearchFieldMetadataMaps: 'flat-maps:search-field-metadata',- flatLogicFunctionMaps: 'flat-maps:logic-function',- featureFlagsMap: 'feature-flag:feature-flags-map',- rolesPermissions: 'metadata:permissions:roles-permissions',- userWorkspaceRoleMap: 'metadata:permissions:user-workspace-role-map',- apiKeyRoleMap: 'metadata:permissions:api-key-role-map',- flatApplicationMaps: 'flat-maps:flatApplicationMaps',- flatRoleMaps: 'flat-maps:role',- flatRoleTargetMaps: 'flat-maps:role-target',- flatRolePermissionFlagMaps: 'flat-maps:role-permission-flag',- flatPermissionFlagMaps: 'flat-maps:permission-flag-catalog',- flatObjectPermissionMaps: 'flat-maps:object-permission',- flatFieldPermissionMaps: 'flat-maps:field-permission',- ORMEntityMetadatas: 'orm:entity-metadatas',- flatAgentMaps: 'flat-maps:agent',- flatSkillMaps: 'flat-maps:skill',- flatCommandMenuItemMaps: 'flat-maps:command-menu-item',- flatNavigationMenuItemMaps: 'flat-maps:navigation-menu-item',- flatRoleTargetByAgentIdMaps: 'flat-maps:flatRoleTargetByAgentId',- flatPageLayoutMaps: 'flat-maps:page-layout',- flatPageLayoutWidgetMaps: 'flat-maps:page-layout-widget',- flatPageLayoutTabMaps: 'flat-maps:flatPageLayoutTabMaps',- flatRowLevelPermissionPredicateMaps:- 'flat-maps:row-level-permission-predicate',- flatRowLevelPermissionPredicateGroupMaps:- 'flat-maps:row-level-permission-predicate-group',- flatFrontComponentMaps: 'flat-maps:front-component',- flatWebhookMaps: 'flat-maps:webhook',- flatApplicationVariableMaps: 'flat-maps:application-variable',- flatConnectionProviderMaps: 'flat-maps:connection-provider',- flatWorkspaceMemberMaps: 'flat-maps:workspace-member',- apiKeyMap: 'cache:api-key-map',- applicationVariableMaps: 'cache:application-variable',- graphQLResolverNameMap: 'direct-execution:graphql-resolver-name-map',- currentBillingSubscription: 'billing:subscription',- workflowAutomatedTriggerMaps: 'cache:workflow-automated-trigger',-} as const satisfies Record<WorkspaceCacheKeyName, string>;-export type AdditionalCacheDataMaps = {featureFlagsMap: Record<FeatureFlagKey, boolean>;rolesPermissions: ObjectsPermissionsByRoleId;@@ -98,3 +50,12 @@ export type WorkspaceCacheResultWithHashes<K extends WorkspaceCacheKeyName[]> =data: WorkspaceCacheResult<K>;hashes: { [P in K[number]]: string };};++export type WorkspaceCacheStoredDataMap = Omit<+ WorkspaceCacheDataMap,+ 'flatFieldMetadataMaps'+> & {+ flatFieldMetadataMaps:+ | WorkspaceCacheDataMap['flatFieldMetadataMaps']+ | EncodedFlatFieldMetadataMaps;+};diff --git a/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts b/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.tsindex 3db32ab0d41..9bade6606a3 100644--- a/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts+++ b/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts@@ -1,8 +1,17 @@-export type VersionEntry<T> = {+export type HotVersionEntry<T> = {+ state: 'hot';data: T;lastReadAt: number;};+export type ColdVersionEntry = {+ state: 'cold';+ blob: Buffer;+ lastReadAt: number;+};++export type VersionEntry<T> = HotVersionEntry<T> | ColdVersionEntry;+export type WorkspaceLocalCacheEntry<T> = {versions: Map<string, VersionEntry<T>>;latestHash: string;diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.tsindex dbeb4ecef11..a7c5e5dfb72 100644--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts@@ -11,16 +11,16 @@ describe('computeLocalCacheStats', () => {workspaces: 0,versionsTotal: 0,versionsByCount: { '1': 0, '2': 0, '3': 0, '4': 0, '5+': 0 },- entriesByProvider: {},+ entriesByKeyName: {},});});it('counts entries, distinct workspaces, and total versions', () => {const stats = computeLocalCacheStats(new Map([- ['orm:entity-metadatas:ws-a', entry(1)],- ['flat-maps:field-metadata:ws-a', entry(2)],- ['orm:entity-metadatas:ws-b', entry(1)],+ ['ORMEntityMetadatas:ws-a', entry(1)],+ ['flatFieldMetadataMaps:ws-a', entry(2)],+ ['ORMEntityMetadatas:ws-b', entry(1)],]),);@@ -28,9 +28,9 @@ describe('computeLocalCacheStats', () => {// ws-a and ws-b — the same workspace under two providers counts once.expect(stats.workspaces).toBe(2);expect(stats.versionsTotal).toBe(4);- expect(stats.entriesByProvider).toEqual({- 'orm:entity-metadatas': 2,- 'flat-maps:field-metadata': 1,+ expect(stats.entriesByKeyName).toEqual({+ ORMEntityMetadatas: 2,+ flatFieldMetadataMaps: 1,});});diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.tsnew file mode 100644index 00000000000..1c139fba4ea--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts@@ -0,0 +1,111 @@+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';+import { demoteColdStorageEntries } from 'src/engine/workspace-cache/utils/demote-cold-storage-entries.util';++const FIELD_METADATA = 'flatFieldMetadataMaps';+const ORM = 'ORMEntityMetadatas';++const hotEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({+ versions: new Map([['hash-1', { state: 'hot', data: 'data', lastReadAt }]]),+ latestHash: 'hash-1',+ lastHashCheckedAt: lastReadAt,+});++const coldEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({+ versions: new Map([+ ['hash-1', { state: 'cold', blob: Buffer.from('{}'), lastReadAt }],+ ]),+ latestHash: 'hash-1',+ lastHashCheckedAt: lastReadAt,+});++const serialize = () => Buffer.from('serialized');++const demote = (+ localCache: Map<string, WorkspaceLocalCacheEntry<string>>,+ hotEntriesPerProvider: number,+ overrides?: { serialize?: () => Buffer | undefined },+) =>+ demoteColdStorageEntries({+ localCache,+ hotEntriesPerProvider,+ serialize: overrides?.serialize ?? serialize,+ });++describe('demoteColdStorageEntries', () => {+ it('should demote nothing when the provider is within its hot budget', () => {+ const localCache = new Map([+ [`${FIELD_METADATA}:ws-a`, hotEntry(1)],+ [`${FIELD_METADATA}:ws-b`, hotEntry(2)],+ ]);++ expect(demote(localCache, 2)).toBe(0);+ expect(+ localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),+ ).toMatchObject({ state: 'hot' });+ });++ it('should demote the least recently read entries beyond the hot budget', () => {+ const localCache = new Map([+ [`${FIELD_METADATA}:ws-newest`, hotEntry(9)],+ [`${FIELD_METADATA}:ws-oldest`, hotEntry(1)],+ [`${FIELD_METADATA}:ws-middle`, hotEntry(5)],+ ]);++ expect(demote(localCache, 1)).toBe(2);+ expect(+ localCache.get(`${FIELD_METADATA}:ws-oldest`)?.versions.get('hash-1'),+ ).toMatchObject({ state: 'cold', blob: Buffer.from('serialized') });+ expect(+ localCache.get(`${FIELD_METADATA}:ws-newest`)?.versions.get('hash-1'),+ ).toMatchObject({ state: 'hot' });+ });++ it('should preserve lastReadAt so a demoted entry keeps its place in the eviction order', () => {+ const localCache = new Map([+ [`${FIELD_METADATA}:ws-a`, hotEntry(1)],+ [`${FIELD_METADATA}:ws-b`, hotEntry(2)],+ ]);++ demote(localCache, 1);++ expect(+ localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),+ ).toMatchObject({ lastReadAt: 1 });+ });++ it('should leave the entry hot when serialize declines it', () => {+ const localCache = new Map([+ [`${FIELD_METADATA}:ws-a`, hotEntry(1)],+ [`${FIELD_METADATA}:ws-b`, hotEntry(2)],+ ]);++ expect(demote(localCache, 1, { serialize: () => undefined })).toBe(0);+ expect(+ localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),+ ).toMatchObject({ state: 'hot' });+ });++ it('should budget each provider independently', () => {+ const localCache = new Map([+ [`${FIELD_METADATA}:ws-a`, hotEntry(1)],+ [`${FIELD_METADATA}:ws-b`, hotEntry(2)],+ [`${ORM}:ws-a`, hotEntry(3)],+ ]);++ expect(demote(localCache, 1)).toBe(1);+ expect(localCache.get(`${ORM}:ws-a`)?.versions.get('hash-1')).toMatchObject(+ {+ state: 'hot',+ },+ );+ });++ it('should not count an already cold entry against the hot budget', () => {+ const localCache = new Map([+ [`${FIELD_METADATA}:ws-cold`, coldEntry(1)],+ [`${FIELD_METADATA}:ws-hot`, hotEntry(2)],+ ]);++ expect(demote(localCache, 1)).toBe(0);+ });+});diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.tsindex 0c42624e930..205f59dd963 100644--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts@@ -5,10 +5,10 @@ import {} from 'src/engine/workspace-cache/utils/sweep-local-cache.util';const TTL_MS = 30 * 60 * 1000;-const ORM_PREFIX = 'orm:entity-metadatas';+const ORM_KEY_NAME = 'ORMEntityMetadatas';const entry = (lastReadAt: number): WorkspaceLocalCacheEntry<unknown> => ({- versions: new Map([['h', { data: {}, lastReadAt }]]),+ versions: new Map([['h', { state: 'hot', data: {}, lastReadAt }]]),latestHash: 'h',lastHashCheckedAt: lastReadAt,});@@ -17,7 +17,7 @@ const config = (overrides: Partial<LocalCacheSweepConfig> = {},): LocalCacheSweepConfig => ({ttlMs: TTL_MS,- maxEntriesByPrefix: new Map(),+ maxEntriesByKeyName: new Map(),globalMaxEntries: 6_000,minEvict: 0,...overrides,@@ -26,34 +26,34 @@ const config = (describe('sweepLocalCache', () => {it('drops entries idle past the TTL and keeps recent ones', () => {const cache = new Map([- [`${ORM_PREFIX}:ws-idle`, entry(0)],- [`${ORM_PREFIX}:ws-recent`, entry(TTL_MS)],+ [`${ORM_KEY_NAME}:ws-idle`, entry(0)],+ [`${ORM_KEY_NAME}:ws-recent`, entry(TTL_MS)],]);const evicted = sweepLocalCache(cache, TTL_MS + 1, config());expect(evicted).toBe(1);- expect(cache.has(`${ORM_PREFIX}:ws-idle`)).toBe(false);- expect(cache.has(`${ORM_PREFIX}:ws-recent`)).toBe(true);+ expect(cache.has(`${ORM_KEY_NAME}:ws-idle`)).toBe(false);+ expect(cache.has(`${ORM_KEY_NAME}:ws-recent`)).toBe(true);});it('trims a provider over its cap, evicting the least-recently-read', () => {const cache = new Map([- [`${ORM_PREFIX}:ws-a`, entry(1)],- [`${ORM_PREFIX}:ws-b`, entry(2)],- [`${ORM_PREFIX}:ws-c`, entry(3)],+ [`${ORM_KEY_NAME}:ws-a`, entry(1)],+ [`${ORM_KEY_NAME}:ws-b`, entry(2)],+ [`${ORM_KEY_NAME}:ws-c`, entry(3)],]);const evicted = sweepLocalCache(cache,100,- config({ maxEntriesByPrefix: new Map([[ORM_PREFIX, 2]]) }),+ config({ maxEntriesByKeyName: new Map([[ORM_KEY_NAME, 2]]) }),);expect(evicted).toBe(1);expect(cache.size).toBe(2);- expect(cache.has(`${ORM_PREFIX}:ws-a`)).toBe(false); // oldest- expect(cache.has(`${ORM_PREFIX}:ws-c`)).toBe(true); // newest+ expect(cache.has(`${ORM_KEY_NAME}:ws-a`)).toBe(false); // oldest+ expect(cache.has(`${ORM_KEY_NAME}:ws-c`)).toBe(true); // newest});it('leaves a provider without a cap untouched', () => {@@ -67,7 +67,7 @@ describe('sweepLocalCache', () => {const evicted = sweepLocalCache(cache,100,- config({ maxEntriesByPrefix: new Map([[ORM_PREFIX, 2]]) }),+ config({ maxEntriesByKeyName: new Map([[ORM_KEY_NAME, 2]]) }),);expect(evicted).toBe(0);@@ -97,15 +97,15 @@ describe('sweepLocalCache', () => {it('expires first so the per-provider cap only counts survivors', () => {const cache = new Map([- [`${ORM_PREFIX}:ws-idle`, entry(0)], // expired by TTL- [`${ORM_PREFIX}:ws-a`, entry(TTL_MS + 1)],- [`${ORM_PREFIX}:ws-b`, entry(TTL_MS + 2)],+ [`${ORM_KEY_NAME}:ws-idle`, entry(0)], // expired by TTL+ [`${ORM_KEY_NAME}:ws-a`, entry(TTL_MS + 1)],+ [`${ORM_KEY_NAME}:ws-b`, entry(TTL_MS + 2)],]);const evicted = sweepLocalCache(cache,TTL_MS + 10,- config({ maxEntriesByPrefix: new Map([[ORM_PREFIX, 2]]) }),+ config({ maxEntriesByKeyName: new Map([[ORM_KEY_NAME, 2]]) }),);// The idle one is expired; the two survivors are within the cap, so nothing else goes.diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.tsindex e61b5296989..71a07ad1442 100644--- a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts+++ b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts@@ -1,11 +1,11 @@-import { getProviderFromCacheKey } from 'src/engine/workspace-cache/utils/get-provider-from-cache-key.util';+import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';export type LocalCacheStats = {entries: number;workspaces: number;versionsTotal: number;versionsByCount: Record<string, number>;- entriesByProvider: Record<string, number>;+ entriesByKeyName: Record<string, number>;};export const computeLocalCacheStats = (@@ -19,14 +19,14 @@ export const computeLocalCacheStats = ('4': 0,'5+': 0,};- const entriesByProvider: Record<string, number> = {};+ const entriesByKeyName: Record<string, number> = {};let versionsTotal = 0;for (const [key, entry] of localCache) {workspaceIds.add(key.slice(key.lastIndexOf(':') + 1));- const provider = getProviderFromCacheKey(key);+ const keyName = getKeyNameFromLocalCacheKey(key);- entriesByProvider[provider] = (entriesByProvider[provider] ?? 0) + 1;+ entriesByKeyName[keyName] = (entriesByKeyName[keyName] ?? 0) + 1;const versionCount = entry.versions.size;versionsTotal += versionCount;@@ -40,6 +40,6 @@ export const computeLocalCacheStats = (workspaces: workspaceIds.size,versionsTotal,versionsByCount,- entriesByProvider,+ entriesByKeyName,};};diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.tsnew file mode 100644index 00000000000..eb565c5dfbd--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts@@ -0,0 +1,68 @@+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';+import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';++export const demoteColdStorageEntries = <T>({+ localCache,+ hotEntriesPerProvider,+ serialize,+}: {+ localCache: ReadonlyMap<string, WorkspaceLocalCacheEntry<T>>;+ hotEntriesPerProvider: number;+ serialize: (params: { localKey: string; data: T }) => Buffer | undefined;+}): number => {+ const hotByProvider = new Map<+ string,+ { localKey: string; hash: string; lastReadAt: number }[]+ >();++ for (const [localKey, entry] of localCache) {+ for (const [hash, version] of entry.versions) {+ if (version.state !== 'hot') {+ continue;+ }++ const provider = getKeyNameFromLocalCacheKey(localKey);+ const hot = hotByProvider.get(provider) ?? [];++ hot.push({ localKey, hash, lastReadAt: version.lastReadAt });+ hotByProvider.set(provider, hot);+ }+ }++ let demoted = 0;++ for (const hot of hotByProvider.values()) {+ if (hot.length <= hotEntriesPerProvider) {+ continue;+ }++ hot.sort((a, b) => a.lastReadAt - b.lastReadAt);++ for (const { localKey, hash } of hot.slice(+ 0,+ hot.length - hotEntriesPerProvider,+ )) {+ const entry = localCache.get(localKey);+ const version = entry?.versions.get(hash);++ if (!entry || version?.state !== 'hot') {+ continue;+ }++ const blob = serialize({ localKey, data: version.data });++ if (blob === undefined) {+ continue;+ }++ entry.versions.set(hash, {+ state: 'cold',+ blob,+ lastReadAt: version.lastReadAt,+ });+ demoted += 1;+ }+ }++ return demoted;+};diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util.tsnew file mode 100644index 00000000000..d43767f3a9b--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util.ts@@ -0,0 +1,2 @@+export const getKeyNameFromLocalCacheKey = (localCacheKey: string): string =>+ localCacheKey.slice(0, localCacheKey.lastIndexOf(':'));diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/get-provider-from-cache-key.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/get-provider-from-cache-key.util.tsdeleted file mode 100644index 3a6f48993c8..00000000000--- a/packages/twenty-server/src/engine/workspace-cache/utils/get-provider-from-cache-key.util.ts+++ /dev/null@@ -1,2 +0,0 @@-export const getProviderFromCacheKey = (cacheKey: string): string =>- cacheKey.slice(0, cacheKey.lastIndexOf(':'));diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.tsindex d6b7b87ee4f..9cb7fccb585 100644--- a/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.ts+++ b/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.ts@@ -2,7 +2,7 @@ import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/export type LocalCacheSweepConfig = {ttlMs: number;- maxEntriesByPrefix: ReadonlyMap<string, number>;+ maxEntriesByKeyName: ReadonlyMap<string, number>;globalMaxEntries: number;minEvict: number;};@@ -70,10 +70,10 @@ export const sweepLocalCache = <T>(}}- for (const [prefix, maxEntries] of config.maxEntriesByPrefix) {+ for (const [keyName, maxEntries] of config.maxEntriesByKeyName) {evicted += evictLeastRecentlyRead({localCache,- matches: (key) => key.startsWith(`${prefix}:`),+ matches: (key) => key.startsWith(`${keyName}:`),maxEntries,minEvict: 0,});diff --git a/packages/twenty-shared/src/types/FeatureFlagKey.ts b/packages/twenty-shared/src/types/FeatureFlagKey.tsindex d9beb16545c..92a89ee1fb3 100644--- a/packages/twenty-shared/src/types/FeatureFlagKey.ts+++ b/packages/twenty-shared/src/types/FeatureFlagKey.ts@@ -12,4 +12,5 @@ export enum FeatureFlagKey {IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED',IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED = 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED',+ IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED = 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED',}