Skip to content

The open alternative to Salesforce, designed for AI.

perf(server): compact and cold-store workspace cache payloads behind a feature flag (#23880)

Charles Bochetc6200c7a143b
diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql b/packages/twenty-client-sdk/src/metadata/generated/schema.graphql
index 31423102e55..e598895d21c 100644
--- a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql
+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.graphql
@@ -1739,6 +1739,7 @@ enum FeatureFlagKey {
IS_SETTINGS_DISCOVERY_HERO_ENABLED
IS_WORKFLOW_VERSION_IN_CORE_ENABLED
IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED
+ IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED
}
type WorkspaceUrls {
diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.ts b/packages/twenty-client-sdk/src/metadata/generated/schema.ts
index dcab22580af..6edd8d4bcdb 100644
--- a/packages/twenty-client-sdk/src/metadata/generated/schema.ts
+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.ts
@@ -1395,7 +1395,7 @@ export interface FeatureFlag {
__typename: 'FeatureFlag'
}
-export type FeatureFlagKey = 'IS_AI_CHAT_PAGE_ENABLED' | 'IS_APP_CLAIMING_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_CALENDAR_WEEK_VIEW_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_LIST_VIEW_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' | 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED' | 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED'
+export type FeatureFlagKey = 'IS_AI_CHAT_PAGE_ENABLED' | 'IS_APP_CLAIMING_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_CALENDAR_WEEK_VIEW_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_LIST_VIEW_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' | 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED' | 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED' | 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED'
export interface WorkspaceUrls {
customUrl?: Scalars['String']
@@ -9653,7 +9653,8 @@ export const enumFeatureFlagKey = {
IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED: 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' as const,
IS_SETTINGS_DISCOVERY_HERO_ENABLED: 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' as const,
IS_WORKFLOW_VERSION_IN_CORE_ENABLED: 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED' as const,
- IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED: 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED' as const
+ IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED: 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED' as const,
+ IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED: 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED' as const
}
export const enumIdentityProviderType = {
diff --git a/packages/twenty-front/src/generated-admin/graphql.ts b/packages/twenty-front/src/generated-admin/graphql.ts
index 1ce0033c05f..781160320ee 100644
--- a/packages/twenty-front/src/generated-admin/graphql.ts
+++ b/packages/twenty-front/src/generated-admin/graphql.ts
@@ -380,7 +380,8 @@ export enum FeatureFlagKey {
IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',
IS_UNIQUE_INDEXES_ENABLED = 'IS_UNIQUE_INDEXES_ENABLED',
IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED = 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED',
- IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED'
+ IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED',
+ IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED = 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED'
}
export type FindApplicationRegistrationInstalledWorkspacesInput = {
diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.ts
index f911b15cba1..d4f5ec7b18a 100644
--- a/packages/twenty-front/src/generated-metadata/graphql.ts
+++ b/packages/twenty-front/src/generated-metadata/graphql.ts
@@ -1824,7 +1824,8 @@ export enum FeatureFlagKey {
IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',
IS_UNIQUE_INDEXES_ENABLED = 'IS_UNIQUE_INDEXES_ENABLED',
IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED = 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED',
- IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED'
+ IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED',
+ IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED = 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED'
}
export type Field = {
diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/constants/flat-field-metadata-cache-codec.constant.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/constants/flat-field-metadata-cache-codec.constant.ts
new file mode 100644
index 00000000000..3415f40282e
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/constants/flat-field-metadata-cache-codec.constant.ts
@@ -0,0 +1,86 @@
+export const FLAT_FIELD_METADATA_SHORT_CODE_BY_KEY = {
+ applicationId: '$0',
+ applicationUniversalIdentifier: '$1',
+ calendarEndViewIds: '$2',
+ calendarEndViewUniversalIdentifiers: '$3',
+ calendarViewIds: '$4',
+ calendarViewUniversalIdentifiers: '$5',
+ createdAt: '$6',
+ defaultValue: '$7',
+ description: '$8',
+ fieldPermissionIds: '$9',
+ fieldPermissionUniversalIdentifiers: '$a',
+ icon: '$b',
+ id: '$c',
+ isActive: '$d',
+ isLabelSyncedWithName: '$e',
+ isNullable: '$f',
+ isSystem: '$g',
+ isSystemSideEffect: '$h',
+ isUIEditable: '$i',
+ isUIReadOnly: '$j',
+ isUnique: '$k',
+ kanbanAggregateOperationViewIds: '$l',
+ kanbanAggregateOperationViewUniversalIdentifiers: '$m',
+ label: '$n',
+ mainGroupByFieldMetadataViewIds: '$o',
+ mainGroupByFieldMetadataViewUniversalIdentifiers: '$p',
+ morphId: '$q',
+ name: '$r',
+ objectMetadataId: '$s',
+ objectMetadataUniversalIdentifier: '$t',
+ options: '$u',
+ overrides: '$v',
+ relationTargetFieldMetadataId: '$w',
+ relationTargetFieldMetadataUniversalIdentifier: '$x',
+ relationTargetObjectMetadataId: '$y',
+ relationTargetObjectMetadataUniversalIdentifier: '$z',
+ searchFieldMetadataIds: '$A',
+ searchFieldMetadataUniversalIdentifiers: '$B',
+ settings: '$C',
+ type: '$D',
+ universalIdentifier: '$E',
+ universalSettings: '$F',
+ updatedAt: '$G',
+ viewFieldIds: '$H',
+ viewFieldUniversalIdentifiers: '$I',
+ viewFilterIds: '$J',
+ viewFilterUniversalIdentifiers: '$K',
+ viewSortIds: '$L',
+ viewSortUniversalIdentifiers: '$M',
+ workspaceId: '$N',
+} as const satisfies Record<string, string>;
+
+export const FLAT_FIELD_METADATA_SHORT_CODE_LOOKUP = new Map<string, string>(
+ Object.entries(FLAT_FIELD_METADATA_SHORT_CODE_BY_KEY),
+);
+
+export const FLAT_FIELD_METADATA_KEY_LOOKUP = new Map<string, string>(
+ Object.entries(FLAT_FIELD_METADATA_SHORT_CODE_BY_KEY).map(
+ ([key, shortCode]) => [shortCode, key],
+ ),
+);
+
+export const FLAT_FIELD_METADATA_EMPTY_ARRAY_KEYS = [
+ 'kanbanAggregateOperationViewIds',
+ 'calendarViewIds',
+ 'calendarEndViewIds',
+ 'mainGroupByFieldMetadataViewIds',
+ 'viewFieldIds',
+ 'viewFilterIds',
+ 'fieldPermissionIds',
+ 'viewFieldUniversalIdentifiers',
+ 'viewFilterUniversalIdentifiers',
+ 'kanbanAggregateOperationViewUniversalIdentifiers',
+ 'calendarViewUniversalIdentifiers',
+ 'calendarEndViewUniversalIdentifiers',
+ 'mainGroupByFieldMetadataViewUniversalIdentifiers',
+ 'viewSortIds',
+ 'viewSortUniversalIdentifiers',
+ 'searchFieldMetadataIds',
+ 'searchFieldMetadataUniversalIdentifiers',
+ 'fieldPermissionUniversalIdentifiers',
+] as const satisfies readonly (keyof typeof FLAT_FIELD_METADATA_SHORT_CODE_BY_KEY)[];
+
+export const FLAT_FIELD_METADATA_EMPTY_ARRAY_KEY_SET: ReadonlySet<string> =
+ new Set(FLAT_FIELD_METADATA_EMPTY_ARRAY_KEYS);
diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/services/workspace-flat-field-metadata-map-cache.service.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/services/workspace-flat-field-metadata-map-cache.service.ts
index 704357cbdb6..d948fb1d020 100644
--- a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/services/workspace-flat-field-metadata-map-cache.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/services/workspace-flat-field-metadata-map-cache.service.ts
@@ -10,6 +10,9 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/
import { createEmptyFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/constant/create-empty-flat-entity-maps.constant';
import { FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
import { FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { type EncodedFlatFieldMetadataMaps } from 'src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type';
+import { decodeFlatFieldMetadataMapsFromCache } from 'src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util';
+import { encodeFlatFieldMetadataMapsForCache } from 'src/engine/metadata-modules/flat-field-metadata/utils/encode-flat-field-metadata-maps-for-cache.util';
import { fromFieldMetadataEntityToFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/from-field-metadata-entity-to-flat-field-metadata.util';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { computeUniqueFieldMetadataIdsFromIndexEntities } from 'src/engine/metadata-modules/index-metadata/utils/compute-unique-field-metadata-ids-from-index-entities.util';
@@ -30,7 +33,8 @@ import { ViewSortEntity } from 'src/engine/metadata-modules/view-sort/entities/v
@Injectable()
@WorkspaceCache('flatFieldMetadataMaps')
export class WorkspaceFlatFieldMetadataMapCacheService extends WorkspaceCacheProvider<
- FlatEntityMaps<FlatFieldMetadata>
+ FlatEntityMaps<FlatFieldMetadata>,
+ EncodedFlatFieldMetadataMaps
> {
constructor(
@InjectRepository(FieldMetadataEntity)
@@ -57,6 +61,18 @@ export class WorkspaceFlatFieldMetadataMapCacheService extends WorkspaceCachePro
super();
}
+ override encodeForCacheStorage(
+ data: FlatEntityMaps<FlatFieldMetadata>,
+ ): EncodedFlatFieldMetadataMaps {
+ return encodeFlatFieldMetadataMapsForCache(data);
+ }
+
+ override decodeFromCacheStorage(
+ rawData: EncodedFlatFieldMetadataMaps,
+ ): FlatEntityMaps<FlatFieldMetadata> {
+ return decodeFlatFieldMetadataMapsFromCache(rawData);
+ }
+
async computeForCache(
workspaceId: string,
): Promise<FlatEntityMaps<FlatFieldMetadata>> {
diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type.ts
new file mode 100644
index 00000000000..7994eefa9d5
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type.ts
@@ -0,0 +1,23 @@
+import { type FLAT_FIELD_METADATA_SHORT_CODE_BY_KEY } from 'src/engine/metadata-modules/flat-field-metadata/constants/flat-field-metadata-cache-codec.constant';
+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+
+type ShortCodeByKey = typeof FLAT_FIELD_METADATA_SHORT_CODE_BY_KEY;
+
+export type EncodedFlatFieldMetadata = {
+ [Key in keyof FlatFieldMetadata as Key extends keyof ShortCodeByKey
+ ? ShortCodeByKey[Key]
+ : Key]?: FlatFieldMetadata[Key];
+};
+
+export type EncodedFlatFieldMetadataMaps = {
+ byUniversalIdentifier: Record<string, EncodedFlatFieldMetadata>;
+ universalIdentifierById: FlatEntityMaps<FlatFieldMetadata>['universalIdentifierById'];
+ universalIdentifiersByApplicationId: FlatEntityMaps<FlatFieldMetadata>['universalIdentifiersByApplicationId'];
+};
+
+export type EncodableFlatFieldMetadataMaps = {
+ byUniversalIdentifier: Partial<Record<string, Partial<FlatFieldMetadata>>>;
+ universalIdentifierById: FlatEntityMaps<FlatFieldMetadata>['universalIdentifierById'];
+ universalIdentifiersByApplicationId: FlatEntityMaps<FlatFieldMetadata>['universalIdentifiersByApplicationId'];
+};
diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/__tests__/decode-flat-field-metadata-maps-from-cache.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/__tests__/decode-flat-field-metadata-maps-from-cache.util.spec.ts
new file mode 100644
index 00000000000..447fe05eb4a
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/__tests__/decode-flat-field-metadata-maps-from-cache.util.spec.ts
@@ -0,0 +1,60 @@
+import { type EncodedFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type';
+import { decodeFlatFieldMetadataMapsFromCache } from 'src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util';
+import { encodeFlatFieldMetadataMapsForCache } from 'src/engine/metadata-modules/flat-field-metadata/utils/encode-flat-field-metadata-maps-for-cache.util';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+
+const decodeOne = (
+ encodedFlatFieldMetadata: EncodedFlatFieldMetadata,
+): Partial<FlatFieldMetadata> | undefined =>
+ decodeFlatFieldMetadataMapsFromCache({
+ byUniversalIdentifier: { 'field-uid': encodedFlatFieldMetadata },
+ universalIdentifierById: {},
+ universalIdentifiersByApplicationId: {},
+ }).byUniversalIdentifier['field-uid'];
+
+const encodeOne = (flatFieldMetadata: Partial<FlatFieldMetadata>) =>
+ encodeFlatFieldMetadataMapsForCache({
+ byUniversalIdentifier: { 'field-uid': flatFieldMetadata },
+ universalIdentifierById: {},
+ universalIdentifiersByApplicationId: {},
+ });
+
+describe('decodeFlatFieldMetadataMapsFromCache', () => {
+ it('should restore a short code to its full key', () => {
+ expect(decodeOne({ $r: 'firstName' })).toMatchObject({
+ name: 'firstName',
+ });
+ });
+
+ it('should restore an omitted relation array as empty', () => {
+ expect(decodeOne({})).toMatchObject({ viewFieldIds: [] });
+ });
+
+ it('should keep a decoded relation array over the empty default', () => {
+ expect(decodeOne({ $H: ['view-field-1'] })).toMatchObject({
+ viewFieldIds: ['view-field-1'],
+ });
+ });
+
+ it('should round trip a field through JSON, as the cache storage layer does', () => {
+ const encoded = JSON.parse(
+ JSON.stringify(
+ encodeOne({
+ name: 'firstName',
+ viewFieldIds: ['view-field-1'],
+ viewFilterIds: [],
+ }),
+ ),
+ );
+
+ expect(
+ decodeFlatFieldMetadataMapsFromCache(encoded).byUniversalIdentifier,
+ ).toMatchObject({
+ 'field-uid': {
+ name: 'firstName',
+ viewFieldIds: ['view-field-1'],
+ viewFilterIds: [],
+ },
+ });
+ });
+});
diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/__tests__/encode-flat-field-metadata-maps-for-cache.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/__tests__/encode-flat-field-metadata-maps-for-cache.util.spec.ts
new file mode 100644
index 00000000000..7ca91ebc2c0
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/__tests__/encode-flat-field-metadata-maps-for-cache.util.spec.ts
@@ -0,0 +1,28 @@
+import { type EncodedFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { encodeFlatFieldMetadataMapsForCache } from 'src/engine/metadata-modules/flat-field-metadata/utils/encode-flat-field-metadata-maps-for-cache.util';
+
+const encodeOne = (
+ flatFieldMetadata: Partial<FlatFieldMetadata>,
+): EncodedFlatFieldMetadata =>
+ encodeFlatFieldMetadataMapsForCache({
+ byUniversalIdentifier: { 'field-uid': flatFieldMetadata },
+ universalIdentifierById: {},
+ universalIdentifiersByApplicationId: {},
+ }).byUniversalIdentifier['field-uid'];
+
+describe('encodeFlatFieldMetadataMapsForCache', () => {
+ it('should replace a mapped key with its short code', () => {
+ expect(encodeOne({ name: 'firstName' })).toEqual({ $r: 'firstName' });
+ });
+
+ it('should drop a relation array that is empty', () => {
+ expect(encodeOne({ viewFieldIds: [] })).toEqual({});
+ });
+
+ it('should keep a relation array that is populated', () => {
+ expect(encodeOne({ viewFieldIds: ['view-field-1'] })).toEqual({
+ $H: ['view-field-1'],
+ });
+ });
+});
diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.ts
new file mode 100644
index 00000000000..049aac12d7b
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.ts
@@ -0,0 +1,52 @@
+import {
+ FLAT_FIELD_METADATA_EMPTY_ARRAY_KEYS,
+ FLAT_FIELD_METADATA_KEY_LOOKUP,
+} from 'src/engine/metadata-modules/flat-field-metadata/constants/flat-field-metadata-cache-codec.constant';
+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
+import {
+ type EncodedFlatFieldMetadata,
+ type EncodedFlatFieldMetadataMaps,
+} from 'src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+
+const decodeFlatFieldMetadata = (
+ encoded: EncodedFlatFieldMetadata | Partial<FlatFieldMetadata>,
+): FlatFieldMetadata => {
+ const decoded: Partial<
+ Record<string, FlatFieldMetadata[keyof FlatFieldMetadata]>
+ > = {};
+
+ for (const [shortCode, value] of Object.entries(encoded)) {
+ decoded[FLAT_FIELD_METADATA_KEY_LOOKUP.get(shortCode) ?? shortCode] = value;
+ }
+
+ for (const key of FLAT_FIELD_METADATA_EMPTY_ARRAY_KEYS) {
+ if (!(key in decoded)) {
+ decoded[key] = [];
+ }
+ }
+
+ return decoded as FlatFieldMetadata;
+};
+
+export const decodeFlatFieldMetadataMapsFromCache = (
+ encoded: EncodedFlatFieldMetadataMaps | FlatEntityMaps<FlatFieldMetadata>,
+): FlatEntityMaps<FlatFieldMetadata> => {
+ const byUniversalIdentifier: FlatEntityMaps<FlatFieldMetadata>['byUniversalIdentifier'] =
+ {};
+
+ for (const [universalIdentifier, encodedFlatFieldMetadata] of Object.entries(
+ encoded.byUniversalIdentifier,
+ )) {
+ byUniversalIdentifier[universalIdentifier] = decodeFlatFieldMetadata(
+ encodedFlatFieldMetadata,
+ );
+ }
+
+ return {
+ byUniversalIdentifier,
+ universalIdentifierById: encoded.universalIdentifierById,
+ universalIdentifiersByApplicationId:
+ encoded.universalIdentifiersByApplicationId,
+ };
+};
diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/encode-flat-field-metadata-maps-for-cache.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/encode-flat-field-metadata-maps-for-cache.util.ts
new file mode 100644
index 00000000000..b6a88024f37
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/encode-flat-field-metadata-maps-for-cache.util.ts
@@ -0,0 +1,58 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import {
+ FLAT_FIELD_METADATA_EMPTY_ARRAY_KEY_SET,
+ FLAT_FIELD_METADATA_SHORT_CODE_LOOKUP,
+} from 'src/engine/metadata-modules/flat-field-metadata/constants/flat-field-metadata-cache-codec.constant';
+import {
+ type EncodableFlatFieldMetadataMaps,
+ type EncodedFlatFieldMetadata,
+ type EncodedFlatFieldMetadataMaps,
+} from 'src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+
+const encodeFlatFieldMetadata = (
+ flatFieldMetadata: Partial<FlatFieldMetadata>,
+): EncodedFlatFieldMetadata => {
+ const encoded: Partial<
+ Record<string, FlatFieldMetadata[keyof FlatFieldMetadata]>
+ > = {};
+
+ for (const [key, value] of Object.entries(flatFieldMetadata)) {
+ if (
+ FLAT_FIELD_METADATA_EMPTY_ARRAY_KEY_SET.has(key) &&
+ Array.isArray(value) &&
+ value.length === 0
+ ) {
+ continue;
+ }
+
+ encoded[FLAT_FIELD_METADATA_SHORT_CODE_LOOKUP.get(key) ?? key] = value;
+ }
+
+ return encoded;
+};
+
+export const encodeFlatFieldMetadataMapsForCache = (
+ flatEntityMaps: EncodableFlatFieldMetadataMaps,
+): EncodedFlatFieldMetadataMaps => {
+ const byUniversalIdentifier: Record<string, EncodedFlatFieldMetadata> = {};
+
+ for (const [universalIdentifier, flatFieldMetadata] of Object.entries(
+ flatEntityMaps.byUniversalIdentifier,
+ )) {
+ if (!isDefined(flatFieldMetadata)) {
+ continue;
+ }
+
+ byUniversalIdentifier[universalIdentifier] =
+ encodeFlatFieldMetadata(flatFieldMetadata);
+ }
+
+ return {
+ byUniversalIdentifier,
+ universalIdentifierById: flatEntityMaps.universalIdentifierById,
+ universalIdentifiersByApplicationId:
+ flatEntityMaps.universalIdentifiersByApplicationId,
+ };
+};
diff --git a/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts b/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts
index 3f6975d4b2a..b66e181afc3 100644
--- a/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts
@@ -252,6 +252,7 @@ describe('WorkspaceEntityManager', () => {
IS_SETTINGS_DISCOVERY_HERO_ENABLED: false,
IS_WORKFLOW_VERSION_IN_CORE_ENABLED: false,
IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED: false,
+ IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED: false,
},
userWorkspaceRoleMap: {},
apiKeyRoleMap: {},
diff --git a/packages/twenty-server/src/engine/workspace-cache/interfaces/workspace-cache-provider.service.ts b/packages/twenty-server/src/engine/workspace-cache/interfaces/workspace-cache-provider.service.ts
index 99884c1eced..38b6d4d6a92 100644
--- a/packages/twenty-server/src/engine/workspace-cache/interfaces/workspace-cache-provider.service.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/interfaces/workspace-cache-provider.service.ts
@@ -10,6 +10,15 @@ type WorkspaceCacheDataType = WorkspaceCacheDataMap[WorkspaceCacheKeyName];
@Injectable()
export abstract class WorkspaceCacheProvider<
T extends WorkspaceCacheDataType = WorkspaceCacheDataType,
+ TEncoded = T,
> {
abstract computeForCache(workspaceId: string): Promise<T>;
+
+ encodeForCacheStorage(data: T): T | TEncoded {
+ return data;
+ }
+
+ decodeFromCacheStorage(rawData: T | TEncoded): T {
+ return rawData as T;
+ }
}
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
index ec689162ab6..340af1a684a 100644
--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
@@ -16,7 +16,7 @@ import {
computeLocalCacheStats,
} from 'src/engine/workspace-cache/utils/compute-local-cache-stats.util';
import { deepSizeBytes } from 'src/engine/workspace-cache/utils/deep-size-bytes.util';
-import { getProviderFromCacheKey } from 'src/engine/workspace-cache/utils/get-provider-from-cache-key.util';
+import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';
type LocalCache = ReadonlyMap<
string,
@@ -40,7 +40,7 @@ export class WorkspaceCacheMetricsService {
private readonly redisWriteDurationHistogram: Histogram;
private localCache?: LocalCache;
- private cacheSizeByProvider: Record<string, number> = {};
+ private cacheSizeByKeyName: Record<string, number> = {};
private cacheSizeTotalBytes = 0;
private sizeSampler?: ReturnType<typeof setInterval>;
private sizeStartupTimer?: ReturnType<typeof setTimeout>;
@@ -148,14 +148,14 @@ export class WorkspaceCacheMetricsService {
return;
}
- const perProvider: Record<
+ const perKeyName: Record<
string,
{ count: number; sampledBytes: number; sampled: number }
> = {};
for (const [key, entry] of localCache) {
- const provider = getProviderFromCacheKey(key);
- const stats = (perProvider[provider] ??= {
+ const keyName = getKeyNameFromLocalCacheKey(key);
+ const stats = (perKeyName[keyName] ??= {
count: 0,
sampledBytes: 0,
sampled: 0,
@@ -168,7 +168,10 @@ export class WorkspaceCacheMetricsService {
let entryBytes = 0;
for (const version of entry.versions.values()) {
- entryBytes += deepSizeBytes(version.data, SIZE_WALK_NODE_CAP);
+ entryBytes +=
+ version.state === 'cold'
+ ? version.blob.byteLength
+ : deepSizeBytes(version.data, SIZE_WALK_NODE_CAP);
}
stats.sampledBytes += entryBytes;
@@ -177,20 +180,20 @@ export class WorkspaceCacheMetricsService {
}
}
- const byProvider: Record<string, number> = {};
+ const byKeyName: Record<string, number> = {};
let total = 0;
- for (const [provider, stats] of Object.entries(perProvider)) {
+ for (const [keyName, stats] of Object.entries(perKeyName)) {
const estimate =
stats.sampled === 0
? 0
: Math.round((stats.sampledBytes / stats.sampled) * stats.count);
- byProvider[provider] = estimate;
+ byKeyName[keyName] = estimate;
total += estimate;
}
- this.cacheSizeByProvider = byProvider;
+ this.cacheSizeByKeyName = byKeyName;
this.cacheSizeTotalBytes = total;
}
@@ -246,9 +249,9 @@ export class WorkspaceCacheMetricsService {
unit: 'By',
},
callback: async () =>
- Object.entries(this.cacheSizeByProvider).map(([provider, value]) => ({
+ Object.entries(this.cacheSizeByKeyName).map(([keyName, value]) => ({
value,
- attributes: { provider },
+ attributes: { provider: keyName },
})),
});
this.metricsService.createMultiObservableGauge({
@@ -258,10 +261,10 @@ export class WorkspaceCacheMetricsService {
'Entries in the local workspace metadata cache per provider',
},
callback: async () =>
- Object.entries(this.getStats().entriesByProvider).map(
- ([provider, value]) => ({
+ Object.entries(this.getStats().entriesByKeyName).map(
+ ([keyName, value]) => ({
value,
- attributes: { provider },
+ attributes: { provider: keyName },
}),
),
});
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
index 2c25a8e71b6..2436ed6b05d 100644
--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
@@ -9,6 +9,7 @@ import { DiscoveryService, Reflector } from '@nestjs/core';
import * as Sentry from '@sentry/node';
import crypto from 'crypto';
+import { FeatureFlagKey } from 'twenty-shared/types';
import { isDefined, isValidUuid } from 'twenty-shared/utils';
import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';
@@ -29,14 +30,18 @@ import {
WorkspaceCacheExceptionCode,
} from 'src/engine/workspace-cache/exceptions/workspace-cache.exception';
import {
- WORKSPACE_CACHE_KEYS_V2,
WorkspaceCacheKeyName,
type WorkspaceCacheDataMap,
type WorkspaceCacheResult,
type WorkspaceCacheResultWithHashes,
+ type WorkspaceCacheStoredDataMap,
} from 'src/engine/workspace-cache/types/workspace-cache-key.type';
-import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
+import {
+ type VersionEntry,
+ type WorkspaceLocalCacheEntry,
+} from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
import { combineCacheHashes } from 'src/engine/workspace-cache/utils/combine-cache-hashes.util';
+import { demoteColdStorageEntries } from 'src/engine/workspace-cache/utils/demote-cold-storage-entries.util';
import { sweepLocalCache } from 'src/engine/workspace-cache/utils/sweep-local-cache.util';
const LOCAL_TTL_MS = 100; // 100ms
@@ -48,11 +53,14 @@ const MAX_LOCAL_CACHE_ENTRIES = 6_000;
const MIN_EVICT_KEYS = 100;
const LOCAL_ENTRY_TTL_MS = 30 * 60 * 1000; // 30 minutes idle
const LOCAL_CACHE_SWEEP_INTERVAL_MS = 60 * 1000;
+const HOT_ENTRIES_PER_PROVIDER = 64;
// Per-provider entry caps, keyed by local cache key prefix (ORM graphs are ~5 MB each).
-const MAX_LOCAL_ENTRIES_BY_PREFIX = new Map<string, number>([
- [WORKSPACE_CACHE_KEYS_V2.ORMEntityMetadatas, 128],
+const MAX_LOCAL_ENTRIES_BY_KEY_NAME = new Map<string, number>([
+ ['ORMEntityMetadatas', 128],
+ ['flatFieldMetadataMaps', 256],
]);
type CacheDataType = WorkspaceCacheDataMap[WorkspaceCacheKeyName];
+type StoredCacheDataType = WorkspaceCacheStoredDataMap[WorkspaceCacheKeyName];
type CacheEntriesResult = {
data: Partial<WorkspaceCacheDataMap>;
@@ -75,7 +83,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
private lastLocalCacheSweepAt: number | undefined;
private readonly workspaceCacheProviders = new Map<
WorkspaceCacheKeyName,
- WorkspaceCacheProvider<CacheDataType>
+ WorkspaceCacheProvider<CacheDataType, StoredCacheDataType>
>();
private readonly localDataOnlyKeys = new Set<WorkspaceCacheKeyName>();
private readonly memoizer = new PromiseMemoizer<CacheEntriesResult>(
@@ -426,15 +434,31 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
return [`${baseKey}:data`, `${baseKey}:hash`];
});
- const allValues = await this.cacheStorage.mget<CacheDataType | string>(
- allKeys,
- );
+ const [allValues, isCompactStorageEnabled] = await Promise.all([
+ this.cacheStorage.mget<CacheDataType | string>(allKeys),
+ this.resolveCompactStorageEnabled(workspaceId),
+ ]);
for (const [index, keyName] of cacheKeyNames.entries()) {
- const data = allValues[index * 2] as CacheDataType | undefined;
+ const rawData = allValues[index * 2] as CacheDataType | undefined;
const hash = allValues[index * 2 + 1] as string | undefined;
- if (isDefined(data) && isDefined(hash)) {
+ if (isDefined(rawData) && isDefined(hash)) {
+ let data: CacheDataType;
+
+ try {
+ data = isCompactStorageEnabled
+ ? this.getProviderOrThrow(keyName).decodeFromCacheStorage(rawData)
+ : rawData;
+ } catch (error) {
+ this.logger.warn(
+ `Failed to decode cached ${keyName} for workspace ${workspaceId}, recomputing`,
+ error,
+ );
+ missingInRedis.push(keyName);
+ continue;
+ }
+
Object.assign(redisEntries.data, { [keyName]: data });
redisEntries.hashes[keyName] = hash;
this.setInLocalCache(workspaceId, keyName, data, hash);
@@ -499,7 +523,20 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
const computed = await Promise.all(computePromises);
- const redisEntries: Array<{ key: string; value: unknown }> = [];
+ const recomputedFeatureFlagsMap = computed.find(
+ ({ keyName }) => keyName === 'featureFlagsMap',
+ )?.data as WorkspaceCacheDataMap['featureFlagsMap'] | undefined;
+
+ const isCompactStorageEnabled = isDefined(recomputedFeatureFlagsMap)
+ ? recomputedFeatureFlagsMap[
+ FeatureFlagKey.IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED
+ ]
+ : await this.resolveCompactStorageEnabled(workspaceId);
+
+ const redisEntries: Array<{
+ key: string;
+ value: StoredCacheDataType | string;
+ }> = [];
const bootstrapHashEntries: Array<{ key: string; value: string }> = [];
for (const { keyName, data, hash, isAdopted } of computed) {
@@ -518,7 +555,12 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
}
if (!isLocalDataOnly) {
- redisEntries.push({ key: `${baseKey}:data`, value: data });
+ redisEntries.push({
+ key: `${baseKey}:data`,
+ value: isCompactStorageEnabled
+ ? this.getProviderOrThrow(keyName).encodeForCacheStorage(data)
+ : data,
+ });
}
this.setInLocalCache(workspaceId, keyName, data, hash);
@@ -562,8 +604,9 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
const version = entry?.versions.get(entry.latestHash);
if (isDefined(entry) && isDefined(version)) {
- version.lastReadAt = Date.now();
- Object.assign(result.data, { [keyName]: version.data });
+ const data = this.readVersion({ keyName, entry, version });
+
+ Object.assign(result.data, { [keyName]: data });
result.hashes[keyName] = entry.latestHash;
this.cleanupStaleVersions(entry);
}
@@ -613,7 +656,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
this.localCache.set(localKey, entry);
}
- entry.versions.set(hash, { data, lastReadAt: Date.now() });
+ entry.versions.set(hash, { state: 'hot', data, lastReadAt: Date.now() });
entry.latestHash = hash;
entry.lastHashCheckedAt = Date.now();
@@ -633,7 +676,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
const evicted = sweepLocalCache(this.localCache, now, {
ttlMs: LOCAL_ENTRY_TTL_MS,
- maxEntriesByPrefix: MAX_LOCAL_ENTRIES_BY_PREFIX,
+ maxEntriesByKeyName: MAX_LOCAL_ENTRIES_BY_KEY_NAME,
globalMaxEntries: MAX_LOCAL_CACHE_ENTRIES,
minEvict: MIN_EVICT_KEYS,
});
@@ -641,6 +684,105 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
if (evicted > 0) {
this.cacheMetricsService.recordEviction(evicted);
}
+
+ this.demoteColdEntries();
+ }
+
+ private demoteColdEntries(): void {
+ demoteColdStorageEntries({
+ localCache: this.localCache,
+ hotEntriesPerProvider: HOT_ENTRIES_PER_PROVIDER,
+ serialize: ({ localKey, data }) => {
+ const separatorIndex = localKey.lastIndexOf(':');
+ const workspaceId = localKey.slice(separatorIndex + 1);
+ const keyName = localKey.slice(
+ 0,
+ separatorIndex,
+ ) as WorkspaceCacheKeyName;
+
+ if (
+ this.localDataOnlyKeys.has(keyName) ||
+ keyName === 'featureFlagsMap' ||
+ !this.isCompactStorageEnabledInLocalCache(workspaceId)
+ ) {
+ return undefined;
+ }
+
+ return Buffer.from(
+ JSON.stringify(
+ this.getProviderOrThrow(keyName).encodeForCacheStorage(data),
+ ),
+ 'utf8',
+ );
+ },
+ });
+ }
+
+ private isCompactStorageEnabledInLocalCache(
+ workspaceId: string,
+ ): boolean | undefined {
+ const entry = this.localCache.get(
+ this.buildCacheKey(workspaceId, 'featureFlagsMap'),
+ );
+ const version = entry?.versions.get(entry.latestHash);
+
+ if (version?.state !== 'hot') {
+ return undefined;
+ }
+
+ return (version.data as WorkspaceCacheDataMap['featureFlagsMap'])[
+ FeatureFlagKey.IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED
+ ];
+ }
+
+ // A restarted pod reads an already-encoded Redis payload with an empty local
+ // cache, so the flag has to resolve from Redis before any decode happens.
+ private async resolveCompactStorageEnabled(
+ workspaceId: string,
+ ): Promise<boolean> {
+ const localValue = this.isCompactStorageEnabledInLocalCache(workspaceId);
+
+ if (isDefined(localValue)) {
+ return localValue;
+ }
+
+ const featureFlagsMap = await this.cacheStorage.get<
+ WorkspaceCacheDataMap['featureFlagsMap']
+ >(`${this.buildCacheKey(workspaceId, 'featureFlagsMap')}:data`);
+
+ return (
+ featureFlagsMap?.[
+ FeatureFlagKey.IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED
+ ] ?? false
+ );
+ }
+
+ private readVersion({
+ keyName,
+ entry,
+ version,
+ }: {
+ keyName: WorkspaceCacheKeyName;
+ entry: WorkspaceLocalCacheEntry<CacheDataType>;
+ version: VersionEntry<CacheDataType>;
+ }): CacheDataType {
+ if (version.state === 'hot') {
+ version.lastReadAt = Date.now();
+
+ return version.data;
+ }
+
+ const data = this.getProviderOrThrow(keyName).decodeFromCacheStorage(
+ JSON.parse(version.blob.toString('utf8')),
+ );
+
+ entry.versions.set(entry.latestHash, {
+ state: 'hot',
+ data,
+ lastReadAt: Date.now(),
+ });
+
+ return data;
}
private cleanupStaleVersions(
@@ -677,7 +819,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
private getProviderOrThrow(
keyName: WorkspaceCacheKeyName,
- ): WorkspaceCacheProvider<CacheDataType> {
+ ): WorkspaceCacheProvider<CacheDataType, StoredCacheDataType> {
const provider = this.workspaceCacheProviders.get(keyName);
if (!isDefined(provider)) {
@@ -691,6 +833,6 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
workspaceId: string,
keyName: WorkspaceCacheKeyName,
): string {
- return `${WORKSPACE_CACHE_KEYS_V2[keyName]}:${workspaceId}`;
+ return `${keyName}:${workspaceId}`;
}
}
diff --git a/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts b/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts
index d2499955014..60f48214029 100644
--- a/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts
@@ -4,6 +4,7 @@ import {
} from 'twenty-shared/types';
import { type EntityMetadata } from 'typeorm';
+import { type EncodedFlatFieldMetadataMaps } from 'src/engine/metadata-modules/flat-field-metadata/types/encoded-flat-field-metadata-maps.type';
import { type ResolverNameMapEntry } from 'src/engine/api/graphql/direct-execution/utils/build-resolver-name-map.util';
import { type FlatApiKey } from 'src/engine/core-modules/api-key/types/flat-api-key.type';
import { type ApplicationVariableCacheMaps } from 'src/engine/core-modules/application/application-variable/types/application-variable-cache-maps.type';
@@ -17,55 +18,6 @@ import { type UserWorkspaceRoleMap } from 'src/engine/metadata-modules/role-targ
import { type FlatRowLevelPermissionPredicateGroupMaps } from 'src/engine/metadata-modules/row-level-permission-predicate/types/flat-row-level-permission-predicate-group-maps.type';
import { type FlatRowLevelPermissionPredicateMaps } from 'src/engine/metadata-modules/row-level-permission-predicate/types/flat-row-level-permission-predicate-maps.type';
-export const WORKSPACE_CACHE_KEYS_V2 = {
- flatObjectMetadataMaps: 'flat-maps:object-metadata',
- flatFieldMetadataMaps: 'flat-maps:field-metadata',
- flatIndexMaps: 'flat-maps:index',
- flatViewMaps: 'flat-maps:view',
- flatViewFieldMaps: 'flat-maps:view-field',
- flatViewFieldGroupMaps: 'flat-maps:view-field-group',
- flatViewGroupMaps: 'flat-maps:view-group',
- flatViewFilterMaps: 'flat-maps:view-filter',
- flatViewFilterGroupMaps: 'flat-maps:view-filter-group',
- flatViewSortMaps: 'flat-maps:view-sort',
- flatSearchFieldMetadataMaps: 'flat-maps:search-field-metadata',
- flatLogicFunctionMaps: 'flat-maps:logic-function',
- featureFlagsMap: 'feature-flag:feature-flags-map',
- rolesPermissions: 'metadata:permissions:roles-permissions',
- userWorkspaceRoleMap: 'metadata:permissions:user-workspace-role-map',
- apiKeyRoleMap: 'metadata:permissions:api-key-role-map',
- flatApplicationMaps: 'flat-maps:flatApplicationMaps',
- flatRoleMaps: 'flat-maps:role',
- flatRoleTargetMaps: 'flat-maps:role-target',
- flatRolePermissionFlagMaps: 'flat-maps:role-permission-flag',
- flatPermissionFlagMaps: 'flat-maps:permission-flag-catalog',
- flatObjectPermissionMaps: 'flat-maps:object-permission',
- flatFieldPermissionMaps: 'flat-maps:field-permission',
- ORMEntityMetadatas: 'orm:entity-metadatas',
- flatAgentMaps: 'flat-maps:agent',
- flatSkillMaps: 'flat-maps:skill',
- flatCommandMenuItemMaps: 'flat-maps:command-menu-item',
- flatNavigationMenuItemMaps: 'flat-maps:navigation-menu-item',
- flatRoleTargetByAgentIdMaps: 'flat-maps:flatRoleTargetByAgentId',
- flatPageLayoutMaps: 'flat-maps:page-layout',
- flatPageLayoutWidgetMaps: 'flat-maps:page-layout-widget',
- flatPageLayoutTabMaps: 'flat-maps:flatPageLayoutTabMaps',
- flatRowLevelPermissionPredicateMaps:
- 'flat-maps:row-level-permission-predicate',
- flatRowLevelPermissionPredicateGroupMaps:
- 'flat-maps:row-level-permission-predicate-group',
- flatFrontComponentMaps: 'flat-maps:front-component',
- flatWebhookMaps: 'flat-maps:webhook',
- flatApplicationVariableMaps: 'flat-maps:application-variable',
- flatConnectionProviderMaps: 'flat-maps:connection-provider',
- flatWorkspaceMemberMaps: 'flat-maps:workspace-member',
- apiKeyMap: 'cache:api-key-map',
- applicationVariableMaps: 'cache:application-variable',
- graphQLResolverNameMap: 'direct-execution:graphql-resolver-name-map',
- currentBillingSubscription: 'billing:subscription',
- workflowAutomatedTriggerMaps: 'cache:workflow-automated-trigger',
-} as const satisfies Record<WorkspaceCacheKeyName, string>;
-
export type AdditionalCacheDataMaps = {
featureFlagsMap: Record<FeatureFlagKey, boolean>;
rolesPermissions: ObjectsPermissionsByRoleId;
@@ -98,3 +50,12 @@ export type WorkspaceCacheResultWithHashes<K extends WorkspaceCacheKeyName[]> =
data: WorkspaceCacheResult<K>;
hashes: { [P in K[number]]: string };
};
+
+export type WorkspaceCacheStoredDataMap = Omit<
+ WorkspaceCacheDataMap,
+ 'flatFieldMetadataMaps'
+> & {
+ flatFieldMetadataMaps:
+ | WorkspaceCacheDataMap['flatFieldMetadataMaps']
+ | EncodedFlatFieldMetadataMaps;
+};
diff --git a/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts b/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts
index 3db32ab0d41..9bade6606a3 100644
--- a/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts
@@ -1,8 +1,17 @@
-export type VersionEntry<T> = {
+export type HotVersionEntry<T> = {
+ state: 'hot';
data: T;
lastReadAt: number;
};
+export type ColdVersionEntry = {
+ state: 'cold';
+ blob: Buffer;
+ lastReadAt: number;
+};
+
+export type VersionEntry<T> = HotVersionEntry<T> | ColdVersionEntry;
+
export type WorkspaceLocalCacheEntry<T> = {
versions: Map<string, VersionEntry<T>>;
latestHash: string;
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts
index dbeb4ecef11..a7c5e5dfb72 100644
--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts
@@ -11,16 +11,16 @@ describe('computeLocalCacheStats', () => {
workspaces: 0,
versionsTotal: 0,
versionsByCount: { '1': 0, '2': 0, '3': 0, '4': 0, '5+': 0 },
- entriesByProvider: {},
+ entriesByKeyName: {},
});
});
it('counts entries, distinct workspaces, and total versions', () => {
const stats = computeLocalCacheStats(
new Map([
- ['orm:entity-metadatas:ws-a', entry(1)],
- ['flat-maps:field-metadata:ws-a', entry(2)],
- ['orm:entity-metadatas:ws-b', entry(1)],
+ ['ORMEntityMetadatas:ws-a', entry(1)],
+ ['flatFieldMetadataMaps:ws-a', entry(2)],
+ ['ORMEntityMetadatas:ws-b', entry(1)],
]),
);
@@ -28,9 +28,9 @@ describe('computeLocalCacheStats', () => {
// ws-a and ws-b — the same workspace under two providers counts once.
expect(stats.workspaces).toBe(2);
expect(stats.versionsTotal).toBe(4);
- expect(stats.entriesByProvider).toEqual({
- 'orm:entity-metadatas': 2,
- 'flat-maps:field-metadata': 1,
+ expect(stats.entriesByKeyName).toEqual({
+ ORMEntityMetadatas: 2,
+ flatFieldMetadataMaps: 1,
});
});
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts
new file mode 100644
index 00000000000..1c139fba4ea
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts
@@ -0,0 +1,111 @@
+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
+import { demoteColdStorageEntries } from 'src/engine/workspace-cache/utils/demote-cold-storage-entries.util';
+
+const FIELD_METADATA = 'flatFieldMetadataMaps';
+const ORM = 'ORMEntityMetadatas';
+
+const hotEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({
+ versions: new Map([['hash-1', { state: 'hot', data: 'data', lastReadAt }]]),
+ latestHash: 'hash-1',
+ lastHashCheckedAt: lastReadAt,
+});
+
+const coldEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({
+ versions: new Map([
+ ['hash-1', { state: 'cold', blob: Buffer.from('{}'), lastReadAt }],
+ ]),
+ latestHash: 'hash-1',
+ lastHashCheckedAt: lastReadAt,
+});
+
+const serialize = () => Buffer.from('serialized');
+
+const demote = (
+ localCache: Map<string, WorkspaceLocalCacheEntry<string>>,
+ hotEntriesPerProvider: number,
+ overrides?: { serialize?: () => Buffer | undefined },
+) =>
+ demoteColdStorageEntries({
+ localCache,
+ hotEntriesPerProvider,
+ serialize: overrides?.serialize ?? serialize,
+ });
+
+describe('demoteColdStorageEntries', () => {
+ it('should demote nothing when the provider is within its hot budget', () => {
+ const localCache = new Map([
+ [`${FIELD_METADATA}:ws-a`, hotEntry(1)],
+ [`${FIELD_METADATA}:ws-b`, hotEntry(2)],
+ ]);
+
+ expect(demote(localCache, 2)).toBe(0);
+ expect(
+ localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),
+ ).toMatchObject({ state: 'hot' });
+ });
+
+ it('should demote the least recently read entries beyond the hot budget', () => {
+ const localCache = new Map([
+ [`${FIELD_METADATA}:ws-newest`, hotEntry(9)],
+ [`${FIELD_METADATA}:ws-oldest`, hotEntry(1)],
+ [`${FIELD_METADATA}:ws-middle`, hotEntry(5)],
+ ]);
+
+ expect(demote(localCache, 1)).toBe(2);
+ expect(
+ localCache.get(`${FIELD_METADATA}:ws-oldest`)?.versions.get('hash-1'),
+ ).toMatchObject({ state: 'cold', blob: Buffer.from('serialized') });
+ expect(
+ localCache.get(`${FIELD_METADATA}:ws-newest`)?.versions.get('hash-1'),
+ ).toMatchObject({ state: 'hot' });
+ });
+
+ it('should preserve lastReadAt so a demoted entry keeps its place in the eviction order', () => {
+ const localCache = new Map([
+ [`${FIELD_METADATA}:ws-a`, hotEntry(1)],
+ [`${FIELD_METADATA}:ws-b`, hotEntry(2)],
+ ]);
+
+ demote(localCache, 1);
+
+ expect(
+ localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),
+ ).toMatchObject({ lastReadAt: 1 });
+ });
+
+ it('should leave the entry hot when serialize declines it', () => {
+ const localCache = new Map([
+ [`${FIELD_METADATA}:ws-a`, hotEntry(1)],
+ [`${FIELD_METADATA}:ws-b`, hotEntry(2)],
+ ]);
+
+ expect(demote(localCache, 1, { serialize: () => undefined })).toBe(0);
+ expect(
+ localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),
+ ).toMatchObject({ state: 'hot' });
+ });
+
+ it('should budget each provider independently', () => {
+ const localCache = new Map([
+ [`${FIELD_METADATA}:ws-a`, hotEntry(1)],
+ [`${FIELD_METADATA}:ws-b`, hotEntry(2)],
+ [`${ORM}:ws-a`, hotEntry(3)],
+ ]);
+
+ expect(demote(localCache, 1)).toBe(1);
+ expect(localCache.get(`${ORM}:ws-a`)?.versions.get('hash-1')).toMatchObject(
+ {
+ state: 'hot',
+ },
+ );
+ });
+
+ it('should not count an already cold entry against the hot budget', () => {
+ const localCache = new Map([
+ [`${FIELD_METADATA}:ws-cold`, coldEntry(1)],
+ [`${FIELD_METADATA}:ws-hot`, hotEntry(2)],
+ ]);
+
+ expect(demote(localCache, 1)).toBe(0);
+ });
+});
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts
index 0c42624e930..205f59dd963 100644
--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts
@@ -5,10 +5,10 @@ import {
} from 'src/engine/workspace-cache/utils/sweep-local-cache.util';
const TTL_MS = 30 * 60 * 1000;
-const ORM_PREFIX = 'orm:entity-metadatas';
+const ORM_KEY_NAME = 'ORMEntityMetadatas';
const entry = (lastReadAt: number): WorkspaceLocalCacheEntry<unknown> => ({
- versions: new Map([['h', { data: {}, lastReadAt }]]),
+ versions: new Map([['h', { state: 'hot', data: {}, lastReadAt }]]),
latestHash: 'h',
lastHashCheckedAt: lastReadAt,
});
@@ -17,7 +17,7 @@ const config = (
overrides: Partial<LocalCacheSweepConfig> = {},
): LocalCacheSweepConfig => ({
ttlMs: TTL_MS,
- maxEntriesByPrefix: new Map(),
+ maxEntriesByKeyName: new Map(),
globalMaxEntries: 6_000,
minEvict: 0,
...overrides,
@@ -26,34 +26,34 @@ const config = (
describe('sweepLocalCache', () => {
it('drops entries idle past the TTL and keeps recent ones', () => {
const cache = new Map([
- [`${ORM_PREFIX}:ws-idle`, entry(0)],
- [`${ORM_PREFIX}:ws-recent`, entry(TTL_MS)],
+ [`${ORM_KEY_NAME}:ws-idle`, entry(0)],
+ [`${ORM_KEY_NAME}:ws-recent`, entry(TTL_MS)],
]);
const evicted = sweepLocalCache(cache, TTL_MS + 1, config());
expect(evicted).toBe(1);
- expect(cache.has(`${ORM_PREFIX}:ws-idle`)).toBe(false);
- expect(cache.has(`${ORM_PREFIX}:ws-recent`)).toBe(true);
+ expect(cache.has(`${ORM_KEY_NAME}:ws-idle`)).toBe(false);
+ expect(cache.has(`${ORM_KEY_NAME}:ws-recent`)).toBe(true);
});
it('trims a provider over its cap, evicting the least-recently-read', () => {
const cache = new Map([
- [`${ORM_PREFIX}:ws-a`, entry(1)],
- [`${ORM_PREFIX}:ws-b`, entry(2)],
- [`${ORM_PREFIX}:ws-c`, entry(3)],
+ [`${ORM_KEY_NAME}:ws-a`, entry(1)],
+ [`${ORM_KEY_NAME}:ws-b`, entry(2)],
+ [`${ORM_KEY_NAME}:ws-c`, entry(3)],
]);
const evicted = sweepLocalCache(
cache,
100,
- config({ maxEntriesByPrefix: new Map([[ORM_PREFIX, 2]]) }),
+ config({ maxEntriesByKeyName: new Map([[ORM_KEY_NAME, 2]]) }),
);
expect(evicted).toBe(1);
expect(cache.size).toBe(2);
- expect(cache.has(`${ORM_PREFIX}:ws-a`)).toBe(false); // oldest
- expect(cache.has(`${ORM_PREFIX}:ws-c`)).toBe(true); // newest
+ expect(cache.has(`${ORM_KEY_NAME}:ws-a`)).toBe(false); // oldest
+ expect(cache.has(`${ORM_KEY_NAME}:ws-c`)).toBe(true); // newest
});
it('leaves a provider without a cap untouched', () => {
@@ -67,7 +67,7 @@ describe('sweepLocalCache', () => {
const evicted = sweepLocalCache(
cache,
100,
- config({ maxEntriesByPrefix: new Map([[ORM_PREFIX, 2]]) }),
+ config({ maxEntriesByKeyName: new Map([[ORM_KEY_NAME, 2]]) }),
);
expect(evicted).toBe(0);
@@ -97,15 +97,15 @@ describe('sweepLocalCache', () => {
it('expires first so the per-provider cap only counts survivors', () => {
const cache = new Map([
- [`${ORM_PREFIX}:ws-idle`, entry(0)], // expired by TTL
- [`${ORM_PREFIX}:ws-a`, entry(TTL_MS + 1)],
- [`${ORM_PREFIX}:ws-b`, entry(TTL_MS + 2)],
+ [`${ORM_KEY_NAME}:ws-idle`, entry(0)], // expired by TTL
+ [`${ORM_KEY_NAME}:ws-a`, entry(TTL_MS + 1)],
+ [`${ORM_KEY_NAME}:ws-b`, entry(TTL_MS + 2)],
]);
const evicted = sweepLocalCache(
cache,
TTL_MS + 10,
- config({ maxEntriesByPrefix: new Map([[ORM_PREFIX, 2]]) }),
+ config({ maxEntriesByKeyName: new Map([[ORM_KEY_NAME, 2]]) }),
);
// The idle one is expired; the two survivors are within the cap, so nothing else goes.
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts
index e61b5296989..71a07ad1442 100644
--- a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts
@@ -1,11 +1,11 @@
-import { getProviderFromCacheKey } from 'src/engine/workspace-cache/utils/get-provider-from-cache-key.util';
+import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';
export type LocalCacheStats = {
entries: number;
workspaces: number;
versionsTotal: number;
versionsByCount: Record<string, number>;
- entriesByProvider: Record<string, number>;
+ entriesByKeyName: Record<string, number>;
};
export const computeLocalCacheStats = (
@@ -19,14 +19,14 @@ export const computeLocalCacheStats = (
'4': 0,
'5+': 0,
};
- const entriesByProvider: Record<string, number> = {};
+ const entriesByKeyName: Record<string, number> = {};
let versionsTotal = 0;
for (const [key, entry] of localCache) {
workspaceIds.add(key.slice(key.lastIndexOf(':') + 1));
- const provider = getProviderFromCacheKey(key);
+ const keyName = getKeyNameFromLocalCacheKey(key);
- entriesByProvider[provider] = (entriesByProvider[provider] ?? 0) + 1;
+ entriesByKeyName[keyName] = (entriesByKeyName[keyName] ?? 0) + 1;
const versionCount = entry.versions.size;
versionsTotal += versionCount;
@@ -40,6 +40,6 @@ export const computeLocalCacheStats = (
workspaces: workspaceIds.size,
versionsTotal,
versionsByCount,
- entriesByProvider,
+ entriesByKeyName,
};
};
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts
new file mode 100644
index 00000000000..eb565c5dfbd
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts
@@ -0,0 +1,68 @@
+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
+import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';
+
+export const demoteColdStorageEntries = <T>({
+ localCache,
+ hotEntriesPerProvider,
+ serialize,
+}: {
+ localCache: ReadonlyMap<string, WorkspaceLocalCacheEntry<T>>;
+ hotEntriesPerProvider: number;
+ serialize: (params: { localKey: string; data: T }) => Buffer | undefined;
+}): number => {
+ const hotByProvider = new Map<
+ string,
+ { localKey: string; hash: string; lastReadAt: number }[]
+ >();
+
+ for (const [localKey, entry] of localCache) {
+ for (const [hash, version] of entry.versions) {
+ if (version.state !== 'hot') {
+ continue;
+ }
+
+ const provider = getKeyNameFromLocalCacheKey(localKey);
+ const hot = hotByProvider.get(provider) ?? [];
+
+ hot.push({ localKey, hash, lastReadAt: version.lastReadAt });
+ hotByProvider.set(provider, hot);
+ }
+ }
+
+ let demoted = 0;
+
+ for (const hot of hotByProvider.values()) {
+ if (hot.length <= hotEntriesPerProvider) {
+ continue;
+ }
+
+ hot.sort((a, b) => a.lastReadAt - b.lastReadAt);
+
+ for (const { localKey, hash } of hot.slice(
+ 0,
+ hot.length - hotEntriesPerProvider,
+ )) {
+ const entry = localCache.get(localKey);
+ const version = entry?.versions.get(hash);
+
+ if (!entry || version?.state !== 'hot') {
+ continue;
+ }
+
+ const blob = serialize({ localKey, data: version.data });
+
+ if (blob === undefined) {
+ continue;
+ }
+
+ entry.versions.set(hash, {
+ state: 'cold',
+ blob,
+ lastReadAt: version.lastReadAt,
+ });
+ demoted += 1;
+ }
+ }
+
+ return demoted;
+};
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util.ts
new file mode 100644
index 00000000000..d43767f3a9b
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util.ts
@@ -0,0 +1,2 @@
+export const getKeyNameFromLocalCacheKey = (localCacheKey: string): string =>
+ localCacheKey.slice(0, localCacheKey.lastIndexOf(':'));
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/get-provider-from-cache-key.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/get-provider-from-cache-key.util.ts
deleted file mode 100644
index 3a6f48993c8..00000000000
--- a/packages/twenty-server/src/engine/workspace-cache/utils/get-provider-from-cache-key.util.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-export const getProviderFromCacheKey = (cacheKey: string): string =>
- cacheKey.slice(0, cacheKey.lastIndexOf(':'));
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.ts
index d6b7b87ee4f..9cb7fccb585 100644
--- a/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.ts
@@ -2,7 +2,7 @@ import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/
export type LocalCacheSweepConfig = {
ttlMs: number;
- maxEntriesByPrefix: ReadonlyMap<string, number>;
+ maxEntriesByKeyName: ReadonlyMap<string, number>;
globalMaxEntries: number;
minEvict: number;
};
@@ -70,10 +70,10 @@ export const sweepLocalCache = <T>(
}
}
- for (const [prefix, maxEntries] of config.maxEntriesByPrefix) {
+ for (const [keyName, maxEntries] of config.maxEntriesByKeyName) {
evicted += evictLeastRecentlyRead({
localCache,
- matches: (key) => key.startsWith(`${prefix}:`),
+ matches: (key) => key.startsWith(`${keyName}:`),
maxEntries,
minEvict: 0,
});
diff --git a/packages/twenty-shared/src/types/FeatureFlagKey.ts b/packages/twenty-shared/src/types/FeatureFlagKey.ts
index d9beb16545c..92a89ee1fb3 100644
--- a/packages/twenty-shared/src/types/FeatureFlagKey.ts
+++ b/packages/twenty-shared/src/types/FeatureFlagKey.ts
@@ -12,4 +12,5 @@ export enum FeatureFlagKey {
IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',
IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED',
IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED = 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED',
+ IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED = 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED',
}