The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
perf(server): remove the compact-storage flag, slice cache demotion, use a plain LIMIT on find-many (#23980)
Charles Bochet9d5edecefdcf
diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql b/packages/twenty-client-sdk/src/metadata/generated/schema.graphqlindex 47e24571e66..f4512620618 100644--- a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.graphql@@ -1738,7 +1738,6 @@ enum FeatureFlagKey {IS_SETTINGS_DISCOVERY_HERO_ENABLEDIS_WORKFLOW_VERSION_IN_CORE_ENABLEDIS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED- IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLEDIS_NATIVE_CALL_RECORDING_TABS_ENABLED}diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.ts b/packages/twenty-client-sdk/src/metadata/generated/schema.tsindex ab55861b711..527b490e83a 100644--- a/packages/twenty-client-sdk/src/metadata/generated/schema.ts+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.ts@@ -1395,7 +1395,7 @@ export interface FeatureFlag {__typename: 'FeatureFlag'}-export type FeatureFlagKey = 'IS_APP_CLAIMING_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_CALENDAR_WEEK_VIEW_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_LIST_VIEW_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' | 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED' | 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED' | 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED' | 'IS_NATIVE_CALL_RECORDING_TABS_ENABLED'+export type FeatureFlagKey = 'IS_APP_CLAIMING_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_CALENDAR_WEEK_VIEW_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_LIST_VIEW_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' | 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED' | 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED' | 'IS_NATIVE_CALL_RECORDING_TABS_ENABLED'export interface WorkspaceUrls {customUrl?: Scalars['String']@@ -9653,7 +9653,6 @@ export const enumFeatureFlagKey = {IS_SETTINGS_DISCOVERY_HERO_ENABLED: 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' as const,IS_WORKFLOW_VERSION_IN_CORE_ENABLED: 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED' as const,IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED: 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED' as const,- IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED: 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED' as const,IS_NATIVE_CALL_RECORDING_TABS_ENABLED: 'IS_NATIVE_CALL_RECORDING_TABS_ENABLED' as const}diff --git a/packages/twenty-front/src/generated-admin/graphql.ts b/packages/twenty-front/src/generated-admin/graphql.tsindex 3f4e1637e83..f256de0368a 100644--- a/packages/twenty-front/src/generated-admin/graphql.ts+++ b/packages/twenty-front/src/generated-admin/graphql.ts@@ -380,8 +380,7 @@ export enum FeatureFlagKey {IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',IS_UNIQUE_INDEXES_ENABLED = 'IS_UNIQUE_INDEXES_ENABLED',IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED = 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED',- IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED',- IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED = 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED'+ IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED'}export type FindApplicationRegistrationInstalledWorkspacesInput = {diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.tsindex 32417f0d9a9..0e4f46e684d 100644--- a/packages/twenty-front/src/generated-metadata/graphql.ts+++ b/packages/twenty-front/src/generated-metadata/graphql.ts@@ -1824,8 +1824,7 @@ export enum FeatureFlagKey {IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',IS_UNIQUE_INDEXES_ENABLED = 'IS_UNIQUE_INDEXES_ENABLED',IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED = 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED',- IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED',- IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED = 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED'+ IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED'}export type Field = {diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.tsindex 66ae1e5f985..e87c325dec5 100644--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts@@ -1,5 +1,7 @@import { Injectable } from '@nestjs/common';+import { msg } from '@lingui/core/macro';+import { isDefined } from 'class-validator';import {QUERY_MAX_RECORDS,@@ -30,6 +32,7 @@ import {} from 'src/engine/api/common/types/common-query-args.type';import { CommonSelectedFieldsResult } from 'src/engine/api/common/types/common-selected-fields-result.type';import { buildCursorPage } from 'src/engine/api/utils/build-cursor-page.util';+import { getNonToOneJoinAliases } from 'src/engine/api/common/utils/get-non-to-one-join-aliases.util';import { getPageInfo } from 'src/engine/api/common/utils/get-page-info.util';import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper';import { buildColumnsToSelect } from 'src/engine/api/graphql/graphql-query-runner/utils/build-columns-to-select';@@ -163,12 +166,27 @@ export class CommonFindManyQueryRunnerService extends CommonBaseQueryRunnerServiflatFieldMetadataMaps,});- if (isDefined(args.offset)) {- queryBuilder.skip(args.offset);+ queryBuilder.setFindOptions({ select: columnsToSelect });++ // A join that can duplicate root rows makes a row-level LIMIT return fewer records than+ // asked, so it is rejected rather than paginated with take/skip, which drops the LIMIT+ // from the scan.+ const nonToOneJoinAliases = getNonToOneJoinAliases(queryBuilder);++ if (nonToOneJoinAliases.length > 0) {+ throw new CommonQueryRunnerException(+ `Cannot filter or order through ${nonToOneJoinAliases.join(', ')}: only to-one relations are supported`,+ CommonQueryRunnerExceptionCode.INVALID_QUERY_INPUT,+ {+ userFriendlyMessage: msg`Filtering or ordering through this relation is not supported.`,+ },+ );}- queryBuilder.setFindOptions({ select: columnsToSelect });- queryBuilder.take(limit + 1);+ if (isDefined(args.offset)) {+ queryBuilder.offset(args.offset);+ }+ queryBuilder.limit(limit + 1);// Add order columns AFTER setFindOptions (setFindOptions clears addSelect)// Pass columnsToSelect so we only add columns that aren't already selecteddiff --git a/packages/twenty-server/src/engine/api/common/utils/get-non-to-one-join-aliases.util.ts b/packages/twenty-server/src/engine/api/common/utils/get-non-to-one-join-aliases.util.tsnew file mode 100644index 00000000000..2a7ccb58252--- /dev/null+++ b/packages/twenty-server/src/engine/api/common/utils/get-non-to-one-join-aliases.util.ts@@ -0,0 +1,23 @@+import { type Alias } from 'typeorm/query-builder/Alias';+import { type RelationMetadata } from 'typeorm/metadata/RelationMetadata';++import { isDefined } from 'twenty-shared/utils';++type QueryBuilderWithJoinAttributes = {+ expressionMap: {+ joinAttributes: {+ alias: Pick<Alias, 'name'>;+ relation?: Pick<RelationMetadata, 'isOneToMany' | 'isManyToMany'>;+ }[];+ };+};++export const getNonToOneJoinAliases = (+ queryBuilder: QueryBuilderWithJoinAttributes,+): string[] =>+ queryBuilder.expressionMap.joinAttributes+ .filter(+ ({ relation }) =>+ !isDefined(relation) || relation.isOneToMany || relation.isManyToMany,+ )+ .map(({ alias }) => alias.name);diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.tsindex 8aa0d5b1ca2..5a44aa383a0 100644--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts@@ -72,6 +72,7 @@ export enum MetricsKeys {AiChatTurnCompleted = 'ai-chat/turn-completed',AiChatTurnFailed = 'ai-chat/turn-failed',WorkspaceMetadataCacheLocalEviction = 'workspace-metadata-cache/local-eviction',+ WorkspaceMetadataCacheDemoted = 'workspace-metadata-cache/demoted',SdkClientGenerationSucceeded = 'sdk-client-generation/succeeded',SdkClientGenerationFailed = 'sdk-client-generation/failed',SdkClientGenerationDurationMs = 'sdk-client-generation/duration-ms',diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.tsindex 049aac12d7b..24fb04607e2 100644--- a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.ts@@ -30,7 +30,7 @@ const decodeFlatFieldMetadata = (};export const decodeFlatFieldMetadataMapsFromCache = (- encoded: EncodedFlatFieldMetadataMaps | FlatEntityMaps<FlatFieldMetadata>,+ encoded: EncodedFlatFieldMetadataMaps,): FlatEntityMaps<FlatFieldMetadata> => {const byUniversalIdentifier: FlatEntityMaps<FlatFieldMetadata>['byUniversalIdentifier'] ={};diff --git a/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts b/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.tsindex c3e7a368af1..84397d9e241 100644--- a/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts+++ b/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts@@ -251,7 +251,6 @@ describe('WorkspaceEntityManager', () => {IS_SETTINGS_DISCOVERY_HERO_ENABLED: false,IS_WORKFLOW_VERSION_IN_CORE_ENABLED: false,IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED: false,- IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED: false,IS_NATIVE_CALL_RECORDING_TABS_ENABLED: false,},userWorkspaceRoleMap: {},diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.tsindex 340af1a684a..4153044c3c3 100644--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts@@ -38,6 +38,8 @@ export class WorkspaceCacheMetricsService {private readonly logger = new Logger(WorkspaceCacheMetricsService.name);private readonly recomputeDurationHistogram: Histogram;private readonly redisWriteDurationHistogram: Histogram;+ private readonly demotionDurationHistogram: Histogram;+ private readonly hydrationDurationHistogram: Histogram;private localCache?: LocalCache;private cacheSizeByKeyName: Record<string, number> = {};@@ -46,6 +48,7 @@ export class WorkspaceCacheMetricsService {private sizeStartupTimer?: ReturnType<typeof setTimeout>;private statsCache?: { computedAt: number } & LocalCacheStats;private sizeSampleInFlight = false;+ private demotionBacklog = 0;constructor(private readonly metricsService: MetricsService) {const meter = this.metricsService.getMeter();@@ -68,6 +71,24 @@ export class WorkspaceCacheMetricsService {advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },},);+ this.demotionDurationHistogram = meter.createHistogram(+ 'twenty_workspace_cache_demotion_slice_duration_seconds',+ {+ description:+ 'Event loop time consumed by one cold-storage demotion slice',+ unit: 's',+ advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },+ },+ );+ this.hydrationDurationHistogram = meter.createHistogram(+ 'twenty_workspace_cache_hydration_duration_seconds',+ {+ description:+ 'Wall-clock time to hydrate one cold cache entry back into objects on read',+ unit: 's',+ advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },+ },+ );}start(localCache: LocalCache): void {@@ -93,6 +114,32 @@ export class WorkspaceCacheMetricsService {this.redisWriteDurationHistogram.record(seconds);}+ recordDemotionSlice({+ durationSeconds,+ demoted,+ remaining,+ }: {+ durationSeconds: number;+ demoted: number;+ remaining: number;+ }): void {+ this.demotionBacklog = remaining;++ if (demoted === 0) {+ return;+ }++ this.demotionDurationHistogram.record(durationSeconds);+ this.metricsService.incrementCounterBy({+ key: MetricsKeys.WorkspaceMetadataCacheDemoted,+ amount: demoted,+ });+ }++ recordHydration(seconds: number, cacheKey: WorkspaceCacheKeyName): void {+ this.hydrationDurationHistogram.record(seconds, { cache_key: cacheKey });+ }+recordEviction(amount: number): void {this.metricsService.incrementCounterBy({key: MetricsKeys.WorkspaceMetadataCacheLocalEviction,@@ -268,5 +315,63 @@ export class WorkspaceCacheMetricsService {}),),});+ this.metricsService.createMultiObservableGauge({+ metricName: 'twenty_workspace_cache_local_versions_by_state',+ options: {+ description:+ 'Local workspace metadata cache versions by storage state and provider',+ },+ callback: async () => {+ const stats = this.getStats();++ return [+ ...Object.entries(stats.hotVersionsByKeyName).map(+ ([keyName, value]) => ({+ value,+ attributes: { provider: keyName, state: 'hot' },+ }),+ ),+ ...Object.entries(stats.coldVersionsByKeyName).map(+ ([keyName, value]) => ({+ value,+ attributes: { provider: keyName, state: 'cold' },+ }),+ ),+ ];+ },+ });+ this.metricsService.createObservableGauge({+ metricName: 'twenty_workspace_cache_local_hot_versions',+ options: {+ description:+ 'Total hot (live object) versions in the local workspace metadata cache',+ },+ callback: async () => this.getStats().hotVersionsTotal,+ });+ this.metricsService.createObservableGauge({+ metricName: 'twenty_workspace_cache_local_cold_versions',+ options: {+ description:+ 'Total cold (serialized buffer) versions in the local workspace metadata cache',+ },+ callback: async () => this.getStats().coldVersionsTotal,+ });+ this.metricsService.createObservableGauge({+ metricName: 'twenty_workspace_cache_local_cold_bytes',+ options: {+ description:+ 'Exact retained bytes held as cold buffers in the local workspace metadata cache',+ unit: 'By',+ },+ callback: async () => this.getStats().coldBytesTotal,+ });+ this.metricsService.createObservableGauge({+ metricName: 'twenty_workspace_cache_demotion_backlog',+ options: {+ description:+ 'Entries still awaiting demotion when the last demotion slice ran out of budget',+ },+ callback: async () => this.demotionBacklog,+ });}}diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.tsindex 2436ed6b05d..ae6ee098608 100644--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts@@ -9,7 +9,6 @@ import { DiscoveryService, Reflector } from '@nestjs/core';import * as Sentry from '@sentry/node';import crypto from 'crypto';-import { FeatureFlagKey } from 'twenty-shared/types';import { isDefined, isValidUuid } from 'twenty-shared/utils';import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';@@ -54,6 +53,8 @@ const MIN_EVICT_KEYS = 100;const LOCAL_ENTRY_TTL_MS = 30 * 60 * 1000; // 30 minutes idleconst LOCAL_CACHE_SWEEP_INTERVAL_MS = 60 * 1000;const HOT_ENTRIES_PER_PROVIDER = 64;+const DEMOTION_INTERVAL_MS = 250;+const DEMOTION_BUDGET_MS = 10;// Per-provider entry caps, keyed by local cache key prefix (ORM graphs are ~5 MB each).const MAX_LOCAL_ENTRIES_BY_KEY_NAME = new Map<string, number>([['ORMEntityMetadatas', 128],@@ -80,7 +81,8 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {string,WorkspaceLocalCacheEntry<CacheDataType>>();- private lastLocalCacheSweepAt: number | undefined;+ private sweepTimer?: ReturnType<typeof setInterval>;+ private demotionTimer?: ReturnType<typeof setInterval>;private readonly workspaceCacheProviders = new Map<WorkspaceCacheKeyName,WorkspaceCacheProvider<CacheDataType, StoredCacheDataType>@@ -135,12 +137,33 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {}this.cacheMetricsService.start(this.localCache);+ this.startMaintenanceTimers();}onModuleDestroy(): void {+ if (isDefined(this.sweepTimer)) {+ clearInterval(this.sweepTimer);+ }+ if (isDefined(this.demotionTimer)) {+ clearInterval(this.demotionTimer);+ }this.cacheMetricsService.stop();}+ private startMaintenanceTimers(): void {+ this.sweepTimer = setInterval(+ () => this.sweepLocalCache(),+ LOCAL_CACHE_SWEEP_INTERVAL_MS,+ );+ this.sweepTimer.unref();++ this.demotionTimer = setInterval(+ () => this.demoteColdEntries(),+ DEMOTION_INTERVAL_MS,+ );+ this.demotionTimer.unref();+ }+public async getOrRecompute<const K extends WorkspaceCacheKeyName[]>(workspaceId: string,cacheKeyNames: K,@@ -159,7 +182,6 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {workspaceId: string,cacheKeyNames: K,): Promise<WorkspaceCacheResultWithHashes<K>> {- this.sweepLocalCacheIfDue();this.assertValidCacheParameters(workspaceId, cacheKeyNames);const memoKey =@@ -270,10 +292,6 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {strategy: 'mint',});- // Invalidation writes entries too, so run the sweep here as well — otherwise an- // invalidate-only workload never enforces the per-provider caps.- this.sweepLocalCacheIfDue();-// Clear memoizer again after recomputation to evict any stale entries// cached by concurrent getOrRecompute calls during the flush window.await this.memoizer.clearKeys(`${workspaceId}-`);@@ -434,10 +452,9 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {return [`${baseKey}:data`, `${baseKey}:hash`];});- const [allValues, isCompactStorageEnabled] = await Promise.all([- this.cacheStorage.mget<CacheDataType | string>(allKeys),- this.resolveCompactStorageEnabled(workspaceId),- ]);+ const allValues = await this.cacheStorage.mget<CacheDataType | string>(+ allKeys,+ );for (const [index, keyName] of cacheKeyNames.entries()) {const rawData = allValues[index * 2] as CacheDataType | undefined;@@ -447,9 +464,8 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {let data: CacheDataType;try {- data = isCompactStorageEnabled- ? this.getProviderOrThrow(keyName).decodeFromCacheStorage(rawData)- : rawData;+ data =+ this.getProviderOrThrow(keyName).decodeFromCacheStorage(rawData);} catch (error) {this.logger.warn(`Failed to decode cached ${keyName} for workspace ${workspaceId}, recomputing`,@@ -523,16 +539,6 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {const computed = await Promise.all(computePromises);- const recomputedFeatureFlagsMap = computed.find(- ({ keyName }) => keyName === 'featureFlagsMap',- )?.data as WorkspaceCacheDataMap['featureFlagsMap'] | undefined;-- const isCompactStorageEnabled = isDefined(recomputedFeatureFlagsMap)- ? recomputedFeatureFlagsMap[- FeatureFlagKey.IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED- ]- : await this.resolveCompactStorageEnabled(workspaceId);-const redisEntries: Array<{key: string;value: StoredCacheDataType | string;@@ -557,9 +563,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {if (!isLocalDataOnly) {redisEntries.push({key: `${baseKey}:data`,- value: isCompactStorageEnabled- ? this.getProviderOrThrow(keyName).encodeForCacheStorage(data)- : data,+ value: this.getProviderOrThrow(keyName).encodeForCacheStorage(data),});}@@ -663,17 +667,9 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {this.cleanupStaleVersions(entry);}- private sweepLocalCacheIfDue(): void {+ private sweepLocalCache(): void {const now = Date.now();- if (- isDefined(this.lastLocalCacheSweepAt) &&- now - this.lastLocalCacheSweepAt < LOCAL_CACHE_SWEEP_INTERVAL_MS- ) {- return;- }- this.lastLocalCacheSweepAt = now;-const evicted = sweepLocalCache(this.localCache, now, {ttlMs: LOCAL_ENTRY_TTL_MS,maxEntriesByKeyName: MAX_LOCAL_ENTRIES_BY_KEY_NAME,@@ -684,27 +680,23 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {if (evicted > 0) {this.cacheMetricsService.recordEviction(evicted);}-- this.demoteColdEntries();}private demoteColdEntries(): void {- demoteColdStorageEntries({+ const startedAt = performance.now();++ const { demoted, remaining } = demoteColdStorageEntries({localCache: this.localCache,hotEntriesPerProvider: HOT_ENTRIES_PER_PROVIDER,+ budgetMs: DEMOTION_BUDGET_MS,serialize: ({ localKey, data }) => {const separatorIndex = localKey.lastIndexOf(':');- const workspaceId = localKey.slice(separatorIndex + 1);const keyName = localKey.slice(0,separatorIndex,) as WorkspaceCacheKeyName;- if (- this.localDataOnlyKeys.has(keyName) ||- keyName === 'featureFlagsMap' ||- !this.isCompactStorageEnabledInLocalCache(workspaceId)- ) {+ if (this.localDataOnlyKeys.has(keyName)) {return undefined;}@@ -716,45 +708,12 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {);},});- }- private isCompactStorageEnabledInLocalCache(- workspaceId: string,- ): boolean | undefined {- const entry = this.localCache.get(- this.buildCacheKey(workspaceId, 'featureFlagsMap'),- );- const version = entry?.versions.get(entry.latestHash);-- if (version?.state !== 'hot') {- return undefined;- }-- return (version.data as WorkspaceCacheDataMap['featureFlagsMap'])[- FeatureFlagKey.IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED- ];- }-- // A restarted pod reads an already-encoded Redis payload with an empty local- // cache, so the flag has to resolve from Redis before any decode happens.- private async resolveCompactStorageEnabled(- workspaceId: string,- ): Promise<boolean> {- const localValue = this.isCompactStorageEnabledInLocalCache(workspaceId);-- if (isDefined(localValue)) {- return localValue;- }-- const featureFlagsMap = await this.cacheStorage.get<- WorkspaceCacheDataMap['featureFlagsMap']- >(`${this.buildCacheKey(workspaceId, 'featureFlagsMap')}:data`);-- return (- featureFlagsMap?.[- FeatureFlagKey.IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED- ] ?? false- );+ this.cacheMetricsService.recordDemotionSlice({+ durationSeconds: (performance.now() - startedAt) / 1000,+ demoted,+ remaining,+ });}private readVersion({@@ -772,6 +731,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {return version.data;}+ const hydrateStartedAt = performance.now();const data = this.getProviderOrThrow(keyName).decodeFromCacheStorage(JSON.parse(version.blob.toString('utf8')),);@@ -782,6 +742,11 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {lastReadAt: Date.now(),});+ this.cacheMetricsService.recordHydration(+ (performance.now() - hydrateStartedAt) / 1000,+ keyName,+ );+return data;}diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.tsindex a7c5e5dfb72..c3b3ade2459 100644--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts@@ -1,6 +1,19 @@import { computeLocalCacheStats } from 'src/engine/workspace-cache/utils/compute-local-cache-stats.util';-const entry = (versionCount: number) => ({ versions: { size: versionCount } });+const entry = (versionCount: number, state: 'hot' | 'cold' = 'hot') => ({+ versions: new Map(+ Array.from({ length: versionCount }, (_, index) => [+ `hash-${index}`,+ state === 'hot'+ ? { state: 'hot' as const, data: 'data', lastReadAt: index }+ : {+ state: 'cold' as const,+ blob: Buffer.alloc(100),+ lastReadAt: index,+ },+ ]),+ ),+});describe('computeLocalCacheStats', () => {it('returns zeros for an empty cache', () => {@@ -12,6 +25,12 @@ describe('computeLocalCacheStats', () => {versionsTotal: 0,versionsByCount: { '1': 0, '2': 0, '3': 0, '4': 0, '5+': 0 },entriesByKeyName: {},+ hotVersionsByKeyName: {},+ coldVersionsByKeyName: {},+ coldBytesByKeyName: {},+ hotVersionsTotal: 0,+ coldVersionsTotal: 0,+ coldBytesTotal: 0,});});@@ -53,4 +72,26 @@ describe('computeLocalCacheStats', () => {'5+': 2,});});++ it('splits versions by storage state and sums exact cold bytes', () => {+ const stats = computeLocalCacheStats(+ new Map([+ ['flatFieldMetadataMaps:ws-a', entry(2, 'cold')],+ ['flatFieldMetadataMaps:ws-b', entry(1, 'hot')],+ ['ORMEntityMetadatas:ws-a', entry(1, 'hot')],+ ]),+ );++ expect(stats.hotVersionsTotal).toBe(2);+ expect(stats.coldVersionsTotal).toBe(2);+ expect(stats.coldBytesTotal).toBe(200);+ expect(stats.coldVersionsByKeyName).toEqual({+ flatFieldMetadataMaps: 2,+ ORMEntityMetadatas: 0,+ });+ expect(stats.hotVersionsByKeyName).toEqual({+ flatFieldMetadataMaps: 1,+ ORMEntityMetadatas: 1,+ });+ });});diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.tsindex 1c139fba4ea..57e56f74c63 100644--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts@@ -20,16 +20,24 @@ const coldEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({const serialize = () => Buffer.from('serialized');+const NO_BUDGET_LIMIT = Number.POSITIVE_INFINITY;+const demote = (localCache: Map<string, WorkspaceLocalCacheEntry<string>>,hotEntriesPerProvider: number,- overrides?: { serialize?: () => Buffer | undefined },+ overrides?: {+ serialize?: () => Buffer | undefined;+ budgetMs?: number;+ now?: () => number;+ },) =>demoteColdStorageEntries({localCache,hotEntriesPerProvider,+ budgetMs: overrides?.budgetMs ?? NO_BUDGET_LIMIT,serialize: overrides?.serialize ?? serialize,- });+ now: overrides?.now,+ }).demoted;describe('demoteColdStorageEntries', () => {it('should demote nothing when the provider is within its hot budget', () => {@@ -108,4 +116,85 @@ describe('demoteColdStorageEntries', () => {expect(demote(localCache, 1)).toBe(0);});++ describe('time budget', () => {+ const clockAdvancingPerEntry = () => {+ let elapsed = 0;++ return () => {+ const current = elapsed;++ elapsed += 10;++ return current;+ };+ };++ const buildCache = (count: number) =>+ new Map(+ Array.from({ length: count }, (_, index) => [+ `${FIELD_METADATA}:ws-${index}`,+ hotEntry(index),+ ]),+ );++ it('should stop once the budget is spent instead of draining the backlog', () => {+ const localCache = buildCache(10);++ const result = demoteColdStorageEntries({+ localCache,+ hotEntriesPerProvider: 0,+ budgetMs: 25,+ serialize,+ now: clockAdvancingPerEntry(),+ });++ expect(result.demoted).toBe(2);+ expect(result.remaining).toBe(8);+ });++ it('should demote coldest first so an interrupted slice leaves the hottest behind', () => {+ const localCache = buildCache(10);++ demoteColdStorageEntries({+ localCache,+ hotEntriesPerProvider: 0,+ budgetMs: 25,+ serialize,+ now: clockAdvancingPerEntry(),+ });++ expect(+ localCache.get(`${FIELD_METADATA}:ws-0`)?.versions.get('hash-1'),+ ).toMatchObject({ state: 'cold' });+ expect(+ localCache.get(`${FIELD_METADATA}:ws-9`)?.versions.get('hash-1'),+ ).toMatchObject({ state: 'hot' });+ });++ it('should converge across successive slices without a cursor', () => {+ const localCache = buildCache(10);+ let slices = 0;+ let remaining = Number.POSITIVE_INFINITY;++ while (remaining > 0 && slices < 10) {+ remaining = demoteColdStorageEntries({+ localCache,+ hotEntriesPerProvider: 0,+ budgetMs: 25,+ serialize,+ now: clockAdvancingPerEntry(),+ }).remaining;+ slices += 1;+ }++ expect(remaining).toBe(0);+ expect(slices).toBe(5);+ expect(+ [...localCache.values()].every(+ (entry) => entry.versions.get('hash-1')?.state === 'cold',+ ),+ ).toBe(true);+ });+ });});diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.tsindex 71a07ad1442..c7259055c6d 100644--- a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts+++ b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts@@ -1,3 +1,4 @@+import { type VersionEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';export type LocalCacheStats = {@@ -6,10 +7,21 @@ export type LocalCacheStats = {versionsTotal: number;versionsByCount: Record<string, number>;entriesByKeyName: Record<string, number>;+ hotVersionsByKeyName: Record<string, number>;+ coldVersionsByKeyName: Record<string, number>;+ coldBytesByKeyName: Record<string, number>;+ hotVersionsTotal: number;+ coldVersionsTotal: number;+ coldBytesTotal: number;};+type StatsInput = ReadonlyMap<+ string,+ { versions: ReadonlyMap<string, VersionEntry<unknown>> }+>;+export const computeLocalCacheStats = (- localCache: ReadonlyMap<string, { versions: { size: number } }>,+ localCache: StatsInput,): LocalCacheStats => {const workspaceIds = new Set<string>();const versionsByCount: Record<string, number> = {@@ -20,19 +32,42 @@ export const computeLocalCacheStats = ('5+': 0,};const entriesByKeyName: Record<string, number> = {};+ const hotVersionsByKeyName: Record<string, number> = {};+ const coldVersionsByKeyName: Record<string, number> = {};+ const coldBytesByKeyName: Record<string, number> = {};let versionsTotal = 0;+ let hotVersionsTotal = 0;+ let coldVersionsTotal = 0;+ let coldBytesTotal = 0;for (const [key, entry] of localCache) {workspaceIds.add(key.slice(key.lastIndexOf(':') + 1));const keyName = getKeyNameFromLocalCacheKey(key);entriesByKeyName[keyName] = (entriesByKeyName[keyName] ?? 0) + 1;+ hotVersionsByKeyName[keyName] ??= 0;+ coldVersionsByKeyName[keyName] ??= 0;+ coldBytesByKeyName[keyName] ??= 0;+const versionCount = entry.versions.size;versionsTotal += versionCount;const bucket = versionCount >= 5 ? '5+' : String(versionCount);versionsByCount[bucket] = (versionsByCount[bucket] ?? 0) + 1;++ for (const version of entry.versions.values()) {+ if (version.state === 'cold') {+ coldVersionsByKeyName[keyName] += 1;+ coldVersionsTotal += 1;+ coldBytesByKeyName[keyName] += version.blob.byteLength;+ coldBytesTotal += version.blob.byteLength;+ continue;+ }++ hotVersionsByKeyName[keyName] += 1;+ hotVersionsTotal += 1;+ }}return {@@ -41,5 +76,11 @@ export const computeLocalCacheStats = (versionsTotal,versionsByCount,entriesByKeyName,+ hotVersionsByKeyName,+ coldVersionsByKeyName,+ coldBytesByKeyName,+ hotVersionsTotal,+ coldVersionsTotal,+ coldBytesTotal,};};diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.tsindex eb565c5dfbd..d48d1426c9f 100644--- a/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts+++ b/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts@@ -1,15 +1,27 @@+import { isDefined } from 'twenty-shared/utils';+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';+export type DemoteColdStorageResult = {+ demoted: number;+ remaining: number;+};+export const demoteColdStorageEntries = <T>({localCache,hotEntriesPerProvider,+ budgetMs,serialize,+ now = () => performance.now(),}: {localCache: ReadonlyMap<string, WorkspaceLocalCacheEntry<T>>;hotEntriesPerProvider: number;+ budgetMs: number;serialize: (params: { localKey: string; data: T }) => Buffer | undefined;-}): number => {+ now?: () => number;+}): DemoteColdStorageResult => {+ const startedAt = now();const hotByProvider = new Map<string,{ localKey: string; hash: string; lastReadAt: number }[]@@ -29,7 +41,8 @@ export const demoteColdStorageEntries = <T>({}}- let demoted = 0;+ const candidates: { localKey: string; hash: string; lastReadAt: number }[] =+ [];for (const hot of hotByProvider.values()) {if (hot.length <= hotEntriesPerProvider) {@@ -37,32 +50,40 @@ export const demoteColdStorageEntries = <T>({}hot.sort((a, b) => a.lastReadAt - b.lastReadAt);+ candidates.push(...hot.slice(0, hot.length - hotEntriesPerProvider));+ }- for (const { localKey, hash } of hot.slice(- 0,- hot.length - hotEntriesPerProvider,- )) {- const entry = localCache.get(localKey);- const version = entry?.versions.get(hash);+ candidates.sort((a, b) => a.lastReadAt - b.lastReadAt);- if (!entry || version?.state !== 'hot') {- continue;- }+ let demoted = 0;+ let index = 0;- const blob = serialize({ localKey, data: version.data });+ for (const { localKey, hash } of candidates) {+ if (now() - startedAt >= budgetMs) {+ break;+ }+ index += 1;- if (blob === undefined) {- continue;- }+ const entry = localCache.get(localKey);+ const version = entry?.versions.get(hash);++ if (!isDefined(entry) || version?.state !== 'hot') {+ continue;+ }++ const blob = serialize({ localKey, data: version.data });- entry.versions.set(hash, {- state: 'cold',- blob,- lastReadAt: version.lastReadAt,- });- demoted += 1;+ if (!isDefined(blob)) {+ continue;}++ entry.versions.set(hash, {+ state: 'cold',+ blob,+ lastReadAt: version.lastReadAt,+ });+ demoted += 1;}- return demoted;+ return { demoted, remaining: candidates.length - index };};diff --git a/packages/twenty-shared/src/types/FeatureFlagKey.ts b/packages/twenty-shared/src/types/FeatureFlagKey.tsindex c026bc7404d..7f9d0a688af 100644--- a/packages/twenty-shared/src/types/FeatureFlagKey.ts+++ b/packages/twenty-shared/src/types/FeatureFlagKey.ts@@ -11,6 +11,5 @@ export enum FeatureFlagKey {IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED',IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED = 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED',- IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED = 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED',IS_NATIVE_CALL_RECORDING_TABS_ENABLED = 'IS_NATIVE_CALL_RECORDING_TABS_ENABLED',}