The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
feat(server): instrument the local metadata cache and cap heavy providers by entry count (#23778)
Charles Bochet6abeb7b5e50c
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.ts b/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.tsdeleted file mode 100644index 7b9d3106b51..00000000000--- a/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.ts+++ /dev/null@@ -1,736 +0,0 @@-import { DiscoveryService, Reflector } from '@nestjs/core';-import { Test, type TestingModule } from '@nestjs/testing';--import * as Sentry from '@sentry/node';--import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';--import { type CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';-import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';-import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';-import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';-import {- WORKSPACE_CACHE_KEY,- WORKSPACE_CACHE_OPTIONS,-} from 'src/engine/workspace-cache/decorators/workspace-cache.decorator';-import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';--jest.mock('@sentry/node', () => ({- startSpan: jest.fn(),-}));--const WORKSPACE_ID = '20202020-0000-4000-8000-000000000000';--class MockFeatureFlagsCacheProvider extends WorkspaceCacheProvider<{- testData: string;-}> {- async computeForCache(_workspaceId: string) {- return { testData: 'computed-value' };- }-}--class MockRolesPermissionsCacheProvider extends WorkspaceCacheProvider<{- testData: string;-}> {- async computeForCache(_workspaceId: string) {- return { testData: 'computed-value' };- }-}--class MockOrmEntityMetadatasCacheProvider extends WorkspaceCacheProvider<{- testData: string;-}> {- async computeForCache(_workspaceId: string) {- return { testData: 'orm-computed-value' };- }-}--describe('WorkspaceCacheService', () => {- let service: WorkspaceCacheService;- let cacheStorageService: jest.Mocked<CacheStorageService>;- let discoveryService: jest.Mocked<DiscoveryService>;- let reflector: jest.Mocked<Reflector>;- let mockProvider: MockFeatureFlagsCacheProvider;-- beforeEach(async () => {- jest.useFakeTimers();- jest- .mocked(Sentry.startSpan)- .mockImplementation((_options, callback) => callback({} as never));-- mockProvider = new MockFeatureFlagsCacheProvider();-- const module: TestingModule = await Test.createTestingModule({- providers: [- WorkspaceCacheService,- {- provide: CacheStorageNamespace.EngineWorkspace,- useValue: {- mget: jest.fn(),- mset: jest.fn(),- mdel: jest.fn(),- setIfAbsent: jest.fn(),- },- },- {- provide: DiscoveryService,- useValue: {- getProviders: jest.fn(),- },- },- {- provide: Reflector,- useValue: {- get: jest.fn(),- },- },- {- provide: MetricsService,- useValue: {- incrementCounterBy: jest.fn(),- getMeter: jest.fn().mockReturnValue({- createHistogram: jest.fn().mockReturnValue({ record: jest.fn() }),- }),- },- },- {- provide: TwentyConfigService,- useValue: {- get: jest.fn().mockReturnValue(604800),- },- },- ],- }).compile();-- service = module.get<WorkspaceCacheService>(WorkspaceCacheService);- cacheStorageService = module.get(CacheStorageNamespace.EngineWorkspace);- discoveryService = module.get(DiscoveryService);- reflector = module.get(Reflector);- });-- afterEach(() => {- jest.useRealTimers();- jest.clearAllMocks();- });-- describe('onModuleInit', () => {- it('should register workspace cache providers', async () => {- discoveryService.getProviders.mockReturnValue([- { instance: mockProvider },- ] as any);-- reflector.get.mockImplementation((key, target) => {- if (- key === WORKSPACE_CACHE_KEY &&- target === MockFeatureFlagsCacheProvider- ) {- return 'featureFlagsMap';- }-- return undefined;- });-- await service.onModuleInit();-- expect(discoveryService.getProviders).toHaveBeenCalled();- expect(reflector.get).toHaveBeenCalled();- });-- it('should skip non-object instances', async () => {- discoveryService.getProviders.mockReturnValue([- { instance: null },- { instance: undefined },- { instance: 'string-value' },- ] as any);-- await service.onModuleInit();-- expect(reflector.get).not.toHaveBeenCalled();- });-- it('should skip instances without workspace cache key metadata', async () => {- discoveryService.getProviders.mockReturnValue([- { instance: mockProvider },- ] as any);-- reflector.get.mockReturnValue(undefined);-- await service.onModuleInit();-- expect(reflector.get).toHaveBeenCalled();- });- });-- describe('getOrRecompute', () => {- beforeEach(async () => {- discoveryService.getProviders.mockReturnValue([- { instance: mockProvider },- ] as any);-- reflector.get.mockImplementation((key, target) => {- if (- key === WORKSPACE_CACHE_KEY &&- target === MockFeatureFlagsCacheProvider- ) {- return 'featureFlagsMap';- }-- return undefined;- });-- await service.onModuleInit();- });-- it('should compute and cache data when redis cache is empty', async () => {- cacheStorageService.mget.mockResolvedValue([undefined]);- cacheStorageService.mset.mockResolvedValue(undefined);-- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({- testData: 'fresh-computed-value',- });-- const result = await service.getOrRecompute(WORKSPACE_ID, [- 'featureFlagsMap',- ]);-- expect(result).toEqual({- featureFlagsMap: { testData: 'fresh-computed-value' },- });- expect(mockProvider.computeForCache).toHaveBeenCalledWith(WORKSPACE_ID);- expect(cacheStorageService.mset).toHaveBeenCalled();- expect(Sentry.startSpan).toHaveBeenCalledWith(- {- name: 'compute workspace metadata cache entry from provider',- op: 'cache.recompute',- onlyIfParent: true,- attributes: {- 'cache.key_name': 'featureFlagsMap',- 'cache.recompute.strategy': 'recover',- 'cache.local_data_only': false,- },- },- expect.any(Function),- );- });-- it('should return data from redis when available', async () => {- const cachedData = { FLAG_A: true, FLAG_B: false };- const cachedHash = 'some-hash-from-redis';-- cacheStorageService.mget- // First call: validateLocalHashAgainstRedisHash checks hash- .mockResolvedValueOnce([undefined])- // Second call: fetchDataFromRedis fetches data and hash atomically- .mockResolvedValueOnce([cachedData, cachedHash]);-- const result = await service.getOrRecompute(WORKSPACE_ID, [- 'featureFlagsMap',- ]);-- expect(result).toEqual({ featureFlagsMap: cachedData });- expect(Sentry.startSpan).not.toHaveBeenCalled();- });-- it('should use local cache when within TTL staleness window', async () => {- cacheStorageService.mget.mockResolvedValue([undefined]);- cacheStorageService.mset.mockResolvedValue(undefined);-- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({- testData: 'computed-value',- });-- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);-- jest.advanceTimersByTime(50);-- const result = await service.getOrRecompute(WORKSPACE_ID, [- 'featureFlagsMap',- ]);-- expect(result).toEqual({- featureFlagsMap: { testData: 'computed-value' },- });-- expect(mockProvider.computeForCache).toHaveBeenCalledTimes(1);- });-- it('should recheck redis when local cache exceeds TTL staleness window', async () => {- const initialData = { testData: 'initial-value' };- const updatedData = { testData: 'updated-value' };-- cacheStorageService.mget.mockResolvedValue([undefined]);- cacheStorageService.mset.mockResolvedValue(undefined);-- jest- .spyOn(mockProvider, 'computeForCache')- .mockResolvedValue(initialData);-- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);-- // Advance past the local staleness TTL (100ms) and memoizer TTL (10s)- jest.advanceTimersByTime(15_000);-- cacheStorageService.mget.mockResolvedValue([updatedData]);-- jest- .spyOn(mockProvider, 'computeForCache')- .mockResolvedValue(updatedData);-- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);-- // Verify Redis was rechecked after TTL expired- // Each getOrRecompute triggers: 1 hash check + 1 atomic data/hash fetch = 2 mget calls- // Total: 4 mget calls (2 per getOrRecompute)- expect(cacheStorageService.mget).toHaveBeenCalledTimes(4);- });- });-- describe('local cache expiration sweep', () => {- it('should run at most once per minute', async () => {- const expirationSweepSpy = jest.spyOn(- service as unknown as {- evictExpiredLocalEntries: (now: number) => void;- },- 'evictExpiredLocalEntries',- );-- await expect(- service.getOrRecompute('invalid-workspace-id', ['featureFlagsMap']),- ).rejects.toThrow();- await expect(- service.getOrRecompute('invalid-workspace-id', ['featureFlagsMap']),- ).rejects.toThrow();-- expect(expirationSweepSpy).toHaveBeenCalledTimes(1);-- jest.advanceTimersByTime(60_000);-- await expect(- service.getOrRecompute('invalid-workspace-id', ['featureFlagsMap']),- ).rejects.toThrow();-- expect(expirationSweepSpy).toHaveBeenCalledTimes(2);- });- });-- describe('invalidateAndRecompute', () => {- beforeEach(async () => {- discoveryService.getProviders.mockReturnValue([- { instance: mockProvider },- ] as any);-- reflector.get.mockImplementation((key, target) => {- if (- key === WORKSPACE_CACHE_KEY &&- target === MockFeatureFlagsCacheProvider- ) {- return 'featureFlagsMap';- }-- return undefined;- });-- await service.onModuleInit();- });-- it('should delete from redis, mark local cache as stale, and recompute', async () => {- cacheStorageService.mdel.mockResolvedValue(undefined);- cacheStorageService.mset.mockResolvedValue(undefined);-- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({- testData: 'recomputed-value',- });-- await service.invalidateAndRecompute(WORKSPACE_ID, ['featureFlagsMap']);-- expect(cacheStorageService.mdel).toHaveBeenCalledWith([- 'feature-flag:feature-flags-map:20202020-0000-4000-8000-000000000000:data',- 'feature-flag:feature-flags-map:20202020-0000-4000-8000-000000000000:hash',- ]);- expect(mockProvider.computeForCache).toHaveBeenCalledWith(WORKSPACE_ID);- expect(cacheStorageService.mset).toHaveBeenCalled();- expect(Sentry.startSpan).toHaveBeenCalledWith(- {- name: 'invalidate and recompute workspace metadata cache',- op: 'cache.invalidate',- onlyIfParent: true,- attributes: { 'cache.key_count': 1 },- },- expect.any(Function),- );- });-- it('should invalidate multiple cache keys at once', async () => {- const secondMockProvider = new MockRolesPermissionsCacheProvider();-- discoveryService.getProviders.mockReturnValue([- { instance: mockProvider },- { instance: secondMockProvider },- ] as any);-- reflector.get.mockImplementation((key, target) => {- if (key === WORKSPACE_CACHE_KEY) {- if (target === MockFeatureFlagsCacheProvider) {- return 'featureFlagsMap';- }- if (target === MockRolesPermissionsCacheProvider) {- return 'rolesPermissions';- }- }-- return undefined;- });-- await service.onModuleInit();-- cacheStorageService.mdel.mockResolvedValue(undefined);- cacheStorageService.mset.mockResolvedValue(undefined);-- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({- testData: 'recomputed-value',- });- jest.spyOn(secondMockProvider, 'computeForCache').mockResolvedValue({- testData: 'recomputed-value',- });-- await service.invalidateAndRecompute(WORKSPACE_ID, [- 'featureFlagsMap',- 'rolesPermissions',- ]);-- expect(cacheStorageService.mdel).toHaveBeenCalledWith(- expect.arrayContaining([- 'feature-flag:feature-flags-map:20202020-0000-4000-8000-000000000000:data',- 'feature-flag:feature-flags-map:20202020-0000-4000-8000-000000000000:hash',- 'metadata:permissions:roles-permissions:20202020-0000-4000-8000-000000000000:data',- 'metadata:permissions:roles-permissions:20202020-0000-4000-8000-000000000000:hash',- ]),- );- });-- it('should keep old versions in local cache after invalidation for race condition safety', async () => {- cacheStorageService.mget.mockResolvedValue([undefined]);- cacheStorageService.mset.mockResolvedValue(undefined);- cacheStorageService.mdel.mockResolvedValue(undefined);-- const initialData = { testData: 'initial-value' };- const recomputedData = { testData: 'recomputed-value' };-- jest- .spyOn(mockProvider, 'computeForCache')- .mockResolvedValue(initialData);-- // First, populate the cache- const firstResult = await service.getOrRecompute(WORKSPACE_ID, [- 'featureFlagsMap',- ]);-- expect(firstResult).toEqual({ featureFlagsMap: initialData });-- // Now invalidate and recompute- jest- .spyOn(mockProvider, 'computeForCache')- .mockResolvedValue(recomputedData);-- await service.invalidateAndRecompute(WORKSPACE_ID, ['featureFlagsMap']);-- // The new value should be returned- const secondResult = await service.getOrRecompute(WORKSPACE_ID, [- 'featureFlagsMap',- ]);-- expect(secondResult).toEqual({ featureFlagsMap: recomputedData });- });- });-- describe('flush', () => {- it('should delete from redis and mark local cache as stale', async () => {- cacheStorageService.mdel.mockResolvedValue(undefined);-- await service.flush(WORKSPACE_ID, ['featureFlagsMap']);-- expect(cacheStorageService.mdel).toHaveBeenCalledWith([- 'feature-flag:feature-flags-map:20202020-0000-4000-8000-000000000000:data',- 'feature-flag:feature-flags-map:20202020-0000-4000-8000-000000000000:hash',- ]);- });-- it('should handle empty cache keys array', async () => {- cacheStorageService.mdel.mockResolvedValue(undefined);-- await service.flush(WORKSPACE_ID, []);-- expect(cacheStorageService.mdel).toHaveBeenCalledWith([]);- });-- it('should force staleness on local cache entries without deleting them', async () => {- cacheStorageService.mget.mockResolvedValue([undefined]);- cacheStorageService.mset.mockResolvedValue(undefined);- cacheStorageService.mdel.mockResolvedValue(undefined);-- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({- testData: 'computed-value',- });-- discoveryService.getProviders.mockReturnValue([- { instance: mockProvider },- ] as any);-- reflector.get.mockImplementation((key, target) => {- if (- key === WORKSPACE_CACHE_KEY &&- target === MockFeatureFlagsCacheProvider- ) {- return 'featureFlagsMap';- }-- return undefined;- });-- await service.onModuleInit();-- // Populate the cache- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);-- // Flush the cache (marks as stale, doesn't delete)- await service.flush(WORKSPACE_ID, ['featureFlagsMap']);-- // Advance time slightly (but still within memoizer TTL)- jest.advanceTimersByTime(50);-- // Next call should check Redis since local cache is marked stale- cacheStorageService.mget.mockResolvedValue(['some-hash']);-- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);-- // Should have made additional mget calls to check Redis- expect(cacheStorageService.mget.mock.calls.length).toBeGreaterThan(1);- });- });-- describe('versioning behavior', () => {- beforeEach(async () => {- discoveryService.getProviders.mockReturnValue([- { instance: mockProvider },- ] as any);-- reflector.get.mockImplementation((key, target) => {- if (- key === WORKSPACE_CACHE_KEY &&- target === MockFeatureFlagsCacheProvider- ) {- return 'featureFlagsMap';- }-- return undefined;- });-- await service.onModuleInit();- });-- it('should store multiple versions when data is recomputed', async () => {- cacheStorageService.mget.mockResolvedValue([undefined]);- cacheStorageService.mset.mockResolvedValue(undefined);- cacheStorageService.mdel.mockResolvedValue(undefined);-- const firstData = { testData: 'first-value' };- const secondData = { testData: 'second-value' };-- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue(firstData);-- // First computation- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);-- // Invalidate and recompute with new data- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue(secondData);-- await service.invalidateAndRecompute(WORKSPACE_ID, ['featureFlagsMap']);-- // Should return the latest version- const result = await service.getOrRecompute(WORKSPACE_ID, [- 'featureFlagsMap',- ]);-- expect(result).toEqual({ featureFlagsMap: secondData });- });-- it('should cleanup stale versions after TTL expires', async () => {- cacheStorageService.mget.mockResolvedValue([undefined]);- cacheStorageService.mset.mockResolvedValue(undefined);- cacheStorageService.mdel.mockResolvedValue(undefined);-- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({- testData: 'value-1',- });-- // Create first version- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);-- // Create multiple versions by invalidating- for (let i = 2; i <= 4; i++) {- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({- testData: `value-${i}`,- });- await service.invalidateAndRecompute(WORKSPACE_ID, ['featureFlagsMap']);- }-- // Advance time past the stale version TTL (5000ms)- jest.advanceTimersByTime(6_000);-- // Advance past memoizer TTL as well- jest.advanceTimersByTime(15_000);-- // Trigger a read which should cleanup stale versions- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({- testData: 'latest-value',- });-- const result = await service.getOrRecompute(WORKSPACE_ID, [- 'featureFlagsMap',- ]);-- expect(result).toEqual({ featureFlagsMap: { testData: 'latest-value' } });- });- });-- describe('localDataOnly hash recovery', () => {- let localDataOnlyProvider: MockOrmEntityMetadatasCacheProvider;- const ormHashKey = `orm:entity-metadatas:${WORKSPACE_ID}:hash`;- const ormDataKey = `orm:entity-metadatas:${WORKSPACE_ID}:data`;-- beforeEach(async () => {- localDataOnlyProvider = new MockOrmEntityMetadatasCacheProvider();-- discoveryService.getProviders.mockReturnValue([- { instance: localDataOnlyProvider },- ] as any);-- reflector.get.mockImplementation((key, target) => {- if (target !== MockOrmEntityMetadatasCacheProvider) {- return undefined;- }-- if (key === WORKSPACE_CACHE_KEY) {- return 'ORMEntityMetadatas';- }-- if (key === WORKSPACE_CACHE_OPTIONS) {- return { localDataOnly: true };- }-- return undefined;- });-- await service.onModuleInit();- });-- it('should adopt the existing redis hash when recomputing instead of minting a new one', async () => {- cacheStorageService.mget.mockResolvedValue(['hash-from-another-pod']);- cacheStorageService.mset.mockResolvedValue(undefined);-- const computeSpy = jest.spyOn(localDataOnlyProvider, 'computeForCache');-- const result = await service.getOrRecompute(WORKSPACE_ID, [- 'ORMEntityMetadatas',- ]);-- expect(result).toEqual({- ORMEntityMetadatas: { testData: 'orm-computed-value' },- });- expect(computeSpy).toHaveBeenCalledTimes(1);- expect(cacheStorageService.mset).not.toHaveBeenCalled();- });-- it('should treat the local copy as valid on later reads after adopting the redis hash', async () => {- cacheStorageService.mget.mockResolvedValue(['hash-from-another-pod']);- cacheStorageService.mset.mockResolvedValue(undefined);-- const computeSpy = jest.spyOn(localDataOnlyProvider, 'computeForCache');-- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);-- jest.advanceTimersByTime(15_000);-- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);-- expect(computeSpy).toHaveBeenCalledTimes(1);- expect(cacheStorageService.mset).not.toHaveBeenCalled();- });-- it('should recompute and adopt the new hash when another pod invalidated the key', async () => {- cacheStorageService.mget.mockResolvedValue(['hash-v1']);- cacheStorageService.mset.mockResolvedValue(undefined);-- const computeSpy = jest.spyOn(localDataOnlyProvider, 'computeForCache');-- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);-- jest.advanceTimersByTime(15_000);-- cacheStorageService.mget.mockResolvedValue(['hash-v2']);-- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);-- expect(computeSpy).toHaveBeenCalledTimes(2);- expect(cacheStorageService.mset).not.toHaveBeenCalled();- });-- it('should mint a hash and persist it only if still absent when redis has none', async () => {- cacheStorageService.mget.mockResolvedValue([undefined]);- cacheStorageService.setIfAbsent.mockResolvedValue(true);-- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);-- expect(cacheStorageService.setIfAbsent).toHaveBeenCalledWith(- ormHashKey,- expect.any(String),- 604800000,- );- expect(cacheStorageService.mset).not.toHaveBeenCalled();- });-- it('should converge on a hash written concurrently by another pod instead of overwriting it', async () => {- cacheStorageService.mget.mockResolvedValue([undefined]);- cacheStorageService.setIfAbsent.mockResolvedValue(false);-- const computeSpy = jest.spyOn(localDataOnlyProvider, 'computeForCache');-- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);-- expect(cacheStorageService.setIfAbsent).toHaveBeenCalledTimes(1);- expect(cacheStorageService.mset).not.toHaveBeenCalled();-- jest.advanceTimersByTime(15_000);-- cacheStorageService.mget.mockResolvedValue(['winner-hash']);-- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);-- expect(computeSpy).toHaveBeenCalledTimes(2);- expect(cacheStorageService.setIfAbsent).toHaveBeenCalledTimes(1);-- jest.advanceTimersByTime(15_000);-- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);-- expect(computeSpy).toHaveBeenCalledTimes(2);- });-- it('should mint a new hash on invalidateAndRecompute', async () => {- cacheStorageService.mdel.mockResolvedValue(undefined);- cacheStorageService.mset.mockResolvedValue(undefined);-- await service.invalidateAndRecompute(WORKSPACE_ID, [- 'ORMEntityMetadatas',- ]);-- expect(cacheStorageService.mdel).toHaveBeenCalledWith([- ormDataKey,- ormHashKey,- ]);- expect(cacheStorageService.mset).toHaveBeenCalledWith([- { key: ormHashKey, value: expect.any(String) },- ]);- });- });-});diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.tsnew file mode 100644index 00000000000..f5519f1e992--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts@@ -0,0 +1,254 @@+import { Injectable, Logger } from '@nestjs/common';++import { type Histogram } from '@opentelemetry/api';++import { isDefined } from 'twenty-shared/utils';++import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';+import {+ type WorkspaceCacheDataMap,+ type WorkspaceCacheKeyName,+} from 'src/engine/workspace-cache/types/workspace-cache-key.type';+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';+import {+ type LocalCacheStats,+ computeLocalCacheStats,+} from 'src/engine/workspace-cache/utils/compute-local-cache-stats.util';+import { deepSizeBytes } from 'src/engine/workspace-cache/utils/deep-size-bytes.util';++type LocalCache = ReadonlyMap<+ string,+ WorkspaceLocalCacheEntry<WorkspaceCacheDataMap[WorkspaceCacheKeyName]>+>;++const CACHE_DURATION_BUCKETS_SECONDS = [+ 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,+];+const STATS_TTL_MS = 5_000;+const SIZE_REFRESH_MS = 5 * 60 * 1000;+const SIZE_STARTUP_DELAY_MS = 30 * 1000;+const SIZE_SAMPLE_PER_PROVIDER = 3;+const SIZE_WALK_NODE_CAP = 300_000;++// Cache observability, split from WorkspaceCacheService (which drives it via start()/stop()).+@Injectable()+export class WorkspaceCacheMetricsService {+ private readonly logger = new Logger(WorkspaceCacheMetricsService.name);+ private readonly recomputeDurationHistogram: Histogram;+ private readonly redisWriteDurationHistogram: Histogram;++ private localCache?: LocalCache;+ private cacheSizeByProvider: Record<string, number> = {};+ private cacheSizeTotalBytes = 0;+ private sizeSampler?: ReturnType<typeof setInterval>;+ private sizeStartupTimer?: ReturnType<typeof setTimeout>;+ private statsCache?: { computedAt: number } & LocalCacheStats;+ private sizeSampleInFlight = false;++ constructor(private readonly metricsService: MetricsService) {+ const meter = this.metricsService.getMeter();++ this.recomputeDurationHistogram = meter.createHistogram(+ 'twenty_workspace_cache_recompute_duration_seconds',+ {+ description:+ 'Wall-clock time to compute one workspace metadata cache entry from its provider',+ unit: 's',+ advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },+ },+ );+ this.redisWriteDurationHistogram = meter.createHistogram(+ 'twenty_workspace_cache_redis_write_duration_seconds',+ {+ description:+ 'Wall-clock time to serialize and write recomputed cache entries to Redis',+ unit: 's',+ advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },+ },+ );+ }++ start(localCache: LocalCache): void {+ this.localCache = localCache;+ this.registerGauges();+ this.scheduleSizeSampler();+ }++ stop(): void {+ if (isDefined(this.sizeStartupTimer)) {+ clearTimeout(this.sizeStartupTimer);+ }+ if (isDefined(this.sizeSampler)) {+ clearInterval(this.sizeSampler);+ }+ }++ recordRecompute(seconds: number, cacheKey: WorkspaceCacheKeyName): void {+ this.recomputeDurationHistogram.record(seconds, { cache_key: cacheKey });+ }++ recordRedisWrite(seconds: number): void {+ this.redisWriteDurationHistogram.record(seconds);+ }++ recordEviction(amount: number): void {+ this.metricsService.incrementCounterBy({+ key: MetricsKeys.WorkspaceMetadataCacheLocalEviction,+ amount,+ });+ }++ private getStats(): LocalCacheStats {+ const now = Date.now();++ if (+ isDefined(this.statsCache) &&+ now - this.statsCache.computedAt < STATS_TTL_MS+ ) {+ return this.statsCache;+ }++ const stats = computeLocalCacheStats(this.localCache ?? new Map());++ this.statsCache = { computedAt: now, ...stats };++ return this.statsCache;+ }++ private scheduleSizeSampler(): void {+ const sample = (): void => {+ if (this.sizeSampleInFlight) {+ return;+ }++ this.sizeSampleInFlight = true;+ this.refreshSizeBreakdown()+ .catch((error) =>+ this.logger.error('Failed to sample local cache size', error),+ )+ .finally(() => {+ this.sizeSampleInFlight = false;+ });+ };++ // Prime once after startup so the gauges aren't 0 until the first 5-minute interval.+ this.sizeStartupTimer = setTimeout(sample, SIZE_STARTUP_DELAY_MS);+ this.sizeStartupTimer.unref();++ this.sizeSampler = setInterval(sample, SIZE_REFRESH_MS);+ this.sizeSampler.unref();+ }++ private async refreshSizeBreakdown(): Promise<void> {+ const localCache = this.localCache;++ if (!isDefined(localCache)) {+ return;+ }++ const perProvider: Record<+ string,+ { count: number; sampledBytes: number; sampled: number }+ > = {};++ for (const [key, entry] of localCache) {+ const provider = key.slice(0, key.lastIndexOf(':'));+ const stats = (perProvider[provider] ??= {+ count: 0,+ sampledBytes: 0,+ sampled: 0,+ });++ stats.count += 1;++ if (stats.sampled < SIZE_SAMPLE_PER_PROVIDER && entry.versions.size > 0) {+ // Size every retained version, not just the latest — stale versions still occupy heap.+ let entryBytes = 0;++ for (const version of entry.versions.values()) {+ entryBytes += deepSizeBytes(version.data, SIZE_WALK_NODE_CAP);+ }++ stats.sampledBytes += entryBytes;+ stats.sampled += 1;+ await new Promise((resolve) => setImmediate(resolve));+ }+ }++ const byProvider: Record<string, number> = {};+ let total = 0;++ for (const [provider, stats] of Object.entries(perProvider)) {+ const estimate =+ stats.sampled === 0+ ? 0+ : Math.round((stats.sampledBytes / stats.sampled) * stats.count);++ byProvider[provider] = estimate;+ total += estimate;+ }++ this.cacheSizeByProvider = byProvider;+ this.cacheSizeTotalBytes = total;+ }++ private registerGauges(): void {+ this.metricsService.createObservableGauge({+ metricName: 'twenty_workspace_cache_local_entries',+ options: {+ description: 'Entries in the per-pod local workspace metadata cache',+ },+ callback: async () => this.getStats().entries,+ });+ this.metricsService.createObservableGauge({+ metricName: 'twenty_workspace_cache_local_workspaces',+ options: {+ description:+ 'Distinct workspaces held in the per-pod local workspace metadata cache',+ },+ callback: async () => this.getStats().workspaces,+ });+ this.metricsService.createObservableGauge({+ metricName: 'twenty_workspace_cache_local_versions_total',+ options: {+ description:+ 'Total versions across local workspace metadata cache entries',+ },+ callback: async () => this.getStats().versionsTotal,+ });+ this.metricsService.createObservableGauge({+ metricName: 'twenty_workspace_cache_local_bytes_estimate',+ options: {+ description:+ 'Estimated retained bytes (deep-size, includes local-only providers) in the local workspace metadata cache',+ unit: 'By',+ },+ callback: async () => this.cacheSizeTotalBytes,+ });+ this.metricsService.createMultiObservableGauge({+ metricName: 'twenty_workspace_cache_local_entries_by_version_count',+ options: {+ description:+ 'Local workspace metadata cache entries bucketed by version count',+ },+ callback: async () =>+ Object.entries(this.getStats().versionsByCount).map(+ ([versions, value]) => ({ value, attributes: { versions } }),+ ),+ });+ this.metricsService.createMultiObservableGauge({+ metricName: 'twenty_workspace_cache_local_bytes_by_provider',+ options: {+ description:+ 'Estimated retained bytes in the local workspace metadata cache per provider',+ unit: 'By',+ },+ callback: async () =>+ Object.entries(this.cacheSizeByProvider).map(([provider, value]) => ({+ value,+ attributes: { provider },+ })),+ });+ }+}diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.tsindex b363d7d5a65..2c25a8e71b6 100644--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts@@ -1,7 +1,11 @@-import { Injectable, Logger, OnModuleInit } from '@nestjs/common';+import {+ Injectable,+ Logger,+ OnModuleDestroy,+ OnModuleInit,+} from '@nestjs/common';import { DiscoveryService, Reflector } from '@nestjs/core';-import { type Histogram } from '@opentelemetry/api';import * as Sentry from '@sentry/node';import crypto from 'crypto';@@ -12,10 +16,9 @@ import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/woimport { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';-import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';-import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';import { PromiseMemoizer } from 'src/engine/twenty-orm/storage/promise-memoizer.storage';+import { WorkspaceCacheMetricsService } from 'src/engine/workspace-cache/services/workspace-cache-metrics.service';import {WORKSPACE_CACHE_KEY,WORKSPACE_CACHE_OPTIONS,@@ -34,20 +37,21 @@ import {} from 'src/engine/workspace-cache/types/workspace-cache-key.type';import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';import { combineCacheHashes } from 'src/engine/workspace-cache/utils/combine-cache-hashes.util';+import { sweepLocalCache } from 'src/engine/workspace-cache/utils/sweep-local-cache.util';const LOCAL_TTL_MS = 100; // 100ms-const LOCAL_ENTRY_TTL_MS = 30 * 60 * 1000; // 30 minutes-const LOCAL_CACHE_EXPIRATION_SWEEP_INTERVAL_MS = 60 * 1000;const MEMOIZER_TTL_MS = 10_000; // 10 secondsconst STALE_VERSION_TTL_MS = 5_000; // 5 secondsconst MAX_LOCAL_STALE_VERSIONS = 5; // 5 stale versions-// Sized against 4 GiB pods (--max-old-space-size=3500): 7,500 sat at the heap ceiling+// Sized against 4 GiB pods (--max-old-space-size=3500): 7,500 sat at the heap ceiling.const MAX_LOCAL_CACHE_ENTRIES = 6_000;const MIN_EVICT_KEYS = 100;-const CACHE_DURATION_BUCKETS_SECONDS = [- 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,-];-+const LOCAL_ENTRY_TTL_MS = 30 * 60 * 1000; // 30 minutes idle+const LOCAL_CACHE_SWEEP_INTERVAL_MS = 60 * 1000;+// Per-provider entry caps, keyed by local cache key prefix (ORM graphs are ~5 MB each).+const MAX_LOCAL_ENTRIES_BY_PREFIX = new Map<string, number>([+ [WORKSPACE_CACHE_KEYS_V2.ORMEntityMetadatas, 128],+]);type CacheDataType = WorkspaceCacheDataMap[WorkspaceCacheKeyName];type CacheEntriesResult = {@@ -63,11 +67,12 @@ type RecomputeHashResolution =};@Injectable()-export class WorkspaceCacheService implements OnModuleInit {+export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {private readonly localCache = new Map<string,WorkspaceLocalCacheEntry<CacheDataType>>();+ private lastLocalCacheSweepAt: number | undefined;private readonly workspaceCacheProviders = new Map<WorkspaceCacheKeyName,WorkspaceCacheProvider<CacheDataType>@@ -76,46 +81,17 @@ export class WorkspaceCacheService implements OnModuleInit {private readonly memoizer = new PromiseMemoizer<CacheEntriesResult>(MEMOIZER_TTL_MS,);- private lastLocalCacheExpirationSweepAt: number | undefined;private readonly logger = new Logger(WorkspaceCacheService.name);- private readonly recomputeDurationHistogram: Histogram;- private readonly redisWriteDurationHistogram: Histogram;-constructor(@InjectCacheStorage(CacheStorageNamespace.EngineWorkspace)private readonly cacheStorage: CacheStorageService,private readonly discoveryService: DiscoveryService,private readonly reflector: Reflector,- private readonly metricsService: MetricsService,+ private readonly cacheMetricsService: WorkspaceCacheMetricsService,private readonly twentyConfigService: TwentyConfigService,- ) {- const meter = this.metricsService.getMeter();-- this.recomputeDurationHistogram = meter.createHistogram(- 'twenty_workspace_cache_recompute_duration_seconds',- {- description:- 'Wall-clock time to compute one workspace metadata cache entry from its provider',- unit: 's',- advice: {- explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS,- },- },- );- this.redisWriteDurationHistogram = meter.createHistogram(- 'twenty_workspace_cache_redis_write_duration_seconds',- {- description:- 'Wall-clock time to serialize and write recomputed cache entries to Redis',- unit: 's',- advice: {- explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS,- },- },- );- }+ ) {}async onModuleInit() {const providers = this.discoveryService.getProviders();@@ -149,6 +125,12 @@ export class WorkspaceCacheService implements OnModuleInit {}}}++ this.cacheMetricsService.start(this.localCache);+ }++ onModuleDestroy(): void {+ this.cacheMetricsService.stop();}public async getOrRecompute<const K extends WorkspaceCacheKeyName[]>(@@ -169,7 +151,7 @@ export class WorkspaceCacheService implements OnModuleInit {workspaceId: string,cacheKeyNames: K,): Promise<WorkspaceCacheResultWithHashes<K>> {- this.evictExpiredLocalEntriesIfNeeded();+ this.sweepLocalCacheIfDue();this.assertValidCacheParameters(workspaceId, cacheKeyNames);const memoKey =@@ -280,6 +262,10 @@ export class WorkspaceCacheService implements OnModuleInit {strategy: 'mint',});+ // Invalidation writes entries too, so run the sweep here as well — otherwise an+ // invalidate-only workload never enforces the per-provider caps.+ this.sweepLocalCacheIfDue();+// Clear memoizer again after recomputation to evict any stale entries// cached by concurrent getOrRecompute calls during the flush window.await this.memoizer.clearKeys(`${workspaceId}-`);@@ -504,9 +490,9 @@ export class WorkspaceCacheService implements OnModuleInit {isAdopted: isDefined(adoptableHash),};} finally {- this.recomputeDurationHistogram.record(+ this.cacheMetricsService.recordRecompute((performance.now() - computeStartedAt) / 1000,- { cache_key: keyName },+ keyName,);}});@@ -544,7 +530,7 @@ export class WorkspaceCacheService implements OnModuleInit {try {await this.cacheStorage.mset(redisEntries);} finally {- this.redisWriteDurationHistogram.record(+ this.cacheMetricsService.recordRedisWrite((performance.now() - redisWriteStartedAt) / 1000,);}@@ -632,31 +618,29 @@ export class WorkspaceCacheService implements OnModuleInit {entry.lastHashCheckedAt = Date.now();this.cleanupStaleVersions(entry);- this.evictLRUEntriesIfNeeded();}- private evictLRUEntriesIfNeeded(): void {- if (this.localCache.size <= MAX_LOCAL_CACHE_ENTRIES) {+ private sweepLocalCacheIfDue(): void {+ const now = Date.now();++ if (+ isDefined(this.lastLocalCacheSweepAt) &&+ now - this.lastLocalCacheSweepAt < LOCAL_CACHE_SWEEP_INTERVAL_MS+ ) {return;}+ this.lastLocalCacheSweepAt = now;- const entries = [...this.localCache.entries()].sort(- (a, b) => a[1].lastHashCheckedAt - b[1].lastHashCheckedAt,- );-- const toEvict = entries.slice(- 0,- Math.max(MIN_EVICT_KEYS, this.localCache.size - MAX_LOCAL_CACHE_ENTRIES),- );+ const evicted = sweepLocalCache(this.localCache, now, {+ ttlMs: LOCAL_ENTRY_TTL_MS,+ maxEntriesByPrefix: MAX_LOCAL_ENTRIES_BY_PREFIX,+ globalMaxEntries: MAX_LOCAL_CACHE_ENTRIES,+ minEvict: MIN_EVICT_KEYS,+ });- for (const [key] of toEvict) {- this.localCache.delete(key);+ if (evicted > 0) {+ this.cacheMetricsService.recordEviction(evicted);}-- this.metricsService.incrementCounterBy({- key: MetricsKeys.WorkspaceMetadataCacheLocalEviction,- amount: toEvict.length,- });}private cleanupStaleVersions(@@ -691,41 +675,6 @@ export class WorkspaceCacheService implements OnModuleInit {}}- private evictExpiredLocalEntriesIfNeeded(): void {- const now = Date.now();-- if (- isDefined(this.lastLocalCacheExpirationSweepAt) &&- now - this.lastLocalCacheExpirationSweepAt <- LOCAL_CACHE_EXPIRATION_SWEEP_INTERVAL_MS- ) {- return;- }-- this.evictExpiredLocalEntries(now);- this.lastLocalCacheExpirationSweepAt = now;- }-- private evictExpiredLocalEntries(now: number): void {- for (const [localKey, entry] of this.localCache) {- for (const [hash, version] of entry.versions) {- if (now - version.lastReadAt > LOCAL_ENTRY_TTL_MS) {- entry.versions.delete(hash);- }- }-- if (entry.versions.size === 0) {- this.localCache.delete(localKey);- continue;- }-- if (!entry.versions.has(entry.latestHash)) {- // Latest was evicted; drop the entire entry to avoid serving stale data.- this.localCache.delete(localKey);- }- }- }-private getProviderOrThrow(keyName: WorkspaceCacheKeyName,): WorkspaceCacheProvider<CacheDataType> {diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.tsnew file mode 100644index 00000000000..890c1199692--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts@@ -0,0 +1,51 @@+import { computeLocalCacheStats } from 'src/engine/workspace-cache/utils/compute-local-cache-stats.util';++const entry = (versionCount: number) => ({ versions: { size: versionCount } });++describe('computeLocalCacheStats', () => {+ it('returns zeros for an empty cache', () => {+ const stats = computeLocalCacheStats(new Map());++ expect(stats).toEqual({+ entries: 0,+ workspaces: 0,+ versionsTotal: 0,+ versionsByCount: { '1': 0, '2': 0, '3': 0, '4': 0, '5+': 0 },+ });+ });++ it('counts entries, distinct workspaces, and total versions', () => {+ const stats = computeLocalCacheStats(+ new Map([+ ['orm:entity-metadatas:ws-a', entry(1)],+ ['flat-maps:field-metadata:ws-a', entry(2)],+ ['orm:entity-metadatas:ws-b', entry(1)],+ ]),+ );++ expect(stats.entries).toBe(3);+ // ws-a and ws-b — the same workspace under two providers counts once.+ expect(stats.workspaces).toBe(2);+ expect(stats.versionsTotal).toBe(4);+ });++ it('buckets entries by version count and folds 5+ together', () => {+ const stats = computeLocalCacheStats(+ new Map([+ ['p:ws-1', entry(1)],+ ['p:ws-2', entry(2)],+ ['p:ws-3', entry(4)],+ ['p:ws-4', entry(5)],+ ['p:ws-5', entry(9)],+ ]),+ );++ expect(stats.versionsByCount).toEqual({+ '1': 1,+ '2': 1,+ '3': 0,+ '4': 1,+ '5+': 2,+ });+ });+});diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/deep-size-bytes.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/deep-size-bytes.util.spec.tsnew file mode 100644index 00000000000..c2a57591d3f--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/deep-size-bytes.util.spec.ts@@ -0,0 +1,54 @@+import { deepSizeBytes } from 'src/engine/workspace-cache/utils/deep-size-bytes.util';++const CAP = 300_000;++describe('deepSizeBytes', () => {+ it('sizes primitives', () => {+ expect(deepSizeBytes(42, CAP)).toBe(8);+ expect(deepSizeBytes(true, CAP)).toBe(4);+ expect(deepSizeBytes(null, CAP)).toBe(0);+ expect(deepSizeBytes('ab', CAP)).toBe(12 + 2 * 2);+ });++ it('walks nested objects, arrays, Maps and Sets', () => {+ const withArray = deepSizeBytes({ items: [1, 2, 3] }, CAP);+ const withoutArray = deepSizeBytes({ items: [] }, CAP);++ expect(withArray).toBeGreaterThan(withoutArray);+ expect(deepSizeBytes(new Map([['k', 'v']]), CAP)).toBeGreaterThan(0);+ expect(deepSizeBytes(new Set([1, 2, 3]), CAP)).toBeGreaterThan(0);+ });++ it('is cycle-safe: a self-referential graph terminates and counts each object once', () => {+ const a: Record<string, unknown> = {};+ const b: Record<string, unknown> = { a };++ a.b = b;++ const cyclic = deepSizeBytes(a, CAP);+ // Same two objects, no cycle — the cyclic walk must not double-count.+ const acyclic = deepSizeBytes({ b: { value: 1 } }, CAP);++ expect(Number.isFinite(cyclic)).toBe(true);+ expect(cyclic).toBeGreaterThan(0);+ expect(cyclic).toBeLessThan(acyclic + 100);+ });++ it('stops at the node cap so a large graph cannot run unbounded', () => {+ const big = Array.from({ length: 10_000 }, (_, index) => ({ index }));++ const capped = deepSizeBytes(big, 5);+ const full = deepSizeBytes(big, CAP);++ expect(capped).toBeLessThan(full);+ });++ it('counts primitive nodes toward the cap, so a huge flat array is bounded too', () => {+ const manyNumbers = Array.from({ length: 10_000 }, (_, index) => index);++ const capped = deepSizeBytes(manyNumbers, 10);+ const full = deepSizeBytes(manyNumbers, CAP);++ expect(capped).toBeLessThan(full);+ });+});diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.tsnew file mode 100644index 00000000000..0c42624e930--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts@@ -0,0 +1,115 @@+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';+import {+ type LocalCacheSweepConfig,+ sweepLocalCache,+} from 'src/engine/workspace-cache/utils/sweep-local-cache.util';++const TTL_MS = 30 * 60 * 1000;+const ORM_PREFIX = 'orm:entity-metadatas';++const entry = (lastReadAt: number): WorkspaceLocalCacheEntry<unknown> => ({+ versions: new Map([['h', { data: {}, lastReadAt }]]),+ latestHash: 'h',+ lastHashCheckedAt: lastReadAt,+});++const config = (+ overrides: Partial<LocalCacheSweepConfig> = {},+): LocalCacheSweepConfig => ({+ ttlMs: TTL_MS,+ maxEntriesByPrefix: new Map(),+ globalMaxEntries: 6_000,+ minEvict: 0,+ ...overrides,+});++describe('sweepLocalCache', () => {+ it('drops entries idle past the TTL and keeps recent ones', () => {+ const cache = new Map([+ [`${ORM_PREFIX}:ws-idle`, entry(0)],+ [`${ORM_PREFIX}:ws-recent`, entry(TTL_MS)],+ ]);++ const evicted = sweepLocalCache(cache, TTL_MS + 1, config());++ expect(evicted).toBe(1);+ expect(cache.has(`${ORM_PREFIX}:ws-idle`)).toBe(false);+ expect(cache.has(`${ORM_PREFIX}:ws-recent`)).toBe(true);+ });++ it('trims a provider over its cap, evicting the least-recently-read', () => {+ const cache = new Map([+ [`${ORM_PREFIX}:ws-a`, entry(1)],+ [`${ORM_PREFIX}:ws-b`, entry(2)],+ [`${ORM_PREFIX}:ws-c`, entry(3)],+ ]);++ const evicted = sweepLocalCache(+ cache,+ 100,+ config({ maxEntriesByPrefix: new Map([[ORM_PREFIX, 2]]) }),+ );++ expect(evicted).toBe(1);+ expect(cache.size).toBe(2);+ expect(cache.has(`${ORM_PREFIX}:ws-a`)).toBe(false); // oldest+ expect(cache.has(`${ORM_PREFIX}:ws-c`)).toBe(true); // newest+ });++ it('leaves a provider without a cap untouched', () => {+ const cache = new Map(+ Array.from({ length: 5 }, (_, index) => [+ `feature-flag:feature-flags-map:ws-${index}`,+ entry(index + 1),+ ]),+ );++ const evicted = sweepLocalCache(+ cache,+ 100,+ config({ maxEntriesByPrefix: new Map([[ORM_PREFIX, 2]]) }),+ );++ expect(evicted).toBe(0);+ expect(cache.size).toBe(5);+ });++ it('trims the global total over the cap, batching by minEvict', () => {+ const cache = new Map(+ Array.from({ length: 10 }, (_, index) => [+ `feature-flag:feature-flags-map:ws-${index}`,+ entry(index + 1),+ ]),+ );++ const evicted = sweepLocalCache(+ cache,+ 100,+ config({ globalMaxEntries: 6, minEvict: 4 }),+ );++ // 10 > 6, and minEvict=4 means at least 4 go — the four least-recently-read.+ expect(evicted).toBe(4);+ expect(cache.size).toBe(6);+ expect(cache.has('feature-flag:feature-flags-map:ws-0')).toBe(false);+ expect(cache.has('feature-flag:feature-flags-map:ws-9')).toBe(true);+ });++ it('expires first so the per-provider cap only counts survivors', () => {+ const cache = new Map([+ [`${ORM_PREFIX}:ws-idle`, entry(0)], // expired by TTL+ [`${ORM_PREFIX}:ws-a`, entry(TTL_MS + 1)],+ [`${ORM_PREFIX}:ws-b`, entry(TTL_MS + 2)],+ ]);++ const evicted = sweepLocalCache(+ cache,+ TTL_MS + 10,+ config({ maxEntriesByPrefix: new Map([[ORM_PREFIX, 2]]) }),+ );++ // The idle one is expired; the two survivors are within the cap, so nothing else goes.+ expect(evicted).toBe(1);+ expect(cache.size).toBe(2);+ });+});diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.tsnew file mode 100644index 00000000000..d6d34ebb2cc--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts@@ -0,0 +1,37 @@+export type LocalCacheStats = {+ entries: number;+ workspaces: number;+ versionsTotal: number;+ versionsByCount: Record<string, number>;+};++export const computeLocalCacheStats = (+ localCache: ReadonlyMap<string, { versions: { size: number } }>,+): LocalCacheStats => {+ const workspaceIds = new Set<string>();+ const versionsByCount: Record<string, number> = {+ '1': 0,+ '2': 0,+ '3': 0,+ '4': 0,+ '5+': 0,+ };+ let versionsTotal = 0;++ for (const [key, entry] of localCache) {+ workspaceIds.add(key.slice(key.lastIndexOf(':') + 1));+ const versionCount = entry.versions.size;++ versionsTotal += versionCount;+ const bucket = versionCount >= 5 ? '5+' : String(versionCount);++ versionsByCount[bucket] = (versionsByCount[bucket] ?? 0) + 1;+ }++ return {+ entries: localCache.size,+ workspaces: workspaceIds.size,+ versionsTotal,+ versionsByCount,+ };+};diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/deep-size-bytes.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/deep-size-bytes.util.tsnew file mode 100644index 00000000000..f00068cf5c1--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/utils/deep-size-bytes.util.ts@@ -0,0 +1,75 @@+export const deepSizeBytes = (root: unknown, nodeCap: number): number => {+ const seen = new WeakSet<object>();+ const stack: unknown[] = [root];+ let bytes = 0;+ let visited = 0;++ while (stack.length > 0 && visited < nodeCap) {+ visited += 1;++ const value = stack.pop();++ if (typeof value === 'string') {+ bytes += 12 + value.length * 2;+ continue;+ }+ if (typeof value === 'number') {+ bytes += 8;+ continue;+ }+ if (typeof value === 'boolean') {+ bytes += 4;+ continue;+ }+ if (+ value === null ||+ (typeof value !== 'object' && typeof value !== 'function')+ ) {+ continue;+ }+ if (seen.has(value)) {+ continue;+ }+ seen.add(value);++ if (Array.isArray(value)) {+ bytes += 16 + value.length * 8;+ for (const item of value) {+ stack.push(item);+ }+ continue;+ }++ if (value instanceof Map) {+ bytes += 48 + value.size * 16;+ for (const [mapKey, mapValue] of value) {+ stack.push(mapKey);+ stack.push(mapValue);+ }+ continue;+ }++ if (value instanceof Set) {+ bytes += 48 + value.size * 8;+ for (const item of value) {+ stack.push(item);+ }+ continue;+ }++ bytes += 32;+ let entries: [string, unknown][] = [];++ try {+ entries = Object.entries(value);+ } catch {+ entries = [];+ }+ for (const [entryKey, child] of entries) {+ bytes += entryKey.length * 2 + 8;+ stack.push(child);+ }+ }++ return bytes;+};diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.tsnew file mode 100644index 00000000000..d6b7b87ee4f--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.ts@@ -0,0 +1,90 @@+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';++export type LocalCacheSweepConfig = {+ ttlMs: number;+ maxEntriesByPrefix: ReadonlyMap<string, number>;+ globalMaxEntries: number;+ minEvict: number;+};++const lastReadAtOf = <T>(entry: WorkspaceLocalCacheEntry<T>): number =>+ entry.versions.get(entry.latestHash)?.lastReadAt ?? 0;++const evictLeastRecentlyRead = <T>({+ localCache,+ matches,+ maxEntries,+ minEvict,+}: {+ localCache: Map<string, WorkspaceLocalCacheEntry<T>>;+ matches: (key: string) => boolean;+ maxEntries: number;+ minEvict: number;+}): number => {+ const matching: [string, WorkspaceLocalCacheEntry<T>][] = [];++ for (const keyEntry of localCache) {+ if (matches(keyEntry[0])) {+ matching.push(keyEntry);+ }+ }++ if (matching.length <= maxEntries) {+ return 0;+ }++ matching.sort((a, b) => lastReadAtOf(a[1]) - lastReadAtOf(b[1]));++ const evictCount = Math.min(+ matching.length,+ Math.max(minEvict, matching.length - maxEntries),+ );++ for (let index = 0; index < evictCount; index += 1) {+ localCache.delete(matching[index][0]);+ }++ return evictCount;+};++// Mutates the cache: expire versions idle past the TTL (dropping any entry left without a current+// version), then trim each capped provider and the global total to budget by least-recently-read.+// Returns the number of entries evicted.+export const sweepLocalCache = <T>(+ localCache: Map<string, WorkspaceLocalCacheEntry<T>>,+ now: number,+ config: LocalCacheSweepConfig,+): number => {+ let evicted = 0;++ for (const [localKey, entry] of localCache) {+ for (const [hash, version] of entry.versions) {+ if (now - version.lastReadAt > config.ttlMs) {+ entry.versions.delete(hash);+ }+ }++ if (entry.versions.size === 0 || !entry.versions.has(entry.latestHash)) {+ localCache.delete(localKey);+ evicted += 1;+ }+ }++ for (const [prefix, maxEntries] of config.maxEntriesByPrefix) {+ evicted += evictLeastRecentlyRead({+ localCache,+ matches: (key) => key.startsWith(`${prefix}:`),+ maxEntries,+ minEvict: 0,+ });+ }++ evicted += evictLeastRecentlyRead({+ localCache,+ matches: () => true,+ maxEntries: config.globalMaxEntries,+ minEvict: config.minEvict,+ });++ return evicted;+};diff --git a/packages/twenty-server/src/engine/workspace-cache/workspace-cache.module.ts b/packages/twenty-server/src/engine/workspace-cache/workspace-cache.module.tsindex a873207fc54..53498de1d76 100644--- a/packages/twenty-server/src/engine/workspace-cache/workspace-cache.module.ts+++ b/packages/twenty-server/src/engine/workspace-cache/workspace-cache.module.ts@@ -3,11 +3,12 @@ import { DiscoveryModule } from '@nestjs/core';import { CacheStorageModule } from 'src/engine/core-modules/cache-storage/cache-storage.module';import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';+import { WorkspaceCacheMetricsService } from 'src/engine/workspace-cache/services/workspace-cache-metrics.service';import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';@Module({imports: [CacheStorageModule, DiscoveryModule, MetricsModule],- providers: [WorkspaceCacheService],+ providers: [WorkspaceCacheService, WorkspaceCacheMetricsService],exports: [WorkspaceCacheService],})export class WorkspaceCacheModule {}