Skip to content

The open alternative to Salesforce, designed for AI.

feat(server): instrument the local metadata cache and cap heavy providers by entry count (#23778)

Charles Bochet6abeb7b5e50c
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.ts b/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.ts
deleted file mode 100644
index 7b9d3106b51..00000000000
--- a/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.ts
+++ /dev/null
@@ -1,736 +0,0 @@
-import { DiscoveryService, Reflector } from '@nestjs/core';
-import { Test, type TestingModule } from '@nestjs/testing';
-
-import * as Sentry from '@sentry/node';
-
-import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';
-
-import { type CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
-import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
-import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
-import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
-import {
- WORKSPACE_CACHE_KEY,
- WORKSPACE_CACHE_OPTIONS,
-} from 'src/engine/workspace-cache/decorators/workspace-cache.decorator';
-import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
-
-jest.mock('@sentry/node', () => ({
- startSpan: jest.fn(),
-}));
-
-const WORKSPACE_ID = '20202020-0000-4000-8000-000000000000';
-
-class MockFeatureFlagsCacheProvider extends WorkspaceCacheProvider<{
- testData: string;
-}> {
- async computeForCache(_workspaceId: string) {
- return { testData: 'computed-value' };
- }
-}
-
-class MockRolesPermissionsCacheProvider extends WorkspaceCacheProvider<{
- testData: string;
-}> {
- async computeForCache(_workspaceId: string) {
- return { testData: 'computed-value' };
- }
-}
-
-class MockOrmEntityMetadatasCacheProvider extends WorkspaceCacheProvider<{
- testData: string;
-}> {
- async computeForCache(_workspaceId: string) {
- return { testData: 'orm-computed-value' };
- }
-}
-
-describe('WorkspaceCacheService', () => {
- let service: WorkspaceCacheService;
- let cacheStorageService: jest.Mocked<CacheStorageService>;
- let discoveryService: jest.Mocked<DiscoveryService>;
- let reflector: jest.Mocked<Reflector>;
- let mockProvider: MockFeatureFlagsCacheProvider;
-
- beforeEach(async () => {
- jest.useFakeTimers();
- jest
- .mocked(Sentry.startSpan)
- .mockImplementation((_options, callback) => callback({} as never));
-
- mockProvider = new MockFeatureFlagsCacheProvider();
-
- const module: TestingModule = await Test.createTestingModule({
- providers: [
- WorkspaceCacheService,
- {
- provide: CacheStorageNamespace.EngineWorkspace,
- useValue: {
- mget: jest.fn(),
- mset: jest.fn(),
- mdel: jest.fn(),
- setIfAbsent: jest.fn(),
- },
- },
- {
- provide: DiscoveryService,
- useValue: {
- getProviders: jest.fn(),
- },
- },
- {
- provide: Reflector,
- useValue: {
- get: jest.fn(),
- },
- },
- {
- provide: MetricsService,
- useValue: {
- incrementCounterBy: jest.fn(),
- getMeter: jest.fn().mockReturnValue({
- createHistogram: jest.fn().mockReturnValue({ record: jest.fn() }),
- }),
- },
- },
- {
- provide: TwentyConfigService,
- useValue: {
- get: jest.fn().mockReturnValue(604800),
- },
- },
- ],
- }).compile();
-
- service = module.get<WorkspaceCacheService>(WorkspaceCacheService);
- cacheStorageService = module.get(CacheStorageNamespace.EngineWorkspace);
- discoveryService = module.get(DiscoveryService);
- reflector = module.get(Reflector);
- });
-
- afterEach(() => {
- jest.useRealTimers();
- jest.clearAllMocks();
- });
-
- describe('onModuleInit', () => {
- it('should register workspace cache providers', async () => {
- discoveryService.getProviders.mockReturnValue([
- { instance: mockProvider },
- ] as any);
-
- reflector.get.mockImplementation((key, target) => {
- if (
- key === WORKSPACE_CACHE_KEY &&
- target === MockFeatureFlagsCacheProvider
- ) {
- return 'featureFlagsMap';
- }
-
- return undefined;
- });
-
- await service.onModuleInit();
-
- expect(discoveryService.getProviders).toHaveBeenCalled();
- expect(reflector.get).toHaveBeenCalled();
- });
-
- it('should skip non-object instances', async () => {
- discoveryService.getProviders.mockReturnValue([
- { instance: null },
- { instance: undefined },
- { instance: 'string-value' },
- ] as any);
-
- await service.onModuleInit();
-
- expect(reflector.get).not.toHaveBeenCalled();
- });
-
- it('should skip instances without workspace cache key metadata', async () => {
- discoveryService.getProviders.mockReturnValue([
- { instance: mockProvider },
- ] as any);
-
- reflector.get.mockReturnValue(undefined);
-
- await service.onModuleInit();
-
- expect(reflector.get).toHaveBeenCalled();
- });
- });
-
- describe('getOrRecompute', () => {
- beforeEach(async () => {
- discoveryService.getProviders.mockReturnValue([
- { instance: mockProvider },
- ] as any);
-
- reflector.get.mockImplementation((key, target) => {
- if (
- key === WORKSPACE_CACHE_KEY &&
- target === MockFeatureFlagsCacheProvider
- ) {
- return 'featureFlagsMap';
- }
-
- return undefined;
- });
-
- await service.onModuleInit();
- });
-
- it('should compute and cache data when redis cache is empty', async () => {
- cacheStorageService.mget.mockResolvedValue([undefined]);
- cacheStorageService.mset.mockResolvedValue(undefined);
-
- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({
- testData: 'fresh-computed-value',
- });
-
- const result = await service.getOrRecompute(WORKSPACE_ID, [
- 'featureFlagsMap',
- ]);
-
- expect(result).toEqual({
- featureFlagsMap: { testData: 'fresh-computed-value' },
- });
- expect(mockProvider.computeForCache).toHaveBeenCalledWith(WORKSPACE_ID);
- expect(cacheStorageService.mset).toHaveBeenCalled();
- expect(Sentry.startSpan).toHaveBeenCalledWith(
- {
- name: 'compute workspace metadata cache entry from provider',
- op: 'cache.recompute',
- onlyIfParent: true,
- attributes: {
- 'cache.key_name': 'featureFlagsMap',
- 'cache.recompute.strategy': 'recover',
- 'cache.local_data_only': false,
- },
- },
- expect.any(Function),
- );
- });
-
- it('should return data from redis when available', async () => {
- const cachedData = { FLAG_A: true, FLAG_B: false };
- const cachedHash = 'some-hash-from-redis';
-
- cacheStorageService.mget
- // First call: validateLocalHashAgainstRedisHash checks hash
- .mockResolvedValueOnce([undefined])
- // Second call: fetchDataFromRedis fetches data and hash atomically
- .mockResolvedValueOnce([cachedData, cachedHash]);
-
- const result = await service.getOrRecompute(WORKSPACE_ID, [
- 'featureFlagsMap',
- ]);
-
- expect(result).toEqual({ featureFlagsMap: cachedData });
- expect(Sentry.startSpan).not.toHaveBeenCalled();
- });
-
- it('should use local cache when within TTL staleness window', async () => {
- cacheStorageService.mget.mockResolvedValue([undefined]);
- cacheStorageService.mset.mockResolvedValue(undefined);
-
- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({
- testData: 'computed-value',
- });
-
- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);
-
- jest.advanceTimersByTime(50);
-
- const result = await service.getOrRecompute(WORKSPACE_ID, [
- 'featureFlagsMap',
- ]);
-
- expect(result).toEqual({
- featureFlagsMap: { testData: 'computed-value' },
- });
-
- expect(mockProvider.computeForCache).toHaveBeenCalledTimes(1);
- });
-
- it('should recheck redis when local cache exceeds TTL staleness window', async () => {
- const initialData = { testData: 'initial-value' };
- const updatedData = { testData: 'updated-value' };
-
- cacheStorageService.mget.mockResolvedValue([undefined]);
- cacheStorageService.mset.mockResolvedValue(undefined);
-
- jest
- .spyOn(mockProvider, 'computeForCache')
- .mockResolvedValue(initialData);
-
- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);
-
- // Advance past the local staleness TTL (100ms) and memoizer TTL (10s)
- jest.advanceTimersByTime(15_000);
-
- cacheStorageService.mget.mockResolvedValue([updatedData]);
-
- jest
- .spyOn(mockProvider, 'computeForCache')
- .mockResolvedValue(updatedData);
-
- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);
-
- // Verify Redis was rechecked after TTL expired
- // Each getOrRecompute triggers: 1 hash check + 1 atomic data/hash fetch = 2 mget calls
- // Total: 4 mget calls (2 per getOrRecompute)
- expect(cacheStorageService.mget).toHaveBeenCalledTimes(4);
- });
- });
-
- describe('local cache expiration sweep', () => {
- it('should run at most once per minute', async () => {
- const expirationSweepSpy = jest.spyOn(
- service as unknown as {
- evictExpiredLocalEntries: (now: number) => void;
- },
- 'evictExpiredLocalEntries',
- );
-
- await expect(
- service.getOrRecompute('invalid-workspace-id', ['featureFlagsMap']),
- ).rejects.toThrow();
- await expect(
- service.getOrRecompute('invalid-workspace-id', ['featureFlagsMap']),
- ).rejects.toThrow();
-
- expect(expirationSweepSpy).toHaveBeenCalledTimes(1);
-
- jest.advanceTimersByTime(60_000);
-
- await expect(
- service.getOrRecompute('invalid-workspace-id', ['featureFlagsMap']),
- ).rejects.toThrow();
-
- expect(expirationSweepSpy).toHaveBeenCalledTimes(2);
- });
- });
-
- describe('invalidateAndRecompute', () => {
- beforeEach(async () => {
- discoveryService.getProviders.mockReturnValue([
- { instance: mockProvider },
- ] as any);
-
- reflector.get.mockImplementation((key, target) => {
- if (
- key === WORKSPACE_CACHE_KEY &&
- target === MockFeatureFlagsCacheProvider
- ) {
- return 'featureFlagsMap';
- }
-
- return undefined;
- });
-
- await service.onModuleInit();
- });
-
- it('should delete from redis, mark local cache as stale, and recompute', async () => {
- cacheStorageService.mdel.mockResolvedValue(undefined);
- cacheStorageService.mset.mockResolvedValue(undefined);
-
- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({
- testData: 'recomputed-value',
- });
-
- await service.invalidateAndRecompute(WORKSPACE_ID, ['featureFlagsMap']);
-
- expect(cacheStorageService.mdel).toHaveBeenCalledWith([
- 'feature-flag:feature-flags-map:20202020-0000-4000-8000-000000000000:data',
- 'feature-flag:feature-flags-map:20202020-0000-4000-8000-000000000000:hash',
- ]);
- expect(mockProvider.computeForCache).toHaveBeenCalledWith(WORKSPACE_ID);
- expect(cacheStorageService.mset).toHaveBeenCalled();
- expect(Sentry.startSpan).toHaveBeenCalledWith(
- {
- name: 'invalidate and recompute workspace metadata cache',
- op: 'cache.invalidate',
- onlyIfParent: true,
- attributes: { 'cache.key_count': 1 },
- },
- expect.any(Function),
- );
- });
-
- it('should invalidate multiple cache keys at once', async () => {
- const secondMockProvider = new MockRolesPermissionsCacheProvider();
-
- discoveryService.getProviders.mockReturnValue([
- { instance: mockProvider },
- { instance: secondMockProvider },
- ] as any);
-
- reflector.get.mockImplementation((key, target) => {
- if (key === WORKSPACE_CACHE_KEY) {
- if (target === MockFeatureFlagsCacheProvider) {
- return 'featureFlagsMap';
- }
- if (target === MockRolesPermissionsCacheProvider) {
- return 'rolesPermissions';
- }
- }
-
- return undefined;
- });
-
- await service.onModuleInit();
-
- cacheStorageService.mdel.mockResolvedValue(undefined);
- cacheStorageService.mset.mockResolvedValue(undefined);
-
- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({
- testData: 'recomputed-value',
- });
- jest.spyOn(secondMockProvider, 'computeForCache').mockResolvedValue({
- testData: 'recomputed-value',
- });
-
- await service.invalidateAndRecompute(WORKSPACE_ID, [
- 'featureFlagsMap',
- 'rolesPermissions',
- ]);
-
- expect(cacheStorageService.mdel).toHaveBeenCalledWith(
- expect.arrayContaining([
- 'feature-flag:feature-flags-map:20202020-0000-4000-8000-000000000000:data',
- 'feature-flag:feature-flags-map:20202020-0000-4000-8000-000000000000:hash',
- 'metadata:permissions:roles-permissions:20202020-0000-4000-8000-000000000000:data',
- 'metadata:permissions:roles-permissions:20202020-0000-4000-8000-000000000000:hash',
- ]),
- );
- });
-
- it('should keep old versions in local cache after invalidation for race condition safety', async () => {
- cacheStorageService.mget.mockResolvedValue([undefined]);
- cacheStorageService.mset.mockResolvedValue(undefined);
- cacheStorageService.mdel.mockResolvedValue(undefined);
-
- const initialData = { testData: 'initial-value' };
- const recomputedData = { testData: 'recomputed-value' };
-
- jest
- .spyOn(mockProvider, 'computeForCache')
- .mockResolvedValue(initialData);
-
- // First, populate the cache
- const firstResult = await service.getOrRecompute(WORKSPACE_ID, [
- 'featureFlagsMap',
- ]);
-
- expect(firstResult).toEqual({ featureFlagsMap: initialData });
-
- // Now invalidate and recompute
- jest
- .spyOn(mockProvider, 'computeForCache')
- .mockResolvedValue(recomputedData);
-
- await service.invalidateAndRecompute(WORKSPACE_ID, ['featureFlagsMap']);
-
- // The new value should be returned
- const secondResult = await service.getOrRecompute(WORKSPACE_ID, [
- 'featureFlagsMap',
- ]);
-
- expect(secondResult).toEqual({ featureFlagsMap: recomputedData });
- });
- });
-
- describe('flush', () => {
- it('should delete from redis and mark local cache as stale', async () => {
- cacheStorageService.mdel.mockResolvedValue(undefined);
-
- await service.flush(WORKSPACE_ID, ['featureFlagsMap']);
-
- expect(cacheStorageService.mdel).toHaveBeenCalledWith([
- 'feature-flag:feature-flags-map:20202020-0000-4000-8000-000000000000:data',
- 'feature-flag:feature-flags-map:20202020-0000-4000-8000-000000000000:hash',
- ]);
- });
-
- it('should handle empty cache keys array', async () => {
- cacheStorageService.mdel.mockResolvedValue(undefined);
-
- await service.flush(WORKSPACE_ID, []);
-
- expect(cacheStorageService.mdel).toHaveBeenCalledWith([]);
- });
-
- it('should force staleness on local cache entries without deleting them', async () => {
- cacheStorageService.mget.mockResolvedValue([undefined]);
- cacheStorageService.mset.mockResolvedValue(undefined);
- cacheStorageService.mdel.mockResolvedValue(undefined);
-
- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({
- testData: 'computed-value',
- });
-
- discoveryService.getProviders.mockReturnValue([
- { instance: mockProvider },
- ] as any);
-
- reflector.get.mockImplementation((key, target) => {
- if (
- key === WORKSPACE_CACHE_KEY &&
- target === MockFeatureFlagsCacheProvider
- ) {
- return 'featureFlagsMap';
- }
-
- return undefined;
- });
-
- await service.onModuleInit();
-
- // Populate the cache
- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);
-
- // Flush the cache (marks as stale, doesn't delete)
- await service.flush(WORKSPACE_ID, ['featureFlagsMap']);
-
- // Advance time slightly (but still within memoizer TTL)
- jest.advanceTimersByTime(50);
-
- // Next call should check Redis since local cache is marked stale
- cacheStorageService.mget.mockResolvedValue(['some-hash']);
-
- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);
-
- // Should have made additional mget calls to check Redis
- expect(cacheStorageService.mget.mock.calls.length).toBeGreaterThan(1);
- });
- });
-
- describe('versioning behavior', () => {
- beforeEach(async () => {
- discoveryService.getProviders.mockReturnValue([
- { instance: mockProvider },
- ] as any);
-
- reflector.get.mockImplementation((key, target) => {
- if (
- key === WORKSPACE_CACHE_KEY &&
- target === MockFeatureFlagsCacheProvider
- ) {
- return 'featureFlagsMap';
- }
-
- return undefined;
- });
-
- await service.onModuleInit();
- });
-
- it('should store multiple versions when data is recomputed', async () => {
- cacheStorageService.mget.mockResolvedValue([undefined]);
- cacheStorageService.mset.mockResolvedValue(undefined);
- cacheStorageService.mdel.mockResolvedValue(undefined);
-
- const firstData = { testData: 'first-value' };
- const secondData = { testData: 'second-value' };
-
- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue(firstData);
-
- // First computation
- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);
-
- // Invalidate and recompute with new data
- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue(secondData);
-
- await service.invalidateAndRecompute(WORKSPACE_ID, ['featureFlagsMap']);
-
- // Should return the latest version
- const result = await service.getOrRecompute(WORKSPACE_ID, [
- 'featureFlagsMap',
- ]);
-
- expect(result).toEqual({ featureFlagsMap: secondData });
- });
-
- it('should cleanup stale versions after TTL expires', async () => {
- cacheStorageService.mget.mockResolvedValue([undefined]);
- cacheStorageService.mset.mockResolvedValue(undefined);
- cacheStorageService.mdel.mockResolvedValue(undefined);
-
- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({
- testData: 'value-1',
- });
-
- // Create first version
- await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);
-
- // Create multiple versions by invalidating
- for (let i = 2; i <= 4; i++) {
- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({
- testData: `value-${i}`,
- });
- await service.invalidateAndRecompute(WORKSPACE_ID, ['featureFlagsMap']);
- }
-
- // Advance time past the stale version TTL (5000ms)
- jest.advanceTimersByTime(6_000);
-
- // Advance past memoizer TTL as well
- jest.advanceTimersByTime(15_000);
-
- // Trigger a read which should cleanup stale versions
- jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({
- testData: 'latest-value',
- });
-
- const result = await service.getOrRecompute(WORKSPACE_ID, [
- 'featureFlagsMap',
- ]);
-
- expect(result).toEqual({ featureFlagsMap: { testData: 'latest-value' } });
- });
- });
-
- describe('localDataOnly hash recovery', () => {
- let localDataOnlyProvider: MockOrmEntityMetadatasCacheProvider;
- const ormHashKey = `orm:entity-metadatas:${WORKSPACE_ID}:hash`;
- const ormDataKey = `orm:entity-metadatas:${WORKSPACE_ID}:data`;
-
- beforeEach(async () => {
- localDataOnlyProvider = new MockOrmEntityMetadatasCacheProvider();
-
- discoveryService.getProviders.mockReturnValue([
- { instance: localDataOnlyProvider },
- ] as any);
-
- reflector.get.mockImplementation((key, target) => {
- if (target !== MockOrmEntityMetadatasCacheProvider) {
- return undefined;
- }
-
- if (key === WORKSPACE_CACHE_KEY) {
- return 'ORMEntityMetadatas';
- }
-
- if (key === WORKSPACE_CACHE_OPTIONS) {
- return { localDataOnly: true };
- }
-
- return undefined;
- });
-
- await service.onModuleInit();
- });
-
- it('should adopt the existing redis hash when recomputing instead of minting a new one', async () => {
- cacheStorageService.mget.mockResolvedValue(['hash-from-another-pod']);
- cacheStorageService.mset.mockResolvedValue(undefined);
-
- const computeSpy = jest.spyOn(localDataOnlyProvider, 'computeForCache');
-
- const result = await service.getOrRecompute(WORKSPACE_ID, [
- 'ORMEntityMetadatas',
- ]);
-
- expect(result).toEqual({
- ORMEntityMetadatas: { testData: 'orm-computed-value' },
- });
- expect(computeSpy).toHaveBeenCalledTimes(1);
- expect(cacheStorageService.mset).not.toHaveBeenCalled();
- });
-
- it('should treat the local copy as valid on later reads after adopting the redis hash', async () => {
- cacheStorageService.mget.mockResolvedValue(['hash-from-another-pod']);
- cacheStorageService.mset.mockResolvedValue(undefined);
-
- const computeSpy = jest.spyOn(localDataOnlyProvider, 'computeForCache');
-
- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);
-
- jest.advanceTimersByTime(15_000);
-
- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);
-
- expect(computeSpy).toHaveBeenCalledTimes(1);
- expect(cacheStorageService.mset).not.toHaveBeenCalled();
- });
-
- it('should recompute and adopt the new hash when another pod invalidated the key', async () => {
- cacheStorageService.mget.mockResolvedValue(['hash-v1']);
- cacheStorageService.mset.mockResolvedValue(undefined);
-
- const computeSpy = jest.spyOn(localDataOnlyProvider, 'computeForCache');
-
- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);
-
- jest.advanceTimersByTime(15_000);
-
- cacheStorageService.mget.mockResolvedValue(['hash-v2']);
-
- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);
-
- expect(computeSpy).toHaveBeenCalledTimes(2);
- expect(cacheStorageService.mset).not.toHaveBeenCalled();
- });
-
- it('should mint a hash and persist it only if still absent when redis has none', async () => {
- cacheStorageService.mget.mockResolvedValue([undefined]);
- cacheStorageService.setIfAbsent.mockResolvedValue(true);
-
- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);
-
- expect(cacheStorageService.setIfAbsent).toHaveBeenCalledWith(
- ormHashKey,
- expect.any(String),
- 604800000,
- );
- expect(cacheStorageService.mset).not.toHaveBeenCalled();
- });
-
- it('should converge on a hash written concurrently by another pod instead of overwriting it', async () => {
- cacheStorageService.mget.mockResolvedValue([undefined]);
- cacheStorageService.setIfAbsent.mockResolvedValue(false);
-
- const computeSpy = jest.spyOn(localDataOnlyProvider, 'computeForCache');
-
- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);
-
- expect(cacheStorageService.setIfAbsent).toHaveBeenCalledTimes(1);
- expect(cacheStorageService.mset).not.toHaveBeenCalled();
-
- jest.advanceTimersByTime(15_000);
-
- cacheStorageService.mget.mockResolvedValue(['winner-hash']);
-
- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);
-
- expect(computeSpy).toHaveBeenCalledTimes(2);
- expect(cacheStorageService.setIfAbsent).toHaveBeenCalledTimes(1);
-
- jest.advanceTimersByTime(15_000);
-
- await service.getOrRecompute(WORKSPACE_ID, ['ORMEntityMetadatas']);
-
- expect(computeSpy).toHaveBeenCalledTimes(2);
- });
-
- it('should mint a new hash on invalidateAndRecompute', async () => {
- cacheStorageService.mdel.mockResolvedValue(undefined);
- cacheStorageService.mset.mockResolvedValue(undefined);
-
- await service.invalidateAndRecompute(WORKSPACE_ID, [
- 'ORMEntityMetadatas',
- ]);
-
- expect(cacheStorageService.mdel).toHaveBeenCalledWith([
- ormDataKey,
- ormHashKey,
- ]);
- expect(cacheStorageService.mset).toHaveBeenCalledWith([
- { key: ormHashKey, value: expect.any(String) },
- ]);
- });
- });
-});
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
new file mode 100644
index 00000000000..f5519f1e992
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
@@ -0,0 +1,254 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { type Histogram } from '@opentelemetry/api';
+
+import { isDefined } from 'twenty-shared/utils';
+
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
+import {
+ type WorkspaceCacheDataMap,
+ type WorkspaceCacheKeyName,
+} from 'src/engine/workspace-cache/types/workspace-cache-key.type';
+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
+import {
+ type LocalCacheStats,
+ computeLocalCacheStats,
+} from 'src/engine/workspace-cache/utils/compute-local-cache-stats.util';
+import { deepSizeBytes } from 'src/engine/workspace-cache/utils/deep-size-bytes.util';
+
+type LocalCache = ReadonlyMap<
+ string,
+ WorkspaceLocalCacheEntry<WorkspaceCacheDataMap[WorkspaceCacheKeyName]>
+>;
+
+const CACHE_DURATION_BUCKETS_SECONDS = [
+ 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
+];
+const STATS_TTL_MS = 5_000;
+const SIZE_REFRESH_MS = 5 * 60 * 1000;
+const SIZE_STARTUP_DELAY_MS = 30 * 1000;
+const SIZE_SAMPLE_PER_PROVIDER = 3;
+const SIZE_WALK_NODE_CAP = 300_000;
+
+// Cache observability, split from WorkspaceCacheService (which drives it via start()/stop()).
+@Injectable()
+export class WorkspaceCacheMetricsService {
+ private readonly logger = new Logger(WorkspaceCacheMetricsService.name);
+ private readonly recomputeDurationHistogram: Histogram;
+ private readonly redisWriteDurationHistogram: Histogram;
+
+ private localCache?: LocalCache;
+ private cacheSizeByProvider: Record<string, number> = {};
+ private cacheSizeTotalBytes = 0;
+ private sizeSampler?: ReturnType<typeof setInterval>;
+ private sizeStartupTimer?: ReturnType<typeof setTimeout>;
+ private statsCache?: { computedAt: number } & LocalCacheStats;
+ private sizeSampleInFlight = false;
+
+ constructor(private readonly metricsService: MetricsService) {
+ const meter = this.metricsService.getMeter();
+
+ this.recomputeDurationHistogram = meter.createHistogram(
+ 'twenty_workspace_cache_recompute_duration_seconds',
+ {
+ description:
+ 'Wall-clock time to compute one workspace metadata cache entry from its provider',
+ unit: 's',
+ advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },
+ },
+ );
+ this.redisWriteDurationHistogram = meter.createHistogram(
+ 'twenty_workspace_cache_redis_write_duration_seconds',
+ {
+ description:
+ 'Wall-clock time to serialize and write recomputed cache entries to Redis',
+ unit: 's',
+ advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },
+ },
+ );
+ }
+
+ start(localCache: LocalCache): void {
+ this.localCache = localCache;
+ this.registerGauges();
+ this.scheduleSizeSampler();
+ }
+
+ stop(): void {
+ if (isDefined(this.sizeStartupTimer)) {
+ clearTimeout(this.sizeStartupTimer);
+ }
+ if (isDefined(this.sizeSampler)) {
+ clearInterval(this.sizeSampler);
+ }
+ }
+
+ recordRecompute(seconds: number, cacheKey: WorkspaceCacheKeyName): void {
+ this.recomputeDurationHistogram.record(seconds, { cache_key: cacheKey });
+ }
+
+ recordRedisWrite(seconds: number): void {
+ this.redisWriteDurationHistogram.record(seconds);
+ }
+
+ recordEviction(amount: number): void {
+ this.metricsService.incrementCounterBy({
+ key: MetricsKeys.WorkspaceMetadataCacheLocalEviction,
+ amount,
+ });
+ }
+
+ private getStats(): LocalCacheStats {
+ const now = Date.now();
+
+ if (
+ isDefined(this.statsCache) &&
+ now - this.statsCache.computedAt < STATS_TTL_MS
+ ) {
+ return this.statsCache;
+ }
+
+ const stats = computeLocalCacheStats(this.localCache ?? new Map());
+
+ this.statsCache = { computedAt: now, ...stats };
+
+ return this.statsCache;
+ }
+
+ private scheduleSizeSampler(): void {
+ const sample = (): void => {
+ if (this.sizeSampleInFlight) {
+ return;
+ }
+
+ this.sizeSampleInFlight = true;
+ this.refreshSizeBreakdown()
+ .catch((error) =>
+ this.logger.error('Failed to sample local cache size', error),
+ )
+ .finally(() => {
+ this.sizeSampleInFlight = false;
+ });
+ };
+
+ // Prime once after startup so the gauges aren't 0 until the first 5-minute interval.
+ this.sizeStartupTimer = setTimeout(sample, SIZE_STARTUP_DELAY_MS);
+ this.sizeStartupTimer.unref();
+
+ this.sizeSampler = setInterval(sample, SIZE_REFRESH_MS);
+ this.sizeSampler.unref();
+ }
+
+ private async refreshSizeBreakdown(): Promise<void> {
+ const localCache = this.localCache;
+
+ if (!isDefined(localCache)) {
+ return;
+ }
+
+ const perProvider: Record<
+ string,
+ { count: number; sampledBytes: number; sampled: number }
+ > = {};
+
+ for (const [key, entry] of localCache) {
+ const provider = key.slice(0, key.lastIndexOf(':'));
+ const stats = (perProvider[provider] ??= {
+ count: 0,
+ sampledBytes: 0,
+ sampled: 0,
+ });
+
+ stats.count += 1;
+
+ if (stats.sampled < SIZE_SAMPLE_PER_PROVIDER && entry.versions.size > 0) {
+ // Size every retained version, not just the latest — stale versions still occupy heap.
+ let entryBytes = 0;
+
+ for (const version of entry.versions.values()) {
+ entryBytes += deepSizeBytes(version.data, SIZE_WALK_NODE_CAP);
+ }
+
+ stats.sampledBytes += entryBytes;
+ stats.sampled += 1;
+ await new Promise((resolve) => setImmediate(resolve));
+ }
+ }
+
+ const byProvider: Record<string, number> = {};
+ let total = 0;
+
+ for (const [provider, stats] of Object.entries(perProvider)) {
+ const estimate =
+ stats.sampled === 0
+ ? 0
+ : Math.round((stats.sampledBytes / stats.sampled) * stats.count);
+
+ byProvider[provider] = estimate;
+ total += estimate;
+ }
+
+ this.cacheSizeByProvider = byProvider;
+ this.cacheSizeTotalBytes = total;
+ }
+
+ private registerGauges(): void {
+ this.metricsService.createObservableGauge({
+ metricName: 'twenty_workspace_cache_local_entries',
+ options: {
+ description: 'Entries in the per-pod local workspace metadata cache',
+ },
+ callback: async () => this.getStats().entries,
+ });
+ this.metricsService.createObservableGauge({
+ metricName: 'twenty_workspace_cache_local_workspaces',
+ options: {
+ description:
+ 'Distinct workspaces held in the per-pod local workspace metadata cache',
+ },
+ callback: async () => this.getStats().workspaces,
+ });
+ this.metricsService.createObservableGauge({
+ metricName: 'twenty_workspace_cache_local_versions_total',
+ options: {
+ description:
+ 'Total versions across local workspace metadata cache entries',
+ },
+ callback: async () => this.getStats().versionsTotal,
+ });
+ this.metricsService.createObservableGauge({
+ metricName: 'twenty_workspace_cache_local_bytes_estimate',
+ options: {
+ description:
+ 'Estimated retained bytes (deep-size, includes local-only providers) in the local workspace metadata cache',
+ unit: 'By',
+ },
+ callback: async () => this.cacheSizeTotalBytes,
+ });
+ this.metricsService.createMultiObservableGauge({
+ metricName: 'twenty_workspace_cache_local_entries_by_version_count',
+ options: {
+ description:
+ 'Local workspace metadata cache entries bucketed by version count',
+ },
+ callback: async () =>
+ Object.entries(this.getStats().versionsByCount).map(
+ ([versions, value]) => ({ value, attributes: { versions } }),
+ ),
+ });
+ this.metricsService.createMultiObservableGauge({
+ metricName: 'twenty_workspace_cache_local_bytes_by_provider',
+ options: {
+ description:
+ 'Estimated retained bytes in the local workspace metadata cache per provider',
+ unit: 'By',
+ },
+ callback: async () =>
+ Object.entries(this.cacheSizeByProvider).map(([provider, value]) => ({
+ value,
+ attributes: { provider },
+ })),
+ });
+ }
+}
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
index b363d7d5a65..2c25a8e71b6 100644
--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
@@ -1,7 +1,11 @@
-import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+import {
+ Injectable,
+ Logger,
+ OnModuleDestroy,
+ OnModuleInit,
+} from '@nestjs/common';
import { DiscoveryService, Reflector } from '@nestjs/core';
-import { type Histogram } from '@opentelemetry/api';
import * as Sentry from '@sentry/node';
import crypto from 'crypto';
@@ -12,10 +16,9 @@ import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/wo
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
-import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
-import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { PromiseMemoizer } from 'src/engine/twenty-orm/storage/promise-memoizer.storage';
+import { WorkspaceCacheMetricsService } from 'src/engine/workspace-cache/services/workspace-cache-metrics.service';
import {
WORKSPACE_CACHE_KEY,
WORKSPACE_CACHE_OPTIONS,
@@ -34,20 +37,21 @@ import {
} from 'src/engine/workspace-cache/types/workspace-cache-key.type';
import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
import { combineCacheHashes } from 'src/engine/workspace-cache/utils/combine-cache-hashes.util';
+import { sweepLocalCache } from 'src/engine/workspace-cache/utils/sweep-local-cache.util';
const LOCAL_TTL_MS = 100; // 100ms
-const LOCAL_ENTRY_TTL_MS = 30 * 60 * 1000; // 30 minutes
-const LOCAL_CACHE_EXPIRATION_SWEEP_INTERVAL_MS = 60 * 1000;
const MEMOIZER_TTL_MS = 10_000; // 10 seconds
const STALE_VERSION_TTL_MS = 5_000; // 5 seconds
const MAX_LOCAL_STALE_VERSIONS = 5; // 5 stale versions
-// Sized against 4 GiB pods (--max-old-space-size=3500): 7,500 sat at the heap ceiling
+// Sized against 4 GiB pods (--max-old-space-size=3500): 7,500 sat at the heap ceiling.
const MAX_LOCAL_CACHE_ENTRIES = 6_000;
const MIN_EVICT_KEYS = 100;
-const CACHE_DURATION_BUCKETS_SECONDS = [
- 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
-];
-
+const LOCAL_ENTRY_TTL_MS = 30 * 60 * 1000; // 30 minutes idle
+const LOCAL_CACHE_SWEEP_INTERVAL_MS = 60 * 1000;
+// Per-provider entry caps, keyed by local cache key prefix (ORM graphs are ~5 MB each).
+const MAX_LOCAL_ENTRIES_BY_PREFIX = new Map<string, number>([
+ [WORKSPACE_CACHE_KEYS_V2.ORMEntityMetadatas, 128],
+]);
type CacheDataType = WorkspaceCacheDataMap[WorkspaceCacheKeyName];
type CacheEntriesResult = {
@@ -63,11 +67,12 @@ type RecomputeHashResolution =
};
@Injectable()
-export class WorkspaceCacheService implements OnModuleInit {
+export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
private readonly localCache = new Map<
string,
WorkspaceLocalCacheEntry<CacheDataType>
>();
+ private lastLocalCacheSweepAt: number | undefined;
private readonly workspaceCacheProviders = new Map<
WorkspaceCacheKeyName,
WorkspaceCacheProvider<CacheDataType>
@@ -76,46 +81,17 @@ export class WorkspaceCacheService implements OnModuleInit {
private readonly memoizer = new PromiseMemoizer<CacheEntriesResult>(
MEMOIZER_TTL_MS,
);
- private lastLocalCacheExpirationSweepAt: number | undefined;
private readonly logger = new Logger(WorkspaceCacheService.name);
- private readonly recomputeDurationHistogram: Histogram;
- private readonly redisWriteDurationHistogram: Histogram;
-
constructor(
@InjectCacheStorage(CacheStorageNamespace.EngineWorkspace)
private readonly cacheStorage: CacheStorageService,
private readonly discoveryService: DiscoveryService,
private readonly reflector: Reflector,
- private readonly metricsService: MetricsService,
+ private readonly cacheMetricsService: WorkspaceCacheMetricsService,
private readonly twentyConfigService: TwentyConfigService,
- ) {
- const meter = this.metricsService.getMeter();
-
- this.recomputeDurationHistogram = meter.createHistogram(
- 'twenty_workspace_cache_recompute_duration_seconds',
- {
- description:
- 'Wall-clock time to compute one workspace metadata cache entry from its provider',
- unit: 's',
- advice: {
- explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS,
- },
- },
- );
- this.redisWriteDurationHistogram = meter.createHistogram(
- 'twenty_workspace_cache_redis_write_duration_seconds',
- {
- description:
- 'Wall-clock time to serialize and write recomputed cache entries to Redis',
- unit: 's',
- advice: {
- explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS,
- },
- },
- );
- }
+ ) {}
async onModuleInit() {
const providers = this.discoveryService.getProviders();
@@ -149,6 +125,12 @@ export class WorkspaceCacheService implements OnModuleInit {
}
}
}
+
+ this.cacheMetricsService.start(this.localCache);
+ }
+
+ onModuleDestroy(): void {
+ this.cacheMetricsService.stop();
}
public async getOrRecompute<const K extends WorkspaceCacheKeyName[]>(
@@ -169,7 +151,7 @@ export class WorkspaceCacheService implements OnModuleInit {
workspaceId: string,
cacheKeyNames: K,
): Promise<WorkspaceCacheResultWithHashes<K>> {
- this.evictExpiredLocalEntriesIfNeeded();
+ this.sweepLocalCacheIfDue();
this.assertValidCacheParameters(workspaceId, cacheKeyNames);
const memoKey =
@@ -280,6 +262,10 @@ export class WorkspaceCacheService implements OnModuleInit {
strategy: 'mint',
});
+ // Invalidation writes entries too, so run the sweep here as well — otherwise an
+ // invalidate-only workload never enforces the per-provider caps.
+ this.sweepLocalCacheIfDue();
+
// Clear memoizer again after recomputation to evict any stale entries
// cached by concurrent getOrRecompute calls during the flush window.
await this.memoizer.clearKeys(`${workspaceId}-`);
@@ -504,9 +490,9 @@ export class WorkspaceCacheService implements OnModuleInit {
isAdopted: isDefined(adoptableHash),
};
} finally {
- this.recomputeDurationHistogram.record(
+ this.cacheMetricsService.recordRecompute(
(performance.now() - computeStartedAt) / 1000,
- { cache_key: keyName },
+ keyName,
);
}
});
@@ -544,7 +530,7 @@ export class WorkspaceCacheService implements OnModuleInit {
try {
await this.cacheStorage.mset(redisEntries);
} finally {
- this.redisWriteDurationHistogram.record(
+ this.cacheMetricsService.recordRedisWrite(
(performance.now() - redisWriteStartedAt) / 1000,
);
}
@@ -632,31 +618,29 @@ export class WorkspaceCacheService implements OnModuleInit {
entry.lastHashCheckedAt = Date.now();
this.cleanupStaleVersions(entry);
- this.evictLRUEntriesIfNeeded();
}
- private evictLRUEntriesIfNeeded(): void {
- if (this.localCache.size <= MAX_LOCAL_CACHE_ENTRIES) {
+ private sweepLocalCacheIfDue(): void {
+ const now = Date.now();
+
+ if (
+ isDefined(this.lastLocalCacheSweepAt) &&
+ now - this.lastLocalCacheSweepAt < LOCAL_CACHE_SWEEP_INTERVAL_MS
+ ) {
return;
}
+ this.lastLocalCacheSweepAt = now;
- const entries = [...this.localCache.entries()].sort(
- (a, b) => a[1].lastHashCheckedAt - b[1].lastHashCheckedAt,
- );
-
- const toEvict = entries.slice(
- 0,
- Math.max(MIN_EVICT_KEYS, this.localCache.size - MAX_LOCAL_CACHE_ENTRIES),
- );
+ const evicted = sweepLocalCache(this.localCache, now, {
+ ttlMs: LOCAL_ENTRY_TTL_MS,
+ maxEntriesByPrefix: MAX_LOCAL_ENTRIES_BY_PREFIX,
+ globalMaxEntries: MAX_LOCAL_CACHE_ENTRIES,
+ minEvict: MIN_EVICT_KEYS,
+ });
- for (const [key] of toEvict) {
- this.localCache.delete(key);
+ if (evicted > 0) {
+ this.cacheMetricsService.recordEviction(evicted);
}
-
- this.metricsService.incrementCounterBy({
- key: MetricsKeys.WorkspaceMetadataCacheLocalEviction,
- amount: toEvict.length,
- });
}
private cleanupStaleVersions(
@@ -691,41 +675,6 @@ export class WorkspaceCacheService implements OnModuleInit {
}
}
- private evictExpiredLocalEntriesIfNeeded(): void {
- const now = Date.now();
-
- if (
- isDefined(this.lastLocalCacheExpirationSweepAt) &&
- now - this.lastLocalCacheExpirationSweepAt <
- LOCAL_CACHE_EXPIRATION_SWEEP_INTERVAL_MS
- ) {
- return;
- }
-
- this.evictExpiredLocalEntries(now);
- this.lastLocalCacheExpirationSweepAt = now;
- }
-
- private evictExpiredLocalEntries(now: number): void {
- for (const [localKey, entry] of this.localCache) {
- for (const [hash, version] of entry.versions) {
- if (now - version.lastReadAt > LOCAL_ENTRY_TTL_MS) {
- entry.versions.delete(hash);
- }
- }
-
- if (entry.versions.size === 0) {
- this.localCache.delete(localKey);
- continue;
- }
-
- if (!entry.versions.has(entry.latestHash)) {
- // Latest was evicted; drop the entire entry to avoid serving stale data.
- this.localCache.delete(localKey);
- }
- }
- }
-
private getProviderOrThrow(
keyName: WorkspaceCacheKeyName,
): WorkspaceCacheProvider<CacheDataType> {
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts
new file mode 100644
index 00000000000..890c1199692
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts
@@ -0,0 +1,51 @@
+import { computeLocalCacheStats } from 'src/engine/workspace-cache/utils/compute-local-cache-stats.util';
+
+const entry = (versionCount: number) => ({ versions: { size: versionCount } });
+
+describe('computeLocalCacheStats', () => {
+ it('returns zeros for an empty cache', () => {
+ const stats = computeLocalCacheStats(new Map());
+
+ expect(stats).toEqual({
+ entries: 0,
+ workspaces: 0,
+ versionsTotal: 0,
+ versionsByCount: { '1': 0, '2': 0, '3': 0, '4': 0, '5+': 0 },
+ });
+ });
+
+ it('counts entries, distinct workspaces, and total versions', () => {
+ const stats = computeLocalCacheStats(
+ new Map([
+ ['orm:entity-metadatas:ws-a', entry(1)],
+ ['flat-maps:field-metadata:ws-a', entry(2)],
+ ['orm:entity-metadatas:ws-b', entry(1)],
+ ]),
+ );
+
+ expect(stats.entries).toBe(3);
+ // ws-a and ws-b — the same workspace under two providers counts once.
+ expect(stats.workspaces).toBe(2);
+ expect(stats.versionsTotal).toBe(4);
+ });
+
+ it('buckets entries by version count and folds 5+ together', () => {
+ const stats = computeLocalCacheStats(
+ new Map([
+ ['p:ws-1', entry(1)],
+ ['p:ws-2', entry(2)],
+ ['p:ws-3', entry(4)],
+ ['p:ws-4', entry(5)],
+ ['p:ws-5', entry(9)],
+ ]),
+ );
+
+ expect(stats.versionsByCount).toEqual({
+ '1': 1,
+ '2': 1,
+ '3': 0,
+ '4': 1,
+ '5+': 2,
+ });
+ });
+});
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/deep-size-bytes.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/deep-size-bytes.util.spec.ts
new file mode 100644
index 00000000000..c2a57591d3f
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/deep-size-bytes.util.spec.ts
@@ -0,0 +1,54 @@
+import { deepSizeBytes } from 'src/engine/workspace-cache/utils/deep-size-bytes.util';
+
+const CAP = 300_000;
+
+describe('deepSizeBytes', () => {
+ it('sizes primitives', () => {
+ expect(deepSizeBytes(42, CAP)).toBe(8);
+ expect(deepSizeBytes(true, CAP)).toBe(4);
+ expect(deepSizeBytes(null, CAP)).toBe(0);
+ expect(deepSizeBytes('ab', CAP)).toBe(12 + 2 * 2);
+ });
+
+ it('walks nested objects, arrays, Maps and Sets', () => {
+ const withArray = deepSizeBytes({ items: [1, 2, 3] }, CAP);
+ const withoutArray = deepSizeBytes({ items: [] }, CAP);
+
+ expect(withArray).toBeGreaterThan(withoutArray);
+ expect(deepSizeBytes(new Map([['k', 'v']]), CAP)).toBeGreaterThan(0);
+ expect(deepSizeBytes(new Set([1, 2, 3]), CAP)).toBeGreaterThan(0);
+ });
+
+ it('is cycle-safe: a self-referential graph terminates and counts each object once', () => {
+ const a: Record<string, unknown> = {};
+ const b: Record<string, unknown> = { a };
+
+ a.b = b;
+
+ const cyclic = deepSizeBytes(a, CAP);
+ // Same two objects, no cycle — the cyclic walk must not double-count.
+ const acyclic = deepSizeBytes({ b: { value: 1 } }, CAP);
+
+ expect(Number.isFinite(cyclic)).toBe(true);
+ expect(cyclic).toBeGreaterThan(0);
+ expect(cyclic).toBeLessThan(acyclic + 100);
+ });
+
+ it('stops at the node cap so a large graph cannot run unbounded', () => {
+ const big = Array.from({ length: 10_000 }, (_, index) => ({ index }));
+
+ const capped = deepSizeBytes(big, 5);
+ const full = deepSizeBytes(big, CAP);
+
+ expect(capped).toBeLessThan(full);
+ });
+
+ it('counts primitive nodes toward the cap, so a huge flat array is bounded too', () => {
+ const manyNumbers = Array.from({ length: 10_000 }, (_, index) => index);
+
+ const capped = deepSizeBytes(manyNumbers, 10);
+ const full = deepSizeBytes(manyNumbers, CAP);
+
+ expect(capped).toBeLessThan(full);
+ });
+});
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts
new file mode 100644
index 00000000000..0c42624e930
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts
@@ -0,0 +1,115 @@
+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
+import {
+ type LocalCacheSweepConfig,
+ sweepLocalCache,
+} from 'src/engine/workspace-cache/utils/sweep-local-cache.util';
+
+const TTL_MS = 30 * 60 * 1000;
+const ORM_PREFIX = 'orm:entity-metadatas';
+
+const entry = (lastReadAt: number): WorkspaceLocalCacheEntry<unknown> => ({
+ versions: new Map([['h', { data: {}, lastReadAt }]]),
+ latestHash: 'h',
+ lastHashCheckedAt: lastReadAt,
+});
+
+const config = (
+ overrides: Partial<LocalCacheSweepConfig> = {},
+): LocalCacheSweepConfig => ({
+ ttlMs: TTL_MS,
+ maxEntriesByPrefix: new Map(),
+ globalMaxEntries: 6_000,
+ minEvict: 0,
+ ...overrides,
+});
+
+describe('sweepLocalCache', () => {
+ it('drops entries idle past the TTL and keeps recent ones', () => {
+ const cache = new Map([
+ [`${ORM_PREFIX}:ws-idle`, entry(0)],
+ [`${ORM_PREFIX}:ws-recent`, entry(TTL_MS)],
+ ]);
+
+ const evicted = sweepLocalCache(cache, TTL_MS + 1, config());
+
+ expect(evicted).toBe(1);
+ expect(cache.has(`${ORM_PREFIX}:ws-idle`)).toBe(false);
+ expect(cache.has(`${ORM_PREFIX}:ws-recent`)).toBe(true);
+ });
+
+ it('trims a provider over its cap, evicting the least-recently-read', () => {
+ const cache = new Map([
+ [`${ORM_PREFIX}:ws-a`, entry(1)],
+ [`${ORM_PREFIX}:ws-b`, entry(2)],
+ [`${ORM_PREFIX}:ws-c`, entry(3)],
+ ]);
+
+ const evicted = sweepLocalCache(
+ cache,
+ 100,
+ config({ maxEntriesByPrefix: new Map([[ORM_PREFIX, 2]]) }),
+ );
+
+ expect(evicted).toBe(1);
+ expect(cache.size).toBe(2);
+ expect(cache.has(`${ORM_PREFIX}:ws-a`)).toBe(false); // oldest
+ expect(cache.has(`${ORM_PREFIX}:ws-c`)).toBe(true); // newest
+ });
+
+ it('leaves a provider without a cap untouched', () => {
+ const cache = new Map(
+ Array.from({ length: 5 }, (_, index) => [
+ `feature-flag:feature-flags-map:ws-${index}`,
+ entry(index + 1),
+ ]),
+ );
+
+ const evicted = sweepLocalCache(
+ cache,
+ 100,
+ config({ maxEntriesByPrefix: new Map([[ORM_PREFIX, 2]]) }),
+ );
+
+ expect(evicted).toBe(0);
+ expect(cache.size).toBe(5);
+ });
+
+ it('trims the global total over the cap, batching by minEvict', () => {
+ const cache = new Map(
+ Array.from({ length: 10 }, (_, index) => [
+ `feature-flag:feature-flags-map:ws-${index}`,
+ entry(index + 1),
+ ]),
+ );
+
+ const evicted = sweepLocalCache(
+ cache,
+ 100,
+ config({ globalMaxEntries: 6, minEvict: 4 }),
+ );
+
+ // 10 > 6, and minEvict=4 means at least 4 go — the four least-recently-read.
+ expect(evicted).toBe(4);
+ expect(cache.size).toBe(6);
+ expect(cache.has('feature-flag:feature-flags-map:ws-0')).toBe(false);
+ expect(cache.has('feature-flag:feature-flags-map:ws-9')).toBe(true);
+ });
+
+ it('expires first so the per-provider cap only counts survivors', () => {
+ const cache = new Map([
+ [`${ORM_PREFIX}:ws-idle`, entry(0)], // expired by TTL
+ [`${ORM_PREFIX}:ws-a`, entry(TTL_MS + 1)],
+ [`${ORM_PREFIX}:ws-b`, entry(TTL_MS + 2)],
+ ]);
+
+ const evicted = sweepLocalCache(
+ cache,
+ TTL_MS + 10,
+ config({ maxEntriesByPrefix: new Map([[ORM_PREFIX, 2]]) }),
+ );
+
+ // The idle one is expired; the two survivors are within the cap, so nothing else goes.
+ expect(evicted).toBe(1);
+ expect(cache.size).toBe(2);
+ });
+});
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts
new file mode 100644
index 00000000000..d6d34ebb2cc
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts
@@ -0,0 +1,37 @@
+export type LocalCacheStats = {
+ entries: number;
+ workspaces: number;
+ versionsTotal: number;
+ versionsByCount: Record<string, number>;
+};
+
+export const computeLocalCacheStats = (
+ localCache: ReadonlyMap<string, { versions: { size: number } }>,
+): LocalCacheStats => {
+ const workspaceIds = new Set<string>();
+ const versionsByCount: Record<string, number> = {
+ '1': 0,
+ '2': 0,
+ '3': 0,
+ '4': 0,
+ '5+': 0,
+ };
+ let versionsTotal = 0;
+
+ for (const [key, entry] of localCache) {
+ workspaceIds.add(key.slice(key.lastIndexOf(':') + 1));
+ const versionCount = entry.versions.size;
+
+ versionsTotal += versionCount;
+ const bucket = versionCount >= 5 ? '5+' : String(versionCount);
+
+ versionsByCount[bucket] = (versionsByCount[bucket] ?? 0) + 1;
+ }
+
+ return {
+ entries: localCache.size,
+ workspaces: workspaceIds.size,
+ versionsTotal,
+ versionsByCount,
+ };
+};
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/deep-size-bytes.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/deep-size-bytes.util.ts
new file mode 100644
index 00000000000..f00068cf5c1
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/deep-size-bytes.util.ts
@@ -0,0 +1,75 @@
+export const deepSizeBytes = (root: unknown, nodeCap: number): number => {
+ const seen = new WeakSet<object>();
+ const stack: unknown[] = [root];
+ let bytes = 0;
+ let visited = 0;
+
+ while (stack.length > 0 && visited < nodeCap) {
+ visited += 1;
+
+ const value = stack.pop();
+
+ if (typeof value === 'string') {
+ bytes += 12 + value.length * 2;
+ continue;
+ }
+ if (typeof value === 'number') {
+ bytes += 8;
+ continue;
+ }
+ if (typeof value === 'boolean') {
+ bytes += 4;
+ continue;
+ }
+ if (
+ value === null ||
+ (typeof value !== 'object' && typeof value !== 'function')
+ ) {
+ continue;
+ }
+ if (seen.has(value)) {
+ continue;
+ }
+ seen.add(value);
+
+ if (Array.isArray(value)) {
+ bytes += 16 + value.length * 8;
+ for (const item of value) {
+ stack.push(item);
+ }
+ continue;
+ }
+
+ if (value instanceof Map) {
+ bytes += 48 + value.size * 16;
+ for (const [mapKey, mapValue] of value) {
+ stack.push(mapKey);
+ stack.push(mapValue);
+ }
+ continue;
+ }
+
+ if (value instanceof Set) {
+ bytes += 48 + value.size * 8;
+ for (const item of value) {
+ stack.push(item);
+ }
+ continue;
+ }
+
+ bytes += 32;
+ let entries: [string, unknown][] = [];
+
+ try {
+ entries = Object.entries(value);
+ } catch {
+ entries = [];
+ }
+ for (const [entryKey, child] of entries) {
+ bytes += entryKey.length * 2 + 8;
+ stack.push(child);
+ }
+ }
+
+ return bytes;
+};
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.ts
new file mode 100644
index 00000000000..d6b7b87ee4f
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/sweep-local-cache.util.ts
@@ -0,0 +1,90 @@
+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
+
+export type LocalCacheSweepConfig = {
+ ttlMs: number;
+ maxEntriesByPrefix: ReadonlyMap<string, number>;
+ globalMaxEntries: number;
+ minEvict: number;
+};
+
+const lastReadAtOf = <T>(entry: WorkspaceLocalCacheEntry<T>): number =>
+ entry.versions.get(entry.latestHash)?.lastReadAt ?? 0;
+
+const evictLeastRecentlyRead = <T>({
+ localCache,
+ matches,
+ maxEntries,
+ minEvict,
+}: {
+ localCache: Map<string, WorkspaceLocalCacheEntry<T>>;
+ matches: (key: string) => boolean;
+ maxEntries: number;
+ minEvict: number;
+}): number => {
+ const matching: [string, WorkspaceLocalCacheEntry<T>][] = [];
+
+ for (const keyEntry of localCache) {
+ if (matches(keyEntry[0])) {
+ matching.push(keyEntry);
+ }
+ }
+
+ if (matching.length <= maxEntries) {
+ return 0;
+ }
+
+ matching.sort((a, b) => lastReadAtOf(a[1]) - lastReadAtOf(b[1]));
+
+ const evictCount = Math.min(
+ matching.length,
+ Math.max(minEvict, matching.length - maxEntries),
+ );
+
+ for (let index = 0; index < evictCount; index += 1) {
+ localCache.delete(matching[index][0]);
+ }
+
+ return evictCount;
+};
+
+// Mutates the cache: expire versions idle past the TTL (dropping any entry left without a current
+// version), then trim each capped provider and the global total to budget by least-recently-read.
+// Returns the number of entries evicted.
+export const sweepLocalCache = <T>(
+ localCache: Map<string, WorkspaceLocalCacheEntry<T>>,
+ now: number,
+ config: LocalCacheSweepConfig,
+): number => {
+ let evicted = 0;
+
+ for (const [localKey, entry] of localCache) {
+ for (const [hash, version] of entry.versions) {
+ if (now - version.lastReadAt > config.ttlMs) {
+ entry.versions.delete(hash);
+ }
+ }
+
+ if (entry.versions.size === 0 || !entry.versions.has(entry.latestHash)) {
+ localCache.delete(localKey);
+ evicted += 1;
+ }
+ }
+
+ for (const [prefix, maxEntries] of config.maxEntriesByPrefix) {
+ evicted += evictLeastRecentlyRead({
+ localCache,
+ matches: (key) => key.startsWith(`${prefix}:`),
+ maxEntries,
+ minEvict: 0,
+ });
+ }
+
+ evicted += evictLeastRecentlyRead({
+ localCache,
+ matches: () => true,
+ maxEntries: config.globalMaxEntries,
+ minEvict: config.minEvict,
+ });
+
+ return evicted;
+};
diff --git a/packages/twenty-server/src/engine/workspace-cache/workspace-cache.module.ts b/packages/twenty-server/src/engine/workspace-cache/workspace-cache.module.ts
index a873207fc54..53498de1d76 100644
--- a/packages/twenty-server/src/engine/workspace-cache/workspace-cache.module.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/workspace-cache.module.ts
@@ -3,11 +3,12 @@ import { DiscoveryModule } from '@nestjs/core';
import { CacheStorageModule } from 'src/engine/core-modules/cache-storage/cache-storage.module';
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
+import { WorkspaceCacheMetricsService } from 'src/engine/workspace-cache/services/workspace-cache-metrics.service';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
@Module({
imports: [CacheStorageModule, DiscoveryModule, MetricsModule],
- providers: [WorkspaceCacheService],
+ providers: [WorkspaceCacheService, WorkspaceCacheMetricsService],
exports: [WorkspaceCacheService],
})
export class WorkspaceCacheModule {}