The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
perf(server): cut Sentry instrumentation overhead and stop packing the working set (#24019)
Charles Bochet9f840e4fb73d
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.tsindex 5a44aa383a0..6617bd9f843 100644--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts@@ -72,7 +72,7 @@ export enum MetricsKeys {AiChatTurnCompleted = 'ai-chat/turn-completed',AiChatTurnFailed = 'ai-chat/turn-failed',WorkspaceMetadataCacheLocalEviction = 'workspace-metadata-cache/local-eviction',- WorkspaceMetadataCacheDemoted = 'workspace-metadata-cache/demoted',+ WorkspaceMetadataCachePacked = 'workspace-metadata-cache/packed',SdkClientGenerationSucceeded = 'sdk-client-generation/succeeded',SdkClientGenerationFailed = 'sdk-client-generation/failed',SdkClientGenerationDurationMs = 'sdk-client-generation/duration-ms',diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.tsindex 4153044c3c3..71e29e3e340 100644--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts@@ -38,8 +38,8 @@ export class WorkspaceCacheMetricsService {private readonly logger = new Logger(WorkspaceCacheMetricsService.name);private readonly recomputeDurationHistogram: Histogram;private readonly redisWriteDurationHistogram: Histogram;- private readonly demotionDurationHistogram: Histogram;- private readonly hydrationDurationHistogram: Histogram;+ private readonly packingDurationHistogram: Histogram;+ private readonly unpackingDurationHistogram: Histogram;private localCache?: LocalCache;private cacheSizeByKeyName: Record<string, number> = {};@@ -48,7 +48,7 @@ export class WorkspaceCacheMetricsService {private sizeStartupTimer?: ReturnType<typeof setTimeout>;private statsCache?: { computedAt: number } & LocalCacheStats;private sizeSampleInFlight = false;- private demotionBacklog = 0;+ private packingBacklog = 0;constructor(private readonly metricsService: MetricsService) {const meter = this.metricsService.getMeter();@@ -71,20 +71,19 @@ export class WorkspaceCacheMetricsService {advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },},);- this.demotionDurationHistogram = meter.createHistogram(- 'twenty_workspace_cache_demotion_slice_duration_seconds',+ this.packingDurationHistogram = meter.createHistogram(+ 'twenty_workspace_cache_packing_run_duration_seconds',{- description:- 'Event loop time consumed by one cold-storage demotion slice',+ description: 'Event loop time consumed by one packing run',unit: 's',advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },},);- this.hydrationDurationHistogram = meter.createHistogram(- 'twenty_workspace_cache_hydration_duration_seconds',+ this.unpackingDurationHistogram = meter.createHistogram(+ 'twenty_workspace_cache_unpacking_duration_seconds',{description:- 'Wall-clock time to hydrate one cold cache entry back into objects on read',+ 'Wall-clock time to unpack one packed cache version back into objects on read',unit: 's',advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },},@@ -114,30 +113,30 @@ export class WorkspaceCacheMetricsService {this.redisWriteDurationHistogram.record(seconds);}- recordDemotionSlice({+ recordPackingRun({durationSeconds,- demoted,- remaining,+ packed,+ pending,}: {durationSeconds: number;- demoted: number;- remaining: number;+ packed: number;+ pending: number;}): void {- this.demotionBacklog = remaining;+ this.packingBacklog = pending;- if (demoted === 0) {+ if (packed === 0) {return;}- this.demotionDurationHistogram.record(durationSeconds);+ this.packingDurationHistogram.record(durationSeconds);this.metricsService.incrementCounterBy({- key: MetricsKeys.WorkspaceMetadataCacheDemoted,- amount: demoted,+ key: MetricsKeys.WorkspaceMetadataCachePacked,+ amount: packed,});}- recordHydration(seconds: number, cacheKey: WorkspaceCacheKeyName): void {- this.hydrationDurationHistogram.record(seconds, { cache_key: cacheKey });+ recordUnpacking(seconds: number, cacheKey: WorkspaceCacheKeyName): void {+ this.unpackingDurationHistogram.record(seconds, { cache_key: cacheKey });}recordEviction(amount: number): void {@@ -216,7 +215,7 @@ export class WorkspaceCacheMetricsService {for (const version of entry.versions.values()) {entryBytes +=- version.state === 'cold'+ version.state === 'packed'? version.blob.byteLength: deepSizeBytes(version.data, SIZE_WALK_NODE_CAP);}@@ -319,59 +318,59 @@ export class WorkspaceCacheMetricsService {metricName: 'twenty_workspace_cache_local_versions_by_state',options: {description:- 'Local workspace metadata cache versions by storage state and provider',+ 'Local workspace metadata cache versions by tier (live or packed) and provider',},callback: async () => {const stats = this.getStats();return [- ...Object.entries(stats.hotVersionsByKeyName).map(+ ...Object.entries(stats.liveVersionsByKeyName).map(([keyName, value]) => ({value,- attributes: { provider: keyName, state: 'hot' },+ attributes: { provider: keyName, state: 'live' },}),),- ...Object.entries(stats.coldVersionsByKeyName).map(+ ...Object.entries(stats.packedVersionsByKeyName).map(([keyName, value]) => ({value,- attributes: { provider: keyName, state: 'cold' },+ attributes: { provider: keyName, state: 'packed' },}),),];},});this.metricsService.createObservableGauge({- metricName: 'twenty_workspace_cache_local_hot_versions',+ metricName: 'twenty_workspace_cache_local_live_versions',options: {description:- 'Total hot (live object) versions in the local workspace metadata cache',+ 'Total live (object graph) versions in the local workspace metadata cache',},- callback: async () => this.getStats().hotVersionsTotal,+ callback: async () => this.getStats().liveVersionsTotal,});this.metricsService.createObservableGauge({- metricName: 'twenty_workspace_cache_local_cold_versions',+ metricName: 'twenty_workspace_cache_local_packed_versions',options: {description:- 'Total cold (serialized buffer) versions in the local workspace metadata cache',+ 'Total packed (serialized buffer) versions in the local workspace metadata cache',},- callback: async () => this.getStats().coldVersionsTotal,+ callback: async () => this.getStats().packedVersionsTotal,});this.metricsService.createObservableGauge({- metricName: 'twenty_workspace_cache_local_cold_bytes',+ metricName: 'twenty_workspace_cache_local_packed_bytes',options: {description:- 'Exact retained bytes held as cold buffers in the local workspace metadata cache',+ 'Exact retained bytes held as packed buffers in the local workspace metadata cache',unit: 'By',},- callback: async () => this.getStats().coldBytesTotal,+ callback: async () => this.getStats().packedBytesTotal,});this.metricsService.createObservableGauge({- metricName: 'twenty_workspace_cache_demotion_backlog',+ metricName: 'twenty_workspace_cache_packing_backlog',options: {description:- 'Entries still awaiting demotion when the last demotion slice ran out of budget',+ 'Versions currently eligible for packing, whether or not the last run reached them',},- callback: async () => this.demotionBacklog,+ callback: async () => this.packingBacklog,});}}diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.tsindex ae6ee098608..2a2c1d08a93 100644--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts@@ -40,7 +40,8 @@ import {type WorkspaceLocalCacheEntry,} from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';import { combineCacheHashes } from 'src/engine/workspace-cache/utils/combine-cache-hashes.util';-import { demoteColdStorageEntries } from 'src/engine/workspace-cache/utils/demote-cold-storage-entries.util';+import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';+import { packIdleVersions } from 'src/engine/workspace-cache/utils/pack-idle-versions.util';import { sweepLocalCache } from 'src/engine/workspace-cache/utils/sweep-local-cache.util';const LOCAL_TTL_MS = 100; // 100ms@@ -52,9 +53,9 @@ const MAX_LOCAL_CACHE_ENTRIES = 6_000;const MIN_EVICT_KEYS = 100;const LOCAL_ENTRY_TTL_MS = 30 * 60 * 1000; // 30 minutes idleconst LOCAL_CACHE_SWEEP_INTERVAL_MS = 60 * 1000;-const HOT_ENTRIES_PER_PROVIDER = 64;-const DEMOTION_INTERVAL_MS = 250;-const DEMOTION_BUDGET_MS = 10;+const PACKING_INTERVAL_MS = 500;+const MAX_ENTRY_VERSIONS_PER_PACKING_RUN = 2;+const MIN_IDLE_BEFORE_PACKING_MS = 60 * 1000;// Per-provider entry caps, keyed by local cache key prefix (ORM graphs are ~5 MB each).const MAX_LOCAL_ENTRIES_BY_KEY_NAME = new Map<string, number>([['ORMEntityMetadatas', 128],@@ -82,7 +83,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {WorkspaceLocalCacheEntry<CacheDataType>>();private sweepTimer?: ReturnType<typeof setInterval>;- private demotionTimer?: ReturnType<typeof setInterval>;+ private packingTimer?: ReturnType<typeof setInterval>;private readonly workspaceCacheProviders = new Map<WorkspaceCacheKeyName,WorkspaceCacheProvider<CacheDataType, StoredCacheDataType>@@ -144,8 +145,8 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {if (isDefined(this.sweepTimer)) {clearInterval(this.sweepTimer);}- if (isDefined(this.demotionTimer)) {- clearInterval(this.demotionTimer);+ if (isDefined(this.packingTimer)) {+ clearInterval(this.packingTimer);}this.cacheMetricsService.stop();}@@ -157,11 +158,11 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {);this.sweepTimer.unref();- this.demotionTimer = setInterval(- () => this.demoteColdEntries(),- DEMOTION_INTERVAL_MS,+ this.packingTimer = setInterval(+ () => this.runPacking(),+ PACKING_INTERVAL_MS,);- this.demotionTimer.unref();+ this.packingTimer.unref();}public async getOrRecompute<const K extends WorkspaceCacheKeyName[]>(@@ -608,7 +609,12 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {const version = entry?.versions.get(entry.latestHash);if (isDefined(entry) && isDefined(version)) {- const data = this.readVersion({ keyName, entry, version });+ const data = this.readVersion({+ keyName,+ entry,+ hash: entry.latestHash,+ version,+ });Object.assign(result.data, { [keyName]: data });result.hashes[keyName] = entry.latestHash;@@ -660,7 +666,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {this.localCache.set(localKey, entry);}- entry.versions.set(hash, { state: 'hot', data, lastReadAt: Date.now() });+ entry.versions.set(hash, { state: 'live', data, lastReadAt: Date.now() });entry.latestHash = hash;entry.lastHashCheckedAt = Date.now();@@ -682,24 +688,22 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {}}- private demoteColdEntries(): void {+ private runPacking(): void {const startedAt = performance.now();- const { demoted, remaining } = demoteColdStorageEntries({+ const { packed, pending } = packIdleVersions({localCache: this.localCache,- hotEntriesPerProvider: HOT_ENTRIES_PER_PROVIDER,- budgetMs: DEMOTION_BUDGET_MS,- serialize: ({ localKey, data }) => {- const separatorIndex = localKey.lastIndexOf(':');- const keyName = localKey.slice(- 0,- separatorIndex,+ minIdleMs: MIN_IDLE_BEFORE_PACKING_MS,+ maxEntryVersionsPerRun: MAX_ENTRY_VERSIONS_PER_PACKING_RUN,+ isPackable: (localKey) =>+ !this.localDataOnlyKeys.has(+ getKeyNameFromLocalCacheKey(localKey) as WorkspaceCacheKeyName,+ ),+ pack: ({ localKey, data }) => {+ const keyName = getKeyNameFromLocalCacheKey(+ localKey,) as WorkspaceCacheKeyName;- if (this.localDataOnlyKeys.has(keyName)) {- return undefined;- }-return Buffer.from(JSON.stringify(this.getProviderOrThrow(keyName).encodeForCacheStorage(data),@@ -709,41 +713,43 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {},});- this.cacheMetricsService.recordDemotionSlice({+ this.cacheMetricsService.recordPackingRun({durationSeconds: (performance.now() - startedAt) / 1000,- demoted,- remaining,+ packed,+ pending,});}private readVersion({keyName,entry,+ hash,version,}: {keyName: WorkspaceCacheKeyName;entry: WorkspaceLocalCacheEntry<CacheDataType>;+ hash: string;version: VersionEntry<CacheDataType>;}): CacheDataType {- if (version.state === 'hot') {+ if (version.state === 'live') {version.lastReadAt = Date.now();return version.data;}- const hydrateStartedAt = performance.now();+ const unpackStartedAt = performance.now();const data = this.getProviderOrThrow(keyName).decodeFromCacheStorage(JSON.parse(version.blob.toString('utf8')),);- entry.versions.set(entry.latestHash, {- state: 'hot',+ entry.versions.set(hash, {+ state: 'live',data,lastReadAt: Date.now(),});- this.cacheMetricsService.recordHydration(- (performance.now() - hydrateStartedAt) / 1000,+ this.cacheMetricsService.recordUnpacking(+ (performance.now() - unpackStartedAt) / 1000,keyName,);diff --git a/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts b/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.tsindex 9bade6606a3..0085ab1d62d 100644--- a/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts+++ b/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts@@ -1,16 +1,16 @@-export type HotVersionEntry<T> = {- state: 'hot';+export type LiveVersionEntry<T> = {+ state: 'live';data: T;lastReadAt: number;};-export type ColdVersionEntry = {- state: 'cold';+export type PackedVersionEntry = {+ state: 'packed';blob: Buffer;lastReadAt: number;};-export type VersionEntry<T> = HotVersionEntry<T> | ColdVersionEntry;+export type VersionEntry<T> = LiveVersionEntry<T> | PackedVersionEntry;export type WorkspaceLocalCacheEntry<T> = {versions: Map<string, VersionEntry<T>>;diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.tsindex c3b3ade2459..c7163f544f5 100644--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts@@ -1,13 +1,13 @@import { computeLocalCacheStats } from 'src/engine/workspace-cache/utils/compute-local-cache-stats.util';-const entry = (versionCount: number, state: 'hot' | 'cold' = 'hot') => ({+const entry = (versionCount: number, state: 'live' | 'packed' = 'live') => ({versions: new Map(Array.from({ length: versionCount }, (_, index) => [`hash-${index}`,- state === 'hot'- ? { state: 'hot' as const, data: 'data', lastReadAt: index }+ state === 'live'+ ? { state: 'live' as const, data: 'data', lastReadAt: index }: {- state: 'cold' as const,+ state: 'packed' as const,blob: Buffer.alloc(100),lastReadAt: index,},@@ -25,12 +25,12 @@ describe('computeLocalCacheStats', () => {versionsTotal: 0,versionsByCount: { '1': 0, '2': 0, '3': 0, '4': 0, '5+': 0 },entriesByKeyName: {},- hotVersionsByKeyName: {},- coldVersionsByKeyName: {},- coldBytesByKeyName: {},- hotVersionsTotal: 0,- coldVersionsTotal: 0,- coldBytesTotal: 0,+ liveVersionsByKeyName: {},+ packedVersionsByKeyName: {},+ packedBytesByKeyName: {},+ liveVersionsTotal: 0,+ packedVersionsTotal: 0,+ packedBytesTotal: 0,});});@@ -73,23 +73,23 @@ describe('computeLocalCacheStats', () => {});});- it('splits versions by storage state and sums exact cold bytes', () => {+ it('splits versions by storage state and sums exact packed bytes', () => {const stats = computeLocalCacheStats(new Map([- ['flatFieldMetadataMaps:ws-a', entry(2, 'cold')],- ['flatFieldMetadataMaps:ws-b', entry(1, 'hot')],- ['ORMEntityMetadatas:ws-a', entry(1, 'hot')],+ ['flatFieldMetadataMaps:ws-a', entry(2, 'packed')],+ ['flatFieldMetadataMaps:ws-b', entry(1, 'live')],+ ['ORMEntityMetadatas:ws-a', entry(1, 'live')],]),);- expect(stats.hotVersionsTotal).toBe(2);- expect(stats.coldVersionsTotal).toBe(2);- expect(stats.coldBytesTotal).toBe(200);- expect(stats.coldVersionsByKeyName).toEqual({+ expect(stats.liveVersionsTotal).toBe(2);+ expect(stats.packedVersionsTotal).toBe(2);+ expect(stats.packedBytesTotal).toBe(200);+ expect(stats.packedVersionsByKeyName).toEqual({flatFieldMetadataMaps: 2,ORMEntityMetadatas: 0,});- expect(stats.hotVersionsByKeyName).toEqual({+ expect(stats.liveVersionsByKeyName).toEqual({flatFieldMetadataMaps: 1,ORMEntityMetadatas: 1,});diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.tsdeleted file mode 100644index 57e56f74c63..00000000000--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts+++ /dev/null@@ -1,200 +0,0 @@-import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';-import { demoteColdStorageEntries } from 'src/engine/workspace-cache/utils/demote-cold-storage-entries.util';--const FIELD_METADATA = 'flatFieldMetadataMaps';-const ORM = 'ORMEntityMetadatas';--const hotEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({- versions: new Map([['hash-1', { state: 'hot', data: 'data', lastReadAt }]]),- latestHash: 'hash-1',- lastHashCheckedAt: lastReadAt,-});--const coldEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({- versions: new Map([- ['hash-1', { state: 'cold', blob: Buffer.from('{}'), lastReadAt }],- ]),- latestHash: 'hash-1',- lastHashCheckedAt: lastReadAt,-});--const serialize = () => Buffer.from('serialized');--const NO_BUDGET_LIMIT = Number.POSITIVE_INFINITY;--const demote = (- localCache: Map<string, WorkspaceLocalCacheEntry<string>>,- hotEntriesPerProvider: number,- overrides?: {- serialize?: () => Buffer | undefined;- budgetMs?: number;- now?: () => number;- },-) =>- demoteColdStorageEntries({- localCache,- hotEntriesPerProvider,- budgetMs: overrides?.budgetMs ?? NO_BUDGET_LIMIT,- serialize: overrides?.serialize ?? serialize,- now: overrides?.now,- }).demoted;--describe('demoteColdStorageEntries', () => {- it('should demote nothing when the provider is within its hot budget', () => {- const localCache = new Map([- [`${FIELD_METADATA}:ws-a`, hotEntry(1)],- [`${FIELD_METADATA}:ws-b`, hotEntry(2)],- ]);-- expect(demote(localCache, 2)).toBe(0);- expect(- localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),- ).toMatchObject({ state: 'hot' });- });-- it('should demote the least recently read entries beyond the hot budget', () => {- const localCache = new Map([- [`${FIELD_METADATA}:ws-newest`, hotEntry(9)],- [`${FIELD_METADATA}:ws-oldest`, hotEntry(1)],- [`${FIELD_METADATA}:ws-middle`, hotEntry(5)],- ]);-- expect(demote(localCache, 1)).toBe(2);- expect(- localCache.get(`${FIELD_METADATA}:ws-oldest`)?.versions.get('hash-1'),- ).toMatchObject({ state: 'cold', blob: Buffer.from('serialized') });- expect(- localCache.get(`${FIELD_METADATA}:ws-newest`)?.versions.get('hash-1'),- ).toMatchObject({ state: 'hot' });- });-- it('should preserve lastReadAt so a demoted entry keeps its place in the eviction order', () => {- const localCache = new Map([- [`${FIELD_METADATA}:ws-a`, hotEntry(1)],- [`${FIELD_METADATA}:ws-b`, hotEntry(2)],- ]);-- demote(localCache, 1);-- expect(- localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),- ).toMatchObject({ lastReadAt: 1 });- });-- it('should leave the entry hot when serialize declines it', () => {- const localCache = new Map([- [`${FIELD_METADATA}:ws-a`, hotEntry(1)],- [`${FIELD_METADATA}:ws-b`, hotEntry(2)],- ]);-- expect(demote(localCache, 1, { serialize: () => undefined })).toBe(0);- expect(- localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),- ).toMatchObject({ state: 'hot' });- });-- it('should budget each provider independently', () => {- const localCache = new Map([- [`${FIELD_METADATA}:ws-a`, hotEntry(1)],- [`${FIELD_METADATA}:ws-b`, hotEntry(2)],- [`${ORM}:ws-a`, hotEntry(3)],- ]);-- expect(demote(localCache, 1)).toBe(1);- expect(localCache.get(`${ORM}:ws-a`)?.versions.get('hash-1')).toMatchObject(- {- state: 'hot',- },- );- });-- it('should not count an already cold entry against the hot budget', () => {- const localCache = new Map([- [`${FIELD_METADATA}:ws-cold`, coldEntry(1)],- [`${FIELD_METADATA}:ws-hot`, hotEntry(2)],- ]);-- expect(demote(localCache, 1)).toBe(0);- });-- describe('time budget', () => {- const clockAdvancingPerEntry = () => {- let elapsed = 0;-- return () => {- const current = elapsed;-- elapsed += 10;-- return current;- };- };-- const buildCache = (count: number) =>- new Map(- Array.from({ length: count }, (_, index) => [- `${FIELD_METADATA}:ws-${index}`,- hotEntry(index),- ]),- );-- it('should stop once the budget is spent instead of draining the backlog', () => {- const localCache = buildCache(10);-- const result = demoteColdStorageEntries({- localCache,- hotEntriesPerProvider: 0,- budgetMs: 25,- serialize,- now: clockAdvancingPerEntry(),- });-- expect(result.demoted).toBe(2);- expect(result.remaining).toBe(8);- });-- it('should demote coldest first so an interrupted slice leaves the hottest behind', () => {- const localCache = buildCache(10);-- demoteColdStorageEntries({- localCache,- hotEntriesPerProvider: 0,- budgetMs: 25,- serialize,- now: clockAdvancingPerEntry(),- });-- expect(- localCache.get(`${FIELD_METADATA}:ws-0`)?.versions.get('hash-1'),- ).toMatchObject({ state: 'cold' });- expect(- localCache.get(`${FIELD_METADATA}:ws-9`)?.versions.get('hash-1'),- ).toMatchObject({ state: 'hot' });- });-- it('should converge across successive slices without a cursor', () => {- const localCache = buildCache(10);- let slices = 0;- let remaining = Number.POSITIVE_INFINITY;-- while (remaining > 0 && slices < 10) {- remaining = demoteColdStorageEntries({- localCache,- hotEntriesPerProvider: 0,- budgetMs: 25,- serialize,- now: clockAdvancingPerEntry(),- }).remaining;- slices += 1;- }-- expect(remaining).toBe(0);- expect(slices).toBe(5);- expect(- [...localCache.values()].every(- (entry) => entry.versions.get('hash-1')?.state === 'cold',- ),- ).toBe(true);- });- });-});diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/pack-idle-versions.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/pack-idle-versions.util.spec.tsnew file mode 100644index 00000000000..145b7da8602--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/pack-idle-versions.util.spec.ts@@ -0,0 +1,148 @@+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';+import { packIdleVersions } from 'src/engine/workspace-cache/utils/pack-idle-versions.util';++const FIELD_METADATA = 'flatFieldMetadataMaps';+const ORM = 'ORMEntityMetadatas';++const NOW_EPOCH_MS = 1_000_000;+const IDLE_MS = 60_000;+const IDLE = NOW_EPOCH_MS - 90_000;+const RECENT = NOW_EPOCH_MS - 1_000;++const liveEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({+ versions: new Map([['hash-1', { state: 'live', data: 'data', lastReadAt }]]),+ latestHash: 'hash-1',+ lastHashCheckedAt: lastReadAt,+});++const packedEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({+ versions: new Map([+ ['hash-1', { state: 'packed', blob: Buffer.from('{}'), lastReadAt }],+ ]),+ latestHash: 'hash-1',+ lastHashCheckedAt: lastReadAt,+});++const pack = () => Buffer.from('packed');++const run = (+ localCache: Map<string, WorkspaceLocalCacheEntry<string>>,+ maxEntryVersionsPerRun: number,+ overrides?: {+ pack?: () => Buffer | undefined;+ isPackable?: (localKey: string) => boolean;+ },+) =>+ packIdleVersions({+ localCache,+ minIdleMs: IDLE_MS,+ maxEntryVersionsPerRun,+ isPackable: overrides?.isPackable ?? (() => true),+ pack: overrides?.pack ?? pack,+ nowEpochMs: () => NOW_EPOCH_MS,+ });++const stateOf = (+ localCache: Map<string, WorkspaceLocalCacheEntry<string>>,+ localKey: string,+) => localCache.get(localKey)?.versions.get('hash-1')?.state;++describe('packIdleVersions', () => {+ it('should pack a version that has gone idle', () => {+ const localCache = new Map([[`${FIELD_METADATA}:ws-a`, liveEntry(IDLE)]]);++ expect(run(localCache, 2).packed).toBe(1);+ expect(stateOf(localCache, `${FIELD_METADATA}:ws-a`)).toBe('packed');+ });++ it('should leave a version read inside the idle window live', () => {+ const localCache = new Map([[`${FIELD_METADATA}:ws-a`, liveEntry(RECENT)]]);++ expect(run(localCache, 2).packed).toBe(0);+ expect(stateOf(localCache, `${FIELD_METADATA}:ws-a`)).toBe('live');+ });++ it('should pack no more than one run allows, coldest first', () => {+ const localCache = new Map([+ [`${FIELD_METADATA}:ws-coldest`, liveEntry(IDLE - 2_000)],+ [`${FIELD_METADATA}:ws-colder`, liveEntry(IDLE - 1_000)],+ [`${ORM}:ws-warm`, liveEntry(IDLE)],+ ]);++ const result = run(localCache, 2);++ expect(result.packed).toBe(2);+ expect(result.pending).toBe(1);+ expect(stateOf(localCache, `${FIELD_METADATA}:ws-coldest`)).toBe('packed');+ expect(stateOf(localCache, `${FIELD_METADATA}:ws-colder`)).toBe('packed');+ expect(stateOf(localCache, `${ORM}:ws-warm`)).toBe('live');+ });++ it('should converge across successive runs', () => {+ const localCache = new Map(+ Array.from({ length: 5 }, (_, index) => [+ `${FIELD_METADATA}:ws-${index}`,+ liveEntry(IDLE - index),+ ]),+ );+ let pending = Number.POSITIVE_INFINITY;+ let runs = 0;++ while (pending > 0 && runs < 10) {+ pending = run(localCache, 2).pending;+ runs += 1;+ }++ expect(runs).toBe(3);+ expect(+ [...localCache.values()].every(+ (entry) => entry.versions.get('hash-1')?.state === 'packed',+ ),+ ).toBe(true);+ });++ it('should preserve lastReadAt so a packed version keeps its place in the eviction order', () => {+ const localCache = new Map([[`${FIELD_METADATA}:ws-a`, liveEntry(IDLE)]]);++ run(localCache, 2);++ expect(+ localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),+ ).toMatchObject({ lastReadAt: IDLE });+ });++ it('should leave the version live when pack declines it', () => {+ const localCache = new Map([[`${FIELD_METADATA}:ws-a`, liveEntry(IDLE)]]);++ expect(run(localCache, 2, { pack: () => undefined }).packed).toBe(0);+ expect(stateOf(localCache, `${FIELD_METADATA}:ws-a`)).toBe('live');+ });++ it('should not let an unpackable provider starve the versions behind it', () => {+ const localCache = new Map([+ [`${ORM}:ws-coldest`, liveEntry(IDLE - 2_000)],+ [`${ORM}:ws-colder`, liveEntry(IDLE - 1_000)],+ [`${FIELD_METADATA}:ws-warm`, liveEntry(IDLE)],+ ]);++ const result = run(localCache, 2, {+ isPackable: (localKey) => !localKey.startsWith(ORM),+ });++ expect(result.packed).toBe(1);+ expect(result.pending).toBe(0);+ expect(stateOf(localCache, `${FIELD_METADATA}:ws-warm`)).toBe('packed');+ expect(stateOf(localCache, `${ORM}:ws-coldest`)).toBe('live');+ });++ it('should ignore versions that are already packed', () => {+ const localCache = new Map([+ [`${FIELD_METADATA}:ws-packed`, packedEntry(IDLE)],+ ]);++ const result = run(localCache, 2);++ expect(result.packed).toBe(0);+ expect(result.pending).toBe(0);+ });+});diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.tsindex 205f59dd963..1bdf4428b89 100644--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts@@ -8,7 +8,7 @@ const TTL_MS = 30 * 60 * 1000;const ORM_KEY_NAME = 'ORMEntityMetadatas';const entry = (lastReadAt: number): WorkspaceLocalCacheEntry<unknown> => ({- versions: new Map([['h', { state: 'hot', data: {}, lastReadAt }]]),+ versions: new Map([['h', { state: 'live', data: {}, lastReadAt }]]),latestHash: 'h',lastHashCheckedAt: lastReadAt,});diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.tsindex c7259055c6d..34c4bd18420 100644--- a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts+++ b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts@@ -7,12 +7,12 @@ export type LocalCacheStats = {versionsTotal: number;versionsByCount: Record<string, number>;entriesByKeyName: Record<string, number>;- hotVersionsByKeyName: Record<string, number>;- coldVersionsByKeyName: Record<string, number>;- coldBytesByKeyName: Record<string, number>;- hotVersionsTotal: number;- coldVersionsTotal: number;- coldBytesTotal: number;+ liveVersionsByKeyName: Record<string, number>;+ packedVersionsByKeyName: Record<string, number>;+ packedBytesByKeyName: Record<string, number>;+ liveVersionsTotal: number;+ packedVersionsTotal: number;+ packedBytesTotal: number;};type StatsInput = ReadonlyMap<@@ -32,22 +32,22 @@ export const computeLocalCacheStats = ('5+': 0,};const entriesByKeyName: Record<string, number> = {};- const hotVersionsByKeyName: Record<string, number> = {};- const coldVersionsByKeyName: Record<string, number> = {};- const coldBytesByKeyName: Record<string, number> = {};+ const liveVersionsByKeyName: Record<string, number> = {};+ const packedVersionsByKeyName: Record<string, number> = {};+ const packedBytesByKeyName: Record<string, number> = {};let versionsTotal = 0;- let hotVersionsTotal = 0;- let coldVersionsTotal = 0;- let coldBytesTotal = 0;+ let liveVersionsTotal = 0;+ let packedVersionsTotal = 0;+ let packedBytesTotal = 0;for (const [key, entry] of localCache) {workspaceIds.add(key.slice(key.lastIndexOf(':') + 1));const keyName = getKeyNameFromLocalCacheKey(key);entriesByKeyName[keyName] = (entriesByKeyName[keyName] ?? 0) + 1;- hotVersionsByKeyName[keyName] ??= 0;- coldVersionsByKeyName[keyName] ??= 0;- coldBytesByKeyName[keyName] ??= 0;+ liveVersionsByKeyName[keyName] ??= 0;+ packedVersionsByKeyName[keyName] ??= 0;+ packedBytesByKeyName[keyName] ??= 0;const versionCount = entry.versions.size;@@ -57,16 +57,16 @@ export const computeLocalCacheStats = (versionsByCount[bucket] = (versionsByCount[bucket] ?? 0) + 1;for (const version of entry.versions.values()) {- if (version.state === 'cold') {- coldVersionsByKeyName[keyName] += 1;- coldVersionsTotal += 1;- coldBytesByKeyName[keyName] += version.blob.byteLength;- coldBytesTotal += version.blob.byteLength;+ if (version.state === 'packed') {+ packedVersionsByKeyName[keyName] += 1;+ packedVersionsTotal += 1;+ packedBytesByKeyName[keyName] += version.blob.byteLength;+ packedBytesTotal += version.blob.byteLength;continue;}- hotVersionsByKeyName[keyName] += 1;- hotVersionsTotal += 1;+ liveVersionsByKeyName[keyName] += 1;+ liveVersionsTotal += 1;}}@@ -76,11 +76,11 @@ export const computeLocalCacheStats = (versionsTotal,versionsByCount,entriesByKeyName,- hotVersionsByKeyName,- coldVersionsByKeyName,- coldBytesByKeyName,- hotVersionsTotal,- coldVersionsTotal,- coldBytesTotal,+ liveVersionsByKeyName,+ packedVersionsByKeyName,+ packedBytesByKeyName,+ liveVersionsTotal,+ packedVersionsTotal,+ packedBytesTotal,};};diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.tsdeleted file mode 100644index d48d1426c9f..00000000000--- a/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts+++ /dev/null@@ -1,89 +0,0 @@-import { isDefined } from 'twenty-shared/utils';--import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';-import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';--export type DemoteColdStorageResult = {- demoted: number;- remaining: number;-};--export const demoteColdStorageEntries = <T>({- localCache,- hotEntriesPerProvider,- budgetMs,- serialize,- now = () => performance.now(),-}: {- localCache: ReadonlyMap<string, WorkspaceLocalCacheEntry<T>>;- hotEntriesPerProvider: number;- budgetMs: number;- serialize: (params: { localKey: string; data: T }) => Buffer | undefined;- now?: () => number;-}): DemoteColdStorageResult => {- const startedAt = now();- const hotByProvider = new Map<- string,- { localKey: string; hash: string; lastReadAt: number }[]- >();-- for (const [localKey, entry] of localCache) {- for (const [hash, version] of entry.versions) {- if (version.state !== 'hot') {- continue;- }-- const provider = getKeyNameFromLocalCacheKey(localKey);- const hot = hotByProvider.get(provider) ?? [];-- hot.push({ localKey, hash, lastReadAt: version.lastReadAt });- hotByProvider.set(provider, hot);- }- }-- const candidates: { localKey: string; hash: string; lastReadAt: number }[] =- [];-- for (const hot of hotByProvider.values()) {- if (hot.length <= hotEntriesPerProvider) {- continue;- }-- hot.sort((a, b) => a.lastReadAt - b.lastReadAt);- candidates.push(...hot.slice(0, hot.length - hotEntriesPerProvider));- }-- candidates.sort((a, b) => a.lastReadAt - b.lastReadAt);-- let demoted = 0;- let index = 0;-- for (const { localKey, hash } of candidates) {- if (now() - startedAt >= budgetMs) {- break;- }- index += 1;-- const entry = localCache.get(localKey);- const version = entry?.versions.get(hash);-- if (!isDefined(entry) || version?.state !== 'hot') {- continue;- }-- const blob = serialize({ localKey, data: version.data });-- if (!isDefined(blob)) {- continue;- }-- entry.versions.set(hash, {- state: 'cold',- blob,- lastReadAt: version.lastReadAt,- });- demoted += 1;- }-- return { demoted, remaining: candidates.length - index };-};diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/pack-idle-versions.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/pack-idle-versions.util.tsnew file mode 100644index 00000000000..8762f222d97--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/utils/pack-idle-versions.util.ts@@ -0,0 +1,75 @@+import { isDefined } from 'twenty-shared/utils';++import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';++export type PackIdleVersionsResult = {+ packed: number;+ pending: number;+};++export const packIdleVersions = <T>({+ localCache,+ minIdleMs,+ maxEntryVersionsPerRun,+ isPackable,+ pack,+ nowEpochMs = () => Date.now(),+}: {+ localCache: ReadonlyMap<string, WorkspaceLocalCacheEntry<T>>;+ minIdleMs: number;+ maxEntryVersionsPerRun: number;+ isPackable: (localKey: string) => boolean;+ pack: (params: { localKey: string; data: T }) => Buffer | undefined;+ nowEpochMs?: () => number;+}): PackIdleVersionsResult => {+ const idleSince = nowEpochMs() - minIdleMs;+ const candidates: { localKey: string; hash: string; lastReadAt: number }[] =+ [];++ for (const [localKey, entry] of localCache) {+ if (!isPackable(localKey)) {+ continue;+ }++ for (const [hash, version] of entry.versions) {+ if (version.state === 'live' && version.lastReadAt <= idleSince) {+ candidates.push({ localKey, hash, lastReadAt: version.lastReadAt });+ }+ }+ }++ candidates.sort((a, b) => a.lastReadAt - b.lastReadAt);++ let packed = 0;++ for (const { localKey, hash } of candidates) {+ if (packed >= maxEntryVersionsPerRun) {+ break;+ }++ const entry = localCache.get(localKey);+ const version = entry?.versions.get(hash);++ if (!isDefined(entry) || version?.state !== 'live') {+ continue;+ }++ const blob = pack({ localKey, data: version.data });++ if (!isDefined(blob)) {+ continue;+ }++ entry.versions.set(hash, {+ state: 'packed',+ blob,+ lastReadAt: version.lastReadAt,+ });+ packed += 1;+ }++ return {+ packed,+ pending: candidates.length - packed,+ };+};diff --git a/packages/twenty-server/src/instrument.ts b/packages/twenty-server/src/instrument.tsindex f367c747e85..67dc40b8f71 100644--- a/packages/twenty-server/src/instrument.ts+++ b/packages/twenty-server/src/instrument.ts@@ -9,6 +9,7 @@ import {MeterProvider,PeriodicExportingMetricReader,} from '@opentelemetry/sdk-metrics';+import { isNonEmptyString } from '@sniptt/guards';import * as Sentry from '@sentry/node';import { nodeProfilingIntegration } from '@sentry/profiling-node';@@ -24,11 +25,37 @@ const meterDrivers = parseArrayEnvVar([],);+const parseSampleRate = ({+ value,+ fallback,+}: {+ value: string | undefined;+ fallback: number;+}) => {+ if (!isNonEmptyString(value?.trim())) {+ return fallback;+ }++ const parsed = Number(value);++ return Number.isFinite(parsed) && parsed >= 0 && parsed <= 1+ ? parsed+ : fallback;+};+if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.SENTRY) {+ const tracesSampleRate = parseSampleRate({+ value: process.env.SENTRY_TRACES_SAMPLE_RATE,+ fallback: 0.1,+ });+Sentry.init({environment: process.env.SENTRY_ENVIRONMENT,release: process.env.APP_VERSION,dsn: process.env.SENTRY_DSN,+ defaultIntegrations: Sentry.getDefaultIntegrations({+ tracesSampleRate,+ }).filter((integration) => integration.name !== 'Modules'),integrations: [Sentry.redisIntegration(),Sentry.httpIntegration(),@@ -55,8 +82,11 @@ if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.SENTRY) {}),nodeProfilingIntegration(),],- tracesSampleRate: 0.1,- profilesSampleRate: 0.3,+ tracesSampleRate,+ profilesSampleRate: parseSampleRate({+ value: process.env.SENTRY_PROFILES_SAMPLE_RATE,+ fallback: 0.01,+ }),sendDefaultPii: true,debug: process.env.NODE_ENV === NodeEnvironment.DEVELOPMENT,beforeSendSpan: (span) => {