Skip to content

The open alternative to Salesforce, designed for AI.

perf(server): cut Sentry instrumentation overhead and stop packing the working set (#24019)

Charles Bochet9f840e4fb73d
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
index 5a44aa383a0..6617bd9f843 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
@@ -72,7 +72,7 @@ export enum MetricsKeys {
AiChatTurnCompleted = 'ai-chat/turn-completed',
AiChatTurnFailed = 'ai-chat/turn-failed',
WorkspaceMetadataCacheLocalEviction = 'workspace-metadata-cache/local-eviction',
- WorkspaceMetadataCacheDemoted = 'workspace-metadata-cache/demoted',
+ WorkspaceMetadataCachePacked = 'workspace-metadata-cache/packed',
SdkClientGenerationSucceeded = 'sdk-client-generation/succeeded',
SdkClientGenerationFailed = 'sdk-client-generation/failed',
SdkClientGenerationDurationMs = 'sdk-client-generation/duration-ms',
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
index 4153044c3c3..71e29e3e340 100644
--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
@@ -38,8 +38,8 @@ export class WorkspaceCacheMetricsService {
private readonly logger = new Logger(WorkspaceCacheMetricsService.name);
private readonly recomputeDurationHistogram: Histogram;
private readonly redisWriteDurationHistogram: Histogram;
- private readonly demotionDurationHistogram: Histogram;
- private readonly hydrationDurationHistogram: Histogram;
+ private readonly packingDurationHistogram: Histogram;
+ private readonly unpackingDurationHistogram: Histogram;
private localCache?: LocalCache;
private cacheSizeByKeyName: Record<string, number> = {};
@@ -48,7 +48,7 @@ export class WorkspaceCacheMetricsService {
private sizeStartupTimer?: ReturnType<typeof setTimeout>;
private statsCache?: { computedAt: number } & LocalCacheStats;
private sizeSampleInFlight = false;
- private demotionBacklog = 0;
+ private packingBacklog = 0;
constructor(private readonly metricsService: MetricsService) {
const meter = this.metricsService.getMeter();
@@ -71,20 +71,19 @@ export class WorkspaceCacheMetricsService {
advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },
},
);
- this.demotionDurationHistogram = meter.createHistogram(
- 'twenty_workspace_cache_demotion_slice_duration_seconds',
+ this.packingDurationHistogram = meter.createHistogram(
+ 'twenty_workspace_cache_packing_run_duration_seconds',
{
- description:
- 'Event loop time consumed by one cold-storage demotion slice',
+ description: 'Event loop time consumed by one packing run',
unit: 's',
advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },
},
);
- this.hydrationDurationHistogram = meter.createHistogram(
- 'twenty_workspace_cache_hydration_duration_seconds',
+ this.unpackingDurationHistogram = meter.createHistogram(
+ 'twenty_workspace_cache_unpacking_duration_seconds',
{
description:
- 'Wall-clock time to hydrate one cold cache entry back into objects on read',
+ 'Wall-clock time to unpack one packed cache version back into objects on read',
unit: 's',
advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },
},
@@ -114,30 +113,30 @@ export class WorkspaceCacheMetricsService {
this.redisWriteDurationHistogram.record(seconds);
}
- recordDemotionSlice({
+ recordPackingRun({
durationSeconds,
- demoted,
- remaining,
+ packed,
+ pending,
}: {
durationSeconds: number;
- demoted: number;
- remaining: number;
+ packed: number;
+ pending: number;
}): void {
- this.demotionBacklog = remaining;
+ this.packingBacklog = pending;
- if (demoted === 0) {
+ if (packed === 0) {
return;
}
- this.demotionDurationHistogram.record(durationSeconds);
+ this.packingDurationHistogram.record(durationSeconds);
this.metricsService.incrementCounterBy({
- key: MetricsKeys.WorkspaceMetadataCacheDemoted,
- amount: demoted,
+ key: MetricsKeys.WorkspaceMetadataCachePacked,
+ amount: packed,
});
}
- recordHydration(seconds: number, cacheKey: WorkspaceCacheKeyName): void {
- this.hydrationDurationHistogram.record(seconds, { cache_key: cacheKey });
+ recordUnpacking(seconds: number, cacheKey: WorkspaceCacheKeyName): void {
+ this.unpackingDurationHistogram.record(seconds, { cache_key: cacheKey });
}
recordEviction(amount: number): void {
@@ -216,7 +215,7 @@ export class WorkspaceCacheMetricsService {
for (const version of entry.versions.values()) {
entryBytes +=
- version.state === 'cold'
+ version.state === 'packed'
? version.blob.byteLength
: deepSizeBytes(version.data, SIZE_WALK_NODE_CAP);
}
@@ -319,59 +318,59 @@ export class WorkspaceCacheMetricsService {
metricName: 'twenty_workspace_cache_local_versions_by_state',
options: {
description:
- 'Local workspace metadata cache versions by storage state and provider',
+ 'Local workspace metadata cache versions by tier (live or packed) and provider',
},
callback: async () => {
const stats = this.getStats();
return [
- ...Object.entries(stats.hotVersionsByKeyName).map(
+ ...Object.entries(stats.liveVersionsByKeyName).map(
([keyName, value]) => ({
value,
- attributes: { provider: keyName, state: 'hot' },
+ attributes: { provider: keyName, state: 'live' },
}),
),
- ...Object.entries(stats.coldVersionsByKeyName).map(
+ ...Object.entries(stats.packedVersionsByKeyName).map(
([keyName, value]) => ({
value,
- attributes: { provider: keyName, state: 'cold' },
+ attributes: { provider: keyName, state: 'packed' },
}),
),
];
},
});
this.metricsService.createObservableGauge({
- metricName: 'twenty_workspace_cache_local_hot_versions',
+ metricName: 'twenty_workspace_cache_local_live_versions',
options: {
description:
- 'Total hot (live object) versions in the local workspace metadata cache',
+ 'Total live (object graph) versions in the local workspace metadata cache',
},
- callback: async () => this.getStats().hotVersionsTotal,
+ callback: async () => this.getStats().liveVersionsTotal,
});
this.metricsService.createObservableGauge({
- metricName: 'twenty_workspace_cache_local_cold_versions',
+ metricName: 'twenty_workspace_cache_local_packed_versions',
options: {
description:
- 'Total cold (serialized buffer) versions in the local workspace metadata cache',
+ 'Total packed (serialized buffer) versions in the local workspace metadata cache',
},
- callback: async () => this.getStats().coldVersionsTotal,
+ callback: async () => this.getStats().packedVersionsTotal,
});
this.metricsService.createObservableGauge({
- metricName: 'twenty_workspace_cache_local_cold_bytes',
+ metricName: 'twenty_workspace_cache_local_packed_bytes',
options: {
description:
- 'Exact retained bytes held as cold buffers in the local workspace metadata cache',
+ 'Exact retained bytes held as packed buffers in the local workspace metadata cache',
unit: 'By',
},
- callback: async () => this.getStats().coldBytesTotal,
+ callback: async () => this.getStats().packedBytesTotal,
});
this.metricsService.createObservableGauge({
- metricName: 'twenty_workspace_cache_demotion_backlog',
+ metricName: 'twenty_workspace_cache_packing_backlog',
options: {
description:
- 'Entries still awaiting demotion when the last demotion slice ran out of budget',
+ 'Versions currently eligible for packing, whether or not the last run reached them',
},
- callback: async () => this.demotionBacklog,
+ callback: async () => this.packingBacklog,
});
}
}
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
index ae6ee098608..2a2c1d08a93 100644
--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
@@ -40,7 +40,8 @@ import {
type WorkspaceLocalCacheEntry,
} from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
import { combineCacheHashes } from 'src/engine/workspace-cache/utils/combine-cache-hashes.util';
-import { demoteColdStorageEntries } from 'src/engine/workspace-cache/utils/demote-cold-storage-entries.util';
+import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';
+import { packIdleVersions } from 'src/engine/workspace-cache/utils/pack-idle-versions.util';
import { sweepLocalCache } from 'src/engine/workspace-cache/utils/sweep-local-cache.util';
const LOCAL_TTL_MS = 100; // 100ms
@@ -52,9 +53,9 @@ const MAX_LOCAL_CACHE_ENTRIES = 6_000;
const MIN_EVICT_KEYS = 100;
const LOCAL_ENTRY_TTL_MS = 30 * 60 * 1000; // 30 minutes idle
const LOCAL_CACHE_SWEEP_INTERVAL_MS = 60 * 1000;
-const HOT_ENTRIES_PER_PROVIDER = 64;
-const DEMOTION_INTERVAL_MS = 250;
-const DEMOTION_BUDGET_MS = 10;
+const PACKING_INTERVAL_MS = 500;
+const MAX_ENTRY_VERSIONS_PER_PACKING_RUN = 2;
+const MIN_IDLE_BEFORE_PACKING_MS = 60 * 1000;
// Per-provider entry caps, keyed by local cache key prefix (ORM graphs are ~5 MB each).
const MAX_LOCAL_ENTRIES_BY_KEY_NAME = new Map<string, number>([
['ORMEntityMetadatas', 128],
@@ -82,7 +83,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
WorkspaceLocalCacheEntry<CacheDataType>
>();
private sweepTimer?: ReturnType<typeof setInterval>;
- private demotionTimer?: ReturnType<typeof setInterval>;
+ private packingTimer?: ReturnType<typeof setInterval>;
private readonly workspaceCacheProviders = new Map<
WorkspaceCacheKeyName,
WorkspaceCacheProvider<CacheDataType, StoredCacheDataType>
@@ -144,8 +145,8 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
if (isDefined(this.sweepTimer)) {
clearInterval(this.sweepTimer);
}
- if (isDefined(this.demotionTimer)) {
- clearInterval(this.demotionTimer);
+ if (isDefined(this.packingTimer)) {
+ clearInterval(this.packingTimer);
}
this.cacheMetricsService.stop();
}
@@ -157,11 +158,11 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
);
this.sweepTimer.unref();
- this.demotionTimer = setInterval(
- () => this.demoteColdEntries(),
- DEMOTION_INTERVAL_MS,
+ this.packingTimer = setInterval(
+ () => this.runPacking(),
+ PACKING_INTERVAL_MS,
);
- this.demotionTimer.unref();
+ this.packingTimer.unref();
}
public async getOrRecompute<const K extends WorkspaceCacheKeyName[]>(
@@ -608,7 +609,12 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
const version = entry?.versions.get(entry.latestHash);
if (isDefined(entry) && isDefined(version)) {
- const data = this.readVersion({ keyName, entry, version });
+ const data = this.readVersion({
+ keyName,
+ entry,
+ hash: entry.latestHash,
+ version,
+ });
Object.assign(result.data, { [keyName]: data });
result.hashes[keyName] = entry.latestHash;
@@ -660,7 +666,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
this.localCache.set(localKey, entry);
}
- entry.versions.set(hash, { state: 'hot', data, lastReadAt: Date.now() });
+ entry.versions.set(hash, { state: 'live', data, lastReadAt: Date.now() });
entry.latestHash = hash;
entry.lastHashCheckedAt = Date.now();
@@ -682,24 +688,22 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
}
}
- private demoteColdEntries(): void {
+ private runPacking(): void {
const startedAt = performance.now();
- const { demoted, remaining } = demoteColdStorageEntries({
+ const { packed, pending } = packIdleVersions({
localCache: this.localCache,
- hotEntriesPerProvider: HOT_ENTRIES_PER_PROVIDER,
- budgetMs: DEMOTION_BUDGET_MS,
- serialize: ({ localKey, data }) => {
- const separatorIndex = localKey.lastIndexOf(':');
- const keyName = localKey.slice(
- 0,
- separatorIndex,
+ minIdleMs: MIN_IDLE_BEFORE_PACKING_MS,
+ maxEntryVersionsPerRun: MAX_ENTRY_VERSIONS_PER_PACKING_RUN,
+ isPackable: (localKey) =>
+ !this.localDataOnlyKeys.has(
+ getKeyNameFromLocalCacheKey(localKey) as WorkspaceCacheKeyName,
+ ),
+ pack: ({ localKey, data }) => {
+ const keyName = getKeyNameFromLocalCacheKey(
+ localKey,
) as WorkspaceCacheKeyName;
- if (this.localDataOnlyKeys.has(keyName)) {
- return undefined;
- }
-
return Buffer.from(
JSON.stringify(
this.getProviderOrThrow(keyName).encodeForCacheStorage(data),
@@ -709,41 +713,43 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
},
});
- this.cacheMetricsService.recordDemotionSlice({
+ this.cacheMetricsService.recordPackingRun({
durationSeconds: (performance.now() - startedAt) / 1000,
- demoted,
- remaining,
+ packed,
+ pending,
});
}
private readVersion({
keyName,
entry,
+ hash,
version,
}: {
keyName: WorkspaceCacheKeyName;
entry: WorkspaceLocalCacheEntry<CacheDataType>;
+ hash: string;
version: VersionEntry<CacheDataType>;
}): CacheDataType {
- if (version.state === 'hot') {
+ if (version.state === 'live') {
version.lastReadAt = Date.now();
return version.data;
}
- const hydrateStartedAt = performance.now();
+ const unpackStartedAt = performance.now();
const data = this.getProviderOrThrow(keyName).decodeFromCacheStorage(
JSON.parse(version.blob.toString('utf8')),
);
- entry.versions.set(entry.latestHash, {
- state: 'hot',
+ entry.versions.set(hash, {
+ state: 'live',
data,
lastReadAt: Date.now(),
});
- this.cacheMetricsService.recordHydration(
- (performance.now() - hydrateStartedAt) / 1000,
+ this.cacheMetricsService.recordUnpacking(
+ (performance.now() - unpackStartedAt) / 1000,
keyName,
);
diff --git a/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts b/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts
index 9bade6606a3..0085ab1d62d 100644
--- a/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/types/workspace-local-cache-entry.type.ts
@@ -1,16 +1,16 @@
-export type HotVersionEntry<T> = {
- state: 'hot';
+export type LiveVersionEntry<T> = {
+ state: 'live';
data: T;
lastReadAt: number;
};
-export type ColdVersionEntry = {
- state: 'cold';
+export type PackedVersionEntry = {
+ state: 'packed';
blob: Buffer;
lastReadAt: number;
};
-export type VersionEntry<T> = HotVersionEntry<T> | ColdVersionEntry;
+export type VersionEntry<T> = LiveVersionEntry<T> | PackedVersionEntry;
export type WorkspaceLocalCacheEntry<T> = {
versions: Map<string, VersionEntry<T>>;
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts
index c3b3ade2459..c7163f544f5 100644
--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts
@@ -1,13 +1,13 @@
import { computeLocalCacheStats } from 'src/engine/workspace-cache/utils/compute-local-cache-stats.util';
-const entry = (versionCount: number, state: 'hot' | 'cold' = 'hot') => ({
+const entry = (versionCount: number, state: 'live' | 'packed' = 'live') => ({
versions: new Map(
Array.from({ length: versionCount }, (_, index) => [
`hash-${index}`,
- state === 'hot'
- ? { state: 'hot' as const, data: 'data', lastReadAt: index }
+ state === 'live'
+ ? { state: 'live' as const, data: 'data', lastReadAt: index }
: {
- state: 'cold' as const,
+ state: 'packed' as const,
blob: Buffer.alloc(100),
lastReadAt: index,
},
@@ -25,12 +25,12 @@ describe('computeLocalCacheStats', () => {
versionsTotal: 0,
versionsByCount: { '1': 0, '2': 0, '3': 0, '4': 0, '5+': 0 },
entriesByKeyName: {},
- hotVersionsByKeyName: {},
- coldVersionsByKeyName: {},
- coldBytesByKeyName: {},
- hotVersionsTotal: 0,
- coldVersionsTotal: 0,
- coldBytesTotal: 0,
+ liveVersionsByKeyName: {},
+ packedVersionsByKeyName: {},
+ packedBytesByKeyName: {},
+ liveVersionsTotal: 0,
+ packedVersionsTotal: 0,
+ packedBytesTotal: 0,
});
});
@@ -73,23 +73,23 @@ describe('computeLocalCacheStats', () => {
});
});
- it('splits versions by storage state and sums exact cold bytes', () => {
+ it('splits versions by storage state and sums exact packed bytes', () => {
const stats = computeLocalCacheStats(
new Map([
- ['flatFieldMetadataMaps:ws-a', entry(2, 'cold')],
- ['flatFieldMetadataMaps:ws-b', entry(1, 'hot')],
- ['ORMEntityMetadatas:ws-a', entry(1, 'hot')],
+ ['flatFieldMetadataMaps:ws-a', entry(2, 'packed')],
+ ['flatFieldMetadataMaps:ws-b', entry(1, 'live')],
+ ['ORMEntityMetadatas:ws-a', entry(1, 'live')],
]),
);
- expect(stats.hotVersionsTotal).toBe(2);
- expect(stats.coldVersionsTotal).toBe(2);
- expect(stats.coldBytesTotal).toBe(200);
- expect(stats.coldVersionsByKeyName).toEqual({
+ expect(stats.liveVersionsTotal).toBe(2);
+ expect(stats.packedVersionsTotal).toBe(2);
+ expect(stats.packedBytesTotal).toBe(200);
+ expect(stats.packedVersionsByKeyName).toEqual({
flatFieldMetadataMaps: 2,
ORMEntityMetadatas: 0,
});
- expect(stats.hotVersionsByKeyName).toEqual({
+ expect(stats.liveVersionsByKeyName).toEqual({
flatFieldMetadataMaps: 1,
ORMEntityMetadatas: 1,
});
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts
deleted file mode 100644
index 57e56f74c63..00000000000
--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts
+++ /dev/null
@@ -1,200 +0,0 @@
-import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
-import { demoteColdStorageEntries } from 'src/engine/workspace-cache/utils/demote-cold-storage-entries.util';
-
-const FIELD_METADATA = 'flatFieldMetadataMaps';
-const ORM = 'ORMEntityMetadatas';
-
-const hotEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({
- versions: new Map([['hash-1', { state: 'hot', data: 'data', lastReadAt }]]),
- latestHash: 'hash-1',
- lastHashCheckedAt: lastReadAt,
-});
-
-const coldEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({
- versions: new Map([
- ['hash-1', { state: 'cold', blob: Buffer.from('{}'), lastReadAt }],
- ]),
- latestHash: 'hash-1',
- lastHashCheckedAt: lastReadAt,
-});
-
-const serialize = () => Buffer.from('serialized');
-
-const NO_BUDGET_LIMIT = Number.POSITIVE_INFINITY;
-
-const demote = (
- localCache: Map<string, WorkspaceLocalCacheEntry<string>>,
- hotEntriesPerProvider: number,
- overrides?: {
- serialize?: () => Buffer | undefined;
- budgetMs?: number;
- now?: () => number;
- },
-) =>
- demoteColdStorageEntries({
- localCache,
- hotEntriesPerProvider,
- budgetMs: overrides?.budgetMs ?? NO_BUDGET_LIMIT,
- serialize: overrides?.serialize ?? serialize,
- now: overrides?.now,
- }).demoted;
-
-describe('demoteColdStorageEntries', () => {
- it('should demote nothing when the provider is within its hot budget', () => {
- const localCache = new Map([
- [`${FIELD_METADATA}:ws-a`, hotEntry(1)],
- [`${FIELD_METADATA}:ws-b`, hotEntry(2)],
- ]);
-
- expect(demote(localCache, 2)).toBe(0);
- expect(
- localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),
- ).toMatchObject({ state: 'hot' });
- });
-
- it('should demote the least recently read entries beyond the hot budget', () => {
- const localCache = new Map([
- [`${FIELD_METADATA}:ws-newest`, hotEntry(9)],
- [`${FIELD_METADATA}:ws-oldest`, hotEntry(1)],
- [`${FIELD_METADATA}:ws-middle`, hotEntry(5)],
- ]);
-
- expect(demote(localCache, 1)).toBe(2);
- expect(
- localCache.get(`${FIELD_METADATA}:ws-oldest`)?.versions.get('hash-1'),
- ).toMatchObject({ state: 'cold', blob: Buffer.from('serialized') });
- expect(
- localCache.get(`${FIELD_METADATA}:ws-newest`)?.versions.get('hash-1'),
- ).toMatchObject({ state: 'hot' });
- });
-
- it('should preserve lastReadAt so a demoted entry keeps its place in the eviction order', () => {
- const localCache = new Map([
- [`${FIELD_METADATA}:ws-a`, hotEntry(1)],
- [`${FIELD_METADATA}:ws-b`, hotEntry(2)],
- ]);
-
- demote(localCache, 1);
-
- expect(
- localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),
- ).toMatchObject({ lastReadAt: 1 });
- });
-
- it('should leave the entry hot when serialize declines it', () => {
- const localCache = new Map([
- [`${FIELD_METADATA}:ws-a`, hotEntry(1)],
- [`${FIELD_METADATA}:ws-b`, hotEntry(2)],
- ]);
-
- expect(demote(localCache, 1, { serialize: () => undefined })).toBe(0);
- expect(
- localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),
- ).toMatchObject({ state: 'hot' });
- });
-
- it('should budget each provider independently', () => {
- const localCache = new Map([
- [`${FIELD_METADATA}:ws-a`, hotEntry(1)],
- [`${FIELD_METADATA}:ws-b`, hotEntry(2)],
- [`${ORM}:ws-a`, hotEntry(3)],
- ]);
-
- expect(demote(localCache, 1)).toBe(1);
- expect(localCache.get(`${ORM}:ws-a`)?.versions.get('hash-1')).toMatchObject(
- {
- state: 'hot',
- },
- );
- });
-
- it('should not count an already cold entry against the hot budget', () => {
- const localCache = new Map([
- [`${FIELD_METADATA}:ws-cold`, coldEntry(1)],
- [`${FIELD_METADATA}:ws-hot`, hotEntry(2)],
- ]);
-
- expect(demote(localCache, 1)).toBe(0);
- });
-
- describe('time budget', () => {
- const clockAdvancingPerEntry = () => {
- let elapsed = 0;
-
- return () => {
- const current = elapsed;
-
- elapsed += 10;
-
- return current;
- };
- };
-
- const buildCache = (count: number) =>
- new Map(
- Array.from({ length: count }, (_, index) => [
- `${FIELD_METADATA}:ws-${index}`,
- hotEntry(index),
- ]),
- );
-
- it('should stop once the budget is spent instead of draining the backlog', () => {
- const localCache = buildCache(10);
-
- const result = demoteColdStorageEntries({
- localCache,
- hotEntriesPerProvider: 0,
- budgetMs: 25,
- serialize,
- now: clockAdvancingPerEntry(),
- });
-
- expect(result.demoted).toBe(2);
- expect(result.remaining).toBe(8);
- });
-
- it('should demote coldest first so an interrupted slice leaves the hottest behind', () => {
- const localCache = buildCache(10);
-
- demoteColdStorageEntries({
- localCache,
- hotEntriesPerProvider: 0,
- budgetMs: 25,
- serialize,
- now: clockAdvancingPerEntry(),
- });
-
- expect(
- localCache.get(`${FIELD_METADATA}:ws-0`)?.versions.get('hash-1'),
- ).toMatchObject({ state: 'cold' });
- expect(
- localCache.get(`${FIELD_METADATA}:ws-9`)?.versions.get('hash-1'),
- ).toMatchObject({ state: 'hot' });
- });
-
- it('should converge across successive slices without a cursor', () => {
- const localCache = buildCache(10);
- let slices = 0;
- let remaining = Number.POSITIVE_INFINITY;
-
- while (remaining > 0 && slices < 10) {
- remaining = demoteColdStorageEntries({
- localCache,
- hotEntriesPerProvider: 0,
- budgetMs: 25,
- serialize,
- now: clockAdvancingPerEntry(),
- }).remaining;
- slices += 1;
- }
-
- expect(remaining).toBe(0);
- expect(slices).toBe(5);
- expect(
- [...localCache.values()].every(
- (entry) => entry.versions.get('hash-1')?.state === 'cold',
- ),
- ).toBe(true);
- });
- });
-});
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/pack-idle-versions.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/pack-idle-versions.util.spec.ts
new file mode 100644
index 00000000000..145b7da8602
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/pack-idle-versions.util.spec.ts
@@ -0,0 +1,148 @@
+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
+import { packIdleVersions } from 'src/engine/workspace-cache/utils/pack-idle-versions.util';
+
+const FIELD_METADATA = 'flatFieldMetadataMaps';
+const ORM = 'ORMEntityMetadatas';
+
+const NOW_EPOCH_MS = 1_000_000;
+const IDLE_MS = 60_000;
+const IDLE = NOW_EPOCH_MS - 90_000;
+const RECENT = NOW_EPOCH_MS - 1_000;
+
+const liveEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({
+ versions: new Map([['hash-1', { state: 'live', data: 'data', lastReadAt }]]),
+ latestHash: 'hash-1',
+ lastHashCheckedAt: lastReadAt,
+});
+
+const packedEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({
+ versions: new Map([
+ ['hash-1', { state: 'packed', blob: Buffer.from('{}'), lastReadAt }],
+ ]),
+ latestHash: 'hash-1',
+ lastHashCheckedAt: lastReadAt,
+});
+
+const pack = () => Buffer.from('packed');
+
+const run = (
+ localCache: Map<string, WorkspaceLocalCacheEntry<string>>,
+ maxEntryVersionsPerRun: number,
+ overrides?: {
+ pack?: () => Buffer | undefined;
+ isPackable?: (localKey: string) => boolean;
+ },
+) =>
+ packIdleVersions({
+ localCache,
+ minIdleMs: IDLE_MS,
+ maxEntryVersionsPerRun,
+ isPackable: overrides?.isPackable ?? (() => true),
+ pack: overrides?.pack ?? pack,
+ nowEpochMs: () => NOW_EPOCH_MS,
+ });
+
+const stateOf = (
+ localCache: Map<string, WorkspaceLocalCacheEntry<string>>,
+ localKey: string,
+) => localCache.get(localKey)?.versions.get('hash-1')?.state;
+
+describe('packIdleVersions', () => {
+ it('should pack a version that has gone idle', () => {
+ const localCache = new Map([[`${FIELD_METADATA}:ws-a`, liveEntry(IDLE)]]);
+
+ expect(run(localCache, 2).packed).toBe(1);
+ expect(stateOf(localCache, `${FIELD_METADATA}:ws-a`)).toBe('packed');
+ });
+
+ it('should leave a version read inside the idle window live', () => {
+ const localCache = new Map([[`${FIELD_METADATA}:ws-a`, liveEntry(RECENT)]]);
+
+ expect(run(localCache, 2).packed).toBe(0);
+ expect(stateOf(localCache, `${FIELD_METADATA}:ws-a`)).toBe('live');
+ });
+
+ it('should pack no more than one run allows, coldest first', () => {
+ const localCache = new Map([
+ [`${FIELD_METADATA}:ws-coldest`, liveEntry(IDLE - 2_000)],
+ [`${FIELD_METADATA}:ws-colder`, liveEntry(IDLE - 1_000)],
+ [`${ORM}:ws-warm`, liveEntry(IDLE)],
+ ]);
+
+ const result = run(localCache, 2);
+
+ expect(result.packed).toBe(2);
+ expect(result.pending).toBe(1);
+ expect(stateOf(localCache, `${FIELD_METADATA}:ws-coldest`)).toBe('packed');
+ expect(stateOf(localCache, `${FIELD_METADATA}:ws-colder`)).toBe('packed');
+ expect(stateOf(localCache, `${ORM}:ws-warm`)).toBe('live');
+ });
+
+ it('should converge across successive runs', () => {
+ const localCache = new Map(
+ Array.from({ length: 5 }, (_, index) => [
+ `${FIELD_METADATA}:ws-${index}`,
+ liveEntry(IDLE - index),
+ ]),
+ );
+ let pending = Number.POSITIVE_INFINITY;
+ let runs = 0;
+
+ while (pending > 0 && runs < 10) {
+ pending = run(localCache, 2).pending;
+ runs += 1;
+ }
+
+ expect(runs).toBe(3);
+ expect(
+ [...localCache.values()].every(
+ (entry) => entry.versions.get('hash-1')?.state === 'packed',
+ ),
+ ).toBe(true);
+ });
+
+ it('should preserve lastReadAt so a packed version keeps its place in the eviction order', () => {
+ const localCache = new Map([[`${FIELD_METADATA}:ws-a`, liveEntry(IDLE)]]);
+
+ run(localCache, 2);
+
+ expect(
+ localCache.get(`${FIELD_METADATA}:ws-a`)?.versions.get('hash-1'),
+ ).toMatchObject({ lastReadAt: IDLE });
+ });
+
+ it('should leave the version live when pack declines it', () => {
+ const localCache = new Map([[`${FIELD_METADATA}:ws-a`, liveEntry(IDLE)]]);
+
+ expect(run(localCache, 2, { pack: () => undefined }).packed).toBe(0);
+ expect(stateOf(localCache, `${FIELD_METADATA}:ws-a`)).toBe('live');
+ });
+
+ it('should not let an unpackable provider starve the versions behind it', () => {
+ const localCache = new Map([
+ [`${ORM}:ws-coldest`, liveEntry(IDLE - 2_000)],
+ [`${ORM}:ws-colder`, liveEntry(IDLE - 1_000)],
+ [`${FIELD_METADATA}:ws-warm`, liveEntry(IDLE)],
+ ]);
+
+ const result = run(localCache, 2, {
+ isPackable: (localKey) => !localKey.startsWith(ORM),
+ });
+
+ expect(result.packed).toBe(1);
+ expect(result.pending).toBe(0);
+ expect(stateOf(localCache, `${FIELD_METADATA}:ws-warm`)).toBe('packed');
+ expect(stateOf(localCache, `${ORM}:ws-coldest`)).toBe('live');
+ });
+
+ it('should ignore versions that are already packed', () => {
+ const localCache = new Map([
+ [`${FIELD_METADATA}:ws-packed`, packedEntry(IDLE)],
+ ]);
+
+ const result = run(localCache, 2);
+
+ expect(result.packed).toBe(0);
+ expect(result.pending).toBe(0);
+ });
+});
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts
index 205f59dd963..1bdf4428b89 100644
--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/sweep-local-cache.util.spec.ts
@@ -8,7 +8,7 @@ const TTL_MS = 30 * 60 * 1000;
const ORM_KEY_NAME = 'ORMEntityMetadatas';
const entry = (lastReadAt: number): WorkspaceLocalCacheEntry<unknown> => ({
- versions: new Map([['h', { state: 'hot', data: {}, lastReadAt }]]),
+ versions: new Map([['h', { state: 'live', data: {}, lastReadAt }]]),
latestHash: 'h',
lastHashCheckedAt: lastReadAt,
});
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts
index c7259055c6d..34c4bd18420 100644
--- a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts
@@ -7,12 +7,12 @@ export type LocalCacheStats = {
versionsTotal: number;
versionsByCount: Record<string, number>;
entriesByKeyName: Record<string, number>;
- hotVersionsByKeyName: Record<string, number>;
- coldVersionsByKeyName: Record<string, number>;
- coldBytesByKeyName: Record<string, number>;
- hotVersionsTotal: number;
- coldVersionsTotal: number;
- coldBytesTotal: number;
+ liveVersionsByKeyName: Record<string, number>;
+ packedVersionsByKeyName: Record<string, number>;
+ packedBytesByKeyName: Record<string, number>;
+ liveVersionsTotal: number;
+ packedVersionsTotal: number;
+ packedBytesTotal: number;
};
type StatsInput = ReadonlyMap<
@@ -32,22 +32,22 @@ export const computeLocalCacheStats = (
'5+': 0,
};
const entriesByKeyName: Record<string, number> = {};
- const hotVersionsByKeyName: Record<string, number> = {};
- const coldVersionsByKeyName: Record<string, number> = {};
- const coldBytesByKeyName: Record<string, number> = {};
+ const liveVersionsByKeyName: Record<string, number> = {};
+ const packedVersionsByKeyName: Record<string, number> = {};
+ const packedBytesByKeyName: Record<string, number> = {};
let versionsTotal = 0;
- let hotVersionsTotal = 0;
- let coldVersionsTotal = 0;
- let coldBytesTotal = 0;
+ let liveVersionsTotal = 0;
+ let packedVersionsTotal = 0;
+ let packedBytesTotal = 0;
for (const [key, entry] of localCache) {
workspaceIds.add(key.slice(key.lastIndexOf(':') + 1));
const keyName = getKeyNameFromLocalCacheKey(key);
entriesByKeyName[keyName] = (entriesByKeyName[keyName] ?? 0) + 1;
- hotVersionsByKeyName[keyName] ??= 0;
- coldVersionsByKeyName[keyName] ??= 0;
- coldBytesByKeyName[keyName] ??= 0;
+ liveVersionsByKeyName[keyName] ??= 0;
+ packedVersionsByKeyName[keyName] ??= 0;
+ packedBytesByKeyName[keyName] ??= 0;
const versionCount = entry.versions.size;
@@ -57,16 +57,16 @@ export const computeLocalCacheStats = (
versionsByCount[bucket] = (versionsByCount[bucket] ?? 0) + 1;
for (const version of entry.versions.values()) {
- if (version.state === 'cold') {
- coldVersionsByKeyName[keyName] += 1;
- coldVersionsTotal += 1;
- coldBytesByKeyName[keyName] += version.blob.byteLength;
- coldBytesTotal += version.blob.byteLength;
+ if (version.state === 'packed') {
+ packedVersionsByKeyName[keyName] += 1;
+ packedVersionsTotal += 1;
+ packedBytesByKeyName[keyName] += version.blob.byteLength;
+ packedBytesTotal += version.blob.byteLength;
continue;
}
- hotVersionsByKeyName[keyName] += 1;
- hotVersionsTotal += 1;
+ liveVersionsByKeyName[keyName] += 1;
+ liveVersionsTotal += 1;
}
}
@@ -76,11 +76,11 @@ export const computeLocalCacheStats = (
versionsTotal,
versionsByCount,
entriesByKeyName,
- hotVersionsByKeyName,
- coldVersionsByKeyName,
- coldBytesByKeyName,
- hotVersionsTotal,
- coldVersionsTotal,
- coldBytesTotal,
+ liveVersionsByKeyName,
+ packedVersionsByKeyName,
+ packedBytesByKeyName,
+ liveVersionsTotal,
+ packedVersionsTotal,
+ packedBytesTotal,
};
};
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts
deleted file mode 100644
index d48d1426c9f..00000000000
--- a/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts
+++ /dev/null
@@ -1,89 +0,0 @@
-import { isDefined } from 'twenty-shared/utils';
-
-import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
-import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';
-
-export type DemoteColdStorageResult = {
- demoted: number;
- remaining: number;
-};
-
-export const demoteColdStorageEntries = <T>({
- localCache,
- hotEntriesPerProvider,
- budgetMs,
- serialize,
- now = () => performance.now(),
-}: {
- localCache: ReadonlyMap<string, WorkspaceLocalCacheEntry<T>>;
- hotEntriesPerProvider: number;
- budgetMs: number;
- serialize: (params: { localKey: string; data: T }) => Buffer | undefined;
- now?: () => number;
-}): DemoteColdStorageResult => {
- const startedAt = now();
- const hotByProvider = new Map<
- string,
- { localKey: string; hash: string; lastReadAt: number }[]
- >();
-
- for (const [localKey, entry] of localCache) {
- for (const [hash, version] of entry.versions) {
- if (version.state !== 'hot') {
- continue;
- }
-
- const provider = getKeyNameFromLocalCacheKey(localKey);
- const hot = hotByProvider.get(provider) ?? [];
-
- hot.push({ localKey, hash, lastReadAt: version.lastReadAt });
- hotByProvider.set(provider, hot);
- }
- }
-
- const candidates: { localKey: string; hash: string; lastReadAt: number }[] =
- [];
-
- for (const hot of hotByProvider.values()) {
- if (hot.length <= hotEntriesPerProvider) {
- continue;
- }
-
- hot.sort((a, b) => a.lastReadAt - b.lastReadAt);
- candidates.push(...hot.slice(0, hot.length - hotEntriesPerProvider));
- }
-
- candidates.sort((a, b) => a.lastReadAt - b.lastReadAt);
-
- let demoted = 0;
- let index = 0;
-
- for (const { localKey, hash } of candidates) {
- if (now() - startedAt >= budgetMs) {
- break;
- }
- index += 1;
-
- const entry = localCache.get(localKey);
- const version = entry?.versions.get(hash);
-
- if (!isDefined(entry) || version?.state !== 'hot') {
- continue;
- }
-
- const blob = serialize({ localKey, data: version.data });
-
- if (!isDefined(blob)) {
- continue;
- }
-
- entry.versions.set(hash, {
- state: 'cold',
- blob,
- lastReadAt: version.lastReadAt,
- });
- demoted += 1;
- }
-
- return { demoted, remaining: candidates.length - index };
-};
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/pack-idle-versions.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/pack-idle-versions.util.ts
new file mode 100644
index 00000000000..8762f222d97
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/pack-idle-versions.util.ts
@@ -0,0 +1,75 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
+
+export type PackIdleVersionsResult = {
+ packed: number;
+ pending: number;
+};
+
+export const packIdleVersions = <T>({
+ localCache,
+ minIdleMs,
+ maxEntryVersionsPerRun,
+ isPackable,
+ pack,
+ nowEpochMs = () => Date.now(),
+}: {
+ localCache: ReadonlyMap<string, WorkspaceLocalCacheEntry<T>>;
+ minIdleMs: number;
+ maxEntryVersionsPerRun: number;
+ isPackable: (localKey: string) => boolean;
+ pack: (params: { localKey: string; data: T }) => Buffer | undefined;
+ nowEpochMs?: () => number;
+}): PackIdleVersionsResult => {
+ const idleSince = nowEpochMs() - minIdleMs;
+ const candidates: { localKey: string; hash: string; lastReadAt: number }[] =
+ [];
+
+ for (const [localKey, entry] of localCache) {
+ if (!isPackable(localKey)) {
+ continue;
+ }
+
+ for (const [hash, version] of entry.versions) {
+ if (version.state === 'live' && version.lastReadAt <= idleSince) {
+ candidates.push({ localKey, hash, lastReadAt: version.lastReadAt });
+ }
+ }
+ }
+
+ candidates.sort((a, b) => a.lastReadAt - b.lastReadAt);
+
+ let packed = 0;
+
+ for (const { localKey, hash } of candidates) {
+ if (packed >= maxEntryVersionsPerRun) {
+ break;
+ }
+
+ const entry = localCache.get(localKey);
+ const version = entry?.versions.get(hash);
+
+ if (!isDefined(entry) || version?.state !== 'live') {
+ continue;
+ }
+
+ const blob = pack({ localKey, data: version.data });
+
+ if (!isDefined(blob)) {
+ continue;
+ }
+
+ entry.versions.set(hash, {
+ state: 'packed',
+ blob,
+ lastReadAt: version.lastReadAt,
+ });
+ packed += 1;
+ }
+
+ return {
+ packed,
+ pending: candidates.length - packed,
+ };
+};
diff --git a/packages/twenty-server/src/instrument.ts b/packages/twenty-server/src/instrument.ts
index f367c747e85..67dc40b8f71 100644
--- a/packages/twenty-server/src/instrument.ts
+++ b/packages/twenty-server/src/instrument.ts
@@ -9,6 +9,7 @@ import {
MeterProvider,
PeriodicExportingMetricReader,
} from '@opentelemetry/sdk-metrics';
+import { isNonEmptyString } from '@sniptt/guards';
import * as Sentry from '@sentry/node';
import { nodeProfilingIntegration } from '@sentry/profiling-node';
@@ -24,11 +25,37 @@ const meterDrivers = parseArrayEnvVar(
[],
);
+const parseSampleRate = ({
+ value,
+ fallback,
+}: {
+ value: string | undefined;
+ fallback: number;
+}) => {
+ if (!isNonEmptyString(value?.trim())) {
+ return fallback;
+ }
+
+ const parsed = Number(value);
+
+ return Number.isFinite(parsed) && parsed >= 0 && parsed <= 1
+ ? parsed
+ : fallback;
+};
+
if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.SENTRY) {
+ const tracesSampleRate = parseSampleRate({
+ value: process.env.SENTRY_TRACES_SAMPLE_RATE,
+ fallback: 0.1,
+ });
+
Sentry.init({
environment: process.env.SENTRY_ENVIRONMENT,
release: process.env.APP_VERSION,
dsn: process.env.SENTRY_DSN,
+ defaultIntegrations: Sentry.getDefaultIntegrations({
+ tracesSampleRate,
+ }).filter((integration) => integration.name !== 'Modules'),
integrations: [
Sentry.redisIntegration(),
Sentry.httpIntegration(),
@@ -55,8 +82,11 @@ if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.SENTRY) {
}),
nodeProfilingIntegration(),
],
- tracesSampleRate: 0.1,
- profilesSampleRate: 0.3,
+ tracesSampleRate,
+ profilesSampleRate: parseSampleRate({
+ value: process.env.SENTRY_PROFILES_SAMPLE_RATE,
+ fallback: 0.01,
+ }),
sendDefaultPii: true,
debug: process.env.NODE_ENV === NodeEnvironment.DEVELOPMENT,
beforeSendSpan: (span) => {