Skip to content

The open alternative to Salesforce, designed for AI.

perf(server): remove the compact-storage flag, slice cache demotion, use a plain LIMIT on find-many (#23980)

Charles Bochet9d5edecefdcf
diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql b/packages/twenty-client-sdk/src/metadata/generated/schema.graphql
index 47e24571e66..f4512620618 100644
--- a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql
+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.graphql
@@ -1738,7 +1738,6 @@ enum FeatureFlagKey {
IS_SETTINGS_DISCOVERY_HERO_ENABLED
IS_WORKFLOW_VERSION_IN_CORE_ENABLED
IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED
- IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED
IS_NATIVE_CALL_RECORDING_TABS_ENABLED
}
diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.ts b/packages/twenty-client-sdk/src/metadata/generated/schema.ts
index ab55861b711..527b490e83a 100644
--- a/packages/twenty-client-sdk/src/metadata/generated/schema.ts
+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.ts
@@ -1395,7 +1395,7 @@ export interface FeatureFlag {
__typename: 'FeatureFlag'
}
-export type FeatureFlagKey = 'IS_APP_CLAIMING_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_CALENDAR_WEEK_VIEW_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_LIST_VIEW_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' | 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED' | 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED' | 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED' | 'IS_NATIVE_CALL_RECORDING_TABS_ENABLED'
+export type FeatureFlagKey = 'IS_APP_CLAIMING_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_CALENDAR_WEEK_VIEW_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_LIST_VIEW_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' | 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED' | 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED' | 'IS_NATIVE_CALL_RECORDING_TABS_ENABLED'
export interface WorkspaceUrls {
customUrl?: Scalars['String']
@@ -9653,7 +9653,6 @@ export const enumFeatureFlagKey = {
IS_SETTINGS_DISCOVERY_HERO_ENABLED: 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' as const,
IS_WORKFLOW_VERSION_IN_CORE_ENABLED: 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED' as const,
IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED: 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED' as const,
- IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED: 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED' as const,
IS_NATIVE_CALL_RECORDING_TABS_ENABLED: 'IS_NATIVE_CALL_RECORDING_TABS_ENABLED' as const
}
diff --git a/packages/twenty-front/src/generated-admin/graphql.ts b/packages/twenty-front/src/generated-admin/graphql.ts
index 3f4e1637e83..f256de0368a 100644
--- a/packages/twenty-front/src/generated-admin/graphql.ts
+++ b/packages/twenty-front/src/generated-admin/graphql.ts
@@ -380,8 +380,7 @@ export enum FeatureFlagKey {
IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',
IS_UNIQUE_INDEXES_ENABLED = 'IS_UNIQUE_INDEXES_ENABLED',
IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED = 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED',
- IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED',
- IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED = 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED'
+ IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED'
}
export type FindApplicationRegistrationInstalledWorkspacesInput = {
diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.ts
index 32417f0d9a9..0e4f46e684d 100644
--- a/packages/twenty-front/src/generated-metadata/graphql.ts
+++ b/packages/twenty-front/src/generated-metadata/graphql.ts
@@ -1824,8 +1824,7 @@ export enum FeatureFlagKey {
IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',
IS_UNIQUE_INDEXES_ENABLED = 'IS_UNIQUE_INDEXES_ENABLED',
IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED = 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED',
- IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED',
- IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED = 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED'
+ IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED'
}
export type Field = {
diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts
index 66ae1e5f985..e87c325dec5 100644
--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts
+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts
@@ -1,5 +1,7 @@
import { Injectable } from '@nestjs/common';
+import { msg } from '@lingui/core/macro';
+
import { isDefined } from 'class-validator';
import {
QUERY_MAX_RECORDS,
@@ -30,6 +32,7 @@ import {
} from 'src/engine/api/common/types/common-query-args.type';
import { CommonSelectedFieldsResult } from 'src/engine/api/common/types/common-selected-fields-result.type';
import { buildCursorPage } from 'src/engine/api/utils/build-cursor-page.util';
+import { getNonToOneJoinAliases } from 'src/engine/api/common/utils/get-non-to-one-join-aliases.util';
import { getPageInfo } from 'src/engine/api/common/utils/get-page-info.util';
import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper';
import { buildColumnsToSelect } from 'src/engine/api/graphql/graphql-query-runner/utils/build-columns-to-select';
@@ -163,12 +166,27 @@ export class CommonFindManyQueryRunnerService extends CommonBaseQueryRunnerServi
flatFieldMetadataMaps,
});
- if (isDefined(args.offset)) {
- queryBuilder.skip(args.offset);
+ queryBuilder.setFindOptions({ select: columnsToSelect });
+
+ // A join that can duplicate root rows makes a row-level LIMIT return fewer records than
+ // asked, so it is rejected rather than paginated with take/skip, which drops the LIMIT
+ // from the scan.
+ const nonToOneJoinAliases = getNonToOneJoinAliases(queryBuilder);
+
+ if (nonToOneJoinAliases.length > 0) {
+ throw new CommonQueryRunnerException(
+ `Cannot filter or order through ${nonToOneJoinAliases.join(', ')}: only to-one relations are supported`,
+ CommonQueryRunnerExceptionCode.INVALID_QUERY_INPUT,
+ {
+ userFriendlyMessage: msg`Filtering or ordering through this relation is not supported.`,
+ },
+ );
}
- queryBuilder.setFindOptions({ select: columnsToSelect });
- queryBuilder.take(limit + 1);
+ if (isDefined(args.offset)) {
+ queryBuilder.offset(args.offset);
+ }
+ queryBuilder.limit(limit + 1);
// Add order columns AFTER setFindOptions (setFindOptions clears addSelect)
// Pass columnsToSelect so we only add columns that aren't already selected
diff --git a/packages/twenty-server/src/engine/api/common/utils/get-non-to-one-join-aliases.util.ts b/packages/twenty-server/src/engine/api/common/utils/get-non-to-one-join-aliases.util.ts
new file mode 100644
index 00000000000..2a7ccb58252
--- /dev/null
+++ b/packages/twenty-server/src/engine/api/common/utils/get-non-to-one-join-aliases.util.ts
@@ -0,0 +1,23 @@
+import { type Alias } from 'typeorm/query-builder/Alias';
+import { type RelationMetadata } from 'typeorm/metadata/RelationMetadata';
+
+import { isDefined } from 'twenty-shared/utils';
+
+type QueryBuilderWithJoinAttributes = {
+ expressionMap: {
+ joinAttributes: {
+ alias: Pick<Alias, 'name'>;
+ relation?: Pick<RelationMetadata, 'isOneToMany' | 'isManyToMany'>;
+ }[];
+ };
+};
+
+export const getNonToOneJoinAliases = (
+ queryBuilder: QueryBuilderWithJoinAttributes,
+): string[] =>
+ queryBuilder.expressionMap.joinAttributes
+ .filter(
+ ({ relation }) =>
+ !isDefined(relation) || relation.isOneToMany || relation.isManyToMany,
+ )
+ .map(({ alias }) => alias.name);
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
index 8aa0d5b1ca2..5a44aa383a0 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
@@ -72,6 +72,7 @@ export enum MetricsKeys {
AiChatTurnCompleted = 'ai-chat/turn-completed',
AiChatTurnFailed = 'ai-chat/turn-failed',
WorkspaceMetadataCacheLocalEviction = 'workspace-metadata-cache/local-eviction',
+ WorkspaceMetadataCacheDemoted = 'workspace-metadata-cache/demoted',
SdkClientGenerationSucceeded = 'sdk-client-generation/succeeded',
SdkClientGenerationFailed = 'sdk-client-generation/failed',
SdkClientGenerationDurationMs = 'sdk-client-generation/duration-ms',
diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.ts
index 049aac12d7b..24fb04607e2 100644
--- a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/decode-flat-field-metadata-maps-from-cache.util.ts
@@ -30,7 +30,7 @@ const decodeFlatFieldMetadata = (
};
export const decodeFlatFieldMetadataMapsFromCache = (
- encoded: EncodedFlatFieldMetadataMaps | FlatEntityMaps<FlatFieldMetadata>,
+ encoded: EncodedFlatFieldMetadataMaps,
): FlatEntityMaps<FlatFieldMetadata> => {
const byUniversalIdentifier: FlatEntityMaps<FlatFieldMetadata>['byUniversalIdentifier'] =
{};
diff --git a/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts b/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts
index c3e7a368af1..84397d9e241 100644
--- a/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts
@@ -251,7 +251,6 @@ describe('WorkspaceEntityManager', () => {
IS_SETTINGS_DISCOVERY_HERO_ENABLED: false,
IS_WORKFLOW_VERSION_IN_CORE_ENABLED: false,
IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED: false,
- IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED: false,
IS_NATIVE_CALL_RECORDING_TABS_ENABLED: false,
},
userWorkspaceRoleMap: {},
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
index 340af1a684a..4153044c3c3 100644
--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
@@ -38,6 +38,8 @@ export class WorkspaceCacheMetricsService {
private readonly logger = new Logger(WorkspaceCacheMetricsService.name);
private readonly recomputeDurationHistogram: Histogram;
private readonly redisWriteDurationHistogram: Histogram;
+ private readonly demotionDurationHistogram: Histogram;
+ private readonly hydrationDurationHistogram: Histogram;
private localCache?: LocalCache;
private cacheSizeByKeyName: Record<string, number> = {};
@@ -46,6 +48,7 @@ export class WorkspaceCacheMetricsService {
private sizeStartupTimer?: ReturnType<typeof setTimeout>;
private statsCache?: { computedAt: number } & LocalCacheStats;
private sizeSampleInFlight = false;
+ private demotionBacklog = 0;
constructor(private readonly metricsService: MetricsService) {
const meter = this.metricsService.getMeter();
@@ -68,6 +71,24 @@ export class WorkspaceCacheMetricsService {
advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },
},
);
+ this.demotionDurationHistogram = meter.createHistogram(
+ 'twenty_workspace_cache_demotion_slice_duration_seconds',
+ {
+ description:
+ 'Event loop time consumed by one cold-storage demotion slice',
+ unit: 's',
+ advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },
+ },
+ );
+ this.hydrationDurationHistogram = meter.createHistogram(
+ 'twenty_workspace_cache_hydration_duration_seconds',
+ {
+ description:
+ 'Wall-clock time to hydrate one cold cache entry back into objects on read',
+ unit: 's',
+ advice: { explicitBucketBoundaries: CACHE_DURATION_BUCKETS_SECONDS },
+ },
+ );
}
start(localCache: LocalCache): void {
@@ -93,6 +114,32 @@ export class WorkspaceCacheMetricsService {
this.redisWriteDurationHistogram.record(seconds);
}
+ recordDemotionSlice({
+ durationSeconds,
+ demoted,
+ remaining,
+ }: {
+ durationSeconds: number;
+ demoted: number;
+ remaining: number;
+ }): void {
+ this.demotionBacklog = remaining;
+
+ if (demoted === 0) {
+ return;
+ }
+
+ this.demotionDurationHistogram.record(durationSeconds);
+ this.metricsService.incrementCounterBy({
+ key: MetricsKeys.WorkspaceMetadataCacheDemoted,
+ amount: demoted,
+ });
+ }
+
+ recordHydration(seconds: number, cacheKey: WorkspaceCacheKeyName): void {
+ this.hydrationDurationHistogram.record(seconds, { cache_key: cacheKey });
+ }
+
recordEviction(amount: number): void {
this.metricsService.incrementCounterBy({
key: MetricsKeys.WorkspaceMetadataCacheLocalEviction,
@@ -268,5 +315,63 @@ export class WorkspaceCacheMetricsService {
}),
),
});
+ this.metricsService.createMultiObservableGauge({
+ metricName: 'twenty_workspace_cache_local_versions_by_state',
+ options: {
+ description:
+ 'Local workspace metadata cache versions by storage state and provider',
+ },
+ callback: async () => {
+ const stats = this.getStats();
+
+ return [
+ ...Object.entries(stats.hotVersionsByKeyName).map(
+ ([keyName, value]) => ({
+ value,
+ attributes: { provider: keyName, state: 'hot' },
+ }),
+ ),
+ ...Object.entries(stats.coldVersionsByKeyName).map(
+ ([keyName, value]) => ({
+ value,
+ attributes: { provider: keyName, state: 'cold' },
+ }),
+ ),
+ ];
+ },
+ });
+ this.metricsService.createObservableGauge({
+ metricName: 'twenty_workspace_cache_local_hot_versions',
+ options: {
+ description:
+ 'Total hot (live object) versions in the local workspace metadata cache',
+ },
+ callback: async () => this.getStats().hotVersionsTotal,
+ });
+ this.metricsService.createObservableGauge({
+ metricName: 'twenty_workspace_cache_local_cold_versions',
+ options: {
+ description:
+ 'Total cold (serialized buffer) versions in the local workspace metadata cache',
+ },
+ callback: async () => this.getStats().coldVersionsTotal,
+ });
+ this.metricsService.createObservableGauge({
+ metricName: 'twenty_workspace_cache_local_cold_bytes',
+ options: {
+ description:
+ 'Exact retained bytes held as cold buffers in the local workspace metadata cache',
+ unit: 'By',
+ },
+ callback: async () => this.getStats().coldBytesTotal,
+ });
+ this.metricsService.createObservableGauge({
+ metricName: 'twenty_workspace_cache_demotion_backlog',
+ options: {
+ description:
+ 'Entries still awaiting demotion when the last demotion slice ran out of budget',
+ },
+ callback: async () => this.demotionBacklog,
+ });
}
}
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
index 2436ed6b05d..ae6ee098608 100644
--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
@@ -9,7 +9,6 @@ import { DiscoveryService, Reflector } from '@nestjs/core';
import * as Sentry from '@sentry/node';
import crypto from 'crypto';
-import { FeatureFlagKey } from 'twenty-shared/types';
import { isDefined, isValidUuid } from 'twenty-shared/utils';
import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';
@@ -54,6 +53,8 @@ const MIN_EVICT_KEYS = 100;
const LOCAL_ENTRY_TTL_MS = 30 * 60 * 1000; // 30 minutes idle
const LOCAL_CACHE_SWEEP_INTERVAL_MS = 60 * 1000;
const HOT_ENTRIES_PER_PROVIDER = 64;
+const DEMOTION_INTERVAL_MS = 250;
+const DEMOTION_BUDGET_MS = 10;
// Per-provider entry caps, keyed by local cache key prefix (ORM graphs are ~5 MB each).
const MAX_LOCAL_ENTRIES_BY_KEY_NAME = new Map<string, number>([
['ORMEntityMetadatas', 128],
@@ -80,7 +81,8 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
string,
WorkspaceLocalCacheEntry<CacheDataType>
>();
- private lastLocalCacheSweepAt: number | undefined;
+ private sweepTimer?: ReturnType<typeof setInterval>;
+ private demotionTimer?: ReturnType<typeof setInterval>;
private readonly workspaceCacheProviders = new Map<
WorkspaceCacheKeyName,
WorkspaceCacheProvider<CacheDataType, StoredCacheDataType>
@@ -135,12 +137,33 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
}
this.cacheMetricsService.start(this.localCache);
+ this.startMaintenanceTimers();
}
onModuleDestroy(): void {
+ if (isDefined(this.sweepTimer)) {
+ clearInterval(this.sweepTimer);
+ }
+ if (isDefined(this.demotionTimer)) {
+ clearInterval(this.demotionTimer);
+ }
this.cacheMetricsService.stop();
}
+ private startMaintenanceTimers(): void {
+ this.sweepTimer = setInterval(
+ () => this.sweepLocalCache(),
+ LOCAL_CACHE_SWEEP_INTERVAL_MS,
+ );
+ this.sweepTimer.unref();
+
+ this.demotionTimer = setInterval(
+ () => this.demoteColdEntries(),
+ DEMOTION_INTERVAL_MS,
+ );
+ this.demotionTimer.unref();
+ }
+
public async getOrRecompute<const K extends WorkspaceCacheKeyName[]>(
workspaceId: string,
cacheKeyNames: K,
@@ -159,7 +182,6 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
workspaceId: string,
cacheKeyNames: K,
): Promise<WorkspaceCacheResultWithHashes<K>> {
- this.sweepLocalCacheIfDue();
this.assertValidCacheParameters(workspaceId, cacheKeyNames);
const memoKey =
@@ -270,10 +292,6 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
strategy: 'mint',
});
- // Invalidation writes entries too, so run the sweep here as well — otherwise an
- // invalidate-only workload never enforces the per-provider caps.
- this.sweepLocalCacheIfDue();
-
// Clear memoizer again after recomputation to evict any stale entries
// cached by concurrent getOrRecompute calls during the flush window.
await this.memoizer.clearKeys(`${workspaceId}-`);
@@ -434,10 +452,9 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
return [`${baseKey}:data`, `${baseKey}:hash`];
});
- const [allValues, isCompactStorageEnabled] = await Promise.all([
- this.cacheStorage.mget<CacheDataType | string>(allKeys),
- this.resolveCompactStorageEnabled(workspaceId),
- ]);
+ const allValues = await this.cacheStorage.mget<CacheDataType | string>(
+ allKeys,
+ );
for (const [index, keyName] of cacheKeyNames.entries()) {
const rawData = allValues[index * 2] as CacheDataType | undefined;
@@ -447,9 +464,8 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
let data: CacheDataType;
try {
- data = isCompactStorageEnabled
- ? this.getProviderOrThrow(keyName).decodeFromCacheStorage(rawData)
- : rawData;
+ data =
+ this.getProviderOrThrow(keyName).decodeFromCacheStorage(rawData);
} catch (error) {
this.logger.warn(
`Failed to decode cached ${keyName} for workspace ${workspaceId}, recomputing`,
@@ -523,16 +539,6 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
const computed = await Promise.all(computePromises);
- const recomputedFeatureFlagsMap = computed.find(
- ({ keyName }) => keyName === 'featureFlagsMap',
- )?.data as WorkspaceCacheDataMap['featureFlagsMap'] | undefined;
-
- const isCompactStorageEnabled = isDefined(recomputedFeatureFlagsMap)
- ? recomputedFeatureFlagsMap[
- FeatureFlagKey.IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED
- ]
- : await this.resolveCompactStorageEnabled(workspaceId);
-
const redisEntries: Array<{
key: string;
value: StoredCacheDataType | string;
@@ -557,9 +563,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
if (!isLocalDataOnly) {
redisEntries.push({
key: `${baseKey}:data`,
- value: isCompactStorageEnabled
- ? this.getProviderOrThrow(keyName).encodeForCacheStorage(data)
- : data,
+ value: this.getProviderOrThrow(keyName).encodeForCacheStorage(data),
});
}
@@ -663,17 +667,9 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
this.cleanupStaleVersions(entry);
}
- private sweepLocalCacheIfDue(): void {
+ private sweepLocalCache(): void {
const now = Date.now();
- if (
- isDefined(this.lastLocalCacheSweepAt) &&
- now - this.lastLocalCacheSweepAt < LOCAL_CACHE_SWEEP_INTERVAL_MS
- ) {
- return;
- }
- this.lastLocalCacheSweepAt = now;
-
const evicted = sweepLocalCache(this.localCache, now, {
ttlMs: LOCAL_ENTRY_TTL_MS,
maxEntriesByKeyName: MAX_LOCAL_ENTRIES_BY_KEY_NAME,
@@ -684,27 +680,23 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
if (evicted > 0) {
this.cacheMetricsService.recordEviction(evicted);
}
-
- this.demoteColdEntries();
}
private demoteColdEntries(): void {
- demoteColdStorageEntries({
+ const startedAt = performance.now();
+
+ const { demoted, remaining } = demoteColdStorageEntries({
localCache: this.localCache,
hotEntriesPerProvider: HOT_ENTRIES_PER_PROVIDER,
+ budgetMs: DEMOTION_BUDGET_MS,
serialize: ({ localKey, data }) => {
const separatorIndex = localKey.lastIndexOf(':');
- const workspaceId = localKey.slice(separatorIndex + 1);
const keyName = localKey.slice(
0,
separatorIndex,
) as WorkspaceCacheKeyName;
- if (
- this.localDataOnlyKeys.has(keyName) ||
- keyName === 'featureFlagsMap' ||
- !this.isCompactStorageEnabledInLocalCache(workspaceId)
- ) {
+ if (this.localDataOnlyKeys.has(keyName)) {
return undefined;
}
@@ -716,45 +708,12 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
);
},
});
- }
- private isCompactStorageEnabledInLocalCache(
- workspaceId: string,
- ): boolean | undefined {
- const entry = this.localCache.get(
- this.buildCacheKey(workspaceId, 'featureFlagsMap'),
- );
- const version = entry?.versions.get(entry.latestHash);
-
- if (version?.state !== 'hot') {
- return undefined;
- }
-
- return (version.data as WorkspaceCacheDataMap['featureFlagsMap'])[
- FeatureFlagKey.IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED
- ];
- }
-
- // A restarted pod reads an already-encoded Redis payload with an empty local
- // cache, so the flag has to resolve from Redis before any decode happens.
- private async resolveCompactStorageEnabled(
- workspaceId: string,
- ): Promise<boolean> {
- const localValue = this.isCompactStorageEnabledInLocalCache(workspaceId);
-
- if (isDefined(localValue)) {
- return localValue;
- }
-
- const featureFlagsMap = await this.cacheStorage.get<
- WorkspaceCacheDataMap['featureFlagsMap']
- >(`${this.buildCacheKey(workspaceId, 'featureFlagsMap')}:data`);
-
- return (
- featureFlagsMap?.[
- FeatureFlagKey.IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED
- ] ?? false
- );
+ this.cacheMetricsService.recordDemotionSlice({
+ durationSeconds: (performance.now() - startedAt) / 1000,
+ demoted,
+ remaining,
+ });
}
private readVersion({
@@ -772,6 +731,7 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
return version.data;
}
+ const hydrateStartedAt = performance.now();
const data = this.getProviderOrThrow(keyName).decodeFromCacheStorage(
JSON.parse(version.blob.toString('utf8')),
);
@@ -782,6 +742,11 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
lastReadAt: Date.now(),
});
+ this.cacheMetricsService.recordHydration(
+ (performance.now() - hydrateStartedAt) / 1000,
+ keyName,
+ );
+
return data;
}
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts
index a7c5e5dfb72..c3b3ade2459 100644
--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/compute-local-cache-stats.util.spec.ts
@@ -1,6 +1,19 @@
import { computeLocalCacheStats } from 'src/engine/workspace-cache/utils/compute-local-cache-stats.util';
-const entry = (versionCount: number) => ({ versions: { size: versionCount } });
+const entry = (versionCount: number, state: 'hot' | 'cold' = 'hot') => ({
+ versions: new Map(
+ Array.from({ length: versionCount }, (_, index) => [
+ `hash-${index}`,
+ state === 'hot'
+ ? { state: 'hot' as const, data: 'data', lastReadAt: index }
+ : {
+ state: 'cold' as const,
+ blob: Buffer.alloc(100),
+ lastReadAt: index,
+ },
+ ]),
+ ),
+});
describe('computeLocalCacheStats', () => {
it('returns zeros for an empty cache', () => {
@@ -12,6 +25,12 @@ describe('computeLocalCacheStats', () => {
versionsTotal: 0,
versionsByCount: { '1': 0, '2': 0, '3': 0, '4': 0, '5+': 0 },
entriesByKeyName: {},
+ hotVersionsByKeyName: {},
+ coldVersionsByKeyName: {},
+ coldBytesByKeyName: {},
+ hotVersionsTotal: 0,
+ coldVersionsTotal: 0,
+ coldBytesTotal: 0,
});
});
@@ -53,4 +72,26 @@ describe('computeLocalCacheStats', () => {
'5+': 2,
});
});
+
+ it('splits versions by storage state and sums exact cold bytes', () => {
+ const stats = computeLocalCacheStats(
+ new Map([
+ ['flatFieldMetadataMaps:ws-a', entry(2, 'cold')],
+ ['flatFieldMetadataMaps:ws-b', entry(1, 'hot')],
+ ['ORMEntityMetadatas:ws-a', entry(1, 'hot')],
+ ]),
+ );
+
+ expect(stats.hotVersionsTotal).toBe(2);
+ expect(stats.coldVersionsTotal).toBe(2);
+ expect(stats.coldBytesTotal).toBe(200);
+ expect(stats.coldVersionsByKeyName).toEqual({
+ flatFieldMetadataMaps: 2,
+ ORMEntityMetadatas: 0,
+ });
+ expect(stats.hotVersionsByKeyName).toEqual({
+ flatFieldMetadataMaps: 1,
+ ORMEntityMetadatas: 1,
+ });
+ });
});
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts
index 1c139fba4ea..57e56f74c63 100644
--- a/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/__tests__/demote-cold-storage-entries.util.spec.ts
@@ -20,16 +20,24 @@ const coldEntry = (lastReadAt: number): WorkspaceLocalCacheEntry<string> => ({
const serialize = () => Buffer.from('serialized');
+const NO_BUDGET_LIMIT = Number.POSITIVE_INFINITY;
+
const demote = (
localCache: Map<string, WorkspaceLocalCacheEntry<string>>,
hotEntriesPerProvider: number,
- overrides?: { serialize?: () => Buffer | undefined },
+ overrides?: {
+ serialize?: () => Buffer | undefined;
+ budgetMs?: number;
+ now?: () => number;
+ },
) =>
demoteColdStorageEntries({
localCache,
hotEntriesPerProvider,
+ budgetMs: overrides?.budgetMs ?? NO_BUDGET_LIMIT,
serialize: overrides?.serialize ?? serialize,
- });
+ now: overrides?.now,
+ }).demoted;
describe('demoteColdStorageEntries', () => {
it('should demote nothing when the provider is within its hot budget', () => {
@@ -108,4 +116,85 @@ describe('demoteColdStorageEntries', () => {
expect(demote(localCache, 1)).toBe(0);
});
+
+ describe('time budget', () => {
+ const clockAdvancingPerEntry = () => {
+ let elapsed = 0;
+
+ return () => {
+ const current = elapsed;
+
+ elapsed += 10;
+
+ return current;
+ };
+ };
+
+ const buildCache = (count: number) =>
+ new Map(
+ Array.from({ length: count }, (_, index) => [
+ `${FIELD_METADATA}:ws-${index}`,
+ hotEntry(index),
+ ]),
+ );
+
+ it('should stop once the budget is spent instead of draining the backlog', () => {
+ const localCache = buildCache(10);
+
+ const result = demoteColdStorageEntries({
+ localCache,
+ hotEntriesPerProvider: 0,
+ budgetMs: 25,
+ serialize,
+ now: clockAdvancingPerEntry(),
+ });
+
+ expect(result.demoted).toBe(2);
+ expect(result.remaining).toBe(8);
+ });
+
+ it('should demote coldest first so an interrupted slice leaves the hottest behind', () => {
+ const localCache = buildCache(10);
+
+ demoteColdStorageEntries({
+ localCache,
+ hotEntriesPerProvider: 0,
+ budgetMs: 25,
+ serialize,
+ now: clockAdvancingPerEntry(),
+ });
+
+ expect(
+ localCache.get(`${FIELD_METADATA}:ws-0`)?.versions.get('hash-1'),
+ ).toMatchObject({ state: 'cold' });
+ expect(
+ localCache.get(`${FIELD_METADATA}:ws-9`)?.versions.get('hash-1'),
+ ).toMatchObject({ state: 'hot' });
+ });
+
+ it('should converge across successive slices without a cursor', () => {
+ const localCache = buildCache(10);
+ let slices = 0;
+ let remaining = Number.POSITIVE_INFINITY;
+
+ while (remaining > 0 && slices < 10) {
+ remaining = demoteColdStorageEntries({
+ localCache,
+ hotEntriesPerProvider: 0,
+ budgetMs: 25,
+ serialize,
+ now: clockAdvancingPerEntry(),
+ }).remaining;
+ slices += 1;
+ }
+
+ expect(remaining).toBe(0);
+ expect(slices).toBe(5);
+ expect(
+ [...localCache.values()].every(
+ (entry) => entry.versions.get('hash-1')?.state === 'cold',
+ ),
+ ).toBe(true);
+ });
+ });
});
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts
index 71a07ad1442..c7259055c6d 100644
--- a/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/compute-local-cache-stats.util.ts
@@ -1,3 +1,4 @@
+import { type VersionEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';
export type LocalCacheStats = {
@@ -6,10 +7,21 @@ export type LocalCacheStats = {
versionsTotal: number;
versionsByCount: Record<string, number>;
entriesByKeyName: Record<string, number>;
+ hotVersionsByKeyName: Record<string, number>;
+ coldVersionsByKeyName: Record<string, number>;
+ coldBytesByKeyName: Record<string, number>;
+ hotVersionsTotal: number;
+ coldVersionsTotal: number;
+ coldBytesTotal: number;
};
+type StatsInput = ReadonlyMap<
+ string,
+ { versions: ReadonlyMap<string, VersionEntry<unknown>> }
+>;
+
export const computeLocalCacheStats = (
- localCache: ReadonlyMap<string, { versions: { size: number } }>,
+ localCache: StatsInput,
): LocalCacheStats => {
const workspaceIds = new Set<string>();
const versionsByCount: Record<string, number> = {
@@ -20,19 +32,42 @@ export const computeLocalCacheStats = (
'5+': 0,
};
const entriesByKeyName: Record<string, number> = {};
+ const hotVersionsByKeyName: Record<string, number> = {};
+ const coldVersionsByKeyName: Record<string, number> = {};
+ const coldBytesByKeyName: Record<string, number> = {};
let versionsTotal = 0;
+ let hotVersionsTotal = 0;
+ let coldVersionsTotal = 0;
+ let coldBytesTotal = 0;
for (const [key, entry] of localCache) {
workspaceIds.add(key.slice(key.lastIndexOf(':') + 1));
const keyName = getKeyNameFromLocalCacheKey(key);
entriesByKeyName[keyName] = (entriesByKeyName[keyName] ?? 0) + 1;
+ hotVersionsByKeyName[keyName] ??= 0;
+ coldVersionsByKeyName[keyName] ??= 0;
+ coldBytesByKeyName[keyName] ??= 0;
+
const versionCount = entry.versions.size;
versionsTotal += versionCount;
const bucket = versionCount >= 5 ? '5+' : String(versionCount);
versionsByCount[bucket] = (versionsByCount[bucket] ?? 0) + 1;
+
+ for (const version of entry.versions.values()) {
+ if (version.state === 'cold') {
+ coldVersionsByKeyName[keyName] += 1;
+ coldVersionsTotal += 1;
+ coldBytesByKeyName[keyName] += version.blob.byteLength;
+ coldBytesTotal += version.blob.byteLength;
+ continue;
+ }
+
+ hotVersionsByKeyName[keyName] += 1;
+ hotVersionsTotal += 1;
+ }
}
return {
@@ -41,5 +76,11 @@ export const computeLocalCacheStats = (
versionsTotal,
versionsByCount,
entriesByKeyName,
+ hotVersionsByKeyName,
+ coldVersionsByKeyName,
+ coldBytesByKeyName,
+ hotVersionsTotal,
+ coldVersionsTotal,
+ coldBytesTotal,
};
};
diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts
index eb565c5dfbd..d48d1426c9f 100644
--- a/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/utils/demote-cold-storage-entries.util.ts
@@ -1,15 +1,27 @@
+import { isDefined } from 'twenty-shared/utils';
+
import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';
+export type DemoteColdStorageResult = {
+ demoted: number;
+ remaining: number;
+};
+
export const demoteColdStorageEntries = <T>({
localCache,
hotEntriesPerProvider,
+ budgetMs,
serialize,
+ now = () => performance.now(),
}: {
localCache: ReadonlyMap<string, WorkspaceLocalCacheEntry<T>>;
hotEntriesPerProvider: number;
+ budgetMs: number;
serialize: (params: { localKey: string; data: T }) => Buffer | undefined;
-}): number => {
+ now?: () => number;
+}): DemoteColdStorageResult => {
+ const startedAt = now();
const hotByProvider = new Map<
string,
{ localKey: string; hash: string; lastReadAt: number }[]
@@ -29,7 +41,8 @@ export const demoteColdStorageEntries = <T>({
}
}
- let demoted = 0;
+ const candidates: { localKey: string; hash: string; lastReadAt: number }[] =
+ [];
for (const hot of hotByProvider.values()) {
if (hot.length <= hotEntriesPerProvider) {
@@ -37,32 +50,40 @@ export const demoteColdStorageEntries = <T>({
}
hot.sort((a, b) => a.lastReadAt - b.lastReadAt);
+ candidates.push(...hot.slice(0, hot.length - hotEntriesPerProvider));
+ }
- for (const { localKey, hash } of hot.slice(
- 0,
- hot.length - hotEntriesPerProvider,
- )) {
- const entry = localCache.get(localKey);
- const version = entry?.versions.get(hash);
+ candidates.sort((a, b) => a.lastReadAt - b.lastReadAt);
- if (!entry || version?.state !== 'hot') {
- continue;
- }
+ let demoted = 0;
+ let index = 0;
- const blob = serialize({ localKey, data: version.data });
+ for (const { localKey, hash } of candidates) {
+ if (now() - startedAt >= budgetMs) {
+ break;
+ }
+ index += 1;
- if (blob === undefined) {
- continue;
- }
+ const entry = localCache.get(localKey);
+ const version = entry?.versions.get(hash);
+
+ if (!isDefined(entry) || version?.state !== 'hot') {
+ continue;
+ }
+
+ const blob = serialize({ localKey, data: version.data });
- entry.versions.set(hash, {
- state: 'cold',
- blob,
- lastReadAt: version.lastReadAt,
- });
- demoted += 1;
+ if (!isDefined(blob)) {
+ continue;
}
+
+ entry.versions.set(hash, {
+ state: 'cold',
+ blob,
+ lastReadAt: version.lastReadAt,
+ });
+ demoted += 1;
}
- return demoted;
+ return { demoted, remaining: candidates.length - index };
};
diff --git a/packages/twenty-shared/src/types/FeatureFlagKey.ts b/packages/twenty-shared/src/types/FeatureFlagKey.ts
index c026bc7404d..7f9d0a688af 100644
--- a/packages/twenty-shared/src/types/FeatureFlagKey.ts
+++ b/packages/twenty-shared/src/types/FeatureFlagKey.ts
@@ -11,6 +11,5 @@ export enum FeatureFlagKey {
IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',
IS_WORKFLOW_VERSION_IN_CORE_ENABLED = 'IS_WORKFLOW_VERSION_IN_CORE_ENABLED',
IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED = 'IS_WORKFLOW_DISPATCH_FROM_CORE_ENABLED',
- IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED = 'IS_WORKSPACE_CACHE_COMPACT_STORAGE_ENABLED',
IS_NATIVE_CALL_RECORDING_TABS_ENABLED = 'IS_NATIVE_CALL_RECORDING_TABS_ENABLED',
}