The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Rekey metadata caches when flat map hashes change (#23164)
Weiko0108a3476569
diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-config/constants/metadata-graphql-operations-to-cache.constant.ts b/packages/twenty-server/src/engine/api/graphql/graphql-config/constants/metadata-graphql-operations-to-cache.constant.tsnew file mode 100644index 00000000000..70900ecd6d6--- /dev/null+++ b/packages/twenty-server/src/engine/api/graphql/graphql-config/constants/metadata-graphql-operations-to-cache.constant.ts@@ -0,0 +1,32 @@+import { type CachedOperationConfig } from 'src/engine/api/graphql/graphql-config/hooks/use-cached-metadata';+import { FIND_ALL_VIEWS_GRAPHQL_OPERATION } from 'src/engine/metadata-modules/view/constants/find-all-views-graphql-operation.constant';++export const METADATA_GRAPHQL_OPERATIONS_TO_CACHE: Record<+ string,+ CachedOperationConfig+> = {+ ObjectMetadataItems: {+ scope: 'workspace',+ dependencies: [+ 'flatObjectMetadataMaps',+ 'flatFieldMetadataMaps',+ 'flatIndexMaps',+ 'flatSearchFieldMetadataMaps',+ 'flatApplicationMaps',+ ],+ },+ [FIND_ALL_VIEWS_GRAPHQL_OPERATION]: {+ scope: 'userWorkspace',+ dependencies: [+ 'flatViewMaps',+ 'flatViewFieldMaps',+ 'flatViewFieldGroupMaps',+ 'flatViewFilterMaps',+ 'flatViewFilterGroupMaps',+ 'flatViewSortMaps',+ 'flatViewGroupMaps',+ 'flatObjectMetadataMaps',+ 'flatApplicationMaps',+ ],+ },+};diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/__tests__/use-cached-metadata.spec.ts b/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/__tests__/use-cached-metadata.spec.tsindex ae26f4cff68..835737d0fae 100644--- a/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/__tests__/use-cached-metadata.spec.ts+++ b/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/__tests__/use-cached-metadata.spec.ts@@ -4,6 +4,7 @@ import { type Request } from 'express';import { useCachedMetadata } from 'src/engine/api/graphql/graphql-config/hooks/use-cached-metadata';jest.mock('@sentry/node', () => ({+ captureException: jest.fn(),getCurrentScope: jest.fn(),setTags: jest.fn(),startSpan: jest.fn(),@@ -40,18 +41,25 @@ describe('useCachedMetadata', () => {},locale: 'en',userWorkspaceId: 'user-workspace-id',- workspace: { id: 'workspace-id', metadataVersion: 3 },+ workspace: { id: 'workspace-id' },...overrides,}) as Request;const createPlugin = ({cacheGetter = jest.fn().mockResolvedValue(undefined),cacheSetter = jest.fn(),+ dependencyHashGetter = jest.fn().mockResolvedValue('dependency-hash'),} = {}) =>useCachedMetadata({cacheGetter,cacheSetter,- operationsToCache: ['FindAllViews'],+ operationsToCache: {+ FindAllViews: {+ scope: 'userWorkspace',+ dependencies: ['flatViewMaps'],+ },+ },+ dependencyHashGetter,});beforeEach(() => {@@ -133,6 +141,92 @@ describe('useCachedMetadata', () => {expect(mockSpan.setAttribute).toHaveBeenNthCalledWith(2, 'cache.hit', true);});+ it('resolves dependency hashes once per request and caches the response under that key', async () => {+ const cacheGetter = jest.fn().mockResolvedValue(undefined);+ const cacheSetter = jest.fn();+ const dependencyHashGetter = jest.fn().mockResolvedValue('dependency-hash');+ const plugin = createPlugin({+ cacheGetter,+ cacheSetter,+ dependencyHashGetter,+ });+ const request = createRequest();+ const serverContext = { req: request };+ const responseBody = { data: { views: [] } };++ await plugin.onRequest?.({+ endResponse: jest.fn(),+ serverContext,+ } as never);+ await plugin.onResponse?.({+ response: Response.json(responseBody),+ serverContext,+ } as never);++ expect(dependencyHashGetter).toHaveBeenCalledTimes(1);+ expect(dependencyHashGetter).toHaveBeenCalledWith('workspace-id', [+ 'flatViewMaps',+ ]);++ const cacheKey = cacheGetter.mock.calls[0][0];++ expect(cacheKey).toContain('dependency-hash');+ expect(cacheKey).toContain('user-workspace-id');+ expect(cacheSetter).toHaveBeenCalledWith(cacheKey, responseBody);+ });++ it('shares cache entries across users for workspace-scoped operations', async () => {+ const cacheGetter = jest.fn().mockResolvedValue(undefined);+ const plugin = useCachedMetadata({+ cacheGetter,+ cacheSetter: jest.fn(),+ operationsToCache: {+ FindAllViews: { scope: 'workspace', dependencies: ['flatViewMaps'] },+ },+ dependencyHashGetter: jest.fn().mockResolvedValue('dependency-hash'),+ });+ const request = createRequest();+ const serverContext = { req: request };++ await plugin.onRequest?.({+ endResponse: jest.fn(),+ serverContext,+ } as never);++ const cacheKey = cacheGetter.mock.calls[0][0];++ expect(cacheKey).not.toContain('user-workspace-id');+ expect(cacheKey).toContain(':en:');+ });++ it('serves the request uncached when dependency hashes cannot be resolved', async () => {+ const cacheGetter = jest.fn();+ const cacheSetter = jest.fn();+ const dependencyHashGetter = jest+ .fn()+ .mockRejectedValue(new Error('cache storage unavailable'));+ const plugin = createPlugin({+ cacheGetter,+ cacheSetter,+ dependencyHashGetter,+ });+ const request = createRequest();+ const serverContext = { req: request };++ await plugin.onRequest?.({+ endResponse: jest.fn(),+ serverContext,+ } as never);+ await plugin.onResponse?.({+ response: Response.json({ data: { views: [] } }),+ serverContext,+ } as never);++ expect(cacheGetter).not.toHaveBeenCalled();+ expect(cacheSetter).not.toHaveBeenCalled();+ expect(Sentry.captureException).toHaveBeenCalledTimes(1);+ });+it('does not trace client-controlled operations outside the cache allowlist', async () => {const cacheGetter = jest.fn();const plugin = createPlugin({ cacheGetter });@@ -157,4 +251,25 @@ describe('useCachedMetadata', () => {expect(Sentry.setTags).not.toHaveBeenCalled();expect(Sentry.startSpan).not.toHaveBeenCalled();});++ it('ignores operation names inherited from Object.prototype', async () => {+ const cacheGetter = jest.fn();+ const dependencyHashGetter = jest.fn();+ const plugin = createPlugin({ cacheGetter, dependencyHashGetter });+ const request = createRequest({+ body: {+ operationName: 'constructor',+ query: 'query { views { id } }',+ },+ });+ const serverContext = { req: request };++ await plugin.onRequest?.({+ endResponse: jest.fn(),+ serverContext,+ } as never);++ expect(cacheGetter).not.toHaveBeenCalled();+ expect(dependencyHashGetter).not.toHaveBeenCalled();+ });});diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.ts b/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.tsindex 8bc509fb891..227ff839bb7 100644--- a/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.ts+++ b/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.ts@@ -5,48 +5,100 @@ import { type Request } from 'express';import { type Plugin } from 'graphql-yoga';import { isDefined } from 'twenty-shared/utils';-import { InternalServerError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';+import { type WorkspaceCacheKeyName } from 'src/engine/workspace-cache/types/workspace-cache-key.type';++export type CachedOperationConfig = {+ dependencies: WorkspaceCacheKeyName[];+ scope: 'workspace' | 'userWorkspace';+};export type CacheMetadataPluginConfig = {// oxlint-disable-next-line typescript/no-explicit-anycacheGetter: (key: string) => any;// oxlint-disable-next-line typescript/no-explicit-anycacheSetter: (key: string, value: any) => void;- operationsToCache: string[];+ operationsToCache: Record<string, CachedOperationConfig>;+ dependencyHashGetter: (+ workspaceId: string,+ cacheKeyNames: WorkspaceCacheKeyName[],+ ) => Promise<string>;};export function useCachedMetadata(config: CacheMetadataPluginConfig): Plugin {- const computeCacheKey = ({+ const computeCacheKey = async ({operationName,+ operationConfig,+ workspaceId,request,}: {operationName: string;- request: Pick<Request, 'workspace' | 'locale' | 'body' | 'userWorkspaceId'>;- }) => {- const workspace = request.workspace;-- if (!isDefined(workspace)) {- throw new InternalServerError('Workspace is not defined');- }-- const workspaceMetadataVersion = workspace.metadataVersion ?? '0';- const locale = request.locale;+ operationConfig: CachedOperationConfig;+ workspaceId: string;+ request: Pick<Request, 'locale' | 'body' | 'userWorkspaceId'>;+ }): Promise<string> => {+ const dependencyHash = await config.dependencyHashGetter(+ workspaceId,+ operationConfig.dependencies,+ );const queryHash = createHash('sha256').update(request.body.query)+ .update(JSON.stringify(request.body.variables ?? null)).digest('hex');+ const userScopeSegment =+ operationConfig.scope === 'userWorkspace'+ ? `:${request.userWorkspaceId}`+ : '';- if (operationName === 'FindAllViews') {- return `graphql:operations:${operationName}:${workspace.id}:${workspaceMetadataVersion}:${request.userWorkspaceId}:${queryHash}`;- }-- return `graphql:operations:${operationName}:${workspace.id}:${workspaceMetadataVersion}:${locale}:${queryHash}`;+ return `graphql:operations:${operationName}:${workspaceId}:${dependencyHash}${userScopeSegment}:${request.locale}:${queryHash}`;};// oxlint-disable-next-line typescript/no-explicit-anyconst getOperationName = (serverContext: any) =>serverContext?.req?.body?.operationName;+ const getOperationCacheConfig = (+ operationName: unknown,+ ): CachedOperationConfig | undefined =>+ typeof operationName === 'string' &&+ Object.prototype.hasOwnProperty.call(+ config.operationsToCache,+ operationName,+ )+ ? config.operationsToCache[operationName]+ : undefined;+const cacheHitRequests = new WeakSet<Request>();+ const requestCacheKeys = new WeakMap<Request, string | null>();++ const resolveCacheKey = async ({+ operationName,+ operationConfig,+ workspaceId,+ request,+ }: {+ operationName: string;+ operationConfig: CachedOperationConfig;+ workspaceId: string;+ request: Request;+ }): Promise<string | null> => {+ let cacheKey: string | null;++ try {+ cacheKey = await computeCacheKey({+ operationName,+ operationConfig,+ workspaceId,+ request,+ });+ } catch (error) {+ Sentry.captureException(error);+ cacheKey = null;+ }++ requestCacheKeys.set(request, cacheKey);++ return cacheKey;+ };const getCachedResponse = ({cacheKey,@@ -81,24 +133,33 @@ export function useCachedMetadata(config: CacheMetadataPluginConfig): Plugin {onRequest: async ({ endResponse, serverContext }) => {// TODO: we should probably override the graphql-yoga request type to include the workspace and localeconst request = (serverContext as unknown as { req: Request }).req;+ const workspaceId = request.workspace?.id;- if (!request.workspace?.id) {+ if (!workspaceId) {return;}const operationName = getOperationName(serverContext);+ const operationConfig = getOperationCacheConfig(operationName);- if (!config.operationsToCache.includes(operationName)) {+ if (!isDefined(operationConfig)) {return;}Sentry.setTags({ operationName, operation: 'query' });Sentry.getCurrentScope().setTransactionName(operationName);- const cacheKey = computeCacheKey({+ const cacheKey = await resolveCacheKey({operationName,+ operationConfig,+ workspaceId,request,});++ if (!isDefined(cacheKey)) {+ return;+ }+const cachedResponse = await getCachedResponse({cacheKey,operationName,@@ -122,7 +183,7 @@ export function useCachedMetadata(config: CacheMetadataPluginConfig): Plugin {const operationName = getOperationName(serverContext);- if (!config.operationsToCache.includes(operationName)) {+ if (!isDefined(getOperationCacheConfig(operationName))) {return;}@@ -130,10 +191,11 @@ export function useCachedMetadata(config: CacheMetadataPluginConfig): Plugin {return;}- const cacheKey = computeCacheKey({- operationName,- request,- });+ const cacheKey = requestCacheKeys.get(request);++ if (!isDefined(cacheKey)) {+ return;+ }const cachedResponse = await getCachedResponse({cacheKey,diff --git a/packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts b/packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.tsindex 5bce5825446..8dddc34e073 100644--- a/packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts+++ b/packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts@@ -16,6 +16,8 @@ import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twentimport { DataloaderModule } from 'src/engine/dataloaders/dataloader.module';import { DataloaderService } from 'src/engine/dataloaders/dataloader.service';import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module';+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';+import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';@Module({imports: [@@ -27,6 +29,7 @@ import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-enginDataloaderModule,MetricsModule,I18nModule,+ WorkspaceCacheModule,],inject: [TwentyConfigService,@@ -36,6 +39,7 @@ import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-enginMetricsService,I18nService,FeatureFlagService,+ WorkspaceCacheService,],}),MetadataEngineModule,diff --git a/packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts b/packages/twenty-server/src/engine/api/graphql/metadata.module-factory.tsindex 575cbace329..0b370d9e352 100644--- a/packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts+++ b/packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts@@ -4,6 +4,7 @@ import GraphQLJSON from 'graphql-type-json';import { NodeEnvironment } from 'src/engine/core-modules/twenty-config/interfaces/node-environment.interface';+import { METADATA_GRAPHQL_OPERATIONS_TO_CACHE } from 'src/engine/api/graphql/graphql-config/constants/metadata-graphql-operations-to-cache.constant';import { useCachedMetadata } from 'src/engine/api/graphql/graphql-config/hooks/use-cached-metadata';import { MetadataGraphQLApiModule } from 'src/engine/api/graphql/metadata-graphql-api.module';import { type CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';@@ -19,6 +20,7 @@ import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.serimport { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';import { type DataloaderService } from 'src/engine/dataloaders/dataloader.service';import { renderApolloPlayground } from 'src/engine/utils/render-apollo-playground.util';+import { type WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';export const metadataModuleFactory = async (twentyConfigService: TwentyConfigService,@@ -28,6 +30,7 @@ export const metadataModuleFactory = async (metricsService: MetricsService,i18nService: I18nService,_featureFlagService: FeatureFlagService,+ workspaceCacheService: WorkspaceCacheService,): Promise<YogaDriverConfig> => {const config: YogaDriverConfig = {autoSchemaFile: true,@@ -51,7 +54,11 @@ export const metadataModuleFactory = async (useCachedMetadata({cacheGetter: cacheStorageService.get.bind(cacheStorageService),cacheSetter: cacheStorageService.set.bind(cacheStorageService),- operationsToCache: ['ObjectMetadataItems', 'FindAllViews'],+ operationsToCache: METADATA_GRAPHQL_OPERATIONS_TO_CACHE,+ dependencyHashGetter:+ workspaceCacheService.getOrRecomputeCombinedHash.bind(+ workspaceCacheService,+ ),}),useDisableIntrospectionAndSuggestionsForUnauthenticatedUsers(twentyConfigService.get('NODE_ENV') === NodeEnvironment.PRODUCTION,diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-graphql-schema-sdl/constants/schema-sdl-cache-dependencies.constant.ts b/packages/twenty-server/src/engine/api/graphql/workspace-graphql-schema-sdl/constants/schema-sdl-cache-dependencies.constant.tsnew file mode 100644index 00000000000..3063062c3c4--- /dev/null+++ b/packages/twenty-server/src/engine/api/graphql/workspace-graphql-schema-sdl/constants/schema-sdl-cache-dependencies.constant.ts@@ -0,0 +1,6 @@+export const SCHEMA_SDL_CACHE_DEPENDENCIES = [+ 'flatObjectMetadataMaps',+ 'flatFieldMetadataMaps',+ 'flatIndexMaps',+ 'flatApplicationMaps',+] as const;diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-graphql-schema-sdl/workspace-graphql-schema-sdl.service.ts b/packages/twenty-server/src/engine/api/graphql/workspace-graphql-schema-sdl/workspace-graphql-schema-sdl.service.tsindex cd5d0b55471..94997778354 100644--- a/packages/twenty-server/src/engine/api/graphql/workspace-graphql-schema-sdl/workspace-graphql-schema-sdl.service.ts+++ b/packages/twenty-server/src/engine/api/graphql/workspace-graphql-schema-sdl/workspace-graphql-schema-sdl.service.ts@@ -17,7 +17,9 @@ import { getSubFlatEntityMapsByApplicationIdsOrThrow } from 'src/engine/metadataimport { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';import { type FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type';import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';+import { SCHEMA_SDL_CACHE_DEPENDENCIES } from 'src/engine/api/graphql/workspace-graphql-schema-sdl/constants/schema-sdl-cache-dependencies.constant';import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';+import { combineCacheHashes } from 'src/engine/workspace-cache/utils/combine-cache-hashes.util';import { TWENTY_STANDARD_APPLICATION } from 'src/engine/workspace-manager/twenty-standard-application/constants/twenty-standard-applications';export type WorkspaceGraphqlSchemaSDLResult = {@@ -46,21 +48,20 @@ export class WorkspaceGraphqlSchemaSDLService {}const {- flatObjectMetadataMaps: allFlatObjectMetadataMaps,- flatFieldMetadataMaps: allFlatFieldMetadataMaps,- flatIndexMaps: allFlatIndexMaps,- flatApplicationMaps,- } = await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(- {- workspaceId: workspace.id,- flatMapsKeys: [- 'flatObjectMetadataMaps',- 'flatFieldMetadataMaps',- 'flatIndexMaps',- 'flatApplicationMaps',- ],+ data: {+ flatObjectMetadataMaps: allFlatObjectMetadataMaps,+ flatFieldMetadataMaps: allFlatFieldMetadataMaps,+ flatIndexMaps: allFlatIndexMaps,+ flatApplicationMaps,},- );+ hashes,+ } =+ await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMapsWithHashes(+ {+ workspaceId: workspace.id,+ flatMapsKeys: [...SCHEMA_SDL_CACHE_DEPENDENCIES],+ },+ );if (!isDefined(allFlatObjectMetadataMaps)) {throw new FlatEntityMapsException(@@ -113,28 +114,20 @@ export class WorkspaceGraphqlSchemaSDLService {}}- let metadataVersion =- await this.workspaceCacheStorageService.getMetadataVersion(workspace.id);-- if (!isDefined(metadataVersion)) {- metadataVersion = isDefined(workspace.metadataVersion)- ? workspace.metadataVersion- : 0;- await this.workspaceCacheStorageService.setMetadataVersion(- workspace.id,- metadataVersion,- );- }+ const metadataCacheHash = combineCacheHashes(+ hashes,+ SCHEMA_SDL_CACHE_DEPENDENCIES,+ );let sdl = await this.workspaceCacheStorageService.getGraphQLTypeDefs(workspace.id,- metadataVersion,+ metadataCacheHash,applicationId,);let usedScalarNames =await this.workspaceCacheStorageService.getGraphQLUsedScalarNames(workspace.id,- metadataVersion,+ metadataCacheHash,applicationId,);@@ -152,13 +145,13 @@ export class WorkspaceGraphqlSchemaSDLService {await this.workspaceCacheStorageService.setGraphQLTypeDefs(workspace.id,- metadataVersion,+ metadataCacheHash,sdl,applicationId,);await this.workspaceCacheStorageService.setGraphQLUsedScalarNames(workspace.id,- metadataVersion,+ metadataCacheHash,usedScalarNames,applicationId,);diff --git a/packages/twenty-server/src/engine/core-modules/cache-storage/services/__tests__/cache-storage.service.spec.ts b/packages/twenty-server/src/engine/core-modules/cache-storage/services/__tests__/cache-storage.service.spec.tsnew file mode 100644index 00000000000..fc68a09cf6a--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/cache-storage/services/__tests__/cache-storage.service.spec.ts@@ -0,0 +1,112 @@+import { type Cache } from '@nestjs/cache-manager';++import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';+import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';++const prefixKey = (key: string) =>+ `${CacheStorageNamespace.IntegrationTests}:${CacheStorageNamespace.EngineWorkspace}:${key}`;++describe('CacheStorageService', () => {+ describe('mset', () => {+ const createRedisCacheMock = () => {+ const storeMset = jest.fn().mockResolvedValue(undefined);+ const cache = {+ store: { name: 'redis', mset: storeMset },+ set: jest.fn(),+ } as unknown as Cache;++ return { cache, storeMset };+ };++ it('commits all same-ttl entries in a single atomic store call', async () => {+ const { cache, storeMset } = createRedisCacheMock();+ const cacheStorageService = new CacheStorageService(+ cache,+ CacheStorageNamespace.EngineWorkspace,+ );++ await cacheStorageService.mset<unknown>([+ { key: 'flat-maps:field-metadata:workspace-id:hash', value: 'hash-1' },+ {+ key: 'flat-maps:field-metadata:workspace-id:data',+ value: { byId: {} },+ },+ ]);++ expect(storeMset).toHaveBeenCalledTimes(1);+ expect(storeMset).toHaveBeenCalledWith(+ [+ [prefixKey('flat-maps:field-metadata:workspace-id:hash'), 'hash-1'],+ [+ prefixKey('flat-maps:field-metadata:workspace-id:data'),+ { byId: {} },+ ],+ ],+ undefined,+ );+ });++ it('groups entries by ttl into one atomic store call per ttl', async () => {+ const { cache, storeMset } = createRedisCacheMock();+ const cacheStorageService = new CacheStorageService(+ cache,+ CacheStorageNamespace.EngineWorkspace,+ );++ await cacheStorageService.mset([+ { key: 'first', value: 1, ttl: 1000 },+ { key: 'second', value: 2 },+ { key: 'third', value: 3, ttl: 1000 },+ ]);++ expect(storeMset).toHaveBeenCalledTimes(2);+ expect(storeMset).toHaveBeenCalledWith(+ [+ [prefixKey('first'), 1],+ [prefixKey('third'), 3],+ ],+ 1000,+ );+ expect(storeMset).toHaveBeenCalledWith(+ [[prefixKey('second'), 2]],+ undefined,+ );+ });++ it('does not call the store for empty entries', async () => {+ const { cache, storeMset } = createRedisCacheMock();+ const cacheStorageService = new CacheStorageService(+ cache,+ CacheStorageNamespace.EngineWorkspace,+ );++ await cacheStorageService.mset([]);++ expect(storeMset).not.toHaveBeenCalled();+ });++ it('falls back to sequential sets on non-redis stores', async () => {+ const cache = {+ store: { name: 'memory' },+ set: jest.fn().mockResolvedValue(undefined),+ } as unknown as Cache;+ const cacheStorageService = new CacheStorageService(+ cache,+ CacheStorageNamespace.EngineWorkspace,+ );++ await cacheStorageService.mset([+ { key: 'first', value: 1, ttl: 500 },+ { key: 'second', value: 2 },+ ]);++ expect(cache.set).toHaveBeenNthCalledWith(1, prefixKey('first'), 1, 500);+ expect(cache.set).toHaveBeenNthCalledWith(+ 2,+ prefixKey('second'),+ 2,+ undefined,+ );+ });+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.ts b/packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.tsindex 0f5f49d3505..0068776c94d 100644--- a/packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.ts+++ b/packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.ts@@ -29,8 +29,8 @@ export class CacheStorageService {value: T,ttl: Milliseconds,): Promise<boolean> {- if (this.isRedisCache()) {- const result = await (this.cache as RedisCache).store.client.set(+ if (this.isRedisCache(this.cache)) {+ const result = await this.cache.store.client.set(this.getKey(key),JSON.stringify(value),ttl > 0 ? { NX: true, PX: ttl } : { NX: true },@@ -59,10 +59,10 @@ export class CacheStorageService {return;}- if (this.isRedisCache()) {+ if (this.isRedisCache(this.cache)) {const prefixedKeys = keys.map((k) => this.getKey(k));- await (this.cache as RedisCache).store.client.del(prefixedKeys);+ await this.cache.store.client.del(prefixedKeys);return;}@@ -71,11 +71,9 @@ export class CacheStorageService {}async mget<T = unknown>(keys: string[]): Promise<(T | undefined)[]> {- if (this.isRedisCache()) {+ if (this.isRedisCache(this.cache)) {const prefixedKeys = keys.map((k) => this.getKey(k));- const values = await (this.cache as RedisCache).store.client.mGet(- prefixedKeys,- );+ const values = await this.cache.store.client.mGet(prefixedKeys);return values.map((v) => {if (v === null || v === undefined) return undefined;@@ -97,9 +95,29 @@ export class CacheStorageService {return;}- await Promise.all(- entries.map(({ key, value, ttl }) => this.set(key, value, ttl)),- );+ if (this.isRedisCache(this.cache)) {+ const redisStore = this.cache.store;+ const entriesByTtl = new Map<Milliseconds | undefined, [string, T][]>();++ for (const { key, value, ttl } of entries) {+ const ttlGroup = entriesByTtl.get(ttl) ?? [];++ ttlGroup.push([this.getKey(key), value]);+ entriesByTtl.set(ttl, ttlGroup);+ }++ await Promise.all(+ [...entriesByTtl.entries()].map(([ttl, ttlGroupEntries]) =>+ redisStore.mset(ttlGroupEntries, ttl),+ ),+ );++ return;+ }++ for (const { key, value, ttl } of entries) {+ await this.set(key, value, ttl);+ }}async setAdd(key: string, value: string[], ttl?: Milliseconds) {@@ -107,17 +125,11 @@ export class CacheStorageService {return;}- if (this.isRedisCache()) {- await (this.cache as RedisCache).store.client.sAdd(- this.getKey(key),- value,- );+ if (this.isRedisCache(this.cache)) {+ await this.cache.store.client.sAdd(this.getKey(key), value);if (ttl) {- await (this.cache as RedisCache).store.client.expire(- this.getKey(key),- ttl / 1000,- );+ await this.cache.store.client.expire(this.getKey(key), ttl / 1000);}return;@@ -137,11 +149,8 @@ export class CacheStorageService {return 0;}- if (this.isRedisCache()) {- return (this.cache as RedisCache).store.client.sRem(- this.getKey(key),- values,- );+ if (this.isRedisCache(this.cache)) {+ return this.cache.store.client.sRem(this.getKey(key), values);}const existing = await this.get<string[]>(key);@@ -165,11 +174,8 @@ export class CacheStorageService {}async setPop(key: string, size = 1) {- if (this.isRedisCache()) {- return (this.cache as RedisCache).store.client.sPop(- this.getKey(key),- size,- );+ if (this.isRedisCache(this.cache)) {+ return this.cache.store.client.sPop(this.getKey(key), size);}const res = await this.get<string[]>(key);@@ -184,10 +190,8 @@ export class CacheStorageService {}async getSetLength(key: string) {- if (this.isRedisCache()) {- return await (this.cache as RedisCache).store.client.sCard(- this.getKey(key),- );+ if (this.isRedisCache(this.cache)) {+ return await this.cache.store.client.sCard(this.getKey(key));}const res = await this.get<string[]>(key);@@ -196,8 +200,8 @@ export class CacheStorageService {}async setMembers(key: string): Promise<string[]> {- if (this.isRedisCache()) {- return (this.cache as RedisCache).store.client.sMembers(this.getKey(key));+ if (this.isRedisCache(this.cache)) {+ return this.cache.store.client.sMembers(this.getKey(key));}return (await this.get<string[]>(key)) ?? [];@@ -208,11 +212,11 @@ export class CacheStorageService {}async flushByPattern(scanPattern: string): Promise<void> {- if (!this.isRedisCache()) {+ if (!this.isRedisCache(this.cache)) {throw new Error('flushByPattern is only supported with Redis cache');}- const redisClient = (this.cache as RedisCache).store.client;+ const redisClient = this.cache.store.client;let cursor = 0;do {@@ -233,13 +237,13 @@ export class CacheStorageService {}async scanAndCountSetMembers(scanPattern: string): Promise<number> {- if (!this.isRedisCache()) {+ if (!this.isRedisCache(this.cache)) {throw new Error('scanAndCountSetMembers is only supported with Redis cache',);}- const redisClient = (this.cache as RedisCache).store.client;+ const redisClient = this.cache.store.client;let cursor = 0;let totalCount = 0;@@ -274,11 +278,11 @@ export class CacheStorageService {}async acquireLock(key: string, ttl = 1000): Promise<boolean> {- if (!this.isRedisCache()) {+ if (!this.isRedisCache(this.cache)) {throw new Error('acquireLock is only supported with Redis cache');}- const redisClient = (this.cache as RedisCache).store.client;+ const redisClient = this.cache.store.client;const result = await redisClient.set(this.getKey(key), 'lock', {NX: true,@@ -289,7 +293,7 @@ export class CacheStorageService {}async releaseLock(key: string): Promise<void> {- if (!this.isRedisCache()) {+ if (!this.isRedisCache(this.cache)) {throw new Error('releaseLock is only supported with Redis cache');}@@ -297,11 +301,8 @@ export class CacheStorageService {}async incrBy(key: string, increment: number): Promise<number> {- if (this.isRedisCache()) {- return (this.cache as RedisCache).store.client.incrBy(- this.getKey(key),- increment,- );+ if (this.isRedisCache(this.cache)) {+ return this.cache.store.client.incrBy(this.getKey(key), increment);}const current = (await this.get<number>(key)) ?? 0;@@ -313,11 +314,11 @@ export class CacheStorageService {}async hashGetValues(key: string): Promise<string[]> {- if (!this.isRedisCache()) {+ if (!this.isRedisCache(this.cache)) {throw new Error('hashGetValues is only supported with Redis cache');}- const redisClient = (this.cache as RedisCache).store.client;+ const redisClient = this.cache.store.client;return redisClient.hVals(this.getKey(key));}@@ -331,11 +332,11 @@ export class CacheStorageService {field: string;value: string;}): Promise<number> {- if (!this.isRedisCache()) {+ if (!this.isRedisCache(this.cache)) {throw new Error('hashSet is only supported with Redis cache');}- const redisClient = (this.cache as RedisCache).store.client;+ const redisClient = this.cache.store.client;return redisClient.hSet(this.getKey(key), field, value);}@@ -349,11 +350,11 @@ export class CacheStorageService {field: string;value: string;}): Promise<number> {- if (!this.isRedisCache()) {+ if (!this.isRedisCache(this.cache)) {throw new Error('hashSetIfExists is only supported with Redis cache');}- const redisClient = (this.cache as RedisCache).store.client;+ const redisClient = this.cache.store.client;const script = `if redis.call('EXISTS', KEYS[1]) == 1 then@@ -379,11 +380,11 @@ end`;value: string;ttlMs: Milliseconds;}): Promise<void> {- if (!this.isRedisCache()) {+ if (!this.isRedisCache(this.cache)) {throw new Error('hashSetWithExpire is only supported with Redis cache');}- const redisClient = (this.cache as RedisCache).store.client;+ const redisClient = this.cache.store.client;const prefixedKey = this.getKey(key);await redisClient@@ -400,21 +401,18 @@ end`;key: string;field: string;}): Promise<number> {- if (!this.isRedisCache()) {+ if (!this.isRedisCache(this.cache)) {throw new Error('hashDelete is only supported with Redis cache');}- const redisClient = (this.cache as RedisCache).store.client;+ const redisClient = this.cache.store.client;return redisClient.hDel(this.getKey(key), field);}async expire(key: string, ttlMs: Milliseconds): Promise<boolean> {- if (this.isRedisCache()) {- return (this.cache as RedisCache).store.client.expire(- this.getKey(key),- ttlMs / 1000,- );+ if (this.isRedisCache(this.cache)) {+ return this.cache.store.client.expire(this.getKey(key), ttlMs / 1000);}const existing = await this.get(key);@@ -428,9 +426,9 @@ end`;return false;}- private isRedisCache() {+ private isRedisCache(cache: Cache): cache is RedisCache {// oxlint-disable-next-line typescript/no-explicit-any- return (this.cache.store as any)?.name === 'redis';+ return (cache.store as any)?.name === 'redis';}private getKey(key: string) {diff --git a/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts b/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.tsindex 5c8fcfbd0b5..e1301ecbd05 100644--- a/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts@@ -306,7 +306,6 @@ describe('WorkspaceService', () => {).toHaveBeenCalledWith(mockWorkspace.id);expect(workspaceCacheStorageService.flush).toHaveBeenCalledWith(mockWorkspace.id,- mockWorkspace.metadataVersion,);expect(messageQueueService.add).toHaveBeenCalled();expect(workspaceRepository.delete).toHaveBeenCalledWith(mockWorkspace.id);diff --git a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.tsindex f046524561d..fcfc7864ed1 100644--- a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts+++ b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts@@ -589,10 +589,7 @@ export class WorkspaceService {await this.workspaceDataSourceService.deleteWorkspaceDBSchema(workspace.id);- await this.workspaceCacheStorageService.flush(- workspace.id,- workspace.metadataVersion,- );+ await this.workspaceCacheStorageService.flush(workspace.id);await this.flatEntityMapsCacheService.flushFlatEntityMaps({workspaceId: workspace.id,});diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service.tsindex 40124869203..e8580a6904f 100644--- a/packages/twenty-server/src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service.ts@@ -3,7 +3,10 @@ import { Injectable } from '@nestjs/common';import { ALL_FLAT_ENTITY_MAPS_PROPERTIES } from 'src/engine/metadata-modules/flat-entity/constant/all-flat-entity-maps-properties.constant';import { AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type';import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';-import { type WorkspaceCacheDataMap } from 'src/engine/workspace-cache/types/workspace-cache-key.type';+import {+ type WorkspaceCacheDataMap,+ type WorkspaceCacheResultWithHashes,+} from 'src/engine/workspace-cache/types/workspace-cache-key.type';export type FlatEntityMapsCacheKeyName =| keyof AllFlatEntityMaps@@ -22,7 +25,23 @@ export class WorkspaceManyOrAllFlatEntityMapsCacheService {workspaceId: string;flatMapsKeys?: T;}): Promise<Pick<WorkspaceCacheDataMap, T[number]>> {- return await this.workspaceCacheService.getOrRecompute(+ const { data } = await this.getOrRecomputeManyOrAllFlatEntityMapsWithHashes(+ { flatMapsKeys, workspaceId },+ );++ return data;+ }++ public async getOrRecomputeManyOrAllFlatEntityMapsWithHashes<+ T extends FlatEntityMapsCacheKeyName[] = (keyof AllFlatEntityMaps)[],+ >({+ flatMapsKeys,+ workspaceId,+ }: {+ workspaceId: string;+ flatMapsKeys?: T;+ }): Promise<WorkspaceCacheResultWithHashes<T>> {+ return await this.workspaceCacheService.getOrRecomputeWithHashes(workspaceId,(flatMapsKeys ??ALL_FLAT_ENTITY_MAPS_PROPERTIES) as (keyof WorkspaceCacheDataMap)[],diff --git a/packages/twenty-server/src/engine/middlewares/middleware.service.ts b/packages/twenty-server/src/engine/middlewares/middleware.service.tsindex cb4dbe9a52b..0f16d8b329c 100644--- a/packages/twenty-server/src/engine/middlewares/middleware.service.ts+++ b/packages/twenty-server/src/engine/middlewares/middleware.service.ts@@ -12,6 +12,7 @@ import { getAuthExceptionRestStatus } from 'src/engine/core-modules/auth/utils/gimport { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';import { ErrorCode } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service';+import { type FlatWorkspace } from 'src/engine/core-modules/workspace/types/flat-workspace.type';import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';import { INTERNAL_SERVER_ERROR } from 'src/engine/middlewares/constants/default-error-message.constant';import { bindDataToRequestObject } from 'src/engine/utils/bind-data-to-request-object.util';@@ -100,9 +101,7 @@ export class MiddlewareService {public async hydrateRestRequest(request: Request) {const data = await this.accessTokenService.validateTokenByRequest(request);const metadataVersion = data.workspace- ? await this.workspaceStorageCacheService.getMetadataVersion(- data.workspace.id,- )+ ? await this.getOrSeedMetadataVersion(data.workspace): undefined;if (!data.workspace) {@@ -127,14 +126,32 @@ export class MiddlewareService {const data = await this.accessTokenService.validateTokenByRequest(request);const metadataVersion = data.workspace- ? await this.workspaceStorageCacheService.getMetadataVersion(- data.workspace.id,- )+ ? await this.getOrSeedMetadataVersion(data.workspace): undefined;bindDataToRequestObject(data, request, metadataVersion);}+ private async getOrSeedMetadataVersion(+ workspace: Pick<FlatWorkspace, 'id' | 'metadataVersion'>,+ ): Promise<number | undefined> {+ const cachedMetadataVersion =+ await this.workspaceStorageCacheService.getMetadataVersion(workspace.id);++ if (isDefined(cachedMetadataVersion)) {+ return cachedMetadataVersion;+ }++ if (isDefined(workspace.metadataVersion)) {+ await this.workspaceStorageCacheService.setMetadataVersion(+ workspace.id,+ workspace.metadataVersion,+ );+ }++ return workspace.metadataVersion;+ }+private hasErrorStatus(error: unknown): error is { status: number } {return isDefined((error as { status: number })?.status);}diff --git a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.tsindex e1f02b35b7d..159c3892862 100644--- a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts+++ b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts@@ -2,21 +2,18 @@ import { Injectable } from '@nestjs/common';import crypto from 'crypto';-import { isDefined } from 'twenty-shared/utils';-import { type FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface';import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';-export const METADATA_VERSIONED_WORKSPACE_CACHE_KEY = {+export const HASH_KEYED_WORKSPACE_CACHE_KEYS = {GraphQLTypeDefs: 'graphql:type-defs',- MetadataVersion: 'metadata:workspace-metadata-version',GraphQLUsedScalarNames: 'graphql:used-scalar-names',- ORMEntitySchemas: 'orm:entity-schemas',} as const;export const WORKSPACE_CACHE_KEYS = {+ MetadataVersion: 'metadata:workspace-metadata-version',GraphQLOperations: 'graphql:operations',GraphQLFeatureFlag: 'graphql:feature-flag',FeatureFlagMap: 'feature-flag:feature-flag-map',@@ -47,7 +44,7 @@ export class WorkspaceCacheStorageService {metadataVersion: number,): Promise<void> {return this.cacheStorageService.set<number>(- `${METADATA_VERSIONED_WORKSPACE_CACHE_KEY.MetadataVersion}:${workspaceId}`,+ `${WORKSPACE_CACHE_KEYS.MetadataVersion}:${workspaceId}`,metadataVersion,TTL_ONE_WEEK,);@@ -55,20 +52,20 @@ export class WorkspaceCacheStorageService {getMetadataVersion(workspaceId: string): Promise<number | undefined> {return this.cacheStorageService.get<number>(- `${METADATA_VERSIONED_WORKSPACE_CACHE_KEY.MetadataVersion}:${workspaceId}`,+ `${WORKSPACE_CACHE_KEYS.MetadataVersion}:${workspaceId}`,);}setGraphQLTypeDefs(workspaceId: string,- metadataVersion: number,+ metadataCacheHash: string,typeDefs: string,applicationId?: string,): Promise<void> {const applicationSuffix = applicationId ? `:${applicationId}` : '';return this.cacheStorageService.set<string>(- `${METADATA_VERSIONED_WORKSPACE_CACHE_KEY.GraphQLTypeDefs}:${workspaceId}:${metadataVersion}${applicationSuffix}`,+ `${HASH_KEYED_WORKSPACE_CACHE_KEYS.GraphQLTypeDefs}:${workspaceId}:${metadataCacheHash}${applicationSuffix}`,typeDefs,TTL_ONE_WEEK,);@@ -76,26 +73,26 @@ export class WorkspaceCacheStorageService {getGraphQLTypeDefs(workspaceId: string,- metadataVersion: number,+ metadataCacheHash: string,applicationId?: string,): Promise<string | undefined> {const applicationSuffix = applicationId ? `:${applicationId}` : '';return this.cacheStorageService.get<string>(- `${METADATA_VERSIONED_WORKSPACE_CACHE_KEY.GraphQLTypeDefs}:${workspaceId}:${metadataVersion}${applicationSuffix}`,+ `${HASH_KEYED_WORKSPACE_CACHE_KEYS.GraphQLTypeDefs}:${workspaceId}:${metadataCacheHash}${applicationSuffix}`,);}setGraphQLUsedScalarNames(workspaceId: string,- metadataVersion: number,+ metadataCacheHash: string,usedScalarNames: string[],applicationId?: string,): Promise<void> {const applicationSuffix = applicationId ? `:${applicationId}` : '';return this.cacheStorageService.set<string[]>(- `${METADATA_VERSIONED_WORKSPACE_CACHE_KEY.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersion}${applicationSuffix}`,+ `${HASH_KEYED_WORKSPACE_CACHE_KEYS.GraphQLUsedScalarNames}:${workspaceId}:${metadataCacheHash}${applicationSuffix}`,usedScalarNames,TTL_ONE_WEEK,);@@ -103,13 +100,13 @@ export class WorkspaceCacheStorageService {getGraphQLUsedScalarNames(workspaceId: string,- metadataVersion: number,+ metadataCacheHash: string,applicationId?: string,): Promise<string[] | undefined> {const applicationSuffix = applicationId ? `:${applicationId}` : '';return this.cacheStorageService.get<string[]>(- `${METADATA_VERSIONED_WORKSPACE_CACHE_KEY.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersion}${applicationSuffix}`,+ `${HASH_KEYED_WORKSPACE_CACHE_KEYS.GraphQLUsedScalarNames}:${workspaceId}:${metadataCacheHash}${applicationSuffix}`,);}@@ -169,36 +166,16 @@ export class WorkspaceCacheStorageService {);}- async flushVersionedMetadata(- workspaceId: string,- metadataVersion?: number,- ): Promise<void> {- const { MetadataVersion, ...versionedCacheKeys } =- METADATA_VERSIONED_WORKSPACE_CACHE_KEY;-- await Promise.all([- this.cacheStorageService.del(`${MetadataVersion}:${workspaceId}`),- ...Object.values(versionedCacheKeys).flatMap((key) =>- isDefined(metadataVersion)- ? [- this.cacheStorageService.del(- `${key}:${workspaceId}:${metadataVersion}`,- ),- this.cacheStorageService.flushByPattern(- `${key}:${workspaceId}:${metadataVersion}:*`,- ),- ]- : [- this.cacheStorageService.flushByPattern(- `${key}:${workspaceId}:*`,- ),- ],+ async flushHashKeyedWorkspaceCache(workspaceId: string): Promise<void> {+ await Promise.all(+ Object.values(HASH_KEYED_WORKSPACE_CACHE_KEYS).map((key) =>+ this.cacheStorageService.flushByPattern(`${key}:${workspaceId}:*`),),- ]);+ );}- async flush(workspaceId: string, metadataVersion?: number): Promise<void> {- await this.flushVersionedMetadata(workspaceId, metadataVersion);+ async flush(workspaceId: string): Promise<void> {+ await this.flushHashKeyedWorkspaceCache(workspaceId);await Promise.all(Object.values(WORKSPACE_CACHE_KEYS).map(diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.tsindex f5339163c28..72f00b49601 100644--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts@@ -29,8 +29,10 @@ import {WorkspaceCacheKeyName,type WorkspaceCacheDataMap,type WorkspaceCacheResult,+ type WorkspaceCacheResultWithHashes,} from 'src/engine/workspace-cache/types/workspace-cache-key.type';import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';+import { combineCacheHashes } from 'src/engine/workspace-cache/utils/combine-cache-hashes.util';const LOCAL_TTL_MS = 100; // 100msconst LOCAL_ENTRY_TTL_MS = 30 * 60 * 1000; // 30 minutes@@ -43,6 +45,11 @@ const MIN_EVICT_KEYS = 100;type CacheDataType = WorkspaceCacheDataMap[WorkspaceCacheKeyName];+type CacheEntriesResult = {+ data: Partial<WorkspaceCacheDataMap>;+ hashes: Partial<Record<WorkspaceCacheKeyName, string>>;+};+type RecomputeHashResolution =| { strategy: 'mint' }| {@@ -61,9 +68,9 @@ export class WorkspaceCacheService implements OnModuleInit {WorkspaceCacheProvider<CacheDataType>>();private readonly localDataOnlyKeys = new Set<WorkspaceCacheKeyName>();- private readonly memoizer = new PromiseMemoizer<- Partial<WorkspaceCacheDataMap>- >(MEMOIZER_TTL_MS);+ private readonly memoizer = new PromiseMemoizer<CacheEntriesResult>(+ MEMOIZER_TTL_MS,+ );private readonly logger = new Logger(WorkspaceCacheService.name);@@ -114,18 +121,22 @@ export class WorkspaceCacheService implements OnModuleInit {workspaceId: string,cacheKeyNames: K,): Promise<WorkspaceCacheResult<K>> {- this.evictExpiredLocalEntries();+ const { data } = await this.getOrRecomputeWithHashes(+ workspaceId,+ cacheKeyNames,+ );- if (- !isDefined(workspaceId) ||- cacheKeyNames.length === 0 ||- !isValidUuid(workspaceId)- ) {- throw new WorkspaceCacheException(- 'Invalid parameters: workspace ID and cache key names are required',- WorkspaceCacheExceptionCode.INVALID_PARAMETERS,- );- }+ return data;+ }++ public async getOrRecomputeWithHashes<+ const K extends WorkspaceCacheKeyName[],+ >(+ workspaceId: string,+ cacheKeyNames: K,+ ): Promise<WorkspaceCacheResultWithHashes<K>> {+ this.evictExpiredLocalEntries();+ this.assertValidCacheParameters(workspaceId, cacheKeyNames);const memoKey =`${workspaceId}-${[...cacheKeyNames].sort().join(',')}` as const;@@ -138,10 +149,10 @@ export class WorkspaceCacheService implements OnModuleInit {workspaceId,cacheKeyNames,);- const freshData = this.getFromLocalCache(workspaceId, freshKeys);+ const freshEntries = this.getFromLocalCache(workspaceId, freshKeys);if (staleKeys.length === 0) {- return freshData;+ return freshEntries;}// Stage 2: Validate ttl stale keys against Redis hash@@ -154,32 +165,66 @@ export class WorkspaceCacheService implements OnModuleInit {workspaceId,staleKeys,);- const validatedData = this.getFromLocalCache(workspaceId, validKeys);+ const validatedEntries = this.getFromLocalCache(workspaceId, validKeys);// Stage 3: Fetch data from Redis- const { redisData, missingInRedis } = await this.fetchDataFromRedis(+ const { redisEntries, missingInRedis } = await this.fetchDataFromRedis(workspaceId,keysNeedingDataFromRedis,);// Stage 4: Recompute remainingconst keysToRecompute = [...keysNeedingRecompute, ...missingInRedis];- const recomputedData = await this.recomputeDataFromProvider(+ const recomputedEntries = await this.recomputeDataFromProvider(workspaceId,keysToRecompute,{ strategy: 'recover', adoptableHashes },);return {- ...freshData,- ...validatedData,- ...redisData,- ...recomputedData,+ data: {+ ...freshEntries.data,+ ...validatedEntries.data,+ ...redisEntries.data,+ ...recomputedEntries.data,+ },+ hashes: {+ ...freshEntries.hashes,+ ...validatedEntries.hashes,+ ...redisEntries.hashes,+ ...recomputedEntries.hashes,+ },};},);- return result as WorkspaceCacheResult<K>;+ return result as WorkspaceCacheResultWithHashes<K>;+ }++ public async getOrRecomputeCombinedHash(+ workspaceId: string,+ cacheKeyNames: WorkspaceCacheKeyName[],+ ): Promise<string> {+ this.assertValidCacheParameters(workspaceId, cacheKeyNames);++ const cachedHashes = await this.getCacheHashes(workspaceId, cacheKeyNames);+ const missingKeys = cacheKeyNames.filter(+ (cacheKeyName) => !isDefined(cachedHashes[cacheKeyName]),+ );++ if (missingKeys.length === 0) {+ return combineCacheHashes(cachedHashes, cacheKeyNames);+ }++ const { hashes: recomputedHashes } = await this.getOrRecomputeWithHashes(+ workspaceId,+ missingKeys,+ );++ return combineCacheHashes(+ { ...cachedHashes, ...recomputedHashes },+ cacheKeyNames,+ );}public async invalidateAndRecompute(@@ -242,6 +287,22 @@ export class WorkspaceCacheService implements OnModuleInit {this.deleteFromLocalCache(workspaceId, cacheKeyNames);}+ private assertValidCacheParameters(+ workspaceId: string,+ cacheKeyNames: WorkspaceCacheKeyName[],+ ): void {+ if (+ !isDefined(workspaceId) ||+ cacheKeyNames.length === 0 ||+ !isValidUuid(workspaceId)+ ) {+ throw new WorkspaceCacheException(+ 'Invalid parameters: workspace ID and cache key names are required',+ WorkspaceCacheExceptionCode.INVALID_PARAMETERS,+ );+ }+ }+private checkLocalTTL<K extends WorkspaceCacheKeyName>(workspaceId: string,cacheKeyNames: readonly K[],@@ -328,14 +389,14 @@ export class WorkspaceCacheService implements OnModuleInit {workspaceId: string,cacheKeyNames: WorkspaceCacheKeyName[],): Promise<{- redisData: Partial<WorkspaceCacheDataMap>;+ redisEntries: CacheEntriesResult;missingInRedis: WorkspaceCacheKeyName[];}> {- const redisData: Partial<WorkspaceCacheDataMap> = {};+ const redisEntries: CacheEntriesResult = { data: {}, hashes: {} };const missingInRedis: WorkspaceCacheKeyName[] = [];if (cacheKeyNames.length === 0) {- return { redisData, missingInRedis };+ return { redisEntries, missingInRedis };}// Interleave data and hash keys for atomic fetch: [data1, hash1, data2, hash2, ...]@@ -354,22 +415,23 @@ export class WorkspaceCacheService implements OnModuleInit {const hash = allValues[index * 2 + 1] as string | undefined;if (isDefined(data) && isDefined(hash)) {- Object.assign(redisData, { [keyName]: data });+ Object.assign(redisEntries.data, { [keyName]: data });+ redisEntries.hashes[keyName] = hash;this.setInLocalCache(workspaceId, keyName, data, hash);} else {missingInRedis.push(keyName);}}- return { redisData, missingInRedis };+ return { redisEntries, missingInRedis };}private async recomputeDataFromProvider(workspaceId: string,cacheKeyNames: WorkspaceCacheKeyName[],hashResolution: RecomputeHashResolution,- ): Promise<Partial<WorkspaceCacheDataMap>> {- const result: Partial<WorkspaceCacheDataMap> = {};+ ): Promise<CacheEntriesResult> {+ const result: CacheEntriesResult = { data: {}, hashes: {} };if (cacheKeyNames.length === 0) {return result;@@ -412,7 +474,8 @@ export class WorkspaceCacheService implements OnModuleInit {const bootstrapHashEntries: Array<{ key: string; value: string }> = [];for (const { keyName, data, hash, isAdopted } of computed) {- Object.assign(result, { [keyName]: data });+ Object.assign(result.data, { [keyName]: data });+ result.hashes[keyName] = hash;const baseKey = this.buildCacheKey(workspaceId, keyName);const isLocalDataOnly = this.localDataOnlyKeys.has(keyName);@@ -453,8 +516,8 @@ export class WorkspaceCacheService implements OnModuleInit {private getFromLocalCache(workspaceId: string,workspaceCacheKeyNames: WorkspaceCacheKeyName[],- ): Partial<WorkspaceCacheDataMap> {- const result: Partial<WorkspaceCacheDataMap> = {};+ ): CacheEntriesResult {+ const result: CacheEntriesResult = { data: {}, hashes: {} };for (const keyName of workspaceCacheKeyNames) {const localKey = this.buildCacheKey(workspaceId, keyName);@@ -463,7 +526,8 @@ export class WorkspaceCacheService implements OnModuleInit {if (isDefined(entry) && isDefined(version)) {version.lastReadAt = Date.now();- Object.assign(result, { [keyName]: version.data });+ Object.assign(result.data, { [keyName]: version.data });+ result.hashes[keyName] = entry.latestHash;this.cleanupStaleVersions(entry);}}diff --git a/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts b/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.tsindex c2ce745097b..d2499955014 100644--- a/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts+++ b/packages/twenty-server/src/engine/workspace-cache/types/workspace-cache-key.type.ts@@ -92,3 +92,9 @@ export type WorkspaceCacheKeyName = keyof WorkspaceCacheDataMap;export type WorkspaceCacheResult<K extends WorkspaceCacheKeyName[]> = {[P in K[number]]: WorkspaceCacheDataMap[P];};++export type WorkspaceCacheResultWithHashes<K extends WorkspaceCacheKeyName[]> =+ {+ data: WorkspaceCacheResult<K>;+ hashes: { [P in K[number]]: string };+ };diff --git a/packages/twenty-server/src/engine/workspace-cache/utils/combine-cache-hashes.util.ts b/packages/twenty-server/src/engine/workspace-cache/utils/combine-cache-hashes.util.tsnew file mode 100644index 00000000000..6576a514683--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-cache/utils/combine-cache-hashes.util.ts@@ -0,0 +1,26 @@+import { createHash } from 'crypto';++import { isDefined } from 'twenty-shared/utils';++import { type WorkspaceCacheKeyName } from 'src/engine/workspace-cache/types/workspace-cache-key.type';++export const combineCacheHashes = (+ hashes: Partial<Record<WorkspaceCacheKeyName, string>>,+ cacheKeyNames: readonly WorkspaceCacheKeyName[],+): string => {+ if (cacheKeyNames.length === 0) {+ throw new Error('Cannot combine cache hashes without cache key names');+ }++ const orderedHashes = [...cacheKeyNames].sort().map((cacheKeyName) => {+ const hash = hashes[cacheKeyName];++ if (!isDefined(hash)) {+ throw new Error(`Missing cache hash for "${cacheKeyName}"`);+ }++ return hash;+ });++ return createHash('sha256').update(orderedHashes.join(':')).digest('hex');+};diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/services/dev-seeder.service.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/services/dev-seeder.service.tsindex cd7cc09e5a7..760abb9a5f6 100644--- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/services/dev-seeder.service.ts+++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/services/dev-seeder.service.ts@@ -191,7 +191,7 @@ export class DevSeederService {light,});- await this.workspaceCacheStorageService.flush(workspaceId, undefined);+ await this.workspaceCacheStorageService.flush(workspaceId);}private async seedCoreSchema({diff --git a/packages/twenty-server/test/integration/metadata/suites/graphql-operation-cache/metadata-graphql-operations-cache-dependencies.integration-spec.ts b/packages/twenty-server/test/integration/metadata/suites/graphql-operation-cache/metadata-graphql-operations-cache-dependencies.integration-spec.tsnew file mode 100644index 00000000000..fbe48b4677f--- /dev/null+++ b/packages/twenty-server/test/integration/metadata/suites/graphql-operation-cache/metadata-graphql-operations-cache-dependencies.integration-spec.ts@@ -0,0 +1,295 @@+import { gql } from 'graphql-tag';+import { makeMetadataAPIRequest } from 'test/integration/metadata/suites/utils/make-metadata-api-request.util';+import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';++import { type ASTNode } from 'graphql';++import { METADATA_GRAPHQL_OPERATIONS_TO_CACHE } from 'src/engine/api/graphql/graphql-config/constants/metadata-graphql-operations-to-cache.constant';+import { FIND_ALL_VIEWS_GRAPHQL_OPERATION } from 'src/engine/metadata-modules/view/constants/find-all-views-graphql-operation.constant';+import { type WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';+import { type WorkspaceCacheKeyName } from 'src/engine/workspace-cache/types/workspace-cache-key.type';++// Mirrors OBJECT_METADATA_FRAGMENT from twenty-front+// (packages/twenty-front/src/modules/object-metadata/graphql/fragment.ts):+// the recorded resolver reads are only as complete as this selection set.+const OBJECT_METADATA_ITEMS_QUERY = gql`+ query ObjectMetadataItems {+ objects(paging: { first: 1000 }) {+ edges {+ node {+ id+ universalIdentifier+ nameSingular+ namePlural+ labelSingular+ labelPlural+ color+ description+ icon+ isRemote+ isActive+ isSystem+ isUIEditable+ isUICreatable+ createdAt+ updatedAt+ labelIdentifierFieldMetadataId+ imageIdentifierFieldMetadataId+ applicationId+ shortcut+ isLabelSyncedWithName+ isSearchable+ duplicateCriteria+ searchFieldMetadataList {+ id+ fieldMetadataId+ tsVectorFieldMetadataId+ position+ }+ indexMetadataList {+ id+ name+ indexWhereClause+ indexType+ isUnique+ isCustom+ indexFieldMetadataList {+ id+ fieldMetadataId+ subFieldName+ order+ }+ }+ fieldsList {+ id+ universalIdentifier+ type+ name+ label+ description+ icon+ isActive+ isSystem+ isUIEditable+ isNullable+ isUnique+ defaultValue+ options+ settings+ isLabelSyncedWithName+ morphId+ applicationId+ relation {+ type+ sourceObjectMetadata {+ id+ nameSingular+ namePlural+ }+ targetObjectMetadata {+ id+ nameSingular+ namePlural+ }+ sourceFieldMetadata {+ id+ name+ }+ targetFieldMetadata {+ id+ name+ }+ }+ morphRelations {+ type+ sourceObjectMetadata {+ id+ nameSingular+ namePlural+ }+ targetObjectMetadata {+ id+ nameSingular+ namePlural+ }+ sourceFieldMetadata {+ id+ name+ }+ targetFieldMetadata {+ id+ name+ }+ }+ }+ }+ }+ }+ }+`;++const OBJECT_METADATA_ITEMS_BASELINE_QUERY = gql`+ query ObjectMetadataItems {+ objects(paging: { first: 1 }) {+ edges {+ node {+ id+ }+ }+ }+ }+`;++// Mirrors VIEW_FRAGMENT from twenty-front+// (packages/twenty-front/src/modules/views/graphql/fragments/viewFragment.ts).+const FIND_ALL_VIEWS_QUERY = gql`+ query FindAllViews {+ getViews {+ id+ name+ objectMetadataId+ type+ key+ icon+ position+ isCompact+ openRecordIn+ kanbanAggregateOperation+ kanbanAggregateOperationFieldMetadataId+ mainGroupByFieldMetadataId+ shouldHideEmptyGroups+ kanbanColumnWidth+ anyFieldFilterValue+ calendarFieldMetadataId+ calendarEndFieldMetadataId+ calendarLayout+ visibility+ createdByUserWorkspaceId+ isActive+ viewFields {+ id+ fieldMetadataId+ isVisible+ size+ position+ aggregateOperation+ }+ viewFieldGroups {+ id+ name+ position+ isVisible+ viewId+ isActive+ viewFields {+ id+ fieldMetadataId+ isVisible+ size+ position+ aggregateOperation+ }+ }+ viewFilters {+ id+ fieldMetadataId+ operand+ value+ viewFilterGroupId+ positionInViewFilterGroup+ subFieldName+ }+ viewFilterGroups {+ id+ parentViewFilterGroupId+ logicalOperator+ positionInViewFilterGroup+ }+ viewSorts {+ id+ fieldMetadataId+ direction+ }+ viewGroups {+ id+ isVisible+ fieldValue+ position+ viewId+ }+ }+ }+`;++const FIND_ALL_VIEWS_BASELINE_QUERY = gql`+ query FindAllViews {+ getViews {+ id+ }+ }+`;++describe('metadata GraphQL operations cache dependencies', () => {+ const recordAccessedCacheKeys = async (+ query: ASTNode,+ ): Promise<Set<WorkspaceCacheKeyName>> => {+ const workspaceCacheService =+ getAppProviderByClassName<WorkspaceCacheService>('WorkspaceCacheService');+ const spy = jest.spyOn(workspaceCacheService, 'getOrRecomputeWithHashes');++ try {+ const response = await makeMetadataAPIRequest({ query });++ expect(response.body.errors).toBeUndefined();++ return new Set<WorkspaceCacheKeyName>(+ spy.mock.calls.flatMap(([, cacheKeyNames]) => cacheKeyNames),+ );+ } finally {+ spy.mockRestore();+ }+ };++ const findUndeclaredDependencies = async ({+ operationName,+ fullQuery,+ baselineQuery,+ }: {+ operationName: string;+ fullQuery: ASTNode;+ baselineQuery: ASTNode;+ }): Promise<WorkspaceCacheKeyName[]> => {+ const declaredDependencies = new Set<WorkspaceCacheKeyName>(+ METADATA_GRAPHQL_OPERATIONS_TO_CACHE[operationName].dependencies,+ );+ const requestInfrastructureKeys =+ await recordAccessedCacheKeys(baselineQuery);+ const accessedCacheKeys = await recordAccessedCacheKeys(fullQuery);++ return [...accessedCacheKeys].filter(+ (cacheKeyName) =>+ !declaredDependencies.has(cacheKeyName) &&+ !requestInfrastructureKeys.has(cacheKeyName),+ );+ };++ it('declares every flat map the ObjectMetadataItems resolvers read', async () => {+ const undeclaredDependencies = await findUndeclaredDependencies({+ operationName: 'ObjectMetadataItems',+ fullQuery: OBJECT_METADATA_ITEMS_QUERY,+ baselineQuery: OBJECT_METADATA_ITEMS_BASELINE_QUERY,+ });++ expect(undeclaredDependencies).toEqual([]);+ });++ it('declares every flat map the FindAllViews resolvers read', async () => {+ const undeclaredDependencies = await findUndeclaredDependencies({+ operationName: FIND_ALL_VIEWS_GRAPHQL_OPERATION,+ fullQuery: FIND_ALL_VIEWS_QUERY,+ baselineQuery: FIND_ALL_VIEWS_BASELINE_QUERY,+ });++ expect(undeclaredDependencies).toEqual([]);+ });+});diff --git a/packages/twenty-server/test/integration/utils/get-app-provider-by-class-name.util.ts b/packages/twenty-server/test/integration/utils/get-app-provider-by-class-name.util.tsnew file mode 100644index 00000000000..d6f8e31d554--- /dev/null+++ b/packages/twenty-server/test/integration/utils/get-app-provider-by-class-name.util.ts@@ -0,0 +1,30 @@+type ProviderWrapper = { instance: unknown };+type ContainerModule = { providers: Map<unknown, ProviderWrapper> };++export const getAppProviderByClassName = <T>(className: string): T => {+ if (!global.app) {+ throw new Error(+ 'global.app is not set: integration test globalSetup has not run',+ );+ }++ const container = (+ global.app as unknown as {+ container: { getModules: () => Map<string, ContainerModule> };+ }+ ).container;++ for (const containerModule of container.getModules().values()) {+ for (const [token, wrapper] of containerModule.providers) {+ if (+ typeof token === 'function' &&+ token.name === className &&+ wrapper.instance+ ) {+ return wrapper.instance as T;+ }+ }+ }++ throw new Error(`Provider "${className}" not found in application container`);+};