Skip to content

The open alternative to Salesforce, designed for AI.

fix(server): stop workspace commands retaining every visited workspace in memory (#26342)

martmull256d82db8f62
diff --git a/packages/twenty-server/src/database/commands/command-runners/__tests__/workspace-iterator.service.spec.ts b/packages/twenty-server/src/database/commands/command-runners/__tests__/workspace-iterator.service.spec.ts
new file mode 100644
index 00000000000..e0230129746
--- /dev/null
+++ b/packages/twenty-server/src/database/commands/command-runners/__tests__/workspace-iterator.service.spec.ts
@@ -0,0 +1,98 @@
+import { Logger } from '@nestjs/common';
+
+import { type DataSource, type Repository } from 'typeorm';
+
+import { type CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
+import { WorkspaceIteratorService } from 'src/database/commands/command-runners/workspace-iterator.service';
+import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
+import { type WorkspaceOrmManager } from 'src/engine/twenty-orm/workspace-orm.manager';
+import { type WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
+
+describe('WorkspaceIteratorService', () => {
+ let workspaceCacheService: jest.Mocked<
+ Pick<WorkspaceCacheService, 'evictWorkspaceFromLocalCache'>
+ >;
+ let service: WorkspaceIteratorService;
+
+ beforeEach(() => {
+ jest.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
+ jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
+ jest.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined);
+
+ const workspaceRepository = {
+ findOne: jest
+ .fn()
+ .mockResolvedValue({ databaseSchema: 'workspace_schema' }),
+ } as unknown as Repository<WorkspaceEntity>;
+ const workspaceOrmManager = {
+ executeInWorkspaceContext: jest.fn((fn: () => Promise<void>) => fn()),
+ } as unknown as WorkspaceOrmManager;
+ const commandShutdownService = {
+ isShutdownRequested: jest.fn().mockReturnValue(false),
+ listenToShutdownSignals: jest.fn(),
+ } as unknown as CommandShutdownService;
+
+ workspaceCacheService = {
+ evictWorkspaceFromLocalCache: jest.fn().mockResolvedValue(undefined),
+ };
+
+ service = new WorkspaceIteratorService(
+ workspaceRepository,
+ {} as DataSource,
+ workspaceOrmManager,
+ commandShutdownService,
+ workspaceCacheService as unknown as WorkspaceCacheService,
+ );
+ });
+
+ afterEach(() => {
+ jest.restoreAllMocks();
+ });
+
+ it('evicts each workspace from the local cache once its callback has run', async () => {
+ const events: string[] = [];
+
+ workspaceCacheService.evictWorkspaceFromLocalCache.mockImplementation(
+ async (workspaceId) => {
+ events.push(`evict:${workspaceId}`);
+ },
+ );
+
+ const report = await service.iterate({
+ workspaceIds: ['workspace-1', 'workspace-2'],
+ callback: async ({ workspaceId }) => {
+ events.push(`callback:${workspaceId}`);
+ },
+ });
+
+ expect(report.success.map(({ workspaceId }) => workspaceId)).toEqual([
+ 'workspace-1',
+ 'workspace-2',
+ ]);
+ expect(events).toEqual([
+ 'callback:workspace-1',
+ 'evict:workspace-1',
+ 'callback:workspace-2',
+ 'evict:workspace-2',
+ ]);
+ });
+
+ it('still evicts a workspace whose callback failed', async () => {
+ const report = await service.iterate({
+ workspaceIds: ['workspace-1'],
+ callback: async () => {
+ throw new Error('callback failed');
+ },
+ });
+
+ expect(report.fail).toEqual([
+ { workspaceId: 'workspace-1', error: new Error('callback failed') },
+ ]);
+ expect(
+ workspaceCacheService.evictWorkspaceFromLocalCache,
+ ).toHaveBeenCalledTimes(1);
+ expect(
+ workspaceCacheService.evictWorkspaceFromLocalCache,
+ ).toHaveBeenCalledWith('workspace-1');
+ });
+});
diff --git a/packages/twenty-server/src/database/commands/command-runners/workspace-iterator.module.ts b/packages/twenty-server/src/database/commands/command-runners/workspace-iterator.module.ts
index a49596f7bfa..3eecd2b5d10 100644
--- a/packages/twenty-server/src/database/commands/command-runners/workspace-iterator.module.ts
+++ b/packages/twenty-server/src/database/commands/command-runners/workspace-iterator.module.ts
@@ -4,9 +4,14 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { CommandShutdownModule } from 'src/database/commands/command-runners/command-shutdown.module';
import { WorkspaceIteratorService } from 'src/database/commands/command-runners/workspace-iterator.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
+import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
@Module({
- imports: [CommandShutdownModule, TypeOrmModule.forFeature([WorkspaceEntity])],
+ imports: [
+ CommandShutdownModule,
+ TypeOrmModule.forFeature([WorkspaceEntity]),
+ WorkspaceCacheModule,
+ ],
providers: [WorkspaceIteratorService],
exports: [WorkspaceIteratorService],
})
diff --git a/packages/twenty-server/src/database/commands/command-runners/workspace-iterator.service.ts b/packages/twenty-server/src/database/commands/command-runners/workspace-iterator.service.ts
index 84e19f068a3..6c8c7264c66 100644
--- a/packages/twenty-server/src/database/commands/command-runners/workspace-iterator.service.ts
+++ b/packages/twenty-server/src/database/commands/command-runners/workspace-iterator.service.ts
@@ -15,6 +15,7 @@ import { activationStatusIn } from 'src/database/commands/command-runners/utils/
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceOrmManager } from 'src/engine/twenty-orm/workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import { WorkspaceMigrationRunnerException } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/exceptions/workspace-migration-runner.exception';
export type WorkspaceIteratorShard = {
@@ -64,6 +65,7 @@ export class WorkspaceIteratorService {
private readonly coreDataSource: DataSource,
private readonly workspaceOrmManager: WorkspaceOrmManager,
private readonly commandShutdownService: CommandShutdownService,
+ private readonly workspaceCacheService: WorkspaceCacheService,
) {}
listenToShutdownSignals(): void {
@@ -138,6 +140,10 @@ export class WorkspaceIteratorService {
report.success.push({ workspaceId });
} catch (error: unknown) {
report.fail.push({ error: error as Error, workspaceId });
+ } finally {
+ await this.workspaceCacheService.evictWorkspaceFromLocalCache(
+ workspaceId,
+ );
}
}
diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts
index 2934e220ef8..20cc91d85a0 100644
--- a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts
@@ -1,8 +1,7 @@
import { Injectable } from '@nestjs/common';
import { makeExecutableSchema } from '@graphql-tools/schema';
-import { GraphQLSchema } from 'graphql';
-import { gql } from 'graphql-tag';
+import { GraphQLSchema, parse } from 'graphql';
import { isDefined } from 'twenty-shared/utils';
import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service';
@@ -56,10 +55,11 @@ export class WorkspaceSchemaFactory {
const scalarsResolvers =
this.scalarsExplorerService.getScalarResolvers(usedScalarNames);
+ // graphql-tag keeps every document it parses in a process-wide cache keyed
+ // by source, so each workspace SDL fed through it would stay resident for
+ // the lifetime of the process.
const executableSchema = makeExecutableSchema({
- typeDefs: gql`
- ${sdl}
- `,
+ typeDefs: parse(sdl, { noLocation: true }),
resolvers: {
...scalarsResolvers,
...autoGeneratedResolvers,
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.ts b/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.ts
new file mode 100644
index 00000000000..4f832a4508e
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.ts
@@ -0,0 +1,126 @@
+import { type DiscoveryService, Reflector } from '@nestjs/core';
+
+import { type DataSource } from 'typeorm';
+
+import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';
+
+import { type CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
+import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+import { WorkspaceCache } from 'src/engine/workspace-cache/decorators/workspace-cache.decorator';
+import { type WorkspaceCacheMetricsService } from 'src/engine/workspace-cache/services/workspace-cache-metrics.service';
+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
+
+@WorkspaceCache('apiKeyRoleMap', { packingPonderation: 1 })
+class ApiKeyRoleMapTestProvider extends WorkspaceCacheProvider<
+ Record<string, string>
+> {
+ computeForCache(): Record<string, string> {
+ return {};
+ }
+}
+
+const WORKSPACE_1_ID = '20202020-1c25-4d02-bf25-6aeccf7ea419';
+const WORKSPACE_2_ID = '3b8e6458-5fc1-4e63-8563-008ccddaa6db';
+
+const buildCacheKey = (workspaceId: string, suffix: 'hash' | 'data') =>
+ `apiKeyRoleMap:${workspaceId}:${suffix}`;
+
+describe('WorkspaceCacheService', () => {
+ const redisEntries = new Map<string, unknown>([
+ [buildCacheKey(WORKSPACE_1_ID, 'hash'), 'hash-1'],
+ [buildCacheKey(WORKSPACE_1_ID, 'data'), { 'api-key-1': 'role-1' }],
+ [buildCacheKey(WORKSPACE_2_ID, 'hash'), 'hash-2'],
+ [buildCacheKey(WORKSPACE_2_ID, 'data'), { 'api-key-2': 'role-2' }],
+ ]);
+
+ let cacheStorage: jest.Mocked<
+ Pick<CacheStorageService, 'mget' | 'mset' | 'mdel' | 'setIfAbsent'>
+ >;
+ let service: WorkspaceCacheService;
+
+ const countDataFetches = (workspaceId: string) =>
+ cacheStorage.mget.mock.calls.filter(([keys]) =>
+ keys.includes(buildCacheKey(workspaceId, 'data')),
+ ).length;
+
+ beforeEach(async () => {
+ jest.useFakeTimers();
+
+ cacheStorage = {
+ mget: jest.fn(async (keys: string[]) =>
+ keys.map((key) => redisEntries.get(key)),
+ ),
+ mset: jest.fn(),
+ mdel: jest.fn(),
+ setIfAbsent: jest.fn(),
+ } as unknown as typeof cacheStorage;
+
+ const discoveryService = {
+ getProviders: () => [{ instance: new ApiKeyRoleMapTestProvider() }],
+ } as unknown as DiscoveryService;
+ const cacheMetricsService = {
+ start: jest.fn(),
+ stop: jest.fn(),
+ recordRecompute: jest.fn(),
+ recordRedisWrite: jest.fn(),
+ recordEviction: jest.fn(),
+ recordPackingRun: jest.fn(),
+ recordUnpacking: jest.fn(),
+ } as unknown as WorkspaceCacheMetricsService;
+ const twentyConfigService = {
+ get: jest.fn().mockReturnValue(3600),
+ } as unknown as TwentyConfigService;
+
+ service = new WorkspaceCacheService(
+ cacheStorage as unknown as CacheStorageService,
+ {} as DataSource,
+ discoveryService,
+ new Reflector(),
+ cacheMetricsService,
+ twentyConfigService,
+ );
+
+ await service.onModuleInit();
+ });
+
+ afterEach(() => {
+ service.onModuleDestroy();
+ jest.useRealTimers();
+ });
+
+ describe('evictWorkspaceFromLocalCache', () => {
+ it('drops the local copy so the next read fetches the data again', async () => {
+ await service.getOrRecompute(WORKSPACE_1_ID, ['apiKeyRoleMap']);
+
+ // Past the memoizer and local TTLs, a read only re-checks the hash.
+ jest.setSystemTime(Date.now() + 11_000);
+
+ await service.getOrRecompute(WORKSPACE_1_ID, ['apiKeyRoleMap']);
+
+ expect(countDataFetches(WORKSPACE_1_ID)).toBe(1);
+
+ await service.evictWorkspaceFromLocalCache(WORKSPACE_1_ID);
+
+ const { apiKeyRoleMap } = await service.getOrRecompute(WORKSPACE_1_ID, [
+ 'apiKeyRoleMap',
+ ]);
+
+ expect(apiKeyRoleMap).toEqual({ 'api-key-1': 'role-1' });
+ expect(countDataFetches(WORKSPACE_1_ID)).toBe(2);
+ });
+
+ it('leaves Redis and the other workspaces untouched', async () => {
+ await service.getOrRecompute(WORKSPACE_1_ID, ['apiKeyRoleMap']);
+ await service.getOrRecompute(WORKSPACE_2_ID, ['apiKeyRoleMap']);
+
+ await service.evictWorkspaceFromLocalCache(WORKSPACE_1_ID);
+
+ jest.setSystemTime(Date.now() + 11_000);
+
+ await service.getOrRecompute(WORKSPACE_2_ID, ['apiKeyRoleMap']);
+
+ expect(countDataFetches(WORKSPACE_2_ID)).toBe(1);
+ expect(cacheStorage.mdel).not.toHaveBeenCalled();
+ });
+ });
+});
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
index 9a61ddf52e9..36094606644 100644
--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts
@@ -358,6 +358,18 @@ export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
this.deleteFromLocalCache(workspaceId, cacheKeyNames);
}
+ public async evictWorkspaceFromLocalCache(
+ workspaceId: string,
+ ): Promise<void> {
+ await this.memoizer.clearKeys(`${workspaceId}-`);
+
+ for (const localKey of this.localCache.keys()) {
+ if (localKey.endsWith(`:${workspaceId}`)) {
+ this.localCache.delete(localKey);
+ }
+ }
+ }
+
private assertValidCacheParameters(
workspaceId: string,
cacheKeyNames: WorkspaceCacheKeyName[],