Skip to content

The open alternative to Salesforce, designed for AI.

30.8 KB
Raw Blame
import {
  Injectable,
  Logger,
  OnModuleDestroy,
  OnModuleInit,
} from '@nestjs/common';
import { DiscoveryService, Reflector } from '@nestjs/core';
import { InjectDataSource } from '@nestjs/typeorm';

import * as Sentry from '@sentry/node';
import crypto from 'crypto';

import { DataSource } from 'typeorm';

import { isDefined, isValidUuid } from 'twenty-shared/utils';

import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';
import { WorkspaceDerivedCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-derived-cache-provider.service';
import { PUBLISH_WORKSPACE_CACHE_SCRIPT } from 'src/engine/workspace-cache/constants/publish-workspace-cache-script.constant';

import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { PromiseMemoizer } from 'src/engine/twenty-orm/storage/promise-memoizer.storage';
import {
  WORKSPACE_CACHE_KEY,
  WORKSPACE_CACHE_OPTIONS,
  WorkspaceCacheOptions,
} from 'src/engine/workspace-cache/decorators/workspace-cache.decorator';
import { WORKSPACE_DERIVED_CACHE_KEY } from 'src/engine/workspace-cache/decorators/workspace-derived-cache.decorator';
import {
  WorkspaceCacheException,
  WorkspaceCacheExceptionCode,
} from 'src/engine/workspace-cache/exceptions/workspace-cache.exception';
import { WorkspaceCacheMetricsService } from 'src/engine/workspace-cache/services/workspace-cache-metrics.service';
import { WorkspaceCacheRowsBatchLoader } from 'src/engine/workspace-cache/services/workspace-cache-rows-batch-loader';
import {
  WorkspaceCacheKeyName,
  type WorkspaceCacheDataMap,
  type WorkspaceCacheOrDerivedCacheDataMap,
  type RemovedWorkspaceCacheKeyName,
  type WorkspaceCacheOrDerivedCacheKeyName,
  type WorkspaceCacheResult,
  type WorkspaceCacheResultWithHashes,
  type WorkspaceCacheStoredDataMap,
  type WorkspaceDerivedCacheKeyName,
} from 'src/engine/workspace-cache/types/workspace-cache-key.type';
import {
  type VersionEntry,
  type WorkspaceLocalCacheEntry,
} from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
import { combineCacheHashes } from 'src/engine/workspace-cache/utils/combine-cache-hashes.util';
import { getKeyNameFromLocalCacheKey } from 'src/engine/workspace-cache/utils/get-key-name-from-local-cache-key.util';
import { packIdleVersions } from 'src/engine/workspace-cache/utils/pack-idle-versions.util';
import { isWorkspaceDerivedCacheKeyName } from 'src/engine/workspace-cache/utils/is-workspace-derived-cache-key-name.util';
import {
  deserializeCacheBlob,
  serializeCacheBlob,
} from 'src/engine/workspace-cache/utils/serialize-cache-blob.util';
import { sweepLocalCache } from 'src/engine/workspace-cache/utils/sweep-local-cache.util';

const LOCAL_TTL_MS = 100;
const MEMOIZER_TTL_MS = 10_000;
const STALE_VERSION_TTL_MS = 5_000;
const MAX_LOCAL_STALE_VERSIONS = 5;
// Sized against 4 GiB pods (--max-old-space-size=3500): 7,500 sat at the heap ceiling.
const MAX_LOCAL_CACHE_ENTRIES = 6_000;
const MIN_EVICT_KEYS = 100;
const LOCAL_ENTRY_TTL_MS = 30 * 60 * 1000;
const LOCAL_CACHE_SWEEP_INTERVAL_MS = 60 * 1000;
const PACKING_INTERVAL_MS = 500;
const PACKING_PONDERATION_BUDGET = 64;
const MIN_IDLE_BEFORE_PACKING_MS = 60 * 1000;
// Per-cache-key entry caps
const MAX_LOCAL_ENTRIES_BY_KEY_NAME = new Map<string, number>([
  ['flatFieldMetadataMaps', 256],
  ['flatFieldMetadataMapsOrm', 512],
]);
type CacheDataType = WorkspaceCacheDataMap[WorkspaceCacheKeyName];
type StoredCacheDataType = WorkspaceCacheStoredDataMap[WorkspaceCacheKeyName];

type CacheEntriesResult = {
  data: Partial<WorkspaceCacheDataMap>;
  hashes: Partial<Record<WorkspaceCacheKeyName, string>>;
};

type KeyToRecompute = {
  keyName: WorkspaceCacheKeyName;
  expectedHash?: string;
};

type RecomputedCacheEntriesResult = CacheEntriesResult & {
  hasUnpublishedEntries: boolean;
};

type CacheOrDerivedCacheEntriesResult = {
  data: Partial<WorkspaceCacheOrDerivedCacheDataMap>;
  hashes: Partial<Record<WorkspaceCacheOrDerivedCacheKeyName, string>>;
};

@Injectable()
export class WorkspaceCacheService implements OnModuleInit, OnModuleDestroy {
  private readonly localCache = new Map<
    string,
    WorkspaceLocalCacheEntry<CacheDataType>
  >();
  private sweepTimer?: ReturnType<typeof setInterval>;
  private packingTimer?: ReturnType<typeof setInterval>;
  private readonly workspaceCacheProviders = new Map<
    WorkspaceCacheKeyName,
    WorkspaceCacheProvider<CacheDataType, StoredCacheDataType>
  >();
  private readonly workspaceDerivedCacheProviders = new Map<
    WorkspaceDerivedCacheKeyName,
    WorkspaceDerivedCacheProvider
  >();
  private readonly localDataOnlyKeys = new Set<WorkspaceCacheKeyName>();
  private readonly packingPonderationByKey = new Map<
    WorkspaceCacheKeyName,
    number
  >();
  private readonly memoizer = new PromiseMemoizer<CacheEntriesResult>(
    MEMOIZER_TTL_MS,
  );

  private readonly logger = new Logger(WorkspaceCacheService.name);

  constructor(
    @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace)
    private readonly cacheStorage: CacheStorageService,
    @InjectDataSource()
    private readonly coreDataSource: DataSource,
    private readonly discoveryService: DiscoveryService,
    private readonly reflector: Reflector,
    private readonly cacheMetricsService: WorkspaceCacheMetricsService,
    private readonly twentyConfigService: TwentyConfigService,
  ) {}

  async onModuleInit() {
    const providers = this.discoveryService.getProviders();

    for (const wrapper of providers) {
      const { instance } = wrapper;

      if (!isDefined(instance) || typeof instance !== 'object') {
        continue;
      }

      const workspaceCacheKeyName = this.reflector.get<WorkspaceCacheKeyName>(
        WORKSPACE_CACHE_KEY,
        instance.constructor,
      );

      if (
        isDefined(workspaceCacheKeyName) &&
        instance instanceof WorkspaceCacheProvider
      ) {
        this.workspaceCacheProviders.set(workspaceCacheKeyName, instance);

        const options: WorkspaceCacheOptions | undefined =
          this.reflector.get<WorkspaceCacheOptions>(
            WORKSPACE_CACHE_OPTIONS,
            instance.constructor,
          );

        if (isDefined(options)) {
          if (options.localDataOnly) {
            this.localDataOnlyKeys.add(workspaceCacheKeyName);
          }

          this.packingPonderationByKey.set(
            workspaceCacheKeyName,
            options.packingPonderation,
          );
        }
      }

      const workspaceDerivedCacheKeyName =
        this.reflector.get<WorkspaceDerivedCacheKeyName>(
          WORKSPACE_DERIVED_CACHE_KEY,
          instance.constructor,
        );

      if (
        isDefined(workspaceDerivedCacheKeyName) &&
        instance instanceof WorkspaceDerivedCacheProvider
      ) {
        this.workspaceDerivedCacheProviders.set(
          workspaceDerivedCacheKeyName,
          instance,
        );
      }
    }

    this.cacheMetricsService.start(this.localCache);
    this.startMaintenanceTimers();
  }

  onModuleDestroy(): void {
    if (isDefined(this.sweepTimer)) {
      clearInterval(this.sweepTimer);
    }
    if (isDefined(this.packingTimer)) {
      clearInterval(this.packingTimer);
    }
    this.cacheMetricsService.stop();
  }

  private startMaintenanceTimers(): void {
    this.sweepTimer = setInterval(
      () => this.sweepLocalCache(),
      LOCAL_CACHE_SWEEP_INTERVAL_MS,
    );
    this.sweepTimer.unref();

    this.packingTimer = setInterval(
      () => this.runPacking(),
      PACKING_INTERVAL_MS,
    );
    this.packingTimer.unref();
  }

  public async getOrRecompute<
    const K extends WorkspaceCacheOrDerivedCacheKeyName[],
  >(workspaceId: string, cacheKeyNames: K): Promise<WorkspaceCacheResult<K>> {
    const { data } = await this.getOrRecomputeWithHashes(
      workspaceId,
      cacheKeyNames,
    );

    return data;
  }

  public async getOrRecomputeWithHashes<
    const K extends WorkspaceCacheOrDerivedCacheKeyName[],
  >(
    workspaceId: string,
    cacheKeyNames: K,
  ): Promise<WorkspaceCacheResultWithHashes<K>> {
    this.assertValidCacheParameters(workspaceId, cacheKeyNames);

    const providerEntries = await this.getOrRecomputeProviderEntries(
      workspaceId,
      this.getSourceCacheKeyNames(cacheKeyNames),
    );

    if (!cacheKeyNames.some(isWorkspaceDerivedCacheKeyName)) {
      return providerEntries as WorkspaceCacheResultWithHashes<K>;
    }

    const result: CacheOrDerivedCacheEntriesResult = { data: {}, hashes: {} };

    for (const cacheKeyName of cacheKeyNames) {
      if (!isWorkspaceDerivedCacheKeyName(cacheKeyName)) {
        Object.assign(result.data, {
          [cacheKeyName]: providerEntries.data[cacheKeyName],
        });
        result.hashes[cacheKeyName] = providerEntries.hashes[cacheKeyName];
        continue;
      }

      const derivedCacheProvider =
        this.getDerivedCacheProviderOrThrow(cacheKeyName);
      const { sourceKeyName } = derivedCacheProvider;
      const source = providerEntries.data[sourceKeyName];

      if (!isDefined(source)) {
        throw new WorkspaceCacheException(
          `Source cache "${sourceKeyName}" of derived cache "${cacheKeyName}" is not loaded`,
          WorkspaceCacheExceptionCode.INTERNAL_SERVER_ERROR,
        );
      }

      Object.assign(result.data, {
        [cacheKeyName]: derivedCacheProvider.getFromSource(source),
      });
      result.hashes[cacheKeyName] = providerEntries.hashes[sourceKeyName];
    }

    return result as WorkspaceCacheResultWithHashes<K>;
  }

  private getSourceCacheKeyNames(
    cacheKeyNames: readonly WorkspaceCacheOrDerivedCacheKeyName[],
  ): WorkspaceCacheKeyName[] {
    return [
      ...new Set(
        cacheKeyNames.map((cacheKeyName) =>
          isWorkspaceDerivedCacheKeyName(cacheKeyName)
            ? this.getDerivedCacheProviderOrThrow(cacheKeyName).sourceKeyName
            : cacheKeyName,
        ),
      ),
    ];
  }

  private getDerivedCacheProviderOrThrow(
    derivedKeyName: WorkspaceDerivedCacheKeyName,
  ): WorkspaceDerivedCacheProvider {
    const derivedCacheProvider =
      this.workspaceDerivedCacheProviders.get(derivedKeyName);

    if (!isDefined(derivedCacheProvider)) {
      throw new WorkspaceCacheException(
        `Derived cache provider with key name "${derivedKeyName}" not found`,
        WorkspaceCacheExceptionCode.INTERNAL_SERVER_ERROR,
      );
    }

    return derivedCacheProvider;
  }

  private async getOrRecomputeProviderEntries(
    workspaceId: string,
    cacheKeyNames: WorkspaceCacheKeyName[],
  ): Promise<CacheEntriesResult> {
    const memoKey =
      `${workspaceId}-${[...cacheKeyNames].sort().join(',')}` as const;

    const result = await this.memoizer.memoizePromiseAndExecute(
      memoKey,
      async () => {
        const { freshKeys, staleKeys } = this.checkLocalTTL(
          workspaceId,
          cacheKeyNames,
        );
        const freshEntries = this.getFromLocalCache(workspaceId, freshKeys);

        if (staleKeys.length === 0) {
          return freshEntries;
        }

        const { validKeys, keysNeedingDataFromRedis, keysNeedingRecompute } =
          await this.validateLocalHashAgainstRedisHash(workspaceId, staleKeys);
        const validatedEntries = this.getFromLocalCache(workspaceId, validKeys);

        const { redisEntries, missingInRedis } = await this.fetchDataFromRedis(
          workspaceId,
          keysNeedingDataFromRedis,
        );

        const recomputedEntries = await this.recomputeDataFromProvider({
          workspaceId,
          keysToRecompute: [...keysNeedingRecompute, ...missingInRedis],
        });

        if (recomputedEntries.hasUnpublishedEntries) {
          await this.memoizer.clearKey(memoKey);
        }

        return {
          data: {
            ...freshEntries.data,
            ...validatedEntries.data,
            ...redisEntries.data,
            ...recomputedEntries.data,
          },
          hashes: {
            ...freshEntries.hashes,
            ...validatedEntries.hashes,
            ...redisEntries.hashes,
            ...recomputedEntries.hashes,
          },
        };
      },
    );

    return result as CacheEntriesResult;
  }

  public async getOrRecomputeCombinedHash(
    workspaceId: string,
    cacheKeyNames: WorkspaceCacheOrDerivedCacheKeyName[],
  ): Promise<string> {
    this.assertValidCacheParameters(workspaceId, cacheKeyNames);

    const sourceCacheKeyNames = this.getSourceCacheKeyNames(cacheKeyNames);

    const cachedHashes = await this.getCacheHashes(
      workspaceId,
      sourceCacheKeyNames,
    );
    const missingKeys = sourceCacheKeyNames.filter(
      (cacheKeyName) => !isDefined(cachedHashes[cacheKeyName]),
    );

    const sourceHashes =
      missingKeys.length === 0
        ? cachedHashes
        : {
            ...cachedHashes,
            ...(
              await this.getOrRecomputeProviderEntries(workspaceId, missingKeys)
            ).hashes,
          };

    const cacheOrDerivedCacheHashes: Partial<
      Record<WorkspaceCacheOrDerivedCacheKeyName, string>
    > = { ...sourceHashes };

    for (const cacheKeyName of cacheKeyNames) {
      if (isWorkspaceDerivedCacheKeyName(cacheKeyName)) {
        cacheOrDerivedCacheHashes[cacheKeyName] =
          sourceHashes[
            this.getDerivedCacheProviderOrThrow(cacheKeyName).sourceKeyName
          ];
      }
    }

    return combineCacheHashes(cacheOrDerivedCacheHashes, cacheKeyNames);
  }

  private collectRowsRequirements(cacheKeyNames: WorkspaceCacheKeyName[]) {
    return cacheKeyNames.map(
      (keyName) => this.getProviderOrThrow(keyName).rowsRequirement,
    );
  }

  public async invalidateAndRecompute(
    workspaceId: string,
    cacheKeyNames: WorkspaceCacheKeyName[],
  ): Promise<void> {
    return Sentry.startSpan(
      {
        name: 'invalidate and recompute workspace metadata cache',
        op: 'cache.invalidate',
        onlyIfParent: true,
        attributes: { 'cache.key_count': cacheKeyNames.length },
      },
      async () => {
        const keysToRecompute = await this.invalidate(
          workspaceId,
          cacheKeyNames,
        );

        await this.recomputeDataFromProvider({ workspaceId, keysToRecompute });
        await this.memoizer.clearKeys(`${workspaceId}-`);
      },
    );
  }

  public async getCacheHashes(
    workspaceId: string,
    cacheKeyNames: WorkspaceCacheKeyName[],
  ): Promise<Partial<Record<WorkspaceCacheKeyName, string>>> {
    if (cacheKeyNames.length === 0) {
      return {};
    }

    const hashKeys = cacheKeyNames.map(
      (keyName) => `${this.buildCacheKey(workspaceId, keyName)}:hash`,
    );

    const hashes = await this.cacheStorage.mget<string>(hashKeys);

    const result: Partial<Record<WorkspaceCacheKeyName, string>> = {};

    for (const [index, keyName] of cacheKeyNames.entries()) {
      if (isDefined(hashes[index])) {
        result[keyName] = hashes[index];
      }
    }

    return result;
  }

  public async flush(
    workspaceId: string,
    cacheKeyNames: (
      | WorkspaceCacheOrDerivedCacheKeyName
      | RemovedWorkspaceCacheKeyName
    )[],
  ): Promise<void> {
    const storedCacheKeyNames = cacheKeyNames.filter(
      (cacheKeyName): cacheKeyName is WorkspaceCacheKeyName =>
        cacheKeyName !== 'ORMEntityMetadatas' &&
        !isWorkspaceDerivedCacheKeyName(cacheKeyName),
    );

    await this.invalidate(workspaceId, storedCacheKeyNames);
  }

  private async invalidate(
    workspaceId: string,
    cacheKeyNames: WorkspaceCacheKeyName[],
  ): Promise<KeyToRecompute[]> {
    if (cacheKeyNames.length === 0) {
      return [];
    }

    const invalidatedKeys = cacheKeyNames.map((keyName) => ({
      keyName,
      expectedHash: crypto.randomUUID(),
    }));

    await this.cacheStorage.msetAndMdel({
      entries: invalidatedKeys.map(({ keyName, expectedHash }) => ({
        key: `${this.buildCacheKey(workspaceId, keyName)}:hash`,
        value: expectedHash,
        ttl: this.twentyConfigService.get('CACHE_STORAGE_TTL') * 1000,
      })),
      keysToDelete: cacheKeyNames.map(
        (keyName) => `${this.buildCacheKey(workspaceId, keyName)}:data`,
      ),
    });
    this.deleteFromLocalCache(workspaceId, cacheKeyNames);
    await this.memoizer.clearKeys(`${workspaceId}-`);

    return invalidatedKeys;
  }

  public async evictWorkspaceFromLocalCache(
    workspaceId: string,
  ): Promise<void> {
    await this.memoizer.clearKeys(`${workspaceId}-`);

    for (const localKey of this.localCache.keys()) {
      if (localKey.endsWith(`:${workspaceId}`)) {
        this.localCache.delete(localKey);
      }
    }
  }

  private assertValidCacheParameters(
    workspaceId: string,
    cacheKeyNames: WorkspaceCacheOrDerivedCacheKeyName[],
  ): void {
    if (
      !isDefined(workspaceId) ||
      cacheKeyNames.length === 0 ||
      !isValidUuid(workspaceId)
    ) {
      throw new WorkspaceCacheException(
        'Invalid parameters: workspace ID and cache key names are required',
        WorkspaceCacheExceptionCode.INVALID_PARAMETERS,
      );
    }
  }

  private checkLocalTTL<K extends WorkspaceCacheKeyName>(
    workspaceId: string,
    cacheKeyNames: readonly K[],
  ): { freshKeys: K[]; staleKeys: K[] } {
    const freshKeys: K[] = [];
    const staleKeys: K[] = [];
    const now = Date.now();

    for (const keyName of cacheKeyNames) {
      const localKey = this.buildCacheKey(workspaceId, keyName);
      const cached = this.localCache.get(localKey);

      if (isDefined(cached) && now - cached.lastHashCheckedAt < LOCAL_TTL_MS) {
        freshKeys.push(keyName);
      } else {
        staleKeys.push(keyName);
      }
    }

    return { freshKeys, staleKeys };
  }

  private async validateLocalHashAgainstRedisHash(
    workspaceId: string,
    cacheKeyNames: WorkspaceCacheKeyName[],
  ): Promise<{
    validKeys: WorkspaceCacheKeyName[];
    keysNeedingDataFromRedis: WorkspaceCacheKeyName[];
    keysNeedingRecompute: KeyToRecompute[];
  }> {
    const validKeys: WorkspaceCacheKeyName[] = [];
    const keysNeedingDataFromRedis: WorkspaceCacheKeyName[] = [];
    const keysNeedingRecompute: KeyToRecompute[] = [];

    if (cacheKeyNames.length === 0) {
      return {
        validKeys,
        keysNeedingDataFromRedis,
        keysNeedingRecompute,
      };
    }

    const hashKeys = cacheKeyNames.map(
      (keyName) => `${this.buildCacheKey(workspaceId, keyName)}:hash`,
    );

    const redisHashes = await this.cacheStorage.mget<string>(hashKeys);

    for (const [index, keyName] of cacheKeyNames.entries()) {
      const redisHash = redisHashes[index];
      const localKey = this.buildCacheKey(workspaceId, keyName);
      const localEntry = this.localCache.get(localKey);

      if (
        isDefined(localEntry) &&
        isDefined(redisHash) &&
        localEntry.latestHash === redisHash
      ) {
        localEntry.lastHashCheckedAt = Date.now();
        validKeys.push(keyName);
      } else if (this.localDataOnlyKeys.has(keyName)) {
        keysNeedingRecompute.push({ keyName, expectedHash: redisHash });
      } else {
        keysNeedingDataFromRedis.push(keyName);
      }
    }

    return {
      validKeys,
      keysNeedingDataFromRedis,
      keysNeedingRecompute,
    };
  }

  private async fetchDataFromRedis(
    workspaceId: string,
    cacheKeyNames: WorkspaceCacheKeyName[],
  ): Promise<{
    redisEntries: CacheEntriesResult;
    missingInRedis: KeyToRecompute[];
  }> {
    const redisEntries: CacheEntriesResult = { data: {}, hashes: {} };
    const missingInRedis: KeyToRecompute[] = [];

    if (cacheKeyNames.length === 0) {
      return { redisEntries, missingInRedis };
    }

    const allKeys = cacheKeyNames.flatMap((keyName) => {
      const baseKey = this.buildCacheKey(workspaceId, keyName);

      return [`${baseKey}:data`, `${baseKey}:hash`];
    });

    const allValues = await this.cacheStorage.mget<CacheDataType | string>(
      allKeys,
    );

    for (const [index, keyName] of cacheKeyNames.entries()) {
      const rawData = allValues[index * 2] as CacheDataType | undefined;
      const hash = allValues[index * 2 + 1] as string | undefined;

      if (isDefined(rawData) && isDefined(hash)) {
        let data: CacheDataType;

        try {
          data = this.getProviderOrThrow(keyName).expandFromStorage(rawData);
        } catch (error) {
          this.logger.warn(
            `Failed to expand cached ${keyName} for workspace ${workspaceId}, recomputing`,
            error,
          );
          missingInRedis.push({ keyName, expectedHash: hash });
          continue;
        }

        Object.assign(redisEntries.data, { [keyName]: data });
        redisEntries.hashes[keyName] = hash;
        this.setInLocalCache(workspaceId, keyName, data, hash);
      } else {
        missingInRedis.push({ keyName, expectedHash: hash });
      }
    }

    return { redisEntries, missingInRedis };
  }

  private async recomputeDataFromProvider({
    workspaceId,
    keysToRecompute,
  }: {
    workspaceId: string;
    keysToRecompute: KeyToRecompute[];
  }): Promise<RecomputedCacheEntriesResult> {
    const result: RecomputedCacheEntriesResult = {
      data: {},
      hashes: {},
      hasUnpublishedEntries: false,
    };

    if (keysToRecompute.length === 0) {
      return result;
    }

    const rowsBatchLoader = new WorkspaceCacheRowsBatchLoader(
      this.coreDataSource,
      workspaceId,
    );

    await rowsBatchLoader.loadRows(
      this.collectRowsRequirements(
        keysToRecompute.map(({ keyName }) => keyName),
      ),
    );

    const computePromises = keysToRecompute.map(async (keyToRecompute) => {
      const { keyName, expectedHash } = keyToRecompute;
      const provider = this.getProviderOrThrow(keyName);
      const isLocalDataOnly = this.localDataOnlyKeys.has(keyName);
      const computeStartedAt = performance.now();

      try {
        const data = await Sentry.startSpan(
          {
            name: 'compute workspace metadata cache entry from provider',
            op: 'cache.recompute',
            onlyIfParent: true,
            attributes: {
              'cache.key_name': keyName,
              'cache.local_data_only': isLocalDataOnly,
            },
          },
          () =>
            provider.computeForCache({
              workspaceId,
              rows: rowsBatchLoader.readRows(provider.rowsRequirement),
            }),
        );

        return {
          keyName,
          data,
          expectedHash,
          hashToPublish: expectedHash ?? crypto.randomUUID(),
        };
      } finally {
        this.cacheMetricsService.recordRecompute(
          (performance.now() - computeStartedAt) / 1000,
          keyName,
        );
      }
    });

    const settledComputes = await Promise.allSettled(computePromises);
    const { computed, computeFailures } = settledComputes.reduce<{
      computed: Awaited<(typeof computePromises)[number]>[];
      computeFailures: { keyName: WorkspaceCacheKeyName; reason: unknown }[];
    }>(
      (acc, settled, index) => {
        if (settled.status === 'fulfilled') {
          acc.computed.push(settled.value);
        } else {
          acc.computeFailures.push({
            keyName: keysToRecompute[index].keyName,
            reason: settled.reason,
          });
        }

        return acc;
      },
      { computed: [], computeFailures: [] },
    );

    if (computed.length > 0) {
      const redisWriteStartedAt = performance.now();
      const cacheTtlMs =
        this.twentyConfigService.get('CACHE_STORAGE_TTL') * 1000;
      const publishArgumentsByEntry = computed.map(
        ({ keyName, data, expectedHash, hashToPublish }) => {
          const serializedExpectedHash = isDefined(expectedHash)
            ? JSON.stringify(expectedHash)
            : '';
          const serializedHashToPublish = JSON.stringify(hashToPublish);
          const serializedData = this.localDataOnlyKeys.has(keyName)
            ? ''
            : JSON.stringify(
                this.getProviderOrThrow(keyName).compactForStorage(data),
              );

          return [
            serializedExpectedHash,
            serializedHashToPublish,
            serializedData,
          ];
        },
      );
      let published: number[];

      try {
        published = await this.cacheStorage.runScript<number[]>({
          script: PUBLISH_WORKSPACE_CACHE_SCRIPT,
          keys: this.buildCacheEntryKeys(
            workspaceId,
            computed.map(({ keyName }) => keyName),
          ),
          args: [String(cacheTtlMs), ...publishArgumentsByEntry.flat()],
        });
      } finally {
        this.cacheMetricsService.recordRedisWrite(
          (performance.now() - redisWriteStartedAt) / 1000,
        );
      }

      for (const [
        index,
        { keyName, data, hashToPublish },
      ] of computed.entries()) {
        Object.assign(result.data, { [keyName]: data });
        result.hashes[keyName] = hashToPublish;

        if (published[index] === 1) {
          this.setInLocalCache(workspaceId, keyName, data, hashToPublish);
        } else {
          result.hasUnpublishedEntries = true;
        }
      }
    }

    if (computeFailures.length > 0) {
      computeFailures.forEach(({ keyName, reason }) =>
        this.logger.error(
          `Failed to compute cache key '${keyName}': ${reason}`,
        ),
      );
      throw computeFailures[0].reason;
    }

    return result;
  }

  private getFromLocalCache(
    workspaceId: string,
    workspaceCacheKeyNames: WorkspaceCacheKeyName[],
  ): CacheEntriesResult {
    const result: CacheEntriesResult = { data: {}, hashes: {} };

    for (const keyName of workspaceCacheKeyNames) {
      const localKey = this.buildCacheKey(workspaceId, keyName);
      const entry = this.localCache.get(localKey);
      const version = entry?.versions.get(entry.latestHash);

      if (isDefined(entry) && isDefined(version)) {
        const data = this.readVersion({
          keyName,
          entry,
          hash: entry.latestHash,
          version,
        });

        Object.assign(result.data, { [keyName]: data });
        result.hashes[keyName] = entry.latestHash;
        this.cleanupStaleVersions(entry);
      }
    }

    return result;
  }

  private deleteFromLocalCache(
    workspaceId: string,
    cacheKeyNames: WorkspaceCacheKeyName[],
  ): void {
    for (const keyName of cacheKeyNames) {
      const localKey = this.buildCacheKey(workspaceId, keyName);
      const entry = this.localCache.get(localKey);

      if (isDefined(entry)) {
        entry.lastHashCheckedAt = 0;
      }
    }
  }

  private buildCacheEntryKeys(
    workspaceId: string,
    cacheKeyNames: WorkspaceCacheKeyName[],
  ): string[] {
    return cacheKeyNames.flatMap((keyName) => {
      const baseKey = this.buildCacheKey(workspaceId, keyName);

      return [`${baseKey}:hash`, `${baseKey}:data`];
    });
  }

  private setInLocalCache(
    workspaceId: string,
    keyName: WorkspaceCacheKeyName,
    data: CacheDataType,
    hash: string,
  ): void {
    const localKey = this.buildCacheKey(workspaceId, keyName);
    let entry = this.localCache.get(localKey);

    if (!isDefined(entry)) {
      entry = { versions: new Map(), latestHash: '', lastHashCheckedAt: 0 };
      this.localCache.set(localKey, entry);
    }

    entry.versions.set(hash, { state: 'live', data, lastReadAt: Date.now() });
    entry.latestHash = hash;
    entry.lastHashCheckedAt = Date.now();

    this.cleanupStaleVersions(entry);
  }

  private sweepLocalCache(): void {
    const now = Date.now();

    const evicted = sweepLocalCache(this.localCache, now, {
      ttlMs: LOCAL_ENTRY_TTL_MS,
      maxEntriesByKeyName: MAX_LOCAL_ENTRIES_BY_KEY_NAME,
      globalMaxEntries: MAX_LOCAL_CACHE_ENTRIES,
      minEvict: MIN_EVICT_KEYS,
    });

    if (evicted > 0) {
      this.cacheMetricsService.recordEviction(evicted);
    }
  }

  private runPacking(): void {
    const startedAt = performance.now();

    const { packed, pending } = packIdleVersions({
      localCache: this.localCache,
      minIdleMs: MIN_IDLE_BEFORE_PACKING_MS,
      ponderationBudget: PACKING_PONDERATION_BUDGET,
      ponderationOf: (localKey) =>
        this.packingPonderationByKey.get(
          getKeyNameFromLocalCacheKey(localKey) as WorkspaceCacheKeyName,
        )!,
      isPackable: (localKey) =>
        !this.localDataOnlyKeys.has(
          getKeyNameFromLocalCacheKey(localKey) as WorkspaceCacheKeyName,
        ),
      pack: ({ localKey, data }) => {
        const keyName = getKeyNameFromLocalCacheKey(
          localKey,
        ) as WorkspaceCacheKeyName;

        return serializeCacheBlob(
          this.getProviderOrThrow(keyName).compactForStorage(data),
        );
      },
    });

    this.cacheMetricsService.recordPackingRun({
      durationSeconds: (performance.now() - startedAt) / 1000,
      packed,
      pending,
    });
  }

  private readVersion({
    keyName,
    entry,
    hash,
    version,
  }: {
    keyName: WorkspaceCacheKeyName;
    entry: WorkspaceLocalCacheEntry<CacheDataType>;
    hash: string;
    version: VersionEntry<CacheDataType>;
  }): CacheDataType {
    if (version.state === 'live') {
      version.lastReadAt = Date.now();

      return version.data;
    }

    const unpackStartedAt = performance.now();
    const data = this.getProviderOrThrow(keyName).expandFromStorage(
      deserializeCacheBlob(version.blob),
    );

    entry.versions.set(hash, {
      state: 'live',
      data,
      lastReadAt: Date.now(),
    });

    this.cacheMetricsService.recordUnpacking(
      (performance.now() - unpackStartedAt) / 1000,
      keyName,
    );

    return data;
  }

  private cleanupStaleVersions(
    entry: WorkspaceLocalCacheEntry<CacheDataType>,
  ): void {
    const now = Date.now();

    for (const [hash, version] of entry.versions) {
      if (
        hash !== entry.latestHash &&
        now - version.lastReadAt > STALE_VERSION_TTL_MS
      ) {
        entry.versions.delete(hash);
      }
    }

    if (entry.versions.size >= MAX_LOCAL_STALE_VERSIONS) {
      const sorted = [...entry.versions.entries()]
        .filter(([hash]) => hash !== entry.latestHash)
        .sort((entryA, entryB) => entryA[1].lastReadAt - entryB[1].lastReadAt);

      while (
        entry.versions.size >= MAX_LOCAL_STALE_VERSIONS &&
        sorted.length > 0
      ) {
        const oldestEntry = sorted.shift();

        if (isDefined(oldestEntry)) {
          entry.versions.delete(oldestEntry[0]);
        }
      }
    }
  }

  private getProviderOrThrow(
    keyName: WorkspaceCacheKeyName,
  ): WorkspaceCacheProvider<CacheDataType, StoredCacheDataType> {
    const provider = this.workspaceCacheProviders.get(keyName);

    if (!isDefined(provider)) {
      throw new Error(`Cache provider with key name "${keyName}" not found`);
    }

    return provider;
  }

  private buildCacheKey(
    workspaceId: string,
    keyName: WorkspaceCacheKeyName,
  ): string {
    return `${keyName}:${workspaceId}`;
  }
}