The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Split metadata loader and filter responsibilities (#23954)
Félix Malfaitda02b26ef9b6
diff --git a/packages/twenty-server/src/engine/dataloaders/__tests__/dataloader.service.spec.ts b/packages/twenty-server/src/engine/dataloaders/__tests__/dataloader.service.spec.tsnew file mode 100644index 00000000000..ca642b87b57--- /dev/null+++ b/packages/twenty-server/src/engine/dataloaders/__tests__/dataloader.service.spec.ts@@ -0,0 +1,43 @@+import { DataloaderService } from 'src/engine/dataloaders/dataloader.service';+import { type FieldMetadataConnectionLoaderFactory } from 'src/engine/dataloaders/factories/field-metadata-connection-loader.factory';+import { type IndexMetadataConnectionLoaderFactory } from 'src/engine/dataloaders/factories/index-metadata-connection-loader.factory';++describe('DataloaderService', () => {+ it('delegates metadata connection loader creation to domain factories', () => {+ const fieldMetadataConnectionLoader = {} as ReturnType<+ FieldMetadataConnectionLoaderFactory['create']+ >;+ const indexMetadataConnectionLoader = {} as ReturnType<+ IndexMetadataConnectionLoaderFactory['create']+ >;+ const fieldMetadataConnectionLoaderFactory = {+ create: jest.fn(() => fieldMetadataConnectionLoader),+ } as unknown as FieldMetadataConnectionLoaderFactory;+ const indexMetadataConnectionLoaderFactory = {+ create: jest.fn(() => indexMetadataConnectionLoader),+ } as unknown as IndexMetadataConnectionLoaderFactory;+ const service = new DataloaderService(+ {} as never,+ {} as never,+ {} as never,+ {} as never,+ fieldMetadataConnectionLoaderFactory,+ indexMetadataConnectionLoaderFactory,+ );++ const loaders = service.createLoaders();++ expect(fieldMetadataConnectionLoaderFactory.create).toHaveBeenCalledTimes(+ 1,+ );+ expect(indexMetadataConnectionLoaderFactory.create).toHaveBeenCalledTimes(+ 1,+ );+ expect(loaders.fieldMetadataConnectionLoader).toBe(+ fieldMetadataConnectionLoader,+ );+ expect(loaders.indexMetadataConnectionLoader).toBe(+ indexMetadataConnectionLoader,+ );+ });+});diff --git a/packages/twenty-server/src/engine/dataloaders/dataloader.interface.ts b/packages/twenty-server/src/engine/dataloaders/dataloader.interface.tsindex 6b8fd3258e3..9f54b5c6c2c 100644--- a/packages/twenty-server/src/engine/dataloaders/dataloader.interface.ts+++ b/packages/twenty-server/src/engine/dataloaders/dataloader.interface.ts@@ -1,12 +1,12 @@import type DataLoader from 'dataloader';+import { type FieldMetadataConnectionLoaderPayload } from 'src/engine/dataloaders/factories/field-metadata-connection-loader.factory';+import { type IndexMetadataConnectionLoaderPayload } from 'src/engine/dataloaders/factories/index-metadata-connection-loader.factory';import {type ApplicationTranslationCatalogLoaderPayload,type FieldMetadataLoaderPayload,- type FieldMetadataConnectionLoaderPayload,type IndexFieldMetadataLoaderPayload,type IndexMetadataLoaderPayload,- type IndexMetadataConnectionLoaderPayload,type SearchFieldMetadataLoaderPayload,type IsConfiguredLoaderPayload,type MorphRelationLoaderPayload,diff --git a/packages/twenty-server/src/engine/dataloaders/dataloader.module.ts b/packages/twenty-server/src/engine/dataloaders/dataloader.module.tsindex 79db79368c5..5692e0ff50b 100644--- a/packages/twenty-server/src/engine/dataloaders/dataloader.module.ts+++ b/packages/twenty-server/src/engine/dataloaders/dataloader.module.ts@@ -3,6 +3,8 @@ import { Module } from '@nestjs/common';import { ApplicationRegistrationVariableModule } from 'src/engine/core-modules/application/application-registration-variable/application-registration-variable.module';import { ApplicationTranslationModule } from 'src/engine/core-modules/application/application-translation/application-translation.module';import { DataloaderService } from 'src/engine/dataloaders/dataloader.service';+import { FieldMetadataConnectionLoaderFactory } from 'src/engine/dataloaders/factories/field-metadata-connection-loader.factory';+import { IndexMetadataConnectionLoaderFactory } from 'src/engine/dataloaders/factories/index-metadata-connection-loader.factory';import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';@@ -13,7 +15,11 @@ import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadataApplicationRegistrationVariableModule,ApplicationTranslationModule,],- providers: [DataloaderService],+ providers: [+ DataloaderService,+ FieldMetadataConnectionLoaderFactory,+ IndexMetadataConnectionLoaderFactory,+ ],exports: [DataloaderService],})export class DataloaderModule {}diff --git a/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts b/packages/twenty-server/src/engine/dataloaders/dataloader.service.tsindex 0feb04da0ad..70a59f26b32 100644--- a/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts+++ b/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts@@ -9,14 +9,12 @@ import { type IndexMetadataInterface } from 'src/engine/metadata-modules/index-mimport { ApplicationRegistrationVariableService } from 'src/engine/core-modules/application/application-registration-variable/application-registration-variable.service';import { ApplicationTranslationCacheService } from 'src/engine/core-modules/application/application-translation/application-translation-cache.service';-import { type FlatApplicationCacheMaps } from 'src/engine/core-modules/application/types/flat-application-cache-maps.type';import { I18nService } from 'src/engine/core-modules/i18n/i18n.service';import { type IDataloaders } from 'src/engine/dataloaders/dataloader.interface';+import { FieldMetadataConnectionLoaderFactory } from 'src/engine/dataloaders/factories/field-metadata-connection-loader.factory';+import { IndexMetadataConnectionLoaderFactory } from 'src/engine/dataloaders/factories/index-metadata-connection-loader.factory';import { filterMorphRelationDuplicateFields } from 'src/engine/dataloaders/utils/filter-morph-relation-duplicate-fields.util';-import {- FIELD_FILTER_COLUMN_BY_FILTER_FIELD,- type FieldFilterInput,-} from 'src/engine/metadata-modules/field-metadata/dtos/field-filter.input';+import { loadApplicationCatalogsByRegistrationId } from 'src/engine/dataloaders/utils/load-application-catalogs-by-registration-id.util';import { type FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto';import { RelationDTO } from 'src/engine/metadata-modules/field-metadata/dtos/relation.dto';import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';@@ -43,17 +41,9 @@ import { fromFlatViewFilterGroupToViewFilterGroupDto } from 'src/engine/metadataimport { fromFlatViewGroupToViewGroupDto } from 'src/engine/metadata-modules/view-group/utils/from-flat-view-group-to-view-group-dto.util';import { fromFlatViewSortToViewSortDto } from 'src/engine/metadata-modules/view-sort/utils/from-flat-view-sort-to-view-sort-dto.util';import { type IndexFieldMetadataDTO } from 'src/engine/metadata-modules/index-metadata/dtos/index-field-metadata.dto';-import {- INDEX_FILTER_COLUMN_BY_FILTER_FIELD,- type IndexFilterInput,-} from 'src/engine/metadata-modules/index-metadata/dtos/index-filter.input';import { type IndexMetadataDTO } from 'src/engine/metadata-modules/index-metadata/dtos/index-metadata.dto';import { ObjectMetadataDTO } from 'src/engine/metadata-modules/object-metadata/dtos/object-metadata.dto';import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';-import { type CursorConnection } from 'src/engine/metadata-modules/pagination/dtos/cursor-connection-type.factory';-import { type CursorPagingInput } from 'src/engine/metadata-modules/pagination/dtos/cursor-paging.input';-import { applyMetadataFilterToItems } from 'src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-query-builder.util';-import { findManyItemsWithCursorPagination } from 'src/engine/metadata-modules/pagination/utils/find-many-items-with-cursor-pagination.util';import { type SearchFieldMetadataDTO } from 'src/engine/metadata-modules/search-field-metadata/dtos/search-field-metadata.dto';export type RelationMetadataLoaderPayload = {@@ -79,23 +69,11 @@ export type FieldMetadataLoaderPayload = {locale?: keyof typeof APP_LOCALES;};-export type FieldMetadataConnectionLoaderPayload =- FieldMetadataLoaderPayload & {- filter: FieldFilterInput;- paging: CursorPagingInput;- };-export type IndexMetadataLoaderPayload = {workspaceId: string;objectMetadata: Pick<ObjectMetadataEntity, 'id'>;};-export type IndexMetadataConnectionLoaderPayload =- IndexMetadataLoaderPayload & {- filter: IndexFilterInput;- paging: CursorPagingInput;- };-export type IndexFieldMetadataLoaderPayload = {workspaceId: string;objectMetadata: Pick<ObjectMetadataEntity, 'id'>;@@ -168,6 +146,8 @@ export class DataloaderService {private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,private readonly applicationRegistrationVariableService: ApplicationRegistrationVariableService,private readonly applicationTranslationCacheService: ApplicationTranslationCacheService,+ private readonly fieldMetadataConnectionLoaderFactory: FieldMetadataConnectionLoaderFactory,+ private readonly indexMetadataConnectionLoaderFactory: IndexMetadataConnectionLoaderFactory,) {}createLoaders(): IDataloaders {@@ -175,10 +155,10 @@ export class DataloaderService {const morphRelationLoader = this.createMorphRelationLoader();const fieldMetadataLoader = this.createFieldMetadataLoader();const fieldMetadataConnectionLoader =- this.createFieldMetadataConnectionLoader();+ this.fieldMetadataConnectionLoaderFactory.create();const indexMetadataLoader = this.createIndexMetadataLoader();const indexMetadataConnectionLoader =- this.createIndexMetadataConnectionLoader();+ this.indexMetadataConnectionLoaderFactory.create();const indexFieldMetadataLoader = this.createIndexFieldMetadataLoader();const searchFieldMetadataLoader = this.createSearchFieldMetadataLoader();const objectMetadataLoader = this.createObjectMetadataLoader();@@ -345,59 +325,6 @@ export class DataloaderService {);}- private createIndexMetadataConnectionLoader() {- return new DataLoader<- IndexMetadataConnectionLoaderPayload,- CursorConnection<IndexMetadataDTO>- >(async (dataLoaderParams: IndexMetadataConnectionLoaderPayload[]) => {- const workspaceId = dataLoaderParams[0].workspaceId;- const { flatIndexMaps, flatObjectMetadataMaps } =- await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(- {- workspaceId,- flatMapsKeys: ['flatIndexMaps', 'flatObjectMetadataMaps'],- },- );-- return dataLoaderParams.map(({ objectMetadata, paging, filter }) => {- const flatObjectMetadata = findFlatEntityByIdInFlatEntityMapsOrThrow({- flatEntityId: objectMetadata.id,- flatEntityMaps: flatObjectMetadataMaps,- });- const flatIndexMetadatas =- findManyFlatEntityByIdInFlatEntityMapsOrThrow({- flatEntityIds: flatObjectMetadata.indexMetadataIds,- flatEntityMaps: flatIndexMaps,- });- const filteredFlatIndexMetadatas = applyMetadataFilterToItems({- items: flatIndexMetadatas,- filter,- columnByFilterField: INDEX_FILTER_COLUMN_BY_FILTER_FIELD,- });- const connection = findManyItemsWithCursorPagination({- items: filteredFlatIndexMetadatas,- paging,- });-- return {- ...connection,- edges: connection.edges.map((edge) => ({- ...edge,- node: {- ...edge.node,- indexFieldMetadatas: edge.node.flatIndexFieldMetadatas,- createdAt: new Date(edge.node.createdAt),- updatedAt: new Date(edge.node.updatedAt),- indexWhereClause: edge.node.indexWhereClause ?? undefined,- objectMetadataId: objectMetadata.id,- workspaceId,- },- })),- };- });- });- }-private createSearchFieldMetadataLoader() {return new DataLoader<SearchFieldMetadataLoaderPayload,@@ -481,12 +408,14 @@ export class DataloaderService {);const applicationCatalogByRegistrationId =- await this.loadApplicationCatalogByRegistrationId({+ await loadApplicationCatalogsByRegistrationId({applicationIds: objectFlatFieldMetadatasList.flat().map((flatFieldMetadata) => flatFieldMetadata.applicationId),flatApplicationMaps,locale: safeLocale,+ applicationTranslationCacheService:+ this.applicationTranslationCacheService,});const fieldMetadataCollection = objectFlatFieldMetadatasList.map(@@ -569,133 +498,6 @@ export class DataloaderService {);}- private createFieldMetadataConnectionLoader() {- return new DataLoader<- FieldMetadataConnectionLoaderPayload,- CursorConnection<FieldMetadataDTO>- >(async (dataLoaderParams: FieldMetadataConnectionLoaderPayload[]) => {- const locale = dataLoaderParams[0].locale;- const safeLocale = locale ?? SOURCE_LOCALE;- const i18nInstance = this.i18nService.getI18nInstance(safeLocale);- const workspaceId = dataLoaderParams[0].workspaceId;- const {- flatFieldMetadataMaps,- flatObjectMetadataMaps,- flatApplicationMaps,- } =- await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(- {- workspaceId,- flatMapsKeys: [- 'flatFieldMetadataMaps',- 'flatObjectMetadataMaps',- 'flatApplicationMaps',- ],- },- );-- const connections = dataLoaderParams.map(- ({ objectMetadata, paging, filter }) => {- const flatObjectMetadata = findFlatEntityByIdInFlatEntityMapsOrThrow({- flatEntityId: objectMetadata.id,- flatEntityMaps: flatObjectMetadataMaps,- });- const flatFieldMetadatas =- findManyFlatEntityByIdInFlatEntityMapsOrThrow({- flatEntityIds: flatObjectMetadata.fieldIds,- flatEntityMaps: flatFieldMetadataMaps,- });- const filteredFlatFieldMetadatas = applyMetadataFilterToItems({- items: filterMorphRelationDuplicateFields(flatFieldMetadatas),- filter,- columnByFilterField: FIELD_FILTER_COLUMN_BY_FILTER_FIELD,- });-- return findManyItemsWithCursorPagination({- items: filteredFlatFieldMetadatas,- paging,- });- },- );- const selectedFlatFieldMetadatas = connections.flatMap((connection) =>- connection.edges.map(({ node }) => node),- );- const applicationCatalogByRegistrationId =- await this.loadApplicationCatalogByRegistrationId({- applicationIds: selectedFlatFieldMetadatas.map(- (flatFieldMetadata) => flatFieldMetadata.applicationId,- ),- flatApplicationMaps,- locale: safeLocale,- });-- return connections.map((connection) => ({- ...connection,- edges: connection.edges.map((edge) => {- const flatFieldMetadata = edge.node;- const applicationRegistrationId =- flatApplicationMaps.byId[flatFieldMetadata.applicationId]- ?.applicationRegistrationId;- const applicationCatalog = isDefined(applicationRegistrationId)- ? applicationCatalogByRegistrationId.get(applicationRegistrationId)- : undefined;- const overrides = flatFieldMetadata.overrides ?? undefined;- const i18nContext = {- locale,- i18nInstance,- isStandardApp: belongsToTwentyStandardApp(flatFieldMetadata),- applicationCatalog,- };- const overriddenFlatFieldMetadata =- ALL_OVERRIDABLE_PROPERTIES_BY_METADATA_NAME.fieldMetadata.reduce(- (acc, property) => ({- ...acc,- [property]: resolveEffectiveEntityProperty({- metadataName: 'fieldMetadata',- baseValue: flatFieldMetadata[property],- overrides,- property,- i18nContext,- }),- }),- flatFieldMetadata,- );- let renamedFlatFieldMetadata = overriddenFlatFieldMetadata;-- if (- isFlatFieldMetadataOfType(- overriddenFlatFieldMetadata,- FieldMetadataType.MORPH_RELATION,- )- ) {- const relationTargetObjectMetadata =- findFlatEntityByIdInFlatEntityMapsOrThrow({- flatEntityId:- overriddenFlatFieldMetadata.relationTargetObjectMetadataId,- flatEntityMaps: flatObjectMetadataMaps,- });-- renamedFlatFieldMetadata = {- ...overriddenFlatFieldMetadata,- name: getMorphNameFromMorphFieldMetadataName({- morphRelationFlatFieldMetadata: overriddenFlatFieldMetadata,- nameSingular: relationTargetObjectMetadata.nameSingular,- namePlural: relationTargetObjectMetadata.namePlural,- }),- };- }-- return {- ...edge,- node: fromFlatFieldMetadataToFieldMetadataDto(- renamedFlatFieldMetadata,- ),- };- }),- }));- });- }-private createIndexFieldMetadataLoader() {return new DataLoader<IndexFieldMetadataLoaderPayload,@@ -1112,10 +914,12 @@ export class DataloaderService {getTwentyStandardApplicationIdOrThrow(flatApplicationMaps);const catalogByRegistrationId =- await this.loadApplicationCatalogByRegistrationId({+ await loadApplicationCatalogsByRegistrationId({applicationIds: params.map((param) => param.applicationId),flatApplicationMaps,locale,+ applicationTranslationCacheService:+ this.applicationTranslationCacheService,});return params.map((param) => {@@ -1133,42 +937,4 @@ export class DataloaderService {});});}-- private async loadApplicationCatalogByRegistrationId({- applicationIds,- flatApplicationMaps,- locale,- }: {- applicationIds: string[];- flatApplicationMaps: FlatApplicationCacheMaps;- locale: keyof typeof APP_LOCALES;- }): Promise<Map<string, Record<string, string>>> {- const registrationIds = [- ...new Set(- applicationIds- .map(- (applicationId) =>- flatApplicationMaps.byId[applicationId]- ?.applicationRegistrationId,- )- .filter(isDefined),- ),- ];-- const catalogByRegistrationId = new Map<string, Record<string, string>>();-- await Promise.all(- registrationIds.map(async (applicationRegistrationId) => {- const catalog =- await this.applicationTranslationCacheService.getCatalog({- applicationRegistrationId,- locale,- });-- catalogByRegistrationId.set(applicationRegistrationId, catalog);- }),- );-- return catalogByRegistrationId;- }}diff --git a/packages/twenty-server/src/engine/dataloaders/factories/field-metadata-connection-loader.factory.ts b/packages/twenty-server/src/engine/dataloaders/factories/field-metadata-connection-loader.factory.tsnew file mode 100644index 00000000000..c4fc7c39edb--- /dev/null+++ b/packages/twenty-server/src/engine/dataloaders/factories/field-metadata-connection-loader.factory.ts@@ -0,0 +1,179 @@+import { Injectable } from '@nestjs/common';++import DataLoader from 'dataloader';+import { type APP_LOCALES, SOURCE_LOCALE } from 'twenty-shared/translations';+import { FieldMetadataType } from 'twenty-shared/types';+import { isDefined } from 'twenty-shared/utils';++import { ApplicationTranslationCacheService } from 'src/engine/core-modules/application/application-translation/application-translation-cache.service';+import { I18nService } from 'src/engine/core-modules/i18n/i18n.service';+import { loadApplicationCatalogsByRegistrationId } from 'src/engine/dataloaders/utils/load-application-catalogs-by-registration-id.util';+import {+ FIELD_FILTER_COLUMN_BY_FILTER_FIELD,+ type FieldFilterInput,+} from 'src/engine/metadata-modules/field-metadata/dtos/field-filter.input';+import { type FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto';+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';+import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps-or-throw.util';+import { findManyFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-many-flat-entity-by-id-in-flat-entity-maps-or-throw.util';+import { ALL_OVERRIDABLE_PROPERTIES_BY_METADATA_NAME } from 'src/engine/metadata-modules/flat-entity/constant/all-overridable-properties-by-metadata-name.constant';+import { fromFlatFieldMetadataToFieldMetadataDto } from 'src/engine/metadata-modules/flat-field-metadata/utils/from-flat-field-metadata-to-field-metadata-dto.util';+import { isFlatFieldMetadataOfType } from 'src/engine/metadata-modules/flat-field-metadata/utils/is-flat-field-metadata-of-type.util';+import { belongsToTwentyStandardApp } from 'src/engine/metadata-modules/utils/belongs-to-twenty-standard-app.util';+import { resolveEffectiveEntityProperty } from 'src/engine/metadata-modules/utils/resolve-effective-entity-property.util';+import { getMorphNameFromMorphFieldMetadataName } from 'src/engine/metadata-modules/flat-object-metadata/utils/get-morph-name-from-morph-field-metadata-name.util';+import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';+import { type CursorConnection } from 'src/engine/metadata-modules/pagination/dtos/cursor-connection-type.factory';+import { type CursorPagingInput } from 'src/engine/metadata-modules/pagination/dtos/cursor-paging.input';+import { applyMetadataFilterToItems } from 'src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-items.util';+import { findManyItemsWithCursorPagination } from 'src/engine/metadata-modules/pagination/utils/find-many-items-with-cursor-pagination.util';+import { filterMorphRelationDuplicateFields } from 'src/engine/dataloaders/utils/filter-morph-relation-duplicate-fields.util';++export type FieldMetadataConnectionLoaderPayload = {+ workspaceId: string;+ objectMetadata: Pick<ObjectMetadataEntity, 'id'>;+ locale?: keyof typeof APP_LOCALES;+ filter: FieldFilterInput;+ paging: CursorPagingInput;+};++@Injectable()+export class FieldMetadataConnectionLoaderFactory {+ constructor(+ private readonly i18nService: I18nService,+ private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,+ private readonly applicationTranslationCacheService: ApplicationTranslationCacheService,+ ) {}++ create(): DataLoader<+ FieldMetadataConnectionLoaderPayload,+ CursorConnection<FieldMetadataDTO>+ > {+ return new DataLoader<+ FieldMetadataConnectionLoaderPayload,+ CursorConnection<FieldMetadataDTO>+ >(async (dataLoaderParams: FieldMetadataConnectionLoaderPayload[]) => {+ const locale = dataLoaderParams[0].locale;+ const safeLocale = locale ?? SOURCE_LOCALE;+ const i18nInstance = this.i18nService.getI18nInstance(safeLocale);+ const workspaceId = dataLoaderParams[0].workspaceId;+ const {+ flatFieldMetadataMaps,+ flatObjectMetadataMaps,+ flatApplicationMaps,+ } =+ await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(+ {+ workspaceId,+ flatMapsKeys: [+ 'flatFieldMetadataMaps',+ 'flatObjectMetadataMaps',+ 'flatApplicationMaps',+ ],+ },+ );++ const connections = dataLoaderParams.map(+ ({ objectMetadata, paging, filter }) => {+ const flatObjectMetadata = findFlatEntityByIdInFlatEntityMapsOrThrow({+ flatEntityId: objectMetadata.id,+ flatEntityMaps: flatObjectMetadataMaps,+ });+ const flatFieldMetadatas =+ findManyFlatEntityByIdInFlatEntityMapsOrThrow({+ flatEntityIds: flatObjectMetadata.fieldIds,+ flatEntityMaps: flatFieldMetadataMaps,+ });+ const filteredFlatFieldMetadatas = applyMetadataFilterToItems({+ items: filterMorphRelationDuplicateFields(flatFieldMetadatas),+ filter,+ columnByFilterField: FIELD_FILTER_COLUMN_BY_FILTER_FIELD,+ });++ return findManyItemsWithCursorPagination({+ items: filteredFlatFieldMetadatas,+ paging,+ });+ },+ );+ const selectedFlatFieldMetadatas = connections.flatMap((connection) =>+ connection.edges.map(({ node }) => node),+ );+ const applicationCatalogByRegistrationId =+ await loadApplicationCatalogsByRegistrationId({+ applicationIds: selectedFlatFieldMetadatas.map(+ (flatFieldMetadata) => flatFieldMetadata.applicationId,+ ),+ flatApplicationMaps,+ locale: safeLocale,+ applicationTranslationCacheService:+ this.applicationTranslationCacheService,+ });++ return connections.map((connection) => ({+ ...connection,+ edges: connection.edges.map((edge) => {+ const flatFieldMetadata = edge.node;+ const applicationRegistrationId =+ flatApplicationMaps.byId[flatFieldMetadata.applicationId]+ ?.applicationRegistrationId;+ const applicationCatalog = isDefined(applicationRegistrationId)+ ? applicationCatalogByRegistrationId.get(applicationRegistrationId)+ : undefined;+ const overrides = flatFieldMetadata.overrides ?? undefined;+ const i18nContext = {+ locale,+ i18nInstance,+ isStandardApp: belongsToTwentyStandardApp(flatFieldMetadata),+ applicationCatalog,+ };+ const overriddenFlatFieldMetadata =+ ALL_OVERRIDABLE_PROPERTIES_BY_METADATA_NAME.fieldMetadata.reduce(+ (acc, property) => ({+ ...acc,+ [property]: resolveEffectiveEntityProperty({+ metadataName: 'fieldMetadata',+ baseValue: flatFieldMetadata[property],+ overrides,+ property,+ i18nContext,+ }),+ }),+ flatFieldMetadata,+ );+ let renamedFlatFieldMetadata = overriddenFlatFieldMetadata;++ if (+ isFlatFieldMetadataOfType(+ overriddenFlatFieldMetadata,+ FieldMetadataType.MORPH_RELATION,+ )+ ) {+ const relationTargetObjectMetadata =+ findFlatEntityByIdInFlatEntityMapsOrThrow({+ flatEntityId:+ overriddenFlatFieldMetadata.relationTargetObjectMetadataId,+ flatEntityMaps: flatObjectMetadataMaps,+ });++ renamedFlatFieldMetadata = {+ ...overriddenFlatFieldMetadata,+ name: getMorphNameFromMorphFieldMetadataName({+ morphRelationFlatFieldMetadata: overriddenFlatFieldMetadata,+ nameSingular: relationTargetObjectMetadata.nameSingular,+ namePlural: relationTargetObjectMetadata.namePlural,+ }),+ };+ }++ return {+ ...edge,+ node: fromFlatFieldMetadataToFieldMetadataDto(+ renamedFlatFieldMetadata,+ ),+ };+ }),+ }));+ });+ }+}diff --git a/packages/twenty-server/src/engine/dataloaders/factories/index-metadata-connection-loader.factory.ts b/packages/twenty-server/src/engine/dataloaders/factories/index-metadata-connection-loader.factory.tsnew file mode 100644index 00000000000..b502a5caa28--- /dev/null+++ b/packages/twenty-server/src/engine/dataloaders/factories/index-metadata-connection-loader.factory.ts@@ -0,0 +1,87 @@+import { Injectable } from '@nestjs/common';++import DataLoader from 'dataloader';++import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';+import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps-or-throw.util';+import { findManyFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-many-flat-entity-by-id-in-flat-entity-maps-or-throw.util';+import {+ INDEX_FILTER_COLUMN_BY_FILTER_FIELD,+ type IndexFilterInput,+} from 'src/engine/metadata-modules/index-metadata/dtos/index-filter.input';+import { type IndexMetadataDTO } from 'src/engine/metadata-modules/index-metadata/dtos/index-metadata.dto';+import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';+import { type CursorConnection } from 'src/engine/metadata-modules/pagination/dtos/cursor-connection-type.factory';+import { type CursorPagingInput } from 'src/engine/metadata-modules/pagination/dtos/cursor-paging.input';+import { applyMetadataFilterToItems } from 'src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-items.util';+import { findManyItemsWithCursorPagination } from 'src/engine/metadata-modules/pagination/utils/find-many-items-with-cursor-pagination.util';++export type IndexMetadataConnectionLoaderPayload = {+ workspaceId: string;+ objectMetadata: Pick<ObjectMetadataEntity, 'id'>;+ filter: IndexFilterInput;+ paging: CursorPagingInput;+};++@Injectable()+export class IndexMetadataConnectionLoaderFactory {+ constructor(+ private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,+ ) {}++ create(): DataLoader<+ IndexMetadataConnectionLoaderPayload,+ CursorConnection<IndexMetadataDTO>+ > {+ return new DataLoader<+ IndexMetadataConnectionLoaderPayload,+ CursorConnection<IndexMetadataDTO>+ >(async (dataLoaderParams: IndexMetadataConnectionLoaderPayload[]) => {+ const workspaceId = dataLoaderParams[0].workspaceId;+ const { flatIndexMaps, flatObjectMetadataMaps } =+ await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(+ {+ workspaceId,+ flatMapsKeys: ['flatIndexMaps', 'flatObjectMetadataMaps'],+ },+ );++ return dataLoaderParams.map(({ objectMetadata, paging, filter }) => {+ const flatObjectMetadata = findFlatEntityByIdInFlatEntityMapsOrThrow({+ flatEntityId: objectMetadata.id,+ flatEntityMaps: flatObjectMetadataMaps,+ });+ const flatIndexMetadatas =+ findManyFlatEntityByIdInFlatEntityMapsOrThrow({+ flatEntityIds: flatObjectMetadata.indexMetadataIds,+ flatEntityMaps: flatIndexMaps,+ });+ const filteredFlatIndexMetadatas = applyMetadataFilterToItems({+ items: flatIndexMetadatas,+ filter,+ columnByFilterField: INDEX_FILTER_COLUMN_BY_FILTER_FIELD,+ });+ const connection = findManyItemsWithCursorPagination({+ items: filteredFlatIndexMetadatas,+ paging,+ });++ return {+ ...connection,+ edges: connection.edges.map((edge) => ({+ ...edge,+ node: {+ ...edge.node,+ indexFieldMetadatas: edge.node.flatIndexFieldMetadatas,+ createdAt: new Date(edge.node.createdAt),+ updatedAt: new Date(edge.node.updatedAt),+ indexWhereClause: edge.node.indexWhereClause ?? undefined,+ objectMetadataId: objectMetadata.id,+ workspaceId,+ },+ })),+ };+ });+ });+ }+}diff --git a/packages/twenty-server/src/engine/dataloaders/utils/__tests__/load-application-catalogs-by-registration-id.util.spec.ts b/packages/twenty-server/src/engine/dataloaders/utils/__tests__/load-application-catalogs-by-registration-id.util.spec.tsnew file mode 100644index 00000000000..154f54101b9--- /dev/null+++ b/packages/twenty-server/src/engine/dataloaders/utils/__tests__/load-application-catalogs-by-registration-id.util.spec.ts@@ -0,0 +1,55 @@+import { type ApplicationTranslationCacheService } from 'src/engine/core-modules/application/application-translation/application-translation-cache.service';+import { type FlatApplicationCacheMaps } from 'src/engine/core-modules/application/types/flat-application-cache-maps.type';+import { loadApplicationCatalogsByRegistrationId } from 'src/engine/dataloaders/utils/load-application-catalogs-by-registration-id.util';++describe('loadApplicationCatalogsByRegistrationId', () => {+ it('loads each unique application registration catalog once', async () => {+ const getCatalog = jest.fn(+ async ({+ applicationRegistrationId,+ }: {+ applicationRegistrationId: string;+ }) => ({+ applicationRegistrationId,+ }),+ );+ const applicationTranslationCacheService = {+ getCatalog,+ } as unknown as ApplicationTranslationCacheService;+ const flatApplicationMaps = {+ byId: {+ 'application-1': { applicationRegistrationId: 'registration-1' },+ 'application-2': { applicationRegistrationId: 'registration-1' },+ 'application-3': { applicationRegistrationId: 'registration-2' },+ },+ } as unknown as FlatApplicationCacheMaps;++ const catalogs = await loadApplicationCatalogsByRegistrationId({+ applicationIds: [+ 'application-1',+ 'application-2',+ 'application-3',+ 'unknown-application',+ ],+ flatApplicationMaps,+ locale: 'en',+ applicationTranslationCacheService,+ });++ expect(getCatalog).toHaveBeenCalledTimes(2);+ expect(getCatalog).toHaveBeenCalledWith({+ applicationRegistrationId: 'registration-1',+ locale: 'en',+ });+ expect(getCatalog).toHaveBeenCalledWith({+ applicationRegistrationId: 'registration-2',+ locale: 'en',+ });+ expect(catalogs).toEqual(+ new Map([+ ['registration-1', { applicationRegistrationId: 'registration-1' }],+ ['registration-2', { applicationRegistrationId: 'registration-2' }],+ ]),+ );+ });+});diff --git a/packages/twenty-server/src/engine/dataloaders/utils/load-application-catalogs-by-registration-id.util.ts b/packages/twenty-server/src/engine/dataloaders/utils/load-application-catalogs-by-registration-id.util.tsnew file mode 100644index 00000000000..336b2e4edb0--- /dev/null+++ b/packages/twenty-server/src/engine/dataloaders/utils/load-application-catalogs-by-registration-id.util.ts@@ -0,0 +1,43 @@+import { type APP_LOCALES } from 'twenty-shared/translations';+import { isDefined } from 'twenty-shared/utils';++import { ApplicationTranslationCacheService } from 'src/engine/core-modules/application/application-translation/application-translation-cache.service';+import { type FlatApplicationCacheMaps } from 'src/engine/core-modules/application/types/flat-application-cache-maps.type';++export const loadApplicationCatalogsByRegistrationId = async ({+ applicationIds,+ flatApplicationMaps,+ locale,+ applicationTranslationCacheService,+}: {+ applicationIds: string[];+ flatApplicationMaps: FlatApplicationCacheMaps;+ locale: keyof typeof APP_LOCALES;+ applicationTranslationCacheService: ApplicationTranslationCacheService;+}): Promise<Map<string, Record<string, string>>> => {+ const registrationIds = [+ ...new Set(+ applicationIds+ .map(+ (applicationId) =>+ flatApplicationMaps.byId[applicationId]?.applicationRegistrationId,+ )+ .filter(isDefined),+ ),+ ];++ const catalogByRegistrationId = new Map<string, Record<string, string>>();++ await Promise.all(+ registrationIds.map(async (applicationRegistrationId) => {+ const catalog = await applicationTranslationCacheService.getCatalog({+ applicationRegistrationId,+ locale,+ });++ catalogByRegistrationId.set(applicationRegistrationId, catalog);+ }),+ );++ return catalogByRegistrationId;+};diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/dtos/field-filter.input.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/dtos/field-filter.input.tsindex 21f25e37d7e..7258d981c96 100644--- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/dtos/field-filter.input.ts+++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/dtos/field-filter.input.ts@@ -1,8 +1,8 @@import { Field, InputType } from '@nestjs/graphql';import { BooleanFieldComparisonInput } from 'src/engine/metadata-modules/pagination/dtos/boolean-field-comparison.input';-import { type MetadataFilterColumn } from 'src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-query-builder.util';import { UUIDFilterComparisonInput } from 'src/engine/metadata-modules/pagination/dtos/uuid-filter-comparison.input';+import { type MetadataFilterColumn } from 'src/engine/metadata-modules/pagination/types/metadata-filter-column.type';@InputType('FieldFilter')export class FieldFilterInput {diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/dtos/index-filter.input.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/dtos/index-filter.input.tsindex ba17657f7c5..f4cd4c3192d 100644--- a/packages/twenty-server/src/engine/metadata-modules/index-metadata/dtos/index-filter.input.ts+++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/dtos/index-filter.input.ts@@ -2,7 +2,7 @@ import { Field, InputType } from '@nestjs/graphql';import { BooleanFieldComparisonInput } from 'src/engine/metadata-modules/pagination/dtos/boolean-field-comparison.input';import { UUIDFilterComparisonInput } from 'src/engine/metadata-modules/pagination/dtos/uuid-filter-comparison.input';-import { type MetadataFilterColumn } from 'src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-query-builder.util';+import { type MetadataFilterColumn } from 'src/engine/metadata-modules/pagination/types/metadata-filter-column.type';@InputType('IndexFilter')export class IndexFilterInput {diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/dtos/object-filter.input.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/dtos/object-filter.input.tsindex 3747f4bee23..fce8cc2c9ee 100644--- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/dtos/object-filter.input.ts+++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/dtos/object-filter.input.ts@@ -1,8 +1,8 @@import { Field, InputType } from '@nestjs/graphql';import { BooleanFieldComparisonInput } from 'src/engine/metadata-modules/pagination/dtos/boolean-field-comparison.input';-import { type MetadataFilterColumn } from 'src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-query-builder.util';import { UUIDFilterComparisonInput } from 'src/engine/metadata-modules/pagination/dtos/uuid-filter-comparison.input';+import { type MetadataFilterColumn } from 'src/engine/metadata-modules/pagination/types/metadata-filter-column.type';@InputType('ObjectFilter')export class ObjectFilterInput {diff --git a/packages/twenty-server/src/engine/metadata-modules/pagination/types/metadata-filter-column.type.ts b/packages/twenty-server/src/engine/metadata-modules/pagination/types/metadata-filter-column.type.tsnew file mode 100644index 00000000000..63eaf0dc40c--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/pagination/types/metadata-filter-column.type.ts@@ -0,0 +1,6 @@+// invertBooleanValues supports deprecated boolean aliases whose backing+// column has the opposite meaning (isUIReadOnly filters run against+// isUIEditable, since the legacy column is no longer written).+export type MetadataFilterColumn =+ | { column: string; type: 'uuid' }+ | { column: string; type: 'boolean'; invertBooleanValues?: true };diff --git a/packages/twenty-server/src/engine/metadata-modules/pagination/types/metadata-filter-comparison.type.ts b/packages/twenty-server/src/engine/metadata-modules/pagination/types/metadata-filter-comparison.type.tsnew file mode 100644index 00000000000..e84a5cca413--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/pagination/types/metadata-filter-comparison.type.ts@@ -0,0 +1,5 @@+import { type BooleanFieldComparisonInput } from 'src/engine/metadata-modules/pagination/dtos/boolean-field-comparison.input';+import { type UUIDFilterComparisonInput } from 'src/engine/metadata-modules/pagination/dtos/uuid-filter-comparison.input';++export type MetadataFilterComparison = UUIDFilterComparisonInput &+ BooleanFieldComparisonInput;diff --git a/packages/twenty-server/src/engine/metadata-modules/pagination/types/metadata-filter-shape.type.ts b/packages/twenty-server/src/engine/metadata-modules/pagination/types/metadata-filter-shape.type.tsnew file mode 100644index 00000000000..1223dd5421f--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/pagination/types/metadata-filter-shape.type.ts@@ -0,0 +1,4 @@+export type MetadataFilterShape<TFilter> = {+ and?: TFilter[];+ or?: TFilter[];+};diff --git a/packages/twenty-server/src/engine/metadata-modules/pagination/utils/__tests__/apply-metadata-filter-to-items.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/pagination/utils/__tests__/apply-metadata-filter-to-items.util.spec.tsnew file mode 100644index 00000000000..48d75c56b13--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/pagination/utils/__tests__/apply-metadata-filter-to-items.util.spec.ts@@ -0,0 +1,98 @@+import { UserInputError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';+import { applyMetadataFilterToItems } from 'src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-items.util';++describe('applyMetadataFilterToItems', () => {+ it('applies nested filters to batched in-memory relations', () => {+ type TestFilter = {+ and?: TestFilter[];+ id?: { gt?: string; lt?: string };+ isActive?: { is?: boolean };+ };++ const items = [+ { id: 'a', isActive: true },+ { id: 'b', isActive: false },+ { id: 'c', isActive: true },+ ];++ expect(+ applyMetadataFilterToItems<(typeof items)[number], TestFilter>({+ items,+ filter: {+ and: [{ isActive: { is: true } }],+ id: { gt: 'a', lt: 'c' },+ },+ columnByFilterField: {+ id: { column: 'id', type: 'uuid' },+ isActive: { column: 'isActive', type: 'boolean' },+ },+ }),+ ).toEqual([]);+ });++ it('matches UUID filters regardless of operand casing', () => {+ type UuidFilter = {+ and?: UuidFilter[];+ or?: UuidFilter[];+ id?: { eq?: string; neq?: string; in?: string[]; notIn?: string[] };+ };++ const items = [{ id: '00000000-0000-4000-8000-00000000000a' }];+ const columnByFilterField = {+ id: { column: 'id', type: 'uuid' },+ } as const;+ const upperCasedId = items[0].id.toUpperCase();++ const matching = (filter: UuidFilter) =>+ applyMetadataFilterToItems<(typeof items)[number], UuidFilter>({+ items,+ filter,+ columnByFilterField,+ });++ expect(matching({ id: { eq: upperCasedId } })).toEqual(items);+ expect(matching({ id: { in: [upperCasedId] } })).toEqual(items);+ expect(matching({ id: { neq: upperCasedId } })).toEqual([]);+ expect(matching({ id: { notIn: [upperCasedId] } })).toEqual([]);+ });++ it('inverts aliased boolean columns', () => {+ type ReadOnlyFilter = {+ and?: ReadOnlyFilter[];+ or?: ReadOnlyFilter[];+ isUIReadOnly?: { is?: boolean };+ };++ const items = [{ id: 'a', isUIEditable: false }];++ expect(+ applyMetadataFilterToItems<(typeof items)[number], ReadOnlyFilter>({+ items,+ filter: { isUIReadOnly: { is: true } },+ columnByFilterField: {+ isUIReadOnly: {+ column: 'isUIEditable',+ type: 'boolean',+ invertBooleanValues: true,+ },+ },+ }),+ ).toEqual(items);+ });++ it('rejects unknown filter fields', () => {+ type UnknownFieldFilter = {+ and?: UnknownFieldFilter[];+ or?: UnknownFieldFilter[];+ unknownField?: { eq?: string };+ };++ expect(() =>+ applyMetadataFilterToItems<{ id: string }, UnknownFieldFilter>({+ items: [{ id: 'a' }],+ filter: { unknownField: { eq: 'a' } },+ columnByFilterField: {},+ }),+ ).toThrow(UserInputError);+ });+});diff --git a/packages/twenty-server/src/engine/metadata-modules/pagination/utils/__tests__/apply-metadata-filter-to-query-builder.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/pagination/utils/__tests__/apply-metadata-filter-to-query-builder.util.spec.tsindex 49bcbcf2a5e..9dbe5e245aa 100644--- a/packages/twenty-server/src/engine/metadata-modules/pagination/utils/__tests__/apply-metadata-filter-to-query-builder.util.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/pagination/utils/__tests__/apply-metadata-filter-to-query-builder.util.spec.ts@@ -1,10 +1,7 @@import { Brackets, type WhereExpressionBuilder } from 'typeorm';import { UserInputError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';-import {- applyMetadataFilterToItems,- applyMetadataFilterToQueryBuilder,-} from 'src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-query-builder.util';+import { applyMetadataFilterToQueryBuilder } from 'src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-query-builder.util';class FakeWhereBuilder {conditions: string[] = [];@@ -174,18 +171,9 @@ describe('applyMetadataFilterToQueryBuilder', () => {expect(() => applyFilter({ id: { is: true } })).toThrow(UserInputError);});- it('normalizes UUID operands for database and in-memory filters', () => {- type UuidFilter = {- and?: UuidFilter[];- or?: UuidFilter[];- id?: { eq?: string; in?: string[] };- };-+ it('normalizes UUID operands for database filters', () => {const lowerCaseUuid = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa';const upperCaseUuid = lowerCaseUuid.toUpperCase();- const filter: UuidFilter = {- id: { eq: upperCaseUuid, in: [upperCaseUuid] },- };const whereBuilder = applyFilter({id: { eq: upperCaseUuid, in: [upperCaseUuid] },});@@ -194,69 +182,6 @@ describe('applyMetadataFilterToQueryBuilder', () => {metadataFilterParameter0: lowerCaseUuid,metadataFilterParameter1: [lowerCaseUuid],});- expect(- applyMetadataFilterToItems({- items: [{ id: lowerCaseUuid }],- filter,- columnByFilterField: {- id: { column: 'id', type: 'uuid' },- },- }),- ).toEqual([{ id: lowerCaseUuid }]);- });-- it('applies the same filter semantics to batched in-memory relations', () => {- type TestFilter = {- and?: TestFilter[];- id?: { gt?: string; lt?: string };- isActive?: { is?: boolean };- };-- const items = [- { id: 'a', isActive: true },- { id: 'b', isActive: false },- { id: 'c', isActive: true },- ];-- expect(- applyMetadataFilterToItems<(typeof items)[number], TestFilter>({- items,- filter: {- and: [{ isActive: { is: true } }],- id: { gt: 'a', lt: 'c' },- },- columnByFilterField: {- id: { column: 'id', type: 'uuid' },- isActive: { column: 'isActive', type: 'boolean' },- },- }),- ).toEqual([]);- });-- it('matches uuid filters regardless of operand casing', () => {- type UuidFilter = {- and?: UuidFilter[];- or?: UuidFilter[];- id?: { eq?: string; neq?: string; in?: string[]; notIn?: string[] };- };-- const items = [{ id: '00000000-0000-4000-8000-00000000000a' }];- const columnByFilterField = {- id: { column: 'id', type: 'uuid' },- } as const;- const upperCasedId = items[0].id.toUpperCase();-- const matching = (filter: UuidFilter) =>- applyMetadataFilterToItems<(typeof items)[number], UuidFilter>({- items,- filter,- columnByFilterField,- });-- expect(matching({ id: { eq: upperCasedId } })).toEqual(items);- expect(matching({ id: { in: [upperCasedId] } })).toEqual(items);- expect(matching({ id: { neq: upperCasedId } })).toEqual([]);- expect(matching({ id: { notIn: [upperCasedId] } })).toEqual([]);});it('ignores undefined filter values', () => {diff --git a/packages/twenty-server/src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-items.util.ts b/packages/twenty-server/src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-items.util.tsnew file mode 100644index 00000000000..8c4a91c573e--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-items.util.ts@@ -0,0 +1,222 @@+import { isDefined } from 'twenty-shared/utils';++import { UserInputError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';+import { type MetadataFilterColumn } from 'src/engine/metadata-modules/pagination/types/metadata-filter-column.type';+import { type MetadataFilterComparison } from 'src/engine/metadata-modules/pagination/types/metadata-filter-comparison.type';+import { type MetadataFilterShape } from 'src/engine/metadata-modules/pagination/types/metadata-filter-shape.type';+import { normalizeMetadataFilterComparison } from 'src/engine/metadata-modules/pagination/utils/normalize-metadata-filter-comparison.util';++const matchesSqlLikePattern = (value: string, pattern: string): boolean => {+ const escapedPattern = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');+ const regularExpressionPattern = escapedPattern+ .replace(/%/g, '.*')+ .replace(/_/g, '.');++ return new RegExp(`^${regularExpressionPattern}$`).test(value);+};++const matchesMetadataComparison = ({+ value,+ comparison,+ column,+}: {+ value: unknown;+ comparison: MetadataFilterComparison;+ column: MetadataFilterColumn;+}): boolean => {+ const normalizedComparison = normalizeMetadataFilterComparison({+ comparison,+ column,+ });+ const normalizedValue =+ column.type === 'uuid' && typeof value === 'string'+ ? value.toLowerCase()+ : value;++ const stringValue =+ typeof normalizedValue === 'string' ? normalizedValue : undefined;+ const simpleComparisons: [+ keyof MetadataFilterComparison,+ (comparisonValue: unknown) => boolean,+ ][] = [+ ['eq', (comparisonValue) => normalizedValue === comparisonValue],+ ['neq', (comparisonValue) => normalizedValue !== comparisonValue],+ [+ 'gt',+ (comparisonValue) =>+ isDefined(stringValue) &&+ typeof comparisonValue === 'string' &&+ stringValue > comparisonValue,+ ],+ [+ 'gte',+ (comparisonValue) =>+ isDefined(stringValue) &&+ typeof comparisonValue === 'string' &&+ stringValue >= comparisonValue,+ ],+ [+ 'lt',+ (comparisonValue) =>+ isDefined(stringValue) &&+ typeof comparisonValue === 'string' &&+ stringValue < comparisonValue,+ ],+ [+ 'lte',+ (comparisonValue) =>+ isDefined(stringValue) &&+ typeof comparisonValue === 'string' &&+ stringValue <= comparisonValue,+ ],+ ];++ for (const [comparisonKey, matches] of simpleComparisons) {+ const comparisonValue = normalizedComparison[comparisonKey];++ if (isDefined(comparisonValue) && !matches(comparisonValue)) {+ return false;+ }+ }++ const likeComparisons: [keyof MetadataFilterComparison, boolean, boolean][] =+ [+ ['like', false, false],+ ['notLike', false, true],+ ['iLike', true, false],+ ['notILike', true, true],+ ];++ for (const [comparisonKey, caseInsensitive, negate] of likeComparisons) {+ const comparisonValue = normalizedComparison[comparisonKey];++ if (!isDefined(comparisonValue)) {+ continue;+ }++ if (!isDefined(stringValue) || typeof comparisonValue !== 'string') {+ return false;+ }++ const candidate = caseInsensitive ? stringValue.toLowerCase() : stringValue;+ const pattern = caseInsensitive+ ? comparisonValue.toLowerCase()+ : comparisonValue;+ const matches = matchesSqlLikePattern(candidate, pattern);++ if (negate ? matches : !matches) {+ return false;+ }+ }++ if (+ isDefined(normalizedComparison.in) &&+ !normalizedComparison.in.includes(normalizedValue as string)+ ) {+ return false;+ }++ if (+ isDefined(normalizedComparison.notIn) &&+ normalizedComparison.notIn.includes(normalizedValue as string)+ ) {+ return false;+ }++ if (+ normalizedComparison.is !== undefined &&+ (normalizedComparison.is === null+ ? normalizedValue !== null && normalizedValue !== undefined+ : normalizedValue !== normalizedComparison.is)+ ) {+ return false;+ }++ if (+ normalizedComparison.isNot !== undefined &&+ (normalizedComparison.isNot === null+ ? normalizedValue === null || normalizedValue === undefined+ : normalizedValue === normalizedComparison.isNot)+ ) {+ return false;+ }++ return true;+};++export const applyMetadataFilterToItems = <+ TEntity extends { id: string },+ TFilter extends MetadataFilterShape<TFilter>,+>({+ items,+ filter,+ columnByFilterField,+}: {+ items: TEntity[];+ filter: TFilter;+ columnByFilterField: Record<string, MetadataFilterColumn>;+}): TEntity[] =>+ items.filter((item) => {+ for (const [filterField, filterValue] of Object.entries(filter)) {+ if (!isDefined(filterValue)) {+ continue;+ }++ if (filterField === 'and') {+ if (+ !(filterValue as TFilter[]).every(+ (subFilter) =>+ applyMetadataFilterToItems({+ items: [item],+ filter: subFilter,+ columnByFilterField,+ }).length === 1,+ )+ ) {+ return false;+ }++ continue;+ }++ if (filterField === 'or') {+ const subFilters = filterValue as TFilter[];++ if (+ subFilters.length > 0 &&+ !subFilters.some(+ (subFilter) =>+ applyMetadataFilterToItems({+ items: [item],+ filter: subFilter,+ columnByFilterField,+ }).length === 1,+ )+ ) {+ return false;+ }++ continue;+ }++ const column = columnByFilterField[filterField];++ if (!isDefined(column)) {+ throw new UserInputError(`Unknown filter field: ${filterField}`);+ }++ const value: unknown = Reflect.get(item, column.column);++ if (+ !matchesMetadataComparison({+ value,+ comparison: filterValue as MetadataFilterComparison,+ column,+ })+ ) {+ return false;+ }+ }++ return true;+ });diff --git a/packages/twenty-server/src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-query-builder.util.ts b/packages/twenty-server/src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-query-builder.util.tsindex 96471b157bc..60398500844 100644--- a/packages/twenty-server/src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-query-builder.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/pagination/utils/apply-metadata-filter-to-query-builder.util.ts@@ -2,35 +2,10 @@ import { isDefined } from 'twenty-shared/utils';import { Brackets, type WhereExpressionBuilder } from 'typeorm';import { UserInputError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';--export type MetadataFilterComparison = {- eq?: unknown;- neq?: unknown;- gt?: unknown;- gte?: unknown;- lt?: unknown;- lte?: unknown;- like?: unknown;- notLike?: unknown;- iLike?: unknown;- notILike?: unknown;- in?: unknown[];- notIn?: unknown[];- is?: boolean | null;- isNot?: boolean | null;-};--// invertBooleanValues supports deprecated boolean aliases whose backing-// column has the opposite meaning (isUIReadOnly filters run against-// isUIEditable, since the legacy column is no longer written).-export type MetadataFilterColumn =- | { column: string; type: 'uuid' }- | { column: string; type: 'boolean'; invertBooleanValues?: true };--type MetadataFilterShape<TFilter> = {- and?: TFilter[];- or?: TFilter[];-};+import { type MetadataFilterColumn } from 'src/engine/metadata-modules/pagination/types/metadata-filter-column.type';+import { type MetadataFilterComparison } from 'src/engine/metadata-modules/pagination/types/metadata-filter-comparison.type';+import { type MetadataFilterShape } from 'src/engine/metadata-modules/pagination/types/metadata-filter-shape.type';+import { normalizeMetadataFilterComparison } from 'src/engine/metadata-modules/pagination/utils/normalize-metadata-filter-comparison.util';type ParameterCounter = { value: number };@@ -46,89 +21,6 @@ const buildBooleanCondition = (return `${column} ${operator} ${value ? 'TRUE' : 'FALSE'}`;};-const invertBooleanComparisonValue = (- value: boolean | null | undefined,-): boolean | null | undefined => {- if (typeof value === 'boolean') {- return !value;- }-- return value;-};--const normalizeMetadataComparison = ({- comparison,- column,-}: {- comparison: MetadataFilterComparison;- column: MetadataFilterColumn;-}): MetadataFilterComparison => {- const normalizedComparison = { ...comparison };-- if (column.type === 'boolean' && column.invertBooleanValues) {- normalizedComparison.is = invertBooleanComparisonValue(comparison.is);- normalizedComparison.isNot = invertBooleanComparisonValue(comparison.isNot);- }-- if (column.type === 'uuid') {- if (- (normalizedComparison.is !== undefined &&- normalizedComparison.is !== null) ||- (normalizedComparison.isNot !== undefined &&- normalizedComparison.isNot !== null)- ) {- throw new UserInputError(- 'UUID is/isNot comparisons only support null values',- );- }-- // Postgres compares uuid columns canonically, but the in-memory matcher- // compares raw strings, so operands are lowercased to keep both paths- // agreeing on differently-cased but equal ids.- return lowercaseComparisonOperands(normalizedComparison);- }-- return normalizedComparison;-};--const lowercaseValue = (value: unknown): unknown =>- typeof value === 'string' ? value.toLowerCase() : value;--const lowercaseComparisonOperands = (- comparison: MetadataFilterComparison,-): MetadataFilterComparison => {- const lowercasedComparison: MetadataFilterComparison = { ...comparison };-- for (const operand of [- 'eq',- 'neq',- 'gt',- 'gte',- 'lt',- 'lte',- 'like',- 'notLike',- 'iLike',- 'notILike',- ] as const) {- if (isDefined(lowercasedComparison[operand])) {- lowercasedComparison[operand] = lowercaseValue(- lowercasedComparison[operand],- );- }- }-- for (const operand of ['in', 'notIn'] as const) {- const values = lowercasedComparison[operand];-- if (isDefined(values)) {- lowercasedComparison[operand] = values.map(lowercaseValue);- }- }-- return lowercasedComparison;-};-const applyComparisonToQueryBuilder = ({whereBuilder,column,@@ -291,7 +183,7 @@ export const applyMetadataFilterToQueryBuilder = <}const { column } = filterColumn;- const comparison = normalizeMetadataComparison({+ const comparison = normalizeMetadataFilterComparison({comparison: filterValue as MetadataFilterComparison,column: filterColumn,});@@ -304,216 +196,3 @@ export const applyMetadataFilterToQueryBuilder = <});}};--const matchesSqlLikePattern = (value: string, pattern: string): boolean => {- const escapedPattern = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');- const regularExpressionPattern = escapedPattern- .replace(/%/g, '.*')- .replace(/_/g, '.');-- return new RegExp(`^${regularExpressionPattern}$`).test(value);-};--const matchesMetadataComparison = ({- value,- comparison,- column,-}: {- value: unknown;- comparison: MetadataFilterComparison;- column: MetadataFilterColumn;-}): boolean => {- const normalizedComparison = normalizeMetadataComparison({- comparison,- column,- });- const normalizedValue =- column.type === 'uuid' ? lowercaseValue(value) : value;-- const stringValue =- typeof normalizedValue === 'string' ? normalizedValue : undefined;- const simpleComparisons: [- keyof MetadataFilterComparison,- (comparisonValue: unknown) => boolean,- ][] = [- ['eq', (comparisonValue) => normalizedValue === comparisonValue],- ['neq', (comparisonValue) => normalizedValue !== comparisonValue],- [- 'gt',- (comparisonValue) =>- isDefined(stringValue) &&- typeof comparisonValue === 'string' &&- stringValue > comparisonValue,- ],- [- 'gte',- (comparisonValue) =>- isDefined(stringValue) &&- typeof comparisonValue === 'string' &&- stringValue >= comparisonValue,- ],- [- 'lt',- (comparisonValue) =>- isDefined(stringValue) &&- typeof comparisonValue === 'string' &&- stringValue < comparisonValue,- ],- [- 'lte',- (comparisonValue) =>- isDefined(stringValue) &&- typeof comparisonValue === 'string' &&- stringValue <= comparisonValue,- ],- ];-- for (const [comparisonKey, matches] of simpleComparisons) {- const comparisonValue = normalizedComparison[comparisonKey];-- if (isDefined(comparisonValue) && !matches(comparisonValue)) {- return false;- }- }-- const likeComparisons: [keyof MetadataFilterComparison, boolean, boolean][] =- [- ['like', false, false],- ['notLike', false, true],- ['iLike', true, false],- ['notILike', true, true],- ];-- for (const [comparisonKey, caseInsensitive, negate] of likeComparisons) {- const comparisonValue = normalizedComparison[comparisonKey];-- if (!isDefined(comparisonValue)) {- continue;- }-- if (!isDefined(stringValue) || typeof comparisonValue !== 'string') {- return false;- }-- const candidate = caseInsensitive ? stringValue.toLowerCase() : stringValue;- const pattern = caseInsensitive- ? comparisonValue.toLowerCase()- : comparisonValue;- const matches = matchesSqlLikePattern(candidate, pattern);-- if (negate ? matches : !matches) {- return false;- }- }-- if (- isDefined(normalizedComparison.in) &&- !normalizedComparison.in.includes(normalizedValue)- ) {- return false;- }-- if (- isDefined(normalizedComparison.notIn) &&- normalizedComparison.notIn.includes(normalizedValue)- ) {- return false;- }-- if (- normalizedComparison.is !== undefined &&- (normalizedComparison.is === null- ? normalizedValue !== null && normalizedValue !== undefined- : normalizedValue !== normalizedComparison.is)- ) {- return false;- }-- if (- normalizedComparison.isNot !== undefined &&- (normalizedComparison.isNot === null- ? normalizedValue === null || normalizedValue === undefined- : normalizedValue === normalizedComparison.isNot)- ) {- return false;- }-- return true;-};--export const applyMetadataFilterToItems = <- TEntity extends { id: string },- TFilter extends MetadataFilterShape<TFilter>,->({- items,- filter,- columnByFilterField,-}: {- items: TEntity[];- filter: TFilter;- columnByFilterField: Record<string, MetadataFilterColumn>;-}): TEntity[] =>- items.filter((item) => {- for (const [filterField, filterValue] of Object.entries(filter)) {- if (!isDefined(filterValue)) {- continue;- }-- if (filterField === 'and') {- if (- !(filterValue as TFilter[]).every(- (subFilter) =>- applyMetadataFilterToItems({- items: [item],- filter: subFilter,- columnByFilterField,- }).length === 1,- )- ) {- return false;- }-- continue;- }-- if (filterField === 'or') {- const subFilters = filterValue as TFilter[];-- if (- subFilters.length > 0 &&- !subFilters.some(- (subFilter) =>- applyMetadataFilterToItems({- items: [item],- filter: subFilter,- columnByFilterField,- }).length === 1,- )- ) {- return false;- }-- continue;- }-- const column = columnByFilterField[filterField];-- if (!isDefined(column)) {- throw new UserInputError(`Unknown filter field: ${filterField}`);- }-- const value: unknown = Reflect.get(item, column.column);-- if (- !matchesMetadataComparison({- value,- comparison: filterValue as MetadataFilterComparison,- column,- })- ) {- return false;- }- }-- return true;- });diff --git a/packages/twenty-server/src/engine/metadata-modules/pagination/utils/normalize-metadata-filter-comparison.util.ts b/packages/twenty-server/src/engine/metadata-modules/pagination/utils/normalize-metadata-filter-comparison.util.tsnew file mode 100644index 00000000000..c4a9b91a0ea--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/pagination/utils/normalize-metadata-filter-comparison.util.ts@@ -0,0 +1,87 @@+import { isDefined } from 'twenty-shared/utils';++import { UserInputError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';+import { type MetadataFilterColumn } from 'src/engine/metadata-modules/pagination/types/metadata-filter-column.type';+import { type MetadataFilterComparison } from 'src/engine/metadata-modules/pagination/types/metadata-filter-comparison.type';++const invertBooleanComparisonValue = (+ value: boolean | null | undefined,+): boolean | null | undefined => {+ if (typeof value === 'boolean') {+ return !value;+ }++ return value;+};++const lowercaseComparisonOperands = (+ comparison: MetadataFilterComparison,+): MetadataFilterComparison => {+ const lowercasedComparison: MetadataFilterComparison = { ...comparison };++ for (const operand of [+ 'eq',+ 'neq',+ 'gt',+ 'gte',+ 'lt',+ 'lte',+ 'like',+ 'notLike',+ 'iLike',+ 'notILike',+ ] as const) {+ const value = lowercasedComparison[operand];++ if (isDefined(value)) {+ lowercasedComparison[operand] = value.toLowerCase();+ }+ }++ for (const operand of ['in', 'notIn'] as const) {+ const values = lowercasedComparison[operand];++ if (isDefined(values)) {+ lowercasedComparison[operand] = values.map((value) =>+ value.toLowerCase(),+ );+ }+ }++ return lowercasedComparison;+};++export const normalizeMetadataFilterComparison = ({+ comparison,+ column,+}: {+ comparison: MetadataFilterComparison;+ column: MetadataFilterColumn;+}): MetadataFilterComparison => {+ const normalizedComparison = { ...comparison };++ if (column.type === 'boolean' && column.invertBooleanValues) {+ normalizedComparison.is = invertBooleanComparisonValue(comparison.is);+ normalizedComparison.isNot = invertBooleanComparisonValue(comparison.isNot);+ }++ if (column.type === 'uuid') {+ if (+ (normalizedComparison.is !== undefined &&+ normalizedComparison.is !== null) ||+ (normalizedComparison.isNot !== undefined &&+ normalizedComparison.isNot !== null)+ ) {+ throw new UserInputError(+ 'UUID is/isNot comparisons only support null values',+ );+ }++ // Postgres compares uuid columns canonically, but the in-memory matcher+ // compares raw strings, so operands are lowercased to keep both paths+ // agreeing on differently-cased but equal ids.+ return lowercaseComparisonOperands(normalizedComparison);+ }++ return normalizedComparison;+};