The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
feat(server): route findDuplicates through the ORM v2 read path (#24088)
Charles Bochet587ade034ac7
diff --git a/.github/workflows/ci-server.yaml b/.github/workflows/ci-server.yamlindex 7e4f9f2f2fb..e1e01d37587 100644--- a/.github/workflows/ci-server.yaml+++ b/.github/workflows/ci-server.yaml@@ -311,7 +311,7 @@ jobs:SHARD_COUNTER: 16AUTH_COOKIE_SESSIONS_ENABLED: ${{ contains(github.event.pull_request.labels.*.name, 'ci:auth-cookie-sessions') }}TEST_FORCED_FEATURE_FLAGS: ${{ matrix.orm-v2 && 'IS_ORM_V2_READ_PATH_ENABLED' || '' }}- ORM_V2_TEST_PATH_PATTERN: '(find-many|find-one|composite-field-pagination|filter-by-relation-field|order-by-relation-field|nested-relation|soft-deleted-relation|relation-join-rls)'+ ORM_V2_TEST_PATH_PATTERN: '(find-many|find-one|find-duplicates|composite-field-pagination|filter-by-relation-field|order-by-relation-field|nested-relation|soft-deleted-relation|relation-join-rls)'steps:- name: Fetch custom Github Actions and base branch historyuses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-duplicates-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-duplicates-query-runner.service.tsindex 94dc4515907..26da09dd009 100644--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-duplicates-query-runner.service.ts+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-duplicates-query-runner.service.ts@@ -45,7 +45,6 @@ export class CommonFindDuplicatesQueryRunnerService extends CommonBaseQueryRunnequeryRunnerContext: CommonExtendedQueryRunnerContext,): Promise<CommonFindDuplicatesOutputItem[]> {const {- repository,flatObjectMetadata,flatObjectMetadataMaps,flatFieldMetadataMaps,@@ -55,7 +54,9 @@ export class CommonFindDuplicatesQueryRunnerService extends CommonBaseQueryRunnerolePermissionConfig,} = queryRunnerContext;- const existingRecordsQueryBuilder = repository.createQueryBuilder(+ const readRepository = this.getReadRepository(queryRunnerContext);++ const existingRecordsQueryBuilder = readRepository.createQueryBuilder(flatObjectMetadata.nameSingular,);@@ -110,9 +111,8 @@ export class CommonFindDuplicatesQueryRunnerService extends CommonBaseQueryRunne};}- const duplicateRecordsQueryBuilder = repository.createQueryBuilder(- flatObjectMetadata.nameSingular,- );+ const duplicateRecordsQueryBuilder =+ readRepository.createQueryBuilder(flatObjectMetadata.nameSingular);commonQueryParser.applyFilterToBuilder(duplicateRecordsQueryBuilder,diff --git a/packages/twenty-server/src/engine/twenty-orm-v2/README.md b/packages/twenty-server/src/engine/twenty-orm-v2/README.mdindex 20efe35b05a..86d56a8e063 100644--- a/packages/twenty-server/src/engine/twenty-orm-v2/README.md+++ b/packages/twenty-server/src/engine/twenty-orm-v2/README.md@@ -93,16 +93,18 @@ findMany round trip.## Covered so far-`createQueryBuilder`, `clone`, `where` / `andWhere` / `orWhere` (strings and bracket-factories), `setParameters`, `setFindOptions({ select })`, `addSelect`, `orderBy` /-`addOrderBy`, `leftJoin` on to-one relations, `withDeleted`, `limit` / `offset`,-`getMany`, `getOne`, `getRawOne`, `getQuery`, `getQueryAndParameters`.--The surface stops there on purpose: anything findMany cannot reach (`getCount`,-`getRawMany`, `groupBy`, `take` / `skip`) is left to v1 until a runner needs it.--Wired into `CommonFindManyQueryRunnerService` and `CommonFindOneQueryRunnerService`.-Other runners keep `repository`.+`createQueryBuilder`, `clone`, `where` / `andWhere` / `orWhere` (strings, bracket+factories, and object literals like `{ id: In([...]) }` — `in` and equality only),+`setParameters`, `setFindOptions({ select })`, `addSelect`, `orderBy` / `addOrderBy`,+`leftJoin` on to-one relations, `withDeleted`, `limit` / `offset`, `take` / `skip`+(aliases of `limit` / `offset` — safe because v2 never joins to-many),+`getMany`, `getOne`, `getRawOne`, `getCount`, `getQuery`, `getQueryAndParameters`.++The surface stops there on purpose: anything the wired runners cannot reach+(`getRawMany`, `groupBy`) is left to v1 until a runner needs it.++Wired into `CommonFindManyQueryRunnerService`, `CommonFindOneQueryRunnerService` and+`CommonFindDuplicatesQueryRunnerService`. Other runners keep `repository`.## Not covered yetdiff --git a/packages/twenty-server/src/engine/twenty-orm-v2/query-builder/__tests__/workspace-select-query-builder-v2.spec.ts b/packages/twenty-server/src/engine/twenty-orm-v2/query-builder/__tests__/workspace-select-query-builder-v2.spec.tsindex dca6366d914..66b3b9884d7 100644--- a/packages/twenty-server/src/engine/twenty-orm-v2/query-builder/__tests__/workspace-select-query-builder-v2.spec.ts+++ b/packages/twenty-server/src/engine/twenty-orm-v2/query-builder/__tests__/workspace-select-query-builder-v2.spec.ts@@ -1,4 +1,5 @@import { FieldMetadataType } from 'twenty-shared/types';+import { In, LessThan } from 'typeorm';import { RelationType } from 'src/engine/metadata-modules/field-metadata/interfaces/relation-type.interface';import { type CompiledStatement } from 'src/engine/twenty-orm-v2/sql/utils/compile-named-parameters.util';@@ -543,4 +544,89 @@ describe('WorkspaceSelectQueryBuilderV2', () => {expect(executedStatements[0].text).toContain('$1');expect(executedStatements[0].values).toEqual(['Ada']);});++ it('should map take and skip to LIMIT and OFFSET parameters', () => {+ const { queryBuilder } = buildQueryBuilder();++ queryBuilder+ .setFindOptions({ select: { id: true } })+ .take(31)+ .skip(60);++ const [text, values] = queryBuilder.getQueryAndParameters();++ expect(text).toContain('LIMIT $1');+ expect(text).toContain('OFFSET $2');+ expect(values).toEqual([31, 60]);+ });++ it('should build a COUNT statement that ignores projection, order and pagination', async () => {+ const { queryBuilder, executedStatements } = buildQueryBuilder({+ rows: [{ count: '7' }],+ });++ queryBuilder+ .setFindOptions({ select: { id: true } })+ .orderBy('"person"."id"', 'ASC')+ .take(10);++ const count = await queryBuilder.getCount();++ expect(count).toBe(7);+ expect(executedStatements[0].text).toBe(+ 'SELECT COUNT(1) AS "count" ' ++ `FROM "${SCHEMA_NAME}"."person" AS "person" ` ++ 'WHERE "person"."deletedAt" IS NULL',+ );+ });++ it('should render an object-literal where with In as a bound IN clause', () => {+ const { queryBuilder } = buildQueryBuilder();++ queryBuilder+ .setFindOptions({ select: { id: true } })+ .where({ id: In(['a', 'b']) });++ const [text, values] = queryBuilder.getQueryAndParameters();++ expect(text).toContain('"person"."id" IN ($1, $2)');+ expect(values).toEqual(['a', 'b']);+ });++ it('should allocate unique parameter names across object-literal where clauses', () => {+ const { queryBuilder } = buildQueryBuilder();++ queryBuilder+ .setFindOptions({ select: { id: true } })+ .where({ id: In(['a']) })+ .andWhere({ companyId: In(['b']) });++ const [, values] = queryBuilder.getQueryAndParameters();++ expect(values).toEqual(['a', 'b']);+ });++ it('should reject an object-literal where on an unknown column', () => {+ const { queryBuilder } = buildQueryBuilder();++ expect(() => queryBuilder.where({ missing: In(['x']) })).toThrow(+ TwentyOrmV2Exception,+ );+ });++ it('should reject a non-in operator in an object-literal where', () => {+ const { queryBuilder } = buildQueryBuilder();++ expect(() => queryBuilder.where({ id: LessThan('x') })).toThrow(+ TwentyOrmV2Exception,+ );+ });++ it('should reject a plain value in an object-literal where', () => {+ const { queryBuilder } = buildQueryBuilder();++ expect(() => queryBuilder.where({ companyId: 'company-1' })).toThrow(+ TwentyOrmV2Exception,+ );+ });});diff --git a/packages/twenty-server/src/engine/twenty-orm-v2/query-builder/types/query-builder-v2.type.ts b/packages/twenty-server/src/engine/twenty-orm-v2/query-builder/types/query-builder-v2.type.tsindex 2c594652404..2be319851ae 100644--- a/packages/twenty-server/src/engine/twenty-orm-v2/query-builder/types/query-builder-v2.type.ts+++ b/packages/twenty-server/src/engine/twenty-orm-v2/query-builder/types/query-builder-v2.type.ts@@ -4,17 +4,21 @@ export type WhereFactoryLike = {whereFactory: (queryBuilder: WhereExpressionLike) => void;};+export type ObjectWhereLike = Record<string, unknown>;++export type WhereConditionLike = string | WhereFactoryLike | ObjectWhereLike;+export type WhereExpressionLike = {where: (- condition: string | WhereFactoryLike,+ condition: WhereConditionLike,parameters?: Record<string, unknown>,) => WhereExpressionLike;andWhere: (- condition: string | WhereFactoryLike,+ condition: WhereConditionLike,parameters?: Record<string, unknown>,) => WhereExpressionLike;orWhere: (- condition: string | WhereFactoryLike,+ condition: WhereConditionLike,parameters?: Record<string, unknown>,) => WhereExpressionLike;};@@ -26,6 +30,13 @@ export const isWhereFactoryLike = (condition !== null &&typeof (condition as WhereFactoryLike).whereFactory === 'function';+export const isObjectWhereLike = (+ condition: unknown,+): condition is ObjectWhereLike =>+ typeof condition === 'object' &&+ condition !== null &&+ Object.getPrototypeOf(condition) === Object.prototype;+export const isNegatedWhereFactoryLike = (condition: unknown): boolean =>isWhereFactoryLike(condition) && InstanceChecker.isNotBrackets(condition);diff --git a/packages/twenty-server/src/engine/twenty-orm-v2/query-builder/workspace-select-query-builder-v2.ts b/packages/twenty-server/src/engine/twenty-orm-v2/query-builder/workspace-select-query-builder-v2.tsindex 96ecac9f466..953600e38cd 100644--- a/packages/twenty-server/src/engine/twenty-orm-v2/query-builder/workspace-select-query-builder-v2.ts+++ b/packages/twenty-server/src/engine/twenty-orm-v2/query-builder/workspace-select-query-builder-v2.ts@@ -1,5 +1,6 @@import { type ObjectsPermissions } from 'twenty-shared/types';import { isDefined } from 'twenty-shared/utils';+import { FindOperator } from 'typeorm';import { RelationType } from 'src/engine/metadata-modules/field-metadata/interfaces/relation-type.interface';@@ -11,10 +12,12 @@ import { type QueryExecutorV2 } from 'src/engine/twenty-orm-v2/executor/types/quimport {type ExpressionMapLike,type FindOptionsLike,+ type ObjectWhereLike,type OrderByConditionLike,+ type WhereConditionLike,type WhereExpressionLike,- type WhereFactoryLike,isNegatedWhereFactoryLike,+ isObjectWhereLike,isWhereFactoryLike,} from 'src/engine/twenty-orm-v2/query-builder/types/query-builder-v2.type';import { buildOrderByClauses } from 'src/engine/twenty-orm-v2/sql/utils/build-order-by-clauses.util';@@ -23,6 +26,7 @@ import { compileNamedParameters } from 'src/engine/twenty-orm-v2/sql/utils/compiimport {RESERVED_PARAMETER_NAMES,buildColumnNameByResultAlias,+ buildCountStatement,buildPaginationParameters,buildProjection,buildSelectStatement,@@ -38,6 +42,8 @@ import {} from 'src/engine/twenty-orm-v2/sql/utils/build-select-statement.util';import { type WorkspaceTableShape } from 'src/engine/twenty-orm-v2/table-shape/types/workspace-table-shape.type';+let objectWhereParameterSequence = 0;+export type QueryBuilderV2Context = {tableShape: WorkspaceTableShape;executor: QueryExecutorV2;@@ -117,7 +123,7 @@ export class WorkspaceSelectQueryBuilderV2 implements WhereExpressionLike {}where(- condition: string | WhereFactoryLike,+ condition: WhereConditionLike,parameters?: Record<string, unknown>,): this {this.whereClauses.length = 0;@@ -127,14 +133,14 @@ export class WorkspaceSelectQueryBuilderV2 implements WhereExpressionLike {}andWhere(- condition: string | WhereFactoryLike,+ condition: WhereConditionLike,parameters?: Record<string, unknown>,): this {return this.appendWhere('and', condition, parameters);}orWhere(- condition: string | WhereFactoryLike,+ condition: WhereConditionLike,parameters?: Record<string, unknown>,): this {return this.appendWhere('or', condition, parameters);@@ -282,6 +288,14 @@ export class WorkspaceSelectQueryBuilderV2 implements WhereExpressionLike {return this;}+ take(count: number): this {+ return this.limit(count);+ }++ skip(count: number): this {+ return this.offset(count);+ }+getQuery(): string {return this.buildSelectStatement().sql;}@@ -354,6 +368,16 @@ export class WorkspaceSelectQueryBuilderV2 implements WhereExpressionLike {}}+ async getCount(): Promise<number> {+ this.context.onBeforeExecute(this);++ const sql = buildCountStatement(this.toSelectStatementState());+ const compiled = compileNamedParameters(sql, this.parameters);+ const rows = await this.context.executor.execute(compiled);++ return Number(rows[0]?.count ?? 0);+ }+addJoinCondition(alias: string, condition: string): this {const joinClause = this.joinClauses.find((candidate) => candidate.alias === alias,@@ -396,7 +420,7 @@ export class WorkspaceSelectQueryBuilderV2 implements WhereExpressionLike {private appendWhere(operator: 'and' | 'or',- condition: string | WhereFactoryLike,+ condition: WhereConditionLike,parameters?: Record<string, unknown>,): this {if (isDefined(parameters)) {@@ -429,6 +453,19 @@ export class WorkspaceSelectQueryBuilderV2 implements WhereExpressionLike {return this;}+ if (isObjectWhereLike(condition)) {+ const { sql, parameters: objectParameters } =+ this.buildObjectWhereClause(condition);++ this.setParameters(objectParameters);++ if (sql.length > 0) {+ this.whereClauses.push({ operator, sql: `(${sql})` });+ }++ return this;+ }+if (condition.length > 0) {this.whereClauses.push({ operator, sql: `(${condition})` });}@@ -436,6 +473,39 @@ export class WorkspaceSelectQueryBuilderV2 implements WhereExpressionLike {return this;}+ private buildObjectWhereClause(where: ObjectWhereLike): {+ sql: string;+ parameters: Record<string, unknown>;+ } {+ const conditions: string[] = [];+ const parameters: Record<string, unknown> = {};++ for (const [columnName, value] of Object.entries(where)) {+ if (!isDefined(this.tableShape.columnShapeByColumnName[columnName])) {+ throw new TwentyOrmV2Exception(+ `Column "${columnName}" does not exist on "${this.tableShape.nameSingular}"`,+ TwentyOrmV2ExceptionCode.UNKNOWN_COLUMN,+ );+ }++ if (!(value instanceof FindOperator) || value.type !== 'in') {+ throw new TwentyOrmV2Exception(+ `Object where only supports the "in" operator on "${columnName}"`,+ TwentyOrmV2ExceptionCode.UNSUPPORTED_OPERATION,+ );+ }++ const parameterName = `ormV2ObjectWhere_${objectWhereParameterSequence++}`;++ conditions.push(+ `${quoteColumn(this.alias, columnName)} IN (:...${parameterName})`,+ );+ parameters[parameterName] = value.value;+ }++ return { sql: conditions.join(' AND '), parameters };+ }+private appendOrderBy(orderByOrExpression: OrderByConditionLike | string,direction: 'ASC' | 'DESC',diff --git a/packages/twenty-server/src/engine/twenty-orm-v2/sql/utils/build-select-statement.util.ts b/packages/twenty-server/src/engine/twenty-orm-v2/sql/utils/build-select-statement.util.tsindex 1cec3f6986e..72356a6173e 100644--- a/packages/twenty-server/src/engine/twenty-orm-v2/sql/utils/build-select-statement.util.ts+++ b/packages/twenty-server/src/engine/twenty-orm-v2/sql/utils/build-select-statement.util.ts@@ -230,6 +230,19 @@ export const buildSelectStatement = (state: SelectStatementState): string => {.join(' ');};+export const buildCountStatement = (state: SelectStatementState): string => {+ const whereExpression = buildWhereExpression(state);++ return [+ 'SELECT COUNT(1) AS "count"',+ buildFromClause(state),+ buildJoinClause(state),+ whereExpression.length > 0 ? `WHERE ${whereExpression}` : '',+ ]+ .filter((part) => part.length > 0)+ .join(' ');+};+// Only the projected columns of the main alias become entity properties: anything else// in the row, such as a relation order-by column, is raw join output.export const mapRowToEntity = <T extends Record<string, unknown>>(