The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Fix Kanban group-by queries with a record field or long column names (#26300)
Raphaël Bosi70c093869de7
diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants.tsindex d7e0d60fdee..a788780d6d4 100644--- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants.ts+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants.ts@@ -1,3 +1,5 @@export const RECORDS_PER_GROUP_LIMIT = 10;export const RELATIONS_PER_RECORD_LIMIT = 5;-export const SUB_QUERY_PREFIX = 'sub_query_';+export const SUB_QUERY_ALIAS = 'sub_query';+export const POSTGRES_FUNCTION_MAX_ARGUMENTS = 100;+export const JSONB_BUILD_OBJECT_MAX_PAIRS = POSTGRES_FUNCTION_MAX_ARGUMENTS / 2;diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.tsindex 3b6f534deeb..cf929749240 100644--- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts@@ -21,10 +21,12 @@ import { formatResultWithGroupByDimensionValues } from 'src/engine/api/graphql/gimport {RECORDS_PER_GROUP_LIMIT,RELATIONS_PER_RECORD_LIMIT,- SUB_QUERY_PREFIX,+ SUB_QUERY_ALIAS,} from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants';import { buildGroupByRecordConditions } from 'src/engine/api/graphql/graphql-query-runner/group-by/utils/build-group-by-record-conditions.util';+import { buildRecordJsonObjectSql } from 'src/engine/api/graphql/graphql-query-runner/group-by/utils/build-record-json-object-sql.util';import { getGroupLimit } from 'src/engine/api/graphql/graphql-query-runner/group-by/utils/get-group-limit.util';+import { buildColumnResultAlias } from 'src/engine/twenty-orm/sql/utils/build-column-result-alias.util';import { buildColumnsToSelect } from 'src/engine/api/graphql/graphql-query-runner/utils/build-columns-to-select';import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';import { type OrmFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/orm-flat-field-metadata.type';@@ -172,10 +174,19 @@ export class GroupByWithRecordsService {subQueryBuilder.select([]);- for (const columnName of Object.keys(columnsToSelect)) {+ const subQueryAliasByColumnName = Object.fromEntries(+ Object.keys(columnsToSelect).map((columnName) => [+ columnName,+ buildColumnResultAlias(SUB_QUERY_ALIAS, columnName),+ ]),+ );++ for (const [columnName, subQueryAlias] of Object.entries(+ subQueryAliasByColumnName,+ )) {subQueryBuilder.addSelect(`"${objectAlias}"."${columnName}"`,- `${SUB_QUERY_PREFIX}${columnName}`,+ subQueryAlias,);}@@ -213,20 +224,16 @@ export class GroupByWithRecordsService {const pageStart = offsetForRecords;const pageEnd = offsetForRecords + RECORDS_PER_GROUP_LIMIT;- const recordColumnsToSelect = [- ...Object.keys(columnsToSelect).map(- (columnName) => `"${SUB_QUERY_PREFIX}${columnName}" AS "${columnName}"`,- ),- ...groupByDefinitions.map(- (groupByDefinition) => `"${groupByDefinition.alias}"`,- ),- ].join(', ');+ const recordJsonObjectSql = buildRecordJsonObjectSql({+ subQueryAliasByColumnName,+ groupByDefinitions,+ });const pageFilter = `record_row_number > ${pageStart} AND record_row_number <= ${pageEnd}`;const sql =`SELECT ${groupByAliases}, ` +- `JSON_AGG(CASE WHEN ${pageFilter} THEN (SELECT ROW_TO_JSON(record) FROM (SELECT ${recordColumnsToSelect}) AS record) END) ` ++ `JSONB_AGG(${recordJsonObjectSql} ORDER BY record_row_number) ` +`FILTER (WHERE ${pageFilter}) AS "records" ` +`FROM (${subQueryBuilder.getQuery()}) AS "ranked_records" ` +`GROUP BY ${groupByAliases}`;diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/__tests__/build-record-json-object-sql.util.spec.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/__tests__/build-record-json-object-sql.util.spec.tsnew file mode 100644index 00000000000..52dabf0a620--- /dev/null+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/__tests__/build-record-json-object-sql.util.spec.ts@@ -0,0 +1,119 @@+import { type GroupByDefinition } from 'src/engine/api/common/common-query-runners/types/group-by-definition.type';+import { buildRecordJsonObjectSql } from 'src/engine/api/graphql/graphql-query-runner/group-by/utils/build-record-json-object-sql.util';+import {+ JSONB_BUILD_OBJECT_MAX_PAIRS,+ SUB_QUERY_ALIAS,+} from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants';+import { buildColumnResultAlias } from 'src/engine/twenty-orm/sql/utils/build-column-result-alias.util';++const buildGroupByDefinition = (+ overrides: Partial<GroupByDefinition> = {},+): GroupByDefinition => ({+ columnNameWithQuotes: '"company"."companyId"',+ expression: '"company"."companyId"',+ alias: 'company_companyId',+ ...overrides,+});++const buildSubQueryAliases = (columnNames: string[]): Record<string, string> =>+ Object.fromEntries(+ columnNames.map((columnName) => [+ columnName,+ buildColumnResultAlias(SUB_QUERY_ALIAS, columnName),+ ]),+ );++describe('buildRecordJsonObjectSql', () => {+ it('should emit a single JSONB_BUILD_OBJECT for a short column list', () => {+ const sql = buildRecordJsonObjectSql({+ subQueryAliasByColumnName: buildSubQueryAliases(['id', 'name']),+ groupByDefinitions: [buildGroupByDefinition()],+ });++ expect(sql).toBe(+ `JSONB_BUILD_OBJECT('id', "sub_query_id", 'name', "sub_query_name", ` ++ `'company_companyId', "company_companyId")`,+ );+ });++ it('should keep a column named record as a literal key and never as an identifier', () => {+ const sql = buildRecordJsonObjectSql({+ subQueryAliasByColumnName: buildSubQueryAliases(['id', 'record']),+ groupByDefinitions: [buildGroupByDefinition()],+ });++ expect(sql).toContain(`'record', "sub_query_record"`);+ expect(sql).not.toContain('"record"');+ });++ it('should chunk past the postgres function argument limit', () => {+ const columnCount = JSONB_BUILD_OBJECT_MAX_PAIRS + 2;+ const columnNames = Array.from(+ { length: columnCount },+ (_unused, index) => `column${index}`,+ );++ const sql = buildRecordJsonObjectSql({+ subQueryAliasByColumnName: buildSubQueryAliases(columnNames),+ groupByDefinitions: [buildGroupByDefinition()],+ });++ const chunks = sql.split(' || ');++ expect(chunks).toHaveLength(2);+ expect(chunks[0].split(', ').length / 2).toBe(JSONB_BUILD_OBJECT_MAX_PAIRS);+ expect(chunks[1].split(', ').length / 2).toBe(+ columnCount + 1 - JSONB_BUILD_OBJECT_MAX_PAIRS,+ );+ expect(+ chunks.every((chunkSql) => chunkSql.startsWith('JSONB_BUILD_OBJECT(')),+ ).toBe(true);+ });++ it('should keep the full column name as the key when its alias was hashed', () => {+ const longColumnName = `shippingAddress${'X'.repeat(50)}AddressPostcode`;+ const subQueryAliasByColumnName = buildSubQueryAliases([longColumnName]);+ const hashedAlias = subQueryAliasByColumnName[longColumnName];++ const sql = buildRecordJsonObjectSql({+ subQueryAliasByColumnName,+ groupByDefinitions: [buildGroupByDefinition()],+ });++ expect(hashedAlias).not.toContain(longColumnName);+ expect(hashedAlias.length).toBeLessThanOrEqual(63);+ expect(sql).toContain(`'${longColumnName}', "${hashedAlias}"`);+ });++ it('should give colliding composite subfield columns distinct aliases', () => {+ const fieldName = 'a'.repeat(40);+ const subQueryAliasByColumnName = buildSubQueryAliases([+ `${fieldName}AddressStreet1`,+ `${fieldName}AddressStreet2`,+ ]);++ const [firstAlias, secondAlias] = Object.values(subQueryAliasByColumnName);++ expect(firstAlias).not.toBe(secondAlias);+ expect(firstAlias.length).toBeLessThanOrEqual(63);+ expect(secondAlias.length).toBeLessThanOrEqual(63);+ });++ it('should escape single quotes in a column name', () => {+ const sql = buildRecordJsonObjectSql({+ subQueryAliasByColumnName: { "it's": 'sub_query_quoted' },+ groupByDefinitions: [],+ });++ expect(sql).toBe(`JSONB_BUILD_OBJECT('it''s', "sub_query_quoted")`);+ });++ it('should emit an empty object when there is nothing to select', () => {+ const sql = buildRecordJsonObjectSql({+ subQueryAliasByColumnName: {},+ groupByDefinitions: [],+ });++ expect(sql).toBe('JSONB_BUILD_OBJECT()');+ });+});diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/build-record-json-object-sql.util.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/build-record-json-object-sql.util.tsnew file mode 100644index 00000000000..f35d05d5232--- /dev/null+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/build-record-json-object-sql.util.ts@@ -0,0 +1,38 @@+import chunk from 'lodash.chunk';+import { isNonEmptyArray } from 'twenty-shared/utils';++import { type GroupByDefinition } from 'src/engine/api/common/common-query-runners/types/group-by-definition.type';+import { JSONB_BUILD_OBJECT_MAX_PAIRS } from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants';+import {+ escapeIdentifier,+ escapeLiteral,+} from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';++export const buildRecordJsonObjectSql = ({+ subQueryAliasByColumnName,+ groupByDefinitions,+}: {+ subQueryAliasByColumnName: Record<string, string>;+ groupByDefinitions: GroupByDefinition[];+}): string => {+ const keyValueEntries = [+ ...Object.entries(subQueryAliasByColumnName).map(+ ([columnName, subQueryAlias]) =>+ `${escapeLiteral(columnName)}, ${escapeIdentifier(subQueryAlias)}`,+ ),+ ...groupByDefinitions.map(+ (groupByDefinition) =>+ `${escapeLiteral(groupByDefinition.alias)}, ${escapeIdentifier(+ groupByDefinition.alias,+ )}`,+ ),+ ];++ if (!isNonEmptyArray(keyValueEntries)) {+ return 'JSONB_BUILD_OBJECT()';+ }++ return chunk(keyValueEntries, JSONB_BUILD_OBJECT_MAX_PAIRS)+ .map((entries) => `JSONB_BUILD_OBJECT(${entries.join(', ')})`)+ .join(' || ');+};diff --git a/packages/twenty-server/test/integration/graphql/suites/group-by-with-many-record-fields.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/group-by-with-many-record-fields.integration-spec.tsindex e36b2a27ab0..d0ceb1b37cd 100644--- a/packages/twenty-server/test/integration/graphql/suites/group-by-with-many-record-fields.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/group-by-with-many-record-fields.integration-spec.ts@@ -12,6 +12,8 @@ import { updateOneObjectMetadata } from 'test/integration/metadata/suites/objectimport { FieldMetadataType, RelationType } from 'twenty-shared/types';import { isDefined } from 'twenty-shared/utils';+import { JSONB_BUILD_OBJECT_MAX_PAIRS } from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants';+const OBJECT_NAME_SINGULAR = 'groupByManyFields';const OBJECT_NAME_PLURAL = 'groupByManyFieldsRecords';const ADDRESS_FIELD_NAMES = [@@ -22,6 +24,7 @@ const ADDRESS_FIELD_NAMES = ['warehouseAddress','returnAddress',];+const ROW_ALIAS_SHADOWING_FIELD_NAME = 'record';const ADDRESS_VALUES = Object.fromEntries(ADDRESS_FIELD_NAMES.map((fieldName) => [fieldName,@@ -42,14 +45,23 @@ const RECORDS = [11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1].map((position) => ({name: `Record ${position}`,position,companyId: null,+ [ROW_ALIAS_SHADOWING_FIELD_NAME]: `Shadowing value ${position}`,...ADDRESS_VALUES,}));const ORDERED_RECORDS = [...RECORDS].reverse();+const ADDRESS_SUBFIELD_COUNT = Object.keys(+ ADDRESS_VALUES[ADDRESS_FIELD_NAMES[0]],+).length;+const SCALAR_FIELD_COUNT =+ Object.keys(RECORDS[0]).length - ADDRESS_FIELD_NAMES.length;+const SELECTED_RECORD_COLUMN_COUNT =+ SCALAR_FIELD_COUNT + ADDRESS_FIELD_NAMES.length * ADDRESS_SUBFIELD_COUNT;const RECORD_GQL_FIELDS = `idnamepositioncompanyId+ ${ROW_ALIAS_SHADOWING_FIELD_NAME}${ADDRESS_FIELD_NAMES.map((fieldName) => `${fieldName} {@@ -99,6 +111,17 @@ describe('group-by with more than 50 selected record columns', () => {expect(fieldErrors).toBeUndefined();}+ await createOneFieldMetadata({+ expectToFail: false,+ input: {+ objectMetadataId,+ type: FieldMetadataType.TEXT,+ name: ROW_ALIAS_SHADOWING_FIELD_NAME,+ label: ROW_ALIAS_SHADOWING_FIELD_NAME,+ isLabelSyncedWithName: false,+ },+ });+const { objects } = await findManyObjectMetadata({expectToFail: false,input: { filter: {}, paging: { first: 100 } },@@ -151,6 +174,12 @@ describe('group-by with more than 50 selected record columns', () => {});});+ it('selects more record columns than one JSONB_BUILD_OBJECT call accepts', () => {+ expect(SELECTED_RECORD_COLUMN_COUNT).toBeGreaterThan(+ JSONB_BUILD_OBJECT_MAX_PAIRS,+ );+ });+it.each([{ offsetForRecords: 0, expectedRecords: ORDERED_RECORDS.slice(0, 10) },{ offsetForRecords: 10, expectedRecords: ORDERED_RECORDS.slice(10) },