Skip to content

The open alternative to Salesforce, designed for AI.

Fix Kanban group-by queries with a record field or long column names (#26300)

Raphaël Bosi70c093869de7
diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants.ts
index d7e0d60fdee..a788780d6d4 100644
--- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants.ts
+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants.ts
@@ -1,3 +1,5 @@
export const RECORDS_PER_GROUP_LIMIT = 10;
export const RELATIONS_PER_RECORD_LIMIT = 5;
-export const SUB_QUERY_PREFIX = 'sub_query_';
+export const SUB_QUERY_ALIAS = 'sub_query';
+export const POSTGRES_FUNCTION_MAX_ARGUMENTS = 100;
+export const JSONB_BUILD_OBJECT_MAX_PAIRS = POSTGRES_FUNCTION_MAX_ARGUMENTS / 2;
diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts
index 3b6f534deeb..cf929749240 100644
--- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts
+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts
@@ -21,10 +21,12 @@ import { formatResultWithGroupByDimensionValues } from 'src/engine/api/graphql/g
import {
RECORDS_PER_GROUP_LIMIT,
RELATIONS_PER_RECORD_LIMIT,
- SUB_QUERY_PREFIX,
+ SUB_QUERY_ALIAS,
} from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants';
import { buildGroupByRecordConditions } from 'src/engine/api/graphql/graphql-query-runner/group-by/utils/build-group-by-record-conditions.util';
+import { buildRecordJsonObjectSql } from 'src/engine/api/graphql/graphql-query-runner/group-by/utils/build-record-json-object-sql.util';
import { getGroupLimit } from 'src/engine/api/graphql/graphql-query-runner/group-by/utils/get-group-limit.util';
+import { buildColumnResultAlias } from 'src/engine/twenty-orm/sql/utils/build-column-result-alias.util';
import { buildColumnsToSelect } from 'src/engine/api/graphql/graphql-query-runner/utils/build-columns-to-select';
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
import { type OrmFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/orm-flat-field-metadata.type';
@@ -172,10 +174,19 @@ export class GroupByWithRecordsService {
subQueryBuilder.select([]);
- for (const columnName of Object.keys(columnsToSelect)) {
+ const subQueryAliasByColumnName = Object.fromEntries(
+ Object.keys(columnsToSelect).map((columnName) => [
+ columnName,
+ buildColumnResultAlias(SUB_QUERY_ALIAS, columnName),
+ ]),
+ );
+
+ for (const [columnName, subQueryAlias] of Object.entries(
+ subQueryAliasByColumnName,
+ )) {
subQueryBuilder.addSelect(
`"${objectAlias}"."${columnName}"`,
- `${SUB_QUERY_PREFIX}${columnName}`,
+ subQueryAlias,
);
}
@@ -213,20 +224,16 @@ export class GroupByWithRecordsService {
const pageStart = offsetForRecords;
const pageEnd = offsetForRecords + RECORDS_PER_GROUP_LIMIT;
- const recordColumnsToSelect = [
- ...Object.keys(columnsToSelect).map(
- (columnName) => `"${SUB_QUERY_PREFIX}${columnName}" AS "${columnName}"`,
- ),
- ...groupByDefinitions.map(
- (groupByDefinition) => `"${groupByDefinition.alias}"`,
- ),
- ].join(', ');
+ const recordJsonObjectSql = buildRecordJsonObjectSql({
+ subQueryAliasByColumnName,
+ groupByDefinitions,
+ });
const pageFilter = `record_row_number > ${pageStart} AND record_row_number <= ${pageEnd}`;
const sql =
`SELECT ${groupByAliases}, ` +
- `JSON_AGG(CASE WHEN ${pageFilter} THEN (SELECT ROW_TO_JSON(record) FROM (SELECT ${recordColumnsToSelect}) AS record) END) ` +
+ `JSONB_AGG(${recordJsonObjectSql} ORDER BY record_row_number) ` +
`FILTER (WHERE ${pageFilter}) AS "records" ` +
`FROM (${subQueryBuilder.getQuery()}) AS "ranked_records" ` +
`GROUP BY ${groupByAliases}`;
diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/__tests__/build-record-json-object-sql.util.spec.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/__tests__/build-record-json-object-sql.util.spec.ts
new file mode 100644
index 00000000000..52dabf0a620
--- /dev/null
+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/__tests__/build-record-json-object-sql.util.spec.ts
@@ -0,0 +1,119 @@
+import { type GroupByDefinition } from 'src/engine/api/common/common-query-runners/types/group-by-definition.type';
+import { buildRecordJsonObjectSql } from 'src/engine/api/graphql/graphql-query-runner/group-by/utils/build-record-json-object-sql.util';
+import {
+ JSONB_BUILD_OBJECT_MAX_PAIRS,
+ SUB_QUERY_ALIAS,
+} from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants';
+import { buildColumnResultAlias } from 'src/engine/twenty-orm/sql/utils/build-column-result-alias.util';
+
+const buildGroupByDefinition = (
+ overrides: Partial<GroupByDefinition> = {},
+): GroupByDefinition => ({
+ columnNameWithQuotes: '"company"."companyId"',
+ expression: '"company"."companyId"',
+ alias: 'company_companyId',
+ ...overrides,
+});
+
+const buildSubQueryAliases = (columnNames: string[]): Record<string, string> =>
+ Object.fromEntries(
+ columnNames.map((columnName) => [
+ columnName,
+ buildColumnResultAlias(SUB_QUERY_ALIAS, columnName),
+ ]),
+ );
+
+describe('buildRecordJsonObjectSql', () => {
+ it('should emit a single JSONB_BUILD_OBJECT for a short column list', () => {
+ const sql = buildRecordJsonObjectSql({
+ subQueryAliasByColumnName: buildSubQueryAliases(['id', 'name']),
+ groupByDefinitions: [buildGroupByDefinition()],
+ });
+
+ expect(sql).toBe(
+ `JSONB_BUILD_OBJECT('id', "sub_query_id", 'name', "sub_query_name", ` +
+ `'company_companyId', "company_companyId")`,
+ );
+ });
+
+ it('should keep a column named record as a literal key and never as an identifier', () => {
+ const sql = buildRecordJsonObjectSql({
+ subQueryAliasByColumnName: buildSubQueryAliases(['id', 'record']),
+ groupByDefinitions: [buildGroupByDefinition()],
+ });
+
+ expect(sql).toContain(`'record', "sub_query_record"`);
+ expect(sql).not.toContain('"record"');
+ });
+
+ it('should chunk past the postgres function argument limit', () => {
+ const columnCount = JSONB_BUILD_OBJECT_MAX_PAIRS + 2;
+ const columnNames = Array.from(
+ { length: columnCount },
+ (_unused, index) => `column${index}`,
+ );
+
+ const sql = buildRecordJsonObjectSql({
+ subQueryAliasByColumnName: buildSubQueryAliases(columnNames),
+ groupByDefinitions: [buildGroupByDefinition()],
+ });
+
+ const chunks = sql.split(' || ');
+
+ expect(chunks).toHaveLength(2);
+ expect(chunks[0].split(', ').length / 2).toBe(JSONB_BUILD_OBJECT_MAX_PAIRS);
+ expect(chunks[1].split(', ').length / 2).toBe(
+ columnCount + 1 - JSONB_BUILD_OBJECT_MAX_PAIRS,
+ );
+ expect(
+ chunks.every((chunkSql) => chunkSql.startsWith('JSONB_BUILD_OBJECT(')),
+ ).toBe(true);
+ });
+
+ it('should keep the full column name as the key when its alias was hashed', () => {
+ const longColumnName = `shippingAddress${'X'.repeat(50)}AddressPostcode`;
+ const subQueryAliasByColumnName = buildSubQueryAliases([longColumnName]);
+ const hashedAlias = subQueryAliasByColumnName[longColumnName];
+
+ const sql = buildRecordJsonObjectSql({
+ subQueryAliasByColumnName,
+ groupByDefinitions: [buildGroupByDefinition()],
+ });
+
+ expect(hashedAlias).not.toContain(longColumnName);
+ expect(hashedAlias.length).toBeLessThanOrEqual(63);
+ expect(sql).toContain(`'${longColumnName}', "${hashedAlias}"`);
+ });
+
+ it('should give colliding composite subfield columns distinct aliases', () => {
+ const fieldName = 'a'.repeat(40);
+ const subQueryAliasByColumnName = buildSubQueryAliases([
+ `${fieldName}AddressStreet1`,
+ `${fieldName}AddressStreet2`,
+ ]);
+
+ const [firstAlias, secondAlias] = Object.values(subQueryAliasByColumnName);
+
+ expect(firstAlias).not.toBe(secondAlias);
+ expect(firstAlias.length).toBeLessThanOrEqual(63);
+ expect(secondAlias.length).toBeLessThanOrEqual(63);
+ });
+
+ it('should escape single quotes in a column name', () => {
+ const sql = buildRecordJsonObjectSql({
+ subQueryAliasByColumnName: { "it's": 'sub_query_quoted' },
+ groupByDefinitions: [],
+ });
+
+ expect(sql).toBe(`JSONB_BUILD_OBJECT('it''s', "sub_query_quoted")`);
+ });
+
+ it('should emit an empty object when there is nothing to select', () => {
+ const sql = buildRecordJsonObjectSql({
+ subQueryAliasByColumnName: {},
+ groupByDefinitions: [],
+ });
+
+ expect(sql).toBe('JSONB_BUILD_OBJECT()');
+ });
+});
diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/build-record-json-object-sql.util.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/build-record-json-object-sql.util.ts
new file mode 100644
index 00000000000..f35d05d5232
--- /dev/null
+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/build-record-json-object-sql.util.ts
@@ -0,0 +1,38 @@
+import chunk from 'lodash.chunk';
+import { isNonEmptyArray } from 'twenty-shared/utils';
+
+import { type GroupByDefinition } from 'src/engine/api/common/common-query-runners/types/group-by-definition.type';
+import { JSONB_BUILD_OBJECT_MAX_PAIRS } from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants';
+import {
+ escapeIdentifier,
+ escapeLiteral,
+} from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';
+
+export const buildRecordJsonObjectSql = ({
+ subQueryAliasByColumnName,
+ groupByDefinitions,
+}: {
+ subQueryAliasByColumnName: Record<string, string>;
+ groupByDefinitions: GroupByDefinition[];
+}): string => {
+ const keyValueEntries = [
+ ...Object.entries(subQueryAliasByColumnName).map(
+ ([columnName, subQueryAlias]) =>
+ `${escapeLiteral(columnName)}, ${escapeIdentifier(subQueryAlias)}`,
+ ),
+ ...groupByDefinitions.map(
+ (groupByDefinition) =>
+ `${escapeLiteral(groupByDefinition.alias)}, ${escapeIdentifier(
+ groupByDefinition.alias,
+ )}`,
+ ),
+ ];
+
+ if (!isNonEmptyArray(keyValueEntries)) {
+ return 'JSONB_BUILD_OBJECT()';
+ }
+
+ return chunk(keyValueEntries, JSONB_BUILD_OBJECT_MAX_PAIRS)
+ .map((entries) => `JSONB_BUILD_OBJECT(${entries.join(', ')})`)
+ .join(' || ');
+};
diff --git a/packages/twenty-server/test/integration/graphql/suites/group-by-with-many-record-fields.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/group-by-with-many-record-fields.integration-spec.ts
index e36b2a27ab0..d0ceb1b37cd 100644
--- a/packages/twenty-server/test/integration/graphql/suites/group-by-with-many-record-fields.integration-spec.ts
+++ b/packages/twenty-server/test/integration/graphql/suites/group-by-with-many-record-fields.integration-spec.ts
@@ -12,6 +12,8 @@ import { updateOneObjectMetadata } from 'test/integration/metadata/suites/object
import { FieldMetadataType, RelationType } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
+import { JSONB_BUILD_OBJECT_MAX_PAIRS } from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.constants';
+
const OBJECT_NAME_SINGULAR = 'groupByManyFields';
const OBJECT_NAME_PLURAL = 'groupByManyFieldsRecords';
const ADDRESS_FIELD_NAMES = [
@@ -22,6 +24,7 @@ const ADDRESS_FIELD_NAMES = [
'warehouseAddress',
'returnAddress',
];
+const ROW_ALIAS_SHADOWING_FIELD_NAME = 'record';
const ADDRESS_VALUES = Object.fromEntries(
ADDRESS_FIELD_NAMES.map((fieldName) => [
fieldName,
@@ -42,14 +45,23 @@ const RECORDS = [11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1].map((position) => ({
name: `Record ${position}`,
position,
companyId: null,
+ [ROW_ALIAS_SHADOWING_FIELD_NAME]: `Shadowing value ${position}`,
...ADDRESS_VALUES,
}));
const ORDERED_RECORDS = [...RECORDS].reverse();
+const ADDRESS_SUBFIELD_COUNT = Object.keys(
+ ADDRESS_VALUES[ADDRESS_FIELD_NAMES[0]],
+).length;
+const SCALAR_FIELD_COUNT =
+ Object.keys(RECORDS[0]).length - ADDRESS_FIELD_NAMES.length;
+const SELECTED_RECORD_COLUMN_COUNT =
+ SCALAR_FIELD_COUNT + ADDRESS_FIELD_NAMES.length * ADDRESS_SUBFIELD_COUNT;
const RECORD_GQL_FIELDS = `
id
name
position
companyId
+ ${ROW_ALIAS_SHADOWING_FIELD_NAME}
${ADDRESS_FIELD_NAMES.map(
(fieldName) => `
${fieldName} {
@@ -99,6 +111,17 @@ describe('group-by with more than 50 selected record columns', () => {
expect(fieldErrors).toBeUndefined();
}
+ await createOneFieldMetadata({
+ expectToFail: false,
+ input: {
+ objectMetadataId,
+ type: FieldMetadataType.TEXT,
+ name: ROW_ALIAS_SHADOWING_FIELD_NAME,
+ label: ROW_ALIAS_SHADOWING_FIELD_NAME,
+ isLabelSyncedWithName: false,
+ },
+ });
+
const { objects } = await findManyObjectMetadata({
expectToFail: false,
input: { filter: {}, paging: { first: 100 } },
@@ -151,6 +174,12 @@ describe('group-by with more than 50 selected record columns', () => {
});
});
+ it('selects more record columns than one JSONB_BUILD_OBJECT call accepts', () => {
+ expect(SELECTED_RECORD_COLUMN_COUNT).toBeGreaterThan(
+ JSONB_BUILD_OBJECT_MAX_PAIRS,
+ );
+ });
+
it.each([
{ offsetForRecords: 0, expectedRecords: ORDERED_RECORDS.slice(0, 10) },
{ offsetForRecords: 10, expectedRecords: ORDERED_RECORDS.slice(10) },