The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix: REST cursor encoding for nested order_by composite fields (#20974)
Matt Van Horn6ac797a69cb8
diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-duplicates-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-duplicates-query-runner.service.tsindex 887539bb13d..d7337a2877f 100644--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-duplicates-query-runner.service.ts+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-duplicates-query-runner.service.ts@@ -135,6 +135,8 @@ export class CommonFindDuplicatesQueryRunnerService extends CommonBaseQueryRunne[{ id: OrderByDirection.AscNullsFirst }],QUERY_MAX_RECORDS,true,+ flatObjectMetadata,+ flatFieldMetadataMaps,);return {diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.tsindex f1f0c139164..f719a64f296 100644--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts@@ -185,6 +185,8 @@ export class CommonFindManyQueryRunnerService extends CommonBaseQueryRunnerServiorderByWithIdCondition,limit,isForwardPagination,+ flatObjectMetadata,+ flatFieldMetadataMaps,);if (!isForwardPagination) {diff --git a/packages/twenty-server/src/engine/api/common/utils/get-page-info.util.ts b/packages/twenty-server/src/engine/api/common/utils/get-page-info.util.tsindex bb78bb4129c..ed44fede070 100644--- a/packages/twenty-server/src/engine/api/common/utils/get-page-info.util.ts+++ b/packages/twenty-server/src/engine/api/common/utils/get-page-info.util.ts@@ -7,12 +7,17 @@ import {encodeCursor,getPaginationInfo,} from 'src/engine/api/graphql/graphql-query-runner/utils/cursors.util';+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';export const getPageInfo = (records: ObjectRecord[],orderBy: ObjectRecordOrderBy,limit: number,isForwardPagination: boolean,+ flatObjectMetadata: FlatObjectMetadata,+ flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,): CommonPageInfo => {const { hasNextPage, hasPreviousPage, hasMoreRecords } = getPaginationInfo(records,@@ -25,10 +30,22 @@ export const getPageInfo = (}const startCursor =- records.length > 0 ? encodeCursor(records[0], orderBy) : null;+ records.length > 0+ ? encodeCursor({+ objectRecord: records[0],+ order: orderBy,+ flatObjectMetadata,+ flatFieldMetadataMaps,+ })+ : null;const endCursor =records.length > 0- ? encodeCursor(records[records.length - 1], orderBy)+ ? encodeCursor({+ objectRecord: records[records.length - 1],+ order: orderBy,+ flatObjectMetadata,+ flatFieldMetadataMaps,+ }): null;return { startCursor, endCursor, hasNextPage, hasPreviousPage };diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper.tsindex 8c9529bd369..ebdc61df092 100644--- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper.ts+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper.ts@@ -69,6 +69,12 @@ export class ObjectRecordsToGraphqlConnectionHelper {hasPreviousPage: boolean;depth?: number;}): IConnection<T> {+ const objectMetadataId = this.objectIdByNameSingular[objectName];+ const flatObjectMetadata = findFlatEntityByIdInFlatEntityMapsOrThrow({+ flatEntityId: objectMetadataId,+ flatEntityMaps: this.flatObjectMetadataMaps,+ });+const edges = (objectRecords ?? []).map((objectRecord) => ({node: this.processRecord({objectRecord,@@ -80,7 +86,12 @@ export class ObjectRecordsToGraphqlConnectionHelper {order,depth,}),- cursor: encodeCursor(objectRecord, order),+ cursor: encodeCursor({+ objectRecord,+ order,+ flatObjectMetadata,+ flatFieldMetadataMaps: this.flatFieldMetadataMaps,+ }),}));const aggregatedFieldsValues = this.extractAggregatedFieldsValues({diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/__tests__/cursors.util.spec.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/__tests__/cursors.util.spec.tsnew file mode 100644index 00000000000..0f608e8d01c--- /dev/null+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/__tests__/cursors.util.spec.ts@@ -0,0 +1,181 @@+import { FieldMetadataType, OrderByDirection } from 'twenty-shared/types';++import { type ObjectRecordOrderBy } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';++import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';+import { decodeCursor, encodeCursor } from '../cursors.util';++const buildMockField = (+ id: string,+ name: string,+ type: FieldMetadataType,+): FlatFieldMetadata =>+ ({+ id,+ universalIdentifier: id,+ name,+ type,+ objectMetadataId: 'obj-id',+ workspaceId: 'ws-id',+ label: name,+ isNullable: true,+ isLabelSyncedWithName: true,+ createdAt: new Date(),+ updatedAt: new Date(),+ viewFieldIds: [],+ viewFilterIds: [],+ kanbanAggregateOperationViewIds: [],+ calendarViewIds: [],+ applicationId: null,+ }) as unknown as FlatFieldMetadata;++const nameField = buildMockField('name-id', 'name', FieldMetadataType.TEXT);+const fullNameField = buildMockField(+ 'fullname-id',+ 'fullName',+ FieldMetadataType.FULL_NAME,+);++const flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata> = {+ byUniversalIdentifier: {+ 'name-id': nameField,+ 'fullname-id': fullNameField,+ },+ universalIdentifierById: {+ 'name-id': 'name-id',+ 'fullname-id': 'fullname-id',+ },+ universalIdentifiersByApplicationId: {},+};++const flatObjectMetadata: FlatObjectMetadata = {+ id: 'obj-id',+ universalIdentifier: 'obj-id',+ workspaceId: 'ws-id',+ nameSingular: 'person',+ namePlural: 'people',+ labelSingular: 'Person',+ labelPlural: 'People',+ targetTableName: 'person',+ isCustom: false,+ isRemote: false,+ isActive: true,+ isSystem: false,+ isAuditLogged: false,+ isSearchable: false,+ icon: 'Icon123',+ createdAt: new Date(),+ updatedAt: new Date(),+ fieldIds: ['name-id', 'fullname-id'],+ indexMetadataIds: [],+ viewIds: [],+ applicationId: null,+} as unknown as FlatObjectMetadata;++const callEncodeCursor = (+ record: Record<string, unknown>,+ orderBy: Parameters<typeof encodeCursor>[0]['order'],+) =>+ encodeCursor({+ objectRecord: record as never,+ order: orderBy,+ flatObjectMetadata,+ flatFieldMetadataMaps,+ });++describe('encodeCursor', () => {+ it('should encode scalar fields from the orderBy', () => {+ const record = { id: 'abc', name: 'John', age: 30 };+ const orderBy = [{ name: OrderByDirection.AscNullsLast }];++ const decoded = decodeCursor(callEncodeCursor(record, orderBy));++ expect(decoded).toEqual({ name: 'John', id: 'abc' });+ });++ it('should always include id even if not in orderBy', () => {+ const record = { id: 'abc', name: 'John' };+ const orderBy = [{ name: OrderByDirection.AscNullsLast }];++ const decoded = decodeCursor(callEncodeCursor(record, orderBy));++ expect(decoded).toHaveProperty('id', 'abc');+ });++ it('should only include ordered sub-fields for composite fields', () => {+ const record = {+ id: 'abc',+ fullName: { firstName: 'Katherine', lastName: 'Abbott' },+ };+ const orderBy = [+ { fullName: { firstName: OrderByDirection.AscNullsLast } },+ ];++ const decoded = decodeCursor(callEncodeCursor(record, orderBy));++ expect(decoded).toEqual({+ fullName: { firstName: 'Katherine' },+ id: 'abc',+ });+ expect(decoded.fullName).not.toHaveProperty('lastName');+ });++ it('should include all sub-fields when all are in the orderBy', () => {+ const record = {+ id: 'abc',+ fullName: { firstName: 'Katherine', lastName: 'Abbott' },+ };+ const orderBy = [+ {+ fullName: {+ firstName: OrderByDirection.AscNullsLast,+ lastName: OrderByDirection.AscNullsLast,+ },+ },+ ];++ const decoded = decodeCursor(callEncodeCursor(record, orderBy));++ expect(decoded).toEqual({+ fullName: { firstName: 'Katherine', lastName: 'Abbott' },+ id: 'abc',+ });+ });++ it('should deep-merge two separate entries for the same composite parent', () => {+ const record = {+ id: 'abc',+ fullName: { firstName: 'Katherine', lastName: 'Watts' },+ };+ const orderBy: ObjectRecordOrderBy = [+ { fullName: { firstName: OrderByDirection.AscNullsFirst } },+ { fullName: { lastName: OrderByDirection.DescNullsLast } },+ ];++ const decoded = decodeCursor(callEncodeCursor(record, orderBy));++ expect(decoded).toEqual({+ fullName: { firstName: 'Katherine', lastName: 'Watts' },+ id: 'abc',+ });+ });++ it('should not filter sub-fields for scalar fields', () => {+ const record = { id: 'abc', name: 'John' };+ const orderBy = [{ name: OrderByDirection.AscNullsLast }];++ const decoded = decodeCursor(callEncodeCursor(record, orderBy));++ expect(decoded).toEqual({ name: 'John', id: 'abc' });+ });++ it('should handle undefined orderBy', () => {+ const record = { id: 'abc', name: 'John' };++ const decoded = decodeCursor(callEncodeCursor(record, undefined));++ expect(decoded).toEqual({ id: 'abc' });+ });+});diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/cursors.util.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/cursors.util.tsindex 99810f919c0..0063026e5aa 100644--- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/cursors.util.ts+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/cursors.util.ts@@ -1,4 +1,5 @@import { type ObjectRecord } from 'twenty-shared/types';+import { isDefined, isPlainObject } from 'twenty-shared/utils';import { type ObjectRecordOrderBy } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';import { type FindManyResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';@@ -8,6 +9,12 @@ import {CommonQueryRunnerExceptionCode,} from 'src/engine/api/common/common-query-runners/errors/common-query-runner.exception';import { STANDARD_ERROR_MESSAGE } from 'src/engine/api/common/common-query-runners/errors/standard-error-message.constant';+import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';+import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';+import { buildFieldMapsFromFlatObjectMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-from-flat-object-metadata.util';+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';export interface CursorData {// oxlint-disable-next-line @typescripttypescript/no-explicit-any@@ -26,20 +33,62 @@ export const decodeCursor = <T = CursorData>(cursor: string): T => {}};-export const encodeCursor = <T extends ObjectRecord = ObjectRecord>(- objectRecord: T,- order: ObjectRecordOrderBy | undefined,-): string => {+export const encodeCursor = <T extends ObjectRecord = ObjectRecord>({+ objectRecord,+ order,+ flatObjectMetadata,+ flatFieldMetadataMaps,+}: {+ objectRecord: T;+ order: ObjectRecordOrderBy | undefined;+ flatObjectMetadata: FlatObjectMetadata;+ flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;+}): string => {+ const { fieldIdByName } = buildFieldMapsFromFlatObjectMetadata(+ flatFieldMetadataMaps,+ flatObjectMetadata,+ );+// oxlint-disable-next-line @typescripttypescript/no-explicit-anyconst orderByValues: Record<string, any> = {};- const orderBy = order?.reduce((acc, orderBy) => ({ ...acc, ...orderBy }), {});+ for (const orderByEntry of order ?? []) {+ for (const [key, value] of Object.entries(orderByEntry)) {+ const fieldMetadataId = fieldIdByName[key];+ const fieldMetadata = findFlatEntityByIdInFlatEntityMaps({+ flatEntityMaps: flatFieldMetadataMaps,+ flatEntityId: fieldMetadataId,+ });++ if (!isDefined(fieldMetadata)) {+ continue;+ }- const orderByKeys = Object.keys(orderBy ?? {});+ if (+ isCompositeFieldMetadataType(fieldMetadata.type) &&+ isPlainObject(value) &&+ isDefined(value)+ ) {+ const compositeOrderByKeys = Object.keys(+ value as Record<string, unknown>,+ );+ const existingCompositeValue: Record<string, unknown> =+ orderByValues[key] ?? {};+ const recordCompositeValue = objectRecord[key] as+ | Record<string, unknown>+ | null+ | undefined;- orderByKeys?.forEach((key) => {- orderByValues[key] = objectRecord[key];- });+ for (const subKey of compositeOrderByKeys) {+ existingCompositeValue[subKey] = recordCompositeValue?.[subKey];+ }++ orderByValues[key] = existingCompositeValue;+ } else {+ orderByValues[key] = objectRecord[key];+ }+ }+ }const cursorData: CursorData = {...orderByValues,diff --git a/packages/twenty-server/src/engine/api/utils/__tests__/compute-cursor-arg-filter.utils.spec.ts b/packages/twenty-server/src/engine/api/utils/__tests__/compute-cursor-arg-filter.utils.spec.tsindex 6a37348d73f..4f756e966bc 100644--- a/packages/twenty-server/src/engine/api/utils/__tests__/compute-cursor-arg-filter.utils.spec.ts+++ b/packages/twenty-server/src/engine/api/utils/__tests__/compute-cursor-arg-filter.utils.spec.ts@@ -252,6 +252,51 @@ describe('computeCursorArgFilter', () => {]);});+ it('should handle dotted composite cursor keys', () => {+ const cursor = {+ 'fullName.firstName': 'John',+ 'fullName.lastName': 'Doe',+ } as any;+ const orderBy = [+ {+ fullName: {+ firstName: OrderByDirection.AscNullsLast,+ lastName: OrderByDirection.AscNullsLast,+ },+ },+ ];++ const result = computeCursorArgFilter(+ cursor,+ orderBy,+ flatObjectMetadata,+ flatFieldMetadataMaps,+ true,+ );++ expect(result).toEqual([+ {+ fullName: {+ firstName: { gt: 'John' },+ },+ },+ {+ and: [+ {+ fullName: {+ firstName: { eq: 'John' },+ },+ },+ {+ fullName: {+ lastName: { gt: 'Doe' },+ },+ },+ ],+ },+ ]);+ });+it('should handle composite field with backward pagination', () => {const cursor = {fullName: { firstName: 'John', lastName: 'Doe' },diff --git a/packages/twenty-server/src/engine/api/utils/build-cursor-where-condition.utils.ts b/packages/twenty-server/src/engine/api/utils/build-cursor-where-condition.utils.tsindex 6727179dc72..95f1326cf12 100644--- a/packages/twenty-server/src/engine/api/utils/build-cursor-where-condition.utils.ts+++ b/packages/twenty-server/src/engine/api/utils/build-cursor-where-condition.utils.ts@@ -24,7 +24,7 @@ import { buildFieldMapsFromFlatObjectMetadata } from 'src/engine/metadata-moduleimport { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';type BuildCursorWhereConditionParams = {- cursorKey: keyof ObjectRecord;+ cursorKey: string;cursorValue:| ObjectRecordCursorLeafScalarValue| ObjectRecordCursorLeafCompositeValue;@@ -49,7 +49,10 @@ export const buildCursorWhereCondition = ({flatObjectMetadata,);- const fieldMetadataId = fieldIdByName[cursorKey];+ const [fieldKey, ...subFieldPath] = cursorKey.split('.');+ const compositeSubFieldKey = subFieldPath.join('.');+ const fieldMetadataKey = fieldKey as keyof ObjectRecord;+ const fieldMetadataId = fieldIdByName[fieldMetadataKey];const fieldMetadata = findFlatEntityByIdInFlatEntityMaps({flatEntityMaps: flatFieldMetadataMaps,@@ -58,16 +61,29 @@ export const buildCursorWhereCondition = ({if (!fieldMetadata) {throw new GraphqlQueryRunnerException(- `Field metadata not found for key: ${cursorKey}`,+ `Field metadata not found for key: ${String(cursorKey)}`,GraphqlQueryRunnerExceptionCode.INVALID_CURSOR,{ userFriendlyMessage: STANDARD_ERROR_MESSAGE },);}if (isCompositeFieldMetadataType(fieldMetadata.type)) {+ if (compositeSubFieldKey.length > 0) {+ return buildCursorCompositeFieldWhereCondition({+ fieldType: fieldMetadata.type,+ fieldKey: fieldMetadataKey,+ orderBy,+ cursorValue: {+ [compositeSubFieldKey]: cursorValue,+ } as ObjectRecordCursorLeafCompositeValue,+ isForwardPagination,+ isEqualityCondition,+ });+ }+return buildCursorCompositeFieldWhereCondition({fieldType: fieldMetadata.type,- fieldKey: cursorKey,+ fieldKey: fieldMetadataKey,orderBy,cursorValue: cursorValue as ObjectRecordCursorLeafCompositeValue,isForwardPagination,@@ -76,11 +92,14 @@ export const buildCursorWhereCondition = ({}if (isEqualityCondition) {- return { [cursorKey]: { eq: cursorValue } };+ return { [fieldMetadataKey]: { eq: cursorValue } };}- const keyOrderBy = validateAndGetOrderByForScalarField(cursorKey, orderBy);- const orderByDirection = keyOrderBy[cursorKey];+ const keyOrderBy = validateAndGetOrderByForScalarField(+ fieldMetadataKey,+ orderBy,+ );+ const orderByDirection = keyOrderBy[fieldMetadataKey];if (!isDefined(orderByDirection)) {throw new GraphqlQueryRunnerException(@@ -93,5 +112,5 @@ export const buildCursorWhereCondition = ({const isAscending = isAscendingOrder(orderByDirection);const computedOperator = computeOperator(isAscending, isForwardPagination);- return { [cursorKey]: { [computedOperator]: cursorValue } };+ return { [fieldMetadataKey]: { [computedOperator]: cursorValue } };};diff --git a/packages/twenty-server/src/engine/api/utils/validate-and-get-order-by.utils.ts b/packages/twenty-server/src/engine/api/utils/validate-and-get-order-by.utils.tsindex 6ebc47337d6..eb94ac13feb 100644--- a/packages/twenty-server/src/engine/api/utils/validate-and-get-order-by.utils.ts+++ b/packages/twenty-server/src/engine/api/utils/validate-and-get-order-by.utils.ts@@ -75,9 +75,9 @@ export const validateAndGetOrderByForCompositeField = (key: keyof ObjectRecord,orderBy: ObjectRecordOrderBy,): ObjectRecordOrderByForCompositeField => {- const keyOrderBy = orderBy.find((order) => key in order);+ const matchingOrderBys = orderBy.filter((order) => key in order);- if (!isDefined(keyOrderBy)) {+ if (matchingOrderBys.length === 0) {throw new GraphqlQueryRunnerException('Invalid cursor',GraphqlQueryRunnerExceptionCode.INVALID_CURSOR,@@ -85,7 +85,20 @@ export const validateAndGetOrderByForCompositeField = ();}- if (!isOrderByForCompositeField(keyOrderBy, key)) {+ // Merge all orderBy entries for the same composite field key so that+ // separate { fullName: { firstName } } and { fullName: { lastName } } entries+ // are treated as a single composite orderBy+ const mergedValue = matchingOrderBys.reduce(+ (acc, orderByEntry) => ({+ ...acc,+ ...(orderByEntry[key as string] as Record<string, OrderByDirection>),+ }),+ {} as Record<string, OrderByDirection>,+ );++ const mergedOrderBy = { [key as string]: mergedValue };++ if (!isOrderByForCompositeField(mergedOrderBy, key)) {throw new GraphqlQueryRunnerException('Expected composite field order by',GraphqlQueryRunnerExceptionCode.INVALID_CURSOR,@@ -93,7 +106,7 @@ export const validateAndGetOrderByForCompositeField = ();}- return keyOrderBy;+ return mergedOrderBy;};export const countRelationFieldsInOrderBy = (