Skip to content

The open alternative to Salesforce, designed for AI.

fix: REST cursor encoding for nested order_by composite fields (#20974)

Matt Van Horn6ac797a69cb8
diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-duplicates-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-duplicates-query-runner.service.ts
index 887539bb13d..d7337a2877f 100644
--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-duplicates-query-runner.service.ts
+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-duplicates-query-runner.service.ts
@@ -135,6 +135,8 @@ export class CommonFindDuplicatesQueryRunnerService extends CommonBaseQueryRunne
[{ id: OrderByDirection.AscNullsFirst }],
QUERY_MAX_RECORDS,
true,
+ flatObjectMetadata,
+ flatFieldMetadataMaps,
);
return {
diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts
index f1f0c139164..f719a64f296 100644
--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts
+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-find-many-query-runner.service.ts
@@ -185,6 +185,8 @@ export class CommonFindManyQueryRunnerService extends CommonBaseQueryRunnerServi
orderByWithIdCondition,
limit,
isForwardPagination,
+ flatObjectMetadata,
+ flatFieldMetadataMaps,
);
if (!isForwardPagination) {
diff --git a/packages/twenty-server/src/engine/api/common/utils/get-page-info.util.ts b/packages/twenty-server/src/engine/api/common/utils/get-page-info.util.ts
index bb78bb4129c..ed44fede070 100644
--- a/packages/twenty-server/src/engine/api/common/utils/get-page-info.util.ts
+++ b/packages/twenty-server/src/engine/api/common/utils/get-page-info.util.ts
@@ -7,12 +7,17 @@ import {
encodeCursor,
getPaginationInfo,
} from 'src/engine/api/graphql/graphql-query-runner/utils/cursors.util';
+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
export const getPageInfo = (
records: ObjectRecord[],
orderBy: ObjectRecordOrderBy,
limit: number,
isForwardPagination: boolean,
+ flatObjectMetadata: FlatObjectMetadata,
+ flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
): CommonPageInfo => {
const { hasNextPage, hasPreviousPage, hasMoreRecords } = getPaginationInfo(
records,
@@ -25,10 +30,22 @@ export const getPageInfo = (
}
const startCursor =
- records.length > 0 ? encodeCursor(records[0], orderBy) : null;
+ records.length > 0
+ ? encodeCursor({
+ objectRecord: records[0],
+ order: orderBy,
+ flatObjectMetadata,
+ flatFieldMetadataMaps,
+ })
+ : null;
const endCursor =
records.length > 0
- ? encodeCursor(records[records.length - 1], orderBy)
+ ? encodeCursor({
+ objectRecord: records[records.length - 1],
+ order: orderBy,
+ flatObjectMetadata,
+ flatFieldMetadataMaps,
+ })
: null;
return { startCursor, endCursor, hasNextPage, hasPreviousPage };
diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper.ts
index 8c9529bd369..ebdc61df092 100644
--- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper.ts
+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper.ts
@@ -69,6 +69,12 @@ export class ObjectRecordsToGraphqlConnectionHelper {
hasPreviousPage: boolean;
depth?: number;
}): IConnection<T> {
+ const objectMetadataId = this.objectIdByNameSingular[objectName];
+ const flatObjectMetadata = findFlatEntityByIdInFlatEntityMapsOrThrow({
+ flatEntityId: objectMetadataId,
+ flatEntityMaps: this.flatObjectMetadataMaps,
+ });
+
const edges = (objectRecords ?? []).map((objectRecord) => ({
node: this.processRecord({
objectRecord,
@@ -80,7 +86,12 @@ export class ObjectRecordsToGraphqlConnectionHelper {
order,
depth,
}),
- cursor: encodeCursor(objectRecord, order),
+ cursor: encodeCursor({
+ objectRecord,
+ order,
+ flatObjectMetadata,
+ flatFieldMetadataMaps: this.flatFieldMetadataMaps,
+ }),
}));
const aggregatedFieldsValues = this.extractAggregatedFieldsValues({
diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/__tests__/cursors.util.spec.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/__tests__/cursors.util.spec.ts
new file mode 100644
index 00000000000..0f608e8d01c
--- /dev/null
+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/__tests__/cursors.util.spec.ts
@@ -0,0 +1,181 @@
+import { FieldMetadataType, OrderByDirection } from 'twenty-shared/types';
+
+import { type ObjectRecordOrderBy } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
+
+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
+import { decodeCursor, encodeCursor } from '../cursors.util';
+
+const buildMockField = (
+ id: string,
+ name: string,
+ type: FieldMetadataType,
+): FlatFieldMetadata =>
+ ({
+ id,
+ universalIdentifier: id,
+ name,
+ type,
+ objectMetadataId: 'obj-id',
+ workspaceId: 'ws-id',
+ label: name,
+ isNullable: true,
+ isLabelSyncedWithName: true,
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ viewFieldIds: [],
+ viewFilterIds: [],
+ kanbanAggregateOperationViewIds: [],
+ calendarViewIds: [],
+ applicationId: null,
+ }) as unknown as FlatFieldMetadata;
+
+const nameField = buildMockField('name-id', 'name', FieldMetadataType.TEXT);
+const fullNameField = buildMockField(
+ 'fullname-id',
+ 'fullName',
+ FieldMetadataType.FULL_NAME,
+);
+
+const flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata> = {
+ byUniversalIdentifier: {
+ 'name-id': nameField,
+ 'fullname-id': fullNameField,
+ },
+ universalIdentifierById: {
+ 'name-id': 'name-id',
+ 'fullname-id': 'fullname-id',
+ },
+ universalIdentifiersByApplicationId: {},
+};
+
+const flatObjectMetadata: FlatObjectMetadata = {
+ id: 'obj-id',
+ universalIdentifier: 'obj-id',
+ workspaceId: 'ws-id',
+ nameSingular: 'person',
+ namePlural: 'people',
+ labelSingular: 'Person',
+ labelPlural: 'People',
+ targetTableName: 'person',
+ isCustom: false,
+ isRemote: false,
+ isActive: true,
+ isSystem: false,
+ isAuditLogged: false,
+ isSearchable: false,
+ icon: 'Icon123',
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ fieldIds: ['name-id', 'fullname-id'],
+ indexMetadataIds: [],
+ viewIds: [],
+ applicationId: null,
+} as unknown as FlatObjectMetadata;
+
+const callEncodeCursor = (
+ record: Record<string, unknown>,
+ orderBy: Parameters<typeof encodeCursor>[0]['order'],
+) =>
+ encodeCursor({
+ objectRecord: record as never,
+ order: orderBy,
+ flatObjectMetadata,
+ flatFieldMetadataMaps,
+ });
+
+describe('encodeCursor', () => {
+ it('should encode scalar fields from the orderBy', () => {
+ const record = { id: 'abc', name: 'John', age: 30 };
+ const orderBy = [{ name: OrderByDirection.AscNullsLast }];
+
+ const decoded = decodeCursor(callEncodeCursor(record, orderBy));
+
+ expect(decoded).toEqual({ name: 'John', id: 'abc' });
+ });
+
+ it('should always include id even if not in orderBy', () => {
+ const record = { id: 'abc', name: 'John' };
+ const orderBy = [{ name: OrderByDirection.AscNullsLast }];
+
+ const decoded = decodeCursor(callEncodeCursor(record, orderBy));
+
+ expect(decoded).toHaveProperty('id', 'abc');
+ });
+
+ it('should only include ordered sub-fields for composite fields', () => {
+ const record = {
+ id: 'abc',
+ fullName: { firstName: 'Katherine', lastName: 'Abbott' },
+ };
+ const orderBy = [
+ { fullName: { firstName: OrderByDirection.AscNullsLast } },
+ ];
+
+ const decoded = decodeCursor(callEncodeCursor(record, orderBy));
+
+ expect(decoded).toEqual({
+ fullName: { firstName: 'Katherine' },
+ id: 'abc',
+ });
+ expect(decoded.fullName).not.toHaveProperty('lastName');
+ });
+
+ it('should include all sub-fields when all are in the orderBy', () => {
+ const record = {
+ id: 'abc',
+ fullName: { firstName: 'Katherine', lastName: 'Abbott' },
+ };
+ const orderBy = [
+ {
+ fullName: {
+ firstName: OrderByDirection.AscNullsLast,
+ lastName: OrderByDirection.AscNullsLast,
+ },
+ },
+ ];
+
+ const decoded = decodeCursor(callEncodeCursor(record, orderBy));
+
+ expect(decoded).toEqual({
+ fullName: { firstName: 'Katherine', lastName: 'Abbott' },
+ id: 'abc',
+ });
+ });
+
+ it('should deep-merge two separate entries for the same composite parent', () => {
+ const record = {
+ id: 'abc',
+ fullName: { firstName: 'Katherine', lastName: 'Watts' },
+ };
+ const orderBy: ObjectRecordOrderBy = [
+ { fullName: { firstName: OrderByDirection.AscNullsFirst } },
+ { fullName: { lastName: OrderByDirection.DescNullsLast } },
+ ];
+
+ const decoded = decodeCursor(callEncodeCursor(record, orderBy));
+
+ expect(decoded).toEqual({
+ fullName: { firstName: 'Katherine', lastName: 'Watts' },
+ id: 'abc',
+ });
+ });
+
+ it('should not filter sub-fields for scalar fields', () => {
+ const record = { id: 'abc', name: 'John' };
+ const orderBy = [{ name: OrderByDirection.AscNullsLast }];
+
+ const decoded = decodeCursor(callEncodeCursor(record, orderBy));
+
+ expect(decoded).toEqual({ name: 'John', id: 'abc' });
+ });
+
+ it('should handle undefined orderBy', () => {
+ const record = { id: 'abc', name: 'John' };
+
+ const decoded = decodeCursor(callEncodeCursor(record, undefined));
+
+ expect(decoded).toEqual({ id: 'abc' });
+ });
+});
diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/cursors.util.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/cursors.util.ts
index 99810f919c0..0063026e5aa 100644
--- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/cursors.util.ts
+++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/cursors.util.ts
@@ -1,4 +1,5 @@
import { type ObjectRecord } from 'twenty-shared/types';
+import { isDefined, isPlainObject } from 'twenty-shared/utils';
import { type ObjectRecordOrderBy } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import { type FindManyResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
@@ -8,6 +9,12 @@ import {
CommonQueryRunnerExceptionCode,
} from 'src/engine/api/common/common-query-runners/errors/common-query-runner.exception';
import { STANDARD_ERROR_MESSAGE } from 'src/engine/api/common/common-query-runners/errors/standard-error-message.constant';
+import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
+import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { buildFieldMapsFromFlatObjectMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-from-flat-object-metadata.util';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
export interface CursorData {
// oxlint-disable-next-line @typescripttypescript/no-explicit-any
@@ -26,20 +33,62 @@ export const decodeCursor = <T = CursorData>(cursor: string): T => {
}
};
-export const encodeCursor = <T extends ObjectRecord = ObjectRecord>(
- objectRecord: T,
- order: ObjectRecordOrderBy | undefined,
-): string => {
+export const encodeCursor = <T extends ObjectRecord = ObjectRecord>({
+ objectRecord,
+ order,
+ flatObjectMetadata,
+ flatFieldMetadataMaps,
+}: {
+ objectRecord: T;
+ order: ObjectRecordOrderBy | undefined;
+ flatObjectMetadata: FlatObjectMetadata;
+ flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
+}): string => {
+ const { fieldIdByName } = buildFieldMapsFromFlatObjectMetadata(
+ flatFieldMetadataMaps,
+ flatObjectMetadata,
+ );
+
// oxlint-disable-next-line @typescripttypescript/no-explicit-any
const orderByValues: Record<string, any> = {};
- const orderBy = order?.reduce((acc, orderBy) => ({ ...acc, ...orderBy }), {});
+ for (const orderByEntry of order ?? []) {
+ for (const [key, value] of Object.entries(orderByEntry)) {
+ const fieldMetadataId = fieldIdByName[key];
+ const fieldMetadata = findFlatEntityByIdInFlatEntityMaps({
+ flatEntityMaps: flatFieldMetadataMaps,
+ flatEntityId: fieldMetadataId,
+ });
+
+ if (!isDefined(fieldMetadata)) {
+ continue;
+ }
- const orderByKeys = Object.keys(orderBy ?? {});
+ if (
+ isCompositeFieldMetadataType(fieldMetadata.type) &&
+ isPlainObject(value) &&
+ isDefined(value)
+ ) {
+ const compositeOrderByKeys = Object.keys(
+ value as Record<string, unknown>,
+ );
+ const existingCompositeValue: Record<string, unknown> =
+ orderByValues[key] ?? {};
+ const recordCompositeValue = objectRecord[key] as
+ | Record<string, unknown>
+ | null
+ | undefined;
- orderByKeys?.forEach((key) => {
- orderByValues[key] = objectRecord[key];
- });
+ for (const subKey of compositeOrderByKeys) {
+ existingCompositeValue[subKey] = recordCompositeValue?.[subKey];
+ }
+
+ orderByValues[key] = existingCompositeValue;
+ } else {
+ orderByValues[key] = objectRecord[key];
+ }
+ }
+ }
const cursorData: CursorData = {
...orderByValues,
diff --git a/packages/twenty-server/src/engine/api/utils/__tests__/compute-cursor-arg-filter.utils.spec.ts b/packages/twenty-server/src/engine/api/utils/__tests__/compute-cursor-arg-filter.utils.spec.ts
index 6a37348d73f..4f756e966bc 100644
--- a/packages/twenty-server/src/engine/api/utils/__tests__/compute-cursor-arg-filter.utils.spec.ts
+++ b/packages/twenty-server/src/engine/api/utils/__tests__/compute-cursor-arg-filter.utils.spec.ts
@@ -252,6 +252,51 @@ describe('computeCursorArgFilter', () => {
]);
});
+ it('should handle dotted composite cursor keys', () => {
+ const cursor = {
+ 'fullName.firstName': 'John',
+ 'fullName.lastName': 'Doe',
+ } as any;
+ const orderBy = [
+ {
+ fullName: {
+ firstName: OrderByDirection.AscNullsLast,
+ lastName: OrderByDirection.AscNullsLast,
+ },
+ },
+ ];
+
+ const result = computeCursorArgFilter(
+ cursor,
+ orderBy,
+ flatObjectMetadata,
+ flatFieldMetadataMaps,
+ true,
+ );
+
+ expect(result).toEqual([
+ {
+ fullName: {
+ firstName: { gt: 'John' },
+ },
+ },
+ {
+ and: [
+ {
+ fullName: {
+ firstName: { eq: 'John' },
+ },
+ },
+ {
+ fullName: {
+ lastName: { gt: 'Doe' },
+ },
+ },
+ ],
+ },
+ ]);
+ });
+
it('should handle composite field with backward pagination', () => {
const cursor = {
fullName: { firstName: 'John', lastName: 'Doe' },
diff --git a/packages/twenty-server/src/engine/api/utils/build-cursor-where-condition.utils.ts b/packages/twenty-server/src/engine/api/utils/build-cursor-where-condition.utils.ts
index 6727179dc72..95f1326cf12 100644
--- a/packages/twenty-server/src/engine/api/utils/build-cursor-where-condition.utils.ts
+++ b/packages/twenty-server/src/engine/api/utils/build-cursor-where-condition.utils.ts
@@ -24,7 +24,7 @@ import { buildFieldMapsFromFlatObjectMetadata } from 'src/engine/metadata-module
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
type BuildCursorWhereConditionParams = {
- cursorKey: keyof ObjectRecord;
+ cursorKey: string;
cursorValue:
| ObjectRecordCursorLeafScalarValue
| ObjectRecordCursorLeafCompositeValue;
@@ -49,7 +49,10 @@ export const buildCursorWhereCondition = ({
flatObjectMetadata,
);
- const fieldMetadataId = fieldIdByName[cursorKey];
+ const [fieldKey, ...subFieldPath] = cursorKey.split('.');
+ const compositeSubFieldKey = subFieldPath.join('.');
+ const fieldMetadataKey = fieldKey as keyof ObjectRecord;
+ const fieldMetadataId = fieldIdByName[fieldMetadataKey];
const fieldMetadata = findFlatEntityByIdInFlatEntityMaps({
flatEntityMaps: flatFieldMetadataMaps,
@@ -58,16 +61,29 @@ export const buildCursorWhereCondition = ({
if (!fieldMetadata) {
throw new GraphqlQueryRunnerException(
- `Field metadata not found for key: ${cursorKey}`,
+ `Field metadata not found for key: ${String(cursorKey)}`,
GraphqlQueryRunnerExceptionCode.INVALID_CURSOR,
{ userFriendlyMessage: STANDARD_ERROR_MESSAGE },
);
}
if (isCompositeFieldMetadataType(fieldMetadata.type)) {
+ if (compositeSubFieldKey.length > 0) {
+ return buildCursorCompositeFieldWhereCondition({
+ fieldType: fieldMetadata.type,
+ fieldKey: fieldMetadataKey,
+ orderBy,
+ cursorValue: {
+ [compositeSubFieldKey]: cursorValue,
+ } as ObjectRecordCursorLeafCompositeValue,
+ isForwardPagination,
+ isEqualityCondition,
+ });
+ }
+
return buildCursorCompositeFieldWhereCondition({
fieldType: fieldMetadata.type,
- fieldKey: cursorKey,
+ fieldKey: fieldMetadataKey,
orderBy,
cursorValue: cursorValue as ObjectRecordCursorLeafCompositeValue,
isForwardPagination,
@@ -76,11 +92,14 @@ export const buildCursorWhereCondition = ({
}
if (isEqualityCondition) {
- return { [cursorKey]: { eq: cursorValue } };
+ return { [fieldMetadataKey]: { eq: cursorValue } };
}
- const keyOrderBy = validateAndGetOrderByForScalarField(cursorKey, orderBy);
- const orderByDirection = keyOrderBy[cursorKey];
+ const keyOrderBy = validateAndGetOrderByForScalarField(
+ fieldMetadataKey,
+ orderBy,
+ );
+ const orderByDirection = keyOrderBy[fieldMetadataKey];
if (!isDefined(orderByDirection)) {
throw new GraphqlQueryRunnerException(
@@ -93,5 +112,5 @@ export const buildCursorWhereCondition = ({
const isAscending = isAscendingOrder(orderByDirection);
const computedOperator = computeOperator(isAscending, isForwardPagination);
- return { [cursorKey]: { [computedOperator]: cursorValue } };
+ return { [fieldMetadataKey]: { [computedOperator]: cursorValue } };
};
diff --git a/packages/twenty-server/src/engine/api/utils/validate-and-get-order-by.utils.ts b/packages/twenty-server/src/engine/api/utils/validate-and-get-order-by.utils.ts
index 6ebc47337d6..eb94ac13feb 100644
--- a/packages/twenty-server/src/engine/api/utils/validate-and-get-order-by.utils.ts
+++ b/packages/twenty-server/src/engine/api/utils/validate-and-get-order-by.utils.ts
@@ -75,9 +75,9 @@ export const validateAndGetOrderByForCompositeField = (
key: keyof ObjectRecord,
orderBy: ObjectRecordOrderBy,
): ObjectRecordOrderByForCompositeField => {
- const keyOrderBy = orderBy.find((order) => key in order);
+ const matchingOrderBys = orderBy.filter((order) => key in order);
- if (!isDefined(keyOrderBy)) {
+ if (matchingOrderBys.length === 0) {
throw new GraphqlQueryRunnerException(
'Invalid cursor',
GraphqlQueryRunnerExceptionCode.INVALID_CURSOR,
@@ -85,7 +85,20 @@ export const validateAndGetOrderByForCompositeField = (
);
}
- if (!isOrderByForCompositeField(keyOrderBy, key)) {
+ // Merge all orderBy entries for the same composite field key so that
+ // separate { fullName: { firstName } } and { fullName: { lastName } } entries
+ // are treated as a single composite orderBy
+ const mergedValue = matchingOrderBys.reduce(
+ (acc, orderByEntry) => ({
+ ...acc,
+ ...(orderByEntry[key as string] as Record<string, OrderByDirection>),
+ }),
+ {} as Record<string, OrderByDirection>,
+ );
+
+ const mergedOrderBy = { [key as string]: mergedValue };
+
+ if (!isOrderByForCompositeField(mergedOrderBy, key)) {
throw new GraphqlQueryRunnerException(
'Expected composite field order by',
GraphqlQueryRunnerExceptionCode.INVALID_CURSOR,
@@ -93,7 +106,7 @@ export const validateAndGetOrderByForCompositeField = (
);
}
- return keyOrderBy;
+ return mergedOrderBy;
};
export const countRelationFieldsInOrderBy = (