Skip to content

The open alternative to Salesforce, designed for AI.

perf(upsert) - tighten createMany upsert candidate lookup to avoid broad OR scans (#22721)

Etienne7475b5f16fdd
diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/types/conflicting-field-group.type.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/types/conflicting-field-group.type.ts
index fb2d1741f58..315c46a24f3 100644
--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/types/conflicting-field-group.type.ts
+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/types/conflicting-field-group.type.ts
@@ -1,3 +1,5 @@
+export type ConflictingFieldValue = string | number | boolean;
+
export type ConflictingProperty = {
fullPath: string;
column: string;
diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/__tests__/build-where-conditions.util.spec.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/__tests__/build-where-conditions.util.spec.ts
index 814cdc3eb40..4b71ced899c 100644
--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/__tests__/build-where-conditions.util.spec.ts
+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/__tests__/build-where-conditions.util.spec.ts
@@ -28,7 +28,7 @@ describe('buildWhereConditions', () => {
expect(where).toEqual([]);
});
- it('builds a single where condition for a flat field using all defined values', () => {
+ it('collapses a single-column group into one IN condition with defined values', () => {
const groups: ConflictingFieldGroup[] = [
{
baseFields: ['uniqueText'],
@@ -41,16 +41,36 @@ describe('buildWhereConditions', () => {
const where = buildWhereConditions(records, groups);
expect(where).toHaveLength(1);
- const condition = where[0];
- expect(Object.keys(condition)).toEqual(['uniqueText']);
-
- const operator = condition.uniqueText;
+ const operator = where[0].uniqueText;
expect(operator.type.toLowerCase()).toBe('in');
expect(operator.value).toEqual(['alpha', 'beta']);
});
+ it('deduplicates values within a single-column IN condition', () => {
+ const groups: ConflictingFieldGroup[] = [
+ {
+ baseFields: ['uniqueText'],
+ conflictingProperties: [
+ { fullPath: 'uniqueText', column: 'uniqueText' },
+ ],
+ },
+ ];
+
+ const where = buildWhereConditions(
+ [
+ { uniqueText: 'alpha' },
+ { uniqueText: 'alpha' },
+ { uniqueText: 'beta' },
+ ],
+ groups,
+ );
+
+ expect(where).toHaveLength(1);
+ expect(where[0].uniqueText.value).toEqual(['alpha', 'beta']);
+ });
+
it('skips adding a condition when all values for a field are undefined', () => {
const groups: ConflictingFieldGroup[] = [
{
@@ -66,7 +86,7 @@ describe('buildWhereConditions', () => {
expect(where).toEqual([]);
});
- it('builds conditions for nested paths', () => {
+ it('collapses a nested single-column path into one IN condition', () => {
const groups: ConflictingFieldGroup[] = [
{
baseFields: ['emailsField'],
@@ -82,17 +102,14 @@ describe('buildWhereConditions', () => {
const where = buildWhereConditions(records, groups);
expect(where).toHaveLength(1);
- const condition = where[0];
-
- expect(Object.keys(condition)).toEqual(['emailsFieldPrimaryEmail']);
- const operator = condition.emailsFieldPrimaryEmail;
+ const operator = where[0].emailsFieldPrimaryEmail;
expect(operator.type.toLowerCase()).toBe('in');
expect(operator.value).toEqual(['[email protected]', '[email protected]']);
});
- it('builds multiple conditions when multiple conflicting fields are provided', () => {
+ it('builds one IN condition per single-column conflicting field group', () => {
const groups: ConflictingFieldGroup[] = [
{
baseFields: ['uniqueText'],
@@ -119,17 +136,170 @@ describe('buildWhereConditions', () => {
'emailsFieldPrimaryEmail',
'uniqueText',
]);
+ });
- const uniqueTextOperator = where.find((c) => 'uniqueText' in c)?.uniqueText;
+ it('preserves numeric values instead of coercing them to strings', () => {
+ const groups: ConflictingFieldGroup[] = [
+ {
+ baseFields: ['externalId'],
+ conflictingProperties: [
+ { fullPath: 'externalId', column: 'externalId' },
+ ],
+ },
+ ];
- const emailOperator = where.find(
- (c) => 'emailsFieldPrimaryEmail' in c,
- )?.emailsFieldPrimaryEmail;
+ const where = buildWhereConditions(
+ [{ externalId: 42 }, { externalId: 43 }],
+ groups,
+ );
- expect(uniqueTextOperator?.value).toEqual(['alpha', 'beta']);
- expect(emailOperator?.value).toEqual([
+ expect(where).toHaveLength(1);
+ expect(where[0].externalId.value).toEqual([42, 43]);
+ });
+
+ it('builds composite group conditions with all properties ANDed together per record', () => {
+ const groups: ConflictingFieldGroup[] = [
+ {
+ baseFields: ['customerId', 'environment'],
+ conflictingProperties: [
+ { fullPath: 'customerId', column: 'customerId' },
+ { fullPath: 'environment', column: 'environment' },
+ ],
+ },
+ ];
+
+ const where = buildWhereConditions(
+ [
+ { customerId: 'customer-1', environment: 'prod' },
+ { customerId: 'customer-2', environment: 'staging' },
+ ],
+ groups,
+ );
+
+ expect(where).toHaveLength(2);
+
+ expect(where).toEqual([
+ {
+ customerId: expect.objectContaining({ value: 'customer-1' }),
+ environment: expect.objectContaining({ value: 'prod' }),
+ },
+ {
+ customerId: expect.objectContaining({ value: 'customer-2' }),
+ environment: expect.objectContaining({ value: 'staging' }),
+ },
]);
+
+ where.forEach((condition) => {
+ expect(condition.customerId.type.toLowerCase()).toBe('equal');
+ expect(condition.environment.type.toLowerCase()).toBe('equal');
+ });
+ });
+
+ it('skips a composite group for a record missing part of the key', () => {
+ const groups: ConflictingFieldGroup[] = [
+ {
+ baseFields: ['customerId', 'environment'],
+ conflictingProperties: [
+ { fullPath: 'customerId', column: 'customerId' },
+ { fullPath: 'environment', column: 'environment' },
+ ],
+ },
+ ];
+
+ const where = buildWhereConditions(
+ [
+ { customerId: 'customer-1', environment: 'prod' },
+ { customerId: 'customer-2' },
+ ],
+ groups,
+ );
+
+ expect(where).toHaveLength(1);
+ expect(where[0]).toEqual({
+ customerId: expect.objectContaining({ value: 'customer-1' }),
+ environment: expect.objectContaining({ value: 'prod' }),
+ });
+ });
+
+ it('deduplicates identical composite conditions across records', () => {
+ const groups: ConflictingFieldGroup[] = [
+ {
+ baseFields: ['customerId', 'environment'],
+ conflictingProperties: [
+ { fullPath: 'customerId', column: 'customerId' },
+ { fullPath: 'environment', column: 'environment' },
+ ],
+ },
+ ];
+
+ const where = buildWhereConditions(
+ [
+ { customerId: 'customer-1', environment: 'prod' },
+ { customerId: 'customer-1', environment: 'prod' },
+ ],
+ groups,
+ );
+
+ expect(where).toHaveLength(1);
+
+ expect(where[0]).toEqual({
+ customerId: expect.objectContaining({ value: 'customer-1' }),
+ environment: expect.objectContaining({ value: 'prod' }),
+ });
+ });
+
+ it('does not merge distinct composite tuples that share separator-like characters', () => {
+ const groups: ConflictingFieldGroup[] = [
+ {
+ baseFields: ['customerId', 'environment'],
+ conflictingProperties: [
+ { fullPath: 'customerId', column: 'customerId' },
+ { fullPath: 'environment', column: 'environment' },
+ ],
+ },
+ ];
+
+ const where = buildWhereConditions(
+ [
+ { customerId: 'a:b', environment: 'c' },
+ { customerId: 'a', environment: 'b:c' },
+ ],
+ groups,
+ );
+
+ expect(where).toHaveLength(2);
+ });
+
+ it('ORs single-column and composite groups together', () => {
+ const groups: ConflictingFieldGroup[] = [
+ {
+ baseFields: ['id'],
+ conflictingProperties: [{ fullPath: 'id', column: 'id' }],
+ },
+ {
+ baseFields: ['customerId', 'environment'],
+ conflictingProperties: [
+ { fullPath: 'customerId', column: 'customerId' },
+ { fullPath: 'environment', column: 'environment' },
+ ],
+ },
+ ];
+
+ const where = buildWhereConditions(
+ [{ id: 'record-1', customerId: 'customer-1', environment: 'prod' }],
+ groups,
+ );
+
+ expect(where).toHaveLength(2);
+
+ const idCondition = where.find((condition) => 'id' in condition);
+ const compositeCondition = where.find(
+ (condition) => 'customerId' in condition,
+ );
+
+ expect(idCondition?.id.type.toLowerCase()).toBe('in');
+ expect(idCondition?.id.value).toEqual(['record-1']);
+ expect(compositeCondition?.customerId.type.toLowerCase()).toBe('equal');
+ expect(compositeCondition?.environment.type.toLowerCase()).toBe('equal');
});
});
diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/build-where-conditions.util.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/build-where-conditions.util.ts
index 4ffcab0ee6c..4652809ae4d 100644
--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/build-where-conditions.util.ts
+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/build-where-conditions.util.ts
@@ -1,27 +1,115 @@
import { type ObjectRecord } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
-import { type FindOperator, In } from 'typeorm';
+import { Equal, In, type FindOperator } from 'typeorm';
-import { type ConflictingFieldGroup } from 'src/engine/api/common/common-query-runners/common-create-many-query-runner/types/conflicting-field-group.type';
+import {
+ type ConflictingFieldGroup,
+ type ConflictingFieldValue,
+ type ConflictingProperty,
+} from 'src/engine/api/common/common-query-runners/common-create-many-query-runner/types/conflicting-field-group.type';
import { getValueFromPath } from 'src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/get-value-from-path.util';
+type WhereCondition = Record<string, FindOperator<ConflictingFieldValue>>;
+
+const buildCompositeConditionKey = (
+ conditionEntries: [string, ConflictingFieldValue][],
+): string => {
+ const sortedEntries = [...conditionEntries].sort(([columnA], [columnB]) =>
+ columnA.localeCompare(columnB),
+ );
+
+ return JSON.stringify(sortedEntries);
+};
+
+const buildSingleColumnCondition = (
+ records: Partial<ObjectRecord>[],
+ conflictingProperty: ConflictingProperty,
+): WhereCondition | undefined => {
+ const distinctValues = [
+ ...new Set(
+ records
+ .map((record) => getValueFromPath(record, conflictingProperty.fullPath))
+ .filter(isDefined),
+ ),
+ ];
+
+ if (distinctValues.length === 0) {
+ return undefined;
+ }
+
+ return { [conflictingProperty.column]: In(distinctValues) };
+};
+
+const buildCompositeConditionEntries = (
+ record: Partial<ObjectRecord>,
+ conflictingProperties: ConflictingProperty[],
+): [string, ConflictingFieldValue][] | undefined => {
+ const conditionEntries: [string, ConflictingFieldValue][] = [];
+
+ for (const conflictingProperty of conflictingProperties) {
+ const fieldValue = getValueFromPath(record, conflictingProperty.fullPath);
+
+ if (!isDefined(fieldValue)) {
+ return undefined;
+ }
+
+ conditionEntries.push([conflictingProperty.column, fieldValue]);
+ }
+
+ return conditionEntries;
+};
+
export const buildWhereConditions = (
records: Partial<ObjectRecord>[],
conflictingFieldGroups: ConflictingFieldGroup[],
-): Record<string, FindOperator<string>>[] => {
- const whereConditions: Record<string, FindOperator<string>>[] = [];
-
- for (const conflictingProperty of conflictingFieldGroups.flatMap(
- (group) => group.conflictingProperties,
- )) {
- const fieldValues = records
- .map((record) => getValueFromPath(record, conflictingProperty.fullPath))
- .filter(isDefined);
-
- if (fieldValues.length > 0) {
- whereConditions.push({
- [conflictingProperty.column]: In(fieldValues),
- });
+): WhereCondition[] => {
+ const whereConditions: WhereCondition[] = [];
+ const seenCompositeConditionKeys = new Set<string>();
+
+ for (const conflictingFieldGroup of conflictingFieldGroups) {
+ const { conflictingProperties } = conflictingFieldGroup;
+
+ if (conflictingProperties.length === 1) {
+ const condition = buildSingleColumnCondition(
+ records,
+ conflictingProperties[0],
+ );
+
+ if (isDefined(condition)) {
+ whereConditions.push(condition);
+ }
+
+ continue;
+ }
+
+ for (const record of records) {
+ const conditionEntries = buildCompositeConditionEntries(
+ record,
+ conflictingProperties,
+ );
+
+ if (!isDefined(conditionEntries)) {
+ continue;
+ }
+
+ const conditionKey = buildCompositeConditionKey(conditionEntries);
+
+ if (seenCompositeConditionKeys.has(conditionKey)) {
+ continue;
+ }
+
+ seenCompositeConditionKeys.add(conditionKey);
+
+ whereConditions.push(
+ conditionEntries.reduce<WhereCondition>(
+ (accumulator, [column, value]) => {
+ accumulator[column] = Equal(value);
+
+ return accumulator;
+ },
+ {},
+ ),
+ );
}
}
diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/get-value-from-path.util.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/get-value-from-path.util.ts
index 35bd1db396f..411dc35fb64 100644
--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/get-value-from-path.util.ts
+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-create-many-query-runner/utils/get-value-from-path.util.ts
@@ -1,9 +1,11 @@
import { type ObjectRecord } from 'twenty-shared/types';
+import { type ConflictingFieldValue } from 'src/engine/api/common/common-query-runners/common-create-many-query-runner/types/conflicting-field-group.type';
+
export const getValueFromPath = (
record: Partial<ObjectRecord>,
path: string,
-): string | undefined => {
+): ConflictingFieldValue | undefined => {
const pathParts = path.split('.');
if (pathParts.length === 1) {