The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Serialize timeline activity coalescing (#24659)
Félix Malfait618fe5001a3f
diff --git a/packages/twenty-server/src/modules/timeline/repositories/__tests__/timeline-activity.repository.spec.ts b/packages/twenty-server/src/modules/timeline/repositories/__tests__/timeline-activity.repository.spec.tsindex 618e2260d72..0a0089b3b03 100644--- a/packages/twenty-server/src/modules/timeline/repositories/__tests__/timeline-activity.repository.spec.ts+++ b/packages/twenty-server/src/modules/timeline/repositories/__tests__/timeline-activity.repository.spec.ts@@ -43,7 +43,18 @@ describe('TimelineActivityRepository', () => {executeInWorkspaceContext: jest.fn(async (callback: () => Promise<void>) => callback(),),- getRepository: jest.fn().mockResolvedValue(workspaceRepository),+ runInWorkspaceTransaction: jest.fn(+ async (+ callback: (transactionScope: {+ getRepository: () => typeof workspaceRepository;+ executeRawQuery: () => Promise<never[]>;+ }) => Promise<void>,+ ) =>+ callback({+ getRepository: () => workspaceRepository,+ executeRawQuery: jest.fn().mockResolvedValue([]),+ }),+ ),} as unknown as GlobalWorkspaceOrmManager;const repository = new TimelineActivityRepository(globalWorkspaceOrmManager,@@ -78,4 +89,85 @@ describe('TimelineActivityRepository', () => {},);});++ it('locks merge identities in a stable order before reading recent rows', async () => {+ const executeRawQuery = jest.fn().mockResolvedValue([]);+ const workspaceRepository = {+ find: jest.fn().mockResolvedValue([]),+ update: jest.fn().mockResolvedValue(undefined),+ insert: jest.fn().mockResolvedValue(undefined),+ };+ const globalWorkspaceOrmManager = {+ executeInWorkspaceContext: jest.fn(+ async (callback: () => Promise<void>) => callback(),+ ),+ runInWorkspaceTransaction: jest.fn(+ async (+ callback: (transactionScope: {+ getRepository: () => typeof workspaceRepository;+ executeRawQuery: typeof executeRawQuery;+ }) => Promise<void>,+ ) =>+ callback({+ getRepository: () => workspaceRepository,+ executeRawQuery,+ }),+ ),+ } as unknown as GlobalWorkspaceOrmManager;+ const repository = new TimelineActivityRepository(+ globalWorkspaceOrmManager,+ );++ await repository.upsertTimelineActivities({+ objectSingularName: 'person',+ workspaceId: WORKSPACE_ID,+ payloads: [+ {+ happensAt: new Date('2026-08-23T09:00:00.000Z'),+ properties: {},+ recordId: 'record-z',+ workspaceMemberId: WORKSPACE_MEMBER_ID,+ timelineActivityTypeId: TIMELINE_ACTIVITY_TYPE_ID,+ timelineActivityTypeSnapshot: TIMELINE_ACTIVITY_TYPE_SNAPSHOT,+ },+ {+ happensAt: new Date('2026-08-23T09:00:00.000Z'),+ properties: {},+ recordId: 'record-a',+ workspaceMemberId: WORKSPACE_MEMBER_ID,+ timelineActivityTypeId: TIMELINE_ACTIVITY_TYPE_ID,+ timelineActivityTypeSnapshot: TIMELINE_ACTIVITY_TYPE_SNAPSHOT,+ },+ ],+ });++ const lockStatement = `SELECT pg_advisory_xact_lock(hashtextextended("lockName", 0))+ FROM unnest($1::text[]) WITH ORDINALITY AS "locks"("lockName", "ordinality")+ ORDER BY "ordinality"`;++ expect(executeRawQuery).toHaveBeenCalledTimes(1);+ expect(executeRawQuery).toHaveBeenCalledWith(lockStatement, [+ [+ JSON.stringify([+ 'timeline-activity-merge',+ WORKSPACE_ID,+ 'person',+ 'record-a',+ WORKSPACE_MEMBER_ID,+ TIMELINE_ACTIVITY_TYPE_ID,+ ]),+ JSON.stringify([+ 'timeline-activity-merge',+ WORKSPACE_ID,+ 'person',+ 'record-z',+ WORKSPACE_MEMBER_ID,+ TIMELINE_ACTIVITY_TYPE_ID,+ ]),+ ],+ ]);+ expect(+ workspaceRepository.find.mock.invocationCallOrder[0],+ ).toBeGreaterThan(executeRawQuery.mock.invocationCallOrder[0]);+ });});diff --git a/packages/twenty-server/src/modules/timeline/repositories/timeline-activity.repository.ts b/packages/twenty-server/src/modules/timeline/repositories/timeline-activity.repository.tsindex 4893a99c469..cbcc7d9e390 100644--- a/packages/twenty-server/src/modules/timeline/repositories/timeline-activity.repository.ts+++ b/packages/twenty-server/src/modules/timeline/repositories/timeline-activity.repository.ts@@ -2,10 +2,12 @@ import { Injectable } from '@nestjs/common';import { type ObjectRecord } from 'twenty-shared/types';import { isDefined } from 'twenty-shared/utils';-import { In, MoreThan } from 'typeorm';+import { In, MoreThan, type ObjectLiteral } from 'typeorm';import { objectRecordDiffMerge } from 'src/engine/core-modules/event-emitter/utils/object-record-diff-merge';import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';+import { type WorkspaceTransactionScope } from 'src/engine/twenty-orm/global-workspace-datasource/types/workspace-transaction-scope.type';+import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';import { type TimelineActivityPayload } from 'src/modules/timeline/types/timeline-activity-payload';import {@@ -22,6 +24,10 @@ type TimelineActivityPayloadWorkspaceIdAndObjectSingularName = {objectSingularName: string;};+const ACQUIRE_TIMELINE_ACTIVITY_MERGE_LOCK = `SELECT pg_advisory_xact_lock(hashtextextended("lockName", 0))+ FROM unnest($1::text[]) WITH ORDINALITY AS "locks"("lockName", "ordinality")+ ORDER BY "ordinality"`;+@Injectable()export class TimelineActivityRepository {constructor(@@ -36,115 +42,166 @@ export class TimelineActivityRepository {const authContext = buildSystemAuthContext(workspaceId);await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {- const recentTimelineActivities = await this.findRecentTimelineActivities({- objectSingularName,- workspaceId,- payloads,- });-- const payloadsToInsert: TimelineActivityPayloadWorkspaceIdAndObjectSingularName['payloads'] =- [];- const mergesToApply: {- id: string;- properties: Partial<ObjectRecord>;- workspaceMemberId: string | undefined;- timelineActivityTypeSnapshot?: TimelineActivityPayload['timelineActivityTypeSnapshot'];- }[] = [];-- const timelineActivityPropertyName =- await this.getTimelineActivityPropertyName(objectSingularName);-- // Bucketed once so matching a payload stays constant time: the recent- // window is scoped to this batch but is not capped in size.- const recentTimelineActivitiesByMergeKey = new Map<- string,- (typeof recentTimelineActivities)[number][]- >();-- for (const timelineActivity of recentTimelineActivities) {- const mergeKey = buildTimelineActivityMergeKey({- recordId: timelineActivity[timelineActivityPropertyName],- workspaceMemberId: timelineActivity.workspaceMemberId,- timelineActivityTypeId: timelineActivity.timelineActivityTypeId,- timelineActivityTypeSnapshot:- timelineActivity.timelineActivityTypeSnapshot,- });-- const bucket = recentTimelineActivitiesByMergeKey.get(mergeKey);-- if (isDefined(bucket)) {- bucket.push(timelineActivity);- } else {- recentTimelineActivitiesByMergeKey.set(mergeKey, [timelineActivity]);- }- }-- for (const payload of payloads) {- const recentTimelineActivity = buildTimelineActivityMergeKeyCandidates({- recordId: payload.recordId,- workspaceMemberId: payload.workspaceMemberId,- timelineActivityTypeId: payload.timelineActivityTypeId,- timelineActivityTypeSnapshot: payload.timelineActivityTypeSnapshot,- })- .flatMap(- (mergeKey) =>- recentTimelineActivitiesByMergeKey.get(mergeKey) ?? [],- )- .find(- (timelineActivity) =>- !isDefined(payload.linkedRecordId) ||- timelineActivity.linkedRecordId === payload.linkedRecordId,- );-- if (isDefined(recentTimelineActivity)) {- mergesToApply.push({- id: recentTimelineActivity.id,- properties: objectRecordDiffMerge(- recentTimelineActivity.properties,- payload.properties,- ),- workspaceMemberId: payload.workspaceMemberId,- ...(!isDefined(- recentTimelineActivity.timelineActivityTypeSnapshot,- ) && {+ await this.globalWorkspaceOrmManager.runInWorkspaceTransaction(+ async (transactionScope) => {+ await this.acquireMergeLocks({+ transactionScope,+ objectSingularName,+ workspaceId,+ payloads,+ });++ const timelineActivityRepository =+ transactionScope.getRepository<ObjectLiteral>('timelineActivity', {+ shouldBypassPermissionChecks: true,+ });++ const recentTimelineActivities =+ await this.findRecentTimelineActivities({+ timelineActivityRepository,+ objectSingularName,+ payloads,+ });++ const payloadsToInsert: TimelineActivityPayloadWorkspaceIdAndObjectSingularName['payloads'] =+ [];+ const mergesToApply: {+ id: string;+ properties: Partial<ObjectRecord>;+ workspaceMemberId: string | undefined;+ timelineActivityTypeSnapshot?: TimelineActivityPayload['timelineActivityTypeSnapshot'];+ }[] = [];++ const timelineActivityPropertyName =+ this.getTimelineActivityPropertyName(objectSingularName);++ // Bucketed once so matching a payload stays constant time: the recent+ // window is scoped to this batch but is not capped in size.+ const recentTimelineActivitiesByMergeKey = new Map<+ string,+ (typeof recentTimelineActivities)[number][]+ >();++ for (const timelineActivity of recentTimelineActivities) {+ const mergeKey = buildTimelineActivityMergeKey({+ recordId: timelineActivity[timelineActivityPropertyName],+ workspaceMemberId: timelineActivity.workspaceMemberId,+ timelineActivityTypeId: timelineActivity.timelineActivityTypeId,timelineActivityTypeSnapshot:- payload.timelineActivityTypeSnapshot,+ timelineActivity.timelineActivityTypeSnapshot,+ });++ const bucket = recentTimelineActivitiesByMergeKey.get(mergeKey);++ if (isDefined(bucket)) {+ bucket.push(timelineActivity);+ } else {+ recentTimelineActivitiesByMergeKey.set(mergeKey, [+ timelineActivity,+ ]);+ }+ }++ for (const payload of payloads) {+ const recentTimelineActivity =+ buildTimelineActivityMergeKeyCandidates({+ recordId: payload.recordId,+ workspaceMemberId: payload.workspaceMemberId,+ timelineActivityTypeId: payload.timelineActivityTypeId,+ timelineActivityTypeSnapshot:+ payload.timelineActivityTypeSnapshot,+ })+ .flatMap(+ (mergeKey) =>+ recentTimelineActivitiesByMergeKey.get(mergeKey) ?? [],+ )+ .find(+ (timelineActivity) =>+ !isDefined(payload.linkedRecordId) ||+ timelineActivity.linkedRecordId === payload.linkedRecordId,+ );++ if (isDefined(recentTimelineActivity)) {+ mergesToApply.push({+ id: recentTimelineActivity.id,+ properties: objectRecordDiffMerge(+ recentTimelineActivity.properties,+ payload.properties,+ ),+ workspaceMemberId: payload.workspaceMemberId,+ ...(!isDefined(+ recentTimelineActivity.timelineActivityTypeSnapshot,+ ) && {+ timelineActivityTypeSnapshot:+ payload.timelineActivityTypeSnapshot,+ }),+ });+ } else {+ payloadsToInsert.push(payload);+ }+ }++ await Promise.all([+ this.updateTimelineActivities({+ timelineActivityRepository,+ merges: mergesToApply,}),- });- } else {- payloadsToInsert.push(payload);- }- }-- await Promise.all([- this.updateTimelineActivities({ merges: mergesToApply, workspaceId }),- this.insertTimelineActivities({- objectSingularName,- payloads: payloadsToInsert,- workspaceId,- }),- ]);+ this.insertTimelineActivities({+ timelineActivityRepository,+ objectSingularName,+ payloads: payloadsToInsert,+ }),+ ]);+ },+ );}, authContext);}- private async findRecentTimelineActivities({+ private async acquireMergeLocks({+ transactionScope,objectSingularName,workspaceId,payloads,- }: TimelineActivityPayloadWorkspaceIdAndObjectSingularName) {- const timelineActivityTypeORMRepository =- await this.globalWorkspaceOrmManager.getRepository(- workspaceId,- 'timelineActivity',- {- shouldBypassPermissionChecks: true,- },- );+ }: {+ transactionScope: WorkspaceTransactionScope;+ objectSingularName: string;+ workspaceId: string;+ payloads: TimelineActivityPayloadWorkspaceIdAndObjectSingularName['payloads'];+ }) {+ const lockNames = [+ ...new Set(+ payloads.map((payload) =>+ JSON.stringify([+ 'timeline-activity-merge',+ workspaceId,+ objectSingularName,+ payload.recordId,+ payload.workspaceMemberId ?? null,+ payload.timelineActivityTypeId,+ ]),+ ),+ ),+ ].sort();++ await transactionScope.executeRawQuery(+ ACQUIRE_TIMELINE_ACTIVITY_MERGE_LOCK,+ [lockNames],+ );+ }+ private async findRecentTimelineActivities({+ timelineActivityRepository,+ objectSingularName,+ payloads,+ }: {+ timelineActivityRepository: WorkspaceRepository<ObjectLiteral>;+ objectSingularName: string;+ payloads: TimelineActivityPayloadWorkspaceIdAndObjectSingularName['payloads'];+ }) {const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000);const timelineActivityPropertyName =- await this.getTimelineActivityPropertyName(objectSingularName);+ this.getTimelineActivityPropertyName(objectSingularName);const whereConditions: Record<string, unknown> = {[timelineActivityPropertyName]: In(@@ -159,7 +216,7 @@ export class TimelineActivityRepository {// The where clause is already scoped to this batch payloads and to the merge// window, so every candidate is fetched: taking a single row would let only// one payload of a multi record batch merge.- return await timelineActivityTypeORMRepository.find({+ return await timelineActivityRepository.find({where: {...whereConditions,timelineActivityTypeId: In(@@ -171,27 +228,22 @@ export class TimelineActivityRepository {}public async insertTimelineActivities({+ timelineActivityRepository,objectSingularName,- workspaceId,payloads,- }: TimelineActivityPayloadWorkspaceIdAndObjectSingularName) {+ }: {+ timelineActivityRepository: WorkspaceRepository<ObjectLiteral>;+ objectSingularName: string;+ payloads: TimelineActivityPayloadWorkspaceIdAndObjectSingularName['payloads'];+ }) {if (payloads.length === 0) {return;}- const timelineActivityTypeORMRepository =- await this.globalWorkspaceOrmManager.getRepository(- workspaceId,- 'timelineActivity',- {- shouldBypassPermissionChecks: true,- },- );-const timelineActivityPropertyName =- await this.getTimelineActivityPropertyName(objectSingularName);+ this.getTimelineActivityPropertyName(objectSingularName);- return timelineActivityTypeORMRepository.insert(+ return timelineActivityRepository.insert(payloads.map((payload) => ({happensAt: payload.happensAt,timelineActivityTypeId: payload.timelineActivityTypeId,@@ -207,34 +259,25 @@ export class TimelineActivityRepository {}private async updateTimelineActivities({+ timelineActivityRepository,merges,- workspaceId,}: {+ timelineActivityRepository: WorkspaceRepository<ObjectLiteral>;merges: {id: string;properties: Partial<ObjectRecord>;workspaceMemberId: string | undefined;timelineActivityTypeSnapshot?: TimelineActivityPayload['timelineActivityTypeSnapshot'];}[];- workspaceId: string;}) {if (merges.length === 0) {return;}- const timelineActivityTypeORMRepository =- await this.globalWorkspaceOrmManager.getRepository(- workspaceId,- 'timelineActivity',- {- shouldBypassPermissionChecks: true,- },- );-await Promise.all(merges.map(({ id, properties, workspaceMemberId, timelineActivityTypeSnapshot }) =>- timelineActivityTypeORMRepository.update(id, {+ timelineActivityRepository.update(id, {properties,workspaceMemberId,...(isDefined(timelineActivityTypeSnapshot) && {@@ -245,7 +288,7 @@ export class TimelineActivityRepository {);}- private async getTimelineActivityPropertyName(objectSingularName: string) {+ private getTimelineActivityPropertyName(objectSingularName: string) {return `${buildTimelineActivityRelatedMorphFieldMetadataName(objectSingularName)}Id`;}}