The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Keep ClickHouse stalls from freezing the timeline queue (#26183)
neo7731598258e70e7
diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.tsindex 61752e597b2..f2f254cb2fc 100644--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts@@ -35,6 +35,8 @@ export class EntityEventsToDbListener {constructor(@InjectMessageQueue(MessageQueue.entityEventsToDbQueue)private readonly entityEventsToDbQueueService: MessageQueueService,+ @InjectMessageQueue(MessageQueue.eventLogQueue)+ private readonly eventLogQueueService: MessageQueueService,@InjectMessageQueue(MessageQueue.webhookQueue)private readonly webhookQueueService: MessageQueueService,@InjectMessageQueue(MessageQueue.triggerQueue)@@ -141,7 +143,7 @@ export class EntityEventsToDbListener {return;}- await this.entityEventsToDbQueueService.add<WorkspaceEventBatch<T>>(+ await this.eventLogQueueService.add<WorkspaceEventBatch<T>>(CreateEventLogFromInternalEvent.name,batchEvent,{ retryLimit: 1 },diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts b/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.tsindex f2bd49fa12f..38eb747bbc2 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts@@ -13,8 +13,10 @@ import {type GenericTrackEvent,} from 'src/engine/core-modules/event-logs/emit/events/workspace-event/track';-const common = (): Record<EventCommonPropertiesType, string> => ({- timestamp: formatDateTimeForClickHouse(new Date()),+const common = (+ occurredAt: Date = new Date(),+): Record<EventCommonPropertiesType, string> => ({+ timestamp: formatDateTimeForClickHouse(occurredAt),version: '1',});@@ -33,6 +35,7 @@ export function makePageview(export function makeTrackEvent<T extends TrackEventName>(event: T,properties: TrackEventProperties<T>,+ occurredAt?: Date,): GenericTrackEvent<T> {const schema = eventsRegistry.get(event);@@ -44,6 +47,6 @@ export function makeTrackEvent<T extends TrackEventName>(type: 'track',event,properties,- ...common(),+ ...common(occurredAt),});}diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/emit/build-event-envelope.ts b/packages/twenty-server/src/engine/core-modules/event-logs/emit/build-event-envelope.tsindex 816d90474ef..3caef92cf99 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/emit/build-event-envelope.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/emit/build-event-envelope.ts@@ -23,9 +23,10 @@ export const buildWorkspaceEventEnvelope = <T extends TrackEventName>(contextFields: EventContextFields,event: T,properties: TrackEventProperties<T>,+ occurredAt?: Date,): WorkspaceEventEnvelope => ({table: 'workspaceEvent',- row: { ...contextFields, ...makeTrackEvent(event, properties) },+ row: { ...contextFields, ...makeTrackEvent(event, properties, occurredAt) },});export const buildObjectEventEnvelope = <T extends TrackEventName>(diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.tsindex 402fa8481ea..5e4d6223fed 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts@@ -35,13 +35,13 @@ const OBJECT_EVENT_BY_SUFFIX = {'.upserted': OBJECT_RECORD_UPSERTED_EVENT,} as const;-@Processor(MessageQueue.entityEventsToDbQueue)+@Processor(MessageQueue.eventLogQueue)export class CreateEventLogFromInternalEvent {private readonly logger = new Logger(CreateEventLogFromInternalEvent.name);constructor(private readonly workspaceEventSinkService: WorkspaceEventSinkService,- @InjectMessageQueue(MessageQueue.entityEventsToDbQueue)+ @InjectMessageQueue(MessageQueue.eventLogQueue)private readonly messageQueueService: MessageQueueService,) {}diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.tsindex cb43c999be9..c448383182a 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts@@ -14,6 +14,7 @@ import {EVENT_SINKS,type EventSink,} from 'src/engine/core-modules/event-logs/ingest/event-sink';+import { ForwardEventLogFromEntityEventsToDbQueue } from 'src/engine/core-modules/event-logs/ingest/forward-event-log-from-entity-events-to-db-queue';import { WorkspaceEventSinkService } from 'src/engine/core-modules/event-logs/ingest/workspace-event-sink.service';import { EventLogLiveModule } from 'src/engine/core-modules/event-logs/live/event-log-live.module';import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';@@ -59,6 +60,7 @@ const eventSinksProvider = {eventSinksProvider,WorkspaceEventSinkService,CreateEventLogFromInternalEvent,+ ForwardEventLogFromEntityEventsToDbQueue,],exports: [WorkspaceEventSinkService],})diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/forward-event-log-from-entity-events-to-db-queue.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/forward-event-log-from-entity-events-to-db-queue.tsnew file mode 100644index 00000000000..238b975e3c2--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/forward-event-log-from-entity-events-to-db-queue.ts@@ -0,0 +1,26 @@+import {+ CreateEventLogFromInternalEvent,+ type CreateEventLogFromInternalEventData,+} from 'src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event';+import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';+import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';+import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';++@Processor(MessageQueue.entityEventsToDbQueue)+export class ForwardEventLogFromEntityEventsToDbQueue {+ constructor(+ @InjectMessageQueue(MessageQueue.eventLogQueue)+ private readonly eventLogQueueService: MessageQueueService,+ ) {}++ @Process(CreateEventLogFromInternalEvent.name)+ async handle(batch: CreateEventLogFromInternalEventData): Promise<void> {+ await this.eventLogQueueService.add<CreateEventLogFromInternalEventData>(+ CreateEventLogFromInternalEvent.name,+ batch,+ { retryLimit: 1 },+ );+ }+}diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.tsindex df92072f4d4..663530130bd 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts@@ -148,6 +148,15 @@ export const MESSAGE_QUEUE_WORKER_CONFIG: Record<boundedShutdownDrain: false,},},+ [MessageQueue.eventLogQueue]: {+ priority: 1,+ workerOptions: {+ concurrency: 3,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },[MessageQueue.workflowQueue]: {priority: 2,workerOptions: {diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.tsindex 8d4d1969ae3..c69434ab4f5 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts@@ -17,6 +17,7 @@ export enum MessageQueue {billingQueue = 'billing-queue',workspaceQueue = 'workspace-queue',entityEventsToDbQueue = 'entity-events-to-db-queue',+ eventLogQueue = 'event-log-queue',workflowQueue = 'workflow-queue',delayedJobsQueue = 'delayed-jobs-queue',deleteCascadeQueue = 'delete-cascade-queue',diff --git a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook.job.ts b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook.job.tsindex eaa78780db3..b8f12a289cc 100644--- a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook.job.ts+++ b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook.job.ts@@ -1,8 +1,15 @@+import { Logger } from '@nestjs/common';+import crypto from 'crypto';import { ensureAbsoluteUrl } from 'twenty-shared/utils';+import {+ buildWorkspaceEventEnvelope,+ computeEventContextFields,+} from 'src/engine/core-modules/event-logs/emit/build-event-envelope';import { EventLogEmitterService } from 'src/engine/core-modules/event-logs/emit/event-log-emitter.service';+import { type TrackEventProperties } from 'src/engine/core-modules/event-logs/emit/events.type';import { WEBHOOK_RESPONSE_EVENT } from 'src/engine/core-modules/event-logs/emit/events/workspace-event/webhook/webhook-response';import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';@@ -12,8 +19,16 @@ import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.import { SecureHttpClientService } from 'src/engine/core-modules/secure-http-client/secure-http-client.service';import { type WebhookJobData } from 'src/engine/metadata-modules/webhook/types/webhook-job-data.type';+type WebhookResponse = {+ workspaceId: string;+ respondedAt: Date;+ properties: TrackEventProperties<typeof WEBHOOK_RESPONSE_EVENT>;+};+@Processor(MessageQueue.webhookQueue)export class CallWebhookJob {+ private readonly logger = new Logger(CallWebhookJob.name);+constructor(private readonly eventLogEmitterService: EventLogEmitterService,private readonly metricsService: MetricsService,@@ -33,22 +48,38 @@ export class CallWebhookJob {@Process(CallWebhookJob.name)async handle(webhookJobEvents: WebhookJobData[]): Promise<void> {- await Promise.all(- webhookJobEvents.map(- async (webhookJobEvent) => await this.callWebhook(webhookJobEvent),+ const webhookResponses = await Promise.all(+ webhookJobEvents.map((webhookJobEvent) =>+ this.callWebhook(webhookJobEvent),+ ),+ );++ void this.recordWebhookResponses(webhookResponses).catch((error) =>+ this.logger.error('Failed to record webhook responses', error),+ );+ }++ private async recordWebhookResponses(+ webhookResponses: WebhookResponse[],+ ): Promise<void> {+ await this.eventLogEmitterService.dispatch(+ webhookResponses.map(({ workspaceId, respondedAt, properties }) =>+ buildWorkspaceEventEnvelope(+ computeEventContextFields({ workspaceId }),+ WEBHOOK_RESPONSE_EVENT,+ properties,+ respondedAt,+ ),),);}- private async callWebhook(data: WebhookJobData): Promise<void> {+ private async callWebhook(data: WebhookJobData): Promise<WebhookResponse> {const commonPayload = {url: data.targetUrl,webhookId: data.webhookId,eventName: data.eventName,};- const eventLogContext = this.eventLogEmitterService.createContext({- workspaceId: data.workspaceId,- });try {const headers: Record<string, string> = {@@ -89,30 +120,38 @@ export class CallWebhookJob {const success = response.status >= 200 && response.status < 300;- void eventLogContext.insertWorkspaceEvent(WEBHOOK_RESPONSE_EVENT, {- status: response.status,- success,- ...commonPayload,- });-void this.metricsService.incrementCounterForEvent({key: MetricsKeys.JobWebhookCallCompleted,shouldStoreInCache: false,});++ return {+ workspaceId: data.workspaceId,+ respondedAt: new Date(),+ properties: {+ status: response.status,+ success,+ ...commonPayload,+ },+ };} catch (err) {const isSSRFBlocked =err instanceof Error &&err.message.includes('internal IP address') &&err.message.includes('is not allowed');- void eventLogContext.insertWorkspaceEvent(WEBHOOK_RESPONSE_EVENT, {- success: false,- ...commonPayload,- ...(err.response && { status: err.response.status }),- ...(isSSRFBlocked && {- error: 'Webhook URL resolves to a private/internal IP address',- }),- });+ return {+ workspaceId: data.workspaceId,+ respondedAt: new Date(),+ properties: {+ success: false,+ ...commonPayload,+ ...(err.response && { status: err.response.status }),+ ...(isSSRFBlocked && {+ error: 'Webhook URL resolves to a private/internal IP address',+ }),+ },+ };}}}diff --git a/packages/twenty-server/test/integration/graphql/suites/event-logs/object-event-write.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/event-logs/object-event-write.integration-spec.tsindex 79ab16108b8..b3840dcd02a 100644--- a/packages/twenty-server/test/integration/graphql/suites/event-logs/object-event-write.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/event-logs/object-event-write.integration-spec.ts@@ -17,7 +17,7 @@ import { makeMetadataAPIRequest } from 'test/integration/metadata/suites/utils/mimport { deleteAllRecords } from 'test/integration/utils/delete-all-records';// End-to-end write path: creating a record fires an objectEvent through-// entityEventsToDbQueue -> the consumer -> the ClickHouse sink.+// eventLogQueue -> the consumer -> the ClickHouse sink.describe('Object event write (integration)', () => {let clickHouseClient: ClickHouseClient;let personObjectMetadataId: string;diff --git a/packages/twenty-server/test/integration/graphql/suites/event-logs/webhook-response-write.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/event-logs/webhook-response-write.integration-spec.tsnew file mode 100644index 00000000000..234a0ea48b1--- /dev/null+++ b/packages/twenty-server/test/integration/graphql/suites/event-logs/webhook-response-write.integration-spec.ts@@ -0,0 +1,183 @@+import process from 'process';++import {+ type ClickHouseClient,+ ClickHouseLogLevel,+ createClient,+} from '@clickhouse/client';+import { gql } from 'graphql-tag';+import { v4 as uuidv4 } from 'uuid';++import { WEBHOOK_RESPONSE_EVENT } from 'src/engine/core-modules/event-logs/emit/events/workspace-event/webhook/webhook-response';+import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';++import { makeGraphqlAPIRequest } from 'test/integration/graphql/utils/make-graphql-api-request.util';+import {+ createWebhook,+ createWebhookReceiver,+ deleteWebhook,+} from 'test/integration/metadata/suites/utils/webhook-test.util';+import { makeAdminPanelAPIRequest } from 'test/integration/twenty-config/utils/make-admin-panel-api-request.util';+import { expectEventually } from 'test/integration/utils/expect-eventually.util';++const WEBHOOK_RECEIVER_PORT = 4319;+const UNREACHABLE_WEBHOOK_PORT = 4321;++const CREATE_CONFIG_VARIABLE_MUTATION = gql`+ mutation CreateDatabaseConfigVariable($key: String!, $value: JSON!) {+ createDatabaseConfigVariable(key: $key, value: $value)+ }+`;++const DELETE_CONFIG_VARIABLE_MUTATION = gql`+ mutation DeleteDatabaseConfigVariable($key: String!) {+ deleteDatabaseConfigVariable(key: $key)+ }+`;++const CREATE_PERSON_MUTATION = gql`+ mutation CreatePerson($data: PersonCreateInput!) {+ createPerson(data: $data) {+ id+ }+ }+`;++const DESTROY_PERSON_MUTATION = gql`+ mutation DestroyPerson($id: ID!) {+ destroyPerson(id: $id) {+ id+ }+ }+`;++type WebhookResponseRow = {+ timestamp: string;+ properties: {+ webhookId: string;+ success: boolean;+ status?: number | string;+ };+};++describe('Webhook response event write (integration)', () => {+ let clickHouseClient: ClickHouseClient;+ let receiver: Awaited<ReturnType<typeof createWebhookReceiver>>;+ let reachableWebhookId: string;+ let unreachableWebhookId: string;+ let personId: string | undefined;++ const queryWebhookResponseRows = async () => {+ const result = await clickHouseClient.query({+ query: `+ SELECT timestamp, properties+ FROM workspaceEvent+ WHERE event = '${WEBHOOK_RESPONSE_EVENT}' AND workspaceId = '${SEED_APPLE_WORKSPACE_ID}'+ `,+ format: 'JSONEachRow',+ });++ const rows = await result.json<WebhookResponseRow>();++ return rows.filter(({ properties }) =>+ [reachableWebhookId, unreachableWebhookId].includes(properties.webhookId),+ );+ };++ beforeAll(async () => {+ jest.useRealTimers();++ clickHouseClient = createClient({+ url: process.env.CLICKHOUSE_URL,+ log: { level: ClickHouseLogLevel.OFF },+ });++ await makeAdminPanelAPIRequest({+ query: CREATE_CONFIG_VARIABLE_MUTATION,+ variables: { key: 'OUTBOUND_HTTP_ALLOWED_INTERNAL_HOSTS', value: ['*'] },+ });++ receiver = await createWebhookReceiver(WEBHOOK_RECEIVER_PORT);++ const reachableWebhookResponse = await createWebhook({+ targetUrl: `http://127.0.0.1:${WEBHOOK_RECEIVER_PORT}/webhook`,+ operations: ['person.created'],+ description: 'Webhook response write test (reachable)',+ });++ expect(reachableWebhookResponse.body.errors).toBeUndefined();+ reachableWebhookId = reachableWebhookResponse.body.data.createWebhook.id;++ const unreachableWebhookResponse = await createWebhook({+ targetUrl: `http://127.0.0.1:${UNREACHABLE_WEBHOOK_PORT}/webhook`,+ operations: ['person.created'],+ description: 'Webhook response write test (unreachable)',+ });++ expect(unreachableWebhookResponse.body.errors).toBeUndefined();+ unreachableWebhookId =+ unreachableWebhookResponse.body.data.createWebhook.id;+ });++ afterAll(async () => {+ await receiver.close();+ await deleteWebhook(reachableWebhookId).catch(() => {});+ await deleteWebhook(unreachableWebhookId).catch(() => {});++ if (personId) {+ await makeGraphqlAPIRequest({+ query: DESTROY_PERSON_MUTATION,+ variables: { id: personId },+ }).catch(() => {});+ }++ await makeAdminPanelAPIRequest({+ query: DELETE_CONFIG_VARIABLE_MUTATION,+ variables: { key: 'OUTBOUND_HTTP_ALLOWED_INTERNAL_HOSTS' },+ }).catch(() => {});++ await clickHouseClient.command({+ query: `ALTER TABLE workspaceEvent DELETE WHERE event = '${WEBHOOK_RESPONSE_EVENT}' AND workspaceId = '${SEED_APPLE_WORKSPACE_ID}'`,+ });+ await clickHouseClient.close();++ jest.useFakeTimers();+ });++ it('records one response row per webhook call, successful and failed alike', async () => {+ const createPersonResponse = await makeGraphqlAPIRequest({+ query: CREATE_PERSON_MUTATION,+ variables: {+ data: {+ name: {+ firstName: 'WebhookResponse',+ lastName: `Test-${uuidv4().slice(0, 8)}`,+ },+ },+ },+ });++ expect(createPersonResponse.body.errors).toBeUndefined();+ personId = createPersonResponse.body.data.createPerson.id;++ await expectEventually(+ async () => {+ expect(await queryWebhookResponseRows()).toHaveLength(2);+ },+ { timeoutMs: 30_000, intervalMs: 500 },+ );++ const rows = await queryWebhookResponseRows();+ const reachableRow = rows.find(+ ({ properties }) => properties.webhookId === reachableWebhookId,+ );+ const unreachableRow = rows.find(+ ({ properties }) => properties.webhookId === unreachableWebhookId,+ );++ expect(receiver.receivedPayloads).toHaveLength(1);+ expect(reachableRow?.properties.success).toBe(true);+ expect(Number(reachableRow?.properties.status)).toBe(200);+ expect(unreachableRow?.properties.success).toBe(false);+ }, 60_000);+});