Skip to content

The open alternative to Salesforce, designed for AI.

Keep ClickHouse stalls from freezing the timeline queue (#26183)

neo7731598258e70e7
diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts
index 61752e597b2..f2f254cb2fc 100644
--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts
@@ -35,6 +35,8 @@ export class EntityEventsToDbListener {
constructor(
@InjectMessageQueue(MessageQueue.entityEventsToDbQueue)
private readonly entityEventsToDbQueueService: MessageQueueService,
+ @InjectMessageQueue(MessageQueue.eventLogQueue)
+ private readonly eventLogQueueService: MessageQueueService,
@InjectMessageQueue(MessageQueue.webhookQueue)
private readonly webhookQueueService: MessageQueueService,
@InjectMessageQueue(MessageQueue.triggerQueue)
@@ -141,7 +143,7 @@ export class EntityEventsToDbListener {
return;
}
- await this.entityEventsToDbQueueService.add<WorkspaceEventBatch<T>>(
+ await this.eventLogQueueService.add<WorkspaceEventBatch<T>>(
CreateEventLogFromInternalEvent.name,
batchEvent,
{ retryLimit: 1 },
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts b/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts
index f2bd49fa12f..38eb747bbc2 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts
@@ -13,8 +13,10 @@ import {
type GenericTrackEvent,
} from 'src/engine/core-modules/event-logs/emit/events/workspace-event/track';
-const common = (): Record<EventCommonPropertiesType, string> => ({
- timestamp: formatDateTimeForClickHouse(new Date()),
+const common = (
+ occurredAt: Date = new Date(),
+): Record<EventCommonPropertiesType, string> => ({
+ timestamp: formatDateTimeForClickHouse(occurredAt),
version: '1',
});
@@ -33,6 +35,7 @@ export function makePageview(
export function makeTrackEvent<T extends TrackEventName>(
event: T,
properties: TrackEventProperties<T>,
+ occurredAt?: Date,
): GenericTrackEvent<T> {
const schema = eventsRegistry.get(event);
@@ -44,6 +47,6 @@ export function makeTrackEvent<T extends TrackEventName>(
type: 'track',
event,
properties,
- ...common(),
+ ...common(occurredAt),
});
}
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/emit/build-event-envelope.ts b/packages/twenty-server/src/engine/core-modules/event-logs/emit/build-event-envelope.ts
index 816d90474ef..3caef92cf99 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/emit/build-event-envelope.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/emit/build-event-envelope.ts
@@ -23,9 +23,10 @@ export const buildWorkspaceEventEnvelope = <T extends TrackEventName>(
contextFields: EventContextFields,
event: T,
properties: TrackEventProperties<T>,
+ occurredAt?: Date,
): WorkspaceEventEnvelope => ({
table: 'workspaceEvent',
- row: { ...contextFields, ...makeTrackEvent(event, properties) },
+ row: { ...contextFields, ...makeTrackEvent(event, properties, occurredAt) },
});
export const buildObjectEventEnvelope = <T extends TrackEventName>(
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts
index 402fa8481ea..5e4d6223fed 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts
@@ -35,13 +35,13 @@ const OBJECT_EVENT_BY_SUFFIX = {
'.upserted': OBJECT_RECORD_UPSERTED_EVENT,
} as const;
-@Processor(MessageQueue.entityEventsToDbQueue)
+@Processor(MessageQueue.eventLogQueue)
export class CreateEventLogFromInternalEvent {
private readonly logger = new Logger(CreateEventLogFromInternalEvent.name);
constructor(
private readonly workspaceEventSinkService: WorkspaceEventSinkService,
- @InjectMessageQueue(MessageQueue.entityEventsToDbQueue)
+ @InjectMessageQueue(MessageQueue.eventLogQueue)
private readonly messageQueueService: MessageQueueService,
) {}
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts
index cb43c999be9..c448383182a 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts
@@ -14,6 +14,7 @@ import {
EVENT_SINKS,
type EventSink,
} from 'src/engine/core-modules/event-logs/ingest/event-sink';
+import { ForwardEventLogFromEntityEventsToDbQueue } from 'src/engine/core-modules/event-logs/ingest/forward-event-log-from-entity-events-to-db-queue';
import { WorkspaceEventSinkService } from 'src/engine/core-modules/event-logs/ingest/workspace-event-sink.service';
import { EventLogLiveModule } from 'src/engine/core-modules/event-logs/live/event-log-live.module';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
@@ -59,6 +60,7 @@ const eventSinksProvider = {
eventSinksProvider,
WorkspaceEventSinkService,
CreateEventLogFromInternalEvent,
+ ForwardEventLogFromEntityEventsToDbQueue,
],
exports: [WorkspaceEventSinkService],
})
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/forward-event-log-from-entity-events-to-db-queue.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/forward-event-log-from-entity-events-to-db-queue.ts
new file mode 100644
index 00000000000..238b975e3c2
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/forward-event-log-from-entity-events-to-db-queue.ts
@@ -0,0 +1,26 @@
+import {
+ CreateEventLogFromInternalEvent,
+ type CreateEventLogFromInternalEventData,
+} from 'src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event';
+import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
+import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
+import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
+
+@Processor(MessageQueue.entityEventsToDbQueue)
+export class ForwardEventLogFromEntityEventsToDbQueue {
+ constructor(
+ @InjectMessageQueue(MessageQueue.eventLogQueue)
+ private readonly eventLogQueueService: MessageQueueService,
+ ) {}
+
+ @Process(CreateEventLogFromInternalEvent.name)
+ async handle(batch: CreateEventLogFromInternalEventData): Promise<void> {
+ await this.eventLogQueueService.add<CreateEventLogFromInternalEventData>(
+ CreateEventLogFromInternalEvent.name,
+ batch,
+ { retryLimit: 1 },
+ );
+ }
+}
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts
index df92072f4d4..663530130bd 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts
@@ -148,6 +148,15 @@ export const MESSAGE_QUEUE_WORKER_CONFIG: Record<
boundedShutdownDrain: false,
},
},
+ [MessageQueue.eventLogQueue]: {
+ priority: 1,
+ workerOptions: {
+ concurrency: 3,
+ lockDuration: 30_000,
+ maxStalledCount: 1,
+ boundedShutdownDrain: false,
+ },
+ },
[MessageQueue.workflowQueue]: {
priority: 2,
workerOptions: {
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts
index 8d4d1969ae3..c69434ab4f5 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts
@@ -17,6 +17,7 @@ export enum MessageQueue {
billingQueue = 'billing-queue',
workspaceQueue = 'workspace-queue',
entityEventsToDbQueue = 'entity-events-to-db-queue',
+ eventLogQueue = 'event-log-queue',
workflowQueue = 'workflow-queue',
delayedJobsQueue = 'delayed-jobs-queue',
deleteCascadeQueue = 'delete-cascade-queue',
diff --git a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook.job.ts b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook.job.ts
index eaa78780db3..b8f12a289cc 100644
--- a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook.job.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook.job.ts
@@ -1,8 +1,15 @@
+import { Logger } from '@nestjs/common';
+
import crypto from 'crypto';
import { ensureAbsoluteUrl } from 'twenty-shared/utils';
+import {
+ buildWorkspaceEventEnvelope,
+ computeEventContextFields,
+} from 'src/engine/core-modules/event-logs/emit/build-event-envelope';
import { EventLogEmitterService } from 'src/engine/core-modules/event-logs/emit/event-log-emitter.service';
+import { type TrackEventProperties } from 'src/engine/core-modules/event-logs/emit/events.type';
import { WEBHOOK_RESPONSE_EVENT } from 'src/engine/core-modules/event-logs/emit/events/workspace-event/webhook/webhook-response';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
@@ -12,8 +19,16 @@ import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.
import { SecureHttpClientService } from 'src/engine/core-modules/secure-http-client/secure-http-client.service';
import { type WebhookJobData } from 'src/engine/metadata-modules/webhook/types/webhook-job-data.type';
+type WebhookResponse = {
+ workspaceId: string;
+ respondedAt: Date;
+ properties: TrackEventProperties<typeof WEBHOOK_RESPONSE_EVENT>;
+};
+
@Processor(MessageQueue.webhookQueue)
export class CallWebhookJob {
+ private readonly logger = new Logger(CallWebhookJob.name);
+
constructor(
private readonly eventLogEmitterService: EventLogEmitterService,
private readonly metricsService: MetricsService,
@@ -33,22 +48,38 @@ export class CallWebhookJob {
@Process(CallWebhookJob.name)
async handle(webhookJobEvents: WebhookJobData[]): Promise<void> {
- await Promise.all(
- webhookJobEvents.map(
- async (webhookJobEvent) => await this.callWebhook(webhookJobEvent),
+ const webhookResponses = await Promise.all(
+ webhookJobEvents.map((webhookJobEvent) =>
+ this.callWebhook(webhookJobEvent),
+ ),
+ );
+
+ void this.recordWebhookResponses(webhookResponses).catch((error) =>
+ this.logger.error('Failed to record webhook responses', error),
+ );
+ }
+
+ private async recordWebhookResponses(
+ webhookResponses: WebhookResponse[],
+ ): Promise<void> {
+ await this.eventLogEmitterService.dispatch(
+ webhookResponses.map(({ workspaceId, respondedAt, properties }) =>
+ buildWorkspaceEventEnvelope(
+ computeEventContextFields({ workspaceId }),
+ WEBHOOK_RESPONSE_EVENT,
+ properties,
+ respondedAt,
+ ),
),
);
}
- private async callWebhook(data: WebhookJobData): Promise<void> {
+ private async callWebhook(data: WebhookJobData): Promise<WebhookResponse> {
const commonPayload = {
url: data.targetUrl,
webhookId: data.webhookId,
eventName: data.eventName,
};
- const eventLogContext = this.eventLogEmitterService.createContext({
- workspaceId: data.workspaceId,
- });
try {
const headers: Record<string, string> = {
@@ -89,30 +120,38 @@ export class CallWebhookJob {
const success = response.status >= 200 && response.status < 300;
- void eventLogContext.insertWorkspaceEvent(WEBHOOK_RESPONSE_EVENT, {
- status: response.status,
- success,
- ...commonPayload,
- });
-
void this.metricsService.incrementCounterForEvent({
key: MetricsKeys.JobWebhookCallCompleted,
shouldStoreInCache: false,
});
+
+ return {
+ workspaceId: data.workspaceId,
+ respondedAt: new Date(),
+ properties: {
+ status: response.status,
+ success,
+ ...commonPayload,
+ },
+ };
} catch (err) {
const isSSRFBlocked =
err instanceof Error &&
err.message.includes('internal IP address') &&
err.message.includes('is not allowed');
- void eventLogContext.insertWorkspaceEvent(WEBHOOK_RESPONSE_EVENT, {
- success: false,
- ...commonPayload,
- ...(err.response && { status: err.response.status }),
- ...(isSSRFBlocked && {
- error: 'Webhook URL resolves to a private/internal IP address',
- }),
- });
+ return {
+ workspaceId: data.workspaceId,
+ respondedAt: new Date(),
+ properties: {
+ success: false,
+ ...commonPayload,
+ ...(err.response && { status: err.response.status }),
+ ...(isSSRFBlocked && {
+ error: 'Webhook URL resolves to a private/internal IP address',
+ }),
+ },
+ };
}
}
}
diff --git a/packages/twenty-server/test/integration/graphql/suites/event-logs/object-event-write.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/event-logs/object-event-write.integration-spec.ts
index 79ab16108b8..b3840dcd02a 100644
--- a/packages/twenty-server/test/integration/graphql/suites/event-logs/object-event-write.integration-spec.ts
+++ b/packages/twenty-server/test/integration/graphql/suites/event-logs/object-event-write.integration-spec.ts
@@ -17,7 +17,7 @@ import { makeMetadataAPIRequest } from 'test/integration/metadata/suites/utils/m
import { deleteAllRecords } from 'test/integration/utils/delete-all-records';
// End-to-end write path: creating a record fires an objectEvent through
-// entityEventsToDbQueue -> the consumer -> the ClickHouse sink.
+// eventLogQueue -> the consumer -> the ClickHouse sink.
describe('Object event write (integration)', () => {
let clickHouseClient: ClickHouseClient;
let personObjectMetadataId: string;
diff --git a/packages/twenty-server/test/integration/graphql/suites/event-logs/webhook-response-write.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/event-logs/webhook-response-write.integration-spec.ts
new file mode 100644
index 00000000000..234a0ea48b1
--- /dev/null
+++ b/packages/twenty-server/test/integration/graphql/suites/event-logs/webhook-response-write.integration-spec.ts
@@ -0,0 +1,183 @@
+import process from 'process';
+
+import {
+ type ClickHouseClient,
+ ClickHouseLogLevel,
+ createClient,
+} from '@clickhouse/client';
+import { gql } from 'graphql-tag';
+import { v4 as uuidv4 } from 'uuid';
+
+import { WEBHOOK_RESPONSE_EVENT } from 'src/engine/core-modules/event-logs/emit/events/workspace-event/webhook/webhook-response';
+import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';
+
+import { makeGraphqlAPIRequest } from 'test/integration/graphql/utils/make-graphql-api-request.util';
+import {
+ createWebhook,
+ createWebhookReceiver,
+ deleteWebhook,
+} from 'test/integration/metadata/suites/utils/webhook-test.util';
+import { makeAdminPanelAPIRequest } from 'test/integration/twenty-config/utils/make-admin-panel-api-request.util';
+import { expectEventually } from 'test/integration/utils/expect-eventually.util';
+
+const WEBHOOK_RECEIVER_PORT = 4319;
+const UNREACHABLE_WEBHOOK_PORT = 4321;
+
+const CREATE_CONFIG_VARIABLE_MUTATION = gql`
+ mutation CreateDatabaseConfigVariable($key: String!, $value: JSON!) {
+ createDatabaseConfigVariable(key: $key, value: $value)
+ }
+`;
+
+const DELETE_CONFIG_VARIABLE_MUTATION = gql`
+ mutation DeleteDatabaseConfigVariable($key: String!) {
+ deleteDatabaseConfigVariable(key: $key)
+ }
+`;
+
+const CREATE_PERSON_MUTATION = gql`
+ mutation CreatePerson($data: PersonCreateInput!) {
+ createPerson(data: $data) {
+ id
+ }
+ }
+`;
+
+const DESTROY_PERSON_MUTATION = gql`
+ mutation DestroyPerson($id: ID!) {
+ destroyPerson(id: $id) {
+ id
+ }
+ }
+`;
+
+type WebhookResponseRow = {
+ timestamp: string;
+ properties: {
+ webhookId: string;
+ success: boolean;
+ status?: number | string;
+ };
+};
+
+describe('Webhook response event write (integration)', () => {
+ let clickHouseClient: ClickHouseClient;
+ let receiver: Awaited<ReturnType<typeof createWebhookReceiver>>;
+ let reachableWebhookId: string;
+ let unreachableWebhookId: string;
+ let personId: string | undefined;
+
+ const queryWebhookResponseRows = async () => {
+ const result = await clickHouseClient.query({
+ query: `
+ SELECT timestamp, properties
+ FROM workspaceEvent
+ WHERE event = '${WEBHOOK_RESPONSE_EVENT}' AND workspaceId = '${SEED_APPLE_WORKSPACE_ID}'
+ `,
+ format: 'JSONEachRow',
+ });
+
+ const rows = await result.json<WebhookResponseRow>();
+
+ return rows.filter(({ properties }) =>
+ [reachableWebhookId, unreachableWebhookId].includes(properties.webhookId),
+ );
+ };
+
+ beforeAll(async () => {
+ jest.useRealTimers();
+
+ clickHouseClient = createClient({
+ url: process.env.CLICKHOUSE_URL,
+ log: { level: ClickHouseLogLevel.OFF },
+ });
+
+ await makeAdminPanelAPIRequest({
+ query: CREATE_CONFIG_VARIABLE_MUTATION,
+ variables: { key: 'OUTBOUND_HTTP_ALLOWED_INTERNAL_HOSTS', value: ['*'] },
+ });
+
+ receiver = await createWebhookReceiver(WEBHOOK_RECEIVER_PORT);
+
+ const reachableWebhookResponse = await createWebhook({
+ targetUrl: `http://127.0.0.1:${WEBHOOK_RECEIVER_PORT}/webhook`,
+ operations: ['person.created'],
+ description: 'Webhook response write test (reachable)',
+ });
+
+ expect(reachableWebhookResponse.body.errors).toBeUndefined();
+ reachableWebhookId = reachableWebhookResponse.body.data.createWebhook.id;
+
+ const unreachableWebhookResponse = await createWebhook({
+ targetUrl: `http://127.0.0.1:${UNREACHABLE_WEBHOOK_PORT}/webhook`,
+ operations: ['person.created'],
+ description: 'Webhook response write test (unreachable)',
+ });
+
+ expect(unreachableWebhookResponse.body.errors).toBeUndefined();
+ unreachableWebhookId =
+ unreachableWebhookResponse.body.data.createWebhook.id;
+ });
+
+ afterAll(async () => {
+ await receiver.close();
+ await deleteWebhook(reachableWebhookId).catch(() => {});
+ await deleteWebhook(unreachableWebhookId).catch(() => {});
+
+ if (personId) {
+ await makeGraphqlAPIRequest({
+ query: DESTROY_PERSON_MUTATION,
+ variables: { id: personId },
+ }).catch(() => {});
+ }
+
+ await makeAdminPanelAPIRequest({
+ query: DELETE_CONFIG_VARIABLE_MUTATION,
+ variables: { key: 'OUTBOUND_HTTP_ALLOWED_INTERNAL_HOSTS' },
+ }).catch(() => {});
+
+ await clickHouseClient.command({
+ query: `ALTER TABLE workspaceEvent DELETE WHERE event = '${WEBHOOK_RESPONSE_EVENT}' AND workspaceId = '${SEED_APPLE_WORKSPACE_ID}'`,
+ });
+ await clickHouseClient.close();
+
+ jest.useFakeTimers();
+ });
+
+ it('records one response row per webhook call, successful and failed alike', async () => {
+ const createPersonResponse = await makeGraphqlAPIRequest({
+ query: CREATE_PERSON_MUTATION,
+ variables: {
+ data: {
+ name: {
+ firstName: 'WebhookResponse',
+ lastName: `Test-${uuidv4().slice(0, 8)}`,
+ },
+ },
+ },
+ });
+
+ expect(createPersonResponse.body.errors).toBeUndefined();
+ personId = createPersonResponse.body.data.createPerson.id;
+
+ await expectEventually(
+ async () => {
+ expect(await queryWebhookResponseRows()).toHaveLength(2);
+ },
+ { timeoutMs: 30_000, intervalMs: 500 },
+ );
+
+ const rows = await queryWebhookResponseRows();
+ const reachableRow = rows.find(
+ ({ properties }) => properties.webhookId === reachableWebhookId,
+ );
+ const unreachableRow = rows.find(
+ ({ properties }) => properties.webhookId === unreachableWebhookId,
+ );
+
+ expect(receiver.receivedPayloads).toHaveLength(1);
+ expect(reachableRow?.properties.success).toBe(true);
+ expect(Number(reachableRow?.properties.status)).toBe(200);
+ expect(unreachableRow?.properties.success).toBe(false);
+ }, 60_000);
+});