The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Configure async ClickHouse inserts for pageview events (#23274)
Weikob036d67ec9c6
diff --git a/packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.ts b/packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.tsindex 08e8f3cfa4e..1f32368f4d0 100644--- a/packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.ts+++ b/packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.ts@@ -112,6 +112,25 @@ describe('ClickHouseService', () => {});});+ it('should allow overriding the asynchronous insert busy timeout', async () => {+ const testData = [{ id: 1, name: 'test' }];+ const result = await service.insert('test_table', testData, {+ asyncInsertBusyTimeoutMaxMs: 100,+ });++ expect(result).toEqual({ success: true });+ expect(mockClickHouseClient.insert).toHaveBeenCalledWith({+ table: 'test_table',+ values: testData,+ format: 'JSONEachRow',+ clickhouse_settings: {+ async_insert: 1,+ async_insert_busy_timeout_max_ms: 100,+ wait_for_async_insert: 1,+ },+ });+ });+it('should return failure when clickhouse client is not defined', async () => {(service as any).mainClient = undefined;diff --git a/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts b/packages/twenty-server/src/database/clickHouse/clickHouse.service.tsindex 8210d196192..9a5bae2f966 100644--- a/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts+++ b/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts@@ -13,6 +13,11 @@ import {import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';+export type ClickHouseInsertOptions = {+ clientId?: string;+ asyncInsertBusyTimeoutMaxMs?: number;+};+@Injectable()export class ClickHouseService implements OnModuleInit, OnModuleDestroy {private mainClient: ClickHouseClient | undefined;@@ -135,11 +140,11 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy {public async insert<T extends Record<string, any>>(table: string,values: T[],- clientId?: string,+ options: ClickHouseInsertOptions = {},): Promise<{ success: boolean }> {try {- const client = clientId- ? await this.connectToClient(clientId)+ const client = options.clientId+ ? await this.connectToClient(options.clientId): this.mainClient;if (!client) {@@ -149,6 +154,7 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy {await this.insertInChunks(client, table, values, {chunkSize: 1000,maxMemoryMB: 4,+ asyncInsertBusyTimeoutMaxMs: options.asyncInsertBusyTimeoutMaxMs,});return { success: true };@@ -260,7 +266,11 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy {client: ClickHouseClient,table: string,values: T[],- options: { chunkSize?: number; maxMemoryMB?: number } = {},+ options: {+ chunkSize?: number;+ maxMemoryMB?: number;+ asyncInsertBusyTimeoutMaxMs?: number;+ } = {},): Promise<void> {const chunkSize = options.chunkSize ?? 1000;const maxMemoryMB = options.maxMemoryMB;@@ -276,6 +286,12 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy {format: 'JSONEachRow',clickhouse_settings: {async_insert: 1,+ ...(options.asyncInsertBusyTimeoutMaxMs !== undefined+ ? {+ async_insert_busy_timeout_max_ms:+ options.asyncInsertBusyTimeoutMaxMs,+ }+ : {}),wait_for_async_insert: 1,},});diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.tsindex 126a8c6c443..ca51de72292 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts@@ -43,8 +43,14 @@ describe('ClickHouseEventSink', () => {await sink.write([first, second, applicationLog]);expect(insert).toHaveBeenCalledTimes(2);- expect(insert).toHaveBeenCalledWith('pageview', [first.row, second.row]);- expect(insert).toHaveBeenCalledWith('applicationLog', [applicationLog.row]);+ expect(insert).toHaveBeenCalledWith('pageview', [first.row, second.row], {+ asyncInsertBusyTimeoutMaxMs: 100,+ });+ expect(insert).toHaveBeenCalledWith(+ 'applicationLog',+ [applicationLog.row],+ undefined,+ );});it('no-ops when ClickHouse is not configured', async () => {diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.tsindex acbbe146e5e..e154fa3f430 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts@@ -1,8 +1,22 @@import { Injectable } from '@nestjs/common';-import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';+import {+ type ClickHouseInsertOptions,+ ClickHouseService,+} from 'src/database/clickHouse/clickHouse.service';import { type EventSink } from 'src/engine/core-modules/event-logs/ingest/event-sink';-import { type WorkspaceEventEnvelope } from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type';+import {+ type WorkspaceEventEnvelope,+ type WorkspaceEventTable,+} from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type';++const CLICKHOUSE_INSERT_OPTIONS_BY_TABLE: Partial<+ Record<WorkspaceEventTable, ClickHouseInsertOptions>+> = {+ pageview: {+ asyncInsertBusyTimeoutMaxMs: 100,+ },+};@Injectable()export class ClickHouseEventSink implements EventSink {@@ -13,7 +27,10 @@ export class ClickHouseEventSink implements EventSink {return;}- const rowsByTable = new Map<string, Record<string, unknown>[]>();+ const rowsByTable = new Map<+ WorkspaceEventTable,+ Record<string, unknown>[]+ >();for (const event of events) {const rows = rowsByTable.get(event.table) ?? [];@@ -24,7 +41,11 @@ export class ClickHouseEventSink implements EventSink {await Promise.all([...rowsByTable.entries()].map(async ([table, rows]) => {- const result = await this.clickHouseService.insert(table, rows);+ const result = await this.clickHouseService.insert(+ table,+ rows,+ CLICKHOUSE_INSERT_OPTIONS_BY_TABLE[table],+ );if (!result.success) {throw new Error(