Skip to content

The open alternative to Salesforce, designed for AI.

Configure async ClickHouse inserts for pageview events (#23274)

Weikob036d67ec9c6
diff --git a/packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.ts b/packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.ts
index 08e8f3cfa4e..1f32368f4d0 100644
--- a/packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.ts
+++ b/packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.ts
@@ -112,6 +112,25 @@ describe('ClickHouseService', () => {
});
});
+ it('should allow overriding the asynchronous insert busy timeout', async () => {
+ const testData = [{ id: 1, name: 'test' }];
+ const result = await service.insert('test_table', testData, {
+ asyncInsertBusyTimeoutMaxMs: 100,
+ });
+
+ expect(result).toEqual({ success: true });
+ expect(mockClickHouseClient.insert).toHaveBeenCalledWith({
+ table: 'test_table',
+ values: testData,
+ format: 'JSONEachRow',
+ clickhouse_settings: {
+ async_insert: 1,
+ async_insert_busy_timeout_max_ms: 100,
+ wait_for_async_insert: 1,
+ },
+ });
+ });
+
it('should return failure when clickhouse client is not defined', async () => {
(service as any).mainClient = undefined;
diff --git a/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts b/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts
index 8210d196192..9a5bae2f966 100644
--- a/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts
+++ b/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts
@@ -13,6 +13,11 @@ import {
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+export type ClickHouseInsertOptions = {
+ clientId?: string;
+ asyncInsertBusyTimeoutMaxMs?: number;
+};
+
@Injectable()
export class ClickHouseService implements OnModuleInit, OnModuleDestroy {
private mainClient: ClickHouseClient | undefined;
@@ -135,11 +140,11 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy {
public async insert<T extends Record<string, any>>(
table: string,
values: T[],
- clientId?: string,
+ options: ClickHouseInsertOptions = {},
): Promise<{ success: boolean }> {
try {
- const client = clientId
- ? await this.connectToClient(clientId)
+ const client = options.clientId
+ ? await this.connectToClient(options.clientId)
: this.mainClient;
if (!client) {
@@ -149,6 +154,7 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy {
await this.insertInChunks(client, table, values, {
chunkSize: 1000,
maxMemoryMB: 4,
+ asyncInsertBusyTimeoutMaxMs: options.asyncInsertBusyTimeoutMaxMs,
});
return { success: true };
@@ -260,7 +266,11 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy {
client: ClickHouseClient,
table: string,
values: T[],
- options: { chunkSize?: number; maxMemoryMB?: number } = {},
+ options: {
+ chunkSize?: number;
+ maxMemoryMB?: number;
+ asyncInsertBusyTimeoutMaxMs?: number;
+ } = {},
): Promise<void> {
const chunkSize = options.chunkSize ?? 1000;
const maxMemoryMB = options.maxMemoryMB;
@@ -276,6 +286,12 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy {
format: 'JSONEachRow',
clickhouse_settings: {
async_insert: 1,
+ ...(options.asyncInsertBusyTimeoutMaxMs !== undefined
+ ? {
+ async_insert_busy_timeout_max_ms:
+ options.asyncInsertBusyTimeoutMaxMs,
+ }
+ : {}),
wait_for_async_insert: 1,
},
});
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts
index 126a8c6c443..ca51de72292 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts
@@ -43,8 +43,14 @@ describe('ClickHouseEventSink', () => {
await sink.write([first, second, applicationLog]);
expect(insert).toHaveBeenCalledTimes(2);
- expect(insert).toHaveBeenCalledWith('pageview', [first.row, second.row]);
- expect(insert).toHaveBeenCalledWith('applicationLog', [applicationLog.row]);
+ expect(insert).toHaveBeenCalledWith('pageview', [first.row, second.row], {
+ asyncInsertBusyTimeoutMaxMs: 100,
+ });
+ expect(insert).toHaveBeenCalledWith(
+ 'applicationLog',
+ [applicationLog.row],
+ undefined,
+ );
});
it('no-ops when ClickHouse is not configured', async () => {
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts
index acbbe146e5e..e154fa3f430 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts
@@ -1,8 +1,22 @@
import { Injectable } from '@nestjs/common';
-import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
+import {
+ type ClickHouseInsertOptions,
+ ClickHouseService,
+} from 'src/database/clickHouse/clickHouse.service';
import { type EventSink } from 'src/engine/core-modules/event-logs/ingest/event-sink';
-import { type WorkspaceEventEnvelope } from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type';
+import {
+ type WorkspaceEventEnvelope,
+ type WorkspaceEventTable,
+} from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type';
+
+const CLICKHOUSE_INSERT_OPTIONS_BY_TABLE: Partial<
+ Record<WorkspaceEventTable, ClickHouseInsertOptions>
+> = {
+ pageview: {
+ asyncInsertBusyTimeoutMaxMs: 100,
+ },
+};
@Injectable()
export class ClickHouseEventSink implements EventSink {
@@ -13,7 +27,10 @@ export class ClickHouseEventSink implements EventSink {
return;
}
- const rowsByTable = new Map<string, Record<string, unknown>[]>();
+ const rowsByTable = new Map<
+ WorkspaceEventTable,
+ Record<string, unknown>[]
+ >();
for (const event of events) {
const rows = rowsByTable.get(event.table) ?? [];
@@ -24,7 +41,11 @@ export class ClickHouseEventSink implements EventSink {
await Promise.all(
[...rowsByTable.entries()].map(async ([table, rows]) => {
- const result = await this.clickHouseService.insert(table, rows);
+ const result = await this.clickHouseService.insert(
+ table,
+ rows,
+ CLICKHOUSE_INSERT_OPTIONS_BY_TABLE[table],
+ );
if (!result.success) {
throw new Error(