Skip to content

The open alternative to Salesforce, designed for AI.

Retry event-log inserts on transient ClickHouse network errors without Sentry noise (#24760)

Charles Bochetd422be8da10d
diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json
index f398789bd3c..623afa071c7 100644
--- a/packages/twenty-server/package.json
+++ b/packages/twenty-server/package.json
@@ -11,7 +11,7 @@
"worker:prod": "node dist/queue-worker/queue-worker",
"database:init:prod": "node dist/database/scripts/setup-db.js && yarn database:migrate:prod --force --include-slow",
"database:migrate:prod": "node dist/command/command run-instance-commands",
- "clickhouse:migrate:prod": "node dist/database/clickHouse/migrations/run-migrations.js",
+ "clickhouse:migrate:prod": "node dist/database/clickhouse/migrations/run-migrations.js",
"upgrade:background": "./scripts/upgrade-background.sh start",
"upgrade:background:logs": "./scripts/upgrade-background.sh logs",
"upgrade:background:stop": "./scripts/upgrade-background.sh stop",
diff --git a/packages/twenty-server/project.json b/packages/twenty-server/project.json
index 5295834d741..039a5a1fcd0 100644
--- a/packages/twenty-server/project.json
+++ b/packages/twenty-server/project.json
@@ -286,14 +286,14 @@
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
- "command": "nx ts-node-no-deps-transpile-only -- src/database/clickHouse/migrations/run-migrations.ts"
+ "command": "nx ts-node-no-deps-transpile-only -- src/database/clickhouse/migrations/run-migrations.ts"
}
},
"clickhouse:seed": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
- "command": "nx ts-node-no-deps-transpile-only -- src/database/clickHouse/seeds/run-seeds.ts"
+ "command": "nx ts-node-no-deps-transpile-only -- src/database/clickhouse/seeds/run-seeds.ts"
}
},
"lingui:extract": {
diff --git a/packages/twenty-server/src/app.module.ts b/packages/twenty-server/src/app.module.ts
index a574c627706..7991dcef2dd 100644
--- a/packages/twenty-server/src/app.module.ts
+++ b/packages/twenty-server/src/app.module.ts
@@ -38,7 +38,7 @@ import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/
import { UnhandledExceptionFilter } from 'src/filters/unhandled-exception.filter';
import { ModulesModule } from 'src/modules/modules.module';
-import { ClickHouseModule } from './database/clickHouse/clickHouse.module';
+import { ClickHouseModule } from './database/clickhouse/clickhouse.module';
import { CoreEngineModule } from './engine/core-modules/core-engine.module';
import { I18nModule } from './engine/core-modules/i18n/i18n.module';
diff --git a/packages/twenty-server/src/database/clickHouse/clickHouse.util.ts b/packages/twenty-server/src/database/clickHouse/clickHouse.util.ts
deleted file mode 100644
index 0da29614049..00000000000
--- a/packages/twenty-server/src/database/clickHouse/clickHouse.util.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Formats a Date or ISO string for ClickHouse DateTime64(3) type.
- * ClickHouse expects: YYYY-MM-DD HH:mm:ss.SSS (no 'T' separator, no 'Z' suffix)
- * JavaScript toISOString() returns: YYYY-MM-DDTHH:mm:ss.SSSZ
- */
-export const formatDateTimeForClickHouse = (date: Date | string): string => {
- const iso = typeof date === 'string' ? date : date.toISOString();
-
- // Extract date (YYYY-MM-DD) and time with milliseconds (HH:mm:ss.SSS)
- return `${iso.slice(0, 10)} ${iso.slice(11, 23)}`;
-};
-
-export const formatDateForClickHouse = (date: Date): string =>
- date.toISOString().slice(0, 10);
-
-// ClickHouse returns DateTime64 values as naive strings (YYYY-MM-DD HH:mm:ss.SSS) in UTC.
-// Parse them as UTC explicitly, otherwise `new Date` assumes the server's local timezone.
-export const parseClickHouseDateTime = (value: string): Date =>
- new Date(`${value.replace(' ', 'T')}Z`);
diff --git a/packages/twenty-server/src/database/clickHouse/clickHouse.module.ts b/packages/twenty-server/src/database/clickhouse/clickhouse.module.ts
similarity index 83%
rename from packages/twenty-server/src/database/clickHouse/clickHouse.module.ts
rename to packages/twenty-server/src/database/clickhouse/clickhouse.module.ts
index 203b4863884..20596722260 100644
--- a/packages/twenty-server/src/database/clickHouse/clickHouse.module.ts
+++ b/packages/twenty-server/src/database/clickhouse/clickhouse.module.ts
@@ -2,7 +2,7 @@ import { Module } from '@nestjs/common';
import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
-import { ClickHouseService } from './clickHouse.service';
+import { ClickHouseService } from './clickhouse.service';
@Module({
imports: [TwentyConfigModule],
diff --git a/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts b/packages/twenty-server/src/database/clickhouse/clickhouse.service.ts
similarity index 100%
rename from packages/twenty-server/src/database/clickHouse/clickHouse.service.ts
rename to packages/twenty-server/src/database/clickhouse/clickhouse.service.ts
diff --git a/packages/twenty-server/src/database/clickHouse/migrations/001-create-workspace-event-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/001-create-workspace-event-table.sql
similarity index 100%
rename from packages/twenty-server/src/database/clickHouse/migrations/001-create-workspace-event-table.sql
rename to packages/twenty-server/src/database/clickhouse/migrations/001-create-workspace-event-table.sql
diff --git a/packages/twenty-server/src/database/clickHouse/migrations/002-create-pageview-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/002-create-pageview-table.sql
similarity index 100%
rename from packages/twenty-server/src/database/clickHouse/migrations/002-create-pageview-table.sql
rename to packages/twenty-server/src/database/clickhouse/migrations/002-create-pageview-table.sql
diff --git a/packages/twenty-server/src/database/clickHouse/migrations/003-create-object-event-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/003-create-object-event-table.sql
similarity index 100%
rename from packages/twenty-server/src/database/clickHouse/migrations/003-create-object-event-table.sql
rename to packages/twenty-server/src/database/clickhouse/migrations/003-create-object-event-table.sql
diff --git a/packages/twenty-server/src/database/clickHouse/migrations/004-create-usage-event-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/004-create-usage-event-table.sql
similarity index 100%
rename from packages/twenty-server/src/database/clickHouse/migrations/004-create-usage-event-table.sql
rename to packages/twenty-server/src/database/clickhouse/migrations/004-create-usage-event-table.sql
diff --git a/packages/twenty-server/src/database/clickHouse/migrations/005-create-application-log-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/005-create-application-log-table.sql
similarity index 100%
rename from packages/twenty-server/src/database/clickHouse/migrations/005-create-application-log-table.sql
rename to packages/twenty-server/src/database/clickhouse/migrations/005-create-application-log-table.sql
diff --git a/packages/twenty-server/src/database/clickHouse/migrations/006-update-usage-event-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/006-update-usage-event-table.sql
similarity index 100%
rename from packages/twenty-server/src/database/clickHouse/migrations/006-update-usage-event-table.sql
rename to packages/twenty-server/src/database/clickhouse/migrations/006-update-usage-event-table.sql
diff --git a/packages/twenty-server/src/database/clickHouse/migrations/run-migrations.ts b/packages/twenty-server/src/database/clickhouse/migrations/run-migrations.ts
similarity index 100%
rename from packages/twenty-server/src/database/clickHouse/migrations/run-migrations.ts
rename to packages/twenty-server/src/database/clickhouse/migrations/run-migrations.ts
diff --git a/packages/twenty-server/src/database/clickHouse/seeds/fixtures.ts b/packages/twenty-server/src/database/clickhouse/seeds/fixtures.ts
similarity index 98%
rename from packages/twenty-server/src/database/clickHouse/seeds/fixtures.ts
rename to packages/twenty-server/src/database/clickhouse/seeds/fixtures.ts
index 4ddcec9aac0..1ba4f5cfb43 100644
--- a/packages/twenty-server/src/database/clickHouse/seeds/fixtures.ts
+++ b/packages/twenty-server/src/database/clickhouse/seeds/fixtures.ts
@@ -4,7 +4,7 @@ import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/event-logs/
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/event-logs/emit/events/workspace-event/custom-domain/custom-domain-activated';
import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/event-logs/emit/events/workspace-event/custom-domain/custom-domain-deactivated';
import { type GenericTrackEvent } from 'src/engine/core-modules/event-logs/emit/events/workspace-event/track';
-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';
+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';
import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';
import { USER_WORKSPACE_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/core/utils/seed-user-workspaces.util';
diff --git a/packages/twenty-server/src/database/clickHouse/seeds/run-seeds.ts b/packages/twenty-server/src/database/clickhouse/seeds/run-seeds.ts
similarity index 100%
rename from packages/twenty-server/src/database/clickHouse/seeds/run-seeds.ts
rename to packages/twenty-server/src/database/clickhouse/seeds/run-seeds.ts
diff --git a/packages/twenty-server/src/database/clickhouse/utils/format-date-for-clickhouse.util.ts b/packages/twenty-server/src/database/clickhouse/utils/format-date-for-clickhouse.util.ts
new file mode 100644
index 00000000000..4e7944eabb4
--- /dev/null
+++ b/packages/twenty-server/src/database/clickhouse/utils/format-date-for-clickhouse.util.ts
@@ -0,0 +1,2 @@
+export const formatDateForClickHouse = (date: Date): string =>
+ date.toISOString().slice(0, 10);
diff --git a/packages/twenty-server/src/database/clickhouse/utils/format-date-time-for-clickhouse.util.ts b/packages/twenty-server/src/database/clickhouse/utils/format-date-time-for-clickhouse.util.ts
new file mode 100644
index 00000000000..38389bd006e
--- /dev/null
+++ b/packages/twenty-server/src/database/clickhouse/utils/format-date-time-for-clickhouse.util.ts
@@ -0,0 +1,7 @@
+// ClickHouse DateTime64(3) expects YYYY-MM-DD HH:mm:ss.SSS (no 'T' separator,
+// no 'Z' suffix), while toISOString() returns YYYY-MM-DDTHH:mm:ss.SSSZ.
+export const formatDateTimeForClickHouse = (date: Date | string): string => {
+ const iso = typeof date === 'string' ? date : date.toISOString();
+
+ return `${iso.slice(0, 10)} ${iso.slice(11, 23)}`;
+};
diff --git a/packages/twenty-server/src/database/clickhouse/utils/is-transient-clickhouse-network-error.util.ts b/packages/twenty-server/src/database/clickhouse/utils/is-transient-clickhouse-network-error.util.ts
new file mode 100644
index 00000000000..23f3f95ff53
--- /dev/null
+++ b/packages/twenty-server/src/database/clickhouse/utils/is-transient-clickhouse-network-error.util.ts
@@ -0,0 +1,32 @@
+const TRANSIENT_NETWORK_ERROR_CODES = new Set([
+ 'ECONNRESET',
+ 'ECONNABORTED',
+ 'EPIPE',
+ 'ETIMEDOUT',
+]);
+
+const TRANSIENT_NETWORK_ERROR_MESSAGES = new Set([
+ 'Timeout error.',
+ 'socket hang up',
+]);
+
+export const isTransientClickHouseNetworkError = (
+ error: unknown,
+): error is Error => {
+ if (!(error instanceof Error)) {
+ return false;
+ }
+
+ const code = (error as NodeJS.ErrnoException).code;
+
+ if (
+ (code !== undefined && TRANSIENT_NETWORK_ERROR_CODES.has(code)) ||
+ TRANSIENT_NETWORK_ERROR_MESSAGES.has(error.message)
+ ) {
+ return true;
+ }
+
+ return isTransientClickHouseNetworkError(
+ (error as Error & { cause?: unknown }).cause,
+ );
+};
diff --git a/packages/twenty-server/src/database/clickhouse/utils/parse-clickhouse-date-time.util.ts b/packages/twenty-server/src/database/clickhouse/utils/parse-clickhouse-date-time.util.ts
new file mode 100644
index 00000000000..11dc58cf0a7
--- /dev/null
+++ b/packages/twenty-server/src/database/clickhouse/utils/parse-clickhouse-date-time.util.ts
@@ -0,0 +1,4 @@
+// ClickHouse returns DateTime64 values as naive strings (YYYY-MM-DD HH:mm:ss.SSS) in UTC.
+// Parse them as UTC explicitly, otherwise `new Date` assumes the server's local timezone.
+export const parseClickHouseDateTime = (value: string): Date =>
+ new Date(`${value.replace(' ', 'T')}Z`);
diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts
index bdc75658d32..86876782e47 100644
--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts
@@ -122,6 +122,7 @@ export class EntityEventsToDbListener {
>(
UpsertTimelineActivityFromInternalEvent.name,
batchEvent as WorkspaceEventBatch<ObjectRecordNonDestructiveEvent>,
+ { retryLimit: 1 },
),
);
}
@@ -131,6 +132,7 @@ export class EntityEventsToDbListener {
this.entityEventsToDbQueueService.add<WorkspaceEventBatch<T>>(
CreateEventLogFromInternalEvent.name,
batchEvent,
+ { retryLimit: 1 },
),
);
}
diff --git a/packages/twenty-server/src/engine/core-modules/billing/billing.module.ts b/packages/twenty-server/src/engine/core-modules/billing/billing.module.ts
index cd4868a7977..334748874f1 100644
--- a/packages/twenty-server/src/engine/core-modules/billing/billing.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing/billing.module.ts
@@ -3,7 +3,7 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
-import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';
+import { ClickHouseModule } from 'src/database/clickhouse/clickhouse.module';
import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module';
import { CoreEntityCacheModule } from 'src/engine/core-entity-cache/core-entity-cache.module';
import { BillingGaugeService } from 'src/engine/core-modules/billing/billing-gauge.service';
diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts
index 91af87b5f77..c5a93543858 100644
--- a/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts
@@ -5,8 +5,8 @@ import { Injectable, Logger } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
-import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';
+import { ClickHouseService } from 'src/database/clickhouse/clickhouse.service';
+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';
import { CoreEntityCacheService } from 'src/engine/core-entity-cache/services/core-entity-cache.service';
import {
BillingException,
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/event-log-cleanup.module.ts b/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/event-log-cleanup.module.ts
index 18f0b79d994..62773d20dd4 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/event-log-cleanup.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/event-log-cleanup.module.ts
@@ -3,7 +3,7 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
-import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';
+import { ClickHouseModule } from 'src/database/clickhouse/clickhouse.module';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { EventLogCleanupCronCommand } from 'src/engine/core-modules/event-logs/cleanup/commands/event-log-cleanup.cron.command';
import { EventLogCleanupCronJob } from 'src/engine/core-modules/event-logs/cleanup/crons/event-log-cleanup.cron.job';
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/services/event-log-cleanup.service.ts b/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/services/event-log-cleanup.service.ts
index 14bd99cebf3..160e0cfcf86 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/services/event-log-cleanup.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/services/event-log-cleanup.service.ts
@@ -4,8 +4,8 @@ import { Injectable, Logger } from '@nestjs/common';
import { EventLogTable } from 'twenty-shared/types';
-import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';
+import { ClickHouseService } from 'src/database/clickhouse/clickhouse.service';
+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';
import { getClickHouseTableName } from 'src/engine/core-modules/event-logs/registry/event-log-registry';
export type EventLogCleanupParams = {
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts b/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts
index 7ebbd74d866..f2bd49fa12f 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts
@@ -1,4 +1,4 @@
-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';
+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';
import { type EventCommonPropertiesType } from 'src/engine/core-modules/event-logs/emit/common.type';
import {
type TrackEventName,
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/event-logs-viewer.module.ts b/packages/twenty-server/src/engine/core-modules/event-logs/event-logs-viewer.module.ts
index f163209e698..ce242fa2543 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/event-logs-viewer.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/event-logs-viewer.module.ts
@@ -10,7 +10,7 @@ import { EventLogEmitterResolver } from 'src/engine/core-modules/event-logs/emit
import { EventLogLiveModule } from 'src/engine/core-modules/event-logs/live/event-log-live.module';
import { GuardRedirectModule } from 'src/engine/core-modules/guard-redirect/guard-redirect.module';
import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module';
-import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';
+import { ClickHouseModule } from 'src/database/clickhouse/clickhouse.module';
import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
import { SubscriptionsModule } from 'src/engine/subscriptions/subscriptions.module';
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/event-logs.service.ts b/packages/twenty-server/src/engine/core-modules/event-logs/event-logs.service.ts
index 039ecdc4c23..7f47a239d02 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/event-logs.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/event-logs.service.ts
@@ -7,8 +7,8 @@ import { EventLogTable } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { Repository } from 'typeorm';
-import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';
+import { ClickHouseService } from 'src/database/clickhouse/clickhouse.service';
+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';
import { BillingService } from 'src/engine/core-modules/billing/services/billing.service';
import { EnterprisePlanService } from 'src/engine/core-modules/enterprise/services/enterprise-plan.service';
import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts
index 7265cec1a6c..f9fef9805e0 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts
@@ -1,4 +1,4 @@
-import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
+import { ClickHouseService } from 'src/database/clickhouse/clickhouse.service';
import { ClickHouseEventSink } from 'src/engine/core-modules/event-logs/ingest/clickhouse-event.sink';
import { type WorkspaceEventEnvelope } from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type';
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts
index fab5aae3d72..60fadc83656 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts
@@ -3,7 +3,7 @@ import { Injectable } from '@nestjs/common';
import {
type ClickHouseInsertOptions,
ClickHouseService,
-} from 'src/database/clickHouse/clickHouse.service';
+} from 'src/database/clickhouse/clickhouse.service';
import { type EventSink } from 'src/engine/core-modules/event-logs/ingest/event-sink';
import {
type WorkspaceEventEnvelope,
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.spec.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.spec.ts
index 08afea7736f..141653ef526 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.spec.ts
@@ -2,24 +2,40 @@ import { type ObjectRecordEvent } from 'twenty-shared/database-events';
import { CreateEventLogFromInternalEvent } from 'src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event';
import { WorkspaceEventSinkService } from 'src/engine/core-modules/event-logs/ingest/workspace-event-sink.service';
+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
describe('CreateEventLogFromInternalEvent', () => {
- it('ingests object events (persist + live fan-out) through the sink pipeline', async () => {
- const ingest = jest.fn().mockResolvedValue(undefined);
+ let ingest: jest.Mock;
+ let addToQueue: jest.Mock;
+ let handler: CreateEventLogFromInternalEvent;
+
+ const batch = {
+ name: 'company.created',
+ workspaceId: 'workspace-1',
+ objectMetadata: { id: 'object-metadata-1' },
+ events: [{ recordId: 'record-1', userId: 'user-1', properties: {} }],
+ } as unknown as WorkspaceEventBatch<ObjectRecordEvent>;
- const handler = new CreateEventLogFromInternalEvent({
- isEnabled: () => true,
- ingest,
- } as unknown as WorkspaceEventSinkService);
+ const transientError = Object.assign(
+ new Error('Failed to insert 1 objectEvent row(s) into ClickHouse'),
+ { cause: new Error('Timeout error.') },
+ );
- const batch = {
- name: 'company.created',
- workspaceId: 'workspace-1',
- objectMetadata: { id: 'object-metadata-1' },
- events: [{ recordId: 'record-1', userId: 'user-1', properties: {} }],
- } as unknown as WorkspaceEventBatch<ObjectRecordEvent>;
+ beforeEach(() => {
+ ingest = jest.fn().mockResolvedValue(undefined);
+ addToQueue = jest.fn().mockResolvedValue(undefined);
+ handler = new CreateEventLogFromInternalEvent(
+ {
+ isEnabled: () => true,
+ ingest,
+ } as unknown as WorkspaceEventSinkService,
+ { add: addToQueue } as unknown as MessageQueueService,
+ );
+ });
+
+ it('ingests object events (persist + live fan-out) through the sink pipeline', async () => {
await handler.handle(batch);
const ingestedEnvelopes = ingest.mock.calls[0]?.[0];
@@ -27,4 +43,36 @@ describe('CreateEventLogFromInternalEvent', () => {
expect(ingestedEnvelopes).toHaveLength(1);
expect(ingestedEnvelopes[0].table).toBe('objectEvent');
});
+
+ it('requeues once on a transient ClickHouse network error instead of failing', async () => {
+ ingest.mockRejectedValue(transientError);
+
+ await expect(handler.handle(batch)).resolves.toBeUndefined();
+
+ expect(addToQueue).toHaveBeenCalledWith(
+ CreateEventLogFromInternalEvent.name,
+ expect.objectContaining({ transientErrorRetryCount: 1 }),
+ );
+ });
+
+ it('drops the batch after the transient retry budget is exhausted', async () => {
+ ingest.mockRejectedValue(transientError);
+
+ await expect(
+ handler.handle({ ...batch, transientErrorRetryCount: 1 }),
+ ).resolves.toBeUndefined();
+
+ expect(addToQueue).not.toHaveBeenCalled();
+ });
+
+ it('rethrows non-transient ingestion errors so they fail the job and reach Sentry', async () => {
+ const error = new Error(
+ 'Failed to insert 1 objectEvent row(s) into ClickHouse: (total) memory limit exceeded',
+ );
+
+ ingest.mockRejectedValue(error);
+
+ await expect(handler.handle(batch)).rejects.toBe(error);
+ expect(addToQueue).not.toHaveBeenCalled();
+ });
});
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts
index 5d3c15d463f..402fa8481ea 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts
@@ -1,6 +1,9 @@
+import { Logger } from '@nestjs/common';
+
import { type ObjectRecordEvent } from 'twenty-shared/database-events';
import { isDefined } from 'twenty-shared/utils';
+import { isTransientClickHouseNetworkError } from 'src/database/clickhouse/utils/is-transient-clickhouse-network-error.util';
import { WorkspaceEventSinkService } from 'src/engine/core-modules/event-logs/ingest/workspace-event-sink.service';
import { type WorkspaceEventEnvelope } from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type';
import {
@@ -11,11 +14,20 @@ import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/event-logs/
import { OBJECT_RECORD_DELETED_EVENT } from 'src/engine/core-modules/event-logs/emit/events/object-event/object-record-delete';
import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/event-logs/emit/events/object-event/object-record-updated';
import { OBJECT_RECORD_UPSERTED_EVENT } from 'src/engine/core-modules/event-logs/emit/events/object-event/object-record-upserted';
+import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
+const MAX_TRANSIENT_ERROR_RETRIES = 1;
+
+export type CreateEventLogFromInternalEventData =
+ WorkspaceEventBatch<ObjectRecordEvent> & {
+ transientErrorRetryCount?: number;
+ };
+
const OBJECT_EVENT_BY_SUFFIX = {
'.created': OBJECT_RECORD_CREATED_EVENT,
'.updated': OBJECT_RECORD_UPDATED_EVENT,
@@ -25,12 +37,16 @@ const OBJECT_EVENT_BY_SUFFIX = {
@Processor(MessageQueue.entityEventsToDbQueue)
export class CreateEventLogFromInternalEvent {
+ private readonly logger = new Logger(CreateEventLogFromInternalEvent.name);
+
constructor(
private readonly workspaceEventSinkService: WorkspaceEventSinkService,
+ @InjectMessageQueue(MessageQueue.entityEventsToDbQueue)
+ private readonly messageQueueService: MessageQueueService,
) {}
@Process(CreateEventLogFromInternalEvent.name)
- async handle(batch: WorkspaceEventBatch<ObjectRecordEvent>): Promise<void> {
+ async handle(batch: CreateEventLogFromInternalEventData): Promise<void> {
if (!this.workspaceEventSinkService.isEnabled()) {
return;
}
@@ -41,7 +57,40 @@ export class CreateEventLogFromInternalEvent {
return;
}
- await this.workspaceEventSinkService.ingest(envelopes);
+ try {
+ await this.workspaceEventSinkService.ingest(envelopes);
+ } catch (error) {
+ if (!isTransientClickHouseNetworkError(error)) {
+ throw error;
+ }
+
+ await this.handleTransientError(batch, envelopes.length, error);
+ }
+ }
+
+ private async handleTransientError(
+ batch: CreateEventLogFromInternalEventData,
+ envelopeCount: number,
+ error: Error,
+ ): Promise<void> {
+ const transientErrorRetryCount = batch.transientErrorRetryCount ?? 0;
+
+ if (transientErrorRetryCount >= MAX_TRANSIENT_ERROR_RETRIES) {
+ this.logger.warn(
+ `Dropping ${envelopeCount} event log envelope(s) for workspace ${batch.workspaceId} after ${transientErrorRetryCount + 1} transient ClickHouse network errors: ${error.message}`,
+ );
+
+ return;
+ }
+
+ this.logger.warn(
+ `Requeuing ${envelopeCount} event log envelope(s) for workspace ${batch.workspaceId} after a transient ClickHouse network error: ${error.message}`,
+ );
+
+ await this.messageQueueService.add<CreateEventLogFromInternalEventData>(
+ CreateEventLogFromInternalEvent.name,
+ { ...batch, transientErrorRetryCount: transientErrorRetryCount + 1 },
+ );
}
private toEnvelopes(
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts
index 709ea9bd291..cb43c999be9 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts
@@ -2,7 +2,7 @@ import { Logger, Module } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
-import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';
+import { ClickHouseModule } from 'src/database/clickhouse/clickhouse.module';
import { ClickHouseEventSink } from 'src/engine/core-modules/event-logs/ingest/clickhouse-event.sink';
import { ConsoleEventSink } from 'src/engine/core-modules/event-logs/ingest/console-event.sink';
import { CreateEventLogFromInternalEvent } from 'src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event';
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/producers/application-log/build-application-log-envelopes.ts b/packages/twenty-server/src/engine/core-modules/event-logs/producers/application-log/build-application-log-envelopes.ts
index bac7678f77d..f1024aa4dbd 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/producers/application-log/build-application-log-envelopes.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/producers/application-log/build-application-log-envelopes.ts
@@ -1,4 +1,4 @@
-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';
+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';
import { type ApplicationLogEntry } from 'src/engine/core-modules/event-logs/producers/application-log/application-log-entry.interface';
import { type WorkspaceEventEnvelope } from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type';
diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/utils/normalize-event-log-records.ts b/packages/twenty-server/src/engine/core-modules/event-logs/utils/normalize-event-log-records.ts
index 46800953a22..235cff11437 100644
--- a/packages/twenty-server/src/engine/core-modules/event-logs/utils/normalize-event-log-records.ts
+++ b/packages/twenty-server/src/engine/core-modules/event-logs/utils/normalize-event-log-records.ts
@@ -1,6 +1,6 @@
import { EventLogTable } from 'twenty-shared/types';
-import { parseClickHouseDateTime } from 'src/database/clickHouse/clickHouse.util';
+import { parseClickHouseDateTime } from 'src/database/clickhouse/utils/parse-clickhouse-date-time.util';
import { type EventLogRecord } from 'src/engine/core-modules/event-logs/dtos/event-log-result.dto';
import { EVENT_LOG_TYPES } from 'src/engine/core-modules/event-logs/registry/event-log-registry';
diff --git a/packages/twenty-server/src/engine/core-modules/usage/services/usage-analytics.service.ts b/packages/twenty-server/src/engine/core-modules/usage/services/usage-analytics.service.ts
index f0fdf897f0d..1bb689afde6 100644
--- a/packages/twenty-server/src/engine/core-modules/usage/services/usage-analytics.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage/services/usage-analytics.service.ts
@@ -2,8 +2,8 @@
import { Injectable } from '@nestjs/common';
-import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';
+import { ClickHouseService } from 'src/database/clickhouse/clickhouse.service';
+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';
import { fillUsageTimeSeriesGaps } from 'src/engine/core-modules/usage/utils/fill-usage-time-series-gaps.util';
import { toDisplayCredits } from 'src/engine/core-modules/usage/utils/to-display-credits.util';
import { toDollars } from 'src/engine/core-modules/usage/utils/to-dollars.util';
diff --git a/packages/twenty-server/src/engine/core-modules/usage/usage.module.ts b/packages/twenty-server/src/engine/core-modules/usage/usage.module.ts
index 7a577c5b2ba..7fb3923a50f 100644
--- a/packages/twenty-server/src/engine/core-modules/usage/usage.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage/usage.module.ts
@@ -3,7 +3,7 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
-import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';
+import { ClickHouseModule } from 'src/database/clickhouse/clickhouse.module';
import { EventLogEmitterModule } from 'src/engine/core-modules/event-logs/emit/event-log-emitter.module';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { UsageEventListener } from 'src/engine/core-modules/usage/listeners/usage-event.listener';
diff --git a/packages/twenty-server/src/engine/core-modules/usage/utils/build-usage-event-envelopes.ts b/packages/twenty-server/src/engine/core-modules/usage/utils/build-usage-event-envelopes.ts
index b493f9a8b6b..ec8322d666a 100644
--- a/packages/twenty-server/src/engine/core-modules/usage/utils/build-usage-event-envelopes.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage/utils/build-usage-event-envelopes.ts
@@ -1,6 +1,6 @@
/* @license Enterprise */
-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';
+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';
import { type WorkspaceEventEnvelope } from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type';
import { type UsageEvent } from 'src/engine/core-modules/usage/types/usage-event.type';
diff --git a/packages/twenty-server/test/integration/graphql/suites/event-logs/event-logs.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/event-logs/event-logs.integration-spec.ts
index 39fbee4a380..37e548c0747 100644
--- a/packages/twenty-server/test/integration/graphql/suites/event-logs/event-logs.integration-spec.ts
+++ b/packages/twenty-server/test/integration/graphql/suites/event-logs/event-logs.integration-spec.ts
@@ -7,7 +7,7 @@ import {
} from '@clickhouse/client';
import request from 'supertest';
-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';
+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';
import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';
import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';