The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Retry event-log inserts on transient ClickHouse network errors without Sentry noise (#24760)
Charles Bochetd422be8da10d
diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.jsonindex f398789bd3c..623afa071c7 100644--- a/packages/twenty-server/package.json+++ b/packages/twenty-server/package.json@@ -11,7 +11,7 @@"worker:prod": "node dist/queue-worker/queue-worker","database:init:prod": "node dist/database/scripts/setup-db.js && yarn database:migrate:prod --force --include-slow","database:migrate:prod": "node dist/command/command run-instance-commands",- "clickhouse:migrate:prod": "node dist/database/clickHouse/migrations/run-migrations.js",+ "clickhouse:migrate:prod": "node dist/database/clickhouse/migrations/run-migrations.js","upgrade:background": "./scripts/upgrade-background.sh start","upgrade:background:logs": "./scripts/upgrade-background.sh logs","upgrade:background:stop": "./scripts/upgrade-background.sh stop",diff --git a/packages/twenty-server/project.json b/packages/twenty-server/project.jsonindex 5295834d741..039a5a1fcd0 100644--- a/packages/twenty-server/project.json+++ b/packages/twenty-server/project.json@@ -286,14 +286,14 @@"executor": "nx:run-commands","options": {"cwd": "packages/twenty-server",- "command": "nx ts-node-no-deps-transpile-only -- src/database/clickHouse/migrations/run-migrations.ts"+ "command": "nx ts-node-no-deps-transpile-only -- src/database/clickhouse/migrations/run-migrations.ts"}},"clickhouse:seed": {"executor": "nx:run-commands","options": {"cwd": "packages/twenty-server",- "command": "nx ts-node-no-deps-transpile-only -- src/database/clickHouse/seeds/run-seeds.ts"+ "command": "nx ts-node-no-deps-transpile-only -- src/database/clickhouse/seeds/run-seeds.ts"}},"lingui:extract": {diff --git a/packages/twenty-server/src/app.module.ts b/packages/twenty-server/src/app.module.tsindex a574c627706..7991dcef2dd 100644--- a/packages/twenty-server/src/app.module.ts+++ b/packages/twenty-server/src/app.module.ts@@ -38,7 +38,7 @@ import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/import { UnhandledExceptionFilter } from 'src/filters/unhandled-exception.filter';import { ModulesModule } from 'src/modules/modules.module';-import { ClickHouseModule } from './database/clickHouse/clickHouse.module';+import { ClickHouseModule } from './database/clickhouse/clickhouse.module';import { CoreEngineModule } from './engine/core-modules/core-engine.module';import { I18nModule } from './engine/core-modules/i18n/i18n.module';diff --git a/packages/twenty-server/src/database/clickHouse/clickHouse.util.ts b/packages/twenty-server/src/database/clickHouse/clickHouse.util.tsdeleted file mode 100644index 0da29614049..00000000000--- a/packages/twenty-server/src/database/clickHouse/clickHouse.util.ts+++ /dev/null@@ -1,19 +0,0 @@-/**- * Formats a Date or ISO string for ClickHouse DateTime64(3) type.- * ClickHouse expects: YYYY-MM-DD HH:mm:ss.SSS (no 'T' separator, no 'Z' suffix)- * JavaScript toISOString() returns: YYYY-MM-DDTHH:mm:ss.SSSZ- */-export const formatDateTimeForClickHouse = (date: Date | string): string => {- const iso = typeof date === 'string' ? date : date.toISOString();-- // Extract date (YYYY-MM-DD) and time with milliseconds (HH:mm:ss.SSS)- return `${iso.slice(0, 10)} ${iso.slice(11, 23)}`;-};--export const formatDateForClickHouse = (date: Date): string =>- date.toISOString().slice(0, 10);--// ClickHouse returns DateTime64 values as naive strings (YYYY-MM-DD HH:mm:ss.SSS) in UTC.-// Parse them as UTC explicitly, otherwise `new Date` assumes the server's local timezone.-export const parseClickHouseDateTime = (value: string): Date =>- new Date(`${value.replace(' ', 'T')}Z`);diff --git a/packages/twenty-server/src/database/clickHouse/clickHouse.module.ts b/packages/twenty-server/src/database/clickhouse/clickhouse.module.tssimilarity index 83%rename from packages/twenty-server/src/database/clickHouse/clickHouse.module.tsrename to packages/twenty-server/src/database/clickhouse/clickhouse.module.tsindex 203b4863884..20596722260 100644--- a/packages/twenty-server/src/database/clickHouse/clickHouse.module.ts+++ b/packages/twenty-server/src/database/clickhouse/clickhouse.module.ts@@ -2,7 +2,7 @@ import { Module } from '@nestjs/common';import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';-import { ClickHouseService } from './clickHouse.service';+import { ClickHouseService } from './clickhouse.service';@Module({imports: [TwentyConfigModule],diff --git a/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts b/packages/twenty-server/src/database/clickhouse/clickhouse.service.tssimilarity index 100%rename from packages/twenty-server/src/database/clickHouse/clickHouse.service.tsrename to packages/twenty-server/src/database/clickhouse/clickhouse.service.tsdiff --git a/packages/twenty-server/src/database/clickHouse/migrations/001-create-workspace-event-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/001-create-workspace-event-table.sqlsimilarity index 100%rename from packages/twenty-server/src/database/clickHouse/migrations/001-create-workspace-event-table.sqlrename to packages/twenty-server/src/database/clickhouse/migrations/001-create-workspace-event-table.sqldiff --git a/packages/twenty-server/src/database/clickHouse/migrations/002-create-pageview-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/002-create-pageview-table.sqlsimilarity index 100%rename from packages/twenty-server/src/database/clickHouse/migrations/002-create-pageview-table.sqlrename to packages/twenty-server/src/database/clickhouse/migrations/002-create-pageview-table.sqldiff --git a/packages/twenty-server/src/database/clickHouse/migrations/003-create-object-event-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/003-create-object-event-table.sqlsimilarity index 100%rename from packages/twenty-server/src/database/clickHouse/migrations/003-create-object-event-table.sqlrename to packages/twenty-server/src/database/clickhouse/migrations/003-create-object-event-table.sqldiff --git a/packages/twenty-server/src/database/clickHouse/migrations/004-create-usage-event-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/004-create-usage-event-table.sqlsimilarity index 100%rename from packages/twenty-server/src/database/clickHouse/migrations/004-create-usage-event-table.sqlrename to packages/twenty-server/src/database/clickhouse/migrations/004-create-usage-event-table.sqldiff --git a/packages/twenty-server/src/database/clickHouse/migrations/005-create-application-log-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/005-create-application-log-table.sqlsimilarity index 100%rename from packages/twenty-server/src/database/clickHouse/migrations/005-create-application-log-table.sqlrename to packages/twenty-server/src/database/clickhouse/migrations/005-create-application-log-table.sqldiff --git a/packages/twenty-server/src/database/clickHouse/migrations/006-update-usage-event-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/006-update-usage-event-table.sqlsimilarity index 100%rename from packages/twenty-server/src/database/clickHouse/migrations/006-update-usage-event-table.sqlrename to packages/twenty-server/src/database/clickhouse/migrations/006-update-usage-event-table.sqldiff --git a/packages/twenty-server/src/database/clickHouse/migrations/run-migrations.ts b/packages/twenty-server/src/database/clickhouse/migrations/run-migrations.tssimilarity index 100%rename from packages/twenty-server/src/database/clickHouse/migrations/run-migrations.tsrename to packages/twenty-server/src/database/clickhouse/migrations/run-migrations.tsdiff --git a/packages/twenty-server/src/database/clickHouse/seeds/fixtures.ts b/packages/twenty-server/src/database/clickhouse/seeds/fixtures.tssimilarity index 98%rename from packages/twenty-server/src/database/clickHouse/seeds/fixtures.tsrename to packages/twenty-server/src/database/clickhouse/seeds/fixtures.tsindex 4ddcec9aac0..1ba4f5cfb43 100644--- a/packages/twenty-server/src/database/clickHouse/seeds/fixtures.ts+++ b/packages/twenty-server/src/database/clickhouse/seeds/fixtures.ts@@ -4,7 +4,7 @@ import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/event-logs/import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/event-logs/emit/events/workspace-event/custom-domain/custom-domain-activated';import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/event-logs/emit/events/workspace-event/custom-domain/custom-domain-deactivated';import { type GenericTrackEvent } from 'src/engine/core-modules/event-logs/emit/events/workspace-event/track';-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';import { USER_WORKSPACE_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/core/utils/seed-user-workspaces.util';diff --git a/packages/twenty-server/src/database/clickHouse/seeds/run-seeds.ts b/packages/twenty-server/src/database/clickhouse/seeds/run-seeds.tssimilarity index 100%rename from packages/twenty-server/src/database/clickHouse/seeds/run-seeds.tsrename to packages/twenty-server/src/database/clickhouse/seeds/run-seeds.tsdiff --git a/packages/twenty-server/src/database/clickhouse/utils/format-date-for-clickhouse.util.ts b/packages/twenty-server/src/database/clickhouse/utils/format-date-for-clickhouse.util.tsnew file mode 100644index 00000000000..4e7944eabb4--- /dev/null+++ b/packages/twenty-server/src/database/clickhouse/utils/format-date-for-clickhouse.util.ts@@ -0,0 +1,2 @@+export const formatDateForClickHouse = (date: Date): string =>+ date.toISOString().slice(0, 10);diff --git a/packages/twenty-server/src/database/clickhouse/utils/format-date-time-for-clickhouse.util.ts b/packages/twenty-server/src/database/clickhouse/utils/format-date-time-for-clickhouse.util.tsnew file mode 100644index 00000000000..38389bd006e--- /dev/null+++ b/packages/twenty-server/src/database/clickhouse/utils/format-date-time-for-clickhouse.util.ts@@ -0,0 +1,7 @@+// ClickHouse DateTime64(3) expects YYYY-MM-DD HH:mm:ss.SSS (no 'T' separator,+// no 'Z' suffix), while toISOString() returns YYYY-MM-DDTHH:mm:ss.SSSZ.+export const formatDateTimeForClickHouse = (date: Date | string): string => {+ const iso = typeof date === 'string' ? date : date.toISOString();++ return `${iso.slice(0, 10)} ${iso.slice(11, 23)}`;+};diff --git a/packages/twenty-server/src/database/clickhouse/utils/is-transient-clickhouse-network-error.util.ts b/packages/twenty-server/src/database/clickhouse/utils/is-transient-clickhouse-network-error.util.tsnew file mode 100644index 00000000000..23f3f95ff53--- /dev/null+++ b/packages/twenty-server/src/database/clickhouse/utils/is-transient-clickhouse-network-error.util.ts@@ -0,0 +1,32 @@+const TRANSIENT_NETWORK_ERROR_CODES = new Set([+ 'ECONNRESET',+ 'ECONNABORTED',+ 'EPIPE',+ 'ETIMEDOUT',+]);++const TRANSIENT_NETWORK_ERROR_MESSAGES = new Set([+ 'Timeout error.',+ 'socket hang up',+]);++export const isTransientClickHouseNetworkError = (+ error: unknown,+): error is Error => {+ if (!(error instanceof Error)) {+ return false;+ }++ const code = (error as NodeJS.ErrnoException).code;++ if (+ (code !== undefined && TRANSIENT_NETWORK_ERROR_CODES.has(code)) ||+ TRANSIENT_NETWORK_ERROR_MESSAGES.has(error.message)+ ) {+ return true;+ }++ return isTransientClickHouseNetworkError(+ (error as Error & { cause?: unknown }).cause,+ );+};diff --git a/packages/twenty-server/src/database/clickhouse/utils/parse-clickhouse-date-time.util.ts b/packages/twenty-server/src/database/clickhouse/utils/parse-clickhouse-date-time.util.tsnew file mode 100644index 00000000000..11dc58cf0a7--- /dev/null+++ b/packages/twenty-server/src/database/clickhouse/utils/parse-clickhouse-date-time.util.ts@@ -0,0 +1,4 @@+// ClickHouse returns DateTime64 values as naive strings (YYYY-MM-DD HH:mm:ss.SSS) in UTC.+// Parse them as UTC explicitly, otherwise `new Date` assumes the server's local timezone.+export const parseClickHouseDateTime = (value: string): Date =>+ new Date(`${value.replace(' ', 'T')}Z`);diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.tsindex bdc75658d32..86876782e47 100644--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts@@ -122,6 +122,7 @@ export class EntityEventsToDbListener {>(UpsertTimelineActivityFromInternalEvent.name,batchEvent as WorkspaceEventBatch<ObjectRecordNonDestructiveEvent>,+ { retryLimit: 1 },),);}@@ -131,6 +132,7 @@ export class EntityEventsToDbListener {this.entityEventsToDbQueueService.add<WorkspaceEventBatch<T>>(CreateEventLogFromInternalEvent.name,batchEvent,+ { retryLimit: 1 },),);}diff --git a/packages/twenty-server/src/engine/core-modules/billing/billing.module.ts b/packages/twenty-server/src/engine/core-modules/billing/billing.module.tsindex cd4868a7977..334748874f1 100644--- a/packages/twenty-server/src/engine/core-modules/billing/billing.module.ts+++ b/packages/twenty-server/src/engine/core-modules/billing/billing.module.ts@@ -3,7 +3,7 @@import { Module } from '@nestjs/common';import { TypeOrmModule } from '@nestjs/typeorm';-import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';+import { ClickHouseModule } from 'src/database/clickhouse/clickhouse.module';import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module';import { CoreEntityCacheModule } from 'src/engine/core-entity-cache/core-entity-cache.module';import { BillingGaugeService } from 'src/engine/core-modules/billing/billing-gauge.service';diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.tsindex 91af87b5f77..c5a93543858 100644--- a/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts+++ b/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts@@ -5,8 +5,8 @@ import { Injectable, Logger } from '@nestjs/common';import { isDefined } from 'twenty-shared/utils';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';-import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';+import { ClickHouseService } from 'src/database/clickhouse/clickhouse.service';+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';import { CoreEntityCacheService } from 'src/engine/core-entity-cache/services/core-entity-cache.service';import {BillingException,diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/event-log-cleanup.module.ts b/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/event-log-cleanup.module.tsindex 18f0b79d994..62773d20dd4 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/event-log-cleanup.module.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/event-log-cleanup.module.ts@@ -3,7 +3,7 @@import { Module } from '@nestjs/common';import { TypeOrmModule } from '@nestjs/typeorm';-import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';+import { ClickHouseModule } from 'src/database/clickhouse/clickhouse.module';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import { EventLogCleanupCronCommand } from 'src/engine/core-modules/event-logs/cleanup/commands/event-log-cleanup.cron.command';import { EventLogCleanupCronJob } from 'src/engine/core-modules/event-logs/cleanup/crons/event-log-cleanup.cron.job';diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/services/event-log-cleanup.service.ts b/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/services/event-log-cleanup.service.tsindex 14bd99cebf3..160e0cfcf86 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/services/event-log-cleanup.service.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/cleanup/services/event-log-cleanup.service.ts@@ -4,8 +4,8 @@ import { Injectable, Logger } from '@nestjs/common';import { EventLogTable } from 'twenty-shared/types';-import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';+import { ClickHouseService } from 'src/database/clickhouse/clickhouse.service';+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';import { getClickHouseTableName } from 'src/engine/core-modules/event-logs/registry/event-log-registry';export type EventLogCleanupParams = {diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts b/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.tsindex 7ebbd74d866..f2bd49fa12f 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/emit/analytics.utils.ts@@ -1,4 +1,4 @@-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';import { type EventCommonPropertiesType } from 'src/engine/core-modules/event-logs/emit/common.type';import {type TrackEventName,diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/event-logs-viewer.module.ts b/packages/twenty-server/src/engine/core-modules/event-logs/event-logs-viewer.module.tsindex f163209e698..ce242fa2543 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/event-logs-viewer.module.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/event-logs-viewer.module.ts@@ -10,7 +10,7 @@ import { EventLogEmitterResolver } from 'src/engine/core-modules/event-logs/emitimport { EventLogLiveModule } from 'src/engine/core-modules/event-logs/live/event-log-live.module';import { GuardRedirectModule } from 'src/engine/core-modules/guard-redirect/guard-redirect.module';import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module';-import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';+import { ClickHouseModule } from 'src/database/clickhouse/clickhouse.module';import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity';import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';import { SubscriptionsModule } from 'src/engine/subscriptions/subscriptions.module';diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/event-logs.service.ts b/packages/twenty-server/src/engine/core-modules/event-logs/event-logs.service.tsindex 039ecdc4c23..7f47a239d02 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/event-logs.service.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/event-logs.service.ts@@ -7,8 +7,8 @@ import { EventLogTable } from 'twenty-shared/types';import { isDefined } from 'twenty-shared/utils';import { Repository } from 'typeorm';-import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';+import { ClickHouseService } from 'src/database/clickhouse/clickhouse.service';+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';import { BillingService } from 'src/engine/core-modules/billing/services/billing.service';import { EnterprisePlanService } from 'src/engine/core-modules/enterprise/services/enterprise-plan.service';import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity';diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.tsindex 7265cec1a6c..f9fef9805e0 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts@@ -1,4 +1,4 @@-import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';+import { ClickHouseService } from 'src/database/clickhouse/clickhouse.service';import { ClickHouseEventSink } from 'src/engine/core-modules/event-logs/ingest/clickhouse-event.sink';import { type WorkspaceEventEnvelope } from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type';diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.tsindex fab5aae3d72..60fadc83656 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts@@ -3,7 +3,7 @@ import { Injectable } from '@nestjs/common';import {type ClickHouseInsertOptions,ClickHouseService,-} from 'src/database/clickHouse/clickHouse.service';+} from 'src/database/clickhouse/clickhouse.service';import { type EventSink } from 'src/engine/core-modules/event-logs/ingest/event-sink';import {type WorkspaceEventEnvelope,diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.spec.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.spec.tsindex 08afea7736f..141653ef526 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.spec.ts@@ -2,24 +2,40 @@ import { type ObjectRecordEvent } from 'twenty-shared/database-events';import { CreateEventLogFromInternalEvent } from 'src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event';import { WorkspaceEventSinkService } from 'src/engine/core-modules/event-logs/ingest/workspace-event-sink.service';+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';describe('CreateEventLogFromInternalEvent', () => {- it('ingests object events (persist + live fan-out) through the sink pipeline', async () => {- const ingest = jest.fn().mockResolvedValue(undefined);+ let ingest: jest.Mock;+ let addToQueue: jest.Mock;+ let handler: CreateEventLogFromInternalEvent;++ const batch = {+ name: 'company.created',+ workspaceId: 'workspace-1',+ objectMetadata: { id: 'object-metadata-1' },+ events: [{ recordId: 'record-1', userId: 'user-1', properties: {} }],+ } as unknown as WorkspaceEventBatch<ObjectRecordEvent>;- const handler = new CreateEventLogFromInternalEvent({- isEnabled: () => true,- ingest,- } as unknown as WorkspaceEventSinkService);+ const transientError = Object.assign(+ new Error('Failed to insert 1 objectEvent row(s) into ClickHouse'),+ { cause: new Error('Timeout error.') },+ );- const batch = {- name: 'company.created',- workspaceId: 'workspace-1',- objectMetadata: { id: 'object-metadata-1' },- events: [{ recordId: 'record-1', userId: 'user-1', properties: {} }],- } as unknown as WorkspaceEventBatch<ObjectRecordEvent>;+ beforeEach(() => {+ ingest = jest.fn().mockResolvedValue(undefined);+ addToQueue = jest.fn().mockResolvedValue(undefined);+ handler = new CreateEventLogFromInternalEvent(+ {+ isEnabled: () => true,+ ingest,+ } as unknown as WorkspaceEventSinkService,+ { add: addToQueue } as unknown as MessageQueueService,+ );+ });++ it('ingests object events (persist + live fan-out) through the sink pipeline', async () => {await handler.handle(batch);const ingestedEnvelopes = ingest.mock.calls[0]?.[0];@@ -27,4 +43,36 @@ describe('CreateEventLogFromInternalEvent', () => {expect(ingestedEnvelopes).toHaveLength(1);expect(ingestedEnvelopes[0].table).toBe('objectEvent');});++ it('requeues once on a transient ClickHouse network error instead of failing', async () => {+ ingest.mockRejectedValue(transientError);++ await expect(handler.handle(batch)).resolves.toBeUndefined();++ expect(addToQueue).toHaveBeenCalledWith(+ CreateEventLogFromInternalEvent.name,+ expect.objectContaining({ transientErrorRetryCount: 1 }),+ );+ });++ it('drops the batch after the transient retry budget is exhausted', async () => {+ ingest.mockRejectedValue(transientError);++ await expect(+ handler.handle({ ...batch, transientErrorRetryCount: 1 }),+ ).resolves.toBeUndefined();++ expect(addToQueue).not.toHaveBeenCalled();+ });++ it('rethrows non-transient ingestion errors so they fail the job and reach Sentry', async () => {+ const error = new Error(+ 'Failed to insert 1 objectEvent row(s) into ClickHouse: (total) memory limit exceeded',+ );++ ingest.mockRejectedValue(error);++ await expect(handler.handle(batch)).rejects.toBe(error);+ expect(addToQueue).not.toHaveBeenCalled();+ });});diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.tsindex 5d3c15d463f..402fa8481ea 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts@@ -1,6 +1,9 @@+import { Logger } from '@nestjs/common';+import { type ObjectRecordEvent } from 'twenty-shared/database-events';import { isDefined } from 'twenty-shared/utils';+import { isTransientClickHouseNetworkError } from 'src/database/clickhouse/utils/is-transient-clickhouse-network-error.util';import { WorkspaceEventSinkService } from 'src/engine/core-modules/event-logs/ingest/workspace-event-sink.service';import { type WorkspaceEventEnvelope } from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type';import {@@ -11,11 +14,20 @@ import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/event-logs/import { OBJECT_RECORD_DELETED_EVENT } from 'src/engine/core-modules/event-logs/emit/events/object-event/object-record-delete';import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/event-logs/emit/events/object-event/object-record-updated';import { OBJECT_RECORD_UPSERTED_EVENT } from 'src/engine/core-modules/event-logs/emit/events/object-event/object-record-upserted';+import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';+const MAX_TRANSIENT_ERROR_RETRIES = 1;++export type CreateEventLogFromInternalEventData =+ WorkspaceEventBatch<ObjectRecordEvent> & {+ transientErrorRetryCount?: number;+ };+const OBJECT_EVENT_BY_SUFFIX = {'.created': OBJECT_RECORD_CREATED_EVENT,'.updated': OBJECT_RECORD_UPDATED_EVENT,@@ -25,12 +37,16 @@ const OBJECT_EVENT_BY_SUFFIX = {@Processor(MessageQueue.entityEventsToDbQueue)export class CreateEventLogFromInternalEvent {+ private readonly logger = new Logger(CreateEventLogFromInternalEvent.name);+constructor(private readonly workspaceEventSinkService: WorkspaceEventSinkService,+ @InjectMessageQueue(MessageQueue.entityEventsToDbQueue)+ private readonly messageQueueService: MessageQueueService,) {}@Process(CreateEventLogFromInternalEvent.name)- async handle(batch: WorkspaceEventBatch<ObjectRecordEvent>): Promise<void> {+ async handle(batch: CreateEventLogFromInternalEventData): Promise<void> {if (!this.workspaceEventSinkService.isEnabled()) {return;}@@ -41,7 +57,40 @@ export class CreateEventLogFromInternalEvent {return;}- await this.workspaceEventSinkService.ingest(envelopes);+ try {+ await this.workspaceEventSinkService.ingest(envelopes);+ } catch (error) {+ if (!isTransientClickHouseNetworkError(error)) {+ throw error;+ }++ await this.handleTransientError(batch, envelopes.length, error);+ }+ }++ private async handleTransientError(+ batch: CreateEventLogFromInternalEventData,+ envelopeCount: number,+ error: Error,+ ): Promise<void> {+ const transientErrorRetryCount = batch.transientErrorRetryCount ?? 0;++ if (transientErrorRetryCount >= MAX_TRANSIENT_ERROR_RETRIES) {+ this.logger.warn(+ `Dropping ${envelopeCount} event log envelope(s) for workspace ${batch.workspaceId} after ${transientErrorRetryCount + 1} transient ClickHouse network errors: ${error.message}`,+ );++ return;+ }++ this.logger.warn(+ `Requeuing ${envelopeCount} event log envelope(s) for workspace ${batch.workspaceId} after a transient ClickHouse network error: ${error.message}`,+ );++ await this.messageQueueService.add<CreateEventLogFromInternalEventData>(+ CreateEventLogFromInternalEvent.name,+ { ...batch, transientErrorRetryCount: transientErrorRetryCount + 1 },+ );}private toEnvelopes(diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.tsindex 709ea9bd291..cb43c999be9 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/event-log-ingestion.module.ts@@ -2,7 +2,7 @@ import { Logger, Module } from '@nestjs/common';import { isDefined } from 'twenty-shared/utils';-import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';+import { ClickHouseModule } from 'src/database/clickhouse/clickhouse.module';import { ClickHouseEventSink } from 'src/engine/core-modules/event-logs/ingest/clickhouse-event.sink';import { ConsoleEventSink } from 'src/engine/core-modules/event-logs/ingest/console-event.sink';import { CreateEventLogFromInternalEvent } from 'src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event';diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/producers/application-log/build-application-log-envelopes.ts b/packages/twenty-server/src/engine/core-modules/event-logs/producers/application-log/build-application-log-envelopes.tsindex bac7678f77d..f1024aa4dbd 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/producers/application-log/build-application-log-envelopes.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/producers/application-log/build-application-log-envelopes.ts@@ -1,4 +1,4 @@-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';import { type ApplicationLogEntry } from 'src/engine/core-modules/event-logs/producers/application-log/application-log-entry.interface';import { type WorkspaceEventEnvelope } from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type';diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/utils/normalize-event-log-records.ts b/packages/twenty-server/src/engine/core-modules/event-logs/utils/normalize-event-log-records.tsindex 46800953a22..235cff11437 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/utils/normalize-event-log-records.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/utils/normalize-event-log-records.ts@@ -1,6 +1,6 @@import { EventLogTable } from 'twenty-shared/types';-import { parseClickHouseDateTime } from 'src/database/clickHouse/clickHouse.util';+import { parseClickHouseDateTime } from 'src/database/clickhouse/utils/parse-clickhouse-date-time.util';import { type EventLogRecord } from 'src/engine/core-modules/event-logs/dtos/event-log-result.dto';import { EVENT_LOG_TYPES } from 'src/engine/core-modules/event-logs/registry/event-log-registry';diff --git a/packages/twenty-server/src/engine/core-modules/usage/services/usage-analytics.service.ts b/packages/twenty-server/src/engine/core-modules/usage/services/usage-analytics.service.tsindex f0fdf897f0d..1bb689afde6 100644--- a/packages/twenty-server/src/engine/core-modules/usage/services/usage-analytics.service.ts+++ b/packages/twenty-server/src/engine/core-modules/usage/services/usage-analytics.service.ts@@ -2,8 +2,8 @@import { Injectable } from '@nestjs/common';-import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';+import { ClickHouseService } from 'src/database/clickhouse/clickhouse.service';+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';import { fillUsageTimeSeriesGaps } from 'src/engine/core-modules/usage/utils/fill-usage-time-series-gaps.util';import { toDisplayCredits } from 'src/engine/core-modules/usage/utils/to-display-credits.util';import { toDollars } from 'src/engine/core-modules/usage/utils/to-dollars.util';diff --git a/packages/twenty-server/src/engine/core-modules/usage/usage.module.ts b/packages/twenty-server/src/engine/core-modules/usage/usage.module.tsindex 7a577c5b2ba..7fb3923a50f 100644--- a/packages/twenty-server/src/engine/core-modules/usage/usage.module.ts+++ b/packages/twenty-server/src/engine/core-modules/usage/usage.module.ts@@ -3,7 +3,7 @@import { Module } from '@nestjs/common';import { TypeOrmModule } from '@nestjs/typeorm';-import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';+import { ClickHouseModule } from 'src/database/clickhouse/clickhouse.module';import { EventLogEmitterModule } from 'src/engine/core-modules/event-logs/emit/event-log-emitter.module';import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';import { UsageEventListener } from 'src/engine/core-modules/usage/listeners/usage-event.listener';diff --git a/packages/twenty-server/src/engine/core-modules/usage/utils/build-usage-event-envelopes.ts b/packages/twenty-server/src/engine/core-modules/usage/utils/build-usage-event-envelopes.tsindex b493f9a8b6b..ec8322d666a 100644--- a/packages/twenty-server/src/engine/core-modules/usage/utils/build-usage-event-envelopes.ts+++ b/packages/twenty-server/src/engine/core-modules/usage/utils/build-usage-event-envelopes.ts@@ -1,6 +1,6 @@/* @license Enterprise */-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';import { type WorkspaceEventEnvelope } from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type';import { type UsageEvent } from 'src/engine/core-modules/usage/types/usage-event.type';diff --git a/packages/twenty-server/test/integration/graphql/suites/event-logs/event-logs.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/event-logs/event-logs.integration-spec.tsindex 39fbee4a380..37e548c0747 100644--- a/packages/twenty-server/test/integration/graphql/suites/event-logs/event-logs.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/event-logs/event-logs.integration-spec.ts@@ -7,7 +7,7 @@ import {} from '@clickhouse/client';import request from 'supertest';-import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';+import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';