The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
chore(server): add worker memory diagnostics logging (#25216)
Etienne57545214e58c
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex f96581958e1..4e042a536ed 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -4,6 +4,8 @@ import {type OnModuleInit,} from '@nestjs/common';+import v8 from 'v8';+import * as Sentry from '@sentry/node';import {type JobsOptions,@@ -41,6 +43,7 @@ import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/export type BullMQDriverOptions = QueueOptions;const V4_LENGTH = 36;+const BYTES_PER_MEGABYTE = 1024 * 1024;export class BullMQDriverimplements MessageQueueDriver, OnModuleDestroy, OnModuleInit@@ -209,6 +212,10 @@ export class BullMQDriver// TODO: Correctly support for job.idconst timeStart = performance.now();+ // TODO: diagnostic only — remove once the worker OOM root cause is+ // confirmed. Attribute heap growth to job types, including jobs that+ // throw under memory pressure (logged from the finally below).+ const heapUsedBeforeBytes = v8.getHeapStatistics().used_heap_size;const workspaceId = job.data?.workspaceId;const workspaceSuffix = workspaceId? ` [workspace=${workspaceId}]`@@ -217,20 +224,30 @@ export class BullMQDriverthis.logger.log(`Processing job ${job.id} with name ${job.name} on queue ${queueName}${workspaceSuffix}`,);- await handler({- data: job.data,- id: job.id ?? '',- name: job.name,- retryLimit: Math.max(0, (job.opts.attempts ?? 1) - 1),- updateData: (data) => job.updateData(data),- abortSignal,- });- const timeEnd = performance.now();- const executionTime = timeEnd - timeStart;- this.logger.log(- `Job ${job.id} with name ${job.name} processed on queue ${queueName} in ${executionTime.toFixed(2)}ms${workspaceSuffix}`,- );+ let jobSucceeded = false;++ try {+ await handler({+ data: job.data,+ id: job.id ?? '',+ name: job.name,+ retryLimit: Math.max(0, (job.opts.attempts ?? 1) - 1),+ updateData: (data) => job.updateData(data),+ abortSignal,+ });+ jobSucceeded = true;+ } finally {+ const executionTime = performance.now() - timeStart;+ const heapUsedAfterBytes = v8.getHeapStatistics().used_heap_size;+ const heapDeltaMegabytes =+ (heapUsedAfterBytes - heapUsedBeforeBytes) / BYTES_PER_MEGABYTE;+ const heapUsedMegabytes = heapUsedAfterBytes / BYTES_PER_MEGABYTE;++ this.logger.log(+ `Job ${job.id} with name ${job.name} ${jobSucceeded ? 'processed' : 'failed'} on queue ${queueName} in ${executionTime.toFixed(2)}ms${workspaceSuffix} heapDeltaMB=${heapDeltaMegabytes.toFixed(1)} heapUsedMB=${heapUsedMegabytes.toFixed(0)}`,+ );+ }}),workerOptions,);diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.tsindex 71e29e3e340..2c622626f87 100644--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts@@ -1,5 +1,7 @@import { Injectable, Logger } from '@nestjs/common';+import v8 from 'v8';+import { type Histogram } from '@opentelemetry/api';import { isDefined } from 'twenty-shared/utils';@@ -29,8 +31,26 @@ const CACHE_DURATION_BUCKETS_SECONDS = [const STATS_TTL_MS = 5_000;const SIZE_REFRESH_MS = 5 * 60 * 1000;const SIZE_STARTUP_DELAY_MS = 30 * 1000;-const SIZE_SAMPLE_PER_PROVIDER = 3;+// Sampled off the hot path every SIZE_REFRESH_MS. 50 bounds the walk for+// high-count, low-byte providers; the byte-heavy providers are sized in full via+// SIZE_IN_FULL_PROVIDERS so the estimate is exact where it dominates the heap.+const SIZE_SAMPLE_PER_PROVIDER = 50;+// Byte-heavy providers (ORM / field-metadata graphs, ~MB each) are entry-capped+// in WorkspaceCacheService, so every entry is sized rather than extrapolated from+// a sample — extrapolating the largest, highest-variance entries is what made the+// estimate untrustworthy. Their caps (<=512) bound the walk and packed versions+// are sized for free, so the added cost stays small and off the hot path.+const SIZE_IN_FULL_PROVIDERS = new Set([+ 'ORMEntityMetadatas',+ 'flatFieldMetadataMaps',+ 'flatFieldMetadataMapsOrm',+]);const SIZE_WALK_NODE_CAP = 300_000;+const MEMORY_REPORT_INTERVAL_MS = 60 * 1000;+const BYTES_PER_MEGABYTE = 1024 * 1024;++const toMegabytes = (bytes: number): number =>+ Math.round(bytes / BYTES_PER_MEGABYTE);// Cache observability, split from WorkspaceCacheService (which drives it via start()/stop()).@Injectable()@@ -46,6 +66,7 @@ export class WorkspaceCacheMetricsService {private cacheSizeTotalBytes = 0;private sizeSampler?: ReturnType<typeof setInterval>;private sizeStartupTimer?: ReturnType<typeof setTimeout>;+ private memoryReportTimer?: ReturnType<typeof setInterval>;private statsCache?: { computedAt: number } & LocalCacheStats;private sizeSampleInFlight = false;private packingBacklog = 0;@@ -94,6 +115,7 @@ export class WorkspaceCacheMetricsService {this.localCache = localCache;this.registerGauges();this.scheduleSizeSampler();+ this.scheduleMemoryReport();}stop(): void {@@ -103,6 +125,47 @@ export class WorkspaceCacheMetricsService {if (isDefined(this.sizeSampler)) {clearInterval(this.sizeSampler);}+ if (isDefined(this.memoryReportTimer)) {+ clearInterval(this.memoryReportTimer);+ }+ }++ // TODO: diagnostic only — remove once the worker OOM root cause is confirmed.+ // Correlates process/V8 memory with the cache's own size in one log line so we+ // can tell whether the metadata cache tracks the heap climb to OOM (retained,+ // old-space) versus per-job allocation spikes (transient, new-space).+ private scheduleMemoryReport(): void {+ const report = (): void => {+ const memoryUsage = process.memoryUsage();+ const heapSpaces = v8.getHeapSpaceStatistics();+ const oldSpaceUsedBytes =+ heapSpaces.find((space) => space.space_name === 'old_space')+ ?.space_used_size ?? 0;+ const newSpaceUsedBytes =+ heapSpaces.find((space) => space.space_name === 'new_space')+ ?.space_used_size ?? 0;++ const topProviders = Object.entries(this.cacheSizeByKeyName)+ .sort(([, bytesA], [, bytesB]) => bytesB - bytesA)+ .slice(0, 3)+ .map(([keyName, bytes]) => `${keyName}:${toMegabytes(bytes)}MB`)+ .join(',');++ this.logger.log(+ `[WorkspaceCacheMemReport] ` ++ `heapUsedMB=${toMegabytes(memoryUsage.heapUsed)} ` ++ `rssMB=${toMegabytes(memoryUsage.rss)} ` ++ `externalMB=${toMegabytes(memoryUsage.external)} ` ++ `oldSpaceMB=${toMegabytes(oldSpaceUsedBytes)} ` ++ `newSpaceMB=${toMegabytes(newSpaceUsedBytes)} ` ++ `cacheEntries=${this.localCache?.size ?? 0} ` ++ `cacheBytesEstimateMB=${toMegabytes(this.cacheSizeTotalBytes)} ` ++ `top=${topProviders}`,+ );+ };++ this.memoryReportTimer = setInterval(report, MEMORY_REPORT_INTERVAL_MS);+ this.memoryReportTimer.unref();}recordRecompute(seconds: number, cacheKey: WorkspaceCacheKeyName): void {@@ -209,7 +272,12 @@ export class WorkspaceCacheMetricsService {stats.count += 1;- if (stats.sampled < SIZE_SAMPLE_PER_PROVIDER && entry.versions.size > 0) {+ const sizeEveryEntry = SIZE_IN_FULL_PROVIDERS.has(keyName);++ if (+ (sizeEveryEntry || stats.sampled < SIZE_SAMPLE_PER_PROVIDER) &&+ entry.versions.size > 0+ ) {// Size every retained version, not just the latest — stale versions still occupy heap.let entryBytes = 0;diff --git a/packages/twenty-server/src/modules/match-participant/participant-target-reconciliation.service.ts b/packages/twenty-server/src/modules/match-participant/participant-target-reconciliation.service.tsindex b228b15904f..316b5a612c6 100644--- a/packages/twenty-server/src/modules/match-participant/participant-target-reconciliation.service.ts+++ b/packages/twenty-server/src/modules/match-participant/participant-target-reconciliation.service.ts@@ -1,4 +1,6 @@-import { Injectable } from '@nestjs/common';+import { Injectable, Logger } from '@nestjs/common';++import v8 from 'v8';import chunk from 'lodash.chunk';import { QUERY_MAX_RECORDS } from 'twenty-shared/constants';@@ -29,8 +31,14 @@ type TargetWorkspaceEntity = Omit<ExistingTarget, 'parentId' | 'deletedAt'> & {deletedAt: string | null;};+const BYTES_PER_MEGABYTE = 1024 * 1024;+@Injectable()export class ParticipantTargetReconciliationService {+ private readonly logger = new Logger(+ ParticipantTargetReconciliationService.name,+ );+constructor(private readonly workspaceOrmManager: WorkspaceOrmManager) {}public async reconcileParticipantTargets({@@ -144,19 +152,35 @@ export class ParticipantTargetReconciliationService {messageThreadIds: string[];transactionScope?: WorkspaceTransactionScope;}): Promise<void> {+ const uniqueMessageThreadIds = [...new Set(messageThreadIds)];++ if (uniqueMessageThreadIds.length === 0) {+ return;+ }+const messageRepository = await this.getRepository<MessageWorkspaceEntity>('message',transactionScope,);+ // TODO: diagnostic only — remove once the worker OOM root cause is confirmed.+ // Size what one reconcile loads and how much heap it retains, to confirm+ // whether this path (added in #24778) drives worker memory.+ const heapUsedBeforeBytes = v8.getHeapStatistics().used_heap_size;+ let messagesLoaded = 0;+ let participantsLoaded = 0;+for (const messageThreadIdChunk of chunk(- [...new Set(messageThreadIds)],+ uniqueMessageThreadIds,QUERY_MAX_RECORDS,)) {const threadMessages = await messageRepository.find({where: { messageThreadId: In(messageThreadIdChunk) },select: { id: true, messageThreadId: true },});++ messagesLoaded += threadMessages.length;+const messageThreadIdByMessageId = new Map<string, string>();for (const { id, messageThreadId } of threadMessages) {@@ -185,6 +209,8 @@ export class ParticipantTargetReconciliationService {);}+ participantsLoaded += participants.length;+await this.reconcileTargets({parentIds: messageThreadIdChunk,parentFieldName: 'messageThreadId',@@ -197,6 +223,14 @@ export class ParticipantTargetReconciliationService {transactionScope,});}++ const heapDeltaMegabytes =+ (v8.getHeapStatistics().used_heap_size - heapUsedBeforeBytes) /+ BYTES_PER_MEGABYTE;++ this.logger.log(+ `[ReconcileMem] messageThreadTarget threads=${uniqueMessageThreadIds.length} messagesLoaded=${messagesLoaded} participantsLoaded=${participantsLoaded} heapDeltaMB=${heapDeltaMegabytes.toFixed(1)}`,+ );}private async reconcileTargets({