Skip to content

The open alternative to Salesforce, designed for AI.

chore(server): add worker memory diagnostics logging (#25216)

Etienne57545214e58c
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index f96581958e1..4e042a536ed 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -4,6 +4,8 @@ import {
type OnModuleInit,
} from '@nestjs/common';
+import v8 from 'v8';
+
import * as Sentry from '@sentry/node';
import {
type JobsOptions,
@@ -41,6 +43,7 @@ import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/
export type BullMQDriverOptions = QueueOptions;
const V4_LENGTH = 36;
+const BYTES_PER_MEGABYTE = 1024 * 1024;
export class BullMQDriver
implements MessageQueueDriver, OnModuleDestroy, OnModuleInit
@@ -209,6 +212,10 @@ export class BullMQDriver
// TODO: Correctly support for job.id
const timeStart = performance.now();
+ // TODO: diagnostic only — remove once the worker OOM root cause is
+ // confirmed. Attribute heap growth to job types, including jobs that
+ // throw under memory pressure (logged from the finally below).
+ const heapUsedBeforeBytes = v8.getHeapStatistics().used_heap_size;
const workspaceId = job.data?.workspaceId;
const workspaceSuffix = workspaceId
? ` [workspace=${workspaceId}]`
@@ -217,20 +224,30 @@ export class BullMQDriver
this.logger.log(
`Processing job ${job.id} with name ${job.name} on queue ${queueName}${workspaceSuffix}`,
);
- await handler({
- data: job.data,
- id: job.id ?? '',
- name: job.name,
- retryLimit: Math.max(0, (job.opts.attempts ?? 1) - 1),
- updateData: (data) => job.updateData(data),
- abortSignal,
- });
- const timeEnd = performance.now();
- const executionTime = timeEnd - timeStart;
- this.logger.log(
- `Job ${job.id} with name ${job.name} processed on queue ${queueName} in ${executionTime.toFixed(2)}ms${workspaceSuffix}`,
- );
+ let jobSucceeded = false;
+
+ try {
+ await handler({
+ data: job.data,
+ id: job.id ?? '',
+ name: job.name,
+ retryLimit: Math.max(0, (job.opts.attempts ?? 1) - 1),
+ updateData: (data) => job.updateData(data),
+ abortSignal,
+ });
+ jobSucceeded = true;
+ } finally {
+ const executionTime = performance.now() - timeStart;
+ const heapUsedAfterBytes = v8.getHeapStatistics().used_heap_size;
+ const heapDeltaMegabytes =
+ (heapUsedAfterBytes - heapUsedBeforeBytes) / BYTES_PER_MEGABYTE;
+ const heapUsedMegabytes = heapUsedAfterBytes / BYTES_PER_MEGABYTE;
+
+ this.logger.log(
+ `Job ${job.id} with name ${job.name} ${jobSucceeded ? 'processed' : 'failed'} on queue ${queueName} in ${executionTime.toFixed(2)}ms${workspaceSuffix} heapDeltaMB=${heapDeltaMegabytes.toFixed(1)} heapUsedMB=${heapUsedMegabytes.toFixed(0)}`,
+ );
+ }
}),
workerOptions,
);
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
index 71e29e3e340..2c622626f87 100644
--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
@@ -1,5 +1,7 @@
import { Injectable, Logger } from '@nestjs/common';
+import v8 from 'v8';
+
import { type Histogram } from '@opentelemetry/api';
import { isDefined } from 'twenty-shared/utils';
@@ -29,8 +31,26 @@ const CACHE_DURATION_BUCKETS_SECONDS = [
const STATS_TTL_MS = 5_000;
const SIZE_REFRESH_MS = 5 * 60 * 1000;
const SIZE_STARTUP_DELAY_MS = 30 * 1000;
-const SIZE_SAMPLE_PER_PROVIDER = 3;
+// Sampled off the hot path every SIZE_REFRESH_MS. 50 bounds the walk for
+// high-count, low-byte providers; the byte-heavy providers are sized in full via
+// SIZE_IN_FULL_PROVIDERS so the estimate is exact where it dominates the heap.
+const SIZE_SAMPLE_PER_PROVIDER = 50;
+// Byte-heavy providers (ORM / field-metadata graphs, ~MB each) are entry-capped
+// in WorkspaceCacheService, so every entry is sized rather than extrapolated from
+// a sample — extrapolating the largest, highest-variance entries is what made the
+// estimate untrustworthy. Their caps (<=512) bound the walk and packed versions
+// are sized for free, so the added cost stays small and off the hot path.
+const SIZE_IN_FULL_PROVIDERS = new Set([
+ 'ORMEntityMetadatas',
+ 'flatFieldMetadataMaps',
+ 'flatFieldMetadataMapsOrm',
+]);
const SIZE_WALK_NODE_CAP = 300_000;
+const MEMORY_REPORT_INTERVAL_MS = 60 * 1000;
+const BYTES_PER_MEGABYTE = 1024 * 1024;
+
+const toMegabytes = (bytes: number): number =>
+ Math.round(bytes / BYTES_PER_MEGABYTE);
// Cache observability, split from WorkspaceCacheService (which drives it via start()/stop()).
@Injectable()
@@ -46,6 +66,7 @@ export class WorkspaceCacheMetricsService {
private cacheSizeTotalBytes = 0;
private sizeSampler?: ReturnType<typeof setInterval>;
private sizeStartupTimer?: ReturnType<typeof setTimeout>;
+ private memoryReportTimer?: ReturnType<typeof setInterval>;
private statsCache?: { computedAt: number } & LocalCacheStats;
private sizeSampleInFlight = false;
private packingBacklog = 0;
@@ -94,6 +115,7 @@ export class WorkspaceCacheMetricsService {
this.localCache = localCache;
this.registerGauges();
this.scheduleSizeSampler();
+ this.scheduleMemoryReport();
}
stop(): void {
@@ -103,6 +125,47 @@ export class WorkspaceCacheMetricsService {
if (isDefined(this.sizeSampler)) {
clearInterval(this.sizeSampler);
}
+ if (isDefined(this.memoryReportTimer)) {
+ clearInterval(this.memoryReportTimer);
+ }
+ }
+
+ // TODO: diagnostic only — remove once the worker OOM root cause is confirmed.
+ // Correlates process/V8 memory with the cache's own size in one log line so we
+ // can tell whether the metadata cache tracks the heap climb to OOM (retained,
+ // old-space) versus per-job allocation spikes (transient, new-space).
+ private scheduleMemoryReport(): void {
+ const report = (): void => {
+ const memoryUsage = process.memoryUsage();
+ const heapSpaces = v8.getHeapSpaceStatistics();
+ const oldSpaceUsedBytes =
+ heapSpaces.find((space) => space.space_name === 'old_space')
+ ?.space_used_size ?? 0;
+ const newSpaceUsedBytes =
+ heapSpaces.find((space) => space.space_name === 'new_space')
+ ?.space_used_size ?? 0;
+
+ const topProviders = Object.entries(this.cacheSizeByKeyName)
+ .sort(([, bytesA], [, bytesB]) => bytesB - bytesA)
+ .slice(0, 3)
+ .map(([keyName, bytes]) => `${keyName}:${toMegabytes(bytes)}MB`)
+ .join(',');
+
+ this.logger.log(
+ `[WorkspaceCacheMemReport] ` +
+ `heapUsedMB=${toMegabytes(memoryUsage.heapUsed)} ` +
+ `rssMB=${toMegabytes(memoryUsage.rss)} ` +
+ `externalMB=${toMegabytes(memoryUsage.external)} ` +
+ `oldSpaceMB=${toMegabytes(oldSpaceUsedBytes)} ` +
+ `newSpaceMB=${toMegabytes(newSpaceUsedBytes)} ` +
+ `cacheEntries=${this.localCache?.size ?? 0} ` +
+ `cacheBytesEstimateMB=${toMegabytes(this.cacheSizeTotalBytes)} ` +
+ `top=${topProviders}`,
+ );
+ };
+
+ this.memoryReportTimer = setInterval(report, MEMORY_REPORT_INTERVAL_MS);
+ this.memoryReportTimer.unref();
}
recordRecompute(seconds: number, cacheKey: WorkspaceCacheKeyName): void {
@@ -209,7 +272,12 @@ export class WorkspaceCacheMetricsService {
stats.count += 1;
- if (stats.sampled < SIZE_SAMPLE_PER_PROVIDER && entry.versions.size > 0) {
+ const sizeEveryEntry = SIZE_IN_FULL_PROVIDERS.has(keyName);
+
+ if (
+ (sizeEveryEntry || stats.sampled < SIZE_SAMPLE_PER_PROVIDER) &&
+ entry.versions.size > 0
+ ) {
// Size every retained version, not just the latest — stale versions still occupy heap.
let entryBytes = 0;
diff --git a/packages/twenty-server/src/modules/match-participant/participant-target-reconciliation.service.ts b/packages/twenty-server/src/modules/match-participant/participant-target-reconciliation.service.ts
index b228b15904f..316b5a612c6 100644
--- a/packages/twenty-server/src/modules/match-participant/participant-target-reconciliation.service.ts
+++ b/packages/twenty-server/src/modules/match-participant/participant-target-reconciliation.service.ts
@@ -1,4 +1,6 @@
-import { Injectable } from '@nestjs/common';
+import { Injectable, Logger } from '@nestjs/common';
+
+import v8 from 'v8';
import chunk from 'lodash.chunk';
import { QUERY_MAX_RECORDS } from 'twenty-shared/constants';
@@ -29,8 +31,14 @@ type TargetWorkspaceEntity = Omit<ExistingTarget, 'parentId' | 'deletedAt'> & {
deletedAt: string | null;
};
+const BYTES_PER_MEGABYTE = 1024 * 1024;
+
@Injectable()
export class ParticipantTargetReconciliationService {
+ private readonly logger = new Logger(
+ ParticipantTargetReconciliationService.name,
+ );
+
constructor(private readonly workspaceOrmManager: WorkspaceOrmManager) {}
public async reconcileParticipantTargets({
@@ -144,19 +152,35 @@ export class ParticipantTargetReconciliationService {
messageThreadIds: string[];
transactionScope?: WorkspaceTransactionScope;
}): Promise<void> {
+ const uniqueMessageThreadIds = [...new Set(messageThreadIds)];
+
+ if (uniqueMessageThreadIds.length === 0) {
+ return;
+ }
+
const messageRepository = await this.getRepository<MessageWorkspaceEntity>(
'message',
transactionScope,
);
+ // TODO: diagnostic only — remove once the worker OOM root cause is confirmed.
+ // Size what one reconcile loads and how much heap it retains, to confirm
+ // whether this path (added in #24778) drives worker memory.
+ const heapUsedBeforeBytes = v8.getHeapStatistics().used_heap_size;
+ let messagesLoaded = 0;
+ let participantsLoaded = 0;
+
for (const messageThreadIdChunk of chunk(
- [...new Set(messageThreadIds)],
+ uniqueMessageThreadIds,
QUERY_MAX_RECORDS,
)) {
const threadMessages = await messageRepository.find({
where: { messageThreadId: In(messageThreadIdChunk) },
select: { id: true, messageThreadId: true },
});
+
+ messagesLoaded += threadMessages.length;
+
const messageThreadIdByMessageId = new Map<string, string>();
for (const { id, messageThreadId } of threadMessages) {
@@ -185,6 +209,8 @@ export class ParticipantTargetReconciliationService {
);
}
+ participantsLoaded += participants.length;
+
await this.reconcileTargets({
parentIds: messageThreadIdChunk,
parentFieldName: 'messageThreadId',
@@ -197,6 +223,14 @@ export class ParticipantTargetReconciliationService {
transactionScope,
});
}
+
+ const heapDeltaMegabytes =
+ (v8.getHeapStatistics().used_heap_size - heapUsedBeforeBytes) /
+ BYTES_PER_MEGABYTE;
+
+ this.logger.log(
+ `[ReconcileMem] messageThreadTarget threads=${uniqueMessageThreadIds.length} messagesLoaded=${messagesLoaded} participantsLoaded=${participantsLoaded} heapDeltaMB=${heapDeltaMegabytes.toFixed(1)}`,
+ );
}
private async reconcileTargets({