Skip to content

The open alternative to Salesforce, designed for AI.

chore(server): revert worker memory diagnostics logging (#25355)

Etiennefce1926b0e48
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index 17539de2eeb..a3367862de4 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -4,8 +4,6 @@ import {
type OnModuleInit,
} from '@nestjs/common';
-import v8 from 'v8';
-
import * as Sentry from '@sentry/node';
import {
type JobsOptions,
@@ -45,7 +43,6 @@ import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/
export type BullMQDriverOptions = QueueOptions;
const V4_LENGTH = 36;
-const BYTES_PER_MEGABYTE = 1024 * 1024;
export class BullMQDriver
implements MessageQueueDriver, OnModuleDestroy, OnModuleInit
@@ -214,10 +211,6 @@ export class BullMQDriver
// TODO: Correctly support for job.id
const timeStart = performance.now();
- // TODO: diagnostic only — remove once the worker OOM root cause is
- // confirmed. Attribute heap growth to job types, including jobs that
- // throw under memory pressure (logged from the finally below).
- const heapUsedBeforeBytes = v8.getHeapStatistics().used_heap_size;
const workspaceId = job.data?.workspaceId;
const workspaceSuffix = workspaceId
? ` [workspace=${workspaceId}]`
@@ -226,30 +219,20 @@ export class BullMQDriver
this.logger.log(
`Processing job ${job.id} with name ${job.name} on queue ${queueName}${workspaceSuffix}`,
);
+ await handler({
+ data: job.data,
+ id: job.id ?? '',
+ name: job.name,
+ retryLimit: Math.max(0, (job.opts.attempts ?? 1) - 1),
+ updateData: (data) => job.updateData(data),
+ abortSignal,
+ });
+ const timeEnd = performance.now();
+ const executionTime = timeEnd - timeStart;
- let jobSucceeded = false;
-
- try {
- await handler({
- data: job.data,
- id: job.id ?? '',
- name: job.name,
- retryLimit: Math.max(0, (job.opts.attempts ?? 1) - 1),
- updateData: (data) => job.updateData(data),
- abortSignal,
- });
- jobSucceeded = true;
- } finally {
- const executionTime = performance.now() - timeStart;
- const heapUsedAfterBytes = v8.getHeapStatistics().used_heap_size;
- const heapDeltaMegabytes =
- (heapUsedAfterBytes - heapUsedBeforeBytes) / BYTES_PER_MEGABYTE;
- const heapUsedMegabytes = heapUsedAfterBytes / BYTES_PER_MEGABYTE;
-
- this.logger.log(
- `Job ${job.id} with name ${job.name} ${jobSucceeded ? 'processed' : 'failed'} on queue ${queueName} in ${executionTime.toFixed(2)}ms${workspaceSuffix} heapDeltaMB=${heapDeltaMegabytes.toFixed(1)} heapUsedMB=${heapUsedMegabytes.toFixed(0)}`,
- );
- }
+ this.logger.log(
+ `Job ${job.id} with name ${job.name} processed on queue ${queueName} in ${executionTime.toFixed(2)}ms${workspaceSuffix}`,
+ );
}),
workerOptions,
);
diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
index 2c622626f87..71e29e3e340 100644
--- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
+++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache-metrics.service.ts
@@ -1,7 +1,5 @@
import { Injectable, Logger } from '@nestjs/common';
-import v8 from 'v8';
-
import { type Histogram } from '@opentelemetry/api';
import { isDefined } from 'twenty-shared/utils';
@@ -31,26 +29,8 @@ const CACHE_DURATION_BUCKETS_SECONDS = [
const STATS_TTL_MS = 5_000;
const SIZE_REFRESH_MS = 5 * 60 * 1000;
const SIZE_STARTUP_DELAY_MS = 30 * 1000;
-// Sampled off the hot path every SIZE_REFRESH_MS. 50 bounds the walk for
-// high-count, low-byte providers; the byte-heavy providers are sized in full via
-// SIZE_IN_FULL_PROVIDERS so the estimate is exact where it dominates the heap.
-const SIZE_SAMPLE_PER_PROVIDER = 50;
-// Byte-heavy providers (ORM / field-metadata graphs, ~MB each) are entry-capped
-// in WorkspaceCacheService, so every entry is sized rather than extrapolated from
-// a sample — extrapolating the largest, highest-variance entries is what made the
-// estimate untrustworthy. Their caps (<=512) bound the walk and packed versions
-// are sized for free, so the added cost stays small and off the hot path.
-const SIZE_IN_FULL_PROVIDERS = new Set([
- 'ORMEntityMetadatas',
- 'flatFieldMetadataMaps',
- 'flatFieldMetadataMapsOrm',
-]);
+const SIZE_SAMPLE_PER_PROVIDER = 3;
const SIZE_WALK_NODE_CAP = 300_000;
-const MEMORY_REPORT_INTERVAL_MS = 60 * 1000;
-const BYTES_PER_MEGABYTE = 1024 * 1024;
-
-const toMegabytes = (bytes: number): number =>
- Math.round(bytes / BYTES_PER_MEGABYTE);
// Cache observability, split from WorkspaceCacheService (which drives it via start()/stop()).
@Injectable()
@@ -66,7 +46,6 @@ export class WorkspaceCacheMetricsService {
private cacheSizeTotalBytes = 0;
private sizeSampler?: ReturnType<typeof setInterval>;
private sizeStartupTimer?: ReturnType<typeof setTimeout>;
- private memoryReportTimer?: ReturnType<typeof setInterval>;
private statsCache?: { computedAt: number } & LocalCacheStats;
private sizeSampleInFlight = false;
private packingBacklog = 0;
@@ -115,7 +94,6 @@ export class WorkspaceCacheMetricsService {
this.localCache = localCache;
this.registerGauges();
this.scheduleSizeSampler();
- this.scheduleMemoryReport();
}
stop(): void {
@@ -125,47 +103,6 @@ export class WorkspaceCacheMetricsService {
if (isDefined(this.sizeSampler)) {
clearInterval(this.sizeSampler);
}
- if (isDefined(this.memoryReportTimer)) {
- clearInterval(this.memoryReportTimer);
- }
- }
-
- // TODO: diagnostic only — remove once the worker OOM root cause is confirmed.
- // Correlates process/V8 memory with the cache's own size in one log line so we
- // can tell whether the metadata cache tracks the heap climb to OOM (retained,
- // old-space) versus per-job allocation spikes (transient, new-space).
- private scheduleMemoryReport(): void {
- const report = (): void => {
- const memoryUsage = process.memoryUsage();
- const heapSpaces = v8.getHeapSpaceStatistics();
- const oldSpaceUsedBytes =
- heapSpaces.find((space) => space.space_name === 'old_space')
- ?.space_used_size ?? 0;
- const newSpaceUsedBytes =
- heapSpaces.find((space) => space.space_name === 'new_space')
- ?.space_used_size ?? 0;
-
- const topProviders = Object.entries(this.cacheSizeByKeyName)
- .sort(([, bytesA], [, bytesB]) => bytesB - bytesA)
- .slice(0, 3)
- .map(([keyName, bytes]) => `${keyName}:${toMegabytes(bytes)}MB`)
- .join(',');
-
- this.logger.log(
- `[WorkspaceCacheMemReport] ` +
- `heapUsedMB=${toMegabytes(memoryUsage.heapUsed)} ` +
- `rssMB=${toMegabytes(memoryUsage.rss)} ` +
- `externalMB=${toMegabytes(memoryUsage.external)} ` +
- `oldSpaceMB=${toMegabytes(oldSpaceUsedBytes)} ` +
- `newSpaceMB=${toMegabytes(newSpaceUsedBytes)} ` +
- `cacheEntries=${this.localCache?.size ?? 0} ` +
- `cacheBytesEstimateMB=${toMegabytes(this.cacheSizeTotalBytes)} ` +
- `top=${topProviders}`,
- );
- };
-
- this.memoryReportTimer = setInterval(report, MEMORY_REPORT_INTERVAL_MS);
- this.memoryReportTimer.unref();
}
recordRecompute(seconds: number, cacheKey: WorkspaceCacheKeyName): void {
@@ -272,12 +209,7 @@ export class WorkspaceCacheMetricsService {
stats.count += 1;
- const sizeEveryEntry = SIZE_IN_FULL_PROVIDERS.has(keyName);
-
- if (
- (sizeEveryEntry || stats.sampled < SIZE_SAMPLE_PER_PROVIDER) &&
- entry.versions.size > 0
- ) {
+ if (stats.sampled < SIZE_SAMPLE_PER_PROVIDER && entry.versions.size > 0) {
// Size every retained version, not just the latest — stale versions still occupy heap.
let entryBytes = 0;
diff --git a/packages/twenty-server/src/modules/match-participant/participant-target-reconciliation.service.ts b/packages/twenty-server/src/modules/match-participant/participant-target-reconciliation.service.ts
index 316b5a612c6..b228b15904f 100644
--- a/packages/twenty-server/src/modules/match-participant/participant-target-reconciliation.service.ts
+++ b/packages/twenty-server/src/modules/match-participant/participant-target-reconciliation.service.ts
@@ -1,6 +1,4 @@
-import { Injectable, Logger } from '@nestjs/common';
-
-import v8 from 'v8';
+import { Injectable } from '@nestjs/common';
import chunk from 'lodash.chunk';
import { QUERY_MAX_RECORDS } from 'twenty-shared/constants';
@@ -31,14 +29,8 @@ type TargetWorkspaceEntity = Omit<ExistingTarget, 'parentId' | 'deletedAt'> & {
deletedAt: string | null;
};
-const BYTES_PER_MEGABYTE = 1024 * 1024;
-
@Injectable()
export class ParticipantTargetReconciliationService {
- private readonly logger = new Logger(
- ParticipantTargetReconciliationService.name,
- );
-
constructor(private readonly workspaceOrmManager: WorkspaceOrmManager) {}
public async reconcileParticipantTargets({
@@ -152,35 +144,19 @@ export class ParticipantTargetReconciliationService {
messageThreadIds: string[];
transactionScope?: WorkspaceTransactionScope;
}): Promise<void> {
- const uniqueMessageThreadIds = [...new Set(messageThreadIds)];
-
- if (uniqueMessageThreadIds.length === 0) {
- return;
- }
-
const messageRepository = await this.getRepository<MessageWorkspaceEntity>(
'message',
transactionScope,
);
- // TODO: diagnostic only — remove once the worker OOM root cause is confirmed.
- // Size what one reconcile loads and how much heap it retains, to confirm
- // whether this path (added in #24778) drives worker memory.
- const heapUsedBeforeBytes = v8.getHeapStatistics().used_heap_size;
- let messagesLoaded = 0;
- let participantsLoaded = 0;
-
for (const messageThreadIdChunk of chunk(
- uniqueMessageThreadIds,
+ [...new Set(messageThreadIds)],
QUERY_MAX_RECORDS,
)) {
const threadMessages = await messageRepository.find({
where: { messageThreadId: In(messageThreadIdChunk) },
select: { id: true, messageThreadId: true },
});
-
- messagesLoaded += threadMessages.length;
-
const messageThreadIdByMessageId = new Map<string, string>();
for (const { id, messageThreadId } of threadMessages) {
@@ -209,8 +185,6 @@ export class ParticipantTargetReconciliationService {
);
}
- participantsLoaded += participants.length;
-
await this.reconcileTargets({
parentIds: messageThreadIdChunk,
parentFieldName: 'messageThreadId',
@@ -223,14 +197,6 @@ export class ParticipantTargetReconciliationService {
transactionScope,
});
}
-
- const heapDeltaMegabytes =
- (v8.getHeapStatistics().used_heap_size - heapUsedBeforeBytes) /
- BYTES_PER_MEGABYTE;
-
- this.logger.log(
- `[ReconcileMem] messageThreadTarget threads=${uniqueMessageThreadIds.length} messagesLoaded=${messagesLoaded} participantsLoaded=${participantsLoaded} heapDeltaMB=${heapDeltaMegabytes.toFixed(1)}`,
- );
}
private async reconcileTargets({