The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
feat(worker): log queue wait per tenant and record job duration (#29131)
martmull79a13e072fdd
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/logic-function-job-runner.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/logic-function-job-runner.service.tsindex dd5b2de3d2d..a0ace8f0cd5 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/logic-function-job-runner.service.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/logic-function-job-runner.service.ts@@ -21,6 +21,7 @@ import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scopeexport type LogicFunctionJobPayload = {logicFunctionId: string;workspaceId: string;+ applicationUniversalIdentifier?: string;payload?: object;userId?: string;userWorkspaceId?: string;diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.tsindex 1127a38cfcf..64d28285971 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts@@ -132,6 +132,12 @@ export class CronTriggerCronJob {{logicFunctionId: logicFunction.id,workspaceId: activeWorkspace.id,+ ...(isDefined(application)+ ? {+ applicationUniversalIdentifier:+ application.universalIdentifier,+ }+ : {}),payload: {},},{diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.tsindex 7123c1a6acf..fbb19c952f7 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts@@ -167,6 +167,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {byId: {[APPLICATION_ID]: {id: APPLICATION_ID,+ universalIdentifier: 'application-universal-identifier',applicationRegistrationId: 'application-registration-id',defaultRoleId: APPLICATION_ROLE_ID,deletedAt: null,@@ -244,6 +245,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {data: expect.objectContaining({logicFunctionId: LOGIC_FUNCTION_ID,workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: 'application-universal-identifier',userId: 'user-id',userWorkspaceId: 'user-workspace-id',payload: expect.objectContaining(event),diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.tsindex 0c7ae6bcfc0..f1a83127d1a 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts@@ -171,7 +171,10 @@ export class CallDatabaseEventTriggerJobsJob {await this.messageQueueService.bulkAdd<LogicFunctionTriggerJobData>(LogicFunctionTriggerJob.name,logicFunctionPayloads.map((logicFunctionPayload, index) => ({- data: logicFunctionPayload,+ data: {+ ...logicFunctionPayload,+ applicationUniversalIdentifier: application.universalIdentifier,+ },jobId: logicFunctionTriggerJobIds[index],})),{diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.tsindex 9c0a6bca4dc..89919f1188b 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts@@ -4,6 +4,8 @@ import { DEFAULT_JOB_PRIORITY } from 'src/engine/core-modules/message-queue/consimport { QUEUE_JOB_CHANGED_EVENT } from 'src/engine/core-modules/message-queue/constants/queue-job-changed-event.constant';import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { JOB_LATENCY_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant';+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';const mockGetJobs = jest.fn();const mockGetRanges = jest.fn();@@ -353,6 +355,7 @@ describe('BullMQDriver queue wait metric', () => {expect(recordHistogram).toHaveBeenCalledWith(expect.objectContaining({+ key: MetricsKeys.JobLatencyMs,value: 2_000,attributes: { queue: MessageQueue.workflowQueue, job_name: 'job' },}),@@ -367,7 +370,7 @@ describe('BullMQDriver queue wait metric', () => {});expect(recordHistogram).toHaveBeenCalledWith(- expect.objectContaining({ value: 0 }),+ expect.objectContaining({ key: MetricsKeys.JobLatencyMs, value: 0 }),);});@@ -378,7 +381,31 @@ describe('BullMQDriver queue wait metric', () => {attemptsStarted: 2,});- expect(recordHistogram).not.toHaveBeenCalled();+ expect(recordHistogram).not.toHaveBeenCalledWith(+ expect.objectContaining({ key: MetricsKeys.JobLatencyMs }),+ );+ });++ it('records the job duration', async () => {+ await processJob({+ opts: {},+ timestamp: Date.now(),+ attemptsStarted: 1,+ });++ const durationCalls = recordHistogram.mock.calls.filter(+ ([options]) => options.key === MetricsKeys.JobDurationMs,+ );++ expect(durationCalls).toHaveLength(1);+ expect(durationCalls[0][0]).toEqual({+ key: MetricsKeys.JobDurationMs,+ value: expect.any(Number),+ unit: 'ms',+ attributes: { queue: MessageQueue.workflowQueue, job_name: 'job' },+ bucketBoundaries: JOB_LATENCY_MS_BUCKET_BOUNDARIES,+ });+ expect(durationCalls[0][0].value).toBeGreaterThanOrEqual(0);});});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex 11e43dc6260..15d4585bc50 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -43,6 +43,7 @@ import { type MessageQueue } from 'src/engine/core-modules/message-queue/messageimport { type QueueJobChangedEvent } from 'src/engine/core-modules/message-queue/types/queue-job-changed-event.type';import { getJobKey } from 'src/engine/core-modules/message-queue/utils/get-job-key.util';import { buildQueueJobIdWithSuffix } from 'src/engine/core-modules/message-queue/utils/build-queue-job-id-with-suffix.util';+import { buildJobLogContext } from 'src/engine/core-modules/message-queue/utils/build-job-log-context.util';import { computeJobQueueWaitMs } from 'src/engine/core-modules/message-queue/utils/compute-job-queue-wait-ms.util';import { isQueueJobIdAlreadyWaiting } from 'src/engine/core-modules/message-queue/utils/is-queue-job-id-already-waiting.util';import { JOB_LATENCY_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant';@@ -292,28 +293,39 @@ export class BullMQDriver// TODO: Correctly support for job.idconst timeStart = performance.now();- const workspaceId = job.data?.workspaceId;- const workspaceSuffix = workspaceId- ? ` [workspace=${workspaceId}]`- : '';+ const logContext = buildJobLogContext({+ jobData: job.data,+ queueWaitMs,+ });this.logger.log(- `Processing job ${job.id} with name ${job.name} on queue ${queueName}${workspaceSuffix}`,+ `Processing job ${job.id} with name ${job.name} on queue ${queueName}${logContext}`,);- await handler({- data: job.data,- id: job.id ?? '',- name: job.name,- retryLimit: Math.max(0, (job.opts.attempts ?? 1) - 1),- updateData: (data) => job.updateData(data),- updateProgress: (progress) => job.updateProgress(progress),- abortSignal,- });- const timeEnd = performance.now();- const executionTime = timeEnd - timeStart;++ try {+ await handler({+ data: job.data,+ id: job.id ?? '',+ name: job.name,+ retryLimit: Math.max(0, (job.opts.attempts ?? 1) - 1),+ updateData: (data) => job.updateData(data),+ updateProgress: (progress) => job.updateProgress(progress),+ abortSignal,+ });+ } finally {+ this.metricsService.recordHistogram({+ key: MetricsKeys.JobDurationMs,+ value: performance.now() - timeStart,+ unit: 'ms',+ attributes: { queue: queueName, job_name: job.name },+ bucketBoundaries: JOB_LATENCY_MS_BUCKET_BOUNDARIES,+ });+ }++ const executionTime = performance.now() - timeStart;this.logger.log(- `Job ${job.id} with name ${job.name} processed on queue ${queueName} in ${executionTime.toFixed(2)}ms${workspaceSuffix}`,+ `Job ${job.id} with name ${job.name} processed on queue ${queueName} in ${executionTime.toFixed(2)}ms${logContext}`,);}),workerOptions,diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/__tests__/build-job-log-context.util.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/__tests__/build-job-log-context.util.spec.tsnew file mode 100644index 00000000000..fc6da7c53c7--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/__tests__/build-job-log-context.util.spec.ts@@ -0,0 +1,33 @@+import { buildJobLogContext } from 'src/engine/core-modules/message-queue/utils/build-job-log-context.util';++describe('buildJobLogContext', () => {+ it('lists workspace, application and queue wait as key=value pairs', () => {+ expect(+ buildJobLogContext({+ jobData: {+ workspaceId: 'ws-1',+ applicationUniversalIdentifier: 'app-1',+ },+ queueWaitMs: 1_200,+ }),+ ).toBe(' [workspace=ws-1 application=app-1 queueWaitMs=1200]');+ });++ it('keeps a zero queue wait', () => {+ expect(+ buildJobLogContext({ jobData: { workspaceId: 'ws-1' }, queueWaitMs: 0 }),+ ).toBe(' [workspace=ws-1 queueWaitMs=0]');+ });++ it('skips fields that are missing or not strings', () => {+ expect(+ buildJobLogContext({+ jobData: { workspaceId: 42, applicationUniversalIdentifier: '' },+ }),+ ).toBe('');+ });++ it('handles job data that is not an object', () => {+ expect(buildJobLogContext({ jobData: [{ workspaceId: 'ws-1' }] })).toBe('');+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-job-log-context.util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-job-log-context.util.tsnew file mode 100644index 00000000000..b98c449b168--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-job-log-context.util.ts@@ -0,0 +1,29 @@+import { isNonEmptyString } from '@sniptt/guards';+import { isDefined, isPlainObject } from 'twenty-shared/utils';++// key=value pairs so Loki can aggregate queue wait per workspace / application+export const buildJobLogContext = ({+ jobData,+ queueWaitMs,+}: {+ jobData: unknown;+ queueWaitMs?: number;+}): string => {+ const fields: string[] = [];++ if (isPlainObject(jobData)) {+ if (isNonEmptyString(jobData.workspaceId)) {+ fields.push(`workspace=${jobData.workspaceId}`);+ }++ if (isNonEmptyString(jobData.applicationUniversalIdentifier)) {+ fields.push(`application=${jobData.applicationUniversalIdentifier}`);+ }+ }++ if (isDefined(queueWaitMs)) {+ fields.push(`queueWaitMs=${queueWaitMs}`);+ }++ return fields.length > 0 ? ` [${fields.join(' ')}]` : '';+};diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.tsindex 2edf724a138..850052c5358 100644--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts@@ -67,6 +67,7 @@ export enum MetricsKeys {JobStalled = 'job/stalled',JobWaiting = 'job/waiting',JobLatencyMs = 'job/latency-ms',+ JobDurationMs = 'job/duration-ms',AiChatTurnLatencyMs = 'ai-chat/turn-latency-ms',AiChatStepLatencyMs = 'ai-chat/step-latency-ms',AiChatTtftMs = 'ai-chat/ttft-ms',