Skip to content

The open alternative to Salesforce, designed for AI.

feat(worker): log queue wait per tenant and record job duration (#29131)

martmull79a13e072fdd
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/logic-function-job-runner.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/logic-function-job-runner.service.ts
index dd5b2de3d2d..a0ace8f0cd5 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/logic-function-job-runner.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/logic-function-job-runner.service.ts
@@ -21,6 +21,7 @@ import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scope
export type LogicFunctionJobPayload = {
logicFunctionId: string;
workspaceId: string;
+ applicationUniversalIdentifier?: string;
payload?: object;
userId?: string;
userWorkspaceId?: string;
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts
index 1127a38cfcf..64d28285971 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts
@@ -132,6 +132,12 @@ export class CronTriggerCronJob {
{
logicFunctionId: logicFunction.id,
workspaceId: activeWorkspace.id,
+ ...(isDefined(application)
+ ? {
+ applicationUniversalIdentifier:
+ application.universalIdentifier,
+ }
+ : {}),
payload: {},
},
{
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts
index 7123c1a6acf..fbb19c952f7 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts
@@ -167,6 +167,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {
byId: {
[APPLICATION_ID]: {
id: APPLICATION_ID,
+ universalIdentifier: 'application-universal-identifier',
applicationRegistrationId: 'application-registration-id',
defaultRoleId: APPLICATION_ROLE_ID,
deletedAt: null,
@@ -244,6 +245,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {
data: expect.objectContaining({
logicFunctionId: LOGIC_FUNCTION_ID,
workspaceId: WORKSPACE_ID,
+ applicationUniversalIdentifier: 'application-universal-identifier',
userId: 'user-id',
userWorkspaceId: 'user-workspace-id',
payload: expect.objectContaining(event),
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts
index 0c7ae6bcfc0..f1a83127d1a 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts
@@ -171,7 +171,10 @@ export class CallDatabaseEventTriggerJobsJob {
await this.messageQueueService.bulkAdd<LogicFunctionTriggerJobData>(
LogicFunctionTriggerJob.name,
logicFunctionPayloads.map((logicFunctionPayload, index) => ({
- data: logicFunctionPayload,
+ data: {
+ ...logicFunctionPayload,
+ applicationUniversalIdentifier: application.universalIdentifier,
+ },
jobId: logicFunctionTriggerJobIds[index],
})),
{
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
index 9c0a6bca4dc..89919f1188b 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
@@ -4,6 +4,8 @@ import { DEFAULT_JOB_PRIORITY } from 'src/engine/core-modules/message-queue/cons
import { QUEUE_JOB_CHANGED_EVENT } from 'src/engine/core-modules/message-queue/constants/queue-job-changed-event.constant';
import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { JOB_LATENCY_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
const mockGetJobs = jest.fn();
const mockGetRanges = jest.fn();
@@ -353,6 +355,7 @@ describe('BullMQDriver queue wait metric', () => {
expect(recordHistogram).toHaveBeenCalledWith(
expect.objectContaining({
+ key: MetricsKeys.JobLatencyMs,
value: 2_000,
attributes: { queue: MessageQueue.workflowQueue, job_name: 'job' },
}),
@@ -367,7 +370,7 @@ describe('BullMQDriver queue wait metric', () => {
});
expect(recordHistogram).toHaveBeenCalledWith(
- expect.objectContaining({ value: 0 }),
+ expect.objectContaining({ key: MetricsKeys.JobLatencyMs, value: 0 }),
);
});
@@ -378,7 +381,31 @@ describe('BullMQDriver queue wait metric', () => {
attemptsStarted: 2,
});
- expect(recordHistogram).not.toHaveBeenCalled();
+ expect(recordHistogram).not.toHaveBeenCalledWith(
+ expect.objectContaining({ key: MetricsKeys.JobLatencyMs }),
+ );
+ });
+
+ it('records the job duration', async () => {
+ await processJob({
+ opts: {},
+ timestamp: Date.now(),
+ attemptsStarted: 1,
+ });
+
+ const durationCalls = recordHistogram.mock.calls.filter(
+ ([options]) => options.key === MetricsKeys.JobDurationMs,
+ );
+
+ expect(durationCalls).toHaveLength(1);
+ expect(durationCalls[0][0]).toEqual({
+ key: MetricsKeys.JobDurationMs,
+ value: expect.any(Number),
+ unit: 'ms',
+ attributes: { queue: MessageQueue.workflowQueue, job_name: 'job' },
+ bucketBoundaries: JOB_LATENCY_MS_BUCKET_BOUNDARIES,
+ });
+ expect(durationCalls[0][0].value).toBeGreaterThanOrEqual(0);
});
});
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index 11e43dc6260..15d4585bc50 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -43,6 +43,7 @@ import { type MessageQueue } from 'src/engine/core-modules/message-queue/message
import { type QueueJobChangedEvent } from 'src/engine/core-modules/message-queue/types/queue-job-changed-event.type';
import { getJobKey } from 'src/engine/core-modules/message-queue/utils/get-job-key.util';
import { buildQueueJobIdWithSuffix } from 'src/engine/core-modules/message-queue/utils/build-queue-job-id-with-suffix.util';
+import { buildJobLogContext } from 'src/engine/core-modules/message-queue/utils/build-job-log-context.util';
import { computeJobQueueWaitMs } from 'src/engine/core-modules/message-queue/utils/compute-job-queue-wait-ms.util';
import { isQueueJobIdAlreadyWaiting } from 'src/engine/core-modules/message-queue/utils/is-queue-job-id-already-waiting.util';
import { JOB_LATENCY_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant';
@@ -292,28 +293,39 @@ export class BullMQDriver
// TODO: Correctly support for job.id
const timeStart = performance.now();
- const workspaceId = job.data?.workspaceId;
- const workspaceSuffix = workspaceId
- ? ` [workspace=${workspaceId}]`
- : '';
+ const logContext = buildJobLogContext({
+ jobData: job.data,
+ queueWaitMs,
+ });
this.logger.log(
- `Processing job ${job.id} with name ${job.name} on queue ${queueName}${workspaceSuffix}`,
+ `Processing job ${job.id} with name ${job.name} on queue ${queueName}${logContext}`,
);
- await handler({
- data: job.data,
- id: job.id ?? '',
- name: job.name,
- retryLimit: Math.max(0, (job.opts.attempts ?? 1) - 1),
- updateData: (data) => job.updateData(data),
- updateProgress: (progress) => job.updateProgress(progress),
- abortSignal,
- });
- const timeEnd = performance.now();
- const executionTime = timeEnd - timeStart;
+
+ try {
+ await handler({
+ data: job.data,
+ id: job.id ?? '',
+ name: job.name,
+ retryLimit: Math.max(0, (job.opts.attempts ?? 1) - 1),
+ updateData: (data) => job.updateData(data),
+ updateProgress: (progress) => job.updateProgress(progress),
+ abortSignal,
+ });
+ } finally {
+ this.metricsService.recordHistogram({
+ key: MetricsKeys.JobDurationMs,
+ value: performance.now() - timeStart,
+ unit: 'ms',
+ attributes: { queue: queueName, job_name: job.name },
+ bucketBoundaries: JOB_LATENCY_MS_BUCKET_BOUNDARIES,
+ });
+ }
+
+ const executionTime = performance.now() - timeStart;
this.logger.log(
- `Job ${job.id} with name ${job.name} processed on queue ${queueName} in ${executionTime.toFixed(2)}ms${workspaceSuffix}`,
+ `Job ${job.id} with name ${job.name} processed on queue ${queueName} in ${executionTime.toFixed(2)}ms${logContext}`,
);
}),
workerOptions,
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/__tests__/build-job-log-context.util.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/__tests__/build-job-log-context.util.spec.ts
new file mode 100644
index 00000000000..fc6da7c53c7
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/__tests__/build-job-log-context.util.spec.ts
@@ -0,0 +1,33 @@
+import { buildJobLogContext } from 'src/engine/core-modules/message-queue/utils/build-job-log-context.util';
+
+describe('buildJobLogContext', () => {
+ it('lists workspace, application and queue wait as key=value pairs', () => {
+ expect(
+ buildJobLogContext({
+ jobData: {
+ workspaceId: 'ws-1',
+ applicationUniversalIdentifier: 'app-1',
+ },
+ queueWaitMs: 1_200,
+ }),
+ ).toBe(' [workspace=ws-1 application=app-1 queueWaitMs=1200]');
+ });
+
+ it('keeps a zero queue wait', () => {
+ expect(
+ buildJobLogContext({ jobData: { workspaceId: 'ws-1' }, queueWaitMs: 0 }),
+ ).toBe(' [workspace=ws-1 queueWaitMs=0]');
+ });
+
+ it('skips fields that are missing or not strings', () => {
+ expect(
+ buildJobLogContext({
+ jobData: { workspaceId: 42, applicationUniversalIdentifier: '' },
+ }),
+ ).toBe('');
+ });
+
+ it('handles job data that is not an object', () => {
+ expect(buildJobLogContext({ jobData: [{ workspaceId: 'ws-1' }] })).toBe('');
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-job-log-context.util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-job-log-context.util.ts
new file mode 100644
index 00000000000..b98c449b168
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-job-log-context.util.ts
@@ -0,0 +1,29 @@
+import { isNonEmptyString } from '@sniptt/guards';
+import { isDefined, isPlainObject } from 'twenty-shared/utils';
+
+// key=value pairs so Loki can aggregate queue wait per workspace / application
+export const buildJobLogContext = ({
+ jobData,
+ queueWaitMs,
+}: {
+ jobData: unknown;
+ queueWaitMs?: number;
+}): string => {
+ const fields: string[] = [];
+
+ if (isPlainObject(jobData)) {
+ if (isNonEmptyString(jobData.workspaceId)) {
+ fields.push(`workspace=${jobData.workspaceId}`);
+ }
+
+ if (isNonEmptyString(jobData.applicationUniversalIdentifier)) {
+ fields.push(`application=${jobData.applicationUniversalIdentifier}`);
+ }
+ }
+
+ if (isDefined(queueWaitMs)) {
+ fields.push(`queueWaitMs=${queueWaitMs}`);
+ }
+
+ return fields.length > 0 ? ` [${fields.join(' ')}]` : '';
+};
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
index 2edf724a138..850052c5358 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
@@ -67,6 +67,7 @@ export enum MetricsKeys {
JobStalled = 'job/stalled',
JobWaiting = 'job/waiting',
JobLatencyMs = 'job/latency-ms',
+ JobDurationMs = 'job/duration-ms',
AiChatTurnLatencyMs = 'ai-chat/turn-latency-ms',
AiChatStepLatencyMs = 'ai-chat/step-latency-ms',
AiChatTtftMs = 'ai-chat/ttft-ms',