Skip to content

The open alternative to Salesforce, designed for AI.

fix: drain metadata notifications before command shutdown (#26585)

Félix Malfait8155e759a5cd
diff --git a/packages/twenty-server/src/command/command.ts b/packages/twenty-server/src/command/command.ts
index ef9fc10ac85..d008a38fb2c 100644
--- a/packages/twenty-server/src/command/command.ts
+++ b/packages/twenty-server/src/command/command.ts
@@ -2,6 +2,7 @@ import { CommandFactory } from 'nest-commander';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
+import { MetadataEventEmitter } from 'src/engine/subscriptions/metadata-event/metadata-event-emitter';
import { shouldCaptureException } from 'src/engine/utils/global-exception-handler.util';
import { CommandModule } from './command.module';
@@ -28,8 +29,12 @@ async function bootstrap() {
app.useLogger(loggerService);
- await CommandFactory.runApplication(app);
-
- await app.close();
+ try {
+ await CommandFactory.runApplication(app);
+ } finally {
+ // Redis and queues close in onModuleDestroy, before shutdown hooks run.
+ await app.get(MetadataEventEmitter).drain();
+ await app.close();
+ }
}
void bootstrap();
diff --git a/packages/twenty-server/src/engine/subscriptions/metadata-event/metadata-event-emitter.ts b/packages/twenty-server/src/engine/subscriptions/metadata-event/metadata-event-emitter.ts
index 871b39a7fc4..e893be2005a 100644
--- a/packages/twenty-server/src/engine/subscriptions/metadata-event/metadata-event-emitter.ts
+++ b/packages/twenty-server/src/engine/subscriptions/metadata-event/metadata-event-emitter.ts
@@ -1,4 +1,4 @@
-import { Injectable } from '@nestjs/common';
+import { Injectable, Logger } from '@nestjs/common';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { AllMetadataName } from 'twenty-shared/metadata';
@@ -14,6 +14,9 @@ import {
MetadataEvent,
} from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event.type';
+// Shutdown must not hang on a stalled listener: notifications are best effort.
+const DRAIN_TIMEOUT_MS = 30_000;
+
type EmitMetadataEventsArgs = {
metadataEvents: MetadataEvent[];
workspaceId: string;
@@ -22,6 +25,9 @@ type EmitMetadataEventsArgs = {
@Injectable()
export class MetadataEventEmitter {
+ private readonly logger = new Logger(MetadataEventEmitter.name);
+ private readonly pendingEvents = new Set<Promise<unknown>>();
+
constructor(private readonly eventEmitter: EventEmitter2) {}
public emitMetadataEvents({
@@ -63,7 +69,42 @@ export class MetadataEventEmitter {
apiKeyId,
};
- this.eventEmitter.emit(eventName, metadataEventBatch);
+ const pendingEvent = this.eventEmitter
+ .emitAsync(eventName, metadataEventBatch)
+ .catch((error: unknown) => {
+ this.logger.error(
+ `Failed to handle ${eventName} for workspace ${workspaceId}`,
+ error instanceof Error ? error.stack : String(error),
+ );
+ })
+ .finally(() => {
+ this.pendingEvents.delete(pendingEvent);
+ });
+
+ this.pendingEvents.add(pendingEvent);
+ }
+ }
+
+ async drain(timeoutMs = DRAIN_TIMEOUT_MS): Promise<void> {
+ if (this.pendingEvents.size === 0) {
+ return;
+ }
+
+ let timeout: NodeJS.Timeout | undefined;
+
+ const hasTimedOut = await Promise.race([
+ Promise.allSettled([...this.pendingEvents]).then(() => false),
+ new Promise<boolean>((resolve) => {
+ timeout = setTimeout(() => resolve(true), timeoutMs);
+ }),
+ ]);
+
+ clearTimeout(timeout);
+
+ if (hasTimedOut) {
+ this.logger.warn(
+ `Stopped waiting for ${this.pendingEvents.size} metadata event batches after ${timeoutMs}ms`,
+ );
}
}
diff --git a/packages/twenty-server/test/integration/metadata/suites/object-metadata/metadata-event-completion.integration-spec.ts b/packages/twenty-server/test/integration/metadata/suites/object-metadata/metadata-event-completion.integration-spec.ts
new file mode 100644
index 00000000000..406a9cbcbc0
--- /dev/null
+++ b/packages/twenty-server/test/integration/metadata/suites/object-metadata/metadata-event-completion.integration-spec.ts
@@ -0,0 +1,116 @@
+import { createOneObjectMetadata } from 'test/integration/metadata/suites/object-metadata/utils/create-one-object-metadata.util';
+import { deleteOneObjectMetadata } from 'test/integration/metadata/suites/object-metadata/utils/delete-one-object-metadata.util';
+import { updateOneObjectMetadata } from 'test/integration/metadata/suites/object-metadata/utils/update-one-object-metadata.util';
+import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';
+import { isDefined } from 'twenty-shared/utils';
+
+import { type MetadataEventEmitter } from 'src/engine/subscriptions/metadata-event/metadata-event-emitter';
+import { type MetadataEventPublisher } from 'src/engine/subscriptions/metadata-event/metadata-event-publisher';
+
+const OBJECT_NAME = 'metadataNotificationTest';
+
+describe('Metadata event draining before command shutdown', () => {
+ let objectMetadataId: string;
+ let publisher: MetadataEventPublisher;
+ let emitter: MetadataEventEmitter;
+
+ beforeAll(async () => {
+ publisher = getAppProviderByClassName('MetadataEventPublisher');
+ emitter = getAppProviderByClassName('MetadataEventEmitter');
+
+ const { data } = await createOneObjectMetadata({
+ expectToFail: false,
+ input: {
+ nameSingular: OBJECT_NAME,
+ namePlural: `${OBJECT_NAME}s`,
+ labelSingular: 'Metadata notification test',
+ labelPlural: 'Metadata notification tests',
+ icon: 'IconBox',
+ isLabelSyncedWithName: false,
+ },
+ });
+
+ objectMetadataId = data.createOneObject.id;
+ await emitter.drain();
+ });
+
+ afterEach(async () => {
+ jest.restoreAllMocks();
+ await emitter.drain();
+ });
+
+ afterAll(async () => {
+ if (!isDefined(objectMetadataId)) {
+ return;
+ }
+
+ await updateOneObjectMetadata({
+ expectToFail: false,
+ input: {
+ idToUpdate: objectMetadataId,
+ updatePayload: { isActive: false },
+ },
+ });
+ await deleteOneObjectMetadata({
+ expectToFail: false,
+ input: { idToDelete: objectMetadataId },
+ });
+ await emitter.drain();
+ });
+
+ const blockObjectUpdatedPublish = () => {
+ let release!: () => void;
+ const blocked = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ const publish = publisher.publish.bind(publisher);
+
+ jest.spyOn(publisher, 'publish').mockImplementation(async (batch) => {
+ if (batch.metadataName === 'objectMetadata' && batch.type === 'updated') {
+ await blocked;
+ }
+
+ await publish(batch);
+ });
+
+ return release;
+ };
+
+ const updateDescription = (description: string) =>
+ updateOneObjectMetadata({
+ expectToFail: false,
+ gqlFields: 'id description',
+ input: { idToUpdate: objectMetadataId, updatePayload: { description } },
+ });
+
+ it('returns the mutation without waiting and drains the notification on shutdown', async () => {
+ const release = blockObjectUpdatedPublish();
+
+ const { data } = await updateDescription('Pending notification');
+
+ expect(data.updateOneObject.description).toBe('Pending notification');
+
+ let drained = false;
+ const draining = emitter.drain().then(() => {
+ drained = true;
+ });
+
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ expect(drained).toBe(false);
+
+ release();
+ await draining;
+
+ expect(drained).toBe(true);
+ });
+
+ it('stops waiting for a stalled notification after the timeout', async () => {
+ const release = blockObjectUpdatedPublish();
+
+ await updateDescription('Stalled notification');
+
+ await expect(emitter.drain(10)).resolves.toBeUndefined();
+
+ release();
+ });
+});