Skip to content

The open alternative to Salesforce, designed for AI.

fix(server): prevent SSE stream teardown errors from crashing all pods (#21395)

Charles Bochet1c3ae92c04ce
diff --git a/packages/twenty-server/src/engine/subscriptions/event-stream.resolver.ts b/packages/twenty-server/src/engine/subscriptions/event-stream.resolver.ts
index ad5e9ee5444..3a04160fa20 100644
--- a/packages/twenty-server/src/engine/subscriptions/event-stream.resolver.ts
+++ b/packages/twenty-server/src/engine/subscriptions/event-stream.resolver.ts
@@ -5,6 +5,7 @@ import { isDefined } from 'twenty-shared/utils';
import { MetadataResolver } from 'src/engine/api/graphql/graphql-config/decorators/metadata-resolver.decorator';
import { type ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';
+import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter';
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
import { type AuthContextUser } from 'src/engine/core-modules/auth/types/auth-context.type';
@@ -40,6 +41,7 @@ export class EventStreamResolver {
constructor(
private readonly subscriptionService: SubscriptionService,
private readonly eventStreamService: EventStreamService,
+ private readonly exceptionHandlerService: ExceptionHandlerService,
) {}
@Subscription(() => EventSubscriptionDTO, {
@@ -152,6 +154,11 @@ export class EventStreamResolver {
workspaceId: workspace.id,
eventStreamChannelId,
}),
+ onCleanupError: (error) =>
+ this.exceptionHandlerService.captureExceptions([error], {
+ workspace: { id: workspace.id },
+ additionalData: { eventStreamChannelId },
+ }),
});
}
diff --git a/packages/twenty-server/src/engine/subscriptions/utils/wrap-async-iterator-with-lifecycle.ts b/packages/twenty-server/src/engine/subscriptions/utils/wrap-async-iterator-with-lifecycle.ts
index fb182cf5bd5..9b172385fe7 100644
--- a/packages/twenty-server/src/engine/subscriptions/utils/wrap-async-iterator-with-lifecycle.ts
+++ b/packages/twenty-server/src/engine/subscriptions/utils/wrap-async-iterator-with-lifecycle.ts
@@ -5,13 +5,20 @@ type AsyncIteratorLifecycleOptions<T> = {
onHeartbeat?: () => Promise<boolean>;
heartbeatIntervalMs?: number;
onCleanup?: () => Promise<void>;
+ onCleanupError?: (error: unknown) => void;
};
export function wrapAsyncIteratorWithLifecycle<T>(
iterator: AsyncIterableIterator<T>,
options: AsyncIteratorLifecycleOptions<T>,
): AsyncIterableIterator<T> {
- const { initialValue, onHeartbeat, heartbeatIntervalMs, onCleanup } = options;
+ const {
+ initialValue,
+ onHeartbeat,
+ heartbeatIntervalMs,
+ onCleanup,
+ onCleanupError,
+ } = options;
let heartbeatInterval: NodeJS.Timeout | null = null;
let hasYieldedInitialValue = false;
@@ -33,7 +40,11 @@ export function wrapAsyncIteratorWithLifecycle<T>(
heartbeatInterval = null;
}
if (onCleanup) {
- await onCleanup();
+ try {
+ await onCleanup();
+ } catch (error) {
+ onCleanupError?.(error);
+ }
}
};
diff --git a/packages/twenty-server/src/main.ts b/packages/twenty-server/src/main.ts
index 9b65bfa59eb..121505c376c 100644
--- a/packages/twenty-server/src/main.ts
+++ b/packages/twenty-server/src/main.ts
@@ -2,6 +2,7 @@ import { NestFactory } from '@nestjs/core';
import { type NestExpressApplication } from '@nestjs/platform-express';
import fs from 'fs';
+import { inspect } from 'util';
import bytes from 'bytes';
import { useContainer } from 'class-validator';
@@ -11,10 +12,12 @@ import graphqlUploadExpress from 'graphql-upload/graphqlUploadExpress.mjs';
import { NodeEnvironment } from 'src/engine/core-modules/twenty-config/interfaces/node-environment.interface';
import { setPgDateTypeParser } from 'src/database/pg/set-pg-date-type-parser';
+import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
import { getSessionStorageOptions } from 'src/engine/core-modules/session-storage/session-storage.module-factory';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util';
+import { shouldCaptureException } from 'src/engine/utils/global-exception-handler.util';
import { UnhandledExceptionFilter } from 'src/filters/unhandled-exception.filter';
import { AppModule } from './app.module';
@@ -45,6 +48,18 @@ const bootstrap = async () => {
});
const logger = app.get(LoggerService);
const twentyConfigService = app.get(TwentyConfigService);
+ const exceptionHandlerService = app.get(ExceptionHandlerService);
+
+ process.on('unhandledRejection', (reason) => {
+ const error =
+ reason instanceof Error
+ ? reason
+ : new Error(typeof reason === 'string' ? reason : inspect(reason));
+
+ if (shouldCaptureException(error)) {
+ exceptionHandlerService.captureExceptions([error]);
+ }
+ });
const trustProxyRaw = twentyConfigService.get('TRUST_PROXY');
const trustProxy = /^\d+$/.test(trustProxyRaw)