The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import { type SerializableAuthContext } from 'src/engine/core-modules/auth/types/serializable-auth-context.type';
import { WithLock } from 'src/engine/core-modules/cache-lock/with-lock.decorator';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { EVENT_STREAM_TTL_MS } from 'src/engine/subscriptions/constants/event-stream-ttl.constant';
import {
EventStreamException,
EventStreamExceptionCode,
} from 'src/engine/subscriptions/event-stream.exception';
import {
type EventStreamData,
type RecordOrMetadataGqlOperationSignature,
} from 'src/engine/subscriptions/types/event-stream-data.type';
const ACTIVE_STREAM_COUNT_REFRESH_MS = 5 * 60 * 1_000;
const ACTIVE_STREAM_EXPIRATIONS_KEY = 'activeStreamExpirations';
@Injectable()
export class EventStreamService implements OnModuleInit {
private readonly logger = new Logger(EventStreamService.name);
private activeStreamCount = 0;
private activeStreamCountRefreshedAt = 0;
constructor(
@InjectCacheStorage(CacheStorageNamespace.EngineSubscriptions)
private readonly cacheStorageService: CacheStorageService,
private readonly metricsService: MetricsService,
) {}
onModuleInit() {
this.metricsService.createObservableGauge({
metricName: 'twenty_event_streams_live_total',
options: { description: 'Current number of live event streams' },
callback: async () => {
return this.getTotalActiveStreamCount();
},
cacheValue: true,
});
}
async getTotalActiveStreamCount(): Promise<number> {
const now = Date.now();
const isStale =
now - this.activeStreamCountRefreshedAt >= ACTIVE_STREAM_COUNT_REFRESH_MS;
if (isStale) {
this.activeStreamCount =
await this.cacheStorageService.sortedSetRemoveByScoreAndCount(
ACTIVE_STREAM_EXPIRATIONS_KEY,
0,
now,
);
this.activeStreamCountRefreshedAt = now;
}
return this.activeStreamCount;
}
async createEventStream({
workspaceId,
eventStreamChannelId,
authContext,
}: {
workspaceId: string;
eventStreamChannelId: string;
authContext: SerializableAuthContext;
}): Promise<void> {
const key = this.getEventStreamKey(workspaceId, eventStreamChannelId);
const existing = await this.cacheStorageService.get<EventStreamData>(key);
if (isDefined(existing)) {
throw new EventStreamException(
'Event stream already exists',
EventStreamExceptionCode.EVENT_STREAM_ALREADY_EXISTS,
);
}
const streamData: EventStreamData = {
authContext,
workspaceId,
queries: {},
createdAt: Date.now(),
};
await this.cacheStorageService.set(key, streamData, EVENT_STREAM_TTL_MS);
await this.cacheStorageService.setAdd(
this.getActiveStreamsKey(workspaceId),
[eventStreamChannelId],
EVENT_STREAM_TTL_MS,
);
await this.trackActiveStream({
workspaceId,
eventStreamChannelId,
});
}
async destroyEventStream({
workspaceId,
eventStreamChannelId,
}: {
workspaceId: string;
eventStreamChannelId: string;
}): Promise<void> {
const key = this.getEventStreamKey(workspaceId, eventStreamChannelId);
await this.cacheStorageService.del(key);
await this.cacheStorageService.setRemove(
this.getActiveStreamsKey(workspaceId),
[eventStreamChannelId],
);
await this.untrackActiveStreams(workspaceId, [eventStreamChannelId]);
}
async getActiveStreamIds(workspaceId: string): Promise<string[]> {
return this.cacheStorageService.setMembers(
this.getActiveStreamsKey(workspaceId),
);
}
async removeFromActiveStreams(
workspaceId: string,
streamIdsToRemove: string[],
): Promise<void> {
if (streamIdsToRemove.length === 0) {
return;
}
await this.cacheStorageService.setRemove(
this.getActiveStreamsKey(workspaceId),
streamIdsToRemove,
);
await this.untrackActiveStreams(workspaceId, streamIdsToRemove);
}
async getStreamsData(
workspaceId: string,
streamChannelIds: string[],
): Promise<Map<string, EventStreamData | undefined>> {
if (streamChannelIds.length === 0) {
return new Map();
}
const keys = streamChannelIds.map((id) =>
this.getEventStreamKey(workspaceId, id),
);
const values = await this.cacheStorageService.mget<EventStreamData>(keys);
const result = new Map<string, EventStreamData | undefined>();
streamChannelIds.forEach((id, index) => {
result.set(id, values[index]);
});
return result;
}
async isAuthorized({
authContext,
streamData,
}: {
authContext: SerializableAuthContext;
streamData: EventStreamData;
}): Promise<boolean> {
if (isDefined(authContext.userWorkspaceId)) {
return (
streamData.authContext.userWorkspaceId ===
authContext.userWorkspaceId &&
(streamData.authContext.applicationId ?? null) ===
(authContext.applicationId ?? null)
);
}
if (isDefined(authContext.apiKeyId)) {
return streamData.authContext.apiKeyId === authContext.apiKeyId;
}
return false;
}
@WithLock('eventStreamChannelId')
async addQuery({
workspaceId,
eventStreamChannelId,
queryId,
operationSignature,
}: {
workspaceId: string;
eventStreamChannelId: string;
queryId: string;
operationSignature: RecordOrMetadataGqlOperationSignature;
}): Promise<void> {
const key = this.getEventStreamKey(workspaceId, eventStreamChannelId);
const existing = await this.cacheStorageService.get<EventStreamData>(key);
if (!isDefined(existing)) {
return;
}
existing.queries[queryId] = operationSignature;
await this.cacheStorageService.set(key, existing, EVENT_STREAM_TTL_MS);
}
@WithLock('eventStreamChannelId')
async removeQuery({
workspaceId,
eventStreamChannelId,
queryId,
}: {
workspaceId: string;
eventStreamChannelId: string;
queryId: string;
}): Promise<void> {
const key = this.getEventStreamKey(workspaceId, eventStreamChannelId);
const existing = await this.cacheStorageService.get<EventStreamData>(key);
if (isDefined(existing) && isDefined(existing.queries[queryId])) {
delete existing.queries[queryId];
await this.cacheStorageService.set(key, existing, EVENT_STREAM_TTL_MS);
}
}
async refreshEventStreamTTL({
workspaceId,
eventStreamChannelId,
}: {
workspaceId: string;
eventStreamChannelId: string;
}): Promise<boolean> {
const eventStreamKey = this.getEventStreamKey(
workspaceId,
eventStreamChannelId,
);
const activeStreamsKey = this.getActiveStreamsKey(workspaceId);
const [eventStreamRefreshed, activeStreamsRefreshed] = await Promise.all([
this.cacheStorageService.expire(eventStreamKey, EVENT_STREAM_TTL_MS),
this.cacheStorageService.expire(activeStreamsKey, EVENT_STREAM_TTL_MS),
]);
if (eventStreamRefreshed) {
await this.trackActiveStream({
workspaceId,
eventStreamChannelId,
});
}
return eventStreamRefreshed && activeStreamsRefreshed;
}
private async trackActiveStream({
workspaceId,
eventStreamChannelId,
}: {
workspaceId: string;
eventStreamChannelId: string;
}): Promise<void> {
try {
await this.cacheStorageService.sortedSetAdd(
ACTIVE_STREAM_EXPIRATIONS_KEY,
[
{
score: Date.now() + EVENT_STREAM_TTL_MS,
value: this.getActiveStreamExpirationMember(
workspaceId,
eventStreamChannelId,
),
},
],
);
} catch (error) {
this.logger.warn(`Failed to track active event stream: ${error}`);
}
}
private async untrackActiveStreams(
workspaceId: string,
eventStreamChannelIds: string[],
): Promise<void> {
try {
await this.cacheStorageService.sortedSetRemove(
ACTIVE_STREAM_EXPIRATIONS_KEY,
eventStreamChannelIds.map((eventStreamChannelId) =>
this.getActiveStreamExpirationMember(
workspaceId,
eventStreamChannelId,
),
),
);
} catch (error) {
this.logger.warn(`Failed to untrack active event streams: ${error}`);
}
}
private getActiveStreamExpirationMember(
workspaceId: string,
eventStreamChannelId: string,
): string {
return `${workspaceId}:${eventStreamChannelId}`;
}
private getEventStreamKey(
workspaceId: string,
eventStreamId: string,
): string {
return `eventStream:${workspaceId}:${eventStreamId}`;
}
private getActiveStreamsKey(workspaceId: string): string {
return `workspace:${workspaceId}:activeStreams`;
}
async getStreamData(
workspaceId: string,
eventStreamChannelId: string,
): Promise<EventStreamData | undefined> {
const key = this.getEventStreamKey(workspaceId, eventStreamChannelId);
return this.cacheStorageService.get<EventStreamData>(key);
}
}