Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
15.8 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

import {randomUUID} from 'node:crypto';
import {ensureDeletionQueueState} from '@app/api/app/DeletionQueueStartup';
import type {APIConfig} from '@app/api/config/APIConfig';
import {hasDatabaseQueryExecutor, setDatabaseQueryExecutor} from '@app/api/database/CassandraQueryExecution';
import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';
import {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
import {
	jetStreamActivityPublisher,
	renderActivityMetrics,
	shutdownActivityEvents,
	startActivityEvents,
} from '@app/api/infrastructure/activity/ActivityEvents';
import {
	renderSharedListMetrics,
	startSharedListWatch,
	stopSharedListWatch,
} from '@app/api/infrastructure/activity/SharedLists';
import {shutdownStorageChangeFeed} from '@app/api/infrastructure/StorageServiceFactory';
import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
import {ipBanCache} from '@app/api/middleware/IpBanMiddleware';
import {startRequestErrorTelemetry, stopRequestErrorTelemetry} from '@app/api/middleware/RequestErrorTelemetry';
import {initializeServiceSingletons, shutdownReportService} from '@app/api/middleware/ServiceMiddleware';
import {
	closeOwnedKVClient,
	ensureVoiceResourcesInitialized,
	getKVClient,
	getSnowflakeService,
	setInjectedWorkerService,
	shutdownVoiceResources,
} from '@app/api/middleware/ServiceRegistry';
import {
	getCacheService,
	getCustomBadgeCatalog,
	getInstanceConfigRepository,
	getKVAccountDeletionQueue,
	getReportRepository,
	getUserRepository,
	type ServiceSingletonDependencies,
	shutdownInstanceConfigRepository,
	shutdownServiceSingletons,
} from '@app/api/middleware/ServiceSingletons';
import {initializeSearch, shutdownSearch} from '@app/api/SearchFactory';
import {warmupAdminSearchIndexes} from '@app/api/search/SearchWarmup';
import {VisionarySlotInitializer} from '@app/api/stripe/VisionarySlotInitializer';
import {VoiceDataInitializer} from '@app/api/voice/VoiceDataInitializer';
import {JetStreamWorkerQueue} from '@app/api/worker/JetStreamWorkerQueue';
import {WorkerService} from '@app/api/worker/WorkerService';
import {registerMetricsSection} from '@fluxer/hono/src/middleware/Metrics';
import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface';
import {getClient, initCassandra, shutdownCassandra} from '@pkgs/cassandra/src/Client';
import {ensureGeoipDatabaseOnStartup} from '@pkgs/geoip/src/GeoipStartup';
import {JetStreamConnectionManager} from '@pkgs/nats/src/JetStreamConnectionManager';
import {getDefaultPostgresClient, initPostgres, shutdownPostgres} from '@pkgs/postgres/src/Client';

let jsConnectionManager: JetStreamConnectionManager | null = null;
let jobQueue: ApiJobQueue | null = null;
let searchInitRetryTimer: ReturnType<typeof setTimeout> | null = null;
let searchInitStopped = false;
let searchInitInFlight: Promise<void> | null = null;
let markSearchReady: () => void = () => {};
const searchReady = new Promise<void>((resolve) => {
	markSearchReady = resolve;
});
const SEARCH_INIT_RETRY_MIN_MS = 5_000;
const SEARCH_INIT_RETRY_MAX_MS = 60_000;
const unregisterMetricsSections: Array<() => void> = [];

interface RefreshCacheLifecycle {
	initialize(): Promise<void>;
	shutdown(): Promise<void>;
}

const refreshCaches = new Map<RefreshCacheLifecycle, string>();
let refreshCacheShutdownPromise: Promise<void> | null = null;

async function initializeRefreshCache(
	cache: RefreshCacheLifecycle,
	name: string,
	logger: LoggerInterface,
): Promise<void> {
	refreshCaches.set(cache, name);
	await cache.initialize();
	logger.info(`${name} initialized`);
}

async function shutdownRefreshCaches(logger: LoggerInterface): Promise<void> {
	if (refreshCacheShutdownPromise) return await refreshCacheShutdownPromise;
	const caches = [...refreshCaches];
	refreshCaches.clear();
	const shutdown = Promise.all(
		caches.map(async ([cache, name]) => {
			try {
				await cache.shutdown();
				logger.info(`${name} shut down`);
			} catch (error) {
				logger.error({error}, `Error shutting down ${name}`);
			}
		}),
	).then(() => undefined);
	refreshCacheShutdownPromise = shutdown;
	try {
		await shutdown;
	} finally {
		if (refreshCacheShutdownPromise === shutdown) refreshCacheShutdownPromise = null;
	}
}

interface ApiJobQueue {
	jsConnectionManager: JetStreamConnectionManager;
	queue: JetStreamWorkerQueue;
	workerService: WorkerService;
	jobLedger: JobLedgerRepository;
}

export function getApiJobQueue(): ApiJobQueue | null {
	return jobQueue;
}

export function whenSearchReady(): Promise<void> {
	return searchReady;
}

function unsupportedDatabaseBackend(backend: never): never {
	throw new Error(`Unsupported database backend during shutdown: ${String(backend)}`);
}

async function initializeSearchAndWarmup(
	kvClient: ReturnType<typeof getKVClient>,
	logger: LoggerInterface,
): Promise<void> {
	await initializeSearch(getCacheService());
	markSearchReady();
	if (searchInitStopped) return;
	logger.info('Search initialized');
	const warmupLockKey = 'fluxer:search:warmup:admin';
	const warmupLockToken = randomUUID();
	const warmupLockTtlSeconds = 60 * 60;
	const acquiredWarmupLock = await kvClient.acquireLock(warmupLockKey, warmupLockToken, warmupLockTtlSeconds);
	if (!acquiredWarmupLock) {
		logger.info('Another API instance is warming search indexes, skipping warmup');
		return;
	}
	try {
		await warmupAdminSearchIndexes({
			userRepository: getUserRepository(),
			guildRepository: new GuildRepository(),
			reportRepository: getReportRepository(),
			logger,
		});
	} catch (error) {
		logger.error({error}, 'Admin search warmup failed (continuing startup)');
	} finally {
		try {
			await kvClient.releaseLock(warmupLockKey, warmupLockToken);
		} catch (error) {
			logger.warn({error}, 'Failed to release admin search warmup lock');
		}
	}
}

function scheduleSearchInitRetry(
	kvClient: ReturnType<typeof getKVClient>,
	logger: LoggerInterface,
	attempt: number,
): void {
	if (searchInitStopped) return;
	const delay = Math.min(SEARCH_INIT_RETRY_MIN_MS * 2 ** attempt, SEARCH_INIT_RETRY_MAX_MS);
	searchInitRetryTimer = setTimeout(() => {
		searchInitRetryTimer = null;
		searchInitInFlight = initializeSearchAndWarmup(kvClient, logger)
			.catch((error) => {
				logger.error({error, attempt: attempt + 1}, 'Search initialisation failed, retrying in the background');
				scheduleSearchInitRetry(kvClient, logger, attempt + 1);
			})
			.finally(() => {
				searchInitInFlight = null;
			});
	}, delay);
}

export function createInitializer(
	config: APIConfig,
	logger: LoggerInterface,
	dependencies: ServiceSingletonDependencies = {},
): () => Promise<void> {
	return async (): Promise<void> => {
		try {
			logger.info('Initializing API service...');
			const geoipStartupResult = await ensureGeoipDatabaseOnStartup({
				geoip: config.geoip,
				s3Config: {
					endpoint: config.s3.endpoint,
					region: config.s3.region,
					accessKeyId: config.s3.accessKeyId,
					secretAccessKey: config.s3.secretAccessKey,
				},
			});
			if (geoipStartupResult.mode === 's3') {
				logger.info(
					{
						maxmind_db_path: geoipStartupResult.maxmindDbPath,
						city: geoipStartupResult.city,
						s3_bucket: geoipStartupResult.bucket,
						s3_key: geoipStartupResult.key,
					},
					'GeoIP database downloaded from S3',
				);
			}
			if (config.database.backend === 'postgres' && !hasDatabaseQueryExecutor()) {
				await initPostgres(config.postgres, (diagnostic) => logger.error(diagnostic, 'Postgres connection error'));
				const postgres = getDefaultPostgresClient();
				await ensurePostgresKvSchema(postgres);
				setDatabaseQueryExecutor(new PostgresKvQueryExecutor(postgres));
				logger.info('Postgres KV client initialized');
			} else if (config.database.backend === 'postgres') {
				logger.info('Using injected database query executor for tests');
			}
			if (config.database.backend === 'cassandra' && !hasDatabaseQueryExecutor()) {
				const {ensureCassandraSchema, logMissingCassandraKeyspace} = await import(
					'@app/api/database/CassandraSchemaStartup'
				);
				await initCassandra({
					hosts: config.cassandra.hosts.split(',').filter(Boolean),
					port: config.cassandra.port,
					keyspace: config.cassandra.keyspace,
					localDc: config.cassandra.localDc,
					username: config.cassandra.username || undefined,
					password: config.cassandra.password || undefined,
				}).catch((error: unknown) => {
					if (config.instance.selfHosted) {
						logMissingCassandraKeyspace(error, config.cassandra.keyspace, config.cassandra.localDc, logger);
					}
					throw error;
				});
				logger.info('Cassandra client initialized');
				if (config.instance.selfHosted) {
					await ensureCassandraSchema({
						client: getClient(),
						keyspace: config.cassandra.keyspace,
						kv: getKVClient(),
						logger,
					});
				}
			} else if (config.database.backend === 'cassandra') {
				logger.info('Using injected database query executor for tests');
			}
			const kvClient = getKVClient();
			ipBanCache.setRefreshSubscriber(kvClient);
			await initializeRefreshCache(ipBanCache, 'IP ban cache', logger);
			const {urlBlocklistCache} = await import('@app/api/middleware/UrlBlocklistCache');
			urlBlocklistCache.setRefreshSubscriber(kvClient);
			const {getStorageService} = await import('@app/api/middleware/ServiceSingletons');
			urlBlocklistCache.setStorageService(getStorageService());
			await initializeRefreshCache(urlBlocklistCache, 'URL blocklist cache', logger);
			const {fileShaCache} = await import('@app/api/middleware/FileShaCache');
			fileShaCache.setRefreshSubscriber(kvClient);
			await initializeRefreshCache(fileShaCache, 'File SHA blocklist cache', logger);
			const {phraseBlocklistCache} = await import('@app/api/middleware/PhraseBlocklistCache');
			phraseBlocklistCache.setRefreshSubscriber(kvClient);
			await initializeRefreshCache(phraseBlocklistCache, 'Phrase blocklist cache', logger);
			const {bannedAvatarHashCache} = await import('@app/api/middleware/BannedAvatarHashCache');
			bannedAvatarHashCache.setRefreshSubscriber(kvClient);
			await initializeRefreshCache(bannedAvatarHashCache, 'Banned avatar hash cache', logger);
			const {profileSubstringBlocklistCache} = await import('@app/api/middleware/ProfileSubstringBlocklistCache');
			profileSubstringBlocklistCache.setRefreshSubscriber(kvClient);
			await initializeRefreshCache(profileSubstringBlocklistCache, 'Profile substring blocklist cache', logger);
			await initializeServiceSingletons(dependencies);
			logger.info('Service singletons initialized');
			await getCustomBadgeCatalog().initialize();
			logger.info('Custom badge catalog initialized');
			if (!config.dev.testModeEnabled) {
				jsConnectionManager = new JetStreamConnectionManager({
					url: config.nats.jetStreamUrl,
					token: config.nats.authToken || undefined,
					name: 'api-worker',
				});
				await jsConnectionManager.connect();
				const queue = new JetStreamWorkerQueue(jsConnectionManager);
				await queue.ensureStream();
				const jobLedger = new JobLedgerRepository();
				const workerService = new WorkerService(queue, getSnowflakeService(), jobLedger);
				setInjectedWorkerService(workerService);
				jobQueue = {jsConnectionManager, queue, workerService, jobLedger};
				logger.info('JetStream worker service initialized');
				const connection = jsConnectionManager;
				await startActivityEvents({
					publisher: jetStreamActivityPublisher(connection.getJetStreamClient()),
					kv: kvClient,
					jsm: await connection.getJetStreamManager(),
					spoolWhileMissing: !config.instance.selfHosted,
				});
				startRequestErrorTelemetry();
				startSharedListWatch(connection.getJetStreamClient(), {selfHosted: config.instance.selfHosted});
				unregisterMetricsSections.push(
					registerMetricsSection(renderActivityMetrics),
					registerMetricsSection(renderSharedListMetrics),
				);
				logger.info('Activity events initialized');
			}
			await ensureDeletionQueueState(getKVAccountDeletionQueue(), logger);
			logger.info('Initializing search indexes...');
			searchInitStopped = false;
			try {
				await initializeSearchAndWarmup(kvClient, logger);
			} catch (error) {
				logger.error({error}, 'Search initialisation failed, retrying in the background');
				scheduleSearchInitRetry(kvClient, logger, 0);
			}
			if (config.voice.enabled && config.voice.defaultRegion) {
				const voiceDataInitializer = new VoiceDataInitializer();
				await voiceDataInitializer.initialize();
				await ensureVoiceResourcesInitialized();
				logger.info('Voice data initialized');
			}
			if (config.dev.testModeEnabled && config.stripe.enabled) {
				const visionarySlotInitializer = new VisionarySlotInitializer();
				await visionarySlotInitializer.initialize();
				logger.info('Stripe visionary slots initialized');
			}
			if (config.dev.testModeEnabled) {
				const instanceConfigRepository = getInstanceConfigRepository();
				try {
					await instanceConfigRepository.setSsoConfig({
						enabled: false,
						authorizationUrl: null,
						tokenUrl: null,
						clientId: null,
					});
					logger.info('Reset SSO config to disabled for test mode');
				} catch (error) {
					logger.warn({error}, 'Failed to reset SSO config for test mode');
				}
			}
			logger.info('API service initialization complete');
		} catch (error) {
			logger.error({error}, 'API service initialization failed');
			await createShutdown(config, logger)();
			throw error;
		}
	};
}

export function createShutdown(config: APIConfig, logger: LoggerInterface): () => Promise<void> {
	return async (): Promise<void> => {
		logger.info('Shutting down API service...');
		try {
			await shutdownVoiceResources();
			logger.info('Voice resources shut down');
		} catch (error) {
			logger.error({error}, 'Error shutting down voice resources');
		}
		stopRequestErrorTelemetry();
		stopSharedListWatch();
		for (const unregister of unregisterMetricsSections.splice(0)) unregister();
		await shutdownActivityEvents();
		jobQueue = null;
		if (jsConnectionManager) {
			try {
				await jsConnectionManager.drain();
				jsConnectionManager = null;
			} catch (error) {
				logger.error({error}, 'Error draining JetStream worker connection');
			}
		}
		await shutdownStorageChangeFeed();
		setInjectedWorkerService(undefined);
		searchInitStopped = true;
		if (searchInitRetryTimer) {
			clearTimeout(searchInitRetryTimer);
			searchInitRetryTimer = null;
		}
		await searchInitInFlight;
		try {
			await shutdownSearch();
			logger.info('Search service shut down');
		} catch (error) {
			logger.error({error}, 'Error shutting down search service');
		}
		await Promise.all([
			shutdownRefreshCaches(logger),
			shutdownServiceSingletons()
				.then(() => {
					logger.info('Service singletons shut down');
				})
				.catch((error) => {
					logger.error({error}, 'Error shutting down service singletons');
				}),
		]);
		try {
			await shutdownInstanceConfigRepository();
			logger.info('Instance config repository shut down');
		} catch (error) {
			logger.error({error}, 'Error shutting down instance config repository');
		}
		try {
			shutdownReportService();
			logger.info('Report service shut down');
		} catch (error) {
			logger.error({error}, 'Error shutting down report service');
		}
		setDatabaseQueryExecutor(null);
		switch (config.database.backend) {
			case 'postgres':
				try {
					await shutdownPostgres();
					logger.info('Postgres client shut down');
				} catch (error) {
					logger.error({error}, 'Error shutting down Postgres client');
				}
				break;
			case 'cassandra':
				try {
					await shutdownCassandra();
					logger.info('Cassandra client shut down');
				} catch (error) {
					logger.error({error}, 'Error shutting down Cassandra client');
				}
				break;
			default:
				unsupportedDatabaseBackend(config.database.backend);
		}
		try {
			closeOwnedKVClient();
			logger.info('Key-value client closed');
		} catch (error) {
			logger.error({error}, 'Error closing key-value client');
		}
		logger.info('API service shutdown complete');
	};
}