Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
21.8 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

import {Config} from '@app/api/Config';
import {createApiContext} from '@app/api/CreateApiContext';
import {setDatabaseQueryExecutor} from '@app/api/database/CassandraQueryExecution';
import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';
import {
	activityEventsPublishing,
	jetStreamActivityPublisher,
	shutdownActivityEvents,
	startActivityEvents,
} from '@app/api/infrastructure/activity/ActivityEvents';
import {setActivityProcessChannel} from '@app/api/infrastructure/activity/ActivityMeta';
import {startSharedListWatch, stopSharedListWatch} from '@app/api/infrastructure/activity/SharedLists';
import {sizeProcessForHost} from '@app/api/infrastructure/HostResources';
import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService';
import {shutdownStorageChangeFeed} from '@app/api/infrastructure/StorageServiceFactory';
import type {InstanceConfigRepository} from '@app/api/instance/InstanceConfigRepository';
import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
import {Logger} from '@app/api/Logger';
import type {LimitConfigService} from '@app/api/limits/LimitConfigService';
import {startContentBlocklistCaches, stopContentBlocklistCaches} from '@app/api/middleware/ContentBlocklistCaches';
import {
	closeOwnedKVClient,
	createSnowflakeService,
	getKVClient,
	setInjectedSnowflakeService,
	setInjectedWorkerService,
	shutdownVoiceResources,
} from '@app/api/middleware/ServiceRegistry';
import {
	getAdminArchiveService,
	getAdminRepository,
	getCacheService,
	getInstanceConfigRepository,
	getLimitConfigService,
} from '@app/api/middleware/ServiceSingletons';
import {getLabsOperatorsChannelId, isLabsAutoJoinConfigured} from '@app/api/operator_pass/OperatorPassLabsRole';
import {initializeSearch, shutdownSearch} from '@app/api/SearchFactory';
import {getStripeClient} from '@app/api/stripe/StripeClient';
import {accountStateDepsFromContext} from '@app/api/user/services/AccountStateApplier';
import {awaitAll} from '@app/api/utils/ConcurrencyUtils';
import {readOptionalEnv} from '@app/api/utils/IntegerOptions';
import {startAccountActionConsumer, stopAccountActionConsumer} from '@app/api/worker/AccountActionConsumer';
import {queueBlocklistFeedStartupJobs} from '@app/api/worker/BlocklistFeedStartup';
import {CronScheduler} from '@app/api/worker/CronScheduler';
import {JetStreamWorkerQueue, JOBS_STREAM_MAX_AGE_MS} from '@app/api/worker/JetStreamWorkerQueue';
import {queueDeletedUsersBackfill} from '@app/api/worker/tasks/BackfillDeletedUsers';
import {queueFavoriteMemeNsfwBackfill} from '@app/api/worker/tasks/BackfillFavoriteMemeNsfw';
import {queueHiddenProfileBadgesBackfill} from '@app/api/worker/tasks/BackfillHiddenProfileBadges';
import {queuePasskeyTraitBackfill} from '@app/api/worker/tasks/BackfillPasskeyTrait';
import {queueUserAclIndexBackfill} from '@app/api/worker/tasks/BackfillUserAclIndex';
import {singleCommunityJoinPending} from '@app/api/worker/tasks/JoinSingleCommunityMembers';
import {clearWorkerDependencies, setWorkerDependencies} from '@app/api/worker/WorkerContext';
import {
	guildInvitePauseDeps,
	initializeWorkerDependencies,
	type WorkerDependencies,
} from '@app/api/worker/WorkerDependencies';
import {WorkerHeartbeat} from '@app/api/worker/WorkerHeartbeat';
import {
	resolveCronSchedulerEnabled,
	resolveWorkerLanes,
	validateLaneCompleteness,
} from '@app/api/worker/WorkerLaneConfig';
import {createWorkerProcessErrorHandler} from '@app/api/worker/WorkerProcessErrorHandler';
import {WorkerRunner} from '@app/api/worker/WorkerRunner';
import {WorkerService} from '@app/api/worker/WorkerService';
import {workerTasks} from '@app/api/worker/WorkerTaskRegistry';
import {createRegisteredMetricsHandler} from '@fluxer/hono/src/middleware/Metrics';
import {createServer, setupGracefulShutdown} from '@fluxer/hono/src/Server';
import type {ServerType} from '@hono/node-server';
import {BACKGROUND_READ_TIMEOUT_MS, getClient, initCassandra, shutdownCassandra} from '@pkgs/cassandra/src/Client';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import {JetStreamConnectionManager} from '@pkgs/nats/src/JetStreamConnectionManager';
import {getDefaultPostgresClient, initPostgres, shutdownPostgres} from '@pkgs/postgres/src/Client';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {Hono} from 'hono';
import {ms} from 'itty-time';

function startWorkerMetricsServer(port: number): ServerType {
	const app = new Hono();
	app.get('/_metrics', createRegisteredMetricsHandler());
	return createServer(app, {port, onListen: (info) => Logger.info({port: info.port}, 'Worker metrics listening')});
}

function registerCronJobs(cron: CronScheduler, jobsStreamMaxAgeMs: number, kvClient: IKVProvider): void {
	cron.upsert('processAssetDeletionQueue', 'processAssetDeletionQueue', {}, '0 */5 * * * *', {ledger: false});
	if (Config.cachePurge.adapter !== 'none') {
		cron.upsert('processCachePurgeQueue', 'processCachePurgeQueue', {}, '*/10 * * * * *', {ledger: false});
	}
	cron.upsert('processPendingBulkMessageDeletions', 'processPendingBulkMessageDeletions', {}, '0 */10 * * * *', {
		ledger: false,
	});
	cron.upsert('userProcessPendingDeletions', 'userProcessPendingDeletions', {}, '0 * * * * *', {ledger: false});
	cron.upsert('processPremiumStateReconciliationQueue', 'processPremiumStateReconciliationQueue', {}, '0 * * * * *', {
		ledger: false,
		enabled: () => getStripeClient() !== null,
	});
	cron.upsert('scanBillingNotices', 'scanBillingNotices', {}, '0 15 9 * * *', {
		ledger: false,
		enabled: () => getStripeClient() !== null,
	});
	cron.upsert('processExpiredPremiumSweep', 'processExpiredPremiumSweep', {}, '0 0 * * * *', {ledger: false});
	cron.upsert('closePaymentDisputeAccounts', 'closePaymentDisputeAccounts', {}, '0 40 * * * *', {ledger: false});
	if (!Config.instance.selfHosted) {
		cron.upsert('processStorePurchaseRefreshQueue', 'processStorePurchaseRefreshQueue', {}, '0 */5 * * * *', {
			ledger: false,
		});
		cron.upsert('pollGooglePlayVoidedPurchases', 'pollGooglePlayVoidedPurchases', {}, '0 30 4 * * *', {ledger: false});
		cron.upsert('pollAppStoreNotificationHistory', 'pollAppStoreNotificationHistory', {}, '0 45 4 * * *', {
			ledger: false,
		});
		cron.upsert('operatorPassWaitlistCleanup', 'operatorPassWaitlistCleanup', {}, '0 20 * * * *', {ledger: false});
		cron.upsert('operatorPassRemovalExpiry', 'operatorPassRemovalExpiry', {}, '0 */5 * * * *', {ledger: false});
		cron.upsert('operatorPassWallRetention', 'operatorPassWallRetention', {}, '0 40 3 * * *', {ledger: false});
		cron.upsert('operatorPassConsentRetention', 'operatorPassConsentRetention', {}, '0 50 3 * * *', {ledger: false});
		cron.upsert('operatorPassLabsVisionaryRoleSync', 'operatorPassLabsVisionaryRoleSync', {}, '0 55 3 * * *', {
			ledger: false,
			enabled: () => Config.operatorPass.labsVisionaryRoleId !== null,
		});
		cron.upsert('operatorPassLabsGreeting', 'operatorPassLabsGreeting', {}, '15 */5 * * * *', {
			ledger: false,
			enabled: () => getLabsOperatorsChannelId() !== null,
		});
		cron.upsert('operatorPassLabsDelivery', 'operatorPassLabsDelivery', {}, '0 10 4 * * *', {
			ledger: false,
			enabled: isLabsAutoJoinConfigured,
		});
		cron.upsert('operatorPassOfferCompletion', 'operatorPassOfferCompletion', {}, '30 */5 * * * *', {ledger: false});
		cron.upsert('operatorPassFulfilmentSweep', 'operatorPassFulfilmentSweep', {}, '45 */15 * * * *', {
			ledger: false,
			enabled: () => getStripeClient() !== null,
		});
	}
	cron.upsert('processInactivityDeletions', 'processInactivityDeletions', {}, '0 0 */6 * * *', {ledger: false});
	cron.upsert('expireAttachments', 'expireAttachments', {}, '0 0 */12 * * *', {ledger: false});
	cron.upsert('expireReportSnapshots', 'expireReportSnapshots', {}, '0 15 5 * * *', {ledger: false});
	if (jobsStreamMaxAgeMs > 0 && jobsStreamMaxAgeMs <= JOBS_STREAM_MAX_AGE_MS) {
		cron.upsert('expireStaleJobs', 'expireStaleJobs', {}, '0 45 3 * * *', {ledger: false});
	} else {
		Logger.warn(
			{jobsStreamMaxAgeMs},
			'Jobs stream keeps jobs past 7 days, stale jobs stay active until their ledger rows expire',
		);
	}
	cron.upsert('prunePostgresKvTtl', 'prunePostgresKvTtl', {}, '0 */5 * * * *', {ledger: false});
	cron.upsert('sweepVoiceParticipants', 'sweepVoiceParticipants', {}, '30 * * * * *', {
		ledger: false,
		enabled: () => Config.voice.enabled && Config.voice.participantSweepEnabled,
	});
	cron.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *', {ledger: false});
	if (Config.blocklistFeeds.enabled) {
		cron.upsert('syncUrlBlocklists', 'syncUrlBlocklists', {}, '0 0 */6 * * *', {ledger: true});
		cron.upsert('syncFileShaBlocklists', 'syncFileShaBlocklists', {}, '0 0 */12 * * *', {ledger: true});
	}
	cron.upsert(
		'flushUserActivityBuffer',
		'flushUserActivityBuffer',
		{},
		Config.instance.selfHosted ? '0 * * * * *' : '*/10 * * * * *',
		{ledger: false},
	);
	cron.upsert('drainActivitySpool', 'drainActivitySpool', {}, '*/5 * * * * *', {
		ledger: false,
		enabled: activityEventsPublishing,
	});
	cron.upsert('joinSingleCommunityMembers', 'joinSingleCommunityMembers', {}, '0 * * * * *', {
		ledger: false,
		enabled: () => singleCommunityJoinPending(kvClient),
	});
	cron.upsert('liftExpiredGuildInvitePauses', 'liftExpiredGuildInvitePauses', {}, '0 * * * * *', {ledger: false});
	cron.upsert('archiveInactiveThreads', 'archiveInactiveThreads', {}, '0 * * * * *', {
		ledger: false,
		enabled: () => true,
	});
	Logger.info(
		{
			blocklistFeeds: Config.blocklistFeeds.enabled,
			cachePurgeAdapter: Config.cachePurge.adapter,
			selfHosted: Config.instance.selfHosted,
		},
		'Cron jobs registered successfully',
	);
}

export interface WorkerRuntime {
	stop(): Promise<void>;
	stalledComponents(): Array<string>;
}

interface WorkerRuntimeOptions {
	jsConnectionManager: JetStreamConnectionManager;
	queue: JetStreamWorkerQueue;
	workerService: WorkerService;
	jobLedger: JobLedgerRepository;
	dependencies: WorkerDependencies;
	cores: number;
}

export async function startWorkerRuntime(options: WorkerRuntimeOptions): Promise<WorkerRuntime> {
	const {jsConnectionManager, queue, workerService, jobLedger, dependencies} = options;
	const heartbeat = new WorkerHeartbeat({logger: Logger});
	const runners: Array<WorkerRunner> = [];
	let cron: CronScheduler | null = null;
	let stopPromise: Promise<void> | null = null;

	const cleanupStep = async (label: string, fn: () => Promise<void> | void): Promise<void> => {
		try {
			await fn();
		} catch (error) {
			Logger.error({err: error}, `Error during worker shutdown step: ${label}`);
		}
	};

	const performStop = async (): Promise<void> => {
		await cleanupStep('heartbeat', () => heartbeat.stop());
		await cleanupStep('cron', () => cron?.stop());
		await cleanupStep('account actions', stopAccountActionConsumer);
		await cleanupStep('runners', async () => {
			await awaitAll(
				runners.map((runner) => runner.stop()),
				'Failed to stop worker runners',
			);
		});
	};
	const runtime: WorkerRuntime = {
		stop: () => {
			stopPromise ??= performStop();
			return stopPromise;
		},
		stalledComponents: () => heartbeat.stalledComponents(),
	};

	try {
		validateLaneCompleteness(workerTasks);
		Logger.info('Worker lane configuration validated');
		const activeWorkerLanes = resolveWorkerLanes({
			mode: Config.worker.mode,
			laneName: Config.worker.laneName,
			taskName: Config.worker.taskName,
			laneConcurrencyOverrides: Config.worker.laneConcurrencyOverrides,
			availableParallelism: options.cores,
		});
		const cronSchedulerEnabled = resolveCronSchedulerEnabled(Config.worker.mode, Config.worker.enableCronScheduler);
		Logger.info(
			{
				workerMode: Config.worker.mode,
				workerLane: Config.worker.laneName,
				workerTask: Config.worker.taskName,
				lanes: activeWorkerLanes.map((lane) => `${lane.name}(${lane.concurrency})`).join(', '),
				cronSchedulerEnabled,
			},
			'Worker runtime configuration resolved',
		);
		await queue.ensureInfrastructure(activeWorkerLanes);
		Logger.info('JetStream stream and lane consumers verified');
		await queueBlocklistFeedStartupJobs(dependencies.kvClient, workerService, Config.blocklistFeeds.enabled);
		await queuePasskeyTraitBackfill(dependencies.kvClient, workerService);
		await queueUserAclIndexBackfill(dependencies.kvClient, workerService);
		await queueDeletedUsersBackfill(dependencies.kvClient, workerService);
		await queueHiddenProfileBadgesBackfill(dependencies.kvClient, workerService);
		await queueFavoriteMemeNsfwBackfill(dependencies.kvClient, workerService);
		try {
			const normalized = await getInstanceConfigRepository().normalizeStoredBrandingAssets(dependencies.storageService);
			if (normalized > 0) {
				Logger.info({normalized}, 'Normalised stored instance branding assets to references');
			}
		} catch (error) {
			Logger.warn({err: error}, 'Failed to normalise stored instance branding assets');
		}
		if (activeWorkerLanes.some((lane) => lane.name === 'lifecycle')) {
			const apiContext = createApiContext();
			startAccountActionConsumer({
				js: jsConnectionManager.getJetStreamClient(),
				state: accountStateDepsFromContext(
					apiContext,
					getAdminRepository(),
					{
						userCacheService: dependencies.userCacheService,
						guildRepository: dependencies.guildRepository,
					},
					{
						archives: getAdminArchiveService(),
						messageDeletionQueue: dependencies.bulkMessageDeletionQueueService,
						messageDeletionDelayMs: Config.automatedMessageDeletionDelayDays * ms('1 day'),
					},
					dependencies.channelRepository,
				),
				guildInvites: guildInvitePauseDeps(dependencies),
			});
			Logger.info('Account action consumer started');
		}
		cron = new CronScheduler(workerService, Logger, dependencies.kvClient, heartbeat);
		registerCronJobs(cron, queue.getJobsStreamMaxAgeMs(), dependencies.kvClient);
		for (const lane of activeWorkerLanes) {
			const laneTasks: Record<string, WorkerTaskHandler> = {};
			for (const taskType of lane.taskTypes) {
				const handler = workerTasks[taskType];
				if (handler) {
					laneTasks[taskType] = handler;
				}
			}
			const runner = new WorkerRunner({
				tasks: laneTasks,
				retiredTaskTypes: lane.retiredTaskTypes,
				queue,
				consumerName: lane.consumerName,
				laneName: lane.name,
				ledger: jobLedger,
				concurrency: lane.concurrency,
				maxDeliver: lane.maxDeliver,
				ackWaitMs: lane.ackWaitMs,
				heartbeat,
			});
			runners.push(runner);
		}
		if (cronSchedulerEnabled) {
			cron.start();
			Logger.info('Cron scheduler started');
		} else {
			Logger.info('Cron scheduler disabled for this worker process');
		}
		await Promise.all(runners.map((runner) => runner.start()));
		Logger.info(
			{lanes: activeWorkerLanes.map((l) => `${l.name}(${l.concurrency})`).join(', ')},
			'Worker runners started',
		);
		heartbeat.start();
		return runtime;
	} catch (error) {
		await runtime.stop();
		throw error;
	}
}

export async function startWorkerMain(): Promise<void> {
	Logger.info('Starting worker backend...');
	const host = sizeProcessForHost(Logger);
	let cassandraInitialized = false;
	let postgresInitialized = false;
	let jsConnectionManager: JetStreamConnectionManager | null = null;
	let snowflakeService: ISnowflakeService | null = null;
	let limitConfigService: LimitConfigService | null = null;
	let instanceConfigRepository: InstanceConfigRepository | null = null;
	let dependencies: WorkerDependencies | null = null;
	let runtime: WorkerRuntime | null = null;
	let metricsServer: ServerType | null = null;
	let searchInitialized = false;
	let shutdownPromise: Promise<void> | null = null;

	const cleanupStep = async (label: string, fn: () => Promise<void> | void): Promise<void> => {
		try {
			await fn();
		} catch (error) {
			Logger.error({err: error}, `Error during worker shutdown step: ${label}`);
		}
	};

	const performShutdown = async (): Promise<void> => {
		Logger.info('Shutting down worker backend...');
		const voiceShutdown = cleanupStep('voice resources', shutdownVoiceResources);
		await cleanupStep('metrics server', () => {
			metricsServer?.close();
			metricsServer = null;
		});
		await cleanupStep('worker runtime', () => runtime?.stop());
		await voiceShutdown;
		await cleanupStep('storage change feed', shutdownStorageChangeFeed);
		await cleanupStep('shared lists', stopSharedListWatch);
		await cleanupStep('activity events', () => shutdownActivityEvents());
		await cleanupStep('jetstream', async () => {
			await jsConnectionManager?.drain();
			jsConnectionManager = null;
		});
		await cleanupStep('content blocklist caches', stopContentBlocklistCaches);
		await cleanupStep('worker dependencies', () => {
			dependencies = null;
			clearWorkerDependencies();
			setInjectedWorkerService(undefined);
		});
		await cleanupStep('limit config', async () => {
			if (limitConfigService) {
				await limitConfigService.shutdown();
				limitConfigService = null;
			}
		});
		await cleanupStep('instance config repository', async () => {
			if (instanceConfigRepository) {
				await instanceConfigRepository.shutdown();
				instanceConfigRepository = null;
			}
		});
		await cleanupStep('search', async () => {
			if (searchInitialized) {
				await shutdownSearch();
				searchInitialized = false;
			}
		});
		await cleanupStep('cassandra', async () => {
			if (cassandraInitialized) {
				await shutdownCassandra();
				cassandraInitialized = false;
			}
		});
		await cleanupStep('postgres', async () => {
			if (postgresInitialized) {
				setDatabaseQueryExecutor(null);
				await shutdownPostgres();
				postgresInitialized = false;
			}
		});
		await cleanupStep('snowflake', async () => {
			if (snowflakeService) {
				await snowflakeService.shutdown();
				snowflakeService = null;
			}
			setInjectedSnowflakeService(undefined);
		});
		await cleanupStep('key-value client', closeOwnedKVClient);
	};
	const shutdown = (): Promise<void> => {
		shutdownPromise ??= Promise.resolve().then(performShutdown);
		return shutdownPromise;
	};

	try {
		if (Config.database.backend === 'postgres') {
			await initPostgres(
				{
					...Config.postgres,
					synchronousCommit: readOptionalEnv('FLUXER_WORKER_POSTGRES_SYNCHRONOUS_COMMIT')?.trim() ?? 'off',
				},
				(diagnostic) => Logger.error(diagnostic, 'Postgres connection error'),
			);
			postgresInitialized = true;
			const postgres = getDefaultPostgresClient();
			await ensurePostgresKvSchema(postgres);
			setDatabaseQueryExecutor(new PostgresKvQueryExecutor(postgres));
			Logger.info('Postgres KV client initialised for worker backend');
		}
		if (Config.database.backend === 'cassandra') {
			const {ensureCassandraSchema, logMissingCassandraKeyspace} = await import(
				'@app/api/database/CassandraSchemaStartup'
			);
			await initCassandra({
				hosts: Config.cassandra.hosts.split(',').filter(Boolean),
				port: Config.cassandra.port,
				keyspace: Config.cassandra.keyspace,
				localDc: Config.cassandra.localDc,
				username: Config.cassandra.username || undefined,
				password: Config.cassandra.password || undefined,
				readTimeoutMs: BACKGROUND_READ_TIMEOUT_MS,
			}).catch((error: unknown) => {
				if (Config.instance.selfHosted) {
					logMissingCassandraKeyspace(error, Config.cassandra.keyspace, Config.cassandra.localDc, Logger);
				}
				throw error;
			});
			cassandraInitialized = true;
			Logger.info('Cassandra client initialised for worker backend');
			if (Config.instance.selfHosted) {
				await ensureCassandraSchema({
					client: getClient(),
					keyspace: Config.cassandra.keyspace,
					kv: getKVClient(),
					logger: Logger,
				});
			}
		}
		snowflakeService = createSnowflakeService();
		await snowflakeService.initialize();
		setInjectedSnowflakeService(snowflakeService);
		Logger.info('Shared SnowflakeService initialised');
		jsConnectionManager = new JetStreamConnectionManager({
			url: Config.nats.jetStreamUrl,
			token: Config.nats.authToken || undefined,
			name: 'fluxer-worker',
		});
		await jsConnectionManager.connect();
		Logger.info('JetStream connection established');
		const queue = new JetStreamWorkerQueue(jsConnectionManager);
		const jobLedger = new JobLedgerRepository();
		const workerService = new WorkerService(queue, snowflakeService, jobLedger);
		setInjectedWorkerService(workerService);
		instanceConfigRepository = getInstanceConfigRepository();
		limitConfigService = getLimitConfigService();
		try {
			await initializeSearch(getCacheService());
			searchInitialized = true;
			Logger.info('Search initialised for worker backend');
		} catch (error) {
			Logger.error({err: error}, 'Search initialisation failed for worker backend');
			throw error;
		}
		dependencies = await initializeWorkerDependencies(snowflakeService);
		setWorkerDependencies(dependencies);
		await startContentBlocklistCaches({kvClient: dependencies.kvClient, storageService: dependencies.storageService});
		Logger.info('Content blocklist caches initialised for worker backend');
		setActivityProcessChannel('worker');
		await startActivityEvents({
			publisher: jetStreamActivityPublisher(jsConnectionManager.getJetStreamClient()),
			kv: dependencies.kvClient,
			jsm: await jsConnectionManager.getJetStreamManager(),
			spoolWhileMissing: !Config.instance.selfHosted,
		});
		startSharedListWatch(jsConnectionManager.getJetStreamClient(), {selfHosted: Config.instance.selfHosted});
		runtime = await startWorkerRuntime({
			jsConnectionManager,
			queue,
			workerService,
			jobLedger,
			dependencies,
			cores: host.cores,
		});
		if (Config.worker.metricsPort !== undefined) {
			metricsServer = startWorkerMetricsServer(Config.worker.metricsPort);
		}
		setupGracefulShutdown(shutdown, {logger: Logger, timeoutMs: 30000});
		const handleProcessError = createWorkerProcessErrorHandler({
			logger: Logger,
			shutdown,
			exit: (code) => {
				process.exit(code);
			},
		});
		process.on('uncaughtException', (error) => {
			void handleProcessError('uncaughtException', error);
		});
		process.on('unhandledRejection', (reason: unknown) => {
			void handleProcessError('unhandledRejection', reason);
		});
	} catch (error: unknown) {
		Logger.error({err: error}, 'Failed to start worker backend');
		await shutdown();
		process.exit(1);
	}
}