Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

feat(api): run worker lanes and cron in the api when embedded

Hampusfca0d16aa454
diff --git a/fluxer_api/src/App.ts b/fluxer_api/src/App.ts
index c0585e472..358af73f6 100644
--- a/fluxer_api/src/App.ts
+++ b/fluxer_api/src/App.ts
@@ -6,6 +6,7 @@ import {listenOperatorSocket} from '@app/api/admin/operator/OperatorSocket';
import {buildAPIServerOptions, initializeConfig} from '@app/api/Config';
import {sizeProcessForHost} from '@app/api/infrastructure/HostResources';
import {initializeLogger} from '@app/api/Logger';
+import {startEmbeddedWorker, stopEmbeddedWorker} from '@app/api/worker/EmbeddedWorker';
import {Config} from '@app/Config';
import {Logger} from '@app/Logger';
import {createServer, setupGracefulShutdown} from '@fluxer/hono/src/Server';
@@ -25,7 +26,7 @@ async function closeHttpServer(server: {close: (callback: (error?: Error) => voi
async function main(): Promise<void> {
initializeConfig(Config);
initializeLogger(Logger);
- sizeProcessForHost(Logger);
+ const host = sizeProcessForHost(Logger);
const {app, initialize, shutdown} = await createAPIApp({
config: Config,
logger: Logger,
@@ -40,16 +41,28 @@ async function main(): Promise<void> {
const server = createServer(app, buildAPIServerOptions(Config));
Logger.info({port: Config.port}, `Starting Fluxer API on port ${Config.port}`);
const operatorServer = await listenOperatorSocket(app, OPERATOR_SOCKET_PATH, Logger);
- setupGracefulShutdown(
- async () => {
- await closeHttpServer(server);
- if (operatorServer) {
- await closeHttpServer(operatorServer);
- }
- await shutdown();
- },
- {logger: Logger, timeoutMs: 30000},
- );
+ const stop = async (): Promise<void> => {
+ await Promise.all([
+ (async () => {
+ await closeHttpServer(server);
+ if (operatorServer) {
+ await closeHttpServer(operatorServer);
+ }
+ })(),
+ stopEmbeddedWorker(),
+ ]);
+ await shutdown();
+ };
+ setupGracefulShutdown(stop, {logger: Logger, timeoutMs: 30000});
+ if (Config.worker.embedded) {
+ try {
+ await startEmbeddedWorker(host.cores);
+ } catch (error) {
+ Logger.fatal({error}, 'Failed to start the embedded worker');
+ await stop();
+ process.exit(1);
+ }
+ }
}
main().catch((err) => {
diff --git a/fluxer_api/src/api/app/APILifecycle.ts b/fluxer_api/src/api/app/APILifecycle.ts
index 7a883ce22..b9bca9101 100644
--- a/fluxer_api/src/api/app/APILifecycle.ts
+++ b/fluxer_api/src/api/app/APILifecycle.ts
@@ -54,6 +54,7 @@ import {JetStreamConnectionManager} from '@pkgs/nats/src/JetStreamConnectionMana
import {getDefaultPostgresClient, initPostgres, shutdownPostgres} from '@pkgs/postgres/src/Client';
let jsConnectionManager: JetStreamConnectionManager | null = null;
+let jobQueue: ApiJobQueue | null = null;
let searchInitRetryTimer: ReturnType<typeof setTimeout> | null = null;
let searchInitStopped = false;
let searchInitInFlight: Promise<void> | null = null;
@@ -101,6 +102,17 @@ async function shutdownRefreshCaches(logger: LoggerInterface): Promise<void> {
}
}
+interface ApiJobQueue {
+ jsConnectionManager: JetStreamConnectionManager;
+ queue: JetStreamWorkerQueue;
+ workerService: WorkerService;
+ jobLedger: JobLedgerRepository;
+}
+
+export function getApiJobQueue(): ApiJobQueue | null {
+ return jobQueue;
+}
+
function unsupportedDatabaseBackend(backend: never): never {
throw new Error(`Unsupported database backend during shutdown: ${String(backend)}`);
}
@@ -253,9 +265,12 @@ export function createInitializer(
name: 'api-worker',
});
await jsConnectionManager.connect();
- const workerQueue = new JetStreamWorkerQueue(jsConnectionManager);
- await workerQueue.ensureStream();
- setInjectedWorkerService(new WorkerService(workerQueue, getSnowflakeService(), new JobLedgerRepository()));
+ const queue = new JetStreamWorkerQueue(jsConnectionManager);
+ await queue.ensureStream();
+ const jobLedger = new JobLedgerRepository();
+ const workerService = new WorkerService(queue, getSnowflakeService(), jobLedger);
+ setInjectedWorkerService(workerService);
+ jobQueue = {jsConnectionManager, queue, workerService, jobLedger};
logger.info('JetStream worker service initialized');
const connection = jsConnectionManager;
await startActivityEvents({
@@ -328,6 +343,7 @@ export function createShutdown(config: APIConfig, logger: LoggerInterface): () =
stopSharedListWatch();
for (const unregister of unregisterMetricsSections.splice(0)) unregister();
await shutdownActivityEvents();
+ jobQueue = null;
if (jsConnectionManager) {
try {
await jsConnectionManager.drain();
diff --git a/fluxer_api/src/api/app/MiddlewarePipeline.ts b/fluxer_api/src/api/app/MiddlewarePipeline.ts
index 7955318b2..b631a7a1d 100644
--- a/fluxer_api/src/api/app/MiddlewarePipeline.ts
+++ b/fluxer_api/src/api/app/MiddlewarePipeline.ts
@@ -18,6 +18,7 @@ import {TrustedClientIpHeaderMiddleware} from '@app/api/middleware/TrustedClient
import {UserMiddleware} from '@app/api/middleware/UserMiddleware';
import type {HonoApp} from '@app/api/types/HonoEnv';
import {PeerAddressMiddleware} from '@app/api/utils/RequestClientIp';
+import {embeddedWorkerStalledComponents} from '@app/api/worker/EmbeddedWorker';
import {Headers as HttpHeaders} from '@fluxer/constants/src/Headers';
import {InvalidApiOriginError} from '@fluxer/errors/src/domains/core/InvalidApiOriginError';
import {cors} from '@fluxer/hono/src/middleware/Cors';
@@ -79,7 +80,9 @@ export function configureMiddleware(routes: HonoApp, options: MiddlewarePipeline
},
});
routes.use(ConcurrencyLimitMiddleware({maxInflightRequests}));
- routes.get('/_health', async (ctx) => ctx.text('OK'));
+ routes.get('/_health', async (ctx) =>
+ embeddedWorkerStalledComponents().length > 0 ? ctx.text('Worker stalled', 503) : ctx.text('OK'),
+ );
routes.use(IpBanMiddleware);
routes.use(
requestLogger({
diff --git a/fluxer_api/src/api/worker/EmbeddedWorker.ts b/fluxer_api/src/api/worker/EmbeddedWorker.ts
new file mode 100644
index 000000000..3903ff1ac
--- /dev/null
+++ b/fluxer_api/src/api/worker/EmbeddedWorker.ts
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {getApiJobQueue} from '@app/api/app/APILifecycle';
+import {Logger} from '@app/api/Logger';
+import {getSnowflakeService} from '@app/api/middleware/ServiceRegistry';
+import {clearWorkerDependencies, setWorkerDependencies} from '@app/api/worker/WorkerContext';
+import type {WorkerRuntime} from '@app/api/worker/WorkerMain';
+
+let starting: Promise<WorkerRuntime | null> | null = null;
+let stopping: Promise<void> | null = null;
+let runtime: WorkerRuntime | null = null;
+
+async function start(cores: number): Promise<WorkerRuntime | null> {
+ const jobs = getApiJobQueue();
+ if (!jobs) {
+ throw new Error('The embedded worker needs the API JetStream connection');
+ }
+ const [{initializeWorkerDependencies}, {startWorkerRuntime}] = await Promise.all([
+ import('@app/api/worker/WorkerDependencies'),
+ import('@app/api/worker/WorkerMain'),
+ ]);
+ if (stopping) return null;
+ const dependencies = await initializeWorkerDependencies(getSnowflakeService());
+ if (stopping) return null;
+ setWorkerDependencies(dependencies);
+ const started = await startWorkerRuntime({...jobs, dependencies, cores});
+ runtime = started;
+ Logger.info('Embedded worker started');
+ return started;
+}
+
+export function startEmbeddedWorker(cores: number): Promise<void> {
+ starting ??= start(cores);
+ return starting.then(() => undefined);
+}
+
+export function stopEmbeddedWorker(): Promise<void> {
+ stopping ??= (async () => {
+ if (!starting) return;
+ const [result] = await Promise.allSettled([starting]);
+ runtime = null;
+ if (result.status === 'fulfilled' && result.value) {
+ await result.value.stop();
+ }
+ clearWorkerDependencies();
+ Logger.info('Embedded worker stopped');
+ })();
+ return stopping;
+}
+
+export function embeddedWorkerStalledComponents(): Array<string> {
+ return runtime?.stalledComponents() ?? [];
+}