🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Enhance server clustering support and configuration (#866)
Goldflag22c31d53ec7d
diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.ymlindex 45d0124ea..473d46e33 100644--- a/docker-compose.cloud.yml+++ b/docker-compose.cloud.yml@@ -128,6 +128,7 @@ services:- GOOGLE_REDIRECT_URI=${GOOGLE_REDIRECT_URI}- OPENROUTER_API_KEY=${OPENROUTER_API_KEY}- OPENROUTER_MODEL=${OPENROUTER_MODEL}+ - CLUSTER_WORKERS=${CLUSTER_WORKERS:-4}depends_on:clickhouse:condition: service_healthydiff --git a/server/Dockerfile b/server/Dockerfileindex fb18f3b88..1d7d20ee2 100644--- a/server/Dockerfile+++ b/server/Dockerfile@@ -58,4 +58,4 @@ EXPOSE 3001# Use our custom entrypoint scriptENTRYPOINT ["/docker-entrypoint.sh"]-CMD ["node", "dist/index.js"]\ No newline at end of file+CMD ["node", "dist/cluster.js"]diff --git a/server/package.json b/server/package.jsonindex d0701389e..02e081dcc 100644--- a/server/package.json+++ b/server/package.json@@ -6,9 +6,11 @@"type": "module","scripts": {"dev": "tsc && node dist/index.js",+ "dev:cluster": "tsc && node dist/cluster.js","build": "tsc && npm run build:analytics","build:analytics": "cd src/analytics-script && node build.js","start": "node dist/index.js",+ "start:cluster": "node dist/cluster.js","test": "vitest","test:run": "vitest run","test:watch": "vitest --watch",diff --git a/server/src/cluster.ts b/server/src/cluster.tsnew file mode 100644index 000000000..a6deca581--- /dev/null+++ b/server/src/cluster.ts@@ -0,0 +1,143 @@+import cluster from "node:cluster";+import { initializeClickhouse } from "./db/clickhouse/clickhouse.js";+import { initPostgres } from "./db/postgres/initPostgres.js";+import { IS_CLOUD } from "./lib/const.js";+import { createServiceLogger } from "./lib/logger/logger.js";+import { reengagementService } from "./services/reengagement/reengagementService.js";+import { sessionsService } from "./services/sessions/sessionsService.js";+import { telemetryService } from "./services/telemetryService.js";+import { usageService } from "./services/usageService.js";+import { weeklyReportService } from "./services/weekyReports/weeklyReportService.js";++const logger = createServiceLogger("cluster");++// Determine worker count from environment variable+// Default to 0 (single-process mode) if not set. Max 4 workers.+const MAX_WORKERS = 4;+const requestedWorkers = process.env.CLUSTER_WORKERS;+const workerCount =+ requestedWorkers === undefined || requestedWorkers === ""+ ? 0+ : Math.min(parseInt(requestedWorkers, 10), MAX_WORKERS);++if (workerCount === 0) {+ // Single-process mode — no clustering, same as running index.ts directly+ logger.info("CLUSTER_WORKERS=0, running in single-process mode");+ import("./index.js");+} else if (cluster.isPrimary) {+ logger.info(`Primary process ${process.pid} starting with ${workerCount} workers`);++ // Initialize databases once before forking workers+ await Promise.all([initializeClickhouse(), initPostgres()]);+ logger.info("Database initialization complete");++ // Start cron jobs on the primary process only+ telemetryService.startTelemetryCron();+ sessionsService.startCleanupCron();+ usageService.startUsageCheckCron();+ if (IS_CLOUD && process.env.NODE_ENV !== "development") {+ weeklyReportService.startWeeklyReportCron();+ reengagementService.startReengagementCron();+ }++ // Broadcast sitesOverLimit to workers after each usage update+ usageService.onUsageUpdated(() => {+ broadcastSitesOverLimit();+ });++ // Fork workers+ let isShuttingDown = false;++ for (let i = 0; i < workerCount; i++) {+ cluster.fork();+ }++ // Send current sitesOverLimit to a worker when it comes online+ cluster.on("online", (worker) => {+ logger.info(`Worker ${worker.process.pid} is online`);+ const siteIds = Array.from(usageService.getSitesOverLimit());+ worker.send({ type: "sites-over-limit", siteIds });+ });++ // Auto-restart crashed workers (unless shutting down)+ cluster.on("exit", (worker, code, signal) => {+ if (isShuttingDown) {+ logger.info(`Worker ${worker.process.pid} exited during shutdown`);+ return;+ }+ logger.warn(+ `Worker ${worker.process.pid} died (code: ${code}, signal: ${signal}). Restarting...`+ );+ cluster.fork();+ });++ /**+ * Broadcast the current sitesOverLimit set to all alive workers via IPC+ */+ function broadcastSitesOverLimit() {+ const siteIds = Array.from(usageService.getSitesOverLimit());+ for (const id in cluster.workers) {+ const worker = cluster.workers[id];+ if (worker && !worker.isDead()) {+ worker.send({ type: "sites-over-limit", siteIds });+ }+ }+ logger.debug(`Broadcasted ${siteIds.length} sites-over-limit to workers`);+ }++ // Graceful shutdown+ const shutdown = async (signal: string) => {+ if (isShuttingDown) {+ logger.warn(`${signal} received during shutdown, forcing exit...`);+ process.exit(1);+ }++ isShuttingDown = true;+ logger.info(`${signal} received, shutting down gracefully...`);++ // Force exit timeout+ const forceExitTimeout = setTimeout(() => {+ logger.error("Shutdown timeout exceeded, forcing exit...");+ process.exit(1);+ }, 30000);++ // Stop cron jobs+ usageService.stopUsageCheckCron();+ sessionsService.stopCleanupCron();+ telemetryService.stopTelemetryCron();+ if (IS_CLOUD) {+ weeklyReportService.stopWeeklyReportCron();+ reengagementService.stopReengagementCron();+ }++ // Attach exit listeners before sending SIGTERM to avoid a race where+ // a worker exits before the listener is registered (which would hang shutdown).+ const activeWorkers = Object.values(cluster.workers ?? {}).filter(+ (w): w is NonNullable<typeof w> => w != null && !w.isDead()+ );++ const workerExitPromises = activeWorkers.map(+ (worker) =>+ new Promise<void>((resolve) => {+ worker.on("exit", () => resolve());+ })+ );++ // Now signal all workers to shut down+ for (const worker of activeWorkers) {+ worker.process.kill("SIGTERM");+ }++ await Promise.all(workerExitPromises);+ logger.info("All workers exited");++ clearTimeout(forceExitTimeout);+ process.exit(0);+ };++ process.on("SIGTERM", () => shutdown("SIGTERM"));+ process.on("SIGINT", () => shutdown("SIGINT"));+} else {+ // Worker process — start the HTTP server+ import("./index.js");+}diff --git a/server/src/db/postgres/postgres.ts b/server/src/db/postgres/postgres.tsindex 72a8f76e8..181d73087 100644--- a/server/src/db/postgres/postgres.ts+++ b/server/src/db/postgres/postgres.ts@@ -1,3 +1,4 @@+import cluster from "node:cluster";import dotenv from "dotenv";import { drizzle } from "drizzle-orm/postgres-js";import postgres from "postgres";@@ -5,6 +6,17 @@ import * as schema from "./schema.js";dotenv.config();+// Scale connection pool based on clustering mode.+// Primary gets 5 connections (cron/DB ops only). Each worker gets floor(20 / workerCount), min 5.+// Example with 4 workers: total = 4 × 5 (workers) + 5 (primary) = 25, well under PG's default 100.+// Single-process mode (workerCount=0): 20 connections, same as before clustering.+const workerCount = parseInt(process.env.CLUSTER_WORKERS || "0", 10);+const maxConnections = cluster.isWorker+ ? Math.max(5, Math.floor(20 / (workerCount || 1)))+ : cluster.isPrimary && workerCount > 0+ ? 5+ : 20;+// Create postgres connectionconst client = postgres({host: process.env.POSTGRES_HOST || "postgres",@@ -13,7 +25,7 @@ const client = postgres({username: process.env.POSTGRES_USER,password: process.env.POSTGRES_PASSWORD,onnotice: () => {},- max: 20,+ max: maxConnections,});// Create drizzle ORM instancediff --git a/server/src/index.ts b/server/src/index.tsindex cf57e68ef..8218d1831 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -1,3 +1,4 @@+import cluster from "node:cluster";import cors from "@fastify/cors";import fastifyStatic from "@fastify/static";import { toNodeHandler } from "better-auth/node";@@ -126,9 +127,11 @@ import { mapHeaders } from "./lib/auth-utils.js";import { auth } from "./lib/auth.js";import { IS_CLOUD } from "./lib/const.js";import { reengagementService } from "./services/reengagement/reengagementService.js";+import { sessionsService } from "./services/sessions/sessionsService.js";import { telemetryService } from "./services/telemetryService.js";import { handleIdentify } from "./services/tracker/identifyService.js";import { trackEvent } from "./services/tracker/trackEvent.js";+import { usageService } from "./services/usageService.js";import { weeklyReportService } from "./services/weekyReports/weeklyReportService.js";// Pre-composed middleware chains for common auth patterns@@ -396,17 +399,35 @@ server.register(apiRoutes, { prefix: "/api" });const start = async () => {try {- await Promise.all([initializeClickhouse(), initPostgres()]);+ // When running as a cluster worker, the primary process already initialized the databases+ if (!cluster.isWorker) {+ await Promise.all([initializeClickhouse(), initPostgres()]);+ }- telemetryService.startTelemetryCron();- if (IS_CLOUD && process.env.NODE_ENV !== "development") {- weeklyReportService.startWeeklyReportCron();- reengagementService.startReengagementCron();+ // Cron jobs should only run on the primary process (or in single-process mode)+ if (!cluster.isWorker) {+ telemetryService.startTelemetryCron();+ sessionsService.startCleanupCron();+ usageService.startUsageCheckCron();+ if (IS_CLOUD && process.env.NODE_ENV !== "development") {+ weeklyReportService.startWeeklyReportCron();+ reengagementService.startReengagementCron();+ }}// Start the server firstawait server.listen({ port: 3001, host: "0.0.0.0" });- server.log.info("Server is listening on http://0.0.0.0:3001");+ server.log.info(`Server is listening on http://0.0.0.0:3001 (PID: ${process.pid})`);++ // Listen for IPC messages from the cluster primary process+ if (cluster.isWorker) {+ process.on("message", (message: { type: string; siteIds: number[] }) => {+ if (message?.type === "sites-over-limit") {+ usageService.setSitesOverLimit(new Set(message.siteIds));+ server.log.debug(`Received ${message.siteIds.length} sites-over-limit from primary`);+ }+ });+ }// if (process.env.NODE_ENV === "production") {// // Initialize uptime monitoring service in the background (non-blocking)diff --git a/server/src/services/import/importQuotaManager.ts b/server/src/services/import/importQuotaManager.tsindex 8e445cd9c..fb52a7510 100644--- a/server/src/services/import/importQuotaManager.ts+++ b/server/src/services/import/importQuotaManager.ts@@ -1,3 +1,4 @@+import cluster from "node:cluster";import { ImportQuotaTracker } from "./importQuotaTracker.js";import { IS_CLOUD } from "../../lib/const.js";@@ -87,6 +88,6 @@ class ImportQuotaManager {export const importQuotaManager = new ImportQuotaManager();// Background cleanup loop-if (IS_CLOUD) {+if (IS_CLOUD && !cluster.isWorker) {setInterval(() => importQuotaManager.cleanup(), 15 * 60 * 1000);}diff --git a/server/src/services/sessions/sessionsService.ts b/server/src/services/sessions/sessionsService.tsindex 29aa409e9..c27ff8a7d 100644--- a/server/src/services/sessions/sessionsService.ts+++ b/server/src/services/sessions/sessionsService.ts@@ -9,9 +9,7 @@ class SessionsService {private cleanupTask: cron.ScheduledTask | null = null;private logger = createServiceLogger("sessions");- constructor() {- this.initializeCleanupCron();- }+ constructor() {}private initializeCleanupCron() {this.cleanupTask = cron.schedule(@@ -79,6 +77,10 @@ class SessionsService {}// Method to stop the cleanup cron job (useful for graceful shutdown)+ startCleanupCron() {+ this.initializeCleanupCron();+ }+stopCleanupCron() {if (this.cleanupTask) {this.cleanupTask.stop();diff --git a/server/src/services/usageService.ts b/server/src/services/usageService.tsindex 251c49b12..776308d87 100644--- a/server/src/services/usageService.ts+++ b/server/src/services/usageService.ts@@ -10,13 +10,29 @@ import { sendLimitExceededEmail } from "../lib/email/email.js";import { createServiceLogger } from "../lib/logger/logger.js";import { getBestSubscription } from "../lib/subscriptionUtils.js";+type UsageUpdateCallback = () => void;+class UsageService {private sitesOverLimit = new Set<number>();private usageCheckTask: cron.ScheduledTask | null = null;private logger = createServiceLogger("usage-checker");+ private onUsageUpdatedCallbacks: UsageUpdateCallback[] = [];- constructor() {- this.initializeUsageCheckCron();+ constructor() {}++ /**+ * Sets the sitesOverLimit set (used by workers receiving IPC updates from primary)+ */+ public setSitesOverLimit(sites: Set<number>): void {+ this.sitesOverLimit = sites;+ }++ /**+ * Register a callback to be invoked after usage data is updated.+ * Used by the cluster primary to broadcast sitesOverLimit to workers.+ */+ public onUsageUpdated(callback: UsageUpdateCallback): void {+ this.onUsageUpdatedCallbacks.push(callback);}/**@@ -288,11 +304,23 @@ class UsageService {}this.logger.info(`Completed monthly event usage check. ${this.sitesOverLimit.size} sites are over their limit.`);++ // Notify listeners (e.g., cluster primary broadcasts to workers)+ for (const callback of this.onUsageUpdatedCallbacks) {+ callback();+ }} catch (error) {this.logger.error(error as Error, "Error updating monthly usage");}}+ /**+ * Method to start the usage check cron job+ */+ public startUsageCheckCron() {+ this.initializeUsageCheckCron();+ }+/*** Method to stop the usage check cron job (useful for graceful shutdown)*/