Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Enhance server clustering support and configuration (#866)

Goldflag22c31d53ec7d
diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.yml
index 45d0124ea..473d46e33 100644
--- a/docker-compose.cloud.yml
+++ b/docker-compose.cloud.yml
@@ -128,6 +128,7 @@ services:
- GOOGLE_REDIRECT_URI=${GOOGLE_REDIRECT_URI}
- OPENROUTER_API_KEY=${OPENROUTER_API_KEY}
- OPENROUTER_MODEL=${OPENROUTER_MODEL}
+ - CLUSTER_WORKERS=${CLUSTER_WORKERS:-4}
depends_on:
clickhouse:
condition: service_healthy
diff --git a/server/Dockerfile b/server/Dockerfile
index fb18f3b88..1d7d20ee2 100644
--- a/server/Dockerfile
+++ b/server/Dockerfile
@@ -58,4 +58,4 @@ EXPOSE 3001
# Use our custom entrypoint script
ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["node", "dist/index.js"]
\ No newline at end of file
+CMD ["node", "dist/cluster.js"]
diff --git a/server/package.json b/server/package.json
index d0701389e..02e081dcc 100644
--- a/server/package.json
+++ b/server/package.json
@@ -6,9 +6,11 @@
"type": "module",
"scripts": {
"dev": "tsc && node dist/index.js",
+ "dev:cluster": "tsc && node dist/cluster.js",
"build": "tsc && npm run build:analytics",
"build:analytics": "cd src/analytics-script && node build.js",
"start": "node dist/index.js",
+ "start:cluster": "node dist/cluster.js",
"test": "vitest",
"test:run": "vitest run",
"test:watch": "vitest --watch",
diff --git a/server/src/cluster.ts b/server/src/cluster.ts
new file mode 100644
index 000000000..a6deca581
--- /dev/null
+++ b/server/src/cluster.ts
@@ -0,0 +1,143 @@
+import cluster from "node:cluster";
+import { initializeClickhouse } from "./db/clickhouse/clickhouse.js";
+import { initPostgres } from "./db/postgres/initPostgres.js";
+import { IS_CLOUD } from "./lib/const.js";
+import { createServiceLogger } from "./lib/logger/logger.js";
+import { reengagementService } from "./services/reengagement/reengagementService.js";
+import { sessionsService } from "./services/sessions/sessionsService.js";
+import { telemetryService } from "./services/telemetryService.js";
+import { usageService } from "./services/usageService.js";
+import { weeklyReportService } from "./services/weekyReports/weeklyReportService.js";
+
+const logger = createServiceLogger("cluster");
+
+// Determine worker count from environment variable
+// Default to 0 (single-process mode) if not set. Max 4 workers.
+const MAX_WORKERS = 4;
+const requestedWorkers = process.env.CLUSTER_WORKERS;
+const workerCount =
+ requestedWorkers === undefined || requestedWorkers === ""
+ ? 0
+ : Math.min(parseInt(requestedWorkers, 10), MAX_WORKERS);
+
+if (workerCount === 0) {
+ // Single-process mode — no clustering, same as running index.ts directly
+ logger.info("CLUSTER_WORKERS=0, running in single-process mode");
+ import("./index.js");
+} else if (cluster.isPrimary) {
+ logger.info(`Primary process ${process.pid} starting with ${workerCount} workers`);
+
+ // Initialize databases once before forking workers
+ await Promise.all([initializeClickhouse(), initPostgres()]);
+ logger.info("Database initialization complete");
+
+ // Start cron jobs on the primary process only
+ telemetryService.startTelemetryCron();
+ sessionsService.startCleanupCron();
+ usageService.startUsageCheckCron();
+ if (IS_CLOUD && process.env.NODE_ENV !== "development") {
+ weeklyReportService.startWeeklyReportCron();
+ reengagementService.startReengagementCron();
+ }
+
+ // Broadcast sitesOverLimit to workers after each usage update
+ usageService.onUsageUpdated(() => {
+ broadcastSitesOverLimit();
+ });
+
+ // Fork workers
+ let isShuttingDown = false;
+
+ for (let i = 0; i < workerCount; i++) {
+ cluster.fork();
+ }
+
+ // Send current sitesOverLimit to a worker when it comes online
+ cluster.on("online", (worker) => {
+ logger.info(`Worker ${worker.process.pid} is online`);
+ const siteIds = Array.from(usageService.getSitesOverLimit());
+ worker.send({ type: "sites-over-limit", siteIds });
+ });
+
+ // Auto-restart crashed workers (unless shutting down)
+ cluster.on("exit", (worker, code, signal) => {
+ if (isShuttingDown) {
+ logger.info(`Worker ${worker.process.pid} exited during shutdown`);
+ return;
+ }
+ logger.warn(
+ `Worker ${worker.process.pid} died (code: ${code}, signal: ${signal}). Restarting...`
+ );
+ cluster.fork();
+ });
+
+ /**
+ * Broadcast the current sitesOverLimit set to all alive workers via IPC
+ */
+ function broadcastSitesOverLimit() {
+ const siteIds = Array.from(usageService.getSitesOverLimit());
+ for (const id in cluster.workers) {
+ const worker = cluster.workers[id];
+ if (worker && !worker.isDead()) {
+ worker.send({ type: "sites-over-limit", siteIds });
+ }
+ }
+ logger.debug(`Broadcasted ${siteIds.length} sites-over-limit to workers`);
+ }
+
+ // Graceful shutdown
+ const shutdown = async (signal: string) => {
+ if (isShuttingDown) {
+ logger.warn(`${signal} received during shutdown, forcing exit...`);
+ process.exit(1);
+ }
+
+ isShuttingDown = true;
+ logger.info(`${signal} received, shutting down gracefully...`);
+
+ // Force exit timeout
+ const forceExitTimeout = setTimeout(() => {
+ logger.error("Shutdown timeout exceeded, forcing exit...");
+ process.exit(1);
+ }, 30000);
+
+ // Stop cron jobs
+ usageService.stopUsageCheckCron();
+ sessionsService.stopCleanupCron();
+ telemetryService.stopTelemetryCron();
+ if (IS_CLOUD) {
+ weeklyReportService.stopWeeklyReportCron();
+ reengagementService.stopReengagementCron();
+ }
+
+ // Attach exit listeners before sending SIGTERM to avoid a race where
+ // a worker exits before the listener is registered (which would hang shutdown).
+ const activeWorkers = Object.values(cluster.workers ?? {}).filter(
+ (w): w is NonNullable<typeof w> => w != null && !w.isDead()
+ );
+
+ const workerExitPromises = activeWorkers.map(
+ (worker) =>
+ new Promise<void>((resolve) => {
+ worker.on("exit", () => resolve());
+ })
+ );
+
+ // Now signal all workers to shut down
+ for (const worker of activeWorkers) {
+ worker.process.kill("SIGTERM");
+ }
+
+ await Promise.all(workerExitPromises);
+ logger.info("All workers exited");
+
+ clearTimeout(forceExitTimeout);
+ process.exit(0);
+ };
+
+ process.on("SIGTERM", () => shutdown("SIGTERM"));
+ process.on("SIGINT", () => shutdown("SIGINT"));
+} else {
+ // Worker process — start the HTTP server
+ import("./index.js");
+}
diff --git a/server/src/db/postgres/postgres.ts b/server/src/db/postgres/postgres.ts
index 72a8f76e8..181d73087 100644
--- a/server/src/db/postgres/postgres.ts
+++ b/server/src/db/postgres/postgres.ts
@@ -1,3 +1,4 @@
+import cluster from "node:cluster";
import dotenv from "dotenv";
import { drizzle } from "drizzle-orm/postgres-js";
import postgres from "postgres";
@@ -5,6 +6,17 @@ import * as schema from "./schema.js";
dotenv.config();
+// Scale connection pool based on clustering mode.
+// Primary gets 5 connections (cron/DB ops only). Each worker gets floor(20 / workerCount), min 5.
+// Example with 4 workers: total = 4 × 5 (workers) + 5 (primary) = 25, well under PG's default 100.
+// Single-process mode (workerCount=0): 20 connections, same as before clustering.
+const workerCount = parseInt(process.env.CLUSTER_WORKERS || "0", 10);
+const maxConnections = cluster.isWorker
+ ? Math.max(5, Math.floor(20 / (workerCount || 1)))
+ : cluster.isPrimary && workerCount > 0
+ ? 5
+ : 20;
+
// Create postgres connection
const client = postgres({
host: process.env.POSTGRES_HOST || "postgres",
@@ -13,7 +25,7 @@ const client = postgres({
username: process.env.POSTGRES_USER,
password: process.env.POSTGRES_PASSWORD,
onnotice: () => {},
- max: 20,
+ max: maxConnections,
});
// Create drizzle ORM instance
diff --git a/server/src/index.ts b/server/src/index.ts
index cf57e68ef..8218d1831 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -1,3 +1,4 @@
+import cluster from "node:cluster";
import cors from "@fastify/cors";
import fastifyStatic from "@fastify/static";
import { toNodeHandler } from "better-auth/node";
@@ -126,9 +127,11 @@ import { mapHeaders } from "./lib/auth-utils.js";
import { auth } from "./lib/auth.js";
import { IS_CLOUD } from "./lib/const.js";
import { reengagementService } from "./services/reengagement/reengagementService.js";
+import { sessionsService } from "./services/sessions/sessionsService.js";
import { telemetryService } from "./services/telemetryService.js";
import { handleIdentify } from "./services/tracker/identifyService.js";
import { trackEvent } from "./services/tracker/trackEvent.js";
+import { usageService } from "./services/usageService.js";
import { weeklyReportService } from "./services/weekyReports/weeklyReportService.js";
// Pre-composed middleware chains for common auth patterns
@@ -396,17 +399,35 @@ server.register(apiRoutes, { prefix: "/api" });
const start = async () => {
try {
- await Promise.all([initializeClickhouse(), initPostgres()]);
+ // When running as a cluster worker, the primary process already initialized the databases
+ if (!cluster.isWorker) {
+ await Promise.all([initializeClickhouse(), initPostgres()]);
+ }
- telemetryService.startTelemetryCron();
- if (IS_CLOUD && process.env.NODE_ENV !== "development") {
- weeklyReportService.startWeeklyReportCron();
- reengagementService.startReengagementCron();
+ // Cron jobs should only run on the primary process (or in single-process mode)
+ if (!cluster.isWorker) {
+ telemetryService.startTelemetryCron();
+ sessionsService.startCleanupCron();
+ usageService.startUsageCheckCron();
+ if (IS_CLOUD && process.env.NODE_ENV !== "development") {
+ weeklyReportService.startWeeklyReportCron();
+ reengagementService.startReengagementCron();
+ }
}
// Start the server first
await server.listen({ port: 3001, host: "0.0.0.0" });
- server.log.info("Server is listening on http://0.0.0.0:3001");
+ server.log.info(`Server is listening on http://0.0.0.0:3001 (PID: ${process.pid})`);
+
+ // Listen for IPC messages from the cluster primary process
+ if (cluster.isWorker) {
+ process.on("message", (message: { type: string; siteIds: number[] }) => {
+ if (message?.type === "sites-over-limit") {
+ usageService.setSitesOverLimit(new Set(message.siteIds));
+ server.log.debug(`Received ${message.siteIds.length} sites-over-limit from primary`);
+ }
+ });
+ }
// if (process.env.NODE_ENV === "production") {
// // Initialize uptime monitoring service in the background (non-blocking)
diff --git a/server/src/services/import/importQuotaManager.ts b/server/src/services/import/importQuotaManager.ts
index 8e445cd9c..fb52a7510 100644
--- a/server/src/services/import/importQuotaManager.ts
+++ b/server/src/services/import/importQuotaManager.ts
@@ -1,3 +1,4 @@
+import cluster from "node:cluster";
import { ImportQuotaTracker } from "./importQuotaTracker.js";
import { IS_CLOUD } from "../../lib/const.js";
@@ -87,6 +88,6 @@ class ImportQuotaManager {
export const importQuotaManager = new ImportQuotaManager();
// Background cleanup loop
-if (IS_CLOUD) {
+if (IS_CLOUD && !cluster.isWorker) {
setInterval(() => importQuotaManager.cleanup(), 15 * 60 * 1000);
}
diff --git a/server/src/services/sessions/sessionsService.ts b/server/src/services/sessions/sessionsService.ts
index 29aa409e9..c27ff8a7d 100644
--- a/server/src/services/sessions/sessionsService.ts
+++ b/server/src/services/sessions/sessionsService.ts
@@ -9,9 +9,7 @@ class SessionsService {
private cleanupTask: cron.ScheduledTask | null = null;
private logger = createServiceLogger("sessions");
- constructor() {
- this.initializeCleanupCron();
- }
+ constructor() {}
private initializeCleanupCron() {
this.cleanupTask = cron.schedule(
@@ -79,6 +77,10 @@ class SessionsService {
}
// Method to stop the cleanup cron job (useful for graceful shutdown)
+ startCleanupCron() {
+ this.initializeCleanupCron();
+ }
+
stopCleanupCron() {
if (this.cleanupTask) {
this.cleanupTask.stop();
diff --git a/server/src/services/usageService.ts b/server/src/services/usageService.ts
index 251c49b12..776308d87 100644
--- a/server/src/services/usageService.ts
+++ b/server/src/services/usageService.ts
@@ -10,13 +10,29 @@ import { sendLimitExceededEmail } from "../lib/email/email.js";
import { createServiceLogger } from "../lib/logger/logger.js";
import { getBestSubscription } from "../lib/subscriptionUtils.js";
+type UsageUpdateCallback = () => void;
+
class UsageService {
private sitesOverLimit = new Set<number>();
private usageCheckTask: cron.ScheduledTask | null = null;
private logger = createServiceLogger("usage-checker");
+ private onUsageUpdatedCallbacks: UsageUpdateCallback[] = [];
- constructor() {
- this.initializeUsageCheckCron();
+ constructor() {}
+
+ /**
+ * Sets the sitesOverLimit set (used by workers receiving IPC updates from primary)
+ */
+ public setSitesOverLimit(sites: Set<number>): void {
+ this.sitesOverLimit = sites;
+ }
+
+ /**
+ * Register a callback to be invoked after usage data is updated.
+ * Used by the cluster primary to broadcast sitesOverLimit to workers.
+ */
+ public onUsageUpdated(callback: UsageUpdateCallback): void {
+ this.onUsageUpdatedCallbacks.push(callback);
}
/**
@@ -288,11 +304,23 @@ class UsageService {
}
this.logger.info(`Completed monthly event usage check. ${this.sitesOverLimit.size} sites are over their limit.`);
+
+ // Notify listeners (e.g., cluster primary broadcasts to workers)
+ for (const callback of this.onUsageUpdatedCallbacks) {
+ callback();
+ }
} catch (error) {
this.logger.error(error as Error, "Error updating monthly usage");
}
}
+ /**
+ * Method to start the usage check cron job
+ */
+ public startUsageCheckCron() {
+ this.initializeUsageCheckCron();
+ }
+
/**
* Method to stop the usage check cron job (useful for graceful shutdown)
*/