Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix(api): workers answer health checks while they start up

Carl-Gerhard Lindesvärd6e877b4b729f
diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts
index ef5a89d92..20cf9cf0d 100644
--- a/apps/api/src/main.ts
+++ b/apps/api/src/main.ts
@@ -50,6 +50,7 @@ import {
registerSessionScrapeMetrics,
requestLogging,
type SessionMetricsRedis,
+ setBooting,
setShuttingDown,
startKafkaEventsConsumer,
startSchedulers,
@@ -427,6 +428,29 @@ async function main() {
const app = await buildHttpApp(deps);
+ // Listen before the consumers start: joining the Kafka group can take most
+ // of a minute, and a liveness probe that cannot connect meanwhile gets the
+ // container killed. Readiness stays 503 until boot finishes.
+ setBooting(true);
+ const listenOptions = {
+ ...config.listen,
+ idleTimeout: HTTP_IDLE_TIMEOUT_SECONDS,
+ };
+ app.listen(listenOptions, () => {
+ logger.info(
+ {
+ role,
+ port: config.API_PORT,
+ hostname: config.listen.hostname ?? '0.0.0.0',
+ // The running Bun version, asserted against .bun-version by
+ // scripts/doctor.sh — logged so a wrong-runtime incident is one log
+ // line away rather than an inference.
+ bunVersion: Bun.version,
+ },
+ 'API listening'
+ );
+ });
+
let workers: WorkerHandle | undefined;
let consumer: KafkaConsumerHandle | undefined;
@@ -529,24 +553,8 @@ async function main() {
shutdown('SIGINT');
});
- const listenOptions = {
- ...config.listen,
- idleTimeout: HTTP_IDLE_TIMEOUT_SECONDS,
- };
- app.listen(listenOptions, () => {
- logger.info(
- {
- role,
- port: config.API_PORT,
- hostname: config.listen.hostname ?? '0.0.0.0',
- // The running Bun version, asserted against .bun-version by
- // scripts/doctor.sh — logged so a wrong-runtime incident is one log
- // line away rather than an inference.
- bunVersion: Bun.version,
- },
- 'API listening'
- );
- });
+ setBooting(false);
+ logger.info({ role }, 'Boot complete');
// The expiry notifications only matter where something subscribes.
if (role !== 'worker') {
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index d835412ce..7b7b2bf59 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -160,7 +160,7 @@ export {
} from './modules/event/event.service';
export type { IServiceGroup } from './modules/group/group.service';
export { getGscCannibalization } from './modules/gsc/gsc.service';
-export { setShuttingDown } from './modules/health/src/shutdown';
+export { setBooting, setShuttingDown } from './modules/health/src/shutdown';
export type {
BotMatch,
IncomingEventPayload,
diff --git a/packages/core/src/modules/health/src/readiness.test.ts b/packages/core/src/modules/health/src/readiness.test.ts
index c2cbeaa04..7e6bd6c19 100644
--- a/packages/core/src/modules/health/src/readiness.test.ts
+++ b/packages/core/src/modules/health/src/readiness.test.ts
@@ -6,13 +6,41 @@ const fresh = { enabled: true, lastActivityAt: NOW };
test('shutting down wins over everything else', () => {
expect(
- evaluateReadiness({ shuttingDown: true, heartbeat: fresh, now: NOW })
+ evaluateReadiness({
+ booting: false,
+ shuttingDown: true,
+ heartbeat: fresh,
+ now: NOW,
+ })
+ ).toEqual({ ready: false, reason: 'shutting down' });
+});
+
+test('a booting process answers HTTP but is not ready', () => {
+ expect(
+ evaluateReadiness({
+ booting: true,
+ shuttingDown: false,
+ heartbeat: fresh,
+ now: NOW,
+ })
+ ).toEqual({ ready: false, reason: 'booting' });
+});
+
+test('shutting down wins over booting', () => {
+ expect(
+ evaluateReadiness({
+ booting: true,
+ shuttingDown: true,
+ heartbeat: fresh,
+ now: NOW,
+ })
).toEqual({ ready: false, reason: 'shutting down' });
});
test('no events consumer on this instance skips the heartbeat check', () => {
expect(
evaluateReadiness({
+ booting: false,
shuttingDown: false,
// An instance with no events consumer has the heartbeat disabled,
// since lastActivityAt then never moves off boot time.
@@ -24,7 +52,12 @@ test('no events consumer on this instance skips the heartbeat check', () => {
test('a fresh consumer heartbeat is ready', () => {
expect(
- evaluateReadiness({ shuttingDown: false, heartbeat: fresh, now: NOW })
+ evaluateReadiness({
+ booting: false,
+ shuttingDown: false,
+ heartbeat: fresh,
+ now: NOW,
+ })
).toEqual({ ready: true });
});
@@ -32,6 +65,7 @@ test('a stale consumer heartbeat is 503 with the numbers that explain it', () =>
const idleMs = EVENTS_HEARTBEAT_STALE_MS + 1;
expect(
evaluateReadiness({
+ booting: false,
shuttingDown: false,
heartbeat: { enabled: true, lastActivityAt: NOW - idleMs },
now: NOW,
@@ -47,6 +81,7 @@ test('a stale consumer heartbeat is 503 with the numbers that explain it', () =>
test('exactly at the threshold is still ready — the comparison is >', () => {
expect(
evaluateReadiness({
+ booting: false,
shuttingDown: false,
heartbeat: {
enabled: true,
diff --git a/packages/core/src/modules/health/src/readiness.ts b/packages/core/src/modules/health/src/readiness.ts
index d3fbae7e5..ea0aadfe0 100644
--- a/packages/core/src/modules/health/src/readiness.ts
+++ b/packages/core/src/modules/health/src/readiness.ts
@@ -7,7 +7,7 @@
// readiness on a heartbeat that was never real.
import { getEventsHeartbeat } from '../../ingest/src/heartbeat';
-import { isShuttingDown } from './shutdown';
+import { isBooting, isShuttingDown } from './shutdown';
export const EVENTS_HEARTBEAT_STALE_MS = 60_000;
@@ -16,6 +16,7 @@ export type ReadinessResult =
| { ready: false; reason: string; idleMs?: number; thresholdMs?: number };
export interface ReadinessInputs {
+ booting: boolean;
shuttingDown: boolean;
heartbeat: { enabled: boolean; lastActivityAt: number };
now: number;
@@ -23,6 +24,7 @@ export interface ReadinessInputs {
/** Pure: every branch is a value, so the probe's rules are unit-testable. */
export function evaluateReadiness({
+ booting,
shuttingDown,
heartbeat,
now,
@@ -31,6 +33,10 @@ export function evaluateReadiness({
return { ready: false, reason: 'shutting down' };
}
+ if (booting) {
+ return { ready: false, reason: 'booting' };
+ }
+
if (!heartbeat.enabled) {
return { ready: true };
}
@@ -50,6 +56,7 @@ export function evaluateReadiness({
export function currentReadiness(): ReadinessResult {
return evaluateReadiness({
+ booting: isBooting(),
shuttingDown: isShuttingDown(),
heartbeat: getEventsHeartbeat(),
now: Date.now(),
diff --git a/packages/core/src/modules/health/src/shutdown.ts b/packages/core/src/modules/health/src/shutdown.ts
index ee5e235a6..f6e6a49eb 100644
--- a/packages/core/src/modules/health/src/shutdown.ts
+++ b/packages/core/src/modules/health/src/shutdown.ts
@@ -1,9 +1,18 @@
-// One flag, set by main.ts's signal handler, read by the readiness probe — so a
-// draining process is taken out of the load balancer before its in-flight jobs
-// finish rather than after.
+// Set by main.ts, read by the readiness probe: a process that is still starting
+// its consumers, or already draining, is out of the load balancer even though
+// it answers HTTP.
+let booting = false;
let shuttingDown = false;
+export function setBooting(value: boolean): void {
+ booting = value;
+}
+
+export function isBooting(): boolean {
+ return booting;
+}
+
export function setShuttingDown(value: boolean): void {
shuttingDown = value;
}