OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(api): workers answer health checks while they start up
Carl-Gerhard Lindesvärd6e877b4b729f
diff --git a/apps/api/src/main.ts b/apps/api/src/main.tsindex ef5a89d92..20cf9cf0d 100644--- a/apps/api/src/main.ts+++ b/apps/api/src/main.ts@@ -50,6 +50,7 @@ import {registerSessionScrapeMetrics,requestLogging,type SessionMetricsRedis,+ setBooting,setShuttingDown,startKafkaEventsConsumer,startSchedulers,@@ -427,6 +428,29 @@ async function main() {const app = await buildHttpApp(deps);+ // Listen before the consumers start: joining the Kafka group can take most+ // of a minute, and a liveness probe that cannot connect meanwhile gets the+ // container killed. Readiness stays 503 until boot finishes.+ setBooting(true);+ const listenOptions = {+ ...config.listen,+ idleTimeout: HTTP_IDLE_TIMEOUT_SECONDS,+ };+ app.listen(listenOptions, () => {+ logger.info(+ {+ role,+ port: config.API_PORT,+ hostname: config.listen.hostname ?? '0.0.0.0',+ // The running Bun version, asserted against .bun-version by+ // scripts/doctor.sh — logged so a wrong-runtime incident is one log+ // line away rather than an inference.+ bunVersion: Bun.version,+ },+ 'API listening'+ );+ });+let workers: WorkerHandle | undefined;let consumer: KafkaConsumerHandle | undefined;@@ -529,24 +553,8 @@ async function main() {shutdown('SIGINT');});- const listenOptions = {- ...config.listen,- idleTimeout: HTTP_IDLE_TIMEOUT_SECONDS,- };- app.listen(listenOptions, () => {- logger.info(- {- role,- port: config.API_PORT,- hostname: config.listen.hostname ?? '0.0.0.0',- // The running Bun version, asserted against .bun-version by- // scripts/doctor.sh — logged so a wrong-runtime incident is one log- // line away rather than an inference.- bunVersion: Bun.version,- },- 'API listening'- );- });+ setBooting(false);+ logger.info({ role }, 'Boot complete');// The expiry notifications only matter where something subscribes.if (role !== 'worker') {diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex d835412ce..7b7b2bf59 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -160,7 +160,7 @@ export {} from './modules/event/event.service';export type { IServiceGroup } from './modules/group/group.service';export { getGscCannibalization } from './modules/gsc/gsc.service';-export { setShuttingDown } from './modules/health/src/shutdown';+export { setBooting, setShuttingDown } from './modules/health/src/shutdown';export type {BotMatch,IncomingEventPayload,diff --git a/packages/core/src/modules/health/src/readiness.test.ts b/packages/core/src/modules/health/src/readiness.test.tsindex c2cbeaa04..7e6bd6c19 100644--- a/packages/core/src/modules/health/src/readiness.test.ts+++ b/packages/core/src/modules/health/src/readiness.test.ts@@ -6,13 +6,41 @@ const fresh = { enabled: true, lastActivityAt: NOW };test('shutting down wins over everything else', () => {expect(- evaluateReadiness({ shuttingDown: true, heartbeat: fresh, now: NOW })+ evaluateReadiness({+ booting: false,+ shuttingDown: true,+ heartbeat: fresh,+ now: NOW,+ })+ ).toEqual({ ready: false, reason: 'shutting down' });+});++test('a booting process answers HTTP but is not ready', () => {+ expect(+ evaluateReadiness({+ booting: true,+ shuttingDown: false,+ heartbeat: fresh,+ now: NOW,+ })+ ).toEqual({ ready: false, reason: 'booting' });+});++test('shutting down wins over booting', () => {+ expect(+ evaluateReadiness({+ booting: true,+ shuttingDown: true,+ heartbeat: fresh,+ now: NOW,+ })).toEqual({ ready: false, reason: 'shutting down' });});test('no events consumer on this instance skips the heartbeat check', () => {expect(evaluateReadiness({+ booting: false,shuttingDown: false,// An instance with no events consumer has the heartbeat disabled,// since lastActivityAt then never moves off boot time.@@ -24,7 +52,12 @@ test('no events consumer on this instance skips the heartbeat check', () => {test('a fresh consumer heartbeat is ready', () => {expect(- evaluateReadiness({ shuttingDown: false, heartbeat: fresh, now: NOW })+ evaluateReadiness({+ booting: false,+ shuttingDown: false,+ heartbeat: fresh,+ now: NOW,+ })).toEqual({ ready: true });});@@ -32,6 +65,7 @@ test('a stale consumer heartbeat is 503 with the numbers that explain it', () =>const idleMs = EVENTS_HEARTBEAT_STALE_MS + 1;expect(evaluateReadiness({+ booting: false,shuttingDown: false,heartbeat: { enabled: true, lastActivityAt: NOW - idleMs },now: NOW,@@ -47,6 +81,7 @@ test('a stale consumer heartbeat is 503 with the numbers that explain it', () =>test('exactly at the threshold is still ready — the comparison is >', () => {expect(evaluateReadiness({+ booting: false,shuttingDown: false,heartbeat: {enabled: true,diff --git a/packages/core/src/modules/health/src/readiness.ts b/packages/core/src/modules/health/src/readiness.tsindex d3fbae7e5..ea0aadfe0 100644--- a/packages/core/src/modules/health/src/readiness.ts+++ b/packages/core/src/modules/health/src/readiness.ts@@ -7,7 +7,7 @@// readiness on a heartbeat that was never real.import { getEventsHeartbeat } from '../../ingest/src/heartbeat';-import { isShuttingDown } from './shutdown';+import { isBooting, isShuttingDown } from './shutdown';export const EVENTS_HEARTBEAT_STALE_MS = 60_000;@@ -16,6 +16,7 @@ export type ReadinessResult =| { ready: false; reason: string; idleMs?: number; thresholdMs?: number };export interface ReadinessInputs {+ booting: boolean;shuttingDown: boolean;heartbeat: { enabled: boolean; lastActivityAt: number };now: number;@@ -23,6 +24,7 @@ export interface ReadinessInputs {/** Pure: every branch is a value, so the probe's rules are unit-testable. */export function evaluateReadiness({+ booting,shuttingDown,heartbeat,now,@@ -31,6 +33,10 @@ export function evaluateReadiness({return { ready: false, reason: 'shutting down' };}+ if (booting) {+ return { ready: false, reason: 'booting' };+ }+if (!heartbeat.enabled) {return { ready: true };}@@ -50,6 +56,7 @@ export function evaluateReadiness({export function currentReadiness(): ReadinessResult {return evaluateReadiness({+ booting: isBooting(),shuttingDown: isShuttingDown(),heartbeat: getEventsHeartbeat(),now: Date.now(),diff --git a/packages/core/src/modules/health/src/shutdown.ts b/packages/core/src/modules/health/src/shutdown.tsindex ee5e235a6..f6e6a49eb 100644--- a/packages/core/src/modules/health/src/shutdown.ts+++ b/packages/core/src/modules/health/src/shutdown.ts@@ -1,9 +1,18 @@-// One flag, set by main.ts's signal handler, read by the readiness probe — so a-// draining process is taken out of the load balancer before its in-flight jobs-// finish rather than after.+// Set by main.ts, read by the readiness probe: a process that is still starting+// its consumers, or already draining, is out of the load balancer even though+// it answers HTTP.+let booting = false;let shuttingDown = false;+export function setBooting(value: boolean): void {+ booting = value;+}++export function isBooting(): boolean {+ return booting;+}+export function setShuttingDown(value: boolean): void {shuttingDown = value;}