Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.7 KB
Raw Blame
import * as Pyroscope from '@pyroscope/nodejs'
import { Server } from 'http'
import * as schedule from 'node-schedule'
import { Counter } from 'prom-client'
import express from 'ultimate-express'

import { SetupExpressAppOptions, setupCommonRoutes, setupExpressApp } from '~/common/api/router'
import { KafkaProducerWrapper } from '~/common/kafka/producer'
import { PostgresRouter } from '~/common/utils/db/postgres'
import { isTestEnv } from '~/common/utils/env-utils'
import { configureEventLoopYield } from '~/common/utils/event-loop-yield'
import { logger } from '~/common/utils/logger'
import { NodeInstrumentation } from '~/common/utils/node-instrumentation'
import { captureException, shutdown as posthogShutdown } from '~/common/utils/posthog'
import { PubSub } from '~/common/utils/pubsub'
import { closeSharedAgents } from '~/common/utils/request'
import { delay } from '~/common/utils/utils'

import { onShutdown } from '../lifecycle'
import { PluginServerService, RedisPool } from '../types'

export type BaseServerConfig = {
    INTERNAL_API_SECRET: string
    INTERNAL_API_SECRET_FALLBACKS: string
    INSTRUMENT_THREAD_PERFORMANCE: boolean
    HTTP_SERVER_PORT: number
    HTTP_SERVER_DRAIN_TIMEOUT_MS: number
    POD_TERMINATION_ENABLED: boolean
    POD_TERMINATION_BASE_TIMEOUT_MINUTES: number
    POD_TERMINATION_JITTER_MINUTES: number
    CONTINUOUS_PROFILING_ENABLED: boolean
    PYROSCOPE_SERVER_ADDRESS: string
    PYROSCOPE_APPLICATION_NAME: string
    EVENT_LOOP_YIELD_THRESHOLD_MS: number
}

export interface CleanupResources {
    kafkaProducers: KafkaProducerWrapper[]
    redisPools: RedisPool[]
    postgres?: PostgresRouter
    pubsub?: PubSub
    additionalCleanup?: () => void | Promise<void>
}

/** Minimal interface used by index.ts to interact with any server type. */
export interface NodeServer {
    start(): Promise<void>
    stop(error?: Error): Promise<void>
}

const serverStartupTimeMs = new Counter({
    name: 'plugin_server_startup_time_ms',
    help: 'Time taken to start the nodejs service, in milliseconds',
})

/**
 * Manages the lifecycle concerns shared by all server types: HTTP server,
 * process signals, profiling, pod termination, and graceful shutdown.
 *
 * Concrete servers (PluginServer, IngestionGeneralServer) compose this rather
 * than inheriting from it — they own their domain logic and delegate
 * infrastructure lifecycle here.
 */
export class ServerLifecycle {
    services: PluginServerService[] = []
    httpServer?: Server
    expressApp: express.Application
    stopping = false

    private nodeInstrumentation: NodeInstrumentation
    private podTerminationTimer?: NodeJS.Timeout
    private processListeners: Map<string, (...args: any[]) => void> = new Map()
    private openConnections = 0

    constructor(
        private config: BaseServerConfig,
        expressAppOptions: Omit<SetupExpressAppOptions, 'internalApiSecret' | 'internalApiSecretFallbacks'> = {}
    ) {
        this.expressApp = setupExpressApp({
            internalApiSecret: this.config.INTERNAL_API_SECRET,
            internalApiSecretFallbacks: this.config.INTERNAL_API_SECRET_FALLBACKS,
            ...expressAppOptions,
        })
        if (this.config.HTTP_SERVER_DRAIN_TIMEOUT_MS > 0) {
            ;(this.expressApp as ReturnType<typeof express>).uwsApp.filter((_res, change) => {
                this.openConnections += Number(change)
            })
        }
        this.nodeInstrumentation = new NodeInstrumentation(this.config.INSTRUMENT_THREAD_PERFORMANCE)
        configureEventLoopYield(this.config.EVENT_LOOP_YIELD_THRESHOLD_MS)
        this.setupContinuousProfiling()
    }

    async start(startServices: () => Promise<void>, getCleanupResources: () => CleanupResources): Promise<void> {
        const startupTimer = new Date()
        this.setupListeners(getCleanupResources)
        this.nodeInstrumentation.setupThreadPerformanceInterval()

        try {
            await startServices()

            setupCommonRoutes(this.expressApp, this.services)

            if (!isTestEnv()) {
                this.httpServer = this.expressApp.listen(this.config.HTTP_SERVER_PORT, () => {
                    logger.info('🩺', `HTTP server listening on port ${this.config.HTTP_SERVER_PORT}`)
                })
            }

            serverStartupTimeMs.inc(Date.now() - startupTimer.valueOf())
            logger.info('🚀', `All systems go in ${Date.now() - startupTimer.valueOf()}ms`)

            if (this.config.POD_TERMINATION_ENABLED) {
                this.setupPodTermination(getCleanupResources)
            }
        } catch (error: any) {
            captureException(error)
            logger.error('💥', 'Launchpad failure!', { error: error.stack ?? error })
            logger.error('💥', 'Exception while starting server, shutting down!', { error })
            await this.stop(getCleanupResources, error)
        }
    }

    async stop(getCleanupResources: () => CleanupResources, error?: Error): Promise<void> {
        for (const [event, handler] of this.processListeners) {
            process.removeListener(event, handler)
        }
        this.processListeners.clear()

        if (error) {
            logger.error('🤮', `Shutting down due to error`, { error: error.stack })
        }
        if (this.stopping) {
            logger.info('🚨', 'Stop called but already stopping...')
            return
        }

        this.stopping = true

        if (this.podTerminationTimer) {
            clearTimeout(this.podTerminationTimer)
            this.podTerminationTimer = undefined
        }

        this.nodeInstrumentation.cleanup()

        logger.info('💤', ' Shutting down gracefully...')

        await this.drainConnections()
        this.httpServer?.close()
        Object.values(schedule.scheduledJobs).forEach((job) => {
            job.cancel()
        })

        const resources = getCleanupResources()

        logger.info('💤', ' Shutting down services...')
        await Promise.allSettled([
            resources.pubsub?.stop(),
            ...this.services.map((s) => s.onShutdown()),
            posthogShutdown(),
            onShutdown(),
        ])

        if (resources.kafkaProducers.length > 0) {
            logger.info('💤', ' Flushing kafka producers...')
            await Promise.all([...resources.kafkaProducers.map((p) => p.flush()), delay(2000)])
        }

        logger.info('💤', ' Shutting down infrastructure...')
        await Promise.allSettled([
            ...resources.kafkaProducers.map((p) => p.disconnect()),
            ...resources.redisPools.map((p) => p.drain()),
            resources.postgres?.end(),
            closeSharedAgents(),
        ])
        for (const pool of resources.redisPools) {
            await pool.clear()
        }
        await resources.additionalCleanup?.()

        logger.info('💤', ' Shutting down completed. Exiting...')

        process.exit(error ? 1 : 0)
    }

    // uWebSockets close() terminates every open connection at once. A connection stays open until the
    // tail of an ended response leaves its send buffer, and closes on its own once idle.
    async drainConnections(): Promise<void> {
        const timeoutMs = this.config.HTTP_SERVER_DRAIN_TIMEOUT_MS
        if (timeoutMs <= 0) {
            return
        }
        const deadline = Date.now() + timeoutMs
        if (this.openConnections > 0) {
            logger.info('💤', `Waiting for ${this.openConnections} open HTTP connections to close...`)
        }
        while (this.openConnections > 0 && Date.now() < deadline) {
            await delay(100)
        }
        if (this.openConnections > 0) {
            logger.warn('⚠️', `Closing HTTP server with ${this.openConnections} connections still open`)
        }
    }

    private setupPodTermination(getCleanupResources: () => CleanupResources): void {
        const baseTimeoutMs = this.config.POD_TERMINATION_BASE_TIMEOUT_MINUTES * 60 * 1000
        const jitterMs = Math.random() * this.config.POD_TERMINATION_JITTER_MINUTES * 60 * 1000
        const totalTimeoutMs = baseTimeoutMs + jitterMs

        logger.info('⏰', `Pod termination scheduled in ${Math.round(totalTimeoutMs / 1000 / 60)} minutes`)

        this.podTerminationTimer = setTimeout(() => {
            logger.info('⏰', 'Pod termination timeout reached, shutting down gracefully...')
            void this.stop(getCleanupResources)
        }, totalTimeoutMs)
    }

    private setupListeners(getCleanupResources: () => CleanupResources): void {
        for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) {
            const handler = async () => {
                logger.info('👋', `process handling ${signal} event. Stopping...`)
                await this.stop(getCleanupResources)
            }
            this.processListeners.set(signal, handler)
            process.on(signal, handler)
        }

        const rejectionHandler = (error: Error | any) => {
            logger.error('🤮', `Unhandled Promise Rejection`, { error: String(error) })

            captureException(error, {
                extra: { detected_at: `ServerLifecycle on unhandledRejection` },
            })

            void this.stop(getCleanupResources, error)
        }
        this.processListeners.set('unhandledRejection', rejectionHandler)
        process.on('unhandledRejection', rejectionHandler)

        const exceptionHandler = async (error: Error) => {
            // Log before stopping. Without this the process exits silently, so a crash loop
            // shows only a rising restart count and whatever the service logged last.
            logger.error('🤮', `Uncaught Exception`, { error: String(error), stack: error?.stack })

            captureException(error, {
                extra: { detected_at: `ServerLifecycle on uncaughtException` },
            })

            await this.stop(getCleanupResources, error)
        }
        this.processListeners.set('uncaughtException', exceptionHandler)
        process.on('uncaughtException', exceptionHandler)
    }

    private setupContinuousProfiling(): void {
        if (!this.config.CONTINUOUS_PROFILING_ENABLED) {
            logger.info('Continuous profiling is disabled')
            return
        }

        if (!this.config.PYROSCOPE_SERVER_ADDRESS) {
            logger.warn('Continuous profiling is enabled but PYROSCOPE_SERVER_ADDRESS is empty, skipping')
            return
        }

        try {
            const tags = this.collectK8sTags()

            Pyroscope.init({
                serverAddress: this.config.PYROSCOPE_SERVER_ADDRESS,
                appName: this.config.PYROSCOPE_APPLICATION_NAME || 'nodejs',
                tags,
            })

            Pyroscope.start()
            logger.info('Continuous profiling started', {
                serverAddress: this.config.PYROSCOPE_SERVER_ADDRESS,
                appName: this.config.PYROSCOPE_APPLICATION_NAME || 'nodejs',
                tags,
            })
        } catch (error) {
            logger.error('Failed to start continuous profiling', { error })
        }
    }

    private collectK8sTags(): Record<string, string> {
        const k8sTagEnvVars: Record<string, string> = {
            namespace: 'K8S_NAMESPACE',
            pod: 'K8S_POD_NAME',
            node: 'K8S_NODE_NAME',
            pod_template_hash: 'K8S_POD_TEMPLATE_HASH',
            app_instance: 'K8S_APP_INSTANCE',
            app: 'K8S_APP',
            container: 'K8S_CONTAINER_NAME',
            controller_type: 'K8S_CONTROLLER_TYPE',
        }

        const tags: Record<string, string> = { src: 'SDK' }
        for (const [tagName, envVar] of Object.entries(k8sTagEnvVars)) {
            const value = process.env[envVar]
            if (value) {
                tags[tagName] = value
            } else {
                logger.warn(`K8s tag ${tagName} not set (env var ${envVar} is empty)`)
            }
        }
        return tags
    }
}