Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

7.6 KB
Raw Blame
import * as prometheus from 'prom-client'
import express, { NextFunction, Request, Response } from 'ultimate-express'

import { corsMiddleware } from '~/common/api/middleware/cors'
import { httpMetricsMiddleware } from '~/common/api/middleware/http-metrics'
import { createInternalApiAuthMiddleware } from '~/common/api/middleware/internal-api-auth'
import { logger } from '~/common/utils/logger'
import { HealthCheckResultError, PluginServerService } from '~/types'

prometheus.collectDefaultMetrics()

export function initializePrometheusLabels(
    ingestionPipeline: string | null = null,
    ingestionLane: string | null = null
): void {
    const labels: Record<string, string> = {}
    if (ingestionPipeline) {
        labels['ingestion_pipeline'] = ingestionPipeline
    }
    if (ingestionLane) {
        labels['ingestion_lane'] = ingestionLane
    }
    if (Object.keys(labels).length > 0) {
        prometheus.register.setDefaultLabels(labels)
    }
}

export interface SetupExpressAppOptions {
    // Path prefixes exempt from the shared-secret middleware. Used by servers that own auth on those
    // paths themselves, such as recording-api verifying a team-scoped JWT across the whole prefix.
    internalApiAuthExcludedPathPrefixes?: string[]
    internalApiSecret?: string
    // Comma-separated previous secrets still accepted for verification during rotation.
    internalApiSecretFallbacks?: string
    // JSON body size limit, e.g. '20mb'. Overridable in tests.
    jsonBodyLimit?: string
    // Path prefixes that only serve reads. Other methods get a 405 before the body parser buffers anything, so an
    // unauthenticated caller cannot make the server parse a large body for a route that never reads one.
    readOnlyPathPrefixes?: string[]
}

const READ_METHODS = new Set(['GET', 'HEAD', 'OPTIONS'])

export function setupCommonRoutes(
    app: express.Application,
    services: Pick<PluginServerService, 'id' | 'healthcheck'>[]
): express.Application {
    app.get('/_health', buildGetHealth(services))
    app.get('/_ready', buildGetHealth(services))
    app.get('/_metrics', getMetrics)
    app.get('/metrics', getMetrics)

    return app
}

export function setupExpressApp(options: SetupExpressAppOptions = {}): express.Application {
    const app = express()

    // Add CORS middleware before other middleware
    app.use(corsMiddleware)

    // Track HTTP request duration and status codes per route pattern
    app.use(httpMetricsMiddleware)

    // Add internal API authentication middleware for defense-in-depth.
    // Primary protection comes from Contour routing at the infra level.
    app.use(
        createInternalApiAuthMiddleware({
            secret: options.internalApiSecret || '',
            fallbacks: (options.internalApiSecretFallbacks || '')
                .split(',')
                .map((s) => s.trim())
                .filter(Boolean),
            excludedPathPrefixes: options.internalApiAuthExcludedPathPrefixes,
        })
    )

    const readOnlyPathPrefixes = options.readOnlyPathPrefixes ?? []
    if (readOnlyPathPrefixes.length > 0) {
        app.use((req: Request, res: Response, next: NextFunction) => {
            if (!READ_METHODS.has(req.method) && readOnlyPathPrefixes.some((prefix) => req.path.startsWith(prefix))) {
                res.set('Allow', 'GET, HEAD, OPTIONS')
                res.status(405).json({ error: 'Method not allowed' })
                return
            }
            next()
        })
    }

    app.use(
        express.json({
            limit: options.jsonBodyLimit ?? '20mb',
            verify: (req, res, buf) => {
                ;(req as any).rawBody = buf.toString('utf8')
            },
        })
    )

    // ultimate-express reports an oversized body as a plain Error with no
    // status, which the default handler renders as a 500 — indistinguishable
    // from a worker crash for callers deciding whether to retry. Answer 413 so
    // clients (the Rust ingestion consumer) split the payload instead of
    // retrying it verbatim. Also match express-style body-parser errors
    // (status 413 / type entity.too.large) for compatibility.
    app.use((err: any, _req: Request, res: Response, next: NextFunction) => {
        if (err?.message === 'Request entity too large' || err?.status === 413 || err?.type === 'entity.too.large') {
            res.status(413).json({ status: 'error', error: 'Request entity too large' })
            return
        }
        next(err)
    })

    return app
}

export type ModifiedRequest = Request & { rawBody?: string }

const buildGetHealth =
    (services: Pick<PluginServerService, 'id' | 'healthcheck'>[]) => async (req: Request, res: Response) => {
        // Check that all health checks pass. Note that a failure of these
        // _may_ result in the process being terminated by e.g. Kubernetes
        // so the stakes are high.
        //
        // Also, Kubernetes will call this endpoint frequently, on each pod,
        // so we want to make sure it's fast and doesn't put any stress on
        // other services. Ideally it shouldn't make any calls to other
        // services.
        //
        // Here we take all of the health checks we are given, run them in
        // parallel, and return the results. If any of the checks fail, we
        // return a 503 status code, otherwise we return a 200 status code.
        //
        // In all cases we should return a JSON object with the following
        // structure:
        //
        // {
        //   "status": "ok" | "error",
        //   "checks": {
        //     "service1": "ok" | "error",
        //     "service2": "ok" | "error",
        //     ...
        //   }
        // }
        const healthCheckPromises = services.map(async (service) => {
            try {
                const result = await service.healthcheck()
                return { service, result }
            } catch (error) {
                // If healthcheck throws, create an error result
                return {
                    service,
                    result: new HealthCheckResultError(error instanceof Error ? error.message : 'Unknown error', {}),
                }
            }
        })

        const healthChecks = await Promise.all(healthCheckPromises)

        // Convert to response format for API
        const checkResults = healthChecks.map(({ service, result }) => result.toResponse(service.id))

        // Use isError() method to determine status code
        const statusCode = healthChecks.every(({ result }) => !result.isError()) ? 200 : 503

        const checkResultsMapping = Object.fromEntries(
            checkResults.map((result) => [
                result.service,
                result.message ? { status: result.status, message: result.message } : result.status,
            ])
        )

        if (statusCode === 200) {
            logger.debug('💚', 'Server liveness check succeeded')
        } else {
            const failedServices = checkResults.filter((r) => r.status === 'error')
            logger.error('💔', 'Server liveness check failed', {
                failedServices: failedServices.map((s) => ({
                    service: s.service,
                    message: s.message,
                    details: 'details' in s ? s.details : undefined,
                })),
            })
        }

        return res.status(statusCode).json({ status: statusCode === 200 ? 'ok' : 'error', checks: checkResultsMapping })
    }

const getMetrics = async (req: Request, res: Response) => {
    try {
        const metrics = await prometheus.register.metrics()
        res.send(metrics)
    } catch (err) {
        logger.error('🩺', 'Error while collecting metrics', { err })
        res.sendStatus(500)
    }
}