:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(replay): drain in-flight http requests before closing the node server (#122879)
Tue Haulund6e8a19d148d5
diff --git a/nodejs/src/common/config.ts b/nodejs/src/common/config.tsindex ad16d4b7a1c7..e6b0b893c114 100644--- a/nodejs/src/common/config.ts+++ b/nodejs/src/common/config.ts@@ -490,6 +490,8 @@ export function getDefaultCommonConfig(): CommonConfig {// Event loop yield helperEVENT_LOOP_YIELD_THRESHOLD_MS: 200,+ HTTP_SERVER_DRAIN_TIMEOUT_MS: 0,+// Pod terminationPOD_TERMINATION_ENABLED: false,POD_TERMINATION_BASE_TIMEOUT_MINUTES: 30,diff --git a/nodejs/src/servers/base-server.test.ts b/nodejs/src/servers/base-server.test.tsnew file mode 100644index 000000000000..6c1eab08e7aa--- /dev/null+++ b/nodejs/src/servers/base-server.test.ts@@ -0,0 +1,74 @@+import http from 'http'++import { getDefaultConfig } from '~/common/config/config'++import { ServerLifecycle } from './base-server'++describe('ServerLifecycle', () => {+ let lifecycle: ServerLifecycle+ let port: number+ let releaseResponse: () => void+ let requestReceived: Promise<void>++ beforeEach(async () => {+ lifecycle = new ServerLifecycle(+ { ...getDefaultConfig(), HTTP_SERVER_DRAIN_TIMEOUT_MS: 60_000 },+ { internalApiAuthExcludedPathPrefixes: ['/big'] }+ )+ let onRequest: () => void+ requestReceived = new Promise((resolve) => (onRequest = resolve))+ const released = new Promise<void>((resolve) => (releaseResponse = resolve))+ lifecycle.expressApp.get('/big', async (_req, res) => {+ onRequest()+ await released+ res.send(Buffer.alloc(50 * 1024 * 1024))+ })+ await new Promise<void>((resolve) => {+ lifecycle.httpServer = lifecycle.expressApp.listen(0, () => resolve()) as any+ })+ port = (lifecycle.expressApp as any).address().port+ })++ afterEach(() => {+ releaseResponse()+ ;(lifecycle.httpServer as any)?.close()+ })++ it('waits until a sent response has fully reached a slow client', async () => {+ let receivedBytes = 0+ let pausedResponse: http.IncomingMessage | undefined+ const responseEnded = new Promise<void>((resolve) => {+ http.get({ port, path: '/big', agent: false, headers: { connection: 'close' } }, (res) => {+ pausedResponse = res.pause()+ res.on('data', (chunk) => (receivedBytes += chunk.length))+ res.on('end', resolve)+ })+ })+ await requestReceived+ releaseResponse()++ jest.useFakeTimers({ doNotFake: ['nextTick', 'setImmediate', 'queueMicrotask'] })+ try {+ let drained = false+ void lifecycle.drainConnections().then(() => (drained = true))+ await jest.advanceTimersByTimeAsync(10_000)+ expect(drained).toBe(false)++ pausedResponse!.resume()+ await responseEnded+ expect(receivedBytes).toEqual(50 * 1024 * 1024)+ await jest.advanceTimersByTimeAsync(200)+ expect(drained).toBe(true)+ } finally {+ jest.useRealTimers()+ }+ })++ it('does not wait for a request the client aborted', async () => {+ const req = http.get({ port, path: '/big', agent: false }).on('error', () => {})+ await requestReceived+ req.destroy()++ await expect(lifecycle.drainConnections()).resolves.toBeUndefined()+ })+})diff --git a/nodejs/src/servers/base-server.ts b/nodejs/src/servers/base-server.tsindex 9ca83ab943d0..e4d1460875bc 100644--- a/nodejs/src/servers/base-server.ts+++ b/nodejs/src/servers/base-server.ts@@ -24,6 +24,7 @@ export type BaseServerConfig = {INTERNAL_API_SECRET_FALLBACKS: stringINSTRUMENT_THREAD_PERFORMANCE: booleanHTTP_SERVER_PORT: number+ HTTP_SERVER_DRAIN_TIMEOUT_MS: numberPOD_TERMINATION_ENABLED: booleanPOD_TERMINATION_BASE_TIMEOUT_MINUTES: numberPOD_TERMINATION_JITTER_MINUTES: number@@ -69,6 +70,7 @@ export class ServerLifecycle {private nodeInstrumentation: NodeInstrumentationprivate podTerminationTimer?: NodeJS.Timeoutprivate processListeners: Map<string, (...args: any[]) => void> = new Map()+ private openConnections = 0constructor(private config: BaseServerConfig,@@ -79,6 +81,11 @@ export class ServerLifecycle {internalApiSecretFallbacks: this.config.INTERNAL_API_SECRET_FALLBACKS,...expressAppOptions,})+ if (this.config.HTTP_SERVER_DRAIN_TIMEOUT_MS > 0) {+ ;(this.expressApp as ReturnType<typeof express>).uwsApp.filter((_res, change) => {+ this.openConnections += Number(change)+ })+ }this.nodeInstrumentation = new NodeInstrumentation(this.config.INSTRUMENT_THREAD_PERFORMANCE)configureEventLoopYield(this.config.EVENT_LOOP_YIELD_THRESHOLD_MS)this.setupContinuousProfiling()@@ -139,6 +146,7 @@ export class ServerLifecycle {logger.info('💤', ' Shutting down gracefully...')+ await this.drainConnections()this.httpServer?.close()Object.values(schedule.scheduledJobs).forEach((job) => {job.cancel()@@ -176,6 +184,25 @@ export class ServerLifecycle {process.exit(error ? 1 : 0)}+ // uWebSockets close() terminates every open connection at once. A connection stays open until the+ // tail of an ended response leaves its send buffer, and closes on its own once idle.+ async drainConnections(): Promise<void> {+ const timeoutMs = this.config.HTTP_SERVER_DRAIN_TIMEOUT_MS+ if (timeoutMs <= 0) {+ return+ }+ const deadline = Date.now() + timeoutMs+ if (this.openConnections > 0) {+ logger.info('💤', `Waiting for ${this.openConnections} open HTTP connections to close...`)+ }+ while (this.openConnections > 0 && Date.now() < deadline) {+ await delay(100)+ }+ if (this.openConnections > 0) {+ logger.warn('⚠️', `Closing HTTP server with ${this.openConnections} connections still open`)+ }+ }+private setupPodTermination(getCleanupResources: () => CleanupResources): void {const baseTimeoutMs = this.config.POD_TERMINATION_BASE_TIMEOUT_MINUTES * 60 * 1000const jitterMs = Math.random() * this.config.POD_TERMINATION_JITTER_MINUTES * 60 * 1000