Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(replay): drain in-flight http requests before closing the node server (#122879)

Tue Haulund6e8a19d148d5
diff --git a/nodejs/src/common/config.ts b/nodejs/src/common/config.ts
index ad16d4b7a1c7..e6b0b893c114 100644
--- a/nodejs/src/common/config.ts
+++ b/nodejs/src/common/config.ts
@@ -490,6 +490,8 @@ export function getDefaultCommonConfig(): CommonConfig {
// Event loop yield helper
EVENT_LOOP_YIELD_THRESHOLD_MS: 200,
+ HTTP_SERVER_DRAIN_TIMEOUT_MS: 0,
+
// Pod termination
POD_TERMINATION_ENABLED: false,
POD_TERMINATION_BASE_TIMEOUT_MINUTES: 30,
diff --git a/nodejs/src/servers/base-server.test.ts b/nodejs/src/servers/base-server.test.ts
new file mode 100644
index 000000000000..6c1eab08e7aa
--- /dev/null
+++ b/nodejs/src/servers/base-server.test.ts
@@ -0,0 +1,74 @@
+import http from 'http'
+
+import { getDefaultConfig } from '~/common/config/config'
+
+import { ServerLifecycle } from './base-server'
+
+describe('ServerLifecycle', () => {
+ let lifecycle: ServerLifecycle
+ let port: number
+ let releaseResponse: () => void
+ let requestReceived: Promise<void>
+
+ beforeEach(async () => {
+ lifecycle = new ServerLifecycle(
+ { ...getDefaultConfig(), HTTP_SERVER_DRAIN_TIMEOUT_MS: 60_000 },
+ { internalApiAuthExcludedPathPrefixes: ['/big'] }
+ )
+ let onRequest: () => void
+ requestReceived = new Promise((resolve) => (onRequest = resolve))
+ const released = new Promise<void>((resolve) => (releaseResponse = resolve))
+ lifecycle.expressApp.get('/big', async (_req, res) => {
+ onRequest()
+ await released
+ res.send(Buffer.alloc(50 * 1024 * 1024))
+ })
+ await new Promise<void>((resolve) => {
+ lifecycle.httpServer = lifecycle.expressApp.listen(0, () => resolve()) as any
+ })
+ port = (lifecycle.expressApp as any).address().port
+ })
+
+ afterEach(() => {
+ releaseResponse()
+ ;(lifecycle.httpServer as any)?.close()
+ })
+
+ it('waits until a sent response has fully reached a slow client', async () => {
+ let receivedBytes = 0
+ let pausedResponse: http.IncomingMessage | undefined
+ const responseEnded = new Promise<void>((resolve) => {
+ http.get({ port, path: '/big', agent: false, headers: { connection: 'close' } }, (res) => {
+ pausedResponse = res.pause()
+ res.on('data', (chunk) => (receivedBytes += chunk.length))
+ res.on('end', resolve)
+ })
+ })
+ await requestReceived
+ releaseResponse()
+
+ jest.useFakeTimers({ doNotFake: ['nextTick', 'setImmediate', 'queueMicrotask'] })
+ try {
+ let drained = false
+ void lifecycle.drainConnections().then(() => (drained = true))
+ await jest.advanceTimersByTimeAsync(10_000)
+ expect(drained).toBe(false)
+
+ pausedResponse!.resume()
+ await responseEnded
+ expect(receivedBytes).toEqual(50 * 1024 * 1024)
+ await jest.advanceTimersByTimeAsync(200)
+ expect(drained).toBe(true)
+ } finally {
+ jest.useRealTimers()
+ }
+ })
+
+ it('does not wait for a request the client aborted', async () => {
+ const req = http.get({ port, path: '/big', agent: false }).on('error', () => {})
+ await requestReceived
+ req.destroy()
+
+ await expect(lifecycle.drainConnections()).resolves.toBeUndefined()
+ })
+})
diff --git a/nodejs/src/servers/base-server.ts b/nodejs/src/servers/base-server.ts
index 9ca83ab943d0..e4d1460875bc 100644
--- a/nodejs/src/servers/base-server.ts
+++ b/nodejs/src/servers/base-server.ts
@@ -24,6 +24,7 @@ export type BaseServerConfig = {
INTERNAL_API_SECRET_FALLBACKS: string
INSTRUMENT_THREAD_PERFORMANCE: boolean
HTTP_SERVER_PORT: number
+ HTTP_SERVER_DRAIN_TIMEOUT_MS: number
POD_TERMINATION_ENABLED: boolean
POD_TERMINATION_BASE_TIMEOUT_MINUTES: number
POD_TERMINATION_JITTER_MINUTES: number
@@ -69,6 +70,7 @@ export class ServerLifecycle {
private nodeInstrumentation: NodeInstrumentation
private podTerminationTimer?: NodeJS.Timeout
private processListeners: Map<string, (...args: any[]) => void> = new Map()
+ private openConnections = 0
constructor(
private config: BaseServerConfig,
@@ -79,6 +81,11 @@ export class ServerLifecycle {
internalApiSecretFallbacks: this.config.INTERNAL_API_SECRET_FALLBACKS,
...expressAppOptions,
})
+ if (this.config.HTTP_SERVER_DRAIN_TIMEOUT_MS > 0) {
+ ;(this.expressApp as ReturnType<typeof express>).uwsApp.filter((_res, change) => {
+ this.openConnections += Number(change)
+ })
+ }
this.nodeInstrumentation = new NodeInstrumentation(this.config.INSTRUMENT_THREAD_PERFORMANCE)
configureEventLoopYield(this.config.EVENT_LOOP_YIELD_THRESHOLD_MS)
this.setupContinuousProfiling()
@@ -139,6 +146,7 @@ export class ServerLifecycle {
logger.info('💤', ' Shutting down gracefully...')
+ await this.drainConnections()
this.httpServer?.close()
Object.values(schedule.scheduledJobs).forEach((job) => {
job.cancel()
@@ -176,6 +184,25 @@ export class ServerLifecycle {
process.exit(error ? 1 : 0)
}
+ // uWebSockets close() terminates every open connection at once. A connection stays open until the
+ // tail of an ended response leaves its send buffer, and closes on its own once idle.
+ async drainConnections(): Promise<void> {
+ const timeoutMs = this.config.HTTP_SERVER_DRAIN_TIMEOUT_MS
+ if (timeoutMs <= 0) {
+ return
+ }
+ const deadline = Date.now() + timeoutMs
+ if (this.openConnections > 0) {
+ logger.info('💤', `Waiting for ${this.openConnections} open HTTP connections to close...`)
+ }
+ while (this.openConnections > 0 && Date.now() < deadline) {
+ await delay(100)
+ }
+ if (this.openConnections > 0) {
+ logger.warn('⚠️', `Closing HTTP server with ${this.openConnections} connections still open`)
+ }
+ }
+
private setupPodTermination(getCleanupResources: () => CleanupResources): void {
const baseTimeoutMs = this.config.POD_TERMINATION_BASE_TIMEOUT_MINUTES * 60 * 1000
const jitterMs = Math.random() * this.config.POD_TERMINATION_JITTER_MINUTES * 60 * 1000