Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

26.6 KB
Raw Blame
import { AsyncLocalStorage } from 'node:async_hooks'
import http from 'node:http'
import http2 from 'node:http2'
import https from 'node:https'
import net, { AddressInfo } from 'node:net'
import tls from 'node:tls'

import type { ImageFetchOptions } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher'
import { waitForExpect } from '~/tests/helpers/expectations'
import { TestTlsIdentity, createTestTlsIdentity } from '~/tests/helpers/tls'

type RequestModule = typeof import('./request')
type Client = import('undici').Client
type FetchResponseLike = Awaited<ReturnType<RequestModule['fetch']>>
type LegacyResponseLike = Awaited<ReturnType<RequestModule['legacyFetch']>>

const proxyEnvironmentNames = ['HTTPS_PROXY', 'HTTP_PROXY', 'https_proxy', 'http_proxy'] as const

function serverPort(server: http.Server | http2.Http2SecureServer | https.Server): number {
    return (server.address() as AddressInfo).port
}

function listen(server: http.Server | http2.Http2SecureServer | https.Server): Promise<void> {
    return new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))
}

function close(server: http.Server | http2.Http2SecureServer | https.Server): Promise<void> {
    if (!server.listening) {
        return Promise.resolve()
    }
    return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())))
}

describe('secure HTTP/2 requests', () => {
    let requestModule: RequestModule
    let http2Origin: http2.Http2SecureServer
    let http1Origin: https.Server
    let plainOrigin: http.Server
    let connectProxy: http.Server
    let tlsConnectSpy: jest.SpyInstance
    let tlsIdentity: TestTlsIdentity | undefined
    const originalExternalRequestConnections = process.env.EXTERNAL_REQUEST_CONNECTIONS
    const originalH2Connections = process.env.EXTERNAL_REQUEST_H2_CONNECTIONS
    const originalKeepAliveTimeout = process.env.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS
    const keepAliveTimeoutMs = 3000
    const originalProxyEnvironment = Object.fromEntries(
        proxyEnvironmentNames.map((name) => [name, process.env[name]])
    ) as Record<(typeof proxyEnvironmentNames)[number], string | undefined>
    let http2SessionCount = 0
    const http2OriginProtocols: string[] = []
    const http1OriginProtocols: string[] = []
    let http1ConnectionCount = 0
    const proxyAuthorities: string[] = []
    const openSockets = new Set<net.Socket>()
    const openHttp2Sessions = new Set<http2.ServerHttp2Session>()
    const pendingConcurrentResponses: Array<() => void> = []
    const pendingImageResponses: Array<() => void> = []
    const imageBytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00])

    beforeAll(async () => {
        const generatedTlsIdentity = await createTestTlsIdentity('origin.test')
        tlsIdentity = generatedTlsIdentity

        http2Origin = http2.createSecureServer({
            allowHTTP1: true,
            cert: generatedTlsIdentity.certificate,
            key: generatedTlsIdentity.privateKey,
        })
        http2Origin.on('request', (request, response) => {
            http2OriginProtocols.push(request.httpVersion)
            if (request.url?.startsWith('/image-burst-')) {
                response.writeHead(200, { 'content-type': 'image/png' })
                response.write(imageBytes.subarray(0, 1))
                pendingImageResponses.push(() => response.end(imageBytes.subarray(1)))
                if (pendingImageResponses.length === 6) {
                    pendingImageResponses.splice(0).forEach((finish) => finish())
                }
                return
            }
            const finishResponse = (): void => {
                response.writeHead(200, { 'content-type': 'text/plain' })
                response.end(request.url)
            }
            if (request.url?.startsWith('/concurrent-')) {
                pendingConcurrentResponses.push(finishResponse)
                if (pendingConcurrentResponses.length === 2) {
                    pendingConcurrentResponses.splice(0).forEach((finish) => finish())
                }
                return
            }
            finishResponse()
            if (request.url === '/goaway') {
                request.stream.session?.close()
            }
        })
        http2Origin.on('session', (session) => {
            http2SessionCount += 1
            openHttp2Sessions.add(session)
            session.once('close', () => openHttp2Sessions.delete(session))
        })
        await listen(http2Origin)

        http1Origin = https.createServer(
            { cert: generatedTlsIdentity.certificate, key: generatedTlsIdentity.privateKey },
            (request, response) => {
                http1OriginProtocols.push(request.httpVersion)
                response.writeHead(200, { 'content-type': 'text/plain' })
                response.end(request.url)
            }
        )
        http1Origin.on('connection', () => {
            http1ConnectionCount += 1
        })
        await listen(http1Origin)

        plainOrigin = http.createServer((request, response) => {
            response.writeHead(200, { 'content-type': 'text/plain' })
            response.end(request.url)
        })
        await listen(plainOrigin)

        connectProxy = http.createServer()
        connectProxy.on('connection', (socket) => {
            openSockets.add(socket)
            socket.once('close', () => openSockets.delete(socket))
        })
        connectProxy.on('connect', (request, clientSocket, head) => {
            proxyAuthorities.push(request.url ?? '')
            const requestedPort = Number(request.url?.split(':').at(-1))
            const originSocket = net.connect(requestedPort, '127.0.0.1', () => {
                clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n')
                if (head.length > 0) {
                    originSocket.write(head)
                }
                clientSocket.pipe(originSocket)
                originSocket.pipe(clientSocket)
            })
            openSockets.add(originSocket)
            originSocket.once('close', () => {
                openSockets.delete(originSocket)
                clientSocket.destroy()
            })
            clientSocket.once('close', () => originSocket.destroy())
            originSocket.once('error', () => clientSocket.destroy())
            clientSocket.once('error', () => originSocket.destroy())
        })
        await listen(connectProxy)

        const connectWithSystemTrust = tls.connect
        tlsConnectSpy = jest
            .spyOn(tls, 'connect')
            .mockImplementation(((options: tls.ConnectionOptions, callback?: () => void) =>
                connectWithSystemTrust(
                    { ...options, ca: generatedTlsIdentity.certificate },
                    callback
                )) as typeof tls.connect)
        process.env.HTTPS_PROXY = `http://127.0.0.1:${serverPort(connectProxy)}`
        process.env.EXTERNAL_REQUEST_CONNECTIONS = '2'
        process.env.EXTERNAL_REQUEST_H2_CONNECTIONS = '4'
        process.env.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS = String(keepAliveTimeoutMs)
        delete process.env.HTTP_PROXY
        delete process.env.https_proxy
        delete process.env.http_proxy
    })

    beforeEach(() => {
        http2SessionCount = 0
        http2OriginProtocols.length = 0
        http1OriginProtocols.length = 0
        http1ConnectionCount = 0
        proxyAuthorities.length = 0
        jest.resetModules()
        requestModule = require('./request') as RequestModule
    })

    afterEach(async () => {
        pendingImageResponses.splice(0).forEach((finish) => finish())
        await requestModule.closeSharedAgents()
        await waitForExpect(() => expect(openHttp2Sessions.size).toBe(0), 2000)
    })

    afterAll(async () => {
        for (const name of proxyEnvironmentNames) {
            const value = originalProxyEnvironment[name]
            if (value === undefined) {
                delete process.env[name]
            } else {
                process.env[name] = value
            }
        }
        if (originalExternalRequestConnections === undefined) {
            delete process.env.EXTERNAL_REQUEST_CONNECTIONS
        } else {
            process.env.EXTERNAL_REQUEST_CONNECTIONS = originalExternalRequestConnections
        }
        if (originalH2Connections === undefined) {
            delete process.env.EXTERNAL_REQUEST_H2_CONNECTIONS
        } else {
            process.env.EXTERNAL_REQUEST_H2_CONNECTIONS = originalH2Connections
        }
        if (originalKeepAliveTimeout === undefined) {
            delete process.env.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS
        } else {
            process.env.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS = originalKeepAliveTimeout
        }
        tlsConnectSpy.mockRestore()

        for (const session of openHttp2Sessions) {
            session.destroy()
        }
        for (const socket of openSockets) {
            socket.destroy()
        }
        http1Origin.closeAllConnections()
        plainOrigin.closeAllConnections()
        connectProxy.closeAllConnections()
        await Promise.all([close(http2Origin), close(http1Origin), close(plainOrigin), close(connectProxy)])
        await tlsIdentity?.cleanup()
    })

    it('negotiates, multiplexes, defaults, and falls back through a CONNECT proxy', async () => {
        const http2Authority = `origin.test:${serverPort(http2Origin)}`
        const http2Url = `https://${http2Authority}`
        const bufferedResponse = await requestModule.fetch(`${http2Url}/buffered`, { allowH2: true, timeoutMs: 2000 })
        expect(await bufferedResponse.text()).toBe('/buffered')

        const streamedResponse = await requestModule.fetchStreamed(`${http2Url}/streamed`, {
            allowH2: true,
            timeoutMs: 2000,
        })
        expect((await streamedResponse.read(100)).bytes.toString()).toBe('/streamed')

        const concurrentBodies = await Promise.all(
            ['/concurrent-a', '/concurrent-b'].map(async (path) => {
                const response = await requestModule.fetchStreamed(`${http2Url}${path}`, {
                    allowH2: true,
                    timeoutMs: 2000,
                })
                return (await response.read(100)).bytes.toString()
            })
        )
        expect(concurrentBodies).toEqual(['/concurrent-a', '/concurrent-b'])

        const defaultResponse = await requestModule.fetchStreamed(`${http2Url}/default`, { timeoutMs: 2000 })
        expect((await defaultResponse.read(100)).bytes.toString()).toBe('/default')

        const http1Authority = `origin.test:${serverPort(http1Origin)}`
        const fallbackResponse = await requestModule.fetchStreamed(`https://${http1Authority}/fallback`, {
            allowH2: true,
            timeoutMs: 2000,
        })
        expect((await fallbackResponse.read(100)).bytes.toString()).toBe('/fallback')

        expect(http2OriginProtocols).toEqual(['2.0', '2.0', '2.0', '2.0', '1.1'])
        expect(http1OriginProtocols).toEqual(['1.1'])
        // undici multiplexes the concurrent requests on one session, up to the server's max concurrent streams.
        expect(http2SessionCount).toBe(1)
        expect(proxyAuthorities).toEqual([http2Authority, http2Authority, http1Authority])
    }, 10000)

    it('sends a plain http target through the CONNECT tunnel', async () => {
        // The proxy applies its checks to the tunnel. An absolute-form request forwarded to the proxy would take a
        // different path there, so a target that undici would not tunnel by default must still show up as CONNECT.
        const plainAuthority = `origin.test:${serverPort(plainOrigin)}`
        const response = await requestModule.fetch(`http://${plainAuthority}/plain`, { timeoutMs: 2000 })

        expect(await response.text()).toBe('/plain')
        expect(proxyAuthorities).toEqual([plainAuthority])
    }, 10000)

    describe('egress proxy rollout', () => {
        // resetModules gives `request` a fresh copy of the attribution store, so the test has to reach the caller
        // side through that same copy rather than through a top-level import.
        let attribution: AsyncLocalStorage<Record<string, unknown>>

        // The plain origin is reached by IP so the direct route needs no DNS.
        const loadWithProxyTeams = (proxyTeams: string): void => {
            process.env.EXTERNAL_REQUEST_PROXY_TEAMS = proxyTeams
            jest.resetModules()
            requestModule = require('./request') as RequestModule
            attribution = (require('./fetch-attribution') as typeof import('./fetch-attribution')).fetchAttribution
        }

        // A caller that sets no attribution at all, such as the session replay image lane. A team of undefined is a
        // caller that has attribution but no invocation. Both reach the matcher as team 0 once a rollout is set.
        const NO_ATTRIBUTION = 'no-attribution'
        type Caller = number | undefined | typeof NO_ATTRIBUTION

        const fetchAs = (caller: Caller, url: string, options = {}): Promise<FetchResponseLike> => {
            const run = (): Promise<FetchResponseLike> => requestModule.fetch(url, { timeoutMs: 2000, ...options })
            return caller === NO_ATTRIBUTION ? run() : attribution.run({ teamId: caller }, run)
        }

        afterEach(() => {
            delete process.env.EXTERNAL_REQUEST_PROXY_TEAMS
        })

        it.each<[string, Caller, boolean]>([
            // An unset value means the deployment is not in the rollout, so every caller keeps the proxy.
            ['', 2, true],
            ['', NO_ATTRIBUTION, true],
            ['2', 2, true],
            ['2', 3, false],
            ['2', undefined, false],
            ['2', NO_ATTRIBUTION, false],
            ['2,*:1', 3, true],
            ['*', 3, true],
            ['*:1', undefined, true],
        ])(
            'with EXTERNAL_REQUEST_PROXY_TEAMS=%j and caller %s, proxied: %s',
            async (proxyTeams, caller, expectProxied) => {
                loadWithProxyTeams(proxyTeams)
                const authority = `127.0.0.1:${serverPort(plainOrigin)}`

                const response = await fetchAs(caller, `http://${authority}/routing`)

                expect(await response.text()).toBe('/routing')
                expect(proxyAuthorities).toEqual(expectProxied ? [authority] : [])
            },
            10000
        )

        it.each([['*:10'], ['*:bad'], ['2,*:'], ['*:0.1,*:10']])(
            'refuses to start with EXTERNAL_REQUEST_PROXY_TEAMS=%j',
            (proxyTeams) => {
                // A fraction read as a percentage would proxy every team at once, which is what the staged rollout exists
                // to prevent.
                expect(() => loadWithProxyTeams(proxyTeams)).toThrow(/takes a fraction between 0 and 1/)
            }
        )

        // legacyFetch picks its own dispatcher instead of taking one from `fetch`, so it needs its own coverage.
        // A shared agent assigned there directly would pin the helper to one route whatever the rollout says.
        it.each<[Caller, boolean]>([
            [2, true],
            [3, false],
            [NO_ATTRIBUTION, false],
        ])(
            'routes legacyFetch for caller %s through the proxy: %s',
            async (caller, expectProxied) => {
                loadWithProxyTeams('2')
                const authority = `127.0.0.1:${serverPort(plainOrigin)}`
                const run = (): Promise<LegacyResponseLike> => requestModule.legacyFetch(`http://${authority}/legacy`)

                const response = await (caller === NO_ATTRIBUTION ? run() : attribution.run({ teamId: caller }, run))

                expect(await response.text()).toBe('/legacy')
                expect(proxyAuthorities).toEqual(expectProxied ? [authority] : [])
            },
            10000
        )

        it('keeps the two routes on separate HTTP/2 dispatchers for one idle timeout', async () => {
            loadWithProxyTeams('2')
            const authority = `127.0.0.1:${serverPort(plainOrigin)}`
            const options = { allowH2: true, http2IdleTimeoutMs: 42_000 }

            const proxied = await fetchAs(2, `http://${authority}/proxied`, options)
            const direct = await fetchAs(3, `http://${authority}/direct`, options)

            expect(await proxied.text()).toBe('/proxied')
            expect(await direct.text()).toBe('/direct')
            // A single dispatcher map keyed only by the idle timeout would hand the direct request the proxied
            // dispatcher, and the second authority would show up here.
            expect(proxyAuthorities).toEqual([authority])
        }, 10000)
    })

    it('carries a burst to a cold origin on one session', async () => {
        const http2Url = `https://origin.test:${serverPort(http2Origin)}`
        const paths = Array.from({ length: 6 }, (_, index) => `/burst-${index}`)

        const bodies = await Promise.all(
            paths.map(async (path) => {
                const response = await requestModule.fetchStreamed(`${http2Url}${path}`, {
                    allowH2: true,
                    timeoutMs: 2000,
                })
                return (await response.read(100)).bytes.toString()
            })
        )

        expect(bodies).toEqual(paths)
        // The pool cap is 4, so only the cold-start gate can hold six requests on one session.
        expect(http2SessionCount).toBe(1)
    }, 10000)

    it('keeps requests of different connection groups on separate sessions', async () => {
        const http2Url = `https://origin.test:${serverPort(http2Origin)}`
        const send = async (path: string, group: string): Promise<string> => {
            const response = await requestModule.fetch(`${http2Url}${path}`, {
                allowH2: true,
                http2ConnectionGroup: group,
            })
            return await response.text()
        }

        expect(await send('/team-a-1', 'team-a')).toBe('/team-a-1')
        expect(await send('/team-a-2', 'team-a')).toBe('/team-a-2')
        expect(http2SessionCount).toBe(1)

        expect(await send('/team-b-1', 'team-b')).toBe('/team-b-1')
        expect(http2SessionCount).toBe(2)
    }, 10000)

    it('shares one client for six cold image fetches and closes the idle client and its connections', async () => {
        const { HttpImageFetcher } =
            require('~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher') as typeof import('~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher')
        const { Client: UndiciClient } = jest.requireActual<typeof import('undici')>('undici')
        const clients = new Set<Client>()
        const dispatch = UndiciClient.prototype.dispatch
        const dispatchSpy = jest.spyOn(UndiciClient.prototype, 'dispatch').mockImplementation(function (
            this: Client,
            options,
            handler
        ) {
            if (options.path.startsWith('/image-burst-')) {
                clients.add(this)
            }
            return dispatch.call(this, options, handler)
        })
        const fetcher = new HttpImageFetcher(
            { maxUrlLength: 2048, isPublicHost: () => true },
            { headersForGet: () => ({}) }
        )
        const options: ImageFetchOptions = {
            maxBytes: 1000,
            timeoutMs: 2000,
            maxRedirects: 0,
            isDifferentOrigin: () => false,
            scheduleRequest: async (_url, _deadlineMs, request) => ({ ran: true, value: await request() }),
            checkRedirectPolicy: () => Promise.resolve({ allowed: true, tdmrepReservation: false }),
            tdmrepReservation: false,
        }
        const authority = `origin.test:${serverPort(http2Origin)}`
        const fetchBurst = async (): Promise<void> => {
            const results = await Promise.all(
                Array.from({ length: 6 }, (_, index) =>
                    fetcher.fetch(`https://${authority}/image-burst-${index}.png`, options)
                )
            )
            expect(results).toEqual(
                Array.from({ length: 6 }, () => expect.objectContaining({ outcome: 'ok', bytes: imageBytes }))
            )
        }

        try {
            await fetchBurst()

            expect(clients.size).toBe(1)
            expect(http2SessionCount).toBe(1)
            expect(http2OriginProtocols).toEqual(Array(6).fill('2.0'))
            expect(proxyAuthorities).toEqual([authority])
            expect(openHttp2Sessions.size).toBe(1)
            expect(openSockets.size).toBe(2)
            const [idleClient] = clients

            await waitForExpect(() => {
                expect(idleClient.destroyed).toBe(true)
                expect(openHttp2Sessions.size).toBe(0)
                expect(openSockets.size).toBe(0)
            }, keepAliveTimeoutMs * 3)

            await fetchBurst()

            expect(clients.size).toBe(2)
            expect(http2SessionCount).toBe(2)
            expect(http2OriginProtocols).toEqual(Array(12).fill('2.0'))
            expect(proxyAuthorities).toEqual([authority, authority])
            expect(openHttp2Sessions.size).toBe(1)
            expect(openSockets.size).toBe(2)
        } finally {
            dispatchSpy.mockRestore()
        }
    }, 15000)

    it('still fans a burst out to an origin that negotiates HTTP/1.1', async () => {
        const http1Url = `https://origin.test:${serverPort(http1Origin)}`
        const paths = ['/h1-burst-a', '/h1-burst-b', '/h1-burst-c']

        const bodies = await Promise.all(
            paths.map(async (path) => {
                const response = await requestModule.fetchStreamed(`${http1Url}${path}`, {
                    allowH2: true,
                    timeoutMs: 2000,
                })
                return (await response.read(100)).bytes.toString()
            })
        )

        expect(bodies).toEqual(paths)
        expect(http1OriginProtocols).toEqual(['1.1', '1.1', '1.1'])
        // The probe finds HTTP/1.1, so the released requests open their own connections instead of queueing.
        expect(http1ConnectionCount).toBeGreaterThanOrEqual(2)
    }, 10000)

    it('releases a held burst when the probe request fails', async () => {
        const closedPort = await new Promise<number>((resolve) => {
            const probe = net.createServer()
            probe.listen(0, '127.0.0.1', () => {
                const port = (probe.address() as AddressInfo).port
                probe.close(() => resolve(port))
            })
        })
        const deadUrl = `https://origin.test:${closedPort}`

        const results = await Promise.allSettled(
            ['/a', '/b', '/c'].map((path) =>
                requestModule.fetchStreamed(`${deadUrl}${path}`, { allowH2: true, timeoutMs: 2000 })
            )
        )

        // Held requests must not wait for their own timeout, which only starts once they are released.
        expect(results.map((result) => result.status)).toEqual(['rejected', 'rejected', 'rejected'])
    }, 10000)

    it('opens a new session after the origin sends GOAWAY', async () => {
        const http2Url = `https://origin.test:${serverPort(http2Origin)}`
        const goawayResponse = await requestModule.fetchStreamed(`${http2Url}/goaway`, {
            allowH2: true,
            timeoutMs: 2000,
        })
        expect((await goawayResponse.read(100)).bytes.toString()).toBe('/goaway')
        const sessionsBeforeReconnect = http2SessionCount

        const afterGoawayResponse = await requestModule.fetchStreamed(`${http2Url}/after-goaway`, {
            allowH2: true,
            timeoutMs: 2000,
        })
        expect((await afterGoawayResponse.read(100)).bytes.toString()).toBe('/after-goaway')

        expect(http2SessionCount).toBe(sessionsBeforeReconnect + 1)
    }, 10000)

    it('closes an idle session after the keep-alive timeout', async () => {
        const http2Url = `https://origin.test:${serverPort(http2Origin)}`
        const response = await requestModule.fetchStreamed(`${http2Url}/idle`, { allowH2: true, timeoutMs: 2000 })
        expect((await response.read(100)).bytes.toString()).toBe('/idle')
        expect(openHttp2Sessions.size).toBeGreaterThan(0)

        await waitForExpect(() => expect(openHttp2Sessions.size).toBe(0), keepAliveTimeoutMs * 3)
    }, 15000)

    it('keeps a session open for the idle timeout a caller asks for', async () => {
        const http2Url = `https://origin.test:${serverPort(http2Origin)}`
        const [defaultResponse, patientResponse] = await Promise.all([
            requestModule.fetchStreamed(`${http2Url}/default-idle`, { allowH2: true, timeoutMs: 2000 }),
            requestModule.fetchStreamed(`${http2Url}/patient-idle`, {
                allowH2: true,
                timeoutMs: 2000,
                http2IdleTimeoutMs: 60_000,
            }),
        ])
        expect((await defaultResponse.read(100)).bytes.toString()).toBe('/default-idle')
        expect((await patientResponse.read(100)).bytes.toString()).toBe('/patient-idle')
        expect(openHttp2Sessions.size).toBe(2)

        // The default session closes first. The caller's session must outlive it.
        await waitForExpect(() => expect(openHttp2Sessions.size).toBe(1), keepAliveTimeoutMs * 3)
        expect(http2SessionCount).toBe(2)
    }, 15000)

    it('closes open sessions and proxy tunnels when the shared agents shut down', async () => {
        const http2Url = `https://origin.test:${serverPort(http2Origin)}`
        const response = await requestModule.fetchStreamed(`${http2Url}/shutdown`, {
            allowH2: true,
            timeoutMs: 2000,
        })
        expect((await response.read(100)).bytes.toString()).toBe('/shutdown')
        expect(openHttp2Sessions.size).toBeGreaterThan(0)
        const requestedAt = Date.now()

        await requestModule.closeSharedAgents()

        await waitForExpect(() => {
            expect(openHttp2Sessions.size).toBe(0)
            expect(openSockets.size).toBe(0)
        }, 2000)
        // The wait ended well inside the keep-alive timeout, so the shutdown closed the session and not the idle timer.
        expect(Date.now() - requestedAt).toBeLessThan(keepAliveTimeoutMs)

        // Every path rejects the same way after the agents close. undici guards the destroyed HTTP/1.1 and HTTP/2
        // dispatchers. The closed flag covers a timeout that has no dispatcher yet.
        const destroyed = { name: 'ClientDestroyedError', code: 'UND_ERR_DESTROYED' }
        await expect(requestModule.fetch(`${http2Url}/after-close`, { timeoutMs: 2000 })).rejects.toMatchObject(
            destroyed
        )
        await expect(
            requestModule.fetchStreamed(`${http2Url}/after-close`, { allowH2: true, timeoutMs: 2000 })
        ).rejects.toMatchObject(destroyed)
        await expect(
            requestModule.fetchStreamed(`${http2Url}/after-close`, {
                allowH2: true,
                timeoutMs: 2000,
                http2IdleTimeoutMs: 42_000,
            })
        ).rejects.toMatchObject(destroyed)
    }, 10000)
})