:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { LookupAddress } from 'dns'
import dns from 'dns/promises'
import * as ipaddr from 'ipaddr.js'
import { LRUCache } from 'lru-cache'
import { readFileSync } from 'node:fs'
import net from 'node:net'
import { Counter, Gauge, Histogram } from 'prom-client'
// eslint-disable-next-line no-restricted-imports
import {
Agent,
Dispatcher,
type HeadersInit,
Pool,
ProxyAgent,
RequestInfo,
RequestInit,
Response,
request,
errors as undiciErrors,
fetch as undiciFetch,
} from 'undici'
import { URL } from 'url'
import { getExternalRequestConfig } from '~/common/config'
import { buildIntegerMatcherWithPercentage } from '~/common/config/matchers'
import { ColdStartGate } from './cold-start-gate'
import { isProdEnv } from './env-utils'
import { fetchAttribution } from './fetch-attribution'
import { parseJSON } from './json-parse'
import { logger } from './logger'
import { noteOobDomain } from './oob-domains'
const requestConfig = getExternalRequestConfig()
// eslint-disable-next-line no-restricted-imports
export { Response } from 'undici'
const unsafeRequestCounter = new Counter({
name: 'node_request_unsafe',
help: 'Total number of unsafe requests detected and blocked',
labelNames: ['reason'],
})
// The only signal on this side that says which way a request left the pod. Smokescreen records the destination of a
// tunnel it carries, but nothing here otherwise separates a proxied request from a direct one.
const externalRequestRouteCounter = new Counter({
name: 'node_external_request_route_total',
help: 'Third-party requests by egress route, proxy or direct',
labelNames: ['route'],
})
// Gauge tracking the number of external HTTP requests currently in flight.
// This is the primary scaling signal for the cdp-cyclotron-worker: it directly
// measures I/O saturation rather than CPU (which stays low while waiting on responses)
// or batch utilization (which measures demand, not capacity).
const inflightExternalRequests = new Gauge({
name: 'cdp_http_inflight_requests',
help: 'Number of currently inflight external HTTP requests (undici). Use as HPA scaling metric for cdp-cyclotron-worker.',
})
// The connect timeout is a single budget covering DNS resolution, TCP and TLS, and only the total request duration is
// measured elsewhere, so a connect timeout does not say which phase consumed the budget. Resolution is the phase that
// runs in this process, so it is the one that can be measured here. The duration separates a slow resolver path from a
// fast one, and the in-flight gauge separates lookups that queue behind other lookups from lookups that are themselves
// slow. Neither carries the hostname as a label, because destination hostnames are customer-supplied and unbounded.
const dnsLookupDuration = new Histogram({
name: 'node_dns_lookup_duration_ms',
help: 'Duration of dns.lookup() for an external hostname, in milliseconds',
labelNames: ['outcome'],
buckets: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2000, 3000, 5000, 10000],
})
const dnsLookupsInFlight = new Gauge({
name: 'node_dns_lookups_in_flight',
help: 'Number of dns.lookup() calls currently awaiting a result',
})
const dnsNegativeCacheCounter = new Counter({
name: 'node_dns_negative_cache_total',
help: 'Lookups the DNS negative cache skipped because the hostname recently returned ENOTFOUND',
labelNames: ['result'],
})
// Only ENOTFOUND is cached: a timeout or SERVFAIL can be the resolver's fault, and caching it would fail healthy
// destinations while the resolver is under load. The key is the hostname, which is customer-supplied and unbounded, so
// the entry count is capped as well as the age.
const dnsNegativeCache = new LRUCache<string, true>({ max: 10_000, ttl: 10_000 })
// The hosts file matches a name exactly, so 'alias.test.' misses an 'alias.test' entry from Docker's --add-host or
// Kubernetes hostAliases.
function readHostsFileNames(): Set<string> {
try {
return new Set(
readFileSync('/etc/hosts', 'utf8')
.split('\n')
.flatMap((line) => line.replace(/#.*/, '').trim().split(/\s+/).slice(1))
.map((name) => name.toLowerCase())
)
} catch {
return new Set()
}
}
const hostsFileNames = readHostsFileNames()
// The trailing dot skips the cluster search domains, so a missing hostname costs one query instead of one per domain.
// A single-label name such as a Docker service name resolves only through the search domains, so it stays as given.
// dns.lookup returns an IP literal without a DNS query, but it treats '1.1.1.1.' as a name that fails with ENOTFOUND.
function dnsLookupName(hostname: string): string {
return hostname.includes('.') &&
!hostname.endsWith('.') &&
net.isIP(hostname) === 0 &&
!hostsFileNames.has(hostname.toLowerCase())
? `${hostname}.`
: hostname
}
// NOTE: This isn't exactly fetch - it's meant to be very close but limited to only options we actually want to expose
export type FetchOptions = {
method?: string
headers?: HeadersInit
body?: string | Buffer
timeoutMs?: number
// The TLS handshake offers HTTP/2 and HTTP/1.1. The origin selects the protocol. undici lists HTTP/1.1 first, so an
// origin that follows the client's order selects HTTP/1.1. APNs supports only HTTP/2, so it selects HTTP/2. The
// undici connect option `preferH2` lists HTTP/2 first.
allowH2?: boolean
// The time an idle HTTP/2 session to the origin stays open. The default is the keep-alive timeout. An origin that
// negotiates HTTP/1.1 uses the same dispatcher, so its idle sockets also close after this time. Pass a constant:
// each distinct value keeps a dispatcher for the life of the process, and at most 8 values are allowed.
http2IdleTimeoutMs?: number
// Requests with different groups never share an HTTP/2 session. APNs can reject a provider token from one developer
// team on a session that already carried another team's token. Requires `allowH2`.
http2ConnectionGroup?: string
maxResponseBytes?: number
}
export type FetchResponse = {
status: number
headers: Record<string, string>
json: () => Promise<any>
text: () => Promise<string>
dump: () => Promise<void>
}
// These extend Error, not undici's UndiciError: UndiciError defines a code-based Symbol.hasInstance, so every
// UndiciError subclass matches `instanceof` against every other one, which would defeat isFetchResponseRetriable.
export class SecureRequestError extends Error {
constructor(message: string) {
super(message)
this.name = 'SecureRequestError'
}
}
export class InvalidRequestError extends Error {
constructor(message: string) {
super(message)
this.name = 'InvalidRequestError'
}
}
export class ResponseTooLargeError extends Error {
constructor(maxBytes: number) {
super(`Response body is larger than the ${maxBytes} byte limit`)
this.name = 'ResponseTooLargeError'
}
}
export class ResolutionError extends Error {
constructor(message: string) {
super(message)
this.name = 'ResolutionError'
}
}
// Logged only when a check blocks the request. The check runs inside undici's connect flow
// where the thrown error can't carry caller context, so attribution comes from the ALS store;
// it reflects the request that opened the connection (keep-alive reuse skips the check).
function logBlockedUrlValidation(hostname: string, resolvedIps: string[]): void {
logger.warn('[SSRF] Request blocked by URL validation check', {
hostname,
resolvedIps,
...fetchAttribution.getStore(),
})
}
function validateUrl(url: string): URL {
// Raise if the provided URL seems unsafe, otherwise do nothing.
let parsedUrl: URL
try {
parsedUrl = new URL(url)
} catch {
throw new InvalidRequestError('Invalid URL')
}
const { hostname, protocol } = parsedUrl
if (!hostname) {
throw new InvalidRequestError('No hostname')
}
if (!['http:', 'https:'].includes(protocol)) {
throw new InvalidRequestError('Scheme must be either HTTP or HTTPS')
}
return parsedUrl
}
/**
* Validate IP literal hostnames directly. Undici skips the DNS lookup callback
* for IP literals (both IPv4 and IPv6), so staticLookupAsync never runs for them.
* We must check these before passing the URL to undici.
*/
function validateHostnameIPLiteral(hostname: string, allowUnsafe: boolean): void {
// Above the early return on purpose: every caller passes `!isProdEnv()`, so a check below it
// would never run in a test and would reach production unexercised.
noteOobDomain(hostname, 'node_request', fetchAttribution.getStore())
if (allowUnsafe) {
return
}
// Strip brackets from IPv6 literals — URL.hostname includes them for IPv6
const bare = hostname.startsWith('[') && hostname.endsWith(']') ? hostname.slice(1, -1) : hostname
let parsed: ipaddr.IPv4 | ipaddr.IPv6
try {
parsed = ipaddr.parse(bare)
} catch {
// Not an IP literal (it's a regular hostname) — DNS lookup will handle validation
return
}
let ipv4: ipaddr.IPv4 | null = null
if (isIPv4(parsed)) {
ipv4 = parsed
} else if (parsed.isIPv4MappedAddress()) {
ipv4 = parsed.toIPv4Address()
} else {
if (!isGlobalIPv6(parsed)) {
unsafeRequestCounter.inc({ reason: 'internal_ip_literal' })
logBlockedUrlValidation(hostname, [bare])
throw new SecureRequestError('Hostname is not allowed')
}
return
}
if (!isGlobalIPv4(ipv4)) {
unsafeRequestCounter.inc({ reason: 'internal_ip_literal' })
logBlockedUrlValidation(hostname, [bare])
throw new SecureRequestError('Hostname is not allowed')
}
}
function isGlobalIPv4(ip: ipaddr.IPv4): boolean {
const [a, b, c, d] = ip.octets
if (a === 0) {
return false // "This network" (0.0.0.0/8)
}
if (ip.range() !== 'unicast') {
return false // Non-unicast addresses
}
if (a === 127) {
return false // Loopback (127.0.0.0/8)
}
if (a === 169 && b === 254) {
return false // Link-local (169.254.0.0/16)
}
if (a === 255 && b === 255 && c === 255 && d === 255) {
return false // Broadcast
}
return true
}
function isGlobalIPv6(ip: ipaddr.IPv6): boolean {
const range = ip.range()
// Only allow globally routable unicast IPv6 addresses
return range === 'unicast'
}
function isIPv4(addr: ipaddr.IPv4 | ipaddr.IPv6): addr is ipaddr.IPv4 {
return addr.kind().toLowerCase() === 'ipv4'
}
async function staticLookupAsync(hostname: string): Promise<LookupAddress[]> {
let addrinfo: LookupAddress[]
const validAddrinfo: LookupAddress[] = []
const cacheKey = hostname.toLowerCase().replace(/\.$/, '')
if (dnsNegativeCache.has(cacheKey)) {
dnsNegativeCacheCounter.inc({ result: 'hit' })
throw new ResolutionError('Invalid hostname')
}
const lookupStartedAt = performance.now()
dnsLookupsInFlight.inc()
try {
addrinfo = await dns.lookup(dnsLookupName(hostname), { all: true })
dnsLookupDuration.observe({ outcome: 'success' }, performance.now() - lookupStartedAt)
} catch (error) {
dnsLookupDuration.observe({ outcome: 'failure' }, performance.now() - lookupStartedAt)
if ((error as NodeJS.ErrnoException)?.code === 'ENOTFOUND') {
dnsNegativeCache.set(cacheKey, true)
}
throw new ResolutionError('Invalid hostname')
} finally {
dnsLookupsInFlight.dec()
}
// A lookup that started before another one cached the failure can still succeed. The hostname resolves now.
dnsNegativeCache.delete(cacheKey)
const resolvedIps = addrinfo.map((a) => a.address)
for (const addrInfo of addrinfo) {
const parsed = ipaddr.parse(addrInfo.address)
let ipv4: ipaddr.IPv4 | null = null
if (isIPv4(parsed)) {
ipv4 = parsed
} else if (parsed.isIPv4MappedAddress()) {
// IPv6-mapped IPv4 (e.g. ::ffff:169.254.169.254) must be unwrapped and validated
ipv4 = parsed.toIPv4Address()
} else {
// Pure IPv6 — validate directly
const allowUnsafe = !isProdEnv()
if (!allowUnsafe && !isGlobalIPv6(parsed)) {
unsafeRequestCounter.inc({ reason: 'internal_hostname' })
logBlockedUrlValidation(hostname, resolvedIps)
throw new SecureRequestError('Hostname is not allowed')
}
validAddrinfo.push(addrInfo)
continue
}
// TRICKY: We need this for tests and local dev
const allowUnsafe = !isProdEnv()
// Check if the IPv4 address is global
if (!allowUnsafe && !isGlobalIPv4(ipv4)) {
unsafeRequestCounter.inc({ reason: 'internal_hostname' })
logBlockedUrlValidation(hostname, resolvedIps)
throw new SecureRequestError('Hostname is not allowed')
}
validAddrinfo.push(addrInfo)
}
if (validAddrinfo.length === 0) {
unsafeRequestCounter.inc({ reason: 'unable_to_resolve' })
logBlockedUrlValidation(hostname, resolvedIps)
throw new ResolutionError(`Unable to resolve ${hostname}`)
}
return validAddrinfo
}
export const httpStaticLookup: net.LookupFunction = async (hostname, _options, cb) => {
try {
const addrinfo = await staticLookupAsync(hostname)
cb(null, addrinfo)
} catch (err) {
cb(err as Error, '', 4)
}
}
/**
* Legacy function used by parts of the codebase. Generally speaking this should be replaced with secureFetch.
*/
export async function raiseIfUserProvidedUrlUnsafe(url: string): Promise<void> {
const parsedUrl = validateUrl(url)
validateHostnameIPLiteral(parsedUrl.hostname, !isProdEnv())
await staticLookupAsync(parsedUrl.hostname)
}
// Safe way to use the same helpers for talking to internal endpoints such as other services
class InsecureAgent extends Agent {
constructor() {
super({
keepAliveTimeout: requestConfig.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS,
connections: requestConfig.EXTERNAL_REQUEST_CONNECTIONS,
// undici enables HTTP/2 by default. Internal services stay on HTTP/1.1 because nothing here needs multiplexing. HTTP/2 sessions also fail in different ways, with GOAWAY replays and refused streams.
allowH2: false,
connect: {
timeout: requestConfig.EXTERNAL_REQUEST_CONNECT_TIMEOUT_MS,
},
})
}
}
const proxyUrl =
process.env.HTTPS_PROXY || process.env.HTTP_PROXY || process.env.https_proxy || process.env.http_proxy || ''
// The percentage form rolls per call rather than per team, so a destination that fails through the proxy gets a fresh
// draw on its next retry.
const proxyTeamMatcher = buildIntegerMatcherWithPercentage(requestConfig.EXTERNAL_REQUEST_PROXY_TEAMS)
// The parser takes a fraction, so '*:10' reads as "always" rather than the ten percent an operator means by it, and
// '*:bad' reads as "never" while the rollout looks live. Both defeat the staged rollout silently, so fail at startup
// the way EXTERNAL_REQUEST_H2_CONNECTIONS does below. The parser keeps the last '*:' token it reads, so every one of
// them has to hold rather than only the first.
const proxyRolloutPercentages = requestConfig.EXTERNAL_REQUEST_PROXY_TEAMS.split(',')
.map((part) => part.trim())
.filter((part) => part.startsWith('*:'))
.map((part) => part.slice(2))
for (const rolloutPercentage of proxyRolloutPercentages) {
const percentage = Number(rolloutPercentage)
if (rolloutPercentage === '' || !Number.isFinite(percentage) || percentage < 0 || percentage > 1) {
throw new Error(
`EXTERNAL_REQUEST_PROXY_TEAMS takes a fraction between 0 and 1 after '*:', got '${rolloutPercentage}'`
)
}
}
const proxyRolloutConfigured = requestConfig.EXTERNAL_REQUEST_PROXY_TEAMS.trim().length > 0
// Only a deployment in the rollout sets EXTERNAL_REQUEST_PROXY_TEAMS. Everywhere else keeps the behavior from before
// the rollout existed, which the session replay image lane depends on to send customer URLs through the proxy.
// A deployment in the rollout takes the team from the attribution context the URL-validation logs already use, rather
// than from FetchOptions, so that routing stays out of the request shape every caller and its tests assert on.
function useProxyForTeam(): boolean {
if (!proxyUrl) {
return false
}
if (!proxyRolloutConfigured) {
return true
}
const teamId = fetchAttribution.getStore()?.teamId
// A caller with no team never matches a team in the list, but a percentage rollout still covers it.
return proxyTeamMatcher(typeof teamId === 'number' ? teamId : 0)
}
type SecureRoute = 'proxy' | 'direct'
const routeOf = (useProxy: boolean): SecureRoute => (useProxy ? 'proxy' : 'direct')
// A proxied request goes through a CONNECT tunnel. The proxy handles SSRF blocking (private IP rejection) at the
// network level, so we skip the DNS lookup (httpStaticLookup) which would be redundant. NOTE: undici's ProxyAgent
// does not read NO_PROXY, so a carve-out set in the environment does not apply here.
function makeSecureDispatcher({
useProxy,
allowH2,
keepAliveTimeoutMs = requestConfig.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS,
}: {
useProxy: boolean
allowH2: boolean
keepAliveTimeoutMs?: number
}): Dispatcher {
const connections = allowH2
? requestConfig.EXTERNAL_REQUEST_H2_CONNECTIONS
: requestConfig.EXTERNAL_REQUEST_CONNECTIONS
if (useProxy) {
return new ProxyAgent({
uri: proxyUrl,
keepAliveTimeout: keepAliveTimeoutMs,
connections,
// undici 8 tunnels only https targets by default. The proxy applies its checks to the tunnel, so a plain
// http target must not go to the proxy as an absolute-form request instead.
proxyTunnel: true,
allowH2,
requestTls: { allowH2 },
connectTimeout: requestConfig.EXTERNAL_REQUEST_CONNECT_TIMEOUT_MS,
// undici ignores the abort signal of a request until the request has a connection. So the proxy client needs its own limit on the wait for a CONNECT answer. This limit is the connect timeout of the direct route.
clientFactory: (origin, options) =>
new Pool(origin, { ...options, headersTimeout: requestConfig.EXTERNAL_REQUEST_CONNECT_TIMEOUT_MS }),
})
}
return new Agent({
keepAliveTimeout: keepAliveTimeoutMs,
connections,
allowH2,
connect: {
lookup: httpStaticLookup,
timeout: requestConfig.EXTERNAL_REQUEST_CONNECT_TIMEOUT_MS,
},
})
}
const sharedSecureAgent = makeSecureDispatcher({ useProxy: false, allowH2: false })
const sharedSecureProxyAgent = proxyUrl ? makeSecureDispatcher({ useProxy: true, allowH2: false }) : null
const sharedInsecureAgent = new InsecureAgent()
// undici only reads the value when the first HTTP/2 request builds its pool, and treats 0 as unbounded, so a bad
// value would surface late and as a retriable error. Failing here stops the process at startup instead.
if (
!Number.isInteger(requestConfig.EXTERNAL_REQUEST_H2_CONNECTIONS) ||
requestConfig.EXTERNAL_REQUEST_H2_CONNECTIONS < 1
) {
throw new Error(
`EXTERNAL_REQUEST_H2_CONNECTIONS must be a positive integer, got ${process.env.EXTERNAL_REQUEST_H2_CONNECTIONS}`
)
}
type SecureDispatcher = { dispatcher: Dispatcher; gate: ColdStartGate | null; release?: () => void }
type GroupedSecureDispatcher = { dispatcher: Dispatcher; gate: ColdStartGate; pending: number; evicted: boolean }
// undici sets the idle timeout per dispatcher, so each distinct idle timeout gets its own HTTP/2 dispatcher. The two
// routes keep separate maps so a caller cannot exhaust one route's budget with timeouts it only ever uses on the other.
const sharedSecureH2Agents: Record<SecureRoute, Map<number, SecureDispatcher>> = {
direct: new Map(),
proxy: new Map(),
}
const MAX_SECURE_H2_AGENTS = 8
const groupedSecureH2Agents = new Map<string, GroupedSecureDispatcher>()
const MAX_GROUPED_SECURE_H2_AGENTS = 1000
// An evicted dispatcher stays here until its close() finishes, so shutdown can still destroy one that holds an unread
// body.
const drainingGroupedSecureH2Agents = new Set<Dispatcher>()
// Node clamps a setTimeout delay above this value to 1 ms. A session would then close as soon as it goes idle.
const MAX_H2_IDLE_TIMEOUT_MS = 2_147_483_647
let sharedAgentsClosed = false
function assertValidH2IdleTimeout(idleTimeoutMs: number): void {
// InvalidRequestError is not retriable in cdp-fetch, so a value that can never work fails once.
if (!Number.isFinite(idleTimeoutMs) || idleTimeoutMs <= 0 || idleTimeoutMs > MAX_H2_IDLE_TIMEOUT_MS) {
throw new InvalidRequestError(`http2IdleTimeoutMs must be a number between 1 and ${MAX_H2_IDLE_TIMEOUT_MS}`)
}
}
function getSecureH2Agent(
useProxy: boolean,
idleTimeoutMs = requestConfig.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS
): SecureDispatcher {
assertValidH2IdleTimeout(idleTimeoutMs)
const agents = sharedSecureH2Agents[routeOf(useProxy)]
let agent = agents.get(idleTimeoutMs)
if (!agent) {
if (sharedAgentsClosed) {
throw new undiciErrors.ClientDestroyedError()
}
if (agents.size >= MAX_SECURE_H2_AGENTS) {
throw new InvalidRequestError(`http2IdleTimeoutMs takes at most ${MAX_SECURE_H2_AGENTS} distinct values`)
}
agent = {
dispatcher: makeSecureDispatcher({ useProxy, allowH2: true, keepAliveTimeoutMs: idleTimeoutMs }),
gate: new ColdStartGate(idleTimeoutMs),
}
agents.set(idleTimeoutMs, agent)
}
return agent
}
function getGroupedSecureH2Agent(
useProxy: boolean,
group: string,
idleTimeoutMs = requestConfig.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS
): SecureDispatcher {
assertValidH2IdleTimeout(idleTimeoutMs)
const key = `${routeOf(useProxy)}:${idleTimeoutMs}:${group}`
let agent = groupedSecureH2Agents.get(key)
if (agent) {
groupedSecureH2Agents.delete(key)
} else {
if (sharedAgentsClosed) {
throw new undiciErrors.ClientDestroyedError()
}
agent = {
dispatcher: makeSecureDispatcher({ useProxy, allowH2: true, keepAliveTimeoutMs: idleTimeoutMs }),
gate: new ColdStartGate(idleTimeoutMs),
pending: 0,
evicted: false,
}
if (groupedSecureH2Agents.size >= MAX_GROUPED_SECURE_H2_AGENTS) {
const [oldestKey, oldest] = groupedSecureH2Agents.entries().next().value!
groupedSecureH2Agents.delete(oldestKey)
oldest.evicted = true
drainingGroupedSecureH2Agents.add(oldest.dispatcher)
if (oldest.pending === 0) {
closeDrainingAgent(oldest.dispatcher)
}
}
}
groupedSecureH2Agents.set(key, agent)
// A request can wait in the cold-start gate before it reaches undici, and close() rejects a request dispatched
// after it runs. So an evicted dispatcher closes only once every request that picked it has been dispatched.
const picked = agent
picked.pending++
let released = false
return {
dispatcher: picked.dispatcher,
gate: picked.gate,
release: () => {
if (released) {
return
}
released = true
picked.pending--
if (picked.evicted && picked.pending === 0) {
closeDrainingAgent(picked.dispatcher)
}
},
}
}
function closeDrainingAgent(dispatcher: Dispatcher): void {
void dispatcher
.close()
.catch(() => {})
.finally(() => drainingGroupedSecureH2Agents.delete(dispatcher))
}
function getSecureDispatcher(options: {
allowH2?: boolean
http2IdleTimeoutMs?: number
http2ConnectionGroup?: string
}): SecureDispatcher {
const useProxy = useProxyForTeam()
if (options.http2ConnectionGroup !== undefined && !options.allowH2) {
throw new InvalidRequestError('http2ConnectionGroup requires allowH2')
}
// useProxyForTeam only returns true when a proxy URL was configured, so the agent exists whenever it is needed.
const secure = options.allowH2
? options.http2ConnectionGroup !== undefined
? getGroupedSecureH2Agent(useProxy, options.http2ConnectionGroup, options.http2IdleTimeoutMs)
: getSecureH2Agent(useProxy, options.http2IdleTimeoutMs)
: {
dispatcher: useProxy && sharedSecureProxyAgent ? sharedSecureProxyAgent : sharedSecureAgent,
gate: null,
}
// Counted here because every route decision passes through this function. A throw from getSecureH2Agent leaves
// the request uncounted, which is right: it never picked a route.
externalRequestRouteCounter.inc({ route: routeOf(useProxy) })
return secure
}
// The timer only bounds the wait in closeSharedAgents. When close finishes first, an unref'd timer does not keep the
// process alive for the rest of the grace period.
function unrefDelay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms).unref())
}
/**
* Closes the shared dispatchers at shutdown. Idle keep-alive sockets and HTTP/2 sessions close now
* instead of at their idle timeout. `close` waits for in-flight requests to settle. A streamed body
* that no caller reads or discards holds its socket forever, so the grace period bounds the wait and
* `destroy` ends whatever remains.
*/
export async function closeSharedAgents(gracePeriodMs = 5000): Promise<void> {
sharedAgentsClosed = true
const agents = [
sharedSecureAgent,
...(sharedSecureProxyAgent ? [sharedSecureProxyAgent] : []),
sharedInsecureAgent,
...Object.values(sharedSecureH2Agents).flatMap((route) => [...route.values()].map((h2) => h2.dispatcher)),
...[...groupedSecureH2Agents.values()].map((h2) => h2.dispatcher),
...drainingGroupedSecureH2Agents,
]
const stillOpen = new Set(agents)
const closed = Promise.allSettled(agents.map((agent) => agent.close().finally(() => stillOpen.delete(agent)))).then(
() => true
)
const closedInTime = await Promise.race([closed, unrefDelay(gracePeriodMs).then(() => false)])
if (!closedInTime) {
logger.warn('[request] Destroying shared agents with requests still in flight at shutdown', {
gracePeriodMs,
openAgents: stillOpen.size,
})
}
await Promise.allSettled(agents.map((agent) => agent.destroy()))
}
function destroyBody(body: Dispatcher.ResponseData['body']): void {
try {
body.on('error', () => {})
body.destroy()
} catch {
// The body already ended, or another caller destroyed it.
}
}
/**
* The prototype is null because every key comes from the remote server, and `__proto__` on a plain
* object literal is a setter rather than a key.
*/
function flattenHeaders(raw: Dispatcher.ResponseData['headers']): Record<string, string> {
const headers: Record<string, string> = Object.create(null)
for (const [key, value] of Object.entries(raw)) {
const singleValue = Array.isArray(value) ? value[0] : value
if (singleValue) {
headers[key] = singleValue
}
}
return headers
}
/**
* Reads a response body stream and destroys it immediately after to release
* the underlying socket and its off-heap buffers. Without explicit destruction,
* undici holds onto these buffers until GC, and V8 never returns the ~64MB
* ArrayBuffer arenas they live in to the OS.
*/
async function readAndDestroyBody(body: Dispatcher.ResponseData['body']): Promise<string> {
const text = await body.text()
// After text() fully consumes the stream, destroy to release socket buffers.
// At this point the stream is already ended so destroy is a cleanup no-op,
// but it signals undici to release the underlying socket immediately.
destroyBody(body)
return text
}
// Hooks for the cold-start gate. The signal is created before the hold, so time spent held counts against the
// caller's timeout. onHeaders or onError runs once the request settles; onBodyDone runs when the caller has read or
// discarded the body.
type RequestLifecycle = {
signal: AbortSignal | undefined
onHeaders: () => void
onError: () => void
onBodyDone: () => void
}
export async function _fetch(
url: string,
options: FetchOptions = {},
dispatcher: Dispatcher,
defaultTimeoutMs: number = requestConfig.EXTERNAL_REQUEST_TIMEOUT_MS,
lifecycle?: RequestLifecycle
): Promise<FetchResponse> {
let parsed: URL
try {
parsed = new URL(url)
} catch {
throw new Error('Invalid URL')
}
if (!parsed.hostname || !(parsed.protocol === 'http:' || parsed.protocol === 'https:')) {
throw new Error('URL must have HTTP or HTTPS protocol and a valid hostname')
}
options.timeoutMs = options.timeoutMs ?? defaultTimeoutMs
let result: Dispatcher.ResponseData
try {
result = await request(parsed.toString(), {
method: options.method ?? 'GET',
headers: options.headers,
body: options.body,
dispatcher,
// request() does not follow redirects, so a response can never bounce to an unvalidated host
signal: lifecycle?.signal ?? (options.timeoutMs ? AbortSignal.timeout(options.timeoutMs) : undefined),
})
} catch (error) {
lifecycle?.onError()
throw error
}
lifecycle?.onHeaders()
const headers = flattenHeaders(result.headers)
// On first .text()/.json() call, read the full body and destroy the
// stream immediately after. This releases undici's socket buffers
// without waiting for GC.
let bodyPromise: Promise<string> | undefined
const readBody = (): Promise<string> => {
if (!bodyPromise) {
const maxBytes = options.maxResponseBytes
const read =
maxBytes === undefined
? readAndDestroyBody(result.body)
: readCappedBody(result.body, maxBytes, false).then(({ bytes, overLimit }) => {
if (overLimit) {
throw new ResponseTooLargeError(maxBytes)
}
const text = bytes.toString('utf8')
return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text
})
bodyPromise = read.finally(() => lifecycle?.onBodyDone())
}
return bodyPromise
}
return {
status: result.statusCode,
headers,
json: async () => parseJSON(await readBody()),
text: async () => await readBody(),
dump: () => {
if (!bodyPromise) {
bodyPromise = Promise.resolve('')
destroyBody(result.body)
lifecycle?.onBodyDone()
}
return Promise.resolve()
},
}
}
const coldStartGateCounter = new Counter({
name: 'node_request_cold_start_gate_total',
help: 'Requests the HTTP/2 cold-start gate saw: probes to a cold origin, requests held behind a probe, and probes that failed',
labelNames: ['event'],
})
async function gatedLifecycle(
gate: ColdStartGate | null,
origin: string,
signal: AbortSignal | undefined
): Promise<RequestLifecycle | undefined> {
if (!gate) {
return undefined
}
const { probe, release } = await gate.acquire(origin, signal)
coldStartGateCounter.inc({ event: probe ? 'probe' : 'held' })
return {
signal,
onHeaders: () => {
gate.touch(origin)
release()
},
onError: () => {
if (probe) {
coldStartGateCounter.inc({ event: 'probe_failed' })
}
release()
},
onBodyDone: () => gate.touch(origin),
}
}
export async function internalFetch(url: string, options: FetchOptions = {}): Promise<FetchResponse> {
return await _fetch(url, options, sharedInsecureAgent)
}
/**
* Requests to a host we don't run, meaning CDP destinations and anything else pointed at a
* customer-supplied URL. These take the third-party response budget, which is tunable separately
* from the internal-service one that `internalFetch` keeps; see
* DEFAULT_THIRD_PARTY_REQUEST_TIMEOUT_MS.
*/
export async function fetch(url: string, options: FetchOptions = {}): Promise<FetchResponse> {
const parsed = new URL(url)
validateHostnameIPLiteral(parsed.hostname, !isProdEnv())
inflightExternalRequests.inc()
let lifecycle: RequestLifecycle | undefined
let release: (() => void) | undefined
try {
const { dispatcher, gate, release: releaseDispatcher } = getSecureDispatcher(options)
release = releaseDispatcher
options.timeoutMs = options.timeoutMs ?? requestConfig.EXTERNAL_REQUEST_THIRD_PARTY_TIMEOUT_MS
const signal = options.timeoutMs ? AbortSignal.timeout(options.timeoutMs) : undefined
lifecycle = await gatedLifecycle(gate, parsed.origin, signal)
return await _fetch(url, options, dispatcher, requestConfig.EXTERNAL_REQUEST_THIRD_PARTY_TIMEOUT_MS, lifecycle)
} catch (error) {
// Releases the probe when a throw happens before request() runs. The release is idempotent.
lifecycle?.onError()
throw error
} finally {
release?.()
inflightExternalRequests.dec()
}
}
export type StreamedFetchOptions = {
headers?: HeadersInit
timeoutMs: number
allowH2?: boolean
http2IdleTimeoutMs?: number
http2ConnectionGroup?: string
}
export type StreamedResponse = {
status: number
headers: Record<string, string>
headerLines: Array<{ name: string; value: string }>
/**
* Reads at most `maxBytes`, then abandons the rest. `overLimit` says the response had more, and
* `bytes` then contains the first `maxBytes`, so parsers with a bounded-prefix rule can use it.
*/
read: (maxBytes: number, retainPrefixOnOverflow?: boolean) => Promise<{ bytes: Buffer; overLimit: boolean }>
discard: () => void
}
function orderedHeaderLines(raw: unknown): Array<{ name: string; value: string }> {
if (Array.isArray(raw)) {
const lines: Array<{ name: string; value: string }> = []
for (let index = 0; index + 1 < raw.length; index += 2) {
lines.push({ name: String(raw[index]).toLowerCase(), value: String(raw[index + 1]) })
}
return lines
}
if (!raw || typeof raw !== 'object') {
return []
}
return Object.entries(raw).flatMap(([name, value]) =>
(Array.isArray(value) ? value : [value]).flatMap((line) =>
line === undefined ? [] : [{ name: name.toLowerCase(), value: String(line) }]
)
)
}
function flattenHeaderLines(lines: Array<{ name: string; value: string }>): Record<string, string> {
const headers: Record<string, string> = Object.create(null)
for (const { name, value } of lines) {
headers[name] ??= value
}
return headers
}
/**
* Memory here follows the bytes that arrive, never the bytes a response claims.
*
* One buffer sized from `Content-Length` would halve the peak for an honest response, because the
* chunks and the concatenated copy exist together for a moment. It would also let an origin hold
* `maxBytes` for the whole request timeout, once for every request in flight, by declaring a large
* body and then sending almost nothing.
*/
async function readCappedBody(
body: Dispatcher.ResponseData['body'],
maxBytes: number,
retainPrefixOnOverflow: boolean
): Promise<{ bytes: Buffer; overLimit: boolean }> {
const chunks: Buffer[] = []
let total = 0
let overLimit = false
try {
for await (const chunk of body) {
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
const remaining = maxBytes - total
if (buffer.length > remaining) {
if (remaining > 0) {
chunks.push(buffer.subarray(0, remaining))
total += remaining
}
overLimit = true
if (!retainPrefixOnOverflow) {
chunks.length = 0
total = 0
}
break
}
chunks.push(buffer)
total += buffer.length
}
} finally {
destroyBody(body)
}
return { bytes: Buffer.concat(chunks, total), overLimit }
}
/**
* A third-party request whose caller reads the body under a byte limit.
*
* `fetch` above gives the caller `text()`, which buffers a whole body of any size. An origin can
* answer a request for a small image with gigabytes. Here the caller reads the status and the
* headers first, then sets a byte limit or abandons the body.
*
* This does not follow redirects, as `fetch` does not, so a response cannot bounce to a host that no
* check has seen. A caller that follows one must call this again for the new URL.
*
* The caller must call `read` or `discard`, and only one of them. The socket stays held until then.
*/
export async function fetchStreamed(url: string, options: StreamedFetchOptions): Promise<StreamedResponse> {
const parsed = validateUrl(url)
validateHostnameIPLiteral(parsed.hostname, !isProdEnv())
inflightExternalRequests.inc()
let result: Dispatcher.ResponseData
let lifecycle: RequestLifecycle | undefined
let release: (() => void) | undefined
try {
const { dispatcher, gate, release: releaseDispatcher } = getSecureDispatcher(options)
release = releaseDispatcher
const signal = AbortSignal.timeout(options.timeoutMs)
lifecycle = await gatedLifecycle(gate, parsed.origin, signal)
result = await request(parsed.toString(), {
method: 'GET',
headers: options.headers,
dispatcher,
signal,
responseHeaders: 'raw',
})
} catch (error) {
lifecycle?.onError()
inflightExternalRequests.dec()
throw error
} finally {
release?.()
}
lifecycle?.onHeaders()
// The gauge holds until the body is done, not until the headers arrive, because the body takes
// nearly all the time of an image request.
let settled = false
const settle = (): boolean => {
if (settled) {
return false
}
settled = true
lifecycle?.onBodyDone()
inflightExternalRequests.dec()
return true
}
const headerLines = orderedHeaderLines(result.headers)
const headers = flattenHeaderLines(headerLines)
return {
status: result.statusCode,
headers,
headerLines,
read: async (maxBytes: number, retainPrefixOnOverflow = true) => {
if (settled) {
return { bytes: Buffer.alloc(0), overLimit: false }
}
try {
return await readCappedBody(result.body, maxBytes, retainPrefixOnOverflow)
} finally {
settle()
}
},
discard: () => {
if (settle()) {
destroyBody(result.body)
}
},
}
}
// Legacy fetch implementation that exposes the entire fetch implementation
export function legacyFetch(input: RequestInfo, options?: RequestInit): Promise<Response> {
let parsed: URL
try {
parsed = typeof input === 'string' ? new URL(input) : input instanceof URL ? input : new URL(input.url)
} catch {
throw new Error('Invalid URL')
}
if (!parsed.hostname || !(parsed.protocol === 'http:' || parsed.protocol === 'https:')) {
throw new Error('URL must have HTTP or HTTPS protocol and a valid hostname')
}
validateHostnameIPLiteral(parsed.hostname, !isProdEnv())
const requestOptions = options ?? {}
requestOptions.dispatcher = getSecureDispatcher({}).dispatcher
requestOptions.signal = AbortSignal.timeout(requestConfig.EXTERNAL_REQUEST_THIRD_PARTY_TIMEOUT_MS)
return undiciFetch(parsed.toString(), requestOptions)
}