Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(cdp): always use absolute dns lookups and the dns negative cache (#125571)

Daniel Marchuk887a9fa112d8
diff --git a/nodejs/src/common/config.ts b/nodejs/src/common/config.ts
index e6b0b893c114..46c364167caa 100644
--- a/nodejs/src/common/config.ts
+++ b/nodejs/src/common/config.ts
@@ -236,17 +236,6 @@ export type CommonConfig = BaseServerConfig & {
// Takes the buildIntegerMatcherWithPercentage syntax: '2' for team 2 only, '2,*:0.1' for team 2 plus a tenth of
// everyone else's requests, '*' for all.
EXTERNAL_REQUEST_PROXY_TEAMS: string
- // Resolve third-party hostnames as absolute names (with a trailing dot) on the direct route. The resolver then
- // skips the cluster search domains, so a hostname that does not exist costs one query instead of one per search
- // domain. A customer-supplied hostname never names a cluster service, so the search domains never help it.
- EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP: boolean
- // A process-local cache of hostnames that returned ENOTFOUND on the direct route. 'off' disables it. 'shadow'
- // still runs every lookup and only counts the lookups the cache would have skipped. 'enforce' fails those lookups
- // at once with the same ResolutionError. Only ENOTFOUND is cached: a timeout or SERVFAIL can be the resolver's
- // fault, and caching it would fail healthy destinations while the resolver is under load.
- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: string
- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS: number
- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES: number
// PostHog analytics
POSTHOG_API_KEY: string
@@ -287,10 +276,6 @@ export type ExternalRequestConfig = Pick<
| 'EXTERNAL_REQUEST_CONNECTIONS'
| 'EXTERNAL_REQUEST_H2_CONNECTIONS'
| 'EXTERNAL_REQUEST_PROXY_TEAMS'
- | 'EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP'
- | 'EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE'
- | 'EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS'
- | 'EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES'
>
export function getExternalRequestConfig(): ExternalRequestConfig {
@@ -304,14 +289,6 @@ export function getExternalRequestConfig(): ExternalRequestConfig {
EXTERNAL_REQUEST_CONNECTIONS: Number(process.env.EXTERNAL_REQUEST_CONNECTIONS ?? 500),
EXTERNAL_REQUEST_H2_CONNECTIONS: Number(process.env.EXTERNAL_REQUEST_H2_CONNECTIONS ?? 8),
EXTERNAL_REQUEST_PROXY_TEAMS: process.env.EXTERNAL_REQUEST_PROXY_TEAMS ?? '',
- EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP: process.env.EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP === 'true',
- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: process.env.EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE ?? 'off',
- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS: Number(
- process.env.EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS ?? 10_000
- ),
- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES: Number(
- process.env.EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES ?? 10_000
- ),
}
}
@@ -470,10 +447,6 @@ export function getDefaultCommonConfig(): CommonConfig {
EXTERNAL_REQUEST_CONNECTIONS: 500,
EXTERNAL_REQUEST_H2_CONNECTIONS: 8,
EXTERNAL_REQUEST_PROXY_TEAMS: '',
- EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP: false,
- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: 'off',
- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS: 10_000,
- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES: 10_000,
// PostHog analytics
POSTHOG_API_KEY: '',
diff --git a/nodejs/src/common/utils/request.test.ts b/nodejs/src/common/utils/request.test.ts
index 7c4037ac8416..b6dda770c17d 100644
--- a/nodejs/src/common/utils/request.test.ts
+++ b/nodejs/src/common/utils/request.test.ts
@@ -472,88 +472,73 @@ type IsolatedRequest = {
readNegativeCacheCounter: (result: string) => Promise<number>
}
-// request.ts reads the DNS settings once, at module load, so each case loads a fresh copy with its own environment.
-async function withDnsConfig(env: Record<string, string>, run: (isolated: IsolatedRequest) => Promise<void>) {
- const originalValues = Object.fromEntries(Object.keys(env).map((name) => [name, process.env[name]]))
- Object.assign(process.env, env)
- try {
- await jest.isolateModulesAsync(async () => {
- const request: typeof import('./request') = require('./request')
- const lookup = jest.mocked(require('dns/promises').lookup)
- const registry: typeof register = require('prom-client').register
- const readNegativeCacheCounter = async (result: string): Promise<number> =>
- (await registry.getSingleMetric('node_dns_negative_cache_total')!.get()).values.find(
- (value) => value.labels.result === result
- )?.value ?? 0
- try {
- await run({ request, lookup, readNegativeCacheCounter })
- } finally {
- await request.closeSharedAgents(100)
- }
- })
- } finally {
- for (const [name, value] of Object.entries(originalValues)) {
- if (value === undefined) {
- delete process.env[name]
- } else {
- process.env[name] = value
- }
+// The DNS negative cache and the hosts file names are module state, so each case loads a fresh copy of request.ts.
+async function withIsolatedRequest(run: (isolated: IsolatedRequest) => Promise<void>, hostsFile = '') {
+ await jest.isolateModulesAsync(async () => {
+ const actualFs = jest.requireActual('node:fs')
+ jest.doMock('node:fs', () => ({
+ ...actualFs,
+ readFileSync: (path: string, ...args: unknown[]) =>
+ path === '/etc/hosts' ? hostsFile : actualFs.readFileSync(path, ...args),
+ }))
+ const request: typeof import('./request') = require('./request')
+ const lookup = jest.mocked(require('dns/promises').lookup)
+ const registry: typeof register = require('prom-client').register
+ const readNegativeCacheCounter = async (result: string): Promise<number> =>
+ (await registry.getSingleMetric('node_dns_negative_cache_total')!.get()).values.find(
+ (value) => value.labels.result === result
+ )?.value ?? 0
+ try {
+ await run({ request, lookup, readNegativeCacheCounter })
+ } finally {
+ await request.closeSharedAgents(100)
}
- }
+ })
}
const dnsError = (code: string): Error => Object.assign(new Error(`getaddrinfo ${code} example.com`), { code })
-describe('DNS lookup settings', () => {
- it('resolves hostnames as absolute names when absolute lookups are enabled', async () => {
- await withDnsConfig({ EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP: 'true' }, async ({ request, lookup }) => {
+describe('DNS lookups', () => {
+ it('resolves dotted hostnames as absolute names and leaves other names as given', async () => {
+ await withIsolatedRequest(async ({ request, lookup }) => {
lookup.mockResolvedValue([{ address: '1.1.1.1', family: 4 }])
await request.raiseIfUserProvidedUrlUnsafe('https://example.com/path')
await request.raiseIfUserProvidedUrlUnsafe('https://already-absolute.com./path')
await request.raiseIfUserProvidedUrlUnsafe('https://1.1.1.1/path')
+ await request.raiseIfUserProvidedUrlUnsafe('http://maildev:1080/path')
+ await request.raiseIfUserProvidedUrlUnsafe('http://My.Alias.test:1080/path')
expect(lookup.mock.calls.map(([hostname]) => hostname)).toEqual([
'example.com.',
'already-absolute.com.',
'1.1.1.1',
+ 'maildev',
+ 'my.alias.test',
])
- })
+ }, '127.0.0.1 localhost\n# 10.0.0.9 commented.test\n10.1.2.3\tmy.alias.test other.alias.test\n')
})
- it('resolves hostnames as given when absolute lookups are disabled', async () => {
- await withDnsConfig({ EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP: 'false' }, async ({ request, lookup }) => {
- lookup.mockResolvedValue([{ address: '1.1.1.1', family: 4 }])
+ it('skips the lookup for a hostname that recently returned ENOTFOUND', async () => {
+ await withIsolatedRequest(async ({ request, lookup, readNegativeCacheCounter }) => {
+ lookup.mockRejectedValue(dnsError('ENOTFOUND'))
- await request.raiseIfUserProvidedUrlUnsafe('https://example.com/path')
+ await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow(
+ 'Invalid hostname'
+ )
+ await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow(
+ 'Invalid hostname'
+ )
- expect(lookup).toHaveBeenCalledWith('example.com', { all: true })
+ expect(lookup).toHaveBeenCalledTimes(1)
+ expect(await readNegativeCacheCounter('hit')).toEqual(1)
})
})
- it('skips the lookup for a hostname that recently returned ENOTFOUND in enforce mode', async () => {
- await withDnsConfig(
- { EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: 'enforce' },
- async ({ request, lookup, readNegativeCacheCounter }) => {
- lookup.mockRejectedValue(dnsError('ENOTFOUND'))
-
- await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow(
- 'Invalid hostname'
- )
- await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow(
- 'Invalid hostname'
- )
-
- expect(lookup).toHaveBeenCalledTimes(1)
- expect(await readNegativeCacheCounter('hit')).toEqual(1)
- }
- )
- })
-
// A timeout can come from an overloaded resolver rather than from the hostname. Caching it would fail healthy
// destinations for as long as the resolver stays slow.
it.each(['EAI_AGAIN', 'ETIMEOUT'])('does not cache a %s failure', async (code) => {
- await withDnsConfig({ EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: 'enforce' }, async ({ request, lookup }) => {
+ await withIsolatedRequest(async ({ request, lookup }) => {
lookup.mockRejectedValue(dnsError(code))
await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow()
@@ -562,32 +547,6 @@ describe('DNS lookup settings', () => {
expect(lookup).toHaveBeenCalledTimes(2)
})
})
-
- it('still looks up a cached hostname in shadow mode and counts what enforce mode would skip', async () => {
- await withDnsConfig(
- { EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: 'shadow' },
- async ({ request, lookup, readNegativeCacheCounter }) => {
- lookup.mockRejectedValueOnce(dnsError('ENOTFOUND'))
- lookup.mockRejectedValueOnce(dnsError('ENOTFOUND'))
- lookup.mockResolvedValue([{ address: '1.1.1.1', family: 4 }])
-
- await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow()
- await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow()
- await request.raiseIfUserProvidedUrlUnsafe('https://example.com')
- await request.raiseIfUserProvidedUrlUnsafe('https://example.com')
-
- expect(lookup).toHaveBeenCalledTimes(4)
- expect(await readNegativeCacheCounter('shadow_hit')).toEqual(3)
- expect(await readNegativeCacheCounter('shadow_hit_resolved')).toEqual(2)
- }
- )
- })
-
- it('rejects an unknown negative cache mode at startup', async () => {
- await expect(
- withDnsConfig({ EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: 'on' }, () => Promise.resolve())
- ).rejects.toThrow('EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE must be one of off, shadow, enforce')
- })
})
describe('legacyFetch', () => {
diff --git a/nodejs/src/common/utils/request.ts b/nodejs/src/common/utils/request.ts
index d418a74378f2..79fb29d32ccf 100644
--- a/nodejs/src/common/utils/request.ts
+++ b/nodejs/src/common/utils/request.ts
@@ -2,6 +2,7 @@ import { LookupAddress } from 'dns'
import dns from 'dns/promises'
import * as ipaddr from 'ipaddr.js'
import { LRUCache } from 'lru-cache'
+import { readFileSync } from 'node:fs'
import net from 'node:net'
import { Counter, Gauge, Histogram } from 'prom-client'
// eslint-disable-next-line no-restricted-imports
@@ -75,45 +76,42 @@ const dnsLookupsInFlight = new Gauge({
help: 'Number of dns.lookup() calls currently awaiting a result',
})
-// 'shadow_hit_resolved' counts shadow hits whose lookup then succeeded. Enforce mode would have failed those requests
-// wrongly, so this must stay near zero before a deployment moves from shadow to enforce.
const dnsNegativeCacheCounter = new Counter({
name: 'node_dns_negative_cache_total',
- help: 'Lookups the DNS negative cache skipped (hit), would have skipped in shadow mode (shadow_hit), or would have skipped although the hostname resolved (shadow_hit_resolved)',
+ help: 'Lookups the DNS negative cache skipped because the hostname recently returned ENOTFOUND',
labelNames: ['result'],
})
-const DNS_NEGATIVE_CACHE_MODES = ['off', 'shadow', 'enforce'] as const
-type DnsNegativeCacheMode = (typeof DNS_NEGATIVE_CACHE_MODES)[number]
+// Only ENOTFOUND is cached: a timeout or SERVFAIL can be the resolver's fault, and caching it would fail healthy
+// destinations while the resolver is under load. The key is the hostname, which is customer-supplied and unbounded, so
+// the entry count is capped as well as the age.
+const dnsNegativeCache = new LRUCache<string, true>({ max: 10_000, ttl: 10_000 })
-const dnsNegativeCacheMode = requestConfig.EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE as DnsNegativeCacheMode
-if (!DNS_NEGATIVE_CACHE_MODES.includes(dnsNegativeCacheMode)) {
- throw new Error(
- `EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE must be one of ${DNS_NEGATIVE_CACHE_MODES.join(', ')}, got '${dnsNegativeCacheMode}'`
- )
-}
-for (const name of [
- 'EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS',
- 'EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES',
-] as const) {
- if (!Number.isInteger(requestConfig[name]) || requestConfig[name] < 1) {
- throw new Error(`${name} must be a positive integer, got ${process.env[name]}`)
+// The hosts file matches a name exactly, so 'alias.test.' misses an 'alias.test' entry from Docker's --add-host or
+// Kubernetes hostAliases.
+function readHostsFileNames(): Set<string> {
+ try {
+ return new Set(
+ readFileSync('/etc/hosts', 'utf8')
+ .split('\n')
+ .flatMap((line) => line.replace(/#.*/, '').trim().split(/\s+/).slice(1))
+ .map((name) => name.toLowerCase())
+ )
+ } catch {
+ return new Set()
}
}
-// The key is the hostname, which is customer-supplied and unbounded, so the entry count is capped as well as the age.
-const dnsNegativeCache =
- dnsNegativeCacheMode === 'off'
- ? null
- : new LRUCache<string, true>({
- max: requestConfig.EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES,
- ttl: requestConfig.EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS,
- })
-
-// dns.lookup returns an IP literal without a DNS query. With a trailing dot it treats '1.1.1.1.' as a name, and that
-// query fails with ENOTFOUND.
+const hostsFileNames = readHostsFileNames()
+
+// The trailing dot skips the cluster search domains, so a missing hostname costs one query instead of one per domain.
+// A single-label name such as a Docker service name resolves only through the search domains, so it stays as given.
+// dns.lookup returns an IP literal without a DNS query, but it treats '1.1.1.1.' as a name that fails with ENOTFOUND.
function dnsLookupName(hostname: string): string {
- return requestConfig.EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP && !hostname.endsWith('.') && net.isIP(hostname) === 0
+ return hostname.includes('.') &&
+ !hostname.endsWith('.') &&
+ net.isIP(hostname) === 0 &&
+ !hostsFileNames.has(hostname.toLowerCase())
? `${hostname}.`
: hostname
}
@@ -285,13 +283,9 @@ async function staticLookupAsync(hostname: string): Promise<LookupAddress[]> {
let addrinfo: LookupAddress[]
const validAddrinfo: LookupAddress[] = []
const cacheKey = hostname.toLowerCase().replace(/\.$/, '')
- const cachedFailure = dnsNegativeCache?.has(cacheKey) ?? false
- if (cachedFailure) {
- if (dnsNegativeCacheMode === 'enforce') {
- dnsNegativeCacheCounter.inc({ result: 'hit' })
- throw new ResolutionError('Invalid hostname')
- }
- dnsNegativeCacheCounter.inc({ result: 'shadow_hit' })
+ if (dnsNegativeCache.has(cacheKey)) {
+ dnsNegativeCacheCounter.inc({ result: 'hit' })
+ throw new ResolutionError('Invalid hostname')
}
const lookupStartedAt = performance.now()
dnsLookupsInFlight.inc()
@@ -300,23 +294,15 @@ async function staticLookupAsync(hostname: string): Promise<LookupAddress[]> {
dnsLookupDuration.observe({ outcome: 'success' }, performance.now() - lookupStartedAt)
} catch (error) {
dnsLookupDuration.observe({ outcome: 'failure' }, performance.now() - lookupStartedAt)
- // A shadow hit does not refresh the entry, because enforce mode would not have run this lookup. The shadow
- // counts then match what enforce mode would skip.
- if (!cachedFailure && (error as NodeJS.ErrnoException)?.code === 'ENOTFOUND') {
- dnsNegativeCache?.set(cacheKey, true)
+ if ((error as NodeJS.ErrnoException)?.code === 'ENOTFOUND') {
+ dnsNegativeCache.set(cacheKey, true)
}
throw new ResolutionError('Invalid hostname')
} finally {
dnsLookupsInFlight.dec()
}
- if (cachedFailure) {
- // A shadow hit keeps the entry, because enforce mode would not have run this lookup and would keep failing the
- // hostname until the entry expires. Each later success within the TTL then counts here too.
- dnsNegativeCacheCounter.inc({ result: 'shadow_hit_resolved' })
- } else {
- // A lookup that started before another one cached the failure can still succeed. The hostname resolves now.
- dnsNegativeCache?.delete(cacheKey)
- }
+ // A lookup that started before another one cached the failure can still succeed. The hostname resolves now.
+ dnsNegativeCache.delete(cacheKey)
const resolvedIps = addrinfo.map((a) => a.address)
for (const addrInfo of addrinfo) {
const parsed = ipaddr.parse(addrInfo.address)