:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cdp): always use absolute dns lookups and the dns negative cache (#125571)
Daniel Marchuk887a9fa112d8
diff --git a/nodejs/src/common/config.ts b/nodejs/src/common/config.tsindex e6b0b893c114..46c364167caa 100644--- a/nodejs/src/common/config.ts+++ b/nodejs/src/common/config.ts@@ -236,17 +236,6 @@ export type CommonConfig = BaseServerConfig & {// Takes the buildIntegerMatcherWithPercentage syntax: '2' for team 2 only, '2,*:0.1' for team 2 plus a tenth of// everyone else's requests, '*' for all.EXTERNAL_REQUEST_PROXY_TEAMS: string- // Resolve third-party hostnames as absolute names (with a trailing dot) on the direct route. The resolver then- // skips the cluster search domains, so a hostname that does not exist costs one query instead of one per search- // domain. A customer-supplied hostname never names a cluster service, so the search domains never help it.- EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP: boolean- // A process-local cache of hostnames that returned ENOTFOUND on the direct route. 'off' disables it. 'shadow'- // still runs every lookup and only counts the lookups the cache would have skipped. 'enforce' fails those lookups- // at once with the same ResolutionError. Only ENOTFOUND is cached: a timeout or SERVFAIL can be the resolver's- // fault, and caching it would fail healthy destinations while the resolver is under load.- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: string- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS: number- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES: number// PostHog analyticsPOSTHOG_API_KEY: string@@ -287,10 +276,6 @@ export type ExternalRequestConfig = Pick<| 'EXTERNAL_REQUEST_CONNECTIONS'| 'EXTERNAL_REQUEST_H2_CONNECTIONS'| 'EXTERNAL_REQUEST_PROXY_TEAMS'- | 'EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP'- | 'EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE'- | 'EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS'- | 'EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES'>export function getExternalRequestConfig(): ExternalRequestConfig {@@ -304,14 +289,6 @@ export function getExternalRequestConfig(): ExternalRequestConfig {EXTERNAL_REQUEST_CONNECTIONS: Number(process.env.EXTERNAL_REQUEST_CONNECTIONS ?? 500),EXTERNAL_REQUEST_H2_CONNECTIONS: Number(process.env.EXTERNAL_REQUEST_H2_CONNECTIONS ?? 8),EXTERNAL_REQUEST_PROXY_TEAMS: process.env.EXTERNAL_REQUEST_PROXY_TEAMS ?? '',- EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP: process.env.EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP === 'true',- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: process.env.EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE ?? 'off',- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS: Number(- process.env.EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS ?? 10_000- ),- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES: Number(- process.env.EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES ?? 10_000- ),}}@@ -470,10 +447,6 @@ export function getDefaultCommonConfig(): CommonConfig {EXTERNAL_REQUEST_CONNECTIONS: 500,EXTERNAL_REQUEST_H2_CONNECTIONS: 8,EXTERNAL_REQUEST_PROXY_TEAMS: '',- EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP: false,- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: 'off',- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS: 10_000,- EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES: 10_000,// PostHog analyticsPOSTHOG_API_KEY: '',diff --git a/nodejs/src/common/utils/request.test.ts b/nodejs/src/common/utils/request.test.tsindex 7c4037ac8416..b6dda770c17d 100644--- a/nodejs/src/common/utils/request.test.ts+++ b/nodejs/src/common/utils/request.test.ts@@ -472,88 +472,73 @@ type IsolatedRequest = {readNegativeCacheCounter: (result: string) => Promise<number>}-// request.ts reads the DNS settings once, at module load, so each case loads a fresh copy with its own environment.-async function withDnsConfig(env: Record<string, string>, run: (isolated: IsolatedRequest) => Promise<void>) {- const originalValues = Object.fromEntries(Object.keys(env).map((name) => [name, process.env[name]]))- Object.assign(process.env, env)- try {- await jest.isolateModulesAsync(async () => {- const request: typeof import('./request') = require('./request')- const lookup = jest.mocked(require('dns/promises').lookup)- const registry: typeof register = require('prom-client').register- const readNegativeCacheCounter = async (result: string): Promise<number> =>- (await registry.getSingleMetric('node_dns_negative_cache_total')!.get()).values.find(- (value) => value.labels.result === result- )?.value ?? 0- try {- await run({ request, lookup, readNegativeCacheCounter })- } finally {- await request.closeSharedAgents(100)- }- })- } finally {- for (const [name, value] of Object.entries(originalValues)) {- if (value === undefined) {- delete process.env[name]- } else {- process.env[name] = value- }+// The DNS negative cache and the hosts file names are module state, so each case loads a fresh copy of request.ts.+async function withIsolatedRequest(run: (isolated: IsolatedRequest) => Promise<void>, hostsFile = '') {+ await jest.isolateModulesAsync(async () => {+ const actualFs = jest.requireActual('node:fs')+ jest.doMock('node:fs', () => ({+ ...actualFs,+ readFileSync: (path: string, ...args: unknown[]) =>+ path === '/etc/hosts' ? hostsFile : actualFs.readFileSync(path, ...args),+ }))+ const request: typeof import('./request') = require('./request')+ const lookup = jest.mocked(require('dns/promises').lookup)+ const registry: typeof register = require('prom-client').register+ const readNegativeCacheCounter = async (result: string): Promise<number> =>+ (await registry.getSingleMetric('node_dns_negative_cache_total')!.get()).values.find(+ (value) => value.labels.result === result+ )?.value ?? 0+ try {+ await run({ request, lookup, readNegativeCacheCounter })+ } finally {+ await request.closeSharedAgents(100)}- }+ })}const dnsError = (code: string): Error => Object.assign(new Error(`getaddrinfo ${code} example.com`), { code })-describe('DNS lookup settings', () => {- it('resolves hostnames as absolute names when absolute lookups are enabled', async () => {- await withDnsConfig({ EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP: 'true' }, async ({ request, lookup }) => {+describe('DNS lookups', () => {+ it('resolves dotted hostnames as absolute names and leaves other names as given', async () => {+ await withIsolatedRequest(async ({ request, lookup }) => {lookup.mockResolvedValue([{ address: '1.1.1.1', family: 4 }])await request.raiseIfUserProvidedUrlUnsafe('https://example.com/path')await request.raiseIfUserProvidedUrlUnsafe('https://already-absolute.com./path')await request.raiseIfUserProvidedUrlUnsafe('https://1.1.1.1/path')+ await request.raiseIfUserProvidedUrlUnsafe('http://maildev:1080/path')+ await request.raiseIfUserProvidedUrlUnsafe('http://My.Alias.test:1080/path')expect(lookup.mock.calls.map(([hostname]) => hostname)).toEqual(['example.com.','already-absolute.com.','1.1.1.1',+ 'maildev',+ 'my.alias.test',])- })+ }, '127.0.0.1 localhost\n# 10.0.0.9 commented.test\n10.1.2.3\tmy.alias.test other.alias.test\n')})- it('resolves hostnames as given when absolute lookups are disabled', async () => {- await withDnsConfig({ EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP: 'false' }, async ({ request, lookup }) => {- lookup.mockResolvedValue([{ address: '1.1.1.1', family: 4 }])+ it('skips the lookup for a hostname that recently returned ENOTFOUND', async () => {+ await withIsolatedRequest(async ({ request, lookup, readNegativeCacheCounter }) => {+ lookup.mockRejectedValue(dnsError('ENOTFOUND'))- await request.raiseIfUserProvidedUrlUnsafe('https://example.com/path')+ await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow(+ 'Invalid hostname'+ )+ await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow(+ 'Invalid hostname'+ )- expect(lookup).toHaveBeenCalledWith('example.com', { all: true })+ expect(lookup).toHaveBeenCalledTimes(1)+ expect(await readNegativeCacheCounter('hit')).toEqual(1)})})- it('skips the lookup for a hostname that recently returned ENOTFOUND in enforce mode', async () => {- await withDnsConfig(- { EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: 'enforce' },- async ({ request, lookup, readNegativeCacheCounter }) => {- lookup.mockRejectedValue(dnsError('ENOTFOUND'))-- await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow(- 'Invalid hostname'- )- await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow(- 'Invalid hostname'- )-- expect(lookup).toHaveBeenCalledTimes(1)- expect(await readNegativeCacheCounter('hit')).toEqual(1)- }- )- })-// A timeout can come from an overloaded resolver rather than from the hostname. Caching it would fail healthy// destinations for as long as the resolver stays slow.it.each(['EAI_AGAIN', 'ETIMEOUT'])('does not cache a %s failure', async (code) => {- await withDnsConfig({ EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: 'enforce' }, async ({ request, lookup }) => {+ await withIsolatedRequest(async ({ request, lookup }) => {lookup.mockRejectedValue(dnsError(code))await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow()@@ -562,32 +547,6 @@ describe('DNS lookup settings', () => {expect(lookup).toHaveBeenCalledTimes(2)})})-- it('still looks up a cached hostname in shadow mode and counts what enforce mode would skip', async () => {- await withDnsConfig(- { EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: 'shadow' },- async ({ request, lookup, readNegativeCacheCounter }) => {- lookup.mockRejectedValueOnce(dnsError('ENOTFOUND'))- lookup.mockRejectedValueOnce(dnsError('ENOTFOUND'))- lookup.mockResolvedValue([{ address: '1.1.1.1', family: 4 }])-- await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow()- await expect(request.raiseIfUserProvidedUrlUnsafe('https://example.com')).rejects.toThrow()- await request.raiseIfUserProvidedUrlUnsafe('https://example.com')- await request.raiseIfUserProvidedUrlUnsafe('https://example.com')-- expect(lookup).toHaveBeenCalledTimes(4)- expect(await readNegativeCacheCounter('shadow_hit')).toEqual(3)- expect(await readNegativeCacheCounter('shadow_hit_resolved')).toEqual(2)- }- )- })-- it('rejects an unknown negative cache mode at startup', async () => {- await expect(- withDnsConfig({ EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE: 'on' }, () => Promise.resolve())- ).rejects.toThrow('EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE must be one of off, shadow, enforce')- })})describe('legacyFetch', () => {diff --git a/nodejs/src/common/utils/request.ts b/nodejs/src/common/utils/request.tsindex d418a74378f2..79fb29d32ccf 100644--- a/nodejs/src/common/utils/request.ts+++ b/nodejs/src/common/utils/request.ts@@ -2,6 +2,7 @@ import { LookupAddress } from 'dns'import dns from 'dns/promises'import * as ipaddr from 'ipaddr.js'import { LRUCache } from 'lru-cache'+import { readFileSync } from 'node:fs'import net from 'node:net'import { Counter, Gauge, Histogram } from 'prom-client'// eslint-disable-next-line no-restricted-imports@@ -75,45 +76,42 @@ const dnsLookupsInFlight = new Gauge({help: 'Number of dns.lookup() calls currently awaiting a result',})-// 'shadow_hit_resolved' counts shadow hits whose lookup then succeeded. Enforce mode would have failed those requests-// wrongly, so this must stay near zero before a deployment moves from shadow to enforce.const dnsNegativeCacheCounter = new Counter({name: 'node_dns_negative_cache_total',- help: 'Lookups the DNS negative cache skipped (hit), would have skipped in shadow mode (shadow_hit), or would have skipped although the hostname resolved (shadow_hit_resolved)',+ help: 'Lookups the DNS negative cache skipped because the hostname recently returned ENOTFOUND',labelNames: ['result'],})-const DNS_NEGATIVE_CACHE_MODES = ['off', 'shadow', 'enforce'] as const-type DnsNegativeCacheMode = (typeof DNS_NEGATIVE_CACHE_MODES)[number]+// Only ENOTFOUND is cached: a timeout or SERVFAIL can be the resolver's fault, and caching it would fail healthy+// destinations while the resolver is under load. The key is the hostname, which is customer-supplied and unbounded, so+// the entry count is capped as well as the age.+const dnsNegativeCache = new LRUCache<string, true>({ max: 10_000, ttl: 10_000 })-const dnsNegativeCacheMode = requestConfig.EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE as DnsNegativeCacheMode-if (!DNS_NEGATIVE_CACHE_MODES.includes(dnsNegativeCacheMode)) {- throw new Error(- `EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MODE must be one of ${DNS_NEGATIVE_CACHE_MODES.join(', ')}, got '${dnsNegativeCacheMode}'`- )-}-for (const name of [- 'EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS',- 'EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES',-] as const) {- if (!Number.isInteger(requestConfig[name]) || requestConfig[name] < 1) {- throw new Error(`${name} must be a positive integer, got ${process.env[name]}`)+// The hosts file matches a name exactly, so 'alias.test.' misses an 'alias.test' entry from Docker's --add-host or+// Kubernetes hostAliases.+function readHostsFileNames(): Set<string> {+ try {+ return new Set(+ readFileSync('/etc/hosts', 'utf8')+ .split('\n')+ .flatMap((line) => line.replace(/#.*/, '').trim().split(/\s+/).slice(1))+ .map((name) => name.toLowerCase())+ )+ } catch {+ return new Set()}}-// The key is the hostname, which is customer-supplied and unbounded, so the entry count is capped as well as the age.-const dnsNegativeCache =- dnsNegativeCacheMode === 'off'- ? null- : new LRUCache<string, true>({- max: requestConfig.EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_MAX_ENTRIES,- ttl: requestConfig.EXTERNAL_REQUEST_DNS_NEGATIVE_CACHE_TTL_MS,- })--// dns.lookup returns an IP literal without a DNS query. With a trailing dot it treats '1.1.1.1.' as a name, and that-// query fails with ENOTFOUND.+const hostsFileNames = readHostsFileNames()++// The trailing dot skips the cluster search domains, so a missing hostname costs one query instead of one per domain.+// A single-label name such as a Docker service name resolves only through the search domains, so it stays as given.+// dns.lookup returns an IP literal without a DNS query, but it treats '1.1.1.1.' as a name that fails with ENOTFOUND.function dnsLookupName(hostname: string): string {- return requestConfig.EXTERNAL_REQUEST_DNS_ABSOLUTE_LOOKUP && !hostname.endsWith('.') && net.isIP(hostname) === 0+ return hostname.includes('.') &&+ !hostname.endsWith('.') &&+ net.isIP(hostname) === 0 &&+ !hostsFileNames.has(hostname.toLowerCase())? `${hostname}.`: hostname}@@ -285,13 +283,9 @@ async function staticLookupAsync(hostname: string): Promise<LookupAddress[]> {let addrinfo: LookupAddress[]const validAddrinfo: LookupAddress[] = []const cacheKey = hostname.toLowerCase().replace(/\.$/, '')- const cachedFailure = dnsNegativeCache?.has(cacheKey) ?? false- if (cachedFailure) {- if (dnsNegativeCacheMode === 'enforce') {- dnsNegativeCacheCounter.inc({ result: 'hit' })- throw new ResolutionError('Invalid hostname')- }- dnsNegativeCacheCounter.inc({ result: 'shadow_hit' })+ if (dnsNegativeCache.has(cacheKey)) {+ dnsNegativeCacheCounter.inc({ result: 'hit' })+ throw new ResolutionError('Invalid hostname')}const lookupStartedAt = performance.now()dnsLookupsInFlight.inc()@@ -300,23 +294,15 @@ async function staticLookupAsync(hostname: string): Promise<LookupAddress[]> {dnsLookupDuration.observe({ outcome: 'success' }, performance.now() - lookupStartedAt)} catch (error) {dnsLookupDuration.observe({ outcome: 'failure' }, performance.now() - lookupStartedAt)- // A shadow hit does not refresh the entry, because enforce mode would not have run this lookup. The shadow- // counts then match what enforce mode would skip.- if (!cachedFailure && (error as NodeJS.ErrnoException)?.code === 'ENOTFOUND') {- dnsNegativeCache?.set(cacheKey, true)+ if ((error as NodeJS.ErrnoException)?.code === 'ENOTFOUND') {+ dnsNegativeCache.set(cacheKey, true)}throw new ResolutionError('Invalid hostname')} finally {dnsLookupsInFlight.dec()}- if (cachedFailure) {- // A shadow hit keeps the entry, because enforce mode would not have run this lookup and would keep failing the- // hostname until the entry expires. Each later success within the TTL then counts here too.- dnsNegativeCacheCounter.inc({ result: 'shadow_hit_resolved' })- } else {- // A lookup that started before another one cached the failure can still succeed. The hostname resolves now.- dnsNegativeCache?.delete(cacheKey)- }+ // A lookup that started before another one cached the failure can still succeed. The hostname resolves now.+ dnsNegativeCache.delete(cacheKey)const resolvedIps = addrinfo.map((a) => a.address)for (const addrInfo of addrinfo) {const parsed = ipaddr.parse(addrInfo.address)