:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cdp): keep each apple developer team on its own apns connection (#123189)
Daniel Marchuk47fdd2b78cb8
diff --git a/nodejs/src/cdp/services/messaging/push-notification.service.ts b/nodejs/src/cdp/services/messaging/push-notification.service.tsindex 30bbfac6872d..ee19f431d994 100644--- a/nodejs/src/cdp/services/messaging/push-notification.service.ts+++ b/nodejs/src/cdp/services/messaging/push-notification.service.ts@@ -738,7 +738,7 @@ export class PushNotificationService {result,params,subscription,- { apnsHost, bundleId, jwt, templateId },+ { apnsHost, bundleId, jwt, templateId, appleTeamId },addLog)// Apple rejects a small share of requests with a token it accepts on the next one. Retry each device@@ -759,7 +759,7 @@ export class PushNotificationService {result,params,subscription,- { apnsHost, bundleId, jwt, templateId },+ { apnsHost, bundleId, jwt, templateId, appleTeamId },addLog)}@@ -806,7 +806,7 @@ export class PushNotificationService {result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>,params: CyclotronInvocationQueueParametersSendPushNotificationType,subscription: DeviceSubscription,- apns: { apnsHost: string; bundleId: string; jwt: string; templateId: string },+ apns: { apnsHost: string; bundleId: string; jwt: string; templateId: string; appleTeamId: string },addLog: ReturnType<typeof createAddLogFunction>): Promise<{ sent: boolean; unregistered?: boolean; message?: string; error?: PushSendError }> {const payload = params.payload@@ -832,6 +832,7 @@ export class PushNotificationService {// APNs requires HTTP/2allowH2: true,http2IdleTimeoutMs: APNS_IDLE_TIMEOUT_MS,+ http2ConnectionGroup: `apns:${apns.appleTeamId}`,}if (params.timeoutMs !== undefined) {diff --git a/nodejs/src/common/utils/request-http2.test.ts b/nodejs/src/common/utils/request-http2.test.tsindex 9bdf3097be97..6b38654dacf2 100644--- a/nodejs/src/common/utils/request-http2.test.ts+++ b/nodejs/src/common/utils/request-http2.test.ts@@ -385,6 +385,24 @@ describe('secure HTTP/2 requests', () => {expect(http2SessionCount).toBe(1)}, 10000)+ it('keeps requests of different connection groups on separate sessions', async () => {+ const http2Url = `https://origin.test:${serverPort(http2Origin)}`+ const send = async (path: string, group: string): Promise<string> => {+ const response = await requestModule.fetch(`${http2Url}${path}`, {+ allowH2: true,+ http2ConnectionGroup: group,+ })+ return await response.text()+ }++ expect(await send('/team-a-1', 'team-a')).toBe('/team-a-1')+ expect(await send('/team-a-2', 'team-a')).toBe('/team-a-2')+ expect(http2SessionCount).toBe(1)++ expect(await send('/team-b-1', 'team-b')).toBe('/team-b-1')+ expect(http2SessionCount).toBe(2)+ }, 10000)+it('shares one client for six cold image fetches and closes the idle client and its connections', async () => {const { HttpImageFetcher } =require('~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher') as typeof import('~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher')diff --git a/nodejs/src/common/utils/request.ts b/nodejs/src/common/utils/request.tsindex 1e810ebcc6a6..b8ca00ba7b71 100644--- a/nodejs/src/common/utils/request.ts+++ b/nodejs/src/common/utils/request.ts@@ -132,6 +132,9 @@ export type FetchOptions = {// negotiates HTTP/1.1 uses the same dispatcher, so its idle sockets also close after this time. Pass a constant:// each distinct value keeps a dispatcher for the life of the process, and at most 8 values are allowed.http2IdleTimeoutMs?: number+ // Requests with different groups never share an HTTP/2 session. APNs can reject a provider token from one developer+ // team on a session that already carried another team's token. Requires `allowH2`.+ http2ConnectionGroup?: string}export type FetchResponse = {@@ -485,7 +488,8 @@ if ()}-type SecureDispatcher = { dispatcher: Dispatcher; gate: ColdStartGate | null }+type SecureDispatcher = { dispatcher: Dispatcher; gate: ColdStartGate | null; release?: () => void }+type GroupedSecureDispatcher = { dispatcher: Dispatcher; gate: ColdStartGate; pending: number; evicted: boolean }// undici sets the idle timeout per dispatcher, so each distinct idle timeout gets its own HTTP/2 dispatcher. The two// routes keep separate maps so a caller cannot exhaust one route's budget with timeouts it only ever uses on the other.@@ -494,18 +498,27 @@ const sharedSecureH2Agents: Record<SecureRoute, Map<number, SecureDispatcher>> =proxy: new Map(),}const MAX_SECURE_H2_AGENTS = 8+const groupedSecureH2Agents = new Map<string, GroupedSecureDispatcher>()+const MAX_GROUPED_SECURE_H2_AGENTS = 1000+// An evicted dispatcher stays here until its close() finishes, so shutdown can still destroy one that holds an unread+// body.+const drainingGroupedSecureH2Agents = new Set<Dispatcher>()// Node clamps a setTimeout delay above this value to 1 ms. A session would then close as soon as it goes idle.const MAX_H2_IDLE_TIMEOUT_MS = 2_147_483_647let sharedAgentsClosed = false-function getSecureH2Agent(- useProxy: boolean,- idleTimeoutMs = requestConfig.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS-): SecureDispatcher {+function assertValidH2IdleTimeout(idleTimeoutMs: number): void {// InvalidRequestError is not retriable in cdp-fetch, so a value that can never work fails once.if (!Number.isFinite(idleTimeoutMs) || idleTimeoutMs <= 0 || idleTimeoutMs > MAX_H2_IDLE_TIMEOUT_MS) {throw new InvalidRequestError(`http2IdleTimeoutMs must be a number between 1 and ${MAX_H2_IDLE_TIMEOUT_MS}`)}+}++function getSecureH2Agent(+ useProxy: boolean,+ idleTimeoutMs = requestConfig.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS+): SecureDispatcher {+ assertValidH2IdleTimeout(idleTimeoutMs)const agents = sharedSecureH2Agents[routeOf(useProxy)]let agent = agents.get(idleTimeoutMs)if (!agent) {@@ -524,11 +537,79 @@ function getSecureH2Agent(return agent}-function getSecureDispatcher(options: { allowH2?: boolean; http2IdleTimeoutMs?: number }): SecureDispatcher {+function getGroupedSecureH2Agent(+ useProxy: boolean,+ group: string,+ idleTimeoutMs = requestConfig.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS+): SecureDispatcher {+ assertValidH2IdleTimeout(idleTimeoutMs)+ const key = `${routeOf(useProxy)}:${idleTimeoutMs}:${group}`+ let agent = groupedSecureH2Agents.get(key)+ if (agent) {+ groupedSecureH2Agents.delete(key)+ } else {+ if (sharedAgentsClosed) {+ throw new undiciErrors.ClientDestroyedError()+ }+ agent = {+ dispatcher: makeSecureDispatcher({ useProxy, allowH2: true, keepAliveTimeoutMs: idleTimeoutMs }),+ gate: new ColdStartGate(idleTimeoutMs),+ pending: 0,+ evicted: false,+ }+ if (groupedSecureH2Agents.size >= MAX_GROUPED_SECURE_H2_AGENTS) {+ const [oldestKey, oldest] = groupedSecureH2Agents.entries().next().value!+ groupedSecureH2Agents.delete(oldestKey)+ oldest.evicted = true+ drainingGroupedSecureH2Agents.add(oldest.dispatcher)+ if (oldest.pending === 0) {+ closeDrainingAgent(oldest.dispatcher)+ }+ }+ }+ groupedSecureH2Agents.set(key, agent)+ // A request can wait in the cold-start gate before it reaches undici, and close() rejects a request dispatched+ // after it runs. So an evicted dispatcher closes only once every request that picked it has been dispatched.+ const picked = agent+ picked.pending+++ let released = false+ return {+ dispatcher: picked.dispatcher,+ gate: picked.gate,+ release: () => {+ if (released) {+ return+ }+ released = true+ picked.pending--+ if (picked.evicted && picked.pending === 0) {+ closeDrainingAgent(picked.dispatcher)+ }+ },+ }+}++function closeDrainingAgent(dispatcher: Dispatcher): void {+ void dispatcher+ .close()+ .catch(() => {})+ .finally(() => drainingGroupedSecureH2Agents.delete(dispatcher))+}++function getSecureDispatcher(options: {+ allowH2?: boolean+ http2IdleTimeoutMs?: number+ http2ConnectionGroup?: string+}): SecureDispatcher {const useProxy = useProxyForTeam()+ if (options.http2ConnectionGroup !== undefined && !options.allowH2) {+ throw new InvalidRequestError('http2ConnectionGroup requires allowH2')+ }// useProxyForTeam only returns true when a proxy URL was configured, so the agent exists whenever it is needed.const secure = options.allowH2- ? getSecureH2Agent(useProxy, options.http2IdleTimeoutMs)+ ? options.http2ConnectionGroup !== undefined+ ? getGroupedSecureH2Agent(useProxy, options.http2ConnectionGroup, options.http2IdleTimeoutMs)+ : getSecureH2Agent(useProxy, options.http2IdleTimeoutMs): {dispatcher: useProxy && sharedSecureProxyAgent ? sharedSecureProxyAgent : sharedSecureAgent,gate: null,@@ -558,6 +639,8 @@ export async function closeSharedAgents(gracePeriodMs = 5000): Promise<void> {...(sharedSecureProxyAgent ? [sharedSecureProxyAgent] : []),sharedInsecureAgent,...Object.values(sharedSecureH2Agents).flatMap((route) => [...route.values()].map((h2) => h2.dispatcher)),+ ...[...groupedSecureH2Agents.values()].map((h2) => h2.dispatcher),+ ...drainingGroupedSecureH2Agents,]const stillOpen = new Set(agents)const closed = Promise.allSettled(agents.map((agent) => agent.close().finally(() => stillOpen.delete(agent)))).then(@@ -735,8 +818,10 @@ export async function fetch(url: string, options: FetchOptions = {}): Promise<FevalidateHostnameIPLiteral(parsed.hostname, !isProdEnv())inflightExternalRequests.inc()let lifecycle: RequestLifecycle | undefined+ let release: (() => void) | undefinedtry {- const { dispatcher, gate } = getSecureDispatcher(options)+ const { dispatcher, gate, release: releaseDispatcher } = getSecureDispatcher(options)+ release = releaseDispatcheroptions.timeoutMs = options.timeoutMs ?? requestConfig.EXTERNAL_REQUEST_THIRD_PARTY_TIMEOUT_MSconst signal = options.timeoutMs ? AbortSignal.timeout(options.timeoutMs) : undefinedlifecycle = await gatedLifecycle(gate, parsed.origin, signal)@@ -746,6 +831,7 @@ export async function fetch(url: string, options: FetchOptions = {}): Promise<Felifecycle?.onError()throw error} finally {+ release?.()inflightExternalRequests.dec()}}@@ -755,6 +841,7 @@ export type StreamedFetchOptions = {timeoutMs: numberallowH2?: booleanhttp2IdleTimeoutMs?: number+ http2ConnectionGroup?: string}export type StreamedResponse = {@@ -855,8 +942,10 @@ export async function fetchStreamed(url: string, options: StreamedFetchOptions):inflightExternalRequests.inc()let result: Dispatcher.ResponseDatalet lifecycle: RequestLifecycle | undefined+ let release: (() => void) | undefinedtry {- const { dispatcher, gate } = getSecureDispatcher(options)+ const { dispatcher, gate, release: releaseDispatcher } = getSecureDispatcher(options)+ release = releaseDispatcherconst signal = AbortSignal.timeout(options.timeoutMs)lifecycle = await gatedLifecycle(gate, parsed.origin, signal)result = await request(parsed.toString(), {@@ -870,6 +959,8 @@ export async function fetchStreamed(url: string, options: StreamedFetchOptions):lifecycle?.onError()inflightExternalRequests.dec()throw error+ } finally {+ release?.()}lifecycle?.onHeaders()