Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cdp): keep each apple developer team on its own apns connection (#123189)

Daniel Marchuk47fdd2b78cb8
diff --git a/nodejs/src/cdp/services/messaging/push-notification.service.ts b/nodejs/src/cdp/services/messaging/push-notification.service.ts
index 30bbfac6872d..ee19f431d994 100644
--- a/nodejs/src/cdp/services/messaging/push-notification.service.ts
+++ b/nodejs/src/cdp/services/messaging/push-notification.service.ts
@@ -738,7 +738,7 @@ export class PushNotificationService {
result,
params,
subscription,
- { apnsHost, bundleId, jwt, templateId },
+ { apnsHost, bundleId, jwt, templateId, appleTeamId },
addLog
)
// Apple rejects a small share of requests with a token it accepts on the next one. Retry each device
@@ -759,7 +759,7 @@ export class PushNotificationService {
result,
params,
subscription,
- { apnsHost, bundleId, jwt, templateId },
+ { apnsHost, bundleId, jwt, templateId, appleTeamId },
addLog
)
}
@@ -806,7 +806,7 @@ export class PushNotificationService {
result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>,
params: CyclotronInvocationQueueParametersSendPushNotificationType,
subscription: DeviceSubscription,
- apns: { apnsHost: string; bundleId: string; jwt: string; templateId: string },
+ apns: { apnsHost: string; bundleId: string; jwt: string; templateId: string; appleTeamId: string },
addLog: ReturnType<typeof createAddLogFunction>
): Promise<{ sent: boolean; unregistered?: boolean; message?: string; error?: PushSendError }> {
const payload = params.payload
@@ -832,6 +832,7 @@ export class PushNotificationService {
// APNs requires HTTP/2
allowH2: true,
http2IdleTimeoutMs: APNS_IDLE_TIMEOUT_MS,
+ http2ConnectionGroup: `apns:${apns.appleTeamId}`,
}
if (params.timeoutMs !== undefined) {
diff --git a/nodejs/src/common/utils/request-http2.test.ts b/nodejs/src/common/utils/request-http2.test.ts
index 9bdf3097be97..6b38654dacf2 100644
--- a/nodejs/src/common/utils/request-http2.test.ts
+++ b/nodejs/src/common/utils/request-http2.test.ts
@@ -385,6 +385,24 @@ describe('secure HTTP/2 requests', () => {
expect(http2SessionCount).toBe(1)
}, 10000)
+ it('keeps requests of different connection groups on separate sessions', async () => {
+ const http2Url = `https://origin.test:${serverPort(http2Origin)}`
+ const send = async (path: string, group: string): Promise<string> => {
+ const response = await requestModule.fetch(`${http2Url}${path}`, {
+ allowH2: true,
+ http2ConnectionGroup: group,
+ })
+ return await response.text()
+ }
+
+ expect(await send('/team-a-1', 'team-a')).toBe('/team-a-1')
+ expect(await send('/team-a-2', 'team-a')).toBe('/team-a-2')
+ expect(http2SessionCount).toBe(1)
+
+ expect(await send('/team-b-1', 'team-b')).toBe('/team-b-1')
+ expect(http2SessionCount).toBe(2)
+ }, 10000)
+
it('shares one client for six cold image fetches and closes the idle client and its connections', async () => {
const { HttpImageFetcher } =
require('~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher') as typeof import('~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher')
diff --git a/nodejs/src/common/utils/request.ts b/nodejs/src/common/utils/request.ts
index 1e810ebcc6a6..b8ca00ba7b71 100644
--- a/nodejs/src/common/utils/request.ts
+++ b/nodejs/src/common/utils/request.ts
@@ -132,6 +132,9 @@ export type FetchOptions = {
// negotiates HTTP/1.1 uses the same dispatcher, so its idle sockets also close after this time. Pass a constant:
// each distinct value keeps a dispatcher for the life of the process, and at most 8 values are allowed.
http2IdleTimeoutMs?: number
+ // Requests with different groups never share an HTTP/2 session. APNs can reject a provider token from one developer
+ // team on a session that already carried another team's token. Requires `allowH2`.
+ http2ConnectionGroup?: string
}
export type FetchResponse = {
@@ -485,7 +488,8 @@ if (
)
}
-type SecureDispatcher = { dispatcher: Dispatcher; gate: ColdStartGate | null }
+type SecureDispatcher = { dispatcher: Dispatcher; gate: ColdStartGate | null; release?: () => void }
+type GroupedSecureDispatcher = { dispatcher: Dispatcher; gate: ColdStartGate; pending: number; evicted: boolean }
// undici sets the idle timeout per dispatcher, so each distinct idle timeout gets its own HTTP/2 dispatcher. The two
// routes keep separate maps so a caller cannot exhaust one route's budget with timeouts it only ever uses on the other.
@@ -494,18 +498,27 @@ const sharedSecureH2Agents: Record<SecureRoute, Map<number, SecureDispatcher>> =
proxy: new Map(),
}
const MAX_SECURE_H2_AGENTS = 8
+const groupedSecureH2Agents = new Map<string, GroupedSecureDispatcher>()
+const MAX_GROUPED_SECURE_H2_AGENTS = 1000
+// An evicted dispatcher stays here until its close() finishes, so shutdown can still destroy one that holds an unread
+// body.
+const drainingGroupedSecureH2Agents = new Set<Dispatcher>()
// Node clamps a setTimeout delay above this value to 1 ms. A session would then close as soon as it goes idle.
const MAX_H2_IDLE_TIMEOUT_MS = 2_147_483_647
let sharedAgentsClosed = false
-function getSecureH2Agent(
- useProxy: boolean,
- idleTimeoutMs = requestConfig.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS
-): SecureDispatcher {
+function assertValidH2IdleTimeout(idleTimeoutMs: number): void {
// InvalidRequestError is not retriable in cdp-fetch, so a value that can never work fails once.
if (!Number.isFinite(idleTimeoutMs) || idleTimeoutMs <= 0 || idleTimeoutMs > MAX_H2_IDLE_TIMEOUT_MS) {
throw new InvalidRequestError(`http2IdleTimeoutMs must be a number between 1 and ${MAX_H2_IDLE_TIMEOUT_MS}`)
}
+}
+
+function getSecureH2Agent(
+ useProxy: boolean,
+ idleTimeoutMs = requestConfig.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS
+): SecureDispatcher {
+ assertValidH2IdleTimeout(idleTimeoutMs)
const agents = sharedSecureH2Agents[routeOf(useProxy)]
let agent = agents.get(idleTimeoutMs)
if (!agent) {
@@ -524,11 +537,79 @@ function getSecureH2Agent(
return agent
}
-function getSecureDispatcher(options: { allowH2?: boolean; http2IdleTimeoutMs?: number }): SecureDispatcher {
+function getGroupedSecureH2Agent(
+ useProxy: boolean,
+ group: string,
+ idleTimeoutMs = requestConfig.EXTERNAL_REQUEST_KEEP_ALIVE_TIMEOUT_MS
+): SecureDispatcher {
+ assertValidH2IdleTimeout(idleTimeoutMs)
+ const key = `${routeOf(useProxy)}:${idleTimeoutMs}:${group}`
+ let agent = groupedSecureH2Agents.get(key)
+ if (agent) {
+ groupedSecureH2Agents.delete(key)
+ } else {
+ if (sharedAgentsClosed) {
+ throw new undiciErrors.ClientDestroyedError()
+ }
+ agent = {
+ dispatcher: makeSecureDispatcher({ useProxy, allowH2: true, keepAliveTimeoutMs: idleTimeoutMs }),
+ gate: new ColdStartGate(idleTimeoutMs),
+ pending: 0,
+ evicted: false,
+ }
+ if (groupedSecureH2Agents.size >= MAX_GROUPED_SECURE_H2_AGENTS) {
+ const [oldestKey, oldest] = groupedSecureH2Agents.entries().next().value!
+ groupedSecureH2Agents.delete(oldestKey)
+ oldest.evicted = true
+ drainingGroupedSecureH2Agents.add(oldest.dispatcher)
+ if (oldest.pending === 0) {
+ closeDrainingAgent(oldest.dispatcher)
+ }
+ }
+ }
+ groupedSecureH2Agents.set(key, agent)
+ // A request can wait in the cold-start gate before it reaches undici, and close() rejects a request dispatched
+ // after it runs. So an evicted dispatcher closes only once every request that picked it has been dispatched.
+ const picked = agent
+ picked.pending++
+ let released = false
+ return {
+ dispatcher: picked.dispatcher,
+ gate: picked.gate,
+ release: () => {
+ if (released) {
+ return
+ }
+ released = true
+ picked.pending--
+ if (picked.evicted && picked.pending === 0) {
+ closeDrainingAgent(picked.dispatcher)
+ }
+ },
+ }
+}
+
+function closeDrainingAgent(dispatcher: Dispatcher): void {
+ void dispatcher
+ .close()
+ .catch(() => {})
+ .finally(() => drainingGroupedSecureH2Agents.delete(dispatcher))
+}
+
+function getSecureDispatcher(options: {
+ allowH2?: boolean
+ http2IdleTimeoutMs?: number
+ http2ConnectionGroup?: string
+}): SecureDispatcher {
const useProxy = useProxyForTeam()
+ if (options.http2ConnectionGroup !== undefined && !options.allowH2) {
+ throw new InvalidRequestError('http2ConnectionGroup requires allowH2')
+ }
// useProxyForTeam only returns true when a proxy URL was configured, so the agent exists whenever it is needed.
const secure = options.allowH2
- ? getSecureH2Agent(useProxy, options.http2IdleTimeoutMs)
+ ? options.http2ConnectionGroup !== undefined
+ ? getGroupedSecureH2Agent(useProxy, options.http2ConnectionGroup, options.http2IdleTimeoutMs)
+ : getSecureH2Agent(useProxy, options.http2IdleTimeoutMs)
: {
dispatcher: useProxy && sharedSecureProxyAgent ? sharedSecureProxyAgent : sharedSecureAgent,
gate: null,
@@ -558,6 +639,8 @@ export async function closeSharedAgents(gracePeriodMs = 5000): Promise<void> {
...(sharedSecureProxyAgent ? [sharedSecureProxyAgent] : []),
sharedInsecureAgent,
...Object.values(sharedSecureH2Agents).flatMap((route) => [...route.values()].map((h2) => h2.dispatcher)),
+ ...[...groupedSecureH2Agents.values()].map((h2) => h2.dispatcher),
+ ...drainingGroupedSecureH2Agents,
]
const stillOpen = new Set(agents)
const closed = Promise.allSettled(agents.map((agent) => agent.close().finally(() => stillOpen.delete(agent)))).then(
@@ -735,8 +818,10 @@ export async function fetch(url: string, options: FetchOptions = {}): Promise<Fe
validateHostnameIPLiteral(parsed.hostname, !isProdEnv())
inflightExternalRequests.inc()
let lifecycle: RequestLifecycle | undefined
+ let release: (() => void) | undefined
try {
- const { dispatcher, gate } = getSecureDispatcher(options)
+ const { dispatcher, gate, release: releaseDispatcher } = getSecureDispatcher(options)
+ release = releaseDispatcher
options.timeoutMs = options.timeoutMs ?? requestConfig.EXTERNAL_REQUEST_THIRD_PARTY_TIMEOUT_MS
const signal = options.timeoutMs ? AbortSignal.timeout(options.timeoutMs) : undefined
lifecycle = await gatedLifecycle(gate, parsed.origin, signal)
@@ -746,6 +831,7 @@ export async function fetch(url: string, options: FetchOptions = {}): Promise<Fe
lifecycle?.onError()
throw error
} finally {
+ release?.()
inflightExternalRequests.dec()
}
}
@@ -755,6 +841,7 @@ export type StreamedFetchOptions = {
timeoutMs: number
allowH2?: boolean
http2IdleTimeoutMs?: number
+ http2ConnectionGroup?: string
}
export type StreamedResponse = {
@@ -855,8 +942,10 @@ export async function fetchStreamed(url: string, options: StreamedFetchOptions):
inflightExternalRequests.inc()
let result: Dispatcher.ResponseData
let lifecycle: RequestLifecycle | undefined
+ let release: (() => void) | undefined
try {
- const { dispatcher, gate } = getSecureDispatcher(options)
+ const { dispatcher, gate, release: releaseDispatcher } = getSecureDispatcher(options)
+ release = releaseDispatcher
const signal = AbortSignal.timeout(options.timeoutMs)
lifecycle = await gatedLifecycle(gate, parsed.origin, signal)
result = await request(parsed.toString(), {
@@ -870,6 +959,8 @@ export async function fetchStreamed(url: string, options: StreamedFetchOptions):
lifecycle?.onError()
inflightExternalRequests.dec()
throw error
+ } finally {
+ release?.()
}
lifecycle?.onHeaders()