:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { createHash, createSign } from 'crypto'
import { DateTime } from 'luxon'
import { Counter, Histogram } from 'prom-client'
import { setTimeout as sleep } from 'timers/promises'
import {
CyclotronInvocationQueueParametersSendPushNotificationType,
PushNotificationPayloadType,
} from '~/cdp/schema/cyclotron'
import { RedisV2 } from '~/common/redis/redis-v2'
import { instrumented } from '~/common/tracing/tracing-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { FetchOptions, FetchResponse } from '~/common/utils/request'
import type { CyclotronJobInvocationHogFunction, CyclotronJobInvocationResult, IntegrationType } from '../../types'
import { createAddLogFunction } from '../../utils'
import { EncryptedFields } from '../../utils/encryption-utils'
import { createInvocationResult } from '../../utils/invocation-utils'
import {
DeviceSubscription,
NATIVE_PUSH_TEMPLATE_ID,
getDevicePushSubscriptions,
pushRecipientOtherThanRunPerson,
runDistinctIdMatching,
} from '../../utils/push-subscription-utils'
import { IntegrationManagerService } from '../managers/integration-manager.service'
import { PersonsManagerService } from '../managers/persons-manager.service'
import { MessageAssetsService } from './message-assets.service'
import {
NormalizedPushError,
PushPlatform,
PushSendError,
normalizeApnsError,
normalizeFcmError,
} from './push-notification-errors'
const pushNotificationSentCounter = new Counter({
name: 'push_notification_sent_total',
help: 'Total number of push notifications successfully sent',
labelNames: ['platform'],
})
const pushNotificationTokenPrunedCounter = new Counter({
name: 'push_notification_token_pruned_total',
help: 'Total number of device tokens removed after the provider reported them no longer registered',
labelNames: ['platform'],
})
const pushNotificationFailedCounter = new Counter({
name: 'push_notification_failed_total',
help: 'Push notifications that failed to send, by platform and normalized failure reason.',
labelNames: ['platform', 'reason'],
})
const pushNotificationSkippedCounter = new Counter({
name: 'push_notification_skipped_total',
help: 'Push sends not delivered without an outright failure, by platform and reason. no_token = the recipient never registered a device; unregistered = the provider reported the token dead and it was removed; no_person = the distinct id the step named has no person.',
labelNames: ['platform', 'reason'],
})
const pushNotificationSendDurationMs = new Histogram({
name: 'push_notification_send_duration_ms',
help: 'Duration of a push send request to the provider, by platform.',
labelNames: ['platform'],
buckets: [10, 25, 50, 100, 200, 500, 1000, 2000, 5000, 10000],
})
const apnsProviderTokenRefreshCounter = new Counter({
name: 'push_notification_apns_token_refresh_total',
help: 'APNs provider tokens replaced after Apple rejected one. outcome: minted = a new token was signed; adopted = another pod had already replaced it; skipped = a refresh for this key, or the rejected token itself, is inside the refresh window.',
labelNames: ['outcome'],
})
const pushNotificationRescheduledCounter = new Counter({
name: 'push_notification_rescheduled_total',
help: 'Push sends rescheduled after a transient provider failure (throttle / 5xx / network) instead of being dropped. A sustained rate for one platform means that provider is rate-limiting the sending project.',
labelNames: ['platform'],
})
// Apple rate-limits new APNs provider tokens (returns 429 TooManyProviderTokenUpdates if refreshed more
// than once every ~20 min per key) and accepts a token for up to 1 hour. Cache the signed JWT in Valkey
// keyed by the auth key id so the whole fleet reuses one token per key rather than minting one per send.
const APNS_JWT_CACHE_PREFIX = '@posthog/apns-provider-jwt/'
const APNS_JWT_TTL_SECONDS = 45 * 60
// Apple can reject a token it accepted minutes earlier. One new token per key per window keeps a key that
// keeps getting rejected under Apple's refresh limit.
const APNS_JWT_REFRESH_PREFIX = '@posthog/apns-provider-jwt-refresh/'
// A delivery on any pod proves the key works, so a pod that has not sent with it yet can still retry a rejection.
const APNS_KEY_ACCEPTED_PREFIX = '@posthog/apns-key-accepted/'
const APNS_KEY_ACCEPTED_SHARE_INTERVAL_MS = 60 * 1000
const APNS_JWT_REFRESH_WINDOW_SECONDS = 20 * 60
const APNS_REJECTED_TOKEN_CODES = new Set(['InvalidProviderToken', 'ExpiredProviderToken'])
const APNS_JWT_REPLACEMENT_POLLS = 5
const APNS_JWT_REPLACEMENT_POLL_MS = 100
const APNS_JWT_FRESH_CLAIM_MS = 2000
// Keeps a token another pod already put in place of the rejected one.
const APNS_JWT_DROP_IF_REJECTED_SCRIPT = `
local current = redis.call('GET', KEYS[1])
if current == ARGV[1] then
redis.call('DEL', KEYS[1])
return false
end
return current
`
function apnsJwtIssuedAtMs(jwt: string): number | null {
try {
const iat = parseJSON(Buffer.from(jwt.split('.')[1], 'base64url').toString()).iat
return typeof iat === 'number' ? iat * 1000 : null
} catch {
return null
}
}
// Apple asks providers to keep a connection open for hours to days, and treats rapid reconnects as abuse. undici
// sends an HTTP/2 ping every 60 s by default, which keeps the egress proxy tunnel open across the idle window.
const APNS_IDLE_TIMEOUT_MS = 60 * 60 * 1000
// One entry per signing key, so the ceiling is the number of APNs integrations routed through this pod.
// Pruned on write rather than on a timer.
const APNS_JWT_LOCAL_CACHE_MAX = 500
// A push action fans out to its selected channels within one invocation. Cap the count so a workflow
// crafted with a huge channel list can't tie up a worker with an unbounded outbound-request loop; a real
// push targets a handful of provider integrations, not thousands.
const MAX_PUSH_CHANNELS = 10
// Cap a provider-supplied Retry-After. FCM returns one on QUOTA_EXCEEDED and honoring it is the point,
// but a hostile or misconfigured value must not park an invocation for hours.
const MAX_RETRY_AFTER_MS = 5 * 60 * 1000
function stringifyBody(body: unknown): string {
return typeof body === 'string' ? body : JSON.stringify(body)
}
// Retry-After is delta-seconds or an HTTP-date. Return a bounded millisecond delay, or undefined if the
// header is absent or unparseable so the caller falls back to exponential backoff.
function parseRetryAfterMs(response: FetchResponse | null): number | undefined {
const header = response?.headers?.['retry-after']
if (!header) {
return undefined
}
const seconds = Number(header)
const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(header) - Date.now()
if (!Number.isFinite(ms) || ms <= 0) {
return undefined
}
return Math.min(ms, MAX_RETRY_AFTER_MS)
}
// Exponential-ish backoff with jitter for a retry when the provider didn't give a Retry-After.
function backoffAt(baseMs: number, maxMs: number, tries: number): DateTime {
const delay = Math.min(baseMs * tries + Math.floor(Math.random() * baseMs), maxMs)
return DateTime.utc().plus({ milliseconds: delay })
}
// Correlation ids stamped into every notification's data payload. An open happens on the device, so the
// SDK is the only thing that can observe it; these ride along in the payload and come back on the
// captured open event, which is what lets an open be attributed to the workflow and step that sent it.
// Without them an open is only ever a global count.
//
// The key and the shape are fixed by the mobile SDKs: both read the single `posthog` entry out of the
// notification payload and re-emit each of its keys as `$notification_<key>` on `$push_notification_opened`.
// Nothing outside that entry is read, so these ids have to be nested under it rather than sent as
// sibling keys.
const PUSH_CORRELATION_KEY = 'posthog'
// Serialized rather than nested as an object because FCM's `data` map only accepts string values. Both
// SDKs parse this entry from either a dictionary or a JSON string, on either delivery route, so one
// encoding covers FCM and direct APNs.
function pushCorrelationData(invocation: CyclotronJobInvocationHogFunction): Record<string, string> {
const correlation: Record<string, string> = {
workflow_id: invocation.functionId,
invocation_id: invocation.id,
}
// Absent for a push sent outside a workflow step, where there is no step to attribute an open to.
if (invocation.state.actionId) {
correlation.action_id = invocation.state.actionId
}
// Send metrics are attributed to `parentRunId ?? functionId`, so a batch run counts its sends
// against the batch job rather than the workflow. An open has to be attributable the same way or
// the two don't divide into a rate — which is why the email tracking code carries this id too.
if (invocation.parentRunId) {
correlation.parent_run_id = invocation.parentRunId
}
return { [PUSH_CORRELATION_KEY]: JSON.stringify(correlation) }
}
// The retriable failure that stands for a person's devices: the one asking for the longest wait. Only
// one error per channel reaches the reschedule, so the delay every other device asked for is gone by
// then — keeping the earliest would retry inside a window a later device's provider asked us to sit
// out. A failure carrying no Retry-After counts as zero, which is what the backoff already assumes.
function longestRetriableFailure(current: PushSendError | undefined, candidate: Error): PushSendError | undefined {
if (!(candidate instanceof PushSendError) || !candidate.retriable) {
return current
}
if (!current) {
return candidate
}
return (candidate.retryAfterMs ?? 0) > (current.retryAfterMs ?? 0) ? candidate : current
}
// The devices that failed inside a channel that still delivered to another one. That channel reports
// success, so these never reach the outer catch that logs and counts a whole-channel failure, and a
// device failing for good — a token registered against the wrong app, a rejected payload — would be
// left with only a debug-level line. Counting here cannot double-count: a channel that delivered is
// never rescheduled, so no later attempt sees these devices again.
function reportPartialDeviceFailures(failures: PushSendError[], addLog: ReturnType<typeof createAddLogFunction>): void {
for (const failure of failures) {
pushNotificationFailedCounter.labels({ platform: failure.platform, reason: failure.reason }).inc()
addLog(failure.level, failure.message)
}
}
function pushSendError(platform: PushPlatform, err: NormalizedPushError, retryAfterMs?: number): PushSendError {
// Append the raw provider code so the failure surfaced to the hog template stays debuggable, while
// the human-readable sentence leads.
const message = err.code ? `${err.message} [${err.code}]` : err.message
return new PushSendError(message, platform, err.reason, err.level, err.retriable, err.code, retryAfterMs)
}
// Apple sometimes rejects a token that the rest of the fleet keeps sending with, and the refresh limit can
// block a new token for 20 minutes. A key that delivered recently works, so retry instead of failing the send.
class ApnsTokenRejectionError extends PushSendError {
constructor(message: string, code: string, retryAfterMs: number | undefined) {
super(message, 'apns', 'auth_error', 'warn', true, code, retryAfterMs)
}
}
function retriableTokenRejection(error: PushSendError): PushSendError {
if (!error.code || !APNS_REJECTED_TOKEN_CODES.has(error.code)) {
return error
}
return new ApnsTokenRejectionError(
`APNs rejected the provider token, although this key delivered notifications in the last 20 minutes, so the integration credentials are likely fine. [${error.code}]`,
error.code,
error.retryAfterMs
)
}
// apnsAcceptedKeys holds a short prefix of each key fingerprint, never key material.
type PushQueueMetadata = { tries?: number; apnsAcceptedKeys?: string[] }
// Whose devices a send goes to. `properties` is the person snapshot the tokens are read from, and
// `isRunPerson` says it is the run's own person, so a pruned token has to leave the invocation's
// snapshot as well.
type PushRecipient = {
distinctId: string
personId: string | undefined
properties: Record<string, any> | undefined
isRunPerson: boolean
}
export type PushNotificationFetchUtils = {
trackedFetch: (args: { url: string; fetchParams: FetchOptions; templateId: string; teamId?: number }) => Promise<{
fetchError: Error | null
fetchResponse: FetchResponse | null
fetchDuration: number
}>
maxFetchTimeoutMs: number
// Retry budget + backoff for transient provider failures, wired from CDP_FETCH_* so push shares the
// same policy as the generic fetch destinations.
maxRetries: number
backoffBaseMs: number
backoffMaxMs: number
}
export class PushNotificationService {
constructor(
private integrationManager: IntegrationManagerService,
private encryptedFields: EncryptedFields,
private fetchUtils: PushNotificationFetchUtils,
private valkey: RedisV2,
private personsManager: PersonsManagerService,
private messageAssetsService?: MessageAssetsService
) {}
// Both Valkey calls for the APNs token are failOpen, so an outage turns every send into a fresh mint —
// the exact pattern Apple answers with 429 TooManyProviderTokenUpdates. This per-pod copy bounds that
// to one token per pod per TTL. It is a fallback, not the cache: Valkey is still read first, so the
// fleet normally shares one token per key.
private apnsJwtLocalCache = new Map<string, { jwt: string; expiresAtMs: number }>()
private apnsJwtInFlight = new Map<string, Promise<string>>()
// Last time APNs accepted a send signed with each key, so a token rejection for a key that works can be retried.
private apnsKeyAcceptedAtMs = new Map<string, { acceptedAtMs: number; sharedAtMs: number }>()
@instrumented('push-notification.executeSendPushNotification')
async executeSendPushNotification(
invocation: CyclotronJobInvocationHogFunction,
isTest = false
): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {
if (invocation.queueParameters?.type !== 'sendPushNotification') {
throw new Error('Bad invocation')
}
const queueParams = invocation.queueParameters as CyclotronInvocationQueueParametersSendPushNotificationType
// Stamp the correlation ids last so they win over any custom `data` key of the same name. The
// enriched copy is local: `invocation.queueParameters` stays untouched, so a reschedule re-derives
// these rather than persisting them onto the job.
const params: CyclotronInvocationQueueParametersSendPushNotificationType = {
...queueParams,
payload: {
...queueParams.payload,
data: { ...(queueParams.payload?.data ?? {}), ...pushCorrelationData(invocation) },
},
}
const result = createInvocationResult<CyclotronJobInvocationHogFunction>(invocation, {}, { finished: true })
const addLog = createAddLogFunction(result.logs)
// How many times this send has already been rescheduled after a transient failure, carried on the
// invocation's queue metadata across reschedules.
const metadata = (invocation.queueMetadata as PushQueueMetadata) || {}
const tries = metadata.tries ?? 0
// Business metrics are emitted once, at the terminal outcome below, rather than per channel — a
// rescheduled attempt must not re-count the same notification's skips or failures on every retry.
const pushMetric = (metricName: 'push_sent' | 'push_skipped' | 'push_failed', count: number): void => {
// A test send from the editor's "Run test" must not land in the workflow's Metrics tab,
// matching what the email path already does.
if (count <= 0 || isTest) {
return
}
result.metrics.push({
team_id: invocation.teamId,
app_source_id: invocation.parentRunId ?? invocation.functionId,
instance_id: invocation.state.actionId || invocation.id,
metric_kind: 'push',
metric_name: metricName,
count,
})
}
// Fan out to every selected channel here rather than looping sendPushNotification in the hog
// template — a hog function only runs one async call per invocation, so a per-channel loop in
// hog would only ever deliver to the first channel. Each channel is isolated: one failing must
// not abort the others.
// Dedupe so duplicate channel entries can't notify the same device twice, and cap the fan-out so a
// crafted workflow can't occupy the worker with an unbounded loop.
const uniqueIntegrationIds = [...new Set(params.integrationIds)]
const integrationIds = uniqueIntegrationIds.slice(0, MAX_PUSH_CHANNELS)
if (uniqueIntegrationIds.length > MAX_PUSH_CHANNELS) {
addLog(
'warn',
`Push notification has ${uniqueIntegrationIds.length} channels; only the first ${MAX_PUSH_CHANNELS} will be delivered.`
)
}
let successCount = 0
let skippedCount = 0
let otherErrorCount = 0
const errors: PushSendError[] = []
let firstError: string | undefined
const skipMessages: string[] = []
// Which providers took delivery. Only its emptiness is read, to decide whether there is a
// delivered notification worth capturing.
const deliveredPlatforms = new Set<string>()
// Filled before device failures collapse into one error per channel, so a retry carries every key that works.
const apnsAcceptedKeys = new Set(metadata.apnsAcceptedKeys ?? [])
// Resolved once for the send: every channel reads the same person's tokens.
let recipient: PushRecipient | null
try {
recipient = await this.resolveRecipient(invocation, params)
} catch (error) {
// The person service is down or slow. Nothing was sent yet, so retry later instead of dropping the push.
if (tries >= this.fetchUtils.maxRetries) {
throw error
}
const nextTries = tries + 1
result.invocation.queueParameters = invocation.queueParameters
result.invocation.queueMetadata = { ...metadata, tries: nextTries }
result.invocation.queuePriority = nextTries
result.invocation.queueScheduledAt = backoffAt(
this.fetchUtils.backoffBaseMs,
this.fetchUtils.backoffMaxMs,
nextTries
)
result.finished = false
addLog(
'warn',
`Could not look up the person for distinct_id: ${params.distinctId.trim()}; retrying (attempt ${nextTries}/${this.fetchUtils.maxRetries}).`
)
return result
}
if (!recipient) {
addLog('warn', `No person found for distinct_id: ${params.distinctId.trim()}; push notification skipped.`)
pushNotificationSkippedCounter.labels({ platform: 'unknown', reason: 'no_person' }).inc()
}
for (const integrationId of integrationIds) {
// Nobody to read tokens from, so every channel is skipped the way a person with no device is.
if (!recipient) {
skippedCount++
continue
}
try {
const integration = await this.integrationManager.get(integrationId)
if (!integration || integration.team_id !== invocation.teamId) {
throw new Error('Push notification integration not found')
}
let sent: boolean
if (integration.kind === 'firebase') {
sent = await this.executeFcm(result, params, integration, invocation, recipient, skipMessages)
} else if (integration.kind === 'apns') {
sent = await this.executeApns(
result,
params,
integration,
invocation,
recipient,
apnsAcceptedKeys,
skipMessages
)
} else {
throw new Error(`Unsupported push integration kind: ${integration.kind}`)
}
if (sent) {
successCount++
deliveredPlatforms.add(integration.kind === 'firebase' ? 'Firebase' : 'APNs')
} else {
// A channel with no registered device token for this recipient is skipped, not failed.
skippedCount++
}
} catch (error) {
firstError = firstError ?? error.message
if (error instanceof PushSendError) {
// Normalized provider failure: log once at the reason's severity. The failure metric is
// emitted at the terminal outcome below, so a rescheduled retry doesn't over-count it.
errors.push(error)
addLog(error.level, error.message)
} else {
otherErrorCount++
addLog('error', error.message)
}
}
}
// Most recipients have a device on one platform only, so a skip next to a delivery is expected.
const skipLogLevel = successCount > 0 ? 'info' : 'warn'
for (const message of skipMessages) {
addLog(skipLogLevel, message)
}
const errorCount = errors.length + otherErrorCount
const success = successCount > 0 || errorCount === 0
const retriableErrors = errors.filter((e) => e.retriable)
// Nothing delivered and a channel failed transiently: reschedule the whole invocation instead of
// dropping the notification. Only safe when nothing was delivered — once a channel has sent,
// retrying would re-deliver to it. A terminally-failed channel (e.g. a 400) sitting alongside a
// transient one is re-attempted on each retry; that's acceptable, it just re-fails, bounded by the
// budget. `tries` starts at 0, so this allows CDP_FETCH_RETRIES reschedules after the first attempt
// (the notification is tried maxRetries + 1 times in total).
if (!success && retriableErrors.length > 0 && tries < this.fetchUtils.maxRetries) {
const nextTries = tries + 1
// Restore queueParameters — createInvocationResult cleared them. Without this the retry dequeue
// routes on an undefined queue type and resumes the hog VM instead of re-running this send,
// which pops an empty stack and throws a bytecode error, dropping the notification.
result.invocation.queueParameters = invocation.queueParameters
// A retry can land on a pod that has not delivered with this key yet, so carry the evidence that the key works.
result.invocation.queueMetadata = {
...metadata,
tries: nextTries,
...(apnsAcceptedKeys.size > 0 ? { apnsAcceptedKeys: [...apnsAcceptedKeys] } : {}),
}
result.invocation.queuePriority = nextTries
// Retrying re-attempts every failed channel, so wait out the LONGEST Retry-After any channel
// asked for; fall back to exponential backoff when none provided one.
const retryAfterValues = retriableErrors
.map((e) => e.retryAfterMs)
.filter((ms): ms is number => ms !== undefined)
result.invocation.queueScheduledAt =
retryAfterValues.length > 0
? DateTime.utc().plus({ milliseconds: Math.max(...retryAfterValues) })
: backoffAt(this.fetchUtils.backoffBaseMs, this.fetchUtils.backoffMaxMs, nextTries)
result.finished = false
// Count each distinct throttled platform so the metric shows which provider is rate-limiting.
for (const platform of new Set(retriableErrors.map((e) => e.platform))) {
pushNotificationRescheduledCounter.labels({ platform }).inc()
}
addLog(
'warn',
`Push notification throttled or temporarily unavailable; retrying (attempt ${nextTries}/${this.fetchUtils.maxRetries}).`
)
// No business metric or VM-state push here — the eventual terminal attempt reports the outcome.
return result
}
// Terminal outcome: emit the per-notification business metrics once, and label the ops failure
// metric by platform + reason for every channel that failed for good.
pushMetric('push_sent', successCount)
pushMetric('push_skipped', skippedCount)
pushMetric('push_failed', errorCount)
// Billing reads this rather than the metrics above, which a test send suppresses.
result.deliveredToRecipient = successCount > 0
// Captured at the terminal outcome for the same reason the business metrics are: a rescheduled
// attempt returns earlier, so a retried notification produces one asset rather than one per try.
// Only a delivered notification is captured, matching email: an asset is a snapshot of what a
// recipient received, and a skip has no recipient. Skips stay visible as `push_skipped` plus the
// per-channel run log explaining why.
// Skipped for a test send for the same reason the metrics are: the Assets tab should show
// what real recipients were sent, not what an editor preview produced.
if (this.messageAssetsService && recipient && successCount > 0 && !isTest) {
// Best-effort: the notification is already delivered by this point, so a capture failure
// must not fail the invocation. Throwing here would send the whole batch back for a retry
// and deliver every notification in it a second time. Losing an Assets row is the cheaper
// outcome, and it is the same trade the flush path already makes on a Kafka failure.
try {
const assetRow = this.messageAssetsService.buildRowForPush(
invocation,
params,
[...deliveredPlatforms],
{ distinctId: recipient.distinctId, personId: recipient.personId }
)
if (assetRow) {
result.messageAssets.push(assetRow)
}
} catch (err) {
addLog('warn', 'The notification was delivered but could not be captured for the Assets tab.')
logger.warn('⚠️', `failed to capture a sent push notification: ${err}`, { error: String(err) })
}
}
for (const error of errors) {
pushNotificationFailedCounter.labels({ platform: error.platform, reason: error.reason }).inc()
}
if (otherErrorCount > 0) {
pushNotificationFailedCounter.labels({ platform: 'unknown', reason: 'unknown' }).inc(otherErrorCount)
}
if (!success) {
result.error = firstError
}
// Surface the error to the hog template too, otherwise its failure message renders blank.
result.invocation.state.vmState!.stack.push({ success, error: success ? null : (firstError ?? null) })
// The template does not resume after this send, so the step's output is set here. It lets a workflow
// store the channel counts in an output variable and branch on them.
result.execResult = { delivered: successCount, skipped: skippedCount, failed: errorCount }
return result
}
// The run's person snapshot is only the recipient when the step's Distinct ID input names the run's
// own person (the template default). A manual run's person is whoever pressed the button, and a step
// pointed at someone else has to read that person's tokens, so anything else is looked up fresh:
// a push cannot be recalled, and a cached read could predate a device that unregistered.
// Null when the named distinct id has no person, never the run's person in its place: that would
// notify someone the step did not name.
private async resolveRecipient(
invocation: CyclotronJobInvocationHogFunction,
params: CyclotronInvocationQueueParametersSendPushNotificationType
): Promise<PushRecipient | null> {
const { event, person } = invocation.state.globals
// Only a workflow push step may name another person: the opt-out and frequency cap checks run there
// and nowhere else, so other hog code keeps sending to the run's person.
const isWorkflowPushStep =
!!invocation.state.actionId && invocation.hogFunction.template_id === NATIVE_PUSH_TEMPLATE_ID
const distinctId = isWorkflowPushStep
? pushRecipientOtherThanRunPerson(params.distinctId, invocation.state.globals)
: null
if (!distinctId) {
return {
distinctId:
runDistinctIdMatching(params.distinctId, invocation.state.globals) ??
(event.distinct_id || person?.distinct_id || ''),
personId: person?.id,
properties: person?.properties,
isRunPerson: true,
}
}
const named = await this.personsManager.getCyclotronPerson(invocation.teamId, distinctId, 'distinct_id', {
forceFresh: true,
})
if (!named) {
return null
}
return { distinctId, personId: named.id, properties: named.properties, isRunPerson: false }
}
/** Returns true if a notification was handed off to FCM, false if skipped (no device token). */
private async executeFcm(
result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>,
params: CyclotronInvocationQueueParametersSendPushNotificationType,
integration: IntegrationType,
invocation: CyclotronJobInvocationHogFunction,
recipient: PushRecipient,
skipMessages: string[]
): Promise<boolean> {
const addLog = createAddLogFunction(result.logs)
const projectId = integration.config.project_id
const accessToken = integration.sensitive_config.access_token
if (!projectId || !accessToken) {
throw new Error('Firebase integration is missing project_id or access_token')
}
const subscriptions = getDevicePushSubscriptions(recipient.properties, projectId, this.encryptedFields)
if (subscriptions.length === 0) {
skipMessages.push(`No active FCM device token found for distinct_id: ${recipient.distinctId}`)
pushNotificationSkippedCounter.labels({ platform: 'fcm', reason: 'no_token' }).inc()
return false
}
const url = `https://fcm.googleapis.com/v1/projects/${projectId}/messages:send`
const templateId = result.invocation.hogFunction.template_id ?? 'unknown'
let delivered = 0
// A terminal failure on one device must not mask a retriable one on another: whether the step
// is worth re-running depends on any device being retriable, not on which failed last.
let retriableFailure: PushSendError | undefined
const failures: PushSendError[] = []
for (const subscription of subscriptions) {
const outcome = await this.sendOneFcm(result, params, subscription, url, accessToken, templateId, addLog)
if (outcome.sent) {
delivered++
} else if (outcome.unregistered) {
this.pruneDeviceToken(result, invocation, recipient, subscription.propertyKeys, 'fcm')
addLog('warn', `FCM: ${outcome.message}`)
} else if (outcome.error) {
failures.push(outcome.error)
retriableFailure = longestRetriableFailure(retriableFailure, outcome.error)
}
}
if (delivered > 0) {
addLog('info', `Push notification accepted by FCM for ${delivered} of ${subscriptions.length} device(s).`)
reportPartialDeviceFailures(failures, addLog)
// A throw here would re-run the step and push again to every device already delivered to.
// A device that failed retryably loses this one notification instead, which is the better
// of the two: a missed push on one device rather than a duplicate on the others.
return true
}
const failure = retriableFailure ?? failures.at(-1)
if (failure) {
throw failure
}
return false
}
/** One FCM send. Reports the outcome rather than throwing, so the caller can weigh it against the
* other devices of the same person before deciding to retry the step. */
private async sendOneFcm(
result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>,
params: CyclotronInvocationQueueParametersSendPushNotificationType,
subscription: DeviceSubscription,
url: string,
accessToken: string,
templateId: string,
addLog: ReturnType<typeof createAddLogFunction>
): Promise<{ sent: boolean; unregistered?: boolean; message?: string; error?: PushSendError }> {
const fetchParams: FetchOptions = {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify(this.buildFcmMessage(subscription.token, params.payload)),
}
if (params.timeoutMs !== undefined) {
fetchParams.timeoutMs = Math.min(params.timeoutMs, this.fetchUtils.maxFetchTimeoutMs)
}
const { fetchError, fetchResponse, fetchDuration } = await this.fetchUtils.trackedFetch({
url,
fetchParams,
templateId,
teamId: result.invocation.teamId,
})
result.invocation.state.timings.push({
kind: 'async_function',
duration_ms: fetchDuration,
})
let body: unknown = undefined
try {
body = await fetchResponse?.text()
if (typeof body === 'string') {
try {
body = parseJSON(body)
} catch {
// Pass through
}
}
} catch (e) {
addLog('error', `Failed to parse response body: ${e.message}`)
}
const status = fetchResponse?.status
pushNotificationSendDurationMs.labels({ platform: 'fcm' }).observe(fetchDuration)
if (!fetchResponse || (status && status >= 400)) {
const err = normalizeFcmError(status, body, fetchError)
// Keep the raw provider response available for deep debugging, at debug level so it doesn't
// clutter the workflow log — the user-facing explanation is the normalized message below.
// FCM can quote the rejected message back, so a protected push never logs it.
if (result.invocation.hogFunction.metadata?.sensitive !== true) {
addLog('debug', `FCM response ${status ?? '(none)'}: ${stringifyBody(body)}`)
}
if (err.unregistered) {
return { sent: false, unregistered: true, message: err.message }
}
return { sent: false, error: pushSendError('fcm', err, parseRetryAfterMs(fetchResponse)) }
}
pushNotificationSentCounter.labels({ platform: 'fcm' }).inc()
return { sent: true }
}
/** Returns true if a notification was handed off to APNS, false if skipped (no device token). */
private async executeApns(
result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>,
params: CyclotronInvocationQueueParametersSendPushNotificationType,
integration: IntegrationType,
invocation: CyclotronJobInvocationHogFunction,
recipient: PushRecipient,
apnsAcceptedKeys: Set<string>,
skipMessages: string[]
): Promise<boolean> {
const addLog = createAddLogFunction(result.logs)
const signingKey = integration.sensitive_config.signing_key
const keyId = integration.config.key_id
const appleTeamId = integration.config.team_id
const bundleId = integration.config.bundle_id
if (!signingKey || !keyId || !appleTeamId || !bundleId) {
throw new Error('APNS integration is missing required fields: signing_key, key_id, team_id, or bundle_id')
}
const subscriptions = getDevicePushSubscriptions(recipient.properties, bundleId, this.encryptedFields)
if (subscriptions.length === 0) {
skipMessages.push(`No active APNS device token found for distinct_id: ${recipient.distinctId}`)
pushNotificationSkippedCounter.labels({ platform: 'apns', reason: 'no_token' }).inc()
return false
}
let jwt = await this.generateApnsJwt(appleTeamId, keyId, signingKey)
const keyFingerprint = this.apnsKeyFingerprint(appleTeamId, keyId, signingKey)
const keyRef = keyFingerprint.slice(0, 16)
const apnsHost =
integration.config.environment === 'sandbox' ? 'api.sandbox.push.apple.com' : 'api.push.apple.com'
const templateId = result.invocation.hogFunction.template_id ?? 'unknown'
let delivered = 0
// A terminal failure on one device must not mask a retriable one on another: whether the step
// is worth re-running depends on any device being retriable, not on which failed last.
let retriableFailure: PushSendError | undefined
const failures: PushSendError[] = []
let tokenRefreshed = false
for (const subscription of subscriptions) {
let outcome = await this.sendOneApns(
result,
params,
subscription,
{ apnsHost, bundleId, jwt, templateId, appleTeamId },
addLog
)
// Apple rejects a small share of requests with a token it accepts on the next one. Retry each device
// here, because a person whose other device delivered is never rescheduled.
if (outcome.error?.code && APNS_REJECTED_TOKEN_CODES.has(outcome.error.code)) {
const refreshed = tokenRefreshed ? null : await this.refreshApnsJwt(appleTeamId, keyId, signingKey, jwt)
tokenRefreshed = true
if (refreshed) {
jwt = refreshed
}
addLog(
'warn',
refreshed
? 'APNs rejected the provider token. Retrying once with a new token.'
: 'APNs rejected the provider token. Retrying once.'
)
outcome = await this.sendOneApns(
result,
params,
subscription,
{ apnsHost, bundleId, jwt, templateId, appleTeamId },
addLog
)
}
if (outcome.sent) {
delivered++
this.rememberApnsKeyAccepted(keyFingerprint)
} else if (outcome.unregistered) {
this.pruneDeviceToken(result, invocation, recipient, subscription.propertyKeys, 'apns')
addLog('warn', `APNs: ${outcome.message}`)
} else if (outcome.error) {
// Only a provider-token rejection depends on whether the key works, so skip the shared read otherwise.
const keyAccepted =
!!outcome.error.code &&
APNS_REJECTED_TOKEN_CODES.has(outcome.error.code) &&
(apnsAcceptedKeys.has(keyRef) || (await this.apnsKeyAcceptedRecently(keyFingerprint)))
const error = keyAccepted ? retriableTokenRejection(outcome.error) : outcome.error
if (error instanceof ApnsTokenRejectionError) {
apnsAcceptedKeys.add(keyRef)
}
failures.push(error)
retriableFailure = longestRetriableFailure(retriableFailure, error)
}
}
if (delivered > 0) {
addLog('info', `Push notification accepted by APNs for ${delivered} of ${subscriptions.length} device(s).`)
reportPartialDeviceFailures(failures, addLog)
// See executeFcm: retrying after a partial success would push again to the devices already
// delivered to.
return true
}
const failure = retriableFailure ?? failures.at(-1)
if (failure) {
throw failure
}
return false
}
/** One APNs send. Reports the outcome rather than throwing, so the caller can weigh it against the
* other devices of the same person before deciding to retry the step. */
private async sendOneApns(
result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>,
params: CyclotronInvocationQueueParametersSendPushNotificationType,
subscription: DeviceSubscription,
apns: { apnsHost: string; bundleId: string; jwt: string; templateId: string; appleTeamId: string },
addLog: ReturnType<typeof createAddLogFunction>
): Promise<{ sent: boolean; unregistered?: boolean; message?: string; error?: PushSendError }> {
const payload = params.payload
const silent = payload.delivery === 'silent'
const headers: Record<string, string> = {
Authorization: `bearer ${apns.jwt}`,
'apns-topic': apns.bundleId,
'apns-push-type': silent ? 'background' : 'alert',
}
if (payload.collapseKey) {
headers['apns-collapse-id'] = payload.collapseKey
}
if (payload.ttlSeconds !== undefined) {
headers['apns-expiration'] = String(Math.floor(Date.now() / 1000) + payload.ttlSeconds)
}
if (silent) {
// APNs rejects a background push sent at priority 10.
headers['apns-priority'] = '5'
} else if (payload.apns?.interruptionLevel) {
headers['apns-priority'] = payload.apns.interruptionLevel === 'passive' ? '5' : '10'
}
const fetchParams: FetchOptions = {
method: 'POST',
headers,
body: JSON.stringify(this.buildApnsPayload(payload)),
// APNs requires HTTP/2
allowH2: true,
http2IdleTimeoutMs: APNS_IDLE_TIMEOUT_MS,
http2ConnectionGroup: `apns:${apns.appleTeamId}`,
}
if (params.timeoutMs !== undefined) {
fetchParams.timeoutMs = Math.min(params.timeoutMs, this.fetchUtils.maxFetchTimeoutMs)
}
const { fetchError, fetchResponse, fetchDuration } = await this.fetchUtils.trackedFetch({
url: `https://${apns.apnsHost}/3/device/${subscription.token}`,
fetchParams,
templateId: apns.templateId,
teamId: result.invocation.teamId,
})
result.invocation.state.timings.push({
kind: 'async_function',
duration_ms: fetchDuration,
})
let body: unknown = undefined
try {
body = await fetchResponse?.text()
if (typeof body === 'string' && body.length > 0) {
try {
body = parseJSON(body)
} catch {
// Pass through
}
}
} catch (e) {
addLog('error', `Failed to parse response body: ${e.message}`)
}
const status = fetchResponse?.status
pushNotificationSendDurationMs.labels({ platform: 'apns' }).observe(fetchDuration)
if (!fetchResponse || (status && status >= 400)) {
const err = normalizeApnsError(status, body, fetchError)
if (result.invocation.hogFunction.metadata?.sensitive !== true) {
addLog('debug', `APNs response ${status ?? '(none)'}: ${stringifyBody(body)}`)
}
// Apple signals a dead token with 410 / reason "Unregistered". Prune it and skip rather than
// retrying a token that will never work; other reasons are surfaced as failures.
if (err.unregistered) {
return { sent: false, unregistered: true, message: err.message }
}
return { sent: false, error: pushSendError('apns', err, parseRetryAfterMs(fetchResponse)) }
}
pushNotificationSentCounter.labels({ platform: 'apns' }).inc()
return { sent: true }
}
// Key the cache on a fingerprint of the signing-key material, not on the team id / key id — those
// come from tenant-controlled integration config, so another team could set matching values to
// collide with or poison this shared cache entry. Hashing the signing key makes the key unforgeable
// while still letting the fleet reuse one token per signing key.
private apnsKeyFingerprint(teamId: string, keyId: string, signingKey: string): string {
return createHash('sha256').update(`${teamId}:${keyId}:${signingKey}`).digest('hex')
}
private async generateApnsJwt(teamId: string, keyId: string, signingKey: string): Promise<string> {
const cacheKey = `${APNS_JWT_CACHE_PREFIX}${this.apnsKeyFingerprint(teamId, keyId, signingKey)}`
const inFlight = this.apnsJwtInFlight.get(cacheKey)
if (inFlight) {
return inFlight
}
const lookup = this.resolveApnsJwt(cacheKey, teamId, keyId, signingKey).finally(() =>
this.apnsJwtInFlight.delete(cacheKey)
)
this.apnsJwtInFlight.set(cacheKey, lookup)
return lookup
}
private async refreshApnsJwt(
teamId: string,
keyId: string,
signingKey: string,
rejectedJwt: string
): Promise<string | null> {
const fingerprint = this.apnsKeyFingerprint(teamId, keyId, signingKey)
const cacheKey = `${APNS_JWT_CACHE_PREFIX}${fingerprint}`
const replacement = await this.readReplacementApnsJwt(cacheKey, rejectedJwt)
if (replacement) {
return replacement
}
// Apple answers a second token inside its refresh window with TooManyProviderTokenUpdates for the rest of
// that window, so a new token cannot help while the rejected one is still young.
const rejectedIssuedAtMs = apnsJwtIssuedAtMs(rejectedJwt)
if (rejectedIssuedAtMs !== null && Date.now() - rejectedIssuedAtMs < APNS_JWT_REFRESH_WINDOW_SECONDS * 1000) {
apnsProviderTokenRefreshCounter.labels({ outcome: 'skipped' }).inc()
return null
}
// Fails closed: without Valkey a pod cannot tell whether the fleet already refreshed this key.
const refreshKey = `${APNS_JWT_REFRESH_PREFIX}${fingerprint}`
const claimed = await this.valkey.useClient({ name: 'apns-jwt-refresh', failOpen: true }, (client) =>
client.set(refreshKey, String(Date.now()), 'EX', APNS_JWT_REFRESH_WINDOW_SECONDS, 'NX')
)
if (claimed !== 'OK') {
const claimedAtMs = Number(
await this.valkey.useClient({ name: 'apns-jwt-refresh-read', failOpen: true }, (client) =>
client.get(refreshKey)
)
)
// Only a recent claim can still be signing. A key Apple keeps rejecting must not make every send wait.
const polls = Date.now() - claimedAtMs < APNS_JWT_FRESH_CLAIM_MS ? APNS_JWT_REPLACEMENT_POLLS : 0
for (let attempt = 0; attempt < polls; attempt++) {
await sleep(APNS_JWT_REPLACEMENT_POLL_MS)
const adopted = await this.readReplacementApnsJwt(cacheKey, rejectedJwt)
if (adopted) {
return adopted
}
}
apnsProviderTokenRefreshCounter.labels({ outcome: 'skipped' }).inc()
return null
}
if (this.apnsJwtLocalCache.get(cacheKey)?.jwt === rejectedJwt) {
this.apnsJwtLocalCache.delete(cacheKey)
}
const current = await this.valkey.useClient({ name: 'apns-jwt-drop', failOpen: true }, (client) =>
client.eval(APNS_JWT_DROP_IF_REJECTED_SCRIPT, 1, cacheKey, rejectedJwt)
)
if (typeof current === 'string' && current !== rejectedJwt) {
this.rememberApnsJwtLocally(cacheKey, current)
apnsProviderTokenRefreshCounter.labels({ outcome: 'adopted' }).inc()
return current
}
const jwt = await this.generateApnsJwt(teamId, keyId, signingKey)
if (jwt === rejectedJwt) {
return null
}
apnsProviderTokenRefreshCounter.labels({ outcome: 'minted' }).inc()
return jwt
}
private async readReplacementApnsJwt(cacheKey: string, rejectedJwt: string): Promise<string | null> {
const current = await this.valkey.useClient({ name: 'apns-jwt-read', failOpen: true }, (client) =>
client.get(cacheKey)
)
if (!current || current === rejectedJwt) {
return null
}
this.rememberApnsJwtLocally(cacheKey, current)
apnsProviderTokenRefreshCounter.labels({ outcome: 'adopted' }).inc()
return current
}
private async resolveApnsJwt(cacheKey: string, teamId: string, keyId: string, signingKey: string): Promise<string> {
const cached = await this.valkey.useClient({ name: 'apns-jwt-read', failOpen: true }, (client) =>
client.get(cacheKey)
)
if (cached) {
if (this.apnsJwtLocalCache.get(cacheKey)?.jwt !== cached) {
this.rememberApnsJwtLocally(cacheKey, cached)
}
return cached
}
const local = this.apnsJwtLocalCache.get(cacheKey)
if (local && local.expiresAtMs > Date.now()) {
return local.jwt
}
const header = Buffer.from(JSON.stringify({ alg: 'ES256', kid: keyId })).toString('base64url')
const now = Math.floor(Date.now() / 1000)
const claims = Buffer.from(JSON.stringify({ iss: teamId, iat: now })).toString('base64url')
const signingInput = `${header}.${claims}`
const sign = createSign('SHA256')
sign.update(signingInput)
const signature = sign.sign({ key: signingKey, dsaEncoding: 'ieee-p1363' }, 'base64url')
const minted = `${signingInput}.${signature}`
// ES256 signatures are randomized, so pods that miss together mint different tokens. Keep the first.
const stored = await this.valkey.useClient({ name: 'apns-jwt-write', failOpen: true }, (client) =>
client.set(cacheKey, minted, 'EX', APNS_JWT_TTL_SECONDS, 'NX')
)
let jwt = minted
if (stored !== 'OK') {
const winner = await this.valkey.useClient({ name: 'apns-jwt-read', failOpen: true }, (client) =>
client.get(cacheKey)
)
jwt = winner ?? minted
}
this.rememberApnsJwtLocally(cacheKey, jwt)
return jwt
}
private rememberApnsKeyAccepted(fingerprint: string): void {
const nowMs = Date.now()
let sharedAtMs = this.apnsKeyAcceptedAtMs.get(fingerprint)?.sharedAtMs ?? 0
if (nowMs - sharedAtMs >= APNS_KEY_ACCEPTED_SHARE_INTERVAL_MS) {
sharedAtMs = nowMs
void this.valkey.useClient({ name: 'apns-key-accepted-write', failOpen: true }, (client) =>
client.set(`${APNS_KEY_ACCEPTED_PREFIX}${fingerprint}`, '1', 'EX', APNS_JWT_REFRESH_WINDOW_SECONDS)
)
}
this.apnsKeyAcceptedAtMs.delete(fingerprint)
this.apnsKeyAcceptedAtMs.set(fingerprint, { acceptedAtMs: nowMs, sharedAtMs })
if (this.apnsKeyAcceptedAtMs.size > APNS_JWT_LOCAL_CACHE_MAX) {
const oldest = this.apnsKeyAcceptedAtMs.keys().next().value
if (oldest !== undefined) {
this.apnsKeyAcceptedAtMs.delete(oldest)
}
}
}
private async apnsKeyAcceptedRecently(fingerprint: string): Promise<boolean> {
const acceptedAtMs = this.apnsKeyAcceptedAtMs.get(fingerprint)?.acceptedAtMs
if (acceptedAtMs !== undefined && Date.now() - acceptedAtMs < APNS_JWT_REFRESH_WINDOW_SECONDS * 1000) {
return true
}
const shared = await this.valkey.useClient({ name: 'apns-key-accepted-read', failOpen: true }, (client) =>
client.get(`${APNS_KEY_ACCEPTED_PREFIX}${fingerprint}`)
)
return shared !== null && shared !== undefined
}
private rememberApnsJwtLocally(cacheKey: string, jwt: string): void {
const nowMs = Date.now()
// An adopted token can be older than this pod's read of it, and Apple rejects one past an hour.
const issuedAtMs = apnsJwtIssuedAtMs(jwt) ?? nowMs
this.apnsJwtLocalCache.set(cacheKey, { jwt, expiresAtMs: issuedAtMs + APNS_JWT_TTL_SECONDS * 1000 })
if (this.apnsJwtLocalCache.size <= APNS_JWT_LOCAL_CACHE_MAX) {
return
}
for (const [key, entry] of this.apnsJwtLocalCache) {
if (entry.expiresAtMs <= nowMs) {
this.apnsJwtLocalCache.delete(key)
}
}
// Map iterates in insertion order, so this drops the least recently minted. Safe because Valkey
// is the real cache — a dropped entry only costs a signature during an outage.
for (const key of this.apnsJwtLocalCache.keys()) {
if (this.apnsJwtLocalCache.size <= APNS_JWT_LOCAL_CACHE_MAX) {
break
}
this.apnsJwtLocalCache.delete(key)
}
}
private buildApnsPayload(payload: PushNotificationPayloadType): Record<string, unknown> {
if (payload.delivery === 'silent') {
// Apple drops a background push whose `aps` carries anything the user would see or hear.
return { ...(payload.data ?? {}), aps: { 'content-available': 1 } }
}
const alert: Record<string, unknown> = { title: payload.title ?? '' }
if (payload.body) {
alert.body = payload.body
}
if (payload.apns?.subtitle) {
alert.subtitle = payload.apns.subtitle
}
const aps: Record<string, unknown> = { alert }
if (payload.apns) {
if (payload.apns.sound) {
aps.sound = payload.apns.sound
}
if (payload.apns.badge !== undefined) {
aps.badge = payload.apns.badge
}
if (payload.apns.category) {
aps.category = payload.apns.category
}
if (payload.apns.threadId) {
aps['thread-id'] = payload.apns.threadId
}
if (payload.apns.interruptionLevel) {
aps['interruption-level'] = payload.apns.interruptionLevel
}
if (payload.apns.relevanceScore !== undefined) {
aps['relevance-score'] = payload.apns.relevanceScore
}
if (payload.apns.contentAvailable) {
aps['content-available'] = 1
}
if (payload.apns.mutableContent || payload.image) {
aps['mutable-content'] = 1
}
if (payload.apns.targetContentId) {
aps['target-content-id'] = payload.apns.targetContentId
}
} else if (payload.image) {
aps['mutable-content'] = 1
}
// Spread user data first so the reserved `aps` and `image_url` keys always win — a custom data
// key named `aps` must not overwrite the notification payload.
const result: Record<string, unknown> = { ...(payload.data ?? {}), aps }
if (payload.image) {
result['image_url'] = payload.image
}
return result
}
// Remove a device token the provider reported as no longer registered. Emitted as a $unset person
// update (flushed with the invocation's captured events) so we stop retrying dead tokens and don't
// accumulate stale ones on the person.
private pruneDeviceToken(
result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>,
invocation: CyclotronJobInvocationHogFunction,
recipient: PushRecipient,
propertyKeys: string[],
platform: 'fcm' | 'apns'
): void {
// Only the keys holding this device's token. Unsetting the app's whole subscription here would
// take every other device of the person with it.
result.capturedPostHogEvents.push({
team_id: invocation.teamId,
event: '$set',
distinct_id: recipient.distinctId,
timestamp: new Date().toISOString(),
properties: { $unset: propertyKeys },
})
// Drop the keys from the recipient read by the rest of this send, and from this invocation's own
// snapshot when they are the same person. A reschedule carries `state` forward unchanged, so a
// retry would read the same dead token, send to it again and queue the $unset again. A looked-up
// recipient is read fresh on the retry instead. `person` is replaced rather than edited in place
// because the object can be shared with the sibling invocations built from the same event.
if (recipient.properties) {
const properties = { ...recipient.properties }
for (const propertyKey of propertyKeys) {
delete properties[propertyKey]
}
recipient.properties = properties
}
const globals = result.invocation.state.globals
if (recipient.isRunPerson && globals.person?.properties) {
globals.person = { ...globals.person, properties: recipient.properties ?? {} }
}
pushNotificationTokenPrunedCounter.labels({ platform }).inc()
// A dead token is a non-delivery, not a failure to fix. Record it in the reason-labeled skip
// series too (not just the token-removal counter) so the skip metric accounts for it.
pushNotificationSkippedCounter.labels({ platform, reason: 'unregistered' }).inc()
}
private buildFcmMessage(token: string, payload: PushNotificationPayloadType): Record<string, unknown> {
const delivery = payload.delivery ?? 'notification'
const message: Record<string, unknown> = { token }
// Without a notification block Android hands the message to the app's own handler, even in the
// background, instead of drawing it itself.
if (delivery === 'notification') {
const notification: Record<string, string> = { title: payload.title ?? '' }
if (payload.body) {
notification.body = payload.body
}
if (payload.image) {
notification.image = payload.image
}
message.notification = notification
}
if (payload.data) {
message.data = payload.data
}
// Android-specific config
if (payload.android || payload.collapseKey || payload.ttlSeconds !== undefined) {
const android: Record<string, unknown> = {}
if (payload.collapseKey) {
android.collapse_key = payload.collapseKey
}
if (payload.ttlSeconds !== undefined) {
android.ttl = `${payload.ttlSeconds}s`
}
if (payload.android) {
if (payload.android.priority) {
android.priority = payload.android.priority.toUpperCase()
}
const androidNotification: Record<string, string> = {}
if (payload.android.channelId) {
androidNotification.channel_id = payload.android.channelId
}
if (payload.android.sound) {
androidNotification.sound = payload.android.sound
}
if (payload.android.tag) {
androidNotification.tag = payload.android.tag
}
if (payload.android.icon) {
androidNotification.icon = payload.android.icon
}
if (payload.android.color) {
androidNotification.color = payload.android.color
}
if (payload.android.clickAction) {
androidNotification.click_action = payload.android.clickAction
}
if (delivery === 'notification' && Object.keys(androidNotification).length > 0) {
android.notification = androidNotification
}
}
message.android = android
}
if (delivery === 'silent') {
const headers: Record<string, string> = { 'apns-push-type': 'background', 'apns-priority': '5' }
if (payload.collapseKey) {
headers['apns-collapse-id'] = payload.collapseKey
}
if (payload.ttlSeconds !== undefined) {
headers['apns-expiration'] = String(Math.floor(Date.now() / 1000) + payload.ttlSeconds)
}
message.apns = { headers, payload: { aps: { 'content-available': 1 } } }
return { message }
}
// APNS overrides (for iOS devices via FCM). Android data-only still shows the notification on iOS,
// and with no notification block to build it from the alert has to be spelled out here.
if (payload.apns || delivery === 'android_data_only') {
const apnsOverrides = payload.apns ?? {}
const aps: Record<string, unknown> = {}
if (apnsOverrides.sound) {
aps.sound = apnsOverrides.sound
}
if (apnsOverrides.badge !== undefined) {
aps.badge = apnsOverrides.badge
}
if (apnsOverrides.category) {
aps.category = apnsOverrides.category
}
if (apnsOverrides.threadId) {
aps['thread-id'] = apnsOverrides.threadId
}
if (apnsOverrides.interruptionLevel) {
aps['interruption-level'] = apnsOverrides.interruptionLevel
}
if (apnsOverrides.relevanceScore !== undefined) {
aps['relevance-score'] = apnsOverrides.relevanceScore
}
if (apnsOverrides.subtitle || delivery === 'android_data_only') {
// FCM's message.notification has no `subtitle`. For an iOS device delivered via FCM the
// subtitle must live in the APNS alert; include title/body so overriding the alert keeps them.
aps.alert = {
title: payload.title ?? '',
...(payload.body ? { body: payload.body } : {}),
...(apnsOverrides.subtitle ? { subtitle: apnsOverrides.subtitle } : {}),
}
}
const dataOnlyImage = delivery === 'android_data_only' ? payload.image : undefined
if (apnsOverrides.mutableContent || dataOnlyImage) {
aps['mutable-content'] = 1
}
if (apnsOverrides.targetContentId) {
aps['target-content-id'] = apnsOverrides.targetContentId
}
message.apns = {
payload: { aps },
...(dataOnlyImage ? { fcm_options: { image: dataOnlyImage } } : {}),
}
// Set collapse ID via APNS header if collapseKey is set
if (payload.collapseKey) {
;(message.apns as Record<string, unknown>).headers = {
'apns-collapse-id': payload.collapseKey,
}
}
if (payload.ttlSeconds !== undefined) {
const headers = ((message.apns as Record<string, unknown>).headers as Record<string, string>) ?? {}
headers['apns-expiration'] = String(Math.floor(Date.now() / 1000) + payload.ttlSeconds)
;(message.apns as Record<string, unknown>).headers = headers
}
}
return { message }
}
}