:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { PluginEvent, Properties } from '~/plugin-scaffold'
import { ClickHouseEvent, PipelineEvent, PostIngestionEvent, RawClickHouseEvent } from '~/types'
import { personInitialAndUTMProperties, sanitizeString } from './db/utils'
import { chainToElements } from './elements-chain'
import { parseJSON } from './json-parse'
import { clickHouseTimestampToDateTime, normalizeSessionId } from './utils'
/** Parse an event row SELECTed from ClickHouse into a more malleable form. */
export function parseRawClickHouseEvent(rawEvent: RawClickHouseEvent): ClickHouseEvent {
return {
...rawEvent,
timestamp: clickHouseTimestampToDateTime(rawEvent.timestamp),
created_at: clickHouseTimestampToDateTime(rawEvent.created_at),
properties: rawEvent.properties ? parseJSON(rawEvent.properties) : {},
elements_chain: rawEvent.elements_chain ? chainToElements(rawEvent.elements_chain, rawEvent.team_id) : null,
person_created_at: rawEvent.person_created_at
? clickHouseTimestampToDateTime(rawEvent.person_created_at)
: null,
person_properties: rawEvent.person_properties ? parseJSON(rawEvent.person_properties) : {},
group0_properties: rawEvent.group0_properties ? parseJSON(rawEvent.group0_properties) : {},
group1_properties: rawEvent.group1_properties ? parseJSON(rawEvent.group1_properties) : {},
group2_properties: rawEvent.group2_properties ? parseJSON(rawEvent.group2_properties) : {},
group3_properties: rawEvent.group3_properties ? parseJSON(rawEvent.group3_properties) : {},
group4_properties: rawEvent.group4_properties ? parseJSON(rawEvent.group4_properties) : {},
group0_created_at: rawEvent.group0_created_at
? clickHouseTimestampToDateTime(rawEvent.group0_created_at)
: null,
group1_created_at: rawEvent.group1_created_at
? clickHouseTimestampToDateTime(rawEvent.group1_created_at)
: null,
group2_created_at: rawEvent.group2_created_at
? clickHouseTimestampToDateTime(rawEvent.group2_created_at)
: null,
group3_created_at: rawEvent.group3_created_at
? clickHouseTimestampToDateTime(rawEvent.group3_created_at)
: null,
group4_created_at: rawEvent.group4_created_at
? clickHouseTimestampToDateTime(rawEvent.group4_created_at)
: null,
}
}
/**
* Elements parsing can be really slow so it is only done when required by the caller.
* It mutates the event which is not ideal but the performance gains of lazy loading it were deemed worth it.
*/
export function mutatePostIngestionEventWithElementsList(event: PostIngestionEvent): void {
if (event.elementsList) {
// Don't set if already done before
return
}
event.elementsList = event.properties['$elements_chain']
? chainToElements(event.properties['$elements_chain'], event.teamId)
: []
event.elementsList = event.elementsList.map((element) => ({
...element,
attr_class: element.attributes?.attr__class ?? element.attr_class,
$el_text: element.text,
}))
}
/// Does normalization steps involving the $process_person_profile property. This is currently a separate
/// function because `normalizeEvent` is called from multiple places, some early in the pipeline,
/// and we want to have one trusted place where `$process_person_profile` is handled and passed through
/// all of the processing steps.
///
/// If `formPipelineEvent` is removed this can easily be combined with `normalizeEvent`.
export function normalizeProcessPerson<T extends PipelineEvent | PluginEvent>(event: T, processPerson: boolean): T {
const properties = event.properties ?? {}
if (!processPerson || event.event === '$groupidentify') {
delete event.$set
delete event.$set_once
// In an abundance of caution and future proofing, we delete the $unset field from the
// event if it is set. As of this writing we only *read* $unset out of `properties`, but
// we may as well future-proof this code path.
delete (event as any)['$unset']
delete properties.$set
delete properties.$set_once
delete properties.$unset
}
if (!processPerson) {
// Recorded for clarity and so that the property exists if it is ever sent elsewhere,
// e.g. for migrations.
properties.$process_person_profile = false
} else {
// Removed as it is the default, note that we have record the `person_mode` column
// in ClickHouse for all events.
delete properties.$process_person_profile
}
event.properties = properties
return event
}
/**
* Sanitizes event inputs and merges top-level $set/$set_once into properties.
* Does NOT call personInitialAndUTMProperties - that's done in normalizeEvent
* which should only be called once after transformations.
*
* This split ensures:
* - Transformations see clean events without pre-computed $set/$set_once from UTM/browser fields
* - Transformations can add properties that become person properties
* - personInitialAndUTMProperties runs only once, after transformations
*/
export function sanitizeEvent<T extends PipelineEvent | PluginEvent>(event: T): T {
event.distinct_id = sanitizeString(String(event.distinct_id))
const properties = event.properties ?? {}
if (event['$set']) {
properties['$set'] = { ...properties['$set'], ...event['$set'] }
}
if (event['$set_once']) {
properties['$set_once'] = { ...properties['$set_once'], ...event['$set_once'] }
}
if (!properties['$ip'] && event.ip) {
// if $ip wasn't sent with the event, then add what we got from capture
properties['$ip'] = event.ip
}
// For safety while PluginEvent still has an `ip` field
event.ip = null
if (event.sent_at) {
properties['$sent_at'] = event.sent_at
}
if (typeof properties['$session_id'] === 'string') {
properties['$session_id'] = normalizeSessionId(properties['$session_id'])
}
event.properties = properties
return event
}
/**
* Fills the event's own $os from the $os_name some SDKs report instead:
* https://github.com/PostHog/posthog-js-lite/issues/244.
*
* This describes the event, not the person. Call it after transformations: the user agent populator
* plugin skips an event that already carries $os, so filling $os earlier would stop it deriving
* $browser, $device, and the rest.
*
* Server events are skipped so a server's host OS does not land in the event's own $os. The person
* side is handled separately: personInitialAndUTMProperties skips lifting $os and $os_version and
* skips folding $os_name into the sticky $initial_os for $is_server events. The host OS still reaches
* the person under the raw $os_name and $initial_os_name keys.
*/
export function normalizeOsAlias(properties: Properties): void {
if (properties['$is_server'] === true) {
return
}
const osName = properties['$os_name']
if (osName !== undefined && !('$os' in properties)) {
properties['$os'] = osName
}
}
/**
* Full event normalization including person property mapping.
* This should only be called ONCE per event, after any transformations.
* Calling it multiple times is wasteful as personInitialAndUTMProperties
* does significant work iterating properties.
*/
export function normalizeEvent<T extends PipelineEvent | PluginEvent>(event: T): T {
event = sanitizeEvent(event)
if (!['$snapshot', '$performance_event'].includes(event.event)) {
normalizeOsAlias(event.properties!)
event.properties = personInitialAndUTMProperties(event.properties!)
}
return event
}