:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import { UUIDT } from '~/common/utils/utils'
import { PipelineResultType } from '~/ingestion/framework/results'
import { PluginEvent } from '~/plugin-scaffold'
import { createTestEventHeaders } from '~/tests/helpers/event-headers'
import { createTestTeam } from '~/tests/helpers/team'
import { createNormalizeEventStep } from './normalize-event-step'
describe('normalizeEventStep wrapper', () => {
const team = createTestTeam()
describe('distinctId conversion', () => {
it('converts number distinct_id to string', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 123 as any,
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'test event',
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.normalizedEvent.distinct_id).toBe('123')
expect(typeof result.value.normalizedEvent.distinct_id).toBe('string')
}
})
it('converts boolean distinct_id to string', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: true as any,
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'test event',
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.normalizedEvent.distinct_id).toBe('true')
expect(typeof result.value.normalizedEvent.distinct_id).toBe('string')
}
})
})
it('initializes empty properties object when missing', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'test event',
uuid: uuid,
// No properties field
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.normalizedEvent.properties).toBeDefined()
expect(typeof result.value.normalizedEvent.properties).toBe('object')
}
})
it('merges $set with priority: root level overrides properties', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'test event',
properties: {
$set: { key2: 'value3', key3: 'value4' },
},
$set: { key1: 'value1', key2: 'value2' },
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
// Root level $set should override properties.$set for same keys
expect(result.value.normalizedEvent.properties!.$set).toEqual({
key1: 'value1',
key2: 'value2', // overridden from root level
key3: 'value4',
})
}
})
it('merges $set_once with priority: root level overrides properties', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'test event',
properties: {
$set_once: { key2_once: 'value3', key3_once: 'value4' },
},
$set_once: { key1_once: 'value1', key2_once: 'value2' },
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
// Root level $set_once should override properties.$set_once for same keys
expect(result.value.normalizedEvent.properties!.$set_once).toEqual({
key1_once: 'value1',
key2_once: 'value2', // overridden from root level
key3_once: 'value4',
})
}
})
it('normalizes with processPerson=true and preserves person properties', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'default event',
properties: {
$set: {
a: 5,
},
$browser: 'Chrome',
},
$set: {
someProp: 'value',
},
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
// Since processPerson=true, person properties should be preserved and merged
expect(result.value.normalizedEvent).toEqual({
...event,
properties: {
$browser: 'Chrome',
$set: {
someProp: 'value',
a: 5,
$browser: 'Chrome',
},
$set_once: {
$initial_browser: 'Chrome',
},
},
})
expect(result.value.timestamp).toEqual(DateTime.fromISO(event.timestamp!, { zone: 'utc' }))
expect(result.value.headers).toEqual(createTestEventHeaders())
}
})
it('replaces null byte with unicode replacement character in distinct_id (processPerson=true)', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: '\u0000foo',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'default event',
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.normalizedEvent).toEqual({
...event,
distinct_id: '\uFFFDfoo',
properties: {},
})
expect(result.value.timestamp).toEqual(DateTime.fromISO(event.timestamp!, { zone: 'utc' }))
expect(result.value.headers).toEqual(createTestEventHeaders())
}
})
it('normalizes events with processPerson=false by dropping person-related properties', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: '$$heatmap',
properties: {
$set: {
a: 5,
},
$set_once: {
b: 10,
},
$unset: ['c'],
$browser: 'Chrome',
},
$set: {
someProp: 'value',
},
$set_once: {
foo: 'bar',
},
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: false,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
// The step always uses processPerson=false, so person-related properties should be dropped
expect(result.value.normalizedEvent).toEqual({
...event,
// $set and $set_once at root level should be gone
$set: undefined,
$set_once: undefined,
properties: {
$browser: 'Chrome',
$process_person_profile: false,
},
})
expect(result.value.timestamp).toEqual(DateTime.fromISO(event.timestamp!, { zone: 'utc' }))
expect(result.value.headers).toEqual(createTestEventHeaders())
}
})
it('merges $set from root level into properties.$set', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'test event',
properties: {
$set: {
propA: 'valueA',
},
},
$set: {
propB: 'valueB',
},
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
// Root level $set should merge into properties.$set
expect(result.value.normalizedEvent.properties!.$set).toEqual({
propA: 'valueA',
propB: 'valueB',
})
}
})
it('merges $set_once from root level into properties.$set_once', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'test event',
properties: {
$set_once: {
onceA: 'valueA',
},
},
$set_once: {
onceB: 'valueB',
},
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
// Root level $set_once should merge into properties.$set_once
expect(result.value.normalizedEvent.properties!.$set_once).toEqual({
onceA: 'valueA',
onceB: 'valueB',
})
}
})
it('adds $ip from event.ip to properties if not already present', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: '192.168.1.1',
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'test event',
properties: {},
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.normalizedEvent.properties!.$ip).toBe('192.168.1.1')
// ip field should be set to null
expect(result.value.normalizedEvent.ip).toBe(null)
}
})
it('does not override existing $ip in properties', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: '192.168.1.1',
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'test event',
properties: {
$ip: '10.0.0.1',
},
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
// Should keep the original $ip from properties
expect(result.value.normalizedEvent.properties!.$ip).toBe('10.0.0.1')
}
})
it('adds $sent_at to properties from event.sent_at', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
sent_at: '2020-02-23T02:14:00Z',
event: 'test event',
properties: {},
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.normalizedEvent.properties!.$sent_at).toBe('2020-02-23T02:14:00Z')
}
})
it('deletes $unset for processPerson=false', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'test event',
properties: {
$unset: ['prop1', 'prop2'],
},
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: false,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
// $unset should be removed for processPerson=false
expect(result.value.normalizedEvent.properties!.$unset).toBeUndefined()
}
})
it('handles $groupidentify event with processPerson=true by removing person properties', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: '$groupidentify',
properties: {
$set: {
a: 5,
},
},
$set: {
someProp: 'value',
},
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
// $groupidentify always has person properties removed
expect(result.value.normalizedEvent.$set).toBeUndefined()
expect(result.value.normalizedEvent.properties!.$set).toBeUndefined()
// But $process_person_profile is not added since processPerson=true
expect(result.value.normalizedEvent.properties!.$process_person_profile).toBeUndefined()
}
})
it('removes $process_person_profile when processPerson=true', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'test event',
properties: {
$process_person_profile: false,
other: 'value',
},
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: true,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
// $process_person_profile should be removed when processPerson=true (it's the default)
expect(result.value.normalizedEvent.properties!.$process_person_profile).toBeUndefined()
expect(result.value.normalizedEvent.properties!.other).toBe('value')
}
})
it('passes through additional input fields to the output', async () => {
const uuid = new UUIDT().toString()
const event: PluginEvent = {
distinct_id: 'my_id',
ip: null,
site_url: 'http://localhost',
team_id: team.id,
now: '2020-02-23T02:15:00Z',
timestamp: '2020-02-23T02:15:00Z',
event: 'test event',
uuid: uuid,
}
const step = createNormalizeEventStep()
const input = {
event,
headers: createTestEventHeaders(),
processPerson: false,
// Additional fields
customField: 'custom value',
anotherField: 123,
}
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
// Verify additional fields are passed through
expect((result.value as any).customField).toBe('custom value')
expect((result.value as any).anotherField).toBe(123)
// Verify normalized event is present
expect(result.value.normalizedEvent).toBeDefined()
expect(result.value.timestamp).toBeDefined()
}
})
})