:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import { GroupTypeManager } from '~/common/groups/group-type-manager'
import { TeamManager } from '~/common/utils/team-manager'
import { castTimestampOrNow } from '~/common/utils/utils'
import { GroupStoreForBatch } from '~/ingestion/common/groups/group-store-for-batch'
import { PipelineResultType } from '~/ingestion/framework/results'
import { createTestTeam } from '~/tests/helpers/team'
import { PreIngestionEvent, ProjectId, Team, TimestampFormat } from '~/types'
import { createProcessGroupsStep } from './process-groups-step'
const createTestPreIngestionEvent = (overrides: Partial<PreIngestionEvent> = {}): PreIngestionEvent => ({
eventUuid: 'test-uuid',
event: '$pageview',
teamId: 1,
projectId: 1 as ProjectId,
distinctId: 'test-distinct-id',
properties: {},
timestamp: castTimestampOrNow('2023-01-01T00:00:00.000Z', TimestampFormat.ISO),
...overrides,
})
type TestInput = {
preparedEvent: PreIngestionEvent
team: Team
processPerson: boolean
groupStoreForBatch: GroupStoreForBatch
}
describe('createProcessGroupsStep', () => {
let mockTeamManager: jest.Mocked<Pick<TeamManager, 'setTeamIngestedEvent'>>
let mockGroupTypeManager: jest.Mocked<Pick<GroupTypeManager, 'fetchGroupTypeIndex'>>
let mockGroupStore: jest.Mocked<Pick<GroupStoreForBatch, 'upsertGroup'>>
beforeEach(() => {
jest.clearAllMocks()
mockTeamManager = { setTeamIngestedEvent: jest.fn().mockResolvedValue(undefined) }
mockGroupTypeManager = { fetchGroupTypeIndex: jest.fn().mockResolvedValue(null) }
mockGroupStore = { upsertGroup: jest.fn().mockResolvedValue(undefined) }
})
const createInput = (overrides: Partial<TestInput> = {}): TestInput => ({
preparedEvent: createTestPreIngestionEvent(),
team: createTestTeam(),
processPerson: true,
groupStoreForBatch: mockGroupStore as unknown as GroupStoreForBatch,
...overrides,
})
const createStep = (skipUpdate = false) =>
createProcessGroupsStep<TestInput>(
mockTeamManager as unknown as TeamManager,
mockGroupTypeManager as unknown as GroupTypeManager,
{ SKIP_UPDATE_EVENT_AND_PROPERTIES_STEP: skipUpdate }
)
it.each([
{
desc: 'processPerson=false skips all group operations',
processPerson: false,
properties: { $groups: { org: 'posthog' } },
expectGroupTypeIndexCalls: 0,
},
{
desc: 'processPerson=true resolves group types via fetchGroupTypeIndex',
processPerson: true,
properties: { $groups: { org: 'posthog' } },
expectGroupTypeIndexCalls: 1,
},
{
desc: 'processPerson=true with no $groups skips group resolution',
processPerson: true,
properties: {},
expectGroupTypeIndexCalls: 0,
},
])('$desc', async ({ processPerson, properties, expectGroupTypeIndexCalls }) => {
const step = createStep()
const result = await step(
createInput({ processPerson, preparedEvent: createTestPreIngestionEvent({ properties }) })
)
expect(result.type).toBe(PipelineResultType.OK)
expect(mockGroupTypeManager.fetchGroupTypeIndex).toHaveBeenCalledTimes(expectGroupTypeIndexCalls)
})
it('enriches properties with $group_N when $groups is present', async () => {
mockGroupTypeManager.fetchGroupTypeIndex
.mockResolvedValueOnce(0) // org
.mockResolvedValueOnce(1) // project
const step = createStep()
const input = createInput({
preparedEvent: createTestPreIngestionEvent({
properties: { $groups: { org: 'posthog', project: 'posthog-js' } },
}),
})
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.preparedEvent.properties).toEqual({
$groups: { org: 'posthog', project: 'posthog-js' },
$group_0: 'posthog',
$group_1: 'posthog-js',
})
}
})
it('calls upsertGroup for $groupidentify events', async () => {
mockGroupTypeManager.fetchGroupTypeIndex.mockResolvedValue(0)
const step = createStep()
const result = await step(
createInput({
preparedEvent: createTestPreIngestionEvent({
event: '$groupidentify',
properties: {
$group_type: 'organization',
$group_key: 'org::5',
$group_set: { foo: 'bar' },
},
}),
})
)
expect(result.type).toBe(PipelineResultType.OK)
expect(mockGroupStore.upsertGroup).toHaveBeenCalledWith(1, 1, 0, 'org::5', { foo: 'bar' }, expect.any(DateTime))
})
it.each([
{
desc: 'missing $group_type',
properties: { $group_key: 'org::5', $group_set: { foo: 'bar' } },
},
{
desc: 'missing $group_key',
properties: { $group_type: 'organization', $group_set: { foo: 'bar' } },
},
])('does not call upsertGroup for $groupidentify when $desc', async ({ properties }) => {
mockGroupTypeManager.fetchGroupTypeIndex.mockResolvedValue(0)
const step = createStep()
const result = await step(
createInput({
preparedEvent: createTestPreIngestionEvent({ event: '$groupidentify', properties }),
})
)
expect(result.type).toBe(PipelineResultType.OK)
expect(mockGroupStore.upsertGroup).not.toHaveBeenCalled()
})
it.each([
{ desc: 'string', groupSet: 'some-random-token', receivedType: 'string' },
{ desc: 'number', groupSet: 42, receivedType: 'number' },
{ desc: 'boolean', groupSet: true, receivedType: 'boolean' },
{ desc: 'array', groupSet: ['a', 'b'], receivedType: 'array' },
{
desc: 'array and $group_type has no callable toString',
groupSet: ['a'],
receivedType: 'array',
groupType: { toString: null },
groupTypeInWarning: '{"toString":null}',
},
])(
'drops the event with an invalid_group_set warning when $group_set is a $desc',
async ({ groupSet, receivedType, groupType = 'organization', groupTypeInWarning = 'organization' }) => {
mockGroupTypeManager.fetchGroupTypeIndex.mockResolvedValue(0)
const step = createStep()
const result = await step(
createInput({
preparedEvent: createTestPreIngestionEvent({
event: '$groupidentify',
properties: {
$group_type: groupType,
$group_key: 'org::5',
$group_set: groupSet,
},
}),
})
)
expect(result.type).toBe(PipelineResultType.DROP)
expect(mockGroupStore.upsertGroup).not.toHaveBeenCalled()
expect(result.warnings).toEqual([
expect.objectContaining({
type: 'invalid_group_set',
details: expect.objectContaining({
eventUuid: 'test-uuid',
distinctId: 'test-distinct-id',
groupType: groupTypeInWarning,
groupKey: 'org::5',
receivedType,
}),
}),
])
}
)
it('upserts with empty properties when $group_set is null', async () => {
mockGroupTypeManager.fetchGroupTypeIndex.mockResolvedValue(0)
const step = createStep()
const result = await step(
createInput({
preparedEvent: createTestPreIngestionEvent({
event: '$groupidentify',
properties: {
$group_type: 'organization',
$group_key: 'org::5',
$group_set: null,
},
}),
})
)
expect(result.type).toBe(PipelineResultType.OK)
expect(result.warnings).toEqual([])
expect(mockGroupStore.upsertGroup).toHaveBeenCalledWith(1, 1, 0, 'org::5', {}, expect.any(DateTime))
})
it('does not call upsertGroup when group type index is null', async () => {
mockGroupTypeManager.fetchGroupTypeIndex.mockResolvedValue(null)
const step = createStep()
const result = await step(
createInput({
preparedEvent: createTestPreIngestionEvent({
event: '$groupidentify',
properties: {
$group_type: 'organization',
$group_key: 'org::5',
$group_set: { foo: 'bar' },
},
}),
})
)
expect(result.type).toBe(PipelineResultType.OK)
expect(mockGroupStore.upsertGroup).not.toHaveBeenCalled()
})
it('does not call upsertGroup for non-$groupidentify events', async () => {
const step = createStep()
await step(createInput())
expect(mockGroupStore.upsertGroup).not.toHaveBeenCalled()
})
it('skips updateGroupsAndFirstEvent when SKIP_UPDATE_EVENT_AND_PROPERTIES_STEP=true', async () => {
const step = createStep(true)
await step(createInput())
expect(mockTeamManager.setTeamIngestedEvent).not.toHaveBeenCalled()
})
it('calls updateGroupsAndFirstEvent when SKIP_UPDATE_EVENT_AND_PROPERTIES_STEP=false', async () => {
const step = createStep()
await step(createInput())
expect(mockTeamManager.setTeamIngestedEvent).toHaveBeenCalled()
})
it('swallows updateGroupsAndFirstEvent errors and continues processing', async () => {
mockTeamManager.setTeamIngestedEvent.mockRejectedValue(new Error('DB error'))
const step = createStep()
const result = await step(createInput())
expect(result.type).toBe(PipelineResultType.OK)
})
it('skips updateGroupsAndFirstEvent for $$plugin_metrics events', async () => {
const step = createStep()
await step(
createInput({
preparedEvent: createTestPreIngestionEvent({ event: '$$plugin_metrics' }),
})
)
expect(mockTeamManager.setTeamIngestedEvent).not.toHaveBeenCalled()
})
})