Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

21.5 KB
Raw Blame
import { mockProducer } from '~/tests/helpers/mocks/producer.mock'

import { DateTime } from 'luxon'

import { KAFKA_PERSON, KAFKA_PERSON_DISTINCT_ID, KAFKA_PERSON_MERGE_EVENTS } from '~/common/config/kafka-topics'
import { INGESTION_WARNINGS_OUTPUT } from '~/common/outputs'
import { PERSONS_OUTPUT, PERSON_DISTINCT_IDS_OUTPUT, PERSON_MERGE_EVENTS_OUTPUT } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'
import { PostgresPersonRepository } from '~/common/persons/repositories/postgres-person-repository'
import { UUIDT } from '~/common/utils/utils'
import { BatchWritingPersonsStore } from '~/ingestion/common/persons/batch-writing-person-store'
import { BatchBoundPersonsStore } from '~/ingestion/common/persons/persons-store-for-batch'
import { PipelineResultType, isOkResult } from '~/ingestion/framework/results'
import { PluginEvent, Properties } from '~/plugin-scaffold'
import { createTestEventHeaders } from '~/tests/helpers/event-headers'
import { IngestionTestInfra, createIngestionTestInfra } from '~/tests/helpers/ingestion-e2e'
import { createOrganization, createTeam, getTeam } from '~/tests/helpers/sql'
import { EventHeaders, InternalPerson, PropertiesLastOperation, PropertiesLastUpdatedAt, Team } from '~/types'

import { createNormalizeEventStep } from './normalize-event-step'
import { createNormalizeProcessPersonFlagStep } from './normalize-process-person-flag-step'
import { ProcessPersonlessInput, createProcessPersonlessStep } from './process-personless-step'

function createPersonOutputs(_infra: IngestionTestInfra) {
    return new IngestionOutputs({
        [PERSONS_OUTPUT]: new SingleIngestionOutput(PERSONS_OUTPUT, KAFKA_PERSON, mockProducer, 'test'),
        [PERSON_DISTINCT_IDS_OUTPUT]: new SingleIngestionOutput(
            PERSON_DISTINCT_IDS_OUTPUT,
            KAFKA_PERSON_DISTINCT_ID,
            mockProducer,
            'test'
        ),
    })
}

async function createPerson(
    infra: IngestionTestInfra,
    createdAt: DateTime,
    properties: Properties,
    propertiesLastUpdatedAt: PropertiesLastUpdatedAt,
    propertiesLastOperation: PropertiesLastOperation,
    teamId: number,
    isUserId: number | null,
    isIdentified: boolean,
    uuid: string,
    primaryDistinctId: { distinctId: string; version?: number },
    extraDistinctIds?: { distinctId: string; version?: number }[]
): Promise<InternalPerson> {
    const personRepository = new PostgresPersonRepository(infra.postgres)
    const result = await personRepository.createPerson(
        createdAt,
        properties,
        propertiesLastUpdatedAt,
        propertiesLastOperation,
        teamId,
        isUserId,
        isIdentified,
        uuid,
        primaryDistinctId,
        extraDistinctIds
    )
    if (!result.success) {
        throw new Error('Failed to create person')
    }
    const personOutputs = createPersonOutputs(infra)
    await Promise.all(result.messages.map((msg) => personOutputs.produce(msg.output, { value: msg.value, key: null })))
    return result.person
}

describe('createProcessPersonlessStep', () => {
    let infra: IngestionTestInfra
    let teamId: number
    let team: Team
    let pluginEvent: PluginEvent
    let timestamp: DateTime
    let personsStore: BatchWritingPersonsStore

    beforeEach(async () => {
        infra = await createIngestionTestInfra()
        const organizationId = await createOrganization(infra.postgres)
        teamId = await createTeam(infra.postgres, organizationId)
        team = (await getTeam(infra.postgres, teamId))!

        const personRepository = new PostgresPersonRepository(infra.postgres)
        const storeOutputs = new IngestionOutputs({
            [PERSONS_OUTPUT]: new SingleIngestionOutput(PERSONS_OUTPUT, KAFKA_PERSON, mockProducer, 'test'),
            [PERSON_DISTINCT_IDS_OUTPUT]: new SingleIngestionOutput(
                PERSON_DISTINCT_IDS_OUTPUT,
                KAFKA_PERSON_DISTINCT_ID,
                mockProducer,
                'test'
            ),
            [INGESTION_WARNINGS_OUTPUT]: new SingleIngestionOutput(
                INGESTION_WARNINGS_OUTPUT,
                'ingestion_warnings_test',
                mockProducer,
                'test'
            ),
            [PERSON_MERGE_EVENTS_OUTPUT]: new SingleIngestionOutput(
                PERSON_MERGE_EVENTS_OUTPUT,
                KAFKA_PERSON_MERGE_EVENTS,
                mockProducer,
                'test'
            ),
        })
        personsStore = new BatchWritingPersonsStore(personRepository, storeOutputs)

        pluginEvent = {
            distinct_id: 'test-user-123',
            ip: null,
            site_url: 'http://localhost',
            team_id: teamId,
            now: '2020-02-23T02:15:00Z',
            timestamp: '2020-02-23T02:15:00Z',
            event: '$pageview',
            properties: {},
            uuid: new UUIDT().toString(),
        }
        timestamp = DateTime.fromISO(pluginEvent.timestamp!)
    })

    afterEach(async () => {
        await infra.close()
    })

    const createInput = (overrides: Partial<ProcessPersonlessInput> = {}): ProcessPersonlessInput => ({
        normalizedEvent: pluginEvent,
        team,
        timestamp,
        processPerson: false,
        processPersonExplicitlyTrue: false,
        forceDisablePersonProcessing: false,
        personsStoreForBatch: new BatchBoundPersonsStore(personsStore, 0),
        ...overrides,
    })

    // Builds the step with the flag-called personless default enabled for all teams. The
    // production default is '' (opt-in per team), so tests opt in explicitly here.
    const buildStep = () => createProcessPersonlessStep('*')

    it('passes through when processPerson is true', async () => {
        const step = buildStep()
        const input = createInput({ processPerson: true })

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.OK)
        if (isOkResult(result)) {
            expect(result.value.personlessPerson).toBeUndefined()
        }
    })

    describe('$feature_flag_called personless default', () => {
        const flagCalledEvent = (properties: Properties = {}): PluginEvent => ({
            ...pluginEvent,
            event: '$feature_flag_called',
            properties: { $feature_flag: 'new-homepage', $feature_flag_response: 'test', ...properties },
        })

        it('keeps the event personful when a person already exists', async () => {
            const personUuid = new UUIDT().toString()

            await createPerson(infra, timestamp, { name: 'John' }, {}, {}, teamId, null, false, personUuid, {
                distinctId: pluginEvent.distinct_id,
            })

            const step = buildStep()
            const result = await step(createInput({ processPerson: true, normalizedEvent: flagCalledEvent() }))

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                expect(result.value.processPerson).toBe(true)
                expect(result.value.personlessPerson).toBeUndefined()
            }
        })

        it('keeps the event personful when it carries group keys', async () => {
            const fetchForCheckingSpy = jest.spyOn(personsStore, 'fetchForChecking')

            const step = buildStep()
            const result = await step(
                createInput({
                    processPerson: true,
                    normalizedEvent: flagCalledEvent({ $groups: { organization: 'org-1' } }),
                })
            )

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                expect(result.value.processPerson).toBe(true)
                expect(result.value.personlessPerson).toBeUndefined()
            }
            expect(fetchForCheckingSpy).not.toHaveBeenCalled()
        })

        it('still defaults to personless when $groups is empty', async () => {
            const step = buildStep()
            const result = await step(
                createInput({ processPerson: true, normalizedEvent: flagCalledEvent({ $groups: {} }) })
            )

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                expect(result.value.processPerson).toBe(false)
                expect(result.value.personlessPerson).toBeDefined()
            }
        })

        it('keeps the event personful when the default is disabled via config', async () => {
            const fetchForCheckingSpy = jest.spyOn(personsStore, 'fetchForChecking')

            const step = createProcessPersonlessStep('')
            const result = await step(createInput({ processPerson: true, normalizedEvent: flagCalledEvent() }))

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                expect(result.value.processPerson).toBe(true)
                expect(result.value.personlessPerson).toBeUndefined()
            }
            expect(fetchForCheckingSpy).not.toHaveBeenCalled()
        })

        it('keeps the event personful when the team is excluded', async () => {
            const fetchForCheckingSpy = jest.spyOn(personsStore, 'fetchForChecking')

            const step = createProcessPersonlessStep('*', `${teamId}`)
            const result = await step(createInput({ processPerson: true, normalizedEvent: flagCalledEvent() }))

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                expect(result.value.processPerson).toBe(true)
                expect(result.value.personlessPerson).toBeUndefined()
            }
            expect(fetchForCheckingSpy).not.toHaveBeenCalled()
        })

        it('applies the default when the team is in the configured team list', async () => {
            const step = createProcessPersonlessStep(`${teamId}`)
            const result = await step(createInput({ processPerson: true, normalizedEvent: flagCalledEvent() }))

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                expect(result.value.processPerson).toBe(false)
                expect(result.value.personlessPerson).toBeDefined()
            }
        })

        it('keeps the event personful when $process_person_profile was explicitly true', async () => {
            const fetchForCheckingSpy = jest.spyOn(personsStore, 'fetchForChecking')

            const step = buildStep()
            const result = await step(
                createInput({
                    processPerson: true,
                    processPersonExplicitlyTrue: true,
                    normalizedEvent: flagCalledEvent(),
                })
            )

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                expect(result.value.processPerson).toBe(true)
                expect(result.value.personlessPerson).toBeUndefined()
            }
            expect(fetchForCheckingSpy).not.toHaveBeenCalled()
        })

        it('defaults to personless when no person exists', async () => {
            const step = buildStep()
            const result = await step(
                createInput({
                    processPerson: true,
                    normalizedEvent: flagCalledEvent({ $set: { email: '[email protected]' } }),
                })
            )

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                expect(result.value.processPerson).toBe(false)
                expect(result.value.personlessPerson).toBeDefined()
                expect(result.value.personlessPerson!.properties).toEqual({})
                // The event is re-normalized as personless: $set stripped, personless stamp added.
                expect(result.value.normalizedEvent.properties?.$set).toBeUndefined()
                expect(result.value.normalizedEvent.properties?.$process_person_profile).toBe(false)
            }
        })

        it('takes the generic personless path, not the defaulting branch, when already force-disabled', async () => {
            const fetchForCheckingSpy = jest.spyOn(personsStore, 'fetchForChecking')

            const step = buildStep()
            const result = await step(
                createInput({ normalizedEvent: flagCalledEvent(), forceDisablePersonProcessing: true })
            )

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                expect(result.value.personlessPerson).toBeDefined()
            }
            expect(fetchForCheckingSpy).not.toHaveBeenCalled()
        })

        // Pipes an event through the same normalization steps that precede the personless
        // step in the real subpipeline, so these tests cover the boundary where
        // normalizeProcessPerson strips the explicit $process_person_profile property.
        const runThroughNormalization = async (event: PluginEvent): Promise<ProcessPersonlessInput> => {
            const normalizeFlagStep = createNormalizeProcessPersonFlagStep<{
                event: PluginEvent
                headers: EventHeaders
                team: Team
            }>()
            const flagResult = await normalizeFlagStep({ event, team, headers: createTestEventHeaders() })
            if (!isOkResult(flagResult)) {
                throw new Error('expected normalize flag step to return ok')
            }

            const normalizeEventStep = createNormalizeEventStep<typeof flagResult.value>()
            const normalizeResult = await normalizeEventStep(flagResult.value)
            if (!isOkResult(normalizeResult)) {
                throw new Error('expected normalize event step to return ok')
            }
            return { ...normalizeResult.value, personsStoreForBatch: new BatchBoundPersonsStore(personsStore, 0) }
        }

        it('keeps an explicit-true event personful after normalization strips the property', async () => {
            const personlessStep = buildStep()
            const normalized = await runThroughNormalization(flagCalledEvent({ $process_person_profile: true }))

            // Normalization removes the explicit-true property for personful events, so the
            // personless step must rely on the captured processPersonExplicitlyTrue flag.
            expect(normalized.normalizedEvent.properties?.$process_person_profile).toBeUndefined()

            const result = await personlessStep(normalized)

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                expect(result.value.processPerson).toBe(true)
                expect(result.value.personlessPerson).toBeUndefined()
            }
        })

        it('defaults an event without the property to personless through normalization', async () => {
            const personlessStep = buildStep()
            const normalized = await runThroughNormalization(flagCalledEvent())

            const result = await personlessStep(normalized)

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                expect(result.value.processPerson).toBe(false)
                expect(result.value.personlessPerson).toBeDefined()
                expect(result.value.normalizedEvent.properties?.$process_person_profile).toBe(false)
            }
        })

        it('skips the defaulting branch for explicit $process_person_profile=false events', async () => {
            const personlessStep = buildStep()
            const normalized = await runThroughNormalization(flagCalledEvent({ $process_person_profile: false }))

            const result = await personlessStep(normalized)

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                expect(result.value.processPerson).toBe(false)
                expect(result.value.personlessPerson).toBeDefined()
            }
        })
    })

    describe('basic personless functionality', () => {
        it('returns fake person when no existing person found', async () => {
            const step = buildStep()
            const result = await step(createInput())

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                const person = result.value.personlessPerson!
                expect(person.team_id).toBe(teamId)
                expect(person.properties).toEqual({})
                expect(person.uuid).toBeDefined()
                expect(person.created_at.toISO()).toBe('1970-01-01T00:00:05.000Z')
                expect(person.force_upgrade).toBeUndefined()
            }
        })

        it('returns existing person with empty properties when person exists', async () => {
            const personUuid = new UUIDT().toString()

            await createPerson(infra, timestamp, { name: 'John' }, {}, {}, teamId, null, false, personUuid, {
                distinctId: pluginEvent.distinct_id,
            })

            const step = buildStep()
            const result = await step(createInput())

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                const person = result.value.personlessPerson!
                expect(person.uuid).toBe(personUuid)
                expect(person.properties).toEqual({})
                expect(person.force_upgrade).toBeUndefined()
            }
        })
    })

    describe('force_upgrade logic', () => {
        it('sets force_upgrade=true when event timestamp > person.created_at + 1 minute', async () => {
            const personUuid = new UUIDT().toString()
            const personCreatedAt = DateTime.fromISO('2020-02-23T02:00:00Z')

            await createPerson(infra, personCreatedAt, {}, {}, {}, teamId, null, false, personUuid, {
                distinctId: pluginEvent.distinct_id,
            })

            const step = buildStep()
            const result = await step(createInput())

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                const person = result.value.personlessPerson!
                expect(person.uuid).toBe(personUuid)
                expect(person.force_upgrade).toBe(true)
            }
        })

        it('does NOT set force_upgrade when within 1-minute grace period', async () => {
            const personUuid = new UUIDT().toString()
            const personCreatedAt = DateTime.fromISO('2020-02-23T02:14:30Z')

            await createPerson(infra, personCreatedAt, {}, {}, {}, teamId, null, false, personUuid, {
                distinctId: pluginEvent.distinct_id,
            })

            const step = buildStep()
            const result = await step(createInput())

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                const person = result.value.personlessPerson!
                expect(person.uuid).toBe(personUuid)
                expect(person.force_upgrade).toBeUndefined()
            }
        })

        it('ignores force_upgrade when team.person_processing_opt_out=true', async () => {
            team.person_processing_opt_out = true

            const personUuid = new UUIDT().toString()
            const personCreatedAt = DateTime.fromISO('2020-02-23T02:00:00Z')

            await createPerson(infra, personCreatedAt, {}, {}, {}, teamId, null, false, personUuid, {
                distinctId: pluginEvent.distinct_id,
            })

            const step = buildStep()
            const result = await step(createInput())

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                const person = result.value.personlessPerson!
                expect(person.uuid).toBe(personUuid)
                expect(person.force_upgrade).toBeUndefined()
            }
        })
    })

    describe('forceDisablePersonProcessing', () => {
        it('skips all DB operations and returns fake person immediately when true', async () => {
            const fetchForCheckingSpy = jest.spyOn(personsStore, 'fetchForChecking')

            const step = buildStep()
            const result = await step(createInput({ forceDisablePersonProcessing: true }))

            expect(result.type).toBe(PipelineResultType.OK)
            if (isOkResult(result)) {
                const person = result.value.personlessPerson!
                expect(person.team_id).toBe(teamId)
                expect(person.properties).toEqual({})
                expect(person.created_at.toISO()).toBe('1970-01-01T00:00:05.000Z')
            }

            expect(fetchForCheckingSpy).not.toHaveBeenCalled()
        })

        it('performs normal processing when false', async () => {
            const fetchForCheckingSpy = jest.spyOn(personsStore, 'fetchForChecking')

            const step = buildStep()
            await step(createInput())

            expect(fetchForCheckingSpy).toHaveBeenCalled()
        })

        it('works with different distinct IDs', async () => {
            const distinctIds = ['user-1', 'user-2', 'user-3']
            const step = buildStep()

            for (const distinctId of distinctIds) {
                const result = await step(
                    createInput({
                        normalizedEvent: { ...pluginEvent, distinct_id: distinctId },
                        forceDisablePersonProcessing: true,
                    })
                )

                expect(result.type).toBe(PipelineResultType.OK)
                if (isOkResult(result)) {
                    expect(result.value.personlessPerson!.team_id).toBe(teamId)
                    expect(result.value.personlessPerson!.properties).toEqual({})
                }
            }
        })
    })
})