Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.9 KB
Raw Blame
import { DateTime } from 'luxon'

import { GroupTypeManager } from '~/common/groups/group-type-manager'
import { asPostgresParameterText, sanitizeString } from '~/common/utils/db/utils'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
import { TeamManager } from '~/common/utils/team-manager'
import { GroupStoreForBatch } from '~/ingestion/common/groups/group-store-for-batch'
import { PipelineWarning } from '~/ingestion/framework/pipeline.interface'
import { drop, ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { Properties } from '~/plugin-scaffold'
import { PreIngestionEvent, ProjectId, Team, TeamId } from '~/types'

import { EventPipelineRunnerOptions } from './event-pipeline-options'
import { addGroupProperties } from './groups'

const EVENTS_WITHOUT_EVENT_DEFINITION = ['$$plugin_metrics']

export interface ProcessGroupsStepInput {
    preparedEvent: PreIngestionEvent
    team: Team
    processPerson: boolean
    groupStoreForBatch: GroupStoreForBatch
}

export type ProcessGroupsStepResult<TInput> = TInput

export function createProcessGroupsStep<TInput extends ProcessGroupsStepInput>(
    teamManager: TeamManager,
    groupTypeManager: GroupTypeManager,
    options: Pick<EventPipelineRunnerOptions, 'SKIP_UPDATE_EVENT_AND_PROPERTIES_STEP'>
): ProcessingStep<TInput, ProcessGroupsStepResult<TInput>> {
    return async function processGroupsStep(input: TInput) {
        const { preparedEvent, team, processPerson, groupStoreForBatch } = input

        if (!options.SKIP_UPDATE_EVENT_AND_PROPERTIES_STEP) {
            try {
                await updateGroupsAndFirstEvent(teamManager, groupTypeManager, team, preparedEvent)
            } catch (err) {
                captureException(err, { tags: { team_id: team.id } })
                logger.warn('⚠️', 'Failed to update groups and first event for event ', {
                    event: preparedEvent.event,
                    properties: preparedEvent.properties,
                    err,
                })
            }
        }

        if (processPerson) {
            preparedEvent.properties = await addGroupProperties(
                team.id,
                team.project_id,
                preparedEvent.properties,
                groupTypeManager,
                DateTime.fromISO(preparedEvent.timestamp)
            )

            if (preparedEvent.event === '$groupidentify') {
                const invalidGroupSetWarning = validateGroupSet(preparedEvent)
                if (invalidGroupSetWarning) {
                    return drop('invalid_group_set', [], [invalidGroupSetWarning])
                }
                await upsertGroup(
                    groupTypeManager,
                    groupStoreForBatch,
                    team.id,
                    team.project_id,
                    preparedEvent.properties,
                    DateTime.fromISO(preparedEvent.timestamp)
                )
            }
        }

        return ok(input)
    }
}

async function updateGroupsAndFirstEvent(
    teamManager: TeamManager,
    groupTypeManager: GroupTypeManager,
    team: Team,
    preparedEvent: PreIngestionEvent
): Promise<void> {
    if (EVENTS_WITHOUT_EVENT_DEFINITION.includes(preparedEvent.event)) {
        return
    }

    const promises: Promise<unknown>[] = [teamManager.setTeamIngestedEvent(team, preparedEvent.properties)]

    if (preparedEvent.event === '$groupidentify') {
        const { $group_type: groupType, $group_set: groupPropertiesToSet } = preparedEvent.properties
        if (groupType != null && groupPropertiesToSet != null) {
            promises.push(
                groupTypeManager.fetchGroupTypeIndex(
                    team.id,
                    team.project_id,
                    groupType,
                    DateTime.fromISO(preparedEvent.timestamp)
                )
            )
        }
    }

    await Promise.all(promises)
}

// Group properties must be a plain JSON object — anything else (string, number,
// array, ...) would reach Postgres as an invalid jsonb parameter and poison the
// whole write batch.
function validateGroupSet(preparedEvent: PreIngestionEvent): PipelineWarning | null {
    const { $group_type: groupType, $group_key: groupKey, $group_set: groupPropertiesToSet } = preparedEvent.properties

    if (
        groupPropertiesToSet == null ||
        (typeof groupPropertiesToSet === 'object' && !Array.isArray(groupPropertiesToSet))
    ) {
        return null
    }

    return {
        type: 'invalid_group_set',
        details: {
            eventUuid: preparedEvent.eventUuid,
            distinctId: preparedEvent.distinctId,
            groupType: asPostgresParameterText(groupType),
            groupKey: sanitizeString(asPostgresParameterText(groupKey)),
            receivedType: Array.isArray(groupPropertiesToSet) ? 'array' : typeof groupPropertiesToSet,
        },
        // No `key`: the limiter's bucket map never evicts, so a client-supplied
        // key (like $group_key) would let a sender bypass the per-team debounce
        // and grow the map without bound. Debounce per team and type instead.
    }
}

async function upsertGroup(
    groupTypeManager: GroupTypeManager,
    groupStore: GroupStoreForBatch,
    teamId: TeamId,
    projectId: ProjectId,
    properties: Properties,
    timestamp: DateTime
): Promise<void> {
    if (!properties['$group_type'] || !properties['$group_key']) {
        return
    }

    const { $group_type: groupType, $group_key: groupKey, $group_set: groupPropertiesToSet } = properties
    const groupTypeIndex = await groupTypeManager.fetchGroupTypeIndex(teamId, projectId, groupType, timestamp)
    if (groupTypeIndex !== null) {
        await groupStore.upsertGroup(
            teamId,
            projectId,
            groupTypeIndex,
            sanitizeString(asPostgresParameterText(groupKey)),
            groupPropertiesToSet || {},
            timestamp
        )
    }
}