Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.6 KB
Raw Blame
import { EventSchemaEnforcementManager } from '~/common/utils/event-schema-enforcement-manager'
import { drop, ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { EventSchemaEnforcement, PipelineEvent, Team } from '~/types'

import { isValidClickHouseDateTime } from './clickhouse-datetime-parser'

/**
 * Checks if a value can be coerced to the given PostHog property type.
 * See: PropertySwapper._field_type_to_property_call in posthog/hogql/transforms/property_types.py
 */
function canCoerceToType(value: unknown, propertyType: string): boolean {
    if (value === null || value === undefined) {
        return false
    }

    switch (propertyType) {
        case 'String':
            // Everything can be coerced to string
            return true

        case 'Numeric':
            // Accepts: numbers, numeric strings
            // Rejects: Infinity, -Infinity, NaN, booleans (these become null in ClickHouse)
            if (typeof value === 'number') {
                return Number.isFinite(value)
            }
            if (typeof value === 'string') {
                const trimmed = value.trim()
                const num = Number(trimmed)
                return trimmed !== '' && Number.isFinite(num)
            }
            return false

        case 'Boolean':
            // Accepts: booleans, "true"/"false" strings (case sensitive - ClickHouse transform only matches lowercase)
            if (typeof value === 'boolean') {
                return true
            }
            if (typeof value === 'string') {
                return value === 'true' || value === 'false'
            }
            return false

        case 'DateTime':
            return isValidClickHouseDateTime(value)

        case 'Array':
            return Array.isArray(value)

        case 'Object':
            return typeof value === 'object'

        default:
            // Unknown type - allow by default
            return true
    }
}

export interface SchemaValidationError {
    propertyName: string
    reason: 'missing_required' | 'type_mismatch'
    expectedTypes?: string[]
    actualValue?: unknown
}

export interface SchemaValidationResult {
    valid: boolean
    errors: SchemaValidationError[]
}

/**
 * Validates an event's properties against an enforced schema.
 * Only required properties are validated - optional properties are not included in the schema.
 */
export function validateEventAgainstSchema(
    eventProperties: Record<string, unknown> | undefined,
    schema: EventSchemaEnforcement
): SchemaValidationResult {
    const errors: SchemaValidationError[] = []

    for (const [propertyName, propertyTypes] of schema.required_properties) {
        const value = eventProperties?.[propertyName]

        if (value === null || value === undefined) {
            errors.push({
                propertyName,
                reason: 'missing_required',
            })
            continue
        }

        if (!propertyTypes.some((type) => canCoerceToType(value, type))) {
            errors.push({
                propertyName,
                reason: 'type_mismatch',
                expectedTypes: propertyTypes,
                actualValue: value,
            })
        }
    }

    return {
        valid: errors.length === 0,
        errors,
    }
}

/**
 * Creates a processing step that validates events against enforced schemas.
 * Events that fail validation are dropped and an ingestion warning is emitted.
 * When `enabled` is false the step passes every event through untouched, so
 * pipelines can pipe it unconditionally.
 *
 * @param schemaManager - Manager for fetching enforced schemas (uses caching internally)
 * @param enabled - Schema enforcement opt-in; when false the step is a no-op
 */
export function createValidateEventSchemaStep<T extends { event: PipelineEvent; team: Team }>(
    schemaManager: EventSchemaEnforcementManager,
    enabled: boolean
): ProcessingStep<T, T> {
    return async function validateEventSchemaStep(input) {
        if (!enabled) {
            return ok(input)
        }

        const { event, team } = input

        const enforcedSchemas = await schemaManager.getSchemas(team.id)
        if (enforcedSchemas.size === 0) {
            return ok(input)
        }

        const schema = enforcedSchemas.get(event.event)
        if (!schema) {
            return ok(input)
        }

        const validationResult = validateEventAgainstSchema(event.properties, schema)

        if (!validationResult.valid) {
            return drop(
                'schema_validation_failed',
                [],
                [
                    {
                        type: 'schema_validation_failed',
                        details: {
                            eventUuid: event.uuid,
                            eventName: event.event,
                            distinctId: event.distinct_id,
                            errors: validationResult.errors.map((err) => ({
                                property: err.propertyName,
                                reason: err.reason,
                                expectedTypes: err.expectedTypes,
                                actualValue:
                                    err.actualValue !== undefined
                                        ? typeof err.actualValue === 'object'
                                            ? JSON.stringify(err.actualValue)
                                            : String(err.actualValue)
                                        : undefined,
                            })),
                        },
                    },
                ]
            )
        }

        return ok(input)
    }
}