:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { EventSchemaEnforcementManager } from '~/common/utils/event-schema-enforcement-manager'
import { drop, ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { EventSchemaEnforcement, PipelineEvent, Team } from '~/types'
import { isValidClickHouseDateTime } from './clickhouse-datetime-parser'
/**
* Checks if a value can be coerced to the given PostHog property type.
* See: PropertySwapper._field_type_to_property_call in posthog/hogql/transforms/property_types.py
*/
function canCoerceToType(value: unknown, propertyType: string): boolean {
if (value === null || value === undefined) {
return false
}
switch (propertyType) {
case 'String':
// Everything can be coerced to string
return true
case 'Numeric':
// Accepts: numbers, numeric strings
// Rejects: Infinity, -Infinity, NaN, booleans (these become null in ClickHouse)
if (typeof value === 'number') {
return Number.isFinite(value)
}
if (typeof value === 'string') {
const trimmed = value.trim()
const num = Number(trimmed)
return trimmed !== '' && Number.isFinite(num)
}
return false
case 'Boolean':
// Accepts: booleans, "true"/"false" strings (case sensitive - ClickHouse transform only matches lowercase)
if (typeof value === 'boolean') {
return true
}
if (typeof value === 'string') {
return value === 'true' || value === 'false'
}
return false
case 'DateTime':
return isValidClickHouseDateTime(value)
case 'Array':
return Array.isArray(value)
case 'Object':
return typeof value === 'object'
default:
// Unknown type - allow by default
return true
}
}
export interface SchemaValidationError {
propertyName: string
reason: 'missing_required' | 'type_mismatch'
expectedTypes?: string[]
actualValue?: unknown
}
export interface SchemaValidationResult {
valid: boolean
errors: SchemaValidationError[]
}
/**
* Validates an event's properties against an enforced schema.
* Only required properties are validated - optional properties are not included in the schema.
*/
export function validateEventAgainstSchema(
eventProperties: Record<string, unknown> | undefined,
schema: EventSchemaEnforcement
): SchemaValidationResult {
const errors: SchemaValidationError[] = []
for (const [propertyName, propertyTypes] of schema.required_properties) {
const value = eventProperties?.[propertyName]
if (value === null || value === undefined) {
errors.push({
propertyName,
reason: 'missing_required',
})
continue
}
if (!propertyTypes.some((type) => canCoerceToType(value, type))) {
errors.push({
propertyName,
reason: 'type_mismatch',
expectedTypes: propertyTypes,
actualValue: value,
})
}
}
return {
valid: errors.length === 0,
errors,
}
}
/**
* Creates a processing step that validates events against enforced schemas.
* Events that fail validation are dropped and an ingestion warning is emitted.
* When `enabled` is false the step passes every event through untouched, so
* pipelines can pipe it unconditionally.
*
* @param schemaManager - Manager for fetching enforced schemas (uses caching internally)
* @param enabled - Schema enforcement opt-in; when false the step is a no-op
*/
export function createValidateEventSchemaStep<T extends { event: PipelineEvent; team: Team }>(
schemaManager: EventSchemaEnforcementManager,
enabled: boolean
): ProcessingStep<T, T> {
return async function validateEventSchemaStep(input) {
if (!enabled) {
return ok(input)
}
const { event, team } = input
const enforcedSchemas = await schemaManager.getSchemas(team.id)
if (enforcedSchemas.size === 0) {
return ok(input)
}
const schema = enforcedSchemas.get(event.event)
if (!schema) {
return ok(input)
}
const validationResult = validateEventAgainstSchema(event.properties, schema)
if (!validationResult.valid) {
return drop(
'schema_validation_failed',
[],
[
{
type: 'schema_validation_failed',
details: {
eventUuid: event.uuid,
eventName: event.event,
distinctId: event.distinct_id,
errors: validationResult.errors.map((err) => ({
property: err.propertyName,
reason: err.reason,
expectedTypes: err.expectedTypes,
actualValue:
err.actualValue !== undefined
? typeof err.actualValue === 'object'
? JSON.stringify(err.actualValue)
: String(err.actualValue)
: undefined,
})),
},
},
]
)
}
return ok(input)
}
}