:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import math
from collections.abc import Mapping
from typing import Literal, Self
from pydantic import ValidationError
from posthog.schema import (
ExperimentQueryResponse,
ExperimentStatsBaseValidated,
ExperimentVariantResultBayesian,
ExperimentVariantResultFrequentist,
)
from posthog.dataclasses import frozen
StatsMethod = Literal["bayesian", "frequentist"]
VariantResult = ExperimentVariantResultBayesian | ExperimentVariantResultFrequentist
_INTERVAL_KEYS = ("credible_interval", "confidence_interval")
# A funnel's stored stats carry sample sessions per step, which can be large, and no reader of a stored row needs them.
_UNREAD_FIELDS = frozenset({"step_sessions"})
_BASELINE_FIELDS = frozenset(ExperimentStatsBaseValidated.model_fields) - _UNREAD_FIELDS
_VARIANT_FIELDS_BY_METHOD = {
"bayesian": frozenset(ExperimentVariantResultBayesian.model_fields) - _UNREAD_FIELDS,
"frequentist": frozenset(ExperimentVariantResultFrequentist.model_fields) - _UNREAD_FIELDS,
}
def _declared_fields_only(value: object, fields: frozenset[str]) -> object:
if not isinstance(value, Mapping):
return value
return {key: item for key, item in value.items() if key in fields}
def _declared_variant_fields_only(variant: object) -> object:
if not isinstance(variant, Mapping):
return variant
method = variant.get("method")
fields = _VARIANT_FIELDS_BY_METHOD.get(method) if isinstance(method, str) else None
# A variant without a known method has no model to validate against, so it fails validation as it is.
return variant if fields is None else _declared_fields_only(variant, fields)
def _mean(total: float, number_of_samples: int) -> float | None:
return total / number_of_samples if number_of_samples else None
def _is_finite_interval(value: object) -> bool:
return (
isinstance(value, list | tuple)
and len(value) == 2
and all(isinstance(bound, int | float) and math.isfinite(bound) for bound in value)
)
def _without_non_finite_intervals(variant: object) -> object:
if not isinstance(variant, Mapping):
return variant
return {
key: None if key in _INTERVAL_KEYS and not _is_finite_interval(value) else value
for key, value in variant.items()
}
def parse_stored_response(raw: object) -> ExperimentQueryResponse | None:
"""The baseline and variant results of a stored result row, or None when the row does not parse."""
if not isinstance(raw, Mapping):
return None
# A stored row is a full response dump in the schema of its write time. It can carry keys that the
# current schema forbids: the cache fields of a cached response at the top level, and fields that a
# later schema renamed or removed inside the baseline and the variants. The models reject unknown
# keys, so keep only the keys that they declare. Schema drift then does not discard the row.
baseline = _declared_fields_only(raw.get("baseline"), _BASELINE_FIELDS)
variant_results = raw.get("variant_results")
if isinstance(variant_results, list):
# sanitize_non_finite stores an overflowed interval bound as null, which the schema rejects.
# Drop such an interval so that the counts and means of the day still render.
variant_results = [
_without_non_finite_intervals(_declared_variant_fields_only(variant)) for variant in variant_results
]
try:
return ExperimentQueryResponse.model_validate({"baseline": baseline, "variant_results": variant_results})
except ValidationError:
return None
@frozen
class Interval:
low: float
high: float
@property
def midpoint(self) -> float:
# The results page shows this as the variant's difference (`getDelta` in MetricsView/shared/utils.ts).
return (self.low + self.high) / 2
@frozen
class BaselineStats:
key: str
number_of_samples: int
mean: float | None
@classmethod
def from_schema(cls, baseline: ExperimentStatsBaseValidated) -> Self:
return cls(
key=baseline.key,
number_of_samples=baseline.number_of_samples,
mean=_mean(baseline.sum, baseline.number_of_samples),
)
@frozen
class VariantStats:
key: str
number_of_samples: int
mean: float | None
interval: Interval | None
chance_to_win: float | None
p_value: float | None
significant: bool | None
@classmethod
def from_schema(cls, variant: VariantResult) -> Self:
chance_to_win: float | None = None
p_value: float | None = None
if isinstance(variant, ExperimentVariantResultBayesian):
bounds = variant.credible_interval
chance_to_win = variant.chance_to_win
else:
bounds = variant.confidence_interval
p_value = variant.p_value
return cls(
key=variant.key,
number_of_samples=variant.number_of_samples,
mean=_mean(variant.sum, variant.number_of_samples),
interval=None if bounds is None else Interval(low=bounds[0], high=bounds[1]),
chance_to_win=chance_to_win,
p_value=p_value,
significant=variant.significant,
)
@frozen
class MetricResultStats:
method: StatsMethod | None
baseline: BaselineStats
variants: tuple[VariantStats, ...]
@classmethod
def from_response(cls, response: ExperimentQueryResponse) -> Self | None:
if response.baseline is None:
return None
variant_results = response.variant_results or []
return cls(
method=variant_results[0].method if variant_results else None,
baseline=BaselineStats.from_schema(response.baseline),
variants=tuple(VariantStats.from_schema(variant) for variant in variant_results),
)
@classmethod
def from_stored(cls, raw: object) -> Self | None:
"""Return None instead of raising when the row does not parse or has no baseline."""
response = parse_stored_response(raw)
return cls.from_response(response) if response is not None else None
def variant(self, key: str) -> VariantStats | None:
return next((variant for variant in self.variants if variant.key == key), None)
@frozen
class TimeseriesDay:
date: str
result: MetricResultStats | None
@frozen
class MetricTimeseries:
status: str
days: tuple[TimeseriesDay, ...]
@classmethod
def from_stored(cls, *, status: str, timeseries: Mapping[str, object]) -> Self:
return cls(
status=status,
days=tuple(
TimeseriesDay(date=date, result=MetricResultStats.from_stored(timeseries[date]))
for date in sorted(timeseries)
),
)
@property
def first_result(self) -> MetricResultStats | None:
return next((day.result for day in self.days if day.result is not None), None)