Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): explain metrics that read zero in every variant (#126797)

Marcel Poelker8057b2ab338a
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 7cdf94d9bc3e..85078624d37a 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -11427,9 +11427,9 @@ snapshots:
scenes-app-experiments--experiment-frequentist-two-variants--light:
hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmII
scenes-app-experiments--experiment-health-debug-tab--dark:
- hash: v1.k794b7964.32832ef3ce0f1022d129240ea341f976f33cf255235c6087276d52f7c7b9f6cf.vED1QUQ3zvRyI8X34z6AuD0HovPU9T-JBoCSZkIbiY0
+ hash: v1.k794b7964.bd266bf7b03a0d96fb3447e669fb154d381d3920a8b7043f6b996f46789edf45.GZi7irxblhuJDTcQ4ILtuf9V9inrGgHKC4ID1shnfF4
scenes-app-experiments--experiment-health-debug-tab--light:
- hash: v1.k794b7964.87903c4dee6ea41ecd16e0aae00f3a9d4784b394fa221e3b3516992ddacd6b3d.ww5bg6uopL7e_XLA9pdodLWT5aiZFCBsQRbuQ_0d2NI
+ hash: v1.k794b7964.22d2b1422db4cfd1e2b802d020a14727f577c80a0a881eea96c67d9fce3ddbb1.vEPNwvdLLCrf9rSaG29yCHMqAN5UbMBpWLea3BzQqOk
scenes-app-experiments--experiment-recordings-dropped-metric--dark:
hash: v1.k794b7964.9ce950830524de179a4d761af104781fdc05347d59f18de814e72bc40a486158.T1SiGxL_fGBQT61rjwuJe9MPz1NPcjeZBRv6rKiSzrE
scenes-app-experiments--experiment-recordings-dropped-metric--light:
diff --git a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
index 7c17df644c30..a4a16a27b328 100644
--- a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
+++ b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
@@ -88,6 +88,8 @@ const HEALTH_INPUTS: ExperimentHealthInputsApi = {
latest_calculation: {
finished_at: '2025-01-27T09:30:00Z',
failures: [],
+ hours_of_data: 500,
+ zero_metrics: [],
},
flag_history: {
checked: true,
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 457c87957086..e6eb67a9e96e 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -10063,6 +10063,7 @@ export namespace Schemas {
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `metric_cannot_compile` - Metric Cannot Compile
* * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort
* * `exposure_filter_excludes_events` - Exposure Filter Excludes Events
@@ -10089,6 +10090,7 @@ export namespace Schemas {
| "metric_event_not_ingested"
| "metric_cannot_compile"
| "metric_calculation_failed"
+ | "metric_all_zero"
| "exposure_criteria_cannot_match"
| "exposure_filter_on_dynamic_cohort"
| "exposure_filter_excludes_events"
@@ -10145,6 +10147,7 @@ export namespace Schemas {
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `metric_cannot_compile` - Metric Cannot Compile
* * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort
* * `exposure_filter_excludes_events` - Exposure Filter Excludes Events
@@ -10211,6 +10214,7 @@ export namespace Schemas {
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `metric_cannot_compile` - Metric Cannot Compile
* * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filters` - Exposure Filters
* * `early_significance` - Early Significance
@@ -10234,6 +10238,7 @@ export namespace Schemas {
| "metric_event_not_ingested"
| "metric_cannot_compile"
| "metric_calculation_failed"
+ | "metric_all_zero"
| "exposure_criteria_cannot_match"
| "exposure_filters"
| "early_significance"
@@ -10270,6 +10275,7 @@ export namespace Schemas {
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `metric_cannot_compile` - Metric Cannot Compile
* * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filters` - Exposure Filters
* * `early_significance` - Early Significance
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index 0462e3eaf827..bc19fa497ee9 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -202,6 +202,7 @@ class ExperimentHealthFindingCode(LabeledStrEnum):
METRIC_EVENT_NOT_INGESTED = "metric_event_not_ingested"
METRIC_CANNOT_COMPILE = "metric_cannot_compile"
METRIC_CALCULATION_FAILED = "metric_calculation_failed"
+ METRIC_ALL_ZERO = "metric_all_zero"
EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"
EXPOSURE_FILTER_ON_DYNAMIC_COHORT = "exposure_filter_on_dynamic_cohort"
EXPOSURE_FILTER_EXCLUDES_EVENTS = "exposure_filter_excludes_events"
@@ -312,6 +313,7 @@ class ExperimentHealthCheckKey(LabeledStrEnum):
METRIC_EVENT_NOT_INGESTED = "metric_event_not_ingested"
METRIC_CANNOT_COMPILE = "metric_cannot_compile"
METRIC_CALCULATION_FAILED = "metric_calculation_failed"
+ METRIC_ALL_ZERO = "metric_all_zero"
EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"
EXPOSURE_FILTERS = "exposure_filters"
EARLY_SIGNIFICANCE = "early_significance"
diff --git a/products/experiments/backend/health/checks/metric_all_zero.py b/products/experiments/backend/health/checks/metric_all_zero.py
new file mode 100644
index 000000000000..dfa63dca537a
--- /dev/null
+++ b/products/experiments/backend/health/checks/metric_all_zero.py
@@ -0,0 +1,271 @@
+"""
+Metrics that read zero in every variant, so the results page shows "Not enough data yet" at any sample size. Pure
+functions, no I/O.
+
+The stored result holds only the zero, so the check names a cause only where the metric's settings or the project's
+event definitions show one, and needs no scan of the events. In every other case the words say what the zero means and
+name the usual cause: the exposure and the event come from different SDKs under different ids, so the people who send
+the event aren't the people who were exposed.
+"""
+
+from collections.abc import Mapping
+from typing import assert_never
+
+from posthog.schema import FunnelConversionWindowTimeUnit
+
+from posthog.dataclasses import frozen
+
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFinding,
+ ExperimentHealthFindingCode,
+ ExperimentHealthFindingSeverity,
+)
+from products.experiments.backend.health.checks.events_received import joined, quoted
+from products.experiments.backend.health.context import HealthContext, ZeroMetric
+from products.experiments.backend.health.skips import (
+ NOT_ON_ARCHIVED,
+ RUNS_AFTER_LAUNCH,
+ CheckOutcome,
+ no_calculated_metric,
+ not_applicable,
+ not_checked,
+)
+from products.experiments.backend.health.split_changes import format_change_time
+from products.experiments.backend.hogql_queries.base_query_utils import (
+ conversion_window_to_seconds,
+ quantile_exact_position,
+)
+
+# pinned: the subcodes and `evidence.metric_N_cause`. The page sends the subcode as `finding_subcode` on its events.
+BLANK_EVENT = "blank_event"
+EVENT_STOPPED = "event_stopped"
+OUTLIER_BOUND = "outlier_bound"
+
+# With less of either, most zeros are outcomes that take time, so neither gate works alone.
+MIN_EXPOSURES = 1000
+MIN_HOURS_OF_DATA = 72
+# An empty event name or a bound of 0 reads 0 at any traffic, so these causes need only enough data to read.
+MIN_EXPOSURES_WITH_A_SETTING_CAUSE = 100
+MIN_HOURS_WITH_A_SETTING_CAUSE = 12
+
+# A change to these settings recalculates the metric over the events up to the end, so they stay a warning after it.
+_FIXED_BY_A_SETTING = frozenset({BLANK_EVENT, OUTLIER_BOUND})
+_BOUND_LABEL = '"Ignore zeros when calculating upper bound"'
+_SECONDS_PER_HOUR = 3600
+
+
+@frozen
+class _Worded:
+ metric: ZeroMetric
+ # None when the settings and the event definitions show no cause.
+ cause: str | None
+ # Without the usual causes, which the finding says once for all the metrics that have no cause.
+ detail: str
+
+
+def metric_all_zero(ctx: HealthContext) -> CheckOutcome:
+ if ctx.archived:
+ return not_applicable(NOT_ON_ARCHIVED)
+ if not ctx.is_launched:
+ return not_applicable(RUNS_AFTER_LAUNCH)
+ calculation = ctx.latest_calculation
+ if calculation is None:
+ return not_checked("Couldn't check because the latest calculation couldn't be read")
+ if calculation.metric_count == 0:
+ return no_calculated_metric(ctx)
+ hours_of_data = calculation.hours_of_data
+ if calculation.finished_at is None or hours_of_data is None:
+ return not_applicable("Runs once a calculation of the metrics has finished")
+ worded = [
+ item
+ for metric in calculation.zero_metrics
+ if not _explained_elsewhere(metric, ctx) and (item := _word(metric, ctx, hours_of_data)) is not None
+ ]
+ if not worded:
+ return None
+ worded.sort(key=lambda item: (item.metric.role != "primary", item.cause is None))
+ # Both sides of a ratio can read zero, and each is its own item.
+ count = len({item.metric.metric_id for item in worded})
+ return ExperimentHealthFinding(
+ code=ExperimentHealthFindingCode.METRIC_ALL_ZERO,
+ subcode=worded[0].cause,
+ severity=_severity(worded, ctx),
+ title=f"{count} metrics read zero in every variant" if count > 1 else "A metric reads zero in every variant",
+ detail=_detail(worded),
+ evidence=_evidence(worded),
+ # No action opens a given metric.
+ actions=(),
+ diagnostic_ref=None,
+ )
+
+
+def _explained_elsewhere(metric: ZeroMetric, ctx: HealthContext) -> bool:
+ # `metric_event_not_ingested` reports an event the project never received, until the experiment ends.
+ # `metric_calculation_failed` and `zero_exposures` need no rule here: a failed metric has no zero to read, and a
+ # result with exposures isn't zero exposures.
+ return not ctx.has_ended and any(ctx.events_received.get(event) is False for event in metric.events)
+
+
+def _hours_that_count(metric: ZeroMetric, hours_of_data: float) -> float:
+ """The hours of data in which the metric can count an outcome. A retention metric counts a return only from the
+ start of its retention window after the start event, so its hours begin there."""
+ definition = metric.definition
+ start = definition.get("retention_window_start")
+ unit = definition.get("retention_window_unit")
+ if definition.get("metric_type") != "retention" or start is None or unit is None:
+ return hours_of_data
+ return hours_of_data - conversion_window_to_seconds(start, FunnelConversionWindowTimeUnit(unit)) / _SECONDS_PER_HOUR
+
+
+def _upper_bound(metric: ZeroMetric) -> float | None:
+ """The percentile of the upper outlier bound over the side that reads zero, when the bound counts zeros."""
+ definition = metric.definition
+ match metric.ratio_side:
+ case None:
+ handling = definition if definition.get("metric_type") == "mean" else None
+ case "numerator":
+ handling = definition.get("numerator_outlier_handling")
+ case "denominator":
+ handling = definition.get("denominator_outlier_handling")
+ case _:
+ assert_never(metric.ratio_side)
+ if not handling or handling.get("ignore_zeros"):
+ return None
+ return handling.get("upper_bound_percentile")
+
+
+def _values_a_zero_bound_hides(metric: ZeroMetric) -> int:
+ """How many exposures can have a value while the upper outlier bound is still 0. Zero without a bound that counts
+ zeros, and when the bound picks the largest value."""
+ percentile = _upper_bound(metric)
+ # The queries compute one bound per breakdown group, and the stored totals don't hold the size of each group.
+ if percentile is None or _has_breakdowns(metric):
+ return 0
+ return metric.exposures - quantile_exact_position(percentile, metric.exposures) - 1
+
+
+def _has_breakdowns(metric: ZeroMetric) -> bool:
+ breakdown_filter = metric.definition.get("breakdownFilter")
+ return isinstance(breakdown_filter, Mapping) and bool(breakdown_filter.get("breakdowns"))
+
+
+def _word(metric: ZeroMetric, ctx: HealthContext, hours_of_data: float) -> _Worded | None:
+ """The cause and the words for a metric that passes the gate of its cause, or None."""
+ hours = _hours_that_count(metric, hours_of_data)
+ reads_zero = _reads_zero(metric)
+ readable = metric.exposures >= MIN_EXPOSURES_WITH_A_SETTING_CAUSE and hours >= MIN_HOURS_WITH_A_SETTING_CAUSE
+ passes_gates = metric.role == "primary" and metric.exposures >= MIN_EXPOSURES and hours >= MIN_HOURS_OF_DATA
+ if metric.has_blank_event:
+ if not readable:
+ return None
+ detail = f"{reads_zero} It has an event with an empty name, which matches no events. Edit the metric and choose an event."
+ return _Worded(metric=metric, cause=BLANK_EVENT, detail=detail)
+ if metric.stopped_events:
+ # An event that stopped before launch sends no value for a bound to clamp, so the bound is no cause here.
+ if not passes_gates:
+ return None
+ return _Worded(metric=metric, cause=EVENT_STOPPED, detail=f"{reads_zero} {_stopped(metric, ctx)}")
+ hidden = _values_a_zero_bound_hides(metric)
+ if hidden > 0 and readable:
+ return _Worded(metric=metric, cause=OUTLIER_BOUND, detail=f"{reads_zero} {_bound(hidden, metric.exposures)}")
+ if passes_gates:
+ return _Worded(metric=metric, cause=None, detail=reads_zero)
+ return None
+
+
+def _reads_zero(metric: ZeroMetric) -> str:
+ subject = f'{metric.role} metric "{metric.name}"'
+ subject = f"The {subject}" if metric.ratio_side is None else f"The {metric.ratio_side} of the {subject}"
+ # A retention result counts only the exposed users who did its start event.
+ counted = "users who did its start event" if metric.definition.get("metric_type") == "retention" else "exposures"
+ earlier = (
+ f" {metric.reached_earlier_step:,} reached an earlier step, but nobody finished the funnel."
+ if metric.reached_earlier_step
+ else ""
+ )
+ return f"{subject} reads 0 for all {metric.exposures:,} {counted}, in every variant.{earlier}"
+
+
+def _stopped(metric: ZeroMetric, ctx: HealthContext) -> str:
+ stopped = [
+ (event, format_change_time(last_seen, ctx.team_timezone)) for event, last_seen in metric.stopped_events.items()
+ ]
+ if len(stopped) == 1:
+ event, last_seen = stopped[0]
+ return (
+ f'The event "{event}" last arrived on {last_seen}, before the experiment started. Check that your code '
+ "still sends it, or edit the metric to use an event that still arrives."
+ )
+ named = joined([f'"{event}" on {last_seen}' for event, last_seen in stopped])
+ return (
+ f"These events last arrived before the experiment started: {named}. Check that your code still sends them, or "
+ "edit the metric to use events that still arrive."
+ )
+
+
+def _bound(hidden: int, exposures: int) -> str:
+ return (
+ f"Its upper outlier bound is taken over all {exposures:,} exposures, including those with no value. When at "
+ f"most {hidden:,} of them have a value, the bound is 0, so every value becomes 0. Turn on {_BOUND_LABEL} on "
+ "the metric."
+ )
+
+
+def _detail(worded: list[_Worded]) -> str:
+ sentences = [item.detail for item in worded]
+ without_a_cause = [item.metric for item in worded if item.cause is None]
+ if without_a_cause:
+ sentences.extend(_usual_causes(without_a_cause))
+ return " ".join(dict.fromkeys(sentences))
+
+
+def _usual_causes(metrics: list[ZeroMetric]) -> list[str]:
+ events = tuple(dict.fromkeys(event for metric in metrics for event in metric.events))
+ tables = tuple(dict.fromkeys(table for metric in metrics for table in metric.warehouse_tables))
+ # A metric on actions names no event, but it still reads events.
+ reads_events = bool(events) or any(not metric.warehouse_tables for metric in metrics)
+ counted = joined([*(["an event"] if reads_events else []), *(["a warehouse row"] if tables else [])], "or")
+ sentences = [
+ f"PostHog counts {counted} only for the person or group that was exposed, and only after their first exposure."
+ ]
+ if reads_events:
+ event = f"the event {quoted(events)}" if len(events) == 1 else "the metric events"
+ sentences.append(
+ f"Check that your code sends {event} with the same ID as the exposure, which a server-side event or another "
+ "SDK often doesn't."
+ )
+ if tables:
+ sentences.append(
+ f"Check that the data warehouse join key of {quoted(tables)} holds the same ID as the exposure."
+ )
+ if any(metric.has_filters for metric in metrics):
+ sentences.append("Check that the metric filters match the data you send.")
+ thresholds = [
+ f'"{metric.name}" ({threshold:g})'
+ for metric in metrics
+ if (threshold := metric.definition.get("threshold")) is not None
+ and metric.definition.get("metric_type") == "mean"
+ ]
+ if thresholds:
+ sentences.append(f"Check that the exposed people or groups can reach the threshold of {joined(thresholds)}.")
+ sentences.append("A rare outcome can also read 0 at this sample size.")
+ return sentences
+
+
+def _severity(worded: list[_Worded], ctx: HealthContext) -> ExperimentHealthFindingSeverity:
+ primary = [item for item in worded if item.metric.role == "primary"]
+ # After the end, no change to the code can add events. A setting can still bring back the results.
+ if primary and (not ctx.has_ended or any(item.cause in _FIXED_BY_A_SETTING for item in primary)):
+ return ExperimentHealthFindingSeverity.WARNING
+ return ExperimentHealthFindingSeverity.INFO
+
+
+def _evidence(worded: list[_Worded]) -> dict[str, str | int | None]:
+ evidence: dict[str, str | int | None] = {}
+ for number, item in enumerate(worded, start=1):
+ evidence[f"metric_{number}"] = item.metric.name
+ evidence[f"metric_{number}_role"] = item.metric.role
+ evidence[f"metric_{number}_side"] = item.metric.ratio_side
+ evidence[f"metric_{number}_exposures"] = item.metric.exposures
+ evidence[f"metric_{number}_cause"] = item.cause
+ return evidence
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index f0ffc30500f9..75da26b2e8ec 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -1,4 +1,4 @@
-from collections.abc import Mapping
+from collections.abc import Iterable, Mapping
from datetime import date, datetime, timedelta
from enum import StrEnum
from functools import reduce
@@ -29,7 +29,7 @@ from posthog.models.activity_logging.activity_log import ActivityLog
from posthog.models.team.team import Team
from products.cohorts.backend.models.cohort import Cohort
-from products.event_definitions.backend.models import EventDefinition, PropertyDefinition, effective_project_id_expr
+from products.event_definitions.backend.models import PropertyDefinition, effective_project_id_expr
from products.experiments.backend.facade.contracts import (
ExperimentHealthFlagChangeKind,
ExperimentHealthMetricCompileErrorKind,
@@ -47,7 +47,7 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (
from products.experiments.backend.logic.result_stats import MetricResultStats
from products.experiments.backend.metric_resolution import MetricRole, saved_metric_link_role, saved_metric_links
from products.experiments.backend.metric_utils import get_default_metric_title
-from products.experiments.backend.metric_validation import defined_event_names, extract_entity_nodes
+from products.experiments.backend.metric_validation import defined_event_names, event_last_seen, extract_entity_nodes
from products.experiments.backend.models.experiment import Experiment, ExperimentToSavedMetric
from products.experiments.backend.running_time_target import RunningTimeTarget
from products.feature_flags.backend.facade.edits import (
@@ -361,6 +361,37 @@ class MetricCalculationFailure:
exposures_path: ExposuresPath | None
+RatioSide = Literal["numerator", "denominator"]
+
+
+@frozen
+class ZeroMetric:
+ """A metric, or one side of a ratio metric, whose latest calculation reads zero in every variant, which the results
+ page shows as "Not enough data yet" at any sample size."""
+
+ metric_id: str
+ name: str
+ role: MetricRole
+ # The persons or groups of the result, across the variants.
+ exposures: int
+ # The side of a ratio that reads zero, each side its own `ZeroMetric`. None for the other metric types.
+ ratio_side: RatioSide | None
+ # The effective definition now, which holds the outlier handling and the retention window.
+ definition: Mapping[str, Any]
+ # The persons or groups that reached a step of a funnel before the last one. Zero for the other metric types.
+ reached_earlier_step: int
+ # The events that the zero reads by name, without the other side of a ratio.
+ events: tuple[str, ...]
+ # The data warehouse tables that the zero reads, without the other side of a ratio.
+ warehouse_tables: tuple[str, ...]
+ # Whether the zero reads an event node with a blank name, which matches no event. "All events" stores no name.
+ has_blank_event: bool
+ # Whether the zero reads a node with property filters.
+ has_filters: bool
+ # Of `events`, those that the project last received before launch, by the time of the last one.
+ stopped_events: Mapping[str, datetime]
+
+
@frozen
class LatestCalculation:
"""The latest finished calculation of the experiment's metrics, whose results and errors the results page
@@ -375,6 +406,11 @@ class LatestCalculation:
completed_count: int
# Where the queries of those results read the exposures, when there is a failure to compare them with.
completed_exposures_paths: frozenset[ExposuresPath]
+ # Of those results, the ones that read zero in every variant.
+ zero_metrics: tuple[ZeroMetric, ...] = ()
+ # The hours from launch to the end of the data of this calculation. None when no calculation has finished. The
+ # page can show a calculation from days ago until its refresh finishes.
+ hours_of_data: float | None = None
@frozen
@@ -920,25 +956,28 @@ def _received(
}
+def ingestion_proof_events(experiment_events: Iterable[str]) -> set[str]:
+ """The events whose `last_seen_at` can show that ingestion records the project: the flag calls and the events of
+ the experiment."""
+ return {DEFAULT_EXPOSURE_EVENT, EXPERIMENT_EXPOSURE_EVENT, *experiment_events}
+
+
+def ingestion_records_project(last_seen: Mapping[str, datetime]) -> bool:
+ """Whether ingestion writes the project's definitions now, from `last_seen` of `ingestion_proof_events`. A missing
+ or stale definition proves nothing without it. Ingestion can be set to skip a team, and then writes no definition at
+ all. It refreshes `last_seen_at` of an event that it sees about once an hour. So a flag call or an event of the
+ experiment seen within the margin shows that it records the project."""
+ recent = timezone.now() - _DEFINITION_RECREATION_MARGIN
+ return any(seen >= recent for seen in last_seen.values())
+
+
def _ingestion_records_project(
experiment: Experiment, metric_events: tuple[MetricEvents, ...], exposure_events: ExposureEvents | None
) -> bool:
- """Whether ingestion writes the project's definitions now, which a missing definition needs to prove anything.
- Ingestion can be set to skip a team, and then writes no definition at all. It refreshes `last_seen_at` of an event
- that it sees about once an hour. So a flag call or an event of the experiment seen within the margin shows that it
- records the project."""
- names = {DEFAULT_EXPOSURE_EVENT, EXPERIMENT_EXPOSURE_EVENT, *(event for m in metric_events for event in m.events)}
+ names = ingestion_proof_events(event for m in metric_events for event in m.events)
if exposure_events is not None and exposure_events.custom_event is not None:
names.add(exposure_events.custom_event)
- return (
- EventDefinition.objects.alias(effective_project_id=effective_project_id_expr())
- .filter(
- effective_project_id=experiment.team.project_id,
- name__in=names,
- last_seen_at__gte=timezone.now() - _DEFINITION_RECREATION_MARGIN,
- )
- .exists()
- )
+ return ingestion_records_project(event_last_seen(experiment.team.project_id, names))
def _deletions_since(experiment: Experiment) -> datetime:
diff --git a/products/experiments/backend/health/inputs_payload.py b/products/experiments/backend/health/inputs_payload.py
index 50ab6b106bff..1045cd444823 100644
--- a/products/experiments/backend/health/inputs_payload.py
+++ b/products/experiments/backend/health/inputs_payload.py
@@ -234,6 +234,17 @@ def _health_inputs_latest_calculation(calculation: LatestCalculation) -> dict[st
}
for failure in calculation.failures
],
+ "hours_of_data": calculation.hours_of_data,
+ "zero_metrics": [
+ {
+ "name": metric.name,
+ "role": metric.role,
+ "exposures": metric.exposures,
+ "ratio_side": metric.ratio_side,
+ "stopped_events": list(metric.stopped_events),
+ }
+ for metric in calculation.zero_metrics
+ ],
}
diff --git a/products/experiments/backend/health/latest_calculation.py b/products/experiments/backend/health/latest_calculation.py
index 1445722070d5..acd589f3bae0 100644
--- a/products/experiments/backend/health/latest_calculation.py
+++ b/products/experiments/backend/health/latest_calculation.py
@@ -1,11 +1,24 @@
import re
+from collections.abc import Mapping
+from datetime import datetime, timedelta
from typing import Any
+from posthog.schema import ExperimentStatsBaseValidated
+
+from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
from products.experiments.backend.facade.enums import ExposuresPath
-from products.experiments.backend.health.context import LatestCalculation, MetricCalculationFailure
+from products.experiments.backend.health.context import (
+ LatestCalculation,
+ MetricCalculationFailure,
+ RatioSide,
+ ZeroMetric,
+ ingestion_proof_events,
+ ingestion_records_project,
+)
from products.experiments.backend.health.metric_compiles import can_hold_evaluation_rules, without_available_fields
+from products.experiments.backend.logic.result_stats import VariantResult, parse_stored_response
from products.experiments.backend.metric_calculation.config import (
MetricCalculationConfig,
build_calculation_configs,
@@ -13,6 +26,7 @@ from products.experiments.backend.metric_calculation.config import (
team_experiments_configs,
)
from products.experiments.backend.metric_calculation.results import MetricResultStore
+from products.experiments.backend.metric_validation import event_last_seen, metric_nodes
from products.experiments.backend.models.experiment import Experiment, ExperimentMetricResult
from products.experiments.backend.recalculation import get_latest_recalculation
from products.experiments.backend.temporal.models import RECALCULATION_WINDOW_TOO_LONG_ERROR_TYPE
@@ -76,6 +90,110 @@ def _load_latest_calculation(experiment: Experiment) -> LatestCalculation:
failures=failures,
completed_count=len(completed),
completed_exposures_paths=results.exposures_paths(completed) if failures else frozenset(),
+ zero_metrics=_zero_metrics(
+ experiment, [(configs[row.metric_uuid], row) for row in results.with_zero_baseline(completed)]
+ ),
+ hours_of_data=(run.query_to - experiment.start_date) / timedelta(hours=1)
+ if run.query_to is not None and experiment.start_date is not None
+ else None,
+ )
+
+
+@frozen
+class _ZeroResult:
+ calculation_config: MetricCalculationConfig
+ # None when the metric itself reads zero, rather than a side of a ratio.
+ ratio_side: RatioSide | None
+ stats: tuple[ExperimentStatsBaseValidated | VariantResult, ...]
+ # The nodes that the zero reads, without the other side of a ratio.
+ nodes: tuple[dict, ...]
+
+ @property
+ def events(self) -> tuple[str, ...]:
+ names = (node.get("event") for node in self.nodes if node.get("kind") == "EventsNode")
+ return tuple(dict.fromkeys(name for name in names if isinstance(name, str) and name.strip()))
+
+ @property
+ def warehouse_tables(self) -> tuple[str, ...]:
+ names = (node.get("table_name") for node in self.nodes if node.get("kind") == "ExperimentDataWarehouseNode")
+ return tuple(dict.fromkeys(name for name in names if isinstance(name, str) and name))
+
+
+def _zero_metrics(
+ experiment: Experiment, completed: list[tuple[MetricCalculationConfig, ExperimentMetricResult]]
+) -> tuple[ZeroMetric, ...]:
+ zero = [result for config, row in completed for result in _zero_results(config, row)]
+ if not zero:
+ return ()
+ events = {event for result in zero for event in result.events}
+ last_seen = event_last_seen(experiment.team.project_id, ingestion_proof_events(events)) if events else {}
+ # While ingestion skips the project, `last_seen_at` stays old for an event that still arrives.
+ if not ingestion_records_project(last_seen):
+ last_seen = {}
+ return tuple(_zero_metric(experiment, result, last_seen) for result in zero)
+
+
+def _zero_results(calculation_config: MetricCalculationConfig, row: ExperimentMetricResult) -> tuple[_ZeroResult, ...]:
+ """The metric, or each side of a ratio, that reads zero in every variant of the row."""
+ response = parse_stored_response(row.result)
+ if response is None or response.baseline is None:
+ return ()
+ stats = (response.baseline, *(response.variant_results or []))
+ definition = calculation_config.definition
+ # `sum_squares` tells a metric of zeros from values that cancel out.
+ numerator_is_zero = all(stat.sum == 0 and stat.sum_squares == 0 for stat in stats)
+ if definition.get("metric_type") != "ratio":
+ if not numerator_is_zero:
+ return ()
+ nodes = tuple(metric_nodes(dict(definition)))
+ return (_ZeroResult(calculation_config=calculation_config, ratio_side=None, stats=stats, nodes=nodes),)
+ sides: list[RatioSide] = []
+ if numerator_is_zero:
+ sides.append("numerator")
+ if all(_denominator_is_zero(stat) for stat in stats):
+ sides.append("denominator")
+ return tuple(
+ _ZeroResult(
+ calculation_config=calculation_config,
+ ratio_side=side,
+ stats=stats,
+ nodes=(node,) if isinstance(node := definition.get(side), dict) else (),
+ )
+ for side in sides
+ )
+
+
+def _denominator_is_zero(stat: ExperimentStatsBaseValidated | VariantResult) -> bool:
+ # A variant without exposures gets zero stats that leave the denominator unset.
+ if stat.number_of_samples == 0:
+ return True
+ return stat.denominator_sum == 0 and not stat.denominator_sum_squares
+
+
+def _zero_metric(experiment: Experiment, result: _ZeroResult, last_seen: Mapping[str, datetime]) -> ZeroMetric:
+ calculation_config = result.calculation_config
+ start_date = experiment.start_date
+ return ZeroMetric(
+ metric_id=calculation_config.metric_id,
+ name=calculation_metric_name(experiment, calculation_config),
+ role=calculation_config.role,
+ exposures=sum(stat.number_of_samples for stat in result.stats),
+ ratio_side=result.ratio_side,
+ definition=calculation_config.definition,
+ # `step_counts` holds the entities that reached each step of the metric, so its first count is the largest.
+ reached_earlier_step=sum(stat.step_counts[0] for stat in result.stats if stat.step_counts),
+ events=result.events,
+ warehouse_tables=result.warehouse_tables,
+ has_blank_event=any(
+ node.get("kind") == "EventsNode" and node.get("event") is not None and not str(node["event"]).strip()
+ for node in result.nodes
+ ),
+ has_filters=any(node.get("properties") for node in result.nodes),
+ stopped_events={
+ event: seen
+ for event in result.events
+ if (seen := last_seen.get(event)) is not None and start_date is not None and seen < start_date
+ },
)
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index 6861344571f6..dd0fe8703103 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -22,6 +22,7 @@ from products.experiments.backend.health.checks.flag_change_findings import flag
from products.experiments.backend.health.checks.flag_state import flag_state
from products.experiments.backend.health.checks.flag_variants import flag_variants_mismatch
from products.experiments.backend.health.checks.forced_variant import forced_variant_release_condition
+from products.experiments.backend.health.checks.metric_all_zero import metric_all_zero
from products.experiments.backend.health.checks.metric_calculation_failed import metric_calculation_failed
from products.experiments.backend.health.checks.metric_cannot_compile import metric_cannot_compile
from products.experiments.backend.health.checks.metric_event_not_ingested import metric_event_not_ingested
@@ -104,6 +105,11 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
name="Failed metric calculations",
run=metric_calculation_failed,
),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.METRIC_ALL_ZERO,
+ name="Metrics at zero in every variant",
+ run=metric_all_zero,
+ ),
HealthCheck(
key=ExperimentHealthCheckKey.EXPOSURE_CRITERIA_CANNOT_MATCH,
name="Exposure criteria that can't match",
diff --git a/products/experiments/backend/health/test/test_metric_all_zero.py b/products/experiments/backend/health/test/test_metric_all_zero.py
new file mode 100644
index 000000000000..1b022a93603b
--- /dev/null
+++ b/products/experiments/backend/health/test/test_metric_all_zero.py
@@ -0,0 +1,561 @@
+from dataclasses import replace
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from posthog.test.base import BaseTest
+from unittest import TestCase
+
+from django.utils import timezone
+
+from parameterized import parameterized
+
+from posthog.models.scoping import team_scope
+
+from products.event_definitions.backend.models import EventDefinition
+from products.experiments.backend.facade.api import get_experiment_health
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFinding,
+ ExperimentHealthFindingCode,
+ ExperimentHealthFindingSeverity,
+ ExperimentHealthReaderAccess,
+)
+from products.experiments.backend.health.checks.metric_all_zero import metric_all_zero
+from products.experiments.backend.health.context import LatestCalculation, ZeroMetric
+from products.experiments.backend.health.skips import CheckSkip
+from products.experiments.backend.health.test.factories import ENDED, RUNNING_CONTEXT
+from products.experiments.backend.metric_calculation.config import get_metric_calculation_config
+from products.experiments.backend.metric_calculation.results import MetricResultStore
+from products.experiments.backend.models.experiment import Experiment, ExperimentMetricsRecalculation
+from products.feature_flags.backend.models.feature_flag import FeatureFlag
+
+CODE = ExperimentHealthFindingCode.METRIC_ALL_ZERO
+WARNING = ExperimentHealthFindingSeverity.WARNING
+INFO = ExperimentHealthFindingSeverity.INFO
+NO_QUERIES = ExperimentHealthReaderAccess(
+ can_read_flag_history=False, can_query_events=False, denied_event_properties=frozenset()
+)
+LAST_ARRIVED = datetime(2026, 10, 1, 9, 30, tzinfo=UTC)
+MEAN = {"metric_type": "mean"}
+BOUNDED = {"metric_type": "mean", "upper_bound_percentile": 0.99}
+
+
+def _metric(**changes: Any) -> ZeroMetric:
+ fields: dict[str, Any] = {
+ "metric_id": "metric",
+ "name": "Purchases",
+ "role": "primary",
+ "exposures": 20000,
+ "ratio_side": None,
+ "definition": MEAN,
+ "reached_earlier_step": 0,
+ "events": ("purchase",),
+ "warehouse_tables": (),
+ "has_blank_event": False,
+ "has_filters": False,
+ "stopped_events": {},
+ }
+ return ZeroMetric(**{**fields, **changes})
+
+
+def _outcome(
+ *metrics: ZeroMetric, hours_of_data: float = 96, **changes: Any
+) -> ExperimentHealthFinding | CheckSkip | None:
+ calculation = LatestCalculation(
+ metric_count=len(metrics) or 1,
+ finished_at=LAST_ARRIVED,
+ failures=(),
+ completed_count=len(metrics),
+ completed_exposures_paths=frozenset(),
+ zero_metrics=metrics,
+ hours_of_data=hours_of_data,
+ )
+ return metric_all_zero(replace(RUNNING_CONTEXT, latest_calculation=calculation, **changes))
+
+
+def _variant(
+ *,
+ samples: int,
+ total: float,
+ squares: float | None = None,
+ denominator: float | None = None,
+ denominator_squares: float | None = None,
+) -> dict:
+ return {
+ "key": "test",
+ "method": "bayesian",
+ "number_of_samples": samples,
+ "sum": total,
+ "sum_squares": total**2 if squares is None else squares,
+ "denominator_sum": denominator,
+ "denominator_sum_squares": denominator_squares,
+ }
+
+
+def _result(
+ *,
+ samples: int,
+ total: float,
+ squares: float | None = None,
+ denominator: float | None = None,
+ denominator_squares: float | None = None,
+) -> dict:
+ variant = _variant(
+ samples=samples,
+ total=total,
+ squares=squares,
+ denominator=denominator,
+ denominator_squares=denominator_squares,
+ )
+ baseline = {key: value for key, value in variant.items() if key != "method"}
+ return {"baseline": {**baseline, "key": "control"}, "variant_results": [variant]}
+
+
+class TestMetricAllZero(TestCase):
+ @parameterized.expand(
+ [
+ ("reads_zero_with_the_usual_causes", _metric(), {}, (None, WARNING, "same ID as the exposure")),
+ (
+ "the_filters_of_the_metric",
+ _metric(has_filters=True),
+ {},
+ (None, WARNING, "Check that the metric filters match"),
+ ),
+ (
+ "an_earlier_funnel_step",
+ _metric(reached_earlier_step=900),
+ {},
+ (None, WARNING, "900 reached an earlier step, but nobody finished the funnel."),
+ ),
+ ("a_side_of_a_ratio", _metric(ratio_side="numerator"), {}, (None, WARNING, "The numerator of the primary")),
+ (
+ "an_event_with_an_empty_name",
+ _metric(has_blank_event=True, events=()),
+ {},
+ ("blank_event", WARNING, "matches no events"),
+ ),
+ (
+ "an_event_that_stopped_before_launch",
+ _metric(stopped_events={"purchase": LAST_ARRIVED}),
+ {},
+ ("event_stopped", WARNING, 'The event "purchase" last arrived on October 1, 2026 at 11:30 AM CEST'),
+ ),
+ (
+ "an_outlier_bound_that_counts_zeros",
+ _metric(definition=BOUNDED),
+ {},
+ ("outlier_bound", WARNING, "When at most 199 of them have a value, the bound is 0"),
+ ),
+ (
+ "a_bound_that_can_hide_one_value",
+ _metric(role="secondary", definition=BOUNDED, exposures=101),
+ {},
+ ("outlier_bound", INFO, "When at most 1 of them have a value"),
+ ),
+ (
+ "a_threshold_nobody_reaches",
+ _metric(definition={"metric_type": "mean", "threshold": 50}),
+ {},
+ (None, WARNING, 'can reach the threshold of "Purchases" (50).'),
+ ),
+ (
+ "a_data_warehouse_table",
+ _metric(events=(), warehouse_tables=("charges",)),
+ {},
+ (None, WARNING, "counts a warehouse row only for the person or group that was exposed"),
+ ),
+ (
+ "after_the_end_nobody_can_add_events",
+ _metric(),
+ ENDED,
+ (None, INFO, "same ID as the exposure"),
+ ),
+ (
+ "after_the_end_a_stopped_event_stays_info",
+ _metric(stopped_events={"purchase": LAST_ARRIVED}),
+ ENDED,
+ ("event_stopped", INFO, "before the experiment started"),
+ ),
+ (
+ "after_the_end_a_setting_still_brings_back_the_results",
+ _metric(definition=BOUNDED),
+ ENDED,
+ ("outlier_bound", WARNING, "Turn on"),
+ ),
+ (
+ "secondary_metrics_report_an_outlier_bound",
+ _metric(role="secondary", definition=BOUNDED),
+ {},
+ ("outlier_bound", INFO, 'The secondary metric "Purchases"'),
+ ),
+ (
+ "secondary_metrics_report_an_empty_event_name",
+ _metric(role="secondary", has_blank_event=True, events=(), exposures=100),
+ {},
+ ("blank_event", INFO, "matches no events"),
+ ),
+ (
+ "an_empty_event_name_comes_before_a_bound",
+ _metric(role="secondary", definition=BOUNDED, has_blank_event=True),
+ {},
+ ("blank_event", INFO, "matches no events"),
+ ),
+ ]
+ )
+ def test_reports(
+ self, _name: str, metric: ZeroMetric, changes: dict[str, Any], expected: tuple[str | None, Any, str]
+ ) -> None:
+ outcome = _outcome(metric, **changes)
+ assert isinstance(outcome, ExperimentHealthFinding)
+ subcode, severity, words = expected
+ self.assertEqual((outcome.code, outcome.subcode, outcome.severity), (CODE, subcode, severity))
+ self.assertIn(words, outcome.detail)
+
+ @parameterized.expand(
+ [
+ ("under_the_exposures_floor", _metric(exposures=999), 96),
+ ("under_the_data_floor", _metric(), 71),
+ ("a_secondary_metric_without_a_bound", _metric(role="secondary"), 96),
+ ("an_empty_event_name_under_its_exposures_floor", _metric(has_blank_event=True, exposures=99), 96),
+ ("an_empty_event_name_in_the_first_hours", _metric(has_blank_event=True), 11),
+ ("a_bound_under_its_exposures_floor", _metric(definition=BOUNDED, exposures=99), 96),
+ ("a_bound_in_the_first_hours", _metric(definition=BOUNDED, exposures=800), 11),
+ (
+ "a_bound_that_ignores_zeros_under_the_floor",
+ _metric(definition={**BOUNDED, "ignore_zeros": True}, exposures=800),
+ 96,
+ ),
+ (
+ "a_bound_on_the_largest_value",
+ _metric(definition={**BOUNDED, "upper_bound_percentile": 1}, exposures=800),
+ 96,
+ ),
+ (
+ "a_bound_per_breakdown_group",
+ _metric(
+ role="secondary",
+ definition={
+ **BOUNDED,
+ "breakdownFilter": {"breakdowns": [{"property": "$browser", "type": "event"}]},
+ },
+ ),
+ 96,
+ ),
+ (
+ "a_bound_that_picks_the_largest_of_its_exposures",
+ _metric(role="secondary", definition=BOUNDED, exposures=100),
+ 96,
+ ),
+ (
+ "a_stopped_event_on_a_secondary_metric",
+ _metric(role="secondary", definition=BOUNDED, stopped_events={"purchase": LAST_ARRIVED}),
+ 96,
+ ),
+ (
+ "a_retention_window_that_starts_later",
+ _metric(
+ definition={
+ "metric_type": "retention",
+ "retention_window_start": 7,
+ "retention_window_end": 8,
+ "retention_window_unit": "day",
+ }
+ ),
+ 96,
+ ),
+ ]
+ )
+ def test_stays_quiet(self, _name: str, metric: ZeroMetric, hours_of_data: float) -> None:
+ self.assertIsNone(_outcome(metric, hours_of_data=hours_of_data))
+
+ def test_leaves_an_event_that_never_arrived_to_the_check_that_reports_it(self) -> None:
+ self.assertIsNone(_outcome(_metric(), events_received={"purchase": False}))
+
+ # `metric_event_not_ingested` stops at the end, so this check is the only one left to explain the zero.
+ outcome = _outcome(_metric(), events_received={"purchase": False}, **ENDED)
+
+ assert isinstance(outcome, ExperimentHealthFinding)
+ self.assertEqual(outcome.severity, ExperimentHealthFindingSeverity.INFO)
+
+ @parameterized.expand([("before_the_gate", 119, False), ("at_the_gate", 120, True)])
+ def test_a_retention_metric_counts_its_hours_from_the_window_start(
+ self, _name: str, hours_of_data: float, reports: bool
+ ) -> None:
+ definition = {
+ "metric_type": "retention",
+ "retention_window_start": 2,
+ "retention_window_end": 3,
+ "retention_window_unit": "day",
+ }
+
+ outcome = _outcome(_metric(definition=definition), hours_of_data=hours_of_data)
+
+ self.assertEqual(isinstance(outcome, ExperimentHealthFinding), reports)
+ if isinstance(outcome, ExperimentHealthFinding):
+ self.assertIn("reads 0 for all 20,000 users who did its start event", outcome.detail)
+
+ def test_the_outlier_bound_on_a_side_of_a_ratio(self) -> None:
+ definition = {"metric_type": "ratio", "denominator_outlier_handling": {"upper_bound_percentile": 0.95}}
+ metric = _metric(role="secondary", ratio_side="denominator", definition=definition, exposures=800)
+
+ outcome = _outcome(metric, hours_of_data=18)
+
+ assert isinstance(outcome, ExperimentHealthFinding)
+ self.assertEqual(outcome.subcode, "outlier_bound")
+ self.assertIn('The denominator of the secondary metric "Purchases" reads 0', outcome.detail)
+ self.assertIn("When at most 39 of them have a value", outcome.detail)
+
+ def test_the_outlier_bound_on_the_other_side_of_a_ratio_is_no_cause(self) -> None:
+ definition = {"metric_type": "ratio", "denominator_outlier_handling": {"upper_bound_percentile": 0.95}}
+
+ self.assertIsNone(_outcome(_metric(role="secondary", ratio_side="numerator", definition=definition)))
+
+ def test_lists_the_primary_metrics_with_a_cause_first(self) -> None:
+ outcome = _outcome(
+ _metric(metric_id="a", name="Secondary", role="secondary", definition=BOUNDED),
+ _metric(metric_id="b", name="Unexplained"),
+ _metric(metric_id="c", name="Bounded", definition=BOUNDED),
+ )
+
+ assert isinstance(outcome, ExperimentHealthFinding)
+ self.assertEqual(outcome.title, "3 metrics read zero in every variant")
+ self.assertEqual(outcome.subcode, "outlier_bound")
+ self.assertEqual(
+ [outcome.evidence[f"metric_{number}"] for number in (1, 2, 3)], ["Bounded", "Unexplained", "Secondary"]
+ )
+ self.assertEqual(outcome.evidence["metric_2_exposures"], 20000)
+ self.assertIsNone(outcome.evidence["metric_2_cause"])
+
+ def test_says_the_usual_causes_once(self) -> None:
+ outcome = _outcome(
+ _metric(metric_id="a", name="Purchases", events=("purchase",)),
+ _metric(metric_id="b", name="Refunds", events=("refund",), has_filters=True),
+ )
+
+ assert isinstance(outcome, ExperimentHealthFinding)
+ self.assertEqual(outcome.detail.count("same ID as the exposure"), 1)
+ self.assertIn("sends the metric events with", outcome.detail)
+ self.assertIn("Check that the metric filters match", outcome.detail)
+ self.assertLess(outcome.detail.index('"Refunds" reads 0'), outcome.detail.index("same ID as the exposure"))
+
+ @parameterized.expand(
+ [
+ ("archived", {"archived": True}, "Doesn't run on archived experiments"),
+ ("draft", {"is_launched": False}, "Runs after launch"),
+ ]
+ )
+ def test_skips(self, _name: str, changes: dict[str, Any], reason: str) -> None:
+ outcome = _outcome(_metric(), **changes)
+ assert isinstance(outcome, CheckSkip)
+ self.assertEqual(outcome.reason, reason)
+
+ def test_skips_without_a_finished_calculation(self) -> None:
+ outcome = _outcome(_metric(), hours_of_data=None) # type: ignore[arg-type]
+ assert isinstance(outcome, CheckSkip)
+ self.assertEqual(outcome.reason, "Runs once a calculation of the metrics has finished")
+
+
+class TestMetricAllZeroFromStoredRuns(BaseTest):
+ def _experiment(self, metrics: list[dict[str, Any]], *, started_days_ago: int = 5) -> Experiment:
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ key="checkout-test",
+ created_by=self.user,
+ filters={
+ "groups": [{"properties": [], "rollout_percentage": 100}],
+ "multivariate": {
+ "variants": [
+ {"key": "control", "rollout_percentage": 50},
+ {"key": "test", "rollout_percentage": 50},
+ ]
+ },
+ },
+ )
+ return Experiment.objects.create(
+ team=self.team,
+ name="Checkout",
+ feature_flag=flag,
+ start_date=timezone.now() - timedelta(days=started_days_ago),
+ metrics=metrics,
+ )
+
+ def _store(
+ self, experiment: Experiment, results: dict[str, dict | None], *, query_to: datetime | None = None
+ ) -> None:
+ query_to = query_to or timezone.now() - timedelta(hours=1)
+ with team_scope(self.team.id):
+ run = ExperimentMetricsRecalculation.objects.create(
+ team=self.team,
+ experiment=experiment,
+ status=ExperimentMetricsRecalculation.Status.IN_PROGRESS,
+ metric_uuids=list(results),
+ query_to=query_to,
+ )
+ store = MetricResultStore(experiment_id=experiment.id)
+ for metric_uuid, result in results.items():
+ calculation_config = get_metric_calculation_config(experiment, metric_uuid)
+ assert calculation_config is not None and experiment.start_date is not None
+ store.record_run_result(
+ str(run.id),
+ calculation_config,
+ window=query_to,
+ query_from=experiment.start_date,
+ result=result or {},
+ query_id=None,
+ )
+ ExperimentMetricsRecalculation.objects.filter(id=run.id).update(
+ status=ExperimentMetricsRecalculation.Status.COMPLETED, completed_at=query_to
+ )
+
+ def _findings(self, experiment: Experiment) -> list[ExperimentHealthFinding]:
+ health = get_experiment_health(
+ team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES, user=self.user
+ )
+ return [finding for finding in health.findings if finding.code == CODE]
+
+ def _mean(self, uuid: str, **changes: Any) -> dict[str, Any]:
+ return {
+ "kind": "ExperimentMetric",
+ "metric_type": "mean",
+ "uuid": uuid,
+ "name": f"Metric {uuid}",
+ "source": {"kind": "EventsNode", "event": "purchase"},
+ **changes,
+ }
+
+ def _ratio(self, uuid: str) -> dict[str, Any]:
+ return {
+ "kind": "ExperimentMetric",
+ "metric_type": "ratio",
+ "uuid": uuid,
+ "name": f"Metric {uuid}",
+ "numerator": {"kind": "EventsNode", "event": "viewed"},
+ "denominator": {"kind": "EventsNode", "event": "added"},
+ }
+
+ def _seen(self, event: str, last_seen_at: datetime) -> None:
+ EventDefinition.objects.create(team=self.team, project=self.team.project, name=event, last_seen_at=last_seen_at)
+
+ @parameterized.expand(
+ [
+ ("zero_in_every_variant", _result(samples=1000, total=0), True),
+ ("a_zero_stored_as_a_float", _result(samples=1000, total=0.0), True),
+ (
+ "a_value_in_one_variant",
+ {**_result(samples=1000, total=0), "variant_results": [_variant(samples=1000, total=1)]},
+ False,
+ ),
+ ("values_that_cancel_out", _result(samples=1000, total=0, squares=4), False),
+ (
+ "a_sum_that_overflowed",
+ {
+ **_result(samples=1000, total=0),
+ "baseline": {
+ "key": "control",
+ "number_of_samples": 1000,
+ "sum": None,
+ "sum_squares": None,
+ },
+ },
+ False,
+ ),
+ ]
+ )
+ def test_reads_the_zero_from_the_run_the_page_shows(self, _name: str, result: dict, reports: bool) -> None:
+ experiment = self._experiment([self._mean("primary")])
+ self._seen("purchase", timezone.now())
+ self._store(experiment, {"primary": result})
+
+ self.assertEqual(bool(self._findings(experiment)), reports)
+
+ @parameterized.expand(
+ [
+ ("numerator", _result(samples=1000, total=0, denominator=20), ["The numerator of the primary"]),
+ ("denominator", _result(samples=1000, total=20, denominator=0), ["The denominator of the primary"]),
+ (
+ "both_sides",
+ _result(samples=1000, total=0, denominator=0),
+ ["The numerator of the primary", "The denominator of the primary"],
+ ),
+ (
+ "a_denominator_with_a_variant_without_exposures",
+ {**_result(samples=1000, total=20, denominator=0), "variant_results": [_variant(samples=0, total=0)]},
+ ["The denominator of the primary"],
+ ),
+ ]
+ )
+ def test_reads_the_side_of_a_ratio(self, _name: str, result: dict, words: list[str]) -> None:
+ experiment = self._experiment([self._ratio("primary")])
+ self._store(experiment, {"primary": result})
+
+ [finding] = self._findings(experiment)
+
+ self.assertEqual(finding.title, "A metric reads zero in every variant")
+ for side in words:
+ self.assertIn(side, finding.detail)
+
+ def test_a_denominator_that_cancels_out_is_no_zero(self) -> None:
+ experiment = self._experiment([self._ratio("primary")])
+ self._store(experiment, {"primary": _result(samples=1000, total=20, denominator=0, denominator_squares=8)})
+
+ self.assertEqual(self._findings(experiment), [])
+
+ def test_names_only_the_event_of_the_side_that_reads_zero(self) -> None:
+ experiment = self._experiment([self._ratio("primary")])
+ assert experiment.start_date is not None
+ self._seen("viewed", timezone.now())
+ self._seen("added", experiment.start_date - timedelta(days=1))
+ self._store(experiment, {"primary": _result(samples=1000, total=0, denominator=20)})
+
+ [finding] = self._findings(experiment)
+
+ self.assertIsNone(finding.subcode)
+ self.assertIn('the event "viewed"', finding.detail)
+
+ @parameterized.expand(
+ [("ingestion_records_the_project", True, "event_stopped"), ("ingestion_skips_it", False, None)]
+ )
+ def test_names_an_event_that_stopped_before_launch(
+ self, _name: str, flag_called_recently: bool, subcode: str | None
+ ) -> None:
+ experiment = self._experiment([self._mean("primary")])
+ assert experiment.start_date is not None
+ self._seen("purchase", experiment.start_date - timedelta(days=1))
+ if flag_called_recently:
+ self._seen("$feature_flag_called", timezone.now())
+ self._store(experiment, {"primary": _result(samples=1000, total=0)})
+
+ [finding] = self._findings(experiment)
+
+ self.assertEqual(finding.subcode, subcode)
+
+ def test_an_event_that_arrived_after_launch_did_not_stop(self) -> None:
+ experiment = self._experiment([self._mean("primary")])
+ assert experiment.start_date is not None
+ self._seen("purchase", experiment.start_date + timedelta(hours=1))
+ self._store(experiment, {"primary": _result(samples=1000, total=0)})
+
+ [finding] = self._findings(experiment)
+
+ self.assertIsNone(finding.subcode)
+
+ @parameterized.expand([("under_the_data_floor", 71, False), ("at_the_data_floor", 72, True)])
+ def test_measures_the_data_at_the_calculation_that_the_page_shows(
+ self, _name: str, hours_of_data: int, reports: bool
+ ) -> None:
+ experiment = self._experiment([self._mean("primary")], started_days_ago=4)
+ assert experiment.start_date is not None
+ self._store(
+ experiment,
+ {"primary": _result(samples=1000, total=0)},
+ query_to=experiment.start_date + timedelta(hours=hours_of_data),
+ )
+
+ self.assertEqual(bool(self._findings(experiment)), reports)
+
+ def test_a_blank_event_name_matches_no_events(self) -> None:
+ experiment = self._experiment([self._mean("primary", source={"kind": "EventsNode", "event": ""})])
+ self._store(experiment, {"primary": _result(samples=1000, total=0)})
+
+ [finding] = self._findings(experiment)
+
+ self.assertEqual(finding.subcode, "blank_event")
diff --git a/products/experiments/backend/hogql_queries/base_query_utils.py b/products/experiments/backend/hogql_queries/base_query_utils.py
index ed3935b127f4..707d86af8ef6 100644
--- a/products/experiments/backend/hogql_queries/base_query_utils.py
+++ b/products/experiments/backend/hogql_queries/base_query_utils.py
@@ -231,6 +231,20 @@ def conversion_window_to_seconds(conversion_window: int, conversion_window_unit:
return conversion_window * multipliers[conversion_window_unit]
+def winsorization_bound_expr(level: float, value: str, *, ignore_zeros: bool = False) -> ast.Expr:
+ """The outlier bound at the percentile `level` of `value` across the entities. `value` is an internal column name,
+ never user input, because it is interpolated into the expression. `quantile_exact_position` mirrors the value that
+ this function picks, so a change of the aggregate here must change that function too."""
+ over = f"if({value} != 0, {value}, null)" if ignore_zeros else value
+ return parse_expr(f"quantileExact({{level}})({over})", placeholders={"level": ast.Constant(value=level)})
+
+
+def quantile_exact_position(level: float, count: int) -> int:
+ """The position, among `count` values in ascending order, of the value that `quantileExact(level)` returns."""
+ # ClickHouse truncates `level * count` computed in double precision, which Python's float multiplication matches.
+ return int(level * count) if level < 1 else count - 1
+
+
def analysis_window_end(end_date: Optional[datetime], as_of: datetime) -> datetime:
"""Upper edge of an experiment's analysis window, evaluated at ``as_of``.
diff --git a/products/experiments/backend/hogql_queries/experiment_mean_query_builder.py b/products/experiments/backend/hogql_queries/experiment_mean_query_builder.py
index 411002710605..3437d67302b1 100644
--- a/products/experiments/backend/hogql_queries/experiment_mean_query_builder.py
+++ b/products/experiments/backend/hogql_queries/experiment_mean_query_builder.py
@@ -8,6 +8,7 @@ from posthog.hogql.parser import parse_expr, parse_select
from products.experiments.backend.hogql_queries.base_query_utils import (
is_session_property_metric,
validate_session_property,
+ winsorization_bound_expr,
)
from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureQueryBuilder
from products.experiments.backend.hogql_queries.metric_source import MetricSourceInfo
@@ -346,24 +347,16 @@ class MeanQueryBuilder:
assert isinstance(self._b.metric, ExperimentMeanMetric)
if self._b.metric.lower_bound_percentile is not None:
- lower_bound_expr = parse_expr(
- "quantileExact({level})(entity_metrics.value)",
- placeholders={"level": ast.Constant(value=self._b.metric.lower_bound_percentile)},
- )
+ lower_bound_expr = winsorization_bound_expr(self._b.metric.lower_bound_percentile, "entity_metrics.value")
else:
lower_bound_expr = parse_expr("min(entity_metrics.value)")
if self._b.metric.upper_bound_percentile is not None:
- if getattr(self._b.metric, "ignore_zeros", False):
- upper_bound_expr = parse_expr(
- "quantileExact({level})(if(entity_metrics.value != 0, entity_metrics.value, null))",
- placeholders={"level": ast.Constant(value=self._b.metric.upper_bound_percentile)},
- )
- else:
- upper_bound_expr = parse_expr(
- "quantileExact({level})(entity_metrics.value)",
- placeholders={"level": ast.Constant(value=self._b.metric.upper_bound_percentile)},
- )
+ upper_bound_expr = winsorization_bound_expr(
+ self._b.metric.upper_bound_percentile,
+ "entity_metrics.value",
+ ignore_zeros=bool(getattr(self._b.metric, "ignore_zeros", False)),
+ )
else:
upper_bound_expr = parse_expr("max(entity_metrics.value)")
diff --git a/products/experiments/backend/hogql_queries/experiment_ratio_query_builder.py b/products/experiments/backend/hogql_queries/experiment_ratio_query_builder.py
index 9fe9454640e6..d89a60d0e862 100644
--- a/products/experiments/backend/hogql_queries/experiment_ratio_query_builder.py
+++ b/products/experiments/backend/hogql_queries/experiment_ratio_query_builder.py
@@ -14,6 +14,7 @@ from posthog.hogql.parser import parse_expr, parse_select
from posthog.dataclasses import frozen
+from products.experiments.backend.hogql_queries.base_query_utils import winsorization_bound_expr
from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureQueryBuilder
from products.experiments.backend.hogql_queries.metric_source import MetricSourceInfo
from products.experiments.backend.hogql_queries.metric_source_precompute import (
@@ -154,24 +155,14 @@ class RatioQueryBuilder:
ignore_zeros = bool(outlier_handling.ignore_zeros) if outlier_handling else False
if lower_pct is not None:
- lower_bound_expr = parse_expr(
- f"quantileExact({{level}})(entity_metrics.{value_field})",
- placeholders={"level": ast.Constant(value=lower_pct)},
- )
+ lower_bound_expr = winsorization_bound_expr(lower_pct, f"entity_metrics.{value_field}")
else:
lower_bound_expr = parse_expr(f"min(entity_metrics.{value_field})")
if upper_pct is not None:
- if ignore_zeros:
- upper_bound_expr = parse_expr(
- f"quantileExact({{level}})(if(entity_metrics.{value_field} != 0, entity_metrics.{value_field}, null))",
- placeholders={"level": ast.Constant(value=upper_pct)},
- )
- else:
- upper_bound_expr = parse_expr(
- f"quantileExact({{level}})(entity_metrics.{value_field})",
- placeholders={"level": ast.Constant(value=upper_pct)},
- )
+ upper_bound_expr = winsorization_bound_expr(
+ upper_pct, f"entity_metrics.{value_field}", ignore_zeros=ignore_zeros
+ )
else:
upper_bound_expr = parse_expr(f"max(entity_metrics.{value_field})")
diff --git a/products/experiments/backend/hogql_queries/test/test_base_query_utils.py b/products/experiments/backend/hogql_queries/test/test_base_query_utils.py
new file mode 100644
index 000000000000..75a83ed233f1
--- /dev/null
+++ b/products/experiments/backend/hogql_queries/test/test_base_query_utils.py
@@ -0,0 +1,27 @@
+from posthog.test.base import BaseTest, ClickhouseTestMixin
+
+from parameterized import parameterized
+
+from posthog.hogql.parser import parse_select
+from posthog.hogql.query import execute_hogql_query
+
+from products.experiments.backend.hogql_queries.base_query_utils import (
+ quantile_exact_position,
+ winsorization_bound_expr,
+)
+
+
+class TestWinsorizationBound(ClickhouseTestMixin, BaseTest):
+ @parameterized.expand([(0.99, 100), (0.99, 101), (0.95, 766), (0.999, 600), (0.5, 7)])
+ def test_quantile_exact_position_tells_when_the_upper_bound_is_zero(self, level: float, count: int) -> None:
+ hidden = count - quantile_exact_position(level, count) - 1
+
+ bounds = []
+ for with_value in (hidden, hidden + 1):
+ query = parse_select(
+ f"SELECT {{bound}} FROM (SELECT if(number >= {count - with_value}, 5, 0) AS value FROM numbers({count}))",
+ placeholders={"bound": winsorization_bound_expr(level, "value")},
+ )
+ bounds.append(execute_hogql_query(query, team=self.team).results[0][0])
+
+ self.assertEqual(bounds, [0, 5])
diff --git a/products/experiments/backend/logic/result_stats.py b/products/experiments/backend/logic/result_stats.py
index a5d758006c5c..829c6bf5fdde 100644
--- a/products/experiments/backend/logic/result_stats.py
+++ b/products/experiments/backend/logic/result_stats.py
@@ -14,6 +14,7 @@ from posthog.schema import (
from posthog.dataclasses import frozen
StatsMethod = Literal["bayesian", "frequentist"]
+VariantResult = ExperimentVariantResultBayesian | ExperimentVariantResultFrequentist
_INTERVAL_KEYS = ("credible_interval", "confidence_interval")
@@ -121,7 +122,7 @@ class VariantStats:
significant: bool | None
@classmethod
- def from_schema(cls, variant: ExperimentVariantResultBayesian | ExperimentVariantResultFrequentist) -> Self:
+ def from_schema(cls, variant: VariantResult) -> Self:
chance_to_win: float | None = None
p_value: float | None = None
if isinstance(variant, ExperimentVariantResultBayesian):
diff --git a/products/experiments/backend/metric_calculation/results.py b/products/experiments/backend/metric_calculation/results.py
index 9b904bb60f6e..d5a8a7d4c8e9 100644
--- a/products/experiments/backend/metric_calculation/results.py
+++ b/products/experiments/backend/metric_calculation/results.py
@@ -141,6 +141,17 @@ class MetricResultStore:
rows = rows.defer("result")
return list(_newest_write_first(rows, "metric_uuid").distinct("metric_uuid"))
+ def with_zero_baseline(self, rows: Iterable[ExperimentMetricResult]) -> list[ExperimentMetricResult]:
+ """Those of these rows whose baseline reads zero on the metric or on its denominator, with `result`, in the order
+ of `rows`. The database applies the filter, so the rows with a value don't load `result`."""
+ by_pk = {row.pk: row for row in rows}
+ loaded = ExperimentMetricResult.objects.filter(
+ Q(result__baseline__sum=0) | Q(result__baseline__denominator_sum=0),
+ experiment_id=self.experiment_id,
+ pk__in=by_pk,
+ ).in_bulk()
+ return [loaded[pk] for pk in by_pk if pk in loaded]
+
def exposures_paths(self, rows: Iterable[ExperimentMetricResult]) -> frozenset[ExposuresPath]:
"""Where the queries of these rows read the exposures. It reads one key of `result`, so the rows of `for_run` can
leave `result` unloaded. A result without the key adds nothing."""
diff --git a/products/experiments/backend/metric_validation.py b/products/experiments/backend/metric_validation.py
index 300fcf80955c..2b13a6728c28 100644
--- a/products/experiments/backend/metric_validation.py
+++ b/products/experiments/backend/metric_validation.py
@@ -8,8 +8,11 @@ event exists) stay with the callers.
"""
from collections.abc import Collection
+from datetime import datetime
from typing import Any
+from django.db.models import QuerySet
+
import pydantic
import structlog
from rest_framework.exceptions import PermissionDenied, ValidationError
@@ -180,6 +183,28 @@ def validate_saved_metric_link_overrides(metadata: dict, *, error_prefix: str) -
raise ValidationError(f"{error_prefix}{_pydantic_error_message(e)}")
+def metric_nodes(metric: dict) -> list[dict]:
+ """The event, action and warehouse nodes that a stored metric definition reads."""
+ nodes: list[dict] = []
+ metric_type = metric.get("metric_type")
+ if metric_type == "mean":
+ if source := metric.get("source"):
+ nodes.append(source)
+ elif metric_type == "funnel":
+ nodes.extend(metric.get("series") or [])
+ elif metric_type == "ratio":
+ if num := metric.get("numerator"):
+ nodes.append(num)
+ if den := metric.get("denominator"):
+ nodes.append(den)
+ elif metric_type == "retention":
+ if se := metric.get("start_event"):
+ nodes.append(se)
+ if ce := metric.get("completion_event"):
+ nodes.append(ce)
+ return nodes
+
+
def extract_entity_nodes(metrics: list[dict] | None) -> tuple[set[str], set[int]]:
"""Extract event names and action IDs from all EventsNode/ActionsNode refs in metrics."""
event_names: set[str] = set()
@@ -188,23 +213,7 @@ def extract_entity_nodes(metrics: list[dict] | None) -> tuple[set[str], set[int]
return event_names, action_ids
for metric in metrics:
- nodes: list[dict] = []
- metric_type = metric.get("metric_type")
- if metric_type == "mean":
- if source := metric.get("source"):
- nodes.append(source)
- elif metric_type == "funnel":
- nodes.extend(metric.get("series") or [])
- elif metric_type == "ratio":
- if num := metric.get("numerator"):
- nodes.append(num)
- if den := metric.get("denominator"):
- nodes.append(den)
- elif metric_type == "retention":
- if se := metric.get("start_event"):
- nodes.append(se)
- if ce := metric.get("completion_event"):
- nodes.append(ce)
+ nodes = metric_nodes(metric)
for node in nodes:
kind = node.get("kind")
@@ -243,16 +252,29 @@ def defined_event_names(project_id: int, event_names: Collection[str], *, seen_o
"""
if not event_names:
return set()
+ definitions = _event_definitions(project_id, event_names)
+ if seen_only:
+ definitions = definitions.filter(last_seen_at__isnull=False)
+ return set(definitions.values_list("name", flat=True))
+
+
+def event_last_seen(project_id: int, event_names: Collection[str]) -> dict[str, datetime]:
+ """When ingestion last saw each of `event_names` in the project, in the scope of `defined_event_names`. A name
+ without a definition, or with one that ingestion hasn't seen, is absent."""
+ if not event_names:
+ return {}
+ definitions = _event_definitions(project_id, event_names).values_list("name", "last_seen_at")
+ return {name: last_seen for name, last_seen in definitions if last_seen is not None}
+
+
+def _event_definitions(project_id: int, event_names: Collection[str]) -> QuerySet[EventDefinition]:
# `COALESCE(project_id, team_id)` falls back to `team_id` (not the caller's team id) on purpose: legacy
# EventDefinitions (project_id IS NULL) belong to the *primary* team, and primary_team.id == project.id by
# convention. This mirrors the picker SQL in posthog/api/event_definition.py so sibling teams can validate
# against legacy primary-team events the picker shows.
- definitions = EventDefinition.objects.alias(effective_project_id=effective_project_id_expr()).filter(
+ return EventDefinition.objects.alias(effective_project_id=effective_project_id_expr()).filter(
effective_project_id=project_id, name__in=event_names
)
- if seen_only:
- definitions = definitions.filter(last_seen_at__isnull=False)
- return set(definitions.values_list("name", flat=True))
def validate_metric_action_ids(
diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.py
index ac4eb401a334..410777a43cdf 100644
--- a/products/experiments/backend/presentation/serializers.py
+++ b/products/experiments/backend/presentation/serializers.py
@@ -3484,6 +3484,21 @@ class ExperimentHealthInputsMetricCalculationFailureSerializer(serializers.Seria
)
+class ExperimentHealthInputsZeroMetricSerializer(serializers.Serializer):
+ name = serializers.CharField(help_text="The name of the metric, as the results page shows it.")
+ role = serializers.CharField(help_text="Whether the metric is a primary or a secondary metric of the experiment.")
+ exposures = serializers.IntegerField(help_text="The persons or groups of the result, across the variants.")
+ ratio_side = serializers.CharField(
+ allow_null=True,
+ help_text="The side of a ratio that reads zero. Each side that reads zero has its own entry. Null for the other "
+ "metric types.",
+ )
+ stopped_events = serializers.ListField(
+ child=serializers.CharField(),
+ help_text="The events of the metric that the project last received before launch.",
+ )
+
+
class ExperimentHealthInputsLatestCalculationSerializer(serializers.Serializer):
finished_at = serializers.DateTimeField(
allow_null=True,
@@ -3492,6 +3507,13 @@ class ExperimentHealthInputsLatestCalculationSerializer(serializers.Serializer):
failures = ExperimentHealthInputsMetricCalculationFailureSerializer(
many=True, help_text="The metrics whose latest calculation failed, with the error that the results page shows."
)
+ hours_of_data = serializers.FloatField(
+ allow_null=True,
+ help_text="The hours from launch to the end of the data of this calculation. Null when none has finished.",
+ )
+ zero_metrics = ExperimentHealthInputsZeroMetricSerializer(
+ many=True, help_text="The metrics that read zero in every variant in this calculation."
+ )
class ExperimentHealthInputsReaderAccessSerializer(serializers.Serializer):
diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.py
index 26acf3d3b6bf..c53b76876d51 100644
--- a/products/experiments/backend/presentation/views.py
+++ b/products/experiments/backend/presentation/views.py
@@ -1032,6 +1032,26 @@ class EnterpriseExperimentsViewSet(
then names the filters. A query that read precomputed exposures doesn't apply those filters,
so it can't get `exposure_criteria`. It is null otherwise.
+ `metric_all_zero` reports a metric that reads zero in every variant, from the same latest
+ finished calculation, so the results page shows "Not enough data yet" at any sample size.
+ Each side of a ratio that is zero in every variant counts on its own. It reads the stored
+ result and the project's event definitions, and no events. A primary metric needs 1,000
+ exposures in the result and 72 hours of data, because most zeros before that are outcomes
+ that take time. A retention metric counts those hours from the start of its retention
+ window. The subcode is the cause of the first metric: `blank_event` (an event with an empty
+ name), `event_stopped` (the project last received the event before launch, while ingestion
+ still records the project), `outlier_bound` (an upper outlier bound that is 0 while some
+ exposures have a value) or null when none shows a cause. `blank_event` and `outlier_bound`
+ also report a secondary metric, and need only 100 exposures and 12 hours of data, because
+ they read 0 at any traffic. A bound on the largest value of the exposures can't be 0 while
+ one has a value, so it is no cause. Neither is a bound on a metric with breakdowns, which
+ gets one bound per breakdown group. Until the end, the check leaves out a metric that uses an
+ event `metric_event_not_ingested` reports. Each metric, or side of a ratio, is in the
+ `evidence` keys `metric_N`, `metric_N_role`, `metric_N_side`, `metric_N_exposures` and
+ `metric_N_cause`. It is a warning for a primary metric. After the end it stays a warning only
+ for a cause that a setting fixes, because the calculation after the fix reads the events up
+ to the end. It is info otherwise.
+
The severity of the `some_conditions_pinned` finding follows the largest share that the
users of the pinned release conditions make up among the users who got one variant from
the flag, in the last 7 days, or since launch or the flag's last save if later. It reads
diff --git a/products/experiments/backend/running_time_target.py b/products/experiments/backend/running_time_target.py
index a055bc08749a..ecd5798ec3d7 100644
--- a/products/experiments/backend/running_time_target.py
+++ b/products/experiments/backend/running_time_target.py
@@ -20,8 +20,6 @@ from posthog.schema import (
ExperimentMeanMetric,
ExperimentQueryResponse,
ExperimentStatsBaseValidated,
- ExperimentVariantResultBayesian,
- ExperimentVariantResultFrequentist,
)
from posthog.dataclasses import frozen
@@ -32,6 +30,7 @@ from products.experiments.backend.hogql_queries.utils import (
metric_variant_to_cuped_data,
metric_variant_to_statistic,
)
+from products.experiments.backend.logic.result_stats import VariantResult
from products.experiments.backend.metric_calculation.config import (
MetricCalculationConfig,
build_primary_calculation_configs,
@@ -56,8 +55,6 @@ from products.experiments.stats.shared.statistics import (
)
from products.feature_flags.backend.facade.edits import Split, parse_variants
-VariantResult = ExperimentVariantResultBayesian | ExperimentVariantResultFrequentist
-
# A funnel's stored stats carry sample sessions per step, which can be large, and the target never reads them.
_STATS_FIELDS = set(ExperimentStatsBaseValidated.model_fields) - {"step_sessions"}
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index ee3ab5a194fe..e07654d60e25 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2306,6 +2306,7 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `metric_cannot_compile` - Metric Cannot Compile
* * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort
* * `exposure_filter_excludes_events` - Exposure Filter Excludes Events
@@ -2335,6 +2336,7 @@ export const ExperimentHealthFindingCodeEnumApi = {
MetricEventNotIngested: 'metric_event_not_ingested',
MetricCannotCompile: 'metric_cannot_compile',
MetricCalculationFailed: 'metric_calculation_failed',
+ MetricAllZero: 'metric_all_zero',
ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
ExposureFilterOnDynamicCohort: 'exposure_filter_on_dynamic_cohort',
ExposureFilterExcludesEvents: 'exposure_filter_excludes_events',
@@ -2407,6 +2409,7 @@ export interface ExperimentHealthFindingApi {
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `metric_cannot_compile` - Metric Cannot Compile
* * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort
* * `exposure_filter_excludes_events` - Exposure Filter Excludes Events
@@ -2466,6 +2469,7 @@ export interface ExperimentHealthFindingApi {
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `metric_cannot_compile` - Metric Cannot Compile
* * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filters` - Exposure Filters
* * `early_significance` - Early Significance
@@ -2492,6 +2496,7 @@ export const ExperimentHealthCheckKeyEnumApi = {
MetricEventNotIngested: 'metric_event_not_ingested',
MetricCannotCompile: 'metric_cannot_compile',
MetricCalculationFailed: 'metric_calculation_failed',
+ MetricAllZero: 'metric_all_zero',
ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
ExposureFilters: 'exposure_filters',
EarlySignificance: 'early_significance',
@@ -2534,6 +2539,7 @@ export interface ExperimentHealthCheckResultApi {
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `metric_cannot_compile` - Metric Cannot Compile
* * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filters` - Exposure Filters
* * `early_significance` - Early Significance
@@ -3343,6 +3349,22 @@ export interface ExperimentHealthInputsMetricCalculationFailureApi {
message: string | null
}
+export interface ExperimentHealthInputsZeroMetricApi {
+ /** The name of the metric, as the results page shows it. */
+ name: string
+ /** Whether the metric is a primary or a secondary metric of the experiment. */
+ role: string
+ /** The persons or groups of the result, across the variants. */
+ exposures: number
+ /**
+ * The side of a ratio that reads zero. Each side that reads zero has its own entry. Null for the other metric types.
+ * @nullable
+ */
+ ratio_side: string | null
+ /** The events of the metric that the project last received before launch. */
+ stopped_events: string[]
+}
+
export interface ExperimentHealthInputsLatestCalculationApi {
/**
* When the latest calculation that the results page shows finished. Null when none has finished.
@@ -3351,6 +3373,13 @@ export interface ExperimentHealthInputsLatestCalculationApi {
finished_at: string | null
/** The metrics whose latest calculation failed, with the error that the results page shows. */
failures: ExperimentHealthInputsMetricCalculationFailureApi[]
+ /**
+ * The hours from launch to the end of the data of this calculation. Null when none has finished.
+ * @nullable
+ */
+ hours_of_data: number | null
+ /** The metrics that read zero in every variant in this calculation. */
+ zero_metrics: ExperimentHealthInputsZeroMetricApi[]
}
/**
diff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.ts
index 17137fc3f978..bcddfbf29978 100644
--- a/products/experiments/frontend/generated/api.ts
+++ b/products/experiments/frontend/generated/api.ts
@@ -777,6 +777,26 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =
* then names the filters. A query that read precomputed exposures doesn't apply those filters,
* so it can't get `exposure_criteria`. It is null otherwise.
*
+ * `metric_all_zero` reports a metric that reads zero in every variant, from the same latest
+ * finished calculation, so the results page shows "Not enough data yet" at any sample size.
+ * Each side of a ratio that is zero in every variant counts on its own. It reads the stored
+ * result and the project's event definitions, and no events. A primary metric needs 1,000
+ * exposures in the result and 72 hours of data, because most zeros before that are outcomes
+ * that take time. A retention metric counts those hours from the start of its retention
+ * window. The subcode is the cause of the first metric: `blank_event` (an event with an empty
+ * name), `event_stopped` (the project last received the event before launch, while ingestion
+ * still records the project), `outlier_bound` (an upper outlier bound that is 0 while some
+ * exposures have a value) or null when none shows a cause. `blank_event` and `outlier_bound`
+ * also report a secondary metric, and need only 100 exposures and 12 hours of data, because
+ * they read 0 at any traffic. A bound on the largest value of the exposures can't be 0 while
+ * one has a value, so it is no cause. Neither is a bound on a metric with breakdowns, which
+ * gets one bound per breakdown group. Until the end, the check leaves out a metric that uses an
+ * event `metric_event_not_ingested` reports. Each metric, or side of a ratio, is in the
+ * `evidence` keys `metric_N`, `metric_N_role`, `metric_N_side`, `metric_N_exposures` and
+ * `metric_N_cause`. It is a warning for a primary metric. After the end it stays a warning only
+ * for a cause that a setting fixes, because the calculation after the fix reads the events up
+ * to the end. It is info otherwise.
+ *
* The severity of the `some_conditions_pinned` finding follows the largest share that the
* users of the pinned release conditions make up among the users who got one variant from
* the flag, in the last 7 days, or since launch or the flag's last save if later. It reads
diff --git a/products/experiments/frontend/health/debug/healthDebugInputs.ts b/products/experiments/frontend/health/debug/healthDebugInputs.ts
index 30e876bfc920..20d5deb311b7 100644
--- a/products/experiments/frontend/health/debug/healthDebugInputs.ts
+++ b/products/experiments/frontend/health/debug/healthDebugInputs.ts
@@ -333,6 +333,12 @@ export function buildMetricFacts(
failure.message ?? 'message hidden, it can hold evaluation rules'
}`,
})),
+ ...(state.inputs?.latest_calculation?.zero_metrics ?? []).map((metric) => ({
+ label: `Zero: ${metric.name} (${metric.role})`,
+ value: `${metric.exposures} exposures${metric.ratio_side ? `, ${metric.ratio_side}` : ''}${
+ metric.stopped_events.length ? `, stopped: ${metric.stopped_events.join(', ')}` : ''
+ }`,
+ })),
]
}
diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.ts
index 2751dc4ad03c..31dbd6dac523 100644
--- a/products/experiments/frontend/health/debug/healthDebugReport.ts
+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts
@@ -247,6 +247,16 @@ function metricCalculationNote({ evidence }: ExperimentHealthFindingApi): string
return `Failed: ${failures.join(', ')}. Calculated at ${evidence.calculated_at}.`
}
+function metricAllZeroNote({ evidence }: ExperimentHealthFindingApi): string {
+ const metrics = Object.entries(evidence)
+ .filter(([key]) => /^metric_\d+$/.test(key))
+ .map(([key, value]) => {
+ const side = evidence[`${key}_side`]
+ return `${value}${side ? ` ${side}` : ''} (${evidence[`${key}_cause`] ?? 'no cause shown'})`
+ })
+ return `Zero in every variant: ${metrics.join(', ')}.`
+}
+
function exposureCriteriaNote({ subcode, evidence }: ExperimentHealthFindingApi): string {
if (subcode === 'filter_on_another_flag') {
return `The query requires ${evidence.flag_key}, a filter requires ${evidence.filter_flag_keys}.`
@@ -385,6 +395,7 @@ export function buildHealthDebugChecks(input: HealthDebugInput): HealthDebugChec
serverOnlyCheck(input, 'metric_event_not_ingested', metricEventNote, ['metrics', 'project']),
serverOnlyCheck(input, 'metric_cannot_compile', metricCompileNote, ['metrics', 'experiment']),
serverOnlyCheck(input, 'metric_calculation_failed', metricCalculationNote, ['metrics']),
+ serverOnlyCheck(input, 'metric_all_zero', metricAllZeroNote, ['metrics', 'project']),
serverOnlyCheck(input, 'exposure_criteria_cannot_match', exposureCriteriaNote, ['experiment', 'project']),
exposureFiltersCheck(input),
serverOnlyCheck(input, 'early_significance', null, ['experiment', 'metrics']),
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index 73450b56e36f..e7e466fef090 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -31,6 +31,7 @@ export type ExperimentHealthFindingCode =
| 'metric_event_not_ingested'
| 'metric_cannot_compile'
| 'metric_calculation_failed'
+ | 'metric_all_zero'
| 'exposure_criteria_cannot_match'
| 'exposure_filter_on_dynamic_cohort'
| 'exposure_filter_excludes_events'
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 74ef2b7602ff..fe69c6dfc7ba 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -48124,6 +48124,7 @@ export namespace Schemas {
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `metric_cannot_compile` - Metric Cannot Compile
* * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort
* * `exposure_filter_excludes_events` - Exposure Filter Excludes Events
@@ -48153,6 +48154,7 @@ export namespace Schemas {
MetricEventNotIngested: 'metric_event_not_ingested',
MetricCannotCompile: 'metric_cannot_compile',
MetricCalculationFailed: 'metric_calculation_failed',
+ MetricAllZero: 'metric_all_zero',
ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
ExposureFilterOnDynamicCohort: 'exposure_filter_on_dynamic_cohort',
ExposureFilterExcludesEvents: 'exposure_filter_excludes_events',
@@ -48225,6 +48227,7 @@ export namespace Schemas {
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `metric_cannot_compile` - Metric Cannot Compile
* * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort
* * `exposure_filter_excludes_events` - Exposure Filter Excludes Events
@@ -48284,6 +48287,7 @@ export namespace Schemas {
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `metric_cannot_compile` - Metric Cannot Compile
* * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filters` - Exposure Filters
* * `early_significance` - Early Significance
@@ -48310,6 +48314,7 @@ export namespace Schemas {
MetricEventNotIngested: 'metric_event_not_ingested',
MetricCannotCompile: 'metric_cannot_compile',
MetricCalculationFailed: 'metric_calculation_failed',
+ MetricAllZero: 'metric_all_zero',
ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
ExposureFilters: 'exposure_filters',
EarlySignificance: 'early_significance',
@@ -48352,6 +48357,7 @@ export namespace Schemas {
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `metric_cannot_compile` - Metric Cannot Compile
* * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filters` - Exposure Filters
* * `early_significance` - Early Significance
@@ -49116,6 +49122,22 @@ export namespace Schemas {
message: string | null;
}
+ export interface ExperimentHealthInputsZeroMetric {
+ /** The name of the metric, as the results page shows it. */
+ name: string;
+ /** Whether the metric is a primary or a secondary metric of the experiment. */
+ role: string;
+ /** The persons or groups of the result, across the variants. */
+ exposures: number;
+ /**
+ * The side of a ratio that reads zero. Each side that reads zero has its own entry. Null for the other metric types.
+ * @nullable
+ */
+ ratio_side: string | null;
+ /** The events of the metric that the project last received before launch. */
+ stopped_events: string[];
+ }
+
export interface ExperimentHealthInputsLatestCalculation {
/**
* When the latest calculation that the results page shows finished. Null when none has finished.
@@ -49124,6 +49146,13 @@ export namespace Schemas {
finished_at: string | null;
/** The metrics whose latest calculation failed, with the error that the results page shows. */
failures: ExperimentHealthInputsMetricCalculationFailure[];
+ /**
+ * The hours from launch to the end of the data of this calculation. Null when none has finished.
+ * @nullable
+ */
+ hours_of_data: number | null;
+ /** The metrics that read zero in every variant in this calculation. */
+ zero_metrics: ExperimentHealthInputsZeroMetric[];
}
export interface ExperimentHealthInputsFlagChange {