:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): explain metrics that read zero in every variant (#126797)
Marcel Poelker8057b2ab338a
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex 7cdf94d9bc3e..85078624d37a 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -11427,9 +11427,9 @@ snapshots:scenes-app-experiments--experiment-frequentist-two-variants--light:hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmIIscenes-app-experiments--experiment-health-debug-tab--dark:- hash: v1.k794b7964.32832ef3ce0f1022d129240ea341f976f33cf255235c6087276d52f7c7b9f6cf.vED1QUQ3zvRyI8X34z6AuD0HovPU9T-JBoCSZkIbiY0+ hash: v1.k794b7964.bd266bf7b03a0d96fb3447e669fb154d381d3920a8b7043f6b996f46789edf45.GZi7irxblhuJDTcQ4ILtuf9V9inrGgHKC4ID1shnfF4scenes-app-experiments--experiment-health-debug-tab--light:- hash: v1.k794b7964.87903c4dee6ea41ecd16e0aae00f3a9d4784b394fa221e3b3516992ddacd6b3d.ww5bg6uopL7e_XLA9pdodLWT5aiZFCBsQRbuQ_0d2NI+ hash: v1.k794b7964.22d2b1422db4cfd1e2b802d020a14727f577c80a0a881eea96c67d9fce3ddbb1.vEPNwvdLLCrf9rSaG29yCHMqAN5UbMBpWLea3BzQqOkscenes-app-experiments--experiment-recordings-dropped-metric--dark:hash: v1.k794b7964.9ce950830524de179a4d761af104781fdc05347d59f18de814e72bc40a486158.T1SiGxL_fGBQT61rjwuJe9MPz1NPcjeZBRv6rKiSzrEscenes-app-experiments--experiment-recordings-dropped-metric--light:diff --git a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsxindex 7c17df644c30..a4a16a27b328 100644--- a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx+++ b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx@@ -88,6 +88,8 @@ const HEALTH_INPUTS: ExperimentHealthInputsApi = {latest_calculation: {finished_at: '2025-01-27T09:30:00Z',failures: [],+ hours_of_data: 500,+ zero_metrics: [],},flag_history: {checked: true,diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex 457c87957086..e6eb67a9e96e 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -10063,6 +10063,7 @@ export namespace Schemas {* * `metric_event_not_ingested` - Metric Event Not Ingested* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed+ * * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort* * `exposure_filter_excludes_events` - Exposure Filter Excludes Events@@ -10089,6 +10090,7 @@ export namespace Schemas {| "metric_event_not_ingested"| "metric_cannot_compile"| "metric_calculation_failed"+ | "metric_all_zero"| "exposure_criteria_cannot_match"| "exposure_filter_on_dynamic_cohort"| "exposure_filter_excludes_events"@@ -10145,6 +10147,7 @@ export namespace Schemas {* * `metric_event_not_ingested` - Metric Event Not Ingested* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed+ * * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort* * `exposure_filter_excludes_events` - Exposure Filter Excludes Events@@ -10211,6 +10214,7 @@ export namespace Schemas {* * `metric_event_not_ingested` - Metric Event Not Ingested* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed+ * * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filters` - Exposure Filters* * `early_significance` - Early Significance@@ -10234,6 +10238,7 @@ export namespace Schemas {| "metric_event_not_ingested"| "metric_cannot_compile"| "metric_calculation_failed"+ | "metric_all_zero"| "exposure_criteria_cannot_match"| "exposure_filters"| "early_significance"@@ -10270,6 +10275,7 @@ export namespace Schemas {* * `metric_event_not_ingested` - Metric Event Not Ingested* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed+ * * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filters` - Exposure Filters* * `early_significance` - Early Significancediff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.pyindex 0462e3eaf827..bc19fa497ee9 100644--- a/products/experiments/backend/facade/contracts.py+++ b/products/experiments/backend/facade/contracts.py@@ -202,6 +202,7 @@ class ExperimentHealthFindingCode(LabeledStrEnum):METRIC_EVENT_NOT_INGESTED = "metric_event_not_ingested"METRIC_CANNOT_COMPILE = "metric_cannot_compile"METRIC_CALCULATION_FAILED = "metric_calculation_failed"+ METRIC_ALL_ZERO = "metric_all_zero"EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"EXPOSURE_FILTER_ON_DYNAMIC_COHORT = "exposure_filter_on_dynamic_cohort"EXPOSURE_FILTER_EXCLUDES_EVENTS = "exposure_filter_excludes_events"@@ -312,6 +313,7 @@ class ExperimentHealthCheckKey(LabeledStrEnum):METRIC_EVENT_NOT_INGESTED = "metric_event_not_ingested"METRIC_CANNOT_COMPILE = "metric_cannot_compile"METRIC_CALCULATION_FAILED = "metric_calculation_failed"+ METRIC_ALL_ZERO = "metric_all_zero"EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"EXPOSURE_FILTERS = "exposure_filters"EARLY_SIGNIFICANCE = "early_significance"diff --git a/products/experiments/backend/health/checks/metric_all_zero.py b/products/experiments/backend/health/checks/metric_all_zero.pynew file mode 100644index 000000000000..dfa63dca537a--- /dev/null+++ b/products/experiments/backend/health/checks/metric_all_zero.py@@ -0,0 +1,271 @@+"""+Metrics that read zero in every variant, so the results page shows "Not enough data yet" at any sample size. Pure+functions, no I/O.++The stored result holds only the zero, so the check names a cause only where the metric's settings or the project's+event definitions show one, and needs no scan of the events. In every other case the words say what the zero means and+name the usual cause: the exposure and the event come from different SDKs under different ids, so the people who send+the event aren't the people who were exposed.+"""++from collections.abc import Mapping+from typing import assert_never++from posthog.schema import FunnelConversionWindowTimeUnit++from posthog.dataclasses import frozen++from products.experiments.backend.facade.contracts import (+ ExperimentHealthFinding,+ ExperimentHealthFindingCode,+ ExperimentHealthFindingSeverity,+)+from products.experiments.backend.health.checks.events_received import joined, quoted+from products.experiments.backend.health.context import HealthContext, ZeroMetric+from products.experiments.backend.health.skips import (+ NOT_ON_ARCHIVED,+ RUNS_AFTER_LAUNCH,+ CheckOutcome,+ no_calculated_metric,+ not_applicable,+ not_checked,+)+from products.experiments.backend.health.split_changes import format_change_time+from products.experiments.backend.hogql_queries.base_query_utils import (+ conversion_window_to_seconds,+ quantile_exact_position,+)++# pinned: the subcodes and `evidence.metric_N_cause`. The page sends the subcode as `finding_subcode` on its events.+BLANK_EVENT = "blank_event"+EVENT_STOPPED = "event_stopped"+OUTLIER_BOUND = "outlier_bound"++# With less of either, most zeros are outcomes that take time, so neither gate works alone.+MIN_EXPOSURES = 1000+MIN_HOURS_OF_DATA = 72+# An empty event name or a bound of 0 reads 0 at any traffic, so these causes need only enough data to read.+MIN_EXPOSURES_WITH_A_SETTING_CAUSE = 100+MIN_HOURS_WITH_A_SETTING_CAUSE = 12++# A change to these settings recalculates the metric over the events up to the end, so they stay a warning after it.+_FIXED_BY_A_SETTING = frozenset({BLANK_EVENT, OUTLIER_BOUND})+_BOUND_LABEL = '"Ignore zeros when calculating upper bound"'+_SECONDS_PER_HOUR = 3600+++@frozen+class _Worded:+ metric: ZeroMetric+ # None when the settings and the event definitions show no cause.+ cause: str | None+ # Without the usual causes, which the finding says once for all the metrics that have no cause.+ detail: str+++def metric_all_zero(ctx: HealthContext) -> CheckOutcome:+ if ctx.archived:+ return not_applicable(NOT_ON_ARCHIVED)+ if not ctx.is_launched:+ return not_applicable(RUNS_AFTER_LAUNCH)+ calculation = ctx.latest_calculation+ if calculation is None:+ return not_checked("Couldn't check because the latest calculation couldn't be read")+ if calculation.metric_count == 0:+ return no_calculated_metric(ctx)+ hours_of_data = calculation.hours_of_data+ if calculation.finished_at is None or hours_of_data is None:+ return not_applicable("Runs once a calculation of the metrics has finished")+ worded = [+ item+ for metric in calculation.zero_metrics+ if not _explained_elsewhere(metric, ctx) and (item := _word(metric, ctx, hours_of_data)) is not None+ ]+ if not worded:+ return None+ worded.sort(key=lambda item: (item.metric.role != "primary", item.cause is None))+ # Both sides of a ratio can read zero, and each is its own item.+ count = len({item.metric.metric_id for item in worded})+ return ExperimentHealthFinding(+ code=ExperimentHealthFindingCode.METRIC_ALL_ZERO,+ subcode=worded[0].cause,+ severity=_severity(worded, ctx),+ title=f"{count} metrics read zero in every variant" if count > 1 else "A metric reads zero in every variant",+ detail=_detail(worded),+ evidence=_evidence(worded),+ # No action opens a given metric.+ actions=(),+ diagnostic_ref=None,+ )+++def _explained_elsewhere(metric: ZeroMetric, ctx: HealthContext) -> bool:+ # `metric_event_not_ingested` reports an event the project never received, until the experiment ends.+ # `metric_calculation_failed` and `zero_exposures` need no rule here: a failed metric has no zero to read, and a+ # result with exposures isn't zero exposures.+ return not ctx.has_ended and any(ctx.events_received.get(event) is False for event in metric.events)+++def _hours_that_count(metric: ZeroMetric, hours_of_data: float) -> float:+ """The hours of data in which the metric can count an outcome. A retention metric counts a return only from the+ start of its retention window after the start event, so its hours begin there."""+ definition = metric.definition+ start = definition.get("retention_window_start")+ unit = definition.get("retention_window_unit")+ if definition.get("metric_type") != "retention" or start is None or unit is None:+ return hours_of_data+ return hours_of_data - conversion_window_to_seconds(start, FunnelConversionWindowTimeUnit(unit)) / _SECONDS_PER_HOUR+++def _upper_bound(metric: ZeroMetric) -> float | None:+ """The percentile of the upper outlier bound over the side that reads zero, when the bound counts zeros."""+ definition = metric.definition+ match metric.ratio_side:+ case None:+ handling = definition if definition.get("metric_type") == "mean" else None+ case "numerator":+ handling = definition.get("numerator_outlier_handling")+ case "denominator":+ handling = definition.get("denominator_outlier_handling")+ case _:+ assert_never(metric.ratio_side)+ if not handling or handling.get("ignore_zeros"):+ return None+ return handling.get("upper_bound_percentile")+++def _values_a_zero_bound_hides(metric: ZeroMetric) -> int:+ """How many exposures can have a value while the upper outlier bound is still 0. Zero without a bound that counts+ zeros, and when the bound picks the largest value."""+ percentile = _upper_bound(metric)+ # The queries compute one bound per breakdown group, and the stored totals don't hold the size of each group.+ if percentile is None or _has_breakdowns(metric):+ return 0+ return metric.exposures - quantile_exact_position(percentile, metric.exposures) - 1+++def _has_breakdowns(metric: ZeroMetric) -> bool:+ breakdown_filter = metric.definition.get("breakdownFilter")+ return isinstance(breakdown_filter, Mapping) and bool(breakdown_filter.get("breakdowns"))+++def _word(metric: ZeroMetric, ctx: HealthContext, hours_of_data: float) -> _Worded | None:+ """The cause and the words for a metric that passes the gate of its cause, or None."""+ hours = _hours_that_count(metric, hours_of_data)+ reads_zero = _reads_zero(metric)+ readable = metric.exposures >= MIN_EXPOSURES_WITH_A_SETTING_CAUSE and hours >= MIN_HOURS_WITH_A_SETTING_CAUSE+ passes_gates = metric.role == "primary" and metric.exposures >= MIN_EXPOSURES and hours >= MIN_HOURS_OF_DATA+ if metric.has_blank_event:+ if not readable:+ return None+ detail = f"{reads_zero} It has an event with an empty name, which matches no events. Edit the metric and choose an event."+ return _Worded(metric=metric, cause=BLANK_EVENT, detail=detail)+ if metric.stopped_events:+ # An event that stopped before launch sends no value for a bound to clamp, so the bound is no cause here.+ if not passes_gates:+ return None+ return _Worded(metric=metric, cause=EVENT_STOPPED, detail=f"{reads_zero} {_stopped(metric, ctx)}")+ hidden = _values_a_zero_bound_hides(metric)+ if hidden > 0 and readable:+ return _Worded(metric=metric, cause=OUTLIER_BOUND, detail=f"{reads_zero} {_bound(hidden, metric.exposures)}")+ if passes_gates:+ return _Worded(metric=metric, cause=None, detail=reads_zero)+ return None+++def _reads_zero(metric: ZeroMetric) -> str:+ subject = f'{metric.role} metric "{metric.name}"'+ subject = f"The {subject}" if metric.ratio_side is None else f"The {metric.ratio_side} of the {subject}"+ # A retention result counts only the exposed users who did its start event.+ counted = "users who did its start event" if metric.definition.get("metric_type") == "retention" else "exposures"+ earlier = (+ f" {metric.reached_earlier_step:,} reached an earlier step, but nobody finished the funnel."+ if metric.reached_earlier_step+ else ""+ )+ return f"{subject} reads 0 for all {metric.exposures:,} {counted}, in every variant.{earlier}"+++def _stopped(metric: ZeroMetric, ctx: HealthContext) -> str:+ stopped = [+ (event, format_change_time(last_seen, ctx.team_timezone)) for event, last_seen in metric.stopped_events.items()+ ]+ if len(stopped) == 1:+ event, last_seen = stopped[0]+ return (+ f'The event "{event}" last arrived on {last_seen}, before the experiment started. Check that your code '+ "still sends it, or edit the metric to use an event that still arrives."+ )+ named = joined([f'"{event}" on {last_seen}' for event, last_seen in stopped])+ return (+ f"These events last arrived before the experiment started: {named}. Check that your code still sends them, or "+ "edit the metric to use events that still arrive."+ )+++def _bound(hidden: int, exposures: int) -> str:+ return (+ f"Its upper outlier bound is taken over all {exposures:,} exposures, including those with no value. When at "+ f"most {hidden:,} of them have a value, the bound is 0, so every value becomes 0. Turn on {_BOUND_LABEL} on "+ "the metric."+ )+++def _detail(worded: list[_Worded]) -> str:+ sentences = [item.detail for item in worded]+ without_a_cause = [item.metric for item in worded if item.cause is None]+ if without_a_cause:+ sentences.extend(_usual_causes(without_a_cause))+ return " ".join(dict.fromkeys(sentences))+++def _usual_causes(metrics: list[ZeroMetric]) -> list[str]:+ events = tuple(dict.fromkeys(event for metric in metrics for event in metric.events))+ tables = tuple(dict.fromkeys(table for metric in metrics for table in metric.warehouse_tables))+ # A metric on actions names no event, but it still reads events.+ reads_events = bool(events) or any(not metric.warehouse_tables for metric in metrics)+ counted = joined([*(["an event"] if reads_events else []), *(["a warehouse row"] if tables else [])], "or")+ sentences = [+ f"PostHog counts {counted} only for the person or group that was exposed, and only after their first exposure."+ ]+ if reads_events:+ event = f"the event {quoted(events)}" if len(events) == 1 else "the metric events"+ sentences.append(+ f"Check that your code sends {event} with the same ID as the exposure, which a server-side event or another "+ "SDK often doesn't."+ )+ if tables:+ sentences.append(+ f"Check that the data warehouse join key of {quoted(tables)} holds the same ID as the exposure."+ )+ if any(metric.has_filters for metric in metrics):+ sentences.append("Check that the metric filters match the data you send.")+ thresholds = [+ f'"{metric.name}" ({threshold:g})'+ for metric in metrics+ if (threshold := metric.definition.get("threshold")) is not None+ and metric.definition.get("metric_type") == "mean"+ ]+ if thresholds:+ sentences.append(f"Check that the exposed people or groups can reach the threshold of {joined(thresholds)}.")+ sentences.append("A rare outcome can also read 0 at this sample size.")+ return sentences+++def _severity(worded: list[_Worded], ctx: HealthContext) -> ExperimentHealthFindingSeverity:+ primary = [item for item in worded if item.metric.role == "primary"]+ # After the end, no change to the code can add events. A setting can still bring back the results.+ if primary and (not ctx.has_ended or any(item.cause in _FIXED_BY_A_SETTING for item in primary)):+ return ExperimentHealthFindingSeverity.WARNING+ return ExperimentHealthFindingSeverity.INFO+++def _evidence(worded: list[_Worded]) -> dict[str, str | int | None]:+ evidence: dict[str, str | int | None] = {}+ for number, item in enumerate(worded, start=1):+ evidence[f"metric_{number}"] = item.metric.name+ evidence[f"metric_{number}_role"] = item.metric.role+ evidence[f"metric_{number}_side"] = item.metric.ratio_side+ evidence[f"metric_{number}_exposures"] = item.metric.exposures+ evidence[f"metric_{number}_cause"] = item.cause+ return evidencediff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex f0ffc30500f9..75da26b2e8ec 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -1,4 +1,4 @@-from collections.abc import Mapping+from collections.abc import Iterable, Mappingfrom datetime import date, datetime, timedeltafrom enum import StrEnumfrom functools import reduce@@ -29,7 +29,7 @@ from posthog.models.activity_logging.activity_log import ActivityLogfrom posthog.models.team.team import Teamfrom products.cohorts.backend.models.cohort import Cohort-from products.event_definitions.backend.models import EventDefinition, PropertyDefinition, effective_project_id_expr+from products.event_definitions.backend.models import PropertyDefinition, effective_project_id_exprfrom products.experiments.backend.facade.contracts import (ExperimentHealthFlagChangeKind,ExperimentHealthMetricCompileErrorKind,@@ -47,7 +47,7 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (from products.experiments.backend.logic.result_stats import MetricResultStatsfrom products.experiments.backend.metric_resolution import MetricRole, saved_metric_link_role, saved_metric_linksfrom products.experiments.backend.metric_utils import get_default_metric_title-from products.experiments.backend.metric_validation import defined_event_names, extract_entity_nodes+from products.experiments.backend.metric_validation import defined_event_names, event_last_seen, extract_entity_nodesfrom products.experiments.backend.models.experiment import Experiment, ExperimentToSavedMetricfrom products.experiments.backend.running_time_target import RunningTimeTargetfrom products.feature_flags.backend.facade.edits import (@@ -361,6 +361,37 @@ class MetricCalculationFailure:exposures_path: ExposuresPath | None+RatioSide = Literal["numerator", "denominator"]+++@frozen+class ZeroMetric:+ """A metric, or one side of a ratio metric, whose latest calculation reads zero in every variant, which the results+ page shows as "Not enough data yet" at any sample size."""++ metric_id: str+ name: str+ role: MetricRole+ # The persons or groups of the result, across the variants.+ exposures: int+ # The side of a ratio that reads zero, each side its own `ZeroMetric`. None for the other metric types.+ ratio_side: RatioSide | None+ # The effective definition now, which holds the outlier handling and the retention window.+ definition: Mapping[str, Any]+ # The persons or groups that reached a step of a funnel before the last one. Zero for the other metric types.+ reached_earlier_step: int+ # The events that the zero reads by name, without the other side of a ratio.+ events: tuple[str, ...]+ # The data warehouse tables that the zero reads, without the other side of a ratio.+ warehouse_tables: tuple[str, ...]+ # Whether the zero reads an event node with a blank name, which matches no event. "All events" stores no name.+ has_blank_event: bool+ # Whether the zero reads a node with property filters.+ has_filters: bool+ # Of `events`, those that the project last received before launch, by the time of the last one.+ stopped_events: Mapping[str, datetime]++@frozenclass LatestCalculation:"""The latest finished calculation of the experiment's metrics, whose results and errors the results page@@ -375,6 +406,11 @@ class LatestCalculation:completed_count: int# Where the queries of those results read the exposures, when there is a failure to compare them with.completed_exposures_paths: frozenset[ExposuresPath]+ # Of those results, the ones that read zero in every variant.+ zero_metrics: tuple[ZeroMetric, ...] = ()+ # The hours from launch to the end of the data of this calculation. None when no calculation has finished. The+ # page can show a calculation from days ago until its refresh finishes.+ hours_of_data: float | None = None@frozen@@ -920,25 +956,28 @@ def _received(}+def ingestion_proof_events(experiment_events: Iterable[str]) -> set[str]:+ """The events whose `last_seen_at` can show that ingestion records the project: the flag calls and the events of+ the experiment."""+ return {DEFAULT_EXPOSURE_EVENT, EXPERIMENT_EXPOSURE_EVENT, *experiment_events}+++def ingestion_records_project(last_seen: Mapping[str, datetime]) -> bool:+ """Whether ingestion writes the project's definitions now, from `last_seen` of `ingestion_proof_events`. A missing+ or stale definition proves nothing without it. Ingestion can be set to skip a team, and then writes no definition at+ all. It refreshes `last_seen_at` of an event that it sees about once an hour. So a flag call or an event of the+ experiment seen within the margin shows that it records the project."""+ recent = timezone.now() - _DEFINITION_RECREATION_MARGIN+ return any(seen >= recent for seen in last_seen.values())++def _ingestion_records_project(experiment: Experiment, metric_events: tuple[MetricEvents, ...], exposure_events: ExposureEvents | None) -> bool:- """Whether ingestion writes the project's definitions now, which a missing definition needs to prove anything.- Ingestion can be set to skip a team, and then writes no definition at all. It refreshes `last_seen_at` of an event- that it sees about once an hour. So a flag call or an event of the experiment seen within the margin shows that it- records the project."""- names = {DEFAULT_EXPOSURE_EVENT, EXPERIMENT_EXPOSURE_EVENT, *(event for m in metric_events for event in m.events)}+ names = ingestion_proof_events(event for m in metric_events for event in m.events)if exposure_events is not None and exposure_events.custom_event is not None:names.add(exposure_events.custom_event)- return (- EventDefinition.objects.alias(effective_project_id=effective_project_id_expr())- .filter(- effective_project_id=experiment.team.project_id,- name__in=names,- last_seen_at__gte=timezone.now() - _DEFINITION_RECREATION_MARGIN,- )- .exists()- )+ return ingestion_records_project(event_last_seen(experiment.team.project_id, names))def _deletions_since(experiment: Experiment) -> datetime:diff --git a/products/experiments/backend/health/inputs_payload.py b/products/experiments/backend/health/inputs_payload.pyindex 50ab6b106bff..1045cd444823 100644--- a/products/experiments/backend/health/inputs_payload.py+++ b/products/experiments/backend/health/inputs_payload.py@@ -234,6 +234,17 @@ def _health_inputs_latest_calculation(calculation: LatestCalculation) -> dict[st}for failure in calculation.failures],+ "hours_of_data": calculation.hours_of_data,+ "zero_metrics": [+ {+ "name": metric.name,+ "role": metric.role,+ "exposures": metric.exposures,+ "ratio_side": metric.ratio_side,+ "stopped_events": list(metric.stopped_events),+ }+ for metric in calculation.zero_metrics+ ],}diff --git a/products/experiments/backend/health/latest_calculation.py b/products/experiments/backend/health/latest_calculation.pyindex 1445722070d5..acd589f3bae0 100644--- a/products/experiments/backend/health/latest_calculation.py+++ b/products/experiments/backend/health/latest_calculation.py@@ -1,11 +1,24 @@import re+from collections.abc import Mapping+from datetime import datetime, timedeltafrom typing import Any+from posthog.schema import ExperimentStatsBaseValidated++from posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exceptionfrom products.experiments.backend.facade.enums import ExposuresPath-from products.experiments.backend.health.context import LatestCalculation, MetricCalculationFailure+from products.experiments.backend.health.context import (+ LatestCalculation,+ MetricCalculationFailure,+ RatioSide,+ ZeroMetric,+ ingestion_proof_events,+ ingestion_records_project,+)from products.experiments.backend.health.metric_compiles import can_hold_evaluation_rules, without_available_fields+from products.experiments.backend.logic.result_stats import VariantResult, parse_stored_responsefrom products.experiments.backend.metric_calculation.config import (MetricCalculationConfig,build_calculation_configs,@@ -13,6 +26,7 @@ from products.experiments.backend.metric_calculation.config import (team_experiments_configs,)from products.experiments.backend.metric_calculation.results import MetricResultStore+from products.experiments.backend.metric_validation import event_last_seen, metric_nodesfrom products.experiments.backend.models.experiment import Experiment, ExperimentMetricResultfrom products.experiments.backend.recalculation import get_latest_recalculationfrom products.experiments.backend.temporal.models import RECALCULATION_WINDOW_TOO_LONG_ERROR_TYPE@@ -76,6 +90,110 @@ def _load_latest_calculation(experiment: Experiment) -> LatestCalculation:failures=failures,completed_count=len(completed),completed_exposures_paths=results.exposures_paths(completed) if failures else frozenset(),+ zero_metrics=_zero_metrics(+ experiment, [(configs[row.metric_uuid], row) for row in results.with_zero_baseline(completed)]+ ),+ hours_of_data=(run.query_to - experiment.start_date) / timedelta(hours=1)+ if run.query_to is not None and experiment.start_date is not None+ else None,+ )+++@frozen+class _ZeroResult:+ calculation_config: MetricCalculationConfig+ # None when the metric itself reads zero, rather than a side of a ratio.+ ratio_side: RatioSide | None+ stats: tuple[ExperimentStatsBaseValidated | VariantResult, ...]+ # The nodes that the zero reads, without the other side of a ratio.+ nodes: tuple[dict, ...]++ @property+ def events(self) -> tuple[str, ...]:+ names = (node.get("event") for node in self.nodes if node.get("kind") == "EventsNode")+ return tuple(dict.fromkeys(name for name in names if isinstance(name, str) and name.strip()))++ @property+ def warehouse_tables(self) -> tuple[str, ...]:+ names = (node.get("table_name") for node in self.nodes if node.get("kind") == "ExperimentDataWarehouseNode")+ return tuple(dict.fromkeys(name for name in names if isinstance(name, str) and name))+++def _zero_metrics(+ experiment: Experiment, completed: list[tuple[MetricCalculationConfig, ExperimentMetricResult]]+) -> tuple[ZeroMetric, ...]:+ zero = [result for config, row in completed for result in _zero_results(config, row)]+ if not zero:+ return ()+ events = {event for result in zero for event in result.events}+ last_seen = event_last_seen(experiment.team.project_id, ingestion_proof_events(events)) if events else {}+ # While ingestion skips the project, `last_seen_at` stays old for an event that still arrives.+ if not ingestion_records_project(last_seen):+ last_seen = {}+ return tuple(_zero_metric(experiment, result, last_seen) for result in zero)+++def _zero_results(calculation_config: MetricCalculationConfig, row: ExperimentMetricResult) -> tuple[_ZeroResult, ...]:+ """The metric, or each side of a ratio, that reads zero in every variant of the row."""+ response = parse_stored_response(row.result)+ if response is None or response.baseline is None:+ return ()+ stats = (response.baseline, *(response.variant_results or []))+ definition = calculation_config.definition+ # `sum_squares` tells a metric of zeros from values that cancel out.+ numerator_is_zero = all(stat.sum == 0 and stat.sum_squares == 0 for stat in stats)+ if definition.get("metric_type") != "ratio":+ if not numerator_is_zero:+ return ()+ nodes = tuple(metric_nodes(dict(definition)))+ return (_ZeroResult(calculation_config=calculation_config, ratio_side=None, stats=stats, nodes=nodes),)+ sides: list[RatioSide] = []+ if numerator_is_zero:+ sides.append("numerator")+ if all(_denominator_is_zero(stat) for stat in stats):+ sides.append("denominator")+ return tuple(+ _ZeroResult(+ calculation_config=calculation_config,+ ratio_side=side,+ stats=stats,+ nodes=(node,) if isinstance(node := definition.get(side), dict) else (),+ )+ for side in sides+ )+++def _denominator_is_zero(stat: ExperimentStatsBaseValidated | VariantResult) -> bool:+ # A variant without exposures gets zero stats that leave the denominator unset.+ if stat.number_of_samples == 0:+ return True+ return stat.denominator_sum == 0 and not stat.denominator_sum_squares+++def _zero_metric(experiment: Experiment, result: _ZeroResult, last_seen: Mapping[str, datetime]) -> ZeroMetric:+ calculation_config = result.calculation_config+ start_date = experiment.start_date+ return ZeroMetric(+ metric_id=calculation_config.metric_id,+ name=calculation_metric_name(experiment, calculation_config),+ role=calculation_config.role,+ exposures=sum(stat.number_of_samples for stat in result.stats),+ ratio_side=result.ratio_side,+ definition=calculation_config.definition,+ # `step_counts` holds the entities that reached each step of the metric, so its first count is the largest.+ reached_earlier_step=sum(stat.step_counts[0] for stat in result.stats if stat.step_counts),+ events=result.events,+ warehouse_tables=result.warehouse_tables,+ has_blank_event=any(+ node.get("kind") == "EventsNode" and node.get("event") is not None and not str(node["event"]).strip()+ for node in result.nodes+ ),+ has_filters=any(node.get("properties") for node in result.nodes),+ stopped_events={+ event: seen+ for event in result.events+ if (seen := last_seen.get(event)) is not None and start_date is not None and seen < start_date+ },)diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.pyindex 6861344571f6..dd0fe8703103 100644--- a/products/experiments/backend/health/registry.py+++ b/products/experiments/backend/health/registry.py@@ -22,6 +22,7 @@ from products.experiments.backend.health.checks.flag_change_findings import flagfrom products.experiments.backend.health.checks.flag_state import flag_statefrom products.experiments.backend.health.checks.flag_variants import flag_variants_mismatchfrom products.experiments.backend.health.checks.forced_variant import forced_variant_release_condition+from products.experiments.backend.health.checks.metric_all_zero import metric_all_zerofrom products.experiments.backend.health.checks.metric_calculation_failed import metric_calculation_failedfrom products.experiments.backend.health.checks.metric_cannot_compile import metric_cannot_compilefrom products.experiments.backend.health.checks.metric_event_not_ingested import metric_event_not_ingested@@ -104,6 +105,11 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (name="Failed metric calculations",run=metric_calculation_failed,),+ HealthCheck(+ key=ExperimentHealthCheckKey.METRIC_ALL_ZERO,+ name="Metrics at zero in every variant",+ run=metric_all_zero,+ ),HealthCheck(key=ExperimentHealthCheckKey.EXPOSURE_CRITERIA_CANNOT_MATCH,name="Exposure criteria that can't match",diff --git a/products/experiments/backend/health/test/test_metric_all_zero.py b/products/experiments/backend/health/test/test_metric_all_zero.pynew file mode 100644index 000000000000..1b022a93603b--- /dev/null+++ b/products/experiments/backend/health/test/test_metric_all_zero.py@@ -0,0 +1,561 @@+from dataclasses import replace+from datetime import UTC, datetime, timedelta+from typing import Any++from posthog.test.base import BaseTest+from unittest import TestCase++from django.utils import timezone++from parameterized import parameterized++from posthog.models.scoping import team_scope++from products.event_definitions.backend.models import EventDefinition+from products.experiments.backend.facade.api import get_experiment_health+from products.experiments.backend.facade.contracts import (+ ExperimentHealthFinding,+ ExperimentHealthFindingCode,+ ExperimentHealthFindingSeverity,+ ExperimentHealthReaderAccess,+)+from products.experiments.backend.health.checks.metric_all_zero import metric_all_zero+from products.experiments.backend.health.context import LatestCalculation, ZeroMetric+from products.experiments.backend.health.skips import CheckSkip+from products.experiments.backend.health.test.factories import ENDED, RUNNING_CONTEXT+from products.experiments.backend.metric_calculation.config import get_metric_calculation_config+from products.experiments.backend.metric_calculation.results import MetricResultStore+from products.experiments.backend.models.experiment import Experiment, ExperimentMetricsRecalculation+from products.feature_flags.backend.models.feature_flag import FeatureFlag++CODE = ExperimentHealthFindingCode.METRIC_ALL_ZERO+WARNING = ExperimentHealthFindingSeverity.WARNING+INFO = ExperimentHealthFindingSeverity.INFO+NO_QUERIES = ExperimentHealthReaderAccess(+ can_read_flag_history=False, can_query_events=False, denied_event_properties=frozenset()+)+LAST_ARRIVED = datetime(2026, 10, 1, 9, 30, tzinfo=UTC)+MEAN = {"metric_type": "mean"}+BOUNDED = {"metric_type": "mean", "upper_bound_percentile": 0.99}+++def _metric(**changes: Any) -> ZeroMetric:+ fields: dict[str, Any] = {+ "metric_id": "metric",+ "name": "Purchases",+ "role": "primary",+ "exposures": 20000,+ "ratio_side": None,+ "definition": MEAN,+ "reached_earlier_step": 0,+ "events": ("purchase",),+ "warehouse_tables": (),+ "has_blank_event": False,+ "has_filters": False,+ "stopped_events": {},+ }+ return ZeroMetric(**{**fields, **changes})+++def _outcome(+ *metrics: ZeroMetric, hours_of_data: float = 96, **changes: Any+) -> ExperimentHealthFinding | CheckSkip | None:+ calculation = LatestCalculation(+ metric_count=len(metrics) or 1,+ finished_at=LAST_ARRIVED,+ failures=(),+ completed_count=len(metrics),+ completed_exposures_paths=frozenset(),+ zero_metrics=metrics,+ hours_of_data=hours_of_data,+ )+ return metric_all_zero(replace(RUNNING_CONTEXT, latest_calculation=calculation, **changes))+++def _variant(+ *,+ samples: int,+ total: float,+ squares: float | None = None,+ denominator: float | None = None,+ denominator_squares: float | None = None,+) -> dict:+ return {+ "key": "test",+ "method": "bayesian",+ "number_of_samples": samples,+ "sum": total,+ "sum_squares": total**2 if squares is None else squares,+ "denominator_sum": denominator,+ "denominator_sum_squares": denominator_squares,+ }+++def _result(+ *,+ samples: int,+ total: float,+ squares: float | None = None,+ denominator: float | None = None,+ denominator_squares: float | None = None,+) -> dict:+ variant = _variant(+ samples=samples,+ total=total,+ squares=squares,+ denominator=denominator,+ denominator_squares=denominator_squares,+ )+ baseline = {key: value for key, value in variant.items() if key != "method"}+ return {"baseline": {**baseline, "key": "control"}, "variant_results": [variant]}+++class TestMetricAllZero(TestCase):+ @parameterized.expand(+ [+ ("reads_zero_with_the_usual_causes", _metric(), {}, (None, WARNING, "same ID as the exposure")),+ (+ "the_filters_of_the_metric",+ _metric(has_filters=True),+ {},+ (None, WARNING, "Check that the metric filters match"),+ ),+ (+ "an_earlier_funnel_step",+ _metric(reached_earlier_step=900),+ {},+ (None, WARNING, "900 reached an earlier step, but nobody finished the funnel."),+ ),+ ("a_side_of_a_ratio", _metric(ratio_side="numerator"), {}, (None, WARNING, "The numerator of the primary")),+ (+ "an_event_with_an_empty_name",+ _metric(has_blank_event=True, events=()),+ {},+ ("blank_event", WARNING, "matches no events"),+ ),+ (+ "an_event_that_stopped_before_launch",+ _metric(stopped_events={"purchase": LAST_ARRIVED}),+ {},+ ("event_stopped", WARNING, 'The event "purchase" last arrived on October 1, 2026 at 11:30 AM CEST'),+ ),+ (+ "an_outlier_bound_that_counts_zeros",+ _metric(definition=BOUNDED),+ {},+ ("outlier_bound", WARNING, "When at most 199 of them have a value, the bound is 0"),+ ),+ (+ "a_bound_that_can_hide_one_value",+ _metric(role="secondary", definition=BOUNDED, exposures=101),+ {},+ ("outlier_bound", INFO, "When at most 1 of them have a value"),+ ),+ (+ "a_threshold_nobody_reaches",+ _metric(definition={"metric_type": "mean", "threshold": 50}),+ {},+ (None, WARNING, 'can reach the threshold of "Purchases" (50).'),+ ),+ (+ "a_data_warehouse_table",+ _metric(events=(), warehouse_tables=("charges",)),+ {},+ (None, WARNING, "counts a warehouse row only for the person or group that was exposed"),+ ),+ (+ "after_the_end_nobody_can_add_events",+ _metric(),+ ENDED,+ (None, INFO, "same ID as the exposure"),+ ),+ (+ "after_the_end_a_stopped_event_stays_info",+ _metric(stopped_events={"purchase": LAST_ARRIVED}),+ ENDED,+ ("event_stopped", INFO, "before the experiment started"),+ ),+ (+ "after_the_end_a_setting_still_brings_back_the_results",+ _metric(definition=BOUNDED),+ ENDED,+ ("outlier_bound", WARNING, "Turn on"),+ ),+ (+ "secondary_metrics_report_an_outlier_bound",+ _metric(role="secondary", definition=BOUNDED),+ {},+ ("outlier_bound", INFO, 'The secondary metric "Purchases"'),+ ),+ (+ "secondary_metrics_report_an_empty_event_name",+ _metric(role="secondary", has_blank_event=True, events=(), exposures=100),+ {},+ ("blank_event", INFO, "matches no events"),+ ),+ (+ "an_empty_event_name_comes_before_a_bound",+ _metric(role="secondary", definition=BOUNDED, has_blank_event=True),+ {},+ ("blank_event", INFO, "matches no events"),+ ),+ ]+ )+ def test_reports(+ self, _name: str, metric: ZeroMetric, changes: dict[str, Any], expected: tuple[str | None, Any, str]+ ) -> None:+ outcome = _outcome(metric, **changes)+ assert isinstance(outcome, ExperimentHealthFinding)+ subcode, severity, words = expected+ self.assertEqual((outcome.code, outcome.subcode, outcome.severity), (CODE, subcode, severity))+ self.assertIn(words, outcome.detail)++ @parameterized.expand(+ [+ ("under_the_exposures_floor", _metric(exposures=999), 96),+ ("under_the_data_floor", _metric(), 71),+ ("a_secondary_metric_without_a_bound", _metric(role="secondary"), 96),+ ("an_empty_event_name_under_its_exposures_floor", _metric(has_blank_event=True, exposures=99), 96),+ ("an_empty_event_name_in_the_first_hours", _metric(has_blank_event=True), 11),+ ("a_bound_under_its_exposures_floor", _metric(definition=BOUNDED, exposures=99), 96),+ ("a_bound_in_the_first_hours", _metric(definition=BOUNDED, exposures=800), 11),+ (+ "a_bound_that_ignores_zeros_under_the_floor",+ _metric(definition={**BOUNDED, "ignore_zeros": True}, exposures=800),+ 96,+ ),+ (+ "a_bound_on_the_largest_value",+ _metric(definition={**BOUNDED, "upper_bound_percentile": 1}, exposures=800),+ 96,+ ),+ (+ "a_bound_per_breakdown_group",+ _metric(+ role="secondary",+ definition={+ **BOUNDED,+ "breakdownFilter": {"breakdowns": [{"property": "$browser", "type": "event"}]},+ },+ ),+ 96,+ ),+ (+ "a_bound_that_picks_the_largest_of_its_exposures",+ _metric(role="secondary", definition=BOUNDED, exposures=100),+ 96,+ ),+ (+ "a_stopped_event_on_a_secondary_metric",+ _metric(role="secondary", definition=BOUNDED, stopped_events={"purchase": LAST_ARRIVED}),+ 96,+ ),+ (+ "a_retention_window_that_starts_later",+ _metric(+ definition={+ "metric_type": "retention",+ "retention_window_start": 7,+ "retention_window_end": 8,+ "retention_window_unit": "day",+ }+ ),+ 96,+ ),+ ]+ )+ def test_stays_quiet(self, _name: str, metric: ZeroMetric, hours_of_data: float) -> None:+ self.assertIsNone(_outcome(metric, hours_of_data=hours_of_data))++ def test_leaves_an_event_that_never_arrived_to_the_check_that_reports_it(self) -> None:+ self.assertIsNone(_outcome(_metric(), events_received={"purchase": False}))++ # `metric_event_not_ingested` stops at the end, so this check is the only one left to explain the zero.+ outcome = _outcome(_metric(), events_received={"purchase": False}, **ENDED)++ assert isinstance(outcome, ExperimentHealthFinding)+ self.assertEqual(outcome.severity, ExperimentHealthFindingSeverity.INFO)++ @parameterized.expand([("before_the_gate", 119, False), ("at_the_gate", 120, True)])+ def test_a_retention_metric_counts_its_hours_from_the_window_start(+ self, _name: str, hours_of_data: float, reports: bool+ ) -> None:+ definition = {+ "metric_type": "retention",+ "retention_window_start": 2,+ "retention_window_end": 3,+ "retention_window_unit": "day",+ }++ outcome = _outcome(_metric(definition=definition), hours_of_data=hours_of_data)++ self.assertEqual(isinstance(outcome, ExperimentHealthFinding), reports)+ if isinstance(outcome, ExperimentHealthFinding):+ self.assertIn("reads 0 for all 20,000 users who did its start event", outcome.detail)++ def test_the_outlier_bound_on_a_side_of_a_ratio(self) -> None:+ definition = {"metric_type": "ratio", "denominator_outlier_handling": {"upper_bound_percentile": 0.95}}+ metric = _metric(role="secondary", ratio_side="denominator", definition=definition, exposures=800)++ outcome = _outcome(metric, hours_of_data=18)++ assert isinstance(outcome, ExperimentHealthFinding)+ self.assertEqual(outcome.subcode, "outlier_bound")+ self.assertIn('The denominator of the secondary metric "Purchases" reads 0', outcome.detail)+ self.assertIn("When at most 39 of them have a value", outcome.detail)++ def test_the_outlier_bound_on_the_other_side_of_a_ratio_is_no_cause(self) -> None:+ definition = {"metric_type": "ratio", "denominator_outlier_handling": {"upper_bound_percentile": 0.95}}++ self.assertIsNone(_outcome(_metric(role="secondary", ratio_side="numerator", definition=definition)))++ def test_lists_the_primary_metrics_with_a_cause_first(self) -> None:+ outcome = _outcome(+ _metric(metric_id="a", name="Secondary", role="secondary", definition=BOUNDED),+ _metric(metric_id="b", name="Unexplained"),+ _metric(metric_id="c", name="Bounded", definition=BOUNDED),+ )++ assert isinstance(outcome, ExperimentHealthFinding)+ self.assertEqual(outcome.title, "3 metrics read zero in every variant")+ self.assertEqual(outcome.subcode, "outlier_bound")+ self.assertEqual(+ [outcome.evidence[f"metric_{number}"] for number in (1, 2, 3)], ["Bounded", "Unexplained", "Secondary"]+ )+ self.assertEqual(outcome.evidence["metric_2_exposures"], 20000)+ self.assertIsNone(outcome.evidence["metric_2_cause"])++ def test_says_the_usual_causes_once(self) -> None:+ outcome = _outcome(+ _metric(metric_id="a", name="Purchases", events=("purchase",)),+ _metric(metric_id="b", name="Refunds", events=("refund",), has_filters=True),+ )++ assert isinstance(outcome, ExperimentHealthFinding)+ self.assertEqual(outcome.detail.count("same ID as the exposure"), 1)+ self.assertIn("sends the metric events with", outcome.detail)+ self.assertIn("Check that the metric filters match", outcome.detail)+ self.assertLess(outcome.detail.index('"Refunds" reads 0'), outcome.detail.index("same ID as the exposure"))++ @parameterized.expand(+ [+ ("archived", {"archived": True}, "Doesn't run on archived experiments"),+ ("draft", {"is_launched": False}, "Runs after launch"),+ ]+ )+ def test_skips(self, _name: str, changes: dict[str, Any], reason: str) -> None:+ outcome = _outcome(_metric(), **changes)+ assert isinstance(outcome, CheckSkip)+ self.assertEqual(outcome.reason, reason)++ def test_skips_without_a_finished_calculation(self) -> None:+ outcome = _outcome(_metric(), hours_of_data=None) # type: ignore[arg-type]+ assert isinstance(outcome, CheckSkip)+ self.assertEqual(outcome.reason, "Runs once a calculation of the metrics has finished")+++class TestMetricAllZeroFromStoredRuns(BaseTest):+ def _experiment(self, metrics: list[dict[str, Any]], *, started_days_ago: int = 5) -> Experiment:+ flag = FeatureFlag.objects.create(+ team=self.team,+ key="checkout-test",+ created_by=self.user,+ filters={+ "groups": [{"properties": [], "rollout_percentage": 100}],+ "multivariate": {+ "variants": [+ {"key": "control", "rollout_percentage": 50},+ {"key": "test", "rollout_percentage": 50},+ ]+ },+ },+ )+ return Experiment.objects.create(+ team=self.team,+ name="Checkout",+ feature_flag=flag,+ start_date=timezone.now() - timedelta(days=started_days_ago),+ metrics=metrics,+ )++ def _store(+ self, experiment: Experiment, results: dict[str, dict | None], *, query_to: datetime | None = None+ ) -> None:+ query_to = query_to or timezone.now() - timedelta(hours=1)+ with team_scope(self.team.id):+ run = ExperimentMetricsRecalculation.objects.create(+ team=self.team,+ experiment=experiment,+ status=ExperimentMetricsRecalculation.Status.IN_PROGRESS,+ metric_uuids=list(results),+ query_to=query_to,+ )+ store = MetricResultStore(experiment_id=experiment.id)+ for metric_uuid, result in results.items():+ calculation_config = get_metric_calculation_config(experiment, metric_uuid)+ assert calculation_config is not None and experiment.start_date is not None+ store.record_run_result(+ str(run.id),+ calculation_config,+ window=query_to,+ query_from=experiment.start_date,+ result=result or {},+ query_id=None,+ )+ ExperimentMetricsRecalculation.objects.filter(id=run.id).update(+ status=ExperimentMetricsRecalculation.Status.COMPLETED, completed_at=query_to+ )++ def _findings(self, experiment: Experiment) -> list[ExperimentHealthFinding]:+ health = get_experiment_health(+ team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES, user=self.user+ )+ return [finding for finding in health.findings if finding.code == CODE]++ def _mean(self, uuid: str, **changes: Any) -> dict[str, Any]:+ return {+ "kind": "ExperimentMetric",+ "metric_type": "mean",+ "uuid": uuid,+ "name": f"Metric {uuid}",+ "source": {"kind": "EventsNode", "event": "purchase"},+ **changes,+ }++ def _ratio(self, uuid: str) -> dict[str, Any]:+ return {+ "kind": "ExperimentMetric",+ "metric_type": "ratio",+ "uuid": uuid,+ "name": f"Metric {uuid}",+ "numerator": {"kind": "EventsNode", "event": "viewed"},+ "denominator": {"kind": "EventsNode", "event": "added"},+ }++ def _seen(self, event: str, last_seen_at: datetime) -> None:+ EventDefinition.objects.create(team=self.team, project=self.team.project, name=event, last_seen_at=last_seen_at)++ @parameterized.expand(+ [+ ("zero_in_every_variant", _result(samples=1000, total=0), True),+ ("a_zero_stored_as_a_float", _result(samples=1000, total=0.0), True),+ (+ "a_value_in_one_variant",+ {**_result(samples=1000, total=0), "variant_results": [_variant(samples=1000, total=1)]},+ False,+ ),+ ("values_that_cancel_out", _result(samples=1000, total=0, squares=4), False),+ (+ "a_sum_that_overflowed",+ {+ **_result(samples=1000, total=0),+ "baseline": {+ "key": "control",+ "number_of_samples": 1000,+ "sum": None,+ "sum_squares": None,+ },+ },+ False,+ ),+ ]+ )+ def test_reads_the_zero_from_the_run_the_page_shows(self, _name: str, result: dict, reports: bool) -> None:+ experiment = self._experiment([self._mean("primary")])+ self._seen("purchase", timezone.now())+ self._store(experiment, {"primary": result})++ self.assertEqual(bool(self._findings(experiment)), reports)++ @parameterized.expand(+ [+ ("numerator", _result(samples=1000, total=0, denominator=20), ["The numerator of the primary"]),+ ("denominator", _result(samples=1000, total=20, denominator=0), ["The denominator of the primary"]),+ (+ "both_sides",+ _result(samples=1000, total=0, denominator=0),+ ["The numerator of the primary", "The denominator of the primary"],+ ),+ (+ "a_denominator_with_a_variant_without_exposures",+ {**_result(samples=1000, total=20, denominator=0), "variant_results": [_variant(samples=0, total=0)]},+ ["The denominator of the primary"],+ ),+ ]+ )+ def test_reads_the_side_of_a_ratio(self, _name: str, result: dict, words: list[str]) -> None:+ experiment = self._experiment([self._ratio("primary")])+ self._store(experiment, {"primary": result})++ [finding] = self._findings(experiment)++ self.assertEqual(finding.title, "A metric reads zero in every variant")+ for side in words:+ self.assertIn(side, finding.detail)++ def test_a_denominator_that_cancels_out_is_no_zero(self) -> None:+ experiment = self._experiment([self._ratio("primary")])+ self._store(experiment, {"primary": _result(samples=1000, total=20, denominator=0, denominator_squares=8)})++ self.assertEqual(self._findings(experiment), [])++ def test_names_only_the_event_of_the_side_that_reads_zero(self) -> None:+ experiment = self._experiment([self._ratio("primary")])+ assert experiment.start_date is not None+ self._seen("viewed", timezone.now())+ self._seen("added", experiment.start_date - timedelta(days=1))+ self._store(experiment, {"primary": _result(samples=1000, total=0, denominator=20)})++ [finding] = self._findings(experiment)++ self.assertIsNone(finding.subcode)+ self.assertIn('the event "viewed"', finding.detail)++ @parameterized.expand(+ [("ingestion_records_the_project", True, "event_stopped"), ("ingestion_skips_it", False, None)]+ )+ def test_names_an_event_that_stopped_before_launch(+ self, _name: str, flag_called_recently: bool, subcode: str | None+ ) -> None:+ experiment = self._experiment([self._mean("primary")])+ assert experiment.start_date is not None+ self._seen("purchase", experiment.start_date - timedelta(days=1))+ if flag_called_recently:+ self._seen("$feature_flag_called", timezone.now())+ self._store(experiment, {"primary": _result(samples=1000, total=0)})++ [finding] = self._findings(experiment)++ self.assertEqual(finding.subcode, subcode)++ def test_an_event_that_arrived_after_launch_did_not_stop(self) -> None:+ experiment = self._experiment([self._mean("primary")])+ assert experiment.start_date is not None+ self._seen("purchase", experiment.start_date + timedelta(hours=1))+ self._store(experiment, {"primary": _result(samples=1000, total=0)})++ [finding] = self._findings(experiment)++ self.assertIsNone(finding.subcode)++ @parameterized.expand([("under_the_data_floor", 71, False), ("at_the_data_floor", 72, True)])+ def test_measures_the_data_at_the_calculation_that_the_page_shows(+ self, _name: str, hours_of_data: int, reports: bool+ ) -> None:+ experiment = self._experiment([self._mean("primary")], started_days_ago=4)+ assert experiment.start_date is not None+ self._store(+ experiment,+ {"primary": _result(samples=1000, total=0)},+ query_to=experiment.start_date + timedelta(hours=hours_of_data),+ )++ self.assertEqual(bool(self._findings(experiment)), reports)++ def test_a_blank_event_name_matches_no_events(self) -> None:+ experiment = self._experiment([self._mean("primary", source={"kind": "EventsNode", "event": ""})])+ self._store(experiment, {"primary": _result(samples=1000, total=0)})++ [finding] = self._findings(experiment)++ self.assertEqual(finding.subcode, "blank_event")diff --git a/products/experiments/backend/hogql_queries/base_query_utils.py b/products/experiments/backend/hogql_queries/base_query_utils.pyindex ed3935b127f4..707d86af8ef6 100644--- a/products/experiments/backend/hogql_queries/base_query_utils.py+++ b/products/experiments/backend/hogql_queries/base_query_utils.py@@ -231,6 +231,20 @@ def conversion_window_to_seconds(conversion_window: int, conversion_window_unit:return conversion_window * multipliers[conversion_window_unit]+def winsorization_bound_expr(level: float, value: str, *, ignore_zeros: bool = False) -> ast.Expr:+ """The outlier bound at the percentile `level` of `value` across the entities. `value` is an internal column name,+ never user input, because it is interpolated into the expression. `quantile_exact_position` mirrors the value that+ this function picks, so a change of the aggregate here must change that function too."""+ over = f"if({value} != 0, {value}, null)" if ignore_zeros else value+ return parse_expr(f"quantileExact({{level}})({over})", placeholders={"level": ast.Constant(value=level)})+++def quantile_exact_position(level: float, count: int) -> int:+ """The position, among `count` values in ascending order, of the value that `quantileExact(level)` returns."""+ # ClickHouse truncates `level * count` computed in double precision, which Python's float multiplication matches.+ return int(level * count) if level < 1 else count - 1++def analysis_window_end(end_date: Optional[datetime], as_of: datetime) -> datetime:"""Upper edge of an experiment's analysis window, evaluated at ``as_of``.diff --git a/products/experiments/backend/hogql_queries/experiment_mean_query_builder.py b/products/experiments/backend/hogql_queries/experiment_mean_query_builder.pyindex 411002710605..3437d67302b1 100644--- a/products/experiments/backend/hogql_queries/experiment_mean_query_builder.py+++ b/products/experiments/backend/hogql_queries/experiment_mean_query_builder.py@@ -8,6 +8,7 @@ from posthog.hogql.parser import parse_expr, parse_selectfrom products.experiments.backend.hogql_queries.base_query_utils import (is_session_property_metric,validate_session_property,+ winsorization_bound_expr,)from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureQueryBuilderfrom products.experiments.backend.hogql_queries.metric_source import MetricSourceInfo@@ -346,24 +347,16 @@ class MeanQueryBuilder:assert isinstance(self._b.metric, ExperimentMeanMetric)if self._b.metric.lower_bound_percentile is not None:- lower_bound_expr = parse_expr(- "quantileExact({level})(entity_metrics.value)",- placeholders={"level": ast.Constant(value=self._b.metric.lower_bound_percentile)},- )+ lower_bound_expr = winsorization_bound_expr(self._b.metric.lower_bound_percentile, "entity_metrics.value")else:lower_bound_expr = parse_expr("min(entity_metrics.value)")if self._b.metric.upper_bound_percentile is not None:- if getattr(self._b.metric, "ignore_zeros", False):- upper_bound_expr = parse_expr(- "quantileExact({level})(if(entity_metrics.value != 0, entity_metrics.value, null))",- placeholders={"level": ast.Constant(value=self._b.metric.upper_bound_percentile)},- )- else:- upper_bound_expr = parse_expr(- "quantileExact({level})(entity_metrics.value)",- placeholders={"level": ast.Constant(value=self._b.metric.upper_bound_percentile)},- )+ upper_bound_expr = winsorization_bound_expr(+ self._b.metric.upper_bound_percentile,+ "entity_metrics.value",+ ignore_zeros=bool(getattr(self._b.metric, "ignore_zeros", False)),+ )else:upper_bound_expr = parse_expr("max(entity_metrics.value)")diff --git a/products/experiments/backend/hogql_queries/experiment_ratio_query_builder.py b/products/experiments/backend/hogql_queries/experiment_ratio_query_builder.pyindex 9fe9454640e6..d89a60d0e862 100644--- a/products/experiments/backend/hogql_queries/experiment_ratio_query_builder.py+++ b/products/experiments/backend/hogql_queries/experiment_ratio_query_builder.py@@ -14,6 +14,7 @@ from posthog.hogql.parser import parse_expr, parse_selectfrom posthog.dataclasses import frozen+from products.experiments.backend.hogql_queries.base_query_utils import winsorization_bound_exprfrom products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureQueryBuilderfrom products.experiments.backend.hogql_queries.metric_source import MetricSourceInfofrom products.experiments.backend.hogql_queries.metric_source_precompute import (@@ -154,24 +155,14 @@ class RatioQueryBuilder:ignore_zeros = bool(outlier_handling.ignore_zeros) if outlier_handling else Falseif lower_pct is not None:- lower_bound_expr = parse_expr(- f"quantileExact({{level}})(entity_metrics.{value_field})",- placeholders={"level": ast.Constant(value=lower_pct)},- )+ lower_bound_expr = winsorization_bound_expr(lower_pct, f"entity_metrics.{value_field}")else:lower_bound_expr = parse_expr(f"min(entity_metrics.{value_field})")if upper_pct is not None:- if ignore_zeros:- upper_bound_expr = parse_expr(- f"quantileExact({{level}})(if(entity_metrics.{value_field} != 0, entity_metrics.{value_field}, null))",- placeholders={"level": ast.Constant(value=upper_pct)},- )- else:- upper_bound_expr = parse_expr(- f"quantileExact({{level}})(entity_metrics.{value_field})",- placeholders={"level": ast.Constant(value=upper_pct)},- )+ upper_bound_expr = winsorization_bound_expr(+ upper_pct, f"entity_metrics.{value_field}", ignore_zeros=ignore_zeros+ )else:upper_bound_expr = parse_expr(f"max(entity_metrics.{value_field})")diff --git a/products/experiments/backend/hogql_queries/test/test_base_query_utils.py b/products/experiments/backend/hogql_queries/test/test_base_query_utils.pynew file mode 100644index 000000000000..75a83ed233f1--- /dev/null+++ b/products/experiments/backend/hogql_queries/test/test_base_query_utils.py@@ -0,0 +1,27 @@+from posthog.test.base import BaseTest, ClickhouseTestMixin++from parameterized import parameterized++from posthog.hogql.parser import parse_select+from posthog.hogql.query import execute_hogql_query++from products.experiments.backend.hogql_queries.base_query_utils import (+ quantile_exact_position,+ winsorization_bound_expr,+)+++class TestWinsorizationBound(ClickhouseTestMixin, BaseTest):+ @parameterized.expand([(0.99, 100), (0.99, 101), (0.95, 766), (0.999, 600), (0.5, 7)])+ def test_quantile_exact_position_tells_when_the_upper_bound_is_zero(self, level: float, count: int) -> None:+ hidden = count - quantile_exact_position(level, count) - 1++ bounds = []+ for with_value in (hidden, hidden + 1):+ query = parse_select(+ f"SELECT {{bound}} FROM (SELECT if(number >= {count - with_value}, 5, 0) AS value FROM numbers({count}))",+ placeholders={"bound": winsorization_bound_expr(level, "value")},+ )+ bounds.append(execute_hogql_query(query, team=self.team).results[0][0])++ self.assertEqual(bounds, [0, 5])diff --git a/products/experiments/backend/logic/result_stats.py b/products/experiments/backend/logic/result_stats.pyindex a5d758006c5c..829c6bf5fdde 100644--- a/products/experiments/backend/logic/result_stats.py+++ b/products/experiments/backend/logic/result_stats.py@@ -14,6 +14,7 @@ from posthog.schema import (from posthog.dataclasses import frozenStatsMethod = Literal["bayesian", "frequentist"]+VariantResult = ExperimentVariantResultBayesian | ExperimentVariantResultFrequentist_INTERVAL_KEYS = ("credible_interval", "confidence_interval")@@ -121,7 +122,7 @@ class VariantStats:significant: bool | None@classmethod- def from_schema(cls, variant: ExperimentVariantResultBayesian | ExperimentVariantResultFrequentist) -> Self:+ def from_schema(cls, variant: VariantResult) -> Self:chance_to_win: float | None = Nonep_value: float | None = Noneif isinstance(variant, ExperimentVariantResultBayesian):diff --git a/products/experiments/backend/metric_calculation/results.py b/products/experiments/backend/metric_calculation/results.pyindex 9b904bb60f6e..d5a8a7d4c8e9 100644--- a/products/experiments/backend/metric_calculation/results.py+++ b/products/experiments/backend/metric_calculation/results.py@@ -141,6 +141,17 @@ class MetricResultStore:rows = rows.defer("result")return list(_newest_write_first(rows, "metric_uuid").distinct("metric_uuid"))+ def with_zero_baseline(self, rows: Iterable[ExperimentMetricResult]) -> list[ExperimentMetricResult]:+ """Those of these rows whose baseline reads zero on the metric or on its denominator, with `result`, in the order+ of `rows`. The database applies the filter, so the rows with a value don't load `result`."""+ by_pk = {row.pk: row for row in rows}+ loaded = ExperimentMetricResult.objects.filter(+ Q(result__baseline__sum=0) | Q(result__baseline__denominator_sum=0),+ experiment_id=self.experiment_id,+ pk__in=by_pk,+ ).in_bulk()+ return [loaded[pk] for pk in by_pk if pk in loaded]+def exposures_paths(self, rows: Iterable[ExperimentMetricResult]) -> frozenset[ExposuresPath]:"""Where the queries of these rows read the exposures. It reads one key of `result`, so the rows of `for_run` canleave `result` unloaded. A result without the key adds nothing."""diff --git a/products/experiments/backend/metric_validation.py b/products/experiments/backend/metric_validation.pyindex 300fcf80955c..2b13a6728c28 100644--- a/products/experiments/backend/metric_validation.py+++ b/products/experiments/backend/metric_validation.py@@ -8,8 +8,11 @@ event exists) stay with the callers."""from collections.abc import Collection+from datetime import datetimefrom typing import Any+from django.db.models import QuerySet+import pydanticimport structlogfrom rest_framework.exceptions import PermissionDenied, ValidationError@@ -180,6 +183,28 @@ def validate_saved_metric_link_overrides(metadata: dict, *, error_prefix: str) -raise ValidationError(f"{error_prefix}{_pydantic_error_message(e)}")+def metric_nodes(metric: dict) -> list[dict]:+ """The event, action and warehouse nodes that a stored metric definition reads."""+ nodes: list[dict] = []+ metric_type = metric.get("metric_type")+ if metric_type == "mean":+ if source := metric.get("source"):+ nodes.append(source)+ elif metric_type == "funnel":+ nodes.extend(metric.get("series") or [])+ elif metric_type == "ratio":+ if num := metric.get("numerator"):+ nodes.append(num)+ if den := metric.get("denominator"):+ nodes.append(den)+ elif metric_type == "retention":+ if se := metric.get("start_event"):+ nodes.append(se)+ if ce := metric.get("completion_event"):+ nodes.append(ce)+ return nodes++def extract_entity_nodes(metrics: list[dict] | None) -> tuple[set[str], set[int]]:"""Extract event names and action IDs from all EventsNode/ActionsNode refs in metrics."""event_names: set[str] = set()@@ -188,23 +213,7 @@ def extract_entity_nodes(metrics: list[dict] | None) -> tuple[set[str], set[int]return event_names, action_idsfor metric in metrics:- nodes: list[dict] = []- metric_type = metric.get("metric_type")- if metric_type == "mean":- if source := metric.get("source"):- nodes.append(source)- elif metric_type == "funnel":- nodes.extend(metric.get("series") or [])- elif metric_type == "ratio":- if num := metric.get("numerator"):- nodes.append(num)- if den := metric.get("denominator"):- nodes.append(den)- elif metric_type == "retention":- if se := metric.get("start_event"):- nodes.append(se)- if ce := metric.get("completion_event"):- nodes.append(ce)+ nodes = metric_nodes(metric)for node in nodes:kind = node.get("kind")@@ -243,16 +252,29 @@ def defined_event_names(project_id: int, event_names: Collection[str], *, seen_o"""if not event_names:return set()+ definitions = _event_definitions(project_id, event_names)+ if seen_only:+ definitions = definitions.filter(last_seen_at__isnull=False)+ return set(definitions.values_list("name", flat=True))+++def event_last_seen(project_id: int, event_names: Collection[str]) -> dict[str, datetime]:+ """When ingestion last saw each of `event_names` in the project, in the scope of `defined_event_names`. A name+ without a definition, or with one that ingestion hasn't seen, is absent."""+ if not event_names:+ return {}+ definitions = _event_definitions(project_id, event_names).values_list("name", "last_seen_at")+ return {name: last_seen for name, last_seen in definitions if last_seen is not None}+++def _event_definitions(project_id: int, event_names: Collection[str]) -> QuerySet[EventDefinition]:# `COALESCE(project_id, team_id)` falls back to `team_id` (not the caller's team id) on purpose: legacy# EventDefinitions (project_id IS NULL) belong to the *primary* team, and primary_team.id == project.id by# convention. This mirrors the picker SQL in posthog/api/event_definition.py so sibling teams can validate# against legacy primary-team events the picker shows.- definitions = EventDefinition.objects.alias(effective_project_id=effective_project_id_expr()).filter(+ return EventDefinition.objects.alias(effective_project_id=effective_project_id_expr()).filter(effective_project_id=project_id, name__in=event_names)- if seen_only:- definitions = definitions.filter(last_seen_at__isnull=False)- return set(definitions.values_list("name", flat=True))def validate_metric_action_ids(diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.pyindex ac4eb401a334..410777a43cdf 100644--- a/products/experiments/backend/presentation/serializers.py+++ b/products/experiments/backend/presentation/serializers.py@@ -3484,6 +3484,21 @@ class ExperimentHealthInputsMetricCalculationFailureSerializer(serializers.Seria)+class ExperimentHealthInputsZeroMetricSerializer(serializers.Serializer):+ name = serializers.CharField(help_text="The name of the metric, as the results page shows it.")+ role = serializers.CharField(help_text="Whether the metric is a primary or a secondary metric of the experiment.")+ exposures = serializers.IntegerField(help_text="The persons or groups of the result, across the variants.")+ ratio_side = serializers.CharField(+ allow_null=True,+ help_text="The side of a ratio that reads zero. Each side that reads zero has its own entry. Null for the other "+ "metric types.",+ )+ stopped_events = serializers.ListField(+ child=serializers.CharField(),+ help_text="The events of the metric that the project last received before launch.",+ )++class ExperimentHealthInputsLatestCalculationSerializer(serializers.Serializer):finished_at = serializers.DateTimeField(allow_null=True,@@ -3492,6 +3507,13 @@ class ExperimentHealthInputsLatestCalculationSerializer(serializers.Serializer):failures = ExperimentHealthInputsMetricCalculationFailureSerializer(many=True, help_text="The metrics whose latest calculation failed, with the error that the results page shows.")+ hours_of_data = serializers.FloatField(+ allow_null=True,+ help_text="The hours from launch to the end of the data of this calculation. Null when none has finished.",+ )+ zero_metrics = ExperimentHealthInputsZeroMetricSerializer(+ many=True, help_text="The metrics that read zero in every variant in this calculation."+ )class ExperimentHealthInputsReaderAccessSerializer(serializers.Serializer):diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.pyindex 26acf3d3b6bf..c53b76876d51 100644--- a/products/experiments/backend/presentation/views.py+++ b/products/experiments/backend/presentation/views.py@@ -1032,6 +1032,26 @@ class EnterpriseExperimentsViewSet(then names the filters. A query that read precomputed exposures doesn't apply those filters,so it can't get `exposure_criteria`. It is null otherwise.+ `metric_all_zero` reports a metric that reads zero in every variant, from the same latest+ finished calculation, so the results page shows "Not enough data yet" at any sample size.+ Each side of a ratio that is zero in every variant counts on its own. It reads the stored+ result and the project's event definitions, and no events. A primary metric needs 1,000+ exposures in the result and 72 hours of data, because most zeros before that are outcomes+ that take time. A retention metric counts those hours from the start of its retention+ window. The subcode is the cause of the first metric: `blank_event` (an event with an empty+ name), `event_stopped` (the project last received the event before launch, while ingestion+ still records the project), `outlier_bound` (an upper outlier bound that is 0 while some+ exposures have a value) or null when none shows a cause. `blank_event` and `outlier_bound`+ also report a secondary metric, and need only 100 exposures and 12 hours of data, because+ they read 0 at any traffic. A bound on the largest value of the exposures can't be 0 while+ one has a value, so it is no cause. Neither is a bound on a metric with breakdowns, which+ gets one bound per breakdown group. Until the end, the check leaves out a metric that uses an+ event `metric_event_not_ingested` reports. Each metric, or side of a ratio, is in the+ `evidence` keys `metric_N`, `metric_N_role`, `metric_N_side`, `metric_N_exposures` and+ `metric_N_cause`. It is a warning for a primary metric. After the end it stays a warning only+ for a cause that a setting fixes, because the calculation after the fix reads the events up+ to the end. It is info otherwise.+The severity of the `some_conditions_pinned` finding follows the largest share that theusers of the pinned release conditions make up among the users who got one variant fromthe flag, in the last 7 days, or since launch or the flag's last save if later. It readsdiff --git a/products/experiments/backend/running_time_target.py b/products/experiments/backend/running_time_target.pyindex a055bc08749a..ecd5798ec3d7 100644--- a/products/experiments/backend/running_time_target.py+++ b/products/experiments/backend/running_time_target.py@@ -20,8 +20,6 @@ from posthog.schema import (ExperimentMeanMetric,ExperimentQueryResponse,ExperimentStatsBaseValidated,- ExperimentVariantResultBayesian,- ExperimentVariantResultFrequentist,)from posthog.dataclasses import frozen@@ -32,6 +30,7 @@ from products.experiments.backend.hogql_queries.utils import (metric_variant_to_cuped_data,metric_variant_to_statistic,)+from products.experiments.backend.logic.result_stats import VariantResultfrom products.experiments.backend.metric_calculation.config import (MetricCalculationConfig,build_primary_calculation_configs,@@ -56,8 +55,6 @@ from products.experiments.stats.shared.statistics import ()from products.feature_flags.backend.facade.edits import Split, parse_variants-VariantResult = ExperimentVariantResultBayesian | ExperimentVariantResultFrequentist-# A funnel's stored stats carry sample sessions per step, which can be large, and the target never reads them._STATS_FIELDS = set(ExperimentStatsBaseValidated.model_fields) - {"step_sessions"}diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.tsindex ee3ab5a194fe..e07654d60e25 100644--- a/products/experiments/frontend/generated/api.schemas.ts+++ b/products/experiments/frontend/generated/api.schemas.ts@@ -2306,6 +2306,7 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]* * `metric_event_not_ingested` - Metric Event Not Ingested* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed+ * * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort* * `exposure_filter_excludes_events` - Exposure Filter Excludes Events@@ -2335,6 +2336,7 @@ export const ExperimentHealthFindingCodeEnumApi = {MetricEventNotIngested: 'metric_event_not_ingested',MetricCannotCompile: 'metric_cannot_compile',MetricCalculationFailed: 'metric_calculation_failed',+ MetricAllZero: 'metric_all_zero',ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',ExposureFilterOnDynamicCohort: 'exposure_filter_on_dynamic_cohort',ExposureFilterExcludesEvents: 'exposure_filter_excludes_events',@@ -2407,6 +2409,7 @@ export interface ExperimentHealthFindingApi {* * `metric_event_not_ingested` - Metric Event Not Ingested* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed+ * * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort* * `exposure_filter_excludes_events` - Exposure Filter Excludes Events@@ -2466,6 +2469,7 @@ export interface ExperimentHealthFindingApi {* * `metric_event_not_ingested` - Metric Event Not Ingested* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed+ * * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filters` - Exposure Filters* * `early_significance` - Early Significance@@ -2492,6 +2496,7 @@ export const ExperimentHealthCheckKeyEnumApi = {MetricEventNotIngested: 'metric_event_not_ingested',MetricCannotCompile: 'metric_cannot_compile',MetricCalculationFailed: 'metric_calculation_failed',+ MetricAllZero: 'metric_all_zero',ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',ExposureFilters: 'exposure_filters',EarlySignificance: 'early_significance',@@ -2534,6 +2539,7 @@ export interface ExperimentHealthCheckResultApi {* * `metric_event_not_ingested` - Metric Event Not Ingested* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed+ * * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filters` - Exposure Filters* * `early_significance` - Early Significance@@ -3343,6 +3349,22 @@ export interface ExperimentHealthInputsMetricCalculationFailureApi {message: string | null}+export interface ExperimentHealthInputsZeroMetricApi {+ /** The name of the metric, as the results page shows it. */+ name: string+ /** Whether the metric is a primary or a secondary metric of the experiment. */+ role: string+ /** The persons or groups of the result, across the variants. */+ exposures: number+ /**+ * The side of a ratio that reads zero. Each side that reads zero has its own entry. Null for the other metric types.+ * @nullable+ */+ ratio_side: string | null+ /** The events of the metric that the project last received before launch. */+ stopped_events: string[]+}+export interface ExperimentHealthInputsLatestCalculationApi {/*** When the latest calculation that the results page shows finished. Null when none has finished.@@ -3351,6 +3373,13 @@ export interface ExperimentHealthInputsLatestCalculationApi {finished_at: string | null/** The metrics whose latest calculation failed, with the error that the results page shows. */failures: ExperimentHealthInputsMetricCalculationFailureApi[]+ /**+ * The hours from launch to the end of the data of this calculation. Null when none has finished.+ * @nullable+ */+ hours_of_data: number | null+ /** The metrics that read zero in every variant in this calculation. */+ zero_metrics: ExperimentHealthInputsZeroMetricApi[]}/**diff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.tsindex 17137fc3f978..bcddfbf29978 100644--- a/products/experiments/frontend/generated/api.ts+++ b/products/experiments/frontend/generated/api.ts@@ -777,6 +777,26 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =* then names the filters. A query that read precomputed exposures doesn't apply those filters,* so it can't get `exposure_criteria`. It is null otherwise.*+ * `metric_all_zero` reports a metric that reads zero in every variant, from the same latest+ * finished calculation, so the results page shows "Not enough data yet" at any sample size.+ * Each side of a ratio that is zero in every variant counts on its own. It reads the stored+ * result and the project's event definitions, and no events. A primary metric needs 1,000+ * exposures in the result and 72 hours of data, because most zeros before that are outcomes+ * that take time. A retention metric counts those hours from the start of its retention+ * window. The subcode is the cause of the first metric: `blank_event` (an event with an empty+ * name), `event_stopped` (the project last received the event before launch, while ingestion+ * still records the project), `outlier_bound` (an upper outlier bound that is 0 while some+ * exposures have a value) or null when none shows a cause. `blank_event` and `outlier_bound`+ * also report a secondary metric, and need only 100 exposures and 12 hours of data, because+ * they read 0 at any traffic. A bound on the largest value of the exposures can't be 0 while+ * one has a value, so it is no cause. Neither is a bound on a metric with breakdowns, which+ * gets one bound per breakdown group. Until the end, the check leaves out a metric that uses an+ * event `metric_event_not_ingested` reports. Each metric, or side of a ratio, is in the+ * `evidence` keys `metric_N`, `metric_N_role`, `metric_N_side`, `metric_N_exposures` and+ * `metric_N_cause`. It is a warning for a primary metric. After the end it stays a warning only+ * for a cause that a setting fixes, because the calculation after the fix reads the events up+ * to the end. It is info otherwise.+ ** The severity of the `some_conditions_pinned` finding follows the largest share that the* users of the pinned release conditions make up among the users who got one variant from* the flag, in the last 7 days, or since launch or the flag's last save if later. It readsdiff --git a/products/experiments/frontend/health/debug/healthDebugInputs.ts b/products/experiments/frontend/health/debug/healthDebugInputs.tsindex 30e876bfc920..20d5deb311b7 100644--- a/products/experiments/frontend/health/debug/healthDebugInputs.ts+++ b/products/experiments/frontend/health/debug/healthDebugInputs.ts@@ -333,6 +333,12 @@ export function buildMetricFacts(failure.message ?? 'message hidden, it can hold evaluation rules'}`,})),+ ...(state.inputs?.latest_calculation?.zero_metrics ?? []).map((metric) => ({+ label: `Zero: ${metric.name} (${metric.role})`,+ value: `${metric.exposures} exposures${metric.ratio_side ? `, ${metric.ratio_side}` : ''}${+ metric.stopped_events.length ? `, stopped: ${metric.stopped_events.join(', ')}` : ''+ }`,+ })),]}diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.tsindex 2751dc4ad03c..31dbd6dac523 100644--- a/products/experiments/frontend/health/debug/healthDebugReport.ts+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts@@ -247,6 +247,16 @@ function metricCalculationNote({ evidence }: ExperimentHealthFindingApi): stringreturn `Failed: ${failures.join(', ')}. Calculated at ${evidence.calculated_at}.`}+function metricAllZeroNote({ evidence }: ExperimentHealthFindingApi): string {+ const metrics = Object.entries(evidence)+ .filter(([key]) => /^metric_\d+$/.test(key))+ .map(([key, value]) => {+ const side = evidence[`${key}_side`]+ return `${value}${side ? ` ${side}` : ''} (${evidence[`${key}_cause`] ?? 'no cause shown'})`+ })+ return `Zero in every variant: ${metrics.join(', ')}.`+}+function exposureCriteriaNote({ subcode, evidence }: ExperimentHealthFindingApi): string {if (subcode === 'filter_on_another_flag') {return `The query requires ${evidence.flag_key}, a filter requires ${evidence.filter_flag_keys}.`@@ -385,6 +395,7 @@ export function buildHealthDebugChecks(input: HealthDebugInput): HealthDebugChecserverOnlyCheck(input, 'metric_event_not_ingested', metricEventNote, ['metrics', 'project']),serverOnlyCheck(input, 'metric_cannot_compile', metricCompileNote, ['metrics', 'experiment']),serverOnlyCheck(input, 'metric_calculation_failed', metricCalculationNote, ['metrics']),+ serverOnlyCheck(input, 'metric_all_zero', metricAllZeroNote, ['metrics', 'project']),serverOnlyCheck(input, 'exposure_criteria_cannot_match', exposureCriteriaNote, ['experiment', 'project']),exposureFiltersCheck(input),serverOnlyCheck(input, 'early_significance', null, ['experiment', 'metrics']),diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.tsindex 73450b56e36f..e7e466fef090 100644--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts@@ -31,6 +31,7 @@ export type ExperimentHealthFindingCode =| 'metric_event_not_ingested'| 'metric_cannot_compile'| 'metric_calculation_failed'+ | 'metric_all_zero'| 'exposure_criteria_cannot_match'| 'exposure_filter_on_dynamic_cohort'| 'exposure_filter_excludes_events'diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 74ef2b7602ff..fe69c6dfc7ba 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -48124,6 +48124,7 @@ export namespace Schemas {* * `metric_event_not_ingested` - Metric Event Not Ingested* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed+ * * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort* * `exposure_filter_excludes_events` - Exposure Filter Excludes Events@@ -48153,6 +48154,7 @@ export namespace Schemas {MetricEventNotIngested: 'metric_event_not_ingested',MetricCannotCompile: 'metric_cannot_compile',MetricCalculationFailed: 'metric_calculation_failed',+ MetricAllZero: 'metric_all_zero',ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',ExposureFilterOnDynamicCohort: 'exposure_filter_on_dynamic_cohort',ExposureFilterExcludesEvents: 'exposure_filter_excludes_events',@@ -48225,6 +48227,7 @@ export namespace Schemas {* * `metric_event_not_ingested` - Metric Event Not Ingested* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed+ * * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort* * `exposure_filter_excludes_events` - Exposure Filter Excludes Events@@ -48284,6 +48287,7 @@ export namespace Schemas {* * `metric_event_not_ingested` - Metric Event Not Ingested* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed+ * * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filters` - Exposure Filters* * `early_significance` - Early Significance@@ -48310,6 +48314,7 @@ export namespace Schemas {MetricEventNotIngested: 'metric_event_not_ingested',MetricCannotCompile: 'metric_cannot_compile',MetricCalculationFailed: 'metric_calculation_failed',+ MetricAllZero: 'metric_all_zero',ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',ExposureFilters: 'exposure_filters',EarlySignificance: 'early_significance',@@ -48352,6 +48357,7 @@ export namespace Schemas {* * `metric_event_not_ingested` - Metric Event Not Ingested* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed+ * * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filters` - Exposure Filters* * `early_significance` - Early Significance@@ -49116,6 +49122,22 @@ export namespace Schemas {message: string | null;}+ export interface ExperimentHealthInputsZeroMetric {+ /** The name of the metric, as the results page shows it. */+ name: string;+ /** Whether the metric is a primary or a secondary metric of the experiment. */+ role: string;+ /** The persons or groups of the result, across the variants. */+ exposures: number;+ /**+ * The side of a ratio that reads zero. Each side that reads zero has its own entry. Null for the other metric types.+ * @nullable+ */+ ratio_side: string | null;+ /** The events of the metric that the project last received before launch. */+ stopped_events: string[];+ }+export interface ExperimentHealthInputsLatestCalculation {/*** When the latest calculation that the results page shows finished. Null when none has finished.@@ -49124,6 +49146,13 @@ export namespace Schemas {finished_at: string | null;/** The metrics whose latest calculation failed, with the error that the results page shows. */failures: ExperimentHealthInputsMetricCalculationFailure[];+ /**+ * The hours from launch to the end of the data of this calculation. Null when none has finished.+ * @nullable+ */+ hours_of_data: number | null;+ /** The metrics that read zero in every variant in this calculation. */+ zero_metrics: ExperimentHealthInputsZeroMetric[];}export interface ExperimentHealthInputsFlagChange {