:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): flag metrics and exposures on events the project never received (#120893)
Marcel Poelker9b0d6147a369
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex 7356a2e2f198..6df65d9cfee8 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -3288,6 +3288,10 @@ snapshots:hash: v1.k794b7964.998711a45ae74b0bc74066f1d70b2b58c809e9d9e0bd7d4d0b541698a312cad3.IPJlc0qZzHtFDsZ5Gb06zukOf9kk_JnT6-OU2V-Pwa0experiments-healthchecksoverview--draft--light:hash: v1.k794b7964.5b098ee13c120e815fdf13e18a59d6234a4c904e747a7cc83fec0e1406cd3491.YrNkI7P5QfXa-TnjviRwcQIIML09K1-POg889feICRg+ experiments-healthchecksoverview--draft-with-events-not-received--dark:+ hash: v1.k794b7964.8fc4d4ec055de978ea383b15138251df99dafb87f0713218485a74da67b31634.XBTJmRJvnfs_9gv6zfC10-5JQAAGTRfTctWZc7T852s+ experiments-healthchecksoverview--draft-with-events-not-received--light:+ hash: v1.k794b7964.207d3f925f5972dbbc0825b2d7f6e9026c362c4b513b2db68b70303715bcd866.J3uCNeI2NrU5MV9AWMYFCI4ftj9WCM-4d0Vvjp_7K-kexperiments-howtoreadtooltip--intervals-bayesian--dark:hash: v1.k794b7964.a06d267cee9cb54bafc732335b35d461d3c1a59250c6daa8becace0fcb8492d2.vR5tecOozn6B51xE8LrByAFVsi03J4w-iUu-j2hQUTUexperiments-howtoreadtooltip--intervals-bayesian--light:@@ -10779,9 +10783,9 @@ snapshots:scenes-app-experiments--experiment-frequentist-two-variants--light:hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmIIscenes-app-experiments--experiment-health-debug-tab--dark:- hash: v1.k794b7964.eae73ea7b85aef09cd37d294cdb0bd2cf191348232c9df704588d2c25499c839.GaCHEbAy8iVYYnI_jir5s_XFuJNNHii1sYqzXPlf3i4+ hash: v1.k794b7964.5cc5e86e5c2379904d044c54858ac88f2b864584975e9ac75bc4c9cc55499be7.n1wcGp5JgWshP7nhURz-X-LyFqi7Ht2RXc6qOq5hLucscenes-app-experiments--experiment-health-debug-tab--light:- hash: v1.k794b7964.f7b8889003f93b6a829f42658ac4c50a895d28d7d417da5c336cd5ad7b918886.OHBwsWdtYlEQAHwZ3Hl24xUUeB8nAWC8gfZ_dGXBT3w+ hash: v1.k794b7964.e7ed4da61b907080726cd60bb44f0253ff715e9ec2ae314ebea1d0537218ccf7.DSB70UsaEOYBs_2j-uGHqfeD99JRnVFpUbPAfPg81UYscenes-app-experiments--experiment-recordings-dropped-metric--dark:hash: v1.k794b7964.e0e7181f2c9e61a8324eff9aa7a9230fef25b3cf37e0b24ef875780186ed25dc.9VEvZ4iVieVJwmDol0gwtAsr3wgZYW5gj2KXAGZJfiwscenes-app-experiments--experiment-recordings-dropped-metric--light:diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex f06017f6d397..9c89b5cd66b0 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -9727,6 +9727,8 @@ export namespace Schemas {* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded* * `multiple_variants_share` - Multiple Variants Share* * `no_metric` - No Metric+ * * `metric_event_not_ingested` - Metric Event Not Ingested+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition@@ -9744,6 +9746,8 @@ export namespace Schemas {| "bias_risk_multiple_excluded"| "multiple_variants_share"| "no_metric"+ | "metric_event_not_ingested"+ | "exposure_criteria_cannot_match"| "srm"| "zero_exposures"| "forced_variant_release_condition"@@ -9791,6 +9795,8 @@ export namespace Schemas {* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded* * `multiple_variants_share` - Multiple Variants Share* * `no_metric` - No Metric+ * * `metric_event_not_ingested` - Metric Event Not Ingested+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition@@ -9844,6 +9850,8 @@ export namespace Schemas {* * `forced_variant_release_condition` - Forced Variant Release Condition* * `new_flag_no_traffic` - New Flag No Traffic* * `no_metric` - No Metric+ * * `metric_event_not_ingested` - Metric Event Not Ingested+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `zero_exposures` - Zero Exposures* * `srm` - Srm* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded@@ -9859,6 +9867,8 @@ export namespace Schemas {| "forced_variant_release_condition"| "new_flag_no_traffic"| "no_metric"+ | "metric_event_not_ingested"+ | "exposure_criteria_cannot_match"| "zero_exposures"| "srm"| "bias_risk_multiple_excluded"@@ -9887,6 +9897,8 @@ export namespace Schemas {* * `forced_variant_release_condition` - Forced Variant Release Condition* * `new_flag_no_traffic` - New Flag No Traffic* * `no_metric` - No Metric+ * * `metric_event_not_ingested` - Metric Event Not Ingested+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `zero_exposures` - Zero Exposures* * `srm` - Srm* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excludeddiff --git a/products/experiments/backend/experiment_service.py b/products/experiments/backend/experiment_service.pyindex 26c9ad768e9b..9901a2196620 100644--- a/products/experiments/backend/experiment_service.py+++ b/products/experiments/backend/experiment_service.py@@ -52,7 +52,6 @@ from posthog.models.user import Userfrom posthog.utils import str_to_boolfrom products.cohorts.backend.models.cohort import Cohort-from products.event_definitions.backend.models import EventDefinition, effective_project_id_exprfrom products.experiments.backend.facade.contracts import ExperimentHealthFindingfrom products.experiments.backend.facade.launch_signals import experiment_launchedfrom products.experiments.backend.flag_cleanup import build_cleanup_prompt, cleanup_plan@@ -82,6 +81,7 @@ from products.experiments.backend.metric_calculation.results import MetricResultfrom products.experiments.backend.metric_resolution import MetricRolefrom products.experiments.backend.metric_utils import filter_metric_group_ids_by_eventfrom products.experiments.backend.metric_validation import (+ defined_event_names,extract_entity_nodes,parse_and_validate_metric,validate_metric_action_ids,@@ -1080,10 +1080,10 @@ class ExperimentService:use, so an update can be checked for what it introduces rather than foreverything it resends. See ``update_experiment``.- Scope must match the picker: the EventDefinition list endpoint is- project-scoped (see posthog/api/event_definition.py), so a user in a- multi-team project can pick an event ingested by a sibling team. We- mirror that scope here to avoid rejecting legitimate selections.+ The lookup is `defined_event_names`, which the experiment health checks+ also read. They count only the definitions that ingestion has seen, but+ this accepts the events Wizard planned, so they can be picked before the+ code that sends them ships."""all_event_names, _ = extract_entity_nodes(metrics)event_names = all_event_names - known_event_names if known_event_names else all_event_names@@ -1091,17 +1091,7 @@ class ExperimentService:returnproject_id = self.team.project_id- # `COALESCE(project_id, team_id)` falls back to `team_id` (not self.team.id)- # on purpose: legacy EventDefinitions (project_id IS NULL) belong to the- # *primary* team, and primary_team.id == project.id by convention. This- # mirrors the picker SQL in posthog/api/event_definition.py so sibling- # teams can validate against legacy primary-team events the picker shows.- existing = set(- EventDefinition.objects.alias(effective_project_id=effective_project_id_expr())- .filter(effective_project_id=project_id, name__in=event_names)- .values_list("name", flat=True)- )- unknown = event_names - existing+ unknown = event_names - defined_event_names(project_id, event_names)if unknown:# Capture the rejected payload shape so we can identify clients# that send malformed metrics (e.g. event="").diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.pyindex 87ecf74bc318..f24127b41a74 100644--- a/products/experiments/backend/facade/contracts.py+++ b/products/experiments/backend/facade/contracts.py@@ -105,6 +105,8 @@ class ExperimentHealthFindingCode(LabeledStrEnum):BIAS_RISK_MULTIPLE_EXCLUDED = "bias_risk_multiple_excluded"MULTIPLE_VARIANTS_SHARE = "multiple_variants_share"NO_METRIC = "no_metric"+ METRIC_EVENT_NOT_INGESTED = "metric_event_not_ingested"+ EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"SRM = "srm", "Sample Ratio Mismatch"ZERO_EXPOSURES = "zero_exposures"FORCED_VARIANT_RELEASE_CONDITION = "forced_variant_release_condition"@@ -193,6 +195,8 @@ class ExperimentHealthCheckKey(LabeledStrEnum):FORCED_VARIANT_RELEASE_CONDITION = "forced_variant_release_condition"NEW_FLAG_NO_TRAFFIC = "new_flag_no_traffic"NO_METRIC = "no_metric"+ METRIC_EVENT_NOT_INGESTED = "metric_event_not_ingested"+ EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"ZERO_EXPOSURES = "zero_exposures"SRM = "srm"BIAS_RISK_MULTIPLE_EXCLUDED = "bias_risk_multiple_excluded"diff --git a/products/experiments/backend/health/checks/events_received.py b/products/experiments/backend/health/checks/events_received.pynew file mode 100644index 000000000000..26ad2323cbb4--- /dev/null+++ b/products/experiments/backend/health/checks/events_received.py@@ -0,0 +1,29 @@+"""What the checks of events the project never received share. Pure functions, no I/O.++Ingestion sets `last_seen_at` on an event definition when it sees the event in the project, so a name without a seen+definition was never received. Wizard writes definitions without it for the events it plans. Before launch that is+normal: the code that sends the event often ships with the launch.+"""++from collections.abc import Sequence++from products.experiments.backend.facade.contracts import ExperimentHealthFindingSeverity+from products.experiments.backend.health.checks.zero_exposures import ZERO_EXPOSURES_GRACE_HOURS+from products.experiments.backend.health.context import HealthContext+++def not_received_severity(ctx: HealthContext, *, affects_results: bool) -> ExperimentHealthFindingSeverity:+ hours = ctx.hours_since_launch+ if affects_results and ctx.is_launched and hours is not None and hours >= ZERO_EXPOSURES_GRACE_HOURS:+ return ExperimentHealthFindingSeverity.WARNING+ return ExperimentHealthFindingSeverity.INFO+++def joined(items: Sequence[str], conjunction: str = "and") -> str:+ if len(items) < 3:+ return f" {conjunction} ".join(items)+ return f"{', '.join(items[:-1])}, {conjunction} {items[-1]}"+++def quoted(names: Sequence[str], conjunction: str = "and") -> str:+ return joined([f'"{name}"' for name in names], conjunction)diff --git a/products/experiments/backend/health/checks/exposure_criteria.py b/products/experiments/backend/health/checks/exposure_criteria.pynew file mode 100644index 000000000000..02d14afdb2c3--- /dev/null+++ b/products/experiments/backend/health/checks/exposure_criteria.py@@ -0,0 +1,124 @@+"""+Exposure criteria that no event can match, read from the criteria alone. Pure functions, no I/O.++The zero exposures check reports an experiment that counted nobody. This check names the cause when the criteria show+it: a custom exposure event that the project never received, or a filter on the flag call events that requires+another flag than the one the exposure query requires.+"""++from posthog.schema import PropertyOperator++from products.experiments.backend.facade.contracts import (+ ExperimentHealthFinding,+ ExperimentHealthFindingActionKind,+ ExperimentHealthFindingCode,+ ExperimentHealthFindingSeverity,+)+from products.experiments.backend.health.checks.events_received import not_received_severity, quoted+from products.experiments.backend.health.context import FlagKeyFilter, HealthContext+from products.experiments.backend.health.skips import (+ NO_FLAG,+ NOT_AFTER_END,+ NOT_ON_ARCHIVED,+ CheckSkip,+ not_applicable,+ not_checked,+)++# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.+EXPOSURE_EVENT_NOT_INGESTED = "exposure_event_not_ingested"+FILTER_ON_ANOTHER_FLAG = "filter_on_another_flag"+++def exposure_criteria_skip(ctx: HealthContext) -> CheckSkip | None:+ if ctx.archived:+ return not_applicable(NOT_ON_ARCHIVED)+ events = ctx.exposure_events+ if events is None:+ return not_checked("Couldn't check because the exposure criteria can't be read")+ if events.custom_event is not None:+ if ctx.has_ended:+ return not_applicable(NOT_AFTER_END)+ if events.custom_event not in ctx.events_received:+ return not_checked("Couldn't check because PostHog keeps no record of this event name")+ return None+ if events.required_flag_key is None:+ if ctx.flag is None:+ return not_applicable(NO_FLAG)+ return not_applicable("Doesn't check exposure criteria that use an action")+ return None+++def exposure_criteria_cannot_match(ctx: HealthContext) -> ExperimentHealthFinding | None:+ events = ctx.exposure_events+ if ctx.archived or events is None:+ return None+ flag_key = events.required_flag_key+ if flag_key is not None:+ other = next((f for f in events.flag_key_filters if _requires_another_flag(f, flag_key)), None)+ if other is not None:+ return _filter_on_another_flag(flag_key, other)+ event = events.custom_event+ if event is not None and not ctx.has_ended and ctx.events_received.get(event) is False:+ return _exposure_event_not_ingested(ctx, event)+ return None+++def _requires_another_flag(flag_filter: FlagKeyFilter, flag_key: str) -> bool:+ # `exact` compares the property with each value as a case-sensitive string. A filter without values matches every+ # event, and the query converts a number to a string, so only string values prove that the filter excludes the key.+ return (+ flag_filter.operator == PropertyOperator.EXACT+ and len(flag_filter.values) > 0+ and all(isinstance(value, str) for value in flag_filter.values)+ and flag_key not in flag_filter.values+ )+++def _filter_on_another_flag(flag_key: str, flag_filter: FlagKeyFilter) -> ExperimentHealthFinding:+ values = [str(value) for value in flag_filter.values]+ required = f'the flag "{values[0]}"' if len(values) == 1 else f"one of the flags {quoted(values, 'or')}"+ return _finding(+ subcode=FILTER_ON_ANOTHER_FLAG,+ severity=ExperimentHealthFindingSeverity.WARNING,+ title="The exposure criteria can't match any event",+ detail=(+ f'The experiment counts the calls of its feature flag "{flag_key}" as exposures, but a filter in the '+ f"exposure criteria requires {required}. No call matches both, so nobody is counted as exposed. "+ f'Remove the filter, or change it to "{flag_key}".'+ ),+ evidence={"flag_key": flag_key, "filter_flag_keys": ", ".join(values)},+ )+++def _exposure_event_not_ingested(ctx: HealthContext, event: str) -> ExperimentHealthFinding:+ advice = (+ "Launch the experiment after your code starts sending it."+ if not ctx.is_launched+ else "Check that your code sends the event with exactly this name, or change the exposure criteria."+ )+ return _finding(+ subcode=EXPOSURE_EVENT_NOT_INGESTED,+ severity=not_received_severity(ctx, affects_results=True),+ title="This project hasn't received the exposure event",+ detail=(+ f'The experiment counts users as exposed when they send the event "{event}", which this project hasn\'t '+ f"received yet. Until it arrives, nobody is counted as exposed. {advice}"+ ),+ evidence={"event": event},+ )+++def _finding(+ *, subcode: str, severity: ExperimentHealthFindingSeverity, title: str, detail: str, evidence: dict[str, str]+) -> ExperimentHealthFinding:+ return ExperimentHealthFinding(+ code=ExperimentHealthFindingCode.EXPOSURE_CRITERIA_CANNOT_MATCH,+ subcode=subcode,+ severity=severity,+ title=title,+ detail=detail,+ evidence=evidence,+ actions=(ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,),+ diagnostic_ref=None,+ )diff --git a/products/experiments/backend/health/checks/metric_event_not_ingested.py b/products/experiments/backend/health/checks/metric_event_not_ingested.pynew file mode 100644index 000000000000..015e50f1626c--- /dev/null+++ b/products/experiments/backend/health/checks/metric_event_not_ingested.py@@ -0,0 +1,91 @@+"""+Metrics that read an event the project never received. Pure functions, no I/O.++Saving a metric checks its event names, but a caller can waive that check, a shared metric and a launch skip it, and+an update checks only the names it adds. A metric on such an event has no data to compare the variants.+"""++from products.experiments.backend.facade.contracts import ExperimentHealthFinding, ExperimentHealthFindingCode+from products.experiments.backend.health.checks.events_received import joined, not_received_severity, quoted+from products.experiments.backend.health.context import HealthContext, MetricEvents+from products.experiments.backend.health.skips import (+ NOT_AFTER_END,+ NOT_ON_ARCHIVED,+ CheckSkip,+ not_applicable,+ not_checked,+)+++def metric_event_not_ingested_skip(ctx: HealthContext) -> CheckSkip | None:+ if ctx.archived:+ return not_applicable(NOT_ON_ARCHIVED)+ if ctx.has_ended:+ return not_applicable(NOT_AFTER_END)+ names = {event for metric in ctx.metric_events for event in metric.events}+ if not names:+ return not_applicable("Runs once a metric uses an event")+ if not names & ctx.events_received.keys():+ return not_checked("Couldn't check because PostHog keeps no record of these event names")+ return None+++def metric_event_not_ingested(ctx: HealthContext) -> ExperimentHealthFinding | None:+ if ctx.archived or ctx.has_ended:+ return None+ affected = [+ (metric, missing)+ for metric in ctx.metric_events+ if (missing := tuple(event for event in metric.events if ctx.events_received.get(event) is False))+ ]+ if not affected:+ return None+ events = sorted({event for _, missing in affected for event in missing})+ return ExperimentHealthFinding(+ code=ExperimentHealthFindingCode.METRIC_EVENT_NOT_INGESTED,+ subcode=None,+ severity=not_received_severity(ctx, affects_results=any(metric.role == "primary" for metric, _ in affected)),+ title=_title(len(affected), len(events)),+ detail=f"{_what(affected)} {_advice(ctx, len(events))}",+ evidence=_evidence(affected),+ # No action opens a given metric.+ actions=(),+ diagnostic_ref=None,+ )+++def _evidence(affected: list[tuple[MetricEvents, tuple[str, ...]]]) -> dict[str, str]:+ evidence: dict[str, str] = {}+ for number, (metric, missing) in enumerate(affected, start=1):+ evidence[f"metric_{number}"] = metric.name+ evidence[f"metric_{number}_role"] = metric.role+ evidence[f"metric_{number}_events"] = ", ".join(missing)+ return evidence+++def _title(metric_count: int, event_count: int) -> str:+ if metric_count > 1:+ return f"{metric_count} metrics use events this project hasn't received"+ return f"A metric uses {'an event' if event_count == 1 else 'events'} this project hasn't received"+++def _what(affected: list[tuple[MetricEvents, tuple[str, ...]]]) -> str:+ if len(affected) == 1:+ metric, missing = affected[0]+ event, arrives = ("the event", "it arrives") if len(missing) == 1 else ("the events", "they arrive")+ return (+ f'The {metric.role} metric "{metric.name}" uses {event} {quoted(missing)}, which this project hasn\'t '+ f"received yet. Until {arrives}, the metric can't compare the variants."+ )+ uses = [f'the {metric.role} metric "{metric.name}" uses {quoted(missing)}' for metric, missing in affected]+ return (+ f"These metrics use events this project hasn't received yet: {joined(uses)}. Until those events arrive, "+ "these metrics can't compare the variants."+ )+++def _advice(ctx: HealthContext, event_count: int) -> str:+ if not ctx.is_launched:+ return f"Launch the experiment after your code starts sending {'it' if event_count == 1 else 'them'}."+ names = "this name" if event_count == 1 else "these names"+ return f"Check that your code sends {'the event' if event_count == 1 else 'the events'} with exactly {names}."diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex 20329e0181be..e3a22b8d5035 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -1,17 +1,30 @@from collections.abc import Mapping-from datetime import datetime+from datetime import datetime, timedeltafrom enum import StrEnumfrom typing import Anyfrom django.conf import settings+from django.utils import timezone-from posthog.schema import MultipleVariantHandling+import pydantic++from posthog.schema import EventPropertyFilter, ExperimentEventExposureConfig, MultipleVariantHandling, PropertyOperatorfrom posthog.dataclasses import frozen+from posthog.models.activity_logging.activity_log import ActivityLog+from posthog.models.team.team import Teamfrom products.experiments.backend.facade.contracts import ExperimentHealthFlagChangeKind-from products.experiments.backend.metric_resolution import saved_metric_link_role, saved_metric_links-from products.experiments.backend.models.experiment import Experiment+from products.experiments.backend.hogql_queries.exposure_query_logic import (+ DEFAULT_EXPOSURE_EVENT,+ EXPERIMENT_EXPOSURE_EVENT,+ get_exposure_event_and_property,+ normalize_to_exposure_criteria,+)+from products.experiments.backend.metric_resolution import MetricRole, saved_metric_link_role, saved_metric_links+from products.experiments.backend.metric_utils import get_default_metric_title+from products.experiments.backend.metric_validation import defined_event_names, extract_entity_nodes+from products.experiments.backend.models.experiment import Experiment, ExperimentToSavedMetricfrom products.feature_flags.backend.facade.enums import FlagEvaluationsModefrom products.feature_flags.backend.facade.filters import condition_aggregation, pinned_variant, reachable_conditionsfrom products.feature_flags.backend.facade.flags import get_organization_flag_evaluations_mode, has_active_flags@@ -226,6 +239,36 @@ class ExposureTotals:counts_groups: bool+@frozen+class MetricEvents:+ """A metric of the experiment and the events that it reads by name."""++ name: str+ role: MetricRole+ events: tuple[str, ...]+++@frozen+class FlagKeyFilter:+ """An event property filter of the exposure criteria on `$feature_flag`, the key of the called flag."""++ operator: PropertyOperator+ values: tuple[str | float | bool, ...]+++@frozen+class ExposureEvents:+ """The events that the exposure criteria count."""++ # The event that the criteria name in place of the flag call events. None when the criteria count the flag call+ # events or an action.+ custom_event: str | None+ # The flag key that the exposure query requires in `$feature_flag`, because the flag call events are not specific+ # to one flag. None when the criteria count a custom event or an action, and when the experiment has no flag.+ required_flag_key: str | None+ flag_key_filters: tuple[FlagKeyFilter, ...]++@frozenclass HealthContext:"""Every input of the health checks, loaded once, so that each check is a pure function of it."""@@ -246,6 +289,13 @@ class HealthContext:baseline_variant_key: str | Noneprimary_metric_count: intsecondary_metric_count: int+ hours_since_launch: float | None+ metric_events: tuple[MetricEvents, ...]+ # None when the stored exposure criteria don't parse.+ exposure_events: ExposureEvents | None+ # Whether the project has received each event that `metric_events` and `exposure_events` name. A name that+ # ingestion never writes an event definition for is absent, because its absence proves nothing.+ events_received: Mapping[str, bool]exposures: ExposureTotals | None# None when not loaded. Only the health endpoint loads it, because it can query the exposures.flag_history: FlagHistory | FlagHistoryNotChecked | None@@ -391,6 +441,106 @@ def _flag_called(experiment: Experiment) -> bool | None:return False+def _metric_events(experiment: Experiment, links: list[ExperimentToSavedMetric]) -> tuple[MetricEvents, ...]:+ named: list[tuple[str | None, MetricRole, object]] = [+ *((None, "primary", metric) for metric in experiment.metrics or []),+ *((None, "secondary", metric) for metric in experiment.metrics_secondary or []),+ *((link.saved_metric.name, saved_metric_link_role(link), link.saved_metric.query) for link in links),+ ]+ metrics: list[MetricEvents] = []+ for shared_name, role, metric in named:+ if not isinstance(metric, dict):+ continue+ events, _ = extract_entity_nodes([metric])+ if events:+ name = shared_name or metric.get("name") or get_default_metric_title(metric)+ metrics.append(MetricEvents(name=name, role=role, events=tuple(sorted(events))))+ return tuple(metrics)+++def _exposure_events(experiment: Experiment) -> ExposureEvents | None:+ try:+ criteria = normalize_to_exposure_criteria(experiment.exposure_criteria)+ except pydantic.ValidationError:+ return None+ flag_key = experiment.feature_flag.key if experiment.feature_flag_id else None+ event, _ = get_exposure_event_and_property(flag_key or "", criteria, default_exposure_event=DEFAULT_EXPOSURE_EVENT)+ reads_flag_calls = event in (DEFAULT_EXPOSURE_EVENT, EXPERIMENT_EXPOSURE_EVENT)+ config = criteria.exposure_config if criteria else None+ properties = config.properties if isinstance(config, ExperimentEventExposureConfig) else []+ return ExposureEvents(+ custom_event=event if event is not None and not reads_flag_calls else None,+ required_flag_key=flag_key if reads_flag_calls else None,+ flag_key_filters=tuple(+ # The exposure query reads a filter without an operator as `exact`, as `property_to_expr` does.+ FlagKeyFilter(operator=prop.operator or PropertyOperator.EXACT, values=_filter_values(prop.value))+ for prop in properties+ if isinstance(prop, EventPropertyFilter) and prop.key == "$feature_flag"+ ),+ )+++def _filter_values(value: list[str | float | bool] | str | float | bool | None) -> tuple[str | float | bool, ...]:+ if isinstance(value, list):+ return tuple(value)+ return () if value is None else (value,)+++# Ingestion writes no event definition for these names, so a missing definition doesn't show that the project never+# received the event. See `EVENTS_WITHOUT_PROPERTIES` and `will_fit_in_postgres_column` in+# rust/property-defs-rs/src/types.rs.+_EVENTS_WITHOUT_DEFINITION = frozenset({"$$plugin_metrics"})+_MAX_EVENT_DEFINITION_NAME_BYTES = 200+++def _has_event_definition(name: str) -> bool:+ return name not in _EVENTS_WITHOUT_DEFINITION and len(name.encode()) <= _MAX_EVENT_DEFINITION_NAME_BYTES+++def _events_received(+ experiment: Experiment, metric_events: tuple[MetricEvents, ...], exposure_events: ExposureEvents | None+) -> dict[str, bool]:+ names = {event for metric in metric_events for event in metric.events}+ if exposure_events is not None and exposure_events.custom_event is not None:+ names.add(exposure_events.custom_event)+ checked = {name for name in names if _has_event_definition(name)}+ project_id = experiment.team.project_id+ received = defined_event_names(project_id, checked, seen_only=True)+ checked -= _deleted_definition_names(project_id, checked - received, since=_deletions_since(experiment))+ return {name: name in received for name in checked}+++# Ingestion writes a deleted definition again when it next sees the event, which takes up to about an hour for an+# event that keeps arriving. A margin of a day also covers events that arrive a few times a day.+_DEFINITION_RECREATION_MARGIN = timedelta(days=1)+++def _deletions_since(experiment: Experiment) -> datetime:+ # An earlier deletion matters only if the event stopped arriving: an event that arrived during the run, or+ # within the margin, has a new definition. So a deletion before then leaves the finding correct.+ recent = timezone.now() - _DEFINITION_RECREATION_MARGIN+ return min(experiment.start_date, recent) if experiment.start_date is not None else recent+++def _deleted_definition_names(project_id: int, names: set[str], *, since: datetime) -> set[str]:+ # Until ingestion writes a deleted definition again, the project can hold the event without a definition.+ if not names:+ return set()+ return set(+ ActivityLog.objects.filter(+ team_id__in=Team.objects.filter(project_id=project_id).values("id"),+ scope="EventDefinition",+ activity="deleted",+ created_at__gte=since,+ # These match the condition of `idx_alog_team_act_scope_usr`, so the lookup reads only the deletions. A+ # deletion made while impersonating a user is not read.+ was_impersonated=False,+ is_system=False,+ detail__name__in=names,+ ).values_list("detail__name", flat=True)+ )++def _configured_baseline_variant_key(stats_config: object) -> str | None:# An empty key counts as unset, as in `get_baseline_variant_key`.key = stats_config.get("baseline_variant_key") if isinstance(stats_config, dict) else None@@ -403,8 +553,11 @@ def load_health_context(*,flag_history: FlagHistory | FlagHistoryNotChecked | None = None,) -> HealthContext:- shared_metric_roles = [saved_metric_link_role(link) for link in saved_metric_links(experiment)]+ links = saved_metric_links(experiment)+ shared_metric_roles = [saved_metric_link_role(link) for link in links]flag_called = _flag_called(experiment)+ metric_events = _metric_events(experiment, links)+ exposure_events = _exposure_events(experiment)return HealthContext(is_launched=experiment.is_launched,is_running=experiment.is_running,@@ -419,6 +572,10 @@ def load_health_context(baseline_variant_key=_configured_baseline_variant_key(experiment.stats_config),primary_metric_count=len(experiment.metrics or []) + shared_metric_roles.count("primary"),secondary_metric_count=len(experiment.metrics_secondary or []) + shared_metric_roles.count("secondary"),+ hours_since_launch=experiment.hours_since_launch,+ metric_events=metric_events,+ exposure_events=exposure_events,+ events_received=_events_received(experiment, metric_events, exposure_events),exposures=exposures,flag_history=flag_history,condition_set_traffic=None,diff --git a/products/experiments/backend/health/loader.py b/products/experiments/backend/health/loader.pyindex ab8e7337afda..132d68a207e9 100644--- a/products/experiments/backend/health/loader.py+++ b/products/experiments/backend/health/loader.py@@ -54,7 +54,7 @@ def load_experiment_health_inputs(def _load_experiment(team_id: int, experiment_id: int) -> Experiment:return (Experiment.objects.select_related("team", "feature_flag")- .prefetch_related("experimenttosavedmetric_set")+ .prefetch_related("experimenttosavedmetric_set__saved_metric").get(id=experiment_id, team_id=team_id))diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.pyindex 588ddfc89cd9..7a654adac7b8 100644--- a/products/experiments/backend/health/registry.py+++ b/products/experiments/backend/health/registry.py@@ -13,6 +13,10 @@ from products.experiments.backend.facade.contracts import (from products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded, bias_risk_skipfrom products.experiments.backend.health.checks.bucketing_changed import bucketing_changed_mid_runfrom products.experiments.backend.health.checks.conditions_changed import release_conditions_changed_mid_run+from products.experiments.backend.health.checks.exposure_criteria import (+ exposure_criteria_cannot_match,+ exposure_criteria_skip,+)from products.experiments.backend.health.checks.flag_change_findings import (flag_change_skip,flag_history_not_readable,@@ -24,6 +28,10 @@ from products.experiments.backend.health.checks.forced_variant import (forced_variant_release_condition,forced_variant_skip,)+from products.experiments.backend.health.checks.metric_event_not_ingested import (+ metric_event_not_ingested,+ metric_event_not_ingested_skip,+)from products.experiments.backend.health.checks.multiple_variants import (multiple_variants_share,multiple_variants_share_skip,@@ -102,6 +110,18 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (find=no_metric,skip=no_metric_skip,),+ HealthCheck(+ key=ExperimentHealthCheckKey.METRIC_EVENT_NOT_INGESTED,+ name="Metric events",+ find=metric_event_not_ingested,+ skip=metric_event_not_ingested_skip,+ ),+ HealthCheck(+ key=ExperimentHealthCheckKey.EXPOSURE_CRITERIA_CANNOT_MATCH,+ name="Exposure criteria",+ find=exposure_criteria_cannot_match,+ skip=exposure_criteria_skip,+ ),)# The exposure query runs these on its answer. They must not include the checks above: the exposure answerdiff --git a/products/experiments/backend/health/test/factories.py b/products/experiments/backend/health/test/factories.pyindex 883ef4e51258..9f31012e9574 100644--- a/products/experiments/backend/health/test/factories.py+++ b/products/experiments/backend/health/test/factories.py@@ -6,6 +6,7 @@ from products.experiments.backend.facade.contracts import ExperimentHealthFlagChfrom products.experiments.backend.health.context import (ChangeWindow,ConditionsChange,+ ExposureEvents,FlagChange,FlagHistory,FlagHistoryNotChecked,@@ -27,6 +28,10 @@ RUNNING_CONTEXT = HealthContext(baseline_variant_key=None,primary_metric_count=1,secondary_metric_count=0,+ hours_since_launch=72,+ metric_events=(),+ exposure_events=ExposureEvents(custom_event=None, required_flag_key="flag", flag_key_filters=()),+ events_received={},exposures=None,flag_history=None,condition_set_traffic=None,diff --git a/products/experiments/backend/health/test/test_events_received.py b/products/experiments/backend/health/test/test_events_received.pynew file mode 100644index 000000000000..71afde9cf034--- /dev/null+++ b/products/experiments/backend/health/test/test_events_received.py@@ -0,0 +1,142 @@+from datetime import timedelta+from typing import Any++from posthog.test.base import BaseTest++from django.db import connection+from django.test.utils import CaptureQueriesContext+from django.utils import timezone++from parameterized import parameterized++from posthog.models.activity_logging.activity_log import ActivityLog, Detail, log_activity+from posthog.models.team.team import Team++from products.event_definitions.backend.models import EventDefinition+from products.experiments.backend.facade.api import get_experiment_health+from products.experiments.backend.facade.contracts import (+ ExperimentHealthFindingCode,+ ExperimentHealthFindingSeverity,+ ExperimentHealthReaderAccess,+)+from products.experiments.backend.models.experiment import Experiment, ExperimentSavedMetric, ExperimentToSavedMetric+from products.feature_flags.backend.models.feature_flag import FeatureFlag++NO_QUERIES = ExperimentHealthReaderAccess(+ can_read_flag_history=False, can_query_events=False, denied_event_properties=frozenset()+)+Code = ExperimentHealthFindingCode+WARNING = ExperimentHealthFindingSeverity.WARNING+++def _mean(event: str, **fields: Any) -> dict[str, Any]:+ return {+ "kind": "ExperimentMetric",+ "metric_type": "mean",+ "source": {"kind": "EventsNode", "event": event},+ **fields,+ }+++class TestEventsReceived(BaseTest):+ @parameterized.expand(+ [+ (+ "custom_exposure_event",+ {"event": "checkout_viewed", "properties": []},+ ("exposure_event_not_ingested", {"event": "checkout_viewed"}),+ ),+ (+ "flag_call_filter_on_another_flag",+ {+ "event": "$feature_flag_called",+ "properties": [+ {"key": "$feature_flag", "type": "event", "value": ["pricing-test"], "operator": "exact"}+ ],+ },+ ("filter_on_another_flag", {"flag_key": "checkout-test", "filter_flag_keys": "pricing-test"}),+ ),+ ]+ )+ def test_reads_the_events_the_project_received(+ self, _name: str, exposure_config: dict[str, Any], expected_exposure: tuple[str, dict[str, str]]+ ) -> None:+ sibling = Team.objects.create(organization=self.organization, project=self.project, name="Sibling")+ EventDefinition.objects.create(team=sibling, project=self.project, name="purchase", last_seen_at=timezone.now())+ # Wizard writes a definition without `last_seen_at` for an event it planned.+ EventDefinition.objects.create(team=self.team, project=self.project, name="signup", last_seen_at=None)+ for name in ("refund", "upgrade"):+ with self.captureOnCommitCallbacks(execute=True):+ log_activity(+ organization_id=self.organization.id,+ team_id=sibling.id,+ user=self.user,+ was_impersonated=False,+ item_id=name,+ scope="EventDefinition",+ activity="deleted",+ detail=Detail(name=name, changes=None),+ )+ ActivityLog.objects.filter(item_id="upgrade").update(created_at=timezone.now() - timedelta(days=4))+ flag = FeatureFlag.objects.create(team=self.team, key="checkout-test", created_by=self.user)+ experiment = Experiment.objects.create(+ team=self.team,+ name="Checkout",+ feature_flag=flag,+ start_date=timezone.now() - timedelta(days=2),+ metrics=[_mean("purchase", uuid="revenue", name="Revenue")],+ metrics_secondary=[+ _mean("signup", uuid="signups"),+ _mean("refund", uuid="refunds"),+ _mean("upgrade", uuid="upgrades"),+ ],+ exposure_criteria={"exposure_config": {"kind": "ExperimentEventExposureConfig", **exposure_config}},+ )+ shared = ExperimentSavedMetric.objects.create(+ team=self.team, name="Checkout started", created_by=self.user, query=_mean("checkout_started", uuid="c")+ )+ ExperimentToSavedMetric.objects.create(experiment=experiment, saved_metric=shared, metadata={"type": "primary"})++ health = get_experiment_health(team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES)++ findings = {finding.code: finding for finding in health.findings}+ metric = findings[Code.METRIC_EVENT_NOT_INGESTED]+ self.assertEqual(metric.severity, WARNING)+ self.assertEqual(+ dict(metric.evidence),+ {+ "metric_1": "Mean signup",+ "metric_1_role": "secondary",+ "metric_1_events": "signup",+ "metric_2": "Mean upgrade",+ "metric_2_role": "secondary",+ "metric_2_events": "upgrade",+ "metric_3": "Checkout started",+ "metric_3_role": "primary",+ "metric_3_events": "checkout_started",+ },+ )+ exposure = findings[Code.EXPOSURE_CRITERIA_CANNOT_MATCH]+ self.assertEqual((exposure.subcode, dict(exposure.evidence)), expected_exposure)+ self.assertEqual(exposure.severity, WARNING)++ def test_shared_metrics_add_no_queries(self) -> None:+ flag = FeatureFlag.objects.create(team=self.team, key="checkout-test", created_by=self.user)+ experiment = Experiment.objects.create(team=self.team, name="Checkout", feature_flag=flag)++ query_counts = []+ for number in range(3):+ shared = ExperimentSavedMetric.objects.create(+ team=self.team,+ name=f"Shared {number}",+ created_by=self.user,+ query=_mean(f"event_{number}", uuid=str(number)),+ )+ ExperimentToSavedMetric.objects.create(+ experiment=experiment, saved_metric=shared, metadata={"type": "primary"}+ )+ with CaptureQueriesContext(connection) as queries:+ get_experiment_health(team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES)+ query_counts.append(len(queries))++ self.assertEqual(len(set(query_counts)), 1, query_counts)diff --git a/products/experiments/backend/health/test/test_exposure_criteria.py b/products/experiments/backend/health/test/test_exposure_criteria.pynew file mode 100644index 000000000000..ec32774cae6d--- /dev/null+++ b/products/experiments/backend/health/test/test_exposure_criteria.py@@ -0,0 +1,82 @@+from dataclasses import replace+from typing import Any++from unittest import TestCase++from parameterized import parameterized++from posthog.schema import PropertyOperator++from products.experiments.backend.facade.contracts import ExperimentHealthFindingSeverity+from products.experiments.backend.health.checks.exposure_criteria import (+ EXPOSURE_EVENT_NOT_INGESTED,+ FILTER_ON_ANOTHER_FLAG,+ exposure_criteria_cannot_match,+)+from products.experiments.backend.health.context import ExposureEvents, FlagKeyFilter, HealthContext+from products.experiments.backend.health.test.factories import DRAFT, ENDED, RUNNING_CONTEXT++Severity = ExperimentHealthFindingSeverity+EXACT = PropertyOperator.EXACT+++def _flag_filter(*values: str | float | bool, operator: PropertyOperator = EXACT, **changes: Any) -> HealthContext:+ events = ExposureEvents(+ custom_event=None,+ required_flag_key="checkout-test",+ flag_key_filters=(FlagKeyFilter(operator=operator, values=values),),+ )+ return replace(RUNNING_CONTEXT, exposure_events=events, **changes)+++def _custom_event(received: dict[str, bool], **changes: Any) -> HealthContext:+ events = ExposureEvents(custom_event="checkout_viewed", required_flag_key=None, flag_key_filters=())+ return replace(RUNNING_CONTEXT, exposure_events=events, events_received=received, **changes)+++class TestExposureCriteriaCannotMatch(TestCase):+ @parameterized.expand(+ [+ ("another_flag", _flag_filter("pricing-test"), (FILTER_ON_ANOTHER_FLAG, Severity.WARNING)),+ (+ "another_flag_in_a_draft",+ _flag_filter("pricing-test", **DRAFT),+ (FILTER_ON_ANOTHER_FLAG, Severity.WARNING),+ ),+ (+ "another_flag_after_the_end",+ _flag_filter("pricing-test", **ENDED),+ (FILTER_ON_ANOTHER_FLAG, Severity.WARNING),+ ),+ ("one_value_is_the_flag", _flag_filter("pricing-test", "checkout-test"), None),+ ("the_key_in_another_case", _flag_filter("Checkout-Test"), (FILTER_ON_ANOTHER_FLAG, Severity.WARNING)),+ ("no_values_match_every_event", _flag_filter(), None),+ ("a_number_can_be_converted", _flag_filter(7), None),+ ("not_exact", _flag_filter("pricing-test", operator=PropertyOperator.IS_NOT), None),+ ("archived", _flag_filter("pricing-test", archived=True), None),+ (+ "custom_event_in_a_draft",+ _custom_event({"checkout_viewed": False}, **DRAFT, hours_since_launch=None),+ (EXPOSURE_EVENT_NOT_INGESTED, Severity.INFO),+ ),+ (+ "custom_event_in_the_first_day",+ _custom_event({"checkout_viewed": False}, hours_since_launch=23.9),+ (EXPOSURE_EVENT_NOT_INGESTED, Severity.INFO),+ ),+ (+ "custom_event_a_day_after_launch",+ _custom_event({"checkout_viewed": False}, hours_since_launch=24),+ (EXPOSURE_EVENT_NOT_INGESTED, Severity.WARNING),+ ),+ ("custom_event_received", _custom_event({"checkout_viewed": True}), None),+ ("custom_event_without_a_definition", _custom_event({}), None),+ ("custom_event_after_the_end", _custom_event({"checkout_viewed": False}, **ENDED), None),+ ]+ )+ def test_exposure_criteria_cannot_match(+ self, _name: str, ctx: HealthContext, expected: tuple[str, Severity] | None+ ) -> None:+ finding = exposure_criteria_cannot_match(ctx)++ self.assertEqual((finding.subcode, finding.severity) if finding else None, expected)diff --git a/products/experiments/backend/health/test/test_metric_event_not_ingested.py b/products/experiments/backend/health/test/test_metric_event_not_ingested.pynew file mode 100644index 000000000000..8337d6fefdb0--- /dev/null+++ b/products/experiments/backend/health/test/test_metric_event_not_ingested.py@@ -0,0 +1,69 @@+from dataclasses import replace+from typing import Any++from unittest import TestCase++from parameterized import parameterized++from products.experiments.backend.facade.contracts import ExperimentHealthFindingSeverity+from products.experiments.backend.health.checks.metric_event_not_ingested import metric_event_not_ingested+from products.experiments.backend.health.context import HealthContext, MetricEvents+from products.experiments.backend.health.test.factories import DRAFT, ENDED, RUNNING_CONTEXT++Severity = ExperimentHealthFindingSeverity+REVENUE = MetricEvents(name="Revenue", role="primary", events=("purchase",))+SIGNUPS = MetricEvents(name="Signups", role="secondary", events=("signup",))+++def _context(*metrics: MetricEvents, received: dict[str, bool], **changes: Any) -> HealthContext:+ return replace(RUNNING_CONTEXT, metric_events=metrics, events_received=received, **changes)+++class TestMetricEventNotIngested(TestCase):+ @parameterized.expand(+ [+ ("draft", _context(REVENUE, received={"purchase": False}, **DRAFT, hours_since_launch=None), Severity.INFO),+ (+ "first_day_after_launch",+ _context(REVENUE, received={"purchase": False}, hours_since_launch=23.9),+ Severity.INFO,+ ),+ (+ "a_day_after_launch",+ _context(REVENUE, received={"purchase": False}, hours_since_launch=24),+ Severity.WARNING,+ ),+ ("only_a_secondary_metric", _context(SIGNUPS, received={"signup": False}), Severity.INFO),+ ("received", _context(REVENUE, received={"purchase": True}), None),+ ("a_name_without_a_definition_is_left_out", _context(REVENUE, received={}), None),+ ("ended", _context(REVENUE, received={"purchase": False}, **ENDED), None),+ ("archived", _context(REVENUE, received={"purchase": False}, archived=True), None),+ ]+ )+ def test_severity_follows_the_state(self, _name: str, ctx: HealthContext, expected: Severity | None) -> None:+ finding = metric_event_not_ingested(ctx)++ self.assertEqual(finding.severity if finding else None, expected)++ def test_names_each_metric_and_its_missing_events(self) -> None:+ funnel = MetricEvents(name="Checkout", role="primary", events=("cart", "pay", "view"))+ ctx = _context(funnel, SIGNUPS, received={"cart": False, "pay": True, "view": False, "signup": False})++ finding = metric_event_not_ingested(ctx)++ assert finding is not None+ self.assertEqual(finding.title, "2 metrics use events this project hasn't received")+ self.assertEqual(+ dict(finding.evidence),+ {+ "metric_1": "Checkout",+ "metric_1_role": "primary",+ "metric_1_events": "cart, view",+ "metric_2": "Signups",+ "metric_2_role": "secondary",+ "metric_2_events": "signup",+ },+ )+ self.assertIn('the primary metric "Checkout" uses "cart" and "view"', finding.detail)+ self.assertIn('the secondary metric "Signups" uses "signup"', finding.detail)+ self.assertNotIn('"pay"', finding.detail)diff --git a/products/experiments/backend/health/test/test_registry.py b/products/experiments/backend/health/test/test_registry.pyindex 98fb6f224292..03c8023826f4 100644--- a/products/experiments/backend/health/test/test_registry.py+++ b/products/experiments/backend/health/test/test_registry.py@@ -16,11 +16,13 @@ from products.experiments.backend.facade.contracts import ()from products.experiments.backend.health import registryfrom products.experiments.backend.health.context import (+ ExposureEvents,ExposureTotals,FlagHistoryNotCheckedReason,FlagState,FlagVariant,HealthContext,+ MetricEvents,)from products.experiments.backend.health.registry import (EXPERIMENT_HEALTH_CHECKS,@@ -59,6 +61,17 @@ Key = ExperimentHealthCheckKeyStatus = ExperimentHealthCheckStatusNOT_LOOKED = (Status.NOT_APPLICABLE, Status.NOT_CHECKED)FLAG_CHANGE_KEYS = (Key.SPLIT_CHANGED_MID_RUN, Key.BUCKETING_CHANGED_MID_RUN, Key.RELEASE_CONDITIONS_CHANGED_MID_RUN)+EVENT_KEYS = (Key.METRIC_EVENT_NOT_INGESTED, Key.EXPOSURE_CRITERIA_CANNOT_MATCH)+++def _events(name: str, received: dict[str, bool]) -> dict[str, Any]:+ return {+ "metric_events": (MetricEvents(name="Metric", role="primary", events=(name,)),),+ "exposure_events": ExposureEvents(custom_event=name, required_flag_key=None, flag_key_filters=()),+ "events_received": received,+ }++EVEN_SPLIT_FLAG = FlagState(active=True,deleted=False,@@ -156,6 +169,8 @@ class TestEvaluate(TestCase):Key.FLAG_HISTORY: Status.NOT_APPLICABLE,**dict.fromkeys(FLAG_CHANGE_KEYS, Status.NOT_APPLICABLE),Key.NO_METRIC: Status.NOT_APPLICABLE,+ Key.METRIC_EVENT_NOT_INGESTED: Status.NOT_APPLICABLE,+ Key.EXPOSURE_CRITERIA_CANNOT_MATCH: Status.PASSED,Key.ZERO_EXPOSURES: Status.NOT_APPLICABLE,Key.SRM: Status.NOT_APPLICABLE,Key.MULTIPLE_VARIANTS_SHARE: Status.NOT_APPLICABLE,@@ -199,6 +214,16 @@ class TestEvaluate(TestCase):{"exposures": replace(NO_EXPOSURES, hours_since_launch=24)},{Key.ZERO_EXPOSURES: Status.FINDING},),+ (+ "events_not_received_are_not_reported_after_the_end",+ {**ENDED, **_events("purchase", {"purchase": False})},+ dict.fromkeys(EVENT_KEYS, Status.NOT_APPLICABLE),+ ),+ (+ "event_names_that_get_no_definition_are_not_checked",+ _events("$$plugin_metrics", {}),+ dict.fromkeys(EVENT_KEYS, Status.NOT_CHECKED),+ ),])def test_check_statuses(self, _name: str, changes: dict[str, Any], expected: dict[Key, Status]) -> None:diff --git a/products/experiments/backend/metric_validation.py b/products/experiments/backend/metric_validation.pyindex bcc3506d158e..d9c12cdcc1b1 100644--- a/products/experiments/backend/metric_validation.py+++ b/products/experiments/backend/metric_validation.py@@ -7,6 +7,7 @@ checks that need a user or that the caller can waive (team ownership, permissionevent exists) stay with the callers."""+from collections.abc import Collectionfrom typing import Anyimport pydantic@@ -27,6 +28,7 @@ from posthog.models.user import Userfrom products.access_control.backend.facade.user_access_control import UserAccessControlfrom products.actions.backend.models.action import Action+from products.event_definitions.backend.models import EventDefinition, effective_project_id_exprfrom products.experiments.backend.hogql_queries.base_query_utils import is_threshold_supported_mathfrom products.experiments.backend.hogql_queries.funnel_validation import FunnelDWValidatorfrom products.experiments.backend.hogql_queries.retention_validation import retention_metric_error@@ -222,6 +224,30 @@ def extract_entity_nodes(metrics: list[dict] | None) -> tuple[set[str], set[int]return event_names, action_ids+def defined_event_names(project_id: int, event_names: Collection[str], *, seen_only: bool = False) -> set[str]:+ """The names in `event_names` that the project has an event definition for.++ Ingestion writes a definition when it first sees an event, and Wizard writes one for each event it plans, so the+ picker lists planned events before the code that sends them ships. `seen_only` leaves out a definition that+ ingestion has not seen yet, which has no `last_seen_at`.++ The scope is the project, as in the event picker: the EventDefinition list endpoint is project-scoped (see+ posthog/api/event_definition.py), so a user in a multi-team project can pick an event ingested by a sibling team.+ """+ if not event_names:+ return set()+ # `COALESCE(project_id, team_id)` falls back to `team_id` (not the caller's team id) on purpose: legacy+ # EventDefinitions (project_id IS NULL) belong to the *primary* team, and primary_team.id == project.id by+ # convention. This mirrors the picker SQL in posthog/api/event_definition.py so sibling teams can validate+ # against legacy primary-team events the picker shows.+ definitions = EventDefinition.objects.alias(effective_project_id=effective_project_id_expr()).filter(+ effective_project_id=project_id, name__in=event_names+ )+ if seen_only:+ definitions = definitions.filter(last_seen_at__isnull=False)+ return set(definitions.values_list("name", flat=True))++def validate_metric_action_ids(metrics: list[dict] | None,team: Team,diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.pyindex afbc548b4f85..99fb5f8116ba 100644--- a/products/experiments/backend/presentation/views.py+++ b/products/experiments/backend/presentation/views.py@@ -925,6 +925,19 @@ class EnterpriseExperimentsViewSet(flag and, with an API key, `activity_log:read`) gets that finding for every runningexperiment, because the changes are not read.+ Two checks read whether the project received an event from its event definitions: a+ definition without `last_seen_at`, such as one Wizard wrote for a planned event, counts as+ not received. A name whose definition was deleted since launch, or in the last day for a+ draft, is not checked, because ingestion writes the definition again only when the event+ arrives. `metric_event_not_ingested` names each metric that+ reads an event the project never received, in the `evidence` keys `metric_N`,+ `metric_N_role` and `metric_N_events`. `exposure_criteria_cannot_match` reports a custom+ exposure event the project never received (`exposure_event_not_ingested`), and a+ `$feature_flag` filter on the flag call events that requires another flag+ (`filter_on_another_flag`). An event not received is info before launch, in the first 24+ hours after launch, and when only secondary metrics read it, and a warning otherwise.+ `filter_on_another_flag` is a warning in every state, and the other findings stop at the end.+The severity of the `some_conditions_pinned` finding follows the largest share that theusers of the pinned release conditions make up among the users who got one variant fromthe flag, in the last 7 days, or since launch or the flag's last save if later. It readsdiff --git a/products/experiments/backend/test/test_experiment_service.py b/products/experiments/backend/test/test_experiment_service.pyindex 28824347db30..58ae7b8db316 100644--- a/products/experiments/backend/test/test_experiment_service.py+++ b/products/experiments/backend/test/test_experiment_service.py@@ -2814,6 +2814,9 @@ class TestExperimentService(APIBaseTest):kwargs.setdefault("metrics", [self._DEFAULT_METRIC])kwargs.setdefault("primary_metrics_ordered_uuids", ["m1"])kwargs.setdefault("allow_unknown_events", True)+ EventDefinition.objects.get_or_create(+ team=self.team, project=self.project, name="$pageview", defaults={"last_seen_at": timezone.now()}+ )return self._service().create_experiment(name=name, feature_flag_key=feature_flag_key, **kwargs)def _create_launchable_experiment_on_flag_of_age(self, age: timedelta, **kwargs: Any) -> Experiment:diff --git a/products/experiments/backend/test/test_presentation_api.py b/products/experiments/backend/test/test_presentation_api.pyindex 14da9bf28115..0c848b069181 100644--- a/products/experiments/backend/test/test_presentation_api.py+++ b/products/experiments/backend/test/test_presentation_api.py@@ -791,6 +791,9 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):flag_edit: tuple[timedelta, dict[str, Any]] | None = None,) -> None:start_date = timezone.now() - timedelta(days=3)+ EventDefinition.objects.create(+ team=self.team, project=self.project, name="$pageview", last_seen_at=timezone.now()+ )flag = FeatureFlag.objects.create(team=self.team,key=f"health-findings-{_name}",diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.tsindex 130663b5cb28..4ada98e070eb 100644--- a/products/experiments/frontend/generated/api.schemas.ts+++ b/products/experiments/frontend/generated/api.schemas.ts@@ -2270,6 +2270,8 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded* * `multiple_variants_share` - Multiple Variants Share* * `no_metric` - No Metric+ * * `metric_event_not_ingested` - Metric Event Not Ingested+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition@@ -2290,6 +2292,8 @@ export const ExperimentHealthFindingCodeEnumApi = {BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',MultipleVariantsShare: 'multiple_variants_share',NoMetric: 'no_metric',+ MetricEventNotIngested: 'metric_event_not_ingested',+ ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',Srm: 'srm',ZeroExposures: 'zero_exposures',ForcedVariantReleaseCondition: 'forced_variant_release_condition',@@ -2353,6 +2357,8 @@ export interface ExperimentHealthFindingApi {* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded* * `multiple_variants_share` - Multiple Variants Share* * `no_metric` - No Metric+ * * `metric_event_not_ingested` - Metric Event Not Ingested+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition@@ -2398,6 +2404,8 @@ export interface ExperimentHealthFindingApi {* * `forced_variant_release_condition` - Forced Variant Release Condition* * `new_flag_no_traffic` - New Flag No Traffic* * `no_metric` - No Metric+ * * `metric_event_not_ingested` - Metric Event Not Ingested+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `zero_exposures` - Zero Exposures* * `srm` - Srm* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded@@ -2416,6 +2424,8 @@ export const ExperimentHealthCheckKeyEnumApi = {ForcedVariantReleaseCondition: 'forced_variant_release_condition',NewFlagNoTraffic: 'new_flag_no_traffic',NoMetric: 'no_metric',+ MetricEventNotIngested: 'metric_event_not_ingested',+ ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',ZeroExposures: 'zero_exposures',Srm: 'srm',BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',@@ -2450,6 +2460,8 @@ export interface ExperimentHealthCheckResultApi {* * `forced_variant_release_condition` - Forced Variant Release Condition* * `new_flag_no_traffic` - New Flag No Traffic* * `no_metric` - No Metric+ * * `metric_event_not_ingested` - Metric Event Not Ingested+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `zero_exposures` - Zero Exposures* * `srm` - Srm* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excludeddiff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.tsindex 0ee63c6ea6d6..8692d7dec371 100644--- a/products/experiments/frontend/generated/api.ts+++ b/products/experiments/frontend/generated/api.ts@@ -713,6 +713,19 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =* flag and, with an API key, `activity_log:read`) gets that finding for every running* experiment, because the changes are not read.*+ * Two checks read whether the project received an event from its event definitions: a+ * definition without `last_seen_at`, such as one Wizard wrote for a planned event, counts as+ * not received. A name whose definition was deleted since launch, or in the last day for a+ * draft, is not checked, because ingestion writes the definition again only when the event+ * arrives. `metric_event_not_ingested` names each metric that+ * reads an event the project never received, in the `evidence` keys `metric_N`,+ * `metric_N_role` and `metric_N_events`. `exposure_criteria_cannot_match` reports a custom+ * exposure event the project never received (`exposure_event_not_ingested`), and a+ * `$feature_flag` filter on the flag call events that requires another flag+ * (`filter_on_another_flag`). An event not received is info before launch, in the first 24+ * hours after launch, and when only secondary metrics read it, and a warning otherwise.+ * `filter_on_another_flag` is a warning in every state, and the other findings stop at the end.+ ** The severity of the `some_conditions_pinned` finding follows the largest share that the* users of the pinned release conditions make up among the users who got one variant from* the flag, in the last 7 days, or since launch or the flag's last save if later. It readsdiff --git a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx b/products/experiments/frontend/health/HealthChecksOverview.stories.tsxindex 329bfa78bbc3..8f52d36e0965 100644--- a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx+++ b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx@@ -64,6 +64,50 @@ export const Draft: Story = {},}+const found = (key: HealthPanelCheck['key'], name: string): HealthPanelCheck => ({+ key,+ name,+ status: 'finding',+ reason: null,+})++export const DraftWithEventsNotReceived: Story = {+ args: {+ checks: [+ ...FLAG_SETUP_CHECKS,+ skipped('no_metric', 'Metrics', 'not_applicable', 'Runs after launch'),+ found('metric_event_not_ingested', 'Metric events'),+ found('exposure_criteria_cannot_match', 'Exposure criteria'),+ ],+ notes: [+ {+ code: 'metric_event_not_ingested',+ subcode: null,+ severity: 'info',+ title: "A metric uses an event this project hasn't received",+ detail: 'The primary metric "Revenue" uses the event "purchase_completed", which this project hasn\'t received yet. Until it arrives, the metric can\'t compare the variants. Launch the experiment after your code starts sending it.',+ evidence: {+ metric_1: 'Revenue',+ metric_1_role: 'primary',+ metric_1_events: 'purchase_completed',+ },+ actions: [],+ },+ {+ code: 'exposure_criteria_cannot_match',+ subcode: 'exposure_event_not_ingested',+ severity: 'info',+ title: "This project hasn't received the exposure event",+ detail: 'The experiment counts users as exposed when they send the event "checkout_viewed", which this project hasn\'t received yet. Until it arrives, nobody is counted as exposed. Launch the experiment after your code starts sending it.',+ evidence: { event: 'checkout_viewed' },+ actions: ['edit_exposure_criteria'],+ },+ ],+ isDraft: true,+ exposureChecksMissing: false,+ },+}+export const ChecksThatCouldNotRun: Story = {args: {checks: [diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.tsindex 4320289f4f84..cc2ab8656f76 100644--- a/products/experiments/frontend/health/debug/healthDebugReport.ts+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts@@ -151,13 +151,14 @@ function share(value: unknown): string | null {function serverOnlyCheck(input: HealthDebugInput,check: ExperimentHealthFindingApi['code'],- note: ((finding: ExperimentHealthFindingApi) => string) | null+ note: ((finding: ExperimentHealthFindingApi) => string) | null,+ reads: HealthDebugDomain[] = ['experiment', 'flag']): HealthDebugCheck {const finding = findingOf(input, check)return {check,source: 'health endpoint',- reads: ['experiment', 'flag'],+ reads,...serverResult(check, input.health, (code) => code === check),browser: null,browserLabel: BROWSER_RULES,@@ -225,6 +226,19 @@ function forcedVariantNote(finding: ExperimentHealthFindingApi): string {return parts.join(' ')}+function metricEventNote({ evidence }: ExperimentHealthFindingApi): string {+ const events = Object.entries(evidence)+ .filter(([key]) => /^metric_\d+_events$/.test(key))+ .map(([, value]) => value)+ return `Not received: ${events.join(', ')}.`+}++function exposureCriteriaNote({ subcode, evidence }: ExperimentHealthFindingApi): string {+ return subcode === 'filter_on_another_flag'+ ? `The query requires ${evidence.flag_key}, a filter requires ${evidence.filter_flag_keys}.`+ : `Not received: ${evidence.event}.`+}+function noMetricCheck(input: HealthDebugInput): HealthDebugCheck {const serverFires = !!findingOf(input, 'no_metric')const differs = !!input.health && input.browserNoMetricsWarning !== serverFires@@ -317,6 +331,8 @@ export function buildHealthDebugChecks(input: HealthDebugInput): HealthDebugChecserverOnlyCheck(input, 'forced_variant_release_condition', forcedVariantNote),serverOnlyCheck(input, 'new_flag_no_traffic', null),noMetricCheck(input),+ serverOnlyCheck(input, 'metric_event_not_ingested', metricEventNote, ['metrics', 'project']),+ serverOnlyCheck(input, 'exposure_criteria_cannot_match', exposureCriteriaNote, ['experiment', 'project']),...exposureChecks(input),]}diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.tsindex e039b19dcffd..cb7ee6db812a 100644--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts@@ -23,6 +23,8 @@ export type ExperimentHealthFindingCode =| 'srm'| 'zero_exposures'| 'no_metric'+ | 'metric_event_not_ingested'+ | 'exposure_criteria_cannot_match'| 'forced_variant_release_condition'| 'flag_variants_mismatch'| 'split_changed_mid_run'diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex c177e394ef39..2063167f66f7 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -44645,6 +44645,8 @@ export namespace Schemas {* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded* * `multiple_variants_share` - Multiple Variants Share* * `no_metric` - No Metric+ * * `metric_event_not_ingested` - Metric Event Not Ingested+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition@@ -44665,6 +44667,8 @@ export namespace Schemas {BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',MultipleVariantsShare: 'multiple_variants_share',NoMetric: 'no_metric',+ MetricEventNotIngested: 'metric_event_not_ingested',+ ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',Srm: 'srm',ZeroExposures: 'zero_exposures',ForcedVariantReleaseCondition: 'forced_variant_release_condition',@@ -44728,6 +44732,8 @@ export namespace Schemas {* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded* * `multiple_variants_share` - Multiple Variants Share* * `no_metric` - No Metric+ * * `metric_event_not_ingested` - Metric Event Not Ingested+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition@@ -44773,6 +44779,8 @@ export namespace Schemas {* * `forced_variant_release_condition` - Forced Variant Release Condition* * `new_flag_no_traffic` - New Flag No Traffic* * `no_metric` - No Metric+ * * `metric_event_not_ingested` - Metric Event Not Ingested+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `zero_exposures` - Zero Exposures* * `srm` - Srm* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded@@ -44791,6 +44799,8 @@ export namespace Schemas {ForcedVariantReleaseCondition: 'forced_variant_release_condition',NewFlagNoTraffic: 'new_flag_no_traffic',NoMetric: 'no_metric',+ MetricEventNotIngested: 'metric_event_not_ingested',+ ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',ZeroExposures: 'zero_exposures',Srm: 'srm',BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',@@ -44825,6 +44835,8 @@ export namespace Schemas {* * `forced_variant_release_condition` - Forced Variant Release Condition* * `new_flag_no_traffic` - New Flag No Traffic* * `no_metric` - No Metric+ * * `metric_event_not_ingested` - Metric Event Not Ingested+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `zero_exposures` - Zero Exposures* * `srm` - Srm* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded