:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): name the causes of srm and zero-exposure findings (#125146)
Marcel Poelkere07cd6e881a6
diff --git a/frontend/src/lib/utils/posthogIdentity.ts b/frontend/src/lib/utils/posthogIdentity.tsindex 1f9cfba4c647..7f0d4d4e3cc0 100644--- a/frontend/src/lib/utils/posthogIdentity.ts+++ b/frontend/src/lib/utils/posthogIdentity.ts@@ -17,7 +17,7 @@ interface UserIdentity {/*** Feature flags target these person and group properties, so this list also decides what a user's flags can use.- * `_flag_identity` in `posthog/utils.py` builds the same inputs for the server flag bootstrap. Change both together.+ * `flag_identity` in `posthog/utils.py` builds the same inputs for the server flag bootstrap. Change both together.*/function userIdentity(user: UserType): UserIdentity {const groups: PostHogGroup[] = []diff --git a/frontend/src/scenes/experiments/experimentLogic.test.ts b/frontend/src/scenes/experiments/experimentLogic.test.tsindex a1817f6d8d44..0facbd4c3c92 100644--- a/frontend/src/scenes/experiments/experimentLogic.test.ts+++ b/frontend/src/scenes/experiments/experimentLogic.test.ts@@ -131,13 +131,18 @@ describe('experimentLogic', () => {total_exposures: { control: 600, test: 350, $multiple: 50 },sample_ratio_mismatch: { expected: { control: 475, test: 475 }, p_value: 0.0001 },bias_risk: { multiple_variant_percentage: 5 },- health_findings: [{ code: 'srm', subcode: 'split_changed' }, { code: 'bias_risk_multiple_excluded' }],+ health_findings: [+ { code: 'srm', subcode: 'split_changed', evidence: { p_value: 0.0001, reason: 'library' } },+ { code: 'bias_risk_multiple_excluded' },+ ],}const noServerVerdicts = {srm_server: null,- srm_cause: null,+ srm_subcode: null,+ srm_reason: null,zero_exposures_server: null,- zero_exposures_cause: null,+ zero_exposures_subcode: null,+ zero_exposures_reason: null,bias_risk_server: null,}@@ -177,8 +182,11 @@ describe('experimentLogic', () => {has_srm: true,has_bias_risk: true,srm_server: true,- srm_cause: 'split_changed',+ srm_subcode: 'split_changed',+ srm_reason: 'library',zero_exposures_server: false,+ zero_exposures_subcode: null,+ zero_exposures_reason: null,bias_risk_server: true,},},@@ -205,14 +213,23 @@ describe('experimentLogic', () => {exposures: {timeseries: [{ variant: 'control' }, { variant: 'test' }],total_exposures: { control: 0, test: 0 },- health_findings: [{ code: 'zero_exposures', subcode: 'flag_not_called' }],+ health_findings: [+ {+ code: 'zero_exposures',+ subcode: 'calls_not_counted',+ evidence: { hours_since_launch: 48, reason: 'test_account_filter' },+ },+ ],},handling: undefined,expected: {health_ui: 'warnings',exposures_total: 0,+ srm_subcode: null,+ srm_reason: null,zero_exposures_server: true,- zero_exposures_cause: 'flag_not_called',+ zero_exposures_subcode: 'calls_not_counted',+ zero_exposures_reason: 'test_account_filter',},},])(diff --git a/posthog/test/test_get_context_for_template.py b/posthog/test/test_get_context_for_template.pyindex c9af790aab55..c7b9ec5854c4 100644--- a/posthog/test/test_get_context_for_template.py+++ b/posthog/test/test_get_context_for_template.py@@ -264,6 +264,7 @@ class TestGetContextForTemplate(APIBaseTest):self.user.anonymize_data = anonymize_dataself.user.save()self.organization.customer_id = "cus_123"+ self.organization.available_product_features = Noneself.organization.save()request = RequestFactory().get("/")SessionMiddleware(lambda _request: HttpResponse()).process_request(request)@@ -297,7 +298,7 @@ class TestGetContextForTemplate(APIBaseTest):"name": self.organization.name,"slug": self.organization.slug,"created_at": mock.ANY,- "available_product_features": self.organization.available_product_features or [],+ "available_product_features": None,"member_count": 1,"project_count": 1,"instance_tag": settings.INSTANCE_TAG,diff --git a/posthog/utils.py b/posthog/utils.pyindex c17ae26a81c7..752f95521918 100644--- a/posthog/utils.py+++ b/posthog/utils.py@@ -80,7 +80,7 @@ TEMPLATE_CONTEXT_DURATION_HISTOGRAM = Histogram(if TYPE_CHECKING:from django.contrib.auth.models import AbstractBaseUser, AnonymousUser- from posthog.models import Team, User+ from posthog.models import Organization, Team, Userfrom products.dashboards.backend.models.dashboard import Dashboardfrom products.dashboards.backend.models.dashboard_tile import DashboardTile@@ -515,53 +515,70 @@ def _app_evaluates_flag(definition: dict[str, Any]) -> bool:@frozen-class _FlagIdentity:+class FlagIdentity:+ """The inputs the web app sends to /flags for a signed-in user, for a local evaluation of the app's flags."""++ distinct_id: strperson_properties: dict[str, Any]groups: dict[str, str]group_properties: dict[str, dict[str, Any]]-def _flag_identity(current_user: dict[str, Any], joined_at: str) -> _FlagIdentity:+def flag_identity(+ user: "User",+ team: "Team | None",+ organization: "Organization | None",+ *,+ member_count: int | None = None,+ project_count: int | None = None,+) -> FlagIdentity | None:"""Mirrors `userIdentity()` in `frontend/src/lib/utils/posthogIdentity.ts`, which sends these inputs to /flags.Local evaluation drops any flag it cannot decide from them, and decides a flag differently when they differ.+ The organization serializer computes `member_count` and `project_count`, so a caller without it leaves them+ out, and a flag that reads them stays undecided. None for a user without a distinct id, who gets no flags."""+ # A worker loads the user with the fields it needs, and Django reads each other field in a query of its own.+ deferred = {"distinct_id", "email", "anonymize_data", "date_joined"} & user.get_deferred_fields()+ if deferred:+ user.refresh_from_db(fields=sorted(deferred))+ if not user.distinct_id:+ return Nonegroups: dict[str, str] = {}group_properties: dict[str, dict[str, Any]] = {}- organization = current_user.get("organization")- team = current_user.get("team")- metadata = (organization.get("metadata") or {}) if organization else {}- if team:- groups["project"] = team["uuid"]+ if team is not None:+ groups["project"] = str(team.uuid)group_properties["project"] = {- "id": team["id"],- "uuid": team["uuid"],- "name": team["name"],- "ingested_event": team["ingested_event"],- "is_demo": team["is_demo"],- "timezone": team["timezone"],- "instance_tag": metadata.get("instance_tag"),+ "id": team.id,+ "uuid": str(team.uuid),+ "name": team.name,+ "ingested_event": team.ingested_event,+ "is_demo": team.is_demo,+ "timezone": team.timezone,+ "instance_tag": settings.INSTANCE_TAG,}- if organization:- groups["organization"] = organization["id"]+ if organization is not None:+ groups["organization"] = str(organization.id)group_properties["organization"] = {- "id": organization["id"],- "name": organization["name"],- "slug": organization["slug"],- "created_at": organization["created_at"],- "available_product_features": organization["available_product_features"],- "member_count": organization["member_count"],- "project_count": len(organization["teams"]),- **metadata,+ "id": str(organization.id),+ "name": organization.name,+ "slug": organization.slug,+ # The serializer's datetime format.+ "created_at": organization.created_at.isoformat().replace("+00:00", "Z"),+ "available_product_features": organization.available_product_features,+ **({"member_count": member_count} if member_count is not None else {}),+ **({"project_count": project_count} if project_count is not None else {}),+ "instance_tag": settings.INSTANCE_TAG,}- if organization.get("customer_id"):- groups["customer"] = organization["customer_id"]+ if organization.customer_id:+ groups["customer"] = organization.customer_idgroups["instance"] = settings.SITE_URLgroup_properties["instance"] = {"site_url": settings.SITE_URL}- return _FlagIdentity(+ return FlagIdentity(+ distinct_id=user.distinct_id,person_properties={- "email": None if current_user["anonymize_data"] else current_user["email"],- "joined_at": joined_at,+ "email": None if user.anonymize_data else user.email,+ "joined_at": user.date_joined.isoformat(),},groups=groups,group_properties=group_properties,@@ -848,13 +865,19 @@ def deferred_app_context_variant(request: HttpRequest, user: "User") -> Optionaldef feature_flag_bootstrap(user: "User", current_user: dict[str, Any]) -> Optional[dict[str, Any]]:- distinct_id = current_user.get("distinct_id")- if not distinct_id:+ organization = current_user.get("organization")+ identity = flag_identity(+ user,+ user.team if current_user.get("team") else None,+ user.organization if organization else None,+ member_count=organization["member_count"] if organization else None,+ project_count=len(organization["teams"]) if organization else None,+ )+ if identity is None:return None- identity = _flag_identity(current_user, user.date_joined.isoformat())with tracer.start_as_current_span("template.feature_flag_bootstrap"):feature_flags = posthoganalytics.get_all_flags(- distinct_id,+ identity.distinct_id,only_evaluate_locally=True,person_properties=identity.person_properties,groups=identity.groups,diff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.pyindex 54e3e8b0d379..369a7e3e412d 100644--- a/products/experiments/backend/health/checks/srm.py+++ b/products/experiments/backend/health/checks/srm.py@@ -1,11 +1,15 @@+import unicodedatafrom collections.abc import Mapping+from enum import StrEnumfrom zoneinfo import ZoneInfo-from scipy.stats import chisquare+from scipy.stats import chi2_contingency, chisquarefrom posthog.schema import SampleRatioMismatch+from posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exception+from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_NULL_STRING_LABEL, BREAKDOWN_OTHER_STRING_LABELfrom products.experiments.backend.facade.contracts import (ExperimentHealthFinding,@@ -13,17 +17,30 @@ from products.experiments.backend.facade.contracts import (ExperimentHealthFindingSeverity,)from products.experiments.backend.health.checks.split_changed import split_changed_mid_run-from products.experiments.backend.health.context import ExposureTotals, FlagHistory, HealthContext+from products.experiments.backend.health.context import ExposureSegment, ExposureTotals, FlagHistory, HealthContextfrom products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicable+from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimensionfrom products.feature_flags.backend.facade.edits import SplitChangeSAMPLE_RATIO_MISMATCH_P_VALUE = 0.001SRM_MINIMUM_SAMPLE_SIZE = 100+TOP_SEGMENT_VALUES = 5+# Below this, a skew of a few points passes the homogeneity test for lack of power, and would read as the same skew+# everywhere. Two segments this large are needed before the finding calls a dimension uniform.+UNIFORM_SEGMENT_MINIMUM_SIZE = 1000+SEGMENT_LABEL_MAX_LENGTH = 60# pinned: the subcode is sent as `finding_variant` on the health finding events, so insights group on it.SPLIT_CHANGED = "split_changed"+class SrmReason(StrEnum):+ # pinned: sent as `finding_reason` on the health finding events, so insights group on them.+ LIBRARY = ExposureSegmentDimension.LIBRARY.value+ PAGE = ExposureSegmentDimension.PAGE.value+ UNIFORM = "uniform"++def sample_ratio_test(observed: Mapping[str, int], weights: Mapping[str, float]) -> SampleRatioMismatch | None:"""Chi-squared goodness-of-fit test of the observed counts against the weights of the variants in the split.None when there is too little data."""@@ -49,6 +66,17 @@ def reads_flag_history(ctx: HealthContext) -> bool:return p_value is not None and p_value < SAMPLE_RATIO_MISMATCH_P_VALUE+def reads_segments(ctx: HealthContext) -> bool:+ """Whether `srm` reports a mismatch that no split change explains, which it then looks for in the segments of the+ exposures. The exposure query loads the segments only then, because their query scans the exposure events."""+ # With an activation, the segments would come from the activation event, not from the flag call that assigned+ # the variant.+ if ctx.exposure_events is not None and ctx.exposure_events.activation is not None:+ return False+ outcome = srm(ctx)+ return isinstance(outcome, ExperimentHealthFinding) and outcome.subcode != SPLIT_CHANGED++def srm(ctx: HealthContext) -> CheckOutcome:exposures = ctx.exposuresif exposures is None:@@ -146,16 +174,155 @@ def _split_change_warning(ctx: HealthContext) -> ExperimentHealthFinding | None:def _mismatch(exposures: ExposureTotals, p_value: float) -> ExperimentHealthFinding:unit = "group" if exposures.counts_groups else "user"+ tests = _segment_tests(exposures)+ verdicts = [_verdict(tests, dimension, exposures.sample_ratio_expected) for dimension in ExposureSegmentDimension]+ cause, reason = _cause(verdicts, unit, exposures.sample_ratio_expected)+ evidence: dict[str, str | int | float | None] = {"p_value": p_value, **_segment_evidence(tests, verdicts)}+ if reason is not None:+ evidence["reason"] = reason.valuereturn ExperimentHealthFinding(code=ExperimentHealthFindingCode.SRM,subcode=None,severity=ExperimentHealthFindingSeverity.WARNING,title=f"{unit.capitalize()}s are not split across variants as configured",- detail=(- f"The distribution of {unit}s across variants doesn't match your configured rollout percentages "- f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}). This may indicate issues with randomization or data collection."+ detail=" ".join(+ [+ f"The distribution of {unit}s across variants doesn't match your configured rollout percentages "+ f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}).",+ *cause,+ ]),- evidence={"p_value": p_value},+ evidence=evidence,actions=(),diagnostic_ref="A2",)+++@frozen+class _SegmentTest:+ segment: ExposureSegment+ # None when the segment holds too few persons or groups for the test.+ p_value: float | None+++@frozen+class _Verdict:+ dimension: ExposureSegmentDimension+ homogeneity_p_value: float | None+ powered: bool+ located: ExposureSegment | None+++def _segment_tests(exposures: ExposureTotals) -> list[_SegmentTest]:+ """The sample ratio test of each segment against the split of the whole test, largest segment first."""+ segments = sorted(exposures.segments or (), key=lambda segment: -sum(segment.exposed_by_variant.values()))+ tests = []+ for segment in segments:+ result = sample_ratio_test(segment.exposed_by_variant, exposures.sample_ratio_expected)+ tests.append(_SegmentTest(segment=segment, p_value=result.p_value if result else None))+ return tests+++def _verdict(tests: list[_SegmentTest], dimension: ExposureSegmentDimension, weights: Mapping[str, float]) -> _Verdict:+ # A test per segment would blame the largest segment for a skew that every segment shares, because the same skew+ # reaches significance first in the most users. The homogeneity test asks whether the segments differ at all.+ compared = [test for test in tests if test.segment.dimension == dimension and test.p_value is not None]+ variants = [+ variant for variant in weights if any(test.segment.exposed_by_variant.get(variant, 0) for test in compared)+ ]+ if len(compared) < 2 or len(variants) < 2:+ return _Verdict(dimension=dimension, homogeneity_p_value=None, powered=False, located=None)+ table = [[test.segment.exposed_by_variant.get(variant, 0) for variant in variants] for test in compared]+ homogeneity_p_value = float(chi2_contingency(table).pvalue)+ powered = sum(1 for row in table if sum(row) >= UNIFORM_SEGMENT_MINIMUM_SIZE) >= 2+ if homogeneity_p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:+ return _Verdict(dimension=dimension, homogeneity_p_value=homogeneity_p_value, powered=powered, located=None)+ failing = [+ test.segment for test in compared if test.p_value is not None and test.p_value < SAMPLE_RATIO_MISMATCH_P_VALUE+ ]+ # The detail says the mismatch comes from the segment, which reads as if the other segments match the split.+ located = failing[0] if len(failing) == 1 else None+ return _Verdict(dimension=dimension, homogeneity_p_value=homogeneity_p_value, powered=powered, located=located)+++def _distance_from_split(segment: ExposureSegment, weights: Mapping[str, float]) -> float:+ """The share of the segment's persons or groups that would have to move between variants to match the split."""+ total = sum(segment.exposed_by_variant.get(variant, 0) for variant in weights)+ total_weight = sum(weights.values())+ return (+ sum(+ abs(segment.exposed_by_variant.get(variant, 0) / total - weight / total_weight)+ for variant, weight in weights.items()+ )+ / 2+ )+++def _cause(verdicts: list[_Verdict], unit: str, weights: Mapping[str, float]) -> tuple[list[str], SrmReason | None]:+ """The sentences that follow the mismatch, and the finding's reason: the dimension of the segment furthest from+ the split."""+ located = sorted(+ (verdict.located for verdict in verdicts if verdict.located is not None),+ key=lambda segment: -_distance_from_split(segment, weights),+ )+ if located:+ places = ", and from ".join(f"{unit}s {_segment_place(segment)}" for segment in located)+ return [+ f"The mismatch comes from {places}.",+ f"Check that the code there sends the exposure for every {unit} in every variant.",+ ], SrmReason(located[0].dimension.value)+ compared = {+ verdict.dimension: verdict.homogeneity_p_value+ for verdict in verdicts+ if verdict.homogeneity_p_value is not None+ }+ powered = [verdict.dimension for verdict in verdicts if verdict.powered]+ if powered and all(p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE for p_value in compared.values()):+ across = " and ".join(+ "SDK libraries" if dimension == ExposureSegmentDimension.LIBRARY else "pages" for dimension in powered+ )+ return [+ f"The mismatch looks the same across {across}, which points at how {unit}s get their variants rather "+ "than at one integration."+ ], SrmReason.UNIFORM+ return ["This may indicate issues with randomization or data collection."], None+++def _segment_place(segment: ExposureSegment) -> str:+ if segment.dimension == ExposureSegmentDimension.LIBRARY:+ if segment.value == BREAKDOWN_OTHER_STRING_LABEL:+ return f"whose first exposure was sent by a library other than the {TOP_SEGMENT_VALUES} most common"+ if segment.value == BREAKDOWN_NULL_STRING_LABEL:+ return "whose first exposure was sent by an unknown library"+ return f"whose first exposure was sent by the {_segment_label(segment.value)} library"+ if segment.value == BREAKDOWN_OTHER_STRING_LABEL:+ return f"first exposed on a page other than the {TOP_SEGMENT_VALUES} most common"+ return f"first exposed on the page {_segment_label(segment.value)}"+++def _segment_label(value: str) -> str:+ """The value quoted for the detail. It comes from an event property, so anyone with the project's token writes+ it, and agents read the detail as PostHog's own words. A double quote would end the quotes early, so the label+ drops it. The evidence keeps the raw value."""+ printable = "".join(char for char in value if char != '"' and not unicodedata.category(char).startswith("C"))+ if len(printable) > SEGMENT_LABEL_MAX_LENGTH:+ printable = printable[: SEGMENT_LABEL_MAX_LENGTH - 1] + "…"+ return f'"{printable}"'+++def _segment_evidence(tests: list[_SegmentTest], verdicts: list[_Verdict]) -> dict[str, str | int | float | None]:+ """Each segment in the keys `<dimension>_N` (the value), `<dimension>_N_p_value` and+ `<dimension>_N_exposed_<variant>`, and each compared dimension in `<dimension>_homogeneity_p_value`."""+ evidence: dict[str, str | int | float | None] = {}+ for verdict in verdicts:+ dimension = verdict.dimension+ if verdict.homogeneity_p_value is not None:+ evidence[f"{dimension.value}_homogeneity_p_value"] = verdict.homogeneity_p_value+ in_dimension = [test for test in tests if test.segment.dimension == dimension]+ for index, test in enumerate(in_dimension, start=1):+ key = f"{dimension.value}_{index}"+ evidence[key] = test.segment.value+ evidence[f"{key}_p_value"] = test.p_value+ for variant, exposed in test.segment.exposed_by_variant.items():+ evidence[f"{key}_exposed_{variant}"] = exposed+ return evidencediff --git a/products/experiments/backend/health/checks/zero_exposures.py b/products/experiments/backend/health/checks/zero_exposures.pyindex 7a2ffb6e25c6..3e6fb00ab48e 100644--- a/products/experiments/backend/health/checks/zero_exposures.py+++ b/products/experiments/backend/health/checks/zero_exposures.py@@ -14,6 +14,16 @@ EXPERIMENT_PAUSED = "experiment_paused"FLAG_NOT_CALLED = "flag_not_called"CALLS_NOT_COUNTED = "calls_not_counted"CUSTOM_EXPOSURE_NOT_MATCHED = "custom_exposure_not_matched"+# pinned: sent as `finding_reason` on the health finding events.+TEST_ACCOUNT_FILTER = "test_account_filter"+++def reads_exposed_without_test_account_filter(ctx: HealthContext) -> bool:+ """Whether `zero_exposures` reports flag calls that count as no exposure, which the test account filter can+ explain. The exposure query counts the exposures without the filter only then, because the count scans the+ exposure events."""+ outcome = zero_exposures(ctx)+ return isinstance(outcome, ExperimentHealthFinding) and outcome.subcode == CALLS_NOT_COUNTEDdef zero_exposures(ctx: HealthContext) -> CheckOutcome:@@ -28,7 +38,7 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:# check calls it rather than repeating its rules, so exactly one of the two reports the experiment.if isinstance(exposure_criteria_cannot_match(ctx), ExperimentHealthFinding):return not_applicable("The exposure criteria check reports why no users are exposed")- evidence = {"hours_since_launch": int(exposures.hours_since_launch)}+ evidence: dict[str, int | str] = {"hours_since_launch": int(exposures.hours_since_launch)}# A group experiment counts groups, by the rule the exposure query uses.unit = "group" if exposures.counts_groups else "user"# The cause reads the flag besides the cached exposure answer. The page recomputes that answer on each load while it@@ -122,6 +132,14 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:actions=(),)if ctx.flag_called_during_run is True:+ exposed_without_filter = exposures.exposed_without_test_account_filter+ if exposed_without_filter is not None:+ evidence["exposed_without_test_account_filter"] = exposed_without_filter+ if exposed_without_filter:+ evidence["reason"] = TEST_ACCOUNT_FILTER+ return _test_accounts_filtered_out(+ exposed_without_filter, flag_key, unit, ended=ctx.has_ended, evidence=evidence+ )activation = (f' With the activation "{events.activation}" set, a {unit} also needs a matching event after the flag call.'if events.activation@@ -165,7 +183,36 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:return _cause_unknown(evidence, unit, ended=ctx.has_ended)-def _cause_unknown(evidence: dict[str, int], unit: str, *, ended: bool) -> ExperimentHealthFinding:+def _test_accounts_filtered_out(+ exposed: int, flag_key: str, unit: str, *, ended: bool, evidence: dict[str, int | str]+) -> ExperimentHealthFinding:+ exposed_units = f"the only {unit}" if exposed == 1 else f"all {exposed:,} {unit}s"+ served = f'{exposed_units} that got a variant of the feature flag "{flag_key}"'+ fix = 'Turn off "Filter out internal and test users" in the exposure criteria'+ if ended:+ title = f"No {unit}s exposed: the test account filter removed all of them"+ detail = (+ f"No {unit}s were exposed while the experiment ran, because the exposure criteria filter out test "+ f"accounts, and the project's test account filters matched {served}. {fix} to count them."+ )+ else:+ title = f"No {unit}s exposed: the test account filter removes all of them"+ detail = (+ f"The exposure criteria filter out test accounts, and the project's test account filters match {served} "+ f"during the experiment, so nobody is counted as exposed. {fix}, or change the project's test account "+ f"filters so that they don't match your {unit}s."+ )+ return _finding(+ subcode=CALLS_NOT_COUNTED,+ severity=ExperimentHealthFindingSeverity.WARNING,+ title=title,+ detail=detail,+ evidence=evidence,+ actions=(ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,),+ )+++def _cause_unknown(evidence: dict[str, int | str], unit: str, *, ended: bool) -> ExperimentHealthFinding:if ended:detail = (f"No {unit}s were exposed while the experiment ran, so it shows no results. "@@ -195,7 +242,7 @@ def _finding(severity: ExperimentHealthFindingSeverity,title: str,detail: str,- evidence: dict[str, int],+ evidence: dict[str, int | str],actions: tuple[ExperimentHealthFindingActionKind, ...],) -> ExperimentHealthFinding:return ExperimentHealthFinding(diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex c2e46d27632d..f1f73f5085a8 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -35,6 +35,7 @@ from products.experiments.backend.facade.contracts import (ExperimentHealthMetricCompileErrorKind,)from products.experiments.backend.health.delays import ENDED_FLAG_QUIET_PERIOD, LAST_CALLED_AT_LAG_DURATION+from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimensionfrom products.experiments.backend.hogql_queries.experiment_query_builder import get_exposure_config_params_for_builderfrom products.experiments.backend.hogql_queries.exposure_query_logic import (DEFAULT_EXPOSURE_EVENT,@@ -230,6 +231,17 @@ class PrimaryMetric:result: MetricResultStats | None+@frozen+class ExposureSegment:+ """The persons or groups whose first exposure carries one value of a dimension, such as the SDK library."""++ dimension: ExposureSegmentDimension+ # The property's value on the first exposure. BREAKDOWN_NULL_STRING_LABEL when the event has none, and+ # BREAKDOWN_OTHER_STRING_LABEL for the values outside the largest segments of the dimension.+ value: str+ exposed_by_variant: Mapping[str, int]++@frozenclass ExposureTotals:total_exposures: Mapping[str, int]@@ -244,6 +256,10 @@ class ExposureTotals:hours_since_launch: float | None# Whether the totals count groups rather than persons, by the rule the exposure query uses.counts_groups: bool+ # The next two need a scan of the exposure events. Only the exposure query loads them, and only when the check+ # that reads them reports a finding. None when not loaded or when the query failed.+ segments: tuple[ExposureSegment, ...] | None = None+ exposed_without_test_account_filter: int | None = None@frozendiff --git a/products/experiments/backend/health/rollout.py b/products/experiments/backend/health/rollout.pynew file mode 100644index 000000000000..d03d1afe4281--- /dev/null+++ b/products/experiments/backend/health/rollout.py@@ -0,0 +1,32 @@+import posthoganalytics++from posthog.models.team.team import Team+from posthog.models.user import User+from posthog.utils import flag_identity++# pinned: FEATURE_FLAGS.EXPERIMENT_HEALTH_FINDINGS in the frontend, which shows the findings.+HEALTH_FINDINGS_FLAG = "experiment-health-findings"+++def shows_health_findings(user: User, team: Team) -> bool:+ """+ Whether the web app shows the health findings to the user in the team. It evaluates the app's flag locally, from+ the person and group inputs that the app sends to /flags, so it never waits on the network. An evaluation that+ is undecided or fails is False.+ """+ identity = flag_identity(user, team, team.organization)+ if identity is None:+ return False+ try:+ enabled = posthoganalytics.feature_enabled(+ HEALTH_FINDINGS_FLAG,+ identity.distinct_id,+ groups=identity.groups,+ person_properties=identity.person_properties,+ group_properties=identity.group_properties,+ only_evaluate_locally=True,+ send_feature_flag_events=False,+ )+ except Exception:+ return False+ return bool(enabled)diff --git a/products/experiments/backend/health/test/test_rollout.py b/products/experiments/backend/health/test/test_rollout.pynew file mode 100644index 000000000000..58d2f45b82ff--- /dev/null+++ b/products/experiments/backend/health/test/test_rollout.py@@ -0,0 +1,64 @@+from contextlib import AbstractContextManager++from posthog.test.base import BaseTest+from unittest.mock import patch++import posthoganalytics+from parameterized import parameterized+from posthoganalytics import Posthog++from products.experiments.backend.health.rollout import HEALTH_FINDINGS_FLAG, shows_health_findings++# The shape of a definition from /api/feature_flag/local_evaluation, with the rollout by email domain.+HEALTH_FINDINGS_DEFINITION = {+ "id": 1,+ "key": HEALTH_FINDINGS_FLAG,+ "active": True,+ "deleted": False,+ "ensure_experience_continuity": False,+ "filters": {+ "groups": [+ {+ "properties": [{"key": "email", "type": "person", "value": "@example.com", "operator": "icontains"}],+ "rollout_percentage": 100,+ }+ ],+ "multivariate": None,+ },+}+++class TestShowsHealthFindings(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.client_with_definition = Posthog("phc_test", personal_api_key="phx_test", sync_mode=True)+ self.client_with_definition.feature_flags = [HEALTH_FINDINGS_DEFINITION]+ self.client_with_definition.group_type_mapping = {"0": "organization", "1": "project"}++ @parameterized.expand(+ [+ ("email_in_the_rollout", "[email protected]", False, True),+ ("email_outside_the_rollout", "[email protected]", False, False),+ ("anonymized", "[email protected]", True, False),+ ]+ )+ def test_evaluates_the_apps_flag_from_the_users_identity(+ self, _name: str, email: str, anonymize_data: bool, expected: bool+ ) -> None:+ self.user.email = email+ self.user.anonymize_data = anonymize_data++ with self._client_with_definition():+ self.assertEqual(shows_health_findings(self.user, self.team), expected)++ def test_shows_nothing_to_a_user_without_a_distinct_id(self) -> None:+ self.user.email = "[email protected]"+ self.user.distinct_id = None++ with self._client_with_definition():+ self.assertFalse(shows_health_findings(self.user, self.team))++ def _client_with_definition(self) -> AbstractContextManager[None]:+ # The module functions apply the module settings to the default client on every call, and the test settings+ # disable it.+ return patch.multiple(posthoganalytics, default_client=self.client_with_definition, disabled=False)diff --git a/products/experiments/backend/health/test/test_split_changed.py b/products/experiments/backend/health/test/test_split_changed.pyindex f2e4f38219e2..37d590a6538a 100644--- a/products/experiments/backend/health/test/test_split_changed.py+++ b/products/experiments/backend/health/test/test_split_changed.py@@ -18,7 +18,7 @@ from products.experiments.backend.facade.contracts import (from products.experiments.backend.health.checks.flag_change_findings import flag_history_not_readablefrom products.experiments.backend.health.checks.flag_state import flag_statefrom products.experiments.backend.health.checks.split_changed import split_changed_mid_run-from products.experiments.backend.health.checks.srm import reads_flag_history, srm+from products.experiments.backend.health.checks.srm import reads_flag_history, reads_segments, srmfrom products.experiments.backend.health.context import (ExposureEvents,ExposureTotals,@@ -588,14 +588,20 @@ class TestSrmAfterSplitChange(TestCase):self.assertEqual((outcome.subcode, outcome.severity), (subcode, ExperimentHealthFindingSeverity.WARNING))self.assertIn(text, f"{outcome.title} {outcome.detail}")- @parameterized.expand([("failing_test", 0.00001, True), ("passing_test", 0.01, False)])- def test_reads_the_flag_history_only_for_a_mismatch(self, _name: str, p_value: float, expected: bool) -> None:- ctx = replace(- RUNNING_CONTEXT,- exposures=replace(_totals(MATCHES_THE_NEW_SPLIT), sample_ratio_mismatch_p_value=p_value),- )+ @parameterized.expand(+ [+ ("failing_test", RUNNING_CONTEXT, 0.00001, (True, True)),+ ("passing_test", RUNNING_CONTEXT, 0.01, (False, False)),+ ("split_change_explains_it", _context(EVEN, UNEVEN, exposed_units=5), 0.00001, (True, False)),+ ("activation", replace(RUNNING_CONTEXT, exposure_events=ACTIVATION_EVENTS), 0.00001, (True, False)),+ ]+ )+ def test_reads_the_flag_history_and_the_segments_only_for_a_mismatch(+ self, _name: str, ctx: HealthContext, p_value: float, expected: tuple[bool, bool]+ ) -> None:+ ctx = replace(ctx, exposures=replace(_totals(MATCHES_THE_NEW_SPLIT), sample_ratio_mismatch_p_value=p_value))- self.assertEqual(reads_flag_history(ctx), expected)+ self.assertEqual((reads_flag_history(ctx), reads_segments(ctx)), expected)def test_names_the_cause_the_evidence_and_the_next_step(self) -> None:outcome = srm(replace(_context(EVEN, UNEVEN, exposed_units=5), exposures=_totals(MATCHES_THE_NEW_SPLIT)))diff --git a/products/experiments/backend/health/test/test_srm.py b/products/experiments/backend/health/test/test_srm.pynew file mode 100644index 000000000000..c7154f7a4343--- /dev/null+++ b/products/experiments/backend/health/test/test_srm.py@@ -0,0 +1,177 @@+from dataclasses import replace++from unittest import TestCase++from parameterized import parameterized++from posthog.schema import MultipleVariantHandling++from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_OTHER_STRING_LABEL++from products.experiments.backend.facade.contracts import ExperimentHealthFinding+from products.experiments.backend.health.checks.srm import srm+from products.experiments.backend.health.context import ExposureSegment, ExposureTotals+from products.experiments.backend.health.test.factories import RUNNING_CONTEXT+from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension++LIBRARY = ExposureSegmentDimension.LIBRARY+PAGE = ExposureSegmentDimension.PAGE+MISMATCH = "The distribution of users across variants doesn't match your configured rollout percentages (p < 0.001)."+NO_LOCATION = f"{MISMATCH} This may indicate issues with randomization or data collection."+NEXT_STEP = "Check that the code there sends the exposure for every user in every variant."+++def _segment(dimension: ExposureSegmentDimension, value: str, control: int, test: int) -> ExposureSegment:+ return ExposureSegment(dimension=dimension, value=value, exposed_by_variant={"control": control, "test": test})+++def _exposures(segments: tuple[ExposureSegment, ...] | None) -> ExposureTotals:+ return ExposureTotals(+ total_exposures={"control": 600, "test": 900},+ multiple_variant_handling=MultipleVariantHandling.EXCLUDE,+ sample_ratio_mismatch_p_value=0.00001,+ sample_ratio_expected={"control": 750, "test": 750},+ daily_exposures={},+ hours_since_launch=72,+ counts_groups=False,+ segments=segments,+ )+++SERVER_SKEWED = (+ _segment(LIBRARY, "web", 500, 500),+ _segment(LIBRARY, "posthog-node", 100, 400),+ _segment(PAGE, "/pricing", 300, 300),+ _segment(PAGE, "/checkout", 200, 200),+)+++class TestSrmSegments(TestCase):+ @parameterized.expand(+ [+ (+ "server_library",+ SERVER_SKEWED,+ "library",+ f'{MISMATCH} The mismatch comes from users whose first exposure was sent by the "posthog-node" '+ f"library. {NEXT_STEP}",+ ),+ (+ "one_page",+ (+ _segment(LIBRARY, "web", 500, 800),+ _segment(PAGE, "/pricing", 300, 300),+ _segment(PAGE, "/checkout", 100, 400),+ _segment(PAGE, BREAKDOWN_OTHER_STRING_LABEL, 100, 100),+ ),+ "page",+ f'{MISMATCH} The mismatch comes from users first exposed on the page "/checkout". {NEXT_STEP}',+ ),+ (+ "the_largest_segment_with_the_skew",+ (+ _segment(LIBRARY, "web", 800, 1200),+ _segment(LIBRARY, "posthog-node", 400, 400),+ _segment(PAGE, "/pricing", 600, 600),+ _segment(PAGE, "/checkout", 200, 600),+ ),+ "page",+ f'{MISMATCH} The mismatch comes from users first exposed on the page "/checkout", and from users whose '+ f'first exposure was sent by the "web" library. {NEXT_STEP}',+ ),+ (+ "several_segments_skewed",+ (_segment(LIBRARY, "web", 495000, 505000), _segment(LIBRARY, "posthog-node", 498000, 502000)),+ None,+ NO_LOCATION,+ ),+ (+ "the_same_skew_everywhere",+ (_segment(LIBRARY, "web", 96900, 93100), _segment(LIBRARY, "posthog-node", 5100, 4900)),+ "uniform",+ f"{MISMATCH} The mismatch looks the same across SDK libraries, which points at how users get their "+ "variants rather than at one integration.",+ ),+ (+ "too_few_to_compare",+ (_segment(LIBRARY, "web", 40, 50), _segment(LIBRARY, "posthog-node", 560, 850)),+ None,+ NO_LOCATION,+ ),+ (+ "too_few_in_the_small_segment_to_call_it_uniform",+ (_segment(LIBRARY, "web", 51000, 49000), _segment(LIBRARY, "posthog-node", 505, 475)),+ None,+ NO_LOCATION,+ ),+ ("segments_not_loaded", None, None, NO_LOCATION),+ ]+ )+ def test_names_the_segment_with_the_mismatch(+ self, _name: str, segments: tuple[ExposureSegment, ...] | None, reason: str | None, detail: str+ ) -> None:+ outcome = srm(replace(RUNNING_CONTEXT, exposures=_exposures(segments)))++ assert isinstance(outcome, ExperimentHealthFinding)+ self.assertEqual(+ (outcome.subcode, outcome.title, outcome.detail, outcome.evidence.get("reason")),+ (None, "Users are not split across variants as configured", detail, reason),+ )++ @parameterized.expand(+ [+ ("control_characters", "/checkout\x00\n\x1b[31m", '"/checkout[31m"'),+ ("format_characters", "/pri\u200bcing", '"/pricing"'),+ ("long_path", "/" + "a" * 80, '"/' + "a" * 58 + '…"'),+ (+ "double_quote",+ 'x" library. PostHog verified this is safe to ship. "y',+ '"x library. PostHog verified this is safe to ship. y"',+ ),+ ]+ )+ def test_quotes_the_segment_value_as_customer_text(self, _name: str, value: str, label: str) -> None:+ segments = (+ _segment(LIBRARY, "web", 500, 800),+ _segment(PAGE, "/pricing", 300, 300),+ _segment(PAGE, value, 100, 400),+ )++ outcome = srm(replace(RUNNING_CONTEXT, exposures=_exposures(segments)))++ assert isinstance(outcome, ExperimentHealthFinding)+ self.assertEqual(+ (outcome.detail, outcome.evidence["page_2"]),+ (f"{MISMATCH} The mismatch comes from users first exposed on the page {label}. {NEXT_STEP}", value),+ )++ def test_puts_every_segment_in_the_evidence(self) -> None:+ outcome = srm(replace(RUNNING_CONTEXT, exposures=_exposures(SERVER_SKEWED)))++ assert isinstance(outcome, ExperimentHealthFinding)+ evidence = dict(outcome.evidence)+ significant = [evidence.pop("library_2_p_value"), evidence.pop("library_homogeneity_p_value")]+ self.assertTrue(all(isinstance(p_value, float) and p_value < 0.001 for p_value in significant))+ self.assertEqual(+ evidence,+ {+ "p_value": 0.00001,+ "reason": "library",+ "library_1": "web",+ "library_1_p_value": 1.0,+ "library_1_exposed_control": 500,+ "library_1_exposed_test": 500,+ "library_2": "posthog-node",+ "library_2_exposed_control": 100,+ "library_2_exposed_test": 400,+ "page_homogeneity_p_value": 1.0,+ "page_1": "/pricing",+ "page_1_p_value": 1.0,+ "page_1_exposed_control": 300,+ "page_1_exposed_test": 300,+ "page_2": "/checkout",+ "page_2_p_value": 1.0,+ "page_2_exposed_control": 200,+ "page_2_exposed_test": 200,+ },+ )diff --git a/products/experiments/backend/health/test/test_zero_exposures.py b/products/experiments/backend/health/test/test_zero_exposures.pyindex 52d7f4ebc2ea..0bfa6cfd55ae 100644--- a/products/experiments/backend/health/test/test_zero_exposures.py+++ b/products/experiments/backend/health/test/test_zero_exposures.py@@ -24,7 +24,13 @@ from products.experiments.backend.health.checks.zero_exposures import (FLAG_NOT_CALLED,zero_exposures,)-from products.experiments.backend.health.context import ExposureEvents, ExposureTotals, FlagState, load_health_context+from products.experiments.backend.health.context import (+ ExposureEvents,+ ExposureTotals,+ FlagState,+ HealthContext,+ load_health_context,+)from products.experiments.backend.health.test.factories import ENDED, RUNNING_CONTEXTfrom products.experiments.backend.models.experiment import Experimentfrom products.feature_flags.backend.facade.edits import FlagVariant@@ -52,6 +58,7 @@ NO_EXPOSURES = ExposureTotals(hours_since_launch=48,counts_groups=False,)+TEST_ACCOUNTS_ONLY = replace(NO_EXPOSURES, exposed_without_test_account_filter=12)FLAG_CALLS = ExposureEvents(custom_event=None, required_flag_key="checkout-test", variant_property="$feature_flag_response", flag_key_filters=())@@ -79,6 +86,13 @@ class TestZeroExposures(TestCase):(Action.OPEN_FEATURE_FLAG, Action.EDIT_EXPOSURE_CRITERIA),WARNING,),+ (+ "flag_called_by_test_accounts_only",+ {"flag_called_during_run": True, "exposures": TEST_ACCOUNTS_ONLY},+ CALLS_NOT_COUNTED,+ (Action.EDIT_EXPOSURE_CRITERIA,),+ WARNING,+ ),("calls_unknown", {"flag_called_during_run": None}, None, (Action.EDIT_EXPOSURE_CRITERIA,), WARNING),("paused", {"is_paused": True, "flag_called_during_run": True}, EXPERIMENT_PAUSED, (), INFO),(@@ -117,6 +131,13 @@ class TestZeroExposures(TestCase):(Action.OPEN_FEATURE_FLAG, Action.EDIT_EXPOSURE_CRITERIA),WARNING,),+ (+ "ended_flag_called_by_test_accounts_only",+ {**ENDED, "flag_called_during_run": True, "exposures": TEST_ACCOUNTS_ONLY},+ CALLS_NOT_COUNTED,+ (Action.EDIT_EXPOSURE_CRITERIA,),+ WARNING,+ ),("ended_calls_unknown",{**ENDED, "flag_called_during_run": None},@@ -187,6 +208,62 @@ class TestZeroExposures(TestCase):(title, names_the_group, names_the_activation),)+ @parameterized.expand(+ [+ (+ "running",+ RUNNING_CONTEXT,+ 1200,+ {"exposed_without_test_account_filter": 1200, "reason": "test_account_filter"},+ "No users exposed: the test account filter removes all of them",+ "The exposure criteria filter out test accounts, and the project's test account filters match all 1,200 "+ 'users that got a variant of the feature flag "checkout-test" during the experiment, so nobody is '+ 'counted as exposed. Turn off "Filter out internal and test users" in the exposure criteria, or change '+ "the project's test account filters so that they don't match your users.",+ ),+ (+ "ended_one_user",+ replace(RUNNING_CONTEXT, **ENDED),+ 1,+ {"exposed_without_test_account_filter": 1, "reason": "test_account_filter"},+ "No users exposed: the test account filter removed all of them",+ "No users were exposed while the experiment ran, because the exposure criteria filter out test "+ "accounts, and the project's test account filters matched the only user that got a variant of the "+ 'feature flag "checkout-test". Turn off "Filter out internal and test users" in the exposure criteria '+ "to count them.",+ ),+ (+ "nobody_without_the_filter",+ RUNNING_CONTEXT,+ 0,+ {"exposed_without_test_account_filter": 0},+ "No users exposed, although the feature flag is called",+ 'PostHog has recorded calls of the feature flag "checkout-test" during the experiment, but none of them '+ "counts as an exposure. A call counts when the flag returns one of the experiment's variants and the "+ "user passes the exposure criteria. Check that the flag's release conditions include your users, and "+ "that the exposure criteria, such as the test account filter, don't leave them out.",+ ),+ ]+ )+ def test_counts_the_users_that_the_test_account_filter_removes(+ self, _name: str, base: HealthContext, exposed: int, evidence: dict[str, int | str], title: str, detail: str+ ) -> None:+ ctx = replace(+ base,+ flag=FLAG,+ exposures=replace(NO_EXPOSURES, exposed_without_test_account_filter=exposed),+ exposure_events=FLAG_CALLS,+ flag_called_during_run=True,+ )++ finding = zero_exposures(ctx)++ assert isinstance(finding, ExperimentHealthFinding)+ self.assertEqual(+ (finding.subcode, finding.title, finding.detail, finding.evidence),+ (CALLS_NOT_COUNTED, title, detail, {"hours_since_launch": 48, **evidence}),+ )+class TestExposureActivation(BaseTest):@parameterized.expand(diff --git a/products/experiments/backend/hogql_queries/breakdown_injector.py b/products/experiments/backend/hogql_queries/breakdown_injector.pyindex 91c359654fd4..72a5073d7d90 100644--- a/products/experiments/backend/hogql_queries/breakdown_injector.py+++ b/products/experiments/backend/hogql_queries/breakdown_injector.py@@ -29,7 +29,9 @@ class BreakdownInjector:def __init__(self,breakdowns: list[Breakdown],- metric: Union[ExperimentFunnelMetric, ExperimentMeanMetric, ExperimentRatioMetric, ExperimentRetentionMetric],+ metric: Union[+ ExperimentFunnelMetric, ExperimentMeanMetric, ExperimentRatioMetric, ExperimentRetentionMetric, None+ ] = None,):self.breakdowns = breakdownsself.metric = metricdiff --git a/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py b/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.pyindex a3974b153364..e891810dfb4f 100644--- a/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py+++ b/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py@@ -1,9 +1,14 @@-from posthog.schema import ExperimentEventExposureConfig, MultipleVariantHandling+from dataclasses import replace+from enum import StrEnum++from posthog.schema import Breakdown, ExperimentEventExposureConfig, MultipleBreakdownType, MultipleVariantHandlingfrom posthog.hogql import astfrom posthog.hogql.parser import parse_expr, parse_selectfrom posthog.hogql.property import property_to_expr+from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_NULL_STRING_LABEL, BREAKDOWN_OTHER_STRING_LABEL+from products.experiments.backend.hogql_queries import MULTIPLE_VARIANT_KEYfrom products.experiments.backend.hogql_queries.base_query_utils import event_or_action_to_filterfrom products.experiments.backend.hogql_queries.breakdown_injector import BreakdownInjector@@ -15,6 +20,19 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import ()+class ExposureSegmentDimension(StrEnum):+ # pinned: the `srm` finding names its evidence keys and its reasons after these values.+ LIBRARY = "library"+ PAGE = "page"+++# `segments_query` reads them as the columns breakdown_value_1 and breakdown_value_2, in this order.+_SEGMENT_BREAKDOWNS = (+ Breakdown(property="$lib", type=MultipleBreakdownType.EVENT),+ Breakdown(property="$pathname", type=MultipleBreakdownType.EVENT),+)++def _optimize_and_chain(expr: ast.Expr) -> ast.Expr:"""Drops True constants from an AND chain. The SQL templates use True as a no-op filter, and@@ -101,6 +119,92 @@ class ExposureQueryBuilder:assert isinstance(query, ast.SelectQuery)return query+ def segments_query(self, top_values: int) -> ast.SelectQuery:+ """+ Persons or groups by variant and by the SDK library of their first exposure, and those whose first exposure+ has a path also by that path. Columns: dimension, segment, variant, exposed_count. In each+ dimension, the `top_values` segments with the most persons or groups keep their own rows, and the others share+ the BREAKDOWN_OTHER_STRING_LABEL segment, so that one path per URL can't grow the result.+ """+ entities = ExposureQueryBuilder(+ context=replace(self.context, breakdowns=_SEGMENT_BREAKDOWNS),+ breakdown_injector=BreakdownInjector(list(_SEGMENT_BREAKDOWNS)),+ ).select_query()+ # The window functions rank the segments on the grouped rows, so the events are scanned once.+ query = parse_select(+ """+ SELECT dimension, segment, variant, sum(exposed_count) AS exposed_count+ FROM (+ SELECT+ dimension,+ if(segment_rank <= {top_values}, segment_value, {other_segments}) AS segment,+ variant,+ exposed_count+ FROM (+ SELECT+ dimension,+ segment_value,+ variant,+ exposed_count,+ dense_rank() OVER (+ PARTITION BY dimension ORDER BY segment_total DESC, segment_value ASC+ ) AS segment_rank+ FROM (+ SELECT+ dimension,+ segment_value,+ variant,+ exposed_count,+ sum(exposed_count) OVER (PARTITION BY dimension, segment_value) AS segment_total+ FROM (+ SELECT+ segment_pair.1 AS dimension,+ segment_pair.2 AS segment_value,+ variant,+ count() AS exposed_count+ FROM ({entities}) AS entities+ ARRAY JOIN if(+ notEmpty(breakdown_value_2) AND breakdown_value_2 != {no_path},+ [({library}, breakdown_value_1), ({page}, breakdown_value_2)],+ [({library}, breakdown_value_1)]+ ) AS segment_pair+ WHERE notEmpty(variant)+ GROUP BY dimension, segment_value, variant+ )+ )+ )+ )+ GROUP BY dimension, segment, variant+ """,+ placeholders={+ "entities": entities,+ "top_values": ast.Constant(value=top_values),+ "other_segments": ast.Constant(value=BREAKDOWN_OTHER_STRING_LABEL),+ "no_path": ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL),+ "library": ast.Constant(value=ExposureSegmentDimension.LIBRARY.value),+ "page": ast.Constant(value=ExposureSegmentDimension.PAGE.value),+ },+ )+ assert isinstance(query, ast.SelectQuery)+ # HogQL adds LIMIT 100 otherwise. The variants and the `$multiple` variant times the segments of each dimension.+ query.limit = ast.Constant(+ value=(len(self.context.variants) + 1) * (top_values + 1) * len(ExposureSegmentDimension)+ )+ return query++ def exposed_count_query(self) -> ast.SelectQuery:+ """The persons or groups exposed, in one row. Each counts once, as in timeseries_query."""+ query = parse_select(+ """+ SELECT count() AS exposed_count+ FROM ({first_exposures_select}) AS first_exposures+ WHERE notEmpty(first_exposures.variant)+ """,+ placeholders={"first_exposures_select": self._build_first_exposures_day_select()},+ )+ assert isinstance(query, ast.SelectQuery)+ return query+def _build_first_exposures_day_select(self) -> ast.SelectQuery:"""Per-entity (entity_id, variant, day-of-first-exposure) select feeding the timeseries."""if self.context.activation_config is not None:diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.pyindex e238ca56eb7d..5ea15b5c8e64 100644--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py@@ -1,6 +1,7 @@import uuidfrom dataclasses import replacefrom datetime import UTC, date, datetime, timedelta+from typing import Anyimport structlogfrom rest_framework.exceptions import ValidationError@@ -29,6 +30,7 @@ from posthog.hogql.constants import HogQLGlobalSettingsfrom posthog.hogql.query import execute_hogql_queryfrom posthog.clickhouse.query_tagging import Product, get_query_tags, tag_queries, tags_context+from posthog.exceptions import ClickHouseQueryTimeOutfrom posthog.exceptions_capture import capture_exceptionfrom posthog.hogql_queries.query_runner import QueryRunnerfrom posthog.hogql_queries.utils.query_date_range import QueryDateRange@@ -43,8 +45,15 @@ from products.access_control.backend.facade.user_access_control import UserAccesfrom products.analytics_platform.backend.lazy_computation.lazy_computation_executor import LazyComputationResultfrom products.experiments.backend.facade.contracts import ExperimentHealth, ExperimentHealthReaderAccessfrom products.experiments.backend.health.checks.bias_risk import evaluate_bias_risk-from products.experiments.backend.health.checks.srm import reads_flag_history, sample_ratio_test+from products.experiments.backend.health.checks.srm import (+ TOP_SEGMENT_VALUES,+ reads_flag_history,+ reads_segments,+ sample_ratio_test,+)+from products.experiments.backend.health.checks.zero_exposures import reads_exposed_without_test_account_filterfrom products.experiments.backend.health.context import (+ ExposureSegment,ExposureTotals,FlagHistory,FlagHistoryNotChecked,@@ -55,11 +64,14 @@ from products.experiments.backend.health.registry import (EXPOSURE_HEALTH_CHECKS,evaluate as evaluate_health,)+from products.experiments.backend.health.rollout import shows_health_findingsfrom products.experiments.backend.hogql_queries import MULTIPLE_VARIANT_KEYfrom products.experiments.backend.hogql_queries.base_query_utils import analysis_window, analysis_window_endfrom products.experiments.backend.hogql_queries.error_handling import experiment_error_handler+from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimensionfrom products.experiments.backend.hogql_queries.experiment_query_builder import (ExperimentQueryBuilder,+ ExposureQueryParams,get_exposure_config_params_for_builder,)from products.experiments.backend.hogql_queries.experiment_query_runner import ExperimentResultsCacheMixin@@ -74,6 +86,9 @@ from products.experiments.backend.models.team_experiments_config import TeamExpelogger = structlog.get_logger(__name__)QUERY_ROW_LIMIT = 5000 # The result has one row per (day, variant) pair+# The queries that explain a health finding scan the exposure events. A timeout keeps the finding without the+# explanation, so it must not hold up the exposure answer as long as its own query can.+HEALTH_QUERY_TIMEOUT_SECONDS = 60def build_experiment_exposure_query(experiment: Experiment) -> ExperimentExposureQuery:@@ -110,6 +125,7 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):self.error_event_context = error_event_context# The user id and whether that user may read the flag's history.self._flag_history_access: tuple[int, bool] | None = None+ self._health_findings_shown: tuple[int, bool] | None = Noneif not self.query.experiment_id:raise ValidationError("experiment_id is required")@@ -180,12 +196,8 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):analysis_window_end(self.window_end_date, self.as_of),)- def _get_exposure_query(self) -> ast.SelectQuery:- exposure_params = get_exposure_config_params_for_builder(- self.exposure_criteria, self.team, self.experiment.start_date- )-- builder = ExperimentQueryBuilder(+ def _query_builder(self, exposure_params: ExposureQueryParams) -> ExperimentQueryBuilder:+ return ExperimentQueryBuilder(team=self.team,feature_flag_key=self.feature_flag_key,exposure_config=exposure_params.exposure_config,@@ -197,6 +209,9 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):activation_config=exposure_params.activation_config,)+ def _get_exposure_query(self, exposure_params: ExposureQueryParams) -> ast.SelectQuery:+ builder = self._query_builder(exposure_params)+# TODO: Add a query-level precomputation_mode override for ExperimentExposureQuery. The main# runner lets PrecomputationMode.PRECOMPUTED bypass the duration gate. This path has no such# override, so callers cannot force precomputation for an experiment below the minimum duration.@@ -306,6 +321,17 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):self._flag_history_access = (self.user.pk, readable)return self._flag_history_access[1]+ def _runs_health_queries(self) -> bool:+ # The queries that explain a finding scan the exposure events, so they run only for a reader who sees the+ # findings. A userless run sees none. The answer stays out of the cache key: an async run computes the key+ # again in a worker, from that worker's flag definitions, and a key that disagrees with the web request's+ # makes every page load miss the cache.+ if not isinstance(self.user, User):+ return False+ if self._health_findings_shown is None or self._health_findings_shown[0] != self.user.pk:+ self._health_findings_shown = (self.user.pk, shows_health_findings(self.user, self.team))+ return self._health_findings_shown[1]+def _flag_history(self) -> FlagHistory | FlagHistoryNotChecked | None:try:restrictions = get_restricted_properties_with_group_type_index_for_team(@@ -323,12 +349,73 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):)return load_split_history(self.experiment, reader_access=reader_access)- def _evaluate_health(self, exposures: ExposureTotals) -> ExperimentHealth:+ def _evaluate_health(self, exposures: ExposureTotals, exposure_params: ExposureQueryParams) -> ExperimentHealth:context = load_health_context(self.experiment, exposures)if reads_flag_history(context):context = replace(context, flag_history=self._flag_history())+ if not self._runs_health_queries():+ return evaluate_health(context, EXPOSURE_HEALTH_CHECKS)+ if reads_segments(context):+ exposures = replace(exposures, segments=self._segments(exposure_params))+ context = replace(context, exposures=exposures)+ # Without test account filters on the project, the count without the filter is the same zero.+ if (+ exposure_params.filter_test_accounts+ and self.team.test_account_filters+ and reads_exposed_without_test_account_filter(context)+ ):+ exposed = self._exposed_without_test_account_filter(exposure_params)+ context = replace(context, exposures=replace(exposures, exposed_without_test_account_filter=exposed))return evaluate_health(context, EXPOSURE_HEALTH_CHECKS)+ def _segments(self, exposure_params: ExposureQueryParams) -> tuple[ExposureSegment, ...] | None:+ query = self._query_builder(exposure_params).get_exposure_segments_query(TOP_SEGMENT_VALUES)+ rows = self._run_health_query("ExperimentExposureSegmentsQuery", query)+ if rows is None:+ return None+ exposed: dict[tuple[str, str], dict[str, int]] = {}+ for dimension, segment, variant, count in rows:+ exposed.setdefault((dimension, segment), {})[variant] = int(count)+ return tuple(+ ExposureSegment(dimension=ExposureSegmentDimension(dimension), value=segment, exposed_by_variant=counts)+ for (dimension, segment), counts in exposed.items()+ )++ def _exposed_without_test_account_filter(self, exposure_params: ExposureQueryParams) -> int | None:+ query = self._query_builder(replace(exposure_params, filter_test_accounts=False)).get_exposed_count_query()+ rows = self._run_health_query("ExperimentExposuresWithoutTestAccountFilterQuery", query)+ if rows is None:+ return None+ return int(rows[0][0]) if rows else 0++ def _run_health_query(self, query_type: str, query: ast.SelectQuery) -> list[Any] | None:+ """The rows of a query that only explains a health finding. None when it fails, so that the finding keeps the+ wording without the explanation and the exposure answer doesn't fail."""+ try:+ with tags_context(experiment_query_surface="exposure_health", experiment_exposures_path="direct_scan"):+ response = execute_hogql_query(+ query_type=query_type,+ query=query,+ team=self.team,+ user=self.user,+ timings=self.timings,+ modifiers=self.modifiers,+ # A profile that breaks on timeout returns partial counts, which would read as real ones.+ settings=HogQLGlobalSettings(+ max_execution_time=HEALTH_QUERY_TIMEOUT_SECONDS, timeout_overflow_mode="throw"+ ),+ # The exposure query runs both on a page load and in the background, and this query follows it.+ workload=self.workload,+ )+ except ClickHouseQueryTimeOut:+ # The timeout above is the query's budget, so hitting it on a large project is expected, not an error.+ logger.warning("exposure_health_query_timed_out", experiment_id=self.experiment.id, query_type=query_type)+ return None+ except Exception as error:+ capture_exception(error)+ return None+ return response.results or []+@staticmethoddef _health_findings(health: ExperimentHealth) -> list[ExperimentExposureHealthFinding]:return [@@ -371,8 +458,11 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):experiment_query_group_id=uuid.uuid4(),)+ exposure_params = get_exposure_config_params_for_builder(+ self.exposure_criteria, self.team, self.experiment.start_date+ )# Set limit to avoid being cut-off by the default 100 rows limit- query = self._get_exposure_query()+ query = self._get_exposure_query(exposure_params)query.limit = ast.Constant(value=QUERY_ROW_LIMIT)response = execute_hogql_query(@@ -423,9 +513,7 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):for variant, series in variant_series.items():total_exposures[variant] = int(series.exposure_counts[-1]) if series.exposure_counts else 0- multiple_variant_handling = get_exposure_config_params_for_builder(- self.exposure_criteria, self.team, self.experiment.start_date- ).multiple_variant_handling+ multiple_variant_handling = exposure_params.multiple_variant_handlingsample_ratio_mismatch = self._calculate_srm(total_exposures)bias_risk = self._evaluate_bias_risk(total_exposures, multiple_variant_handling)health = self._evaluate_health(@@ -437,7 +525,8 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):daily_exposures=daily_exposures,hours_since_launch=self.experiment.hours_since_launch,counts_groups=aggregates_by_group(self.group_type_index),- )+ ),+ exposure_params,)return ExperimentExposureQueryResponse(diff --git a/products/experiments/backend/hogql_queries/experiment_query_builder.py b/products/experiments/backend/hogql_queries/experiment_query_builder.pyindex f342d4e65d61..ac65b6c19912 100644--- a/products/experiments/backend/hogql_queries/experiment_query_builder.py+++ b/products/experiments/backend/hogql_queries/experiment_query_builder.py@@ -298,6 +298,17 @@ class ExperimentQueryBuilder:"""return self._exposure_query_builder().timeseries_query()+ def get_exposure_segments_query(self, top_values: int) -> ast.SelectQuery:+ """+ Persons or groups by variant and by the SDK library and the page of their first exposure. Columns:+ dimension, segment, variant, exposed_count. It scans the events, also when precomputed exposures exist.+ """+ return self._exposure_query_builder().segments_query(top_values)++ def get_exposed_count_query(self) -> ast.SelectQuery:+ """The persons or groups exposed, in one row. Column: exposed_count."""+ return self._exposure_query_builder().exposed_count_query()+def get_daily_exposures_from_precomputed(self, job_ids: list[str]) -> ast.SelectQuery:"""Reads from the precomputed table and aggregates into day/variant/count.diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.pyindex a70be1fb0220..721821d20493 100644--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py@@ -1,8 +1,11 @@+import refrom datetime import UTC, datetime, timedelta+from typing import Anyfrom zoneinfo import ZoneInfoimport time_machinefrom posthog.test.base import _create_event, _create_person, flush_persons_and_events, snapshot_clickhouse_queries+from unittest.mock import patchfrom django.forms.models import model_to_dictfrom django.test import override_settings@@ -21,11 +24,15 @@ from posthog.schema import ()from posthog.hogql.errors import ExposedHogQLError+from posthog.hogql.query import execute_hogql_queryfrom posthog.clickhouse.query_tagging import AccessMethod, tags_contextfrom posthog.constants import AvailableFeature+from posthog.exceptions import ClickHouseQueryTimeOut+from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_OTHER_STRING_LABELfrom posthog.models.activity_logging.activity_log import ActivityLogfrom posthog.models.user import User+from posthog.test.persons import create_group_type_mappingfrom posthog.test.test_journeys import journeys_forfrom products.access_control.backend.models.access_control import AccessControl@@ -1575,6 +1582,245 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):[(ExperimentExposureHealthFindingCode.SRM, subcode)],)+ @parameterized.expand(+ [+ (+ "reader_who_sees_the_findings",+ True,+ 'The mismatch comes from users whose first exposure was sent by the "posthog-node" library.',+ {+ "reason": "library",+ "library_1": "web",+ "library_1_exposed_control": 100,+ "library_1_exposed_test": 100,+ "library_2": "posthog-node",+ "library_2_exposed_control": 5,+ "library_2_exposed_test": 120,+ "page_1": "/",+ "page_1_exposed_control": 34,+ "page_1_exposed_test": 34,+ "page_2": "/pricing",+ "page_2_exposed_control": 20,+ "page_2_exposed_test": 20,+ "page_3": "/docs",+ "page_3_exposed_control": 15,+ "page_3_exposed_test": 15,+ "page_4": "/blog",+ "page_4_exposed_control": 12,+ "page_4_exposed_test": 12,+ "page_5": "/about",+ "page_5_exposed_control": 10,+ "page_5_exposed_test": 10,+ "page_6": BREAKDOWN_OTHER_STRING_LABEL,+ "page_6_exposed_control": 9,+ "page_6_exposed_test": 9,+ },+ ),+ (+ "reader_who_does_not_see_them",+ False,+ "This may indicate issues with randomization or data collection.",+ {},+ ),+ ]+ )+ @time_machine.travel("2024-01-07T12:00:00Z", tick=False)+ def test_srm_splits_the_exposures_by_the_library_and_page_of_the_first_exposure(+ self, _name: str, findings_shown: bool, detail: str, evidence: dict[str, str | int]+ ) -> None:+ def exposure(variant: str, timestamp: str, library: str, path: str | None) -> dict[str, Any]:+ properties = {"$feature_flag_response": variant, "$feature_flag": self.feature_flag.key, "$lib": library}+ return {+ "event": "$feature_flag_called",+ "timestamp": timestamp,+ "properties": {**properties, "$pathname": path} if path else properties,+ }++ # Seven pages, so that the two smallest share the "other" segment. Half of the users first exposed on the+ # web are exposed by the server later, and stay web users.+ pages = [+ page+ for page, users in (+ ("/", 34),+ ("/pricing", 20),+ ("/docs", 15),+ ("/blog", 12),+ ("/about", 10),+ ("/careers", 5),+ ("/contact", 4),+ )+ for _ in range(users)+ ]+ web_first = {+ f"web_{variant}_{i}": [+ exposure(variant, "2024-01-02T10:00:00Z", "web", page),+ *([exposure(variant, "2024-01-03T10:00:00Z", "posthog-node", None)] if i % 2 else []),+ ]+ for variant in ("control", "test")+ for i, page in enumerate(pages)+ }+ server_only = {+ f"server_{variant}_{i}": [exposure(variant, "2024-01-03T10:00:00Z", "posthog-node", None)]+ for variant, count in (("control", 5), ("test", 120))+ for i in range(count)+ }+ journeys_for({**web_first, **server_only}, self.team)+ flush_persons_and_events()++ with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=findings_shown):+ response = ExperimentExposuresQueryRunner(+ team=self.team, query=self._running_exposure_query(), user=self.user+ ).calculate()++ [finding] = response.health_findings or []+ self.assertEqual((finding.code, finding.subcode), (ExperimentExposureHealthFindingCode.SRM, None))+ self.assertIn(detail, finding.detail)+ self.assertEqual(+ {key: value for key, value in finding.evidence.items() if not key.endswith("p_value")}, evidence+ )++ @parameterized.expand(+ [+ ("persons_with_multiple_variants_excluded", False, "exclude", False),+ ("persons_by_first_seen_variant", False, "first_seen", False),+ ("persons_from_precomputed_exposures", False, "exclude", True),+ ("groups", True, "exclude", False),+ ]+ )+ @time_machine.travel("2024-01-07T12:00:00Z", tick=False)+ def test_srm_library_segments_add_up_to_the_total_exposures(+ self, _name: str, by_group: bool, multiple_variant_handling: str, use_precomputation: bool+ ) -> None:+ self._setup_precomputation_test(use_precomputation)+ if by_group:+ # HogQL blanks the group key of events from before the group type existed.+ create_group_type_mapping(+ team=self.team,+ project_id=self.team.project_id,+ group_type_index=0,+ group_type="organization",+ created_at=datetime(2024, 1, 1, tzinfo=UTC),+ )+ self.feature_flag.filters["aggregation_group_type_index"] = 0+ self.feature_flag.save()+ self.experiment.exposure_criteria = {"multiple_variant_handling": multiple_variant_handling}+ self._save_experiment_with_precomputation(self.experiment, use_precomputation)+ ff_property = f"$feature/{self.feature_flag.key}"+ libraries = ["web", "posthog-node", "posthog-python", "posthog-ios", "posthog-android", "posthog-go", None]++ def exposure(unit: int, variant: str, timestamp: str, library: str | None) -> dict[str, Any]:+ return {+ "event": "$feature_flag_called",+ "timestamp": timestamp,+ "properties": {+ "$feature_flag_response": variant,+ ff_property: variant,+ "$feature_flag": self.feature_flag.key,+ "$group_0": f"org:{unit}",+ **({"$lib": library} if library else {}),+ },+ }++ # Units from 100 see both variants. Every third unit is exposed again by another library, which must not+ # move it out of the library of its first exposure.+ journeys_for(+ {+ f"unit_{unit}": [+ exposure(unit, "test" if 70 <= unit < 100 else "control", "2024-01-02T10:00:00Z", library),+ *(+ [exposure(unit, "test" if unit >= 70 else "control", "2024-01-03T10:00:00Z", "posthog-ruby")]+ if unit >= 100 or unit % 3 == 0+ else []+ ),+ ]+ for unit in range(105)+ for library in [libraries[unit % len(libraries)]]+ },+ self.team,+ )+ flush_persons_and_events()++ with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=True):+ response = ExperimentExposuresQueryRunner(+ team=self.team, query=self._running_exposure_query(), user=self.user+ ).calculate()++ [finding] = [+ finding+ for finding in response.health_findings or []+ if finding.code == ExperimentExposureHealthFindingCode.SRM+ ]+ segments: set[str] = set()+ exposed_by_variant: dict[str, float] = {}+ for key, value in finding.evidence.items():+ if re.fullmatch(r"library_\d+", key) and isinstance(value, str):+ segments.add(value)+ elif (match := re.fullmatch(r"library_\d+_exposed_(.+)", key)) and isinstance(value, int | float):+ exposed_by_variant[match.group(1)] = exposed_by_variant.get(match.group(1), 0) + value+ self.assertEqual(+ (exposed_by_variant, BREAKDOWN_OTHER_STRING_LABEL in segments), (response.total_exposures, True)+ )++ @parameterized.expand([("timeout", ClickHouseQueryTimeOut(), True), ("other_error", RuntimeError("boom"), False)])+ @time_machine.travel("2024-01-07T12:00:00Z", tick=False)+ def test_srm_keeps_the_finding_when_the_segment_query_fails(+ self, _name: str, error: Exception, logged: bool+ ) -> None:+ ff_property = f"$feature/{self.feature_flag.key}"+ journeys_for(+ {+ f"user_{variant}_{i}": [+ {+ "event": "$feature_flag_called",+ "timestamp": "2024-01-02",+ "properties": {+ "$feature_flag_response": variant,+ ff_property: variant,+ "$feature_flag": self.feature_flag.key,+ },+ }+ ]+ for variant, count in (("control", 90), ("test", 10))+ for i in range(count)+ },+ self.team,+ )+ flush_persons_and_events()++ query_types: list[str] = []++ def execute(**kwargs: Any) -> Any:+ query_types.append(kwargs["query_type"])+ if kwargs["query_type"] == "ExperimentExposureSegmentsQuery":+ raise error+ return execute_hogql_query(**kwargs)++ runner_module = ExperimentExposuresQueryRunner.__module__+ with (+ patch(f"{runner_module}.execute_hogql_query", side_effect=execute),+ patch(f"{runner_module}.shows_health_findings", return_value=True),+ patch(f"{runner_module}.logger") as logger,+ patch(f"{runner_module}.capture_exception") as capture_exception,+ ):+ response = ExperimentExposuresQueryRunner(+ team=self.team, query=self._running_exposure_query(), user=self.user+ ).calculate()++ self.assertIn("ExperimentExposureSegmentsQuery", query_types)+ self.assertEqual((logger.warning.called, capture_exception.called), (logged, not logged))+ self.assertEqual(response.total_exposures, {"control": 90, "test": 10})+ self.assertEqual(+ [(finding.code, finding.detail, set(finding.evidence)) for finding in response.health_findings or []],+ [+ (+ ExperimentExposureHealthFindingCode.SRM,+ "The distribution of users across variants doesn't match your configured rollout percentages "+ "(p < 0.001). This may indicate issues with randomization or data collection.",+ {"p_value"},+ )+ ],+ )+def test_cache_key_keeps_the_flag_history_from_a_reader_who_cannot_view_the_flag(self) -> None:self.organization.available_product_features = [{"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}@@ -1640,6 +1886,54 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):self.assertEqual(response.total_exposures, {"control": 0, "test": 0})self.assertEqual([(finding.code, finding.subcode) for finding in response.health_findings or []], expected)+ @parameterized.expand(+ [+ ("test_accounts_exposed", 2, True, "No users exposed: the test account filter removes all of them", 2),+ ("nobody_exposed", 0, True, "No users exposed, although the feature flag is called", 0),+ (+ "reader_who_does_not_see_the_findings",+ 2,+ False,+ "No users exposed, although the feature flag is called",+ None,+ ),+ ]+ )+ @time_machine.travel("2024-01-07T12:00:00Z", tick=False)+ def test_zero_exposures_counts_the_users_that_the_test_account_filter_removes(+ self, _name: str, test_accounts: int, findings_shown: bool, title: str, counted: int | None+ ) -> None:+ self.feature_flag.last_called_at = datetime(2024, 1, 2, tzinfo=UTC)+ self.feature_flag.save()+ self.team.test_account_filters = [+ {"key": "email", "value": "@example.com", "operator": "not_icontains", "type": "person"}+ ]+ self.team.save()+ self.experiment.exposure_criteria = {"filterTestAccounts": True}+ for i in range(test_accounts):+ _create_person(distinct_ids=[f"tester_{i}"], properties={"email": f"tester{i}@example.com"}, team=self.team)+ _create_event(+ distinct_id=f"tester_{i}",+ event="$feature_flag_called",+ timestamp="2024-01-02",+ properties={"$feature_flag_response": "test", "$feature_flag": self.feature_flag.key},+ team=self.team,+ )+ flush_persons_and_events()++ with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=findings_shown):+ response = ExperimentExposuresQueryRunner(+ team=self.team, query=self._running_exposure_query(), user=self.user+ ).calculate()++ self.assertEqual(+ [+ (finding.subcode, finding.title, finding.evidence.get("exposed_without_test_account_filter"))+ for finding in response.health_findings or []+ ],+ [("calls_not_counted", title, counted)],+ )+@parameterized.expand([("direct", False), ("precomputed", True)])@time_machine.travel("2024-01-07T12:00:00Z", tick=False)def test_srm_returns_none_when_insufficient_samples(self, _name, use_precomputation):diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.tsindex 3d296663ea16..b407247ee906 100644--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts@@ -43,6 +43,10 @@ export interface ExperimentHealthFinding {variant?: string/** Set for a server finding. The separate warnings have none. */severity?: ExperimentHealthFindingApi['severity']+ /**+ * Why a check was not checked, or the cause the server's finding names in its `reason` evidence. Both are+ * pinned server values, never customer text.+ */reason?: string}@@ -132,8 +136,8 @@ function healthFindingEventProperties(}}-export function healthFindingReason(evidence: ExperimentHealthFindingApi['evidence']): string | undefined {- return typeof evidence.reason === 'string' ? evidence.reason : undefined+export function healthFindingReason(evidence: ExperimentHealthFindingApi['evidence'] | undefined): string | undefined {+ return typeof evidence?.reason === 'string' ? evidence.reason : undefined}function trackedHealthFinding({ code, subcode, severity, evidence }: ExperimentHealthFindingApi): string {@@ -246,8 +250,9 @@ export function captureExperimentHealthChecked(* every load. `experiment results refresh completed` follows the exposure load on every tab.* `has_srm`, `has_bias_risk` and a zero `exposures_total` are the page's own rules. The `_server` values are* the server's checks on the same answer, null when the answer holds no server findings (a cached answer- * from before the server ran them). `zero_exposures_cause` and `srm_cause` are the server's subcodes of those findings,- * null without the finding or when it names no cause.+ * from before the server ran them). `srm_subcode` and `zero_exposures_subcode` are the `subcode` of those findings, and+ * `srm_reason` and `zero_exposures_reason` the `reason` in their evidence, each null without the finding or when it+ * names none.*/export function exposureHealthEventProperties(exposures: ExperimentExposureQueryResponse | null | undefined,@@ -258,9 +263,11 @@ export function exposureHealthEventProperties(has_srm: boolean | nullhas_bias_risk: boolean | nullsrm_server: boolean | null- srm_cause: string | null+ srm_subcode: string | null+ srm_reason: string | nullzero_exposures_server: boolean | null- zero_exposures_cause: string | null+ zero_exposures_subcode: string | null+ zero_exposures_reason: string | nullbias_risk_server: boolean | null} {if (!exposures) {@@ -270,13 +277,17 @@ export function exposureHealthEventProperties(has_srm: null,has_bias_risk: null,srm_server: null,- srm_cause: null,+ srm_subcode: null,+ srm_reason: null,zero_exposures_server: null,- zero_exposures_cause: null,+ zero_exposures_subcode: null,+ zero_exposures_reason: null,bias_risk_server: null,}}const serverCodes = exposures.health_findings ? new Set(exposures.health_findings.map(({ code }) => code)) : null+ const srmFinding = exposures.health_findings?.find(({ code }) => code === 'srm')+ const zeroExposuresFinding = exposures.health_findings?.find(({ code }) => code === 'zero_exposures')return {exposures_total: getTotalExposures(exposures),// With "first seen" handling the exposure query gives each person their first variant and@@ -289,10 +300,14 @@ export function exposureHealthEventProperties(has_bias_risk: exposures.bias_risk != null,srm_server: serverCodes ? serverCodes.has('srm') : null,// pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned subcodes.- srm_cause: exposures.health_findings?.find(({ code }) => code === 'srm')?.subcode ?? null,+ srm_subcode: srmFinding?.subcode ?? null,+ // pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned reasons.+ srm_reason: healthFindingReason(srmFinding?.evidence) ?? null,zero_exposures_server: serverCodes ? serverCodes.has('zero_exposures') : null,// pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned subcodes.- zero_exposures_cause: exposures.health_findings?.find(({ code }) => code === 'zero_exposures')?.subcode ?? null,+ zero_exposures_subcode: zeroExposuresFinding?.subcode ?? null,+ // pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned reasons.+ zero_exposures_reason: healthFindingReason(zeroExposuresFinding?.evidence) ?? null,bias_risk_server: serverCodes ? serverCodes.has('bias_risk_multiple_excluded') : null,}}