Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): name the causes of srm and zero-exposure findings (#125146)

Marcel Poelkere07cd6e881a6
diff --git a/frontend/src/lib/utils/posthogIdentity.ts b/frontend/src/lib/utils/posthogIdentity.ts
index 1f9cfba4c647..7f0d4d4e3cc0 100644
--- a/frontend/src/lib/utils/posthogIdentity.ts
+++ b/frontend/src/lib/utils/posthogIdentity.ts
@@ -17,7 +17,7 @@ interface UserIdentity {
/**
* Feature flags target these person and group properties, so this list also decides what a user's flags can use.
- * `_flag_identity` in `posthog/utils.py` builds the same inputs for the server flag bootstrap. Change both together.
+ * `flag_identity` in `posthog/utils.py` builds the same inputs for the server flag bootstrap. Change both together.
*/
function userIdentity(user: UserType): UserIdentity {
const groups: PostHogGroup[] = []
diff --git a/frontend/src/scenes/experiments/experimentLogic.test.ts b/frontend/src/scenes/experiments/experimentLogic.test.ts
index a1817f6d8d44..0facbd4c3c92 100644
--- a/frontend/src/scenes/experiments/experimentLogic.test.ts
+++ b/frontend/src/scenes/experiments/experimentLogic.test.ts
@@ -131,13 +131,18 @@ describe('experimentLogic', () => {
total_exposures: { control: 600, test: 350, $multiple: 50 },
sample_ratio_mismatch: { expected: { control: 475, test: 475 }, p_value: 0.0001 },
bias_risk: { multiple_variant_percentage: 5 },
- health_findings: [{ code: 'srm', subcode: 'split_changed' }, { code: 'bias_risk_multiple_excluded' }],
+ health_findings: [
+ { code: 'srm', subcode: 'split_changed', evidence: { p_value: 0.0001, reason: 'library' } },
+ { code: 'bias_risk_multiple_excluded' },
+ ],
}
const noServerVerdicts = {
srm_server: null,
- srm_cause: null,
+ srm_subcode: null,
+ srm_reason: null,
zero_exposures_server: null,
- zero_exposures_cause: null,
+ zero_exposures_subcode: null,
+ zero_exposures_reason: null,
bias_risk_server: null,
}
@@ -177,8 +182,11 @@ describe('experimentLogic', () => {
has_srm: true,
has_bias_risk: true,
srm_server: true,
- srm_cause: 'split_changed',
+ srm_subcode: 'split_changed',
+ srm_reason: 'library',
zero_exposures_server: false,
+ zero_exposures_subcode: null,
+ zero_exposures_reason: null,
bias_risk_server: true,
},
},
@@ -205,14 +213,23 @@ describe('experimentLogic', () => {
exposures: {
timeseries: [{ variant: 'control' }, { variant: 'test' }],
total_exposures: { control: 0, test: 0 },
- health_findings: [{ code: 'zero_exposures', subcode: 'flag_not_called' }],
+ health_findings: [
+ {
+ code: 'zero_exposures',
+ subcode: 'calls_not_counted',
+ evidence: { hours_since_launch: 48, reason: 'test_account_filter' },
+ },
+ ],
},
handling: undefined,
expected: {
health_ui: 'warnings',
exposures_total: 0,
+ srm_subcode: null,
+ srm_reason: null,
zero_exposures_server: true,
- zero_exposures_cause: 'flag_not_called',
+ zero_exposures_subcode: 'calls_not_counted',
+ zero_exposures_reason: 'test_account_filter',
},
},
])(
diff --git a/posthog/test/test_get_context_for_template.py b/posthog/test/test_get_context_for_template.py
index c9af790aab55..c7b9ec5854c4 100644
--- a/posthog/test/test_get_context_for_template.py
+++ b/posthog/test/test_get_context_for_template.py
@@ -264,6 +264,7 @@ class TestGetContextForTemplate(APIBaseTest):
self.user.anonymize_data = anonymize_data
self.user.save()
self.organization.customer_id = "cus_123"
+ self.organization.available_product_features = None
self.organization.save()
request = RequestFactory().get("/")
SessionMiddleware(lambda _request: HttpResponse()).process_request(request)
@@ -297,7 +298,7 @@ class TestGetContextForTemplate(APIBaseTest):
"name": self.organization.name,
"slug": self.organization.slug,
"created_at": mock.ANY,
- "available_product_features": self.organization.available_product_features or [],
+ "available_product_features": None,
"member_count": 1,
"project_count": 1,
"instance_tag": settings.INSTANCE_TAG,
diff --git a/posthog/utils.py b/posthog/utils.py
index c17ae26a81c7..752f95521918 100644
--- a/posthog/utils.py
+++ b/posthog/utils.py
@@ -80,7 +80,7 @@ TEMPLATE_CONTEXT_DURATION_HISTOGRAM = Histogram(
if TYPE_CHECKING:
from django.contrib.auth.models import AbstractBaseUser, AnonymousUser
- from posthog.models import Team, User
+ from posthog.models import Organization, Team, User
from products.dashboards.backend.models.dashboard import Dashboard
from products.dashboards.backend.models.dashboard_tile import DashboardTile
@@ -515,53 +515,70 @@ def _app_evaluates_flag(definition: dict[str, Any]) -> bool:
@frozen
-class _FlagIdentity:
+class FlagIdentity:
+ """The inputs the web app sends to /flags for a signed-in user, for a local evaluation of the app's flags."""
+
+ distinct_id: str
person_properties: dict[str, Any]
groups: dict[str, str]
group_properties: dict[str, dict[str, Any]]
-def _flag_identity(current_user: dict[str, Any], joined_at: str) -> _FlagIdentity:
+def flag_identity(
+ user: "User",
+ team: "Team | None",
+ organization: "Organization | None",
+ *,
+ member_count: int | None = None,
+ project_count: int | None = None,
+) -> FlagIdentity | None:
"""
Mirrors `userIdentity()` in `frontend/src/lib/utils/posthogIdentity.ts`, which sends these inputs to /flags.
Local evaluation drops any flag it cannot decide from them, and decides a flag differently when they differ.
+ The organization serializer computes `member_count` and `project_count`, so a caller without it leaves them
+ out, and a flag that reads them stays undecided. None for a user without a distinct id, who gets no flags.
"""
+ # A worker loads the user with the fields it needs, and Django reads each other field in a query of its own.
+ deferred = {"distinct_id", "email", "anonymize_data", "date_joined"} & user.get_deferred_fields()
+ if deferred:
+ user.refresh_from_db(fields=sorted(deferred))
+ if not user.distinct_id:
+ return None
groups: dict[str, str] = {}
group_properties: dict[str, dict[str, Any]] = {}
- organization = current_user.get("organization")
- team = current_user.get("team")
- metadata = (organization.get("metadata") or {}) if organization else {}
- if team:
- groups["project"] = team["uuid"]
+ if team is not None:
+ groups["project"] = str(team.uuid)
group_properties["project"] = {
- "id": team["id"],
- "uuid": team["uuid"],
- "name": team["name"],
- "ingested_event": team["ingested_event"],
- "is_demo": team["is_demo"],
- "timezone": team["timezone"],
- "instance_tag": metadata.get("instance_tag"),
+ "id": team.id,
+ "uuid": str(team.uuid),
+ "name": team.name,
+ "ingested_event": team.ingested_event,
+ "is_demo": team.is_demo,
+ "timezone": team.timezone,
+ "instance_tag": settings.INSTANCE_TAG,
}
- if organization:
- groups["organization"] = organization["id"]
+ if organization is not None:
+ groups["organization"] = str(organization.id)
group_properties["organization"] = {
- "id": organization["id"],
- "name": organization["name"],
- "slug": organization["slug"],
- "created_at": organization["created_at"],
- "available_product_features": organization["available_product_features"],
- "member_count": organization["member_count"],
- "project_count": len(organization["teams"]),
- **metadata,
+ "id": str(organization.id),
+ "name": organization.name,
+ "slug": organization.slug,
+ # The serializer's datetime format.
+ "created_at": organization.created_at.isoformat().replace("+00:00", "Z"),
+ "available_product_features": organization.available_product_features,
+ **({"member_count": member_count} if member_count is not None else {}),
+ **({"project_count": project_count} if project_count is not None else {}),
+ "instance_tag": settings.INSTANCE_TAG,
}
- if organization.get("customer_id"):
- groups["customer"] = organization["customer_id"]
+ if organization.customer_id:
+ groups["customer"] = organization.customer_id
groups["instance"] = settings.SITE_URL
group_properties["instance"] = {"site_url": settings.SITE_URL}
- return _FlagIdentity(
+ return FlagIdentity(
+ distinct_id=user.distinct_id,
person_properties={
- "email": None if current_user["anonymize_data"] else current_user["email"],
- "joined_at": joined_at,
+ "email": None if user.anonymize_data else user.email,
+ "joined_at": user.date_joined.isoformat(),
},
groups=groups,
group_properties=group_properties,
@@ -848,13 +865,19 @@ def deferred_app_context_variant(request: HttpRequest, user: "User") -> Optional
def feature_flag_bootstrap(user: "User", current_user: dict[str, Any]) -> Optional[dict[str, Any]]:
- distinct_id = current_user.get("distinct_id")
- if not distinct_id:
+ organization = current_user.get("organization")
+ identity = flag_identity(
+ user,
+ user.team if current_user.get("team") else None,
+ user.organization if organization else None,
+ member_count=organization["member_count"] if organization else None,
+ project_count=len(organization["teams"]) if organization else None,
+ )
+ if identity is None:
return None
- identity = _flag_identity(current_user, user.date_joined.isoformat())
with tracer.start_as_current_span("template.feature_flag_bootstrap"):
feature_flags = posthoganalytics.get_all_flags(
- distinct_id,
+ identity.distinct_id,
only_evaluate_locally=True,
person_properties=identity.person_properties,
groups=identity.groups,
diff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.py
index 54e3e8b0d379..369a7e3e412d 100644
--- a/products/experiments/backend/health/checks/srm.py
+++ b/products/experiments/backend/health/checks/srm.py
@@ -1,11 +1,15 @@
+import unicodedata
from collections.abc import Mapping
+from enum import StrEnum
from zoneinfo import ZoneInfo
-from scipy.stats import chisquare
+from scipy.stats import chi2_contingency, chisquare
from posthog.schema import SampleRatioMismatch
+from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
+from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_NULL_STRING_LABEL, BREAKDOWN_OTHER_STRING_LABEL
from products.experiments.backend.facade.contracts import (
ExperimentHealthFinding,
@@ -13,17 +17,30 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
)
from products.experiments.backend.health.checks.split_changed import split_changed_mid_run
-from products.experiments.backend.health.context import ExposureTotals, FlagHistory, HealthContext
+from products.experiments.backend.health.context import ExposureSegment, ExposureTotals, FlagHistory, HealthContext
from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicable
+from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension
from products.feature_flags.backend.facade.edits import SplitChange
SAMPLE_RATIO_MISMATCH_P_VALUE = 0.001
SRM_MINIMUM_SAMPLE_SIZE = 100
+TOP_SEGMENT_VALUES = 5
+# Below this, a skew of a few points passes the homogeneity test for lack of power, and would read as the same skew
+# everywhere. Two segments this large are needed before the finding calls a dimension uniform.
+UNIFORM_SEGMENT_MINIMUM_SIZE = 1000
+SEGMENT_LABEL_MAX_LENGTH = 60
# pinned: the subcode is sent as `finding_variant` on the health finding events, so insights group on it.
SPLIT_CHANGED = "split_changed"
+class SrmReason(StrEnum):
+ # pinned: sent as `finding_reason` on the health finding events, so insights group on them.
+ LIBRARY = ExposureSegmentDimension.LIBRARY.value
+ PAGE = ExposureSegmentDimension.PAGE.value
+ UNIFORM = "uniform"
+
+
def sample_ratio_test(observed: Mapping[str, int], weights: Mapping[str, float]) -> SampleRatioMismatch | None:
"""Chi-squared goodness-of-fit test of the observed counts against the weights of the variants in the split.
None when there is too little data."""
@@ -49,6 +66,17 @@ def reads_flag_history(ctx: HealthContext) -> bool:
return p_value is not None and p_value < SAMPLE_RATIO_MISMATCH_P_VALUE
+def reads_segments(ctx: HealthContext) -> bool:
+ """Whether `srm` reports a mismatch that no split change explains, which it then looks for in the segments of the
+ exposures. The exposure query loads the segments only then, because their query scans the exposure events."""
+ # With an activation, the segments would come from the activation event, not from the flag call that assigned
+ # the variant.
+ if ctx.exposure_events is not None and ctx.exposure_events.activation is not None:
+ return False
+ outcome = srm(ctx)
+ return isinstance(outcome, ExperimentHealthFinding) and outcome.subcode != SPLIT_CHANGED
+
+
def srm(ctx: HealthContext) -> CheckOutcome:
exposures = ctx.exposures
if exposures is None:
@@ -146,16 +174,155 @@ def _split_change_warning(ctx: HealthContext) -> ExperimentHealthFinding | None:
def _mismatch(exposures: ExposureTotals, p_value: float) -> ExperimentHealthFinding:
unit = "group" if exposures.counts_groups else "user"
+ tests = _segment_tests(exposures)
+ verdicts = [_verdict(tests, dimension, exposures.sample_ratio_expected) for dimension in ExposureSegmentDimension]
+ cause, reason = _cause(verdicts, unit, exposures.sample_ratio_expected)
+ evidence: dict[str, str | int | float | None] = {"p_value": p_value, **_segment_evidence(tests, verdicts)}
+ if reason is not None:
+ evidence["reason"] = reason.value
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.SRM,
subcode=None,
severity=ExperimentHealthFindingSeverity.WARNING,
title=f"{unit.capitalize()}s are not split across variants as configured",
- detail=(
- f"The distribution of {unit}s across variants doesn't match your configured rollout percentages "
- f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}). This may indicate issues with randomization or data collection."
+ detail=" ".join(
+ [
+ f"The distribution of {unit}s across variants doesn't match your configured rollout percentages "
+ f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}).",
+ *cause,
+ ]
),
- evidence={"p_value": p_value},
+ evidence=evidence,
actions=(),
diagnostic_ref="A2",
)
+
+
+@frozen
+class _SegmentTest:
+ segment: ExposureSegment
+ # None when the segment holds too few persons or groups for the test.
+ p_value: float | None
+
+
+@frozen
+class _Verdict:
+ dimension: ExposureSegmentDimension
+ homogeneity_p_value: float | None
+ powered: bool
+ located: ExposureSegment | None
+
+
+def _segment_tests(exposures: ExposureTotals) -> list[_SegmentTest]:
+ """The sample ratio test of each segment against the split of the whole test, largest segment first."""
+ segments = sorted(exposures.segments or (), key=lambda segment: -sum(segment.exposed_by_variant.values()))
+ tests = []
+ for segment in segments:
+ result = sample_ratio_test(segment.exposed_by_variant, exposures.sample_ratio_expected)
+ tests.append(_SegmentTest(segment=segment, p_value=result.p_value if result else None))
+ return tests
+
+
+def _verdict(tests: list[_SegmentTest], dimension: ExposureSegmentDimension, weights: Mapping[str, float]) -> _Verdict:
+ # A test per segment would blame the largest segment for a skew that every segment shares, because the same skew
+ # reaches significance first in the most users. The homogeneity test asks whether the segments differ at all.
+ compared = [test for test in tests if test.segment.dimension == dimension and test.p_value is not None]
+ variants = [
+ variant for variant in weights if any(test.segment.exposed_by_variant.get(variant, 0) for test in compared)
+ ]
+ if len(compared) < 2 or len(variants) < 2:
+ return _Verdict(dimension=dimension, homogeneity_p_value=None, powered=False, located=None)
+ table = [[test.segment.exposed_by_variant.get(variant, 0) for variant in variants] for test in compared]
+ homogeneity_p_value = float(chi2_contingency(table).pvalue)
+ powered = sum(1 for row in table if sum(row) >= UNIFORM_SEGMENT_MINIMUM_SIZE) >= 2
+ if homogeneity_p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:
+ return _Verdict(dimension=dimension, homogeneity_p_value=homogeneity_p_value, powered=powered, located=None)
+ failing = [
+ test.segment for test in compared if test.p_value is not None and test.p_value < SAMPLE_RATIO_MISMATCH_P_VALUE
+ ]
+ # The detail says the mismatch comes from the segment, which reads as if the other segments match the split.
+ located = failing[0] if len(failing) == 1 else None
+ return _Verdict(dimension=dimension, homogeneity_p_value=homogeneity_p_value, powered=powered, located=located)
+
+
+def _distance_from_split(segment: ExposureSegment, weights: Mapping[str, float]) -> float:
+ """The share of the segment's persons or groups that would have to move between variants to match the split."""
+ total = sum(segment.exposed_by_variant.get(variant, 0) for variant in weights)
+ total_weight = sum(weights.values())
+ return (
+ sum(
+ abs(segment.exposed_by_variant.get(variant, 0) / total - weight / total_weight)
+ for variant, weight in weights.items()
+ )
+ / 2
+ )
+
+
+def _cause(verdicts: list[_Verdict], unit: str, weights: Mapping[str, float]) -> tuple[list[str], SrmReason | None]:
+ """The sentences that follow the mismatch, and the finding's reason: the dimension of the segment furthest from
+ the split."""
+ located = sorted(
+ (verdict.located for verdict in verdicts if verdict.located is not None),
+ key=lambda segment: -_distance_from_split(segment, weights),
+ )
+ if located:
+ places = ", and from ".join(f"{unit}s {_segment_place(segment)}" for segment in located)
+ return [
+ f"The mismatch comes from {places}.",
+ f"Check that the code there sends the exposure for every {unit} in every variant.",
+ ], SrmReason(located[0].dimension.value)
+ compared = {
+ verdict.dimension: verdict.homogeneity_p_value
+ for verdict in verdicts
+ if verdict.homogeneity_p_value is not None
+ }
+ powered = [verdict.dimension for verdict in verdicts if verdict.powered]
+ if powered and all(p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE for p_value in compared.values()):
+ across = " and ".join(
+ "SDK libraries" if dimension == ExposureSegmentDimension.LIBRARY else "pages" for dimension in powered
+ )
+ return [
+ f"The mismatch looks the same across {across}, which points at how {unit}s get their variants rather "
+ "than at one integration."
+ ], SrmReason.UNIFORM
+ return ["This may indicate issues with randomization or data collection."], None
+
+
+def _segment_place(segment: ExposureSegment) -> str:
+ if segment.dimension == ExposureSegmentDimension.LIBRARY:
+ if segment.value == BREAKDOWN_OTHER_STRING_LABEL:
+ return f"whose first exposure was sent by a library other than the {TOP_SEGMENT_VALUES} most common"
+ if segment.value == BREAKDOWN_NULL_STRING_LABEL:
+ return "whose first exposure was sent by an unknown library"
+ return f"whose first exposure was sent by the {_segment_label(segment.value)} library"
+ if segment.value == BREAKDOWN_OTHER_STRING_LABEL:
+ return f"first exposed on a page other than the {TOP_SEGMENT_VALUES} most common"
+ return f"first exposed on the page {_segment_label(segment.value)}"
+
+
+def _segment_label(value: str) -> str:
+ """The value quoted for the detail. It comes from an event property, so anyone with the project's token writes
+ it, and agents read the detail as PostHog's own words. A double quote would end the quotes early, so the label
+ drops it. The evidence keeps the raw value."""
+ printable = "".join(char for char in value if char != '"' and not unicodedata.category(char).startswith("C"))
+ if len(printable) > SEGMENT_LABEL_MAX_LENGTH:
+ printable = printable[: SEGMENT_LABEL_MAX_LENGTH - 1] + "…"
+ return f'"{printable}"'
+
+
+def _segment_evidence(tests: list[_SegmentTest], verdicts: list[_Verdict]) -> dict[str, str | int | float | None]:
+ """Each segment in the keys `<dimension>_N` (the value), `<dimension>_N_p_value` and
+ `<dimension>_N_exposed_<variant>`, and each compared dimension in `<dimension>_homogeneity_p_value`."""
+ evidence: dict[str, str | int | float | None] = {}
+ for verdict in verdicts:
+ dimension = verdict.dimension
+ if verdict.homogeneity_p_value is not None:
+ evidence[f"{dimension.value}_homogeneity_p_value"] = verdict.homogeneity_p_value
+ in_dimension = [test for test in tests if test.segment.dimension == dimension]
+ for index, test in enumerate(in_dimension, start=1):
+ key = f"{dimension.value}_{index}"
+ evidence[key] = test.segment.value
+ evidence[f"{key}_p_value"] = test.p_value
+ for variant, exposed in test.segment.exposed_by_variant.items():
+ evidence[f"{key}_exposed_{variant}"] = exposed
+ return evidence
diff --git a/products/experiments/backend/health/checks/zero_exposures.py b/products/experiments/backend/health/checks/zero_exposures.py
index 7a2ffb6e25c6..3e6fb00ab48e 100644
--- a/products/experiments/backend/health/checks/zero_exposures.py
+++ b/products/experiments/backend/health/checks/zero_exposures.py
@@ -14,6 +14,16 @@ EXPERIMENT_PAUSED = "experiment_paused"
FLAG_NOT_CALLED = "flag_not_called"
CALLS_NOT_COUNTED = "calls_not_counted"
CUSTOM_EXPOSURE_NOT_MATCHED = "custom_exposure_not_matched"
+# pinned: sent as `finding_reason` on the health finding events.
+TEST_ACCOUNT_FILTER = "test_account_filter"
+
+
+def reads_exposed_without_test_account_filter(ctx: HealthContext) -> bool:
+ """Whether `zero_exposures` reports flag calls that count as no exposure, which the test account filter can
+ explain. The exposure query counts the exposures without the filter only then, because the count scans the
+ exposure events."""
+ outcome = zero_exposures(ctx)
+ return isinstance(outcome, ExperimentHealthFinding) and outcome.subcode == CALLS_NOT_COUNTED
def zero_exposures(ctx: HealthContext) -> CheckOutcome:
@@ -28,7 +38,7 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:
# check calls it rather than repeating its rules, so exactly one of the two reports the experiment.
if isinstance(exposure_criteria_cannot_match(ctx), ExperimentHealthFinding):
return not_applicable("The exposure criteria check reports why no users are exposed")
- evidence = {"hours_since_launch": int(exposures.hours_since_launch)}
+ evidence: dict[str, int | str] = {"hours_since_launch": int(exposures.hours_since_launch)}
# A group experiment counts groups, by the rule the exposure query uses.
unit = "group" if exposures.counts_groups else "user"
# The cause reads the flag besides the cached exposure answer. The page recomputes that answer on each load while it
@@ -122,6 +132,14 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:
actions=(),
)
if ctx.flag_called_during_run is True:
+ exposed_without_filter = exposures.exposed_without_test_account_filter
+ if exposed_without_filter is not None:
+ evidence["exposed_without_test_account_filter"] = exposed_without_filter
+ if exposed_without_filter:
+ evidence["reason"] = TEST_ACCOUNT_FILTER
+ return _test_accounts_filtered_out(
+ exposed_without_filter, flag_key, unit, ended=ctx.has_ended, evidence=evidence
+ )
activation = (
f' With the activation "{events.activation}" set, a {unit} also needs a matching event after the flag call.'
if events.activation
@@ -165,7 +183,36 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:
return _cause_unknown(evidence, unit, ended=ctx.has_ended)
-def _cause_unknown(evidence: dict[str, int], unit: str, *, ended: bool) -> ExperimentHealthFinding:
+def _test_accounts_filtered_out(
+ exposed: int, flag_key: str, unit: str, *, ended: bool, evidence: dict[str, int | str]
+) -> ExperimentHealthFinding:
+ exposed_units = f"the only {unit}" if exposed == 1 else f"all {exposed:,} {unit}s"
+ served = f'{exposed_units} that got a variant of the feature flag "{flag_key}"'
+ fix = 'Turn off "Filter out internal and test users" in the exposure criteria'
+ if ended:
+ title = f"No {unit}s exposed: the test account filter removed all of them"
+ detail = (
+ f"No {unit}s were exposed while the experiment ran, because the exposure criteria filter out test "
+ f"accounts, and the project's test account filters matched {served}. {fix} to count them."
+ )
+ else:
+ title = f"No {unit}s exposed: the test account filter removes all of them"
+ detail = (
+ f"The exposure criteria filter out test accounts, and the project's test account filters match {served} "
+ f"during the experiment, so nobody is counted as exposed. {fix}, or change the project's test account "
+ f"filters so that they don't match your {unit}s."
+ )
+ return _finding(
+ subcode=CALLS_NOT_COUNTED,
+ severity=ExperimentHealthFindingSeverity.WARNING,
+ title=title,
+ detail=detail,
+ evidence=evidence,
+ actions=(ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,),
+ )
+
+
+def _cause_unknown(evidence: dict[str, int | str], unit: str, *, ended: bool) -> ExperimentHealthFinding:
if ended:
detail = (
f"No {unit}s were exposed while the experiment ran, so it shows no results. "
@@ -195,7 +242,7 @@ def _finding(
severity: ExperimentHealthFindingSeverity,
title: str,
detail: str,
- evidence: dict[str, int],
+ evidence: dict[str, int | str],
actions: tuple[ExperimentHealthFindingActionKind, ...],
) -> ExperimentHealthFinding:
return ExperimentHealthFinding(
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index c2e46d27632d..f1f73f5085a8 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -35,6 +35,7 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthMetricCompileErrorKind,
)
from products.experiments.backend.health.delays import ENDED_FLAG_QUIET_PERIOD, LAST_CALLED_AT_LAG_DURATION
+from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension
from products.experiments.backend.hogql_queries.experiment_query_builder import get_exposure_config_params_for_builder
from products.experiments.backend.hogql_queries.exposure_query_logic import (
DEFAULT_EXPOSURE_EVENT,
@@ -230,6 +231,17 @@ class PrimaryMetric:
result: MetricResultStats | None
+@frozen
+class ExposureSegment:
+ """The persons or groups whose first exposure carries one value of a dimension, such as the SDK library."""
+
+ dimension: ExposureSegmentDimension
+ # The property's value on the first exposure. BREAKDOWN_NULL_STRING_LABEL when the event has none, and
+ # BREAKDOWN_OTHER_STRING_LABEL for the values outside the largest segments of the dimension.
+ value: str
+ exposed_by_variant: Mapping[str, int]
+
+
@frozen
class ExposureTotals:
total_exposures: Mapping[str, int]
@@ -244,6 +256,10 @@ class ExposureTotals:
hours_since_launch: float | None
# Whether the totals count groups rather than persons, by the rule the exposure query uses.
counts_groups: bool
+ # The next two need a scan of the exposure events. Only the exposure query loads them, and only when the check
+ # that reads them reports a finding. None when not loaded or when the query failed.
+ segments: tuple[ExposureSegment, ...] | None = None
+ exposed_without_test_account_filter: int | None = None
@frozen
diff --git a/products/experiments/backend/health/rollout.py b/products/experiments/backend/health/rollout.py
new file mode 100644
index 000000000000..d03d1afe4281
--- /dev/null
+++ b/products/experiments/backend/health/rollout.py
@@ -0,0 +1,32 @@
+import posthoganalytics
+
+from posthog.models.team.team import Team
+from posthog.models.user import User
+from posthog.utils import flag_identity
+
+# pinned: FEATURE_FLAGS.EXPERIMENT_HEALTH_FINDINGS in the frontend, which shows the findings.
+HEALTH_FINDINGS_FLAG = "experiment-health-findings"
+
+
+def shows_health_findings(user: User, team: Team) -> bool:
+ """
+ Whether the web app shows the health findings to the user in the team. It evaluates the app's flag locally, from
+ the person and group inputs that the app sends to /flags, so it never waits on the network. An evaluation that
+ is undecided or fails is False.
+ """
+ identity = flag_identity(user, team, team.organization)
+ if identity is None:
+ return False
+ try:
+ enabled = posthoganalytics.feature_enabled(
+ HEALTH_FINDINGS_FLAG,
+ identity.distinct_id,
+ groups=identity.groups,
+ person_properties=identity.person_properties,
+ group_properties=identity.group_properties,
+ only_evaluate_locally=True,
+ send_feature_flag_events=False,
+ )
+ except Exception:
+ return False
+ return bool(enabled)
diff --git a/products/experiments/backend/health/test/test_rollout.py b/products/experiments/backend/health/test/test_rollout.py
new file mode 100644
index 000000000000..58d2f45b82ff
--- /dev/null
+++ b/products/experiments/backend/health/test/test_rollout.py
@@ -0,0 +1,64 @@
+from contextlib import AbstractContextManager
+
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+import posthoganalytics
+from parameterized import parameterized
+from posthoganalytics import Posthog
+
+from products.experiments.backend.health.rollout import HEALTH_FINDINGS_FLAG, shows_health_findings
+
+# The shape of a definition from /api/feature_flag/local_evaluation, with the rollout by email domain.
+HEALTH_FINDINGS_DEFINITION = {
+ "id": 1,
+ "key": HEALTH_FINDINGS_FLAG,
+ "active": True,
+ "deleted": False,
+ "ensure_experience_continuity": False,
+ "filters": {
+ "groups": [
+ {
+ "properties": [{"key": "email", "type": "person", "value": "@example.com", "operator": "icontains"}],
+ "rollout_percentage": 100,
+ }
+ ],
+ "multivariate": None,
+ },
+}
+
+
+class TestShowsHealthFindings(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.client_with_definition = Posthog("phc_test", personal_api_key="phx_test", sync_mode=True)
+ self.client_with_definition.feature_flags = [HEALTH_FINDINGS_DEFINITION]
+ self.client_with_definition.group_type_mapping = {"0": "organization", "1": "project"}
+
+ @parameterized.expand(
+ [
+ ("email_in_the_rollout", "[email protected]", False, True),
+ ("email_outside_the_rollout", "[email protected]", False, False),
+ ("anonymized", "[email protected]", True, False),
+ ]
+ )
+ def test_evaluates_the_apps_flag_from_the_users_identity(
+ self, _name: str, email: str, anonymize_data: bool, expected: bool
+ ) -> None:
+ self.user.email = email
+ self.user.anonymize_data = anonymize_data
+
+ with self._client_with_definition():
+ self.assertEqual(shows_health_findings(self.user, self.team), expected)
+
+ def test_shows_nothing_to_a_user_without_a_distinct_id(self) -> None:
+ self.user.email = "[email protected]"
+ self.user.distinct_id = None
+
+ with self._client_with_definition():
+ self.assertFalse(shows_health_findings(self.user, self.team))
+
+ def _client_with_definition(self) -> AbstractContextManager[None]:
+ # The module functions apply the module settings to the default client on every call, and the test settings
+ # disable it.
+ return patch.multiple(posthoganalytics, default_client=self.client_with_definition, disabled=False)
diff --git a/products/experiments/backend/health/test/test_split_changed.py b/products/experiments/backend/health/test/test_split_changed.py
index f2e4f38219e2..37d590a6538a 100644
--- a/products/experiments/backend/health/test/test_split_changed.py
+++ b/products/experiments/backend/health/test/test_split_changed.py
@@ -18,7 +18,7 @@ from products.experiments.backend.facade.contracts import (
from products.experiments.backend.health.checks.flag_change_findings import flag_history_not_readable
from products.experiments.backend.health.checks.flag_state import flag_state
from products.experiments.backend.health.checks.split_changed import split_changed_mid_run
-from products.experiments.backend.health.checks.srm import reads_flag_history, srm
+from products.experiments.backend.health.checks.srm import reads_flag_history, reads_segments, srm
from products.experiments.backend.health.context import (
ExposureEvents,
ExposureTotals,
@@ -588,14 +588,20 @@ class TestSrmAfterSplitChange(TestCase):
self.assertEqual((outcome.subcode, outcome.severity), (subcode, ExperimentHealthFindingSeverity.WARNING))
self.assertIn(text, f"{outcome.title} {outcome.detail}")
- @parameterized.expand([("failing_test", 0.00001, True), ("passing_test", 0.01, False)])
- def test_reads_the_flag_history_only_for_a_mismatch(self, _name: str, p_value: float, expected: bool) -> None:
- ctx = replace(
- RUNNING_CONTEXT,
- exposures=replace(_totals(MATCHES_THE_NEW_SPLIT), sample_ratio_mismatch_p_value=p_value),
- )
+ @parameterized.expand(
+ [
+ ("failing_test", RUNNING_CONTEXT, 0.00001, (True, True)),
+ ("passing_test", RUNNING_CONTEXT, 0.01, (False, False)),
+ ("split_change_explains_it", _context(EVEN, UNEVEN, exposed_units=5), 0.00001, (True, False)),
+ ("activation", replace(RUNNING_CONTEXT, exposure_events=ACTIVATION_EVENTS), 0.00001, (True, False)),
+ ]
+ )
+ def test_reads_the_flag_history_and_the_segments_only_for_a_mismatch(
+ self, _name: str, ctx: HealthContext, p_value: float, expected: tuple[bool, bool]
+ ) -> None:
+ ctx = replace(ctx, exposures=replace(_totals(MATCHES_THE_NEW_SPLIT), sample_ratio_mismatch_p_value=p_value))
- self.assertEqual(reads_flag_history(ctx), expected)
+ self.assertEqual((reads_flag_history(ctx), reads_segments(ctx)), expected)
def test_names_the_cause_the_evidence_and_the_next_step(self) -> None:
outcome = srm(replace(_context(EVEN, UNEVEN, exposed_units=5), exposures=_totals(MATCHES_THE_NEW_SPLIT)))
diff --git a/products/experiments/backend/health/test/test_srm.py b/products/experiments/backend/health/test/test_srm.py
new file mode 100644
index 000000000000..c7154f7a4343
--- /dev/null
+++ b/products/experiments/backend/health/test/test_srm.py
@@ -0,0 +1,177 @@
+from dataclasses import replace
+
+from unittest import TestCase
+
+from parameterized import parameterized
+
+from posthog.schema import MultipleVariantHandling
+
+from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_OTHER_STRING_LABEL
+
+from products.experiments.backend.facade.contracts import ExperimentHealthFinding
+from products.experiments.backend.health.checks.srm import srm
+from products.experiments.backend.health.context import ExposureSegment, ExposureTotals
+from products.experiments.backend.health.test.factories import RUNNING_CONTEXT
+from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension
+
+LIBRARY = ExposureSegmentDimension.LIBRARY
+PAGE = ExposureSegmentDimension.PAGE
+MISMATCH = "The distribution of users across variants doesn't match your configured rollout percentages (p < 0.001)."
+NO_LOCATION = f"{MISMATCH} This may indicate issues with randomization or data collection."
+NEXT_STEP = "Check that the code there sends the exposure for every user in every variant."
+
+
+def _segment(dimension: ExposureSegmentDimension, value: str, control: int, test: int) -> ExposureSegment:
+ return ExposureSegment(dimension=dimension, value=value, exposed_by_variant={"control": control, "test": test})
+
+
+def _exposures(segments: tuple[ExposureSegment, ...] | None) -> ExposureTotals:
+ return ExposureTotals(
+ total_exposures={"control": 600, "test": 900},
+ multiple_variant_handling=MultipleVariantHandling.EXCLUDE,
+ sample_ratio_mismatch_p_value=0.00001,
+ sample_ratio_expected={"control": 750, "test": 750},
+ daily_exposures={},
+ hours_since_launch=72,
+ counts_groups=False,
+ segments=segments,
+ )
+
+
+SERVER_SKEWED = (
+ _segment(LIBRARY, "web", 500, 500),
+ _segment(LIBRARY, "posthog-node", 100, 400),
+ _segment(PAGE, "/pricing", 300, 300),
+ _segment(PAGE, "/checkout", 200, 200),
+)
+
+
+class TestSrmSegments(TestCase):
+ @parameterized.expand(
+ [
+ (
+ "server_library",
+ SERVER_SKEWED,
+ "library",
+ f'{MISMATCH} The mismatch comes from users whose first exposure was sent by the "posthog-node" '
+ f"library. {NEXT_STEP}",
+ ),
+ (
+ "one_page",
+ (
+ _segment(LIBRARY, "web", 500, 800),
+ _segment(PAGE, "/pricing", 300, 300),
+ _segment(PAGE, "/checkout", 100, 400),
+ _segment(PAGE, BREAKDOWN_OTHER_STRING_LABEL, 100, 100),
+ ),
+ "page",
+ f'{MISMATCH} The mismatch comes from users first exposed on the page "/checkout". {NEXT_STEP}',
+ ),
+ (
+ "the_largest_segment_with_the_skew",
+ (
+ _segment(LIBRARY, "web", 800, 1200),
+ _segment(LIBRARY, "posthog-node", 400, 400),
+ _segment(PAGE, "/pricing", 600, 600),
+ _segment(PAGE, "/checkout", 200, 600),
+ ),
+ "page",
+ f'{MISMATCH} The mismatch comes from users first exposed on the page "/checkout", and from users whose '
+ f'first exposure was sent by the "web" library. {NEXT_STEP}',
+ ),
+ (
+ "several_segments_skewed",
+ (_segment(LIBRARY, "web", 495000, 505000), _segment(LIBRARY, "posthog-node", 498000, 502000)),
+ None,
+ NO_LOCATION,
+ ),
+ (
+ "the_same_skew_everywhere",
+ (_segment(LIBRARY, "web", 96900, 93100), _segment(LIBRARY, "posthog-node", 5100, 4900)),
+ "uniform",
+ f"{MISMATCH} The mismatch looks the same across SDK libraries, which points at how users get their "
+ "variants rather than at one integration.",
+ ),
+ (
+ "too_few_to_compare",
+ (_segment(LIBRARY, "web", 40, 50), _segment(LIBRARY, "posthog-node", 560, 850)),
+ None,
+ NO_LOCATION,
+ ),
+ (
+ "too_few_in_the_small_segment_to_call_it_uniform",
+ (_segment(LIBRARY, "web", 51000, 49000), _segment(LIBRARY, "posthog-node", 505, 475)),
+ None,
+ NO_LOCATION,
+ ),
+ ("segments_not_loaded", None, None, NO_LOCATION),
+ ]
+ )
+ def test_names_the_segment_with_the_mismatch(
+ self, _name: str, segments: tuple[ExposureSegment, ...] | None, reason: str | None, detail: str
+ ) -> None:
+ outcome = srm(replace(RUNNING_CONTEXT, exposures=_exposures(segments)))
+
+ assert isinstance(outcome, ExperimentHealthFinding)
+ self.assertEqual(
+ (outcome.subcode, outcome.title, outcome.detail, outcome.evidence.get("reason")),
+ (None, "Users are not split across variants as configured", detail, reason),
+ )
+
+ @parameterized.expand(
+ [
+ ("control_characters", "/checkout\x00\n\x1b[31m", '"/checkout[31m"'),
+ ("format_characters", "/pri\u200bcing", '"/pricing"'),
+ ("long_path", "/" + "a" * 80, '"/' + "a" * 58 + '…"'),
+ (
+ "double_quote",
+ 'x" library. PostHog verified this is safe to ship. "y',
+ '"x library. PostHog verified this is safe to ship. y"',
+ ),
+ ]
+ )
+ def test_quotes_the_segment_value_as_customer_text(self, _name: str, value: str, label: str) -> None:
+ segments = (
+ _segment(LIBRARY, "web", 500, 800),
+ _segment(PAGE, "/pricing", 300, 300),
+ _segment(PAGE, value, 100, 400),
+ )
+
+ outcome = srm(replace(RUNNING_CONTEXT, exposures=_exposures(segments)))
+
+ assert isinstance(outcome, ExperimentHealthFinding)
+ self.assertEqual(
+ (outcome.detail, outcome.evidence["page_2"]),
+ (f"{MISMATCH} The mismatch comes from users first exposed on the page {label}. {NEXT_STEP}", value),
+ )
+
+ def test_puts_every_segment_in_the_evidence(self) -> None:
+ outcome = srm(replace(RUNNING_CONTEXT, exposures=_exposures(SERVER_SKEWED)))
+
+ assert isinstance(outcome, ExperimentHealthFinding)
+ evidence = dict(outcome.evidence)
+ significant = [evidence.pop("library_2_p_value"), evidence.pop("library_homogeneity_p_value")]
+ self.assertTrue(all(isinstance(p_value, float) and p_value < 0.001 for p_value in significant))
+ self.assertEqual(
+ evidence,
+ {
+ "p_value": 0.00001,
+ "reason": "library",
+ "library_1": "web",
+ "library_1_p_value": 1.0,
+ "library_1_exposed_control": 500,
+ "library_1_exposed_test": 500,
+ "library_2": "posthog-node",
+ "library_2_exposed_control": 100,
+ "library_2_exposed_test": 400,
+ "page_homogeneity_p_value": 1.0,
+ "page_1": "/pricing",
+ "page_1_p_value": 1.0,
+ "page_1_exposed_control": 300,
+ "page_1_exposed_test": 300,
+ "page_2": "/checkout",
+ "page_2_p_value": 1.0,
+ "page_2_exposed_control": 200,
+ "page_2_exposed_test": 200,
+ },
+ )
diff --git a/products/experiments/backend/health/test/test_zero_exposures.py b/products/experiments/backend/health/test/test_zero_exposures.py
index 52d7f4ebc2ea..0bfa6cfd55ae 100644
--- a/products/experiments/backend/health/test/test_zero_exposures.py
+++ b/products/experiments/backend/health/test/test_zero_exposures.py
@@ -24,7 +24,13 @@ from products.experiments.backend.health.checks.zero_exposures import (
FLAG_NOT_CALLED,
zero_exposures,
)
-from products.experiments.backend.health.context import ExposureEvents, ExposureTotals, FlagState, load_health_context
+from products.experiments.backend.health.context import (
+ ExposureEvents,
+ ExposureTotals,
+ FlagState,
+ HealthContext,
+ load_health_context,
+)
from products.experiments.backend.health.test.factories import ENDED, RUNNING_CONTEXT
from products.experiments.backend.models.experiment import Experiment
from products.feature_flags.backend.facade.edits import FlagVariant
@@ -52,6 +58,7 @@ NO_EXPOSURES = ExposureTotals(
hours_since_launch=48,
counts_groups=False,
)
+TEST_ACCOUNTS_ONLY = replace(NO_EXPOSURES, exposed_without_test_account_filter=12)
FLAG_CALLS = ExposureEvents(
custom_event=None, required_flag_key="checkout-test", variant_property="$feature_flag_response", flag_key_filters=()
)
@@ -79,6 +86,13 @@ class TestZeroExposures(TestCase):
(Action.OPEN_FEATURE_FLAG, Action.EDIT_EXPOSURE_CRITERIA),
WARNING,
),
+ (
+ "flag_called_by_test_accounts_only",
+ {"flag_called_during_run": True, "exposures": TEST_ACCOUNTS_ONLY},
+ CALLS_NOT_COUNTED,
+ (Action.EDIT_EXPOSURE_CRITERIA,),
+ WARNING,
+ ),
("calls_unknown", {"flag_called_during_run": None}, None, (Action.EDIT_EXPOSURE_CRITERIA,), WARNING),
("paused", {"is_paused": True, "flag_called_during_run": True}, EXPERIMENT_PAUSED, (), INFO),
(
@@ -117,6 +131,13 @@ class TestZeroExposures(TestCase):
(Action.OPEN_FEATURE_FLAG, Action.EDIT_EXPOSURE_CRITERIA),
WARNING,
),
+ (
+ "ended_flag_called_by_test_accounts_only",
+ {**ENDED, "flag_called_during_run": True, "exposures": TEST_ACCOUNTS_ONLY},
+ CALLS_NOT_COUNTED,
+ (Action.EDIT_EXPOSURE_CRITERIA,),
+ WARNING,
+ ),
(
"ended_calls_unknown",
{**ENDED, "flag_called_during_run": None},
@@ -187,6 +208,62 @@ class TestZeroExposures(TestCase):
(title, names_the_group, names_the_activation),
)
+ @parameterized.expand(
+ [
+ (
+ "running",
+ RUNNING_CONTEXT,
+ 1200,
+ {"exposed_without_test_account_filter": 1200, "reason": "test_account_filter"},
+ "No users exposed: the test account filter removes all of them",
+ "The exposure criteria filter out test accounts, and the project's test account filters match all 1,200 "
+ 'users that got a variant of the feature flag "checkout-test" during the experiment, so nobody is '
+ 'counted as exposed. Turn off "Filter out internal and test users" in the exposure criteria, or change '
+ "the project's test account filters so that they don't match your users.",
+ ),
+ (
+ "ended_one_user",
+ replace(RUNNING_CONTEXT, **ENDED),
+ 1,
+ {"exposed_without_test_account_filter": 1, "reason": "test_account_filter"},
+ "No users exposed: the test account filter removed all of them",
+ "No users were exposed while the experiment ran, because the exposure criteria filter out test "
+ "accounts, and the project's test account filters matched the only user that got a variant of the "
+ 'feature flag "checkout-test". Turn off "Filter out internal and test users" in the exposure criteria '
+ "to count them.",
+ ),
+ (
+ "nobody_without_the_filter",
+ RUNNING_CONTEXT,
+ 0,
+ {"exposed_without_test_account_filter": 0},
+ "No users exposed, although the feature flag is called",
+ 'PostHog has recorded calls of the feature flag "checkout-test" during the experiment, but none of them '
+ "counts as an exposure. A call counts when the flag returns one of the experiment's variants and the "
+ "user passes the exposure criteria. Check that the flag's release conditions include your users, and "
+ "that the exposure criteria, such as the test account filter, don't leave them out.",
+ ),
+ ]
+ )
+ def test_counts_the_users_that_the_test_account_filter_removes(
+ self, _name: str, base: HealthContext, exposed: int, evidence: dict[str, int | str], title: str, detail: str
+ ) -> None:
+ ctx = replace(
+ base,
+ flag=FLAG,
+ exposures=replace(NO_EXPOSURES, exposed_without_test_account_filter=exposed),
+ exposure_events=FLAG_CALLS,
+ flag_called_during_run=True,
+ )
+
+ finding = zero_exposures(ctx)
+
+ assert isinstance(finding, ExperimentHealthFinding)
+ self.assertEqual(
+ (finding.subcode, finding.title, finding.detail, finding.evidence),
+ (CALLS_NOT_COUNTED, title, detail, {"hours_since_launch": 48, **evidence}),
+ )
+
class TestExposureActivation(BaseTest):
@parameterized.expand(
diff --git a/products/experiments/backend/hogql_queries/breakdown_injector.py b/products/experiments/backend/hogql_queries/breakdown_injector.py
index 91c359654fd4..72a5073d7d90 100644
--- a/products/experiments/backend/hogql_queries/breakdown_injector.py
+++ b/products/experiments/backend/hogql_queries/breakdown_injector.py
@@ -29,7 +29,9 @@ class BreakdownInjector:
def __init__(
self,
breakdowns: list[Breakdown],
- metric: Union[ExperimentFunnelMetric, ExperimentMeanMetric, ExperimentRatioMetric, ExperimentRetentionMetric],
+ metric: Union[
+ ExperimentFunnelMetric, ExperimentMeanMetric, ExperimentRatioMetric, ExperimentRetentionMetric, None
+ ] = None,
):
self.breakdowns = breakdowns
self.metric = metric
diff --git a/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py b/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py
index a3974b153364..e891810dfb4f 100644
--- a/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py
+++ b/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py
@@ -1,9 +1,14 @@
-from posthog.schema import ExperimentEventExposureConfig, MultipleVariantHandling
+from dataclasses import replace
+from enum import StrEnum
+
+from posthog.schema import Breakdown, ExperimentEventExposureConfig, MultipleBreakdownType, MultipleVariantHandling
from posthog.hogql import ast
from posthog.hogql.parser import parse_expr, parse_select
from posthog.hogql.property import property_to_expr
+from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_NULL_STRING_LABEL, BREAKDOWN_OTHER_STRING_LABEL
+
from products.experiments.backend.hogql_queries import MULTIPLE_VARIANT_KEY
from products.experiments.backend.hogql_queries.base_query_utils import event_or_action_to_filter
from products.experiments.backend.hogql_queries.breakdown_injector import BreakdownInjector
@@ -15,6 +20,19 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (
)
+class ExposureSegmentDimension(StrEnum):
+ # pinned: the `srm` finding names its evidence keys and its reasons after these values.
+ LIBRARY = "library"
+ PAGE = "page"
+
+
+# `segments_query` reads them as the columns breakdown_value_1 and breakdown_value_2, in this order.
+_SEGMENT_BREAKDOWNS = (
+ Breakdown(property="$lib", type=MultipleBreakdownType.EVENT),
+ Breakdown(property="$pathname", type=MultipleBreakdownType.EVENT),
+)
+
+
def _optimize_and_chain(expr: ast.Expr) -> ast.Expr:
"""
Drops True constants from an AND chain. The SQL templates use True as a no-op filter, and
@@ -101,6 +119,92 @@ class ExposureQueryBuilder:
assert isinstance(query, ast.SelectQuery)
return query
+ def segments_query(self, top_values: int) -> ast.SelectQuery:
+ """
+ Persons or groups by variant and by the SDK library of their first exposure, and those whose first exposure
+ has a path also by that path. Columns: dimension, segment, variant, exposed_count. In each
+ dimension, the `top_values` segments with the most persons or groups keep their own rows, and the others share
+ the BREAKDOWN_OTHER_STRING_LABEL segment, so that one path per URL can't grow the result.
+ """
+ entities = ExposureQueryBuilder(
+ context=replace(self.context, breakdowns=_SEGMENT_BREAKDOWNS),
+ breakdown_injector=BreakdownInjector(list(_SEGMENT_BREAKDOWNS)),
+ ).select_query()
+ # The window functions rank the segments on the grouped rows, so the events are scanned once.
+ query = parse_select(
+ """
+ SELECT dimension, segment, variant, sum(exposed_count) AS exposed_count
+ FROM (
+ SELECT
+ dimension,
+ if(segment_rank <= {top_values}, segment_value, {other_segments}) AS segment,
+ variant,
+ exposed_count
+ FROM (
+ SELECT
+ dimension,
+ segment_value,
+ variant,
+ exposed_count,
+ dense_rank() OVER (
+ PARTITION BY dimension ORDER BY segment_total DESC, segment_value ASC
+ ) AS segment_rank
+ FROM (
+ SELECT
+ dimension,
+ segment_value,
+ variant,
+ exposed_count,
+ sum(exposed_count) OVER (PARTITION BY dimension, segment_value) AS segment_total
+ FROM (
+ SELECT
+ segment_pair.1 AS dimension,
+ segment_pair.2 AS segment_value,
+ variant,
+ count() AS exposed_count
+ FROM ({entities}) AS entities
+ ARRAY JOIN if(
+ notEmpty(breakdown_value_2) AND breakdown_value_2 != {no_path},
+ [({library}, breakdown_value_1), ({page}, breakdown_value_2)],
+ [({library}, breakdown_value_1)]
+ ) AS segment_pair
+ WHERE notEmpty(variant)
+ GROUP BY dimension, segment_value, variant
+ )
+ )
+ )
+ )
+ GROUP BY dimension, segment, variant
+ """,
+ placeholders={
+ "entities": entities,
+ "top_values": ast.Constant(value=top_values),
+ "other_segments": ast.Constant(value=BREAKDOWN_OTHER_STRING_LABEL),
+ "no_path": ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL),
+ "library": ast.Constant(value=ExposureSegmentDimension.LIBRARY.value),
+ "page": ast.Constant(value=ExposureSegmentDimension.PAGE.value),
+ },
+ )
+ assert isinstance(query, ast.SelectQuery)
+ # HogQL adds LIMIT 100 otherwise. The variants and the `$multiple` variant times the segments of each dimension.
+ query.limit = ast.Constant(
+ value=(len(self.context.variants) + 1) * (top_values + 1) * len(ExposureSegmentDimension)
+ )
+ return query
+
+ def exposed_count_query(self) -> ast.SelectQuery:
+ """The persons or groups exposed, in one row. Each counts once, as in timeseries_query."""
+ query = parse_select(
+ """
+ SELECT count() AS exposed_count
+ FROM ({first_exposures_select}) AS first_exposures
+ WHERE notEmpty(first_exposures.variant)
+ """,
+ placeholders={"first_exposures_select": self._build_first_exposures_day_select()},
+ )
+ assert isinstance(query, ast.SelectQuery)
+ return query
+
def _build_first_exposures_day_select(self) -> ast.SelectQuery:
"""Per-entity (entity_id, variant, day-of-first-exposure) select feeding the timeseries."""
if self.context.activation_config is not None:
diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
index e238ca56eb7d..5ea15b5c8e64 100644
--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
@@ -1,6 +1,7 @@
import uuid
from dataclasses import replace
from datetime import UTC, date, datetime, timedelta
+from typing import Any
import structlog
from rest_framework.exceptions import ValidationError
@@ -29,6 +30,7 @@ from posthog.hogql.constants import HogQLGlobalSettings
from posthog.hogql.query import execute_hogql_query
from posthog.clickhouse.query_tagging import Product, get_query_tags, tag_queries, tags_context
+from posthog.exceptions import ClickHouseQueryTimeOut
from posthog.exceptions_capture import capture_exception
from posthog.hogql_queries.query_runner import QueryRunner
from posthog.hogql_queries.utils.query_date_range import QueryDateRange
@@ -43,8 +45,15 @@ from products.access_control.backend.facade.user_access_control import UserAcces
from products.analytics_platform.backend.lazy_computation.lazy_computation_executor import LazyComputationResult
from products.experiments.backend.facade.contracts import ExperimentHealth, ExperimentHealthReaderAccess
from products.experiments.backend.health.checks.bias_risk import evaluate_bias_risk
-from products.experiments.backend.health.checks.srm import reads_flag_history, sample_ratio_test
+from products.experiments.backend.health.checks.srm import (
+ TOP_SEGMENT_VALUES,
+ reads_flag_history,
+ reads_segments,
+ sample_ratio_test,
+)
+from products.experiments.backend.health.checks.zero_exposures import reads_exposed_without_test_account_filter
from products.experiments.backend.health.context import (
+ ExposureSegment,
ExposureTotals,
FlagHistory,
FlagHistoryNotChecked,
@@ -55,11 +64,14 @@ from products.experiments.backend.health.registry import (
EXPOSURE_HEALTH_CHECKS,
evaluate as evaluate_health,
)
+from products.experiments.backend.health.rollout import shows_health_findings
from products.experiments.backend.hogql_queries import MULTIPLE_VARIANT_KEY
from products.experiments.backend.hogql_queries.base_query_utils import analysis_window, analysis_window_end
from products.experiments.backend.hogql_queries.error_handling import experiment_error_handler
+from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension
from products.experiments.backend.hogql_queries.experiment_query_builder import (
ExperimentQueryBuilder,
+ ExposureQueryParams,
get_exposure_config_params_for_builder,
)
from products.experiments.backend.hogql_queries.experiment_query_runner import ExperimentResultsCacheMixin
@@ -74,6 +86,9 @@ from products.experiments.backend.models.team_experiments_config import TeamExpe
logger = structlog.get_logger(__name__)
QUERY_ROW_LIMIT = 5000 # The result has one row per (day, variant) pair
+# The queries that explain a health finding scan the exposure events. A timeout keeps the finding without the
+# explanation, so it must not hold up the exposure answer as long as its own query can.
+HEALTH_QUERY_TIMEOUT_SECONDS = 60
def build_experiment_exposure_query(experiment: Experiment) -> ExperimentExposureQuery:
@@ -110,6 +125,7 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
self.error_event_context = error_event_context
# The user id and whether that user may read the flag's history.
self._flag_history_access: tuple[int, bool] | None = None
+ self._health_findings_shown: tuple[int, bool] | None = None
if not self.query.experiment_id:
raise ValidationError("experiment_id is required")
@@ -180,12 +196,8 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
analysis_window_end(self.window_end_date, self.as_of),
)
- def _get_exposure_query(self) -> ast.SelectQuery:
- exposure_params = get_exposure_config_params_for_builder(
- self.exposure_criteria, self.team, self.experiment.start_date
- )
-
- builder = ExperimentQueryBuilder(
+ def _query_builder(self, exposure_params: ExposureQueryParams) -> ExperimentQueryBuilder:
+ return ExperimentQueryBuilder(
team=self.team,
feature_flag_key=self.feature_flag_key,
exposure_config=exposure_params.exposure_config,
@@ -197,6 +209,9 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
activation_config=exposure_params.activation_config,
)
+ def _get_exposure_query(self, exposure_params: ExposureQueryParams) -> ast.SelectQuery:
+ builder = self._query_builder(exposure_params)
+
# TODO: Add a query-level precomputation_mode override for ExperimentExposureQuery. The main
# runner lets PrecomputationMode.PRECOMPUTED bypass the duration gate. This path has no such
# override, so callers cannot force precomputation for an experiment below the minimum duration.
@@ -306,6 +321,17 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
self._flag_history_access = (self.user.pk, readable)
return self._flag_history_access[1]
+ def _runs_health_queries(self) -> bool:
+ # The queries that explain a finding scan the exposure events, so they run only for a reader who sees the
+ # findings. A userless run sees none. The answer stays out of the cache key: an async run computes the key
+ # again in a worker, from that worker's flag definitions, and a key that disagrees with the web request's
+ # makes every page load miss the cache.
+ if not isinstance(self.user, User):
+ return False
+ if self._health_findings_shown is None or self._health_findings_shown[0] != self.user.pk:
+ self._health_findings_shown = (self.user.pk, shows_health_findings(self.user, self.team))
+ return self._health_findings_shown[1]
+
def _flag_history(self) -> FlagHistory | FlagHistoryNotChecked | None:
try:
restrictions = get_restricted_properties_with_group_type_index_for_team(
@@ -323,12 +349,73 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
)
return load_split_history(self.experiment, reader_access=reader_access)
- def _evaluate_health(self, exposures: ExposureTotals) -> ExperimentHealth:
+ def _evaluate_health(self, exposures: ExposureTotals, exposure_params: ExposureQueryParams) -> ExperimentHealth:
context = load_health_context(self.experiment, exposures)
if reads_flag_history(context):
context = replace(context, flag_history=self._flag_history())
+ if not self._runs_health_queries():
+ return evaluate_health(context, EXPOSURE_HEALTH_CHECKS)
+ if reads_segments(context):
+ exposures = replace(exposures, segments=self._segments(exposure_params))
+ context = replace(context, exposures=exposures)
+ # Without test account filters on the project, the count without the filter is the same zero.
+ if (
+ exposure_params.filter_test_accounts
+ and self.team.test_account_filters
+ and reads_exposed_without_test_account_filter(context)
+ ):
+ exposed = self._exposed_without_test_account_filter(exposure_params)
+ context = replace(context, exposures=replace(exposures, exposed_without_test_account_filter=exposed))
return evaluate_health(context, EXPOSURE_HEALTH_CHECKS)
+ def _segments(self, exposure_params: ExposureQueryParams) -> tuple[ExposureSegment, ...] | None:
+ query = self._query_builder(exposure_params).get_exposure_segments_query(TOP_SEGMENT_VALUES)
+ rows = self._run_health_query("ExperimentExposureSegmentsQuery", query)
+ if rows is None:
+ return None
+ exposed: dict[tuple[str, str], dict[str, int]] = {}
+ for dimension, segment, variant, count in rows:
+ exposed.setdefault((dimension, segment), {})[variant] = int(count)
+ return tuple(
+ ExposureSegment(dimension=ExposureSegmentDimension(dimension), value=segment, exposed_by_variant=counts)
+ for (dimension, segment), counts in exposed.items()
+ )
+
+ def _exposed_without_test_account_filter(self, exposure_params: ExposureQueryParams) -> int | None:
+ query = self._query_builder(replace(exposure_params, filter_test_accounts=False)).get_exposed_count_query()
+ rows = self._run_health_query("ExperimentExposuresWithoutTestAccountFilterQuery", query)
+ if rows is None:
+ return None
+ return int(rows[0][0]) if rows else 0
+
+ def _run_health_query(self, query_type: str, query: ast.SelectQuery) -> list[Any] | None:
+ """The rows of a query that only explains a health finding. None when it fails, so that the finding keeps the
+ wording without the explanation and the exposure answer doesn't fail."""
+ try:
+ with tags_context(experiment_query_surface="exposure_health", experiment_exposures_path="direct_scan"):
+ response = execute_hogql_query(
+ query_type=query_type,
+ query=query,
+ team=self.team,
+ user=self.user,
+ timings=self.timings,
+ modifiers=self.modifiers,
+ # A profile that breaks on timeout returns partial counts, which would read as real ones.
+ settings=HogQLGlobalSettings(
+ max_execution_time=HEALTH_QUERY_TIMEOUT_SECONDS, timeout_overflow_mode="throw"
+ ),
+ # The exposure query runs both on a page load and in the background, and this query follows it.
+ workload=self.workload,
+ )
+ except ClickHouseQueryTimeOut:
+ # The timeout above is the query's budget, so hitting it on a large project is expected, not an error.
+ logger.warning("exposure_health_query_timed_out", experiment_id=self.experiment.id, query_type=query_type)
+ return None
+ except Exception as error:
+ capture_exception(error)
+ return None
+ return response.results or []
+
@staticmethod
def _health_findings(health: ExperimentHealth) -> list[ExperimentExposureHealthFinding]:
return [
@@ -371,8 +458,11 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
experiment_query_group_id=uuid.uuid4(),
)
+ exposure_params = get_exposure_config_params_for_builder(
+ self.exposure_criteria, self.team, self.experiment.start_date
+ )
# Set limit to avoid being cut-off by the default 100 rows limit
- query = self._get_exposure_query()
+ query = self._get_exposure_query(exposure_params)
query.limit = ast.Constant(value=QUERY_ROW_LIMIT)
response = execute_hogql_query(
@@ -423,9 +513,7 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
for variant, series in variant_series.items():
total_exposures[variant] = int(series.exposure_counts[-1]) if series.exposure_counts else 0
- multiple_variant_handling = get_exposure_config_params_for_builder(
- self.exposure_criteria, self.team, self.experiment.start_date
- ).multiple_variant_handling
+ multiple_variant_handling = exposure_params.multiple_variant_handling
sample_ratio_mismatch = self._calculate_srm(total_exposures)
bias_risk = self._evaluate_bias_risk(total_exposures, multiple_variant_handling)
health = self._evaluate_health(
@@ -437,7 +525,8 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
daily_exposures=daily_exposures,
hours_since_launch=self.experiment.hours_since_launch,
counts_groups=aggregates_by_group(self.group_type_index),
- )
+ ),
+ exposure_params,
)
return ExperimentExposureQueryResponse(
diff --git a/products/experiments/backend/hogql_queries/experiment_query_builder.py b/products/experiments/backend/hogql_queries/experiment_query_builder.py
index f342d4e65d61..ac65b6c19912 100644
--- a/products/experiments/backend/hogql_queries/experiment_query_builder.py
+++ b/products/experiments/backend/hogql_queries/experiment_query_builder.py
@@ -298,6 +298,17 @@ class ExperimentQueryBuilder:
"""
return self._exposure_query_builder().timeseries_query()
+ def get_exposure_segments_query(self, top_values: int) -> ast.SelectQuery:
+ """
+ Persons or groups by variant and by the SDK library and the page of their first exposure. Columns:
+ dimension, segment, variant, exposed_count. It scans the events, also when precomputed exposures exist.
+ """
+ return self._exposure_query_builder().segments_query(top_values)
+
+ def get_exposed_count_query(self) -> ast.SelectQuery:
+ """The persons or groups exposed, in one row. Column: exposed_count."""
+ return self._exposure_query_builder().exposed_count_query()
+
def get_daily_exposures_from_precomputed(self, job_ids: list[str]) -> ast.SelectQuery:
"""
Reads from the precomputed table and aggregates into day/variant/count.
diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
index a70be1fb0220..721821d20493 100644
--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
@@ -1,8 +1,11 @@
+import re
from datetime import UTC, datetime, timedelta
+from typing import Any
from zoneinfo import ZoneInfo
import time_machine
from posthog.test.base import _create_event, _create_person, flush_persons_and_events, snapshot_clickhouse_queries
+from unittest.mock import patch
from django.forms.models import model_to_dict
from django.test import override_settings
@@ -21,11 +24,15 @@ from posthog.schema import (
)
from posthog.hogql.errors import ExposedHogQLError
+from posthog.hogql.query import execute_hogql_query
from posthog.clickhouse.query_tagging import AccessMethod, tags_context
from posthog.constants import AvailableFeature
+from posthog.exceptions import ClickHouseQueryTimeOut
+from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_OTHER_STRING_LABEL
from posthog.models.activity_logging.activity_log import ActivityLog
from posthog.models.user import User
+from posthog.test.persons import create_group_type_mapping
from posthog.test.test_journeys import journeys_for
from products.access_control.backend.models.access_control import AccessControl
@@ -1575,6 +1582,245 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
[(ExperimentExposureHealthFindingCode.SRM, subcode)],
)
+ @parameterized.expand(
+ [
+ (
+ "reader_who_sees_the_findings",
+ True,
+ 'The mismatch comes from users whose first exposure was sent by the "posthog-node" library.',
+ {
+ "reason": "library",
+ "library_1": "web",
+ "library_1_exposed_control": 100,
+ "library_1_exposed_test": 100,
+ "library_2": "posthog-node",
+ "library_2_exposed_control": 5,
+ "library_2_exposed_test": 120,
+ "page_1": "/",
+ "page_1_exposed_control": 34,
+ "page_1_exposed_test": 34,
+ "page_2": "/pricing",
+ "page_2_exposed_control": 20,
+ "page_2_exposed_test": 20,
+ "page_3": "/docs",
+ "page_3_exposed_control": 15,
+ "page_3_exposed_test": 15,
+ "page_4": "/blog",
+ "page_4_exposed_control": 12,
+ "page_4_exposed_test": 12,
+ "page_5": "/about",
+ "page_5_exposed_control": 10,
+ "page_5_exposed_test": 10,
+ "page_6": BREAKDOWN_OTHER_STRING_LABEL,
+ "page_6_exposed_control": 9,
+ "page_6_exposed_test": 9,
+ },
+ ),
+ (
+ "reader_who_does_not_see_them",
+ False,
+ "This may indicate issues with randomization or data collection.",
+ {},
+ ),
+ ]
+ )
+ @time_machine.travel("2024-01-07T12:00:00Z", tick=False)
+ def test_srm_splits_the_exposures_by_the_library_and_page_of_the_first_exposure(
+ self, _name: str, findings_shown: bool, detail: str, evidence: dict[str, str | int]
+ ) -> None:
+ def exposure(variant: str, timestamp: str, library: str, path: str | None) -> dict[str, Any]:
+ properties = {"$feature_flag_response": variant, "$feature_flag": self.feature_flag.key, "$lib": library}
+ return {
+ "event": "$feature_flag_called",
+ "timestamp": timestamp,
+ "properties": {**properties, "$pathname": path} if path else properties,
+ }
+
+ # Seven pages, so that the two smallest share the "other" segment. Half of the users first exposed on the
+ # web are exposed by the server later, and stay web users.
+ pages = [
+ page
+ for page, users in (
+ ("/", 34),
+ ("/pricing", 20),
+ ("/docs", 15),
+ ("/blog", 12),
+ ("/about", 10),
+ ("/careers", 5),
+ ("/contact", 4),
+ )
+ for _ in range(users)
+ ]
+ web_first = {
+ f"web_{variant}_{i}": [
+ exposure(variant, "2024-01-02T10:00:00Z", "web", page),
+ *([exposure(variant, "2024-01-03T10:00:00Z", "posthog-node", None)] if i % 2 else []),
+ ]
+ for variant in ("control", "test")
+ for i, page in enumerate(pages)
+ }
+ server_only = {
+ f"server_{variant}_{i}": [exposure(variant, "2024-01-03T10:00:00Z", "posthog-node", None)]
+ for variant, count in (("control", 5), ("test", 120))
+ for i in range(count)
+ }
+ journeys_for({**web_first, **server_only}, self.team)
+ flush_persons_and_events()
+
+ with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=findings_shown):
+ response = ExperimentExposuresQueryRunner(
+ team=self.team, query=self._running_exposure_query(), user=self.user
+ ).calculate()
+
+ [finding] = response.health_findings or []
+ self.assertEqual((finding.code, finding.subcode), (ExperimentExposureHealthFindingCode.SRM, None))
+ self.assertIn(detail, finding.detail)
+ self.assertEqual(
+ {key: value for key, value in finding.evidence.items() if not key.endswith("p_value")}, evidence
+ )
+
+ @parameterized.expand(
+ [
+ ("persons_with_multiple_variants_excluded", False, "exclude", False),
+ ("persons_by_first_seen_variant", False, "first_seen", False),
+ ("persons_from_precomputed_exposures", False, "exclude", True),
+ ("groups", True, "exclude", False),
+ ]
+ )
+ @time_machine.travel("2024-01-07T12:00:00Z", tick=False)
+ def test_srm_library_segments_add_up_to_the_total_exposures(
+ self, _name: str, by_group: bool, multiple_variant_handling: str, use_precomputation: bool
+ ) -> None:
+ self._setup_precomputation_test(use_precomputation)
+ if by_group:
+ # HogQL blanks the group key of events from before the group type existed.
+ create_group_type_mapping(
+ team=self.team,
+ project_id=self.team.project_id,
+ group_type_index=0,
+ group_type="organization",
+ created_at=datetime(2024, 1, 1, tzinfo=UTC),
+ )
+ self.feature_flag.filters["aggregation_group_type_index"] = 0
+ self.feature_flag.save()
+ self.experiment.exposure_criteria = {"multiple_variant_handling": multiple_variant_handling}
+ self._save_experiment_with_precomputation(self.experiment, use_precomputation)
+ ff_property = f"$feature/{self.feature_flag.key}"
+ libraries = ["web", "posthog-node", "posthog-python", "posthog-ios", "posthog-android", "posthog-go", None]
+
+ def exposure(unit: int, variant: str, timestamp: str, library: str | None) -> dict[str, Any]:
+ return {
+ "event": "$feature_flag_called",
+ "timestamp": timestamp,
+ "properties": {
+ "$feature_flag_response": variant,
+ ff_property: variant,
+ "$feature_flag": self.feature_flag.key,
+ "$group_0": f"org:{unit}",
+ **({"$lib": library} if library else {}),
+ },
+ }
+
+ # Units from 100 see both variants. Every third unit is exposed again by another library, which must not
+ # move it out of the library of its first exposure.
+ journeys_for(
+ {
+ f"unit_{unit}": [
+ exposure(unit, "test" if 70 <= unit < 100 else "control", "2024-01-02T10:00:00Z", library),
+ *(
+ [exposure(unit, "test" if unit >= 70 else "control", "2024-01-03T10:00:00Z", "posthog-ruby")]
+ if unit >= 100 or unit % 3 == 0
+ else []
+ ),
+ ]
+ for unit in range(105)
+ for library in [libraries[unit % len(libraries)]]
+ },
+ self.team,
+ )
+ flush_persons_and_events()
+
+ with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=True):
+ response = ExperimentExposuresQueryRunner(
+ team=self.team, query=self._running_exposure_query(), user=self.user
+ ).calculate()
+
+ [finding] = [
+ finding
+ for finding in response.health_findings or []
+ if finding.code == ExperimentExposureHealthFindingCode.SRM
+ ]
+ segments: set[str] = set()
+ exposed_by_variant: dict[str, float] = {}
+ for key, value in finding.evidence.items():
+ if re.fullmatch(r"library_\d+", key) and isinstance(value, str):
+ segments.add(value)
+ elif (match := re.fullmatch(r"library_\d+_exposed_(.+)", key)) and isinstance(value, int | float):
+ exposed_by_variant[match.group(1)] = exposed_by_variant.get(match.group(1), 0) + value
+ self.assertEqual(
+ (exposed_by_variant, BREAKDOWN_OTHER_STRING_LABEL in segments), (response.total_exposures, True)
+ )
+
+ @parameterized.expand([("timeout", ClickHouseQueryTimeOut(), True), ("other_error", RuntimeError("boom"), False)])
+ @time_machine.travel("2024-01-07T12:00:00Z", tick=False)
+ def test_srm_keeps_the_finding_when_the_segment_query_fails(
+ self, _name: str, error: Exception, logged: bool
+ ) -> None:
+ ff_property = f"$feature/{self.feature_flag.key}"
+ journeys_for(
+ {
+ f"user_{variant}_{i}": [
+ {
+ "event": "$feature_flag_called",
+ "timestamp": "2024-01-02",
+ "properties": {
+ "$feature_flag_response": variant,
+ ff_property: variant,
+ "$feature_flag": self.feature_flag.key,
+ },
+ }
+ ]
+ for variant, count in (("control", 90), ("test", 10))
+ for i in range(count)
+ },
+ self.team,
+ )
+ flush_persons_and_events()
+
+ query_types: list[str] = []
+
+ def execute(**kwargs: Any) -> Any:
+ query_types.append(kwargs["query_type"])
+ if kwargs["query_type"] == "ExperimentExposureSegmentsQuery":
+ raise error
+ return execute_hogql_query(**kwargs)
+
+ runner_module = ExperimentExposuresQueryRunner.__module__
+ with (
+ patch(f"{runner_module}.execute_hogql_query", side_effect=execute),
+ patch(f"{runner_module}.shows_health_findings", return_value=True),
+ patch(f"{runner_module}.logger") as logger,
+ patch(f"{runner_module}.capture_exception") as capture_exception,
+ ):
+ response = ExperimentExposuresQueryRunner(
+ team=self.team, query=self._running_exposure_query(), user=self.user
+ ).calculate()
+
+ self.assertIn("ExperimentExposureSegmentsQuery", query_types)
+ self.assertEqual((logger.warning.called, capture_exception.called), (logged, not logged))
+ self.assertEqual(response.total_exposures, {"control": 90, "test": 10})
+ self.assertEqual(
+ [(finding.code, finding.detail, set(finding.evidence)) for finding in response.health_findings or []],
+ [
+ (
+ ExperimentExposureHealthFindingCode.SRM,
+ "The distribution of users across variants doesn't match your configured rollout percentages "
+ "(p < 0.001). This may indicate issues with randomization or data collection.",
+ {"p_value"},
+ )
+ ],
+ )
+
def test_cache_key_keeps_the_flag_history_from_a_reader_who_cannot_view_the_flag(self) -> None:
self.organization.available_product_features = [
{"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}
@@ -1640,6 +1886,54 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
self.assertEqual(response.total_exposures, {"control": 0, "test": 0})
self.assertEqual([(finding.code, finding.subcode) for finding in response.health_findings or []], expected)
+ @parameterized.expand(
+ [
+ ("test_accounts_exposed", 2, True, "No users exposed: the test account filter removes all of them", 2),
+ ("nobody_exposed", 0, True, "No users exposed, although the feature flag is called", 0),
+ (
+ "reader_who_does_not_see_the_findings",
+ 2,
+ False,
+ "No users exposed, although the feature flag is called",
+ None,
+ ),
+ ]
+ )
+ @time_machine.travel("2024-01-07T12:00:00Z", tick=False)
+ def test_zero_exposures_counts_the_users_that_the_test_account_filter_removes(
+ self, _name: str, test_accounts: int, findings_shown: bool, title: str, counted: int | None
+ ) -> None:
+ self.feature_flag.last_called_at = datetime(2024, 1, 2, tzinfo=UTC)
+ self.feature_flag.save()
+ self.team.test_account_filters = [
+ {"key": "email", "value": "@example.com", "operator": "not_icontains", "type": "person"}
+ ]
+ self.team.save()
+ self.experiment.exposure_criteria = {"filterTestAccounts": True}
+ for i in range(test_accounts):
+ _create_person(distinct_ids=[f"tester_{i}"], properties={"email": f"tester{i}@example.com"}, team=self.team)
+ _create_event(
+ distinct_id=f"tester_{i}",
+ event="$feature_flag_called",
+ timestamp="2024-01-02",
+ properties={"$feature_flag_response": "test", "$feature_flag": self.feature_flag.key},
+ team=self.team,
+ )
+ flush_persons_and_events()
+
+ with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=findings_shown):
+ response = ExperimentExposuresQueryRunner(
+ team=self.team, query=self._running_exposure_query(), user=self.user
+ ).calculate()
+
+ self.assertEqual(
+ [
+ (finding.subcode, finding.title, finding.evidence.get("exposed_without_test_account_filter"))
+ for finding in response.health_findings or []
+ ],
+ [("calls_not_counted", title, counted)],
+ )
+
@parameterized.expand([("direct", False), ("precomputed", True)])
@time_machine.travel("2024-01-07T12:00:00Z", tick=False)
def test_srm_returns_none_when_insufficient_samples(self, _name, use_precomputation):
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index 3d296663ea16..b407247ee906 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -43,6 +43,10 @@ export interface ExperimentHealthFinding {
variant?: string
/** Set for a server finding. The separate warnings have none. */
severity?: ExperimentHealthFindingApi['severity']
+ /**
+ * Why a check was not checked, or the cause the server's finding names in its `reason` evidence. Both are
+ * pinned server values, never customer text.
+ */
reason?: string
}
@@ -132,8 +136,8 @@ function healthFindingEventProperties(
}
}
-export function healthFindingReason(evidence: ExperimentHealthFindingApi['evidence']): string | undefined {
- return typeof evidence.reason === 'string' ? evidence.reason : undefined
+export function healthFindingReason(evidence: ExperimentHealthFindingApi['evidence'] | undefined): string | undefined {
+ return typeof evidence?.reason === 'string' ? evidence.reason : undefined
}
function trackedHealthFinding({ code, subcode, severity, evidence }: ExperimentHealthFindingApi): string {
@@ -246,8 +250,9 @@ export function captureExperimentHealthChecked(
* every load. `experiment results refresh completed` follows the exposure load on every tab.
* `has_srm`, `has_bias_risk` and a zero `exposures_total` are the page's own rules. The `_server` values are
* the server's checks on the same answer, null when the answer holds no server findings (a cached answer
- * from before the server ran them). `zero_exposures_cause` and `srm_cause` are the server's subcodes of those findings,
- * null without the finding or when it names no cause.
+ * from before the server ran them). `srm_subcode` and `zero_exposures_subcode` are the `subcode` of those findings, and
+ * `srm_reason` and `zero_exposures_reason` the `reason` in their evidence, each null without the finding or when it
+ * names none.
*/
export function exposureHealthEventProperties(
exposures: ExperimentExposureQueryResponse | null | undefined,
@@ -258,9 +263,11 @@ export function exposureHealthEventProperties(
has_srm: boolean | null
has_bias_risk: boolean | null
srm_server: boolean | null
- srm_cause: string | null
+ srm_subcode: string | null
+ srm_reason: string | null
zero_exposures_server: boolean | null
- zero_exposures_cause: string | null
+ zero_exposures_subcode: string | null
+ zero_exposures_reason: string | null
bias_risk_server: boolean | null
} {
if (!exposures) {
@@ -270,13 +277,17 @@ export function exposureHealthEventProperties(
has_srm: null,
has_bias_risk: null,
srm_server: null,
- srm_cause: null,
+ srm_subcode: null,
+ srm_reason: null,
zero_exposures_server: null,
- zero_exposures_cause: null,
+ zero_exposures_subcode: null,
+ zero_exposures_reason: null,
bias_risk_server: null,
}
}
const serverCodes = exposures.health_findings ? new Set(exposures.health_findings.map(({ code }) => code)) : null
+ const srmFinding = exposures.health_findings?.find(({ code }) => code === 'srm')
+ const zeroExposuresFinding = exposures.health_findings?.find(({ code }) => code === 'zero_exposures')
return {
exposures_total: getTotalExposures(exposures),
// With "first seen" handling the exposure query gives each person their first variant and
@@ -289,10 +300,14 @@ export function exposureHealthEventProperties(
has_bias_risk: exposures.bias_risk != null,
srm_server: serverCodes ? serverCodes.has('srm') : null,
// pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned subcodes.
- srm_cause: exposures.health_findings?.find(({ code }) => code === 'srm')?.subcode ?? null,
+ srm_subcode: srmFinding?.subcode ?? null,
+ // pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned reasons.
+ srm_reason: healthFindingReason(srmFinding?.evidence) ?? null,
zero_exposures_server: serverCodes ? serverCodes.has('zero_exposures') : null,
// pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned subcodes.
- zero_exposures_cause: exposures.health_findings?.find(({ code }) => code === 'zero_exposures')?.subcode ?? null,
+ zero_exposures_subcode: zeroExposuresFinding?.subcode ?? null,
+ // pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned reasons.
+ zero_exposures_reason: healthFindingReason(zeroExposuresFinding?.evidence) ?? null,
bias_risk_server: serverCodes ? serverCodes.has('bias_risk_multiple_excluded') : null,
}
}