:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): locate a mismatch by sdk version, cleaned page and screen (#125340)
Marcel Poelker19a94a53524a
diff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.pyindex 369a7e3e412d..f01b91f2289a 100644--- a/products/experiments/backend/health/checks/srm.py+++ b/products/experiments/backend/health/checks/srm.py@@ -1,5 +1,6 @@import unicodedatafrom collections.abc import Mapping+from datetime import datefrom enum import StrEnumfrom zoneinfo import ZoneInfo@@ -29,6 +30,8 @@ TOP_SEGMENT_VALUES = 5# everywhere. Two segments this large are needed before the finding calls a dimension uniform.UNIFORM_SEGMENT_MINIMUM_SIZE = 1000SEGMENT_LABEL_MAX_LENGTH = 60+# The detail names the located segments furthest from the split, up to this many.+NAMED_SEGMENTS = 2# pinned: the subcode is sent as `finding_variant` on the health finding events, so insights group on it.SPLIT_CHANGED = "split_changed"@@ -37,10 +40,20 @@ SPLIT_CHANGED = "split_changed"class SrmReason(StrEnum):# pinned: sent as `finding_reason` on the health finding events, so insights group on them.LIBRARY = ExposureSegmentDimension.LIBRARY.value+ LIBRARY_VERSION = ExposureSegmentDimension.LIBRARY_VERSION.valuePAGE = ExposureSegmentDimension.PAGE.value+ SCREEN = ExposureSegmentDimension.SCREEN.valueUNIFORM = "uniform"+_DIMENSION_PLURALS = {+ ExposureSegmentDimension.LIBRARY: "SDK libraries",+ ExposureSegmentDimension.LIBRARY_VERSION: "SDK versions",+ ExposureSegmentDimension.PAGE: "pages",+ ExposureSegmentDimension.SCREEN: "screens",+}++def sample_ratio_test(observed: Mapping[str, int], weights: Mapping[str, float]) -> SampleRatioMismatch | None:"""Chi-squared goodness-of-fit test of the observed counts against the weights of the variants in the split.None when there is too little data."""@@ -67,14 +80,31 @@ def reads_flag_history(ctx: HealthContext) -> bool:def reads_segments(ctx: HealthContext) -> bool:- """Whether `srm` reports a mismatch that no split change explains, which it then looks for in the segments of the- exposures. The exposure query loads the segments only then, because their query scans the exposure events."""+ """Whether `srm` reports a mismatch that a split change doesn't explain, which it then looks for in the segments+ of the exposures. After a split change, that is a mismatch among the users first exposed after the change, and+ the segments count only them (`segments_first_exposed_after`). The exposure query loads the segments only then,+ because their query scans the exposure events."""# With an activation, the segments would come from the activation event, not from the flag call that assigned# the variant.if ctx.exposure_events is not None and ctx.exposure_events.activation is not None:return Falseoutcome = srm(ctx)- return isinstance(outcome, ExperimentHealthFinding) and outcome.subcode != SPLIT_CHANGED+ if not isinstance(outcome, ExperimentHealthFinding):+ return False+ if outcome.subcode != SPLIT_CHANGED:+ return True+ retest_p_value = outcome.evidence.get("p_value_after_split_change")+ return isinstance(retest_p_value, float) and retest_p_value < SAMPLE_RATIO_MISMATCH_P_VALUE+++def segments_first_exposed_after(ctx: HealthContext) -> date | None:+ """The day after which the segments count the users, in the team's timezone. None for the whole run. After a split+ change, the users first exposed before it got their variants from another split, so they would fail the test+ against the current split in any segment they fill."""+ history = ctx.flag_history+ if not isinstance(history, FlagHistory) or _split_change_warning(ctx) is None:+ return None+ return _last_split_change_day(history)def srm(ctx: HealthContext) -> CheckOutcome:@@ -103,6 +133,15 @@ def srm(ctx: HealthContext) -> CheckOutcome:# retest that passes can't rule out a mismatch with another cause. The retest only adds evidence.unit = "group" if exposures.counts_groups else "user"retest, p_value_after_change = _retest_after_change(ctx, exposures, history, unit)+ evidence: dict[str, str | int | float | None] = {+ "p_value": p_value,+ "p_value_after_split_change": p_value_after_change,+ }+ cause: tuple[str, ...] = ()+ if p_value_after_change is not None and p_value_after_change < SAMPLE_RATIO_MISMATCH_P_VALUE:+ segment_cause = _segment_cause(exposures, unit)+ cause = segment_cause.sentences+ evidence.update(segment_cause.evidence)return ExperimentHealthFinding(code=ExperimentHealthFindingCode.SRM,subcode=SPLIT_CHANGED,@@ -115,10 +154,11 @@ def srm(ctx: HealthContext) -> CheckOutcome:f"The variant split changed after launch while {unit}s were exposed, and some of them got their ""variants from another split, which can cause this.",*retest,+ *cause,f'Follow the steps in "{split_change.title}" to fix the results.',]),- evidence={"p_value": p_value, "p_value_after_split_change": p_value_after_change},+ evidence=evidence,actions=(),diagnostic_ref="A2",)@@ -139,7 +179,7 @@ def _retest_after_change(f"The exposure criteria count {unit}s at an activation, so the {unit}s exposed after the change can't ""be tested alone."], None- changed_on = history.counted(SplitChange)[-1].changed_at.astimezone(ZoneInfo(history.team_timezone)).date()+ changed_on = _last_split_change_day(history)after_change = sample_ratio_test({variant: sum(count for day, count in daily.items() if day > changed_on)@@ -160,6 +200,10 @@ def _retest_after_change(], after_change.p_value+def _last_split_change_day(history: FlagHistory) -> date:+ return history.counted(SplitChange)[-1].changed_at.astimezone(ZoneInfo(history.team_timezone)).date()++def _split_change_warning(ctx: HealthContext) -> ExperimentHealthFinding | None:try:outcome = split_changed_mid_run(ctx)@@ -174,12 +218,7 @@ def _split_change_warning(ctx: HealthContext) -> ExperimentHealthFinding | None:def _mismatch(exposures: ExposureTotals, p_value: float) -> ExperimentHealthFinding:unit = "group" if exposures.counts_groups else "user"- tests = _segment_tests(exposures)- verdicts = [_verdict(tests, dimension, exposures.sample_ratio_expected) for dimension in ExposureSegmentDimension]- cause, reason = _cause(verdicts, unit, exposures.sample_ratio_expected)- evidence: dict[str, str | int | float | None] = {"p_value": p_value, **_segment_evidence(tests, verdicts)}- if reason is not None:- evidence["reason"] = reason.value+ cause = _segment_cause(exposures, unit)return ExperimentHealthFinding(code=ExperimentHealthFindingCode.SRM,subcode=None,@@ -189,10 +228,10 @@ def _mismatch(exposures: ExposureTotals, p_value: float) -> ExperimentHealthFind[f"The distribution of {unit}s across variants doesn't match your configured rollout percentages "f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}).",- *cause,+ *(cause.sentences or ("This may indicate issues with randomization or data collection.",)),]),- evidence=evidence,+ evidence={"p_value": p_value, **cause.evidence},actions=(),diagnostic_ref="A2",)@@ -211,6 +250,29 @@ class _Verdict:homogeneity_p_value: float | Nonepowered: boollocated: ExposureSegment | None+ distance: float+++@frozen+class _SegmentCause:+ """What the segments say about the mismatch: the sentences that follow it, and the reason and the segments as+ evidence. Without a verdict the sentences are empty."""++ sentences: tuple[str, ...]+ evidence: dict[str, str | int | float | None]+++def _segment_cause(exposures: ExposureTotals, unit: str) -> _SegmentCause:+ tests = _segment_tests(exposures)+ verdicts = [_verdict(tests, dimension, exposures.sample_ratio_expected) for dimension in ExposureSegmentDimension]+ sentences, reason = _cause(verdicts, unit)+ evidence = _segment_evidence(tests, verdicts)+ located = _located(verdicts)+ if located:+ evidence["located_dimensions"] = ",".join(segment.dimension.value for segment in located)+ if reason is not None:+ evidence["reason"] = reason.value+ return _SegmentCause(sentences=tuple(sentences), evidence=evidence)def _segment_tests(exposures: ExposureTotals) -> list[_SegmentTest]:@@ -231,18 +293,26 @@ def _verdict(tests: list[_SegmentTest], dimension: ExposureSegmentDimension, weivariant for variant in weights if any(test.segment.exposed_by_variant.get(variant, 0) for test in compared)]if len(compared) < 2 or len(variants) < 2:- return _Verdict(dimension=dimension, homogeneity_p_value=None, powered=False, located=None)+ return _Verdict(dimension=dimension, homogeneity_p_value=None, powered=False, located=None, distance=0.0)table = [[test.segment.exposed_by_variant.get(variant, 0) for variant in variants] for test in compared]homogeneity_p_value = float(chi2_contingency(table).pvalue)powered = sum(1 for row in table if sum(row) >= UNIFORM_SEGMENT_MINIMUM_SIZE) >= 2- if homogeneity_p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:- return _Verdict(dimension=dimension, homogeneity_p_value=homogeneity_p_value, powered=powered, located=None)- failing = [- test.segment for test in compared if test.p_value is not None and test.p_value < SAMPLE_RATIO_MISMATCH_P_VALUE- ]- # The detail says the mismatch comes from the segment, which reads as if the other segments match the split.- located = failing[0] if len(failing) == 1 else None- return _Verdict(dimension=dimension, homogeneity_p_value=homogeneity_p_value, powered=powered, located=located)+ located: ExposureSegment | None = None+ if homogeneity_p_value < SAMPLE_RATIO_MISMATCH_P_VALUE:+ failing = [+ test.segment+ for test in compared+ if test.p_value is not None and test.p_value < SAMPLE_RATIO_MISMATCH_P_VALUE+ ]+ # The detail says the mismatch comes from the segment, which reads as if the other segments match the split.+ located = failing[0] if len(failing) == 1 else None+ return _Verdict(+ dimension=dimension,+ homogeneity_p_value=homogeneity_p_value,+ powered=powered,+ located=located,+ distance=_distance_from_split(located, weights) if located is not None else 0.0,+ )def _distance_from_split(segment: ExposureSegment, weights: Mapping[str, float]) -> float:@@ -258,19 +328,15 @@ def _distance_from_split(segment: ExposureSegment, weights: Mapping[str, float]))-def _cause(verdicts: list[_Verdict], unit: str, weights: Mapping[str, float]) -> tuple[list[str], SrmReason | None]:- """The sentences that follow the mismatch, and the finding's reason: the dimension of the segment furthest from- the split."""- located = sorted(- (verdict.located for verdict in verdicts if verdict.located is not None),- key=lambda segment: -_distance_from_split(segment, weights),- )- if located:- places = ", and from ".join(f"{unit}s {_segment_place(segment)}" for segment in located)+def _cause(verdicts: list[_Verdict], unit: str) -> tuple[list[str], SrmReason | None]:+ """The sentences that follow the mismatch, and the finding's reason. Empty without a verdict."""+ named = _without_libraries_of_located_versions(_located(verdicts))[:NAMED_SEGMENTS]+ if named:+ places = ", and from ".join(f"{unit}s {_segment_place(segment)}" for segment in named)return [f"The mismatch comes from {places}.",f"Check that the code there sends the exposure for every {unit} in every variant.",- ], SrmReason(located[0].dimension.value)+ ], SrmReason(named[0].dimension.value)compared = {verdict.dimension: verdict.homogeneity_p_valuefor verdict in verdicts@@ -278,26 +344,67 @@ def _cause(verdicts: list[_Verdict], unit: str, weights: Mapping[str, float]) ->}powered = [verdict.dimension for verdict in verdicts if verdict.powered]if powered and all(p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE for p_value in compared.values()):- across = " and ".join(- "SDK libraries" if dimension == ExposureSegmentDimension.LIBRARY else "pages" for dimension in powered- )+ across = _joined([_DIMENSION_PLURALS[dimension] for dimension in powered])return [f"The mismatch looks the same across {across}, which points at how {unit}s get their variants rather ""than at one integration."], SrmReason.UNIFORM- return ["This may indicate issues with randomization or data collection."], None+ return [], None+++def _located(verdicts: list[_Verdict]) -> list[ExposureSegment]:+ """The located segments, the one furthest from the split first."""+ ranked = sorted((verdict for verdict in verdicts if verdict.located is not None), key=lambda v: -v.distance)+ return [verdict.located for verdict in ranked if verdict.located is not None]+++def _without_libraries_of_located_versions(located: list[ExposureSegment]) -> list[ExposureSegment]:+ """A located version of a located library says the same more precisely, so the library isn't named too."""+ versioned = {+ segment.library for segment in located if segment.dimension == ExposureSegmentDimension.LIBRARY_VERSION+ }+ return [+ segment+ for segment in located+ if not (segment.dimension == ExposureSegmentDimension.LIBRARY and segment.value in versioned)+ ]+++def _joined(words: list[str]) -> str:+ return " and ".join(filter(None, [", ".join(words[:-1]), words[-1]]))def _segment_place(segment: ExposureSegment) -> str:- if segment.dimension == ExposureSegmentDimension.LIBRARY:- if segment.value == BREAKDOWN_OTHER_STRING_LABEL:- return f"whose first exposure was sent by a library other than the {TOP_SEGMENT_VALUES} most common"- if segment.value == BREAKDOWN_NULL_STRING_LABEL:- return "whose first exposure was sent by an unknown library"- return f"whose first exposure was sent by the {_segment_label(segment.value)} library"- if segment.value == BREAKDOWN_OTHER_STRING_LABEL:- return f"first exposed on a page other than the {TOP_SEGMENT_VALUES} most common"- return f"first exposed on the page {_segment_label(segment.value)}"+ match segment.dimension:+ case ExposureSegmentDimension.LIBRARY:+ if segment.value == BREAKDOWN_OTHER_STRING_LABEL:+ return f"whose first exposure was sent by a library other than the {TOP_SEGMENT_VALUES} most common"+ if segment.value == BREAKDOWN_NULL_STRING_LABEL:+ return "whose first exposure was sent by an unknown library"+ return f"whose first exposure was sent by the {_segment_label(segment.value)} library"+ case ExposureSegmentDimension.LIBRARY_VERSION:+ if segment.value == BREAKDOWN_OTHER_STRING_LABEL:+ return (+ "whose first exposure was sent by a library version other than the "+ f"{TOP_SEGMENT_VALUES} most common"+ )+ library = segment.library+ by_library = (+ "an unknown library"+ if library is None or library == BREAKDOWN_NULL_STRING_LABEL+ else f"the {_segment_label(library)} library"+ )+ if segment.value == BREAKDOWN_NULL_STRING_LABEL:+ return f"whose first exposure was sent by an unknown version of {by_library}"+ return f"whose first exposure was sent by version {_segment_label(segment.value)} of {by_library}"+ case ExposureSegmentDimension.PAGE:+ if segment.value == BREAKDOWN_OTHER_STRING_LABEL:+ return f"first exposed on a page other than the {TOP_SEGMENT_VALUES} most common"+ return f"first exposed on the page {_segment_label(segment.value)}"+ case ExposureSegmentDimension.SCREEN:+ if segment.value == BREAKDOWN_OTHER_STRING_LABEL:+ return f"first exposed on a screen other than the {TOP_SEGMENT_VALUES} most common"+ return f"first exposed on the screen {_segment_label(segment.value)}"def _segment_label(value: str) -> str:@@ -311,8 +418,9 @@ def _segment_label(value: str) -> str:def _segment_evidence(tests: list[_SegmentTest], verdicts: list[_Verdict]) -> dict[str, str | int | float | None]:- """Each segment in the keys `<dimension>_N` (the value), `<dimension>_N_p_value` and- `<dimension>_N_exposed_<variant>`, and each compared dimension in `<dimension>_homogeneity_p_value`."""+ """Each segment in the keys `<dimension>_N` (the value), `<dimension>_N_library` (the library of a version),+ `<dimension>_N_p_value` and `<dimension>_N_exposed_<variant>`, and each compared dimension in+ `<dimension>_homogeneity_p_value`."""evidence: dict[str, str | int | float | None] = {}for verdict in verdicts:dimension = verdict.dimension@@ -322,6 +430,8 @@ def _segment_evidence(tests: list[_SegmentTest], verdicts: list[_Verdict]) -> difor index, test in enumerate(in_dimension, start=1):key = f"{dimension.value}_{index}"evidence[key] = test.segment.value+ if test.segment.library is not None:+ evidence[f"{key}_library"] = test.segment.libraryevidence[f"{key}_p_value"] = test.p_valuefor variant, exposed in test.segment.exposed_by_variant.items():evidence[f"{key}_exposed_{variant}"] = exposeddiff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex f1f73f5085a8..5d6ab4563158 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -237,9 +237,13 @@ class ExposureSegment:dimension: ExposureSegmentDimension# The property's value on the first exposure. BREAKDOWN_NULL_STRING_LABEL when the event has none, and- # BREAKDOWN_OTHER_STRING_LABEL for the values outside the largest segments of the dimension.+ # BREAKDOWN_OTHER_STRING_LABEL for the values outside the largest segments of the dimension. The version in the+ # library version dimension.value: strexposed_by_variant: Mapping[str, int]+ # The library of a library version segment, also BREAKDOWN_NULL_STRING_LABEL when the event has none. None in the+ # other dimensions and in the BREAKDOWN_OTHER_STRING_LABEL segment.+ library: str | None = None@frozendiff --git a/products/experiments/backend/health/test/test_split_changed.py b/products/experiments/backend/health/test/test_split_changed.pyindex 37d590a6538a..6b742c11fc53 100644--- a/products/experiments/backend/health/test/test_split_changed.py+++ b/products/experiments/backend/health/test/test_split_changed.py@@ -18,9 +18,15 @@ from products.experiments.backend.facade.contracts import (from products.experiments.backend.health.checks.flag_change_findings import flag_history_not_readablefrom products.experiments.backend.health.checks.flag_state import flag_statefrom products.experiments.backend.health.checks.split_changed import split_changed_mid_run-from products.experiments.backend.health.checks.srm import reads_flag_history, reads_segments, srm+from products.experiments.backend.health.checks.srm import (+ reads_flag_history,+ reads_segments,+ segments_first_exposed_after,+ srm,+)from products.experiments.backend.health.context import (ExposureEvents,+ ExposureSegment,ExposureTotals,FlagHistoryNotCheckedReason,HealthContext,@@ -35,6 +41,7 @@ from products.experiments.backend.health.skips import (not_applicable,)from products.experiments.backend.health.test.factories import ENDED, RUNNING, RUNNING_CONTEXT, Status, flag_history+from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimensionfrom products.feature_flags.backend.facade.edits import SplitChange, parse_variantsLAUNCHED_AT = datetime(2026, 9, 12, 10, 0, tzinfo=UTC)@@ -534,6 +541,31 @@ class TestSrmAfterSplitChange(TestCase):_totals({"control": 250, "test": 250}),("split_changed", "don't match the current split either (p < 0.001), so look for another cause too."),),+ (+ "retest_fails_in_one_library",+ _context(EVEN, UNEVEN, exposed_units=5),+ replace(+ _totals({"control": 250, "test": 250}),+ segments=(+ ExposureSegment(+ dimension=ExposureSegmentDimension.LIBRARY,+ value="web",+ exposed_by_variant={"control": 800, "test": 200},+ ),+ ExposureSegment(+ dimension=ExposureSegmentDimension.LIBRARY,+ value="posthog-node",+ exposed_by_variant={"control": 50, "test": 450},+ ),+ ),+ ),+ (+ "split_changed",+ "so look for another cause too. The mismatch comes from users whose first exposure was sent by the "+ '"posthog-node" library. Check that the code there sends the exposure for every user in every '+ "variant. Follow the steps in",+ ),+ ),("too_few_for_a_retest",_context(EVEN, UNEVEN, exposed_units=5),@@ -590,18 +622,42 @@ class TestSrmAfterSplitChange(TestCase):@parameterized.expand([- ("failing_test", RUNNING_CONTEXT, 0.00001, (True, True)),- ("passing_test", RUNNING_CONTEXT, 0.01, (False, False)),- ("split_change_explains_it", _context(EVEN, UNEVEN, exposed_units=5), 0.00001, (True, False)),- ("activation", replace(RUNNING_CONTEXT, exposure_events=ACTIVATION_EVENTS), 0.00001, (True, False)),+ ("failing_test", RUNNING_CONTEXT, MATCHES_THE_NEW_SPLIT, 0.00001, (True, True, None)),+ ("passing_test", RUNNING_CONTEXT, MATCHES_THE_NEW_SPLIT, 0.01, (False, False, None)),+ (+ "split_change_explains_it",+ _context(EVEN, UNEVEN, exposed_units=5),+ MATCHES_THE_NEW_SPLIT,+ 0.00001,+ (True, False, FIRST_CHANGE_AT.date()),+ ),+ (+ "split_change_with_a_failed_retest",+ _context(EVEN, UNEVEN, exposed_units=5),+ {"control": 250, "test": 250},+ 0.00001,+ (True, True, FIRST_CHANGE_AT.date()),+ ),+ (+ "activation",+ replace(RUNNING_CONTEXT, exposure_events=ACTIVATION_EVENTS),+ MATCHES_THE_NEW_SPLIT,+ 0.00001,+ (True, False, None),+ ),])def test_reads_the_flag_history_and_the_segments_only_for_a_mismatch(- self, _name: str, ctx: HealthContext, p_value: float, expected: tuple[bool, bool]+ self,+ _name: str,+ ctx: HealthContext,+ after_change: dict[str, int],+ p_value: float,+ expected: tuple[bool, bool, date | None],) -> None:- ctx = replace(ctx, exposures=replace(_totals(MATCHES_THE_NEW_SPLIT), sample_ratio_mismatch_p_value=p_value))+ ctx = replace(ctx, exposures=replace(_totals(after_change), sample_ratio_mismatch_p_value=p_value))- self.assertEqual((reads_flag_history(ctx), reads_segments(ctx)), expected)+ self.assertEqual((reads_flag_history(ctx), reads_segments(ctx), segments_first_exposed_after(ctx)), expected)def test_names_the_cause_the_evidence_and_the_next_step(self) -> None:outcome = srm(replace(_context(EVEN, UNEVEN, exposed_units=5), exposures=_totals(MATCHES_THE_NEW_SPLIT)))diff --git a/products/experiments/backend/health/test/test_srm.py b/products/experiments/backend/health/test/test_srm.pyindex c7154f7a4343..a715bd43b96e 100644--- a/products/experiments/backend/health/test/test_srm.py+++ b/products/experiments/backend/health/test/test_srm.py@@ -6,7 +6,7 @@ from parameterized import parameterizedfrom posthog.schema import MultipleVariantHandling-from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_OTHER_STRING_LABEL+from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_NULL_STRING_LABEL, BREAKDOWN_OTHER_STRING_LABELfrom products.experiments.backend.facade.contracts import ExperimentHealthFindingfrom products.experiments.backend.health.checks.srm import srm@@ -15,7 +15,9 @@ from products.experiments.backend.health.test.factories import RUNNING_CONTEXTfrom products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimensionLIBRARY = ExposureSegmentDimension.LIBRARY+LIBRARY_VERSION = ExposureSegmentDimension.LIBRARY_VERSIONPAGE = ExposureSegmentDimension.PAGE+SCREEN = ExposureSegmentDimension.SCREENMISMATCH = "The distribution of users across variants doesn't match your configured rollout percentages (p < 0.001)."NO_LOCATION = f"{MISMATCH} This may indicate issues with randomization or data collection."NEXT_STEP = "Check that the code there sends the exposure for every user in every variant."@@ -25,6 +27,10 @@ def _segment(dimension: ExposureSegmentDimension, value: str, control: int, testreturn ExposureSegment(dimension=dimension, value=value, exposed_by_variant={"control": control, "test": test})+def _version(library: str, version: str, control: int, test: int) -> ExposureSegment:+ return replace(_segment(LIBRARY_VERSION, version, control, test), library=library)++def _exposures(segments: tuple[ExposureSegment, ...] | None) -> ExposureTotals:return ExposureTotals(total_exposures={"control": 600, "test": 900},@@ -68,7 +74,7 @@ class TestSrmSegments(TestCase):f'{MISMATCH} The mismatch comes from users first exposed on the page "/checkout". {NEXT_STEP}',),(- "the_largest_segment_with_the_skew",+ "the_segment_furthest_from_the_split_first",(_segment(LIBRARY, "web", 800, 1200),_segment(LIBRARY, "posthog-node", 400, 400),@@ -76,8 +82,8 @@ class TestSrmSegments(TestCase):_segment(PAGE, "/checkout", 200, 600),),"page",- f'{MISMATCH} The mismatch comes from users first exposed on the page "/checkout", and from users whose '- f'first exposure was sent by the "web" library. {NEXT_STEP}',+ f'{MISMATCH} The mismatch comes from users first exposed on the page "/checkout", and from users '+ f'whose first exposure was sent by the "web" library. {NEXT_STEP}',),("several_segments_skewed",@@ -85,6 +91,53 @@ class TestSrmSegments(TestCase):None,NO_LOCATION,),+ (+ "at_most_two_segments_named",+ (+ _segment(LIBRARY, "web", 800, 1200),+ _segment(LIBRARY, "posthog-node", 400, 400),+ _version("web", "1.200.0", 400, 400),+ _version("posthog-node", "4.2.1", 100, 400),+ _segment(PAGE, "/pricing", 600, 600),+ _segment(PAGE, "/checkout", 200, 600),+ ),+ "library_version",+ f'{MISMATCH} The mismatch comes from users whose first exposure was sent by version "4.2.1" of the '+ f'"posthog-node" library, and from users first exposed on the page "/checkout". {NEXT_STEP}',+ ),+ (+ "the_version_of_the_located_library",+ (+ *SERVER_SKEWED,+ _version("web", "1.200.0", 500, 500),+ _version("posthog-node", "4.2.1", 100, 400),+ ),+ "library_version",+ f'{MISMATCH} The mismatch comes from users whose first exposure was sent by version "4.2.1" of the '+ f'"posthog-node" library. {NEXT_STEP}',+ ),+ (+ "an_unknown_version_of_the_located_library",+ (+ *SERVER_SKEWED,+ _version("web", "1.200.0", 500, 500),+ _version("posthog-node", BREAKDOWN_NULL_STRING_LABEL, 100, 400),+ ),+ "library_version",+ f"{MISMATCH} The mismatch comes from users whose first exposure was sent by an unknown version of the "+ f'"posthog-node" library. {NEXT_STEP}',+ ),+ (+ "one_screen",+ (+ _segment(LIBRARY, "posthog-ios", 300, 600),+ _segment(LIBRARY, "posthog-android", 300, 600),+ _segment(SCREEN, "Home", 300, 300),+ _segment(SCREEN, "Checkout", 100, 400),+ ),+ "screen",+ f'{MISMATCH} The mismatch comes from users first exposed on the screen "Checkout". {NEXT_STEP}',+ ),("the_same_skew_everywhere",(_segment(LIBRARY, "web", 96900, 93100), _segment(LIBRARY, "posthog-node", 5100, 4900)),@@ -157,6 +210,7 @@ class TestSrmSegments(TestCase):{"p_value": 0.00001,"reason": "library",+ "located_dimensions": "library","library_1": "web","library_1_p_value": 1.0,"library_1_exposed_control": 500,diff --git a/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py b/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.pyindex e891810dfb4f..932f45b3a813 100644--- a/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py+++ b/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py@@ -1,11 +1,12 @@from dataclasses import replace+from datetime import datefrom enum import StrEnumfrom posthog.schema import Breakdown, ExperimentEventExposureConfig, MultipleBreakdownType, MultipleVariantHandlingfrom posthog.hogql import astfrom posthog.hogql.parser import parse_expr, parse_select-from posthog.hogql.property import property_to_expr+from posthog.hogql.property import apply_path_cleaning, property_to_exprfrom posthog.hogql_queries.utils.breakdowns import BREAKDOWN_NULL_STRING_LABEL, BREAKDOWN_OTHER_STRING_LABEL@@ -23,13 +24,17 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (class ExposureSegmentDimension(StrEnum):# pinned: the `srm` finding names its evidence keys and its reasons after these values.LIBRARY = "library"+ LIBRARY_VERSION = "library_version"PAGE = "page"+ SCREEN = "screen"-# `segments_query` reads them as the columns breakdown_value_1 and breakdown_value_2, in this order.+# `segments_query` reads them as the columns breakdown_value_1 to breakdown_value_4, in this order._SEGMENT_BREAKDOWNS = (Breakdown(property="$lib", type=MultipleBreakdownType.EVENT),+ Breakdown(property="$lib_version", type=MultipleBreakdownType.EVENT),Breakdown(property="$pathname", type=MultipleBreakdownType.EVENT),+ Breakdown(property="$screen_name", type=MultipleBreakdownType.EVENT),)@@ -119,12 +124,15 @@ class ExposureQueryBuilder:assert isinstance(query, ast.SelectQuery)return query- def segments_query(self, top_values: int) -> ast.SelectQuery:+ def segments_query(self, top_values: int, first_exposed_after: date | None = None) -> ast.SelectQuery:"""- Persons or groups by variant and by the SDK library of their first exposure, and those whose first exposure- has a path also by that path. Columns: dimension, segment, variant, exposed_count. In each- dimension, the `top_values` segments with the most persons or groups keep their own rows, and the others share- the BREAKDOWN_OTHER_STRING_LABEL segment, so that one path per URL can't grow the result.+ Persons or groups by variant and by the SDK library of their first exposure, by the version of that library,+ and by the page of the exposure, or by its screen when it has no path. The team's path cleaning rules group+ the pages. Columns: dimension, library, segment, variant, exposed_count. `library` holds the library of a+ version segment, and is empty in the other dimensions. In each dimension, the `top_values` segments with the+ most persons or groups keep their own rows, and the others share the BREAKDOWN_OTHER_STRING_LABEL segment, so+ that one path per URL can't grow the result. With `first_exposed_after`, only the persons or groups first+ exposed after that day in the team's timezone count, as in the daily counts of timeseries_query."""entities = ExposureQueryBuilder(context=replace(self.context, breakdowns=_SEGMENT_BREAKDOWNS),@@ -133,56 +141,81 @@ class ExposureQueryBuilder:# The window functions rank the segments on the grouped rows, so the events are scanned once.query = parse_select("""- SELECT dimension, segment, variant, sum(exposed_count) AS exposed_count+ SELECT dimension, library, segment, variant, sum(exposed_count) AS exposed_countFROM (SELECTdimension,+ if(segment_rank <= {top_values}, segment_library, '') AS library,if(segment_rank <= {top_values}, segment_value, {other_segments}) AS segment,variant,exposed_countFROM (SELECTdimension,+ segment_library,segment_value,variant,exposed_count,dense_rank() OVER (- PARTITION BY dimension ORDER BY segment_total DESC, segment_value ASC+ PARTITION BY dimension ORDER BY segment_total DESC, segment_library ASC, segment_value ASC) AS segment_rankFROM (SELECTdimension,+ segment_library,segment_value,variant,exposed_count,- sum(exposed_count) OVER (PARTITION BY dimension, segment_value) AS segment_total+ sum(exposed_count) OVER (+ PARTITION BY dimension, segment_library, segment_value+ ) AS segment_totalFROM (SELECT- segment_pair.1 AS dimension,- segment_pair.2 AS segment_value,+ segment_tuple.1 AS dimension,+ segment_tuple.2 AS segment_library,+ segment_tuple.3 AS segment_value,variant,count() AS exposed_countFROM ({entities}) AS entities- ARRAY JOIN if(- notEmpty(breakdown_value_2) AND breakdown_value_2 != {no_path},- [({library}, breakdown_value_1), ({page}, breakdown_value_2)],- [({library}, breakdown_value_1)]- ) AS segment_pair- WHERE notEmpty(variant)- GROUP BY dimension, segment_value, variant+ ARRAY JOIN arrayConcat(+ [+ ({library}, '', breakdown_value_1),+ ({library_version}, breakdown_value_1, breakdown_value_2)+ ],+ multiIf(+ notEmpty(breakdown_value_3) AND breakdown_value_3 != {no_value},+ [({page}, '', {page_value})],+ notEmpty(breakdown_value_4) AND breakdown_value_4 != {no_value},+ [({screen}, '', breakdown_value_4)],+ []+ )+ ) AS segment_tuple+ WHERE notEmpty(variant) AND {first_exposed}+ GROUP BY dimension, segment_library, segment_value, variant))))- GROUP BY dimension, segment, variant+ GROUP BY dimension, library, segment, variant""",placeholders={"entities": entities,"top_values": ast.Constant(value=top_values),"other_segments": ast.Constant(value=BREAKDOWN_OTHER_STRING_LABEL),- "no_path": ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL),+ "no_value": ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL),"library": ast.Constant(value=ExposureSegmentDimension.LIBRARY.value),+ "library_version": ast.Constant(value=ExposureSegmentDimension.LIBRARY_VERSION.value),"page": ast.Constant(value=ExposureSegmentDimension.PAGE.value),+ "page_value": apply_path_cleaning(ast.Field(chain=["breakdown_value_3"]), self.context.team),+ "screen": ast.Constant(value=ExposureSegmentDimension.SCREEN.value),+ "first_exposed": (+ parse_expr(+ "toDate(toString(first_exposure_time)) > {day}",+ placeholders={"day": ast.Constant(value=first_exposed_after)},+ )+ if first_exposed_after is not None+ else ast.Constant(value=True)+ ),},)assert isinstance(query, ast.SelectQuery)diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.pyindex 5ea15b5c8e64..e71ad1b3d6db 100644--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py@@ -50,6 +50,7 @@ from products.experiments.backend.health.checks.srm import (reads_flag_history,reads_segments,sample_ratio_test,+ segments_first_exposed_after,)from products.experiments.backend.health.checks.zero_exposures import reads_exposed_without_test_account_filterfrom products.experiments.backend.health.context import (@@ -356,7 +357,8 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):if not self._runs_health_queries():return evaluate_health(context, EXPOSURE_HEALTH_CHECKS)if reads_segments(context):- exposures = replace(exposures, segments=self._segments(exposure_params))+ segments = self._segments(exposure_params, segments_first_exposed_after(context))+ exposures = replace(exposures, segments=segments)context = replace(context, exposures=exposures)# Without test account filters on the project, the count without the filter is the same zero.if (@@ -368,17 +370,26 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):context = replace(context, exposures=replace(exposures, exposed_without_test_account_filter=exposed))return evaluate_health(context, EXPOSURE_HEALTH_CHECKS)- def _segments(self, exposure_params: ExposureQueryParams) -> tuple[ExposureSegment, ...] | None:- query = self._query_builder(exposure_params).get_exposure_segments_query(TOP_SEGMENT_VALUES)+ def _segments(+ self, exposure_params: ExposureQueryParams, first_exposed_after: date | None+ ) -> tuple[ExposureSegment, ...] | None:+ query = self._query_builder(exposure_params).get_exposure_segments_query(+ TOP_SEGMENT_VALUES, first_exposed_after+ )rows = self._run_health_query("ExperimentExposureSegmentsQuery", query)if rows is None:return None- exposed: dict[tuple[str, str], dict[str, int]] = {}- for dimension, segment, variant, count in rows:- exposed.setdefault((dimension, segment), {})[variant] = int(count)+ exposed: dict[tuple[str, str, str], dict[str, int]] = {}+ for dimension, library, segment, variant, count in rows:+ exposed.setdefault((dimension, library, segment), {})[variant] = int(count)return tuple(- ExposureSegment(dimension=ExposureSegmentDimension(dimension), value=segment, exposed_by_variant=counts)- for (dimension, segment), counts in exposed.items()+ ExposureSegment(+ dimension=ExposureSegmentDimension(dimension),+ value=segment,+ exposed_by_variant=counts,+ library=library or None,+ )+ for (dimension, library, segment), counts in exposed.items())def _exposed_without_test_account_filter(self, exposure_params: ExposureQueryParams) -> int | None:diff --git a/products/experiments/backend/hogql_queries/experiment_query_builder.py b/products/experiments/backend/hogql_queries/experiment_query_builder.pyindex ac65b6c19912..169e30156840 100644--- a/products/experiments/backend/hogql_queries/experiment_query_builder.py+++ b/products/experiments/backend/hogql_queries/experiment_query_builder.py@@ -1,6 +1,6 @@from collections.abc import Sequencefrom dataclasses import dataclass-from datetime import datetime+from datetime import date, datetimefrom typing import Optional, Unionfrom django.utils import timezone@@ -298,12 +298,13 @@ class ExperimentQueryBuilder:"""return self._exposure_query_builder().timeseries_query()- def get_exposure_segments_query(self, top_values: int) -> ast.SelectQuery:+ def get_exposure_segments_query(self, top_values: int, first_exposed_after: date | None = None) -> ast.SelectQuery:"""- Persons or groups by variant and by the SDK library and the page of their first exposure. Columns:- dimension, segment, variant, exposed_count. It scans the events, also when precomputed exposures exist.+ Persons or groups by variant and by the SDK library, the library version, and the page or screen of their+ first exposure. Columns: dimension, library, segment, variant, exposed_count. It scans the events, also when+ precomputed exposures exist."""- return self._exposure_query_builder().segments_query(top_values)+ return self._exposure_query_builder().segments_query(top_values, first_exposed_after)def get_exposed_count_query(self) -> ast.SelectQuery:"""The persons or groups exposed, in one row. Column: exposed_count."""diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.pyindex 721821d20493..a9acde80de81 100644--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py@@ -1548,6 +1548,67 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):self.team,)flush_persons_and_events()+ self._log_split_change_from_90_10_on_2024_01_03()++ response = ExperimentExposuresQueryRunner(+ team=self.team, query=self._running_exposure_query(), user=self.user if as_user else None+ ).calculate()++ self.assertEqual(+ [(finding.code, finding.subcode) for finding in response.health_findings or []],+ [(ExperimentExposureHealthFindingCode.SRM, subcode)],+ )++ @time_machine.travel("2024-01-07T12:00:00Z", tick=False)+ def test_srm_segments_after_a_split_change_count_only_the_users_first_exposed_after_it(self) -> None:+ ff_property = f"$feature/{self.feature_flag.key}"+ # Web 1.0.0 got the old 90/10 split before the change. After it, web 2.0.0 matches the current 50/50 split+ # and posthog-node 4.2.1 doesn't.+ journeys_for(+ {+ f"user_{library}_{version}_{variant}_{i}": [+ {+ "event": "$feature_flag_called",+ "timestamp": day,+ "properties": {+ "$feature_flag_response": variant,+ ff_property: variant,+ "$feature_flag": self.feature_flag.key,+ "$lib": library,+ "$lib_version": version,+ },+ }+ ]+ for day, library, version, counts in (+ ("2024-01-02", "web", "1.0.0", (("control", 200), ("test", 10))),+ ("2024-01-04", "web", "2.0.0", (("control", 60), ("test", 60))),+ ("2024-01-04", "posthog-node", "4.2.1", (("control", 15), ("test", 105))),+ )+ for variant, count in counts+ for i in range(count)+ },+ self.team,+ )+ flush_persons_and_events()+ self._log_split_change_from_90_10_on_2024_01_03()++ with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=True):+ response = ExperimentExposuresQueryRunner(+ team=self.team, query=self._running_exposure_query(), user=self.user+ ).calculate()++ [finding] = response.health_findings or []+ versions = sorted(+ str(value) for key, value in finding.evidence.items() if re.fullmatch(r"library_version_\d+", key)+ )+ self.assertEqual((finding.subcode, versions), ("split_changed", ["2.0.0", "4.2.1"]))+ self.assertIn(+ 'The mismatch comes from users whose first exposure was sent by version "4.2.1" of the "posthog-node" '+ "library.",+ finding.detail,+ )++ def _log_split_change_from_90_10_on_2024_01_03(self) -> None:before = {**self.feature_flag.filters,"multivariate": {@@ -1573,36 +1634,44 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):created_at=datetime(2024, 1, 3, tzinfo=UTC),)- response = ExperimentExposuresQueryRunner(- team=self.team, query=self._running_exposure_query(), user=self.user if as_user else None- ).calculate()-- self.assertEqual(- [(finding.code, finding.subcode) for finding in response.health_findings or []],- [(ExperimentExposureHealthFindingCode.SRM, subcode)],- )-@parameterized.expand([("reader_who_sees_the_findings",True,- 'The mismatch comes from users whose first exposure was sent by the "posthog-node" library.',+ 'The mismatch comes from users whose first exposure was sent by version "4.2.1" of the "posthog-node" '+ "library.",{- "reason": "library",+ "reason": "library_version",+ "located_dimensions": "library,library_version","library_1": "web","library_1_exposed_control": 100,"library_1_exposed_test": 100,"library_2": "posthog-node","library_2_exposed_control": 5,"library_2_exposed_test": 120,+ "library_3": "posthog-ios",+ "library_3_exposed_control": 10,+ "library_3_exposed_test": 10,+ "library_version_1": "1.200.0",+ "library_version_1_library": "web",+ "library_version_1_exposed_control": 100,+ "library_version_1_exposed_test": 100,+ "library_version_2": "4.2.1",+ "library_version_2_library": "posthog-node",+ "library_version_2_exposed_control": 5,+ "library_version_2_exposed_test": 120,+ "library_version_3": "3.0.0",+ "library_version_3_library": "posthog-ios",+ "library_version_3_exposed_control": 10,+ "library_version_3_exposed_test": 10,"page_1": "/","page_1_exposed_control": 34,"page_1_exposed_test": 34,"page_2": "/pricing","page_2_exposed_control": 20,"page_2_exposed_test": 20,- "page_3": "/docs",+ "page_3": "/docs/:page","page_3_exposed_control": 15,"page_3_exposed_test": 15,"page_4": "/blog",@@ -1614,6 +1683,9 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):"page_6": BREAKDOWN_OTHER_STRING_LABEL,"page_6_exposed_control": 9,"page_6_exposed_test": 9,+ "screen_1": "Checkout",+ "screen_1_exposed_control": 10,+ "screen_1_exposed_test": 10,},),(@@ -1628,43 +1700,59 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):def test_srm_splits_the_exposures_by_the_library_and_page_of_the_first_exposure(self, _name: str, findings_shown: bool, detail: str, evidence: dict[str, str | int]) -> None:- def exposure(variant: str, timestamp: str, library: str, path: str | None) -> dict[str, Any]:- properties = {"$feature_flag_response": variant, "$feature_flag": self.feature_flag.key, "$lib": library}+ def exposure(variant: str, timestamp: str, library: str, version: str, **place: str | None) -> dict[str, Any]:return {"event": "$feature_flag_called","timestamp": timestamp,- "properties": {**properties, "$pathname": path} if path else properties,+ "properties": {+ "$feature_flag_response": variant,+ "$feature_flag": self.feature_flag.key,+ "$lib": library,+ "$lib_version": version,+ **{key: value for key, value in place.items() if value},+ },}- # Seven pages, so that the two smallest share the "other" segment. Half of the users first exposed on the- # web are exposed by the server later, and stay web users.+ # Seven pages, so that the two smallest share the "other" segment, and one of them has a path per document+ # that the team's path cleaning groups. Half of the users first exposed on the web are exposed by the server+ # later, and stay web users.+ self.team.path_cleaning_filters = [{"alias": "/docs/:page", "regex": "/docs/\\d+"}]+ self.team.save()pages = [pagefor page, users in (("/", 34),("/pricing", 20),- ("/docs", 15),+ ("/docs/{i}", 15),("/blog", 12),("/about", 10),("/careers", 5),("/contact", 4),)- for _ in range(users)+ for i in range(users)+ for page in (page.format(i=i),)]web_first = {f"web_{variant}_{i}": [- exposure(variant, "2024-01-02T10:00:00Z", "web", page),- *([exposure(variant, "2024-01-03T10:00:00Z", "posthog-node", None)] if i % 2 else []),+ exposure(variant, "2024-01-02T10:00:00Z", "web", "1.200.0", **{"$pathname": page}),+ *([exposure(variant, "2024-01-03T10:00:00Z", "posthog-node", "4.2.1")] if i % 2 else []),]for variant in ("control", "test")for i, page in enumerate(pages)}server_only = {- f"server_{variant}_{i}": [exposure(variant, "2024-01-03T10:00:00Z", "posthog-node", None)]+ f"server_{variant}_{i}": [exposure(variant, "2024-01-03T10:00:00Z", "posthog-node", "4.2.1")]for variant, count in (("control", 5), ("test", 120))for i in range(count)}- journeys_for({**web_first, **server_only}, self.team)+ mobile = {+ f"mobile_{variant}_{i}": [+ exposure(variant, "2024-01-02T11:00:00Z", "posthog-ios", "3.0.0", **{"$screen_name": "Checkout"})+ ]+ for variant in ("control", "test")+ for i in range(10)+ }+ journeys_for({**web_first, **server_only, **mobile}, self.team)flush_persons_and_events()with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=findings_shown):