Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): locate a mismatch by sdk version, cleaned page and screen (#125340)

Marcel Poelker19a94a53524a
diff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.py
index 369a7e3e412d..f01b91f2289a 100644
--- a/products/experiments/backend/health/checks/srm.py
+++ b/products/experiments/backend/health/checks/srm.py
@@ -1,5 +1,6 @@
import unicodedata
from collections.abc import Mapping
+from datetime import date
from enum import StrEnum
from zoneinfo import ZoneInfo
@@ -29,6 +30,8 @@ TOP_SEGMENT_VALUES = 5
# everywhere. Two segments this large are needed before the finding calls a dimension uniform.
UNIFORM_SEGMENT_MINIMUM_SIZE = 1000
SEGMENT_LABEL_MAX_LENGTH = 60
+# The detail names the located segments furthest from the split, up to this many.
+NAMED_SEGMENTS = 2
# pinned: the subcode is sent as `finding_variant` on the health finding events, so insights group on it.
SPLIT_CHANGED = "split_changed"
@@ -37,10 +40,20 @@ SPLIT_CHANGED = "split_changed"
class SrmReason(StrEnum):
# pinned: sent as `finding_reason` on the health finding events, so insights group on them.
LIBRARY = ExposureSegmentDimension.LIBRARY.value
+ LIBRARY_VERSION = ExposureSegmentDimension.LIBRARY_VERSION.value
PAGE = ExposureSegmentDimension.PAGE.value
+ SCREEN = ExposureSegmentDimension.SCREEN.value
UNIFORM = "uniform"
+_DIMENSION_PLURALS = {
+ ExposureSegmentDimension.LIBRARY: "SDK libraries",
+ ExposureSegmentDimension.LIBRARY_VERSION: "SDK versions",
+ ExposureSegmentDimension.PAGE: "pages",
+ ExposureSegmentDimension.SCREEN: "screens",
+}
+
+
def sample_ratio_test(observed: Mapping[str, int], weights: Mapping[str, float]) -> SampleRatioMismatch | None:
"""Chi-squared goodness-of-fit test of the observed counts against the weights of the variants in the split.
None when there is too little data."""
@@ -67,14 +80,31 @@ def reads_flag_history(ctx: HealthContext) -> bool:
def reads_segments(ctx: HealthContext) -> bool:
- """Whether `srm` reports a mismatch that no split change explains, which it then looks for in the segments of the
- exposures. The exposure query loads the segments only then, because their query scans the exposure events."""
+ """Whether `srm` reports a mismatch that a split change doesn't explain, which it then looks for in the segments
+ of the exposures. After a split change, that is a mismatch among the users first exposed after the change, and
+ the segments count only them (`segments_first_exposed_after`). The exposure query loads the segments only then,
+ because their query scans the exposure events."""
# With an activation, the segments would come from the activation event, not from the flag call that assigned
# the variant.
if ctx.exposure_events is not None and ctx.exposure_events.activation is not None:
return False
outcome = srm(ctx)
- return isinstance(outcome, ExperimentHealthFinding) and outcome.subcode != SPLIT_CHANGED
+ if not isinstance(outcome, ExperimentHealthFinding):
+ return False
+ if outcome.subcode != SPLIT_CHANGED:
+ return True
+ retest_p_value = outcome.evidence.get("p_value_after_split_change")
+ return isinstance(retest_p_value, float) and retest_p_value < SAMPLE_RATIO_MISMATCH_P_VALUE
+
+
+def segments_first_exposed_after(ctx: HealthContext) -> date | None:
+ """The day after which the segments count the users, in the team's timezone. None for the whole run. After a split
+ change, the users first exposed before it got their variants from another split, so they would fail the test
+ against the current split in any segment they fill."""
+ history = ctx.flag_history
+ if not isinstance(history, FlagHistory) or _split_change_warning(ctx) is None:
+ return None
+ return _last_split_change_day(history)
def srm(ctx: HealthContext) -> CheckOutcome:
@@ -103,6 +133,15 @@ def srm(ctx: HealthContext) -> CheckOutcome:
# retest that passes can't rule out a mismatch with another cause. The retest only adds evidence.
unit = "group" if exposures.counts_groups else "user"
retest, p_value_after_change = _retest_after_change(ctx, exposures, history, unit)
+ evidence: dict[str, str | int | float | None] = {
+ "p_value": p_value,
+ "p_value_after_split_change": p_value_after_change,
+ }
+ cause: tuple[str, ...] = ()
+ if p_value_after_change is not None and p_value_after_change < SAMPLE_RATIO_MISMATCH_P_VALUE:
+ segment_cause = _segment_cause(exposures, unit)
+ cause = segment_cause.sentences
+ evidence.update(segment_cause.evidence)
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.SRM,
subcode=SPLIT_CHANGED,
@@ -115,10 +154,11 @@ def srm(ctx: HealthContext) -> CheckOutcome:
f"The variant split changed after launch while {unit}s were exposed, and some of them got their "
"variants from another split, which can cause this.",
*retest,
+ *cause,
f'Follow the steps in "{split_change.title}" to fix the results.',
]
),
- evidence={"p_value": p_value, "p_value_after_split_change": p_value_after_change},
+ evidence=evidence,
actions=(),
diagnostic_ref="A2",
)
@@ -139,7 +179,7 @@ def _retest_after_change(
f"The exposure criteria count {unit}s at an activation, so the {unit}s exposed after the change can't "
"be tested alone."
], None
- changed_on = history.counted(SplitChange)[-1].changed_at.astimezone(ZoneInfo(history.team_timezone)).date()
+ changed_on = _last_split_change_day(history)
after_change = sample_ratio_test(
{
variant: sum(count for day, count in daily.items() if day > changed_on)
@@ -160,6 +200,10 @@ def _retest_after_change(
], after_change.p_value
+def _last_split_change_day(history: FlagHistory) -> date:
+ return history.counted(SplitChange)[-1].changed_at.astimezone(ZoneInfo(history.team_timezone)).date()
+
+
def _split_change_warning(ctx: HealthContext) -> ExperimentHealthFinding | None:
try:
outcome = split_changed_mid_run(ctx)
@@ -174,12 +218,7 @@ def _split_change_warning(ctx: HealthContext) -> ExperimentHealthFinding | None:
def _mismatch(exposures: ExposureTotals, p_value: float) -> ExperimentHealthFinding:
unit = "group" if exposures.counts_groups else "user"
- tests = _segment_tests(exposures)
- verdicts = [_verdict(tests, dimension, exposures.sample_ratio_expected) for dimension in ExposureSegmentDimension]
- cause, reason = _cause(verdicts, unit, exposures.sample_ratio_expected)
- evidence: dict[str, str | int | float | None] = {"p_value": p_value, **_segment_evidence(tests, verdicts)}
- if reason is not None:
- evidence["reason"] = reason.value
+ cause = _segment_cause(exposures, unit)
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.SRM,
subcode=None,
@@ -189,10 +228,10 @@ def _mismatch(exposures: ExposureTotals, p_value: float) -> ExperimentHealthFind
[
f"The distribution of {unit}s across variants doesn't match your configured rollout percentages "
f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}).",
- *cause,
+ *(cause.sentences or ("This may indicate issues with randomization or data collection.",)),
]
),
- evidence=evidence,
+ evidence={"p_value": p_value, **cause.evidence},
actions=(),
diagnostic_ref="A2",
)
@@ -211,6 +250,29 @@ class _Verdict:
homogeneity_p_value: float | None
powered: bool
located: ExposureSegment | None
+ distance: float
+
+
+@frozen
+class _SegmentCause:
+ """What the segments say about the mismatch: the sentences that follow it, and the reason and the segments as
+ evidence. Without a verdict the sentences are empty."""
+
+ sentences: tuple[str, ...]
+ evidence: dict[str, str | int | float | None]
+
+
+def _segment_cause(exposures: ExposureTotals, unit: str) -> _SegmentCause:
+ tests = _segment_tests(exposures)
+ verdicts = [_verdict(tests, dimension, exposures.sample_ratio_expected) for dimension in ExposureSegmentDimension]
+ sentences, reason = _cause(verdicts, unit)
+ evidence = _segment_evidence(tests, verdicts)
+ located = _located(verdicts)
+ if located:
+ evidence["located_dimensions"] = ",".join(segment.dimension.value for segment in located)
+ if reason is not None:
+ evidence["reason"] = reason.value
+ return _SegmentCause(sentences=tuple(sentences), evidence=evidence)
def _segment_tests(exposures: ExposureTotals) -> list[_SegmentTest]:
@@ -231,18 +293,26 @@ def _verdict(tests: list[_SegmentTest], dimension: ExposureSegmentDimension, wei
variant for variant in weights if any(test.segment.exposed_by_variant.get(variant, 0) for test in compared)
]
if len(compared) < 2 or len(variants) < 2:
- return _Verdict(dimension=dimension, homogeneity_p_value=None, powered=False, located=None)
+ return _Verdict(dimension=dimension, homogeneity_p_value=None, powered=False, located=None, distance=0.0)
table = [[test.segment.exposed_by_variant.get(variant, 0) for variant in variants] for test in compared]
homogeneity_p_value = float(chi2_contingency(table).pvalue)
powered = sum(1 for row in table if sum(row) >= UNIFORM_SEGMENT_MINIMUM_SIZE) >= 2
- if homogeneity_p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:
- return _Verdict(dimension=dimension, homogeneity_p_value=homogeneity_p_value, powered=powered, located=None)
- failing = [
- test.segment for test in compared if test.p_value is not None and test.p_value < SAMPLE_RATIO_MISMATCH_P_VALUE
- ]
- # The detail says the mismatch comes from the segment, which reads as if the other segments match the split.
- located = failing[0] if len(failing) == 1 else None
- return _Verdict(dimension=dimension, homogeneity_p_value=homogeneity_p_value, powered=powered, located=located)
+ located: ExposureSegment | None = None
+ if homogeneity_p_value < SAMPLE_RATIO_MISMATCH_P_VALUE:
+ failing = [
+ test.segment
+ for test in compared
+ if test.p_value is not None and test.p_value < SAMPLE_RATIO_MISMATCH_P_VALUE
+ ]
+ # The detail says the mismatch comes from the segment, which reads as if the other segments match the split.
+ located = failing[0] if len(failing) == 1 else None
+ return _Verdict(
+ dimension=dimension,
+ homogeneity_p_value=homogeneity_p_value,
+ powered=powered,
+ located=located,
+ distance=_distance_from_split(located, weights) if located is not None else 0.0,
+ )
def _distance_from_split(segment: ExposureSegment, weights: Mapping[str, float]) -> float:
@@ -258,19 +328,15 @@ def _distance_from_split(segment: ExposureSegment, weights: Mapping[str, float])
)
-def _cause(verdicts: list[_Verdict], unit: str, weights: Mapping[str, float]) -> tuple[list[str], SrmReason | None]:
- """The sentences that follow the mismatch, and the finding's reason: the dimension of the segment furthest from
- the split."""
- located = sorted(
- (verdict.located for verdict in verdicts if verdict.located is not None),
- key=lambda segment: -_distance_from_split(segment, weights),
- )
- if located:
- places = ", and from ".join(f"{unit}s {_segment_place(segment)}" for segment in located)
+def _cause(verdicts: list[_Verdict], unit: str) -> tuple[list[str], SrmReason | None]:
+ """The sentences that follow the mismatch, and the finding's reason. Empty without a verdict."""
+ named = _without_libraries_of_located_versions(_located(verdicts))[:NAMED_SEGMENTS]
+ if named:
+ places = ", and from ".join(f"{unit}s {_segment_place(segment)}" for segment in named)
return [
f"The mismatch comes from {places}.",
f"Check that the code there sends the exposure for every {unit} in every variant.",
- ], SrmReason(located[0].dimension.value)
+ ], SrmReason(named[0].dimension.value)
compared = {
verdict.dimension: verdict.homogeneity_p_value
for verdict in verdicts
@@ -278,26 +344,67 @@ def _cause(verdicts: list[_Verdict], unit: str, weights: Mapping[str, float]) ->
}
powered = [verdict.dimension for verdict in verdicts if verdict.powered]
if powered and all(p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE for p_value in compared.values()):
- across = " and ".join(
- "SDK libraries" if dimension == ExposureSegmentDimension.LIBRARY else "pages" for dimension in powered
- )
+ across = _joined([_DIMENSION_PLURALS[dimension] for dimension in powered])
return [
f"The mismatch looks the same across {across}, which points at how {unit}s get their variants rather "
"than at one integration."
], SrmReason.UNIFORM
- return ["This may indicate issues with randomization or data collection."], None
+ return [], None
+
+
+def _located(verdicts: list[_Verdict]) -> list[ExposureSegment]:
+ """The located segments, the one furthest from the split first."""
+ ranked = sorted((verdict for verdict in verdicts if verdict.located is not None), key=lambda v: -v.distance)
+ return [verdict.located for verdict in ranked if verdict.located is not None]
+
+
+def _without_libraries_of_located_versions(located: list[ExposureSegment]) -> list[ExposureSegment]:
+ """A located version of a located library says the same more precisely, so the library isn't named too."""
+ versioned = {
+ segment.library for segment in located if segment.dimension == ExposureSegmentDimension.LIBRARY_VERSION
+ }
+ return [
+ segment
+ for segment in located
+ if not (segment.dimension == ExposureSegmentDimension.LIBRARY and segment.value in versioned)
+ ]
+
+
+def _joined(words: list[str]) -> str:
+ return " and ".join(filter(None, [", ".join(words[:-1]), words[-1]]))
def _segment_place(segment: ExposureSegment) -> str:
- if segment.dimension == ExposureSegmentDimension.LIBRARY:
- if segment.value == BREAKDOWN_OTHER_STRING_LABEL:
- return f"whose first exposure was sent by a library other than the {TOP_SEGMENT_VALUES} most common"
- if segment.value == BREAKDOWN_NULL_STRING_LABEL:
- return "whose first exposure was sent by an unknown library"
- return f"whose first exposure was sent by the {_segment_label(segment.value)} library"
- if segment.value == BREAKDOWN_OTHER_STRING_LABEL:
- return f"first exposed on a page other than the {TOP_SEGMENT_VALUES} most common"
- return f"first exposed on the page {_segment_label(segment.value)}"
+ match segment.dimension:
+ case ExposureSegmentDimension.LIBRARY:
+ if segment.value == BREAKDOWN_OTHER_STRING_LABEL:
+ return f"whose first exposure was sent by a library other than the {TOP_SEGMENT_VALUES} most common"
+ if segment.value == BREAKDOWN_NULL_STRING_LABEL:
+ return "whose first exposure was sent by an unknown library"
+ return f"whose first exposure was sent by the {_segment_label(segment.value)} library"
+ case ExposureSegmentDimension.LIBRARY_VERSION:
+ if segment.value == BREAKDOWN_OTHER_STRING_LABEL:
+ return (
+ "whose first exposure was sent by a library version other than the "
+ f"{TOP_SEGMENT_VALUES} most common"
+ )
+ library = segment.library
+ by_library = (
+ "an unknown library"
+ if library is None or library == BREAKDOWN_NULL_STRING_LABEL
+ else f"the {_segment_label(library)} library"
+ )
+ if segment.value == BREAKDOWN_NULL_STRING_LABEL:
+ return f"whose first exposure was sent by an unknown version of {by_library}"
+ return f"whose first exposure was sent by version {_segment_label(segment.value)} of {by_library}"
+ case ExposureSegmentDimension.PAGE:
+ if segment.value == BREAKDOWN_OTHER_STRING_LABEL:
+ return f"first exposed on a page other than the {TOP_SEGMENT_VALUES} most common"
+ return f"first exposed on the page {_segment_label(segment.value)}"
+ case ExposureSegmentDimension.SCREEN:
+ if segment.value == BREAKDOWN_OTHER_STRING_LABEL:
+ return f"first exposed on a screen other than the {TOP_SEGMENT_VALUES} most common"
+ return f"first exposed on the screen {_segment_label(segment.value)}"
def _segment_label(value: str) -> str:
@@ -311,8 +418,9 @@ def _segment_label(value: str) -> str:
def _segment_evidence(tests: list[_SegmentTest], verdicts: list[_Verdict]) -> dict[str, str | int | float | None]:
- """Each segment in the keys `<dimension>_N` (the value), `<dimension>_N_p_value` and
- `<dimension>_N_exposed_<variant>`, and each compared dimension in `<dimension>_homogeneity_p_value`."""
+ """Each segment in the keys `<dimension>_N` (the value), `<dimension>_N_library` (the library of a version),
+ `<dimension>_N_p_value` and `<dimension>_N_exposed_<variant>`, and each compared dimension in
+ `<dimension>_homogeneity_p_value`."""
evidence: dict[str, str | int | float | None] = {}
for verdict in verdicts:
dimension = verdict.dimension
@@ -322,6 +430,8 @@ def _segment_evidence(tests: list[_SegmentTest], verdicts: list[_Verdict]) -> di
for index, test in enumerate(in_dimension, start=1):
key = f"{dimension.value}_{index}"
evidence[key] = test.segment.value
+ if test.segment.library is not None:
+ evidence[f"{key}_library"] = test.segment.library
evidence[f"{key}_p_value"] = test.p_value
for variant, exposed in test.segment.exposed_by_variant.items():
evidence[f"{key}_exposed_{variant}"] = exposed
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index f1f73f5085a8..5d6ab4563158 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -237,9 +237,13 @@ class ExposureSegment:
dimension: ExposureSegmentDimension
# The property's value on the first exposure. BREAKDOWN_NULL_STRING_LABEL when the event has none, and
- # BREAKDOWN_OTHER_STRING_LABEL for the values outside the largest segments of the dimension.
+ # BREAKDOWN_OTHER_STRING_LABEL for the values outside the largest segments of the dimension. The version in the
+ # library version dimension.
value: str
exposed_by_variant: Mapping[str, int]
+ # The library of a library version segment, also BREAKDOWN_NULL_STRING_LABEL when the event has none. None in the
+ # other dimensions and in the BREAKDOWN_OTHER_STRING_LABEL segment.
+ library: str | None = None
@frozen
diff --git a/products/experiments/backend/health/test/test_split_changed.py b/products/experiments/backend/health/test/test_split_changed.py
index 37d590a6538a..6b742c11fc53 100644
--- a/products/experiments/backend/health/test/test_split_changed.py
+++ b/products/experiments/backend/health/test/test_split_changed.py
@@ -18,9 +18,15 @@ from products.experiments.backend.facade.contracts import (
from products.experiments.backend.health.checks.flag_change_findings import flag_history_not_readable
from products.experiments.backend.health.checks.flag_state import flag_state
from products.experiments.backend.health.checks.split_changed import split_changed_mid_run
-from products.experiments.backend.health.checks.srm import reads_flag_history, reads_segments, srm
+from products.experiments.backend.health.checks.srm import (
+ reads_flag_history,
+ reads_segments,
+ segments_first_exposed_after,
+ srm,
+)
from products.experiments.backend.health.context import (
ExposureEvents,
+ ExposureSegment,
ExposureTotals,
FlagHistoryNotCheckedReason,
HealthContext,
@@ -35,6 +41,7 @@ from products.experiments.backend.health.skips import (
not_applicable,
)
from products.experiments.backend.health.test.factories import ENDED, RUNNING, RUNNING_CONTEXT, Status, flag_history
+from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension
from products.feature_flags.backend.facade.edits import SplitChange, parse_variants
LAUNCHED_AT = datetime(2026, 9, 12, 10, 0, tzinfo=UTC)
@@ -534,6 +541,31 @@ class TestSrmAfterSplitChange(TestCase):
_totals({"control": 250, "test": 250}),
("split_changed", "don't match the current split either (p < 0.001), so look for another cause too."),
),
+ (
+ "retest_fails_in_one_library",
+ _context(EVEN, UNEVEN, exposed_units=5),
+ replace(
+ _totals({"control": 250, "test": 250}),
+ segments=(
+ ExposureSegment(
+ dimension=ExposureSegmentDimension.LIBRARY,
+ value="web",
+ exposed_by_variant={"control": 800, "test": 200},
+ ),
+ ExposureSegment(
+ dimension=ExposureSegmentDimension.LIBRARY,
+ value="posthog-node",
+ exposed_by_variant={"control": 50, "test": 450},
+ ),
+ ),
+ ),
+ (
+ "split_changed",
+ "so look for another cause too. The mismatch comes from users whose first exposure was sent by the "
+ '"posthog-node" library. Check that the code there sends the exposure for every user in every '
+ "variant. Follow the steps in",
+ ),
+ ),
(
"too_few_for_a_retest",
_context(EVEN, UNEVEN, exposed_units=5),
@@ -590,18 +622,42 @@ class TestSrmAfterSplitChange(TestCase):
@parameterized.expand(
[
- ("failing_test", RUNNING_CONTEXT, 0.00001, (True, True)),
- ("passing_test", RUNNING_CONTEXT, 0.01, (False, False)),
- ("split_change_explains_it", _context(EVEN, UNEVEN, exposed_units=5), 0.00001, (True, False)),
- ("activation", replace(RUNNING_CONTEXT, exposure_events=ACTIVATION_EVENTS), 0.00001, (True, False)),
+ ("failing_test", RUNNING_CONTEXT, MATCHES_THE_NEW_SPLIT, 0.00001, (True, True, None)),
+ ("passing_test", RUNNING_CONTEXT, MATCHES_THE_NEW_SPLIT, 0.01, (False, False, None)),
+ (
+ "split_change_explains_it",
+ _context(EVEN, UNEVEN, exposed_units=5),
+ MATCHES_THE_NEW_SPLIT,
+ 0.00001,
+ (True, False, FIRST_CHANGE_AT.date()),
+ ),
+ (
+ "split_change_with_a_failed_retest",
+ _context(EVEN, UNEVEN, exposed_units=5),
+ {"control": 250, "test": 250},
+ 0.00001,
+ (True, True, FIRST_CHANGE_AT.date()),
+ ),
+ (
+ "activation",
+ replace(RUNNING_CONTEXT, exposure_events=ACTIVATION_EVENTS),
+ MATCHES_THE_NEW_SPLIT,
+ 0.00001,
+ (True, False, None),
+ ),
]
)
def test_reads_the_flag_history_and_the_segments_only_for_a_mismatch(
- self, _name: str, ctx: HealthContext, p_value: float, expected: tuple[bool, bool]
+ self,
+ _name: str,
+ ctx: HealthContext,
+ after_change: dict[str, int],
+ p_value: float,
+ expected: tuple[bool, bool, date | None],
) -> None:
- ctx = replace(ctx, exposures=replace(_totals(MATCHES_THE_NEW_SPLIT), sample_ratio_mismatch_p_value=p_value))
+ ctx = replace(ctx, exposures=replace(_totals(after_change), sample_ratio_mismatch_p_value=p_value))
- self.assertEqual((reads_flag_history(ctx), reads_segments(ctx)), expected)
+ self.assertEqual((reads_flag_history(ctx), reads_segments(ctx), segments_first_exposed_after(ctx)), expected)
def test_names_the_cause_the_evidence_and_the_next_step(self) -> None:
outcome = srm(replace(_context(EVEN, UNEVEN, exposed_units=5), exposures=_totals(MATCHES_THE_NEW_SPLIT)))
diff --git a/products/experiments/backend/health/test/test_srm.py b/products/experiments/backend/health/test/test_srm.py
index c7154f7a4343..a715bd43b96e 100644
--- a/products/experiments/backend/health/test/test_srm.py
+++ b/products/experiments/backend/health/test/test_srm.py
@@ -6,7 +6,7 @@ from parameterized import parameterized
from posthog.schema import MultipleVariantHandling
-from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_OTHER_STRING_LABEL
+from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_NULL_STRING_LABEL, BREAKDOWN_OTHER_STRING_LABEL
from products.experiments.backend.facade.contracts import ExperimentHealthFinding
from products.experiments.backend.health.checks.srm import srm
@@ -15,7 +15,9 @@ from products.experiments.backend.health.test.factories import RUNNING_CONTEXT
from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension
LIBRARY = ExposureSegmentDimension.LIBRARY
+LIBRARY_VERSION = ExposureSegmentDimension.LIBRARY_VERSION
PAGE = ExposureSegmentDimension.PAGE
+SCREEN = ExposureSegmentDimension.SCREEN
MISMATCH = "The distribution of users across variants doesn't match your configured rollout percentages (p < 0.001)."
NO_LOCATION = f"{MISMATCH} This may indicate issues with randomization or data collection."
NEXT_STEP = "Check that the code there sends the exposure for every user in every variant."
@@ -25,6 +27,10 @@ def _segment(dimension: ExposureSegmentDimension, value: str, control: int, test
return ExposureSegment(dimension=dimension, value=value, exposed_by_variant={"control": control, "test": test})
+def _version(library: str, version: str, control: int, test: int) -> ExposureSegment:
+ return replace(_segment(LIBRARY_VERSION, version, control, test), library=library)
+
+
def _exposures(segments: tuple[ExposureSegment, ...] | None) -> ExposureTotals:
return ExposureTotals(
total_exposures={"control": 600, "test": 900},
@@ -68,7 +74,7 @@ class TestSrmSegments(TestCase):
f'{MISMATCH} The mismatch comes from users first exposed on the page "/checkout". {NEXT_STEP}',
),
(
- "the_largest_segment_with_the_skew",
+ "the_segment_furthest_from_the_split_first",
(
_segment(LIBRARY, "web", 800, 1200),
_segment(LIBRARY, "posthog-node", 400, 400),
@@ -76,8 +82,8 @@ class TestSrmSegments(TestCase):
_segment(PAGE, "/checkout", 200, 600),
),
"page",
- f'{MISMATCH} The mismatch comes from users first exposed on the page "/checkout", and from users whose '
- f'first exposure was sent by the "web" library. {NEXT_STEP}',
+ f'{MISMATCH} The mismatch comes from users first exposed on the page "/checkout", and from users '
+ f'whose first exposure was sent by the "web" library. {NEXT_STEP}',
),
(
"several_segments_skewed",
@@ -85,6 +91,53 @@ class TestSrmSegments(TestCase):
None,
NO_LOCATION,
),
+ (
+ "at_most_two_segments_named",
+ (
+ _segment(LIBRARY, "web", 800, 1200),
+ _segment(LIBRARY, "posthog-node", 400, 400),
+ _version("web", "1.200.0", 400, 400),
+ _version("posthog-node", "4.2.1", 100, 400),
+ _segment(PAGE, "/pricing", 600, 600),
+ _segment(PAGE, "/checkout", 200, 600),
+ ),
+ "library_version",
+ f'{MISMATCH} The mismatch comes from users whose first exposure was sent by version "4.2.1" of the '
+ f'"posthog-node" library, and from users first exposed on the page "/checkout". {NEXT_STEP}',
+ ),
+ (
+ "the_version_of_the_located_library",
+ (
+ *SERVER_SKEWED,
+ _version("web", "1.200.0", 500, 500),
+ _version("posthog-node", "4.2.1", 100, 400),
+ ),
+ "library_version",
+ f'{MISMATCH} The mismatch comes from users whose first exposure was sent by version "4.2.1" of the '
+ f'"posthog-node" library. {NEXT_STEP}',
+ ),
+ (
+ "an_unknown_version_of_the_located_library",
+ (
+ *SERVER_SKEWED,
+ _version("web", "1.200.0", 500, 500),
+ _version("posthog-node", BREAKDOWN_NULL_STRING_LABEL, 100, 400),
+ ),
+ "library_version",
+ f"{MISMATCH} The mismatch comes from users whose first exposure was sent by an unknown version of the "
+ f'"posthog-node" library. {NEXT_STEP}',
+ ),
+ (
+ "one_screen",
+ (
+ _segment(LIBRARY, "posthog-ios", 300, 600),
+ _segment(LIBRARY, "posthog-android", 300, 600),
+ _segment(SCREEN, "Home", 300, 300),
+ _segment(SCREEN, "Checkout", 100, 400),
+ ),
+ "screen",
+ f'{MISMATCH} The mismatch comes from users first exposed on the screen "Checkout". {NEXT_STEP}',
+ ),
(
"the_same_skew_everywhere",
(_segment(LIBRARY, "web", 96900, 93100), _segment(LIBRARY, "posthog-node", 5100, 4900)),
@@ -157,6 +210,7 @@ class TestSrmSegments(TestCase):
{
"p_value": 0.00001,
"reason": "library",
+ "located_dimensions": "library",
"library_1": "web",
"library_1_p_value": 1.0,
"library_1_exposed_control": 500,
diff --git a/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py b/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py
index e891810dfb4f..932f45b3a813 100644
--- a/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py
+++ b/products/experiments/backend/hogql_queries/experiment_exposure_query_builder.py
@@ -1,11 +1,12 @@
from dataclasses import replace
+from datetime import date
from enum import StrEnum
from posthog.schema import Breakdown, ExperimentEventExposureConfig, MultipleBreakdownType, MultipleVariantHandling
from posthog.hogql import ast
from posthog.hogql.parser import parse_expr, parse_select
-from posthog.hogql.property import property_to_expr
+from posthog.hogql.property import apply_path_cleaning, property_to_expr
from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_NULL_STRING_LABEL, BREAKDOWN_OTHER_STRING_LABEL
@@ -23,13 +24,17 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (
class ExposureSegmentDimension(StrEnum):
# pinned: the `srm` finding names its evidence keys and its reasons after these values.
LIBRARY = "library"
+ LIBRARY_VERSION = "library_version"
PAGE = "page"
+ SCREEN = "screen"
-# `segments_query` reads them as the columns breakdown_value_1 and breakdown_value_2, in this order.
+# `segments_query` reads them as the columns breakdown_value_1 to breakdown_value_4, in this order.
_SEGMENT_BREAKDOWNS = (
Breakdown(property="$lib", type=MultipleBreakdownType.EVENT),
+ Breakdown(property="$lib_version", type=MultipleBreakdownType.EVENT),
Breakdown(property="$pathname", type=MultipleBreakdownType.EVENT),
+ Breakdown(property="$screen_name", type=MultipleBreakdownType.EVENT),
)
@@ -119,12 +124,15 @@ class ExposureQueryBuilder:
assert isinstance(query, ast.SelectQuery)
return query
- def segments_query(self, top_values: int) -> ast.SelectQuery:
+ def segments_query(self, top_values: int, first_exposed_after: date | None = None) -> ast.SelectQuery:
"""
- Persons or groups by variant and by the SDK library of their first exposure, and those whose first exposure
- has a path also by that path. Columns: dimension, segment, variant, exposed_count. In each
- dimension, the `top_values` segments with the most persons or groups keep their own rows, and the others share
- the BREAKDOWN_OTHER_STRING_LABEL segment, so that one path per URL can't grow the result.
+ Persons or groups by variant and by the SDK library of their first exposure, by the version of that library,
+ and by the page of the exposure, or by its screen when it has no path. The team's path cleaning rules group
+ the pages. Columns: dimension, library, segment, variant, exposed_count. `library` holds the library of a
+ version segment, and is empty in the other dimensions. In each dimension, the `top_values` segments with the
+ most persons or groups keep their own rows, and the others share the BREAKDOWN_OTHER_STRING_LABEL segment, so
+ that one path per URL can't grow the result. With `first_exposed_after`, only the persons or groups first
+ exposed after that day in the team's timezone count, as in the daily counts of timeseries_query.
"""
entities = ExposureQueryBuilder(
context=replace(self.context, breakdowns=_SEGMENT_BREAKDOWNS),
@@ -133,56 +141,81 @@ class ExposureQueryBuilder:
# The window functions rank the segments on the grouped rows, so the events are scanned once.
query = parse_select(
"""
- SELECT dimension, segment, variant, sum(exposed_count) AS exposed_count
+ SELECT dimension, library, segment, variant, sum(exposed_count) AS exposed_count
FROM (
SELECT
dimension,
+ if(segment_rank <= {top_values}, segment_library, '') AS library,
if(segment_rank <= {top_values}, segment_value, {other_segments}) AS segment,
variant,
exposed_count
FROM (
SELECT
dimension,
+ segment_library,
segment_value,
variant,
exposed_count,
dense_rank() OVER (
- PARTITION BY dimension ORDER BY segment_total DESC, segment_value ASC
+ PARTITION BY dimension ORDER BY segment_total DESC, segment_library ASC, segment_value ASC
) AS segment_rank
FROM (
SELECT
dimension,
+ segment_library,
segment_value,
variant,
exposed_count,
- sum(exposed_count) OVER (PARTITION BY dimension, segment_value) AS segment_total
+ sum(exposed_count) OVER (
+ PARTITION BY dimension, segment_library, segment_value
+ ) AS segment_total
FROM (
SELECT
- segment_pair.1 AS dimension,
- segment_pair.2 AS segment_value,
+ segment_tuple.1 AS dimension,
+ segment_tuple.2 AS segment_library,
+ segment_tuple.3 AS segment_value,
variant,
count() AS exposed_count
FROM ({entities}) AS entities
- ARRAY JOIN if(
- notEmpty(breakdown_value_2) AND breakdown_value_2 != {no_path},
- [({library}, breakdown_value_1), ({page}, breakdown_value_2)],
- [({library}, breakdown_value_1)]
- ) AS segment_pair
- WHERE notEmpty(variant)
- GROUP BY dimension, segment_value, variant
+ ARRAY JOIN arrayConcat(
+ [
+ ({library}, '', breakdown_value_1),
+ ({library_version}, breakdown_value_1, breakdown_value_2)
+ ],
+ multiIf(
+ notEmpty(breakdown_value_3) AND breakdown_value_3 != {no_value},
+ [({page}, '', {page_value})],
+ notEmpty(breakdown_value_4) AND breakdown_value_4 != {no_value},
+ [({screen}, '', breakdown_value_4)],
+ []
+ )
+ ) AS segment_tuple
+ WHERE notEmpty(variant) AND {first_exposed}
+ GROUP BY dimension, segment_library, segment_value, variant
)
)
)
)
- GROUP BY dimension, segment, variant
+ GROUP BY dimension, library, segment, variant
""",
placeholders={
"entities": entities,
"top_values": ast.Constant(value=top_values),
"other_segments": ast.Constant(value=BREAKDOWN_OTHER_STRING_LABEL),
- "no_path": ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL),
+ "no_value": ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL),
"library": ast.Constant(value=ExposureSegmentDimension.LIBRARY.value),
+ "library_version": ast.Constant(value=ExposureSegmentDimension.LIBRARY_VERSION.value),
"page": ast.Constant(value=ExposureSegmentDimension.PAGE.value),
+ "page_value": apply_path_cleaning(ast.Field(chain=["breakdown_value_3"]), self.context.team),
+ "screen": ast.Constant(value=ExposureSegmentDimension.SCREEN.value),
+ "first_exposed": (
+ parse_expr(
+ "toDate(toString(first_exposure_time)) > {day}",
+ placeholders={"day": ast.Constant(value=first_exposed_after)},
+ )
+ if first_exposed_after is not None
+ else ast.Constant(value=True)
+ ),
},
)
assert isinstance(query, ast.SelectQuery)
diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
index 5ea15b5c8e64..e71ad1b3d6db 100644
--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
@@ -50,6 +50,7 @@ from products.experiments.backend.health.checks.srm import (
reads_flag_history,
reads_segments,
sample_ratio_test,
+ segments_first_exposed_after,
)
from products.experiments.backend.health.checks.zero_exposures import reads_exposed_without_test_account_filter
from products.experiments.backend.health.context import (
@@ -356,7 +357,8 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
if not self._runs_health_queries():
return evaluate_health(context, EXPOSURE_HEALTH_CHECKS)
if reads_segments(context):
- exposures = replace(exposures, segments=self._segments(exposure_params))
+ segments = self._segments(exposure_params, segments_first_exposed_after(context))
+ exposures = replace(exposures, segments=segments)
context = replace(context, exposures=exposures)
# Without test account filters on the project, the count without the filter is the same zero.
if (
@@ -368,17 +370,26 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
context = replace(context, exposures=replace(exposures, exposed_without_test_account_filter=exposed))
return evaluate_health(context, EXPOSURE_HEALTH_CHECKS)
- def _segments(self, exposure_params: ExposureQueryParams) -> tuple[ExposureSegment, ...] | None:
- query = self._query_builder(exposure_params).get_exposure_segments_query(TOP_SEGMENT_VALUES)
+ def _segments(
+ self, exposure_params: ExposureQueryParams, first_exposed_after: date | None
+ ) -> tuple[ExposureSegment, ...] | None:
+ query = self._query_builder(exposure_params).get_exposure_segments_query(
+ TOP_SEGMENT_VALUES, first_exposed_after
+ )
rows = self._run_health_query("ExperimentExposureSegmentsQuery", query)
if rows is None:
return None
- exposed: dict[tuple[str, str], dict[str, int]] = {}
- for dimension, segment, variant, count in rows:
- exposed.setdefault((dimension, segment), {})[variant] = int(count)
+ exposed: dict[tuple[str, str, str], dict[str, int]] = {}
+ for dimension, library, segment, variant, count in rows:
+ exposed.setdefault((dimension, library, segment), {})[variant] = int(count)
return tuple(
- ExposureSegment(dimension=ExposureSegmentDimension(dimension), value=segment, exposed_by_variant=counts)
- for (dimension, segment), counts in exposed.items()
+ ExposureSegment(
+ dimension=ExposureSegmentDimension(dimension),
+ value=segment,
+ exposed_by_variant=counts,
+ library=library or None,
+ )
+ for (dimension, library, segment), counts in exposed.items()
)
def _exposed_without_test_account_filter(self, exposure_params: ExposureQueryParams) -> int | None:
diff --git a/products/experiments/backend/hogql_queries/experiment_query_builder.py b/products/experiments/backend/hogql_queries/experiment_query_builder.py
index ac65b6c19912..169e30156840 100644
--- a/products/experiments/backend/hogql_queries/experiment_query_builder.py
+++ b/products/experiments/backend/hogql_queries/experiment_query_builder.py
@@ -1,6 +1,6 @@
from collections.abc import Sequence
from dataclasses import dataclass
-from datetime import datetime
+from datetime import date, datetime
from typing import Optional, Union
from django.utils import timezone
@@ -298,12 +298,13 @@ class ExperimentQueryBuilder:
"""
return self._exposure_query_builder().timeseries_query()
- def get_exposure_segments_query(self, top_values: int) -> ast.SelectQuery:
+ def get_exposure_segments_query(self, top_values: int, first_exposed_after: date | None = None) -> ast.SelectQuery:
"""
- Persons or groups by variant and by the SDK library and the page of their first exposure. Columns:
- dimension, segment, variant, exposed_count. It scans the events, also when precomputed exposures exist.
+ Persons or groups by variant and by the SDK library, the library version, and the page or screen of their
+ first exposure. Columns: dimension, library, segment, variant, exposed_count. It scans the events, also when
+ precomputed exposures exist.
"""
- return self._exposure_query_builder().segments_query(top_values)
+ return self._exposure_query_builder().segments_query(top_values, first_exposed_after)
def get_exposed_count_query(self) -> ast.SelectQuery:
"""The persons or groups exposed, in one row. Column: exposed_count."""
diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
index 721821d20493..a9acde80de81 100644
--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
@@ -1548,6 +1548,67 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
self.team,
)
flush_persons_and_events()
+ self._log_split_change_from_90_10_on_2024_01_03()
+
+ response = ExperimentExposuresQueryRunner(
+ team=self.team, query=self._running_exposure_query(), user=self.user if as_user else None
+ ).calculate()
+
+ self.assertEqual(
+ [(finding.code, finding.subcode) for finding in response.health_findings or []],
+ [(ExperimentExposureHealthFindingCode.SRM, subcode)],
+ )
+
+ @time_machine.travel("2024-01-07T12:00:00Z", tick=False)
+ def test_srm_segments_after_a_split_change_count_only_the_users_first_exposed_after_it(self) -> None:
+ ff_property = f"$feature/{self.feature_flag.key}"
+ # Web 1.0.0 got the old 90/10 split before the change. After it, web 2.0.0 matches the current 50/50 split
+ # and posthog-node 4.2.1 doesn't.
+ journeys_for(
+ {
+ f"user_{library}_{version}_{variant}_{i}": [
+ {
+ "event": "$feature_flag_called",
+ "timestamp": day,
+ "properties": {
+ "$feature_flag_response": variant,
+ ff_property: variant,
+ "$feature_flag": self.feature_flag.key,
+ "$lib": library,
+ "$lib_version": version,
+ },
+ }
+ ]
+ for day, library, version, counts in (
+ ("2024-01-02", "web", "1.0.0", (("control", 200), ("test", 10))),
+ ("2024-01-04", "web", "2.0.0", (("control", 60), ("test", 60))),
+ ("2024-01-04", "posthog-node", "4.2.1", (("control", 15), ("test", 105))),
+ )
+ for variant, count in counts
+ for i in range(count)
+ },
+ self.team,
+ )
+ flush_persons_and_events()
+ self._log_split_change_from_90_10_on_2024_01_03()
+
+ with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=True):
+ response = ExperimentExposuresQueryRunner(
+ team=self.team, query=self._running_exposure_query(), user=self.user
+ ).calculate()
+
+ [finding] = response.health_findings or []
+ versions = sorted(
+ str(value) for key, value in finding.evidence.items() if re.fullmatch(r"library_version_\d+", key)
+ )
+ self.assertEqual((finding.subcode, versions), ("split_changed", ["2.0.0", "4.2.1"]))
+ self.assertIn(
+ 'The mismatch comes from users whose first exposure was sent by version "4.2.1" of the "posthog-node" '
+ "library.",
+ finding.detail,
+ )
+
+ def _log_split_change_from_90_10_on_2024_01_03(self) -> None:
before = {
**self.feature_flag.filters,
"multivariate": {
@@ -1573,36 +1634,44 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
created_at=datetime(2024, 1, 3, tzinfo=UTC),
)
- response = ExperimentExposuresQueryRunner(
- team=self.team, query=self._running_exposure_query(), user=self.user if as_user else None
- ).calculate()
-
- self.assertEqual(
- [(finding.code, finding.subcode) for finding in response.health_findings or []],
- [(ExperimentExposureHealthFindingCode.SRM, subcode)],
- )
-
@parameterized.expand(
[
(
"reader_who_sees_the_findings",
True,
- 'The mismatch comes from users whose first exposure was sent by the "posthog-node" library.',
+ 'The mismatch comes from users whose first exposure was sent by version "4.2.1" of the "posthog-node" '
+ "library.",
{
- "reason": "library",
+ "reason": "library_version",
+ "located_dimensions": "library,library_version",
"library_1": "web",
"library_1_exposed_control": 100,
"library_1_exposed_test": 100,
"library_2": "posthog-node",
"library_2_exposed_control": 5,
"library_2_exposed_test": 120,
+ "library_3": "posthog-ios",
+ "library_3_exposed_control": 10,
+ "library_3_exposed_test": 10,
+ "library_version_1": "1.200.0",
+ "library_version_1_library": "web",
+ "library_version_1_exposed_control": 100,
+ "library_version_1_exposed_test": 100,
+ "library_version_2": "4.2.1",
+ "library_version_2_library": "posthog-node",
+ "library_version_2_exposed_control": 5,
+ "library_version_2_exposed_test": 120,
+ "library_version_3": "3.0.0",
+ "library_version_3_library": "posthog-ios",
+ "library_version_3_exposed_control": 10,
+ "library_version_3_exposed_test": 10,
"page_1": "/",
"page_1_exposed_control": 34,
"page_1_exposed_test": 34,
"page_2": "/pricing",
"page_2_exposed_control": 20,
"page_2_exposed_test": 20,
- "page_3": "/docs",
+ "page_3": "/docs/:page",
"page_3_exposed_control": 15,
"page_3_exposed_test": 15,
"page_4": "/blog",
@@ -1614,6 +1683,9 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
"page_6": BREAKDOWN_OTHER_STRING_LABEL,
"page_6_exposed_control": 9,
"page_6_exposed_test": 9,
+ "screen_1": "Checkout",
+ "screen_1_exposed_control": 10,
+ "screen_1_exposed_test": 10,
},
),
(
@@ -1628,43 +1700,59 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
def test_srm_splits_the_exposures_by_the_library_and_page_of_the_first_exposure(
self, _name: str, findings_shown: bool, detail: str, evidence: dict[str, str | int]
) -> None:
- def exposure(variant: str, timestamp: str, library: str, path: str | None) -> dict[str, Any]:
- properties = {"$feature_flag_response": variant, "$feature_flag": self.feature_flag.key, "$lib": library}
+ def exposure(variant: str, timestamp: str, library: str, version: str, **place: str | None) -> dict[str, Any]:
return {
"event": "$feature_flag_called",
"timestamp": timestamp,
- "properties": {**properties, "$pathname": path} if path else properties,
+ "properties": {
+ "$feature_flag_response": variant,
+ "$feature_flag": self.feature_flag.key,
+ "$lib": library,
+ "$lib_version": version,
+ **{key: value for key, value in place.items() if value},
+ },
}
- # Seven pages, so that the two smallest share the "other" segment. Half of the users first exposed on the
- # web are exposed by the server later, and stay web users.
+ # Seven pages, so that the two smallest share the "other" segment, and one of them has a path per document
+ # that the team's path cleaning groups. Half of the users first exposed on the web are exposed by the server
+ # later, and stay web users.
+ self.team.path_cleaning_filters = [{"alias": "/docs/:page", "regex": "/docs/\\d+"}]
+ self.team.save()
pages = [
page
for page, users in (
("/", 34),
("/pricing", 20),
- ("/docs", 15),
+ ("/docs/{i}", 15),
("/blog", 12),
("/about", 10),
("/careers", 5),
("/contact", 4),
)
- for _ in range(users)
+ for i in range(users)
+ for page in (page.format(i=i),)
]
web_first = {
f"web_{variant}_{i}": [
- exposure(variant, "2024-01-02T10:00:00Z", "web", page),
- *([exposure(variant, "2024-01-03T10:00:00Z", "posthog-node", None)] if i % 2 else []),
+ exposure(variant, "2024-01-02T10:00:00Z", "web", "1.200.0", **{"$pathname": page}),
+ *([exposure(variant, "2024-01-03T10:00:00Z", "posthog-node", "4.2.1")] if i % 2 else []),
]
for variant in ("control", "test")
for i, page in enumerate(pages)
}
server_only = {
- f"server_{variant}_{i}": [exposure(variant, "2024-01-03T10:00:00Z", "posthog-node", None)]
+ f"server_{variant}_{i}": [exposure(variant, "2024-01-03T10:00:00Z", "posthog-node", "4.2.1")]
for variant, count in (("control", 5), ("test", 120))
for i in range(count)
}
- journeys_for({**web_first, **server_only}, self.team)
+ mobile = {
+ f"mobile_{variant}_{i}": [
+ exposure(variant, "2024-01-02T11:00:00Z", "posthog-ios", "3.0.0", **{"$screen_name": "Checkout"})
+ ]
+ for variant in ("control", "test")
+ for i in range(10)
+ }
+ journeys_for({**web_first, **server_only, **mobile}, self.team)
flush_persons_and_events()
with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=findings_shown):