Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): warn when many users see multiple variants (#120059)

Marcel Poelkera408ef10853b
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 28e7bcd397a7..cb91fe0a6a5e 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -10671,9 +10671,9 @@ snapshots:
scenes-app-experiments--experiment-frequentist-two-variants--light:
hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmII
scenes-app-experiments--experiment-health-debug-tab--dark:
- hash: v1.k794b7964.89ba753555a9fc78822e75cf52e6783de7ea60c3046eed95150180b32ade9d41.P2cGRvwPp8IuU2bohVDvyURVapAxcTMO7RUmRE0WhjY
+ hash: v1.k794b7964.69bf5c8ee5d81e2dfcb1cfea21ca92353e0d300388424ef0654377b3a0a9b7d9.VIWed-fQfBs_yCyAD1To-hWPxDboIK-e_6RLoEUU_0Y
scenes-app-experiments--experiment-health-debug-tab--light:
- hash: v1.k794b7964.879c2f4f4ad7a1d61402af628b5a578f0e88761c284a1e5bf7f7ef64aa438219.ehJIFQobltKAPwR0e0UXYsAY5pt95g_t-6kui8yerfM
+ hash: v1.k794b7964.71e39d8c8fda127eb38fb0e94ee9e2563d5aadb136c56e4303c91b39fba117fb.fUslsFx1ChV8LWyNjBK54aGTqPqRP_lCEdWBbi5aWmI
scenes-app-experiments--experiment-recordings-dropped-metric--dark:
hash: v1.k794b7964.e0e7181f2c9e61a8324eff9aa7a9230fef25b3cf37e0b24ef875780186ed25dc.9VEvZ4iVieVJwmDol0gwtAsr3wgZYW5gj2KXAGZJfiw
scenes-app-experiments--experiment-recordings-dropped-metric--light:
diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json
index 7ad3e99f4924..96eee38e36c8 100644
--- a/frontend/src/queries/schema.json
+++ b/frontend/src/queries/schema.json
@@ -25756,7 +25756,7 @@
"type": "string"
},
"ExperimentExposureHealthFindingCode": {
- "enum": ["zero_exposures", "srm", "bias_risk_multiple_excluded"],
+ "enum": ["zero_exposures", "srm", "bias_risk_multiple_excluded", "multiple_variants_share"],
"type": "string"
},
"ExperimentExposureHealthFindingSeverity": {
diff --git a/frontend/src/queries/schema/schema-general.ts b/frontend/src/queries/schema/schema-general.ts
index 81acca0f8c4f..1ec8543aa95a 100644
--- a/frontend/src/queries/schema/schema-general.ts
+++ b/frontend/src/queries/schema/schema-general.ts
@@ -6321,7 +6321,11 @@ export interface BiasRisk {
multiple_variant_percentage: number
}
-export type ExperimentExposureHealthFindingCode = 'zero_exposures' | 'srm' | 'bias_risk_multiple_excluded'
+export type ExperimentExposureHealthFindingCode =
+ | 'zero_exposures'
+ | 'srm'
+ | 'bias_risk_multiple_excluded'
+ | 'multiple_variants_share'
export type ExperimentExposureHealthFindingSeverity = 'critical' | 'warning' | 'info'
diff --git a/posthog/schema_enums.py b/posthog/schema_enums.py
index 8b37f50fe30c..625229dff709 100644
--- a/posthog/schema_enums.py
+++ b/posthog/schema_enums.py
@@ -1444,6 +1444,7 @@ class ExperimentExposureHealthFindingCode(StrEnum):
ZERO_EXPOSURES = "zero_exposures"
SRM = "srm"
BIAS_RISK_MULTIPLE_EXCLUDED = "bias_risk_multiple_excluded"
+ MULTIPLE_VARIANTS_SHARE = "multiple_variants_share"
class ExperimentExposureHealthFindingSeverity(StrEnum):
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 7e9fb29a4823..0bada4ae41ea 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -9946,6 +9946,7 @@ export namespace Schemas {
* * `flag_live_after_end` - Flag Live After End
* * `flag_live_before_launch` - Flag Live Before Launch
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
* * `no_metric` - No Metric
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
@@ -9961,6 +9962,7 @@ export namespace Schemas {
| "flag_live_after_end"
| "flag_live_before_launch"
| "bias_risk_multiple_excluded"
+ | "multiple_variants_share"
| "no_metric"
| "srm"
| "zero_exposures"
@@ -10006,6 +10008,7 @@ export namespace Schemas {
* * `flag_live_after_end` - Flag Live After End
* * `flag_live_before_launch` - Flag Live Before Launch
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
* * `no_metric` - No Metric
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
@@ -10388,7 +10391,8 @@ export namespace Schemas {
export type ExperimentExposureHealthFindingCode =
| "zero_exposures"
| "srm"
- | "bias_risk_multiple_excluded";
+ | "bias_risk_multiple_excluded"
+ | "multiple_variants_share";
export type ExperimentExposureHealthFindingSeverity =
| "critical"
| "warning"
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index eaa622e1e671..3c016fd61976 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -103,6 +103,7 @@ class ExperimentHealthFindingCode(LabeledStrEnum):
FLAG_LIVE_AFTER_END = "flag_live_after_end"
FLAG_LIVE_BEFORE_LAUNCH = "flag_live_before_launch"
BIAS_RISK_MULTIPLE_EXCLUDED = "bias_risk_multiple_excluded"
+ MULTIPLE_VARIANTS_SHARE = "multiple_variants_share"
NO_METRIC = "no_metric"
SRM = "srm", "Sample Ratio Mismatch"
ZERO_EXPOSURES = "zero_exposures"
diff --git a/products/experiments/backend/health/checks/multiple_variants.py b/products/experiments/backend/health/checks/multiple_variants.py
new file mode 100644
index 000000000000..871936979a58
--- /dev/null
+++ b/products/experiments/backend/health/checks/multiple_variants.py
@@ -0,0 +1,65 @@
+from posthog.schema import MultipleVariantHandling
+
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFinding,
+ ExperimentHealthFindingCode,
+ ExperimentHealthFindingSeverity,
+)
+from products.experiments.backend.health.checks.bias_risk import MULTIPLE_VARIANT_KEY
+from products.experiments.backend.health.context import HealthContext
+
+# Some users get more than one ID in any experiment, for example one per device, so a low share is expected.
+# The share grows as users return and log in, so a lower line fires on a large part of all experiments. The cases
+# that users report sit above this line.
+MULTIPLE_VARIANTS_SHARE_THRESHOLD = 5.0 # on the 0-100 scale (5.0 = 5%)
+# The floor counts users or groups, not exposures: each of them is a fact, so the finding can fire on a new
+# experiment. The floor stops one or two testers who switch between variants from firing it.
+MULTIPLE_VARIANTS_MINIMUM_COUNT = 5
+
+
+def multiple_variants_share(ctx: HealthContext) -> ExperimentHealthFinding | None:
+ exposures = ctx.exposures
+ # Under "First seen" the exposure query gives each user their first variant, so it counts none of these users.
+ if exposures is None or exposures.multiple_variant_handling != MultipleVariantHandling.EXCLUDE:
+ return None
+ multiple = exposures.total_exposures.get(MULTIPLE_VARIANT_KEY, 0)
+ total = sum(exposures.total_exposures.values())
+ if multiple < MULTIPLE_VARIANTS_MINIMUM_COUNT:
+ return None
+ percentage = multiple / total * 100
+ if percentage <= MULTIPLE_VARIANTS_SHARE_THRESHOLD:
+ return None
+ # The flag gives every member of a group the same variant, so the causes of a person's several IDs do not apply.
+ if exposures.counts_groups:
+ title = "Many groups were exposed to multiple variants"
+ detail = (
+ f"{percentage:.1f}% of groups ({multiple}) were exposed to multiple variants, so the results leave them "
+ "out. The exposures table lists them as $multiple. The flag gives every member of a group the same "
+ "variant, so this usually means that the flag changed after launch, for example with a new variant split "
+ "or a change from matching users to matching groups. It also happens when a person switches to another "
+ "group and your code reads the flag before the flags reload: the event then carries the new group with "
+ "the variant of the old group. Check the flag's history, and after a group switch, read the flag only "
+ "once the flags reload."
+ )
+ else:
+ title = "Many users were exposed to multiple variants"
+ detail = (
+ f"{percentage:.1f}% of users ({multiple}) were exposed to multiple variants, so the results leave them "
+ "out. The exposures table lists them as $multiple. This usually means that one person has more than one "
+ "ID: an anonymous ID before login and another after, an ID for each device, a new ID after reset(), or a "
+ "different ID on your server than in the browser. It also happens when a bootstrapped flag value differs "
+ "from the value PostHog evaluates, or when the flag changes after launch, for example with a new variant "
+ "split. If a variant changes how often users log in, each variant loses a different group of users, and "
+ "the results can be biased. Identify users before your code evaluates the flag, call reset() only at "
+ "logout, and send the same ID from your server."
+ )
+ return ExperimentHealthFinding(
+ code=ExperimentHealthFindingCode.MULTIPLE_VARIANTS_SHARE,
+ subcode=None,
+ severity=ExperimentHealthFindingSeverity.WARNING,
+ title=title,
+ detail=detail,
+ evidence={"multiple_variant_percentage": percentage, "multiple_variant_count": multiple},
+ actions=(),
+ diagnostic_ref="A3",
+ )
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index 27010e18c0bc..a61d93b19c01 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -206,6 +206,8 @@ class ExposureTotals:
# None when the exposure query had too little data for the sample ratio test.
sample_ratio_mismatch_p_value: float | None
hours_since_launch: float | None
+ # Whether the totals count groups rather than persons, by the rule the exposure query uses.
+ counts_groups: bool
@frozen
diff --git a/products/experiments/backend/health/exposure_inputs.py b/products/experiments/backend/health/exposure_inputs.py
index e103ff01f03b..bbd3c6ecacf4 100644
--- a/products/experiments/backend/health/exposure_inputs.py
+++ b/products/experiments/backend/health/exposure_inputs.py
@@ -18,6 +18,7 @@ from products.experiments.backend.hogql_queries.experiment_exposures_query_runne
build_experiment_exposure_query,
)
from products.experiments.backend.hogql_queries.experiment_query_builder import get_exposure_config_params_for_builder
+from products.experiments.backend.hogql_queries.exposure_query_logic import aggregates_by_group
from products.experiments.backend.models.experiment import Experiment
@@ -89,6 +90,7 @@ def load_exposure_inputs(experiment: Experiment, *, user: User | None, can_query
sample_ratio_mismatch_p_value=srm.p_value if srm else None,
# The checks read the hours at computation, not now.
hours_since_launch=(computed_at - experiment.start_date).total_seconds() / 3600 if computed_at else None,
+ counts_groups=aggregates_by_group(runner.group_type_index),
),
findings=tuple(response.health_findings or ()),
error=calculation_error,
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index d6014dda2389..c8be36b2787a 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -10,6 +10,7 @@ from products.experiments.backend.health.checks.flag_change_findings import flag
from products.experiments.backend.health.checks.flag_state import flag_state
from products.experiments.backend.health.checks.flag_variants import flag_variants_mismatch
from products.experiments.backend.health.checks.forced_variant import forced_variant_release_condition
+from products.experiments.backend.health.checks.multiple_variants import multiple_variants_share
from products.experiments.backend.health.checks.no_metric import no_metric
from products.experiments.backend.health.checks.split_changed import split_changed_mid_run
from products.experiments.backend.health.checks.srm import srm
@@ -31,7 +32,12 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
# The exposure query runs these on its answer. They must not include the checks above: the exposure answer
# is cached for a day, and a flag edit would not clear their findings.
-EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (zero_exposures, srm, bias_risk_multiple_excluded)
+EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
+ zero_exposures,
+ srm,
+ bias_risk_multiple_excluded,
+ multiple_variants_share,
+)
def evaluate(ctx: HealthContext, checks: Sequence[HealthCheck]) -> list[ExperimentHealthFinding]:
diff --git a/products/experiments/backend/health/test/test_registry.py b/products/experiments/backend/health/test/test_registry.py
index a44872382a0e..9524f515901d 100644
--- a/products/experiments/backend/health/test/test_registry.py
+++ b/products/experiments/backend/health/test/test_registry.py
@@ -11,6 +11,7 @@ from posthog.schema import MultipleVariantHandling
from products.experiments.backend.facade.contracts import ExperimentHealthFindingCode
from products.experiments.backend.health import registry
from products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded
+from products.experiments.backend.health.checks.multiple_variants import multiple_variants_share
from products.experiments.backend.health.context import ExposureTotals, FlagState, FlagVariant, HealthContext
from products.experiments.backend.health.registry import EXPOSURE_HEALTH_CHECKS, evaluate
from products.experiments.backend.health.test.factories import RUNNING_CONTEXT
@@ -20,6 +21,7 @@ UNEVEN_SPLIT_TOTALS = ExposureTotals(
multiple_variant_handling=MultipleVariantHandling.EXCLUDE,
sample_ratio_mismatch_p_value=None,
hours_since_launch=72,
+ counts_groups=False,
)
RUNNING_UNEVEN_SPLIT = replace(
@@ -39,6 +41,21 @@ RUNNING_UNEVEN_SPLIT = replace(
NO_EXPOSURES = replace(UNEVEN_SPLIT_TOTALS, total_exposures={"control": 0, "test": 0})
ENDED: dict[str, Any] = {"is_running": False, "has_ended": True}
+EVEN_SPLIT_FLAG = FlagState(
+ active=True,
+ deleted=False,
+ release_groups=(),
+ variants=(FlagVariant(key="control", rollout_percentage=50), FlagVariant(key="test", rollout_percentage=50)),
+ reachable_conditions=(),
+)
+
+
+def _totals_with_multiple(multiple: int, total: int) -> ExposureTotals:
+ single = total - multiple
+ return replace(
+ UNEVEN_SPLIT_TOTALS,
+ total_exposures={"control": single - single // 2, "test": single // 2, "$multiple": multiple},
+ )
def _raise(_ctx: HealthContext) -> Any:
@@ -80,6 +97,23 @@ class TestEvaluate(TestCase):
},
[],
),
+ (
+ "multiple_variants_share_on_an_even_split",
+ {"flag": EVEN_SPLIT_FLAG, "exposures": _totals_with_multiple(60, 1000)},
+ [ExperimentHealthFindingCode.MULTIPLE_VARIANTS_SHARE],
+ ),
+ (
+ "multiple_variants_share_after_the_end",
+ {**ENDED, "exposures": _totals_with_multiple(60, 1000)},
+ [ExperimentHealthFindingCode.MULTIPLE_VARIANTS_SHARE],
+ ),
+ ("no_multiple_variants_share_at_the_threshold", {**ENDED, "exposures": _totals_with_multiple(5, 100)}, []),
+ ("no_multiple_variants_share_under_5_users", {**ENDED, "exposures": _totals_with_multiple(4, 40)}, []),
+ (
+ "multiple_variants_share_under_100_exposures",
+ {**ENDED, "exposures": _totals_with_multiple(5, 60)},
+ [ExperimentHealthFindingCode.MULTIPLE_VARIANTS_SHARE],
+ ),
]
)
def test_exposure_checks(
@@ -89,6 +123,18 @@ class TestEvaluate(TestCase):
self.assertEqual([finding.code for finding in findings], expected)
+ @parameterized.expand(
+ [
+ ("persons", False, "Many users were exposed to multiple variants"),
+ ("groups", True, "Many groups were exposed to multiple variants"),
+ ]
+ )
+ def test_multiple_variants_share_names_the_counted_unit(self, _name: str, counts_groups: bool, title: str) -> None:
+ exposures = replace(_totals_with_multiple(60, 1000), counts_groups=counts_groups)
+ findings = evaluate(replace(RUNNING_UNEVEN_SPLIT, **ENDED, exposures=exposures), (multiple_variants_share,))
+
+ self.assertEqual([finding.title for finding in findings], [title])
+
def test_a_failing_check_does_not_hide_the_other_findings(self) -> None:
with patch.object(registry, "capture_exception") as capture:
findings = evaluate(RUNNING_UNEVEN_SPLIT, (_raise, bias_risk_multiple_excluded))
diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
index 6974299bd4a8..5a764d491e84 100644
--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
@@ -46,7 +46,7 @@ from products.experiments.backend.hogql_queries.experiment_query_builder import
get_exposure_config_params_for_builder,
)
from products.experiments.backend.hogql_queries.experiment_query_runner import ExperimentResultsCacheMixin
-from products.experiments.backend.hogql_queries.exposure_query_logic import get_entity_key
+from products.experiments.backend.hogql_queries.exposure_query_logic import aggregates_by_group, get_entity_key
from products.experiments.backend.hogql_queries.precompute_policy import (
ensure_exposures_precomputed,
team_precompute_skip_reason,
@@ -396,6 +396,7 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
multiple_variant_handling=multiple_variant_handling,
sample_ratio_mismatch_p_value=sample_ratio_mismatch.p_value if sample_ratio_mismatch else None,
hours_since_launch=self.experiment.hours_since_launch,
+ counts_groups=aggregates_by_group(self.group_type_index),
)
)
diff --git a/products/experiments/backend/hogql_queries/exposure_query_logic.py b/products/experiments/backend/hogql_queries/exposure_query_logic.py
index 807f273c8cfb..4cad82a94678 100644
--- a/products/experiments/backend/hogql_queries/exposure_query_logic.py
+++ b/products/experiments/backend/hogql_queries/exposure_query_logic.py
@@ -370,7 +370,11 @@ def build_exposure_event_conditions(
]
+def aggregates_by_group(group_type_index: object) -> bool:
+ return isinstance(group_type_index, int)
+
+
def get_entity_key(group_type_index: Optional[int]) -> str:
- if isinstance(group_type_index, int):
+ if aggregates_by_group(group_type_index):
return f"$group_{group_type_index}"
return "person_id"
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index 2e7ab7bef40c..ed704780760b 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2268,6 +2268,7 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]
* * `flag_live_after_end` - Flag Live After End
* * `flag_live_before_launch` - Flag Live Before Launch
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
* * `no_metric` - No Metric
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
@@ -2286,6 +2287,7 @@ export const ExperimentHealthFindingCodeEnumApi = {
FlagLiveAfterEnd: 'flag_live_after_end',
FlagLiveBeforeLaunch: 'flag_live_before_launch',
BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',
+ MultipleVariantsShare: 'multiple_variants_share',
NoMetric: 'no_metric',
Srm: 'srm',
ZeroExposures: 'zero_exposures',
@@ -2347,6 +2349,7 @@ export interface ExperimentHealthFindingApi {
* * `flag_live_after_end` - Flag Live After End
* * `flag_live_before_launch` - Flag Live Before Launch
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
* * `no_metric` - No Metric
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.ts
index 35aabe17efbe..e83114eae102 100644
--- a/products/experiments/frontend/health/debug/healthDebugReport.ts
+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts
@@ -298,6 +298,11 @@ function exposureChecks(input: HealthDebugInput): HealthDebugCheck[] {
exposureCheck('zero_exposures', ['experiment', 'exposures'], null),
exposureCheck('srm', ['exposures'], 'The finding needs p < 0.001.'),
exposureCheck('bias_risk_multiple_excluded', ['exposures'], null),
+ exposureCheck(
+ 'multiple_variants_share',
+ ['exposures'],
+ 'The finding needs more than 5% and at least 5 users or groups in multiple variants, under Exclude only.'
+ ),
]
}
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index 372b6652c090..677f6b3074e1 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -18,6 +18,7 @@ export type ExperimentHealthFindingCode =
| 'flag_live_after_end'
| 'flag_live_before_launch'
| 'bias_risk_multiple_excluded'
+ | 'multiple_variants_share'
| 'srm'
| 'zero_exposures'
| 'no_metric'
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index c4bb995237e4..4009184a7911 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -44826,6 +44826,7 @@ export namespace Schemas {
* * `flag_live_after_end` - Flag Live After End
* * `flag_live_before_launch` - Flag Live Before Launch
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
* * `no_metric` - No Metric
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
@@ -44844,6 +44845,7 @@ export namespace Schemas {
FlagLiveAfterEnd: 'flag_live_after_end',
FlagLiveBeforeLaunch: 'flag_live_before_launch',
BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',
+ MultipleVariantsShare: 'multiple_variants_share',
NoMetric: 'no_metric',
Srm: 'srm',
ZeroExposures: 'zero_exposures',
@@ -44905,6 +44907,7 @@ export namespace Schemas {
* * `flag_live_after_end` - Flag Live After End
* * `flag_live_before_launch` - Flag Live Before Launch
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
* * `no_metric` - No Metric
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
@@ -45212,6 +45215,7 @@ export namespace Schemas {
ZeroExposures: 'zero_exposures',
Srm: 'srm',
BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',
+ MultipleVariantsShare: 'multiple_variants_share',
} as const;
export type ExperimentExposureHealthFindingSeverity = typeof ExperimentExposureHealthFindingSeverity[keyof typeof ExperimentExposureHealthFindingSeverity];