:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): note significance on a small sample (#121972)
Marcel Poelkera45cdd34498d
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex 622b902f992b..5f2fb63924b2 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -3324,6 +3324,10 @@ snapshots:hash: v1.k794b7964.8fc4d4ec055de978ea383b15138251df99dafb87f0713218485a74da67b31634.XBTJmRJvnfs_9gv6zfC10-5JQAAGTRfTctWZc7T852sexperiments-healthchecksoverview--draft-with-events-not-received--light:hash: v1.k794b7964.207d3f925f5972dbbc0825b2d7f6e9026c362c4b513b2db68b70303715bcd866.J3uCNeI2NrU5MV9AWMYFCI4ftj9WCM-4d0Vvjp_7K-k+ experiments-healthchecksoverview--running-with-significance-on-a-small-sample--dark:+ hash: v1.k794b7964.4b8d903f39c6569a954ff966fae2d7e305b3a136578aed58d9205dabe5739c26.jglF4ElvqVB73DiCtuc3bZAv5ESatCaxg3Z1LU9Q-iQ+ experiments-healthchecksoverview--running-with-significance-on-a-small-sample--light:+ hash: v1.k794b7964.d8d89699c9a340a82b74fb99e9c6b890facb29d2c7cad15fcdf251d9e70a2854.G1q2SJtAvjRnHoJIEAwyTQfa0W5DT8KTI25oVpdesI8experiments-howtoreadtooltip--intervals-bayesian--dark:hash: v1.k794b7964.a06d267cee9cb54bafc732335b35d461d3c1a59250c6daa8becace0fcb8492d2.vR5tecOozn6B51xE8LrByAFVsi03J4w-iUu-j2hQUTUexperiments-howtoreadtooltip--intervals-bayesian--light:@@ -10923,9 +10927,9 @@ snapshots:scenes-app-experiments--experiment-frequentist-two-variants--light:hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmIIscenes-app-experiments--experiment-health-debug-tab--dark:- hash: v1.k794b7964.91ba56f50391aab86778f2933317b208ffac976d4ec78239c0a09d724a61b789.ftlKzczALPofIwcDj3S9xS5DI8IZ-VPywrNrFPJw8ow+ hash: v1.k794b7964.df03873a81d660b11a8fbabb7187ae6a8ac51b16fa9e984530b7387478ebdb04.3i56n6We4-RK73B4ma2ipXHZH3gP0KdCGmWzCdq-S5Escenes-app-experiments--experiment-health-debug-tab--light:- hash: v1.k794b7964.ce981e71594609f8b54d3f248837db9ec8ed3352af2bdd8f910f1711e6030a2f.oQwgMeJDgn3NeL_yaRCldOooFmYBBYz0PNVbQvA5STg+ hash: v1.k794b7964.4f995cb939700d5c7474bf9fa87e2685ef5d2b481f82ba25bb3b23d8d9917129.qCBrumZsUJSGWp9aXR3Qqau-QRbmFgUJ7DbV1ELWrnIscenes-app-experiments--experiment-recordings-dropped-metric--dark:hash: v1.k794b7964.e0e7181f2c9e61a8324eff9aa7a9230fef25b3cf37e0b24ef875780186ed25dc.9VEvZ4iVieVJwmDol0gwtAsr3wgZYW5gj2KXAGZJfiwscenes-app-experiments--experiment-recordings-dropped-metric--light:diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex 6aabca591b87..3f2af054c42c 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -9767,6 +9767,7 @@ export namespace Schemas {* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run* * `new_flag_no_traffic` - New Flag No Traffic+ * * `early_significance` - Early Significance*/export type ExperimentHealthFindingCodeEnum =| "flag_off_while_running"@@ -9785,7 +9786,8 @@ export namespace Schemas {| "split_changed_mid_run"| "bucketing_changed_mid_run"| "release_conditions_changed_mid_run"- | "new_flag_no_traffic";+ | "new_flag_no_traffic"+ | "early_significance";/*** * `critical` - Critical severity* * `warning` - Warning severity@@ -9835,6 +9837,7 @@ export namespace Schemas {* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run* * `new_flag_no_traffic` - New Flag No Traffic+ * * `early_significance` - Early Significance*/code: ExperimentHealthFindingCodeEnum;/**@@ -9882,6 +9885,7 @@ export namespace Schemas {* * `no_metric` - No Metric* * `metric_event_not_ingested` - Metric Event Not Ingested* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `early_significance` - Early Significance* * `zero_exposures` - Zero Exposures* * `srm` - Srm* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded@@ -9899,6 +9903,7 @@ export namespace Schemas {| "no_metric"| "metric_event_not_ingested"| "exposure_criteria_cannot_match"+ | "early_significance"| "zero_exposures"| "srm"| "bias_risk_multiple_excluded"@@ -9929,6 +9934,7 @@ export namespace Schemas {* * `no_metric` - No Metric* * `metric_event_not_ingested` - Metric Event Not Ingested* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `early_significance` - Early Significance* * `zero_exposures` - Zero Exposures* * `srm` - Srm* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excludeddiff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.pyindex f24127b41a74..9d4137016f1a 100644--- a/products/experiments/backend/facade/contracts.py+++ b/products/experiments/backend/facade/contracts.py@@ -115,6 +115,7 @@ class ExperimentHealthFindingCode(LabeledStrEnum):BUCKETING_CHANGED_MID_RUN = "bucketing_changed_mid_run"RELEASE_CONDITIONS_CHANGED_MID_RUN = "release_conditions_changed_mid_run"NEW_FLAG_NO_TRAFFIC = "new_flag_no_traffic"+ EARLY_SIGNIFICANCE = "early_significance"class HealthExposureInputsStatus(LabeledStrEnum):@@ -197,6 +198,7 @@ class ExperimentHealthCheckKey(LabeledStrEnum):NO_METRIC = "no_metric"METRIC_EVENT_NOT_INGESTED = "metric_event_not_ingested"EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"+ EARLY_SIGNIFICANCE = "early_significance"ZERO_EXPOSURES = "zero_exposures"SRM = "srm"BIAS_RISK_MULTIPLE_EXCLUDED = "bias_risk_multiple_excluded"diff --git a/products/experiments/backend/health/checks/early_significance.py b/products/experiments/backend/health/checks/early_significance.pynew file mode 100644index 000000000000..85fa4f578fd4--- /dev/null+++ b/products/experiments/backend/health/checks/early_significance.py@@ -0,0 +1,145 @@+"""+A significant result that is smaller than the effect the sample can detect, before the sample reaches the plan. Pure+functions, no I/O.++On a small sample, a difference that reaches significance by chance is larger than the real one, so such a result often+shrinks or goes away as more users arrive. A large difference is reliable on a small sample too, so the check compares+the observed difference with the smallest one the sample can detect, by the running-time estimate's formula.+"""++from posthog.dataclasses import frozen++from products.experiments.backend.facade.contracts import (+ ExperimentHealthFinding,+ ExperimentHealthFindingCode,+ ExperimentHealthFindingSeverity,+)+from products.experiments.backend.health.context import HealthContext+from products.experiments.backend.health.skips import (+ NOT_AFTER_END,+ NOT_ON_ARCHIVED,+ RUNS_AFTER_LAUNCH,+ CheckSkip,+ not_applicable,+ not_checked,+)+from products.experiments.backend.logic.result_stats import MetricResultStats, VariantStats+from products.experiments.backend.running_time_calculator import calculate_detectable_effect, calculate_sample_size++# pinned: sent as `finding_variant` on the health finding events.+BELOW_DETECTABLE_EFFECT = "below_detectable_effect"++# The results page shows results from 50 exposures (`EXPERIMENT_MIN_EXPOSURES_FOR_RESULTS` in+# products/experiments/frontend/constants.ts). Keep the two in step, so that the note never names a result the page hides.+MIN_SAMPLES_FOR_RESULTS = 50+++@frozen+class _Comparison:+ variant: VariantStats+ # Relative to the control, as percentages.+ observed: float+ detectable: float+++def early_significance_skip(ctx: HealthContext) -> CheckSkip | None:+ if ctx.archived:+ return not_applicable(NOT_ON_ARCHIVED)+ if not ctx.is_launched:+ return not_applicable(RUNS_AFTER_LAUNCH)+ # After the end the decision is made, and "wait for the planned sample" can't be followed.+ if ctx.has_ended:+ return not_applicable(NOT_AFTER_END)+ if ctx.primary_metric_count == 0:+ return not_applicable("Runs once the experiment has a primary metric")+ metric = ctx.primary_metric+ if metric is None:+ return not_checked("Couldn't read the results of the primary metric")+ if not metric.is_funnel:+ return not_applicable("Runs on funnel primary metrics only")+ # The page's target in this mode comes from the user's guessed baseline, so a note about the target could say+ # "wait" after the page says the target is reached.+ if metric.manual_running_time:+ return not_applicable("Runs with the automatic running time estimate")+ if metric.result is None:+ return not_applicable("Runs once the primary metric has results")+ if _total_samples(metric.result) < MIN_SAMPLES_FOR_RESULTS:+ return not_applicable("Runs once enough users are exposed")+ if not 0 < (metric.result.baseline.mean or 0) < 1:+ return not_applicable("Runs once the control group's conversion rate is above 0% and below 100%")+ return None+++def early_significance(ctx: HealthContext) -> ExperimentHealthFinding | None:+ metric = ctx.primary_metric+ if early_significance_skip(ctx) is not None or metric is None or metric.result is None:+ return None+ result = metric.result+ control_rate = result.baseline.mean or 0+ samples = _total_samples(result)+ # The running time estimate counts the variants of the flag.+ number_of_variants = len(ctx.flag.variants) if ctx.flag and ctx.flag.variants else len(result.variants) + 1+ planned_samples = calculate_sample_size(+ "funnel", control_rate, metric.minimum_detectable_effect, number_of_variants+ )+ # The running time estimate's "Target reached" compares the same total, so the note and the estimate agree.+ if planned_samples is None or samples >= planned_samples:+ return None+ below_detectable = [+ comparison+ for comparison in _significant_comparisons(result, control_rate)+ if abs(comparison.observed) < comparison.detectable+ ]+ if not below_detectable:+ return None+ comparison = min(+ below_detectable,+ key=lambda comparison: (abs(comparison.observed) / comparison.detectable, comparison.variant.key),+ )+ return ExperimentHealthFinding(+ code=ExperimentHealthFindingCode.EARLY_SIGNIFICANCE,+ subcode=BELOW_DETECTABLE_EFFECT,+ severity=ExperimentHealthFindingSeverity.INFO,+ title="Significant result on a small sample",+ detail=" ".join(+ [+ f'The variant "{comparison.variant.key}" shows a {comparison.observed:+.0f}% difference on the metric '+ f'"{metric.name}", which is significant.',+ f"With {samples:,} users so far, the experiment can reliably detect differences of about "+ f"{comparison.detectable:.0f}% or more, so this difference may shrink or go away as more users arrive.",+ f"The running time estimate needs about {planned_samples:,} users to detect a "+ f"{metric.minimum_detectable_effect:g}% difference.",+ "Wait for that sample before you decide.",+ ]+ ),+ evidence={+ "variant": comparison.variant.key,+ "observed_effect": round(comparison.observed, 1),+ "detectable_effect": round(comparison.detectable, 1),+ "minimum_detectable_effect": metric.minimum_detectable_effect,+ "samples": samples,+ "planned_samples": planned_samples,+ },+ actions=(),+ diagnostic_ref=None,+ )+++def _significant_comparisons(result: MetricResultStats, control_rate: float) -> list[_Comparison]:+ comparisons: list[_Comparison] = []+ for variant in result.variants:+ if variant.significant is not True or variant.interval is None:+ continue+ detectable = calculate_detectable_effect(+ "funnel", control_rate, result.baseline.number_of_samples, variant.number_of_samples+ )+ if detectable is not None:+ # The difference the results page shows, from the analysis itself, so CUPED and the Bayesian model apply.+ comparisons.append(+ _Comparison(variant=variant, observed=100 * variant.interval.midpoint, detectable=detectable)+ )+ return comparisons+++def _total_samples(result: MetricResultStats) -> int:+ return result.baseline.number_of_samples + sum(variant.number_of_samples for variant in result.variants)diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex c2008a737739..67b41721f910 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -28,6 +28,7 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (has_activation_config,normalize_to_exposure_criteria,)+from products.experiments.backend.logic.result_stats import MetricResultStatsfrom products.experiments.backend.metric_resolution import MetricRole, saved_metric_link_role, saved_metric_linksfrom products.experiments.backend.metric_utils import get_default_metric_titlefrom products.experiments.backend.metric_validation import defined_event_names, extract_entity_nodes@@ -197,6 +198,22 @@ class PauseHistory:paused_at: datetime | None+@frozen+class PrimaryMetric:+ """The experiment's first primary metric, which its running-time estimate sizes, with its stored result."""++ name: str+ is_funnel: bool+ # The relative effect, as a percentage, that the experiment plans to detect.+ minimum_detectable_effect: float+ # Whether the running time estimate is in manual mode, where its target comes from a baseline the user entered+ # instead of from the results.+ manual_running_time: bool+ # The current stored result, the one the results page shows after its last calculation. None when no completed+ # result is stored under the metric's current calculation key. Only read for a funnel.+ result: MetricResultStats | None++@frozenclass ExposureTotals:total_exposures: Mapping[str, int]@@ -283,6 +300,9 @@ class HealthContext:condition_set_traffic: ConditionSetTraffic | None# None when not loaded or unknown. Only the health endpoint loads it, and only for a paused experiment.pause_history: PauseHistory | None+ # None when not loaded, unknown, or the experiment calculates no primary metric. Only the health endpoint loads it,+ # and only for a launched experiment.+ primary_metric: PrimaryMetric | None# The variants that the analysis leaves out of the results.excluded_variant_keys: frozenset[str]@@ -567,5 +587,6 @@ def load_health_context(flag_history=flag_history,condition_set_traffic=None,pause_history=None,+ primary_metric=None,excluded_variant_keys=frozenset(experiment.excluded_variants or []),)diff --git a/products/experiments/backend/health/loader.py b/products/experiments/backend/health/loader.pyindex c894b7fb5b3f..c82f70bc7f83 100644--- a/products/experiments/backend/health/loader.py+++ b/products/experiments/backend/health/loader.py@@ -10,6 +10,7 @@ from products.experiments.backend.health.context import HealthContext, load_healfrom products.experiments.backend.health.exposure_gate import load_flag_historyfrom products.experiments.backend.health.exposure_inputs import ExposureInputs, load_exposure_inputsfrom products.experiments.backend.health.pause_history import load_pause_history+from products.experiments.backend.health.primary_metric import load_primary_metricfrom products.experiments.backend.models.experiment import Experimentfrom products.experiments.backend.setup_context import ProjectSetupFacts, build_project_setup_facts@@ -71,4 +72,5 @@ def _health_context(experiment: Experiment, reader_access: ExperimentHealthReadecontext,condition_set_traffic=traffic,pause_history=load_pause_history(experiment, reader_access=reader_access),+ primary_metric=load_primary_metric(experiment),)diff --git a/products/experiments/backend/health/primary_metric.py b/products/experiments/backend/health/primary_metric.pynew file mode 100644index 000000000000..4b49191d9bda--- /dev/null+++ b/products/experiments/backend/health/primary_metric.py@@ -0,0 +1,76 @@+from posthog.exceptions_capture import capture_exception++from products.experiments.backend.health.context import PrimaryMetric+from products.experiments.backend.metric_calculation.config import (+ MetricCalculationConfig,+ build_primary_calculation_configs,+ team_experiments_configs,+)+from products.experiments.backend.metric_calculation.results import MetricResultStore+from products.experiments.backend.metric_resolution import saved_metric_links+from products.experiments.backend.metric_utils import get_default_metric_title+from products.experiments.backend.models.experiment import Experiment+from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig+from products.experiments.backend.setup_context import PRODUCT_DEFAULT_MINIMUM_DETECTABLE_EFFECT+++def load_primary_metric(experiment: Experiment) -> PrimaryMetric | None:+ """+ The first primary metric of a launched experiment with its current stored result, or None. Never raises: the+ checks that read it then report that they couldn't check.+ """+ if not experiment.is_launched:+ return None+ try:+ return _load_primary_metric(experiment)+ except Exception as error:+ capture_exception(error)+ return None+++def _load_primary_metric(experiment: Experiment) -> PrimaryMetric | None:+ team_config = team_experiments_configs([experiment.team_id])[experiment.team_id]+ calculation_config = next(iter(build_primary_calculation_configs(experiment, team_config=team_config)), None)+ if calculation_config is None:+ return None+ is_funnel = calculation_config.definition.get("metric_type") == "funnel"+ return PrimaryMetric(+ name=_metric_name(experiment, calculation_config),+ is_funnel=is_funnel,+ minimum_detectable_effect=_minimum_detectable_effect(experiment, team_config),+ manual_running_time=_manual_running_time(experiment),+ result=MetricResultStore(experiment_id=experiment.id).current_result(calculation_config) if is_funnel else None,+ )+++def _metric_name(experiment: Experiment, calculation_config: MetricCalculationConfig) -> str:+ definition = calculation_config.definition+ # An inline metric wins over a saved metric with the same uuid, as in `get_metrics_for_calculation`.+ inline_uuids = {metric.get("uuid") for metric in experiment.metrics or [] if isinstance(metric, dict)}+ if calculation_config.metric_id not in inline_uuids:+ for link in saved_metric_links(experiment):+ query = link.saved_metric.query+ if isinstance(query, dict) and query.get("uuid") == calculation_config.metric_id:+ return link.saved_metric.name+ return definition.get("name") or get_default_metric_title(definition)+++def _minimum_detectable_effect(experiment: Experiment, team_config: TeamExperimentsConfig) -> float:+ running_time_calculation = experiment.running_time_calculation+ planned = (+ running_time_calculation.get("minimum_detectable_effect")+ if isinstance(running_time_calculation, dict)+ else None+ )+ if isinstance(planned, int | float) and not isinstance(planned, bool) and planned > 0:+ return float(planned)+ return float(team_config.default_minimum_detectable_effect or PRODUCT_DEFAULT_MINIMUM_DETECTABLE_EFFECT)+++def _manual_running_time(experiment: Experiment) -> bool:+ # pinned: the page's `ConversionRateInputType.MANUAL` in frontend/src/types.ts, read by `runningTimeLogic`.+ running_time_calculation = experiment.running_time_calculation+ estimate_config = (+ running_time_calculation.get("exposure_estimate_config") if isinstance(running_time_calculation, dict) else None+ )+ return isinstance(estimate_config, dict) and estimate_config.get("conversionRateInputType") == "manual"diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.pyindex 7a654adac7b8..dd79db50ef57 100644--- a/products/experiments/backend/health/registry.py+++ b/products/experiments/backend/health/registry.py@@ -13,6 +13,7 @@ from products.experiments.backend.facade.contracts import (from products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded, bias_risk_skipfrom products.experiments.backend.health.checks.bucketing_changed import bucketing_changed_mid_runfrom products.experiments.backend.health.checks.conditions_changed import release_conditions_changed_mid_run+from products.experiments.backend.health.checks.early_significance import early_significance, early_significance_skipfrom products.experiments.backend.health.checks.exposure_criteria import (exposure_criteria_cannot_match,exposure_criteria_skip,@@ -122,6 +123,12 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (find=exposure_criteria_cannot_match,skip=exposure_criteria_skip,),+ HealthCheck(+ key=ExperimentHealthCheckKey.EARLY_SIGNIFICANCE,+ name="Sample size behind significant results",+ find=early_significance,+ skip=early_significance_skip,+ ),)# The exposure query runs these on its answer. They must not include the checks above: the exposure answerdiff --git a/products/experiments/backend/health/test/factories.py b/products/experiments/backend/health/test/factories.pyindex 3a18f9ce2cc1..3be50d362e20 100644--- a/products/experiments/backend/health/test/factories.py+++ b/products/experiments/backend/health/test/factories.py@@ -38,6 +38,7 @@ RUNNING_CONTEXT = HealthContext(flag_history=None,condition_set_traffic=None,pause_history=None,+ primary_metric=None,excluded_variant_keys=frozenset(),)diff --git a/products/experiments/backend/health/test/test_early_significance.py b/products/experiments/backend/health/test/test_early_significance.pynew file mode 100644index 000000000000..f1c42fc8f753--- /dev/null+++ b/products/experiments/backend/health/test/test_early_significance.py@@ -0,0 +1,281 @@+from dataclasses import replace+from datetime import timedelta+from typing import Any++from posthog.test.base import BaseTest+from unittest import TestCase++from django.utils import timezone++from parameterized import parameterized++from products.experiments.backend.facade.api import get_experiment_health+from products.experiments.backend.facade.contracts import (+ ExperimentHealthCheckKey,+ ExperimentHealthCheckStatus,+ ExperimentHealthFindingCode,+ ExperimentHealthFindingSeverity,+ ExperimentHealthReaderAccess,+)+from products.experiments.backend.health.checks.early_significance import (+ BELOW_DETECTABLE_EFFECT,+ early_significance,+ early_significance_skip,+)+from products.experiments.backend.health.context import HealthContext, PrimaryMetric+from products.experiments.backend.health.test.factories import ENDED, RUNNING_CONTEXT+from products.experiments.backend.logic.result_stats import BaselineStats, Interval, MetricResultStats, VariantStats+from products.experiments.backend.metric_calculation.config import build_primary_calculation_configs+from products.experiments.backend.models.experiment import (+ Experiment,+ ExperimentMetricResult,+ ExperimentSavedMetric,+ ExperimentToSavedMetric,+)+from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig+from products.feature_flags.backend.models.feature_flag import FeatureFlag++Status = ExperimentHealthCheckStatus+NO_QUERIES = ExperimentHealthReaderAccess(+ can_read_flag_history=False, can_query_events=False, denied_event_properties=frozenset()+)+FUNNEL = {"kind": "ExperimentMetric", "metric_type": "funnel", "series": [{"kind": "EventsNode", "event": "purchase"}]}+++def _context(+ variants: list[tuple[str, int, float, bool | None]],+ *,+ baseline: tuple[int, int] = (1000, 100),+ manual_running_time: bool = False,+ **changes: Any,+) -> HealthContext:+ control_samples, control_conversions = baseline+ control_rate = control_conversions / control_samples+ result = MetricResultStats(+ method="bayesian",+ baseline=BaselineStats(key="control", number_of_samples=control_samples, mean=control_rate),+ # The raw rate equals the control's, so only the analysis interval carries the difference, as under CUPED.+ variants=tuple(+ VariantStats(+ key=key,+ number_of_samples=samples,+ mean=control_rate,+ interval=Interval(low=difference - 0.15, high=difference + 0.15),+ chance_to_win=None,+ p_value=None,+ significant=significant,+ )+ for key, samples, difference, significant in variants+ ),+ )+ metric = PrimaryMetric(+ name="Checkout",+ is_funnel=True,+ minimum_detectable_effect=30,+ manual_running_time=manual_running_time,+ result=result,+ )+ return replace(RUNNING_CONTEXT, primary_metric=metric, **changes)+++# A 10% control rate on 1,000 users per variant detects differences of about 38% or more.+SMALL_SIGNIFICANT_LIFT = _context([("test", 1000, 0.2, True)])+++class TestEarlySignificance(TestCase):+ @parameterized.expand(+ [+ ("significant_and_smaller_than_detectable", SMALL_SIGNIFICANT_LIFT, "test"),+ ("significant_and_larger_than_detectable", _context([("test", 1000, -0.5, True)]), None),+ ("not_significant", _context([("test", 1000, 0.2, False)]), None),+ ("analysis_stopped", _context([("test", 1000, 0.2, None)]), None),+ ("sample_reached_the_plan", _context([("test", 20000, 0.06, True)], baseline=(20000, 2000)), None),+ (+ "uneven_split_past_the_planned_total",+ _context([("test", 500, 0.34, True)], baseline=(9000, 900)),+ None,+ ),+ (+ "names_the_variant_below_detectable",+ _context([("large", 1000, 0.5, True), ("small", 1000, 0.2, True)]),+ "small",+ ),+ ]+ )+ def test_notes_a_significant_difference_the_sample_cannot_detect(+ self, _name: str, ctx: HealthContext, expected_variant: str | None+ ) -> None:+ finding = early_significance(ctx)++ self.assertEqual(finding.evidence["variant"] if finding else None, expected_variant)++ def test_states_the_analysis_difference_and_the_planned_sizes(self) -> None:+ finding = early_significance(SMALL_SIGNIFICANT_LIFT)++ assert finding is not None+ self.assertEqual(+ (finding.code, finding.subcode, finding.severity),+ (+ ExperimentHealthFindingCode.EARLY_SIGNIFICANCE,+ BELOW_DETECTABLE_EFFECT,+ ExperimentHealthFindingSeverity.INFO,+ ),+ )+ self.assertEqual(+ finding.evidence,+ {+ "variant": "test",+ "observed_effect": 20.0,+ "detectable_effect": 37.9,+ "minimum_detectable_effect": 30,+ "samples": 2000,+ "planned_samples": 3201,+ },+ )++ @parameterized.expand(+ [+ ("ended", replace(SMALL_SIGNIFICANT_LIFT, **ENDED), Status.NOT_APPLICABLE),+ ("unreadable_results", replace(SMALL_SIGNIFICANT_LIFT, primary_metric=None), Status.NOT_CHECKED),+ (+ "mean_metric",+ replace(+ SMALL_SIGNIFICANT_LIFT,+ primary_metric=PrimaryMetric(+ name="Revenue",+ is_funnel=False,+ minimum_detectable_effect=30,+ manual_running_time=False,+ result=None,+ ),+ ),+ Status.NOT_APPLICABLE,+ ),+ (+ "manual_running_time",+ _context([("test", 1000, 0.2, True)], manual_running_time=True),+ Status.NOT_APPLICABLE,+ ),+ ("under_50_users", _context([("test", 20, 0.2, True)], baseline=(20, 2)), Status.NOT_APPLICABLE),+ (+ "no_control_conversions",+ _context([("test", 1000, 0.2, True)], baseline=(1000, 0)),+ Status.NOT_APPLICABLE,+ ),+ ]+ )+ def test_says_why_it_did_not_look(self, _name: str, ctx: HealthContext, expected: Status) -> None:+ skip = early_significance_skip(ctx)++ self.assertEqual(skip.status if skip else None, expected)+ self.assertIsNone(early_significance(ctx))+++class TestEarlySignificanceFromStoredResults(BaseTest):+ @parameterized.expand(+ [+ (+ "inline_metric_and_the_experiments_effect",+ False,+ True,+ {"minimum_detectable_effect": 20},+ 15,+ Status.FINDING,+ 20,+ ),+ ("inline_metric_and_the_teams_default_effect", False, True, {}, 15, Status.FINDING, 15),+ ("inline_metric_and_the_product_default_effect", False, True, {}, None, Status.FINDING, 30),+ ("shared_metric", True, True, {}, None, Status.FINDING, 30),+ ("result_of_an_earlier_configuration", False, False, {}, None, Status.NOT_APPLICABLE, None),+ (+ "manual_running_time",+ False,+ True,+ {"exposure_estimate_config": {"conversionRateInputType": "manual"}},+ None,+ Status.NOT_APPLICABLE,+ None,+ ),+ ]+ )+ def test_reads_the_primary_metrics_current_result(+ self,+ _name: str,+ shared: bool,+ under_current_key: bool,+ running_time_calculation: dict[str, Any],+ team_effect: int | None,+ expected: Status,+ expected_effect: int | None,+ ) -> None:+ if team_effect is not None:+ TeamExperimentsConfig.objects.create(team=self.team, default_minimum_detectable_effect=team_effect)+ flag = FeatureFlag.objects.create(+ team=self.team,+ key="checkout-flag",+ created_by=self.user,+ filters={+ "groups": [{"properties": [], "rollout_percentage": 100}],+ "multivariate": {+ "variants": [+ {"key": "control", "rollout_percentage": 50},+ {"key": "test", "rollout_percentage": 50},+ ]+ },+ },+ )+ launched_at = timezone.now() - timedelta(days=3)+ experiment = Experiment.objects.create(+ team=self.team,+ name="checkout",+ feature_flag=flag,+ start_date=launched_at,+ metrics=[] if shared else [{**FUNNEL, "uuid": "checkout-funnel"}],+ running_time_calculation=running_time_calculation,+ )+ if shared:+ saved_metric = ExperimentSavedMetric.objects.create(+ team=self.team, name="Checkout funnel", query={**FUNNEL, "uuid": "checkout-funnel"}+ )+ ExperimentToSavedMetric.objects.create(+ experiment=experiment, saved_metric=saved_metric, metadata={"type": "primary"}+ )+ calculation_key = build_primary_calculation_configs(experiment)[0].calculation_key()+ ExperimentMetricResult.objects.create(+ experiment=experiment,+ metric_uuid="checkout-funnel",+ fingerprint=calculation_key if under_current_key else "0" * 64,+ query_from=launched_at,+ query_to=timezone.now(),+ completed_at=timezone.now(),+ status=ExperimentMetricResult.Status.COMPLETED,+ result={+ "baseline": {"key": "control", "number_of_samples": 1000, "sum": 100, "sum_squares": 100},+ "variant_results": [+ {+ "key": "test",+ "method": "bayesian",+ "number_of_samples": 1000,+ "sum": 120,+ "sum_squares": 120,+ "credible_interval": [0.05, 0.35],+ "chance_to_win": 0.98,+ "significant": True,+ }+ ],+ },+ )++ health = get_experiment_health(team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES)++ check = next(check for check in health.checks if check.key == ExperimentHealthCheckKey.EARLY_SIGNIFICANCE)+ finding = next(+ (finding for finding in health.findings if finding.code == ExperimentHealthFindingCode.EARLY_SIGNIFICANCE),+ None,+ )+ self.assertEqual(+ (check.status, finding.evidence["minimum_detectable_effect"] if finding else None),+ (expected, expected_effect),+ )+ if finding:+ self.assertIn('"Checkout funnel"' if shared else '"purchase conversion"', finding.detail)diff --git a/products/experiments/backend/health/test/test_registry.py b/products/experiments/backend/health/test/test_registry.pyindex 52b61655e243..3fffbba109b6 100644--- a/products/experiments/backend/health/test/test_registry.py+++ b/products/experiments/backend/health/test/test_registry.py@@ -212,6 +212,11 @@ class TestEvaluate(TestCase):{"flag_called": False},{Key.NEW_FLAG_NO_TRAFFIC: Status.NOT_APPLICABLE},),+ (+ "early_significance_needs_the_primary_metrics_results",+ {"primary_metric": None},+ {Key.EARLY_SIGNIFICANCE: Status.NOT_CHECKED},+ ),("zero_exposures_finds_a_day_after_launch",{"exposures": replace(NO_EXPOSURES, hours_since_launch=24)},diff --git a/products/experiments/backend/logic/result_stats.py b/products/experiments/backend/logic/result_stats.pyindex 3f77efb0be2e..c0863b67bc7b 100644--- a/products/experiments/backend/logic/result_stats.py+++ b/products/experiments/backend/logic/result_stats.py@@ -65,6 +65,11 @@ class Interval:low: floathigh: float+ @property+ def midpoint(self) -> float:+ # The results page shows this as the variant's difference (`getDelta` in MetricsView/shared/utils.ts).+ return (self.low + self.high) / 2+@frozenclass BaselineStats:diff --git a/products/experiments/backend/metric_calculation/results.py b/products/experiments/backend/metric_calculation/results.pyindex 8b6ddafebce2..24f592121c18 100644--- a/products/experiments/backend/metric_calculation/results.py+++ b/products/experiments/backend/metric_calculation/results.py@@ -38,6 +38,7 @@ import structlogfrom posthog.dataclasses import frozen+from products.experiments.backend.logic.result_stats import MetricResultStatsfrom products.experiments.backend.metric_calculation.config import MetricCalculationConfig, build_calculation_configsfrom products.experiments.backend.models.experiment import (Experiment,@@ -184,6 +185,11 @@ class MetricResultStore:"""The current result of the metric that `calculation_config` describes. `current_outcomes` defines the rule."""return self.current_outcomes({self.experiment_id: calculation_config}).get(self.experiment_id)+ def current_result(self, calculation_config: MetricCalculationConfig) -> MetricResultStats | None:+ """The statistics of the result that `current_outcome` picks, or None when it has none or doesn't parse."""+ stored = self._current_rows({self.experiment_id: calculation_config}).values_list("result", flat=True).first()+ return MetricResultStats.from_stored(stored)+def previous_completed(self, metric_uuid: str, calculation_key: str, *, before: datetime) -> ExperimentMetricResult | None:@@ -414,19 +420,8 @@ class MetricResultStore:return newest_point + _SYNC_COPY_OFFSET@staticmethod- def current_outcomes(config_by_experiment: Mapping[int, MetricCalculationConfig]) -> dict[int, ResultSummary]:- """For each experiment, the current result of the metric that its calculation config describes, in one query.-- The current result is the completed row whose data ends last among the rows filed under the config's- calculation key, by a recalculation run or by the daily workflows. A row's data ends at its query_to, or at- its completed_at when that comes first. A backfill stamps the current day with the end of that day, but the- row holds data only up to the moment the backfill ran, so a later run of the same day must win over it. A row- under another key was computed from another configuration, for example from the start date before a- relaunch, so it does not count. The end of the data counts, not the newest write, because a backfill writes- past windows after the newer ones.- """- if not config_by_experiment:- return {}+ def _current_rows(config_by_experiment: Mapping[int, MetricCalculationConfig]) -> QuerySet[ExperimentMetricResult]:+ """For each experiment, the row that `current_outcomes` describes."""filed_under_config = Q()for experiment_id, calculation_config in config_by_experiment.items():key = calculation_config.calculation_key()@@ -435,21 +430,36 @@ class MetricResultStore:metric_uuid=calculation_config.metric_id,fingerprint__in=[key, _recalc_fingerprint(key)],)- rows = (+ return (ExperimentMetricResult.objects.filter(filed_under_config, status=_COMPLETED)# Postgres LEAST skips a NULL, so a row without completed_at ranks by its query_to..alias(data_until=Least("query_to", "completed_at")).order_by("experiment_id", F("data_until").desc(), F("completed_at").desc(nulls_last=True), "-id").distinct("experiment_id")- .values(- "experiment_id",- "metric_uuid",- "completed_at",- "query_to",- baseline_samples=KT("result__baseline__number_of_samples"),- baseline_sum=KT("result__baseline__sum"),- variant_results=KeyTransform("variant_results", "result"),- )+ )++ @staticmethod+ def current_outcomes(config_by_experiment: Mapping[int, MetricCalculationConfig]) -> dict[int, ResultSummary]:+ """For each experiment, the current result of the metric that its calculation config describes, in one query.++ The current result is the completed row whose data ends last among the rows filed under the config's+ calculation key, by a recalculation run or by the daily workflows. A row's data ends at its query_to, or at+ its completed_at when that comes first. A backfill stamps the current day with the end of that day, but the+ row holds data only up to the moment the backfill ran, so a later run of the same day must win over it. A row+ under another key was computed from another configuration, for example from the start date before a+ relaunch, so it does not count. The end of the data counts, not the newest write, because a backfill writes+ past windows after the newer ones.+ """+ if not config_by_experiment:+ return {}+ rows = MetricResultStore._current_rows(config_by_experiment).values(+ "experiment_id",+ "metric_uuid",+ "completed_at",+ "query_to",+ baseline_samples=KT("result__baseline__number_of_samples"),+ baseline_sum=KT("result__baseline__sum"),+ variant_results=KeyTransform("variant_results", "result"),)summaries: dict[int, ResultSummary] = {}for row in rows:diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.pyindex 29efc40102f9..f98bac419cef 100644--- a/products/experiments/backend/presentation/views.py+++ b/products/experiments/backend/presentation/views.py@@ -1006,6 +1006,13 @@ class EnterpriseExperimentsViewSet(`running_but_flag_disabled`, as a warning. A caller who can't view the linked flag'shistory gets the warning, because the pause is not read. A draft whose flag is alreadyactive with multiple variants gets `not_started_but_multiple_variants_rolled_out` as info.++ A running experiment whose first primary metric is a funnel gets the info+ `early_significance` (subcode `below_detectable_effect`) when a significant variant's+ difference is smaller than the smallest difference its sample can detect, while that+ sample is below the one the running time estimate plans for the minimum detectable+ effect. It reads the metric's current stored result, the one the results page shows+ after its last calculation, and the running time calculator's formula."""experiment: Experiment = self.get_object()health = get_experiment_health(diff --git a/products/experiments/backend/running_time_calculator.py b/products/experiments/backend/running_time_calculator.pyindex 69ef403665ff..6503a847860f 100644--- a/products/experiments/backend/running_time_calculator.py+++ b/products/experiments/backend/running_time_calculator.py@@ -172,6 +172,37 @@ def calculate_baseline_value(baseline: BaselineStats, metric_type: CalculatorMetreturn None+def calculate_detectable_effect(+ metric_type: CalculatorMetricType,+ baseline_value: float,+ control_samples: int,+ variant_samples: int,+ variance: float | None = None,+) -> float | None:+ """The smallest relative effect, as a percentage, that the sample-size formula detects between a control and a+ variant of these sizes.++ The inverse of :func:`calculate_sample_size`: for two variants of ``n`` samples each, the sample size for the+ returned effect is about ``2 · n``. The formula takes variants of equal size, so ``1/n_control + 1/n_variant``+ replaces its ``2/n`` for an uneven split.+ """+ if control_samples <= 0 or variant_samples <= 0 or baseline_value == 0:+ return None++ if metric_type == "funnel":+ # A conversion rate of 0 or 1 has no binomial variance, so no effect size follows from it.+ if not 0 < baseline_value < 1:+ return None+ variance = baseline_value * (1 - baseline_value)+ elif variance is None:+ variance = calculate_variance(metric_type, baseline_value)+ if variance is None or variance <= 0:+ return None++ d = math.sqrt(SAMPLE_SIZE_Z_FACTOR / 2 * variance * (1 / control_samples + 1 / variant_samples))+ return 100 * d / abs(baseline_value)++def calculate_running_time_days(sample_size: int | None, exposure_rate_per_day: float | None) -> int | None:"""Days to reach ``sample_size`` at ``exposure_rate_per_day`` exposures/day."""if not sample_size or not exposure_rate_per_day or exposure_rate_per_day <= 0:diff --git a/products/experiments/backend/test/test_running_time_calculator.py b/products/experiments/backend/test/test_running_time_calculator.pyindex 0cb87bb173a8..10ff6440539d 100644--- a/products/experiments/backend/test/test_running_time_calculator.py+++ b/products/experiments/backend/test/test_running_time_calculator.py@@ -1,10 +1,13 @@import math+import pytest+from parameterized import parameterizedfrom products.experiments.backend.running_time_calculator import (BaselineStats,calculate_baseline_value,+ calculate_detectable_effect,calculate_running_time_days,calculate_sample_size,calculate_variance,@@ -173,6 +176,25 @@ class TestCalculateSampleSize:assert calculate_sample_size("ratio", 10, 10, 2, variance=-32) is None+class TestCalculateDetectableEffect:+ @parameterized.expand(+ [+ ("mean_count", 4, 5),+ ("mean_sum_or_avg", 50, 5),+ ("funnel", 0.1, 50),+ ("funnel", 0.03, 30),+ ]+ )+ def test_inverts_the_sample_size(self, metric_type, baseline_value, mde):+ sample_size = calculate_sample_size(metric_type, baseline_value, mde, 2)+ assert sample_size is not None+ samples_per_variant = sample_size // 2++ detectable = calculate_detectable_effect(metric_type, baseline_value, samples_per_variant, samples_per_variant)++ assert detectable == pytest.approx(mde, rel=0.01)++class TestCalculateRunningTimeDays:@parameterized.expand([diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.tsindex 58ef181d5a50..43739b79959a 100644--- a/products/experiments/frontend/generated/api.schemas.ts+++ b/products/experiments/frontend/generated/api.schemas.ts@@ -2288,6 +2288,7 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run* * `new_flag_no_traffic` - New Flag No Traffic+ * * `early_significance` - Early Significance*/export type ExperimentHealthFindingCodeEnumApi =(typeof ExperimentHealthFindingCodeEnumApi)[keyof typeof ExperimentHealthFindingCodeEnumApi]@@ -2310,6 +2311,7 @@ export const ExperimentHealthFindingCodeEnumApi = {BucketingChangedMidRun: 'bucketing_changed_mid_run',ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',NewFlagNoTraffic: 'new_flag_no_traffic',+ EarlySignificance: 'early_significance',} as const/**@@ -2374,7 +2376,8 @@ export interface ExperimentHealthFindingApi {* * `split_changed_mid_run` - Split Changed Mid Run* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run- * * `new_flag_no_traffic` - New Flag No Traffic */+ * * `new_flag_no_traffic` - New Flag No Traffic+ * * `early_significance` - Early Significance */code: ExperimentHealthFindingCodeEnumApi/*** The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.@@ -2414,6 +2417,7 @@ export interface ExperimentHealthFindingApi {* * `no_metric` - No Metric* * `metric_event_not_ingested` - Metric Event Not Ingested* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `early_significance` - Early Significance* * `zero_exposures` - Zero Exposures* * `srm` - Srm* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded@@ -2434,6 +2438,7 @@ export const ExperimentHealthCheckKeyEnumApi = {NoMetric: 'no_metric',MetricEventNotIngested: 'metric_event_not_ingested',ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',+ EarlySignificance: 'early_significance',ZeroExposures: 'zero_exposures',Srm: 'srm',BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',@@ -2470,6 +2475,7 @@ export interface ExperimentHealthCheckResultApi {* * `no_metric` - No Metric* * `metric_event_not_ingested` - Metric Event Not Ingested* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `early_significance` - Early Significance* * `zero_exposures` - Zero Exposures* * `srm` - Srm* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excludeddiff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.tsindex 8692d7dec371..2d2e04a74204 100644--- a/products/experiments/frontend/generated/api.ts+++ b/products/experiments/frontend/generated/api.ts@@ -756,6 +756,13 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =* `running_but_flag_disabled`, as a warning. A caller who can't view the linked flag's* history gets the warning, because the pause is not read. A draft whose flag is already* active with multiple variants gets `not_started_but_multiple_variants_rolled_out` as info.+ *+ * A running experiment whose first primary metric is a funnel gets the info+ * `early_significance` (subcode `below_detectable_effect`) when a significant variant's+ * difference is smaller than the smallest difference its sample can detect, while that+ * sample is below the one the running time estimate plans for the minimum detectable+ * effect. It reads the metric's current stored result, the one the results page shows+ * after its last calculation, and the running time calculator's formula.*/export const experimentsHealthRetrieve = async (projectId: string,diff --git a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx b/products/experiments/frontend/health/HealthChecksOverview.stories.tsxindex 8f52d36e0965..57b088b9a8b6 100644--- a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx+++ b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx@@ -108,6 +108,36 @@ export const DraftWithEventsNotReceived: Story = {},}+export const RunningWithSignificanceOnASmallSample: Story = {+ args: {+ checks: [+ ...FLAG_SETUP_CHECKS,+ passed('no_metric', 'Metrics'),+ found('early_significance', 'Sample size behind significant results'),+ ],+ notes: [+ {+ code: 'early_significance',+ subcode: 'below_detectable_effect',+ severity: 'info',+ title: 'Significant result on a small sample',+ detail: 'The variant "test" shows a +20% difference on the metric "Checkout", which is significant. With 2,000 users so far, the experiment can reliably detect differences of about 38% or more, so this difference may shrink or go away as more users arrive. The running time estimate needs about 3,201 users to detect a 30% difference. Wait for that sample before you decide.',+ evidence: {+ variant: 'test',+ observed_effect: 20,+ detectable_effect: 37.9,+ minimum_detectable_effect: 30,+ samples: 2000,+ planned_samples: 3201,+ },+ actions: [],+ },+ ],+ isDraft: false,+ exposureChecksMissing: false,+ },+}+export const ChecksThatCouldNotRun: Story = {args: {checks: [diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.tsindex 50b0c1382259..bf0f6cf6a974 100644--- a/products/experiments/frontend/health/debug/healthDebugReport.ts+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts@@ -333,6 +333,7 @@ export function buildHealthDebugChecks(input: HealthDebugInput): HealthDebugChecnoMetricCheck(input),serverOnlyCheck(input, 'metric_event_not_ingested', metricEventNote, ['metrics', 'project']),serverOnlyCheck(input, 'exposure_criteria_cannot_match', exposureCriteriaNote, ['experiment', 'project']),+ serverOnlyCheck(input, 'early_significance', null, ['experiment', 'metrics']),...exposureChecks(input),]}diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.tsindex 099d14a6e3bb..73ec2da0fdf1 100644--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts@@ -31,6 +31,7 @@ export type ExperimentHealthFindingCode =| 'bucketing_changed_mid_run'| 'release_conditions_changed_mid_run'| 'new_flag_no_traffic'+ | 'early_significance'export interface ExperimentHealthFinding {code: ExperimentHealthFindingCodediff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex b2513f05e214..953eda674e05 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -45213,6 +45213,7 @@ export namespace Schemas {* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run* * `new_flag_no_traffic` - New Flag No Traffic+ * * `early_significance` - Early Significance*/export type ExperimentHealthFindingCodeEnum = typeof ExperimentHealthFindingCodeEnum[keyof typeof ExperimentHealthFindingCodeEnum];@@ -45235,6 +45236,7 @@ export namespace Schemas {BucketingChangedMidRun: 'bucketing_changed_mid_run',ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',NewFlagNoTraffic: 'new_flag_no_traffic',+ EarlySignificance: 'early_significance',} as const;/**@@ -45299,7 +45301,8 @@ export namespace Schemas {* * `split_changed_mid_run` - Split Changed Mid Run* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run- * * `new_flag_no_traffic` - New Flag No Traffic */+ * * `new_flag_no_traffic` - New Flag No Traffic+ * * `early_significance` - Early Significance */code: ExperimentHealthFindingCodeEnum;/*** The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.@@ -45339,6 +45342,7 @@ export namespace Schemas {* * `no_metric` - No Metric* * `metric_event_not_ingested` - Metric Event Not Ingested* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `early_significance` - Early Significance* * `zero_exposures` - Zero Exposures* * `srm` - Srm* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded@@ -45359,6 +45363,7 @@ export namespace Schemas {NoMetric: 'no_metric',MetricEventNotIngested: 'metric_event_not_ingested',ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',+ EarlySignificance: 'early_significance',ZeroExposures: 'zero_exposures',Srm: 'srm',BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',@@ -45395,6 +45400,7 @@ export namespace Schemas {* * `no_metric` - No Metric* * `metric_event_not_ingested` - Metric Event Not Ingested* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `early_significance` - Early Significance* * `zero_exposures` - Zero Exposures* * `srm` - Srm* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded