Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): note significance on a small sample (#121972)

Marcel Poelkera45cdd34498d
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 622b902f992b..5f2fb63924b2 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -3324,6 +3324,10 @@ snapshots:
hash: v1.k794b7964.8fc4d4ec055de978ea383b15138251df99dafb87f0713218485a74da67b31634.XBTJmRJvnfs_9gv6zfC10-5JQAAGTRfTctWZc7T852s
experiments-healthchecksoverview--draft-with-events-not-received--light:
hash: v1.k794b7964.207d3f925f5972dbbc0825b2d7f6e9026c362c4b513b2db68b70303715bcd866.J3uCNeI2NrU5MV9AWMYFCI4ftj9WCM-4d0Vvjp_7K-k
+ experiments-healthchecksoverview--running-with-significance-on-a-small-sample--dark:
+ hash: v1.k794b7964.4b8d903f39c6569a954ff966fae2d7e305b3a136578aed58d9205dabe5739c26.jglF4ElvqVB73DiCtuc3bZAv5ESatCaxg3Z1LU9Q-iQ
+ experiments-healthchecksoverview--running-with-significance-on-a-small-sample--light:
+ hash: v1.k794b7964.d8d89699c9a340a82b74fb99e9c6b890facb29d2c7cad15fcdf251d9e70a2854.G1q2SJtAvjRnHoJIEAwyTQfa0W5DT8KTI25oVpdesI8
experiments-howtoreadtooltip--intervals-bayesian--dark:
hash: v1.k794b7964.a06d267cee9cb54bafc732335b35d461d3c1a59250c6daa8becace0fcb8492d2.vR5tecOozn6B51xE8LrByAFVsi03J4w-iUu-j2hQUTU
experiments-howtoreadtooltip--intervals-bayesian--light:
@@ -10923,9 +10927,9 @@ snapshots:
scenes-app-experiments--experiment-frequentist-two-variants--light:
hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmII
scenes-app-experiments--experiment-health-debug-tab--dark:
- hash: v1.k794b7964.91ba56f50391aab86778f2933317b208ffac976d4ec78239c0a09d724a61b789.ftlKzczALPofIwcDj3S9xS5DI8IZ-VPywrNrFPJw8ow
+ hash: v1.k794b7964.df03873a81d660b11a8fbabb7187ae6a8ac51b16fa9e984530b7387478ebdb04.3i56n6We4-RK73B4ma2ipXHZH3gP0KdCGmWzCdq-S5E
scenes-app-experiments--experiment-health-debug-tab--light:
- hash: v1.k794b7964.ce981e71594609f8b54d3f248837db9ec8ed3352af2bdd8f910f1711e6030a2f.oQwgMeJDgn3NeL_yaRCldOooFmYBBYz0PNVbQvA5STg
+ hash: v1.k794b7964.4f995cb939700d5c7474bf9fa87e2685ef5d2b481f82ba25bb3b23d8d9917129.qCBrumZsUJSGWp9aXR3Qqau-QRbmFgUJ7DbV1ELWrnI
scenes-app-experiments--experiment-recordings-dropped-metric--dark:
hash: v1.k794b7964.e0e7181f2c9e61a8324eff9aa7a9230fef25b3cf37e0b24ef875780186ed25dc.9VEvZ4iVieVJwmDol0gwtAsr3wgZYW5gj2KXAGZJfiw
scenes-app-experiments--experiment-recordings-dropped-metric--light:
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 6aabca591b87..3f2af054c42c 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -9767,6 +9767,7 @@ export namespace Schemas {
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
* * `new_flag_no_traffic` - New Flag No Traffic
+ * * `early_significance` - Early Significance
*/
export type ExperimentHealthFindingCodeEnum =
| "flag_off_while_running"
@@ -9785,7 +9786,8 @@ export namespace Schemas {
| "split_changed_mid_run"
| "bucketing_changed_mid_run"
| "release_conditions_changed_mid_run"
- | "new_flag_no_traffic";
+ | "new_flag_no_traffic"
+ | "early_significance";
/**
* * `critical` - Critical severity
* * `warning` - Warning severity
@@ -9835,6 +9837,7 @@ export namespace Schemas {
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
* * `new_flag_no_traffic` - New Flag No Traffic
+ * * `early_significance` - Early Significance
*/
code: ExperimentHealthFindingCodeEnum;
/**
@@ -9882,6 +9885,7 @@ export namespace Schemas {
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
+ * * `early_significance` - Early Significance
* * `zero_exposures` - Zero Exposures
* * `srm` - Srm
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
@@ -9899,6 +9903,7 @@ export namespace Schemas {
| "no_metric"
| "metric_event_not_ingested"
| "exposure_criteria_cannot_match"
+ | "early_significance"
| "zero_exposures"
| "srm"
| "bias_risk_multiple_excluded"
@@ -9929,6 +9934,7 @@ export namespace Schemas {
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
+ * * `early_significance` - Early Significance
* * `zero_exposures` - Zero Exposures
* * `srm` - Srm
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index f24127b41a74..9d4137016f1a 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -115,6 +115,7 @@ class ExperimentHealthFindingCode(LabeledStrEnum):
BUCKETING_CHANGED_MID_RUN = "bucketing_changed_mid_run"
RELEASE_CONDITIONS_CHANGED_MID_RUN = "release_conditions_changed_mid_run"
NEW_FLAG_NO_TRAFFIC = "new_flag_no_traffic"
+ EARLY_SIGNIFICANCE = "early_significance"
class HealthExposureInputsStatus(LabeledStrEnum):
@@ -197,6 +198,7 @@ class ExperimentHealthCheckKey(LabeledStrEnum):
NO_METRIC = "no_metric"
METRIC_EVENT_NOT_INGESTED = "metric_event_not_ingested"
EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"
+ EARLY_SIGNIFICANCE = "early_significance"
ZERO_EXPOSURES = "zero_exposures"
SRM = "srm"
BIAS_RISK_MULTIPLE_EXCLUDED = "bias_risk_multiple_excluded"
diff --git a/products/experiments/backend/health/checks/early_significance.py b/products/experiments/backend/health/checks/early_significance.py
new file mode 100644
index 000000000000..85fa4f578fd4
--- /dev/null
+++ b/products/experiments/backend/health/checks/early_significance.py
@@ -0,0 +1,145 @@
+"""
+A significant result that is smaller than the effect the sample can detect, before the sample reaches the plan. Pure
+functions, no I/O.
+
+On a small sample, a difference that reaches significance by chance is larger than the real one, so such a result often
+shrinks or goes away as more users arrive. A large difference is reliable on a small sample too, so the check compares
+the observed difference with the smallest one the sample can detect, by the running-time estimate's formula.
+"""
+
+from posthog.dataclasses import frozen
+
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFinding,
+ ExperimentHealthFindingCode,
+ ExperimentHealthFindingSeverity,
+)
+from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.health.skips import (
+ NOT_AFTER_END,
+ NOT_ON_ARCHIVED,
+ RUNS_AFTER_LAUNCH,
+ CheckSkip,
+ not_applicable,
+ not_checked,
+)
+from products.experiments.backend.logic.result_stats import MetricResultStats, VariantStats
+from products.experiments.backend.running_time_calculator import calculate_detectable_effect, calculate_sample_size
+
+# pinned: sent as `finding_variant` on the health finding events.
+BELOW_DETECTABLE_EFFECT = "below_detectable_effect"
+
+# The results page shows results from 50 exposures (`EXPERIMENT_MIN_EXPOSURES_FOR_RESULTS` in
+# products/experiments/frontend/constants.ts). Keep the two in step, so that the note never names a result the page hides.
+MIN_SAMPLES_FOR_RESULTS = 50
+
+
+@frozen
+class _Comparison:
+ variant: VariantStats
+ # Relative to the control, as percentages.
+ observed: float
+ detectable: float
+
+
+def early_significance_skip(ctx: HealthContext) -> CheckSkip | None:
+ if ctx.archived:
+ return not_applicable(NOT_ON_ARCHIVED)
+ if not ctx.is_launched:
+ return not_applicable(RUNS_AFTER_LAUNCH)
+ # After the end the decision is made, and "wait for the planned sample" can't be followed.
+ if ctx.has_ended:
+ return not_applicable(NOT_AFTER_END)
+ if ctx.primary_metric_count == 0:
+ return not_applicable("Runs once the experiment has a primary metric")
+ metric = ctx.primary_metric
+ if metric is None:
+ return not_checked("Couldn't read the results of the primary metric")
+ if not metric.is_funnel:
+ return not_applicable("Runs on funnel primary metrics only")
+ # The page's target in this mode comes from the user's guessed baseline, so a note about the target could say
+ # "wait" after the page says the target is reached.
+ if metric.manual_running_time:
+ return not_applicable("Runs with the automatic running time estimate")
+ if metric.result is None:
+ return not_applicable("Runs once the primary metric has results")
+ if _total_samples(metric.result) < MIN_SAMPLES_FOR_RESULTS:
+ return not_applicable("Runs once enough users are exposed")
+ if not 0 < (metric.result.baseline.mean or 0) < 1:
+ return not_applicable("Runs once the control group's conversion rate is above 0% and below 100%")
+ return None
+
+
+def early_significance(ctx: HealthContext) -> ExperimentHealthFinding | None:
+ metric = ctx.primary_metric
+ if early_significance_skip(ctx) is not None or metric is None or metric.result is None:
+ return None
+ result = metric.result
+ control_rate = result.baseline.mean or 0
+ samples = _total_samples(result)
+ # The running time estimate counts the variants of the flag.
+ number_of_variants = len(ctx.flag.variants) if ctx.flag and ctx.flag.variants else len(result.variants) + 1
+ planned_samples = calculate_sample_size(
+ "funnel", control_rate, metric.minimum_detectable_effect, number_of_variants
+ )
+ # The running time estimate's "Target reached" compares the same total, so the note and the estimate agree.
+ if planned_samples is None or samples >= planned_samples:
+ return None
+ below_detectable = [
+ comparison
+ for comparison in _significant_comparisons(result, control_rate)
+ if abs(comparison.observed) < comparison.detectable
+ ]
+ if not below_detectable:
+ return None
+ comparison = min(
+ below_detectable,
+ key=lambda comparison: (abs(comparison.observed) / comparison.detectable, comparison.variant.key),
+ )
+ return ExperimentHealthFinding(
+ code=ExperimentHealthFindingCode.EARLY_SIGNIFICANCE,
+ subcode=BELOW_DETECTABLE_EFFECT,
+ severity=ExperimentHealthFindingSeverity.INFO,
+ title="Significant result on a small sample",
+ detail=" ".join(
+ [
+ f'The variant "{comparison.variant.key}" shows a {comparison.observed:+.0f}% difference on the metric '
+ f'"{metric.name}", which is significant.',
+ f"With {samples:,} users so far, the experiment can reliably detect differences of about "
+ f"{comparison.detectable:.0f}% or more, so this difference may shrink or go away as more users arrive.",
+ f"The running time estimate needs about {planned_samples:,} users to detect a "
+ f"{metric.minimum_detectable_effect:g}% difference.",
+ "Wait for that sample before you decide.",
+ ]
+ ),
+ evidence={
+ "variant": comparison.variant.key,
+ "observed_effect": round(comparison.observed, 1),
+ "detectable_effect": round(comparison.detectable, 1),
+ "minimum_detectable_effect": metric.minimum_detectable_effect,
+ "samples": samples,
+ "planned_samples": planned_samples,
+ },
+ actions=(),
+ diagnostic_ref=None,
+ )
+
+
+def _significant_comparisons(result: MetricResultStats, control_rate: float) -> list[_Comparison]:
+ comparisons: list[_Comparison] = []
+ for variant in result.variants:
+ if variant.significant is not True or variant.interval is None:
+ continue
+ detectable = calculate_detectable_effect(
+ "funnel", control_rate, result.baseline.number_of_samples, variant.number_of_samples
+ )
+ if detectable is not None:
+ # The difference the results page shows, from the analysis itself, so CUPED and the Bayesian model apply.
+ comparisons.append(
+ _Comparison(variant=variant, observed=100 * variant.interval.midpoint, detectable=detectable)
+ )
+ return comparisons
+
+
+def _total_samples(result: MetricResultStats) -> int:
+ return result.baseline.number_of_samples + sum(variant.number_of_samples for variant in result.variants)
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index c2008a737739..67b41721f910 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -28,6 +28,7 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (
has_activation_config,
normalize_to_exposure_criteria,
)
+from products.experiments.backend.logic.result_stats import MetricResultStats
from products.experiments.backend.metric_resolution import MetricRole, saved_metric_link_role, saved_metric_links
from products.experiments.backend.metric_utils import get_default_metric_title
from products.experiments.backend.metric_validation import defined_event_names, extract_entity_nodes
@@ -197,6 +198,22 @@ class PauseHistory:
paused_at: datetime | None
+@frozen
+class PrimaryMetric:
+ """The experiment's first primary metric, which its running-time estimate sizes, with its stored result."""
+
+ name: str
+ is_funnel: bool
+ # The relative effect, as a percentage, that the experiment plans to detect.
+ minimum_detectable_effect: float
+ # Whether the running time estimate is in manual mode, where its target comes from a baseline the user entered
+ # instead of from the results.
+ manual_running_time: bool
+ # The current stored result, the one the results page shows after its last calculation. None when no completed
+ # result is stored under the metric's current calculation key. Only read for a funnel.
+ result: MetricResultStats | None
+
+
@frozen
class ExposureTotals:
total_exposures: Mapping[str, int]
@@ -283,6 +300,9 @@ class HealthContext:
condition_set_traffic: ConditionSetTraffic | None
# None when not loaded or unknown. Only the health endpoint loads it, and only for a paused experiment.
pause_history: PauseHistory | None
+ # None when not loaded, unknown, or the experiment calculates no primary metric. Only the health endpoint loads it,
+ # and only for a launched experiment.
+ primary_metric: PrimaryMetric | None
# The variants that the analysis leaves out of the results.
excluded_variant_keys: frozenset[str]
@@ -567,5 +587,6 @@ def load_health_context(
flag_history=flag_history,
condition_set_traffic=None,
pause_history=None,
+ primary_metric=None,
excluded_variant_keys=frozenset(experiment.excluded_variants or []),
)
diff --git a/products/experiments/backend/health/loader.py b/products/experiments/backend/health/loader.py
index c894b7fb5b3f..c82f70bc7f83 100644
--- a/products/experiments/backend/health/loader.py
+++ b/products/experiments/backend/health/loader.py
@@ -10,6 +10,7 @@ from products.experiments.backend.health.context import HealthContext, load_heal
from products.experiments.backend.health.exposure_gate import load_flag_history
from products.experiments.backend.health.exposure_inputs import ExposureInputs, load_exposure_inputs
from products.experiments.backend.health.pause_history import load_pause_history
+from products.experiments.backend.health.primary_metric import load_primary_metric
from products.experiments.backend.models.experiment import Experiment
from products.experiments.backend.setup_context import ProjectSetupFacts, build_project_setup_facts
@@ -71,4 +72,5 @@ def _health_context(experiment: Experiment, reader_access: ExperimentHealthReade
context,
condition_set_traffic=traffic,
pause_history=load_pause_history(experiment, reader_access=reader_access),
+ primary_metric=load_primary_metric(experiment),
)
diff --git a/products/experiments/backend/health/primary_metric.py b/products/experiments/backend/health/primary_metric.py
new file mode 100644
index 000000000000..4b49191d9bda
--- /dev/null
+++ b/products/experiments/backend/health/primary_metric.py
@@ -0,0 +1,76 @@
+from posthog.exceptions_capture import capture_exception
+
+from products.experiments.backend.health.context import PrimaryMetric
+from products.experiments.backend.metric_calculation.config import (
+ MetricCalculationConfig,
+ build_primary_calculation_configs,
+ team_experiments_configs,
+)
+from products.experiments.backend.metric_calculation.results import MetricResultStore
+from products.experiments.backend.metric_resolution import saved_metric_links
+from products.experiments.backend.metric_utils import get_default_metric_title
+from products.experiments.backend.models.experiment import Experiment
+from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig
+from products.experiments.backend.setup_context import PRODUCT_DEFAULT_MINIMUM_DETECTABLE_EFFECT
+
+
+def load_primary_metric(experiment: Experiment) -> PrimaryMetric | None:
+ """
+ The first primary metric of a launched experiment with its current stored result, or None. Never raises: the
+ checks that read it then report that they couldn't check.
+ """
+ if not experiment.is_launched:
+ return None
+ try:
+ return _load_primary_metric(experiment)
+ except Exception as error:
+ capture_exception(error)
+ return None
+
+
+def _load_primary_metric(experiment: Experiment) -> PrimaryMetric | None:
+ team_config = team_experiments_configs([experiment.team_id])[experiment.team_id]
+ calculation_config = next(iter(build_primary_calculation_configs(experiment, team_config=team_config)), None)
+ if calculation_config is None:
+ return None
+ is_funnel = calculation_config.definition.get("metric_type") == "funnel"
+ return PrimaryMetric(
+ name=_metric_name(experiment, calculation_config),
+ is_funnel=is_funnel,
+ minimum_detectable_effect=_minimum_detectable_effect(experiment, team_config),
+ manual_running_time=_manual_running_time(experiment),
+ result=MetricResultStore(experiment_id=experiment.id).current_result(calculation_config) if is_funnel else None,
+ )
+
+
+def _metric_name(experiment: Experiment, calculation_config: MetricCalculationConfig) -> str:
+ definition = calculation_config.definition
+ # An inline metric wins over a saved metric with the same uuid, as in `get_metrics_for_calculation`.
+ inline_uuids = {metric.get("uuid") for metric in experiment.metrics or [] if isinstance(metric, dict)}
+ if calculation_config.metric_id not in inline_uuids:
+ for link in saved_metric_links(experiment):
+ query = link.saved_metric.query
+ if isinstance(query, dict) and query.get("uuid") == calculation_config.metric_id:
+ return link.saved_metric.name
+ return definition.get("name") or get_default_metric_title(definition)
+
+
+def _minimum_detectable_effect(experiment: Experiment, team_config: TeamExperimentsConfig) -> float:
+ running_time_calculation = experiment.running_time_calculation
+ planned = (
+ running_time_calculation.get("minimum_detectable_effect")
+ if isinstance(running_time_calculation, dict)
+ else None
+ )
+ if isinstance(planned, int | float) and not isinstance(planned, bool) and planned > 0:
+ return float(planned)
+ return float(team_config.default_minimum_detectable_effect or PRODUCT_DEFAULT_MINIMUM_DETECTABLE_EFFECT)
+
+
+def _manual_running_time(experiment: Experiment) -> bool:
+ # pinned: the page's `ConversionRateInputType.MANUAL` in frontend/src/types.ts, read by `runningTimeLogic`.
+ running_time_calculation = experiment.running_time_calculation
+ estimate_config = (
+ running_time_calculation.get("exposure_estimate_config") if isinstance(running_time_calculation, dict) else None
+ )
+ return isinstance(estimate_config, dict) and estimate_config.get("conversionRateInputType") == "manual"
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index 7a654adac7b8..dd79db50ef57 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -13,6 +13,7 @@ from products.experiments.backend.facade.contracts import (
from products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded, bias_risk_skip
from products.experiments.backend.health.checks.bucketing_changed import bucketing_changed_mid_run
from products.experiments.backend.health.checks.conditions_changed import release_conditions_changed_mid_run
+from products.experiments.backend.health.checks.early_significance import early_significance, early_significance_skip
from products.experiments.backend.health.checks.exposure_criteria import (
exposure_criteria_cannot_match,
exposure_criteria_skip,
@@ -122,6 +123,12 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
find=exposure_criteria_cannot_match,
skip=exposure_criteria_skip,
),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.EARLY_SIGNIFICANCE,
+ name="Sample size behind significant results",
+ find=early_significance,
+ skip=early_significance_skip,
+ ),
)
# The exposure query runs these on its answer. They must not include the checks above: the exposure answer
diff --git a/products/experiments/backend/health/test/factories.py b/products/experiments/backend/health/test/factories.py
index 3a18f9ce2cc1..3be50d362e20 100644
--- a/products/experiments/backend/health/test/factories.py
+++ b/products/experiments/backend/health/test/factories.py
@@ -38,6 +38,7 @@ RUNNING_CONTEXT = HealthContext(
flag_history=None,
condition_set_traffic=None,
pause_history=None,
+ primary_metric=None,
excluded_variant_keys=frozenset(),
)
diff --git a/products/experiments/backend/health/test/test_early_significance.py b/products/experiments/backend/health/test/test_early_significance.py
new file mode 100644
index 000000000000..f1c42fc8f753
--- /dev/null
+++ b/products/experiments/backend/health/test/test_early_significance.py
@@ -0,0 +1,281 @@
+from dataclasses import replace
+from datetime import timedelta
+from typing import Any
+
+from posthog.test.base import BaseTest
+from unittest import TestCase
+
+from django.utils import timezone
+
+from parameterized import parameterized
+
+from products.experiments.backend.facade.api import get_experiment_health
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthCheckKey,
+ ExperimentHealthCheckStatus,
+ ExperimentHealthFindingCode,
+ ExperimentHealthFindingSeverity,
+ ExperimentHealthReaderAccess,
+)
+from products.experiments.backend.health.checks.early_significance import (
+ BELOW_DETECTABLE_EFFECT,
+ early_significance,
+ early_significance_skip,
+)
+from products.experiments.backend.health.context import HealthContext, PrimaryMetric
+from products.experiments.backend.health.test.factories import ENDED, RUNNING_CONTEXT
+from products.experiments.backend.logic.result_stats import BaselineStats, Interval, MetricResultStats, VariantStats
+from products.experiments.backend.metric_calculation.config import build_primary_calculation_configs
+from products.experiments.backend.models.experiment import (
+ Experiment,
+ ExperimentMetricResult,
+ ExperimentSavedMetric,
+ ExperimentToSavedMetric,
+)
+from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig
+from products.feature_flags.backend.models.feature_flag import FeatureFlag
+
+Status = ExperimentHealthCheckStatus
+NO_QUERIES = ExperimentHealthReaderAccess(
+ can_read_flag_history=False, can_query_events=False, denied_event_properties=frozenset()
+)
+FUNNEL = {"kind": "ExperimentMetric", "metric_type": "funnel", "series": [{"kind": "EventsNode", "event": "purchase"}]}
+
+
+def _context(
+ variants: list[tuple[str, int, float, bool | None]],
+ *,
+ baseline: tuple[int, int] = (1000, 100),
+ manual_running_time: bool = False,
+ **changes: Any,
+) -> HealthContext:
+ control_samples, control_conversions = baseline
+ control_rate = control_conversions / control_samples
+ result = MetricResultStats(
+ method="bayesian",
+ baseline=BaselineStats(key="control", number_of_samples=control_samples, mean=control_rate),
+ # The raw rate equals the control's, so only the analysis interval carries the difference, as under CUPED.
+ variants=tuple(
+ VariantStats(
+ key=key,
+ number_of_samples=samples,
+ mean=control_rate,
+ interval=Interval(low=difference - 0.15, high=difference + 0.15),
+ chance_to_win=None,
+ p_value=None,
+ significant=significant,
+ )
+ for key, samples, difference, significant in variants
+ ),
+ )
+ metric = PrimaryMetric(
+ name="Checkout",
+ is_funnel=True,
+ minimum_detectable_effect=30,
+ manual_running_time=manual_running_time,
+ result=result,
+ )
+ return replace(RUNNING_CONTEXT, primary_metric=metric, **changes)
+
+
+# A 10% control rate on 1,000 users per variant detects differences of about 38% or more.
+SMALL_SIGNIFICANT_LIFT = _context([("test", 1000, 0.2, True)])
+
+
+class TestEarlySignificance(TestCase):
+ @parameterized.expand(
+ [
+ ("significant_and_smaller_than_detectable", SMALL_SIGNIFICANT_LIFT, "test"),
+ ("significant_and_larger_than_detectable", _context([("test", 1000, -0.5, True)]), None),
+ ("not_significant", _context([("test", 1000, 0.2, False)]), None),
+ ("analysis_stopped", _context([("test", 1000, 0.2, None)]), None),
+ ("sample_reached_the_plan", _context([("test", 20000, 0.06, True)], baseline=(20000, 2000)), None),
+ (
+ "uneven_split_past_the_planned_total",
+ _context([("test", 500, 0.34, True)], baseline=(9000, 900)),
+ None,
+ ),
+ (
+ "names_the_variant_below_detectable",
+ _context([("large", 1000, 0.5, True), ("small", 1000, 0.2, True)]),
+ "small",
+ ),
+ ]
+ )
+ def test_notes_a_significant_difference_the_sample_cannot_detect(
+ self, _name: str, ctx: HealthContext, expected_variant: str | None
+ ) -> None:
+ finding = early_significance(ctx)
+
+ self.assertEqual(finding.evidence["variant"] if finding else None, expected_variant)
+
+ def test_states_the_analysis_difference_and_the_planned_sizes(self) -> None:
+ finding = early_significance(SMALL_SIGNIFICANT_LIFT)
+
+ assert finding is not None
+ self.assertEqual(
+ (finding.code, finding.subcode, finding.severity),
+ (
+ ExperimentHealthFindingCode.EARLY_SIGNIFICANCE,
+ BELOW_DETECTABLE_EFFECT,
+ ExperimentHealthFindingSeverity.INFO,
+ ),
+ )
+ self.assertEqual(
+ finding.evidence,
+ {
+ "variant": "test",
+ "observed_effect": 20.0,
+ "detectable_effect": 37.9,
+ "minimum_detectable_effect": 30,
+ "samples": 2000,
+ "planned_samples": 3201,
+ },
+ )
+
+ @parameterized.expand(
+ [
+ ("ended", replace(SMALL_SIGNIFICANT_LIFT, **ENDED), Status.NOT_APPLICABLE),
+ ("unreadable_results", replace(SMALL_SIGNIFICANT_LIFT, primary_metric=None), Status.NOT_CHECKED),
+ (
+ "mean_metric",
+ replace(
+ SMALL_SIGNIFICANT_LIFT,
+ primary_metric=PrimaryMetric(
+ name="Revenue",
+ is_funnel=False,
+ minimum_detectable_effect=30,
+ manual_running_time=False,
+ result=None,
+ ),
+ ),
+ Status.NOT_APPLICABLE,
+ ),
+ (
+ "manual_running_time",
+ _context([("test", 1000, 0.2, True)], manual_running_time=True),
+ Status.NOT_APPLICABLE,
+ ),
+ ("under_50_users", _context([("test", 20, 0.2, True)], baseline=(20, 2)), Status.NOT_APPLICABLE),
+ (
+ "no_control_conversions",
+ _context([("test", 1000, 0.2, True)], baseline=(1000, 0)),
+ Status.NOT_APPLICABLE,
+ ),
+ ]
+ )
+ def test_says_why_it_did_not_look(self, _name: str, ctx: HealthContext, expected: Status) -> None:
+ skip = early_significance_skip(ctx)
+
+ self.assertEqual(skip.status if skip else None, expected)
+ self.assertIsNone(early_significance(ctx))
+
+
+class TestEarlySignificanceFromStoredResults(BaseTest):
+ @parameterized.expand(
+ [
+ (
+ "inline_metric_and_the_experiments_effect",
+ False,
+ True,
+ {"minimum_detectable_effect": 20},
+ 15,
+ Status.FINDING,
+ 20,
+ ),
+ ("inline_metric_and_the_teams_default_effect", False, True, {}, 15, Status.FINDING, 15),
+ ("inline_metric_and_the_product_default_effect", False, True, {}, None, Status.FINDING, 30),
+ ("shared_metric", True, True, {}, None, Status.FINDING, 30),
+ ("result_of_an_earlier_configuration", False, False, {}, None, Status.NOT_APPLICABLE, None),
+ (
+ "manual_running_time",
+ False,
+ True,
+ {"exposure_estimate_config": {"conversionRateInputType": "manual"}},
+ None,
+ Status.NOT_APPLICABLE,
+ None,
+ ),
+ ]
+ )
+ def test_reads_the_primary_metrics_current_result(
+ self,
+ _name: str,
+ shared: bool,
+ under_current_key: bool,
+ running_time_calculation: dict[str, Any],
+ team_effect: int | None,
+ expected: Status,
+ expected_effect: int | None,
+ ) -> None:
+ if team_effect is not None:
+ TeamExperimentsConfig.objects.create(team=self.team, default_minimum_detectable_effect=team_effect)
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ key="checkout-flag",
+ created_by=self.user,
+ filters={
+ "groups": [{"properties": [], "rollout_percentage": 100}],
+ "multivariate": {
+ "variants": [
+ {"key": "control", "rollout_percentage": 50},
+ {"key": "test", "rollout_percentage": 50},
+ ]
+ },
+ },
+ )
+ launched_at = timezone.now() - timedelta(days=3)
+ experiment = Experiment.objects.create(
+ team=self.team,
+ name="checkout",
+ feature_flag=flag,
+ start_date=launched_at,
+ metrics=[] if shared else [{**FUNNEL, "uuid": "checkout-funnel"}],
+ running_time_calculation=running_time_calculation,
+ )
+ if shared:
+ saved_metric = ExperimentSavedMetric.objects.create(
+ team=self.team, name="Checkout funnel", query={**FUNNEL, "uuid": "checkout-funnel"}
+ )
+ ExperimentToSavedMetric.objects.create(
+ experiment=experiment, saved_metric=saved_metric, metadata={"type": "primary"}
+ )
+ calculation_key = build_primary_calculation_configs(experiment)[0].calculation_key()
+ ExperimentMetricResult.objects.create(
+ experiment=experiment,
+ metric_uuid="checkout-funnel",
+ fingerprint=calculation_key if under_current_key else "0" * 64,
+ query_from=launched_at,
+ query_to=timezone.now(),
+ completed_at=timezone.now(),
+ status=ExperimentMetricResult.Status.COMPLETED,
+ result={
+ "baseline": {"key": "control", "number_of_samples": 1000, "sum": 100, "sum_squares": 100},
+ "variant_results": [
+ {
+ "key": "test",
+ "method": "bayesian",
+ "number_of_samples": 1000,
+ "sum": 120,
+ "sum_squares": 120,
+ "credible_interval": [0.05, 0.35],
+ "chance_to_win": 0.98,
+ "significant": True,
+ }
+ ],
+ },
+ )
+
+ health = get_experiment_health(team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES)
+
+ check = next(check for check in health.checks if check.key == ExperimentHealthCheckKey.EARLY_SIGNIFICANCE)
+ finding = next(
+ (finding for finding in health.findings if finding.code == ExperimentHealthFindingCode.EARLY_SIGNIFICANCE),
+ None,
+ )
+ self.assertEqual(
+ (check.status, finding.evidence["minimum_detectable_effect"] if finding else None),
+ (expected, expected_effect),
+ )
+ if finding:
+ self.assertIn('"Checkout funnel"' if shared else '"purchase conversion"', finding.detail)
diff --git a/products/experiments/backend/health/test/test_registry.py b/products/experiments/backend/health/test/test_registry.py
index 52b61655e243..3fffbba109b6 100644
--- a/products/experiments/backend/health/test/test_registry.py
+++ b/products/experiments/backend/health/test/test_registry.py
@@ -212,6 +212,11 @@ class TestEvaluate(TestCase):
{"flag_called": False},
{Key.NEW_FLAG_NO_TRAFFIC: Status.NOT_APPLICABLE},
),
+ (
+ "early_significance_needs_the_primary_metrics_results",
+ {"primary_metric": None},
+ {Key.EARLY_SIGNIFICANCE: Status.NOT_CHECKED},
+ ),
(
"zero_exposures_finds_a_day_after_launch",
{"exposures": replace(NO_EXPOSURES, hours_since_launch=24)},
diff --git a/products/experiments/backend/logic/result_stats.py b/products/experiments/backend/logic/result_stats.py
index 3f77efb0be2e..c0863b67bc7b 100644
--- a/products/experiments/backend/logic/result_stats.py
+++ b/products/experiments/backend/logic/result_stats.py
@@ -65,6 +65,11 @@ class Interval:
low: float
high: float
+ @property
+ def midpoint(self) -> float:
+ # The results page shows this as the variant's difference (`getDelta` in MetricsView/shared/utils.ts).
+ return (self.low + self.high) / 2
+
@frozen
class BaselineStats:
diff --git a/products/experiments/backend/metric_calculation/results.py b/products/experiments/backend/metric_calculation/results.py
index 8b6ddafebce2..24f592121c18 100644
--- a/products/experiments/backend/metric_calculation/results.py
+++ b/products/experiments/backend/metric_calculation/results.py
@@ -38,6 +38,7 @@ import structlog
from posthog.dataclasses import frozen
+from products.experiments.backend.logic.result_stats import MetricResultStats
from products.experiments.backend.metric_calculation.config import MetricCalculationConfig, build_calculation_configs
from products.experiments.backend.models.experiment import (
Experiment,
@@ -184,6 +185,11 @@ class MetricResultStore:
"""The current result of the metric that `calculation_config` describes. `current_outcomes` defines the rule."""
return self.current_outcomes({self.experiment_id: calculation_config}).get(self.experiment_id)
+ def current_result(self, calculation_config: MetricCalculationConfig) -> MetricResultStats | None:
+ """The statistics of the result that `current_outcome` picks, or None when it has none or doesn't parse."""
+ stored = self._current_rows({self.experiment_id: calculation_config}).values_list("result", flat=True).first()
+ return MetricResultStats.from_stored(stored)
+
def previous_completed(
self, metric_uuid: str, calculation_key: str, *, before: datetime
) -> ExperimentMetricResult | None:
@@ -414,19 +420,8 @@ class MetricResultStore:
return newest_point + _SYNC_COPY_OFFSET
@staticmethod
- def current_outcomes(config_by_experiment: Mapping[int, MetricCalculationConfig]) -> dict[int, ResultSummary]:
- """For each experiment, the current result of the metric that its calculation config describes, in one query.
-
- The current result is the completed row whose data ends last among the rows filed under the config's
- calculation key, by a recalculation run or by the daily workflows. A row's data ends at its query_to, or at
- its completed_at when that comes first. A backfill stamps the current day with the end of that day, but the
- row holds data only up to the moment the backfill ran, so a later run of the same day must win over it. A row
- under another key was computed from another configuration, for example from the start date before a
- relaunch, so it does not count. The end of the data counts, not the newest write, because a backfill writes
- past windows after the newer ones.
- """
- if not config_by_experiment:
- return {}
+ def _current_rows(config_by_experiment: Mapping[int, MetricCalculationConfig]) -> QuerySet[ExperimentMetricResult]:
+ """For each experiment, the row that `current_outcomes` describes."""
filed_under_config = Q()
for experiment_id, calculation_config in config_by_experiment.items():
key = calculation_config.calculation_key()
@@ -435,21 +430,36 @@ class MetricResultStore:
metric_uuid=calculation_config.metric_id,
fingerprint__in=[key, _recalc_fingerprint(key)],
)
- rows = (
+ return (
ExperimentMetricResult.objects.filter(filed_under_config, status=_COMPLETED)
# Postgres LEAST skips a NULL, so a row without completed_at ranks by its query_to.
.alias(data_until=Least("query_to", "completed_at"))
.order_by("experiment_id", F("data_until").desc(), F("completed_at").desc(nulls_last=True), "-id")
.distinct("experiment_id")
- .values(
- "experiment_id",
- "metric_uuid",
- "completed_at",
- "query_to",
- baseline_samples=KT("result__baseline__number_of_samples"),
- baseline_sum=KT("result__baseline__sum"),
- variant_results=KeyTransform("variant_results", "result"),
- )
+ )
+
+ @staticmethod
+ def current_outcomes(config_by_experiment: Mapping[int, MetricCalculationConfig]) -> dict[int, ResultSummary]:
+ """For each experiment, the current result of the metric that its calculation config describes, in one query.
+
+ The current result is the completed row whose data ends last among the rows filed under the config's
+ calculation key, by a recalculation run or by the daily workflows. A row's data ends at its query_to, or at
+ its completed_at when that comes first. A backfill stamps the current day with the end of that day, but the
+ row holds data only up to the moment the backfill ran, so a later run of the same day must win over it. A row
+ under another key was computed from another configuration, for example from the start date before a
+ relaunch, so it does not count. The end of the data counts, not the newest write, because a backfill writes
+ past windows after the newer ones.
+ """
+ if not config_by_experiment:
+ return {}
+ rows = MetricResultStore._current_rows(config_by_experiment).values(
+ "experiment_id",
+ "metric_uuid",
+ "completed_at",
+ "query_to",
+ baseline_samples=KT("result__baseline__number_of_samples"),
+ baseline_sum=KT("result__baseline__sum"),
+ variant_results=KeyTransform("variant_results", "result"),
)
summaries: dict[int, ResultSummary] = {}
for row in rows:
diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.py
index 29efc40102f9..f98bac419cef 100644
--- a/products/experiments/backend/presentation/views.py
+++ b/products/experiments/backend/presentation/views.py
@@ -1006,6 +1006,13 @@ class EnterpriseExperimentsViewSet(
`running_but_flag_disabled`, as a warning. A caller who can't view the linked flag's
history gets the warning, because the pause is not read. A draft whose flag is already
active with multiple variants gets `not_started_but_multiple_variants_rolled_out` as info.
+
+ A running experiment whose first primary metric is a funnel gets the info
+ `early_significance` (subcode `below_detectable_effect`) when a significant variant's
+ difference is smaller than the smallest difference its sample can detect, while that
+ sample is below the one the running time estimate plans for the minimum detectable
+ effect. It reads the metric's current stored result, the one the results page shows
+ after its last calculation, and the running time calculator's formula.
"""
experiment: Experiment = self.get_object()
health = get_experiment_health(
diff --git a/products/experiments/backend/running_time_calculator.py b/products/experiments/backend/running_time_calculator.py
index 69ef403665ff..6503a847860f 100644
--- a/products/experiments/backend/running_time_calculator.py
+++ b/products/experiments/backend/running_time_calculator.py
@@ -172,6 +172,37 @@ def calculate_baseline_value(baseline: BaselineStats, metric_type: CalculatorMet
return None
+def calculate_detectable_effect(
+ metric_type: CalculatorMetricType,
+ baseline_value: float,
+ control_samples: int,
+ variant_samples: int,
+ variance: float | None = None,
+) -> float | None:
+ """The smallest relative effect, as a percentage, that the sample-size formula detects between a control and a
+ variant of these sizes.
+
+ The inverse of :func:`calculate_sample_size`: for two variants of ``n`` samples each, the sample size for the
+ returned effect is about ``2 · n``. The formula takes variants of equal size, so ``1/n_control + 1/n_variant``
+ replaces its ``2/n`` for an uneven split.
+ """
+ if control_samples <= 0 or variant_samples <= 0 or baseline_value == 0:
+ return None
+
+ if metric_type == "funnel":
+ # A conversion rate of 0 or 1 has no binomial variance, so no effect size follows from it.
+ if not 0 < baseline_value < 1:
+ return None
+ variance = baseline_value * (1 - baseline_value)
+ elif variance is None:
+ variance = calculate_variance(metric_type, baseline_value)
+ if variance is None or variance <= 0:
+ return None
+
+ d = math.sqrt(SAMPLE_SIZE_Z_FACTOR / 2 * variance * (1 / control_samples + 1 / variant_samples))
+ return 100 * d / abs(baseline_value)
+
+
def calculate_running_time_days(sample_size: int | None, exposure_rate_per_day: float | None) -> int | None:
"""Days to reach ``sample_size`` at ``exposure_rate_per_day`` exposures/day."""
if not sample_size or not exposure_rate_per_day or exposure_rate_per_day <= 0:
diff --git a/products/experiments/backend/test/test_running_time_calculator.py b/products/experiments/backend/test/test_running_time_calculator.py
index 0cb87bb173a8..10ff6440539d 100644
--- a/products/experiments/backend/test/test_running_time_calculator.py
+++ b/products/experiments/backend/test/test_running_time_calculator.py
@@ -1,10 +1,13 @@
import math
+import pytest
+
from parameterized import parameterized
from products.experiments.backend.running_time_calculator import (
BaselineStats,
calculate_baseline_value,
+ calculate_detectable_effect,
calculate_running_time_days,
calculate_sample_size,
calculate_variance,
@@ -173,6 +176,25 @@ class TestCalculateSampleSize:
assert calculate_sample_size("ratio", 10, 10, 2, variance=-32) is None
+class TestCalculateDetectableEffect:
+ @parameterized.expand(
+ [
+ ("mean_count", 4, 5),
+ ("mean_sum_or_avg", 50, 5),
+ ("funnel", 0.1, 50),
+ ("funnel", 0.03, 30),
+ ]
+ )
+ def test_inverts_the_sample_size(self, metric_type, baseline_value, mde):
+ sample_size = calculate_sample_size(metric_type, baseline_value, mde, 2)
+ assert sample_size is not None
+ samples_per_variant = sample_size // 2
+
+ detectable = calculate_detectable_effect(metric_type, baseline_value, samples_per_variant, samples_per_variant)
+
+ assert detectable == pytest.approx(mde, rel=0.01)
+
+
class TestCalculateRunningTimeDays:
@parameterized.expand(
[
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index 58ef181d5a50..43739b79959a 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2288,6 +2288,7 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
* * `new_flag_no_traffic` - New Flag No Traffic
+ * * `early_significance` - Early Significance
*/
export type ExperimentHealthFindingCodeEnumApi =
(typeof ExperimentHealthFindingCodeEnumApi)[keyof typeof ExperimentHealthFindingCodeEnumApi]
@@ -2310,6 +2311,7 @@ export const ExperimentHealthFindingCodeEnumApi = {
BucketingChangedMidRun: 'bucketing_changed_mid_run',
ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
NewFlagNoTraffic: 'new_flag_no_traffic',
+ EarlySignificance: 'early_significance',
} as const
/**
@@ -2374,7 +2376,8 @@ export interface ExperimentHealthFindingApi {
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
- * * `new_flag_no_traffic` - New Flag No Traffic */
+ * * `new_flag_no_traffic` - New Flag No Traffic
+ * * `early_significance` - Early Significance */
code: ExperimentHealthFindingCodeEnumApi
/**
* The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.
@@ -2414,6 +2417,7 @@ export interface ExperimentHealthFindingApi {
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
+ * * `early_significance` - Early Significance
* * `zero_exposures` - Zero Exposures
* * `srm` - Srm
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
@@ -2434,6 +2438,7 @@ export const ExperimentHealthCheckKeyEnumApi = {
NoMetric: 'no_metric',
MetricEventNotIngested: 'metric_event_not_ingested',
ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
+ EarlySignificance: 'early_significance',
ZeroExposures: 'zero_exposures',
Srm: 'srm',
BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',
@@ -2470,6 +2475,7 @@ export interface ExperimentHealthCheckResultApi {
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
+ * * `early_significance` - Early Significance
* * `zero_exposures` - Zero Exposures
* * `srm` - Srm
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
diff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.ts
index 8692d7dec371..2d2e04a74204 100644
--- a/products/experiments/frontend/generated/api.ts
+++ b/products/experiments/frontend/generated/api.ts
@@ -756,6 +756,13 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =
* `running_but_flag_disabled`, as a warning. A caller who can't view the linked flag's
* history gets the warning, because the pause is not read. A draft whose flag is already
* active with multiple variants gets `not_started_but_multiple_variants_rolled_out` as info.
+ *
+ * A running experiment whose first primary metric is a funnel gets the info
+ * `early_significance` (subcode `below_detectable_effect`) when a significant variant's
+ * difference is smaller than the smallest difference its sample can detect, while that
+ * sample is below the one the running time estimate plans for the minimum detectable
+ * effect. It reads the metric's current stored result, the one the results page shows
+ * after its last calculation, and the running time calculator's formula.
*/
export const experimentsHealthRetrieve = async (
projectId: string,
diff --git a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
index 8f52d36e0965..57b088b9a8b6 100644
--- a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
+++ b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
@@ -108,6 +108,36 @@ export const DraftWithEventsNotReceived: Story = {
},
}
+export const RunningWithSignificanceOnASmallSample: Story = {
+ args: {
+ checks: [
+ ...FLAG_SETUP_CHECKS,
+ passed('no_metric', 'Metrics'),
+ found('early_significance', 'Sample size behind significant results'),
+ ],
+ notes: [
+ {
+ code: 'early_significance',
+ subcode: 'below_detectable_effect',
+ severity: 'info',
+ title: 'Significant result on a small sample',
+ detail: 'The variant "test" shows a +20% difference on the metric "Checkout", which is significant. With 2,000 users so far, the experiment can reliably detect differences of about 38% or more, so this difference may shrink or go away as more users arrive. The running time estimate needs about 3,201 users to detect a 30% difference. Wait for that sample before you decide.',
+ evidence: {
+ variant: 'test',
+ observed_effect: 20,
+ detectable_effect: 37.9,
+ minimum_detectable_effect: 30,
+ samples: 2000,
+ planned_samples: 3201,
+ },
+ actions: [],
+ },
+ ],
+ isDraft: false,
+ exposureChecksMissing: false,
+ },
+}
+
export const ChecksThatCouldNotRun: Story = {
args: {
checks: [
diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.ts
index 50b0c1382259..bf0f6cf6a974 100644
--- a/products/experiments/frontend/health/debug/healthDebugReport.ts
+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts
@@ -333,6 +333,7 @@ export function buildHealthDebugChecks(input: HealthDebugInput): HealthDebugChec
noMetricCheck(input),
serverOnlyCheck(input, 'metric_event_not_ingested', metricEventNote, ['metrics', 'project']),
serverOnlyCheck(input, 'exposure_criteria_cannot_match', exposureCriteriaNote, ['experiment', 'project']),
+ serverOnlyCheck(input, 'early_significance', null, ['experiment', 'metrics']),
...exposureChecks(input),
]
}
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index 099d14a6e3bb..73ec2da0fdf1 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -31,6 +31,7 @@ export type ExperimentHealthFindingCode =
| 'bucketing_changed_mid_run'
| 'release_conditions_changed_mid_run'
| 'new_flag_no_traffic'
+ | 'early_significance'
export interface ExperimentHealthFinding {
code: ExperimentHealthFindingCode
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index b2513f05e214..953eda674e05 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -45213,6 +45213,7 @@ export namespace Schemas {
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
* * `new_flag_no_traffic` - New Flag No Traffic
+ * * `early_significance` - Early Significance
*/
export type ExperimentHealthFindingCodeEnum = typeof ExperimentHealthFindingCodeEnum[keyof typeof ExperimentHealthFindingCodeEnum];
@@ -45235,6 +45236,7 @@ export namespace Schemas {
BucketingChangedMidRun: 'bucketing_changed_mid_run',
ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
NewFlagNoTraffic: 'new_flag_no_traffic',
+ EarlySignificance: 'early_significance',
} as const;
/**
@@ -45299,7 +45301,8 @@ export namespace Schemas {
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
- * * `new_flag_no_traffic` - New Flag No Traffic */
+ * * `new_flag_no_traffic` - New Flag No Traffic
+ * * `early_significance` - Early Significance */
code: ExperimentHealthFindingCodeEnum;
/**
* The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.
@@ -45339,6 +45342,7 @@ export namespace Schemas {
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
+ * * `early_significance` - Early Significance
* * `zero_exposures` - Zero Exposures
* * `srm` - Srm
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
@@ -45359,6 +45363,7 @@ export namespace Schemas {
NoMetric: 'no_metric',
MetricEventNotIngested: 'metric_event_not_ingested',
ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
+ EarlySignificance: 'early_significance',
ZeroExposures: 'zero_exposures',
Srm: 'srm',
BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',
@@ -45395,6 +45400,7 @@ export namespace Schemas {
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
+ * * `early_significance` - Early Significance
* * `zero_exposures` - Zero Exposures
* * `srm` - Srm
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded