Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): warn when experiments share a feature flag (#128065)

Marcel Poelker539bcb4e533c
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index f469db887f81..8b12e6e0d8af 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -11439,9 +11439,9 @@ snapshots:
scenes-app-experiments--experiment-frequentist-two-variants--light:
hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmII
scenes-app-experiments--experiment-health-debug-tab--dark:
- hash: v1.k794b7964.bd266bf7b03a0d96fb3447e669fb154d381d3920a8b7043f6b996f46789edf45.GZi7irxblhuJDTcQ4ILtuf9V9inrGgHKC4ID1shnfF4
+ hash: v1.k794b7964.4110aac4ac713b4ba9b55eb08ea4a122e2b3e63907edc940bf1ee00e3b4466a4.hwxC9eQVxdcbPqkZjFLdNl-HwkZi_cav1yJ0RJfLuAk
scenes-app-experiments--experiment-health-debug-tab--light:
- hash: v1.k794b7964.22d2b1422db4cfd1e2b802d020a14727f577c80a0a881eea96c67d9fce3ddbb1.vEPNwvdLLCrf9rSaG29yCHMqAN5UbMBpWLea3BzQqOk
+ hash: v1.k794b7964.f25e235992bc6f65e679bc2288df78d2c2c476681844c81692ddf513143bdfc6.--c7yplu92ohebzYTVp__eRhIzPUpr7bmyY39p4oajo
scenes-app-experiments--experiment-recordings-dropped-metric--dark:
hash: v1.k794b7964.9ce950830524de179a4d761af104781fdc05347d59f18de814e72bc40a486158.T1SiGxL_fGBQT61rjwuJe9MPz1NPcjeZBRv6rKiSzrE
scenes-app-experiments--experiment-recordings-dropped-metric--light:
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index a43ebdeb27ec..5390f396bdd5 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -10016,6 +10016,7 @@ export namespace Schemas {
* * `new_flag_no_traffic` - New Flag No Traffic
* * `early_significance` - Early Significance
* * `metrics_not_matured` - Metrics Not Matured
+ * * `flag_shared` - Flag Shared
*/
export type ExperimentHealthFindingCodeEnum =
| "flag_off_while_running"
@@ -10041,7 +10042,8 @@ export namespace Schemas {
| "release_conditions_changed_mid_run"
| "new_flag_no_traffic"
| "early_significance"
- | "metrics_not_matured";
+ | "metrics_not_matured"
+ | "flag_shared";
/**
* * `critical` - Critical severity
* * `warning` - Warning severity
@@ -10098,6 +10100,7 @@ export namespace Schemas {
* * `new_flag_no_traffic` - New Flag No Traffic
* * `early_significance` - Early Significance
* * `metrics_not_matured` - Metrics Not Matured
+ * * `flag_shared` - Flag Shared
*/
code: ExperimentHealthFindingCodeEnum;
/**
@@ -10139,6 +10142,7 @@ export namespace Schemas {
};
/**
* * `flag_state` - Flag State
+ * * `flag_shared` - Flag Shared
* * `flag_history` - Flag History
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
@@ -10163,6 +10167,7 @@ export namespace Schemas {
*/
export type ExperimentHealthCheckKeyEnum =
| "flag_state"
+ | "flag_shared"
| "flag_history"
| "split_changed_mid_run"
| "bucketing_changed_mid_run"
@@ -10200,6 +10205,7 @@ export namespace Schemas {
* Stable identifier of the check. A check can report several finding codes, so the key is not a finding code.
*
* * `flag_state` - Flag State
+ * * `flag_shared` - Flag Shared
* * `flag_history` - Flag History
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index 3bd1c62cf55b..de9d689d1d96 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -251,6 +251,7 @@ class ExperimentHealthFindingCode(LabeledStrEnum):
NEW_FLAG_NO_TRAFFIC = "new_flag_no_traffic"
EARLY_SIGNIFICANCE = "early_significance"
METRICS_NOT_MATURED = "metrics_not_matured"
+ FLAG_SHARED = "flag_shared"
class ExperimentHealthMetricCompileErrorKind(LabeledStrEnum):
@@ -336,6 +337,7 @@ class ExperimentHealthCheckKey(LabeledStrEnum):
# names the check, not a problem: `flag_state` reports four finding codes, and `flag_history` reports its finding
# under `split_changed_mid_run`.
FLAG_STATE = "flag_state"
+ FLAG_SHARED = "flag_shared"
FLAG_HISTORY = "flag_history"
SPLIT_CHANGED_MID_RUN = "split_changed_mid_run"
BUCKETING_CHANGED_MID_RUN = "bucketing_changed_mid_run"
diff --git a/products/experiments/backend/health/checks/flag_state.py b/products/experiments/backend/health/checks/flag_state.py
index ebb3bbfd3bcb..65d5db41bd38 100644
--- a/products/experiments/backend/health/checks/flag_state.py
+++ b/products/experiments/backend/health/checks/flag_state.py
@@ -15,6 +15,9 @@ tells a pause from the experiment from a flag disabled elsewhere, and reports th
An ended experiment is the last exception. Only the server reads the flag's latest call and the other experiments on
the flag. So only the server stays silent when the flag now serves a later experiment, and reports as info a flag that
a draft also uses, a flag with no recent call, and a flag whose calls it can't read yet, each with its own subcode.
+
+A draft whose flag a running experiment uses is an exception too. The flag's state belongs to the running experiment,
+so only the server stays silent there, and the shared-flag check reports the draft.
"""
from datetime import UTC, datetime, timedelta
@@ -104,7 +107,7 @@ def flag_state(ctx: HealthContext) -> CheckOutcome:
if ctx.is_running:
if ctx.is_paused:
- return _flag_disabled(ctx.pause_history)
+ return _flag_disabled(ctx.pause_history, running_others=_running_others(ctx))
if zero_rollout:
traffic = ctx.condition_set_traffic
if traffic is not None and traffic.people_assigned_outside >= _MIN_PEOPLE_ASSIGNED_OUTSIDE:
@@ -136,6 +139,8 @@ def flag_state(ctx: HealthContext) -> CheckOutcome:
return _live_after_end(ctx, ctx.ended_at)
elif not ctx.archived:
# The draft state is also reached again when resetting experiment measurements.
+ if _running_others(ctx):
+ return None
if flag.active and _has_multiple_variants_active(flag) and not zero_rollout:
# Info, because a flag turned on to test the variants before launch is the usual path.
return _finding(
@@ -154,7 +159,11 @@ def flag_state(ctx: HealthContext) -> CheckOutcome:
return None
-def _flag_disabled(pause: PauseHistory | None) -> ExperimentHealthFinding:
+def _running_others(ctx: HealthContext) -> int:
+ return sum(1 for other in ctx.other_experiments_on_flag if other.is_running and not other.archived)
+
+
+def _flag_disabled(pause: PauseHistory | None, *, running_others: int) -> ExperimentHealthFinding:
if pause is not None and pause.paused_at is not None:
# The latest pause or resume decides. A flag enabled and disabled again on the flag page after a pause writes
# no entry, so it still reads as that pause.
@@ -171,6 +180,25 @@ def _flag_disabled(pause: PauseHistory | None) -> ExperimentHealthFinding:
),
evidence={"paused_at": pause.paused_at.astimezone(UTC).isoformat()},
)
+ if running_others:
+ # Only this experiment's own activity log is read, so a pause from another experiment on the flag has no
+ # entry here and reads like a flag disabled on the flag page.
+ others, either, both = (
+ ("Another running experiment uses", "either experiment", "both")
+ if running_others == 1
+ else ("Other running experiments use", "any of these experiments", "all of them")
+ )
+ return _finding(
+ code=ExperimentHealthFindingCode.FLAG_OFF_WHILE_RUNNING,
+ subcode=RUNNING_BUT_FLAG_DISABLED,
+ title="The experiment is running, but its shared feature flag is disabled",
+ detail=(
+ f"The linked feature flag is disabled, so it serves no variants. {others} the same flag, and a pause "
+ f"of {either} or a change on the feature flag page disables it for {both}. Users exposed earlier stay "
+ f"in the results. Resuming {either} enables the flag for {both}. Otherwise, end this experiment with a "
+ "conclusion."
+ ),
+ )
if pause is not None and pause.launched_at >= PAUSE_ENTRIES_SINCE:
return _finding(
code=ExperimentHealthFindingCode.FLAG_OFF_WHILE_RUNNING,
@@ -221,11 +249,11 @@ def _no_rollout(*, observed: bool) -> ExperimentHealthFinding:
def _live_after_end(ctx: HealthContext, ended_at: datetime) -> ExperimentHealthFinding | None:
- others = ctx.other_experiments_on_flag or ()
+ others = ctx.other_experiments_on_flag
# The flag's state belongs to the later experiment, and that experiment's own check reports it.
if any(_serves_later_experiment(other, ended_at) for other in others):
return None
- if any(other.launched_at is None and not other.archived for other in others):
+ if any(other.is_draft and not other.archived for other in others):
return _shared_with_draft(ended_at, ctx.team_timezone)
if ctx.flag_called_after_end is True:
return _finding(
diff --git a/products/experiments/backend/health/checks/shared_flag.py b/products/experiments/backend/health/checks/shared_flag.py
new file mode 100644
index 000000000000..7aae9265702c
--- /dev/null
+++ b/products/experiments/backend/health/checks/shared_flag.py
@@ -0,0 +1,124 @@
+"""
+Other experiments on the experiment's feature flag. Pure function, no I/O.
+
+The API lets several experiments use one flag. The experiment actions that write the flag change it for every
+experiment on it: a pause disables it, and a freeze, a shipped variant or a change of the split rewrites it. A draft
+writes its split, rollout and holdout to the flag on every save.
+
+Each experiment reads exposures only between its own start and end, so an ended experiment shares no exposures with a
+running one. A new experiment on the flag of an ended one keeps each user's variant, as a reset and a new launch do.
+So the check doesn't count ended experiments. On the page of an ended experiment, the flag-state check decides what to
+say about the flag, so this check doesn't run there.
+"""
+
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFinding,
+ ExperimentHealthFindingCode,
+ ExperimentHealthFindingSeverity,
+)
+from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.health.skips import CheckOutcome, CheckSkip, flag_to_check
+
+RUNNING_FLAG_SHARED_WITH_RUNNING = "running_flag_shared_with_running"
+RUNNING_FLAG_SHARED_WITH_DRAFT = "running_flag_shared_with_draft"
+DRAFT_FLAG_SHARED_WITH_RUNNING = "draft_flag_shared_with_running"
+
+
+def flag_shared(ctx: HealthContext) -> CheckOutcome:
+ flag = flag_to_check(ctx, before_launch=True)
+ if isinstance(flag, CheckSkip):
+ return flag
+ # An archived experiment is ended, or a draft that nobody is going to launch.
+ others = [other for other in ctx.other_experiments_on_flag if not other.archived]
+ running = sum(1 for other in others if other.is_running)
+ drafts = sum(1 for other in others if other.is_draft)
+ if not ctx.is_launched:
+ return _draft_shares_with_running(running) if running else None
+ if running:
+ return _running_shares_with_running(running)
+ if drafts:
+ return _running_shares_with_draft(drafts)
+ return None
+
+
+def _running_shares_with_running(others: int) -> ExperimentHealthFinding:
+ both, either, the_others = (
+ ("both", "either one", "the other") if others == 1 else ("all of them", "any of them", "the others")
+ )
+ return _finding(
+ subcode=RUNNING_FLAG_SHARED_WITH_RUNNING,
+ severity=ExperimentHealthFindingSeverity.WARNING,
+ title=(
+ "Another running experiment uses this feature flag"
+ if others == 1
+ else f"{others} other running experiments use this feature flag"
+ ),
+ detail=(
+ f"The experiments read exposures of the same flag, so a user exposed while they run can count in {both}. "
+ f"Pausing {either} disables the flag for {both}. Freezing exposure, shipping a variant or changing the "
+ f"split from {either} changes the flag for {both} too. If you need only one experiment, end {the_others}. "
+ "Ending an experiment doesn't change the flag. To test them separately, end one and run it again on a new "
+ "feature flag."
+ ),
+ evidence={"other_running_experiments": others},
+ )
+
+
+def _draft_shares_with_running(running: int) -> ExperimentHealthFinding:
+ the_running = "the running experiment" if running == 1 else "the running experiments"
+ return _finding(
+ subcode=DRAFT_FLAG_SHARED_WITH_RUNNING,
+ severity=ExperimentHealthFindingSeverity.WARNING,
+ title=(
+ "A running experiment already uses this feature flag"
+ if running == 1
+ else f"{running} running experiments already use this feature flag"
+ ),
+ detail=(
+ f"Saving a change to this draft's split, rollout or holdout changes the flag for {the_running} too. "
+ "After launch, both read exposures of the same flag, and pausing this experiment or shipping a variant "
+ f"from it changes the flag for {the_running}. To test this experiment separately, duplicate it with a new "
+ "feature flag key and delete this draft."
+ ),
+ evidence={"other_running_experiments": running},
+ )
+
+
+def _running_shares_with_draft(drafts: int) -> ExperimentHealthFinding:
+ # Info, because nothing changes the flag until someone edits or launches the draft.
+ the_draft = "the draft" if drafts == 1 else "a draft"
+ return _finding(
+ subcode=RUNNING_FLAG_SHARED_WITH_DRAFT,
+ severity=ExperimentHealthFindingSeverity.INFO,
+ title=(
+ "A draft experiment uses this feature flag"
+ if drafts == 1
+ else f"{drafts} draft experiments use this feature flag"
+ ),
+ detail=(
+ f"Saving a change to {the_draft}'s split, rollout or holdout changes the flag for this experiment too. "
+ f"After {the_draft} launches, both read exposures of the same flag, and pausing either one disables the "
+ f"flag for both. To keep {the_draft} separate, duplicate it with a new feature flag key before launch."
+ ),
+ evidence={"other_draft_experiments": drafts},
+ )
+
+
+def _finding(
+ *,
+ subcode: str,
+ severity: ExperimentHealthFindingSeverity,
+ title: str,
+ detail: str,
+ evidence: dict[str, str | int | float | None],
+) -> ExperimentHealthFinding:
+ return ExperimentHealthFinding(
+ code=ExperimentHealthFindingCode.FLAG_SHARED,
+ subcode=subcode,
+ severity=severity,
+ title=title,
+ detail=detail,
+ evidence=evidence,
+ actions=(),
+ diagnostic_ref="E15",
+ )
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index 840fb975bf68..b45c7969e9a4 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -233,6 +233,14 @@ class ExperimentOnFlag:
ended_at: datetime | None
archived: bool
+ @property
+ def is_draft(self) -> bool:
+ return self.launched_at is None
+
+ @property
+ def is_running(self) -> bool:
+ return self.launched_at is not None and self.ended_at is None
+
@frozen
class PrimaryMetric:
@@ -575,8 +583,8 @@ class HealthContext:
flag_called_after_end: bool | None
# The flag's latest recorded call. None when the experiment has no flag, or the flag has no recorded call.
flag_last_called_at: datetime | None
- # The other non-deleted experiments on the flag. None when not loaded: it is loaded only for an ended experiment.
- other_experiments_on_flag: tuple[ExperimentOnFlag, ...] | None
+ # The other non-deleted experiments on the flag.
+ other_experiments_on_flag: tuple[ExperimentOnFlag, ...]
# Whether the team has an active flag, which posthog-js needs to request flags on page load. None when not read:
# it is read only for a draft whose flag has no recorded call.
team_has_active_flag: bool | None
@@ -772,9 +780,7 @@ def _flag_called_after_end(experiment: Experiment) -> bool | None:
return None if _flag_calls_unrecorded(experiment) else False
-def _other_experiments_on_flag(experiment: Experiment) -> tuple[ExperimentOnFlag, ...] | None:
- if not experiment.is_stopped:
- return None
+def _other_experiments_on_flag(experiment: Experiment) -> tuple[ExperimentOnFlag, ...]:
others = (
live_experiments_for_flag(experiment.feature_flag_id)
.filter(team_id=experiment.team_id)
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index d668a72166fc..c324bffbe738 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -30,6 +30,7 @@ from products.experiments.backend.health.checks.metrics_not_matured import metri
from products.experiments.backend.health.checks.multiple_variants import multiple_variants_share
from products.experiments.backend.health.checks.new_flag_no_traffic import new_flag_no_traffic
from products.experiments.backend.health.checks.no_metric import no_metric
+from products.experiments.backend.health.checks.shared_flag import flag_shared
from products.experiments.backend.health.checks.split_changed import split_changed_mid_run
from products.experiments.backend.health.checks.srm import srm
from products.experiments.backend.health.checks.zero_exposures import zero_exposures
@@ -50,6 +51,11 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
name="Flag state that doesn't match the experiment",
run=flag_state,
),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.FLAG_SHARED,
+ name="Other experiments on the same flag",
+ run=flag_shared,
+ ),
HealthCheck(
key=ExperimentHealthCheckKey.FLAG_HISTORY,
name="Unreadable flag history",
diff --git a/products/experiments/backend/health/test/factories.py b/products/experiments/backend/health/test/factories.py
index fffe0ce44b2d..ea2cb1f6c6b3 100644
--- a/products/experiments/backend/health/test/factories.py
+++ b/products/experiments/backend/health/test/factories.py
@@ -28,7 +28,7 @@ RUNNING_CONTEXT = HealthContext(
flag_called_during_run=None,
flag_called_after_end=None,
flag_last_called_at=None,
- other_experiments_on_flag=None,
+ other_experiments_on_flag=(),
team_has_active_flag=None,
baseline_variant_key=None,
primary_metric_count=1,
diff --git a/products/experiments/backend/health/test/test_flag_state.py b/products/experiments/backend/health/test/test_flag_state.py
index 3de6609a00bb..c31c531774c5 100644
--- a/products/experiments/backend/health/test/test_flag_state.py
+++ b/products/experiments/backend/health/test/test_flag_state.py
@@ -31,7 +31,8 @@ from products.feature_flags.backend.models.feature_flag import FeatureFlag
# The flag documents and cases of the `experimentWarning` test in
# frontend/src/scenes/experiments/experimentLogic.test.ts, so that both sides answer alike. A running
# experiment with a deleted flag is one exception: only the server reports it. An ended experiment is another:
-# only the server reads the flag's calls and the other experiments on the flag.
+# only the server reads the flag's calls and the other experiments on the flag. A draft whose flag a running
+# experiment uses is a third, for the same reason.
MULTIVARIANT = {
"groups": [{"properties": [], "rollout_percentage": 100}],
"multivariate": {
@@ -113,7 +114,7 @@ def _context(
archived=archived,
flag_called_after_end=flag_called_after_end,
flag_last_called_at=flag_last_called_at,
- other_experiments_on_flag=other_experiments if status["has_ended"] else None,
+ other_experiments_on_flag=other_experiments,
flag=parse_flag_state(
active=active,
deleted=deleted,
@@ -249,6 +250,16 @@ class TestFlagState(TestCase):
_context(DRAFT, active=True, filters=MULTIVARIANT, archived=True),
None,
),
+ (
+ "draft_flag_serves_a_running_experiment",
+ _context(
+ DRAFT,
+ active=True,
+ filters=MULTIVARIANT,
+ other_experiments=(_other(launched_at=ENDED_AT - timedelta(days=3)),),
+ ),
+ None,
+ ),
("draft_zero_rollout", _context(DRAFT, active=True, filters=ZERO_ROLLOUT), None),
("draft_flag_disabled", _context(DRAFT, active=False, filters=MULTIVARIANT), None),
(
@@ -319,6 +330,21 @@ class TestFlagState(TestCase):
"The experiment is running, but its feature flag is disabled",
),
),
+ (
+ "disabled_while_another_running_experiment_uses_the_flag",
+ _context(
+ RUNNING,
+ active=False,
+ filters=MULTIVARIANT,
+ pause_history=_pause_history(launched_at=LAUNCHED_AFTER_PAUSE_ENTRIES, paused_at=None),
+ other_experiments=(_other(launched_at=LAUNCHED_AFTER_PAUSE_ENTRIES),),
+ ),
+ (
+ "running_but_flag_disabled",
+ ExperimentHealthFindingSeverity.WARNING,
+ "The experiment is running, but its shared feature flag is disabled",
+ ),
+ ),
(
"launched_before_pauses_wrote_entries",
_context(
diff --git a/products/experiments/backend/health/test/test_shared_flag.py b/products/experiments/backend/health/test/test_shared_flag.py
new file mode 100644
index 000000000000..7d3cefaec644
--- /dev/null
+++ b/products/experiments/backend/health/test/test_shared_flag.py
@@ -0,0 +1,115 @@
+from dataclasses import replace
+from datetime import UTC, datetime
+
+from posthog.test.base import BaseTest
+from unittest import TestCase
+
+from parameterized import parameterized
+
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthCheckStatus,
+ ExperimentHealthFinding,
+ ExperimentHealthFindingSeverity,
+)
+from products.experiments.backend.health.checks.shared_flag import flag_shared
+from products.experiments.backend.health.context import ExperimentOnFlag, load_health_context, parse_flag_state
+from products.experiments.backend.health.skips import CheckSkip
+from products.experiments.backend.health.test.factories import DRAFT, ENDED, RUNNING, RUNNING_CONTEXT, Status
+from products.experiments.backend.models.experiment import Experiment
+from products.feature_flags.backend.models.feature_flag import FeatureFlag
+
+LAUNCHED_AT = datetime(2026, 10, 1, tzinfo=UTC)
+ENDED_AT = datetime(2026, 10, 10, tzinfo=UTC)
+
+RUNNING_OTHER = ExperimentOnFlag(launched_at=LAUNCHED_AT, ended_at=None, archived=False)
+DRAFT_OTHER = ExperimentOnFlag(launched_at=None, ended_at=None, archived=False)
+ENDED_OTHER = ExperimentOnFlag(launched_at=LAUNCHED_AT, ended_at=ENDED_AT, archived=False)
+ARCHIVED_DRAFT_OTHER = ExperimentOnFlag(launched_at=None, ended_at=None, archived=True)
+
+WARNING = ExperimentHealthFindingSeverity.WARNING
+INFO = ExperimentHealthFindingSeverity.INFO
+
+CONTEXT = replace(
+ RUNNING_CONTEXT,
+ flag=parse_flag_state(
+ active=True,
+ deleted=False,
+ groups=[{"properties": [], "rollout_percentage": 100}],
+ variants=[{"key": "control", "rollout_percentage": 50}, {"key": "test", "rollout_percentage": 50}],
+ ),
+)
+
+
+class TestFlagShared(TestCase):
+ @parameterized.expand(
+ [
+ (
+ "running_with_a_running_experiment",
+ RUNNING,
+ (RUNNING_OTHER,),
+ ("running_flag_shared_with_running", WARNING, {"other_running_experiments": 1}),
+ ),
+ (
+ "running_with_running_and_draft_experiments",
+ RUNNING,
+ (DRAFT_OTHER, RUNNING_OTHER, RUNNING_OTHER),
+ ("running_flag_shared_with_running", WARNING, {"other_running_experiments": 2}),
+ ),
+ (
+ "running_with_a_draft",
+ RUNNING,
+ (DRAFT_OTHER, ENDED_OTHER),
+ ("running_flag_shared_with_draft", INFO, {"other_draft_experiments": 1}),
+ ),
+ ("running_after_an_ended_experiment", RUNNING, (ENDED_OTHER,), None),
+ ("running_with_an_archived_draft", RUNNING, (ARCHIVED_DRAFT_OTHER,), None),
+ (
+ "draft_on_the_flag_of_a_running_experiment",
+ DRAFT,
+ (RUNNING_OTHER, DRAFT_OTHER),
+ ("draft_flag_shared_with_running", WARNING, {"other_running_experiments": 1}),
+ ),
+ ("draft_with_another_draft", DRAFT, (DRAFT_OTHER, ENDED_OTHER), None),
+ ]
+ )
+ def test_flag_shared(
+ self,
+ _name: str,
+ status: Status,
+ others: tuple[ExperimentOnFlag, ...],
+ expected: tuple[str, ExperimentHealthFindingSeverity, dict[str, int]] | None,
+ ) -> None:
+ finding = flag_shared(replace(CONTEXT, **status, other_experiments_on_flag=others))
+
+ if expected is None:
+ self.assertIsNone(finding)
+ return
+ assert isinstance(finding, ExperimentHealthFinding)
+ self.assertEqual((finding.subcode, finding.severity, dict(finding.evidence)), expected)
+
+ def test_leaves_an_ended_experiment_to_the_flag_state_check(self) -> None:
+ outcome = flag_shared(replace(CONTEXT, **ENDED, other_experiments_on_flag=(RUNNING_OTHER,)))
+
+ assert isinstance(outcome, CheckSkip)
+ self.assertEqual(outcome.status, ExperimentHealthCheckStatus.NOT_APPLICABLE)
+
+
+class TestFlagSharedLoadsTheOtherExperiments(BaseTest):
+ @parameterized.expand(
+ [
+ ("running_experiment_on_the_flag", False, "running_flag_shared_with_running"),
+ ("deleted_experiment_on_the_flag", True, None),
+ ]
+ )
+ def test_reads_the_other_experiments_of_a_running_experiment(
+ self, _name: str, other_deleted: bool, expected_subcode: str | None
+ ) -> None:
+ flag = FeatureFlag.objects.create(team=self.team, key="checkout-test", created_by=self.user)
+ experiment = Experiment.objects.create(team=self.team, name="exp", feature_flag=flag, start_date=LAUNCHED_AT)
+ Experiment.objects.create(
+ team=self.team, name="other", feature_flag=flag, start_date=LAUNCHED_AT, deleted=other_deleted
+ )
+
+ finding = flag_shared(load_health_context(experiment))
+
+ self.assertEqual(finding.subcode if isinstance(finding, ExperimentHealthFinding) else finding, expected_subcode)
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index 678c92212da6..ae2ca31c8774 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2328,6 +2328,7 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]
* * `new_flag_no_traffic` - New Flag No Traffic
* * `early_significance` - Early Significance
* * `metrics_not_matured` - Metrics Not Matured
+ * * `flag_shared` - Flag Shared
*/
export type ExperimentHealthFindingCodeEnumApi =
(typeof ExperimentHealthFindingCodeEnumApi)[keyof typeof ExperimentHealthFindingCodeEnumApi]
@@ -2357,6 +2358,7 @@ export const ExperimentHealthFindingCodeEnumApi = {
NewFlagNoTraffic: 'new_flag_no_traffic',
EarlySignificance: 'early_significance',
MetricsNotMatured: 'metrics_not_matured',
+ FlagShared: 'flag_shared',
} as const
/**
@@ -2428,7 +2430,8 @@ export interface ExperimentHealthFindingApi {
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
* * `new_flag_no_traffic` - New Flag No Traffic
* * `early_significance` - Early Significance
- * * `metrics_not_matured` - Metrics Not Matured */
+ * * `metrics_not_matured` - Metrics Not Matured
+ * * `flag_shared` - Flag Shared */
code: ExperimentHealthFindingCodeEnumApi
/**
* The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.
@@ -2463,6 +2466,7 @@ export interface ExperimentHealthFindingApi {
/**
* * `flag_state` - Flag State
+ * * `flag_shared` - Flag Shared
* * `flag_history` - Flag History
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
@@ -2490,6 +2494,7 @@ export type ExperimentHealthCheckKeyEnumApi =
export const ExperimentHealthCheckKeyEnumApi = {
FlagState: 'flag_state',
+ FlagShared: 'flag_shared',
FlagHistory: 'flag_history',
SplitChangedMidRun: 'split_changed_mid_run',
BucketingChangedMidRun: 'bucketing_changed_mid_run',
@@ -2533,6 +2538,7 @@ export interface ExperimentHealthCheckResultApi {
/** Stable identifier of the check. A check can report several finding codes, so the key is not a finding code.
*
* * `flag_state` - Flag State
+ * * `flag_shared` - Flag Shared
* * `flag_history` - Flag History
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.ts
index 1054ec6c91a8..9f1e9be35172 100644
--- a/products/experiments/frontend/health/debug/healthDebugReport.ts
+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts
@@ -376,6 +376,7 @@ function exposureChecks(input: HealthDebugInput): HealthDebugCheck[] {
export function buildHealthDebugChecks(input: HealthDebugInput): HealthDebugCheck[] {
return [
flagStateCheck(input),
+ serverOnlyCheck(input, 'flag_shared', null),
serverOnlyCheck(input, 'split_changed_mid_run', splitChangedNote),
serverOnlyCheck(input, 'bucketing_changed_mid_run', bucketingChangedNote),
serverOnlyCheck(input, 'release_conditions_changed_mid_run', conditionsChangedNote),
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index dd1fb0b0219a..8e2d5e92b2cc 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -42,6 +42,7 @@ export type ExperimentHealthFindingCode =
| 'new_flag_no_traffic'
| 'early_significance'
| 'metrics_not_matured'
+ | 'flag_shared'
export interface ExperimentHealthFinding {
code: ExperimentHealthFindingCode
diff --git a/products/experiments/skills/diagnosing-experiment-health/references/bias-and-skew.md b/products/experiments/skills/diagnosing-experiment-health/references/bias-and-skew.md
index 73cbf04dd39a..0450e19e9ae4 100644
--- a/products/experiments/skills/diagnosing-experiment-health/references/bias-and-skew.md
+++ b/products/experiments/skills/diagnosing-experiment-health/references/bias-and-skew.md
@@ -384,6 +384,10 @@ The table shows the page's own rules. When the page shows the health findings, i
- When another experiment on the flag is running, or launched after this one ended, the flag's state belongs to that experiment, and the finding is absent. Do not advise disabling the flag from this experiment, even when the banner says so. Disabling it stops a running experiment.
- A draft experiment on the flag, no recorded call of the flag in the last 3 days, or calls that PostHog can't read yet make the finding info, not a warning. A page that shows the health findings shows no banner then.
+For a draft, the server's finding is absent when another experiment on the flag is running, because the flag's state belongs to that experiment.
+The `flag_shared` finding reports the draft instead.
+Do not advise disabling the flag or starting the draft then (E15 in `mid-run-changes.md`).
+
"Adjust the variant distribution" on a running experiment is a change of the split, with what that does to the people already exposed (E3 in `mid-run-changes.md`).
Say so before the user restores a split.
diff --git a/products/experiments/skills/diagnosing-experiment-health/references/mid-run-changes.md b/products/experiments/skills/diagnosing-experiment-health/references/mid-run-changes.md
index b4ff6ddb0b28..a010f342eeea 100644
--- a/products/experiments/skills/diagnosing-experiment-health/references/mid-run-changes.md
+++ b/products/experiments/skills/diagnosing-experiment-health/references/mid-run-changes.md
@@ -395,6 +395,8 @@ The clean approach is:
The duplicate is a new draft with the metrics and settings of the original, without its holdout and its excluded variants.
3. **Give it a new feature flag key.**
A duplicate without a new key shares the original's flag, so a ship or a pause on one affects both.
+ The `flag_shared` health finding reports a shared flag: a warning on a running experiment when another running experiment uses its flag, and on a draft whose flag a running experiment uses, and info on a running experiment whose flag a draft uses.
+ It doesn't count ended experiments, because each experiment reads exposures only between its own start and end.
4. Launch the new experiment under the new flag.
Why a new key and not a reset on the old flag:
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index e04119629068..863d75e91ed8 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -48090,6 +48090,7 @@ export namespace Schemas {
* * `new_flag_no_traffic` - New Flag No Traffic
* * `early_significance` - Early Significance
* * `metrics_not_matured` - Metrics Not Matured
+ * * `flag_shared` - Flag Shared
*/
export type ExperimentHealthFindingCodeEnum = typeof ExperimentHealthFindingCodeEnum[keyof typeof ExperimentHealthFindingCodeEnum];
@@ -48119,6 +48120,7 @@ export namespace Schemas {
NewFlagNoTraffic: 'new_flag_no_traffic',
EarlySignificance: 'early_significance',
MetricsNotMatured: 'metrics_not_matured',
+ FlagShared: 'flag_shared',
} as const;
/**
@@ -48190,7 +48192,8 @@ export namespace Schemas {
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
* * `new_flag_no_traffic` - New Flag No Traffic
* * `early_significance` - Early Significance
- * * `metrics_not_matured` - Metrics Not Matured */
+ * * `metrics_not_matured` - Metrics Not Matured
+ * * `flag_shared` - Flag Shared */
code: ExperimentHealthFindingCodeEnum;
/**
* The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.
@@ -48225,6 +48228,7 @@ export namespace Schemas {
/**
* * `flag_state` - Flag State
+ * * `flag_shared` - Flag Shared
* * `flag_history` - Flag History
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
@@ -48252,6 +48256,7 @@ export namespace Schemas {
export const ExperimentHealthCheckKeyEnum = {
FlagState: 'flag_state',
+ FlagShared: 'flag_shared',
FlagHistory: 'flag_history',
SplitChangedMidRun: 'split_changed_mid_run',
BucketingChangedMidRun: 'bucketing_changed_mid_run',
@@ -48295,6 +48300,7 @@ export namespace Schemas {
/** Stable identifier of the check. A check can report several finding codes, so the key is not a finding code.
*
* * `flag_state` - Flag State
+ * * `flag_shared` - Flag Shared
* * `flag_history` - Flag History
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run