:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(experiments): quiet the paused and pre-launch flag findings (#120872)
Marcel Poelkerc528d326bb7c
diff --git a/products/experiments/backend/health/checks/flag_state.py b/products/experiments/backend/health/checks/flag_state.pyindex 79ccb8fb7d56..d357fb6383bb 100644--- a/products/experiments/backend/health/checks/flag_state.py+++ b/products/experiments/backend/health/checks/flag_state.py@@ -8,6 +8,9 @@ Change both sides together until the page reads these findings.A running experiment whose flag has a 0% rollout is the exception. The flag service and local evaluation serve novariant then, so people who still get one get it from the customer's own assignment, which only the flag's callsshow. Only the server reports that case.++A paused experiment is the other exception. Only the server reads the experiment's activity log, so only the server+tells a pause from the experiment from a flag disabled elsewhere, and reports the pause with its own subcode."""from datetime import UTC, datetime, timedelta@@ -23,6 +26,7 @@ from products.experiments.backend.health.context import (FlagHistoryNotChecked,FlagState,HealthContext,+ PauseHistory,SplitChange,)from products.experiments.backend.health.skips import NO_FLAG, CheckSkip, not_applicable@@ -43,6 +47,12 @@ NOT_STARTED_BUT_MULTIPLE_VARIANTS_ROLLED_OUT = "not_started_but_multiple_variant# pinned: the page sends these as `finding_variant` on its events, as the keys above. No banner uses them.RUNNING_BUT_FLAG_DELETED = "running_but_flag_deleted"ASSIGNED_OUTSIDE_FLAG_SERVICE = "assigned_outside_flag_service"+PAUSED_FROM_EXPERIMENT = "paused_from_experiment"++# The experiment's Pause and Resume write activity log entries since August 25, 2026, and a pause before then has+# none. So a missing entry shows that the flag was disabled outside the experiment only for an experiment launched+# after that date. The day after the merge leaves time for the deploy.+PAUSE_ENTRIES_SINCE = datetime(2026, 8, 26, tzinfo=UTC)# A bootstrap can come from a server's cached evaluation, and an older client can report a variant it cached before# the rollout went to 0%. Calls in this time after the start of the window don't count as assigned outside the flag@@ -89,15 +99,7 @@ def flag_state(ctx: HealthContext) -> ExperimentHealthFinding | None:if ctx.is_running:if ctx.is_paused:- return _finding(- code=ExperimentHealthFindingCode.FLAG_OFF_WHILE_RUNNING,- subcode=RUNNING_BUT_FLAG_DISABLED,- title="The experiment is paused",- detail=(- "The linked feature flag is disabled while the experiment has not been ended. "- "Resume or end the experiment."- ),- )+ return _flag_disabled(ctx.pause_history)if zero_rollout:traffic = ctx.condition_set_trafficif traffic is not None and traffic.people_assigned_outside >= _MIN_PEOPLE_ASSIGNED_OUTSIDE:@@ -141,19 +143,62 @@ def flag_state(ctx: HealthContext) -> ExperimentHealthFinding | None:elif not ctx.archived:# The draft state is also reached again when resetting experiment measurements.if flag.active and _has_multiple_variants_active(flag) and not zero_rollout:+ # Info, because a flag turned on to test the variants before launch is the usual path.return _finding(code=ExperimentHealthFindingCode.FLAG_LIVE_BEFORE_LAUNCH,subcode=NOT_STARTED_BUT_MULTIPLE_VARIANTS_ROLLED_OUT,- title=_NOT_RUNNING_BUT_EXPOSED_TITLE,+ severity=ExperimentHealthFindingSeverity.INFO,+ title="The feature flag is active before launch",detail=(- "This experiment hasn't launched yet, but the linked feature flag is already active and "- "exposing users to multiple variants. Disable the flag, or start the experiment."+ "The linked feature flag is already active with multiple variants, but the experiment hasn't "+ "launched. The results count exposures from launch on, so a user who gets a variant now enters the "+ "results only when exposed again after launch. This is expected while you test the variants. "+ "If users outside your team get them, launch the experiment or disable the flag."),)return None+def _flag_disabled(pause: PauseHistory | None) -> ExperimentHealthFinding:+ if pause is not None and pause.paused_at is not None:+ # The latest pause or resume decides. A flag enabled and disabled again on the flag page after a pause writes+ # no entry, so it still reads as that pause.+ return _finding(+ code=ExperimentHealthFindingCode.FLAG_OFF_WHILE_RUNNING,+ subcode=PAUSED_FROM_EXPERIMENT,+ severity=ExperimentHealthFindingSeverity.INFO,+ title="The experiment is paused",+ detail=(+ f"The experiment was paused on {format_change_time(pause.paused_at, pause.team_timezone)}, so the "+ "linked feature flag serves no variants. Users exposed before the pause stay in the results. Their "+ "events during the pause can still count toward their variant, although the flag doesn't serve it to "+ "them then. Resume the experiment when you want to expose users again."+ ),+ evidence={"paused_at": pause.paused_at.astimezone(UTC).isoformat()},+ )+ if pause is not None and pause.launched_at >= PAUSE_ENTRIES_SINCE:+ return _finding(+ code=ExperimentHealthFindingCode.FLAG_OFF_WHILE_RUNNING,+ subcode=RUNNING_BUT_FLAG_DISABLED,+ title="The experiment is running, but its feature flag is disabled",+ detail=(+ "The linked feature flag was disabled outside the experiment, for example on the feature flag page, "+ "so it serves no variants. Users exposed earlier stay in the results. Resume the experiment to expose "+ "users again, or end it with a conclusion."+ ),+ )+ # The history is unread, or the experiment launched before pauses wrote entries, so the cause is unknown.+ return _finding(+ code=ExperimentHealthFindingCode.FLAG_OFF_WHILE_RUNNING,+ subcode=RUNNING_BUT_FLAG_DISABLED,+ title="The experiment is paused",+ detail=(+ "The linked feature flag is disabled while the experiment has not been ended. Resume or end the experiment."+ ),+ )++def needs_outside_assignment_check(ctx: HealthContext) -> bool:"""Whether this check reads the flag's calls: a running experiment whose flag has a 0% rollout."""flag = ctx.flagdiff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex 0140e35a5e58..20329e0181be 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -203,6 +203,18 @@ class ConditionSetTraffic:people_assigned_outside: int+@frozen+class PauseHistory:+ """The experiment's latest pause or resume since launch, from its activity log. The experiment's Pause and Resume+ write these entries, from the page, the API or MCP. A flag disabled or enabled anywhere else, such as on the flag+ page, writes none."""++ launched_at: datetime+ team_timezone: str+ # The time of the latest entry when it is a pause. None when it is a resume, or the experiment has no entry.+ paused_at: datetime | None++@frozenclass ExposureTotals:total_exposures: Mapping[str, int]@@ -239,6 +251,8 @@ class HealthContext:flag_history: FlagHistory | FlagHistoryNotChecked | None# None when not loaded or unknown. Only the health endpoint loads it, because it queries the events.condition_set_traffic: ConditionSetTraffic | None+ # None when not loaded or unknown. Only the health endpoint loads it, and only for a paused experiment.+ pause_history: PauseHistory | None# The variants that the analysis leaves out of the results.excluded_variant_keys: frozenset[str]@@ -408,5 +422,6 @@ def load_health_context(exposures=exposures,flag_history=flag_history,condition_set_traffic=None,+ pause_history=None,excluded_variant_keys=frozenset(experiment.excluded_variants or []),)diff --git a/products/experiments/backend/health/loader.py b/products/experiments/backend/health/loader.pyindex 21f569bd8abe..ab8e7337afda 100644--- a/products/experiments/backend/health/loader.py+++ b/products/experiments/backend/health/loader.py@@ -8,6 +8,7 @@ from products.experiments.backend.health.condition_set_traffic import load_condifrom products.experiments.backend.health.context import HealthContext, load_health_contextfrom products.experiments.backend.health.exposure_gate import load_flag_historyfrom products.experiments.backend.health.exposure_inputs import ExposureInputs, load_exposure_inputs+from products.experiments.backend.health.pause_history import load_pause_historyfrom products.experiments.backend.models.experiment import Experimentfrom products.experiments.backend.setup_context import ProjectSetupFacts, build_project_setup_facts@@ -62,4 +63,8 @@ def _health_context(experiment: Experiment, reader_access: ExperimentHealthReadeflag_history = load_flag_history(experiment, reader_access=reader_access)context = load_health_context(experiment, flag_history=flag_history)traffic = load_condition_set_traffic(experiment, context, reader_access=reader_access)- return replace(context, condition_set_traffic=traffic)+ return replace(+ context,+ condition_set_traffic=traffic,+ pause_history=load_pause_history(experiment, reader_access=reader_access),+ )diff --git a/products/experiments/backend/health/pause_history.py b/products/experiments/backend/health/pause_history.pynew file mode 100644index 000000000000..21329c8699ab--- /dev/null+++ b/products/experiments/backend/health/pause_history.py@@ -0,0 +1,51 @@+from posthog.exceptions_capture import capture_exception+from posthog.models.activity_logging.activity_log import ActivityLog++from products.experiments.backend.facade.contracts import ExperimentHealthReaderAccess+from products.experiments.backend.health.context import PauseHistory+from products.experiments.backend.models.experiment import Experiment++# pinned: the `activity` of the entries that `ExperimentService.pause_experiment` and `resume_experiment` write.+_PAUSED = "paused"+_RESUMED = "resumed"+++def load_pause_history(experiment: Experiment, *, reader_access: ExperimentHealthReaderAccess) -> PauseHistory | None:+ """+ The pause history of a paused experiment, or None. Never raises: the flag-state check then reports the disabled+ flag without saying who disabled it.+ """+ # The experiment's own `activity` action shows these entries to a caller with `activity_log:read`.+ # `can_read_flag_history` includes that scope, and also viewer access to the flag, which these entries don't+ # need. A reader without flag access gets the finding of an unknown cause, as when the read fails.+ if not experiment.is_paused or not reader_access.can_read_flag_history:+ return None+ try:+ return _load_pause_history(experiment)+ except Exception as error:+ capture_exception(error)+ return None+++def _load_pause_history(experiment: Experiment) -> PauseHistory | None:+ launched_at = experiment.start_date+ if launched_at is None:+ return None+ # A pause before a reset to draft belongs to an earlier run, so only the entries since launch count.+ latest = (+ ActivityLog.objects.filter(+ team_id=experiment.team_id,+ scope="Experiment",+ item_id=str(experiment.id),+ activity__in=(_PAUSED, _RESUMED),+ created_at__gte=launched_at,+ )+ .order_by("-created_at")+ .values_list("activity", "created_at")+ .first()+ )+ return PauseHistory(+ launched_at=launched_at,+ team_timezone=experiment.team.timezone,+ paused_at=latest[1] if latest is not None and latest[0] == _PAUSED else None,+ )diff --git a/products/experiments/backend/health/test/factories.py b/products/experiments/backend/health/test/factories.pyindex 920121bd06bd..883ef4e51258 100644--- a/products/experiments/backend/health/test/factories.py+++ b/products/experiments/backend/health/test/factories.py@@ -30,6 +30,7 @@ RUNNING_CONTEXT = HealthContext(exposures=None,flag_history=None,condition_set_traffic=None,+ pause_history=None,excluded_variant_keys=frozenset(),)diff --git a/products/experiments/backend/health/test/test_flag_state.py b/products/experiments/backend/health/test/test_flag_state.pyindex c5624b6862e9..3879f59261e3 100644--- a/products/experiments/backend/health/test/test_flag_state.py+++ b/products/experiments/backend/health/test/test_flag_state.py@@ -8,7 +8,12 @@ from parameterized import parameterizedfrom products.experiments.backend.facade.contracts import ExperimentHealthFindingCode, ExperimentHealthFindingSeverityfrom products.experiments.backend.health.checks.flag_state import flag_state-from products.experiments.backend.health.context import ConditionSetTraffic, HealthContext, parse_flag_state+from products.experiments.backend.health.context import (+ ConditionSetTraffic,+ HealthContext,+ PauseHistory,+ parse_flag_state,+)from products.experiments.backend.health.test.factories import DRAFT, ENDED, RUNNING, RUNNING_CONTEXT, Status# The flag documents and cases of the `experimentWarning` test in@@ -55,6 +60,15 @@ def _traffic(people_assigned_outside: int) -> ConditionSetTraffic:)+def _pause_history(*, launched_at: datetime, paused_at: datetime | None) -> PauseHistory:+ return PauseHistory(launched_at=launched_at, team_timezone="Europe/Berlin", paused_at=paused_at)+++LAUNCHED_BEFORE_PAUSE_ENTRIES = datetime(2026, 8, 1, 9, 0, tzinfo=UTC)+LAUNCHED_AFTER_PAUSE_ENTRIES = datetime(2026, 9, 1, 9, 0, tzinfo=UTC)+PAUSED_AT = datetime(2026, 10, 12, 13, 4, tzinfo=UTC)++def _context(status: Status,*,@@ -63,6 +77,7 @@ def _context(deleted: bool = False,archived: bool = False,traffic: ConditionSetTraffic | None = None,+ pause_history: PauseHistory | None = None,) -> HealthContext:return replace(RUNNING_CONTEXT,@@ -76,6 +91,7 @@ def _context(variants=(filters.get("multivariate") or {}).get("variants"),),condition_set_traffic=traffic,+ pause_history=pause_history,)@@ -189,6 +205,88 @@ class TestFlagState(TestCase):self.assertEqual((finding.code, finding.subcode), (code, subcode))self.assertEqual(finding.evidence.get("variant_key"), variant_key)+ @parameterized.expand(+ [+ (+ "paused_from_the_experiment",+ _context(+ RUNNING,+ active=False,+ filters=MULTIVARIANT,+ pause_history=_pause_history(launched_at=LAUNCHED_BEFORE_PAUSE_ENTRIES, paused_at=PAUSED_AT),+ ),+ ("paused_from_experiment", ExperimentHealthFindingSeverity.INFO, "The experiment is paused"),+ ),+ (+ "disabled_outside_the_experiment",+ _context(+ RUNNING,+ active=False,+ filters=MULTIVARIANT,+ pause_history=_pause_history(launched_at=LAUNCHED_AFTER_PAUSE_ENTRIES, paused_at=None),+ ),+ (+ "running_but_flag_disabled",+ ExperimentHealthFindingSeverity.WARNING,+ "The experiment is running, but its feature flag is disabled",+ ),+ ),+ (+ "launched_before_pauses_wrote_entries",+ _context(+ RUNNING,+ active=False,+ filters=MULTIVARIANT,+ pause_history=_pause_history(launched_at=LAUNCHED_BEFORE_PAUSE_ENTRIES, paused_at=None),+ ),+ ("running_but_flag_disabled", ExperimentHealthFindingSeverity.WARNING, "The experiment is paused"),+ ),+ (+ "pause_history_unknown",+ _context(RUNNING, active=False, filters=MULTIVARIANT),+ ("running_but_flag_disabled", ExperimentHealthFindingSeverity.WARNING, "The experiment is paused"),+ ),+ (+ "draft_flag_distributing_variants",+ _context(DRAFT, active=True, filters=MULTIVARIANT),+ (+ "not_started_but_multiple_variants_rolled_out",+ ExperimentHealthFindingSeverity.INFO,+ "The feature flag is active before launch",+ ),+ ),+ ]+ )+ def test_severity_follows_whether_the_state_needs_action(+ self, _name: str, ctx: HealthContext, expected: tuple[str, ExperimentHealthFindingSeverity, str]+ ) -> None:+ finding = flag_state(ctx)++ assert finding is not None+ self.assertEqual((finding.subcode, finding.severity, finding.title), expected)++ def test_a_pause_from_the_experiment_names_its_date(self) -> None:+ finding = flag_state(+ _context(+ RUNNING,+ active=False,+ filters=MULTIVARIANT,+ pause_history=_pause_history(launched_at=LAUNCHED_AFTER_PAUSE_ENTRIES, paused_at=PAUSED_AT),+ )+ )++ assert finding is not None+ self.assertEqual(+ (finding.detail, dict(finding.evidence)),+ (+ "The experiment was paused on October 12, 2026 at 3:04 PM CEST, so the linked feature flag serves no "+ "variants. Users exposed before the pause stay in the results. Their events during the pause can "+ "still count toward their variant, although the flag doesn't serve it to them then. Resume the "+ "experiment when you want to expose users again.",+ {"paused_at": "2026-10-12T13:04:00+00:00"},+ ),+ )+def test_variants_assigned_outside_the_flag_service_are_info(self) -> None:finding = flag_state(_context(RUNNING, active=True, filters=ZERO_ROLLOUT, traffic=_traffic(1234)))diff --git a/products/experiments/backend/health/test/test_pause_history.py b/products/experiments/backend/health/test/test_pause_history.pynew file mode 100644index 000000000000..944c4938fb3d--- /dev/null+++ b/products/experiments/backend/health/test/test_pause_history.py@@ -0,0 +1,77 @@+from dataclasses import replace+from datetime import UTC, datetime, timedelta++from posthog.test.base import BaseTest++from parameterized import parameterized++from posthog.models.activity_logging.activity_log import ActivityLog++from products.experiments.backend.facade.contracts import ExperimentHealthReaderAccess+from products.experiments.backend.health.context import PauseHistory+from products.experiments.backend.health.pause_history import load_pause_history+from products.experiments.backend.models.experiment import Experiment+from products.feature_flags.backend.models.feature_flag import FeatureFlag++LAUNCHED_AT = datetime(2026, 10, 1, 9, 0, tzinfo=UTC)+FULL_ACCESS = ExperimentHealthReaderAccess(+ can_read_flag_history=True, can_query_events=True, denied_event_properties=frozenset()+)+++def _history(paused_after_hours: int | None) -> PauseHistory:+ return PauseHistory(+ launched_at=LAUNCHED_AT,+ team_timezone="UTC",+ paused_at=None if paused_after_hours is None else LAUNCHED_AT + timedelta(hours=paused_after_hours),+ )+++class TestLoadPauseHistory(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ flag = FeatureFlag.objects.create(team=self.team, created_by=self.user, key="paused-flag", active=False)+ self.experiment = Experiment.objects.create(+ team=self.team, name="Paused", feature_flag=flag, start_date=LAUNCHED_AT+ )+ self.other_experiment = Experiment.objects.create(+ team=self.team, name="Other", feature_flag=flag, start_date=LAUNCHED_AT+ )++ @parameterized.expand(+ [+ (+ "latest_entry_is_a_pause",+ [("paused", 1, True), ("resumed", 2, True), ("paused", 3, True)],+ FULL_ACCESS,+ _history(3),+ ),+ ("latest_entry_is_a_resume", [("paused", 1, True), ("resumed", 2, True)], FULL_ACCESS, _history(None)),+ ("pause_before_this_launch", [("paused", -1, True)], FULL_ACCESS, _history(None)),+ ("pause_of_another_experiment", [("resumed", 2, True), ("paused", 3, False)], FULL_ACCESS, _history(None)),+ (+ "reader_without_history_access",+ [("paused", 1, True)],+ replace(FULL_ACCESS, can_read_flag_history=False),+ None,+ ),+ ]+ )+ def test_reads_the_latest_pause_or_resume_since_launch(+ self,+ _name: str,+ entries: list[tuple[str, int, bool]],+ reader_access: ExperimentHealthReaderAccess,+ expected: PauseHistory | None,+ ) -> None:+ for activity, hours_after_launch, own in entries:+ ActivityLog.objects.create(+ team_id=self.team.id,+ organization_id=self.organization.id,+ scope="Experiment",+ item_id=str((self.experiment if own else self.other_experiment).id),+ activity=activity,+ created_at=LAUNCHED_AT + timedelta(hours=hours_after_launch),+ )++ self.assertEqual(load_pause_history(self.experiment, reader_access=reader_access), expected)diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.pyindex 54681a689e56..afbc548b4f85 100644--- a/products/experiments/backend/presentation/views.py+++ b/products/experiments/backend/presentation/views.py@@ -949,6 +949,12 @@ class EnterpriseExperimentsViewSet(is the flag's `last_called_at`, which a periodic task updates, so it trails a call. Thecheck reports `not_checked` when the organization's flag calls skip the events table,and when the project has no active flag, because the web SDK then requests no flags.++ A running experiment whose flag is disabled gets `paused_from_experiment`, as info, when+ its latest pause or resume since launch is a pause of the experiment. Otherwise it gets+ `running_but_flag_disabled`, as a warning. A caller who can't view the linked flag's+ history gets the warning, because the pause is not read. A draft whose flag is already+ active with multiple variants gets `not_started_but_multiple_variants_rolled_out` as info."""experiment: Experiment = self.get_object()health = get_experiment_health(diff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.tsindex 7129106ae19e..0ee63c6ea6d6 100644--- a/products/experiments/frontend/generated/api.ts+++ b/products/experiments/frontend/generated/api.ts@@ -737,6 +737,12 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =* is the flag's `last_called_at`, which a periodic task updates, so it trails a call. The* check reports `not_checked` when the organization's flag calls skip the events table,* and when the project has no active flag, because the web SDK then requests no flags.+ *+ * A running experiment whose flag is disabled gets `paused_from_experiment`, as info, when+ * its latest pause or resume since launch is a pause of the experiment. Otherwise it gets+ * `running_but_flag_disabled`, as a warning. A caller who can't view the linked flag's+ * history gets the warning, because the pause is not read. A draft whose flag is already+ * active with multiple variants gets `not_started_but_multiple_variants_rolled_out` as info.*/export const experimentsHealthRetrieve = async (projectId: string,