Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): note a draft whose feature flag has no traffic yet (#120861)

Marcel Poelker9c9170cbe0c8
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index e44944478189..b81a3edb0279 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -3285,9 +3285,9 @@ snapshots:
experiments-healthchecksoverview--checks-that-could-not-run--light:
hash: v1.k794b7964.8c51277755331ab88e7b5bf4bcdc2c9a3c0c9c87fa5ea946f549b70df6c58ce6.vJaps4AlW1x8oWaPFZMzIky0VccXRtIpAw5elQ8Q424
experiments-healthchecksoverview--draft--dark:
- hash: v1.k794b7964.f9bbebb78696c5c06e4f5572959db8f2874162e0e965c8db58cd4669cc7da9b2.O6UuEfwDD1O2LsejFRqibQsm_1_O3vKAK-bnRQ-J1kA
+ hash: v1.k794b7964.998711a45ae74b0bc74066f1d70b2b58c809e9d9e0bd7d4d0b541698a312cad3.IPJlc0qZzHtFDsZ5Gb06zukOf9kk_JnT6-OU2V-Pwa0
experiments-healthchecksoverview--draft--light:
- hash: v1.k794b7964.b68bab92c07165ef321158a7d33578cc1916c22da7b7653e6a4d1733dc5408c7.H9wWUyEizaXI4E6yp0lk1DzoPLAKSdtVXkzmA59Z4SU
+ hash: v1.k794b7964.5b098ee13c120e815fdf13e18a59d6234a4c904e747a7cc83fec0e1406cd3491.YrNkI7P5QfXa-TnjviRwcQIIML09K1-POg889feICRg
experiments-howtoreadtooltip--intervals-bayesian--dark:
hash: v1.k794b7964.a06d267cee9cb54bafc732335b35d461d3c1a59250c6daa8becace0fcb8492d2.vR5tecOozn6B51xE8LrByAFVsi03J4w-iUu-j2hQUTU
experiments-howtoreadtooltip--intervals-bayesian--light:
@@ -10751,9 +10751,9 @@ snapshots:
scenes-app-experiments--experiment-frequentist-two-variants--light:
hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmII
scenes-app-experiments--experiment-health-debug-tab--dark:
- hash: v1.k794b7964.8a6f16e2e2dd9eddfcd892926ea82e83e96dc8d4d0a7a691b4bef4bd95890e80.fmB2_WJ6xKnT5hBKzZ8t2Ba1jDP0XvVrjbQitJk9FH0
+ hash: v1.k794b7964.eae73ea7b85aef09cd37d294cdb0bd2cf191348232c9df704588d2c25499c839.GaCHEbAy8iVYYnI_jir5s_XFuJNNHii1sYqzXPlf3i4
scenes-app-experiments--experiment-health-debug-tab--light:
- hash: v1.k794b7964.699a38e41fb7d0e936e5db90702e786fdde6692b1cccd8d91acc250672c5653c.b8tZKLOytCcqqZ9N3fvxU0Ztf0ELU-dbQ39SiM9sNSo
+ hash: v1.k794b7964.f7b8889003f93b6a829f42658ac4c50a895d28d7d417da5c336cd5ad7b918886.OHBwsWdtYlEQAHwZ3Hl24xUUeB8nAWC8gfZ_dGXBT3w
scenes-app-experiments--experiment-recordings-dropped-metric--dark:
hash: v1.k794b7964.e0e7181f2c9e61a8324eff9aa7a9230fef25b3cf37e0b24ef875780186ed25dc.9VEvZ4iVieVJwmDol0gwtAsr3wgZYW5gj2KXAGZJfiw
scenes-app-experiments--experiment-recordings-dropped-metric--light:
diff --git a/posthog/models/remote_config.py b/posthog/models/remote_config.py
index 4677741a8a99..48fc5d00ae8f 100644
--- a/posthog/models/remote_config.py
+++ b/posthog/models/remote_config.py
@@ -277,7 +277,7 @@ class RemoteConfig(UUIDTModel):
from posthog.plugins.site import get_decide_site_apps
from products.error_tracking.backend.facade import build_error_tracking_config
- from products.feature_flags.backend.models.feature_flag import FeatureFlag
+ from products.feature_flags.backend.facade.flags import has_active_flags
from products.messaging.backend.facade.api import build_push_config
from products.surveys.backend.api.survey import get_surveys_opt_in, get_surveys_response
from products.web_analytics.backend.remote_config import build_heatmaps_config
@@ -291,7 +291,7 @@ class RemoteConfig(UUIDTModel):
# NOTE: Let's try and keep this tidy! Follow the styling of the values already here...
config = {
"supportedCompression": ["gzip", "gzip-js"],
- "hasFeatureFlags": FeatureFlag.objects.filter(team=team, active=True).exists(),
+ "hasFeatureFlags": has_active_flags(team),
"captureDeadClicks": bool(team.capture_dead_clicks),
"capturePerformance": (
{
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 720489e84d4b..736feb9fb5e3 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -9734,6 +9734,7 @@ export namespace Schemas {
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `new_flag_no_traffic` - New Flag No Traffic
*/
export type ExperimentHealthFindingCodeEnum =
| "flag_off_while_running"
@@ -9749,7 +9750,8 @@ export namespace Schemas {
| "flag_variants_mismatch"
| "split_changed_mid_run"
| "bucketing_changed_mid_run"
- | "release_conditions_changed_mid_run";
+ | "release_conditions_changed_mid_run"
+ | "new_flag_no_traffic";
/**
* * `critical` - Critical severity
* * `warning` - Warning severity
@@ -9796,6 +9798,7 @@ export namespace Schemas {
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `new_flag_no_traffic` - New Flag No Traffic
*/
code: ExperimentHealthFindingCodeEnum;
/**
@@ -9839,6 +9842,7 @@ export namespace Schemas {
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `new_flag_no_traffic` - New Flag No Traffic
* * `no_metric` - No Metric
* * `zero_exposures` - Zero Exposures
* * `srm` - Srm
@@ -9853,6 +9857,7 @@ export namespace Schemas {
| "release_conditions_changed_mid_run"
| "flag_variants_mismatch"
| "forced_variant_release_condition"
+ | "new_flag_no_traffic"
| "no_metric"
| "zero_exposures"
| "srm"
@@ -9880,6 +9885,7 @@ export namespace Schemas {
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `new_flag_no_traffic` - New Flag No Traffic
* * `no_metric` - No Metric
* * `zero_exposures` - Zero Exposures
* * `srm` - Srm
diff --git a/products/experiments/backend/experiment_service.py b/products/experiments/backend/experiment_service.py
index 507fb750d312..021b10c49507 100644
--- a/products/experiments/backend/experiment_service.py
+++ b/products/experiments/backend/experiment_service.py
@@ -1426,8 +1426,9 @@ class ExperimentService:
# Every path saves the launch before it reports it, so an analytics failure must not fail the request.
try:
flag_age = timezone.now() - experiment.feature_flag.created_at
+ health_context = load_health_context(experiment)
# Every launch reports the findings it leaves open, also for a launcher who has no health findings.
- findings = evaluate_health(load_health_context(experiment), EXPERIMENT_HEALTH_CHECKS).findings
+ findings = evaluate_health(health_context, EXPERIMENT_HEALTH_CHECKS).findings
self._report_lifecycle_event(
experiment,
"experiment launched",
@@ -1440,6 +1441,9 @@ class ExperimentService:
and experiment.start_date is not None
and timezone.now() - experiment.start_date > BACKDATED_LAUNCH_THRESHOLD,
"flag_age_seconds": int(flag_age.total_seconds()),
+ # The launch doesn't move `last_called_at`, so this is the state before the launch, and the only
+ # time it is recorded: a later call overwrites it. None when the organization's calls can't be read.
+ "flag_called_before_launch": health_context.flag_called,
"health_finding_codes": [finding.code.value for finding in findings],
# pinned: `code:subcode`, as on `experiment viewed`. The code prefix keeps a subcode
# unambiguous when two codes use the same subcode.
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index 15389bc22be3..87ecf74bc318 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -112,6 +112,7 @@ class ExperimentHealthFindingCode(LabeledStrEnum):
SPLIT_CHANGED_MID_RUN = "split_changed_mid_run"
BUCKETING_CHANGED_MID_RUN = "bucketing_changed_mid_run"
RELEASE_CONDITIONS_CHANGED_MID_RUN = "release_conditions_changed_mid_run"
+ NEW_FLAG_NO_TRAFFIC = "new_flag_no_traffic"
class HealthExposureInputsStatus(LabeledStrEnum):
@@ -190,6 +191,7 @@ class ExperimentHealthCheckKey(LabeledStrEnum):
RELEASE_CONDITIONS_CHANGED_MID_RUN = "release_conditions_changed_mid_run"
FLAG_VARIANTS_MISMATCH = "flag_variants_mismatch"
FORCED_VARIANT_RELEASE_CONDITION = "forced_variant_release_condition"
+ NEW_FLAG_NO_TRAFFIC = "new_flag_no_traffic"
NO_METRIC = "no_metric"
ZERO_EXPOSURES = "zero_exposures"
SRM = "srm"
diff --git a/products/experiments/backend/health/checks/new_flag_no_traffic.py b/products/experiments/backend/health/checks/new_flag_no_traffic.py
new file mode 100644
index 000000000000..cb84a80441bb
--- /dev/null
+++ b/products/experiments/backend/health/checks/new_flag_no_traffic.py
@@ -0,0 +1,72 @@
+"""
+A draft whose feature flag has no recorded call. Pure functions, no I/O.
+
+The experiment's users get their variants from the flag, so a launch before the flag gets traffic starts the run with
+nobody in it. The finding names the missing traffic, not missing code: a flag can be evaluated in code that has no
+users yet, or through a call that records nothing.
+"""
+
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFinding,
+ ExperimentHealthFindingCode,
+ ExperimentHealthFindingSeverity,
+)
+from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.health.skips import (
+ FLAG_DELETED,
+ NO_FLAG,
+ NOT_ON_ARCHIVED,
+ RUNS_BEFORE_LAUNCH,
+ CheckSkip,
+ not_applicable,
+ not_checked,
+)
+
+# `last_called_at` moves only when `sync_feature_flag_last_called` runs, every 30 minutes (`posthog/tasks/scheduled.py`).
+# A run reads the calls up to a buffer before its start (`FEATURE_FLAG_LAST_CALLED_AT_SYNC_*_BUFFER_SECONDS`, 1 or 15
+# minutes by default), so a call waits up to one interval plus that buffer. Update this when the schedule changes.
+LAST_CALLED_AT_LAG = "an hour"
+
+
+def new_flag_no_traffic_skip(ctx: HealthContext) -> CheckSkip | None:
+ if ctx.archived:
+ return not_applicable(NOT_ON_ARCHIVED)
+ # After launch, the zero exposures check reports an experiment without traffic.
+ if ctx.is_launched:
+ return not_applicable(RUNS_BEFORE_LAUNCH)
+ if ctx.flag is None:
+ return not_applicable(NO_FLAG)
+ if ctx.flag.deleted:
+ return not_applicable(FLAG_DELETED)
+ if ctx.flag_called is None:
+ return not_checked(
+ "Couldn't check because this organization's flag calls are stored where this check can't read them"
+ )
+ # Without an active flag the web SDK records no call, so the flag gets traffic only after the launch activates it,
+ # and the finding's advice could not be followed.
+ if ctx.flag_called is False and ctx.team_has_active_flag is not True:
+ return not_checked(
+ "Couldn't check because PostHog's JavaScript SDK only requests flags once a flag in this project is enabled"
+ )
+ return None
+
+
+def new_flag_no_traffic(ctx: HealthContext) -> ExperimentHealthFinding | None:
+ if ctx.archived or ctx.is_launched or ctx.flag is None or ctx.flag.deleted:
+ return None
+ if ctx.flag_called is not False or ctx.team_has_active_flag is not True:
+ return None
+ return ExperimentHealthFinding(
+ code=ExperimentHealthFindingCode.NEW_FLAG_NO_TRAFFIC,
+ subcode=None,
+ severity=ExperimentHealthFindingSeverity.INFO,
+ title="The feature flag has no traffic yet",
+ detail=(
+ "PostHog hasn't recorded any calls to the linked feature flag yet. The experiment only counts users after "
+ "your code evaluates the flag, so launch after the flag starts getting traffic. New calls can take up to "
+ f"{LAST_CALLED_AT_LAG} to show up here."
+ ),
+ evidence={},
+ actions=(),
+ diagnostic_ref=None,
+ )
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index a61d93b19c01..0140e35a5e58 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -3,6 +3,8 @@ from datetime import datetime
from enum import StrEnum
from typing import Any
+from django.conf import settings
+
from posthog.schema import MultipleVariantHandling
from posthog.dataclasses import frozen
@@ -10,7 +12,9 @@ from posthog.dataclasses import frozen
from products.experiments.backend.facade.contracts import ExperimentHealthFlagChangeKind
from products.experiments.backend.metric_resolution import saved_metric_link_role, saved_metric_links
from products.experiments.backend.models.experiment import Experiment
+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
from products.feature_flags.backend.facade.filters import condition_aggregation, pinned_variant, reachable_conditions
+from products.feature_flags.backend.facade.flags import get_organization_flag_evaluations_mode, has_active_flags
@frozen
@@ -220,6 +224,12 @@ class HealthContext:
is_paused: bool
archived: bool
flag: FlagState | None
+ # Whether PostHog has recorded a call of the flag. None when the experiment has no flag, or when PostHog can't
+ # record the calls of this organization's flags.
+ flag_called: bool | None
+ # Whether the team has an active flag, which posthog-js needs to request flags on page load. None when not read:
+ # it is read only for a draft whose flag has no recorded call.
+ team_has_active_flag: bool | None
# The baseline set in the experiment's settings. None when unset: the analysis then picks a variant of the flag.
baseline_variant_key: str | None
primary_metric_count: int
@@ -348,6 +358,25 @@ def _load_flag_state(experiment: Experiment) -> FlagState | None:
)
+def _flag_called(experiment: Experiment) -> bool | None:
+ # django-stubs types the id as int, but an unsaved instance can carry None.
+ feature_flag_id: int | None = experiment.feature_flag_id
+ if feature_flag_id is None:
+ return None
+ if experiment.feature_flag.last_called_at is not None:
+ return True
+ # The task that sets `last_called_at` reads `$feature_flag_called` from the events table, unless it is set to
+ # read flag_evaluations. On FLAG_EVALUATIONS_ONLY, ingestion writes no `$feature_flag_called` to the events table
+ # for the teams in its allowlist, which this side can't read. Their flags keep no `last_called_at` however often
+ # they are called.
+ if settings.FEATURE_FLAG_LAST_CALLED_AT_SYNC_SOURCE != "flag_evaluations" and (
+ get_organization_flag_evaluations_mode(experiment.team.organization_id)
+ == FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY
+ ):
+ return None
+ return False
+
+
def _configured_baseline_variant_key(stats_config: object) -> str | None:
# An empty key counts as unset, as in `get_baseline_variant_key`.
key = stats_config.get("baseline_variant_key") if isinstance(stats_config, dict) else None
@@ -361,6 +390,7 @@ def load_health_context(
flag_history: FlagHistory | FlagHistoryNotChecked | None = None,
) -> HealthContext:
shared_metric_roles = [saved_metric_link_role(link) for link in saved_metric_links(experiment)]
+ flag_called = _flag_called(experiment)
return HealthContext(
is_launched=experiment.is_launched,
is_running=experiment.is_running,
@@ -368,6 +398,10 @@ def load_health_context(
is_paused=experiment.is_paused,
archived=experiment.archived,
flag=_load_flag_state(experiment),
+ flag_called=flag_called,
+ team_has_active_flag=(
+ has_active_flags(experiment.team) if flag_called is False and not experiment.is_launched else None
+ ),
baseline_variant_key=_configured_baseline_variant_key(experiment.stats_config),
primary_metric_count=len(experiment.metrics or []) + shared_metric_roles.count("primary"),
secondary_metric_count=len(experiment.metrics_secondary or []) + shared_metric_roles.count("secondary"),
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index d83a94d8f1fb..588ddfc89cd9 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -28,6 +28,7 @@ from products.experiments.backend.health.checks.multiple_variants import (
multiple_variants_share,
multiple_variants_share_skip,
)
+from products.experiments.backend.health.checks.new_flag_no_traffic import new_flag_no_traffic, new_flag_no_traffic_skip
from products.experiments.backend.health.checks.no_metric import no_metric, no_metric_skip
from products.experiments.backend.health.checks.split_changed import split_changed_mid_run
from products.experiments.backend.health.checks.srm import srm, srm_skip
@@ -89,6 +90,12 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
find=forced_variant_release_condition,
skip=forced_variant_skip,
),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.NEW_FLAG_NO_TRAFFIC,
+ name="Feature flag traffic",
+ find=new_flag_no_traffic,
+ skip=new_flag_no_traffic_skip,
+ ),
HealthCheck(
key=ExperimentHealthCheckKey.NO_METRIC,
name="Metrics",
diff --git a/products/experiments/backend/health/skips.py b/products/experiments/backend/health/skips.py
index b08ec8c26e89..3b765dcb5c0d 100644
--- a/products/experiments/backend/health/skips.py
+++ b/products/experiments/backend/health/skips.py
@@ -13,6 +13,7 @@ class CheckSkip:
RUNS_AFTER_LAUNCH = "Runs after launch"
+RUNS_BEFORE_LAUNCH = "Runs only before launch"
NOT_AFTER_END = "Doesn't run after the experiment ends"
NOT_ON_ARCHIVED = "Doesn't run on archived experiments"
NO_FLAG = "The experiment has no feature flag"
diff --git a/products/experiments/backend/health/test/factories.py b/products/experiments/backend/health/test/factories.py
index acc1c8ac214a..920121bd06bd 100644
--- a/products/experiments/backend/health/test/factories.py
+++ b/products/experiments/backend/health/test/factories.py
@@ -22,6 +22,8 @@ RUNNING_CONTEXT = HealthContext(
is_paused=False,
archived=False,
flag=None,
+ flag_called=None,
+ team_has_active_flag=None,
baseline_variant_key=None,
primary_metric_count=1,
secondary_metric_count=0,
diff --git a/products/experiments/backend/health/test/test_new_flag_no_traffic.py b/products/experiments/backend/health/test/test_new_flag_no_traffic.py
new file mode 100644
index 000000000000..2aa5e1baf623
--- /dev/null
+++ b/products/experiments/backend/health/test/test_new_flag_no_traffic.py
@@ -0,0 +1,107 @@
+from datetime import datetime
+
+from posthog.test.base import BaseTest
+
+from django.test import override_settings
+from django.utils import timezone
+
+from parameterized import parameterized
+
+from products.experiments.backend.facade.api import get_experiment_health
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthCheckKey,
+ ExperimentHealthCheckStatus,
+ ExperimentHealthReaderAccess,
+)
+from products.experiments.backend.models.experiment import Experiment
+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
+from products.feature_flags.backend.models.feature_flag import FeatureFlag
+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
+
+FULL_ACCESS = ExperimentHealthReaderAccess(
+ can_read_flag_history=True, can_query_events=True, denied_event_properties=frozenset()
+)
+Status = ExperimentHealthCheckStatus
+
+
+class TestNewFlagNoTraffic(BaseTest):
+ @parameterized.expand(
+ [
+ ("called", timezone.now(), False, False, FlagEvaluationsMode.EVENTS, "events", Status.PASSED),
+ ("never_called", None, False, True, FlagEvaluationsMode.EVENTS, "events", Status.FINDING),
+ (
+ "never_called_with_its_own_flag_active",
+ None,
+ True,
+ False,
+ FlagEvaluationsMode.EVENTS,
+ "events",
+ Status.FINDING,
+ ),
+ (
+ "never_called_without_an_active_flag",
+ None,
+ False,
+ False,
+ FlagEvaluationsMode.EVENTS,
+ "events",
+ Status.NOT_CHECKED,
+ ),
+ (
+ "calls_kept_out_of_events",
+ None,
+ False,
+ True,
+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ "events",
+ Status.NOT_CHECKED,
+ ),
+ (
+ "calls_kept_out_of_events_with_the_sync_reading_flag_evaluations",
+ None,
+ False,
+ True,
+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ "flag_evaluations",
+ Status.FINDING,
+ ),
+ ]
+ )
+ def test_reads_the_flags_recorded_calls_on_a_draft(
+ self,
+ _name: str,
+ last_called_at: datetime | None,
+ flag_active: bool,
+ another_active_flag: bool,
+ mode: FlagEvaluationsMode,
+ sync_source: str,
+ expected: ExperimentHealthCheckStatus,
+ ) -> None:
+ OrganizationFeatureFlagsConfig.objects.update_or_create(
+ organization=self.organization, defaults={"flag_evaluations_mode": mode}
+ )
+ if another_active_flag:
+ FeatureFlag.objects.create(team=self.team, key="live-flag", created_by=self.user, active=True)
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ key="new-flag",
+ created_by=self.user,
+ active=flag_active,
+ last_called_at=last_called_at,
+ filters={
+ "groups": [{"properties": [], "rollout_percentage": 100}],
+ "multivariate": {
+ "variants": [
+ {"key": "control", "rollout_percentage": 50},
+ {"key": "test", "rollout_percentage": 50},
+ ]
+ },
+ },
+ )
+ experiment = Experiment.objects.create(team=self.team, name="draft", feature_flag=flag)
+
+ with override_settings(FEATURE_FLAG_LAST_CALLED_AT_SYNC_SOURCE=sync_source):
+ health = get_experiment_health(team_id=self.team.id, experiment_id=experiment.id, reader_access=FULL_ACCESS)
+
+ check = next(check for check in health.checks if check.key == ExperimentHealthCheckKey.NEW_FLAG_NO_TRAFFIC)
+ self.assertEqual(check.status, expected)
diff --git a/products/experiments/backend/health/test/test_registry.py b/products/experiments/backend/health/test/test_registry.py
index 6402aa73756c..98fb6f224292 100644
--- a/products/experiments/backend/health/test/test_registry.py
+++ b/products/experiments/backend/health/test/test_registry.py
@@ -189,6 +189,11 @@ class TestEvaluate(TestCase):
},
{Key.MULTIPLE_VARIANTS_SHARE: Status.NOT_CHECKED, Key.BIAS_RISK_MULTIPLE_EXCLUDED: Status.PASSED},
),
+ (
+ "flag_traffic_is_left_to_zero_exposures_after_launch",
+ {"flag_called": False},
+ {Key.NEW_FLAG_NO_TRAFFIC: Status.NOT_APPLICABLE},
+ ),
(
"zero_exposures_finds_a_day_after_launch",
{"exposures": replace(NO_EXPOSURES, hours_since_launch=24)},
diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.py
index 2b7c4cc83b54..54681a689e56 100644
--- a/products/experiments/backend/presentation/views.py
+++ b/products/experiments/backend/presentation/views.py
@@ -944,6 +944,11 @@ class EnterpriseExperimentsViewSet(
flag version that are no local evaluation, from an hour after the window starts. When
every condition that includes users pins the variant whose key one property holds,
`assigned_by_property` replaces `all_conditions_pinned`.
+
+ A draft whose flag has no recorded call gets the info `new_flag_no_traffic`. The record
+ is the flag's `last_called_at`, which a periodic task updates, so it trails a call. The
+ check reports `not_checked` when the organization's flag calls skip the events table,
+ and when the project has no active flag, because the web SDK then requests no flags.
"""
experiment: Experiment = self.get_object()
health = get_experiment_health(
diff --git a/products/experiments/backend/test/test_experiment_service.py b/products/experiments/backend/test/test_experiment_service.py
index f363d332af51..5544c861f002 100644
--- a/products/experiments/backend/test/test_experiment_service.py
+++ b/products/experiments/backend/test/test_experiment_service.py
@@ -2818,7 +2818,9 @@ class TestExperimentService(APIBaseTest):
def _create_launchable_experiment_on_flag_of_age(self, age: timedelta, **kwargs: Any) -> Experiment:
experiment = self._create_launchable_experiment(**kwargs)
- FeatureFlag.objects.filter(pk=experiment.feature_flag_id).update(created_at=timezone.now() - age)
+ FeatureFlag.objects.filter(pk=experiment.feature_flag_id).update(
+ created_at=timezone.now() - age, last_called_at=timezone.now() - timedelta(hours=1)
+ )
experiment.feature_flag.refresh_from_db()
return experiment
@@ -3931,7 +3933,7 @@ class TestExperimentService(APIBaseTest):
"launch_endpoint",
lambda self: self._create_launchable_experiment(name="Path L", feature_flag_key="path-launch-flag"),
lambda service, experiment, request: service.launch_experiment(experiment, request=request),
- [("launch_endpoint", 0, [], [], [])],
+ [("launch_endpoint", 0, False, [], [], [])],
),
(
"launch_endpoint_with_a_qa_override",
@@ -3943,6 +3945,7 @@ class TestExperimentService(APIBaseTest):
(
"launch_endpoint",
0,
+ False,
["forced_variant_release_condition"],
["forced_variant_release_condition:some_conditions_pinned"],
["forced_variant_release_condition:some_conditions_pinned:info:not_measured"],
@@ -3955,7 +3958,7 @@ class TestExperimentService(APIBaseTest):
timedelta(days=3, hours=2), name="Path O", feature_flag_key="path-older-flag"
),
lambda service, experiment, request: service.launch_experiment(experiment, request=request),
- [("launch_endpoint", 266400, [], [], [])],
+ [("launch_endpoint", 266400, True, [], [], [])],
),
(
"create_with_start_date",
@@ -3966,7 +3969,7 @@ class TestExperimentService(APIBaseTest):
start_date=timezone.now(),
event_source=EventSource.API,
),
- [("create_request", 0, ["no_metric"], [], ["no_metric::warning:"])],
+ [("create_request", 0, False, ["no_metric"], [], ["no_metric::warning:"])],
),
(
"create_draft",
@@ -3982,7 +3985,7 @@ class TestExperimentService(APIBaseTest):
lambda service, experiment, request: service.update_experiment(
experiment, {"start_date": timezone.now()}, event_source=EventSource.API
),
- [("update_start_date", 0, [], [], [])],
+ [("update_start_date", 0, False, [], [], [])],
),
(
"update_moves_start_date_of_running",
@@ -4010,6 +4013,7 @@ class TestExperimentService(APIBaseTest):
(
event["launch_path"],
event["flag_age_seconds"],
+ event["flag_called_before_launch"],
event["health_finding_codes"],
event["health_finding_subcodes"],
event["health_findings"],
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index 4de5c0e99585..130663b5cb28 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2277,6 +2277,7 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `new_flag_no_traffic` - New Flag No Traffic
*/
export type ExperimentHealthFindingCodeEnumApi =
(typeof ExperimentHealthFindingCodeEnumApi)[keyof typeof ExperimentHealthFindingCodeEnumApi]
@@ -2296,6 +2297,7 @@ export const ExperimentHealthFindingCodeEnumApi = {
SplitChangedMidRun: 'split_changed_mid_run',
BucketingChangedMidRun: 'bucketing_changed_mid_run',
ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
+ NewFlagNoTraffic: 'new_flag_no_traffic',
} as const
/**
@@ -2357,7 +2359,8 @@ export interface ExperimentHealthFindingApi {
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
- * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run */
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `new_flag_no_traffic` - New Flag No Traffic */
code: ExperimentHealthFindingCodeEnumApi
/**
* The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.
@@ -2393,6 +2396,7 @@ export interface ExperimentHealthFindingApi {
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `new_flag_no_traffic` - New Flag No Traffic
* * `no_metric` - No Metric
* * `zero_exposures` - Zero Exposures
* * `srm` - Srm
@@ -2410,6 +2414,7 @@ export const ExperimentHealthCheckKeyEnumApi = {
ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
FlagVariantsMismatch: 'flag_variants_mismatch',
ForcedVariantReleaseCondition: 'forced_variant_release_condition',
+ NewFlagNoTraffic: 'new_flag_no_traffic',
NoMetric: 'no_metric',
ZeroExposures: 'zero_exposures',
Srm: 'srm',
@@ -2443,6 +2448,7 @@ export interface ExperimentHealthCheckResultApi {
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `new_flag_no_traffic` - New Flag No Traffic
* * `no_metric` - No Metric
* * `zero_exposures` - Zero Exposures
* * `srm` - Srm
diff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.ts
index 4dc0178da9b2..7129106ae19e 100644
--- a/products/experiments/frontend/generated/api.ts
+++ b/products/experiments/frontend/generated/api.ts
@@ -732,6 +732,11 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =
* flag version that are no local evaluation, from an hour after the window starts. When
* every condition that includes users pins the variant whose key one property holds,
* `assigned_by_property` replaces `all_conditions_pinned`.
+ *
+ * A draft whose flag has no recorded call gets the info `new_flag_no_traffic`. The record
+ * is the flag's `last_called_at`, which a periodic task updates, so it trails a call. The
+ * check reports `not_checked` when the organization's flag calls skip the events table,
+ * and when the project has no active flag, because the web SDK then requests no flags.
*/
export const experimentsHealthRetrieve = async (
projectId: string,
diff --git a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
index 59d0e39e40a7..329bfa78bbc3 100644
--- a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
+++ b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
@@ -48,7 +48,17 @@ export const Draft: Story = {
),
skipped('no_metric', 'Metrics', 'not_applicable', 'Runs after launch'),
],
- notes: [],
+ notes: [
+ {
+ code: 'new_flag_no_traffic',
+ subcode: null,
+ severity: 'info',
+ title: 'The feature flag has no traffic yet',
+ detail: "PostHog hasn't recorded any calls to the linked feature flag yet. The experiment only counts users after your code evaluates the flag, so launch after the flag starts getting traffic. New calls can take up to an hour to show up here.",
+ evidence: {},
+ actions: [],
+ },
+ ],
isDraft: true,
exposureChecksMissing: false,
},
diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.ts
index e83114eae102..4320289f4f84 100644
--- a/products/experiments/frontend/health/debug/healthDebugReport.ts
+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts
@@ -315,6 +315,7 @@ export function buildHealthDebugChecks(input: HealthDebugInput): HealthDebugChec
serverOnlyCheck(input, 'release_conditions_changed_mid_run', conditionsChangedNote),
serverOnlyCheck(input, 'flag_variants_mismatch', null),
serverOnlyCheck(input, 'forced_variant_release_condition', forcedVariantNote),
+ serverOnlyCheck(input, 'new_flag_no_traffic', null),
noMetricCheck(input),
...exposureChecks(input),
]
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index 1ea48802d4d3..e039b19dcffd 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -28,6 +28,7 @@ export type ExperimentHealthFindingCode =
| 'split_changed_mid_run'
| 'bucketing_changed_mid_run'
| 'release_conditions_changed_mid_run'
+ | 'new_flag_no_traffic'
export interface ExperimentHealthFinding {
code: ExperimentHealthFindingCode
diff --git a/products/feature_flags/backend/facade/flags.py b/products/feature_flags/backend/facade/flags.py
index f26f2681646f..2ab1c01f4103 100644
--- a/products/feature_flags/backend/facade/flags.py
+++ b/products/feature_flags/backend/facade/flags.py
@@ -1,5 +1,5 @@
-"""Feature-flag gates for this product, and the read of an organization's flag_evaluations mode.
-The module stays light so core code can import it without pulling the facade's heavier logic
+"""Feature-flag gates for this product, the read of an organization's flag_evaluations mode, and whether a team
+has an active flag. The module stays light so core code can import it without pulling the facade's heavier logic
surface onto its import path.
"""
@@ -12,6 +12,7 @@ from posthog.models.instance_setting import get_instance_setting
from posthog.ph_client import feature_enabled_or_false
from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
+from products.feature_flags.backend.models.feature_flag import FeatureFlag
from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
if TYPE_CHECKING:
@@ -20,6 +21,15 @@ if TYPE_CHECKING:
FLAG_EVALUATIONS_HOGQL_TABLE_FEATURE_FLAG = "flag-evaluations-hogql-table"
+def has_active_flags(team: "Team") -> bool:
+ """Whether the team has an active flag. Remote config sends this as `hasFeatureFlags`, and posthog-js requests
+ flags on page load only when it is true. Without an active flag, a browser records no flag call until something
+ else loads flags, for example `identify`."""
+ # `team=`, not `team_id=`: the flag manager rewrites `team_id=` to the project's root team, which changes the
+ # answer for an environment team and costs a join.
+ return FeatureFlag.objects.filter(team=team, active=True).exists()
+
+
def get_organization_flag_evaluations_mode(organization_id: UUID) -> int:
"""The organization's FlagEvaluationsMode value. An organization with no config row reads EVENTS,
which matches the COALESCE in Node ingestion's team query."""
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 15587b5638de..9f2191aa4483 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -44612,6 +44612,7 @@ export namespace Schemas {
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `new_flag_no_traffic` - New Flag No Traffic
*/
export type ExperimentHealthFindingCodeEnum = typeof ExperimentHealthFindingCodeEnum[keyof typeof ExperimentHealthFindingCodeEnum];
@@ -44631,6 +44632,7 @@ export namespace Schemas {
SplitChangedMidRun: 'split_changed_mid_run',
BucketingChangedMidRun: 'bucketing_changed_mid_run',
ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
+ NewFlagNoTraffic: 'new_flag_no_traffic',
} as const;
/**
@@ -44692,7 +44694,8 @@ export namespace Schemas {
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
- * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run */
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `new_flag_no_traffic` - New Flag No Traffic */
code: ExperimentHealthFindingCodeEnum;
/**
* The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.
@@ -44728,6 +44731,7 @@ export namespace Schemas {
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `new_flag_no_traffic` - New Flag No Traffic
* * `no_metric` - No Metric
* * `zero_exposures` - Zero Exposures
* * `srm` - Srm
@@ -44745,6 +44749,7 @@ export namespace Schemas {
ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
FlagVariantsMismatch: 'flag_variants_mismatch',
ForcedVariantReleaseCondition: 'forced_variant_release_condition',
+ NewFlagNoTraffic: 'new_flag_no_traffic',
NoMetric: 'no_metric',
ZeroExposures: 'zero_exposures',
Srm: 'srm',
@@ -44778,6 +44783,7 @@ export namespace Schemas {
* * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `new_flag_no_traffic` - New Flag No Traffic
* * `no_metric` - No Metric
* * `zero_exposures` - Zero Exposures
* * `srm` - Srm