Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): report variants assigned outside the flag service (#119290)

Marcel Poelker1a604a6cb2df
diff --git a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
index 61a84f84a25b..ba3a13b4572d 100644
--- a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
+++ b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
@@ -91,6 +91,7 @@ const HEALTH_INPUTS: ExperimentHealthInputsApi = {
],
calls: 4200,
remotely_evaluated_calls: 4116,
+ people_assigned_outside: 0,
},
reader_access: { can_read_flag_history: true, can_query_events: true, denied_event_properties: [] },
exposures: {
diff --git a/products/experiments/backend/health/checks/conditions_changed.py b/products/experiments/backend/health/checks/conditions_changed.py
index ac4530b50abf..410f4bc11ee6 100644
--- a/products/experiments/backend/health/checks/conditions_changed.py
+++ b/products/experiments/backend/health/checks/conditions_changed.py
@@ -153,7 +153,7 @@ def removed_overrides_to_grade(ctx: HealthContext) -> PinnedConditions | None:
return None
removed = _reported(ctx, history, effect)
pinned = tuple(
- ReachableCondition(number=override.condition_number, pinned_variant=override.variant)
+ ReachableCondition(number=override.condition_number, pinned_variant=override.variant, variant_property=None)
for override in removed
if override.condition_number is not None and override.same_audience
)
diff --git a/products/experiments/backend/health/checks/flag_state.py b/products/experiments/backend/health/checks/flag_state.py
index a01e8d87f3d1..8b04c38caafa 100644
--- a/products/experiments/backend/health/checks/flag_state.py
+++ b/products/experiments/backend/health/checks/flag_state.py
@@ -4,15 +4,27 @@ Conflicting states of an experiment and its feature flag. Pure functions, no I/O
A port of the page's `browserExperimentWarning` selector (frontend/src/scenes/experiments/experimentLogic.tsx)
and the flag helpers it calls (`hasZeroRollout`, `hasMultipleVariantsActive`, `isSingleVariantShipped`).
Change both sides together until the page reads these findings.
+
+A running experiment whose flag has a 0% rollout is the exception. The flag service and local evaluation serve no
+variant then, so people who still get one get it from the customer's own assignment, which only the flag's calls
+show. Only the server reports that case.
"""
+from datetime import UTC, datetime, timedelta
+
from products.experiments.backend.facade.contracts import (
ExperimentHealthFinding,
ExperimentHealthFindingActionKind,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
)
-from products.experiments.backend.health.context import FlagHistoryNotChecked, FlagState, HealthContext, SplitChange
+from products.experiments.backend.health.context import (
+ ConditionSetTraffic,
+ FlagHistoryNotChecked,
+ FlagState,
+ HealthContext,
+ SplitChange,
+)
from products.experiments.backend.health.split_changes import (
SplitChangeShape,
describe_split,
@@ -27,8 +39,19 @@ RUNNING_BUT_NO_ROLLOUT = "running_but_no_rollout"
RUNNING_BUT_SINGLE_VARIANT_SHIPPED = "running_but_single_variant_shipped"
ENDED_BUT_MULTIPLE_VARIANTS_ROLLED_OUT = "ended_but_multiple_variants_rolled_out"
NOT_STARTED_BUT_MULTIPLE_VARIANTS_ROLLED_OUT = "not_started_but_multiple_variants_rolled_out"
-# pinned: the page sends it as `finding_variant` on its events, as the keys above. No banner uses it.
+# pinned: the page sends these as `finding_variant` on its events, as the keys above. No banner uses them.
RUNNING_BUT_FLAG_DELETED = "running_but_flag_deleted"
+ASSIGNED_OUTSIDE_FLAG_SERVICE = "assigned_outside_flag_service"
+
+# A bootstrap can come from a server's cached evaluation, and an older client can report a variant it cached before
+# the rollout went to 0%. Calls in this time after the start of the window don't count as assigned outside the flag
+# service, so that these leftovers don't hide a rollout that really stopped new exposures.
+OUTSIDE_ASSIGNMENT_GRACE = timedelta(hours=1)
+# A few people can get a variant from a developer's local override, or from a client that cached it before the
+# rollout went to 0%.
+_MIN_PEOPLE_ASSIGNED_OUTSIDE = 10
+# A shorter window can still fill up, so an answer under the minimum is asked again sooner.
+_OUTSIDE_ASSIGNMENT_SETTLED_AFTER = timedelta(days=1)
_NOT_RUNNING_BUT_EXPOSED_TITLE = "The experiment is not running, but users are exposed to multiple variants"
@@ -71,6 +94,9 @@ def flag_state(ctx: HealthContext) -> ExperimentHealthFinding | None:
),
)
if zero_rollout:
+ traffic = ctx.condition_set_traffic
+ if traffic is not None and traffic.people_assigned_outside >= _MIN_PEOPLE_ASSIGNED_OUTSIDE:
+ return _assigned_outside(traffic)
return _finding(
code=ExperimentHealthFindingCode.FLAG_OFF_WHILE_RUNNING,
subcode=RUNNING_BUT_NO_ROLLOUT,
@@ -123,6 +149,41 @@ def flag_state(ctx: HealthContext) -> ExperimentHealthFinding | None:
return None
+def needs_outside_assignment_check(ctx: HealthContext) -> bool:
+ """Whether this check reads the flag's calls: a running experiment whose flag has a 0% rollout."""
+ flag = ctx.flag
+ return ctx.is_running and not ctx.is_paused and flag is not None and not flag.deleted and _has_zero_rollout(flag)
+
+
+def outside_assignment_unsettled(ctx: HealthContext, traffic: ConditionSetTraffic, *, now: datetime) -> bool:
+ """Whether `traffic` holds too few people assigned outside the flag service only because its window is young."""
+ return (
+ needs_outside_assignment_check(ctx)
+ and traffic.people_assigned_outside < _MIN_PEOPLE_ASSIGNED_OUTSIDE
+ and now - traffic.since < _OUTSIDE_ASSIGNMENT_SETTLED_AFTER
+ )
+
+
+def _assigned_outside(traffic: ConditionSetTraffic) -> ExperimentHealthFinding:
+ since = traffic.since + OUTSIDE_ASSIGNMENT_GRACE
+ people = traffic.people_assigned_outside
+ return _finding(
+ code=ExperimentHealthFindingCode.FLAG_OFF_WHILE_RUNNING,
+ subcode=ASSIGNED_OUTSIDE_FLAG_SERVICE,
+ severity=ExperimentHealthFindingSeverity.INFO,
+ title="Variants are assigned outside PostHog",
+ detail=(
+ "The linked feature flag has a 0% rollout, so PostHog assigns no variants. But "
+ f"{people:,} {'user' if people == 1 else 'users'} got a variant since "
+ f"{format_change_time(since, traffic.team_timezone)} from flag calls that PostHog didn't evaluate, so "
+ "your own code assigns the variants, directly or through a bootstrap. These users count in the results. "
+ "The flag's release conditions and variant split don't decide who gets which variant. If PostHog should "
+ "assign the variants, increase the rollout and stop assigning them in your code."
+ ),
+ evidence={"people_assigned_outside": people, "assigned_outside_since": since.astimezone(UTC).isoformat()},
+ )
+
+
def _finding(
*,
code: ExperimentHealthFindingCode,
@@ -130,11 +191,12 @@ def _finding(
title: str,
detail: str,
evidence: dict[str, str | int | float | None] | None = None,
+ severity: ExperimentHealthFindingSeverity = ExperimentHealthFindingSeverity.WARNING,
) -> ExperimentHealthFinding:
return ExperimentHealthFinding(
code=code,
subcode=subcode,
- severity=ExperimentHealthFindingSeverity.WARNING,
+ severity=severity,
title=title,
detail=detail,
evidence=evidence or {},
diff --git a/products/experiments/backend/health/checks/forced_variant.py b/products/experiments/backend/health/checks/forced_variant.py
index 3075afa0743e..a828c2ea8aef 100644
--- a/products/experiments/backend/health/checks/forced_variant.py
+++ b/products/experiments/backend/health/checks/forced_variant.py
@@ -20,7 +20,13 @@ from products.experiments.backend.facade.contracts import (
)
from products.experiments.backend.health.checks.flag_state import is_single_variant_shipped, shipped_variant_key
from products.experiments.backend.health.condition_changes import conditions_net_effect
-from products.experiments.backend.health.context import ConditionsChange, FlagHistory, HealthContext, ReachableCondition
+from products.experiments.backend.health.context import (
+ ConditionsChange,
+ FlagHistory,
+ HealthContext,
+ ReachableCondition,
+ VariantProperty,
+)
from products.experiments.backend.health.pinned_share import (
PinnedConditions,
PinnedShareGrade,
@@ -38,6 +44,9 @@ ALL_CONDITIONS_PINNED = "all_conditions_pinned"
# Some conditions pin a variant. The users they match count in that variant's results without random
# assignment, so they bias the comparison unless the analysis excludes them, for example as test users.
SOME_CONDITIONS_PINNED = "some_conditions_pinned"
+# Every condition that includes users pins the variant that one property names, so the code that sets the
+# property assigns the variants. The comparison is valid if that code assigns at random, which PostHog can't see.
+ASSIGNED_BY_PROPERTY = "assigned_by_property"
@frozen
@@ -73,6 +82,18 @@ def forced_variant_release_condition(ctx: HealthContext) -> ExperimentHealthFind
if ctx.is_running and is_single_variant_shipped(flag) and pinned_keys == [shipped_variant_key(flag)]:
return None
set_after_launch = _set_after_launch(ctx, pinned_numbers)
+ variant_property = _shared_variant_property(pins.pinned)
+ if variant_property is not None and len(pinned_keys) > 1:
+ return _finding(
+ subcode=ASSIGNED_BY_PROPERTY,
+ severity=ExperimentHealthFindingSeverity.INFO,
+ title="Variants are assigned outside PostHog",
+ detail=_assigned_by_property_detail(variant_property.key, set_after_launch.sentence),
+ pinned_numbers=pinned_numbers,
+ pinned_keys=pinned_keys,
+ diagnostic_ref="A7b",
+ grade_evidence={"variant_property": variant_property.key, **set_after_launch.evidence},
+ )
return _finding(
subcode=ALL_CONDITIONS_PINNED,
severity=ExperimentHealthFindingSeverity.WARNING,
@@ -178,6 +199,11 @@ def _set_after_launch(ctx: HealthContext, numbers: Sequence[int]) -> _SetAfterLa
)
+def _shared_variant_property(conditions: Sequence[ReachableCondition]) -> VariantProperty | None:
+ properties = {condition.variant_property for condition in conditions}
+ return next(iter(properties)) if len(properties) == 1 else None
+
+
def _quoted(keys: Sequence[str]) -> str:
return ", ".join(f'"{key}"' for key in keys)
@@ -223,6 +249,16 @@ def _all_pinned_detail(
return f"{fact}{set_after_launch.sentence} {why} {fix}"
+def _assigned_by_property_detail(variant_property: str, set_after_launch: str) -> str:
+ return (
+ f'Each release condition that includes users matches the users whose property "{variant_property}" holds a '
+ f'variant key, and serves them that variant. So the code that sets "{variant_property}" assigns the '
+ f"variants, not PostHog.{set_after_launch} If it assigns at random, each variant gets similar users. If it "
+ "assigns by who the users are, for example by plan or country, a difference in the results can come from "
+ "those users and not from the variant."
+ )
+
+
def _some_pinned_detail(
numbers: Sequence[int],
pinned_keys: Sequence[str],
diff --git a/products/experiments/backend/health/condition_set_traffic.py b/products/experiments/backend/health/condition_set_traffic.py
index a9c5384ecec2..f13660bd6aa7 100644
--- a/products/experiments/backend/health/condition_set_traffic.py
+++ b/products/experiments/backend/health/condition_set_traffic.py
@@ -5,6 +5,10 @@ sets served. These are the people the flag served a variant, not the exposures t
criteria count when they name another event, filters or an action, so the finding speaks of the users who got a
variant.
+The flag-state check reads the people who got a variant from a call that the flag service did not evaluate. When
+the flag has a 0% rollout, neither the flag service nor local evaluation serves a variant, so these people got it
+from the customer's own assignment.
+
The flag service names the set in the reason "Matched condition set N", numbered from 1 in the stored order of the
conditions (`get_reason_description` in rust/feature-flags/src/api/types.rs). Only calls of the flag's current
version count, so the numbers match the conditions that the check reads. No call before the flag's last save can
@@ -38,6 +42,11 @@ from posthog.utils import safe_cache_add
from products.experiments.backend.facade.contracts import ExperimentHealthReaderAccess
from products.experiments.backend.health.checks.conditions_changed import removed_overrides_to_grade
+from products.experiments.backend.health.checks.flag_state import (
+ OUTSIDE_ASSIGNMENT_GRACE,
+ needs_outside_assignment_check,
+ outside_assignment_unsettled,
+)
from products.experiments.backend.health.checks.forced_variant import overrides_to_grade
from products.experiments.backend.health.context import ConditionSetTraffic, HealthContext
from products.experiments.backend.health.exposure_gate import UNKNOWN, cache_failed_scan
@@ -66,8 +75,13 @@ _MATCHED_REASON = "^Matched condition set ([0-9]+)$"
# The scan runs without the reader, so it can't apply a reader's property access rules. A reader denied one of
# these gets no share instead of counts derived from it.
_SCANNED_PROPERTIES = frozenset(
- {"$feature_flag", "$feature_flag_response", "$feature_flag_reason", "$feature_flag_version"}
+ {"$feature_flag", "$feature_flag_response", "$feature_flag_reason", "$feature_flag_version", "locally_evaluated"}
)
+# The reason that posthog-node sends for a local evaluation. posthog-python sends `locally_evaluated` instead.
+_LOCALLY_EVALUATED_REASON = "Evaluated locally"
+# Part of the cache key. Raise it when the fields of `ConditionSetTraffic` change, because an answer cached with
+# other fields can't be read back into it.
+_ANSWER_SHAPE = 2
_MAX_CONDITION_SETS = 1000
_SCAN_LOCK_SECONDS = QUERY_TIMEOUT_SECONDS + 5
_WAIT_FOR_SCAN_SECONDS = QUERY_TIMEOUT_SECONDS + 1
@@ -95,6 +109,12 @@ def load_condition_set_traffic(
def needs_condition_set_traffic(ctx: HealthContext) -> bool:
+ """Whether a check reads the flag's calls: the forced-variant or release-condition check to grade by the pinned
+ share, or the flag-state check for a flag with a 0% rollout."""
+ return grades_by_pinned_share(ctx) or needs_outside_assignment_check(ctx)
+
+
+def grades_by_pinned_share(ctx: HealthContext) -> bool:
"""Whether a check grades its finding by the users that pinned conditions served: the forced-variant check for
the overrides in place, and the release-condition check for an override removed after launch."""
return ctx.is_launched and any(pins.weighted for pins in _pins_to_grade(ctx))
@@ -125,7 +145,7 @@ def query_condition_set_traffic(
flag = experiment.feature_flag
query = parse_select(
"""
- SELECT condition_set, count(), sum(calls), sum(remotely_evaluated_calls)
+ SELECT condition_set, count(), sum(calls), sum(remotely_evaluated_calls), countIf(assigned_outside)
FROM (
SELECT
{entity} AS entity,
@@ -136,7 +156,13 @@ def query_condition_set_traffic(
AND match(properties.$feature_flag_reason, {matched_reason})
) AS condition_set,
count() AS calls,
- countIf(isNotNull(toFloat(properties.$feature_flag_version))) AS remotely_evaluated_calls
+ countIf(isNotNull(toFloat(properties.$feature_flag_version))) AS remotely_evaluated_calls,
+ countIf(
+ isNull(toFloat(properties.$feature_flag_version))
+ AND ifNull(toString(properties.locally_evaluated), '') != 'true'
+ AND ifNull(properties.$feature_flag_reason, '') != {locally_evaluated_reason}
+ AND timestamp >= {assigned_outside_since}
+ ) > 0 AS assigned_outside
FROM events
WHERE {flag_calls}
AND timestamp >= {window_start} AND timestamp < {window_end}
@@ -154,6 +180,8 @@ def query_condition_set_traffic(
"entity": ast.Field(chain=[get_entity_key(flag.aggregation_group_type_index)]),
"matched_reason": ast.Constant(value=_MATCHED_REASON),
"flag_version": ast.Constant(value=flag.version),
+ "locally_evaluated_reason": ast.Constant(value=_LOCALLY_EVALUATED_REASON),
+ "assigned_outside_since": ast.Constant(value=window_start + OUTSIDE_ASSIGNMENT_GRACE),
"window_start": ast.Constant(value=window_start),
"window_end": ast.Constant(value=window_end),
"flag_key": ast.Constant(value=flag.key),
@@ -180,18 +208,20 @@ def query_condition_set_traffic(
settings=HogQLGlobalSettings(max_execution_time=QUERY_TIMEOUT_SECONDS, timeout_overflow_mode="throw"),
)
people_by_set: dict[int, int] = {}
- calls = remotely_evaluated_calls = 0
- for condition_set, people, set_calls, set_remotely_evaluated_calls in response.results or []:
+ calls = remotely_evaluated_calls = people_assigned_outside = 0
+ for condition_set, people, set_calls, set_remotely_evaluated_calls, set_assigned_outside in response.results or []:
if condition_set:
people_by_set[int(condition_set)] = int(people)
calls += int(set_calls)
remotely_evaluated_calls += int(set_remotely_evaluated_calls)
+ people_assigned_outside += int(set_assigned_outside)
return ConditionSetTraffic(
since=window_start,
team_timezone=team.timezone,
people_by_set=people_by_set,
calls=calls,
remotely_evaluated_calls=remotely_evaluated_calls,
+ people_assigned_outside=people_assigned_outside,
)
@@ -232,10 +262,9 @@ def _scan(
except Exception as error:
cache_failed_scan(cache_key, error)
return None
+ asked_again_sooner = has_too_few_people(ctx, traffic) or outside_assignment_unsettled(ctx, traffic, now=now)
cache.set(
- cache_key,
- asdict(traffic),
- _TOO_FEW_PEOPLE_CACHE_SECONDS if has_too_few_people(ctx, traffic) else _ANSWER_CACHE_SECONDS,
+ cache_key, asdict(traffic), _TOO_FEW_PEOPLE_CACHE_SECONDS if asked_again_sooner else _ANSWER_CACHE_SECONDS
)
return traffic
@@ -264,6 +293,7 @@ def _cache_key(experiment: Experiment, launched_at: datetime) -> str:
# Every input of the query except the end of the window, which moves with the clock.
inputs = json.dumps(
[
+ _ANSWER_SHAPE,
launched_at.isoformat(),
flag.key,
flag.version,
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index 6157ce49ef45..27010e18c0bc 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -1,6 +1,7 @@
from collections.abc import Mapping
from datetime import datetime
from enum import StrEnum
+from typing import Any
from posthog.schema import MultipleVariantHandling
@@ -9,7 +10,7 @@ from posthog.dataclasses import frozen
from products.experiments.backend.facade.contracts import ExperimentHealthFlagChangeKind
from products.experiments.backend.metric_resolution import saved_metric_link_role, saved_metric_links
from products.experiments.backend.models.experiment import Experiment
-from products.feature_flags.backend.facade.filters import pinned_variant, reachable_conditions
+from products.feature_flags.backend.facade.filters import condition_aggregation, pinned_variant, reachable_conditions
@frozen
@@ -151,11 +152,23 @@ class FlagReleaseGroup:
property_count: int | None
+@frozen
+class VariantProperty:
+ """A property whose value names the variant. A person and a group, or two group types, can have a property with
+ the same key, and those are different properties."""
+
+ key: str
+ group_type_index: int | None
+
+
@frozen
class ReachableCondition:
# Numbered from 1, as the page labels the release conditions ("Set 2").
number: int
pinned_variant: str | None
+ # The person or group property that the condition matches on alone, when its one value is the pinned variant's
+ # key. The variant then comes from whatever sets that property, usually the customer's own assignment.
+ variant_property: VariantProperty | None
@frozen
@@ -180,6 +193,10 @@ class ConditionSetTraffic:
# The calls that returned a variant, and those of them that the flag service evaluated, which alone name a set.
calls: int
remotely_evaluated_calls: int
+ # The people who got a variant in a call that carries no flag version and is not a local evaluation in a server
+ # SDK, from the start of the flag-state check's grace period after `since`. Such a call comes from a bootstrap or
+ # from the customer's own code, so it can serve a variant that the flag's release conditions don't.
+ people_assigned_outside: int
@frozen
@@ -233,12 +250,54 @@ def parse_flag_state(
release_groups=tuple(_release_group(group) for group in conditions),
variants=flag_variants,
reachable_conditions=tuple(
- ReachableCondition(number=index + 1, pinned_variant=pinned_variant(conditions[index], variant_keys))
+ _reachable_condition(index + 1, conditions[index], variant_keys, aggregation_group_type_index)
for index in reachable
),
)
+def _reachable_condition(
+ number: int, condition: Mapping[str, Any], variant_keys: set[str], flag_aggregation: int | None
+) -> ReachableCondition:
+ pinned = pinned_variant(condition, variant_keys)
+ return ReachableCondition(
+ number=number,
+ pinned_variant=pinned,
+ variant_property=None if pinned is None else _variant_property(condition, pinned, flag_aggregation),
+ )
+
+
+def _variant_property(
+ condition: Mapping[str, Any], pinned: str, flag_aggregation: int | None
+) -> VariantProperty | None:
+ properties = condition.get("properties")
+ if not isinstance(properties, list) or len(properties) != 1 or not isinstance(properties[0], dict):
+ return None
+ key = properties[0].get("key")
+ value = properties[0].get("value")
+ values = value if isinstance(value, list) else [value]
+ if (
+ properties[0].get("type") not in ("person", "group")
+ or properties[0].get("operator") not in ("exact", None)
+ or len(values) != 1
+ or not isinstance(key, str)
+ ):
+ return None
+ # The flag service compares `exact` values as lowercase strings.
+ if str(values[0]).lower() != pinned.lower():
+ return None
+ if properties[0].get("type") == "person":
+ return VariantProperty(key=key, group_type_index=None)
+ # A group filter without its own group type reads the group the condition aggregates on.
+ group_type_index = properties[0].get("group_type_index")
+ return VariantProperty(
+ key=key,
+ group_type_index=group_type_index
+ if isinstance(group_type_index, int)
+ else condition_aggregation(condition, flag_aggregation),
+ )
+
+
def _release_group(group: object) -> FlagReleaseGroup:
if not isinstance(group, dict):
return FlagReleaseGroup(rollout_percentage=None, property_count=None)
diff --git a/products/experiments/backend/health/inputs_payload.py b/products/experiments/backend/health/inputs_payload.py
index e10e79f85ea1..b3bbc8ccb00c 100644
--- a/products/experiments/backend/health/inputs_payload.py
+++ b/products/experiments/backend/health/inputs_payload.py
@@ -165,6 +165,7 @@ def _health_inputs_traffic(traffic: ConditionSetTraffic) -> dict[str, Any]:
],
"calls": traffic.calls,
"remotely_evaluated_calls": traffic.remotely_evaluated_calls,
+ "people_assigned_outside": traffic.people_assigned_outside,
}
diff --git a/products/experiments/backend/health/test/test_condition_set_traffic.py b/products/experiments/backend/health/test/test_condition_set_traffic.py
index 249173b1da2f..31dc8333b10a 100644
--- a/products/experiments/backend/health/test/test_condition_set_traffic.py
+++ b/products/experiments/backend/health/test/test_condition_set_traffic.py
@@ -40,6 +40,7 @@ TRAFFIC = ConditionSetTraffic(
people_by_set={1: 30, 2: 970},
calls=2000,
remotely_evaluated_calls=2000,
+ people_assigned_outside=0,
)
TOO_FEW_PEOPLE = ConditionSetTraffic(
since=datetime(2026, 10, 1, tzinfo=UTC),
@@ -47,6 +48,7 @@ TOO_FEW_PEOPLE = ConditionSetTraffic(
people_by_set={1: 3, 2: 50},
calls=100,
remotely_evaluated_calls=100,
+ people_assigned_outside=0,
)
_NO_VARIANT_REASONS = frozenset({"Out of rollout bound", "No matching condition set"})
_STAFF_PINNED_TO_TEST: dict[str, Any] = {
@@ -76,6 +78,16 @@ _HOLDOUT_FILTERS: dict[str, Any] = {
}
+_ZERO_ROLLOUT_FILTERS: dict[str, Any] = {
+ "groups": [{"properties": [], "rollout_percentage": 0}],
+ "multivariate": {
+ "variants": [{"key": "control", "rollout_percentage": 50}, {"key": "test", "rollout_percentage": 50}]
+ },
+}
+NO_ONE_ASSIGNED_OUTSIDE = replace(TRAFFIC, people_by_set={}, remotely_evaluated_calls=0, calls=0)
+ASSIGNED_OUTSIDE = replace(NO_ONE_ASSIGNED_OUTSIDE, calls=900, people_assigned_outside=300)
+
+
def _create_running_experiment(team: Team, user: User, *, launched_at: datetime) -> Experiment:
flag = FeatureFlag.objects.create(
team=team,
@@ -125,6 +137,28 @@ class TestLoadConditionSetTraffic(BaseTest):
self.assertEqual((traffic, query.call_count), (None, 0))
+ @parameterized.expand(
+ [
+ ("pinned_condition", None, 1),
+ ("zero_rollout", _ZERO_ROLLOUT_FILTERS, 1),
+ (
+ "no_pin_and_a_rollout",
+ {**_ZERO_ROLLOUT_FILTERS, "groups": [{"properties": [], "rollout_percentage": 100}]},
+ 0,
+ ),
+ ]
+ )
+ def test_scans_only_for_a_check_that_reads_the_calls(
+ self, _name: str, flag_filters: dict[str, Any] | None, expected_queries: int
+ ) -> None:
+ if flag_filters is not None:
+ FeatureFlag.objects.filter(pk=self.experiment.feature_flag_id).update(filters=flag_filters)
+ self.experiment = Experiment.objects.get(pk=self.experiment.pk)
+
+ _, query = self._load(TRAFFIC)
+
+ self.assertEqual(query.call_count, expected_queries)
+
@parameterized.expand(
[
("since_launch", timedelta(days=2), timedelta(days=3), "launch"),
@@ -159,6 +193,29 @@ class TestLoadConditionSetTraffic(BaseTest):
1,
_HOLDOUT_FILTERS,
),
+ (
+ "no_one_assigned_outside_in_a_young_window_an_hour_later",
+ NO_ONE_ASSIGNED_OUTSIDE,
+ timedelta(hours=1, minutes=1),
+ 1,
+ _ZERO_ROLLOUT_FILTERS,
+ timedelta(hours=2),
+ ),
+ (
+ "no_one_assigned_outside_in_a_full_window_hours_later",
+ NO_ONE_ASSIGNED_OUTSIDE,
+ timedelta(hours=12),
+ 0,
+ _ZERO_ROLLOUT_FILTERS,
+ ),
+ (
+ "assigned_outside_in_a_young_window_hours_later",
+ ASSIGNED_OUTSIDE,
+ timedelta(hours=12),
+ 0,
+ _ZERO_ROLLOUT_FILTERS,
+ timedelta(hours=2),
+ ),
]
)
def test_answer_cache(
@@ -168,10 +225,16 @@ class TestLoadConditionSetTraffic(BaseTest):
next_load_after: timedelta,
expected_queries: int,
flag_filters: dict[str, Any] | None = None,
+ saved_ago: timedelta | None = None,
) -> None:
if flag_filters is not None:
- FeatureFlag.objects.filter(pk=self.experiment.feature_flag_id).update(filters=flag_filters)
+ saved_at = timezone.now() - saved_ago if saved_ago is not None else None
+ FeatureFlag.objects.filter(pk=self.experiment.feature_flag_id).update(
+ filters=flag_filters, **({"updated_at": saved_at} if saved_at is not None else {})
+ )
self.experiment = Experiment.objects.get(pk=self.experiment.pk)
+ if saved_at is not None:
+ answer = replace(answer, since=saved_at)
first, _ = self._load(answer)
with time_machine.travel(timezone.now() + next_load_after):
@@ -283,7 +346,7 @@ class TestQueryConditionSetTraffic(ClickhouseTestMixin, BaseTest):
("person-1", "Matched condition set 1", FLAG_VERSION, 1),
("person-2", "Matched condition set 1", 2, 1),
],
- ({1: 1}, 2, 2),
+ ({1: 1}, 2, 2, 0),
),
(
"calls_evaluated_outside_the_flag_service_name_no_set",
@@ -291,8 +354,11 @@ class TestQueryConditionSetTraffic(ClickhouseTestMixin, BaseTest):
("person-1", "Matched condition set 2", FLAG_VERSION, 1),
("person-2", None, None, 1),
("person-3", "Evaluated locally", None, 1),
+ ("person-4", None, None, 1),
+ ("person-5", None, None, 6.99),
],
- ({2: 1}, 3, 1),
+ ({2: 1}, 5, 1, 1),
+ frozenset({"person-4"}),
),
(
"calls_that_return_no_variant_do_not_count",
@@ -300,7 +366,7 @@ class TestQueryConditionSetTraffic(ClickhouseTestMixin, BaseTest):
("person-1", "Out of rollout bound", FLAG_VERSION, 1),
("person-2", "Matched condition set 2", FLAG_VERSION, 1),
],
- ({2: 1}, 1, 1),
+ ({2: 1}, 1, 1, 0),
),
(
"person_counts_once_under_the_latest_set",
@@ -308,7 +374,7 @@ class TestQueryConditionSetTraffic(ClickhouseTestMixin, BaseTest):
("person-1", "Matched condition set 1", FLAG_VERSION, 2),
("person-1", "Matched condition set 2", FLAG_VERSION, 1),
],
- ({2: 1}, 2, 2),
+ ({2: 1}, 2, 2, 0),
),
(
"calls_outside_the_window",
@@ -316,27 +382,42 @@ class TestQueryConditionSetTraffic(ClickhouseTestMixin, BaseTest):
("person-1", "Matched condition set 1", FLAG_VERSION, 1),
("person-2", "Matched condition set 1", FLAG_VERSION, 8),
],
- ({1: 1}, 1, 1),
+ ({1: 1}, 1, 1, 0),
),
]
)
def test_query_condition_set_traffic(
self,
_name: str,
- calls: list[tuple[str, str | None, int | None, int]],
- expected: tuple[dict[int, int], int, int],
+ calls: list[tuple[str, str | None, int | None, float]],
+ expected: tuple[dict[int, int], int, int, int],
+ locally_evaluated: frozenset[str] = frozenset(),
) -> None:
now = timezone.now()
experiment = _create_running_experiment(self.team, self.user, launched_at=now - timedelta(days=30))
for distinct_id in dict.fromkeys(distinct_id for distinct_id, *_ in calls):
create_person(team=self.team, distinct_ids=[distinct_id])
for distinct_id, reason, version, days_ago in calls:
- self._call_flag(distinct_id, reason, version, now - timedelta(days=days_ago))
+ self._call_flag(
+ distinct_id,
+ reason,
+ version,
+ now - timedelta(days=days_ago),
+ locally_evaluated=distinct_id in locally_evaluated,
+ )
flush_persons_and_events()
traffic = query_condition_set_traffic(experiment, window_start=now - timedelta(days=7), window_end=now)
- self.assertEqual((dict(traffic.people_by_set), traffic.calls, traffic.remotely_evaluated_calls), expected)
+ self.assertEqual(
+ (
+ dict(traffic.people_by_set),
+ traffic.calls,
+ traffic.remotely_evaluated_calls,
+ traffic.people_assigned_outside,
+ ),
+ expected,
+ )
@parameterized.expand(
[
@@ -370,11 +451,14 @@ class TestQueryConditionSetTraffic(ClickhouseTestMixin, BaseTest):
at: datetime,
*,
event: str = "$feature_flag_called",
+ locally_evaluated: bool = False,
) -> None:
properties: dict[str, Any] = {
"$feature_flag": FLAG_KEY,
"$feature_flag_response": False if reason in _NO_VARIANT_REASONS else "test",
}
+ if locally_evaluated:
+ properties["locally_evaluated"] = True
if reason is not None:
properties["$feature_flag_reason"] = reason
if version is not None:
diff --git a/products/experiments/backend/health/test/test_conditions_changed.py b/products/experiments/backend/health/test/test_conditions_changed.py
index bc91d2c47275..82786861d0c5 100644
--- a/products/experiments/backend/health/test/test_conditions_changed.py
+++ b/products/experiments/backend/health/test/test_conditions_changed.py
@@ -87,6 +87,7 @@ def _traffic(people_by_set: dict[int, int]) -> ConditionSetTraffic:
people_by_set=people_by_set,
calls=calls,
remotely_evaluated_calls=calls,
+ people_assigned_outside=0,
)
diff --git a/products/experiments/backend/health/test/test_flag_state.py b/products/experiments/backend/health/test/test_flag_state.py
index ef213d18e3fc..c5624b6862e9 100644
--- a/products/experiments/backend/health/test/test_flag_state.py
+++ b/products/experiments/backend/health/test/test_flag_state.py
@@ -1,13 +1,14 @@
from dataclasses import replace
+from datetime import UTC, datetime
from typing import Any
from unittest import TestCase
from parameterized import parameterized
-from products.experiments.backend.facade.contracts import ExperimentHealthFindingCode
+from products.experiments.backend.facade.contracts import ExperimentHealthFindingCode, ExperimentHealthFindingSeverity
from products.experiments.backend.health.checks.flag_state import flag_state
-from products.experiments.backend.health.context import HealthContext, parse_flag_state
+from products.experiments.backend.health.context import ConditionSetTraffic, HealthContext, parse_flag_state
from products.experiments.backend.health.test.factories import DRAFT, ENDED, RUNNING, RUNNING_CONTEXT, Status
# The flag documents and cases of the `experimentWarning` test in
@@ -43,8 +44,25 @@ SHIPPED_TO_SOME = {
}
+def _traffic(people_assigned_outside: int) -> ConditionSetTraffic:
+ return ConditionSetTraffic(
+ since=datetime(2026, 10, 12, 13, 4, tzinfo=UTC),
+ team_timezone="Europe/Berlin",
+ people_by_set={},
+ calls=3 * people_assigned_outside,
+ remotely_evaluated_calls=0,
+ people_assigned_outside=people_assigned_outside,
+ )
+
+
def _context(
- status: Status, *, active: bool, filters: dict[str, Any], deleted: bool = False, archived: bool = False
+ status: Status,
+ *,
+ active: bool,
+ filters: dict[str, Any],
+ deleted: bool = False,
+ archived: bool = False,
+ traffic: ConditionSetTraffic | None = None,
) -> HealthContext:
return replace(
RUNNING_CONTEXT,
@@ -57,6 +75,7 @@ def _context(
groups=filters.get("groups"),
variants=(filters.get("multivariate") or {}).get("variants"),
),
+ condition_set_traffic=traffic,
)
@@ -84,6 +103,16 @@ class TestFlagState(TestCase):
_context(RUNNING, active=True, filters=ZERO_ROLLOUT),
(ExperimentHealthFindingCode.FLAG_OFF_WHILE_RUNNING, "running_but_no_rollout", None),
),
+ (
+ "running_zero_rollout_with_variants_assigned_outside",
+ _context(RUNNING, active=True, filters=ZERO_ROLLOUT, traffic=_traffic(10)),
+ (ExperimentHealthFindingCode.FLAG_OFF_WHILE_RUNNING, "assigned_outside_flag_service", None),
+ ),
+ (
+ "running_zero_rollout_with_too_few_assigned_outside",
+ _context(RUNNING, active=True, filters=ZERO_ROLLOUT, traffic=_traffic(9)),
+ (ExperimentHealthFindingCode.FLAG_OFF_WHILE_RUNNING, "running_but_no_rollout", None),
+ ),
(
"running_zero_rollout_before_shipped_variant",
_context(RUNNING, active=True, filters=ZERO_ROLLOUT_SHIPPED_VARIANT),
@@ -159,3 +188,21 @@ class TestFlagState(TestCase):
code, subcode, variant_key = expected
self.assertEqual((finding.code, finding.subcode), (code, subcode))
self.assertEqual(finding.evidence.get("variant_key"), variant_key)
+
+ def test_variants_assigned_outside_the_flag_service_are_info(self) -> None:
+ finding = flag_state(_context(RUNNING, active=True, filters=ZERO_ROLLOUT, traffic=_traffic(1234)))
+
+ assert finding is not None
+ self.assertEqual(
+ (finding.severity, finding.title, finding.detail, dict(finding.evidence)),
+ (
+ ExperimentHealthFindingSeverity.INFO,
+ "Variants are assigned outside PostHog",
+ "The linked feature flag has a 0% rollout, so PostHog assigns no variants. But 1,234 users got a "
+ "variant since October 12, 2026 at 4:04 PM CEST from flag calls that PostHog didn't evaluate, so your "
+ "own code assigns the variants, directly or through a bootstrap. These users count in the results. "
+ "The flag's release conditions and variant split don't decide who gets which variant. If PostHog "
+ "should assign the variants, increase the rollout and stop assigning them in your code.",
+ {"people_assigned_outside": 1234, "assigned_outside_since": "2026-10-12T14:04:00+00:00"},
+ ),
+ )
diff --git a/products/experiments/backend/health/test/test_forced_variant.py b/products/experiments/backend/health/test/test_forced_variant.py
index cb48d6e5cd12..3b94b59b8db4 100644
--- a/products/experiments/backend/health/test/test_forced_variant.py
+++ b/products/experiments/backend/health/test/test_forced_variant.py
@@ -9,7 +9,7 @@ from parameterized import parameterized
from products.experiments.backend.facade.contracts import ExperimentHealthFindingSeverity
from products.experiments.backend.health.checks.forced_variant import forced_variant_release_condition
from products.experiments.backend.health.condition_changes import parse_release_conditions
-from products.experiments.backend.health.condition_set_traffic import needs_condition_set_traffic
+from products.experiments.backend.health.condition_set_traffic import grades_by_pinned_share
from products.experiments.backend.health.context import (
ConditionsChange,
ConditionSetTraffic,
@@ -47,6 +47,14 @@ IOS: dict[str, Any] = {
CHANGED_AT = datetime(2026, 9, 12, 13, 4, tzinfo=UTC)
+def _property_is(key: str, value: str, *, group_type_index: int | None = None) -> dict[str, Any]:
+ source = {"type": "person"} if group_type_index is None else {"type": "group", "group_type_index": group_type_index}
+ return {
+ "properties": [{"key": key, "value": [value], "operator": "exact", **source}],
+ "rollout_percentage": 100,
+ }
+
+
def _context(
groups: list[dict[str, Any]],
*,
@@ -88,6 +96,7 @@ def _traffic(people_by_set: dict[int, int], *, remotely_evaluated: float = 1.0)
people_by_set=people_by_set,
calls=calls,
remotely_evaluated_calls=round(calls * remotely_evaluated),
+ people_assigned_outside=0,
)
@@ -115,6 +124,51 @@ class TestForcedVariantReleaseCondition(TestCase):
_context([{**IOS, "variant": "test"}, {**STAFF, "variant": "control"}]),
("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "test, control", None),
),
+ (
+ "variants_mapped_from_a_property",
+ _context(
+ [
+ {**_property_is("arm", "Control"), "variant": "control"},
+ {**_property_is("arm", "test"), "variant": "test"},
+ ]
+ ),
+ ("assigned_by_property", ExperimentHealthFindingSeverity.INFO, "1, 2", "control, test", None),
+ ),
+ (
+ "segments_mapped_onto_variants",
+ _context(
+ [
+ {**_property_is("plan", "free"), "variant": "control"},
+ {**_property_is("plan", "pro"), "variant": "test"},
+ ]
+ ),
+ ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),
+ ),
+ (
+ "variants_mapped_from_a_person_and_a_group_property_of_one_key",
+ _context(
+ [
+ {**_property_is("arm", "control"), "variant": "control"},
+ {**_property_is("arm", "test", group_type_index=0), "variant": "test"},
+ ]
+ ),
+ ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),
+ ),
+ (
+ "one_variant_mapped_from_a_property",
+ _context([{**_property_is("arm", "control"), "variant": "control"}]),
+ ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1", "control", None),
+ ),
+ (
+ "variants_mapped_from_two_properties",
+ _context(
+ [
+ {**_property_is("arm", "control"), "variant": "control"},
+ {**_property_is("cell", "test"), "variant": "test"},
+ ]
+ ),
+ ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),
+ ),
("override_after_a_catch_all_is_unreachable", _context([EVERYONE, {**STAFF, "variant": "test"}]), None),
(
"override_after_a_partial_catch_all_with_early_exit",
@@ -313,8 +367,8 @@ class TestForcedVariantReleaseCondition(TestCase):
),
]
)
- def test_needs_condition_set_traffic(self, _name: str, ctx: HealthContext, expected: bool) -> None:
- self.assertIs(needs_condition_set_traffic(ctx), expected)
+ def test_grades_by_pinned_share(self, _name: str, ctx: HealthContext, expected: bool) -> None:
+ self.assertIs(grades_by_pinned_share(ctx), expected)
@parameterized.expand(
[
@@ -344,3 +398,27 @@ class TestForcedVariantReleaseCondition(TestCase):
else "Then reset the experiment, or move its start date to after the removal."
)
)
+
+ def test_variants_mapped_from_a_property_say_who_assigns_them(self) -> None:
+ finding = forced_variant_release_condition(
+ _context(
+ [
+ {**_property_is("checkout_arm", "control"), "variant": "control"},
+ {**_property_is("checkout_arm", "test"), "variant": "test"},
+ ]
+ )
+ )
+
+ assert finding is not None
+ self.assertEqual(
+ (finding.title, finding.detail, finding.evidence["variant_property"]),
+ (
+ "Variants are assigned outside PostHog",
+ 'Each release condition that includes users matches the users whose property "checkout_arm" holds a '
+ 'variant key, and serves them that variant. So the code that sets "checkout_arm" assigns the '
+ "variants, not PostHog. If it assigns at random, each variant gets similar users. If it assigns by "
+ "who the users are, for example by plan or country, a difference in the results can come from those "
+ "users and not from the variant.",
+ "checkout_arm",
+ ),
+ )
diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.py
index e9512eb4b00a..f1f2fd91b7e7 100644
--- a/products/experiments/backend/presentation/serializers.py
+++ b/products/experiments/backend/presentation/serializers.py
@@ -3238,6 +3238,13 @@ class ExperimentHealthInputsConditionSetTrafficSerializer(serializers.Serializer
"server SDK and a bootstrapped value name none."
)
)
+ people_assigned_outside = serializers.IntegerField(
+ help_text=(
+ "The people who got a variant from a call without a flag version that is no local evaluation in a server "
+ "SDK, from an hour after the start of the window: a bootstrap or the caller's own assignment. With a 0% "
+ "rollout, 10 or more of them turn `running_but_no_rollout` into `assigned_outside_flag_service`."
+ )
+ )
class ExperimentHealthInputsReaderAccessSerializer(serializers.Serializer):
diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.py
index 870b51c8b150..fd3c09cd2cce 100644
--- a/products/experiments/backend/presentation/views.py
+++ b/products/experiments/backend/presentation/views.py
@@ -935,6 +935,13 @@ class EnterpriseExperimentsViewSet(
the caller's own assignment). A pin to a variant with 0% in the split adds an info,
because that variant's results show only the pinned users, unless the experiment
excludes the variant from the analysis.
+
+ Two findings say that the variants are assigned outside PostHog, as info. A running
+ experiment whose flag has a 0% rollout gets `assigned_outside_flag_service` instead of
+ `running_but_no_rollout` when 10 or more people got a variant from flag calls without a
+ flag version that are no local evaluation, from an hour after the window starts. When
+ every condition that includes users pins the variant whose key one property holds,
+ `assigned_by_property` replaces `all_conditions_pinned`.
"""
experiment: Experiment = self.get_object()
findings = get_experiment_health_findings(
diff --git a/products/experiments/backend/test/test_presentation_api.py b/products/experiments/backend/test/test_presentation_api.py
index 205c63175c47..dfe3ea8652a9 100644
--- a/products/experiments/backend/test/test_presentation_api.py
+++ b/products/experiments/backend/test/test_presentation_api.py
@@ -1,3 +1,4 @@
+from dataclasses import replace
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from typing import Any, cast
@@ -106,6 +107,7 @@ _HEALTH_TRAFFIC = ConditionSetTraffic(
people_by_set={1: 120, 2: 880},
calls=2000,
remotely_evaluated_calls=2000,
+ people_assigned_outside=0,
)
_HEALTH_METRIC = {
"kind": "ExperimentMetric",
@@ -964,6 +966,54 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):
[("some_conditions_pinned", expected_severity)],
)
+ @parameterized.expand(
+ [
+ ("variants_assigned_outside", 300, ("assigned_outside_flag_service", "info")),
+ ("no_one_assigned_outside", 0, ("running_but_no_rollout", "warning")),
+ ]
+ )
+ def test_health_reads_the_calls_of_a_flag_with_no_rollout(
+ self, _name: str, people_assigned_outside: int, expected: tuple[str, str]
+ ) -> None:
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ key=f"health-no-rollout-{_name}",
+ created_by=self.user,
+ filters={
+ "groups": [{"properties": [], "rollout_percentage": 0}],
+ "multivariate": {
+ "variants": [
+ {"key": "control", "rollout_percentage": 50},
+ {"key": "test", "rollout_percentage": 50},
+ ]
+ },
+ },
+ )
+ experiment = Experiment.objects.create(
+ team=self.team,
+ name="health-no-rollout",
+ feature_flag=flag,
+ start_date=timezone.now() - timedelta(days=3),
+ metrics=[_HEALTH_METRIC],
+ )
+ traffic = replace(_HEALTH_TRAFFIC, people_by_set={}, people_assigned_outside=people_assigned_outside)
+
+ with patch(
+ "products.experiments.backend.health.condition_set_traffic.query_condition_set_traffic",
+ side_effect=[traffic],
+ ):
+ response = self.client.get(f"/api/projects/{self.team.id}/experiments/{experiment.id}/health/")
+
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.content)
+ self.assertEqual(
+ [
+ (finding["subcode"], finding["severity"])
+ for finding in response.json()["findings"]
+ if finding["code"] == "flag_off_while_running"
+ ],
+ [expected],
+ )
+
@parameterized.expand(
[
("staff_session", True, None, status.HTTP_200_OK, ("read", [120, 880], "ok", "ok")),
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index 3fd756940ce2..2e7ab7bef40c 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -3176,6 +3176,8 @@ export interface ExperimentHealthInputsConditionSetTrafficApi {
calls: number
/** The calls that the flag service evaluated. Only those name a condition set. Local evaluation in a server SDK and a bootstrapped value name none. */
remotely_evaluated_calls: number
+ /** The people who got a variant from a call without a flag version that is no local evaluation in a server SDK, from an hour after the start of the window: a bootstrap or the caller's own assignment. With a 0% rollout, 10 or more of them turn `running_but_no_rollout` into `assigned_outside_flag_service`. */
+ people_assigned_outside: number
}
export interface ExperimentHealthInputsReaderAccessApi {
diff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.ts
index d8d6f1addd09..ad9b77d47b1b 100644
--- a/products/experiments/frontend/generated/api.ts
+++ b/products/experiments/frontend/generated/api.ts
@@ -723,6 +723,13 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =
* the caller's own assignment). A pin to a variant with 0% in the split adds an info,
* because that variant's results show only the pinned users, unless the experiment
* excludes the variant from the analysis.
+ *
+ * Two findings say that the variants are assigned outside PostHog, as info. A running
+ * experiment whose flag has a 0% rollout gets `assigned_outside_flag_service` instead of
+ * `running_but_no_rollout` when 10 or more people got a variant from flag calls without a
+ * flag version that are no local evaluation, from an hour after the window starts. When
+ * every condition that includes users pins the variant whose key one property holds,
+ * `assigned_by_property` replaces `all_conditions_pinned`.
*/
export const experimentsHealthRetrieve = async (
projectId: string,
diff --git a/products/experiments/frontend/health/debug/healthDebugInputs.ts b/products/experiments/frontend/health/debug/healthDebugInputs.ts
index 674fff8da410..d6ea9e6d2d7a 100644
--- a/products/experiments/frontend/health/debug/healthDebugInputs.ts
+++ b/products/experiments/frontend/health/debug/healthDebugInputs.ts
@@ -227,7 +227,7 @@ export function buildTrafficFacts(state: HealthInputsState): SetupFact[] {
{
label: 'Status',
value: serverValue(state, (inputs) => TRAFFIC_STATUS_LABELS[inputs.condition_set_traffic_status]),
- help: 'The forced-variant check reads the calls when the experiment is launched and some, but not all, release conditions pin a variant that has a share of the split and is not excluded from the analysis. The release-condition check reads them when such an override was removed after launch.',
+ help: 'The forced-variant check reads the calls when the experiment is launched and some, but not all, release conditions pin a variant that has a share of the split and is not excluded from the analysis. The release-condition check reads them when such an override was removed after launch. The flag-state check reads them when the experiment is running and the flag has a 0% rollout.',
},
...(traffic
? [
@@ -239,6 +239,11 @@ export function buildTrafficFacts(state: HealthInputsState): SetupFact[] {
help: 'Only these calls name the condition that served them. Local evaluation and bootstrapped values name none.',
},
{ label: 'People served by a named condition', value: formatCount(people) },
+ {
+ label: 'People assigned outside the flag service',
+ value: formatCount(traffic.people_assigned_outside),
+ help: 'People who got a variant from a call without a flag version that is no local evaluation, from an hour after the window starts. With a 0% rollout, 10 or more of them show that the variants are assigned outside PostHog.',
+ },
]
: []),
]
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 0a5773273d50..9ad4e7e93294 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -45580,6 +45580,8 @@ export namespace Schemas {
calls: number;
/** The calls that the flag service evaluated. Only those name a condition set. Local evaluation in a server SDK and a bootstrapped value name none. */
remotely_evaluated_calls: number;
+ /** The people who got a variant from a call without a flag version that is no local evaluation in a server SDK, from an hour after the start of the window: a bootstrap or the caller's own assignment. With a 0% rollout, 10 or more of them turn `running_but_no_rollout` into `assigned_outside_flag_service`. */
+ people_assigned_outside: number;
}
export interface ExperimentHealthInputsReaderAccess {