:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(today): show experiments that need a decision or more users (#117283)
Shy Alter955026975ec0
diff --git a/products/experiments/backend/facade/api.py b/products/experiments/backend/facade/api.pyindex f4a608de04c4..9ea98e6414ac 100644--- a/products/experiments/backend/facade/api.py+++ b/products/experiments/backend/facade/api.py@@ -5,15 +5,19 @@ This module provides the public interface for creating and managing experimentsusing framework-free DTOs, wrapping the existing ExperimentService."""+from datetime import datetimefrom typing import Anyfrom uuid import UUID+import pydantic+import structlogfrom rest_framework.exceptions import ValidationErrorfrom posthog.models.team import Teamfrom posthog.models.user import Userfrom products.experiments.backend.experiment_service import ExperimentService+from products.experiments.backend.flag_enrollment import enrolled_people_by_flagfrom products.experiments.backend.health.inputs_payload import health_inputs_payloadfrom products.experiments.backend.health.loader import load_experiment_health_context, load_experiment_health_inputsfrom products.experiments.backend.health.registry import (@@ -23,11 +27,22 @@ from products.experiments.backend.health.registry import (from products.experiments.backend.hogql_queries.exposure_query_logic import (DEFAULT_EXPOSURE_EVENT,get_exposure_event_and_property,+ is_default_exposure_config,+ normalize_to_exposure_criteria,resolve_default_exposure_event,)from products.experiments.backend.models.experiment import Experiment as ExperimentModel-from .contracts import CreateExperimentInput, Experiment, ExperimentHealth, ExperimentHealthReaderAccess+from .contracts import (+ CreateExperimentInput,+ EndedExperiment,+ Experiment,+ ExperimentHealth,+ ExperimentHealthReaderAccess,+ RunningExperiment,+)++logger = structlog.get_logger(__name__)def create_experiment(*, team: Team, user: User, input_dto: CreateExperimentInput) -> Experiment:@@ -148,6 +163,63 @@ def get_experiment_health_inputs(return health_inputs_payload(inputs)+def list_ended_experiments_without_conclusion(+ *, team_id: int, user_id: int, since: datetime, limit: int+) -> list[EndedExperiment]:+ rows = (+ ExperimentModel.objects.filter(+ team_id=team_id,+ created_by_id=user_id,+ end_date__gte=since,+ conclusion__isnull=True,+ archived=False,+ )+ .exclude(deleted=True)+ .order_by("-end_date")+ .values_list("id", "name", "end_date")[:limit]+ )+ return [EndedExperiment(id=id, name=name, ended_at=ended_at) for id, name, ended_at in rows]+++def _exposes_on_flag_calls(experiment_id: int, exposure_criteria: dict | None) -> bool:+ try:+ criteria = normalize_to_exposure_criteria(exposure_criteria)+ except pydantic.ValidationError:+ logger.warning("experiment_exposure_criteria_invalid", experiment_id=experiment_id, exc_info=True)+ return False+ return criteria is None or (+ criteria.activation_config is None+ and is_default_exposure_config(criteria.exposure_config)+ and not (criteria.exposure_config and criteria.exposure_config.properties)+ )+++def list_running_experiments(+ *, team_id: int, user_id: int, started_after: datetime | None, started_before: datetime, limit: int+) -> list[RunningExperiment]:+ experiments = ExperimentModel.objects.filter(+ team_id=team_id,+ created_by_id=user_id,+ start_date__lte=started_before,+ end_date__isnull=True,+ archived=False,+ ).exclude(deleted=True)+ if started_after is not None:+ experiments = experiments.filter(start_date__gt=started_after)+ rows = experiments.order_by("-start_date").values_list(+ "id", "name", "start_date", "feature_flag__key", "exposure_criteria"+ )[:limit]+ return [+ RunningExperiment(id=id, name=name, started_at=started_at, flag_key=flag_key)+ for id, name, started_at, flag_key, exposure_criteria in rows+ if _exposes_on_flag_calls(id, exposure_criteria)+ ]+++def count_enrolled_people(*, team: Team, user: User, flag_keys: list[str]) -> dict[str, int]:+ return enrolled_people_by_flag(team, user, flag_keys)++def _experiment_model_to_dto(experiment: ExperimentModel) -> Experiment:"""Convert Django model to DTO."""return Experiment(diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.pyindex 9d4137016f1a..ea3a06b2b63c 100644--- a/products/experiments/backend/facade/contracts.py+++ b/products/experiments/backend/facade/contracts.py@@ -351,3 +351,18 @@ class DailyMetricCalculation:metric_uuid: str# The key that files the metric's daily result.calculation_key: str+++@frozen+class EndedExperiment:+ id: int+ name: str+ ended_at: datetime+++@frozen+class RunningExperiment:+ id: int+ name: str+ started_at: datetime+ flag_key: strdiff --git a/products/experiments/backend/flag_enrollment.py b/products/experiments/backend/flag_enrollment.pynew file mode 100644index 000000000000..759d5de5df5f--- /dev/null+++ b/products/experiments/backend/flag_enrollment.py@@ -0,0 +1,43 @@+from posthog.hogql import ast+from posthog.hogql.constants import HogQLGlobalSettings+from posthog.hogql.query import execute_hogql_query++from posthog.models.team import Team+from posthog.models.user import User++from products.feature_flags.backend.facade.enums import FlagEvaluationsMode+from products.feature_flags.backend.facade.flags import get_flag_evaluations_read_mode++QUERY_TIMEOUT_SECONDS = 10+ENROLLED_FROM_EVENTS = """+SELECT properties.$feature_flag AS flag_key, uniq(person_id) AS people+FROM events+WHERE event = '$feature_flag_called'+ AND timestamp >= toStartOfDay(now()) - INTERVAL 14 DAY AND timestamp < toStartOfDay(now())+ AND properties.$feature_flag IN {flag_keys}+ AND toString(properties.$feature_flag_response) NOT IN ('', 'false')+GROUP BY flag_key+"""+ENROLLED_FROM_EVALUATIONS = """+SELECT flag_key, uniq(person_id) AS people+FROM posthog.flag_evaluations+WHERE timestamp >= toStartOfDay(now()) - INTERVAL 14 DAY AND timestamp < toStartOfDay(now())+ AND flag_key IN {flag_keys}+ AND response NOT IN ('', 'false', 'null')+GROUP BY flag_key+"""+++def enrolled_people_by_flag(team: Team, user: User, flag_keys: list[str]) -> dict[str, int]:+ if not flag_keys:+ return {}+ from_events = get_flag_evaluations_read_mode(team.organization_id) == FlagEvaluationsMode.EVENTS+ result = execute_hogql_query(+ ENROLLED_FROM_EVENTS if from_events else ENROLLED_FROM_EVALUATIONS,+ team,+ query_type="experiments_enrolled_people_by_flag",+ placeholders={"flag_keys": ast.Constant(value=flag_keys)},+ user=user,+ settings=HogQLGlobalSettings(max_execution_time=QUERY_TIMEOUT_SECONDS, timeout_overflow_mode="throw"),+ )+ return dict(result.results or [])diff --git a/products/experiments/backend/test/test_facade.py b/products/experiments/backend/test/test_facade.pyindex 1f11cf5e41e5..ccbc1c6d7a06 100644--- a/products/experiments/backend/test/test_facade.py+++ b/products/experiments/backend/test/test_facade.py@@ -1,19 +1,33 @@"""Tests for experiments facade layer."""-from datetime import UTC, datetime+import uuid+from datetime import UTC, datetime, timedeltaimport time_machine-from posthog.test.base import APIBaseTest, BaseTest+from posthog.test.base import (+ APIBaseTest,+ BaseTest,+ ClickhouseTestMixin,+ _create_event,+ _create_flag_evaluations,+ flush_persons_and_events,+)++from django.utils import timezonefrom parameterized import parameterizedfrom posthog.models.organization import Organizationfrom posthog.models.team import Team+from posthog.models.user import User+from posthog.test.persons import create_person-from products.experiments.backend.facade import count_running_experiments_on_feature_flag_called, create_experiment-from products.experiments.backend.facade.contracts import CreateExperimentInput+from products.experiments.backend.facade import api, count_running_experiments_on_feature_flag_called, create_experiment+from products.experiments.backend.facade.contracts import CreateExperimentInput, EndedExperimentfrom products.experiments.backend.models.experiment import Experiment+from products.feature_flags.backend.facade.enums import FlagEvaluationsModefrom products.feature_flags.backend.models.feature_flag import FeatureFlag+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfigPAGEVIEW_EXPOSURE = {"exposure_config": {"kind": "ExperimentEventExposureConfig", "event": "$pageview", "properties": []}@@ -191,3 +205,121 @@ class TestCountRunningExperimentsOnFeatureFlagCalled(BaseTest):)assert count_running_experiments_on_feature_flag_called(self.organization.id) == expected+++@time_machine.travel("2026-10-14T12:00:00Z", tick=False)+class TestHighlightFacts(BaseTest):+ def _experiment(self, name: str, created_by: User | None = None, **fields: object) -> Experiment:+ flag = FeatureFlag.objects.create(team=self.team, key=f"flag-{uuid.uuid4().hex[:8]}", created_by=self.user)+ return Experiment.objects.create(+ team=self.team, name=name, feature_flag=flag, created_by=created_by or self.user, **fields+ )++ def test_lists_ended_experiments_without_a_conclusion(self) -> None:+ now = timezone.now()+ started = now - timedelta(days=30)+ ended_at = now - timedelta(days=1)+ listed = self._experiment("Pricing page layout", start_date=started, end_date=ended_at)+ self._experiment("Concluded", start_date=started, end_date=ended_at, conclusion="won")+ self._experiment("Ended long ago", start_date=started, end_date=now - timedelta(days=10))+ self._experiment("Archived", start_date=started, end_date=ended_at, archived=True)+ self._experiment("Deleted", start_date=started, end_date=ended_at, deleted=True)+ self._experiment("Still running", start_date=started)+ self._experiment(+ "Not yours", start_date=started, end_date=ended_at, created_by=self._create_user("[email protected]")+ )++ assert api.list_ended_experiments_without_conclusion(+ team_id=self.team.pk, user_id=self.user.pk, since=now - timedelta(days=7), limit=20+ ) == [EndedExperiment(id=listed.id, name="Pricing page layout", ended_at=ended_at)]++ @parameterized.expand(+ [("any start", None, ["Checkout copy", "Onboarding tips"]), ("a recent start", 67, ["Checkout copy"])]+ )+ def test_lists_running_experiments_that_started_in_the_window(+ self, _name: str, started_after_days: int | None, expected: list[str]+ ) -> None:+ now = timezone.now()+ listed = {+ experiment.name: experiment.feature_flag.key+ for experiment in (+ self._experiment("Onboarding tips", start_date=now - timedelta(days=80)),+ self._experiment("Checkout copy", start_date=now - timedelta(days=62)),+ )+ }+ self._experiment("Started recently", start_date=now - timedelta(days=10))+ self._experiment("Custom exposure", start_date=now - timedelta(days=62), exposure_criteria=PAGEVIEW_EXPOSURE)+ self._experiment(+ "Activation",+ start_date=now - timedelta(days=62),+ exposure_criteria={"activation_config": {"event": "signed up", "properties": []}},+ )+ self._experiment(+ "Filtered exposure",+ start_date=now - timedelta(days=62),+ exposure_criteria={+ "exposure_config": {+ "kind": "ExperimentEventExposureConfig",+ "event": "$feature_flag_called",+ "properties": [{"key": "plan", "value": ["paid"], "operator": "exact", "type": "person"}],+ }+ },+ )+ self._experiment(+ "Legacy criteria",+ start_date=now - timedelta(days=62),+ exposure_criteria={"multiple_variant_handling": "whatever_this_was"},+ )+ self._experiment("Draft")+ self._experiment("Ended", start_date=now - timedelta(days=80), end_date=now - timedelta(days=1))+ self._experiment(+ "Not yours", start_date=now - timedelta(days=80), created_by=self._create_user("[email protected]")+ )++ found = api.list_running_experiments(+ team_id=self.team.pk,+ user_id=self.user.pk,+ started_after=None if started_after_days is None else now - timedelta(days=started_after_days),+ started_before=now - timedelta(days=60),+ limit=20,+ )++ assert [(e.name, e.flag_key) for e in found] == [(name, listed[name]) for name in expected]+++class TestCountEnrolledPeople(ClickhouseTestMixin, BaseTest):+ def _ago(self, days: int) -> datetime:+ return timezone.now() - timedelta(days=days)++ @parameterized.expand(+ [("events", FlagEvaluationsMode.EVENTS), ("flag_evaluations", FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY)]+ )+ def test_counts_people_who_got_a_variant_in_the_last_14_days(self, _name: str, mode: FlagEvaluationsMode) -> None:+ OrganizationFeatureFlagsConfig.objects.update_or_create(+ organization=self.organization, defaults={"flag_evaluations_mode": mode}+ )+ calls = [+ ("checkout", "test", 2),+ ("checkout", "control", 3),+ ("checkout", "false", 2),+ ("checkout", "test", 20),+ ("other", "test", 2),+ ]+ for flag_key, response, days_ago in calls:+ distinct_id = str(uuid.uuid4())+ if mode == FlagEvaluationsMode.EVENTS:+ create_person(team=self.team, distinct_ids=[distinct_id])+ _create_event(+ team=self.team,+ event="$feature_flag_called",+ distinct_id=distinct_id,+ properties={"$feature_flag": flag_key, "$feature_flag_response": response},+ timestamp=self._ago(days_ago),+ )+ else:+ _create_flag_evaluations(+ self.team.pk, flag_key, timestamp=self._ago(days_ago), distinct_id=distinct_id, response=response+ )+ flush_persons_and_events()++ assert api.count_enrolled_people(team=self.team, user=self.user, flag_keys=["checkout"]) == {"checkout": 2}diff --git a/products/today/backend/logic/highlight_sources/experiments.py b/products/today/backend/logic/highlight_sources/experiments.pynew file mode 100644index 000000000000..e343b01e6798--- /dev/null+++ b/products/today/backend/logic/highlight_sources/experiments.py@@ -0,0 +1,98 @@+import re+from datetime import datetime, timedelta++from django.utils import timezone++from products.experiments.backend.facade.api import (+ count_enrolled_people,+ list_ended_experiments_without_conclusion,+ list_running_experiments,+)+from products.experiments.backend.facade.contracts import EndedExperiment, RunningExperiment++from ...facade.enums import HighlightLabel+from ..highlight_rules import short, users+from ..highlight_types import Candidate, Context, Found, Source++ENDED = "Experiment ‘{experiment}’ ended {when}"+ENDED_DETAIL = "Read the results and decide whether to release the change"+FEW_USERS = "Experiment ‘{experiment}’ reached {enrolled} in the last 14 days"+NO_USERS = "Experiment ‘{experiment}’ reached no users in the last 14 days"+FEW_USERS_DETAIL = "Stop it or increase the rollout"+ENDED_WINDOW_DAYS = 7+LONG_RUNNING_DAYS = 60+JUST_CROSSED_DAYS = 67+MIN_PEOPLE = 100+++def find(context: Context) -> Found:+ now = timezone.now()+ ended = list_ended_experiments_without_conclusion(+ team_id=context.team.id, user_id=context.user.id, since=now - timedelta(days=ENDED_WINDOW_DAYS), limit=20+ )+ running = list_running_experiments(+ team_id=context.team.id,+ user_id=context.user.id,+ started_after=None if context.today.weekday() == 0 else now - timedelta(days=JUST_CROSSED_DAYS),+ started_before=now - timedelta(days=LONG_RUNNING_DAYS),+ limit=100,+ )+ enrolled = context.guard(+ "experiments_enrolled",+ lambda: count_enrolled_people(+ team=context.team, user=context.user, flag_keys=[experiment.flag_key for experiment in running]+ ),+ None,+ )+ counted = [] if enrolled is None else [(experiment, enrolled.get(experiment.flag_key, 0)) for experiment in running]+ return Found(+ items=(+ *(_ended(context, experiment) for experiment in ended),+ *(_few_users(experiment, people) for experiment, people in counted if people < MIN_PEOPLE),+ )+ )+++def _days_since(context: Context, moment: datetime) -> int:+ return (context.today - timezone.localtime(moment, context.team.timezone_info).date()).days+++def _when(days: int) -> str:+ return "today" if days <= 0 else "yesterday" if days == 1 else f"{days} days ago"+++def _name(name: str, limit: int) -> str:+ return short(re.sub(r"(?i)\s+experiment$", "", name) or name, limit)+++def _ended(context: Context, experiment: EndedExperiment) -> Candidate:+ return Candidate(+ key=f"experiment_ended:{experiment.id}",+ label=HighlightLabel.OPPORTUNITY,+ family="experiments",+ importance=0.6,+ relevance=1.0,+ title=ENDED.format(+ experiment=_name(experiment.name, 60), when=_when(_days_since(context, experiment.ended_at))+ ),+ detail=ENDED_DETAIL,+ url=f"/experiments/{experiment.id}",+ )+++def _few_users(experiment: RunningExperiment, enrolled: int) -> Candidate:+ return Candidate(+ key=f"experiment_stale:{experiment.id}",+ label=HighlightLabel.RISK,+ family="experiments",+ importance=0.3,+ relevance=1.0,+ title=(FEW_USERS if enrolled else NO_USERS).format(+ experiment=_name(experiment.name, 50), enrolled=users(enrolled)+ ),+ detail=FEW_USERS_DETAIL,+ url=f"/experiments/{experiment.id}",+ )+++SOURCE = Source(name="experiments", find=find, scopes=("experiment:read",))diff --git a/products/today/backend/tests/test_highlight_experiments.py b/products/today/backend/tests/test_highlight_experiments.pynew file mode 100644index 000000000000..6cdb615791e1--- /dev/null+++ b/products/today/backend/tests/test_highlight_experiments.py@@ -0,0 +1,86 @@+from datetime import UTC, date, datetime, time, timedelta++import time_machine+from posthog.test.base import BaseTest+from unittest.mock import MagicMock, patch++from parameterized import parameterized++from products.experiments.backend.facade.contracts import EndedExperiment, RunningExperiment+from products.today.backend.logic.highlight_registry import load_sources+from products.today.backend.logic.highlight_sources import experiments+from products.today.backend.logic.highlight_types import Context, Found++MONDAY = date(2026, 10, 12)+TUESDAY = date(2026, 10, 13)+NOW = datetime.combine(TUESDAY, time(12), tzinfo=UTC)+ENDED = EndedExperiment(id=7, name="Pricing page layout experiment", ended_at=NOW - timedelta(days=1))+RUNNING = RunningExperiment(+ id=9, name="Onboarding tips", started_at=NOW - timedelta(days=62), flag_key="onboarding-tips"+)+++@time_machine.travel(NOW, tick=False)+class TestExperiments(BaseTest):+ def _find(self, today: date, running: MagicMock, enrolled: MagicMock) -> tuple[Found, Context]:+ registered = next(s for s in load_sources() if s.name == "experiments")+ context = Context(self.team, self.user, today)+ with (+ patch.object(experiments, "list_ended_experiments_without_conclusion", return_value=[ENDED]),+ patch.object(experiments, "list_running_experiments", running),+ patch.object(experiments, "count_enrolled_people", enrolled),+ ):+ return registered.find(context), context++ def test_an_ended_experiment_without_a_decision_is_an_opportunity(self) -> None:+ found, _ = self._find(TUESDAY, MagicMock(return_value=[]), MagicMock(return_value={}))++ self.assertEqual(+ [(c.title, c.detail, c.url) for c in found.items],+ [+ (+ "Experiment ‘Pricing page layout’ ended yesterday",+ "Read the results and decide whether to release the change",+ "/experiments/7",+ )+ ],+ )++ @parameterized.expand([("Monday", MONDAY, None), ("Tuesday", TUESDAY, NOW - timedelta(days=67))])+ def test_only_mondays_look_past_the_week_an_experiment_crosses_60_days(+ self, _name: str, today: date, started_after: datetime | None+ ) -> None:+ running = MagicMock(return_value=[])++ self._find(today, running, MagicMock(return_value={}))++ self.assertEqual(+ (running.call_args.kwargs["started_after"], running.call_args.kwargs["started_before"]),+ (started_after, NOW - timedelta(days=60)),+ )++ @parameterized.expand(+ [+ ("few users", 42, "Experiment ‘Onboarding tips’ reached 42 users in the last 14 days"),+ ("one user", 1, "Experiment ‘Onboarding tips’ reached 1 user in the last 14 days"),+ ("no users", 0, "Experiment ‘Onboarding tips’ reached no users in the last 14 days"),+ ("enough users", 150, None),+ ]+ )+ def test_a_long_running_experiment_with_few_users_is_a_risk(+ self, _name: str, enrolled: int, title: str | None+ ) -> None:+ found, _ = self._find(+ TUESDAY, MagicMock(return_value=[RUNNING]), MagicMock(return_value={"onboarding-tips": enrolled})+ )++ self.assertEqual([c.title for c in found.items][1:], [title] if title else [])++ def test_a_failed_count_keeps_the_ended_reminders(self) -> None:+ found, context = self._find(+ TUESDAY, MagicMock(return_value=[RUNNING]), MagicMock(side_effect=TimeoutError("slow"))+ )++ self.assertEqual(+ ([c.key for c in found.items], context.failures), (["experiment_ended:7"], ["experiments_enrolled"])+ )diff --git a/tach.toml b/tach.tomlindex 7d2fb0a59792..a292cc3f57e7 100644--- a/tach.toml+++ b/tach.toml@@ -1681,6 +1681,7 @@ depends_on = ["products.dashboards","products.data_catalog","products.error_tracking",+ "products.experiments","products.feature_flags","products.product_analytics","products.signals",