Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(today): show experiments that need a decision or more users (#117283)

Shy Alter955026975ec0
diff --git a/products/experiments/backend/facade/api.py b/products/experiments/backend/facade/api.py
index f4a608de04c4..9ea98e6414ac 100644
--- a/products/experiments/backend/facade/api.py
+++ b/products/experiments/backend/facade/api.py
@@ -5,15 +5,19 @@ This module provides the public interface for creating and managing experiments
using framework-free DTOs, wrapping the existing ExperimentService.
"""
+from datetime import datetime
from typing import Any
from uuid import UUID
+import pydantic
+import structlog
from rest_framework.exceptions import ValidationError
from posthog.models.team import Team
from posthog.models.user import User
from products.experiments.backend.experiment_service import ExperimentService
+from products.experiments.backend.flag_enrollment import enrolled_people_by_flag
from products.experiments.backend.health.inputs_payload import health_inputs_payload
from products.experiments.backend.health.loader import load_experiment_health_context, load_experiment_health_inputs
from products.experiments.backend.health.registry import (
@@ -23,11 +27,22 @@ from products.experiments.backend.health.registry import (
from products.experiments.backend.hogql_queries.exposure_query_logic import (
DEFAULT_EXPOSURE_EVENT,
get_exposure_event_and_property,
+ is_default_exposure_config,
+ normalize_to_exposure_criteria,
resolve_default_exposure_event,
)
from products.experiments.backend.models.experiment import Experiment as ExperimentModel
-from .contracts import CreateExperimentInput, Experiment, ExperimentHealth, ExperimentHealthReaderAccess
+from .contracts import (
+ CreateExperimentInput,
+ EndedExperiment,
+ Experiment,
+ ExperimentHealth,
+ ExperimentHealthReaderAccess,
+ RunningExperiment,
+)
+
+logger = structlog.get_logger(__name__)
def create_experiment(*, team: Team, user: User, input_dto: CreateExperimentInput) -> Experiment:
@@ -148,6 +163,63 @@ def get_experiment_health_inputs(
return health_inputs_payload(inputs)
+def list_ended_experiments_without_conclusion(
+ *, team_id: int, user_id: int, since: datetime, limit: int
+) -> list[EndedExperiment]:
+ rows = (
+ ExperimentModel.objects.filter(
+ team_id=team_id,
+ created_by_id=user_id,
+ end_date__gte=since,
+ conclusion__isnull=True,
+ archived=False,
+ )
+ .exclude(deleted=True)
+ .order_by("-end_date")
+ .values_list("id", "name", "end_date")[:limit]
+ )
+ return [EndedExperiment(id=id, name=name, ended_at=ended_at) for id, name, ended_at in rows]
+
+
+def _exposes_on_flag_calls(experiment_id: int, exposure_criteria: dict | None) -> bool:
+ try:
+ criteria = normalize_to_exposure_criteria(exposure_criteria)
+ except pydantic.ValidationError:
+ logger.warning("experiment_exposure_criteria_invalid", experiment_id=experiment_id, exc_info=True)
+ return False
+ return criteria is None or (
+ criteria.activation_config is None
+ and is_default_exposure_config(criteria.exposure_config)
+ and not (criteria.exposure_config and criteria.exposure_config.properties)
+ )
+
+
+def list_running_experiments(
+ *, team_id: int, user_id: int, started_after: datetime | None, started_before: datetime, limit: int
+) -> list[RunningExperiment]:
+ experiments = ExperimentModel.objects.filter(
+ team_id=team_id,
+ created_by_id=user_id,
+ start_date__lte=started_before,
+ end_date__isnull=True,
+ archived=False,
+ ).exclude(deleted=True)
+ if started_after is not None:
+ experiments = experiments.filter(start_date__gt=started_after)
+ rows = experiments.order_by("-start_date").values_list(
+ "id", "name", "start_date", "feature_flag__key", "exposure_criteria"
+ )[:limit]
+ return [
+ RunningExperiment(id=id, name=name, started_at=started_at, flag_key=flag_key)
+ for id, name, started_at, flag_key, exposure_criteria in rows
+ if _exposes_on_flag_calls(id, exposure_criteria)
+ ]
+
+
+def count_enrolled_people(*, team: Team, user: User, flag_keys: list[str]) -> dict[str, int]:
+ return enrolled_people_by_flag(team, user, flag_keys)
+
+
def _experiment_model_to_dto(experiment: ExperimentModel) -> Experiment:
"""Convert Django model to DTO."""
return Experiment(
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index 9d4137016f1a..ea3a06b2b63c 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -351,3 +351,18 @@ class DailyMetricCalculation:
metric_uuid: str
# The key that files the metric's daily result.
calculation_key: str
+
+
+@frozen
+class EndedExperiment:
+ id: int
+ name: str
+ ended_at: datetime
+
+
+@frozen
+class RunningExperiment:
+ id: int
+ name: str
+ started_at: datetime
+ flag_key: str
diff --git a/products/experiments/backend/flag_enrollment.py b/products/experiments/backend/flag_enrollment.py
new file mode 100644
index 000000000000..759d5de5df5f
--- /dev/null
+++ b/products/experiments/backend/flag_enrollment.py
@@ -0,0 +1,43 @@
+from posthog.hogql import ast
+from posthog.hogql.constants import HogQLGlobalSettings
+from posthog.hogql.query import execute_hogql_query
+
+from posthog.models.team import Team
+from posthog.models.user import User
+
+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
+from products.feature_flags.backend.facade.flags import get_flag_evaluations_read_mode
+
+QUERY_TIMEOUT_SECONDS = 10
+ENROLLED_FROM_EVENTS = """
+SELECT properties.$feature_flag AS flag_key, uniq(person_id) AS people
+FROM events
+WHERE event = '$feature_flag_called'
+ AND timestamp >= toStartOfDay(now()) - INTERVAL 14 DAY AND timestamp < toStartOfDay(now())
+ AND properties.$feature_flag IN {flag_keys}
+ AND toString(properties.$feature_flag_response) NOT IN ('', 'false')
+GROUP BY flag_key
+"""
+ENROLLED_FROM_EVALUATIONS = """
+SELECT flag_key, uniq(person_id) AS people
+FROM posthog.flag_evaluations
+WHERE timestamp >= toStartOfDay(now()) - INTERVAL 14 DAY AND timestamp < toStartOfDay(now())
+ AND flag_key IN {flag_keys}
+ AND response NOT IN ('', 'false', 'null')
+GROUP BY flag_key
+"""
+
+
+def enrolled_people_by_flag(team: Team, user: User, flag_keys: list[str]) -> dict[str, int]:
+ if not flag_keys:
+ return {}
+ from_events = get_flag_evaluations_read_mode(team.organization_id) == FlagEvaluationsMode.EVENTS
+ result = execute_hogql_query(
+ ENROLLED_FROM_EVENTS if from_events else ENROLLED_FROM_EVALUATIONS,
+ team,
+ query_type="experiments_enrolled_people_by_flag",
+ placeholders={"flag_keys": ast.Constant(value=flag_keys)},
+ user=user,
+ settings=HogQLGlobalSettings(max_execution_time=QUERY_TIMEOUT_SECONDS, timeout_overflow_mode="throw"),
+ )
+ return dict(result.results or [])
diff --git a/products/experiments/backend/test/test_facade.py b/products/experiments/backend/test/test_facade.py
index 1f11cf5e41e5..ccbc1c6d7a06 100644
--- a/products/experiments/backend/test/test_facade.py
+++ b/products/experiments/backend/test/test_facade.py
@@ -1,19 +1,33 @@
"""Tests for experiments facade layer."""
-from datetime import UTC, datetime
+import uuid
+from datetime import UTC, datetime, timedelta
import time_machine
-from posthog.test.base import APIBaseTest, BaseTest
+from posthog.test.base import (
+ APIBaseTest,
+ BaseTest,
+ ClickhouseTestMixin,
+ _create_event,
+ _create_flag_evaluations,
+ flush_persons_and_events,
+)
+
+from django.utils import timezone
from parameterized import parameterized
from posthog.models.organization import Organization
from posthog.models.team import Team
+from posthog.models.user import User
+from posthog.test.persons import create_person
-from products.experiments.backend.facade import count_running_experiments_on_feature_flag_called, create_experiment
-from products.experiments.backend.facade.contracts import CreateExperimentInput
+from products.experiments.backend.facade import api, count_running_experiments_on_feature_flag_called, create_experiment
+from products.experiments.backend.facade.contracts import CreateExperimentInput, EndedExperiment
from products.experiments.backend.models.experiment import Experiment
+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
from products.feature_flags.backend.models.feature_flag import FeatureFlag
+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
PAGEVIEW_EXPOSURE = {
"exposure_config": {"kind": "ExperimentEventExposureConfig", "event": "$pageview", "properties": []}
@@ -191,3 +205,121 @@ class TestCountRunningExperimentsOnFeatureFlagCalled(BaseTest):
)
assert count_running_experiments_on_feature_flag_called(self.organization.id) == expected
+
+
+@time_machine.travel("2026-10-14T12:00:00Z", tick=False)
+class TestHighlightFacts(BaseTest):
+ def _experiment(self, name: str, created_by: User | None = None, **fields: object) -> Experiment:
+ flag = FeatureFlag.objects.create(team=self.team, key=f"flag-{uuid.uuid4().hex[:8]}", created_by=self.user)
+ return Experiment.objects.create(
+ team=self.team, name=name, feature_flag=flag, created_by=created_by or self.user, **fields
+ )
+
+ def test_lists_ended_experiments_without_a_conclusion(self) -> None:
+ now = timezone.now()
+ started = now - timedelta(days=30)
+ ended_at = now - timedelta(days=1)
+ listed = self._experiment("Pricing page layout", start_date=started, end_date=ended_at)
+ self._experiment("Concluded", start_date=started, end_date=ended_at, conclusion="won")
+ self._experiment("Ended long ago", start_date=started, end_date=now - timedelta(days=10))
+ self._experiment("Archived", start_date=started, end_date=ended_at, archived=True)
+ self._experiment("Deleted", start_date=started, end_date=ended_at, deleted=True)
+ self._experiment("Still running", start_date=started)
+ self._experiment(
+ "Not yours", start_date=started, end_date=ended_at, created_by=self._create_user("[email protected]")
+ )
+
+ assert api.list_ended_experiments_without_conclusion(
+ team_id=self.team.pk, user_id=self.user.pk, since=now - timedelta(days=7), limit=20
+ ) == [EndedExperiment(id=listed.id, name="Pricing page layout", ended_at=ended_at)]
+
+ @parameterized.expand(
+ [("any start", None, ["Checkout copy", "Onboarding tips"]), ("a recent start", 67, ["Checkout copy"])]
+ )
+ def test_lists_running_experiments_that_started_in_the_window(
+ self, _name: str, started_after_days: int | None, expected: list[str]
+ ) -> None:
+ now = timezone.now()
+ listed = {
+ experiment.name: experiment.feature_flag.key
+ for experiment in (
+ self._experiment("Onboarding tips", start_date=now - timedelta(days=80)),
+ self._experiment("Checkout copy", start_date=now - timedelta(days=62)),
+ )
+ }
+ self._experiment("Started recently", start_date=now - timedelta(days=10))
+ self._experiment("Custom exposure", start_date=now - timedelta(days=62), exposure_criteria=PAGEVIEW_EXPOSURE)
+ self._experiment(
+ "Activation",
+ start_date=now - timedelta(days=62),
+ exposure_criteria={"activation_config": {"event": "signed up", "properties": []}},
+ )
+ self._experiment(
+ "Filtered exposure",
+ start_date=now - timedelta(days=62),
+ exposure_criteria={
+ "exposure_config": {
+ "kind": "ExperimentEventExposureConfig",
+ "event": "$feature_flag_called",
+ "properties": [{"key": "plan", "value": ["paid"], "operator": "exact", "type": "person"}],
+ }
+ },
+ )
+ self._experiment(
+ "Legacy criteria",
+ start_date=now - timedelta(days=62),
+ exposure_criteria={"multiple_variant_handling": "whatever_this_was"},
+ )
+ self._experiment("Draft")
+ self._experiment("Ended", start_date=now - timedelta(days=80), end_date=now - timedelta(days=1))
+ self._experiment(
+ "Not yours", start_date=now - timedelta(days=80), created_by=self._create_user("[email protected]")
+ )
+
+ found = api.list_running_experiments(
+ team_id=self.team.pk,
+ user_id=self.user.pk,
+ started_after=None if started_after_days is None else now - timedelta(days=started_after_days),
+ started_before=now - timedelta(days=60),
+ limit=20,
+ )
+
+ assert [(e.name, e.flag_key) for e in found] == [(name, listed[name]) for name in expected]
+
+
+class TestCountEnrolledPeople(ClickhouseTestMixin, BaseTest):
+ def _ago(self, days: int) -> datetime:
+ return timezone.now() - timedelta(days=days)
+
+ @parameterized.expand(
+ [("events", FlagEvaluationsMode.EVENTS), ("flag_evaluations", FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY)]
+ )
+ def test_counts_people_who_got_a_variant_in_the_last_14_days(self, _name: str, mode: FlagEvaluationsMode) -> None:
+ OrganizationFeatureFlagsConfig.objects.update_or_create(
+ organization=self.organization, defaults={"flag_evaluations_mode": mode}
+ )
+ calls = [
+ ("checkout", "test", 2),
+ ("checkout", "control", 3),
+ ("checkout", "false", 2),
+ ("checkout", "test", 20),
+ ("other", "test", 2),
+ ]
+ for flag_key, response, days_ago in calls:
+ distinct_id = str(uuid.uuid4())
+ if mode == FlagEvaluationsMode.EVENTS:
+ create_person(team=self.team, distinct_ids=[distinct_id])
+ _create_event(
+ team=self.team,
+ event="$feature_flag_called",
+ distinct_id=distinct_id,
+ properties={"$feature_flag": flag_key, "$feature_flag_response": response},
+ timestamp=self._ago(days_ago),
+ )
+ else:
+ _create_flag_evaluations(
+ self.team.pk, flag_key, timestamp=self._ago(days_ago), distinct_id=distinct_id, response=response
+ )
+ flush_persons_and_events()
+
+ assert api.count_enrolled_people(team=self.team, user=self.user, flag_keys=["checkout"]) == {"checkout": 2}
diff --git a/products/today/backend/logic/highlight_sources/experiments.py b/products/today/backend/logic/highlight_sources/experiments.py
new file mode 100644
index 000000000000..e343b01e6798
--- /dev/null
+++ b/products/today/backend/logic/highlight_sources/experiments.py
@@ -0,0 +1,98 @@
+import re
+from datetime import datetime, timedelta
+
+from django.utils import timezone
+
+from products.experiments.backend.facade.api import (
+ count_enrolled_people,
+ list_ended_experiments_without_conclusion,
+ list_running_experiments,
+)
+from products.experiments.backend.facade.contracts import EndedExperiment, RunningExperiment
+
+from ...facade.enums import HighlightLabel
+from ..highlight_rules import short, users
+from ..highlight_types import Candidate, Context, Found, Source
+
+ENDED = "Experiment ‘{experiment}’ ended {when}"
+ENDED_DETAIL = "Read the results and decide whether to release the change"
+FEW_USERS = "Experiment ‘{experiment}’ reached {enrolled} in the last 14 days"
+NO_USERS = "Experiment ‘{experiment}’ reached no users in the last 14 days"
+FEW_USERS_DETAIL = "Stop it or increase the rollout"
+ENDED_WINDOW_DAYS = 7
+LONG_RUNNING_DAYS = 60
+JUST_CROSSED_DAYS = 67
+MIN_PEOPLE = 100
+
+
+def find(context: Context) -> Found:
+ now = timezone.now()
+ ended = list_ended_experiments_without_conclusion(
+ team_id=context.team.id, user_id=context.user.id, since=now - timedelta(days=ENDED_WINDOW_DAYS), limit=20
+ )
+ running = list_running_experiments(
+ team_id=context.team.id,
+ user_id=context.user.id,
+ started_after=None if context.today.weekday() == 0 else now - timedelta(days=JUST_CROSSED_DAYS),
+ started_before=now - timedelta(days=LONG_RUNNING_DAYS),
+ limit=100,
+ )
+ enrolled = context.guard(
+ "experiments_enrolled",
+ lambda: count_enrolled_people(
+ team=context.team, user=context.user, flag_keys=[experiment.flag_key for experiment in running]
+ ),
+ None,
+ )
+ counted = [] if enrolled is None else [(experiment, enrolled.get(experiment.flag_key, 0)) for experiment in running]
+ return Found(
+ items=(
+ *(_ended(context, experiment) for experiment in ended),
+ *(_few_users(experiment, people) for experiment, people in counted if people < MIN_PEOPLE),
+ )
+ )
+
+
+def _days_since(context: Context, moment: datetime) -> int:
+ return (context.today - timezone.localtime(moment, context.team.timezone_info).date()).days
+
+
+def _when(days: int) -> str:
+ return "today" if days <= 0 else "yesterday" if days == 1 else f"{days} days ago"
+
+
+def _name(name: str, limit: int) -> str:
+ return short(re.sub(r"(?i)\s+experiment$", "", name) or name, limit)
+
+
+def _ended(context: Context, experiment: EndedExperiment) -> Candidate:
+ return Candidate(
+ key=f"experiment_ended:{experiment.id}",
+ label=HighlightLabel.OPPORTUNITY,
+ family="experiments",
+ importance=0.6,
+ relevance=1.0,
+ title=ENDED.format(
+ experiment=_name(experiment.name, 60), when=_when(_days_since(context, experiment.ended_at))
+ ),
+ detail=ENDED_DETAIL,
+ url=f"/experiments/{experiment.id}",
+ )
+
+
+def _few_users(experiment: RunningExperiment, enrolled: int) -> Candidate:
+ return Candidate(
+ key=f"experiment_stale:{experiment.id}",
+ label=HighlightLabel.RISK,
+ family="experiments",
+ importance=0.3,
+ relevance=1.0,
+ title=(FEW_USERS if enrolled else NO_USERS).format(
+ experiment=_name(experiment.name, 50), enrolled=users(enrolled)
+ ),
+ detail=FEW_USERS_DETAIL,
+ url=f"/experiments/{experiment.id}",
+ )
+
+
+SOURCE = Source(name="experiments", find=find, scopes=("experiment:read",))
diff --git a/products/today/backend/tests/test_highlight_experiments.py b/products/today/backend/tests/test_highlight_experiments.py
new file mode 100644
index 000000000000..6cdb615791e1
--- /dev/null
+++ b/products/today/backend/tests/test_highlight_experiments.py
@@ -0,0 +1,86 @@
+from datetime import UTC, date, datetime, time, timedelta
+
+import time_machine
+from posthog.test.base import BaseTest
+from unittest.mock import MagicMock, patch
+
+from parameterized import parameterized
+
+from products.experiments.backend.facade.contracts import EndedExperiment, RunningExperiment
+from products.today.backend.logic.highlight_registry import load_sources
+from products.today.backend.logic.highlight_sources import experiments
+from products.today.backend.logic.highlight_types import Context, Found
+
+MONDAY = date(2026, 10, 12)
+TUESDAY = date(2026, 10, 13)
+NOW = datetime.combine(TUESDAY, time(12), tzinfo=UTC)
+ENDED = EndedExperiment(id=7, name="Pricing page layout experiment", ended_at=NOW - timedelta(days=1))
+RUNNING = RunningExperiment(
+ id=9, name="Onboarding tips", started_at=NOW - timedelta(days=62), flag_key="onboarding-tips"
+)
+
+
+@time_machine.travel(NOW, tick=False)
+class TestExperiments(BaseTest):
+ def _find(self, today: date, running: MagicMock, enrolled: MagicMock) -> tuple[Found, Context]:
+ registered = next(s for s in load_sources() if s.name == "experiments")
+ context = Context(self.team, self.user, today)
+ with (
+ patch.object(experiments, "list_ended_experiments_without_conclusion", return_value=[ENDED]),
+ patch.object(experiments, "list_running_experiments", running),
+ patch.object(experiments, "count_enrolled_people", enrolled),
+ ):
+ return registered.find(context), context
+
+ def test_an_ended_experiment_without_a_decision_is_an_opportunity(self) -> None:
+ found, _ = self._find(TUESDAY, MagicMock(return_value=[]), MagicMock(return_value={}))
+
+ self.assertEqual(
+ [(c.title, c.detail, c.url) for c in found.items],
+ [
+ (
+ "Experiment ‘Pricing page layout’ ended yesterday",
+ "Read the results and decide whether to release the change",
+ "/experiments/7",
+ )
+ ],
+ )
+
+ @parameterized.expand([("Monday", MONDAY, None), ("Tuesday", TUESDAY, NOW - timedelta(days=67))])
+ def test_only_mondays_look_past_the_week_an_experiment_crosses_60_days(
+ self, _name: str, today: date, started_after: datetime | None
+ ) -> None:
+ running = MagicMock(return_value=[])
+
+ self._find(today, running, MagicMock(return_value={}))
+
+ self.assertEqual(
+ (running.call_args.kwargs["started_after"], running.call_args.kwargs["started_before"]),
+ (started_after, NOW - timedelta(days=60)),
+ )
+
+ @parameterized.expand(
+ [
+ ("few users", 42, "Experiment ‘Onboarding tips’ reached 42 users in the last 14 days"),
+ ("one user", 1, "Experiment ‘Onboarding tips’ reached 1 user in the last 14 days"),
+ ("no users", 0, "Experiment ‘Onboarding tips’ reached no users in the last 14 days"),
+ ("enough users", 150, None),
+ ]
+ )
+ def test_a_long_running_experiment_with_few_users_is_a_risk(
+ self, _name: str, enrolled: int, title: str | None
+ ) -> None:
+ found, _ = self._find(
+ TUESDAY, MagicMock(return_value=[RUNNING]), MagicMock(return_value={"onboarding-tips": enrolled})
+ )
+
+ self.assertEqual([c.title for c in found.items][1:], [title] if title else [])
+
+ def test_a_failed_count_keeps_the_ended_reminders(self) -> None:
+ found, context = self._find(
+ TUESDAY, MagicMock(return_value=[RUNNING]), MagicMock(side_effect=TimeoutError("slow"))
+ )
+
+ self.assertEqual(
+ ([c.key for c in found.items], context.failures), (["experiment_ended:7"], ["experiments_enrolled"])
+ )
diff --git a/tach.toml b/tach.toml
index 7d2fb0a59792..a292cc3f57e7 100644
--- a/tach.toml
+++ b/tach.toml
@@ -1681,6 +1681,7 @@ depends_on = [
"products.dashboards",
"products.data_catalog",
"products.error_tracking",
+ "products.experiments",
"products.feature_flags",
"products.product_analytics",
"products.signals",