:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): add usage-weighted insight picks for anomaly scoring (#122213)
posthog[bot]d5b914436e2a
diff --git a/products/alerts/backend/facade/api.py b/products/alerts/backend/facade/api.pyindex 1a6cdf7fa595..b02b7b166eca 100644--- a/products/alerts/backend/facade/api.py+++ b/products/alerts/backend/facade/api.py@@ -84,6 +84,15 @@ def insight_ids_with_alerts(insight_ids: Collection[int]) -> set[int]:return set(AlertConfiguration.objects.filter(insight_id__in=insight_ids).values_list("insight_id", flat=True))+def insight_ids_with_enabled_alerts(*, team_id: int, insight_ids: Collection[int]) -> set[int]:+ """Which of the team's given insights have at least one enabled alert, in one query."""+ return set(+ AlertConfiguration.objects.filter(team_id=team_id, insight_id__in=insight_ids, enabled=True).values_list(+ "insight_id", flat=True+ )+ )++def delete_insight_alerts(insight_ids: Collection[int]) -> None:"""Delete the alerts pointing at the given insights.diff --git a/products/alerts/backend/test/test_facade_api.py b/products/alerts/backend/test/test_facade_api.pynew file mode 100644index 000000000000..9992ec56e6c5--- /dev/null+++ b/products/alerts/backend/test/test_facade_api.py@@ -0,0 +1,20 @@+from posthog.test.base import BaseTest++from posthog.models.team import Team++from products.alerts.backend.facade.api import insight_ids_with_enabled_alerts+from products.alerts.backend.test.insight_alerts import create_insight_alert+++class TestInsightIdsWithEnabledAlerts(BaseTest):+ def test_returns_only_the_teams_insights_with_an_enabled_alert(self) -> None:+ enabled = create_insight_alert(self.team)+ disabled = create_insight_alert(self.team, enabled=False)+ other_team_alert = create_insight_alert(Team.objects.create(organization=self.organization))++ result = insight_ids_with_enabled_alerts(+ team_id=self.team.pk,+ insight_ids={enabled.insight_id, disabled.insight_id, other_team_alert.insight_id},+ )++ assert result == {enabled.insight_id}diff --git a/products/product_analytics/backend/facade/api.py b/products/product_analytics/backend/facade/api.pyindex 9b234cddf0f6..7642847d547e 100644--- a/products/product_analytics/backend/facade/api.py+++ b/products/product_analytics/backend/facade/api.py@@ -143,6 +143,14 @@ def recent_viewers_by_insight()+def viewer_counts_by_insight(*, team_id: int, insight_ids: Collection[int], since: datetime) -> dict[int, int]:+ """How many people looked at each of these insights since the given time, in one query.++ Shared and embedded views have no viewer, so they are not counted. Insights nobody viewed are left out.+ """+ return logic.viewer_counts_by_insight(team_id=team_id, insight_ids=insight_ids, since=since)++def map_stale_to_latest(stale_variables: dict, latest_variables: list[InsightVariableDefinition]) -> dict:"""Refresh an insight's stored variables against the team's latest variable definitions."""return logic.map_stale_to_latest(stale_variables, latest_variables)diff --git a/products/product_analytics/backend/logic.py b/products/product_analytics/backend/logic.pyindex b80f5e6113fd..75734efb65a8 100644--- a/products/product_analytics/backend/logic.py+++ b/products/product_analytics/backend/logic.py@@ -3,7 +3,7 @@ from datetime import datetimefrom typing import TYPE_CHECKING, Anyfrom uuid import UUID-from django.db.models import OuterRef, QuerySet, Subquery+from django.db.models import Count, OuterRef, QuerySet, Subqueryfrom django.utils.timezone import nowfrom products.product_analytics.backend.facade.contracts import InsightVariableDefinition@@ -117,6 +117,22 @@ def recent_viewers_by_insight(return viewers_by_insight+def viewer_counts_by_insight(*, team_id: int, insight_ids: Collection[int], since: datetime) -> dict[int, int]:+ # InsightViewed holds one row per team, user and insight, so counting rows counts viewers.+ rows = (+ InsightViewed.objects.filter(+ team_id=team_id,+ insight_id__in=insight_ids,+ last_viewed_at__gte=since,+ user__isnull=False,+ )+ .values("insight_id")+ .annotate(viewers=Count("id"))+ .values_list("insight_id", "viewers")+ )+ return dict(rows)++def map_stale_to_latest(stale_variables: dict, latest_variables: list[InsightVariableDefinition]) -> dict:# Keep the variables in an insight up to date based on variable code names that existcurrent_variables = stale_variablesdiff --git a/products/product_analytics/backend/tests/test_facade.py b/products/product_analytics/backend/tests/test_facade.pyindex 2419aab75f0a..a2d4b460c5bd 100644--- a/products/product_analytics/backend/tests/test_facade.py+++ b/products/product_analytics/backend/tests/test_facade.py@@ -19,6 +19,7 @@ from products.product_analytics.backend.facade.api import (insight_variables_for_team,insights_including_soft_deleted_for_team,record_insight_view,+ viewer_counts_by_insight,)from products.product_analytics.backend.facade.queries import run_cached_trends_queryfrom products.product_analytics.backend.models.insight import Insight, InsightViewed@@ -84,6 +85,31 @@ class TestInsightReads(BaseTest):assert {insight.pk for insight in insights} == {deleted_insight.pk, live_insight.pk}+ def test_viewer_counts_count_recent_signed_in_viewers_of_the_team(self) -> None:+ viewed = Insight.objects.create(team=self.team, name="Viewed")+ unviewed = Insight.objects.create(team=self.team, name="Unviewed")+ other_team = Team.objects.create(organization=self.organization)+ other_team_insight = Insight.objects.create(team=other_team, name="Other team")+ recent = now() - timedelta(days=1)+ for viewer in (self.user, self._create_user("[email protected]")):+ InsightViewed.objects.create(team=self.team, user=viewer, insight=viewed, last_viewed_at=recent)+ InsightViewed.objects.create(+ team=self.team,+ user=self._create_user("[email protected]"),+ insight=viewed,+ last_viewed_at=now() - timedelta(days=40),+ )+ InsightViewed.objects.create(team=None, user=None, insight=viewed, last_viewed_at=recent)+ InsightViewed.objects.create(team=other_team, user=self.user, insight=other_team_insight, last_viewed_at=recent)++ counts = viewer_counts_by_insight(+ team_id=self.team.pk,+ insight_ids={viewed.pk, unviewed.pk, other_team_insight.pk},+ since=now() - timedelta(days=30),+ )++ assert counts == {viewed.pk: 2}+class TestRunCachedTrendsQuery(BaseTest):def tearDown(self) -> None:diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.pyindex a6a1b667aef3..2c6b65f5bfe7 100644--- a/products/signals/backend/anomaly_scores/scoring.py+++ b/products/signals/backend/anomaly_scores/scoring.py@@ -147,7 +147,7 @@ def score_series("""if len(values) != len(buckets):raise ValueError("Each value needs a bucket")- completed = [i for i, start in enumerate(buckets) if _bucket_end(start, interval) <= now]+ completed = [i for i, start in enumerate(buckets) if bucket_end(start, interval) <= now]returned = [i for i in completed if after is None or buckets[i] > after][-limits.max_backfill_buckets :]detector = get_detector(scoring.detector)data = np.array(values, dtype=float)@@ -262,7 +262,7 @@ def _interval(query: TrendsQuery) -> IntervalType:return query.interval or IntervalType.DAY-def _bucket_end(start: datetime, interval: IntervalType) -> datetime:+def bucket_end(start: datetime, interval: IntervalType) -> datetime:if interval == IntervalType.HOUR:# An hour is elapsed time, so step in UTC to stay one hour long across DST changes.return (start.astimezone(UTC) + timedelta(hours=1)).astimezone(start.tzinfo)diff --git a/products/signals/backend/anomaly_scores/selection.py b/products/signals/backend/anomaly_scores/selection.pynew file mode 100644index 000000000000..4f87dce53963--- /dev/null+++ b/products/signals/backend/anomaly_scores/selection.py@@ -0,0 +1,155 @@+"""The order in which a scoring tick picks insights, set by the `anomaly-scoring` flag payload.++`weighted` samples insights in proportion to usage × staleness, so most of the per-tick query budget goes to+insights people look at. Weight grows while an insight waits, so an insight nobody views is picked rarely but+never starves. `uniform` picks the stalest insight first, so every insight gets the same share.+"""++import math+import hashlib+from collections.abc import Collection, Sequence+from datetime import datetime, timedelta+from enum import StrEnum+from typing import Any++from posthog.schema import IntervalType++from posthog.dataclasses import frozen++from products.alerts.backend.facade.api import insight_ids_with_enabled_alerts+from products.product_analytics.backend.facade.api import viewer_counts_by_insight+from products.signals.backend.anomaly_scores.scoring import bucket_end++USAGE_WINDOW = timedelta(days=30)+++class Weighting(StrEnum):+ UNIFORM = "uniform"+ WEIGHTED = "weighted"+++@frozen+class SelectionConfig:+ weighting: Weighting = Weighting.WEIGHTED+ # Gives insights nobody views a small chance, so we can see how the tail scores. 0 leaves them out.+ usage_floor: float = 1.0+ alert_bonus: float = 5.0+++@frozen+class InsightUsage:+ viewers: int+ has_enabled_alert: bool+++@frozen+class Candidate:+ insight_id: int+ interval: IntervalType+ # The newest bucket scored for any series of the insight. None when the insight was never scored.+ newest_scored_bucket: datetime | None+ usage: InsightUsage+++def _non_negative(value: Any, default: float) -> float:+ # `bool` is an `int` subclass, and `true` is not a number.+ if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value) or value < 0:+ return default+ return float(value)+++def parse_selection_config(payload: Any) -> SelectionConfig:+ """The selection fields of the `anomaly-scoring` payload. An invalid field gets its default."""+ defaults = SelectionConfig()+ if not isinstance(payload, dict):+ return defaults+ weighting = payload.get("weighting")+ return SelectionConfig(+ weighting=Weighting(weighting) if weighting in tuple(Weighting) else defaults.weighting,+ usage_floor=_non_negative(payload.get("usage_floor"), defaults.usage_floor),+ alert_bonus=_non_negative(payload.get("alert_bonus"), defaults.alert_bonus),+ )+++def insight_usage(*, team_id: int, insight_ids: Collection[int], now: datetime) -> dict[int, InsightUsage]:+ viewers = viewer_counts_by_insight(team_id=team_id, insight_ids=insight_ids, since=now - USAGE_WINDOW)+ alerted = insight_ids_with_enabled_alerts(team_id=team_id, insight_ids=insight_ids)+ return {+ insight_id: InsightUsage(viewers=viewers.get(insight_id, 0), has_enabled_alert=insight_id in alerted)+ for insight_id in insight_ids+ }+++def staleness(candidate: Candidate, *, now: datetime, max_staleness: int) -> int:+ """Completed buckets after the newest scored one, up to ``max_staleness``.++ A never-scored insight counts as ``max_staleness``. Pass the backfill cap: one pick scores at most that+ many buckets, so a longer wait adds nothing to score.+ """+ if candidate.newest_scored_bucket is None:+ return max_staleness+ count = 0+ start = bucket_end(candidate.newest_scored_bucket, candidate.interval)+ while count < max_staleness and bucket_end(start, candidate.interval) <= now:+ count += 1+ start = bucket_end(start, candidate.interval)+ return count+++def weight(candidate: Candidate, *, config: SelectionConfig, now: datetime, max_staleness: int) -> float:+ usage = config.usage_floor + candidate.usage.viewers+ if candidate.usage.has_enabled_alert:+ usage += config.alert_bonus+ return usage * staleness(candidate, now=now, max_staleness=max_staleness)+++def pick_order(+ candidates: Sequence[Candidate],+ *,+ config: SelectionConfig,+ tick_start: datetime,+ max_staleness: int,+) -> list[int]:+ """Insight ids in the order the tick scores them. Any prefix is a valid pick for a smaller budget.++ Candidates with no completed bucket to score are left out. In weighted mode, so are zero-weight candidates.+ """+ if config.weighting == Weighting.UNIFORM:+ return _stalest_first(candidates, now=tick_start, max_staleness=max_staleness)+ return _weighted_sample(candidates, config=config, tick_start=tick_start, max_staleness=max_staleness)+++def _unit_interval(*parts: object) -> float:+ """A uniform number strictly between 0 and 1, fixed by ``parts``."""+ digest = hashlib.sha256(":".join(str(part) for part in parts).encode()).digest()+ # 53 bits fit a float exactly, so the result never rounds to 0 or 1.+ return ((int.from_bytes(digest[:8], "big") >> 11) + 0.5) / 2**53+++def _weighted_sample(+ candidates: Sequence[Candidate], *, config: SelectionConfig, tick_start: datetime, max_staleness: int+) -> list[int]:+ # Efraimidis–Spirakis: sorting by u ** (1 / weight) gives a weighted sample without replacement.+ # log(u) / weight sorts the same way and keeps precision when the weight is large.+ keyed: list[tuple[float, int]] = []+ for candidate in candidates:+ candidate_weight = weight(candidate, config=config, now=tick_start, max_staleness=max_staleness)+ if candidate_weight > 0:+ u = _unit_interval(candidate.insight_id, tick_start.isoformat())+ keyed.append((math.log(u) / candidate_weight, candidate.insight_id))+ keyed.sort(key=lambda item: (-item[0], item[1]))+ return [insight_id for _, insight_id in keyed]+++def _stalest_first(candidates: Sequence[Candidate], *, now: datetime, max_staleness: int) -> list[int]:+ due = [candidate for candidate in candidates if staleness(candidate, now=now, max_staleness=max_staleness) > 0]+ # A hashed order for never-scored insights, so first coverage does not favor any group of insights.+ never_scored = sorted(+ (candidate.insight_id for candidate in due if candidate.newest_scored_bucket is None),+ key=_unit_interval,+ )+ scored = sorted(+ (candidate for candidate in due if candidate.newest_scored_bucket is not None),+ key=lambda candidate: (candidate.newest_scored_bucket, candidate.insight_id),+ )+ return never_scored + [candidate.insight_id for candidate in scored]diff --git a/products/signals/backend/test/test_anomaly_selection.py b/products/signals/backend/test/test_anomaly_selection.pynew file mode 100644index 000000000000..15cdd89711cf--- /dev/null+++ b/products/signals/backend/test/test_anomaly_selection.py@@ -0,0 +1,137 @@+from collections import Counter+from datetime import UTC, datetime, timedelta+from typing import Any++from django.test import SimpleTestCase++from parameterized import parameterized++from posthog.schema import IntervalType++from products.signals.backend.anomaly_scores.selection import (+ Candidate,+ InsightUsage,+ SelectionConfig,+ Weighting,+ parse_selection_config,+ pick_order,+ staleness,+)++TICK = datetime(2026, 10, 15, 12, 0, tzinfo=UTC)+MAX_STALENESS = 30+WEIGHTED = SelectionConfig()+UNIFORM = SelectionConfig(weighting=Weighting.UNIFORM)+++def _candidate(+ insight_id: int,+ *,+ days_since_scored: int | None,+ viewers: int = 0,+ alert: bool = False,+ interval: IntervalType = IntervalType.DAY,+) -> Candidate:+ newest = (+ None if days_since_scored is None else datetime(2026, 10, 15, tzinfo=UTC) - timedelta(days=days_since_scored)+ )+ return Candidate(+ insight_id=insight_id,+ interval=interval,+ newest_scored_bucket=newest,+ usage=InsightUsage(viewers=viewers, has_enabled_alert=alert),+ )+++class TestPickOrder(SimpleTestCase):+ @parameterized.expand(+ [+ # (usage 1 × staleness 1) vs (usage 4 × staleness 1)+ ("viewers", {"days_since_scored": 2}, {"days_since_scored": 2, "viewers": 3}, 0.8),+ # (1 × 1) vs ((1 + 5) × 1)+ ("enabled_alert", {"days_since_scored": 2}, {"days_since_scored": 2, "alert": True}, 6 / 7),+ # A popular insight scored recently, (10 × 1), against a long-waiting unviewed one, (1 × 29).+ ("waiting_overtakes_usage", {"days_since_scored": 2, "viewers": 9}, {"days_since_scored": 30}, 29 / 39),+ # Staleness is capped, and a never-scored insight counts as the cap.+ ("capped_staleness", {"days_since_scored": None}, {"days_since_scored": 90}, 0.5),+ ]+ )+ def test_first_pick_rate_follows_weight(+ self, _name: str, low: dict[str, Any], high: dict[str, Any], expected_high_rate: float+ ) -> None:+ candidates = [_candidate(1, **low), _candidate(2, **high)]+ first_picks = Counter(+ pick_order(+ candidates, config=WEIGHTED, tick_start=TICK + timedelta(seconds=i), max_staleness=MAX_STALENESS+ )[0]+ for i in range(4000)+ )++ assert abs(first_picks[2] / 4000 - expected_high_rate) < 0.03++ def test_uniform_picks_never_scored_then_stalest(self) -> None:+ candidates = [+ _candidate(1, days_since_scored=3, viewers=50),+ _candidate(2, days_since_scored=None),+ _candidate(3, days_since_scored=10),+ _candidate(4, days_since_scored=None, viewers=50),+ ]++ order = pick_order(candidates, config=UNIFORM, tick_start=TICK, max_staleness=MAX_STALENESS)++ assert set(order[:2]) == {2, 4}+ assert order[2:] == [3, 1]++ @parameterized.expand(+ [+ ("nothing_completed_weighted", WEIGHTED, {"days_since_scored": 1, "viewers": 50}),+ ("nothing_completed_uniform", UNIFORM, {"days_since_scored": 1, "viewers": 50}),+ ("unviewed_without_floor", SelectionConfig(usage_floor=0), {"days_since_scored": None}),+ ]+ )+ def test_leaves_out_candidates_with_nothing_to_gain(+ self, _name: str, config: SelectionConfig, excluded: dict[str, Any]+ ) -> None:+ candidates = [_candidate(1, **excluded), _candidate(2, days_since_scored=5, viewers=1)]++ assert pick_order(candidates, config=config, tick_start=TICK, max_staleness=MAX_STALENESS) == [2]++ @parameterized.expand(+ [+ # Oct 14 is ongoing at noon, so only Oct 12 and Oct 13 are completed.+ ("day", IntervalType.DAY, datetime(2026, 10, 11, tzinfo=UTC), 2),+ ("month", IntervalType.MONTH, datetime(2026, 7, 1, tzinfo=UTC), 2),+ ("capped", IntervalType.HOUR, datetime(2026, 10, 1, tzinfo=UTC), MAX_STALENESS),+ ]+ )+ def test_staleness_counts_completed_buckets_after_the_newest_scored_one(+ self, _name: str, interval: IntervalType, newest_scored_bucket: datetime, expected: int+ ) -> None:+ candidate = Candidate(+ insight_id=1,+ interval=interval,+ newest_scored_bucket=newest_scored_bucket,+ usage=InsightUsage(viewers=0, has_enabled_alert=False),+ )++ assert staleness(candidate, now=datetime(2026, 10, 14, 12, tzinfo=UTC), max_staleness=MAX_STALENESS) == expected+++class TestParseSelectionConfig(SimpleTestCase):+ @parameterized.expand(+ [+ ("missing", None, SelectionConfig()),+ (+ "valid",+ {"weighting": "uniform", "usage_floor": 0, "alert_bonus": 10},+ SelectionConfig(weighting=Weighting.UNIFORM, usage_floor=0, alert_bonus=10),+ ),+ (+ "invalid_fields_fall_back",+ {"weighting": "random", "usage_floor": -1, "alert_bonus": True},+ SelectionConfig(),+ ),+ ]+ )+ def test_parse(self, _name: str, payload: Any, expected: SelectionConfig) -> None:+ assert parse_selection_config(payload) == expected