Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): add usage-weighted insight picks for anomaly scoring (#122213)

posthog[bot]d5b914436e2a
diff --git a/products/alerts/backend/facade/api.py b/products/alerts/backend/facade/api.py
index 1a6cdf7fa595..b02b7b166eca 100644
--- a/products/alerts/backend/facade/api.py
+++ b/products/alerts/backend/facade/api.py
@@ -84,6 +84,15 @@ def insight_ids_with_alerts(insight_ids: Collection[int]) -> set[int]:
return set(AlertConfiguration.objects.filter(insight_id__in=insight_ids).values_list("insight_id", flat=True))
+def insight_ids_with_enabled_alerts(*, team_id: int, insight_ids: Collection[int]) -> set[int]:
+ """Which of the team's given insights have at least one enabled alert, in one query."""
+ return set(
+ AlertConfiguration.objects.filter(team_id=team_id, insight_id__in=insight_ids, enabled=True).values_list(
+ "insight_id", flat=True
+ )
+ )
+
+
def delete_insight_alerts(insight_ids: Collection[int]) -> None:
"""Delete the alerts pointing at the given insights.
diff --git a/products/alerts/backend/test/test_facade_api.py b/products/alerts/backend/test/test_facade_api.py
new file mode 100644
index 000000000000..9992ec56e6c5
--- /dev/null
+++ b/products/alerts/backend/test/test_facade_api.py
@@ -0,0 +1,20 @@
+from posthog.test.base import BaseTest
+
+from posthog.models.team import Team
+
+from products.alerts.backend.facade.api import insight_ids_with_enabled_alerts
+from products.alerts.backend.test.insight_alerts import create_insight_alert
+
+
+class TestInsightIdsWithEnabledAlerts(BaseTest):
+ def test_returns_only_the_teams_insights_with_an_enabled_alert(self) -> None:
+ enabled = create_insight_alert(self.team)
+ disabled = create_insight_alert(self.team, enabled=False)
+ other_team_alert = create_insight_alert(Team.objects.create(organization=self.organization))
+
+ result = insight_ids_with_enabled_alerts(
+ team_id=self.team.pk,
+ insight_ids={enabled.insight_id, disabled.insight_id, other_team_alert.insight_id},
+ )
+
+ assert result == {enabled.insight_id}
diff --git a/products/product_analytics/backend/facade/api.py b/products/product_analytics/backend/facade/api.py
index 9b234cddf0f6..7642847d547e 100644
--- a/products/product_analytics/backend/facade/api.py
+++ b/products/product_analytics/backend/facade/api.py
@@ -143,6 +143,14 @@ def recent_viewers_by_insight(
)
+def viewer_counts_by_insight(*, team_id: int, insight_ids: Collection[int], since: datetime) -> dict[int, int]:
+ """How many people looked at each of these insights since the given time, in one query.
+
+ Shared and embedded views have no viewer, so they are not counted. Insights nobody viewed are left out.
+ """
+ return logic.viewer_counts_by_insight(team_id=team_id, insight_ids=insight_ids, since=since)
+
+
def map_stale_to_latest(stale_variables: dict, latest_variables: list[InsightVariableDefinition]) -> dict:
"""Refresh an insight's stored variables against the team's latest variable definitions."""
return logic.map_stale_to_latest(stale_variables, latest_variables)
diff --git a/products/product_analytics/backend/logic.py b/products/product_analytics/backend/logic.py
index b80f5e6113fd..75734efb65a8 100644
--- a/products/product_analytics/backend/logic.py
+++ b/products/product_analytics/backend/logic.py
@@ -3,7 +3,7 @@ from datetime import datetime
from typing import TYPE_CHECKING, Any
from uuid import UUID
-from django.db.models import OuterRef, QuerySet, Subquery
+from django.db.models import Count, OuterRef, QuerySet, Subquery
from django.utils.timezone import now
from products.product_analytics.backend.facade.contracts import InsightVariableDefinition
@@ -117,6 +117,22 @@ def recent_viewers_by_insight(
return viewers_by_insight
+def viewer_counts_by_insight(*, team_id: int, insight_ids: Collection[int], since: datetime) -> dict[int, int]:
+ # InsightViewed holds one row per team, user and insight, so counting rows counts viewers.
+ rows = (
+ InsightViewed.objects.filter(
+ team_id=team_id,
+ insight_id__in=insight_ids,
+ last_viewed_at__gte=since,
+ user__isnull=False,
+ )
+ .values("insight_id")
+ .annotate(viewers=Count("id"))
+ .values_list("insight_id", "viewers")
+ )
+ return dict(rows)
+
+
def map_stale_to_latest(stale_variables: dict, latest_variables: list[InsightVariableDefinition]) -> dict:
# Keep the variables in an insight up to date based on variable code names that exist
current_variables = stale_variables
diff --git a/products/product_analytics/backend/tests/test_facade.py b/products/product_analytics/backend/tests/test_facade.py
index 2419aab75f0a..a2d4b460c5bd 100644
--- a/products/product_analytics/backend/tests/test_facade.py
+++ b/products/product_analytics/backend/tests/test_facade.py
@@ -19,6 +19,7 @@ from products.product_analytics.backend.facade.api import (
insight_variables_for_team,
insights_including_soft_deleted_for_team,
record_insight_view,
+ viewer_counts_by_insight,
)
from products.product_analytics.backend.facade.queries import run_cached_trends_query
from products.product_analytics.backend.models.insight import Insight, InsightViewed
@@ -84,6 +85,31 @@ class TestInsightReads(BaseTest):
assert {insight.pk for insight in insights} == {deleted_insight.pk, live_insight.pk}
+ def test_viewer_counts_count_recent_signed_in_viewers_of_the_team(self) -> None:
+ viewed = Insight.objects.create(team=self.team, name="Viewed")
+ unviewed = Insight.objects.create(team=self.team, name="Unviewed")
+ other_team = Team.objects.create(organization=self.organization)
+ other_team_insight = Insight.objects.create(team=other_team, name="Other team")
+ recent = now() - timedelta(days=1)
+ for viewer in (self.user, self._create_user("[email protected]")):
+ InsightViewed.objects.create(team=self.team, user=viewer, insight=viewed, last_viewed_at=recent)
+ InsightViewed.objects.create(
+ team=self.team,
+ user=self._create_user("[email protected]"),
+ insight=viewed,
+ last_viewed_at=now() - timedelta(days=40),
+ )
+ InsightViewed.objects.create(team=None, user=None, insight=viewed, last_viewed_at=recent)
+ InsightViewed.objects.create(team=other_team, user=self.user, insight=other_team_insight, last_viewed_at=recent)
+
+ counts = viewer_counts_by_insight(
+ team_id=self.team.pk,
+ insight_ids={viewed.pk, unviewed.pk, other_team_insight.pk},
+ since=now() - timedelta(days=30),
+ )
+
+ assert counts == {viewed.pk: 2}
+
class TestRunCachedTrendsQuery(BaseTest):
def tearDown(self) -> None:
diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.py
index a6a1b667aef3..2c6b65f5bfe7 100644
--- a/products/signals/backend/anomaly_scores/scoring.py
+++ b/products/signals/backend/anomaly_scores/scoring.py
@@ -147,7 +147,7 @@ def score_series(
"""
if len(values) != len(buckets):
raise ValueError("Each value needs a bucket")
- completed = [i for i, start in enumerate(buckets) if _bucket_end(start, interval) <= now]
+ completed = [i for i, start in enumerate(buckets) if bucket_end(start, interval) <= now]
returned = [i for i in completed if after is None or buckets[i] > after][-limits.max_backfill_buckets :]
detector = get_detector(scoring.detector)
data = np.array(values, dtype=float)
@@ -262,7 +262,7 @@ def _interval(query: TrendsQuery) -> IntervalType:
return query.interval or IntervalType.DAY
-def _bucket_end(start: datetime, interval: IntervalType) -> datetime:
+def bucket_end(start: datetime, interval: IntervalType) -> datetime:
if interval == IntervalType.HOUR:
# An hour is elapsed time, so step in UTC to stay one hour long across DST changes.
return (start.astimezone(UTC) + timedelta(hours=1)).astimezone(start.tzinfo)
diff --git a/products/signals/backend/anomaly_scores/selection.py b/products/signals/backend/anomaly_scores/selection.py
new file mode 100644
index 000000000000..4f87dce53963
--- /dev/null
+++ b/products/signals/backend/anomaly_scores/selection.py
@@ -0,0 +1,155 @@
+"""The order in which a scoring tick picks insights, set by the `anomaly-scoring` flag payload.
+
+`weighted` samples insights in proportion to usage × staleness, so most of the per-tick query budget goes to
+insights people look at. Weight grows while an insight waits, so an insight nobody views is picked rarely but
+never starves. `uniform` picks the stalest insight first, so every insight gets the same share.
+"""
+
+import math
+import hashlib
+from collections.abc import Collection, Sequence
+from datetime import datetime, timedelta
+from enum import StrEnum
+from typing import Any
+
+from posthog.schema import IntervalType
+
+from posthog.dataclasses import frozen
+
+from products.alerts.backend.facade.api import insight_ids_with_enabled_alerts
+from products.product_analytics.backend.facade.api import viewer_counts_by_insight
+from products.signals.backend.anomaly_scores.scoring import bucket_end
+
+USAGE_WINDOW = timedelta(days=30)
+
+
+class Weighting(StrEnum):
+ UNIFORM = "uniform"
+ WEIGHTED = "weighted"
+
+
+@frozen
+class SelectionConfig:
+ weighting: Weighting = Weighting.WEIGHTED
+ # Gives insights nobody views a small chance, so we can see how the tail scores. 0 leaves them out.
+ usage_floor: float = 1.0
+ alert_bonus: float = 5.0
+
+
+@frozen
+class InsightUsage:
+ viewers: int
+ has_enabled_alert: bool
+
+
+@frozen
+class Candidate:
+ insight_id: int
+ interval: IntervalType
+ # The newest bucket scored for any series of the insight. None when the insight was never scored.
+ newest_scored_bucket: datetime | None
+ usage: InsightUsage
+
+
+def _non_negative(value: Any, default: float) -> float:
+ # `bool` is an `int` subclass, and `true` is not a number.
+ if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value) or value < 0:
+ return default
+ return float(value)
+
+
+def parse_selection_config(payload: Any) -> SelectionConfig:
+ """The selection fields of the `anomaly-scoring` payload. An invalid field gets its default."""
+ defaults = SelectionConfig()
+ if not isinstance(payload, dict):
+ return defaults
+ weighting = payload.get("weighting")
+ return SelectionConfig(
+ weighting=Weighting(weighting) if weighting in tuple(Weighting) else defaults.weighting,
+ usage_floor=_non_negative(payload.get("usage_floor"), defaults.usage_floor),
+ alert_bonus=_non_negative(payload.get("alert_bonus"), defaults.alert_bonus),
+ )
+
+
+def insight_usage(*, team_id: int, insight_ids: Collection[int], now: datetime) -> dict[int, InsightUsage]:
+ viewers = viewer_counts_by_insight(team_id=team_id, insight_ids=insight_ids, since=now - USAGE_WINDOW)
+ alerted = insight_ids_with_enabled_alerts(team_id=team_id, insight_ids=insight_ids)
+ return {
+ insight_id: InsightUsage(viewers=viewers.get(insight_id, 0), has_enabled_alert=insight_id in alerted)
+ for insight_id in insight_ids
+ }
+
+
+def staleness(candidate: Candidate, *, now: datetime, max_staleness: int) -> int:
+ """Completed buckets after the newest scored one, up to ``max_staleness``.
+
+ A never-scored insight counts as ``max_staleness``. Pass the backfill cap: one pick scores at most that
+ many buckets, so a longer wait adds nothing to score.
+ """
+ if candidate.newest_scored_bucket is None:
+ return max_staleness
+ count = 0
+ start = bucket_end(candidate.newest_scored_bucket, candidate.interval)
+ while count < max_staleness and bucket_end(start, candidate.interval) <= now:
+ count += 1
+ start = bucket_end(start, candidate.interval)
+ return count
+
+
+def weight(candidate: Candidate, *, config: SelectionConfig, now: datetime, max_staleness: int) -> float:
+ usage = config.usage_floor + candidate.usage.viewers
+ if candidate.usage.has_enabled_alert:
+ usage += config.alert_bonus
+ return usage * staleness(candidate, now=now, max_staleness=max_staleness)
+
+
+def pick_order(
+ candidates: Sequence[Candidate],
+ *,
+ config: SelectionConfig,
+ tick_start: datetime,
+ max_staleness: int,
+) -> list[int]:
+ """Insight ids in the order the tick scores them. Any prefix is a valid pick for a smaller budget.
+
+ Candidates with no completed bucket to score are left out. In weighted mode, so are zero-weight candidates.
+ """
+ if config.weighting == Weighting.UNIFORM:
+ return _stalest_first(candidates, now=tick_start, max_staleness=max_staleness)
+ return _weighted_sample(candidates, config=config, tick_start=tick_start, max_staleness=max_staleness)
+
+
+def _unit_interval(*parts: object) -> float:
+ """A uniform number strictly between 0 and 1, fixed by ``parts``."""
+ digest = hashlib.sha256(":".join(str(part) for part in parts).encode()).digest()
+ # 53 bits fit a float exactly, so the result never rounds to 0 or 1.
+ return ((int.from_bytes(digest[:8], "big") >> 11) + 0.5) / 2**53
+
+
+def _weighted_sample(
+ candidates: Sequence[Candidate], *, config: SelectionConfig, tick_start: datetime, max_staleness: int
+) -> list[int]:
+ # Efraimidis–Spirakis: sorting by u ** (1 / weight) gives a weighted sample without replacement.
+ # log(u) / weight sorts the same way and keeps precision when the weight is large.
+ keyed: list[tuple[float, int]] = []
+ for candidate in candidates:
+ candidate_weight = weight(candidate, config=config, now=tick_start, max_staleness=max_staleness)
+ if candidate_weight > 0:
+ u = _unit_interval(candidate.insight_id, tick_start.isoformat())
+ keyed.append((math.log(u) / candidate_weight, candidate.insight_id))
+ keyed.sort(key=lambda item: (-item[0], item[1]))
+ return [insight_id for _, insight_id in keyed]
+
+
+def _stalest_first(candidates: Sequence[Candidate], *, now: datetime, max_staleness: int) -> list[int]:
+ due = [candidate for candidate in candidates if staleness(candidate, now=now, max_staleness=max_staleness) > 0]
+ # A hashed order for never-scored insights, so first coverage does not favor any group of insights.
+ never_scored = sorted(
+ (candidate.insight_id for candidate in due if candidate.newest_scored_bucket is None),
+ key=_unit_interval,
+ )
+ scored = sorted(
+ (candidate for candidate in due if candidate.newest_scored_bucket is not None),
+ key=lambda candidate: (candidate.newest_scored_bucket, candidate.insight_id),
+ )
+ return never_scored + [candidate.insight_id for candidate in scored]
diff --git a/products/signals/backend/test/test_anomaly_selection.py b/products/signals/backend/test/test_anomaly_selection.py
new file mode 100644
index 000000000000..15cdd89711cf
--- /dev/null
+++ b/products/signals/backend/test/test_anomaly_selection.py
@@ -0,0 +1,137 @@
+from collections import Counter
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from posthog.schema import IntervalType
+
+from products.signals.backend.anomaly_scores.selection import (
+ Candidate,
+ InsightUsage,
+ SelectionConfig,
+ Weighting,
+ parse_selection_config,
+ pick_order,
+ staleness,
+)
+
+TICK = datetime(2026, 10, 15, 12, 0, tzinfo=UTC)
+MAX_STALENESS = 30
+WEIGHTED = SelectionConfig()
+UNIFORM = SelectionConfig(weighting=Weighting.UNIFORM)
+
+
+def _candidate(
+ insight_id: int,
+ *,
+ days_since_scored: int | None,
+ viewers: int = 0,
+ alert: bool = False,
+ interval: IntervalType = IntervalType.DAY,
+) -> Candidate:
+ newest = (
+ None if days_since_scored is None else datetime(2026, 10, 15, tzinfo=UTC) - timedelta(days=days_since_scored)
+ )
+ return Candidate(
+ insight_id=insight_id,
+ interval=interval,
+ newest_scored_bucket=newest,
+ usage=InsightUsage(viewers=viewers, has_enabled_alert=alert),
+ )
+
+
+class TestPickOrder(SimpleTestCase):
+ @parameterized.expand(
+ [
+ # (usage 1 × staleness 1) vs (usage 4 × staleness 1)
+ ("viewers", {"days_since_scored": 2}, {"days_since_scored": 2, "viewers": 3}, 0.8),
+ # (1 × 1) vs ((1 + 5) × 1)
+ ("enabled_alert", {"days_since_scored": 2}, {"days_since_scored": 2, "alert": True}, 6 / 7),
+ # A popular insight scored recently, (10 × 1), against a long-waiting unviewed one, (1 × 29).
+ ("waiting_overtakes_usage", {"days_since_scored": 2, "viewers": 9}, {"days_since_scored": 30}, 29 / 39),
+ # Staleness is capped, and a never-scored insight counts as the cap.
+ ("capped_staleness", {"days_since_scored": None}, {"days_since_scored": 90}, 0.5),
+ ]
+ )
+ def test_first_pick_rate_follows_weight(
+ self, _name: str, low: dict[str, Any], high: dict[str, Any], expected_high_rate: float
+ ) -> None:
+ candidates = [_candidate(1, **low), _candidate(2, **high)]
+ first_picks = Counter(
+ pick_order(
+ candidates, config=WEIGHTED, tick_start=TICK + timedelta(seconds=i), max_staleness=MAX_STALENESS
+ )[0]
+ for i in range(4000)
+ )
+
+ assert abs(first_picks[2] / 4000 - expected_high_rate) < 0.03
+
+ def test_uniform_picks_never_scored_then_stalest(self) -> None:
+ candidates = [
+ _candidate(1, days_since_scored=3, viewers=50),
+ _candidate(2, days_since_scored=None),
+ _candidate(3, days_since_scored=10),
+ _candidate(4, days_since_scored=None, viewers=50),
+ ]
+
+ order = pick_order(candidates, config=UNIFORM, tick_start=TICK, max_staleness=MAX_STALENESS)
+
+ assert set(order[:2]) == {2, 4}
+ assert order[2:] == [3, 1]
+
+ @parameterized.expand(
+ [
+ ("nothing_completed_weighted", WEIGHTED, {"days_since_scored": 1, "viewers": 50}),
+ ("nothing_completed_uniform", UNIFORM, {"days_since_scored": 1, "viewers": 50}),
+ ("unviewed_without_floor", SelectionConfig(usage_floor=0), {"days_since_scored": None}),
+ ]
+ )
+ def test_leaves_out_candidates_with_nothing_to_gain(
+ self, _name: str, config: SelectionConfig, excluded: dict[str, Any]
+ ) -> None:
+ candidates = [_candidate(1, **excluded), _candidate(2, days_since_scored=5, viewers=1)]
+
+ assert pick_order(candidates, config=config, tick_start=TICK, max_staleness=MAX_STALENESS) == [2]
+
+ @parameterized.expand(
+ [
+ # Oct 14 is ongoing at noon, so only Oct 12 and Oct 13 are completed.
+ ("day", IntervalType.DAY, datetime(2026, 10, 11, tzinfo=UTC), 2),
+ ("month", IntervalType.MONTH, datetime(2026, 7, 1, tzinfo=UTC), 2),
+ ("capped", IntervalType.HOUR, datetime(2026, 10, 1, tzinfo=UTC), MAX_STALENESS),
+ ]
+ )
+ def test_staleness_counts_completed_buckets_after_the_newest_scored_one(
+ self, _name: str, interval: IntervalType, newest_scored_bucket: datetime, expected: int
+ ) -> None:
+ candidate = Candidate(
+ insight_id=1,
+ interval=interval,
+ newest_scored_bucket=newest_scored_bucket,
+ usage=InsightUsage(viewers=0, has_enabled_alert=False),
+ )
+
+ assert staleness(candidate, now=datetime(2026, 10, 14, 12, tzinfo=UTC), max_staleness=MAX_STALENESS) == expected
+
+
+class TestParseSelectionConfig(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("missing", None, SelectionConfig()),
+ (
+ "valid",
+ {"weighting": "uniform", "usage_floor": 0, "alert_bonus": 10},
+ SelectionConfig(weighting=Weighting.UNIFORM, usage_floor=0, alert_bonus=10),
+ ),
+ (
+ "invalid_fields_fall_back",
+ {"weighting": "random", "usage_floor": -1, "alert_bonus": True},
+ SelectionConfig(),
+ ),
+ ]
+ )
+ def test_parse(self, _name: str, payload: Any, expected: SelectionConfig) -> None:
+ assert parse_selection_config(payload) == expected