:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): add per-pick and daily per-team anomaly scoring events (#125556)
posthog[bot]f625900d4500
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.pyindex 7dbb9c0eb255..d222ed8d70c0 100644--- a/products/signals/backend/anomaly_scores/queue.py+++ b/products/signals/backend/anomaly_scores/queue.py@@ -148,6 +148,16 @@ class InsightPick:run: ScoringRun | None = None+@frozen+class TeamCensus:+ """The active insights of one eligible team, for the daily per-team census event."""++ team_id: int+ active: int+ active_reasons: dict[str, int]+ active_intervals: dict[str, int]++@frozenclass Selection:picks: list[InsightPick]@@ -166,6 +176,8 @@ class Selection:# Active insights of the eligible teams, and how many of them have each reason. One insight can have several.active: int = 0active_reasons: dict[str, int] = dataclasses.field(default_factory=dict)+ # One entry for each eligible team, also a team with no active insight.+ census: list[TeamCensus] = dataclasses.field(default_factory=list)@frozen@@ -240,6 +252,21 @@ def reason_counts(insights: Collection[ActiveInsight]) -> dict[str, int]:return {reason.value: counts[reason] for reason in ActivityReason}+def team_census(team_ids: Collection[int], insights: Collection[ActiveInsight]) -> list[TeamCensus]:+ by_team: dict[int, list[ActiveInsight]] = {team_id: [] for team_id in team_ids}+ for insight in insights:+ by_team.setdefault(insight.team_id, []).append(insight)+ return [+ TeamCensus(+ team_id=team_id,+ active=len(team_insights),+ active_reasons=reason_counts(team_insights),+ active_intervals=dict(Counter(insight.interval.value for insight in team_insights)),+ )+ for team_id, team_insights in sorted(by_team.items())+ ]++def scored_state_since(config: AnomalyScoringConfig, interval: IntervalType, now: datetime.datetime) -> datetime.datetime:@@ -463,6 +490,7 @@ def select_insights(insights = active_insights(list(teams), config.intervals, now, config.active_window_days)active = len(insights)active_reasons = reason_counts(insights)+ census = team_census(teams, insights)state: dict[int, dict[str, datetime.datetime]] = {}failed_teams: set[int] = set()# One query per team and interval, so a day or week insight does not read the months a month insight needs.@@ -623,6 +651,7 @@ def select_insights(train_backlog_hard=sum(insight_id not in stale_ready for insight_id in backlog),active=active,active_reasons=active_reasons,+ census=census,)diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.pyindex d922b7ce6714..aa780fb60fe1 100644--- a/products/signals/backend/anomaly_scores/tick.py+++ b/products/signals/backend/anomaly_scores/tick.py@@ -11,6 +11,7 @@ import dataclassesfrom collections import Counterfrom django.conf import settings+from django.core.cache import cachefrom django.utils import timezoneimport structlog@@ -51,6 +52,7 @@ from products.signals.backend.anomaly_scores.queue import (ATTEMPT_COOLDOWN,SKIP_COOLDOWN,InsightPick,+ TeamCensus,mark_attempted,select_insights,)@@ -93,6 +95,8 @@ _INSIGHT_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)_CAPTURE_TIMEOUT = datetime.timedelta(minutes=1)# No batch of insights starts after this time. The picks that are left wait for a later tick.TICK_BUDGET = datetime.timedelta(minutes=10)+# A census key lives past its UTC day, so a tick at the end of the day still reads it.+_CENSUS_TTL = datetime.timedelta(days=2)# The last batch can start at the budget, and the summary capture runs after it.TICK_TIMEOUT = _PLAN_TIMEOUT + TICK_BUDGET + _INSIGHT_TIMEOUT + _CAPTURE_TIMEOUT@@ -176,6 +180,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlreturn AnomalyScoringTickPlan(enabled=False, picks=[], candidates=0, max_concurrency=config.max_concurrency)selection = select_insights(config, parse_selection_config(payload), now, limit)record_candidates(selection.candidates)+ capture_census(selection.census, now)return AnomalyScoringTickPlan(enabled=True,picks=selection.picks,@@ -238,6 +243,7 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsighduration_ms=duration_ms,)record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0, run=pick.run)+ capture_pick(pick, status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, run=pick.run)if status is None:raise# A full cluster is not a fault of the insight, so it keeps the attempt cooldown.@@ -259,9 +265,75 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsighmissing_series=result.missing_series,)record_pick(status=result.status, interval=interval, duration_ms=duration_ms, events=result.events, run=result.run)+ capture_pick(+ pick,+ status=result.status,+ interval=interval,+ duration_ms=duration_ms,+ run=result.run,+ series=result.series,+ events=result.events,+ )return result+def capture_pick(+ pick: InsightPick,+ *,+ status: str,+ interval: str,+ duration_ms: int,+ run: ScoringRun | None,+ series: int = 0,+ events: int = 0,+) -> None:+ """One event per pick, with the ids that the SDK metrics leave out, so cost and outcome break down per project."""+ ph_background_capture()(+ distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,+ event="anomaly scoring pick",+ properties={+ "deployment": settings.CLOUD_DEPLOYMENT,+ "team_id": pick.team_id,+ "insight_id": pick.insight_id,+ "interval": interval,+ "run": run,+ "status": status,+ "duration_ms": duration_ms,+ "series": series,+ "events": events,+ "queries": pick.query_cost,+ },+ )+++def _census_key(team_id: int, day: datetime.date) -> str:+ return f"signals:anomaly_scoring:census:{day.isoformat()}:{team_id}"+++def capture_census(census: list[TeamCensus], now: datetime.datetime) -> None:+ """One event per eligible team per UTC day, from the first tick of the day that sees the team."""+ day = now.astimezone(datetime.UTC).date()+ sent = cache.get_many([_census_key(team.team_id, day) for team in census])+ capture = ph_background_capture()+ for team in census:+ key = _census_key(team.team_id, day)+ # `add` sets the key only when it is missing, so two ticks that overlap send one event.+ if key in sent or not cache.add(key, True, timeout=_CENSUS_TTL.total_seconds()):+ continue+ capture(+ distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,+ event="anomaly scoring team census",+ properties={+ "deployment": settings.CLOUD_DEPLOYMENT,+ "team_id": team.team_id,+ "day": day.isoformat(),+ "active_insights": team.active,+ **{f"active_{reason}": count for reason, count in team.active_reasons.items()},+ **{f"interval_{interval}": count for interval, count in team.active_intervals.items()},+ },+ )++def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsightResult:pick = input.pickinsight = next(diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.pyindex 42c2c43e643e..3602b3214a35 100644--- a/products/signals/backend/test/test_anomaly_scoring_tick.py+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py@@ -47,6 +47,7 @@ from products.signals.backend.anomaly_scores.queue import (ActivityReason,InsightPick,QueryBudget,+ TeamCensus,_cooldown_key,_fleet_budget_key,_project_budget_key,@@ -79,6 +80,7 @@ from products.signals.backend.anomaly_scores.tick import (AnomalyScoringTickWorkflow,ScoreInsightInput,ScoreInsightResult,+ capture_census,plan_tick,run_insight,)@@ -480,10 +482,12 @@ class TestScoreEvents(SimpleTestCase):class TestRunInsight(SimpleTestCase):- def _run(self, error: Exception) -> tuple[str, list[Any], MagicMock]:+ def _run(self, error: Exception) -> tuple[str, list[Any], MagicMock, dict[str, Any]]:tick = "products.signals.backend.anomaly_scores.tick"client = MagicMock()+ capture = MagicMock()with (+ patch(f"{tick}.ph_background_capture", return_value=capture),patch(f"{tick}.insights_including_soft_deleted_for_team",return_value=[Insight(id=7, saved=True, deleted=False)],@@ -499,7 +503,10 @@ class TestRunInsight(SimpleTestCase):except Exception:status = "raised"cooldowns = [call.args[1] for call in mark_attempted.call_args_list]- return status, cooldowns, client.metrics+ [pick_event] = [+ call.kwargs for call in capture.call_args_list if call.kwargs["event"] == "anomaly scoring pick"+ ]+ return status, cooldowns, client.metrics, pick_event["properties"]@parameterized.expand([@@ -512,16 +519,23 @@ class TestRunInsight(SimpleTestCase):def test_a_known_failure_returns_its_status_and_waits_a_day(self, _name: str, error: Exception, expected: str) -> None:- status, cooldowns, metrics = self._run(error)+ status, cooldowns, metrics, pick_event = self._run(error)assert status == expectedassert cooldowns[-1] == SKIP_COOLDOWNmetrics.count.assert_any_call("anomaly_scoring.picks", 1, attributes={"status": expected, "interval": IntervalType.DAY.value})+ assert {key: pick_event[key] for key in ("team_id", "insight_id", "interval", "status", "queries")} == {+ "team_id": 1,+ "insight_id": 7,+ "interval": IntervalType.DAY.value,+ "status": expected,+ "queries": 1,+ }def test_a_full_cluster_returns_at_capacity_and_waits_an_hour(self) -> None:- status, cooldowns, _metrics = self._run(ClickHouseAtCapacity())+ status, cooldowns, _metrics, _pick_event = self._run(ClickHouseAtCapacity())assert status == "at_capacity"assert SKIP_COOLDOWN not in cooldowns@@ -534,10 +548,37 @@ class TestRunInsight(SimpleTestCase):])def test_any_other_failure_raises_and_waits_an_hour(self, _name: str, error: Exception) -> None:- status, cooldowns, _metrics = self._run(error)+ status, cooldowns, _metrics, pick_event = self._run(error)assert status == "raised"assert SKIP_COOLDOWN not in cooldowns+ assert pick_event["status"] == "errored"+++class TestCaptureCensus(SimpleTestCase):+ def test_each_team_gets_one_census_event_per_utc_day(self) -> None:+ census = [+ TeamCensus(team_id=team_id, active=1, active_reasons={"viewed": 1}, active_intervals={"day": 1})+ for team_id in (901, 902)+ ]+ days = [NOW, NOW + timedelta(hours=11), NOW + timedelta(days=1)]+ cache.delete_many(+ [+ f"signals:anomaly_scoring:census:{day.date().isoformat()}:{team_id}"+ for day in days+ for team_id in (901, 902)+ ]+ )+ capture = MagicMock()++ with patch("products.signals.backend.anomaly_scores.tick.ph_background_capture", return_value=capture):+ for now in days:+ capture_census(census, now)++ sent = [+ (call.kwargs["properties"]["day"], call.kwargs["properties"]["team_id"]) for call in capture.call_args_list+ ]+ assert sent == [("2026-10-22", 901), ("2026-10-22", 902), ("2026-10-23", 901), ("2026-10-23", 902)]class TestTrainScoreSplit(BaseTest):@@ -907,6 +948,10 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):assert selection.picks[0].insight_id in {insight.id for insight in large_insights}assert selection.picks[1].insight_id == small_insight.idassert selection.candidates == 4+ assert [(team.team_id, team.active, team.active_intervals) for team in selection.census] == [+ (self.team.id, 3, {"day": 3}),+ (small_team.id, 1, {"day": 1}),+ ]@time_machine.travel(NOW, tick=False)def test_select_insights_skips_a_project_whose_scored_buckets_query_times_out(self) -> None: