Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): add per-pick and daily per-team anomaly scoring events (#125556)

posthog[bot]f625900d4500
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index 7dbb9c0eb255..d222ed8d70c0 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -148,6 +148,16 @@ class InsightPick:
run: ScoringRun | None = None
+@frozen
+class TeamCensus:
+ """The active insights of one eligible team, for the daily per-team census event."""
+
+ team_id: int
+ active: int
+ active_reasons: dict[str, int]
+ active_intervals: dict[str, int]
+
+
@frozen
class Selection:
picks: list[InsightPick]
@@ -166,6 +176,8 @@ class Selection:
# Active insights of the eligible teams, and how many of them have each reason. One insight can have several.
active: int = 0
active_reasons: dict[str, int] = dataclasses.field(default_factory=dict)
+ # One entry for each eligible team, also a team with no active insight.
+ census: list[TeamCensus] = dataclasses.field(default_factory=list)
@frozen
@@ -240,6 +252,21 @@ def reason_counts(insights: Collection[ActiveInsight]) -> dict[str, int]:
return {reason.value: counts[reason] for reason in ActivityReason}
+def team_census(team_ids: Collection[int], insights: Collection[ActiveInsight]) -> list[TeamCensus]:
+ by_team: dict[int, list[ActiveInsight]] = {team_id: [] for team_id in team_ids}
+ for insight in insights:
+ by_team.setdefault(insight.team_id, []).append(insight)
+ return [
+ TeamCensus(
+ team_id=team_id,
+ active=len(team_insights),
+ active_reasons=reason_counts(team_insights),
+ active_intervals=dict(Counter(insight.interval.value for insight in team_insights)),
+ )
+ for team_id, team_insights in sorted(by_team.items())
+ ]
+
+
def scored_state_since(
config: AnomalyScoringConfig, interval: IntervalType, now: datetime.datetime
) -> datetime.datetime:
@@ -463,6 +490,7 @@ def select_insights(
insights = active_insights(list(teams), config.intervals, now, config.active_window_days)
active = len(insights)
active_reasons = reason_counts(insights)
+ census = team_census(teams, insights)
state: dict[int, dict[str, datetime.datetime]] = {}
failed_teams: set[int] = set()
# One query per team and interval, so a day or week insight does not read the months a month insight needs.
@@ -623,6 +651,7 @@ def select_insights(
train_backlog_hard=sum(insight_id not in stale_ready for insight_id in backlog),
active=active,
active_reasons=active_reasons,
+ census=census,
)
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index d922b7ce6714..aa780fb60fe1 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -11,6 +11,7 @@ import dataclasses
from collections import Counter
from django.conf import settings
+from django.core.cache import cache
from django.utils import timezone
import structlog
@@ -51,6 +52,7 @@ from products.signals.backend.anomaly_scores.queue import (
ATTEMPT_COOLDOWN,
SKIP_COOLDOWN,
InsightPick,
+ TeamCensus,
mark_attempted,
select_insights,
)
@@ -93,6 +95,8 @@ _INSIGHT_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)
_CAPTURE_TIMEOUT = datetime.timedelta(minutes=1)
# No batch of insights starts after this time. The picks that are left wait for a later tick.
TICK_BUDGET = datetime.timedelta(minutes=10)
+# A census key lives past its UTC day, so a tick at the end of the day still reads it.
+_CENSUS_TTL = datetime.timedelta(days=2)
# The last batch can start at the budget, and the summary capture runs after it.
TICK_TIMEOUT = _PLAN_TIMEOUT + TICK_BUDGET + _INSIGHT_TIMEOUT + _CAPTURE_TIMEOUT
@@ -176,6 +180,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
return AnomalyScoringTickPlan(enabled=False, picks=[], candidates=0, max_concurrency=config.max_concurrency)
selection = select_insights(config, parse_selection_config(payload), now, limit)
record_candidates(selection.candidates)
+ capture_census(selection.census, now)
return AnomalyScoringTickPlan(
enabled=True,
picks=selection.picks,
@@ -238,6 +243,7 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
duration_ms=duration_ms,
)
record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0, run=pick.run)
+ capture_pick(pick, status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, run=pick.run)
if status is None:
raise
# A full cluster is not a fault of the insight, so it keeps the attempt cooldown.
@@ -259,9 +265,75 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
missing_series=result.missing_series,
)
record_pick(status=result.status, interval=interval, duration_ms=duration_ms, events=result.events, run=result.run)
+ capture_pick(
+ pick,
+ status=result.status,
+ interval=interval,
+ duration_ms=duration_ms,
+ run=result.run,
+ series=result.series,
+ events=result.events,
+ )
return result
+def capture_pick(
+ pick: InsightPick,
+ *,
+ status: str,
+ interval: str,
+ duration_ms: int,
+ run: ScoringRun | None,
+ series: int = 0,
+ events: int = 0,
+) -> None:
+ """One event per pick, with the ids that the SDK metrics leave out, so cost and outcome break down per project."""
+ ph_background_capture()(
+ distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,
+ event="anomaly scoring pick",
+ properties={
+ "deployment": settings.CLOUD_DEPLOYMENT,
+ "team_id": pick.team_id,
+ "insight_id": pick.insight_id,
+ "interval": interval,
+ "run": run,
+ "status": status,
+ "duration_ms": duration_ms,
+ "series": series,
+ "events": events,
+ "queries": pick.query_cost,
+ },
+ )
+
+
+def _census_key(team_id: int, day: datetime.date) -> str:
+ return f"signals:anomaly_scoring:census:{day.isoformat()}:{team_id}"
+
+
+def capture_census(census: list[TeamCensus], now: datetime.datetime) -> None:
+ """One event per eligible team per UTC day, from the first tick of the day that sees the team."""
+ day = now.astimezone(datetime.UTC).date()
+ sent = cache.get_many([_census_key(team.team_id, day) for team in census])
+ capture = ph_background_capture()
+ for team in census:
+ key = _census_key(team.team_id, day)
+ # `add` sets the key only when it is missing, so two ticks that overlap send one event.
+ if key in sent or not cache.add(key, True, timeout=_CENSUS_TTL.total_seconds()):
+ continue
+ capture(
+ distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,
+ event="anomaly scoring team census",
+ properties={
+ "deployment": settings.CLOUD_DEPLOYMENT,
+ "team_id": team.team_id,
+ "day": day.isoformat(),
+ "active_insights": team.active,
+ **{f"active_{reason}": count for reason, count in team.active_reasons.items()},
+ **{f"interval_{interval}": count for interval, count in team.active_intervals.items()},
+ },
+ )
+
+
def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsightResult:
pick = input.pick
insight = next(
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index 42c2c43e643e..3602b3214a35 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -47,6 +47,7 @@ from products.signals.backend.anomaly_scores.queue import (
ActivityReason,
InsightPick,
QueryBudget,
+ TeamCensus,
_cooldown_key,
_fleet_budget_key,
_project_budget_key,
@@ -79,6 +80,7 @@ from products.signals.backend.anomaly_scores.tick import (
AnomalyScoringTickWorkflow,
ScoreInsightInput,
ScoreInsightResult,
+ capture_census,
plan_tick,
run_insight,
)
@@ -480,10 +482,12 @@ class TestScoreEvents(SimpleTestCase):
class TestRunInsight(SimpleTestCase):
- def _run(self, error: Exception) -> tuple[str, list[Any], MagicMock]:
+ def _run(self, error: Exception) -> tuple[str, list[Any], MagicMock, dict[str, Any]]:
tick = "products.signals.backend.anomaly_scores.tick"
client = MagicMock()
+ capture = MagicMock()
with (
+ patch(f"{tick}.ph_background_capture", return_value=capture),
patch(
f"{tick}.insights_including_soft_deleted_for_team",
return_value=[Insight(id=7, saved=True, deleted=False)],
@@ -499,7 +503,10 @@ class TestRunInsight(SimpleTestCase):
except Exception:
status = "raised"
cooldowns = [call.args[1] for call in mark_attempted.call_args_list]
- return status, cooldowns, client.metrics
+ [pick_event] = [
+ call.kwargs for call in capture.call_args_list if call.kwargs["event"] == "anomaly scoring pick"
+ ]
+ return status, cooldowns, client.metrics, pick_event["properties"]
@parameterized.expand(
[
@@ -512,16 +519,23 @@ class TestRunInsight(SimpleTestCase):
def test_a_known_failure_returns_its_status_and_waits_a_day(
self, _name: str, error: Exception, expected: str
) -> None:
- status, cooldowns, metrics = self._run(error)
+ status, cooldowns, metrics, pick_event = self._run(error)
assert status == expected
assert cooldowns[-1] == SKIP_COOLDOWN
metrics.count.assert_any_call(
"anomaly_scoring.picks", 1, attributes={"status": expected, "interval": IntervalType.DAY.value}
)
+ assert {key: pick_event[key] for key in ("team_id", "insight_id", "interval", "status", "queries")} == {
+ "team_id": 1,
+ "insight_id": 7,
+ "interval": IntervalType.DAY.value,
+ "status": expected,
+ "queries": 1,
+ }
def test_a_full_cluster_returns_at_capacity_and_waits_an_hour(self) -> None:
- status, cooldowns, _metrics = self._run(ClickHouseAtCapacity())
+ status, cooldowns, _metrics, _pick_event = self._run(ClickHouseAtCapacity())
assert status == "at_capacity"
assert SKIP_COOLDOWN not in cooldowns
@@ -534,10 +548,37 @@ class TestRunInsight(SimpleTestCase):
]
)
def test_any_other_failure_raises_and_waits_an_hour(self, _name: str, error: Exception) -> None:
- status, cooldowns, _metrics = self._run(error)
+ status, cooldowns, _metrics, pick_event = self._run(error)
assert status == "raised"
assert SKIP_COOLDOWN not in cooldowns
+ assert pick_event["status"] == "errored"
+
+
+class TestCaptureCensus(SimpleTestCase):
+ def test_each_team_gets_one_census_event_per_utc_day(self) -> None:
+ census = [
+ TeamCensus(team_id=team_id, active=1, active_reasons={"viewed": 1}, active_intervals={"day": 1})
+ for team_id in (901, 902)
+ ]
+ days = [NOW, NOW + timedelta(hours=11), NOW + timedelta(days=1)]
+ cache.delete_many(
+ [
+ f"signals:anomaly_scoring:census:{day.date().isoformat()}:{team_id}"
+ for day in days
+ for team_id in (901, 902)
+ ]
+ )
+ capture = MagicMock()
+
+ with patch("products.signals.backend.anomaly_scores.tick.ph_background_capture", return_value=capture):
+ for now in days:
+ capture_census(census, now)
+
+ sent = [
+ (call.kwargs["properties"]["day"], call.kwargs["properties"]["team_id"]) for call in capture.call_args_list
+ ]
+ assert sent == [("2026-10-22", 901), ("2026-10-22", 902), ("2026-10-23", 901), ("2026-10-23", 902)]
class TestTrainScoreSplit(BaseTest):
@@ -907,6 +948,10 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
assert selection.picks[0].insight_id in {insight.id for insight in large_insights}
assert selection.picks[1].insight_id == small_insight.id
assert selection.candidates == 4
+ assert [(team.team_id, team.active, team.active_intervals) for team in selection.census] == [
+ (self.team.id, 3, {"day": 3}),
+ (small_team.id, 1, {"day": 1}),
+ ]
@time_machine.travel(NOW, tick=False)
def test_select_insights_skips_a_project_whose_scored_buckets_query_times_out(self) -> None: