Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): keep score slots for stale anomaly models (#128552)

posthog[bot]b227a2d0a435
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index b156b721a7d9..6f50b4e2c903 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -11,6 +11,7 @@ first: it takes the trained insights from their stored state, longest overdue fi
keep them current. The train pass then takes the other due insights in the `pick_order` order. An insight
whose model is due for a soft retrain goes to the train pass first. When the train pass does not pick it,
the score pass scores it with the old model, so the train budget does not decide the score coverage.
+The first score walk leaves a share of the score cap for these stale models, so ready picks cannot take every slot.
The CPU limits of `spend.py` apply before the query budgets: a walk leaves out the insights of a team over its
daily CPU limit, and every insight of a pass that the fleet limit stops.
@@ -125,6 +126,8 @@ _MAX_AT_CAPACITY_TEAMS = 2
_TEAM_LOADED_TTL = datetime.timedelta(days=1)
# The most due rows that one plan reads. Far more than the picks of one tick.
_MAX_DUE_ROWS = 5000
+# The score pass keeps one in this many of its slots for stale models, so ready picks cannot take every slot.
+_STALE_SCORE_SHARE = 4
# The scorer rejects these, so selection leaves them out instead of picking them on every tick.
_UNSCORED_DISPLAYS = sorted(
display.value for display in {*NON_TIME_SERIES_DISPLAY_TYPES, ChartDisplayType.ACTIONS_LINE_GRAPH_CUMULATIVE}
@@ -202,6 +205,9 @@ class Selection:
picked_train: int = 0
# Score pass picks that score with a model due for a soft retrain.
scored_stale_model: int = 0
+ # Due insights with a stale model that the train pass did not pick. More than `scored_stale_model` means
+ # the score pass had no slot or budget for them.
+ stale_candidates: int = 0
# Due train candidates that the train pass did not pick: soft ones still score with a stale model, hard ones do not.
train_backlog_soft: int = 0
train_backlog_hard: int = 0
@@ -816,7 +822,8 @@ def select_insights(
if config.train_score_split:
score_pass = _score_order(active_by_id, teams, config, now)
score_order_ids, stale_order_ids, score_ready = score_pass.order, score_pass.stale_order, score_pass.ready
- score_picked, score_skipped = walk(score_order_ids, ScoringRun.SCORE, score_cap)
+ stale_reserve = min(len(stale_order_ids), _stale_reserve(score_cap))
+ score_picked, score_skipped = walk(score_order_ids, ScoringRun.SCORE, score_cap - stale_reserve)
# A spent score budget makes the walk load no plan. Plan the head of the order anyway, so that an
# edited insight there still reaches the train pass while the train budget has room.
plans_for(score_order_ids[:score_cap])
@@ -880,9 +887,10 @@ def select_insights(
stale_picked: dict[int, RunPlan] = {}
stale_skipped = 0
stale_cap = score_cap - len(score_picked)
+ stale_candidates = [insight_id for insight_id in stale_order_ids if insight_id not in train_picked]
if config.train_score_split and stale_cap > 0:
stale_picked, stale_skipped = walk(
- [insight_id for insight_id in stale_order_ids if insight_id not in train_picked],
+ stale_candidates,
ScoringRun.SCORE,
stale_cap,
lambda insight_ids: {
@@ -912,6 +920,7 @@ def select_insights(
picked_score=len(score_picked),
picked_train=len(train_picked),
scored_stale_model=len(stale_picked),
+ stale_candidates=len(stale_candidates),
train_backlog_soft=sum(insight_id in stale_ready for insight_id in backlog),
train_backlog_hard=sum(insight_id not in stale_ready for insight_id in backlog),
active=active,
@@ -936,6 +945,11 @@ def select_insights(
)
+def _stale_reserve(score_cap: int) -> int:
+ """The score pass slots that ready picks leave for stale models. A cap of one slot keeps it for ready picks."""
+ return max(1, score_cap // _STALE_SCORE_SHARE) if score_cap > 1 else 0
+
+
def _as_score_step(plan: RunPlan) -> RunPlan:
"""A stale model plan as a score step. A plan that needs a train step stays one, and the score pass passes over it."""
return dataclasses.replace(plan, run=ScoringRun.SCORE) if plan.stale_model else plan
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index f27415fd2a4b..6993df5a9826 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -133,6 +133,7 @@ class AnomalyScoringTickPlan:
picked_score: int = 0
picked_train: int = 0
scored_stale_model: int = 0
+ stale_candidates: int = 0
train_backlog_soft: int = 0
train_backlog_hard: int = 0
active: int = 0
@@ -188,8 +189,10 @@ class AnomalyScoringTickResult:
# Planned picks of the score pass and of the train pass, started or not.
picked_score: int = 0
picked_train: int = 0
- # Planned score pass picks with a model due for a soft retrain, and the due train candidates left unpicked.
+ # Planned score pass picks with a model due for a soft retrain, the stale models that the score pass could take,
+ # and the due train candidates left unpicked.
scored_stale_model: int = 0
+ stale_candidates: int = 0
train_backlog_soft: int = 0
train_backlog_hard: int = 0
# Active insights of the eligible teams, and how many of them have each activity reason.
@@ -255,6 +258,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
picked_score=selection.picked_score,
picked_train=selection.picked_train,
scored_stale_model=selection.scored_stale_model,
+ stale_candidates=selection.stale_candidates,
train_backlog_soft=selection.train_backlog_soft,
train_backlog_hard=selection.train_backlog_hard,
active=selection.active,
@@ -552,6 +556,7 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
picked_score=result.picked_score,
picked_train=result.picked_train,
scored_stale_model=result.scored_stale_model,
+ stale_candidates=result.stale_candidates,
train_backlog_soft=result.train_backlog_soft,
train_backlog_hard=result.train_backlog_hard,
active=result.active,
@@ -594,6 +599,7 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
"picked_score": result.picked_score,
"picked_train": result.picked_train,
"scored_stale_model": result.scored_stale_model,
+ "stale_candidates": result.stale_candidates,
"train_backlog_soft": result.train_backlog_soft,
"train_backlog_hard": result.train_backlog_hard,
"active_insights": result.active,
@@ -703,6 +709,7 @@ class AnomalyScoringTickWorkflow:
picked_score=plan.picked_score,
picked_train=plan.picked_train,
scored_stale_model=plan.scored_stale_model,
+ stale_candidates=plan.stale_candidates,
train_backlog_soft=plan.train_backlog_soft,
train_backlog_hard=plan.train_backlog_hard,
active=plan.active,
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index 776a94318d2e..f1ed9e7b3620 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -1719,14 +1719,23 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
(
"train_budget_spent",
True,
+ 0,
[("past_train_every", ScoringRun.SCORE)],
- (1, 1, 1),
+ (1, 1, 1, 1),
),
(
"train_budget_left",
False,
+ 0,
[("past_age_ceiling", ScoringRun.TRAIN), ("past_train_every", ScoringRun.TRAIN)],
- (0, 0, 0),
+ (0, 0, 0, 0),
+ ),
+ (
+ "ready_picks_fill_the_score_cap",
+ True,
+ 3,
+ [("past_train_every", ScoringRun.SCORE), ("ready", ScoringRun.SCORE)],
+ (1, 1, 1, 1),
),
]
)
@@ -1735,24 +1744,32 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
self,
_name: str,
train_budget_spent: bool,
+ ready_insights: int,
expected: list[tuple[str, ScoringRun]],
- stale_and_backlog: tuple[int, int, int],
+ stale_and_backlog: tuple[int, int, int, int],
) -> None:
self.team.timezone = "UTC"
self.team.save()
dashboard = Dashboard.objects.create(team=self.team, name="Main")
+ trained_days_ago = {"past_train_every": 10, "past_age_ceiling": 30}
+ trained_days_ago.update({f"ready_{index}": 1 for index in range(ready_insights)})
insights = {
name: Insight.objects.create(team=self.team, name=name, query=_trends(interval="day"))
- for name in ("past_train_every", "past_age_ceiling")
+ for name in trained_days_ago
}
for insight in insights.values():
DashboardTile.objects.create(dashboard=dashboard, insight=insight)
cache.delete(_cooldown_key(insight.id))
config = parse_config(
- {**_SPLIT_PAYLOAD, "organization_ids": [str(self.organization.id)], "allow_team_ids": [self.team.id]}
+ {
+ **_SPLIT_PAYLOAD,
+ "organization_ids": [str(self.organization.id)],
+ "allow_team_ids": [self.team.id],
+ "max_score_insights_per_tick": 2,
+ }
)
today = NOW.replace(hour=0)
- for name, trained_days_ago in (("past_train_every", 10), ("past_age_ceiling", 30)):
+ for name, days_ago in trained_days_ago.items():
insight = insights[name]
save_state(
self.team,
@@ -1765,7 +1782,7 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
self.team,
config.limits,
),
- trained_at=NOW - timedelta(days=trained_days_ago),
+ trained_at=NOW - timedelta(days=days_ago),
scored_through=today - timedelta(days=2),
series_keys=("a",),
artifact_key=f"anomaly_models/{insight.id}",
@@ -1779,10 +1796,13 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):
selection = select_insights(config, SelectionConfig(weighting=Weighting.UNIFORM), NOW)
- names = {insight.id: name for name, insight in insights.items()}
+ names = {
+ insight.id: name.split("_")[0] if name.startswith("ready_") else name for name, insight in insights.items()
+ }
assert sorted((names[pick.insight_id], pick.run) for pick in selection.picks) == expected
assert (
selection.scored_stale_model,
+ selection.stale_candidates,
selection.train_backlog_soft,
selection.train_backlog_hard,
) == stale_and_backlog