Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): keep anomaly scores running on a stale model (#123861)

posthog[bot]e4c1e4667555
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.py
index 102daafa728d..581960807f54 100644
--- a/products/signals/backend/anomaly_scores/config.py
+++ b/products/signals/backend/anomaly_scores/config.py
@@ -93,6 +93,20 @@ class ScoringRun(StrEnum):
# Retrain hourly insights once a day, daily once a week, weekly once a month, and monthly once a quarter.
DEFAULT_TRAIN_EVERY_BUCKETS = {IntervalType.HOUR: 24, IntervalType.DAY: 7, IntervalType.WEEK: 4, IntervalType.MONTH: 3}
MAX_TRAIN_EVERY_BUCKETS = {IntervalType.HOUR: 168, IntervalType.DAY: 30, IntervalType.WEEK: 13, IntervalType.MONTH: 12}
+# Past `train_every_buckets`, a model keeps scoring until a train step has budget for it. Past this age it stops
+# scoring, and its insight waits for a train step. A value at or below `train_every_buckets` turns off stale scores.
+DEFAULT_MAX_MODEL_AGE_BUCKETS = {
+ IntervalType.HOUR: 96,
+ IntervalType.DAY: 28,
+ IntervalType.WEEK: 16,
+ IntervalType.MONTH: 12,
+}
+MAX_MODEL_AGE_BUCKETS_LIMIT = {
+ IntervalType.HOUR: 672,
+ IntervalType.DAY: 120,
+ IntervalType.WEEK: 52,
+ IntervalType.MONTH: 48,
+}
# Hourly insights are left out by default: with many eligible insights, the time between two visits of
@@ -121,6 +135,9 @@ class AnomalyScoringConfig:
train_every_buckets: dict[IntervalType, int] = dataclasses.field(
default_factory=lambda: dict(DEFAULT_TRAIN_EVERY_BUCKETS)
)
+ max_model_age_buckets: dict[IntervalType, int] = dataclasses.field(
+ default_factory=lambda: dict(DEFAULT_MAX_MODEL_AGE_BUCKETS)
+ )
# Score step queries per clock hour. Train steps use the budgets above, so a cheap score step never waits for them.
max_score_queries_per_hour: int = 600
max_score_queries_per_project_per_hour: int = 500
@@ -228,6 +245,9 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:
train_every_buckets=_buckets_per_interval(
payload.get("train_every_buckets"), DEFAULT_TRAIN_EVERY_BUCKETS, MAX_TRAIN_EVERY_BUCKETS
),
+ max_model_age_buckets=_buckets_per_interval(
+ payload.get("max_model_age_buckets"), DEFAULT_MAX_MODEL_AGE_BUCKETS, MAX_MODEL_AGE_BUCKETS_LIMIT
+ ),
max_score_queries_per_hour=min(
_positive_int(payload.get("max_score_queries_per_hour"), defaults.max_score_queries_per_hour),
MAX_SCORE_QUERIES_PER_HOUR,
diff --git a/products/signals/backend/anomaly_scores/emission.py b/products/signals/backend/anomaly_scores/emission.py
index dc534d22b66f..4e796a37dc1b 100644
--- a/products/signals/backend/anomaly_scores/emission.py
+++ b/products/signals/backend/anomaly_scores/emission.py
@@ -59,6 +59,12 @@ def _event(
"$anomaly_details": json.dumps(point.details, sort_keys=True),
**({"$anomaly_run": scores.run.value} if scores.run else {}),
**({"$anomaly_model_id": scores.model_id} if scores.model_id else {}),
+ # Tells a score from a model past its retrain age apart from a score from a fresh model.
+ **(
+ {"$anomaly_trained_at": scores.state.trained_at.astimezone(datetime.UTC).isoformat()}
+ if scores.state
+ else {}
+ ),
},
}
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index 90cfa257f136..c8bce20c9c9b 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -7,11 +7,14 @@ makes each pick a train step or a score step, and each kind has its own budgets.
With the split on, the tick runs two passes, each with its own order and pick cap. The score pass goes
first: it takes the trained insights from their stored state, longest overdue first, so cheap score steps
-keep them current. The train pass then takes the other due insights in the `pick_order` order.
+keep them current. The train pass then takes the other due insights in the `pick_order` order. An insight
+whose model is due for a soft retrain goes to the train pass first. When the train pass does not pick it,
+the score pass scores it with the old model, so the train budget does not decide the score coverage.
"""
import datetime
import itertools
+import dataclasses
from collections.abc import Callable, Collection
from typing import Any, cast
@@ -53,7 +56,7 @@ from products.signals.backend.anomaly_scores.selection import (
pick_order,
score_order,
)
-from products.signals.backend.anomaly_scores.state import RunPlan, load_states, plan_run, scored_through_by_insight
+from products.signals.backend.anomaly_scores.state import RunPlan, load_states, plan_run, score_ready_by_insight
from ee.billing.quota_limiting import QuotaLimitingCaches, QuotaResource, list_limited_team_attributes
@@ -139,6 +142,21 @@ class Selection:
# Picks of the score pass and of the train pass. With the split off, every pick is a train pass pick.
picked_score: int = 0
picked_train: int = 0
+ # Score pass picks that score with a model due for a soft retrain.
+ scored_stale_model: int = 0
+ # Due train candidates that the train pass did not pick: soft ones still score with a stale model, hard ones do not.
+ train_backlog_soft: int = 0
+ train_backlog_hard: int = 0
+
+
+@frozen
+class ScorePassOrder:
+ # Due insights with a current model, longest overdue first.
+ order: list[int]
+ # Due insights with a model past a soft retrain trigger, longest overdue first.
+ stale_order: list[int]
+ # Every watched insight with a current model, due or not.
+ ready: set[int]
@frozen
@@ -436,24 +454,31 @@ def select_insights(
planned.update(_run_plans(teams[team_id], team_batch, config, now))
return {insight_id: planned[insight_id] for insight_id in insight_ids}
- def walk(order: list[int], kind: ScoringRun, cap: int) -> tuple[dict[int, RunPlan], int]:
+ def walk(
+ order: list[int],
+ kind: ScoringRun,
+ cap: int,
+ plans: Callable[[list[int]], dict[int, RunPlan]] = plans_for,
+ ) -> tuple[dict[int, RunPlan], int]:
# Read the budget before any cost, so a spent hour loads no insight.
return within_budget(
order,
team_ids,
{kind: remaining_budget(config, {team_ids[insight_id] for insight_id in order}, now, kind)},
cap,
- plans_for,
+ plans,
reserve=lambda team_id, plan: reserve_queries(config, team_id, plan.queries, now, budget_kind(plan.run)),
)
score_order_ids: list[int] = []
+ stale_order_ids: list[int] = []
score_ready: set[int] = set()
score_picked: dict[int, RunPlan] = {}
score_skipped = 0
+ score_cap = config.max_score_insights_per_tick if limit is None else limit
if config.train_score_split:
- score_order_ids, score_ready = _score_order(watched_by_id, teams, config, now)
- score_cap = config.max_score_insights_per_tick if limit is None else limit
+ score_pass = _score_order(watched_by_id, teams, config, now)
+ score_order_ids, stale_order_ids, score_ready = score_pass.order, score_pass.stale_order, score_pass.ready
score_picked, score_skipped = walk(score_order_ids, ScoringRun.SCORE, score_cap)
# A spent score budget makes the walk load no plan. Plan the head of the order anyway, so that an
# edited insight there still reaches the train pass while the train budget has room.
@@ -461,6 +486,7 @@ def select_insights(
# A score-ready insight belongs to the score pass, also when its cap or budget left it out this tick.
# Without this, the train pass loads a plan for every one of them and then passes over it. An insight
# that the score pass planned as a train step, for example after an edit, goes to the train pass.
+ # A stale model goes to the train pass first, and the score pass takes it back below.
score_only = {
insight_id
for insight_id in score_ready
@@ -507,9 +533,25 @@ def select_insights(
train_cap = config.max_train_insights_per_tick
else:
train_cap = config.max_insights_per_tick
- # With the split off, every pick charges the train budgets.
+ # A stale model plans as a train step here. With the split off, every pick charges the train budgets.
train_picked, train_skipped = walk(train_order, ScoringRun.TRAIN, train_cap if limit is None else limit)
+ stale_picked: dict[int, RunPlan] = {}
+ stale_skipped = 0
+ stale_cap = score_cap - len(score_picked)
+ if config.train_score_split and stale_cap > 0:
+ stale_picked, stale_skipped = walk(
+ [insight_id for insight_id in stale_order_ids if insight_id not in train_picked],
+ ScoringRun.SCORE,
+ stale_cap,
+ lambda insight_ids: {
+ insight_id: _as_score_step(plan) for insight_id, plan in plans_for(insight_ids).items()
+ },
+ )
+ score_picked.update(stale_picked)
+
+ stale_ready = set(stale_order_ids)
+ backlog = [insight_id for insight_id in due_by_id if insight_id not in train_picked]
picks = [
InsightPick(
team_id=team_ids[insight_id],
@@ -523,38 +565,59 @@ def select_insights(
]
return Selection(
picks=picks,
- candidates=len({*score_order_ids, *train_order}),
- skipped_for_budget=score_skipped + train_skipped,
+ candidates=len({*score_order_ids, *stale_order_ids, *train_order}),
+ skipped_for_budget=score_skipped + train_skipped + stale_skipped,
picked_score=len(score_picked),
picked_train=len(train_picked),
+ scored_stale_model=len(stale_picked),
+ train_backlog_soft=sum(insight_id in stale_ready for insight_id in backlog),
+ train_backlog_hard=sum(insight_id not in stale_ready for insight_id in backlog),
)
+def _as_score_step(plan: RunPlan) -> RunPlan:
+ """A stale model plan as a score step. A plan that needs a train step stays one, and the score pass passes over it."""
+ return dataclasses.replace(plan, run=ScoringRun.SCORE) if plan.stale_model else plan
+
+
def _score_order(
watched_by_id: dict[int, WatchedInsight],
teams: dict[int, Team],
config: AnomalyScoringConfig,
now: datetime.datetime,
-) -> tuple[list[int], set[int]]:
- """The score pass order, and every watched insight with a score-ready state, due or not.
+) -> ScorePassOrder:
+ """The score pass orders, longest overdue first in each project.
- The order holds the insights with a due bucket after their state, longest overdue first in each project.
+ A stale model is in `stale_order` only, because the train pass gets it first. `ready` leaves out the
+ stale models, so they reach the train pass.
"""
team_orders: list[list[int]] = []
+ stale_orders: list[list[int]] = []
ready: set[int] = set()
for team_id in sorted({insight.team_id for insight in watched_by_id.values()}):
team = teams[team_id]
team_insights = [insight for insight in watched_by_id.values() if insight.team_id == team_id]
- scored = scored_through_by_insight(team, [insight.insight_id for insight in team_insights], config, now)
- ready.update(scored)
+ scored = score_ready_by_insight(team, [insight.insight_id for insight in team_insights], config, now)
+ ready.update(insight_id for insight_id, state in scored.items() if not state.stale_model)
candidates = [
- ScoreCandidate(
- insight_id=insight.insight_id,
- interval=insight.interval,
- scored_through=scored[insight.insight_id].astimezone(team.timezone_info),
+ (
+ scored[insight.insight_id].stale_model,
+ ScoreCandidate(
+ insight_id=insight.insight_id,
+ interval=insight.interval,
+ scored_through=scored[insight.insight_id].scored_through.astimezone(team.timezone_info),
+ ),
)
for insight in team_insights
- if insight.insight_id in scored and is_due(insight, scored[insight.insight_id], team.timezone_info, now)
+ if insight.insight_id in scored
+ and is_due(insight, scored[insight.insight_id].scored_through, team.timezone_info, now)
]
- team_orders.append(score_order(candidates, now=now, max_staleness=config.limits.max_backfill_buckets))
- return _take_turns(team_orders), ready
+ for stale, orders in ((False, team_orders), (True, stale_orders)):
+ orders.append(
+ score_order(
+ [candidate for is_stale, candidate in candidates if is_stale == stale],
+ now=now,
+ max_staleness=config.limits.max_backfill_buckets,
+ )
+ )
+ return ScorePassOrder(order=_take_turns(team_orders), stale_order=_take_turns(stale_orders), ready=ready)
diff --git a/products/signals/backend/anomaly_scores/selection.py b/products/signals/backend/anomaly_scores/selection.py
index f16f2159eaa2..2b73ee15ea88 100644
--- a/products/signals/backend/anomaly_scores/selection.py
+++ b/products/signals/backend/anomaly_scores/selection.py
@@ -117,7 +117,7 @@ def score_order(candidates: Sequence[ScoreCandidate], *, now: datetime, max_stal
the time since the oldest due bucket ended, so every interval waits about as long when the pick cap binds.
A count of due buckets would put hourly insights first, and daily, weekly and monthly insights would wait
that many days, weeks and months. An insight close to ``max_staleness`` due buckets goes first of all,
- because one more due bucket makes its next visit a catch-up train step.
+ because past it the oldest due buckets are never scored, and the insight waits behind the train pass.
"""
near_cap = max_staleness - max(1, max_staleness // 5)
keyed: list[tuple[bool, datetime, int]] = []
diff --git a/products/signals/backend/anomaly_scores/state.py b/products/signals/backend/anomaly_scores/state.py
index 75e127d1ad78..f9d268168f71 100644
--- a/products/signals/backend/anomaly_scores/state.py
+++ b/products/signals/backend/anomaly_scores/state.py
@@ -8,6 +8,7 @@ one blob. A score step fetches only the newest buckets and scores them with the
import datetime
import dataclasses
from collections.abc import Collection
+from enum import StrEnum
from zoneinfo import ZoneInfo
import structlog
@@ -42,6 +43,17 @@ from products.signals.backend.models import AnomalyScoringState
logger = structlog.get_logger(__name__)
+class TrainDue(StrEnum):
+ """Whether a stored model must be replaced before the next score step."""
+
+ NO = "no"
+ # The model is past `train_every_buckets`, or more buckets are due than one visit scores. It still scores,
+ # so the score pass keeps the insight current while the train pass has no budget for a refresh.
+ SOFT = "soft"
+ # No usable model: none stored, a changed identity, too few series, or past `max_model_age_buckets`.
+ HARD = "hard"
+
+
@frozen
class RunPlan:
"""What one pick does and the ClickHouse queries it sends."""
@@ -49,9 +61,21 @@ class RunPlan:
queries: int
# None when the split is off.
run: ScoringRun | None = None
+ # A train step whose model can still score. The score pass changes it to a score step when the train pass
+ # does not pick it.
+ stale_model: bool = False
+
+@frozen
+class ScoreReady:
+ """The stored state of an insight that a score step can read."""
+
+ scored_through: datetime.datetime
+ # A soft retrain is due. The train pass can pick the insight first.
+ stale_model: bool = False
-def _schedule_needs_training(
+
+def _schedule_train_due(
interval: IntervalType,
*,
trained_at: datetime.datetime,
@@ -61,12 +85,18 @@ def _schedule_needs_training(
now: datetime.datetime,
timezone: ZoneInfo,
train_every: int,
+ max_model_age: int,
max_backfill: int,
-) -> bool:
+) -> TrainDue:
"""The retrain triggers that a state row answers alone, without the insight's query."""
- if trained_at + interval_spec(interval).period * train_every <= now:
- return True
- # More due buckets than one visit scores. A train step scores the newest of them, as a first visit does.
+ if missing_series * 2 > series_count:
+ return TrainDue.HARD
+ period = interval_spec(interval).period
+ if trained_at + period * max_model_age <= now:
+ return TrainDue.HARD
+ if trained_at + period * train_every <= now:
+ return TrainDue.SOFT
+ # More due buckets than one visit scores. A score step scores the newest of them, as a first visit does.
due = buckets_to_score(
interval,
now=now,
@@ -74,20 +104,19 @@ def _schedule_needs_training(
timezone=timezone,
limits=dataclasses.replace(DEFAULT_LIMITS, max_backfill_buckets=max_backfill + 1),
)
- if due > max_backfill:
- return True
- return missing_series * 2 > series_count
+ return TrainDue.SOFT if due > max_backfill else TrainDue.NO
-def needs_training(
+def train_due(
state: TrainedState | None,
*,
identity: StateIdentity,
now: datetime.datetime,
timezone: ZoneInfo,
train_every: int,
+ max_model_age: int,
max_backfill: int,
-) -> bool:
+) -> TrainDue:
if (
state is None
or state.identity != identity
@@ -95,8 +124,8 @@ def needs_training(
or not state.artifact_key
or state.scored_through is None
):
- return True
- return _schedule_needs_training(
+ return TrainDue.HARD
+ return _schedule_train_due(
identity.interval,
trained_at=state.trained_at,
scored_through=state.scored_through,
@@ -105,6 +134,7 @@ def needs_training(
now=now,
timezone=timezone,
train_every=train_every,
+ max_model_age=max_model_age,
max_backfill=max_backfill,
)
@@ -124,15 +154,20 @@ def plan_run(
except Exception:
# The scorer fails on the same query and reports the error. A train step charges the larger budget.
return RunPlan(queries=queries, run=ScoringRun.TRAIN)
- trains = not supports_score_step(query) or needs_training(
+ if not supports_score_step(query):
+ return RunPlan(queries=queries, run=ScoringRun.TRAIN)
+ due = train_due(
state,
identity=identity,
now=now,
timezone=team.timezone_info,
train_every=config.train_every_buckets[interval],
+ max_model_age=config.max_model_age_buckets[interval],
max_backfill=config.limits.max_backfill_buckets,
)
- return RunPlan(queries=queries, run=ScoringRun.TRAIN if trains else ScoringRun.SCORE)
+ if due == TrainDue.NO:
+ return RunPlan(queries=queries, run=ScoringRun.SCORE)
+ return RunPlan(queries=queries, run=ScoringRun.TRAIN, stale_model=due == TrainDue.SOFT)
def load_states(team: Team, insights: Collection[Insight]) -> dict[int, TrainedState]:
@@ -144,14 +179,15 @@ def load_states(team: Team, insights: Collection[Insight]) -> dict[int, TrainedS
return {row.insight_id: _from_row(row) for row in rows if profiles.get(row.insight_id) == row.profile}
-def scored_through_by_insight(
+def score_ready_by_insight(
team: Team, insight_ids: Collection[int], config: AnomalyScoringConfig, now: datetime.datetime
-) -> dict[int, datetime.datetime]:
+) -> dict[int, ScoreReady]:
"""Insight id to the newest scored bucket of its stored state, for the rows that a score step can read.
- A row whose retrain is due by age, by due buckets or by missing series is left out, so its insight goes to
- the train pass without a plan. The identity and the profile are not checked here, because they need the
- insight's query. `plan_run` checks them, and the score pass hands an insight that fails them to the train pass.
+ A row with a hard retrain trigger is left out, so its insight goes to the train pass without a plan. A row
+ with a soft trigger stays, and is marked stale. The identity and the profile are not checked here, because
+ they need the insight's query. `plan_run` checks them, and the score pass hands an insight that fails them
+ to the train pass.
"""
if not insight_ids:
return {}
@@ -162,12 +198,12 @@ def scored_through_by_insight(
.exclude(series_keys=[])
.values_list("insight_id", "interval", "trained_at", "scored_through", "missing_series", "series_keys")
)
- scored: dict[int, datetime.datetime] = {}
+ ready: dict[int, ScoreReady] = {}
scorable = {interval.value: interval for interval in config.train_every_buckets}
for insight_id, interval, trained_at, scored_through, missing_series, series_keys in rows:
if scored_through is None or interval not in scorable:
continue
- if _schedule_needs_training(
+ due = _schedule_train_due(
scorable[interval],
trained_at=trained_at,
scored_through=scored_through,
@@ -176,11 +212,15 @@ def scored_through_by_insight(
now=now,
timezone=team.timezone_info,
train_every=config.train_every_buckets[scorable[interval]],
+ max_model_age=config.max_model_age_buckets[scorable[interval]],
max_backfill=config.limits.max_backfill_buckets,
- ):
+ )
+ if due == TrainDue.HARD:
continue
- scored[insight_id] = max(scored_through, scored.get(insight_id, scored_through))
- return scored
+ previous = ready.get(insight_id)
+ if previous is None or scored_through > previous.scored_through:
+ ready[insight_id] = ScoreReady(scored_through=scored_through, stale_model=due == TrainDue.SOFT)
+ return ready
def load_state(team: Team, insight: Insight) -> TrainedState | None:
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index 2b241c772883..dd9be1d251f8 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -113,6 +113,9 @@ class AnomalyScoringTickPlan:
skipped_for_budget: int = 0
picked_score: int = 0
picked_train: int = 0
+ scored_stale_model: int = 0
+ train_backlog_soft: int = 0
+ train_backlog_hard: int = 0
@frozen
@@ -154,6 +157,10 @@ class AnomalyScoringTickResult:
# Planned picks of the score pass and of the train pass, started or not.
picked_score: int = 0
picked_train: int = 0
+ # Planned score pass picks with a model due for a soft retrain, and the due train candidates left unpicked.
+ scored_stale_model: int = 0
+ train_backlog_soft: int = 0
+ train_backlog_hard: int = 0
def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:
@@ -173,6 +180,9 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
skipped_for_budget=selection.skipped_for_budget,
picked_score=selection.picked_score,
picked_train=selection.picked_train,
+ scored_stale_model=selection.scored_stale_model,
+ train_backlog_soft=selection.train_backlog_soft,
+ train_backlog_hard=selection.train_backlog_hard,
)
@@ -329,6 +339,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
run_score=result.run_score,
picked_score=result.picked_score,
picked_train=result.picked_train,
+ scored_stale_model=result.scored_stale_model,
+ train_backlog_soft=result.train_backlog_soft,
+ train_backlog_hard=result.train_backlog_hard,
)
ph_background_capture()(
distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,
@@ -349,6 +362,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
"run_score": result.run_score,
"picked_score": result.picked_score,
"picked_train": result.picked_train,
+ "scored_stale_model": result.scored_stale_model,
+ "train_backlog_soft": result.train_backlog_soft,
+ "train_backlog_hard": result.train_backlog_hard,
},
)
@@ -441,6 +457,9 @@ class AnomalyScoringTickWorkflow:
run_score=sum(pick.run == ScoringRun.SCORE for pick in started_picks),
picked_score=plan.picked_score,
picked_train=plan.picked_train,
+ scored_stale_model=plan.scored_stale_model,
+ train_backlog_soft=plan.train_backlog_soft,
+ train_backlog_hard=plan.train_backlog_hard,
)
await workflow.execute_activity(
capture_anomaly_scoring_tick_activity,
diff --git a/products/signals/backend/test/test_anomaly_scores.py b/products/signals/backend/test/test_anomaly_scores.py
index c66841b2f8e5..4d292d24550f 100644
--- a/products/signals/backend/test/test_anomaly_scores.py
+++ b/products/signals/backend/test/test_anomaly_scores.py
@@ -47,7 +47,7 @@ from products.signals.backend.anomaly_scores.scoring import (
score_series,
trends_query_for,
)
-from products.signals.backend.anomaly_scores.state import needs_training
+from products.signals.backend.anomaly_scores.state import TrainDue, train_due
from products.signals.backend.anomaly_scores.tick import _INSIGHT_TIMEOUT
NEW_YORK = ZoneInfo("America/New_York")
@@ -436,46 +436,48 @@ def _state(**changes: Any) -> TrainedState:
return TrainedState(**fields)
-class TestNeedsTraining(SimpleTestCase):
+class TestTrainDue(SimpleTestCase):
@parameterized.expand(
[
- ("fresh_state", _state(), False),
- ("one_of_two_series_missing", _state(missing_series=1), False),
- ("no_state", None, True),
- ("no_artifact", _state(artifact_key=""), True),
+ ("fresh_state", _state(), TrainDue.NO),
+ ("one_of_two_series_missing", _state(missing_series=1), TrainDue.NO),
+ ("no_state", None, TrainDue.HARD),
+ ("no_artifact", _state(artifact_key=""), TrainDue.HARD),
(
"query_changed",
_state(identity=StateIdentity(interval=IntervalType.DAY, detector_version="v1", query_hash="q0")),
- True,
+ TrainDue.HARD,
),
(
"detector_changed",
_state(identity=StateIdentity(interval=IntervalType.DAY, detector_version="v0", query_hash="q1")),
- True,
+ TrainDue.HARD,
),
- ("trained_too_long_ago", _state(trained_at=datetime(2026, 1, 2, tzinfo=UTC)), True),
- ("gap_over_the_backfill_cap", _state(scored_through=datetime(2025, 12, 1, tzinfo=UTC)), True),
- ("most_series_missing", _state(missing_series=2), True),
+ ("trained_too_long_ago", _state(trained_at=datetime(2026, 1, 2, tzinfo=UTC)), TrainDue.SOFT),
+ ("past_the_model_age_ceiling", _state(trained_at=datetime(2025, 12, 13, tzinfo=UTC)), TrainDue.HARD),
+ ("gap_over_the_backfill_cap", _state(scored_through=datetime(2025, 12, 1, tzinfo=UTC)), TrainDue.SOFT),
+ ("most_series_missing", _state(missing_series=2), TrainDue.HARD),
]
)
- def test_a_visit_trains_when_the_stored_state_cannot_score_it(
- self, _name: str, state: TrainedState | None, trains: bool
+ def test_only_a_missing_or_untrusted_model_stops_score_steps(
+ self, _name: str, state: TrainedState | None, expected: TrainDue
) -> None:
assert (
- needs_training(
+ train_due(
state,
identity=_IDENTITY,
now=datetime(2026, 1, 10, 1, tzinfo=UTC),
timezone=ZoneInfo("UTC"),
train_every=7,
+ max_model_age=28,
max_backfill=30,
)
- is trains
+ == expected
)
- @parameterized.expand([("kept_current", 23, False), ("retrain_due", 24, True)])
- def test_an_hourly_insight_kept_current_trains_only_after_train_every(
- self, _name: str, hours_since_trained: int, trains: bool
+ @parameterized.expand([("kept_current", 23, TrainDue.NO), ("retrain_due", 24, TrainDue.SOFT)])
+ def test_an_hourly_insight_kept_current_is_due_only_after_train_every(
+ self, _name: str, hours_since_trained: int, expected: TrainDue
) -> None:
identity = StateIdentity(interval=IntervalType.HOUR, detector_version="v1", query_hash="q1")
now = datetime(2026, 1, 10, 12, 5, tzinfo=UTC)
@@ -486,8 +488,16 @@ class TestNeedsTraining(SimpleTestCase):
)
assert (
- needs_training(state, identity=identity, now=now, timezone=ZoneInfo("UTC"), train_every=24, max_backfill=30)
- is trains
+ train_due(
+ state,
+ identity=identity,
+ now=now,
+ timezone=ZoneInfo("UTC"),
+ train_every=24,
+ max_model_age=96,
+ max_backfill=30,
+ )
+ == expected
)
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index a42438788036..6abd913e8e4b 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -607,6 +607,8 @@ class TestTrainScoreSplit(BaseTest):
)
for event in scored
] == [("score", "2026-10-20T00:00:00+00:00", 0), ("score", "2026-10-21T00:00:00+00:00", 1)]
+ # A score step names the train time of its model, so a reader can tell a stale-model score apart.
+ assert {event["properties"]["$anomaly_trained_at"] for event in scored} == {trained_at.isoformat()}
model_ids = {event["properties"]["$anomaly_model_id"] for event in [*trained, *scored]}
assert model_ids == {first_key.rsplit("/", 1)[-1]}
state = AnomalyScoringState.objects.for_team(self.team.id).get(insight_id=insight.id)
@@ -969,6 +971,79 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
names = {insight.id: name for name, insight in insights.items()}
assert [(names[pick.insight_id], pick.run) for pick in selection.picks] == expected
+ @parameterized.expand(
+ [
+ (
+ "train_budget_spent",
+ True,
+ [("past_train_every", ScoringRun.SCORE)],
+ (1, 1, 1),
+ ),
+ (
+ "train_budget_left",
+ False,
+ [("past_age_ceiling", ScoringRun.TRAIN), ("past_train_every", ScoringRun.TRAIN)],
+ (0, 0, 0),
+ ),
+ ]
+ )
+ @time_machine.travel(NOW, tick=False)
+ def test_a_model_past_train_every_keeps_scoring_until_the_train_pass_has_budget_for_it(
+ self,
+ _name: str,
+ train_budget_spent: bool,
+ expected: list[tuple[str, ScoringRun]],
+ stale_and_backlog: tuple[int, int, int],
+ ) -> None:
+ self.team.timezone = "UTC"
+ self.team.save()
+ dashboard = Dashboard.objects.create(team=self.team, name="Main")
+ insights = {
+ name: Insight.objects.create(team=self.team, name=name, query=_trends(interval="day"))
+ for name in ("past_train_every", "past_age_ceiling")
+ }
+ for insight in insights.values():
+ DashboardTile.objects.create(dashboard=dashboard, insight=insight)
+ cache.delete(_cooldown_key(insight.id))
+ config = parse_config(
+ {**_SPLIT_PAYLOAD, "organization_ids": [str(self.organization.id)], "allow_team_ids": [self.team.id]}
+ )
+ today = NOW.replace(hour=0)
+ for name, trained_days_ago in (("past_train_every", 10), ("past_age_ceiling", 30)):
+ insight = insights[name]
+ save_state(
+ self.team,
+ insight.id,
+ DEFAULT_PROFILE.name,
+ TrainedState(
+ identity=state_identity(
+ trends_query_for(insight, DEFAULT_PROFILE),
+ interval_scoring(DEFAULT_PROFILE, IntervalType.DAY),
+ self.team,
+ config.limits,
+ ),
+ trained_at=NOW - timedelta(days=trained_days_ago),
+ scored_through=today - timedelta(days=2),
+ series_keys=("a",),
+ artifact_key=f"anomaly_models/{insight.id}",
+ ),
+ )
+ for kind in ScoringRun:
+ cache.delete_many([_fleet_budget_key(NOW, kind), _project_budget_key(self.team.id, NOW, kind)])
+ if train_budget_spent:
+ cache.set(_fleet_budget_key(NOW, ScoringRun.TRAIN), config.max_queries_per_hour)
+
+ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):
+ selection = select_insights(config, SelectionConfig(weighting=Weighting.UNIFORM), NOW)
+
+ names = {insight.id: name for name, insight in insights.items()}
+ assert sorted((names[pick.insight_id], pick.run) for pick in selection.picks) == expected
+ assert (
+ selection.scored_stale_model,
+ selection.train_backlog_soft,
+ selection.train_backlog_hard,
+ ) == stale_and_backlog
+
async def _run_tick(
plan: AnomalyScoringTickPlan, statuses: dict[int, str]