Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): run-aware anomaly scoring tick with separate score and train passes (#123265)

posthog[bot]c171655d0815
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.py
index c0e2a74a1622..102daafa728d 100644
--- a/products/signals/backend/anomaly_scores/config.py
+++ b/products/signals/backend/anomaly_scores/config.py
@@ -108,7 +108,8 @@ class AnomalyScoringConfig:
allow_team_ids: tuple[int, ...] = ()
skip_team_ids: tuple[int, ...] = ()
intervals: tuple[IntervalType, ...] = DEFAULT_TICK_INTERVALS
- # Smooths the spend over the hour, so one tick cannot use the whole query budget.
+ # Smooths the spend over the hour, so one tick cannot use the whole query budget. With the split on,
+ # the two passes use the per-pass caps below instead.
max_insights_per_tick: int = 8
max_concurrency: int = 2
# Scoring queries per clock hour. One pick sends one query per series, see `scoring.query_cost`.
@@ -122,7 +123,10 @@ class AnomalyScoringConfig:
)
# Score step queries per clock hour. Train steps use the budgets above, so a cheap score step never waits for them.
max_score_queries_per_hour: int = 600
- max_score_queries_per_project_per_hour: int = 150
+ max_score_queries_per_project_per_hour: int = 500
+ # Per-tick picks of each pass when the split is on. Score steps are cheap, so the score pass gets most slots.
+ max_score_insights_per_tick: int = 20
+ max_train_insights_per_tick: int = 2
DISABLED = AnomalyScoringConfig()
@@ -133,6 +137,8 @@ MAX_QUERIES_PER_HOUR = 2000
MAX_QUERIES_PER_PROJECT_PER_HOUR = 500
MAX_SCORE_QUERIES_PER_HOUR = 5000
MAX_SCORE_QUERIES_PER_PROJECT_PER_HOUR = 1000
+MAX_SCORE_INSIGHTS_PER_TICK = 200
+MAX_TRAIN_INSIGHTS_PER_TICK = 50
def _positive_int(value: Any, default: int) -> int:
@@ -232,6 +238,18 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:
),
MAX_SCORE_QUERIES_PER_PROJECT_PER_HOUR,
),
+ max_score_insights_per_tick=min(
+ _positive_int(payload.get("max_score_insights_per_tick"), defaults.max_score_insights_per_tick),
+ MAX_SCORE_INSIGHTS_PER_TICK,
+ ),
+ # A payload from before the split sets only `max_insights_per_tick`, and keeps it as its train cap.
+ max_train_insights_per_tick=min(
+ _positive_int(
+ payload.get("max_train_insights_per_tick"),
+ _positive_int(payload.get("max_insights_per_tick"), defaults.max_train_insights_per_tick),
+ ),
+ MAX_TRAIN_INSIGHTS_PER_TICK,
+ ),
)
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index af9c5d9e4f90..90cfa257f136 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -4,6 +4,10 @@ One query per team reads the newest scored bucket of each series back from the `
events. `selection.pick_order` then orders the due insights, and the tick picks from that order the
insights whose queries fit the hourly budgets. With the train and score split on, `state.plan_run`
makes each pick a train step or a score step, and each kind has its own budgets.
+
+With the split on, the tick runs two passes, each with its own order and pick cap. The score pass goes
+first: it takes the trained insights from their stored state, longest overdue first, so cheap score steps
+keep them current. The train pass then takes the other due insights in the `pick_order` order.
"""
import datetime
@@ -43,11 +47,13 @@ from products.signals.backend.anomaly_scores.scoring import bucket_end
from products.signals.backend.anomaly_scores.selection import (
Candidate,
InsightUsage,
+ ScoreCandidate,
SelectionConfig,
insight_usage,
pick_order,
+ score_order,
)
-from products.signals.backend.anomaly_scores.state import RunPlan, load_states, plan_run
+from products.signals.backend.anomaly_scores.state import RunPlan, load_states, plan_run, scored_through_by_insight
from ee.billing.quota_limiting import QuotaLimitingCaches, QuotaResource, list_limited_team_attributes
@@ -130,6 +136,9 @@ class Selection:
candidates: int
# Due insights that the walk reached but skipped, because their queries did not fit a budget.
skipped_for_budget: int = 0
+ # Picks of the score pass and of the train pass. With the split off, every pick is a train pass pick.
+ picked_score: int = 0
+ picked_train: int = 0
@frozen
@@ -324,6 +333,7 @@ def within_budget(
"""The insights of ``order`` whose cost fits both budgets of their kind, up to ``limit``, and the skip count.
A pick that does not fit is skipped and the walk continues, so a large insight does not block smaller ones.
+ A pick of a kind with no entry in ``budgets`` belongs to the other pass, and the walk passes over it.
``budgets`` are snapshots that skip most misses without a cache call. ``reserve`` charges a pick for real.
``plans_for`` loads plans in batches of ``limit``, and only for insights that a snapshot can still fit.
"""
@@ -348,9 +358,10 @@ def within_budget(
team_id = team_ids[insight_id]
plan = plans.get(insight_id)
kind = budget_kind(plan.run if plan else None)
+ if plan is not None and kind not in budgets:
+ continue
if (
plan is None
- or kind not in budgets
or plan.queries > fleets[kind]
or plan.queries > projects[kind].get(team_id, 0)
or not reserve(team_id, plan)
@@ -374,9 +385,15 @@ def _run_plans(
return {insight_id: plans.get(insight_id, missing) for insight_id in insight_ids}
+def _take_turns(team_orders: list[list[int]]) -> list[int]:
+ # Projects take turns, so one large project cannot take every slot of the tick budget.
+ return [insight_id for turn in itertools.zip_longest(*team_orders) for insight_id in turn if insight_id is not None]
+
+
def select_insights(
- config: AnomalyScoringConfig, selection_config: SelectionConfig, now: datetime.datetime, limit: int
+ config: AnomalyScoringConfig, selection_config: SelectionConfig, now: datetime.datetime, limit: int | None = None
) -> Selection:
+ """The picks of the tick. ``limit`` replaces the pick cap of each pass, for a manual run."""
teams = {team.id: team for team in eligible_teams(config)}
insights = watched_insights(list(teams), config.intervals, now)
state: dict[int, dict[str, datetime.datetime]] = {}
@@ -405,14 +422,58 @@ def select_insights(
insight.insight_id: max(state[insight.insight_id].values()) if insight.insight_id in state else None
for insight in insights
}
- due = [
- insight
- for insight in insights
- if is_due(insight, newest_buckets[insight.insight_id], teams[insight.team_id].timezone_info, now)
- ]
- cooling = _cooling_down([insight.insight_id for insight in due])
- due_by_id = {insight.insight_id: insight for insight in due if insight.insight_id not in cooling}
+ cooling = _cooling_down([insight.insight_id for insight in insights])
+ watched_by_id = {insight.insight_id: insight for insight in insights if insight.insight_id not in cooling}
+ team_ids = {insight_id: insight.team_id for insight_id, insight in watched_by_id.items()}
+ # Both passes share the plans, so an insight that the score pass planned is not planned again.
+ planned: dict[int, RunPlan] = {}
+
+ def plans_for(insight_ids: list[int]) -> dict[int, RunPlan]:
+ missing = [insight_id for insight_id in insight_ids if insight_id not in planned]
+ for team_id in sorted({team_ids[insight_id] for insight_id in missing}):
+ team_batch = [insight_id for insight_id in missing if team_ids[insight_id] == team_id]
+ planned.update(_run_plans(teams[team_id], team_batch, config, now))
+ return {insight_id: planned[insight_id] for insight_id in insight_ids}
+
+ def walk(order: list[int], kind: ScoringRun, cap: int) -> tuple[dict[int, RunPlan], int]:
+ # Read the budget before any cost, so a spent hour loads no insight.
+ return within_budget(
+ order,
+ team_ids,
+ {kind: remaining_budget(config, {team_ids[insight_id] for insight_id in order}, now, kind)},
+ cap,
+ plans_for,
+ reserve=lambda team_id, plan: reserve_queries(config, team_id, plan.queries, now, budget_kind(plan.run)),
+ )
+
+ score_order_ids: list[int] = []
+ score_ready: set[int] = set()
+ score_picked: dict[int, RunPlan] = {}
+ score_skipped = 0
+ if config.train_score_split:
+ score_order_ids, score_ready = _score_order(watched_by_id, teams, config, now)
+ score_cap = config.max_score_insights_per_tick if limit is None else limit
+ score_picked, score_skipped = walk(score_order_ids, ScoringRun.SCORE, score_cap)
+ # A spent score budget makes the walk load no plan. Plan the head of the order anyway, so that an
+ # edited insight there still reaches the train pass while the train budget has room.
+ plans_for(score_order_ids[:score_cap])
+ # A score-ready insight belongs to the score pass, also when its cap or budget left it out this tick.
+ # Without this, the train pass loads a plan for every one of them and then passes over it. An insight
+ # that the score pass planned as a train step, for example after an edit, goes to the train pass.
+ score_only = {
+ insight_id
+ for insight_id in score_ready
+ if insight_id not in planned or planned[insight_id].run == ScoringRun.SCORE
+ }
+
+ due_by_id = {
+ insight_id: insight
+ for insight_id, insight in watched_by_id.items()
+ if insight_id not in score_picked
+ and insight_id not in score_only
+ and is_due(insight, newest_buckets[insight_id], teams[insight.team_id].timezone_info, now)
+ }
usage: dict[int, InsightUsage] = {}
for team_id in sorted({insight.team_id for insight in due_by_id.values()}):
due_ids = [insight.insight_id for insight in due_by_id.values() if insight.team_id == team_id]
@@ -431,47 +492,69 @@ def select_insights(
usage=usage[insight.insight_id],
)
)
- team_orders = [
- pick_order(
- candidates_by_team[team_id],
- config=selection_config,
- tick_start=now,
- max_staleness=config.limits.max_backfill_buckets,
- )
- for team_id in sorted(candidates_by_team)
- ]
- # Projects take turns, so one large project cannot take every slot of the tick budget.
- order = [
- insight_id for turn in itertools.zip_longest(*team_orders) for insight_id in turn if insight_id is not None
- ]
- team_ids = {insight_id: due_by_id[insight_id].team_id for insight_id in order}
-
- def plans_for(insight_ids: list[int]) -> dict[int, RunPlan]:
- plans: dict[int, RunPlan] = {}
- for team_id in sorted({team_ids[insight_id] for insight_id in insight_ids}):
- team_batch = [insight_id for insight_id in insight_ids if team_ids[insight_id] == team_id]
- plans.update(_run_plans(teams[team_id], team_batch, config, now))
- return plans
-
- kinds = [ScoringRun.TRAIN, ScoringRun.SCORE] if config.train_score_split else [ScoringRun.TRAIN]
- # Read the budgets before any cost, so a spent hour loads no insight.
- picked, skipped = within_budget(
- order,
- team_ids,
- {kind: remaining_budget(config, set(team_ids.values()), now, kind) for kind in kinds},
- limit,
- plans_for,
- reserve=lambda team_id, plan: reserve_queries(config, team_id, plan.queries, now, budget_kind(plan.run)),
+ train_order = _take_turns(
+ [
+ pick_order(
+ candidates_by_team[team_id],
+ config=selection_config,
+ tick_start=now,
+ max_staleness=config.limits.max_backfill_buckets,
+ )
+ for team_id in sorted(candidates_by_team)
+ ]
)
+ if config.train_score_split:
+ train_cap = config.max_train_insights_per_tick
+ else:
+ train_cap = config.max_insights_per_tick
+ # With the split off, every pick charges the train budgets.
+ train_picked, train_skipped = walk(train_order, ScoringRun.TRAIN, train_cap if limit is None else limit)
+
picks = [
InsightPick(
team_id=team_ids[insight_id],
insight_id=insight_id,
scored_through=state.get(insight_id, {}),
- interval=due_by_id[insight_id].interval,
+ interval=watched_by_id[insight_id].interval,
query_cost=plan.queries,
run=plan.run,
)
- for insight_id, plan in picked.items()
+ for insight_id, plan in [*score_picked.items(), *train_picked.items()]
]
- return Selection(picks=picks, candidates=len(order), skipped_for_budget=skipped)
+ return Selection(
+ picks=picks,
+ candidates=len({*score_order_ids, *train_order}),
+ skipped_for_budget=score_skipped + train_skipped,
+ picked_score=len(score_picked),
+ picked_train=len(train_picked),
+ )
+
+
+def _score_order(
+ watched_by_id: dict[int, WatchedInsight],
+ teams: dict[int, Team],
+ config: AnomalyScoringConfig,
+ now: datetime.datetime,
+) -> tuple[list[int], set[int]]:
+ """The score pass order, and every watched insight with a score-ready state, due or not.
+
+ The order holds the insights with a due bucket after their state, longest overdue first in each project.
+ """
+ team_orders: list[list[int]] = []
+ ready: set[int] = set()
+ for team_id in sorted({insight.team_id for insight in watched_by_id.values()}):
+ team = teams[team_id]
+ team_insights = [insight for insight in watched_by_id.values() if insight.team_id == team_id]
+ scored = scored_through_by_insight(team, [insight.insight_id for insight in team_insights], config, now)
+ ready.update(scored)
+ candidates = [
+ ScoreCandidate(
+ insight_id=insight.insight_id,
+ interval=insight.interval,
+ scored_through=scored[insight.insight_id].astimezone(team.timezone_info),
+ )
+ for insight in team_insights
+ if insight.insight_id in scored and is_due(insight, scored[insight.insight_id], team.timezone_info, now)
+ ]
+ team_orders.append(score_order(candidates, now=now, max_staleness=config.limits.max_backfill_buckets))
+ return _take_turns(team_orders), ready
diff --git a/products/signals/backend/anomaly_scores/selection.py b/products/signals/backend/anomaly_scores/selection.py
index 4f87dce53963..f16f2159eaa2 100644
--- a/products/signals/backend/anomaly_scores/selection.py
+++ b/products/signals/backend/anomaly_scores/selection.py
@@ -1,5 +1,7 @@
-"""The order in which a scoring tick picks insights, set by the `anomaly-scoring` flag payload.
+"""The order in which a scoring tick picks insights.
+The score pass takes the trained insights longest overdue first, see `score_order`. The train pass, and the
+whole tick when the train and score split is off, uses `pick_order`, set by the `anomaly-scoring` flag payload.
`weighted` samples insights in proportion to usage × staleness, so most of the per-tick query budget goes to
insights people look at. Weight grows while an insight waits, so an insight nobody views is picked rarely but
never starves. `uniform` picks the stalest insight first, so every insight gets the same share.
@@ -51,6 +53,14 @@ class Candidate:
usage: InsightUsage
+@frozen
+class ScoreCandidate:
+ insight_id: int
+ interval: IntervalType
+ # The newest scored bucket start of the stored state, in the project timezone.
+ scored_through: datetime
+
+
def _non_negative(value: Any, default: float) -> float:
# `bool` is an `int` subclass, and `true` is not a number.
if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value) or value < 0:
@@ -88,14 +98,38 @@ def staleness(candidate: Candidate, *, now: datetime, max_staleness: int) -> int
"""
if candidate.newest_scored_bucket is None:
return max_staleness
+ return _completed_buckets_after(candidate.newest_scored_bucket, candidate.interval, now, max_staleness)
+
+
+def _completed_buckets_after(newest: datetime, interval: IntervalType, now: datetime, max_count: int) -> int:
count = 0
- start = bucket_end(candidate.newest_scored_bucket, candidate.interval)
- while count < max_staleness and bucket_end(start, candidate.interval) <= now:
+ start = bucket_end(newest, interval)
+ while count < max_count and bucket_end(start, interval) <= now:
count += 1
- start = bucket_end(start, candidate.interval)
+ start = bucket_end(start, interval)
return count
+def score_order(candidates: Sequence[ScoreCandidate], *, now: datetime, max_staleness: int) -> list[int]:
+ """Insight ids with a due bucket, the longest overdue first. Usage has no effect.
+
+ A score step is cheap, so the pass keeps every trained insight current instead of sampling. Overdue is
+ the time since the oldest due bucket ended, so every interval waits about as long when the pick cap binds.
+ A count of due buckets would put hourly insights first, and daily, weekly and monthly insights would wait
+ that many days, weeks and months. An insight close to ``max_staleness`` due buckets goes first of all,
+ because one more due bucket makes its next visit a catch-up train step.
+ """
+ near_cap = max_staleness - max(1, max_staleness // 5)
+ keyed: list[tuple[bool, datetime, int]] = []
+ for candidate in candidates:
+ due = _completed_buckets_after(candidate.scored_through, candidate.interval, now, max_staleness)
+ if due == 0:
+ continue
+ oldest_due_end = bucket_end(bucket_end(candidate.scored_through, candidate.interval), candidate.interval)
+ keyed.append((due < near_cap, oldest_due_end, candidate.insight_id))
+ return [insight_id for _, _, insight_id in sorted(keyed)]
+
+
def weight(candidate: Candidate, *, config: SelectionConfig, now: datetime, max_staleness: int) -> float:
usage = config.usage_floor + candidate.usage.viewers
if candidate.usage.has_enabled_alert:
diff --git a/products/signals/backend/anomaly_scores/state.py b/products/signals/backend/anomaly_scores/state.py
index c46f117169f7..75e127d1ad78 100644
--- a/products/signals/backend/anomaly_scores/state.py
+++ b/products/signals/backend/anomaly_scores/state.py
@@ -51,6 +51,34 @@ class RunPlan:
run: ScoringRun | None = None
+def _schedule_needs_training(
+ interval: IntervalType,
+ *,
+ trained_at: datetime.datetime,
+ scored_through: datetime.datetime,
+ missing_series: int,
+ series_count: int,
+ now: datetime.datetime,
+ timezone: ZoneInfo,
+ train_every: int,
+ max_backfill: int,
+) -> bool:
+ """The retrain triggers that a state row answers alone, without the insight's query."""
+ if trained_at + interval_spec(interval).period * train_every <= now:
+ return True
+ # More due buckets than one visit scores. A train step scores the newest of them, as a first visit does.
+ due = buckets_to_score(
+ interval,
+ now=now,
+ after=scored_through,
+ timezone=timezone,
+ limits=dataclasses.replace(DEFAULT_LIMITS, max_backfill_buckets=max_backfill + 1),
+ )
+ if due > max_backfill:
+ return True
+ return missing_series * 2 > series_count
+
+
def needs_training(
state: TrainedState | None,
*,
@@ -68,20 +96,17 @@ def needs_training(
or state.scored_through is None
):
return True
- interval = identity.interval
- if state.trained_at + interval_spec(interval).period * train_every <= now:
- return True
- # More due buckets than one visit scores. A train step scores the newest of them, as a first visit does.
- due = buckets_to_score(
- interval,
+ return _schedule_needs_training(
+ identity.interval,
+ trained_at=state.trained_at,
+ scored_through=state.scored_through,
+ missing_series=state.missing_series,
+ series_count=len(state.series_keys),
now=now,
- after=state.scored_through,
timezone=timezone,
- limits=dataclasses.replace(DEFAULT_LIMITS, max_backfill_buckets=max_backfill + 1),
+ train_every=train_every,
+ max_backfill=max_backfill,
)
- if due > max_backfill:
- return True
- return state.missing_series * 2 > len(state.series_keys)
def plan_run(
@@ -119,6 +144,45 @@ def load_states(team: Team, insights: Collection[Insight]) -> dict[int, TrainedS
return {row.insight_id: _from_row(row) for row in rows if profiles.get(row.insight_id) == row.profile}
+def scored_through_by_insight(
+ team: Team, insight_ids: Collection[int], config: AnomalyScoringConfig, now: datetime.datetime
+) -> dict[int, datetime.datetime]:
+ """Insight id to the newest scored bucket of its stored state, for the rows that a score step can read.
+
+ A row whose retrain is due by age, by due buckets or by missing series is left out, so its insight goes to
+ the train pass without a plan. The identity and the profile are not checked here, because they need the
+ insight's query. `plan_run` checks them, and the score pass hands an insight that fails them to the train pass.
+ """
+ if not insight_ids:
+ return {}
+ rows = (
+ AnomalyScoringState.objects.for_team(team.id)
+ .filter(insight_id__in=list(insight_ids), scored_through__isnull=False)
+ .exclude(artifact_key="")
+ .exclude(series_keys=[])
+ .values_list("insight_id", "interval", "trained_at", "scored_through", "missing_series", "series_keys")
+ )
+ scored: dict[int, datetime.datetime] = {}
+ scorable = {interval.value: interval for interval in config.train_every_buckets}
+ for insight_id, interval, trained_at, scored_through, missing_series, series_keys in rows:
+ if scored_through is None or interval not in scorable:
+ continue
+ if _schedule_needs_training(
+ scorable[interval],
+ trained_at=trained_at,
+ scored_through=scored_through,
+ missing_series=missing_series,
+ series_count=len(series_keys),
+ now=now,
+ timezone=team.timezone_info,
+ train_every=config.train_every_buckets[scorable[interval]],
+ max_backfill=config.limits.max_backfill_buckets,
+ ):
+ continue
+ scored[insight_id] = max(scored_through, scored.get(insight_id, scored_through))
+ return scored
+
+
def load_state(team: Team, insight: Insight) -> TrainedState | None:
return load_states(team, [insight]).get(insight.id)
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index 62079d017cb2..2b241c772883 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -111,6 +111,8 @@ class AnomalyScoringTickPlan:
max_concurrency: int
limits: ScoringLimits = DEFAULT_LIMITS
skipped_for_budget: int = 0
+ picked_score: int = 0
+ picked_train: int = 0
@frozen
@@ -149,6 +151,9 @@ class AnomalyScoringTickResult:
# Started picks per kind of the train and score split.
run_train: int = 0
run_score: int = 0
+ # Planned picks of the score pass and of the train pass, started or not.
+ picked_score: int = 0
+ picked_train: int = 0
def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:
@@ -157,7 +162,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
config = read_config(payload)
if not config.enabled:
return AnomalyScoringTickPlan(enabled=False, picks=[], candidates=0, max_concurrency=config.max_concurrency)
- selection = select_insights(config, parse_selection_config(payload), now, limit or config.max_insights_per_tick)
+ selection = select_insights(config, parse_selection_config(payload), now, limit)
record_candidates(selection.candidates)
return AnomalyScoringTickPlan(
enabled=True,
@@ -166,6 +171,8 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
max_concurrency=config.max_concurrency,
limits=config.limits,
skipped_for_budget=selection.skipped_for_budget,
+ picked_score=selection.picked_score,
+ picked_train=selection.picked_train,
)
@@ -320,6 +327,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
stopped_at_capacity=result.stopped_at_capacity,
run_train=result.run_train,
run_score=result.run_score,
+ picked_score=result.picked_score,
+ picked_train=result.picked_train,
)
ph_background_capture()(
distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,
@@ -338,6 +347,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
"stopped_at_capacity": result.stopped_at_capacity,
"run_train": result.run_train,
"run_score": result.run_score,
+ "picked_score": result.picked_score,
+ "picked_train": result.picked_train,
},
)
@@ -428,6 +439,8 @@ class AnomalyScoringTickWorkflow:
stopped_at_capacity=statuses[STATUS_AT_CAPACITY] > 0,
run_train=sum(pick.run == ScoringRun.TRAIN for pick in started_picks),
run_score=sum(pick.run == ScoringRun.SCORE for pick in started_picks),
+ picked_score=plan.picked_score,
+ picked_train=plan.picked_train,
)
await workflow.execute_activity(
capture_anomaly_scoring_tick_activity,
diff --git a/products/signals/backend/test/test_anomaly_scores.py b/products/signals/backend/test/test_anomaly_scores.py
index 629bff385ec6..c66841b2f8e5 100644
--- a/products/signals/backend/test/test_anomaly_scores.py
+++ b/products/signals/backend/test/test_anomaly_scores.py
@@ -473,6 +473,23 @@ class TestNeedsTraining(SimpleTestCase):
is trains
)
+ @parameterized.expand([("kept_current", 23, False), ("retrain_due", 24, True)])
+ def test_an_hourly_insight_kept_current_trains_only_after_train_every(
+ self, _name: str, hours_since_trained: int, trains: bool
+ ) -> None:
+ identity = StateIdentity(interval=IntervalType.HOUR, detector_version="v1", query_hash="q1")
+ now = datetime(2026, 1, 10, 12, 5, tzinfo=UTC)
+ state = _state(
+ identity=identity,
+ trained_at=now - timedelta(hours=hours_since_trained),
+ scored_through=datetime(2026, 1, 10, 10, tzinfo=UTC),
+ )
+
+ assert (
+ needs_training(state, identity=identity, now=now, timezone=ZoneInfo("UTC"), train_every=24, max_backfill=30)
+ is trains
+ )
+
class TestMetricScoringQuery(SimpleTestCase):
@parameterized.expand([("without_compare", None), ("with_saved_compare", CompareFilter(compare=True))])
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index fee7064f359d..a42438788036 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -32,7 +32,13 @@ from products.dashboards.backend.models.dashboard_tile import DashboardTile
from products.product_analytics.backend.facade.api import record_insight_view
from products.product_analytics.backend.facade.models import Insight
from products.signals.backend.anomaly_scores.artifacts import clear_artifact_cache
-from products.signals.backend.anomaly_scores.config import DISABLED, AnomalyScoringConfig, ScoringRun, parse_config
+from products.signals.backend.anomaly_scores.config import (
+ DEFAULT_PROFILE,
+ DISABLED,
+ AnomalyScoringConfig,
+ ScoringRun,
+ parse_config,
+)
from products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_events
from products.signals.backend.anomaly_scores.queue import (
SKIP_COOLDOWN,
@@ -55,10 +61,15 @@ from products.signals.backend.anomaly_scores.scoring import (
InsightScores,
ScoredPoint,
ScoredSeries,
+ StateIdentity,
+ TrainedState,
UnsupportedInsightError,
+ interval_scoring,
+ state_identity,
+ trends_query_for,
)
from products.signals.backend.anomaly_scores.selection import SelectionConfig, Weighting
-from products.signals.backend.anomaly_scores.state import RunPlan, load_state, plan_run
+from products.signals.backend.anomaly_scores.state import RunPlan, load_state, plan_run, save_state
from products.signals.backend.anomaly_scores.tick import (
AnomalyScoringTickInput,
AnomalyScoringTickPlan,
@@ -154,6 +165,8 @@ class TestTickConfig(SimpleTestCase):
("max_queries_per_project_per_hour", 501, 500),
("max_score_queries_per_hour", 1_000_000, 5000),
("max_score_queries_per_project_per_hour", 1001, 1000),
+ ("max_score_insights_per_tick", 201, 200),
+ ("max_train_insights_per_tick", 51, 50),
]
)
def test_tick_levers_above_their_maximum_parse_to_the_maximum(self, field: str, value: int, expected: int) -> None:
@@ -161,6 +174,26 @@ class TestTickConfig(SimpleTestCase):
assert getattr(config, field) == expected
+ @parameterized.expand(
+ [
+ ("defaults", {}, 20, 2),
+ # A payload from before the split keeps its per-tick cap for train steps.
+ ("only_the_old_cap", {"max_insights_per_tick": 8}, 20, 8),
+ (
+ "per_pass_caps",
+ {"max_insights_per_tick": 8, "max_score_insights_per_tick": 40, "max_train_insights_per_tick": 3},
+ 40,
+ 3,
+ ),
+ ]
+ )
+ def test_each_pass_gets_its_own_cap(
+ self, _name: str, levers: dict[str, int], score_cap: int, train_cap: int
+ ) -> None:
+ config = parse_config({**_SPLIT_PAYLOAD, **levers})
+
+ assert (config.max_score_insights_per_tick, config.max_train_insights_per_tick) == (score_cap, train_cap)
+
class TestQueryBudget(SimpleTestCase):
@parameterized.expand(
@@ -210,6 +243,18 @@ class TestQueryBudget(SimpleTestCase):
0,
QueryBudget(fleet=0, projects={1: 0}),
),
+ # The train pass passes over a score step, and does not count it as a budget skip.
+ (
+ "other_pass_kind_passed_over",
+ {1: 1, 2: 1, 3: 1},
+ QueryBudget(fleet=100, projects={1: 30}),
+ 10,
+ [1, 3],
+ 0,
+ 0,
+ None,
+ ScoringRun.SCORE,
+ ),
]
)
def test_the_walk_skips_picks_over_budget_and_continues(
@@ -222,10 +267,11 @@ class TestQueryBudget(SimpleTestCase):
skipped: int,
refused_reservations: int,
score_budget: QueryBudget | None = None,
+ second_run: ScoringRun | None = None,
) -> None:
plans = {
1: RunPlan(queries=3),
- 2: RunPlan(queries=3, run=ScoringRun.SCORE if score_budget else None),
+ 2: RunPlan(queries=3, run=ScoringRun.SCORE if score_budget else second_run),
3: RunPlan(queries=2),
}
budgets = {ScoringRun.TRAIN: budget}
@@ -803,6 +849,126 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
assert [pick.insight_id for pick in selection.picks] == [insights[self.team.id].id]
+ @parameterized.expand([("cap_one", 1, ["most_overdue"]), ("cap_two", 2, ["most_overdue", "popular"])])
+ @time_machine.travel(NOW, tick=False)
+ def test_the_score_pass_picks_trained_insights_most_overdue_first_with_the_train_budget_spent(
+ self, _name: str, score_cap: int, expected: list[str]
+ ) -> None:
+ self.team.timezone = "UTC"
+ self.team.save()
+ dashboard = Dashboard.objects.create(team=self.team, name="Main")
+ insights = {
+ name: Insight.objects.create(team=self.team, name=name, query=_trends(interval="day"))
+ for name in ("most_overdue", "popular", "untrained")
+ }
+ for insight in insights.values():
+ DashboardTile.objects.create(dashboard=dashboard, insight=insight)
+ cache.delete(_cooldown_key(insight.id))
+ for viewer_index in range(5):
+ viewer = User.objects.create_and_join(self.organization, f"viewer{viewer_index}@example.com", "password")
+ record_insight_view(insight_id=insights["popular"].id, team_id=self.team.id, user_id=viewer.id)
+ config = parse_config(
+ {
+ **_SPLIT_PAYLOAD,
+ "organization_ids": [str(self.organization.id)],
+ "allow_team_ids": [self.team.id],
+ "max_score_insights_per_tick": score_cap,
+ }
+ )
+ today = NOW.replace(hour=0)
+ for name, scored_through in (
+ ("most_overdue", today - timedelta(days=4)),
+ ("popular", today - timedelta(days=2)),
+ ):
+ insight = insights[name]
+ query = trends_query_for(insight, DEFAULT_PROFILE)
+ identity = state_identity(
+ query, interval_scoring(DEFAULT_PROFILE, IntervalType.DAY), self.team, config.limits
+ )
+ save_state(
+ self.team,
+ insight.id,
+ DEFAULT_PROFILE.name,
+ TrainedState(
+ identity=identity,
+ trained_at=NOW - timedelta(days=1),
+ scored_through=scored_through,
+ series_keys=("a",),
+ artifact_key=f"anomaly_models/{insight.id}",
+ ),
+ )
+ for kind in ScoringRun:
+ cache.delete_many([_fleet_budget_key(NOW, kind), _project_budget_key(self.team.id, NOW, kind)])
+ cache.set(_fleet_budget_key(NOW, ScoringRun.TRAIN), config.max_queries_per_hour)
+
+ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):
+ selection = select_insights(config, SelectionConfig(), NOW)
+
+ names = {insight.id: name for name, insight in insights.items()}
+ assert [(names[pick.insight_id], pick.run) for pick in selection.picks] == [
+ (name, ScoringRun.SCORE) for name in expected
+ ]
+ assert (selection.picked_score, selection.picked_train) == (len(expected), 0)
+
+ @parameterized.expand(
+ [
+ ("score_budget_left", False, [("current", ScoringRun.SCORE), ("edited", ScoringRun.TRAIN)]),
+ ("score_budget_spent", True, [("edited", ScoringRun.TRAIN)]),
+ ]
+ )
+ @time_machine.travel(NOW, tick=False)
+ def test_an_edited_insight_with_a_current_state_goes_to_the_train_pass(
+ self, _name: str, score_budget_spent: bool, expected: list[tuple[str, ScoringRun]]
+ ) -> None:
+ self.team.timezone = "UTC"
+ self.team.save()
+ dashboard = Dashboard.objects.create(team=self.team, name="Main")
+ insights = {
+ name: Insight.objects.create(team=self.team, name=name, query=_trends(interval="day"))
+ for name in ("edited", "current")
+ }
+ for insight in insights.values():
+ DashboardTile.objects.create(dashboard=dashboard, insight=insight)
+ cache.delete(_cooldown_key(insight.id))
+ config = parse_config(
+ {**_SPLIT_PAYLOAD, "organization_ids": [str(self.organization.id)], "allow_team_ids": [self.team.id]}
+ )
+ today = NOW.replace(hour=0)
+ for name, scored_through in (("edited", today - timedelta(days=4)), ("current", today - timedelta(days=2))):
+ insight = insights[name]
+ identity = state_identity(
+ trends_query_for(insight, DEFAULT_PROFILE),
+ interval_scoring(DEFAULT_PROFILE, IntervalType.DAY),
+ self.team,
+ config.limits,
+ )
+ if name == "edited":
+ identity = StateIdentity(
+ interval=identity.interval, detector_version=identity.detector_version, query_hash="before-edit"
+ )
+ save_state(
+ self.team,
+ insight.id,
+ DEFAULT_PROFILE.name,
+ TrainedState(
+ identity=identity,
+ trained_at=NOW - timedelta(days=1),
+ scored_through=scored_through,
+ series_keys=("a",),
+ artifact_key=f"anomaly_models/{insight.id}",
+ ),
+ )
+ for kind in ScoringRun:
+ cache.delete_many([_fleet_budget_key(NOW, kind), _project_budget_key(self.team.id, NOW, kind)])
+ if score_budget_spent:
+ cache.set(_fleet_budget_key(NOW, ScoringRun.SCORE), config.max_score_queries_per_hour)
+
+ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):
+ selection = select_insights(config, SelectionConfig(), NOW)
+
+ names = {insight.id: name for name, insight in insights.items()}
+ assert [(names[pick.insight_id], pick.run) for pick in selection.picks] == expected
+
async def _run_tick(
plan: AnomalyScoringTickPlan, statuses: dict[int, str]
diff --git a/products/signals/backend/test/test_anomaly_selection.py b/products/signals/backend/test/test_anomaly_selection.py
index 15cdd89711cf..84ea8b3e72e0 100644
--- a/products/signals/backend/test/test_anomaly_selection.py
+++ b/products/signals/backend/test/test_anomaly_selection.py
@@ -11,10 +11,12 @@ from posthog.schema import IntervalType
from products.signals.backend.anomaly_scores.selection import (
Candidate,
InsightUsage,
+ ScoreCandidate,
SelectionConfig,
Weighting,
parse_selection_config,
pick_order,
+ score_order,
staleness,
)
@@ -117,6 +119,42 @@ class TestPickOrder(SimpleTestCase):
assert staleness(candidate, now=datetime(2026, 10, 14, 12, tzinfo=UTC), max_staleness=MAX_STALENESS) == expected
+class TestScoreOrder(SimpleTestCase):
+ @parameterized.expand(
+ [
+ (
+ "longest_overdue_first_whatever_the_interval",
+ [
+ # 1 due bucket, which ended 12 hours ago.
+ ScoreCandidate(
+ insight_id=1, interval=IntervalType.DAY, scored_through=datetime(2026, 10, 13, tzinfo=UTC)
+ ),
+ # 5 due buckets, the oldest of which ended 4 hours ago.
+ ScoreCandidate(insight_id=2, interval=IntervalType.HOUR, scored_through=TICK - timedelta(hours=6)),
+ ScoreCandidate(insight_id=3, interval=IntervalType.HOUR, scored_through=TICK - timedelta(hours=3)),
+ # Nothing completed after it.
+ ScoreCandidate(insight_id=4, interval=IntervalType.HOUR, scored_through=TICK - timedelta(hours=1)),
+ ],
+ [1, 2, 3],
+ ),
+ (
+ "close_to_the_backfill_cap_goes_first",
+ [
+ # 2 due buckets, the oldest of which ended 36 hours ago.
+ ScoreCandidate(
+ insight_id=5, interval=IntervalType.DAY, scored_through=datetime(2026, 10, 12, tzinfo=UTC)
+ ),
+ # 26 due buckets of 30, the oldest of which ended 25 hours ago.
+ ScoreCandidate(insight_id=6, interval=IntervalType.HOUR, scored_through=TICK - timedelta(hours=27)),
+ ],
+ [6, 5],
+ ),
+ ]
+ )
+ def test_order(self, _name: str, candidates: list[ScoreCandidate], expected: list[int]) -> None:
+ assert score_order(candidates, now=TICK, max_staleness=MAX_STALENESS) == expected
+
+
class TestParseSelectionConfig(SimpleTestCase):
@parameterized.expand(
[