:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): run-aware anomaly scoring tick with separate score and train passes (#123265)
posthog[bot]c171655d0815
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.pyindex c0e2a74a1622..102daafa728d 100644--- a/products/signals/backend/anomaly_scores/config.py+++ b/products/signals/backend/anomaly_scores/config.py@@ -108,7 +108,8 @@ class AnomalyScoringConfig:allow_team_ids: tuple[int, ...] = ()skip_team_ids: tuple[int, ...] = ()intervals: tuple[IntervalType, ...] = DEFAULT_TICK_INTERVALS- # Smooths the spend over the hour, so one tick cannot use the whole query budget.+ # Smooths the spend over the hour, so one tick cannot use the whole query budget. With the split on,+ # the two passes use the per-pass caps below instead.max_insights_per_tick: int = 8max_concurrency: int = 2# Scoring queries per clock hour. One pick sends one query per series, see `scoring.query_cost`.@@ -122,7 +123,10 @@ class AnomalyScoringConfig:)# Score step queries per clock hour. Train steps use the budgets above, so a cheap score step never waits for them.max_score_queries_per_hour: int = 600- max_score_queries_per_project_per_hour: int = 150+ max_score_queries_per_project_per_hour: int = 500+ # Per-tick picks of each pass when the split is on. Score steps are cheap, so the score pass gets most slots.+ max_score_insights_per_tick: int = 20+ max_train_insights_per_tick: int = 2DISABLED = AnomalyScoringConfig()@@ -133,6 +137,8 @@ MAX_QUERIES_PER_HOUR = 2000MAX_QUERIES_PER_PROJECT_PER_HOUR = 500MAX_SCORE_QUERIES_PER_HOUR = 5000MAX_SCORE_QUERIES_PER_PROJECT_PER_HOUR = 1000+MAX_SCORE_INSIGHTS_PER_TICK = 200+MAX_TRAIN_INSIGHTS_PER_TICK = 50def _positive_int(value: Any, default: int) -> int:@@ -232,6 +238,18 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:),MAX_SCORE_QUERIES_PER_PROJECT_PER_HOUR,),+ max_score_insights_per_tick=min(+ _positive_int(payload.get("max_score_insights_per_tick"), defaults.max_score_insights_per_tick),+ MAX_SCORE_INSIGHTS_PER_TICK,+ ),+ # A payload from before the split sets only `max_insights_per_tick`, and keeps it as its train cap.+ max_train_insights_per_tick=min(+ _positive_int(+ payload.get("max_train_insights_per_tick"),+ _positive_int(payload.get("max_insights_per_tick"), defaults.max_train_insights_per_tick),+ ),+ MAX_TRAIN_INSIGHTS_PER_TICK,+ ),)diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.pyindex af9c5d9e4f90..90cfa257f136 100644--- a/products/signals/backend/anomaly_scores/queue.py+++ b/products/signals/backend/anomaly_scores/queue.py@@ -4,6 +4,10 @@ One query per team reads the newest scored bucket of each series back from the `events. `selection.pick_order` then orders the due insights, and the tick picks from that order theinsights whose queries fit the hourly budgets. With the train and score split on, `state.plan_run`makes each pick a train step or a score step, and each kind has its own budgets.++With the split on, the tick runs two passes, each with its own order and pick cap. The score pass goes+first: it takes the trained insights from their stored state, longest overdue first, so cheap score steps+keep them current. The train pass then takes the other due insights in the `pick_order` order."""import datetime@@ -43,11 +47,13 @@ from products.signals.backend.anomaly_scores.scoring import bucket_endfrom products.signals.backend.anomaly_scores.selection import (Candidate,InsightUsage,+ ScoreCandidate,SelectionConfig,insight_usage,pick_order,+ score_order,)-from products.signals.backend.anomaly_scores.state import RunPlan, load_states, plan_run+from products.signals.backend.anomaly_scores.state import RunPlan, load_states, plan_run, scored_through_by_insightfrom ee.billing.quota_limiting import QuotaLimitingCaches, QuotaResource, list_limited_team_attributes@@ -130,6 +136,9 @@ class Selection:candidates: int# Due insights that the walk reached but skipped, because their queries did not fit a budget.skipped_for_budget: int = 0+ # Picks of the score pass and of the train pass. With the split off, every pick is a train pass pick.+ picked_score: int = 0+ picked_train: int = 0@frozen@@ -324,6 +333,7 @@ def within_budget("""The insights of ``order`` whose cost fits both budgets of their kind, up to ``limit``, and the skip count.A pick that does not fit is skipped and the walk continues, so a large insight does not block smaller ones.+ A pick of a kind with no entry in ``budgets`` belongs to the other pass, and the walk passes over it.``budgets`` are snapshots that skip most misses without a cache call. ``reserve`` charges a pick for real.``plans_for`` loads plans in batches of ``limit``, and only for insights that a snapshot can still fit."""@@ -348,9 +358,10 @@ def within_budget(team_id = team_ids[insight_id]plan = plans.get(insight_id)kind = budget_kind(plan.run if plan else None)+ if plan is not None and kind not in budgets:+ continueif (plan is None- or kind not in budgetsor plan.queries > fleets[kind]or plan.queries > projects[kind].get(team_id, 0)or not reserve(team_id, plan)@@ -374,9 +385,15 @@ def _run_plans(return {insight_id: plans.get(insight_id, missing) for insight_id in insight_ids}+def _take_turns(team_orders: list[list[int]]) -> list[int]:+ # Projects take turns, so one large project cannot take every slot of the tick budget.+ return [insight_id for turn in itertools.zip_longest(*team_orders) for insight_id in turn if insight_id is not None]++def select_insights(- config: AnomalyScoringConfig, selection_config: SelectionConfig, now: datetime.datetime, limit: int+ config: AnomalyScoringConfig, selection_config: SelectionConfig, now: datetime.datetime, limit: int | None = None) -> Selection:+ """The picks of the tick. ``limit`` replaces the pick cap of each pass, for a manual run."""teams = {team.id: team for team in eligible_teams(config)}insights = watched_insights(list(teams), config.intervals, now)state: dict[int, dict[str, datetime.datetime]] = {}@@ -405,14 +422,58 @@ def select_insights(insight.insight_id: max(state[insight.insight_id].values()) if insight.insight_id in state else Nonefor insight in insights}- due = [- insight- for insight in insights- if is_due(insight, newest_buckets[insight.insight_id], teams[insight.team_id].timezone_info, now)- ]- cooling = _cooling_down([insight.insight_id for insight in due])- due_by_id = {insight.insight_id: insight for insight in due if insight.insight_id not in cooling}+ cooling = _cooling_down([insight.insight_id for insight in insights])+ watched_by_id = {insight.insight_id: insight for insight in insights if insight.insight_id not in cooling}+ team_ids = {insight_id: insight.team_id for insight_id, insight in watched_by_id.items()}+ # Both passes share the plans, so an insight that the score pass planned is not planned again.+ planned: dict[int, RunPlan] = {}++ def plans_for(insight_ids: list[int]) -> dict[int, RunPlan]:+ missing = [insight_id for insight_id in insight_ids if insight_id not in planned]+ for team_id in sorted({team_ids[insight_id] for insight_id in missing}):+ team_batch = [insight_id for insight_id in missing if team_ids[insight_id] == team_id]+ planned.update(_run_plans(teams[team_id], team_batch, config, now))+ return {insight_id: planned[insight_id] for insight_id in insight_ids}++ def walk(order: list[int], kind: ScoringRun, cap: int) -> tuple[dict[int, RunPlan], int]:+ # Read the budget before any cost, so a spent hour loads no insight.+ return within_budget(+ order,+ team_ids,+ {kind: remaining_budget(config, {team_ids[insight_id] for insight_id in order}, now, kind)},+ cap,+ plans_for,+ reserve=lambda team_id, plan: reserve_queries(config, team_id, plan.queries, now, budget_kind(plan.run)),+ )++ score_order_ids: list[int] = []+ score_ready: set[int] = set()+ score_picked: dict[int, RunPlan] = {}+ score_skipped = 0+ if config.train_score_split:+ score_order_ids, score_ready = _score_order(watched_by_id, teams, config, now)+ score_cap = config.max_score_insights_per_tick if limit is None else limit+ score_picked, score_skipped = walk(score_order_ids, ScoringRun.SCORE, score_cap)+ # A spent score budget makes the walk load no plan. Plan the head of the order anyway, so that an+ # edited insight there still reaches the train pass while the train budget has room.+ plans_for(score_order_ids[:score_cap])+ # A score-ready insight belongs to the score pass, also when its cap or budget left it out this tick.+ # Without this, the train pass loads a plan for every one of them and then passes over it. An insight+ # that the score pass planned as a train step, for example after an edit, goes to the train pass.+ score_only = {+ insight_id+ for insight_id in score_ready+ if insight_id not in planned or planned[insight_id].run == ScoringRun.SCORE+ }++ due_by_id = {+ insight_id: insight+ for insight_id, insight in watched_by_id.items()+ if insight_id not in score_picked+ and insight_id not in score_only+ and is_due(insight, newest_buckets[insight_id], teams[insight.team_id].timezone_info, now)+ }usage: dict[int, InsightUsage] = {}for team_id in sorted({insight.team_id for insight in due_by_id.values()}):due_ids = [insight.insight_id for insight in due_by_id.values() if insight.team_id == team_id]@@ -431,47 +492,69 @@ def select_insights(usage=usage[insight.insight_id],))- team_orders = [- pick_order(- candidates_by_team[team_id],- config=selection_config,- tick_start=now,- max_staleness=config.limits.max_backfill_buckets,- )- for team_id in sorted(candidates_by_team)- ]- # Projects take turns, so one large project cannot take every slot of the tick budget.- order = [- insight_id for turn in itertools.zip_longest(*team_orders) for insight_id in turn if insight_id is not None- ]- team_ids = {insight_id: due_by_id[insight_id].team_id for insight_id in order}-- def plans_for(insight_ids: list[int]) -> dict[int, RunPlan]:- plans: dict[int, RunPlan] = {}- for team_id in sorted({team_ids[insight_id] for insight_id in insight_ids}):- team_batch = [insight_id for insight_id in insight_ids if team_ids[insight_id] == team_id]- plans.update(_run_plans(teams[team_id], team_batch, config, now))- return plans-- kinds = [ScoringRun.TRAIN, ScoringRun.SCORE] if config.train_score_split else [ScoringRun.TRAIN]- # Read the budgets before any cost, so a spent hour loads no insight.- picked, skipped = within_budget(- order,- team_ids,- {kind: remaining_budget(config, set(team_ids.values()), now, kind) for kind in kinds},- limit,- plans_for,- reserve=lambda team_id, plan: reserve_queries(config, team_id, plan.queries, now, budget_kind(plan.run)),+ train_order = _take_turns(+ [+ pick_order(+ candidates_by_team[team_id],+ config=selection_config,+ tick_start=now,+ max_staleness=config.limits.max_backfill_buckets,+ )+ for team_id in sorted(candidates_by_team)+ ])+ if config.train_score_split:+ train_cap = config.max_train_insights_per_tick+ else:+ train_cap = config.max_insights_per_tick+ # With the split off, every pick charges the train budgets.+ train_picked, train_skipped = walk(train_order, ScoringRun.TRAIN, train_cap if limit is None else limit)+picks = [InsightPick(team_id=team_ids[insight_id],insight_id=insight_id,scored_through=state.get(insight_id, {}),- interval=due_by_id[insight_id].interval,+ interval=watched_by_id[insight_id].interval,query_cost=plan.queries,run=plan.run,)- for insight_id, plan in picked.items()+ for insight_id, plan in [*score_picked.items(), *train_picked.items()]]- return Selection(picks=picks, candidates=len(order), skipped_for_budget=skipped)+ return Selection(+ picks=picks,+ candidates=len({*score_order_ids, *train_order}),+ skipped_for_budget=score_skipped + train_skipped,+ picked_score=len(score_picked),+ picked_train=len(train_picked),+ )+++def _score_order(+ watched_by_id: dict[int, WatchedInsight],+ teams: dict[int, Team],+ config: AnomalyScoringConfig,+ now: datetime.datetime,+) -> tuple[list[int], set[int]]:+ """The score pass order, and every watched insight with a score-ready state, due or not.++ The order holds the insights with a due bucket after their state, longest overdue first in each project.+ """+ team_orders: list[list[int]] = []+ ready: set[int] = set()+ for team_id in sorted({insight.team_id for insight in watched_by_id.values()}):+ team = teams[team_id]+ team_insights = [insight for insight in watched_by_id.values() if insight.team_id == team_id]+ scored = scored_through_by_insight(team, [insight.insight_id for insight in team_insights], config, now)+ ready.update(scored)+ candidates = [+ ScoreCandidate(+ insight_id=insight.insight_id,+ interval=insight.interval,+ scored_through=scored[insight.insight_id].astimezone(team.timezone_info),+ )+ for insight in team_insights+ if insight.insight_id in scored and is_due(insight, scored[insight.insight_id], team.timezone_info, now)+ ]+ team_orders.append(score_order(candidates, now=now, max_staleness=config.limits.max_backfill_buckets))+ return _take_turns(team_orders), readydiff --git a/products/signals/backend/anomaly_scores/selection.py b/products/signals/backend/anomaly_scores/selection.pyindex 4f87dce53963..f16f2159eaa2 100644--- a/products/signals/backend/anomaly_scores/selection.py+++ b/products/signals/backend/anomaly_scores/selection.py@@ -1,5 +1,7 @@-"""The order in which a scoring tick picks insights, set by the `anomaly-scoring` flag payload.+"""The order in which a scoring tick picks insights.+The score pass takes the trained insights longest overdue first, see `score_order`. The train pass, and the+whole tick when the train and score split is off, uses `pick_order`, set by the `anomaly-scoring` flag payload.`weighted` samples insights in proportion to usage × staleness, so most of the per-tick query budget goes toinsights people look at. Weight grows while an insight waits, so an insight nobody views is picked rarely butnever starves. `uniform` picks the stalest insight first, so every insight gets the same share.@@ -51,6 +53,14 @@ class Candidate:usage: InsightUsage+@frozen+class ScoreCandidate:+ insight_id: int+ interval: IntervalType+ # The newest scored bucket start of the stored state, in the project timezone.+ scored_through: datetime++def _non_negative(value: Any, default: float) -> float:# `bool` is an `int` subclass, and `true` is not a number.if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value) or value < 0:@@ -88,14 +98,38 @@ def staleness(candidate: Candidate, *, now: datetime, max_staleness: int) -> int"""if candidate.newest_scored_bucket is None:return max_staleness+ return _completed_buckets_after(candidate.newest_scored_bucket, candidate.interval, now, max_staleness)+++def _completed_buckets_after(newest: datetime, interval: IntervalType, now: datetime, max_count: int) -> int:count = 0- start = bucket_end(candidate.newest_scored_bucket, candidate.interval)- while count < max_staleness and bucket_end(start, candidate.interval) <= now:+ start = bucket_end(newest, interval)+ while count < max_count and bucket_end(start, interval) <= now:count += 1- start = bucket_end(start, candidate.interval)+ start = bucket_end(start, interval)return count+def score_order(candidates: Sequence[ScoreCandidate], *, now: datetime, max_staleness: int) -> list[int]:+ """Insight ids with a due bucket, the longest overdue first. Usage has no effect.++ A score step is cheap, so the pass keeps every trained insight current instead of sampling. Overdue is+ the time since the oldest due bucket ended, so every interval waits about as long when the pick cap binds.+ A count of due buckets would put hourly insights first, and daily, weekly and monthly insights would wait+ that many days, weeks and months. An insight close to ``max_staleness`` due buckets goes first of all,+ because one more due bucket makes its next visit a catch-up train step.+ """+ near_cap = max_staleness - max(1, max_staleness // 5)+ keyed: list[tuple[bool, datetime, int]] = []+ for candidate in candidates:+ due = _completed_buckets_after(candidate.scored_through, candidate.interval, now, max_staleness)+ if due == 0:+ continue+ oldest_due_end = bucket_end(bucket_end(candidate.scored_through, candidate.interval), candidate.interval)+ keyed.append((due < near_cap, oldest_due_end, candidate.insight_id))+ return [insight_id for _, _, insight_id in sorted(keyed)]++def weight(candidate: Candidate, *, config: SelectionConfig, now: datetime, max_staleness: int) -> float:usage = config.usage_floor + candidate.usage.viewersif candidate.usage.has_enabled_alert:diff --git a/products/signals/backend/anomaly_scores/state.py b/products/signals/backend/anomaly_scores/state.pyindex c46f117169f7..75e127d1ad78 100644--- a/products/signals/backend/anomaly_scores/state.py+++ b/products/signals/backend/anomaly_scores/state.py@@ -51,6 +51,34 @@ class RunPlan:run: ScoringRun | None = None+def _schedule_needs_training(+ interval: IntervalType,+ *,+ trained_at: datetime.datetime,+ scored_through: datetime.datetime,+ missing_series: int,+ series_count: int,+ now: datetime.datetime,+ timezone: ZoneInfo,+ train_every: int,+ max_backfill: int,+) -> bool:+ """The retrain triggers that a state row answers alone, without the insight's query."""+ if trained_at + interval_spec(interval).period * train_every <= now:+ return True+ # More due buckets than one visit scores. A train step scores the newest of them, as a first visit does.+ due = buckets_to_score(+ interval,+ now=now,+ after=scored_through,+ timezone=timezone,+ limits=dataclasses.replace(DEFAULT_LIMITS, max_backfill_buckets=max_backfill + 1),+ )+ if due > max_backfill:+ return True+ return missing_series * 2 > series_count++def needs_training(state: TrainedState | None,*,@@ -68,20 +96,17 @@ def needs_training(or state.scored_through is None):return True- interval = identity.interval- if state.trained_at + interval_spec(interval).period * train_every <= now:- return True- # More due buckets than one visit scores. A train step scores the newest of them, as a first visit does.- due = buckets_to_score(- interval,+ return _schedule_needs_training(+ identity.interval,+ trained_at=state.trained_at,+ scored_through=state.scored_through,+ missing_series=state.missing_series,+ series_count=len(state.series_keys),now=now,- after=state.scored_through,timezone=timezone,- limits=dataclasses.replace(DEFAULT_LIMITS, max_backfill_buckets=max_backfill + 1),+ train_every=train_every,+ max_backfill=max_backfill,)- if due > max_backfill:- return True- return state.missing_series * 2 > len(state.series_keys)def plan_run(@@ -119,6 +144,45 @@ def load_states(team: Team, insights: Collection[Insight]) -> dict[int, TrainedSreturn {row.insight_id: _from_row(row) for row in rows if profiles.get(row.insight_id) == row.profile}+def scored_through_by_insight(+ team: Team, insight_ids: Collection[int], config: AnomalyScoringConfig, now: datetime.datetime+) -> dict[int, datetime.datetime]:+ """Insight id to the newest scored bucket of its stored state, for the rows that a score step can read.++ A row whose retrain is due by age, by due buckets or by missing series is left out, so its insight goes to+ the train pass without a plan. The identity and the profile are not checked here, because they need the+ insight's query. `plan_run` checks them, and the score pass hands an insight that fails them to the train pass.+ """+ if not insight_ids:+ return {}+ rows = (+ AnomalyScoringState.objects.for_team(team.id)+ .filter(insight_id__in=list(insight_ids), scored_through__isnull=False)+ .exclude(artifact_key="")+ .exclude(series_keys=[])+ .values_list("insight_id", "interval", "trained_at", "scored_through", "missing_series", "series_keys")+ )+ scored: dict[int, datetime.datetime] = {}+ scorable = {interval.value: interval for interval in config.train_every_buckets}+ for insight_id, interval, trained_at, scored_through, missing_series, series_keys in rows:+ if scored_through is None or interval not in scorable:+ continue+ if _schedule_needs_training(+ scorable[interval],+ trained_at=trained_at,+ scored_through=scored_through,+ missing_series=missing_series,+ series_count=len(series_keys),+ now=now,+ timezone=team.timezone_info,+ train_every=config.train_every_buckets[scorable[interval]],+ max_backfill=config.limits.max_backfill_buckets,+ ):+ continue+ scored[insight_id] = max(scored_through, scored.get(insight_id, scored_through))+ return scored++def load_state(team: Team, insight: Insight) -> TrainedState | None:return load_states(team, [insight]).get(insight.id)diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.pyindex 62079d017cb2..2b241c772883 100644--- a/products/signals/backend/anomaly_scores/tick.py+++ b/products/signals/backend/anomaly_scores/tick.py@@ -111,6 +111,8 @@ class AnomalyScoringTickPlan:max_concurrency: intlimits: ScoringLimits = DEFAULT_LIMITSskipped_for_budget: int = 0+ picked_score: int = 0+ picked_train: int = 0@frozen@@ -149,6 +151,9 @@ class AnomalyScoringTickResult:# Started picks per kind of the train and score split.run_train: int = 0run_score: int = 0+ # Planned picks of the score pass and of the train pass, started or not.+ picked_score: int = 0+ picked_train: int = 0def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:@@ -157,7 +162,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlconfig = read_config(payload)if not config.enabled:return AnomalyScoringTickPlan(enabled=False, picks=[], candidates=0, max_concurrency=config.max_concurrency)- selection = select_insights(config, parse_selection_config(payload), now, limit or config.max_insights_per_tick)+ selection = select_insights(config, parse_selection_config(payload), now, limit)record_candidates(selection.candidates)return AnomalyScoringTickPlan(enabled=True,@@ -166,6 +171,8 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlmax_concurrency=config.max_concurrency,limits=config.limits,skipped_for_budget=selection.skipped_for_budget,+ picked_score=selection.picked_score,+ picked_train=selection.picked_train,)@@ -320,6 +327,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:stopped_at_capacity=result.stopped_at_capacity,run_train=result.run_train,run_score=result.run_score,+ picked_score=result.picked_score,+ picked_train=result.picked_train,)ph_background_capture()(distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,@@ -338,6 +347,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:"stopped_at_capacity": result.stopped_at_capacity,"run_train": result.run_train,"run_score": result.run_score,+ "picked_score": result.picked_score,+ "picked_train": result.picked_train,},)@@ -428,6 +439,8 @@ class AnomalyScoringTickWorkflow:stopped_at_capacity=statuses[STATUS_AT_CAPACITY] > 0,run_train=sum(pick.run == ScoringRun.TRAIN for pick in started_picks),run_score=sum(pick.run == ScoringRun.SCORE for pick in started_picks),+ picked_score=plan.picked_score,+ picked_train=plan.picked_train,)await workflow.execute_activity(capture_anomaly_scoring_tick_activity,diff --git a/products/signals/backend/test/test_anomaly_scores.py b/products/signals/backend/test/test_anomaly_scores.pyindex 629bff385ec6..c66841b2f8e5 100644--- a/products/signals/backend/test/test_anomaly_scores.py+++ b/products/signals/backend/test/test_anomaly_scores.py@@ -473,6 +473,23 @@ class TestNeedsTraining(SimpleTestCase):is trains)+ @parameterized.expand([("kept_current", 23, False), ("retrain_due", 24, True)])+ def test_an_hourly_insight_kept_current_trains_only_after_train_every(+ self, _name: str, hours_since_trained: int, trains: bool+ ) -> None:+ identity = StateIdentity(interval=IntervalType.HOUR, detector_version="v1", query_hash="q1")+ now = datetime(2026, 1, 10, 12, 5, tzinfo=UTC)+ state = _state(+ identity=identity,+ trained_at=now - timedelta(hours=hours_since_trained),+ scored_through=datetime(2026, 1, 10, 10, tzinfo=UTC),+ )++ assert (+ needs_training(state, identity=identity, now=now, timezone=ZoneInfo("UTC"), train_every=24, max_backfill=30)+ is trains+ )+class TestMetricScoringQuery(SimpleTestCase):@parameterized.expand([("without_compare", None), ("with_saved_compare", CompareFilter(compare=True))])diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.pyindex fee7064f359d..a42438788036 100644--- a/products/signals/backend/test/test_anomaly_scoring_tick.py+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py@@ -32,7 +32,13 @@ from products.dashboards.backend.models.dashboard_tile import DashboardTilefrom products.product_analytics.backend.facade.api import record_insight_viewfrom products.product_analytics.backend.facade.models import Insightfrom products.signals.backend.anomaly_scores.artifacts import clear_artifact_cache-from products.signals.backend.anomaly_scores.config import DISABLED, AnomalyScoringConfig, ScoringRun, parse_config+from products.signals.backend.anomaly_scores.config import (+ DEFAULT_PROFILE,+ DISABLED,+ AnomalyScoringConfig,+ ScoringRun,+ parse_config,+)from products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_eventsfrom products.signals.backend.anomaly_scores.queue import (SKIP_COOLDOWN,@@ -55,10 +61,15 @@ from products.signals.backend.anomaly_scores.scoring import (InsightScores,ScoredPoint,ScoredSeries,+ StateIdentity,+ TrainedState,UnsupportedInsightError,+ interval_scoring,+ state_identity,+ trends_query_for,)from products.signals.backend.anomaly_scores.selection import SelectionConfig, Weighting-from products.signals.backend.anomaly_scores.state import RunPlan, load_state, plan_run+from products.signals.backend.anomaly_scores.state import RunPlan, load_state, plan_run, save_statefrom products.signals.backend.anomaly_scores.tick import (AnomalyScoringTickInput,AnomalyScoringTickPlan,@@ -154,6 +165,8 @@ class TestTickConfig(SimpleTestCase):("max_queries_per_project_per_hour", 501, 500),("max_score_queries_per_hour", 1_000_000, 5000),("max_score_queries_per_project_per_hour", 1001, 1000),+ ("max_score_insights_per_tick", 201, 200),+ ("max_train_insights_per_tick", 51, 50),])def test_tick_levers_above_their_maximum_parse_to_the_maximum(self, field: str, value: int, expected: int) -> None:@@ -161,6 +174,26 @@ class TestTickConfig(SimpleTestCase):assert getattr(config, field) == expected+ @parameterized.expand(+ [+ ("defaults", {}, 20, 2),+ # A payload from before the split keeps its per-tick cap for train steps.+ ("only_the_old_cap", {"max_insights_per_tick": 8}, 20, 8),+ (+ "per_pass_caps",+ {"max_insights_per_tick": 8, "max_score_insights_per_tick": 40, "max_train_insights_per_tick": 3},+ 40,+ 3,+ ),+ ]+ )+ def test_each_pass_gets_its_own_cap(+ self, _name: str, levers: dict[str, int], score_cap: int, train_cap: int+ ) -> None:+ config = parse_config({**_SPLIT_PAYLOAD, **levers})++ assert (config.max_score_insights_per_tick, config.max_train_insights_per_tick) == (score_cap, train_cap)+class TestQueryBudget(SimpleTestCase):@parameterized.expand(@@ -210,6 +243,18 @@ class TestQueryBudget(SimpleTestCase):0,QueryBudget(fleet=0, projects={1: 0}),),+ # The train pass passes over a score step, and does not count it as a budget skip.+ (+ "other_pass_kind_passed_over",+ {1: 1, 2: 1, 3: 1},+ QueryBudget(fleet=100, projects={1: 30}),+ 10,+ [1, 3],+ 0,+ 0,+ None,+ ScoringRun.SCORE,+ ),])def test_the_walk_skips_picks_over_budget_and_continues(@@ -222,10 +267,11 @@ class TestQueryBudget(SimpleTestCase):skipped: int,refused_reservations: int,score_budget: QueryBudget | None = None,+ second_run: ScoringRun | None = None,) -> None:plans = {1: RunPlan(queries=3),- 2: RunPlan(queries=3, run=ScoringRun.SCORE if score_budget else None),+ 2: RunPlan(queries=3, run=ScoringRun.SCORE if score_budget else second_run),3: RunPlan(queries=2),}budgets = {ScoringRun.TRAIN: budget}@@ -803,6 +849,126 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):assert [pick.insight_id for pick in selection.picks] == [insights[self.team.id].id]+ @parameterized.expand([("cap_one", 1, ["most_overdue"]), ("cap_two", 2, ["most_overdue", "popular"])])+ @time_machine.travel(NOW, tick=False)+ def test_the_score_pass_picks_trained_insights_most_overdue_first_with_the_train_budget_spent(+ self, _name: str, score_cap: int, expected: list[str]+ ) -> None:+ self.team.timezone = "UTC"+ self.team.save()+ dashboard = Dashboard.objects.create(team=self.team, name="Main")+ insights = {+ name: Insight.objects.create(team=self.team, name=name, query=_trends(interval="day"))+ for name in ("most_overdue", "popular", "untrained")+ }+ for insight in insights.values():+ DashboardTile.objects.create(dashboard=dashboard, insight=insight)+ cache.delete(_cooldown_key(insight.id))+ for viewer_index in range(5):+ viewer = User.objects.create_and_join(self.organization, f"viewer{viewer_index}@example.com", "password")+ record_insight_view(insight_id=insights["popular"].id, team_id=self.team.id, user_id=viewer.id)+ config = parse_config(+ {+ **_SPLIT_PAYLOAD,+ "organization_ids": [str(self.organization.id)],+ "allow_team_ids": [self.team.id],+ "max_score_insights_per_tick": score_cap,+ }+ )+ today = NOW.replace(hour=0)+ for name, scored_through in (+ ("most_overdue", today - timedelta(days=4)),+ ("popular", today - timedelta(days=2)),+ ):+ insight = insights[name]+ query = trends_query_for(insight, DEFAULT_PROFILE)+ identity = state_identity(+ query, interval_scoring(DEFAULT_PROFILE, IntervalType.DAY), self.team, config.limits+ )+ save_state(+ self.team,+ insight.id,+ DEFAULT_PROFILE.name,+ TrainedState(+ identity=identity,+ trained_at=NOW - timedelta(days=1),+ scored_through=scored_through,+ series_keys=("a",),+ artifact_key=f"anomaly_models/{insight.id}",+ ),+ )+ for kind in ScoringRun:+ cache.delete_many([_fleet_budget_key(NOW, kind), _project_budget_key(self.team.id, NOW, kind)])+ cache.set(_fleet_budget_key(NOW, ScoringRun.TRAIN), config.max_queries_per_hour)++ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):+ selection = select_insights(config, SelectionConfig(), NOW)++ names = {insight.id: name for name, insight in insights.items()}+ assert [(names[pick.insight_id], pick.run) for pick in selection.picks] == [+ (name, ScoringRun.SCORE) for name in expected+ ]+ assert (selection.picked_score, selection.picked_train) == (len(expected), 0)++ @parameterized.expand(+ [+ ("score_budget_left", False, [("current", ScoringRun.SCORE), ("edited", ScoringRun.TRAIN)]),+ ("score_budget_spent", True, [("edited", ScoringRun.TRAIN)]),+ ]+ )+ @time_machine.travel(NOW, tick=False)+ def test_an_edited_insight_with_a_current_state_goes_to_the_train_pass(+ self, _name: str, score_budget_spent: bool, expected: list[tuple[str, ScoringRun]]+ ) -> None:+ self.team.timezone = "UTC"+ self.team.save()+ dashboard = Dashboard.objects.create(team=self.team, name="Main")+ insights = {+ name: Insight.objects.create(team=self.team, name=name, query=_trends(interval="day"))+ for name in ("edited", "current")+ }+ for insight in insights.values():+ DashboardTile.objects.create(dashboard=dashboard, insight=insight)+ cache.delete(_cooldown_key(insight.id))+ config = parse_config(+ {**_SPLIT_PAYLOAD, "organization_ids": [str(self.organization.id)], "allow_team_ids": [self.team.id]}+ )+ today = NOW.replace(hour=0)+ for name, scored_through in (("edited", today - timedelta(days=4)), ("current", today - timedelta(days=2))):+ insight = insights[name]+ identity = state_identity(+ trends_query_for(insight, DEFAULT_PROFILE),+ interval_scoring(DEFAULT_PROFILE, IntervalType.DAY),+ self.team,+ config.limits,+ )+ if name == "edited":+ identity = StateIdentity(+ interval=identity.interval, detector_version=identity.detector_version, query_hash="before-edit"+ )+ save_state(+ self.team,+ insight.id,+ DEFAULT_PROFILE.name,+ TrainedState(+ identity=identity,+ trained_at=NOW - timedelta(days=1),+ scored_through=scored_through,+ series_keys=("a",),+ artifact_key=f"anomaly_models/{insight.id}",+ ),+ )+ for kind in ScoringRun:+ cache.delete_many([_fleet_budget_key(NOW, kind), _project_budget_key(self.team.id, NOW, kind)])+ if score_budget_spent:+ cache.set(_fleet_budget_key(NOW, ScoringRun.SCORE), config.max_score_queries_per_hour)++ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):+ selection = select_insights(config, SelectionConfig(), NOW)++ names = {insight.id: name for name, insight in insights.items()}+ assert [(names[pick.insight_id], pick.run) for pick in selection.picks] == expected+async def _run_tick(plan: AnomalyScoringTickPlan, statuses: dict[int, str]diff --git a/products/signals/backend/test/test_anomaly_selection.py b/products/signals/backend/test/test_anomaly_selection.pyindex 15cdd89711cf..84ea8b3e72e0 100644--- a/products/signals/backend/test/test_anomaly_selection.py+++ b/products/signals/backend/test/test_anomaly_selection.py@@ -11,10 +11,12 @@ from posthog.schema import IntervalTypefrom products.signals.backend.anomaly_scores.selection import (Candidate,InsightUsage,+ ScoreCandidate,SelectionConfig,Weighting,parse_selection_config,pick_order,+ score_order,staleness,)@@ -117,6 +119,42 @@ class TestPickOrder(SimpleTestCase):assert staleness(candidate, now=datetime(2026, 10, 14, 12, tzinfo=UTC), max_staleness=MAX_STALENESS) == expected+class TestScoreOrder(SimpleTestCase):+ @parameterized.expand(+ [+ (+ "longest_overdue_first_whatever_the_interval",+ [+ # 1 due bucket, which ended 12 hours ago.+ ScoreCandidate(+ insight_id=1, interval=IntervalType.DAY, scored_through=datetime(2026, 10, 13, tzinfo=UTC)+ ),+ # 5 due buckets, the oldest of which ended 4 hours ago.+ ScoreCandidate(insight_id=2, interval=IntervalType.HOUR, scored_through=TICK - timedelta(hours=6)),+ ScoreCandidate(insight_id=3, interval=IntervalType.HOUR, scored_through=TICK - timedelta(hours=3)),+ # Nothing completed after it.+ ScoreCandidate(insight_id=4, interval=IntervalType.HOUR, scored_through=TICK - timedelta(hours=1)),+ ],+ [1, 2, 3],+ ),+ (+ "close_to_the_backfill_cap_goes_first",+ [+ # 2 due buckets, the oldest of which ended 36 hours ago.+ ScoreCandidate(+ insight_id=5, interval=IntervalType.DAY, scored_through=datetime(2026, 10, 12, tzinfo=UTC)+ ),+ # 26 due buckets of 30, the oldest of which ended 25 hours ago.+ ScoreCandidate(insight_id=6, interval=IntervalType.HOUR, scored_through=TICK - timedelta(hours=27)),+ ],+ [6, 5],+ ),+ ]+ )+ def test_order(self, _name: str, candidates: list[ScoreCandidate], expected: list[int]) -> None:+ assert score_order(candidates, now=TICK, max_staleness=MAX_STALENESS) == expected++class TestParseSelectionConfig(SimpleTestCase):@parameterized.expand([