:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): split anomaly scoring into train and score steps (#123092)
posthog[bot]ee1d430f993d
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yamlindex d0f72595ac04..6afda3bb2514 100644--- a/.semgrep/rules/security/idor-team-scoped-models.yaml+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml@@ -408,6 +408,7 @@ rules:|SignalScoutSuggestionSet|SignalScoutBackgroundBand|SignalSpend+ |AnomalyScoringState|SignalScratchpad|SignalProjectProfile|SingleSessionSummary@@ -880,6 +881,7 @@ rules:|SignalScoutSuggestionSet|SignalScoutBackgroundBand|SignalSpend+ |AnomalyScoringState|SignalScratchpad|SignalProjectProfile|SingleSessionSummarydiff --git a/frontend/src/taxonomy/core-filter-definitions-by-group.json b/frontend/src/taxonomy/core-filter-definitions-by-group.jsonindex d60dfefedda3..9dee9bfc3ed5 100644--- a/frontend/src/taxonomy/core-filter-definitions-by-group.json+++ b/frontend/src/taxonomy/core-filter-definitions-by-group.json@@ -803,6 +803,12 @@"label": "Anomaly scoring profile","type": "String"},+ "$anomaly_run": {+ "description": "How the bucket was scored. `train` fetched the full history of the series, and `score` scored the bucket on stored history. Empty when scoring fetches the full history on every run.",+ "examples": ["train", "score"],+ "label": "Anomaly scoring run",+ "type": "String"+ },"$anomaly_score": {"description": "Score from 0 to 1. It ranks buckets within one series only, so never compare or average it across series. Empty while the series warms up.","label": "Anomaly score",@@ -4745,6 +4751,12 @@"label": "Anomaly scoring profile","type": "String"},+ "$anomaly_run": {+ "description": "How the bucket was scored. `train` fetched the full history of the series, and `score` scored the bucket on stored history. Empty when scoring fetches the full history on every run.",+ "examples": ["train", "score"],+ "label": "Anomaly scoring run",+ "type": "String"+ },"$anomaly_score": {"description": "Score from 0 to 1. It ranks buckets within one series only, so never compare or average it across series. Empty while the series warms up.","label": "Anomaly score",diff --git a/posthog/taxonomy/taxonomy.py b/posthog/taxonomy/taxonomy.pyindex 2218fba9f82f..dbcba028435e 100644--- a/posthog/taxonomy/taxonomy.py+++ b/posthog/taxonomy/taxonomy.py@@ -3376,6 +3376,12 @@ CORE_FILTER_DEFINITIONS_BY_GROUP: dict[str, dict[str, CoreFilterDefinition]] = {"description": "Detector-specific values for this bucket, for example ensemble sub-scores. Its shape depends on the detector.","type": "String",},+ "$anomaly_run": {+ "label": "Anomaly scoring run",+ "description": "How the bucket was scored. `train` fetched the full history of the series, and `score` scored the bucket on stored history. Empty when scoring fetches the full history on every run.",+ "examples": ["train", "score"],+ "type": "String",+ },"$csp_document_url": {"label": "Document URL","description": "The URL of the document where the violation occurred.",diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.pyindex 5d4631dee843..0f8938b4c115 100644--- a/products/signals/backend/anomaly_scores/config.py+++ b/products/signals/backend/anomaly_scores/config.py@@ -6,6 +6,7 @@ field falls back to its default, and no default is unbounded.import uuidimport dataclasses+from enum import StrEnumfrom typing import TYPE_CHECKING, Anyimport structlog@@ -80,6 +81,23 @@ def resolve_profile(team: "Team", insight: "Insight") -> ScoringProfile:return DEFAULT_PROFILE+class ScoringRun(StrEnum):+ """The kind of visit when the train and score split is on."""++ # Fetch the full detector history, score the new buckets, and store the series tail.+ TRAIN = "train"+ # Fetch only the refresh and new buckets, merge them into the stored tail, and score the new buckets.+ SCORE = "score"+++# Retrain hourly insights once a day, daily once a week, weekly once a month, and monthly once a quarter.+DEFAULT_TRAIN_EVERY_BUCKETS = {IntervalType.HOUR: 24, IntervalType.DAY: 7, IntervalType.WEEK: 4, IntervalType.MONTH: 3}+MAX_TRAIN_EVERY_BUCKETS = {IntervalType.HOUR: 168, IntervalType.DAY: 30, IntervalType.WEEK: 13, IntervalType.MONTH: 12}+# Stored buckets that a score step fetches again, so late events reach the stored values.+DEFAULT_REFRESH_BUCKETS = {IntervalType.HOUR: 3, IntervalType.DAY: 3, IntervalType.WEEK: 2, IntervalType.MONTH: 1}+MAX_REFRESH_BUCKETS = {IntervalType.HOUR: 24, IntervalType.DAY: 14, IntervalType.WEEK: 4, IntervalType.MONTH: 3}++# Hourly insights are left out by default: with many eligible insights, the time between two visits of# one insight passes the hourly backfill cap, and the buckets in between are never scored.DEFAULT_TICK_INTERVALS = (IntervalType.DAY, IntervalType.WEEK, IntervalType.MONTH)@@ -100,6 +118,15 @@ class AnomalyScoringConfig:max_queries_per_hour: int = 120max_queries_per_project_per_hour: int = 30limits: ScoringLimits = DEFAULT_LIMITS+ # Off keeps one full fetch per visit and stores no state. On, most visits are cheap score steps.+ train_score_split: bool = False+ train_every_buckets: dict[IntervalType, int] = dataclasses.field(+ default_factory=lambda: dict(DEFAULT_TRAIN_EVERY_BUCKETS)+ )+ refresh_buckets: dict[IntervalType, int] = dataclasses.field(default_factory=lambda: dict(DEFAULT_REFRESH_BUCKETS))+ # Score step queries per clock hour. Train steps use the budgets above, so a cheap score step never waits for them.+ max_score_queries_per_hour: int = 600+ max_score_queries_per_project_per_hour: int = 150DISABLED = AnomalyScoringConfig()@@ -108,6 +135,8 @@ MAX_INSIGHTS_PER_TICK = 50MAX_CONCURRENCY = 8MAX_QUERIES_PER_HOUR = 2000MAX_QUERIES_PER_PROJECT_PER_HOUR = 500+MAX_SCORE_QUERIES_PER_HOUR = 5000+MAX_SCORE_QUERIES_PER_PROJECT_PER_HOUR = 1000def _positive_int(value: Any, default: int) -> int:@@ -144,6 +173,18 @@ def _intervals(value: Any) -> tuple[IntervalType, ...]:return intervals or DEFAULT_TICK_INTERVALS+def _buckets_per_interval(+ value: Any, defaults: dict[IntervalType, int], maxima: dict[IntervalType, int]+) -> dict[IntervalType, int]:+ """An object of interval name to bucket count. A missing or invalid count gets its default, a large one its maximum."""+ if not isinstance(value, dict):+ return dict(defaults)+ return {+ interval: min(_positive_int(value.get(interval.value), default), maxima[interval])+ for interval, default in defaults.items()+ }++def parse_limits(payload: Any) -> ScoringLimits:"""The `limits` object of the payload. An invalid field gets its default, a large one its `MAX_LIMITS` value."""if not isinstance(payload, dict):@@ -181,6 +222,23 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:MAX_QUERIES_PER_PROJECT_PER_HOUR,),limits=parse_limits(payload.get("limits")),+ train_score_split=payload.get("train_score_split") is True,+ train_every_buckets=_buckets_per_interval(+ payload.get("train_every_buckets"), DEFAULT_TRAIN_EVERY_BUCKETS, MAX_TRAIN_EVERY_BUCKETS+ ),+ refresh_buckets=_buckets_per_interval(+ payload.get("refresh_buckets"), DEFAULT_REFRESH_BUCKETS, MAX_REFRESH_BUCKETS+ ),+ max_score_queries_per_hour=min(+ _positive_int(payload.get("max_score_queries_per_hour"), defaults.max_score_queries_per_hour),+ MAX_SCORE_QUERIES_PER_HOUR,+ ),+ max_score_queries_per_project_per_hour=min(+ _positive_int(+ payload.get("max_score_queries_per_project_per_hour"), defaults.max_score_queries_per_project_per_hour+ ),+ MAX_SCORE_QUERIES_PER_PROJECT_PER_HOUR,+ ),)diff --git a/products/signals/backend/anomaly_scores/emission.py b/products/signals/backend/anomaly_scores/emission.pyindex d775a0ba129b..57b1ba67853d 100644--- a/products/signals/backend/anomaly_scores/emission.py+++ b/products/signals/backend/anomaly_scores/emission.py@@ -57,6 +57,7 @@ def _event("$anomaly_profile": scores.profile,"$anomaly_scored_at": scored_at.astimezone(datetime.UTC).isoformat(),"$anomaly_details": json.dumps(point.details, sort_keys=True),+ **({"$anomaly_run": scores.run.value} if scores.run else {}),},}diff --git a/products/signals/backend/anomaly_scores/metrics.py b/products/signals/backend/anomaly_scores/metrics.pyindex 06b3ed442a1e..e98c4ca9d2dd 100644--- a/products/signals/backend/anomaly_scores/metrics.py+++ b/products/signals/backend/anomaly_scores/metrics.py@@ -13,11 +13,11 @@ def _metrics() -> PostHogMetrics | None:return client.metrics if client is not None else None-def record_pick(*, status: str, interval: str, duration_ms: float, events: int) -> None:+def record_pick(*, status: str, interval: str, duration_ms: float, events: int, run: str | None = None) -> None:metrics = _metrics()if metrics is None:return- attributes = {"status": status, "interval": interval}+ attributes = {"status": status, "interval": interval, **({"run": run} if run else {})}metrics.count("anomaly_scoring.picks", 1, attributes=attributes)metrics.histogram("anomaly_scoring.query.duration", duration_ms, unit="ms", attributes=attributes)if events:diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.pyindex 468150a5e115..21121ed3a2d4 100644--- a/products/signals/backend/anomaly_scores/queue.py+++ b/products/signals/backend/anomaly_scores/queue.py@@ -1,8 +1,9 @@"""Pick the watched insights that the next anomaly scoring tick scores.-There is no stored state. One query per team reads the newest scored bucket of each series back-from the `$anomaly_score` events. `selection.pick_order` then orders the due insights, and the tick-picks from that order the insights whose queries fit the hourly budgets.+One query per team reads the newest scored bucket of each series back from the `$anomaly_score`+events. `selection.pick_order` then orders the due insights, and the tick picks from that order the+insights whose queries fit the hourly budgets. With the train and score split on, `state.plan_run`+makes each pick a train step or a score step, and each kind has its own budgets."""import datetime@@ -30,8 +31,8 @@ from products.product_analytics.backend.facade.api import (insights_including_soft_deleted_for_team,watched_trends_insights,)-from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig-from products.signals.backend.anomaly_scores.scoring import bucket_end, query_cost+from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig, ScoringRun+from products.signals.backend.anomaly_scores.scoring import bucket_endfrom products.signals.backend.anomaly_scores.selection import (Candidate,InsightUsage,@@ -39,6 +40,7 @@ from products.signals.backend.anomaly_scores.selection import (insight_usage,pick_order,)+from products.signals.backend.anomaly_scores.state import RunPlan, load_states, plan_runfrom ee.billing.quota_limiting import QuotaLimitingCaches, QuotaResource, list_limited_team_attributes@@ -101,6 +103,10 @@ class InsightPick:# The ClickHouse queries this pick sends. Selection charges them to the hourly budgets, so two ticks# that overlap cannot spend the same budget.query_cost: int = 1+ # None when the train and score split is off.+ run: ScoringRun | None = None+ # Stored buckets that a score step fetches again.+ refresh_buckets: int = 0@frozen@@ -223,23 +229,46 @@ def _hour(now: datetime.datetime) -> str:return _utc(now).strftime("%Y%m%d%H")-def _fleet_budget_key(now: datetime.datetime) -> str:- return f"signals:anomaly_scores:queries:{_hour(now)}"+def budget_kind(run: ScoringRun | None) -> ScoringRun:+ """The budgets a pick charges. Only a score step uses the score budgets."""+ return ScoringRun.SCORE if run == ScoringRun.SCORE else ScoringRun.TRAIN-def _project_budget_key(team_id: int, now: datetime.datetime) -> str:- return f"signals:anomaly_scores:queries:{_hour(now)}:team:{team_id}"+def _budget_prefix(kind: ScoringRun) -> str:+ # Train steps keep the key of the budgets from before the split, so turning the split on keeps the hour's spend.+ return "signals:anomaly_scores:score_queries" if kind == ScoringRun.SCORE else "signals:anomaly_scores:queries"-def remaining_budget(config: AnomalyScoringConfig, team_ids: Collection[int], now: datetime.datetime) -> QueryBudget:- project_keys = {team_id: _project_budget_key(team_id, now) for team_id in team_ids}- spent = cache.get_many([_fleet_budget_key(now), *project_keys.values()])+def _fleet_budget_key(now: datetime.datetime, kind: ScoringRun = ScoringRun.TRAIN) -> str:+ return f"{_budget_prefix(kind)}:{_hour(now)}"+++def _project_budget_key(team_id: int, now: datetime.datetime, kind: ScoringRun = ScoringRun.TRAIN) -> str:+ return f"{_budget_prefix(kind)}:{_hour(now)}:team:{team_id}"+++def _fleet_cap(config: AnomalyScoringConfig, kind: ScoringRun) -> int:+ return config.max_score_queries_per_hour if kind == ScoringRun.SCORE else config.max_queries_per_hour+++def _project_cap(config: AnomalyScoringConfig, kind: ScoringRun) -> int:+ if kind == ScoringRun.SCORE:+ return config.max_score_queries_per_project_per_hour+ return config.max_queries_per_project_per_hour+++def remaining_budget(+ config: AnomalyScoringConfig,+ team_ids: Collection[int],+ now: datetime.datetime,+ kind: ScoringRun = ScoringRun.TRAIN,+) -> QueryBudget:+ fleet_key = _fleet_budget_key(now, kind)+ project_keys = {team_id: _project_budget_key(team_id, now, kind) for team_id in team_ids}+ spent = cache.get_many([fleet_key, *project_keys.values()])return QueryBudget(- fleet=config.max_queries_per_hour - spent.get(_fleet_budget_key(now), 0),- projects={- team_id: config.max_queries_per_project_per_hour - spent.get(key, 0)- for team_id, key in project_keys.items()- },+ fleet=_fleet_cap(config, kind) - spent.get(fleet_key, 0),+ projects={team_id: _project_cap(config, kind) - spent.get(key, 0) for team_id, key in project_keys.items()},)@@ -253,12 +282,18 @@ def _reserve(key: str, queries: int, cap: int) -> bool:return False-def reserve_queries(config: AnomalyScoringConfig, team_id: int, queries: int, now: datetime.datetime) -> bool:- """Charge ``queries`` to both hourly budgets, or to neither when one of them has too few left."""- fleet_key = _fleet_budget_key(now)- if not _reserve(fleet_key, queries, config.max_queries_per_hour):+def reserve_queries(+ config: AnomalyScoringConfig,+ team_id: int,+ queries: int,+ now: datetime.datetime,+ kind: ScoringRun = ScoringRun.TRAIN,+) -> bool:+ """Charge ``queries`` to both hourly budgets of ``kind``, or to neither when one of them has too few left."""+ fleet_key = _fleet_budget_key(now, kind)+ if not _reserve(fleet_key, queries, _fleet_cap(config, kind)):return False- if _reserve(_project_budget_key(team_id, now), queries, config.max_queries_per_project_per_hour):+ if _reserve(_project_budget_key(team_id, now, kind), queries, _project_cap(config, kind)):return Truecache.decr(fleet_key, queries)return False@@ -267,47 +302,62 @@ def reserve_queries(config: AnomalyScoringConfig, team_id: int, queries: int, nodef within_budget(order: list[int],team_ids: dict[int, int],- budget: QueryBudget,+ budgets: dict[ScoringRun, QueryBudget],limit: int,- costs_for: Callable[[list[int]], dict[int, int]],- reserve: Callable[[int, int], bool],-) -> tuple[dict[int, int], int]:- """The insights of ``order`` whose cost fits both budgets, up to ``limit``, with their costs, and the skip count.+ plans_for: Callable[[list[int]], dict[int, RunPlan]],+ reserve: Callable[[int, RunPlan], bool],+) -> tuple[dict[int, RunPlan], int]:+ """The insights of ``order`` whose cost fits both budgets of their kind, up to ``limit``, and the skip count.A pick that does not fit is skipped and the walk continues, so a large insight does not block smaller ones.- ``budget`` is a snapshot that skips most misses without a cache call. ``reserve`` charges a pick for real.- ``costs_for`` loads costs in batches of ``limit``, and only for insights that the snapshot can still fit.+ ``budgets`` are snapshots that skip most misses without a cache call. ``reserve`` charges a pick for real.+ ``plans_for`` loads plans in batches of ``limit``, and only for insights that a snapshot can still fit."""- fleet = budget.fleet- projects = dict(budget.projects)- picked: dict[int, int] = {}+ fleets = {kind: budget.fleet for kind, budget in budgets.items()}+ projects = {kind: dict(budget.projects) for kind, budget in budgets.items()}+ picked: dict[int, RunPlan] = {}skipped = 0batch_size = max(1, limit)for start in range(0, len(order), batch_size):if len(picked) >= limit:breakbatch = order[start : start + batch_size]- affordable = [insight_id for insight_id in batch if fleet > 0 and projects.get(team_ids[insight_id], 0) > 0]- costs = costs_for(affordable) if affordable else {}+ affordable = [+ insight_id+ for insight_id in batch+ if any(fleets[kind] > 0 and projects[kind].get(team_ids[insight_id], 0) > 0 for kind in budgets)+ ]+ plans = plans_for(affordable) if affordable else {}for insight_id in batch:if len(picked) >= limit:breakteam_id = team_ids[insight_id]- cost = costs.get(insight_id)- if cost is None or cost > fleet or cost > projects.get(team_id, 0) or not reserve(team_id, cost):+ plan = plans.get(insight_id)+ kind = budget_kind(plan.run if plan else None)+ if (+ plan is None+ or kind not in budgets+ or plan.queries > fleets[kind]+ or plan.queries > projects[kind].get(team_id, 0)+ or not reserve(team_id, plan)+ ):skipped += 1continue- picked[insight_id] = cost- fleet -= cost- projects[team_id] -= cost+ picked[insight_id] = plan+ fleets[kind] -= plan.queries+ projects[kind][team_id] -= plan.queriesreturn picked, skipped-def _query_costs(team: Team, insight_ids: list[int]) -> dict[int, int]:+def _run_plans(+ team: Team, insight_ids: list[int], config: AnomalyScoringConfig, now: datetime.datetime+) -> dict[int, RunPlan]:insights = insights_including_soft_deleted_for_team(team_id=team.id, insight_ids=insight_ids)- costs = {insight.id: query_cost(insight, team) for insight in insights}+ states = load_states(team, insights) if config.train_score_split else {}+ plans = {insight.id: plan_run(insight, team, states.get(insight.id), config, now) for insight in insights}# An insight that no longer exists sends no query, but one query is the safe charge.- return {insight_id: costs.get(insight_id, 1) for insight_id in insight_ids}+ missing = RunPlan(queries=1, run=ScoringRun.TRAIN if config.train_score_split else None)+ return {insight_id: plans.get(insight_id, missing) for insight_id in insight_ids}def select_insights(@@ -366,21 +416,22 @@ def select_insights(]team_ids = {insight_id: due_by_id[insight_id].team_id for insight_id in order}- def costs_for(insight_ids: list[int]) -> dict[int, int]:- costs: dict[int, int] = {}+ def plans_for(insight_ids: list[int]) -> dict[int, RunPlan]:+ plans: dict[int, RunPlan] = {}for team_id in sorted({team_ids[insight_id] for insight_id in insight_ids}):team_batch = [insight_id for insight_id in insight_ids if team_ids[insight_id] == team_id]- costs.update(_query_costs(teams[team_id], team_batch))- return costs+ plans.update(_run_plans(teams[team_id], team_batch, config, now))+ return plans- # Read the budget before any cost, so a spent hour loads no insight.+ kinds = [ScoringRun.TRAIN, ScoringRun.SCORE] if config.train_score_split else [ScoringRun.TRAIN]+ # Read the budgets before any cost, so a spent hour loads no insight.picked, skipped = within_budget(order,team_ids,- remaining_budget(config, set(team_ids.values()), now),+ {kind: remaining_budget(config, set(team_ids.values()), now, kind) for kind in kinds},limit,- costs_for,- reserve=lambda team_id, cost: reserve_queries(config, team_id, cost, now),+ plans_for,+ reserve=lambda team_id, plan: reserve_queries(config, team_id, plan.queries, now, budget_kind(plan.run)),)picks = [InsightPick(@@ -388,8 +439,10 @@ def select_insights(insight_id=insight_id,scored_through=state.get(insight_id, {}),interval=due_by_id[insight_id].interval,- query_cost=cost,+ query_cost=plan.queries,+ run=plan.run,+ refresh_buckets=plan.refresh_buckets,)- for insight_id, cost in picked.items()+ for insight_id, plan in picked.items()]return Selection(picks=picks, candidates=len(order), skipped_for_budget=skipped)diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.pyindex 990bac80ace1..15b6f8053bad 100644--- a/products/signals/backend/anomaly_scores/scoring.py+++ b/products/signals/backend/anomaly_scores/scoring.py@@ -36,6 +36,7 @@ from products.signals.backend.anomaly_scores.config import (IntervalScoring,ScoringLimits,ScoringProfile,+ ScoringRun,resolve_profile,)@@ -77,6 +78,43 @@ class ScoredSeries:points: list[ScoredPoint]+@frozen+class SeriesWindow:+ """The newest completed values of one series, oldest first. A score step reads them as the history of its new buckets."""++ # Aware bucket start of the first value. Trends zero-fills buckets, so each next value is one interval later.+ first_bucket: datetime+ values: list[float]+++@frozen+class StateIdentity:+ """What a stored state was built from. A score step needs the same identity, or the visit trains."""++ interval: IntervalType+ detector_version: str+ query_hash: str+++@frozen+class TrainedState:+ identity: StateIdentity+ trained_at: datetime+ # The newest scored bucket start over all series, or None when no series had a completed bucket.+ scored_through: datetime | None+ windows: dict[str, SeriesWindow]+ # Stored series that the last score step did not get back.+ missing_series: int = 0+++@frozen+class SplitRun:+ """A visit of the train and score split. A score step needs ``state``; with no matching state, the visit trains."""++ refresh_buckets: int+ state: TrainedState | None = None++@frozenclass InsightScores:interval: IntervalType@@ -84,6 +122,21 @@ class InsightScores:detector: strdetector_version: strseries: list[ScoredSeries]+ # None when the split is off. Then no state is stored.+ run: ScoringRun | None = None+ # The state for the next visit to store.+ state: TrainedState | None = None+ # Series of a score step with no stored window. A later train step scores them.+ unmatched_series: int = 0+++@frozen+class _FetchedSeries:+ series_key: str+ label: str+ breakdown_value: str | None+ values: list[float]+ buckets: list[datetime]def _short_hash(payload: Any) -> str:@@ -96,6 +149,15 @@ def detector_version(scoring: IntervalScoring) -> str:return _short_hash(dataclasses.asdict(scoring))+def query_hash(query: TrendsQuery, timezone: ZoneInfo, limits: ScoringLimits = DEFAULT_LIMITS) -> str:+ """Identifies what the scoring query counts, without its date range, so an edited insight retrains."""+ dumped = build_scoring_query(query, buckets=1, limits=limits).model_dump(mode="json", exclude_none=True)+ # The fetch size changes with every visit. `daysOfWeek` stays, because it changes the buckets.+ (dumped.get("dateRange") or {}).pop("date_from", None)+ # Stored bucket starts step in wall-clock time, so a timezone change needs new windows.+ return _short_hash({"query": dumped, "timezone": timezone.key})++def series_key(insight_id: int, series_definition: dict[str, Any], breakdown_value: str | None) -> str:return _short_hash({"insight": insight_id, "series": series_definition, "breakdown": breakdown_value})@@ -119,7 +181,7 @@ def trends_query_for(insight: Insight, profile: ScoringProfile) -> TrendsQuery:if trends.trendsFilter and trends.trendsFilter.display == ChartDisplayType.CALENDAR_HEATMAP:# The chart has its own runner and shows hours of the week, not a time series.raise UnsupportedInsightError("Calendar heatmaps are not scored")- interval_scoring(profile, _interval(trends))+ interval_scoring(profile, scoring_interval(trends))return trends@@ -170,12 +232,13 @@ def buckets_to_score(return max(count, 1)-def fetch_buckets(query: TrendsQuery, scoring: IntervalScoring, to_score: int) -> int:+def fetch_buckets(query: TrendsQuery, scoring: IntervalScoring, to_score: int, *, history: int | None = None) -> int:"""Completed buckets the scoring query fetches: the history of the oldest bucket to score, then the rest.The count has one bucket more than the scores read, so a first bucket that the date range cuts short is never read.+ ``history`` replaces the detector history, for a score step that reads its history from the stored state."""- count = history_buckets(scoring) + to_score+ count = (history_buckets(scoring) if history is None else history) + to_scoresmoothing = query.trendsFilter.smoothingIntervals if query.trendsFilter else Noneif smoothing and smoothing > _MAX_SMOOTHING_BUCKETS:# The field has no upper bound, and the fetch grows with it.@@ -184,7 +247,7 @@ def fetch_buckets(query: TrendsQuery, scoring: IntervalScoring, to_score: int) -# Trends smooths over the fetched buckets only, so the oldest values need their own trailing window.count += smoothing - 1days_of_week = query.dateRange.daysOfWeek if query.dateRange else None- if days_of_week and _interval(query) == IntervalType.DAY:+ if days_of_week and scoring_interval(query) == IntervalType.DAY:# Trends drops the buckets of the days that the filter leaves out.count = math.ceil(count * 7 / len(set(days_of_week)))return count@@ -197,7 +260,7 @@ def build_scoring_query(query: TrendsQuery, buckets: int, limits: ScoringLimits"""days_of_week = query.dateRange.daysOfWeek if query.dateRange else Noneupdate: dict[str, Any] = {- "dateRange": DateRange(date_from=f"-{buckets}{_DATE_UNITS[_interval(query)]}", daysOfWeek=days_of_week),+ "dateRange": DateRange(date_from=f"-{buckets}{_DATE_UNITS[scoring_interval(query)]}", daysOfWeek=days_of_week),"compareFilter": None,}if query.trendsFilter and query.trendsFilter.display == ChartDisplayType.METRIC:@@ -227,6 +290,20 @@ def scoring_query_settings(queries: int) -> HogQLGlobalSettings:)+def supports_score_step(query: TrendsQuery) -> bool:+ """A score step steps over stored buckets one interval at a time, so the buckets must have no gaps."""+ # Trends drops the buckets of the days that the filter leaves out.+ return not (query.dateRange and query.dateRange.daysOfWeek)+++def state_identity(query: TrendsQuery, scoring: IntervalScoring, team: Team, limits: ScoringLimits) -> StateIdentity:+ return StateIdentity(+ interval=scoring_interval(query),+ detector_version=detector_version(scoring),+ query_hash=query_hash(query, team.timezone_info, limits),+ )++def query_cost(insight: Insight, team: Team) -> int:"""The ClickHouse queries that scoring this insight sends, so the tick can budget them before it runs.@@ -283,6 +360,67 @@ def score_series(return points+def window_buckets(window: SeriesWindow, interval: IntervalType) -> list[datetime]:+ buckets: list[datetime] = []+ start = window.first_bucket+ for _ in window.values:+ buckets.append(start)+ start = bucket_end(start, interval)+ return buckets+++def train_window(+ values: Sequence[float], buckets: Sequence[datetime], *, history: int, interval: IntervalType, now: datetime+) -> SeriesWindow | None:+ """The newest ``history`` completed values of a fetched series, or None when they do not step one interval apart."""+ completed = [i for i, start in enumerate(buckets) if bucket_end(start, interval) <= now][-history:]+ if not completed:+ return None+ window = SeriesWindow(first_bucket=buckets[completed[0]], values=[float(values[i]) for i in completed])+ # A repeated hour at a DST fall-back breaks the steps. The series waits for a train step after the change.+ if window_buckets(window, interval) != [buckets[i] for i in completed]:+ return None+ return window+++def score_window(+ window: SeriesWindow,+ values: Sequence[float],+ buckets: Sequence[datetime],+ *,+ refresh: int,+ scoring: IntervalScoring,+ interval: IntervalType,+ now: datetime,+ limits: ScoringLimits = DEFAULT_LIMITS,+) -> tuple[list[ScoredPoint], SeriesWindow] | None:+ """Merge freshly fetched buckets into a stored window, score the buckets after it, and trim it to its history.++ A fresh value replaces the stored value of its bucket for the newest ``refresh`` stored buckets, so late events+ reach the window. Older fresh buckets can be cut short or smoothed over too few buckets, so they are not read.+ Returns None when the fresh buckets leave a gap after the window. The series then keeps its window.+ """+ stored = window_buckets(window, interval)+ merged = dict(zip(stored, window.values))+ refresh_from = [*stored, bucket_end(stored[-1], interval)][max(0, len(stored) - refresh)]+ fresh = [(start, value) for start, value in zip(buckets, values) if bucket_end(start, interval) <= now]+ for start, value in fresh:+ if start >= refresh_from:+ merged[start] = float(value)+ starts = list(stored)+ next_start = bucket_end(stored[-1], interval)+ while next_start in merged:+ starts.append(next_start)+ next_start = bucket_end(next_start, interval)+ if any(start >= next_start for start, _ in fresh):+ return None++ data = [merged[start] for start in starts]+ points = score_series(data, starts, scoring=scoring, interval=interval, now=now, after=stored[-1], limits=limits)+ keep = history_buckets(scoring)+ return points, SeriesWindow(first_bucket=starts[-keep:][0], values=data[-keep:])++def score_insight(insight: Insight,team: Team,@@ -290,17 +428,166 @@ def score_insight(now: datetime,after: datetime | None = None,limits: ScoringLimits = DEFAULT_LIMITS,+ split: SplitRun | None = None,) -> InsightScores:+ """Score the new buckets of an insight.++ With ``split``, a stored state with the insight's identity makes this a score step, and anything else a+ train step. Both return the state for the next visit.+ """profile = resolve_profile(team, insight)query = trends_query_for(insight, profile)- interval = _interval(query)+ interval = scoring_interval(query)scoring = interval_scoring(profile, interval)timezone = team.timezone_info+ identity = state_identity(query, scoring, team, limits) if split is not None else None+ state = split.state if split is not None else None+ if state is not None and (state.identity != identity or not supports_score_step(query)):+ state = None+ scores = InsightScores(+ interval=interval,+ profile=profile.name,+ detector=str(scoring.detector["type"]),+ detector_version=detector_version(scoring),+ series=[],+ )++ if split is not None and state is not None and identity is not None:+ to_score = buckets_to_score(interval, now=now, after=state.scored_through, timezone=timezone, limits=limits)+ # One bucket more than the score step reads, as in the full fetch.+ buckets = fetch_buckets(query, scoring, to_score, history=split.refresh_buckets + 1)+ fetched = _fetch(insight, team, query, buckets, limits, ScoringRun.SCORE)+ return _score_step(+ scores,+ fetched,+ state,+ refresh=split.refresh_buckets,+ scoring=scoring,+ interval=interval,+ now=now,+ limits=limits,+ )+to_score = buckets_to_score(interval, now=now, after=after, timezone=timezone, limits=limits)- scoring_query = build_scoring_query(query, fetch_buckets(query, scoring, to_score), limits)+ run = ScoringRun.TRAIN if split is not None else None+ fetched = _fetch(insight, team, query, fetch_buckets(query, scoring, to_score), limits, run)+ series = [+ ScoredSeries(+ series_key=line.series_key,+ label=line.label,+ breakdown_value=line.breakdown_value,+ points=score_series(+ line.values,+ line.buckets,+ scoring=scoring,+ interval=interval,+ now=now,+ after=after,+ limits=limits,+ ),+ )+ for line in fetched+ ]+ if identity is None:+ return dataclasses.replace(scores, series=series)+ windows = {+ line.series_key: window+ for line in fetched+ if (+ window := train_window(+ line.values, line.buckets, history=history_buckets(scoring), interval=interval, now=now+ )+ )+ is not None+ }+ return dataclasses.replace(+ scores,+ series=series,+ run=ScoringRun.TRAIN,+ state=TrainedState(+ identity=identity, trained_at=now, scored_through=_scored_through(windows, interval), windows=windows+ ),+ )+++def _score_step(+ scores: InsightScores,+ fetched: list[_FetchedSeries],+ state: TrainedState,+ *,+ refresh: int,+ scoring: IntervalScoring,+ interval: IntervalType,+ now: datetime,+ limits: ScoringLimits,+) -> InsightScores:+ series: list[ScoredSeries] = []+ windows = dict(state.windows)+ unmatched = 0+ returned: set[str] = set()+ for line in fetched:+ returned.add(line.series_key)+ window = state.windows.get(line.series_key)+ # The short range can rank a different top N of breakdown values. A new value waits for a train step.+ scored = (+ None+ if window is None+ else score_window(+ window,+ line.values,+ line.buckets,+ refresh=refresh,+ scoring=scoring,+ interval=interval,+ now=now,+ limits=limits,+ )+ )+ if scored is None:+ unmatched += 1+ continue+ points, windows[line.series_key] = scored+ series.append(+ ScoredSeries(+ series_key=line.series_key, label=line.label, breakdown_value=line.breakdown_value, points=points+ )+ )+ return dataclasses.replace(+ scores,+ series=series,+ run=ScoringRun.SCORE,+ unmatched_series=unmatched,+ # A stored series that is missing from the result keeps its window.+ state=dataclasses.replace(+ state,+ scored_through=_scored_through(windows, interval),+ windows=windows,+ missing_series=len(set(state.windows) - returned),+ ),+ )+++def _scored_through(windows: dict[str, SeriesWindow], interval: IntervalType) -> datetime | None:+ ends = [window_buckets(window, interval)[-1] for window in windows.values() if window.values]+ return max(ends).astimezone(UTC) if ends else None+++def _fetch(+ insight: Insight,+ team: Team,+ query: TrendsQuery,+ buckets: int,+ limits: ScoringLimits,+ run: ScoringRun | None,+) -> list[_FetchedSeries]:+ timezone = team.timezone_info+ scoring_query = build_scoring_query(query, buckets, limits)tag_queries(**get_team_query_tags(team), insight_id=insight.pk)tag_queries(product=Product.SIGNALS, feature=Feature.ALERTING)+ if run is not None:+ # Splits the scoring queries in `query_log`, so train and score cost can be compared.+ tag_queries(name=f"anomaly_scoring_{run.value}")runner = TrendsQueryRunner(query=scoring_query,team=team,@@ -333,38 +620,24 @@ def score_insight(skipped=len(results) - len(current_results),)- series: list[ScoredSeries] = []+ series: list[_FetchedSeries] = []key_counts: dict[str, int] = {}for result in current_results[: limits.max_series]:- buckets = [_parse_bucket(day, timezone) for day in result.get("days") or []]breakdown_value = _breakdown_label(result.get("breakdown_value"))key = series_key(insight.pk, _series_definition(scoring_query, result), breakdown_value)repeats = key_counts.get(key, 0)key_counts[key] = repeats + 1series.append(- ScoredSeries(+ _FetchedSeries(# Lines that differ only by name share a key. Event uuids come from the key, so each repeat needs its own.series_key=key if repeats == 0 else _short_hash({"key": key, "repeat": repeats}),label=str(result.get("label") or ""),breakdown_value=breakdown_value,- points=score_series(- result.get("data") or [],- buckets,- scoring=scoring,- interval=interval,- now=now,- after=after,- limits=limits,- ),+ values=list(result.get("data") or []),+ buckets=[_parse_bucket(day, timezone) for day in result.get("days") or []],))- return InsightScores(- interval=interval,- profile=profile.name,- detector=str(scoring.detector["type"]),- detector_version=detector_version(scoring),- series=series,- )+ return seriesdef _formulas(query: TrendsQuery) -> list[str]:@@ -398,7 +671,7 @@ def _series_definition(query: TrendsQuery, result: dict[str, Any]) -> dict[str,return _series_node_definition(query.series[order])-def _interval(query: TrendsQuery) -> IntervalType:+def scoring_interval(query: TrendsQuery) -> IntervalType:return query.interval or IntervalType.DAYdiff --git a/products/signals/backend/anomaly_scores/state.py b/products/signals/backend/anomaly_scores/state.pynew file mode 100644index 000000000000..48c870bc3d33--- /dev/null+++ b/products/signals/backend/anomaly_scores/state.py@@ -0,0 +1,168 @@+"""The stored state of the train and score split, and the choice between a train step and a score step.++A train step fetches the full detector history and stores the newest values of each series. A score+step fetches only the newest buckets and scores them on the stored values. See `AnomalyScoringState`.+"""++import math+import datetime+import dataclasses+from collections.abc import Collection+from typing import Any+from zoneinfo import ZoneInfo++from posthog.schema import IntervalType++from posthog.dataclasses import frozen+from posthog.interval_specs import interval_spec+from posthog.models import Team++from products.product_analytics.backend.facade.models import Insight+from products.signals.backend.anomaly_scores.config import (+ DEFAULT_LIMITS,+ AnomalyScoringConfig,+ ScoringRun,+ resolve_profile,+)+from products.signals.backend.anomaly_scores.scoring import (+ SeriesWindow,+ StateIdentity,+ TrainedState,+ buckets_to_score,+ interval_scoring,+ query_cost,+ scoring_interval,+ state_identity,+ supports_score_step,+ trends_query_for,+)+from products.signals.backend.models import AnomalyScoringState+++@frozen+class RunPlan:+ """What one pick does and the ClickHouse queries it sends."""++ queries: int+ # None when the split is off.+ run: ScoringRun | None = None+ refresh_buckets: int = 0+++def needs_training(+ state: TrainedState | None,+ *,+ identity: StateIdentity,+ now: datetime.datetime,+ timezone: ZoneInfo,+ train_every: int,+ max_backfill: int,+) -> bool:+ if state is None or state.identity != identity or not state.windows or state.scored_through is None:+ return True+ interval = identity.interval+ if state.trained_at + interval_spec(interval).period * train_every <= now:+ return True+ # More due buckets than one visit scores. A train step scores the newest of them, as a first visit does.+ due = buckets_to_score(+ interval,+ now=now,+ after=state.scored_through,+ timezone=timezone,+ limits=dataclasses.replace(DEFAULT_LIMITS, max_backfill_buckets=max_backfill + 1),+ )+ if due > max_backfill:+ return True+ return state.missing_series * 2 > len(state.windows)+++def plan_run(+ insight: Insight, team: Team, state: TrainedState | None, config: AnomalyScoringConfig, now: datetime.datetime+) -> RunPlan:+ queries = query_cost(insight, team)+ if not config.train_score_split:+ return RunPlan(queries=queries)+ try:+ profile = resolve_profile(team, insight)+ query = trends_query_for(insight, profile)+ interval = scoring_interval(query)+ scoring = interval_scoring(profile, interval)+ identity = state_identity(query, scoring, team, config.limits)+ except Exception:+ # The scorer fails on the same query and reports the error. A train step charges the larger budget.+ return RunPlan(queries=queries, run=ScoringRun.TRAIN)+ refresh = config.refresh_buckets.get(interval, 0)+ trains = not supports_score_step(query) or needs_training(+ state,+ identity=identity,+ now=now,+ timezone=team.timezone_info,+ train_every=config.train_every_buckets[interval],+ max_backfill=config.limits.max_backfill_buckets,+ )+ return RunPlan(queries=queries, run=ScoringRun.TRAIN if trains else ScoringRun.SCORE, refresh_buckets=refresh)+++def load_states(team: Team, insights: Collection[Insight]) -> dict[int, TrainedState]:+ """Insight id to the stored state of the insight's own scoring profile."""+ profiles = {insight.id: resolve_profile(team, insight).name for insight in insights}+ if not profiles:+ return {}+ rows = AnomalyScoringState.objects.for_team(team.id).filter(insight_id__in=list(profiles))+ return {+ row.insight_id: _from_row(row, team.timezone_info)+ for row in rows+ if profiles.get(row.insight_id) == row.profile+ }+++def load_state(team: Team, insight: Insight) -> TrainedState | None:+ return load_states(team, [insight]).get(insight.id)+++def save_state(team: Team, insight_id: int, profile: str, state: TrainedState) -> None:+ fields = {+ "interval": state.identity.interval.value,+ "detector_version": state.identity.detector_version,+ "query_hash": state.identity.query_hash,+ "trained_at": state.trained_at,+ "scored_through": state.scored_through,+ "missing_series": state.missing_series,+ "series": {key: _window_json(window) for key, window in state.windows.items()},+ }+ # The scope filters on the project's canonical team, and save() writes that team for a new row.+ AnomalyScoringState.objects.for_team(team.id).update_or_create(+ insight_id=insight_id, profile=profile, defaults=fields, create_defaults={**fields, "team_id": team.id}+ )+++def _window_json(window: SeriesWindow) -> dict[str, Any]:+ return {+ "first_bucket": window.first_bucket.isoformat(),+ # JSON has no NaN.+ "values": [value if math.isfinite(value) else None for value in window.values],+ }+++def _from_row(row: AnomalyScoringState, timezone: datetime.tzinfo) -> TrainedState:+ windows: dict[str, SeriesWindow] = {}+ for key, stored in (row.series or {}).items():+ try:+ window = SeriesWindow(+ # Bucket starts step in wall-clock time of the project timezone.+ first_bucket=datetime.datetime.fromisoformat(stored["first_bucket"]).astimezone(timezone),+ values=[math.nan if value is None else float(value) for value in stored["values"]],+ )+ except (KeyError, TypeError, ValueError):+ continue+ if window.values:+ windows[key] = window+ return TrainedState(+ identity=StateIdentity(+ interval=IntervalType(row.interval), detector_version=row.detector_version, query_hash=row.query_hash+ ),+ trained_at=row.trained_at,+ scored_through=row.scored_through,+ windows=windows,+ missing_series=row.missing_series,+ )diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.pyindex 81ee7638aefd..104194333bf0 100644--- a/products/signals/backend/anomaly_scores/tick.py+++ b/products/signals/backend/anomaly_scores/tick.py@@ -38,8 +38,10 @@ from products.signals.backend.anomaly_scores.config import (ANOMALY_SCORING_FLAG_DISTINCT_ID,DEFAULT_LIMITS,ScoringLimits,+ ScoringRun,read_config,read_payload,+ resolve_profile,)from products.signals.backend.anomaly_scores.emission import emit, score_eventsfrom products.signals.backend.anomaly_scores.metrics import record_candidates, record_pick@@ -52,10 +54,12 @@ from products.signals.backend.anomaly_scores.queue import ()from products.signals.backend.anomaly_scores.scoring import (SCORING_QUERY_BUDGET_SECONDS,+ SplitRun,UnsupportedInsightError,score_insight,)from products.signals.backend.anomaly_scores.selection import parse_selection_config+from products.signals.backend.anomaly_scores.state import load_state, save_statelogger = structlog.get_logger(__name__)@@ -116,6 +120,11 @@ class ScoreInsightResult:status: strevents: intseries: int = 0+ # What the visit ran. A score pick with no matching state trains. None when the split is off.+ run: ScoringRun | None = None+ # Series of a score step with no stored window, and stored series missing from its result.+ unmatched_series: int = 0+ missing_series: int = 0@frozen@@ -133,6 +142,9 @@ class AnomalyScoringTickResult:queries: int = 0skipped_for_budget: int = 0stopped_at_capacity: bool = False+ # Started picks per kind of the train and score split.+ run_train: int = 0+ run_score: int = 0def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:@@ -197,7 +209,7 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigherror=str(error)[:200],duration_ms=duration_ms,)- record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0)+ record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0, run=pick.run)if status is None:raise# A full cluster is not a fault of the insight, so it keeps the attempt cooldown.@@ -214,8 +226,11 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsighseries=result.series,events=result.events,duration_ms=duration_ms,+ run=result.run,+ unmatched_series=result.unmatched_series,+ missing_series=result.missing_series,)- record_pick(status=result.status, interval=interval, duration_ms=duration_ms, events=result.events)+ record_pick(status=result.status, interval=interval, duration_ms=duration_ms, events=result.events, run=result.run)return result@@ -234,6 +249,12 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsighteam = Team.objects.get(id=pick.team_id)# Set before scoring, so a failure also waits for the cooldown instead of taking a slot on every tick.mark_attempted(pick.insight_id, ATTEMPT_COOLDOWN)+ split = None+ if pick.run is not None:+ split = SplitRun(+ refresh_buckets=pick.refresh_buckets,+ state=load_state(team, insight) if pick.run == ScoringRun.SCORE else None,+ )scores = score_insight(insight,team,@@ -241,13 +262,23 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh# Every series is filtered again on its own newest bucket below. A new series gets up to the backfill cap.after=min(pick.scored_through.values()) if pick.scored_through else None,limits=input.limits,+ split=split,)events = score_events(team_id=team.id, insight_id=insight.id, scores=scores, scored_through=pick.scored_through, scored_at=now)emit(team, events)+ # After the emit: a lost state write only scores buckets again, and their events collapse by uuid.+ # A state write before a failed emit would skip those buckets for good.+ if scores.state is not None:+ save_state(team, insight.id, resolve_profile(team, insight).name, scores.state)return ScoreInsightResult(- status=STATUS_SCORED if events else STATUS_UP_TO_DATE, events=len(events), series=len(scores.series)+ status=STATUS_SCORED if events else STATUS_UP_TO_DATE,+ events=len(events),+ series=len(scores.series),+ run=scores.run,+ unmatched_series=scores.unmatched_series,+ missing_series=scores.state.missing_series if scores.state else 0,)@@ -264,6 +295,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:queries=result.queries,skipped_for_budget=result.skipped_for_budget,stopped_at_capacity=result.stopped_at_capacity,+ run_train=result.run_train,+ run_score=result.run_score,)ph_background_capture()(distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,@@ -280,6 +313,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:"queries": result.queries,"skipped_for_budget": result.skipped_for_budget,"stopped_at_capacity": result.stopped_at_capacity,+ "run_train": result.run_train,+ "run_score": result.run_score,},)@@ -368,6 +403,8 @@ class AnomalyScoringTickWorkflow:queries=sum(pick.query_cost for pick in started_picks),skipped_for_budget=plan.skipped_for_budget,stopped_at_capacity=statuses[STATUS_AT_CAPACITY] > 0,+ run_train=sum(pick.run == ScoringRun.TRAIN for pick in started_picks),+ run_score=sum(pick.run == ScoringRun.SCORE for pick in started_picks),)await workflow.execute_activity(capture_anomaly_scoring_tick_activity,diff --git a/products/signals/backend/migrations/0199_anomalyscoringstate.py b/products/signals/backend/migrations/0199_anomalyscoringstate.pynew file mode 100644index 000000000000..98a7597c3e81--- /dev/null+++ b/products/signals/backend/migrations/0199_anomalyscoringstate.py@@ -0,0 +1,63 @@+# Generated by Django 5.2.18 on 2026-10-24 19:12++import django.db.models.manager+import django.db.models.deletion+from django.db import migrations, models++import posthog.uuidt+++class Migration(migrations.Migration):+ dependencies = [+ ("posthog", "1412_drop_retired_desktop_file_system_tables"),+ ("signals", "0198_useractivitysummary_computed"),+ ]++ operations = [+ migrations.CreateModel(+ name="AnomalyScoringState",+ fields=[+ (+ "id",+ models.UUIDField(+ default=posthog.uuidt.uuid7,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("insight_id", models.BigIntegerField()),+ ("profile", models.CharField(max_length=64)),+ ("interval", models.CharField(max_length=16)),+ ("detector_version", models.CharField(max_length=32)),+ ("query_hash", models.CharField(max_length=32)),+ ("trained_at", models.DateTimeField()),+ ("scored_through", models.DateTimeField(blank=True, null=True)),+ ("missing_series", models.PositiveIntegerField(default=0)),+ ("series", models.JSONField(default=dict)),+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("updated_at", models.DateTimeField(auto_now=True)),+ (+ "team",+ models.ForeignKey(+ db_constraint=False,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="+",+ to="posthog.team",+ ),+ ),+ ],+ options={+ "default_manager_name": "all_teams",+ "constraints": [+ models.UniqueConstraint(+ fields=("team", "insight_id", "profile"),+ name="signals_anomaly_scoring_state_unique",+ )+ ],+ },+ managers=[+ ("all_teams", django.db.models.manager.Manager()),+ ],+ ),+ ]diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txtindex ad1b5ceff741..5e45e928fd34 100644--- a/products/signals/backend/migrations/max_migration.txt+++ b/products/signals/backend/migrations/max_migration.txt@@ -1 +1 @@-0198_useractivitysummary_computed+0199_anomalyscoringstatediff --git a/products/signals/backend/models.py b/products/signals/backend/models.pyindex 6cf3a6f76b6f..430f7bda2b58 100644--- a/products/signals/backend/models.py+++ b/products/signals/backend/models.py@@ -4013,3 +4013,42 @@ class UserActivitySummaryQueueCursor(models.Model):source = models.CharField(max_length=40, primary_key=True)position = models.DateTimeField()updated_at = models.DateTimeField(auto_now=True)+++class AnomalyScoringState(TeamScopedRootMixin, UUIDModel):+ """The stored series tail of one insight and scoring profile of the anomaly scoring tick.++ A train step fetches the full detector history and writes the tail. A score step fetches only the+ newest buckets, merges them into the tail, and scores the new buckets on it. See+ `anomaly_scores/state.py`. `$anomaly_score` events stay output only.+ """++ # See SignalReportRefund.all_teams for rationale.+ all_teams = models.Manager() # noqa: DJ012++ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, db_constraint=False, related_name="+")+ # Not a foreign key: insights belong to another product. A deleted insight leaves a small row behind.+ insight_id = models.BigIntegerField()+ profile = models.CharField(max_length=64)+ # The insight as the state was built for it. A change to any of them makes the next visit train.+ interval = models.CharField(max_length=16)+ detector_version = models.CharField(max_length=32)+ query_hash = models.CharField(max_length=32)+ trained_at = models.DateTimeField()+ # The newest scored bucket start over all series. Null when the insight returned no completed bucket.+ scored_through = models.DateTimeField(null=True, blank=True)+ # Stored series that the last score step did not get back. More than half of them makes the next visit train.+ missing_series = models.PositiveIntegerField(default=0)+ # Series key to `{"first_bucket": ISO bucket start, "values": [...]}`, with the newest values the detector reads.+ series = models.JSONField(default=dict)++ created_at = models.DateTimeField(auto_now_add=True)+ updated_at = models.DateTimeField(auto_now=True)++ class Meta:+ default_manager_name = "all_teams"+ constraints = [+ models.UniqueConstraint(+ fields=["team", "insight_id", "profile"], name="signals_anomaly_scoring_state_unique"+ ),+ ]diff --git a/products/signals/backend/test/test_anomaly_scores.py b/products/signals/backend/test/test_anomaly_scores.pyindex 9b9dcc60337d..f9f6930b5cd8 100644--- a/products/signals/backend/test/test_anomaly_scores.py+++ b/products/signals/backend/test/test_anomaly_scores.py@@ -27,16 +27,25 @@ from posthog.interval_specs import interval_specfrom posthog.models import Teamfrom products.product_analytics.backend.facade.models import Insight-from products.signals.backend.anomaly_scores.config import DEFAULT_LIMITS, DEFAULT_PROFILE, IntervalScoring+from products.signals.backend.anomaly_scores.config import DEFAULT_LIMITS, DEFAULT_PROFILE, IntervalScoring, ScoringRunfrom products.signals.backend.anomaly_scores.scoring import (+ SeriesWindow,+ SplitRun,+ StateIdentity,+ TrainedState,UnsupportedInsightError,+ bucket_end,buckets_to_score,fetch_buckets,+ history_buckets,query_cost,score_insight,score_series,+ score_window,+ train_window,trends_query_for,)+from products.signals.backend.anomaly_scores.state import needs_trainingfrom products.signals.backend.anomaly_scores.tick import _INSIGHT_TIMEOUTNEW_YORK = ZoneInfo("America/New_York")@@ -178,6 +187,68 @@ class TestScoreSeries(SimpleTestCase):assert all(point.score is None and point.flag is None for point in points)+class TestScoreWindow(SimpleTestCase):+ @parameterized.expand(+ [+ ("day", IntervalType.DAY, False),+ ("week", IntervalType.WEEK, False),+ ("hour", IntervalType.HOUR, False),+ ("day_late_events", IntervalType.DAY, True),+ ("hour_late_events", IntervalType.HOUR, True),+ ]+ )+ def test_a_score_step_on_the_stored_window_scores_like_the_full_history(+ self, _name: str, interval: IntervalType, late_events: bool+ ) -> None:+ scoring = DEFAULT_PROFILE.intervals[interval]+ history = history_buckets(scoring)+ count = history + 40+ values = list(np.random.default_rng(2).normal(100, 10, size=count))+ values[-1] = 400.0+ buckets = _buckets(datetime(2025, 1, 6, tzinfo=UTC), interval, count)+ trained_at = bucket_end(buckets[-3], interval) + timedelta(minutes=5)+ scored_at = bucket_end(buckets[-1], interval) + timedelta(minutes=5)+ window = train_window(values[:-1], buckets[:-1], history=history, interval=interval, now=trained_at)+ assert window is not None+ later = list(values)+ if late_events:+ # Events that arrive after the train step change a bucket that the score step fetches again.+ later[-4] += 50.0+ refresh = 3+ # The score step fetches the refresh buckets, one guard bucket, and the two new buckets.+ fetched = slice(count - (refresh + 1 + 2), count)++ scored = score_window(+ window, later[fetched], buckets[fetched], refresh=refresh, scoring=scoring, interval=interval, now=scored_at+ )++ assert scored is not None+ points, updated = scored+ expected = score_series(later, buckets, scoring=scoring, interval=interval, now=scored_at, after=buckets[-3])+ assert [(point.bucket_start, point.score, point.flag) for point in points] == [+ (point.bucket_start, point.score, point.flag) for point in expected+ ]+ assert [point.bucket_start for point in points] == buckets[-2:]+ assert points[-1].flag is True+ assert updated == SeriesWindow(first_bucket=buckets[-history], values=later[-history:])++ def test_fresh_buckets_after_a_gap_leave_the_window_unscored(self) -> None:+ buckets = _buckets(datetime(2026, 1, 1, tzinfo=UTC), IntervalType.DAY, 70)+ window = SeriesWindow(first_bucket=buckets[0], values=[1.0] * 60)++ scored = score_window(+ window,+ [1.0] * 5,+ buckets[62:67],+ refresh=3,+ scoring=DAY_SCORING,+ interval=IntervalType.DAY,+ now=buckets[67],+ )++ assert scored is None++def _insight(series: list[EventsNode], **query: Any) -> Insight:return Insight(id=7, query=TrendsQuery(series=series, **query).model_dump(exclude_none=True))@@ -255,6 +326,130 @@ class TestScoreInsight(SimpleTestCase):if waves == 1:assert limits[0] == 150+ @parameterized.expand([("no_smoothing", None, "-6d"), ("smoothing", 7, "-12d")])+ def test_a_score_step_fetches_only_new_buckets_and_keeps_series_it_did_not_get_back(+ self, _name: str, smoothing: int | None, date_from: str+ ) -> None:+ insight = _insight([EventsNode(event="$pageview")], trendsFilter=TrendsFilter(smoothingIntervals=smoothing))+ team = Team(timezone="UTC")+ rng = np.random.default_rng(3)++ def result(breakdown: str, first_day: datetime, days: int) -> dict[str, Any]:+ return {+ "action": {"order": 0},+ "label": "pageview",+ "breakdown_value": breakdown,+ "days": [(first_day + timedelta(days=day)).strftime("%Y-%m-%d") for day in range(days)],+ "data": list(rng.normal(100, 10, size=days)),+ }++ def run(now: datetime, results: list[dict[str, Any]], split: SplitRun) -> tuple[Any, TrendsQuery]:+ with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:+ runner.return_value.run.return_value = MagicMock(results=results)+ scores = score_insight(insight, team, now=now, split=split)+ return scores, runner.call_args.kwargs["query"]++ trained, _query = run(+ datetime(2026, 1, 10, 0, 30, tzinfo=UTC),+ [+ result("kept", datetime(2025, 10, 1, tzinfo=UTC), 102),+ result("gone", datetime(2025, 10, 1, tzinfo=UTC), 102),+ ],+ SplitRun(refresh_buckets=3),+ )+ assert trained.run == ScoringRun.TRAIN and trained.state is not None+ keys = {series.breakdown_value: series.series_key for series in trained.series}++ scores, query = run(+ datetime(2026, 1, 12, 0, 30, tzinfo=UTC),+ [result("kept", datetime(2026, 1, 6, tzinfo=UTC), 7), result("new", datetime(2026, 1, 6, tzinfo=UTC), 7)],+ SplitRun(refresh_buckets=3, state=trained.state),+ )++ assert query.dateRange is not None and query.dateRange.date_from == date_from+ assert scores.run == ScoringRun.SCORE+ assert [+ (series.breakdown_value, [point.bucket_start.day for point in series.points]) for series in scores.series+ ] == [("kept", [10, 11])]+ assert all(point.score is not None for point in scores.series[0].points)+ assert scores.unmatched_series == 1+ assert scores.state is not None+ assert scores.state.missing_series == 1+ assert scores.state.windows[keys["gone"]] == trained.state.windows[keys["gone"]]+ assert scores.state.scored_through == datetime(2026, 1, 11, tzinfo=UTC)++ def test_a_score_pick_whose_state_no_longer_matches_the_insight_trains(self) -> None:+ stale = TrainedState(+ identity=StateIdentity(interval=IntervalType.DAY, detector_version="old", query_hash="old"),+ trained_at=datetime(2026, 1, 9, tzinfo=UTC),+ scored_through=datetime(2026, 1, 9, tzinfo=UTC),+ windows={},+ )+ with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:+ runner.return_value.run.return_value = MagicMock(results=[])+ scores = score_insight(+ _insight([EventsNode(event="$pageview")]),+ Team(timezone="UTC"),+ now=datetime(2026, 1, 10, tzinfo=UTC),+ split=SplitRun(refresh_buckets=3, state=stale),+ )++ assert scores.run == ScoringRun.TRAIN+ assert runner.call_args.kwargs["query"].dateRange.date_from == "-90d"+++_IDENTITY = StateIdentity(interval=IntervalType.DAY, detector_version="v1", query_hash="q1")+_SCORED_THROUGH = datetime(2026, 1, 9, tzinfo=UTC)+_WINDOW = SeriesWindow(first_bucket=datetime(2025, 11, 11, tzinfo=UTC), values=[1.0] * 60)+++def _state(**changes: Any) -> TrainedState:+ fields: dict[str, Any] = {+ "identity": _IDENTITY,+ "trained_at": datetime(2026, 1, 5, tzinfo=UTC),+ "scored_through": _SCORED_THROUGH,+ "windows": {"a": _WINDOW, "b": _WINDOW},+ **changes,+ }+ return TrainedState(**fields)+++class TestNeedsTraining(SimpleTestCase):+ @parameterized.expand(+ [+ ("fresh_state", _state(), False),+ ("one_of_two_series_missing", _state(missing_series=1), False),+ ("no_state", None, True),+ (+ "query_changed",+ _state(identity=StateIdentity(interval=IntervalType.DAY, detector_version="v1", query_hash="q0")),+ True,+ ),+ (+ "detector_changed",+ _state(identity=StateIdentity(interval=IntervalType.DAY, detector_version="v0", query_hash="q1")),+ True,+ ),+ ("trained_too_long_ago", _state(trained_at=datetime(2026, 1, 2, tzinfo=UTC)), True),+ ("gap_over_the_backfill_cap", _state(scored_through=datetime(2025, 12, 1, tzinfo=UTC)), True),+ ("most_series_missing", _state(missing_series=2), True),+ ]+ )+ def test_a_visit_trains_when_the_stored_state_cannot_score_it(+ self, _name: str, state: TrainedState | None, trains: bool+ ) -> None:+ assert (+ needs_training(+ state,+ identity=_IDENTITY,+ now=datetime(2026, 1, 10, 1, tzinfo=UTC),+ timezone=ZoneInfo("UTC"),+ train_every=7,+ max_backfill=30,+ )+ is trains+ )+class TestMetricScoringQuery(SimpleTestCase):@parameterized.expand([("without_compare", None), ("with_saved_compare", CompareFilter(compare=True))])diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.pyindex f7697e168203..0daeededb270 100644--- a/products/signals/backend/test/test_anomaly_scoring_tick.py+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py@@ -11,6 +11,7 @@ from unittest.mock import MagicMock, patchfrom django.core.cache import cachefrom django.test import SimpleTestCase+import numpy as npfrom parameterized import parameterizedfrom temporalio import activityfrom temporalio.testing import WorkflowEnvironment@@ -27,7 +28,7 @@ from products.dashboards.backend.models.dashboard import Dashboardfrom products.dashboards.backend.models.dashboard_tile import DashboardTilefrom products.product_analytics.backend.facade.api import record_insight_viewfrom products.product_analytics.backend.facade.models import Insight-from products.signals.backend.anomaly_scores.config import DISABLED, AnomalyScoringConfig, parse_config+from products.signals.backend.anomaly_scores.config import DISABLED, AnomalyScoringConfig, ScoringRun, parse_configfrom products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_eventsfrom products.signals.backend.anomaly_scores.queue import (SKIP_COOLDOWN,@@ -53,6 +54,7 @@ from products.signals.backend.anomaly_scores.scoring import (UnsupportedInsightError,)from products.signals.backend.anomaly_scores.selection import SelectionConfig, Weighting+from products.signals.backend.anomaly_scores.state import RunPlanfrom products.signals.backend.anomaly_scores.tick import (AnomalyScoringTickInput,AnomalyScoringTickPlan,@@ -63,6 +65,7 @@ from products.signals.backend.anomaly_scores.tick import (plan_tick,run_insight,)+from products.signals.backend.models import AnomalyScoringStateNEW_YORK = ZoneInfo("America/New_York")ORGANIZATION_ID = "4dc8564d-bd82-1065-2f40-97f7c50f67cf"@@ -121,6 +124,9 @@ class TestTickConfig(SimpleTestCase):"intervals": ["minute", "week"],"max_insights_per_tick": 0,"limits": {"max_backfill_buckets": 100_000, "max_series": -1},+ "train_score_split": "true",+ "train_every_buckets": {"hour": 100_000, "day": 0, "week": 2},+ "refresh_buckets": ["day"],})@@ -129,6 +135,14 @@ class TestTickConfig(SimpleTestCase):assert config.intervals == (IntervalType.WEEK,)assert config.max_insights_per_tick == DISABLED.max_insights_per_tickassert (config.limits.max_backfill_buckets, config.limits.max_series) == (90, 20)+ assert config.train_score_split is False+ assert config.train_every_buckets == {+ IntervalType.HOUR: 168,+ IntervalType.DAY: 7,+ IntervalType.WEEK: 2,+ IntervalType.MONTH: 3,+ }+ assert config.refresh_buckets == DISABLED.refresh_buckets@parameterized.expand([@@ -136,6 +150,8 @@ class TestTickConfig(SimpleTestCase):("max_concurrency", 100, 8),("max_queries_per_hour", 1_000_000, 2000),("max_queries_per_project_per_hour", 501, 500),+ ("max_score_queries_per_hour", 1_000_000, 5000),+ ("max_score_queries_per_project_per_hour", 1001, 1000),])def test_tick_levers_above_their_maximum_parse_to_the_maximum(self, field: str, value: int, expected: int) -> None:@@ -170,6 +186,28 @@ class TestQueryBudget(SimpleTestCase):("tick_cap", {1: 1, 2: 1, 3: 1}, QueryBudget(fleet=100, projects={1: 30}), 1, [1], 0, 0),# Another tick reserved the queries after the snapshot.("reservation_fails", {1: 1, 2: 1, 3: 1}, QueryBudget(fleet=100, projects={1: 30}), 10, [2, 3], 1, 1),+ # The train budget is spent, and the score step of the second insight charges its own budget.+ (+ "score_step_uses_its_own_budget",+ {1: 1, 2: 1, 3: 1},+ QueryBudget(fleet=0, projects={1: 0}),+ 10,+ [2],+ 2,+ 0,+ QueryBudget(fleet=5, projects={1: 5}),+ ),+ # A score step does not fit a spent score budget, even with train budget left.+ (+ "score_budget_spent",+ {1: 1, 2: 1, 3: 1},+ QueryBudget(fleet=100, projects={1: 30}),+ 10,+ [1, 3],+ 1,+ 0,+ QueryBudget(fleet=0, projects={1: 0}),+ ),])def test_the_walk_skips_picks_over_budget_and_continues(@@ -181,20 +219,28 @@ class TestQueryBudget(SimpleTestCase):expected: list[int],skipped: int,refused_reservations: int,+ score_budget: QueryBudget | None = None,) -> None:- costs = {1: 3, 2: 3, 3: 2}- calls: list[int] = []+ plans = {+ 1: RunPlan(queries=3),+ 2: RunPlan(queries=3, run=ScoringRun.SCORE if score_budget else None),+ 3: RunPlan(queries=2),+ }+ budgets = {ScoringRun.TRAIN: budget}+ if score_budget is not None:+ budgets[ScoringRun.SCORE] = score_budget+ calls: list[RunPlan] = []- def reserve(team_id: int, cost: int) -> bool:- calls.append(cost)+ def reserve(team_id: int, plan: RunPlan) -> bool:+ calls.append(plan)return len(calls) > refused_reservationspicked, skipped_count = within_budget(- [1, 2, 3], team_ids, budget, limit, lambda ids: {i: costs[i] for i in ids}, reserve+ [1, 2, 3], team_ids, budgets, limit, lambda ids: {i: plans[i] for i in ids}, reserve)assert (list(picked), skipped_count) == (expected, skipped)- assert picked == {insight_id: costs[insight_id] for insight_id in expected}+ assert picked == {insight_id: plans[insight_id] for insight_id in expected}@parameterized.expand([@@ -207,28 +253,52 @@ class TestQueryBudget(SimpleTestCase):) -> None:batches: list[list[int]] = []- def costs_for(insight_ids: list[int]) -> dict[int, int]:+ def plans_for(insight_ids: list[int]) -> dict[int, RunPlan]:batches.append(insight_ids)- return dict.fromkeys(insight_ids, 1)-- within_budget([1, 2, 3, 4], {1: 1, 2: 2, 3: 1, 4: 2}, budget, 10, costs_for, lambda _team, _cost: True)+ return dict.fromkeys(insight_ids, RunPlan(queries=1))++ within_budget(+ [1, 2, 3, 4],+ {1: 1, 2: 2, 3: 1, 4: 2},+ {ScoringRun.TRAIN: budget},+ 10,+ plans_for,+ lambda _team, _plan: True,+ )assert batches == loaded- def test_a_reservation_takes_both_budgets_or_neither_and_resets_each_hour(self) -> None:- config = AnomalyScoringConfig(enabled=True, max_queries_per_hour=10, max_queries_per_project_per_hour=6)+ @parameterized.expand([(ScoringRun.TRAIN, ScoringRun.SCORE), (ScoringRun.SCORE, ScoringRun.TRAIN)])+ def test_a_reservation_takes_both_budgets_or_neither_and_resets_each_hour(+ self, kind: ScoringRun, other: ScoringRun+ ) -> None:+ config = AnomalyScoringConfig(+ enabled=True,+ max_queries_per_hour=10,+ max_queries_per_project_per_hour=6,+ max_score_queries_per_hour=10,+ max_score_queries_per_project_per_hour=6,+ )next_hour = NOW + timedelta(hours=1)for now in (NOW, next_hour):- cache.delete_many([_fleet_budget_key(now), _project_budget_key(1, now), _project_budget_key(2, now)])+ for budget in ScoringRun:+ cache.delete_many(+ [+ _fleet_budget_key(now, budget),+ _project_budget_key(1, now, budget),+ _project_budget_key(2, now, budget),+ ]+ )reserved = [- reserve_queries(config, team_id, queries, NOW + timedelta(minutes=minute))+ reserve_queries(config, team_id, queries, NOW + timedelta(minutes=minute), kind)for team_id, queries, minute in [(1, 3, 0), (1, 3, 10), (1, 1, 20), (2, 4, 30), (2, 1, 40)]]assert reserved == [True, True, False, True, False]- assert remaining_budget(config, [1, 2], NOW) == QueryBudget(fleet=0, projects={1: 0, 2: 2})- assert remaining_budget(config, [1, 2], next_hour) == QueryBudget(fleet=10, projects={1: 6, 2: 6})+ assert remaining_budget(config, [1, 2], NOW, kind) == QueryBudget(fleet=0, projects={1: 0, 2: 2})+ assert remaining_budget(config, [1, 2], NOW, other) == QueryBudget(fleet=10, projects={1: 6, 2: 6})+ assert remaining_budget(config, [1, 2], next_hour, kind) == QueryBudget(fleet=10, projects={1: 6, 2: 6})class TestTickSelection(SimpleTestCase):@@ -416,6 +486,61 @@ class TestRunInsight(SimpleTestCase):assert SKIP_COOLDOWN not in cooldowns+class TestTrainScoreSplit(BaseTest):+ def _run(self, insight: Insight, pick: InsightPick, now: datetime, data: list[float]) -> list[dict[str, Any]]:+ days = [(now - timedelta(days=len(data) - 1 - day)).strftime("%Y-%m-%d") for day in range(len(data))]+ results = [{"action": {"order": 0}, "label": "pageview", "days": days, "data": data, "breakdown_value": None}]+ with (+ patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner,+ patch("products.signals.backend.anomaly_scores.tick.emit") as emit,+ patch("products.signals.backend.anomaly_scores.tick.mark_attempted"),+ ):+ runner.return_value.run.return_value = MagicMock(results=results)+ run_insight(ScoreInsightInput(pick=pick), now)+ return emit.call_args.args[1]++ def test_a_train_pick_stores_the_tail_that_the_next_score_pick_scores_on(self) -> None:+ self.team.timezone = "UTC"+ self.team.save()+ insight = Insight.objects.create(team=self.team, saved=True, query=_trends(interval="day"))+ values = [float(value) for value in np.random.default_rng(4).normal(100, 5, size=100)]+ # JSON has no NaN, so the oldest stored bucket is stored as null. The detector no longer reads it.+ values[39] = float("nan")+ trained_at = datetime(2026, 10, 20, 0, 30, tzinfo=UTC)++ trained = self._run(+ insight,+ InsightPick(team_id=self.team.id, insight_id=insight.id, scored_through={}, run=ScoringRun.TRAIN),+ trained_at,+ values,+ )+ scored = self._run(+ insight,+ InsightPick(+ team_id=self.team.id,+ insight_id=insight.id,+ scored_through={trained[-1]["properties"]["$anomaly_series_key"]: datetime(2026, 10, 19, tzinfo=UTC)},+ run=ScoringRun.SCORE,+ refresh_buckets=3,+ ),+ trained_at + timedelta(days=2),+ [*values[-4:], 500.0, 100.0],+ )++ assert {event["properties"]["$anomaly_run"] for event in trained} == {"train"}+ assert [+ (+ event["properties"]["$anomaly_run"],+ event["properties"]["$anomaly_bucket_start"],+ event["properties"]["$anomaly_flag"],+ )+ for event in scored+ ] == [("score", "2026-10-20T00:00:00+00:00", 0), ("score", "2026-10-21T00:00:00+00:00", 1)]+ state = AnomalyScoringState.objects.for_team(self.team.id).get(insight_id=insight.id)+ assert state.scored_through == datetime(2026, 10, 21, tzinfo=UTC)+ assert state.trained_at == trained_at++class TestEligibleTeams(BaseTest):@parameterized.expand([