Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): split anomaly scoring into train and score steps (#123092)

posthog[bot]ee1d430f993d
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yaml
index d0f72595ac04..6afda3bb2514 100644
--- a/.semgrep/rules/security/idor-team-scoped-models.yaml
+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml
@@ -408,6 +408,7 @@ rules:
|SignalScoutSuggestionSet
|SignalScoutBackgroundBand
|SignalSpend
+ |AnomalyScoringState
|SignalScratchpad
|SignalProjectProfile
|SingleSessionSummary
@@ -880,6 +881,7 @@ rules:
|SignalScoutSuggestionSet
|SignalScoutBackgroundBand
|SignalSpend
+ |AnomalyScoringState
|SignalScratchpad
|SignalProjectProfile
|SingleSessionSummary
diff --git a/frontend/src/taxonomy/core-filter-definitions-by-group.json b/frontend/src/taxonomy/core-filter-definitions-by-group.json
index d60dfefedda3..9dee9bfc3ed5 100644
--- a/frontend/src/taxonomy/core-filter-definitions-by-group.json
+++ b/frontend/src/taxonomy/core-filter-definitions-by-group.json
@@ -803,6 +803,12 @@
"label": "Anomaly scoring profile",
"type": "String"
},
+ "$anomaly_run": {
+ "description": "How the bucket was scored. `train` fetched the full history of the series, and `score` scored the bucket on stored history. Empty when scoring fetches the full history on every run.",
+ "examples": ["train", "score"],
+ "label": "Anomaly scoring run",
+ "type": "String"
+ },
"$anomaly_score": {
"description": "Score from 0 to 1. It ranks buckets within one series only, so never compare or average it across series. Empty while the series warms up.",
"label": "Anomaly score",
@@ -4745,6 +4751,12 @@
"label": "Anomaly scoring profile",
"type": "String"
},
+ "$anomaly_run": {
+ "description": "How the bucket was scored. `train` fetched the full history of the series, and `score` scored the bucket on stored history. Empty when scoring fetches the full history on every run.",
+ "examples": ["train", "score"],
+ "label": "Anomaly scoring run",
+ "type": "String"
+ },
"$anomaly_score": {
"description": "Score from 0 to 1. It ranks buckets within one series only, so never compare or average it across series. Empty while the series warms up.",
"label": "Anomaly score",
diff --git a/posthog/taxonomy/taxonomy.py b/posthog/taxonomy/taxonomy.py
index 2218fba9f82f..dbcba028435e 100644
--- a/posthog/taxonomy/taxonomy.py
+++ b/posthog/taxonomy/taxonomy.py
@@ -3376,6 +3376,12 @@ CORE_FILTER_DEFINITIONS_BY_GROUP: dict[str, dict[str, CoreFilterDefinition]] = {
"description": "Detector-specific values for this bucket, for example ensemble sub-scores. Its shape depends on the detector.",
"type": "String",
},
+ "$anomaly_run": {
+ "label": "Anomaly scoring run",
+ "description": "How the bucket was scored. `train` fetched the full history of the series, and `score` scored the bucket on stored history. Empty when scoring fetches the full history on every run.",
+ "examples": ["train", "score"],
+ "type": "String",
+ },
"$csp_document_url": {
"label": "Document URL",
"description": "The URL of the document where the violation occurred.",
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.py
index 5d4631dee843..0f8938b4c115 100644
--- a/products/signals/backend/anomaly_scores/config.py
+++ b/products/signals/backend/anomaly_scores/config.py
@@ -6,6 +6,7 @@ field falls back to its default, and no default is unbounded.
import uuid
import dataclasses
+from enum import StrEnum
from typing import TYPE_CHECKING, Any
import structlog
@@ -80,6 +81,23 @@ def resolve_profile(team: "Team", insight: "Insight") -> ScoringProfile:
return DEFAULT_PROFILE
+class ScoringRun(StrEnum):
+ """The kind of visit when the train and score split is on."""
+
+ # Fetch the full detector history, score the new buckets, and store the series tail.
+ TRAIN = "train"
+ # Fetch only the refresh and new buckets, merge them into the stored tail, and score the new buckets.
+ SCORE = "score"
+
+
+# Retrain hourly insights once a day, daily once a week, weekly once a month, and monthly once a quarter.
+DEFAULT_TRAIN_EVERY_BUCKETS = {IntervalType.HOUR: 24, IntervalType.DAY: 7, IntervalType.WEEK: 4, IntervalType.MONTH: 3}
+MAX_TRAIN_EVERY_BUCKETS = {IntervalType.HOUR: 168, IntervalType.DAY: 30, IntervalType.WEEK: 13, IntervalType.MONTH: 12}
+# Stored buckets that a score step fetches again, so late events reach the stored values.
+DEFAULT_REFRESH_BUCKETS = {IntervalType.HOUR: 3, IntervalType.DAY: 3, IntervalType.WEEK: 2, IntervalType.MONTH: 1}
+MAX_REFRESH_BUCKETS = {IntervalType.HOUR: 24, IntervalType.DAY: 14, IntervalType.WEEK: 4, IntervalType.MONTH: 3}
+
+
# Hourly insights are left out by default: with many eligible insights, the time between two visits of
# one insight passes the hourly backfill cap, and the buckets in between are never scored.
DEFAULT_TICK_INTERVALS = (IntervalType.DAY, IntervalType.WEEK, IntervalType.MONTH)
@@ -100,6 +118,15 @@ class AnomalyScoringConfig:
max_queries_per_hour: int = 120
max_queries_per_project_per_hour: int = 30
limits: ScoringLimits = DEFAULT_LIMITS
+ # Off keeps one full fetch per visit and stores no state. On, most visits are cheap score steps.
+ train_score_split: bool = False
+ train_every_buckets: dict[IntervalType, int] = dataclasses.field(
+ default_factory=lambda: dict(DEFAULT_TRAIN_EVERY_BUCKETS)
+ )
+ refresh_buckets: dict[IntervalType, int] = dataclasses.field(default_factory=lambda: dict(DEFAULT_REFRESH_BUCKETS))
+ # Score step queries per clock hour. Train steps use the budgets above, so a cheap score step never waits for them.
+ max_score_queries_per_hour: int = 600
+ max_score_queries_per_project_per_hour: int = 150
DISABLED = AnomalyScoringConfig()
@@ -108,6 +135,8 @@ MAX_INSIGHTS_PER_TICK = 50
MAX_CONCURRENCY = 8
MAX_QUERIES_PER_HOUR = 2000
MAX_QUERIES_PER_PROJECT_PER_HOUR = 500
+MAX_SCORE_QUERIES_PER_HOUR = 5000
+MAX_SCORE_QUERIES_PER_PROJECT_PER_HOUR = 1000
def _positive_int(value: Any, default: int) -> int:
@@ -144,6 +173,18 @@ def _intervals(value: Any) -> tuple[IntervalType, ...]:
return intervals or DEFAULT_TICK_INTERVALS
+def _buckets_per_interval(
+ value: Any, defaults: dict[IntervalType, int], maxima: dict[IntervalType, int]
+) -> dict[IntervalType, int]:
+ """An object of interval name to bucket count. A missing or invalid count gets its default, a large one its maximum."""
+ if not isinstance(value, dict):
+ return dict(defaults)
+ return {
+ interval: min(_positive_int(value.get(interval.value), default), maxima[interval])
+ for interval, default in defaults.items()
+ }
+
+
def parse_limits(payload: Any) -> ScoringLimits:
"""The `limits` object of the payload. An invalid field gets its default, a large one its `MAX_LIMITS` value."""
if not isinstance(payload, dict):
@@ -181,6 +222,23 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:
MAX_QUERIES_PER_PROJECT_PER_HOUR,
),
limits=parse_limits(payload.get("limits")),
+ train_score_split=payload.get("train_score_split") is True,
+ train_every_buckets=_buckets_per_interval(
+ payload.get("train_every_buckets"), DEFAULT_TRAIN_EVERY_BUCKETS, MAX_TRAIN_EVERY_BUCKETS
+ ),
+ refresh_buckets=_buckets_per_interval(
+ payload.get("refresh_buckets"), DEFAULT_REFRESH_BUCKETS, MAX_REFRESH_BUCKETS
+ ),
+ max_score_queries_per_hour=min(
+ _positive_int(payload.get("max_score_queries_per_hour"), defaults.max_score_queries_per_hour),
+ MAX_SCORE_QUERIES_PER_HOUR,
+ ),
+ max_score_queries_per_project_per_hour=min(
+ _positive_int(
+ payload.get("max_score_queries_per_project_per_hour"), defaults.max_score_queries_per_project_per_hour
+ ),
+ MAX_SCORE_QUERIES_PER_PROJECT_PER_HOUR,
+ ),
)
diff --git a/products/signals/backend/anomaly_scores/emission.py b/products/signals/backend/anomaly_scores/emission.py
index d775a0ba129b..57b1ba67853d 100644
--- a/products/signals/backend/anomaly_scores/emission.py
+++ b/products/signals/backend/anomaly_scores/emission.py
@@ -57,6 +57,7 @@ def _event(
"$anomaly_profile": scores.profile,
"$anomaly_scored_at": scored_at.astimezone(datetime.UTC).isoformat(),
"$anomaly_details": json.dumps(point.details, sort_keys=True),
+ **({"$anomaly_run": scores.run.value} if scores.run else {}),
},
}
diff --git a/products/signals/backend/anomaly_scores/metrics.py b/products/signals/backend/anomaly_scores/metrics.py
index 06b3ed442a1e..e98c4ca9d2dd 100644
--- a/products/signals/backend/anomaly_scores/metrics.py
+++ b/products/signals/backend/anomaly_scores/metrics.py
@@ -13,11 +13,11 @@ def _metrics() -> PostHogMetrics | None:
return client.metrics if client is not None else None
-def record_pick(*, status: str, interval: str, duration_ms: float, events: int) -> None:
+def record_pick(*, status: str, interval: str, duration_ms: float, events: int, run: str | None = None) -> None:
metrics = _metrics()
if metrics is None:
return
- attributes = {"status": status, "interval": interval}
+ attributes = {"status": status, "interval": interval, **({"run": run} if run else {})}
metrics.count("anomaly_scoring.picks", 1, attributes=attributes)
metrics.histogram("anomaly_scoring.query.duration", duration_ms, unit="ms", attributes=attributes)
if events:
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index 468150a5e115..21121ed3a2d4 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -1,8 +1,9 @@
"""Pick the watched insights that the next anomaly scoring tick scores.
-There is no stored state. One query per team reads the newest scored bucket of each series back
-from the `$anomaly_score` events. `selection.pick_order` then orders the due insights, and the tick
-picks from that order the insights whose queries fit the hourly budgets.
+One query per team reads the newest scored bucket of each series back from the `$anomaly_score`
+events. `selection.pick_order` then orders the due insights, and the tick picks from that order the
+insights whose queries fit the hourly budgets. With the train and score split on, `state.plan_run`
+makes each pick a train step or a score step, and each kind has its own budgets.
"""
import datetime
@@ -30,8 +31,8 @@ from products.product_analytics.backend.facade.api import (
insights_including_soft_deleted_for_team,
watched_trends_insights,
)
-from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig
-from products.signals.backend.anomaly_scores.scoring import bucket_end, query_cost
+from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig, ScoringRun
+from products.signals.backend.anomaly_scores.scoring import bucket_end
from products.signals.backend.anomaly_scores.selection import (
Candidate,
InsightUsage,
@@ -39,6 +40,7 @@ from products.signals.backend.anomaly_scores.selection import (
insight_usage,
pick_order,
)
+from products.signals.backend.anomaly_scores.state import RunPlan, load_states, plan_run
from ee.billing.quota_limiting import QuotaLimitingCaches, QuotaResource, list_limited_team_attributes
@@ -101,6 +103,10 @@ class InsightPick:
# The ClickHouse queries this pick sends. Selection charges them to the hourly budgets, so two ticks
# that overlap cannot spend the same budget.
query_cost: int = 1
+ # None when the train and score split is off.
+ run: ScoringRun | None = None
+ # Stored buckets that a score step fetches again.
+ refresh_buckets: int = 0
@frozen
@@ -223,23 +229,46 @@ def _hour(now: datetime.datetime) -> str:
return _utc(now).strftime("%Y%m%d%H")
-def _fleet_budget_key(now: datetime.datetime) -> str:
- return f"signals:anomaly_scores:queries:{_hour(now)}"
+def budget_kind(run: ScoringRun | None) -> ScoringRun:
+ """The budgets a pick charges. Only a score step uses the score budgets."""
+ return ScoringRun.SCORE if run == ScoringRun.SCORE else ScoringRun.TRAIN
-def _project_budget_key(team_id: int, now: datetime.datetime) -> str:
- return f"signals:anomaly_scores:queries:{_hour(now)}:team:{team_id}"
+def _budget_prefix(kind: ScoringRun) -> str:
+ # Train steps keep the key of the budgets from before the split, so turning the split on keeps the hour's spend.
+ return "signals:anomaly_scores:score_queries" if kind == ScoringRun.SCORE else "signals:anomaly_scores:queries"
-def remaining_budget(config: AnomalyScoringConfig, team_ids: Collection[int], now: datetime.datetime) -> QueryBudget:
- project_keys = {team_id: _project_budget_key(team_id, now) for team_id in team_ids}
- spent = cache.get_many([_fleet_budget_key(now), *project_keys.values()])
+def _fleet_budget_key(now: datetime.datetime, kind: ScoringRun = ScoringRun.TRAIN) -> str:
+ return f"{_budget_prefix(kind)}:{_hour(now)}"
+
+
+def _project_budget_key(team_id: int, now: datetime.datetime, kind: ScoringRun = ScoringRun.TRAIN) -> str:
+ return f"{_budget_prefix(kind)}:{_hour(now)}:team:{team_id}"
+
+
+def _fleet_cap(config: AnomalyScoringConfig, kind: ScoringRun) -> int:
+ return config.max_score_queries_per_hour if kind == ScoringRun.SCORE else config.max_queries_per_hour
+
+
+def _project_cap(config: AnomalyScoringConfig, kind: ScoringRun) -> int:
+ if kind == ScoringRun.SCORE:
+ return config.max_score_queries_per_project_per_hour
+ return config.max_queries_per_project_per_hour
+
+
+def remaining_budget(
+ config: AnomalyScoringConfig,
+ team_ids: Collection[int],
+ now: datetime.datetime,
+ kind: ScoringRun = ScoringRun.TRAIN,
+) -> QueryBudget:
+ fleet_key = _fleet_budget_key(now, kind)
+ project_keys = {team_id: _project_budget_key(team_id, now, kind) for team_id in team_ids}
+ spent = cache.get_many([fleet_key, *project_keys.values()])
return QueryBudget(
- fleet=config.max_queries_per_hour - spent.get(_fleet_budget_key(now), 0),
- projects={
- team_id: config.max_queries_per_project_per_hour - spent.get(key, 0)
- for team_id, key in project_keys.items()
- },
+ fleet=_fleet_cap(config, kind) - spent.get(fleet_key, 0),
+ projects={team_id: _project_cap(config, kind) - spent.get(key, 0) for team_id, key in project_keys.items()},
)
@@ -253,12 +282,18 @@ def _reserve(key: str, queries: int, cap: int) -> bool:
return False
-def reserve_queries(config: AnomalyScoringConfig, team_id: int, queries: int, now: datetime.datetime) -> bool:
- """Charge ``queries`` to both hourly budgets, or to neither when one of them has too few left."""
- fleet_key = _fleet_budget_key(now)
- if not _reserve(fleet_key, queries, config.max_queries_per_hour):
+def reserve_queries(
+ config: AnomalyScoringConfig,
+ team_id: int,
+ queries: int,
+ now: datetime.datetime,
+ kind: ScoringRun = ScoringRun.TRAIN,
+) -> bool:
+ """Charge ``queries`` to both hourly budgets of ``kind``, or to neither when one of them has too few left."""
+ fleet_key = _fleet_budget_key(now, kind)
+ if not _reserve(fleet_key, queries, _fleet_cap(config, kind)):
return False
- if _reserve(_project_budget_key(team_id, now), queries, config.max_queries_per_project_per_hour):
+ if _reserve(_project_budget_key(team_id, now, kind), queries, _project_cap(config, kind)):
return True
cache.decr(fleet_key, queries)
return False
@@ -267,47 +302,62 @@ def reserve_queries(config: AnomalyScoringConfig, team_id: int, queries: int, no
def within_budget(
order: list[int],
team_ids: dict[int, int],
- budget: QueryBudget,
+ budgets: dict[ScoringRun, QueryBudget],
limit: int,
- costs_for: Callable[[list[int]], dict[int, int]],
- reserve: Callable[[int, int], bool],
-) -> tuple[dict[int, int], int]:
- """The insights of ``order`` whose cost fits both budgets, up to ``limit``, with their costs, and the skip count.
+ plans_for: Callable[[list[int]], dict[int, RunPlan]],
+ reserve: Callable[[int, RunPlan], bool],
+) -> tuple[dict[int, RunPlan], int]:
+ """The insights of ``order`` whose cost fits both budgets of their kind, up to ``limit``, and the skip count.
A pick that does not fit is skipped and the walk continues, so a large insight does not block smaller ones.
- ``budget`` is a snapshot that skips most misses without a cache call. ``reserve`` charges a pick for real.
- ``costs_for`` loads costs in batches of ``limit``, and only for insights that the snapshot can still fit.
+ ``budgets`` are snapshots that skip most misses without a cache call. ``reserve`` charges a pick for real.
+ ``plans_for`` loads plans in batches of ``limit``, and only for insights that a snapshot can still fit.
"""
- fleet = budget.fleet
- projects = dict(budget.projects)
- picked: dict[int, int] = {}
+ fleets = {kind: budget.fleet for kind, budget in budgets.items()}
+ projects = {kind: dict(budget.projects) for kind, budget in budgets.items()}
+ picked: dict[int, RunPlan] = {}
skipped = 0
batch_size = max(1, limit)
for start in range(0, len(order), batch_size):
if len(picked) >= limit:
break
batch = order[start : start + batch_size]
- affordable = [insight_id for insight_id in batch if fleet > 0 and projects.get(team_ids[insight_id], 0) > 0]
- costs = costs_for(affordable) if affordable else {}
+ affordable = [
+ insight_id
+ for insight_id in batch
+ if any(fleets[kind] > 0 and projects[kind].get(team_ids[insight_id], 0) > 0 for kind in budgets)
+ ]
+ plans = plans_for(affordable) if affordable else {}
for insight_id in batch:
if len(picked) >= limit:
break
team_id = team_ids[insight_id]
- cost = costs.get(insight_id)
- if cost is None or cost > fleet or cost > projects.get(team_id, 0) or not reserve(team_id, cost):
+ plan = plans.get(insight_id)
+ kind = budget_kind(plan.run if plan else None)
+ if (
+ plan is None
+ or kind not in budgets
+ or plan.queries > fleets[kind]
+ or plan.queries > projects[kind].get(team_id, 0)
+ or not reserve(team_id, plan)
+ ):
skipped += 1
continue
- picked[insight_id] = cost
- fleet -= cost
- projects[team_id] -= cost
+ picked[insight_id] = plan
+ fleets[kind] -= plan.queries
+ projects[kind][team_id] -= plan.queries
return picked, skipped
-def _query_costs(team: Team, insight_ids: list[int]) -> dict[int, int]:
+def _run_plans(
+ team: Team, insight_ids: list[int], config: AnomalyScoringConfig, now: datetime.datetime
+) -> dict[int, RunPlan]:
insights = insights_including_soft_deleted_for_team(team_id=team.id, insight_ids=insight_ids)
- costs = {insight.id: query_cost(insight, team) for insight in insights}
+ states = load_states(team, insights) if config.train_score_split else {}
+ plans = {insight.id: plan_run(insight, team, states.get(insight.id), config, now) for insight in insights}
# An insight that no longer exists sends no query, but one query is the safe charge.
- return {insight_id: costs.get(insight_id, 1) for insight_id in insight_ids}
+ missing = RunPlan(queries=1, run=ScoringRun.TRAIN if config.train_score_split else None)
+ return {insight_id: plans.get(insight_id, missing) for insight_id in insight_ids}
def select_insights(
@@ -366,21 +416,22 @@ def select_insights(
]
team_ids = {insight_id: due_by_id[insight_id].team_id for insight_id in order}
- def costs_for(insight_ids: list[int]) -> dict[int, int]:
- costs: dict[int, int] = {}
+ def plans_for(insight_ids: list[int]) -> dict[int, RunPlan]:
+ plans: dict[int, RunPlan] = {}
for team_id in sorted({team_ids[insight_id] for insight_id in insight_ids}):
team_batch = [insight_id for insight_id in insight_ids if team_ids[insight_id] == team_id]
- costs.update(_query_costs(teams[team_id], team_batch))
- return costs
+ plans.update(_run_plans(teams[team_id], team_batch, config, now))
+ return plans
- # Read the budget before any cost, so a spent hour loads no insight.
+ kinds = [ScoringRun.TRAIN, ScoringRun.SCORE] if config.train_score_split else [ScoringRun.TRAIN]
+ # Read the budgets before any cost, so a spent hour loads no insight.
picked, skipped = within_budget(
order,
team_ids,
- remaining_budget(config, set(team_ids.values()), now),
+ {kind: remaining_budget(config, set(team_ids.values()), now, kind) for kind in kinds},
limit,
- costs_for,
- reserve=lambda team_id, cost: reserve_queries(config, team_id, cost, now),
+ plans_for,
+ reserve=lambda team_id, plan: reserve_queries(config, team_id, plan.queries, now, budget_kind(plan.run)),
)
picks = [
InsightPick(
@@ -388,8 +439,10 @@ def select_insights(
insight_id=insight_id,
scored_through=state.get(insight_id, {}),
interval=due_by_id[insight_id].interval,
- query_cost=cost,
+ query_cost=plan.queries,
+ run=plan.run,
+ refresh_buckets=plan.refresh_buckets,
)
- for insight_id, cost in picked.items()
+ for insight_id, plan in picked.items()
]
return Selection(picks=picks, candidates=len(order), skipped_for_budget=skipped)
diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.py
index 990bac80ace1..15b6f8053bad 100644
--- a/products/signals/backend/anomaly_scores/scoring.py
+++ b/products/signals/backend/anomaly_scores/scoring.py
@@ -36,6 +36,7 @@ from products.signals.backend.anomaly_scores.config import (
IntervalScoring,
ScoringLimits,
ScoringProfile,
+ ScoringRun,
resolve_profile,
)
@@ -77,6 +78,43 @@ class ScoredSeries:
points: list[ScoredPoint]
+@frozen
+class SeriesWindow:
+ """The newest completed values of one series, oldest first. A score step reads them as the history of its new buckets."""
+
+ # Aware bucket start of the first value. Trends zero-fills buckets, so each next value is one interval later.
+ first_bucket: datetime
+ values: list[float]
+
+
+@frozen
+class StateIdentity:
+ """What a stored state was built from. A score step needs the same identity, or the visit trains."""
+
+ interval: IntervalType
+ detector_version: str
+ query_hash: str
+
+
+@frozen
+class TrainedState:
+ identity: StateIdentity
+ trained_at: datetime
+ # The newest scored bucket start over all series, or None when no series had a completed bucket.
+ scored_through: datetime | None
+ windows: dict[str, SeriesWindow]
+ # Stored series that the last score step did not get back.
+ missing_series: int = 0
+
+
+@frozen
+class SplitRun:
+ """A visit of the train and score split. A score step needs ``state``; with no matching state, the visit trains."""
+
+ refresh_buckets: int
+ state: TrainedState | None = None
+
+
@frozen
class InsightScores:
interval: IntervalType
@@ -84,6 +122,21 @@ class InsightScores:
detector: str
detector_version: str
series: list[ScoredSeries]
+ # None when the split is off. Then no state is stored.
+ run: ScoringRun | None = None
+ # The state for the next visit to store.
+ state: TrainedState | None = None
+ # Series of a score step with no stored window. A later train step scores them.
+ unmatched_series: int = 0
+
+
+@frozen
+class _FetchedSeries:
+ series_key: str
+ label: str
+ breakdown_value: str | None
+ values: list[float]
+ buckets: list[datetime]
def _short_hash(payload: Any) -> str:
@@ -96,6 +149,15 @@ def detector_version(scoring: IntervalScoring) -> str:
return _short_hash(dataclasses.asdict(scoring))
+def query_hash(query: TrendsQuery, timezone: ZoneInfo, limits: ScoringLimits = DEFAULT_LIMITS) -> str:
+ """Identifies what the scoring query counts, without its date range, so an edited insight retrains."""
+ dumped = build_scoring_query(query, buckets=1, limits=limits).model_dump(mode="json", exclude_none=True)
+ # The fetch size changes with every visit. `daysOfWeek` stays, because it changes the buckets.
+ (dumped.get("dateRange") or {}).pop("date_from", None)
+ # Stored bucket starts step in wall-clock time, so a timezone change needs new windows.
+ return _short_hash({"query": dumped, "timezone": timezone.key})
+
+
def series_key(insight_id: int, series_definition: dict[str, Any], breakdown_value: str | None) -> str:
return _short_hash({"insight": insight_id, "series": series_definition, "breakdown": breakdown_value})
@@ -119,7 +181,7 @@ def trends_query_for(insight: Insight, profile: ScoringProfile) -> TrendsQuery:
if trends.trendsFilter and trends.trendsFilter.display == ChartDisplayType.CALENDAR_HEATMAP:
# The chart has its own runner and shows hours of the week, not a time series.
raise UnsupportedInsightError("Calendar heatmaps are not scored")
- interval_scoring(profile, _interval(trends))
+ interval_scoring(profile, scoring_interval(trends))
return trends
@@ -170,12 +232,13 @@ def buckets_to_score(
return max(count, 1)
-def fetch_buckets(query: TrendsQuery, scoring: IntervalScoring, to_score: int) -> int:
+def fetch_buckets(query: TrendsQuery, scoring: IntervalScoring, to_score: int, *, history: int | None = None) -> int:
"""Completed buckets the scoring query fetches: the history of the oldest bucket to score, then the rest.
The count has one bucket more than the scores read, so a first bucket that the date range cuts short is never read.
+ ``history`` replaces the detector history, for a score step that reads its history from the stored state.
"""
- count = history_buckets(scoring) + to_score
+ count = (history_buckets(scoring) if history is None else history) + to_score
smoothing = query.trendsFilter.smoothingIntervals if query.trendsFilter else None
if smoothing and smoothing > _MAX_SMOOTHING_BUCKETS:
# The field has no upper bound, and the fetch grows with it.
@@ -184,7 +247,7 @@ def fetch_buckets(query: TrendsQuery, scoring: IntervalScoring, to_score: int) -
# Trends smooths over the fetched buckets only, so the oldest values need their own trailing window.
count += smoothing - 1
days_of_week = query.dateRange.daysOfWeek if query.dateRange else None
- if days_of_week and _interval(query) == IntervalType.DAY:
+ if days_of_week and scoring_interval(query) == IntervalType.DAY:
# Trends drops the buckets of the days that the filter leaves out.
count = math.ceil(count * 7 / len(set(days_of_week)))
return count
@@ -197,7 +260,7 @@ def build_scoring_query(query: TrendsQuery, buckets: int, limits: ScoringLimits
"""
days_of_week = query.dateRange.daysOfWeek if query.dateRange else None
update: dict[str, Any] = {
- "dateRange": DateRange(date_from=f"-{buckets}{_DATE_UNITS[_interval(query)]}", daysOfWeek=days_of_week),
+ "dateRange": DateRange(date_from=f"-{buckets}{_DATE_UNITS[scoring_interval(query)]}", daysOfWeek=days_of_week),
"compareFilter": None,
}
if query.trendsFilter and query.trendsFilter.display == ChartDisplayType.METRIC:
@@ -227,6 +290,20 @@ def scoring_query_settings(queries: int) -> HogQLGlobalSettings:
)
+def supports_score_step(query: TrendsQuery) -> bool:
+ """A score step steps over stored buckets one interval at a time, so the buckets must have no gaps."""
+ # Trends drops the buckets of the days that the filter leaves out.
+ return not (query.dateRange and query.dateRange.daysOfWeek)
+
+
+def state_identity(query: TrendsQuery, scoring: IntervalScoring, team: Team, limits: ScoringLimits) -> StateIdentity:
+ return StateIdentity(
+ interval=scoring_interval(query),
+ detector_version=detector_version(scoring),
+ query_hash=query_hash(query, team.timezone_info, limits),
+ )
+
+
def query_cost(insight: Insight, team: Team) -> int:
"""The ClickHouse queries that scoring this insight sends, so the tick can budget them before it runs.
@@ -283,6 +360,67 @@ def score_series(
return points
+def window_buckets(window: SeriesWindow, interval: IntervalType) -> list[datetime]:
+ buckets: list[datetime] = []
+ start = window.first_bucket
+ for _ in window.values:
+ buckets.append(start)
+ start = bucket_end(start, interval)
+ return buckets
+
+
+def train_window(
+ values: Sequence[float], buckets: Sequence[datetime], *, history: int, interval: IntervalType, now: datetime
+) -> SeriesWindow | None:
+ """The newest ``history`` completed values of a fetched series, or None when they do not step one interval apart."""
+ completed = [i for i, start in enumerate(buckets) if bucket_end(start, interval) <= now][-history:]
+ if not completed:
+ return None
+ window = SeriesWindow(first_bucket=buckets[completed[0]], values=[float(values[i]) for i in completed])
+ # A repeated hour at a DST fall-back breaks the steps. The series waits for a train step after the change.
+ if window_buckets(window, interval) != [buckets[i] for i in completed]:
+ return None
+ return window
+
+
+def score_window(
+ window: SeriesWindow,
+ values: Sequence[float],
+ buckets: Sequence[datetime],
+ *,
+ refresh: int,
+ scoring: IntervalScoring,
+ interval: IntervalType,
+ now: datetime,
+ limits: ScoringLimits = DEFAULT_LIMITS,
+) -> tuple[list[ScoredPoint], SeriesWindow] | None:
+ """Merge freshly fetched buckets into a stored window, score the buckets after it, and trim it to its history.
+
+ A fresh value replaces the stored value of its bucket for the newest ``refresh`` stored buckets, so late events
+ reach the window. Older fresh buckets can be cut short or smoothed over too few buckets, so they are not read.
+ Returns None when the fresh buckets leave a gap after the window. The series then keeps its window.
+ """
+ stored = window_buckets(window, interval)
+ merged = dict(zip(stored, window.values))
+ refresh_from = [*stored, bucket_end(stored[-1], interval)][max(0, len(stored) - refresh)]
+ fresh = [(start, value) for start, value in zip(buckets, values) if bucket_end(start, interval) <= now]
+ for start, value in fresh:
+ if start >= refresh_from:
+ merged[start] = float(value)
+ starts = list(stored)
+ next_start = bucket_end(stored[-1], interval)
+ while next_start in merged:
+ starts.append(next_start)
+ next_start = bucket_end(next_start, interval)
+ if any(start >= next_start for start, _ in fresh):
+ return None
+
+ data = [merged[start] for start in starts]
+ points = score_series(data, starts, scoring=scoring, interval=interval, now=now, after=stored[-1], limits=limits)
+ keep = history_buckets(scoring)
+ return points, SeriesWindow(first_bucket=starts[-keep:][0], values=data[-keep:])
+
+
def score_insight(
insight: Insight,
team: Team,
@@ -290,17 +428,166 @@ def score_insight(
now: datetime,
after: datetime | None = None,
limits: ScoringLimits = DEFAULT_LIMITS,
+ split: SplitRun | None = None,
) -> InsightScores:
+ """Score the new buckets of an insight.
+
+ With ``split``, a stored state with the insight's identity makes this a score step, and anything else a
+ train step. Both return the state for the next visit.
+ """
profile = resolve_profile(team, insight)
query = trends_query_for(insight, profile)
- interval = _interval(query)
+ interval = scoring_interval(query)
scoring = interval_scoring(profile, interval)
timezone = team.timezone_info
+ identity = state_identity(query, scoring, team, limits) if split is not None else None
+ state = split.state if split is not None else None
+ if state is not None and (state.identity != identity or not supports_score_step(query)):
+ state = None
+ scores = InsightScores(
+ interval=interval,
+ profile=profile.name,
+ detector=str(scoring.detector["type"]),
+ detector_version=detector_version(scoring),
+ series=[],
+ )
+
+ if split is not None and state is not None and identity is not None:
+ to_score = buckets_to_score(interval, now=now, after=state.scored_through, timezone=timezone, limits=limits)
+ # One bucket more than the score step reads, as in the full fetch.
+ buckets = fetch_buckets(query, scoring, to_score, history=split.refresh_buckets + 1)
+ fetched = _fetch(insight, team, query, buckets, limits, ScoringRun.SCORE)
+ return _score_step(
+ scores,
+ fetched,
+ state,
+ refresh=split.refresh_buckets,
+ scoring=scoring,
+ interval=interval,
+ now=now,
+ limits=limits,
+ )
+
to_score = buckets_to_score(interval, now=now, after=after, timezone=timezone, limits=limits)
- scoring_query = build_scoring_query(query, fetch_buckets(query, scoring, to_score), limits)
+ run = ScoringRun.TRAIN if split is not None else None
+ fetched = _fetch(insight, team, query, fetch_buckets(query, scoring, to_score), limits, run)
+ series = [
+ ScoredSeries(
+ series_key=line.series_key,
+ label=line.label,
+ breakdown_value=line.breakdown_value,
+ points=score_series(
+ line.values,
+ line.buckets,
+ scoring=scoring,
+ interval=interval,
+ now=now,
+ after=after,
+ limits=limits,
+ ),
+ )
+ for line in fetched
+ ]
+ if identity is None:
+ return dataclasses.replace(scores, series=series)
+ windows = {
+ line.series_key: window
+ for line in fetched
+ if (
+ window := train_window(
+ line.values, line.buckets, history=history_buckets(scoring), interval=interval, now=now
+ )
+ )
+ is not None
+ }
+ return dataclasses.replace(
+ scores,
+ series=series,
+ run=ScoringRun.TRAIN,
+ state=TrainedState(
+ identity=identity, trained_at=now, scored_through=_scored_through(windows, interval), windows=windows
+ ),
+ )
+
+
+def _score_step(
+ scores: InsightScores,
+ fetched: list[_FetchedSeries],
+ state: TrainedState,
+ *,
+ refresh: int,
+ scoring: IntervalScoring,
+ interval: IntervalType,
+ now: datetime,
+ limits: ScoringLimits,
+) -> InsightScores:
+ series: list[ScoredSeries] = []
+ windows = dict(state.windows)
+ unmatched = 0
+ returned: set[str] = set()
+ for line in fetched:
+ returned.add(line.series_key)
+ window = state.windows.get(line.series_key)
+ # The short range can rank a different top N of breakdown values. A new value waits for a train step.
+ scored = (
+ None
+ if window is None
+ else score_window(
+ window,
+ line.values,
+ line.buckets,
+ refresh=refresh,
+ scoring=scoring,
+ interval=interval,
+ now=now,
+ limits=limits,
+ )
+ )
+ if scored is None:
+ unmatched += 1
+ continue
+ points, windows[line.series_key] = scored
+ series.append(
+ ScoredSeries(
+ series_key=line.series_key, label=line.label, breakdown_value=line.breakdown_value, points=points
+ )
+ )
+ return dataclasses.replace(
+ scores,
+ series=series,
+ run=ScoringRun.SCORE,
+ unmatched_series=unmatched,
+ # A stored series that is missing from the result keeps its window.
+ state=dataclasses.replace(
+ state,
+ scored_through=_scored_through(windows, interval),
+ windows=windows,
+ missing_series=len(set(state.windows) - returned),
+ ),
+ )
+
+
+def _scored_through(windows: dict[str, SeriesWindow], interval: IntervalType) -> datetime | None:
+ ends = [window_buckets(window, interval)[-1] for window in windows.values() if window.values]
+ return max(ends).astimezone(UTC) if ends else None
+
+
+def _fetch(
+ insight: Insight,
+ team: Team,
+ query: TrendsQuery,
+ buckets: int,
+ limits: ScoringLimits,
+ run: ScoringRun | None,
+) -> list[_FetchedSeries]:
+ timezone = team.timezone_info
+ scoring_query = build_scoring_query(query, buckets, limits)
tag_queries(**get_team_query_tags(team), insight_id=insight.pk)
tag_queries(product=Product.SIGNALS, feature=Feature.ALERTING)
+ if run is not None:
+ # Splits the scoring queries in `query_log`, so train and score cost can be compared.
+ tag_queries(name=f"anomaly_scoring_{run.value}")
runner = TrendsQueryRunner(
query=scoring_query,
team=team,
@@ -333,38 +620,24 @@ def score_insight(
skipped=len(results) - len(current_results),
)
- series: list[ScoredSeries] = []
+ series: list[_FetchedSeries] = []
key_counts: dict[str, int] = {}
for result in current_results[: limits.max_series]:
- buckets = [_parse_bucket(day, timezone) for day in result.get("days") or []]
breakdown_value = _breakdown_label(result.get("breakdown_value"))
key = series_key(insight.pk, _series_definition(scoring_query, result), breakdown_value)
repeats = key_counts.get(key, 0)
key_counts[key] = repeats + 1
series.append(
- ScoredSeries(
+ _FetchedSeries(
# Lines that differ only by name share a key. Event uuids come from the key, so each repeat needs its own.
series_key=key if repeats == 0 else _short_hash({"key": key, "repeat": repeats}),
label=str(result.get("label") or ""),
breakdown_value=breakdown_value,
- points=score_series(
- result.get("data") or [],
- buckets,
- scoring=scoring,
- interval=interval,
- now=now,
- after=after,
- limits=limits,
- ),
+ values=list(result.get("data") or []),
+ buckets=[_parse_bucket(day, timezone) for day in result.get("days") or []],
)
)
- return InsightScores(
- interval=interval,
- profile=profile.name,
- detector=str(scoring.detector["type"]),
- detector_version=detector_version(scoring),
- series=series,
- )
+ return series
def _formulas(query: TrendsQuery) -> list[str]:
@@ -398,7 +671,7 @@ def _series_definition(query: TrendsQuery, result: dict[str, Any]) -> dict[str,
return _series_node_definition(query.series[order])
-def _interval(query: TrendsQuery) -> IntervalType:
+def scoring_interval(query: TrendsQuery) -> IntervalType:
return query.interval or IntervalType.DAY
diff --git a/products/signals/backend/anomaly_scores/state.py b/products/signals/backend/anomaly_scores/state.py
new file mode 100644
index 000000000000..48c870bc3d33
--- /dev/null
+++ b/products/signals/backend/anomaly_scores/state.py
@@ -0,0 +1,168 @@
+"""The stored state of the train and score split, and the choice between a train step and a score step.
+
+A train step fetches the full detector history and stores the newest values of each series. A score
+step fetches only the newest buckets and scores them on the stored values. See `AnomalyScoringState`.
+"""
+
+import math
+import datetime
+import dataclasses
+from collections.abc import Collection
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from posthog.schema import IntervalType
+
+from posthog.dataclasses import frozen
+from posthog.interval_specs import interval_spec
+from posthog.models import Team
+
+from products.product_analytics.backend.facade.models import Insight
+from products.signals.backend.anomaly_scores.config import (
+ DEFAULT_LIMITS,
+ AnomalyScoringConfig,
+ ScoringRun,
+ resolve_profile,
+)
+from products.signals.backend.anomaly_scores.scoring import (
+ SeriesWindow,
+ StateIdentity,
+ TrainedState,
+ buckets_to_score,
+ interval_scoring,
+ query_cost,
+ scoring_interval,
+ state_identity,
+ supports_score_step,
+ trends_query_for,
+)
+from products.signals.backend.models import AnomalyScoringState
+
+
+@frozen
+class RunPlan:
+ """What one pick does and the ClickHouse queries it sends."""
+
+ queries: int
+ # None when the split is off.
+ run: ScoringRun | None = None
+ refresh_buckets: int = 0
+
+
+def needs_training(
+ state: TrainedState | None,
+ *,
+ identity: StateIdentity,
+ now: datetime.datetime,
+ timezone: ZoneInfo,
+ train_every: int,
+ max_backfill: int,
+) -> bool:
+ if state is None or state.identity != identity or not state.windows or state.scored_through is None:
+ return True
+ interval = identity.interval
+ if state.trained_at + interval_spec(interval).period * train_every <= now:
+ return True
+ # More due buckets than one visit scores. A train step scores the newest of them, as a first visit does.
+ due = buckets_to_score(
+ interval,
+ now=now,
+ after=state.scored_through,
+ timezone=timezone,
+ limits=dataclasses.replace(DEFAULT_LIMITS, max_backfill_buckets=max_backfill + 1),
+ )
+ if due > max_backfill:
+ return True
+ return state.missing_series * 2 > len(state.windows)
+
+
+def plan_run(
+ insight: Insight, team: Team, state: TrainedState | None, config: AnomalyScoringConfig, now: datetime.datetime
+) -> RunPlan:
+ queries = query_cost(insight, team)
+ if not config.train_score_split:
+ return RunPlan(queries=queries)
+ try:
+ profile = resolve_profile(team, insight)
+ query = trends_query_for(insight, profile)
+ interval = scoring_interval(query)
+ scoring = interval_scoring(profile, interval)
+ identity = state_identity(query, scoring, team, config.limits)
+ except Exception:
+ # The scorer fails on the same query and reports the error. A train step charges the larger budget.
+ return RunPlan(queries=queries, run=ScoringRun.TRAIN)
+ refresh = config.refresh_buckets.get(interval, 0)
+ trains = not supports_score_step(query) or needs_training(
+ state,
+ identity=identity,
+ now=now,
+ timezone=team.timezone_info,
+ train_every=config.train_every_buckets[interval],
+ max_backfill=config.limits.max_backfill_buckets,
+ )
+ return RunPlan(queries=queries, run=ScoringRun.TRAIN if trains else ScoringRun.SCORE, refresh_buckets=refresh)
+
+
+def load_states(team: Team, insights: Collection[Insight]) -> dict[int, TrainedState]:
+ """Insight id to the stored state of the insight's own scoring profile."""
+ profiles = {insight.id: resolve_profile(team, insight).name for insight in insights}
+ if not profiles:
+ return {}
+ rows = AnomalyScoringState.objects.for_team(team.id).filter(insight_id__in=list(profiles))
+ return {
+ row.insight_id: _from_row(row, team.timezone_info)
+ for row in rows
+ if profiles.get(row.insight_id) == row.profile
+ }
+
+
+def load_state(team: Team, insight: Insight) -> TrainedState | None:
+ return load_states(team, [insight]).get(insight.id)
+
+
+def save_state(team: Team, insight_id: int, profile: str, state: TrainedState) -> None:
+ fields = {
+ "interval": state.identity.interval.value,
+ "detector_version": state.identity.detector_version,
+ "query_hash": state.identity.query_hash,
+ "trained_at": state.trained_at,
+ "scored_through": state.scored_through,
+ "missing_series": state.missing_series,
+ "series": {key: _window_json(window) for key, window in state.windows.items()},
+ }
+ # The scope filters on the project's canonical team, and save() writes that team for a new row.
+ AnomalyScoringState.objects.for_team(team.id).update_or_create(
+ insight_id=insight_id, profile=profile, defaults=fields, create_defaults={**fields, "team_id": team.id}
+ )
+
+
+def _window_json(window: SeriesWindow) -> dict[str, Any]:
+ return {
+ "first_bucket": window.first_bucket.isoformat(),
+ # JSON has no NaN.
+ "values": [value if math.isfinite(value) else None for value in window.values],
+ }
+
+
+def _from_row(row: AnomalyScoringState, timezone: datetime.tzinfo) -> TrainedState:
+ windows: dict[str, SeriesWindow] = {}
+ for key, stored in (row.series or {}).items():
+ try:
+ window = SeriesWindow(
+ # Bucket starts step in wall-clock time of the project timezone.
+ first_bucket=datetime.datetime.fromisoformat(stored["first_bucket"]).astimezone(timezone),
+ values=[math.nan if value is None else float(value) for value in stored["values"]],
+ )
+ except (KeyError, TypeError, ValueError):
+ continue
+ if window.values:
+ windows[key] = window
+ return TrainedState(
+ identity=StateIdentity(
+ interval=IntervalType(row.interval), detector_version=row.detector_version, query_hash=row.query_hash
+ ),
+ trained_at=row.trained_at,
+ scored_through=row.scored_through,
+ windows=windows,
+ missing_series=row.missing_series,
+ )
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index 81ee7638aefd..104194333bf0 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -38,8 +38,10 @@ from products.signals.backend.anomaly_scores.config import (
ANOMALY_SCORING_FLAG_DISTINCT_ID,
DEFAULT_LIMITS,
ScoringLimits,
+ ScoringRun,
read_config,
read_payload,
+ resolve_profile,
)
from products.signals.backend.anomaly_scores.emission import emit, score_events
from products.signals.backend.anomaly_scores.metrics import record_candidates, record_pick
@@ -52,10 +54,12 @@ from products.signals.backend.anomaly_scores.queue import (
)
from products.signals.backend.anomaly_scores.scoring import (
SCORING_QUERY_BUDGET_SECONDS,
+ SplitRun,
UnsupportedInsightError,
score_insight,
)
from products.signals.backend.anomaly_scores.selection import parse_selection_config
+from products.signals.backend.anomaly_scores.state import load_state, save_state
logger = structlog.get_logger(__name__)
@@ -116,6 +120,11 @@ class ScoreInsightResult:
status: str
events: int
series: int = 0
+ # What the visit ran. A score pick with no matching state trains. None when the split is off.
+ run: ScoringRun | None = None
+ # Series of a score step with no stored window, and stored series missing from its result.
+ unmatched_series: int = 0
+ missing_series: int = 0
@frozen
@@ -133,6 +142,9 @@ class AnomalyScoringTickResult:
queries: int = 0
skipped_for_budget: int = 0
stopped_at_capacity: bool = False
+ # Started picks per kind of the train and score split.
+ run_train: int = 0
+ run_score: int = 0
def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:
@@ -197,7 +209,7 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
error=str(error)[:200],
duration_ms=duration_ms,
)
- record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0)
+ record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0, run=pick.run)
if status is None:
raise
# A full cluster is not a fault of the insight, so it keeps the attempt cooldown.
@@ -214,8 +226,11 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
series=result.series,
events=result.events,
duration_ms=duration_ms,
+ run=result.run,
+ unmatched_series=result.unmatched_series,
+ missing_series=result.missing_series,
)
- record_pick(status=result.status, interval=interval, duration_ms=duration_ms, events=result.events)
+ record_pick(status=result.status, interval=interval, duration_ms=duration_ms, events=result.events, run=result.run)
return result
@@ -234,6 +249,12 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
team = Team.objects.get(id=pick.team_id)
# Set before scoring, so a failure also waits for the cooldown instead of taking a slot on every tick.
mark_attempted(pick.insight_id, ATTEMPT_COOLDOWN)
+ split = None
+ if pick.run is not None:
+ split = SplitRun(
+ refresh_buckets=pick.refresh_buckets,
+ state=load_state(team, insight) if pick.run == ScoringRun.SCORE else None,
+ )
scores = score_insight(
insight,
team,
@@ -241,13 +262,23 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
# Every series is filtered again on its own newest bucket below. A new series gets up to the backfill cap.
after=min(pick.scored_through.values()) if pick.scored_through else None,
limits=input.limits,
+ split=split,
)
events = score_events(
team_id=team.id, insight_id=insight.id, scores=scores, scored_through=pick.scored_through, scored_at=now
)
emit(team, events)
+ # After the emit: a lost state write only scores buckets again, and their events collapse by uuid.
+ # A state write before a failed emit would skip those buckets for good.
+ if scores.state is not None:
+ save_state(team, insight.id, resolve_profile(team, insight).name, scores.state)
return ScoreInsightResult(
- status=STATUS_SCORED if events else STATUS_UP_TO_DATE, events=len(events), series=len(scores.series)
+ status=STATUS_SCORED if events else STATUS_UP_TO_DATE,
+ events=len(events),
+ series=len(scores.series),
+ run=scores.run,
+ unmatched_series=scores.unmatched_series,
+ missing_series=scores.state.missing_series if scores.state else 0,
)
@@ -264,6 +295,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
queries=result.queries,
skipped_for_budget=result.skipped_for_budget,
stopped_at_capacity=result.stopped_at_capacity,
+ run_train=result.run_train,
+ run_score=result.run_score,
)
ph_background_capture()(
distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,
@@ -280,6 +313,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
"queries": result.queries,
"skipped_for_budget": result.skipped_for_budget,
"stopped_at_capacity": result.stopped_at_capacity,
+ "run_train": result.run_train,
+ "run_score": result.run_score,
},
)
@@ -368,6 +403,8 @@ class AnomalyScoringTickWorkflow:
queries=sum(pick.query_cost for pick in started_picks),
skipped_for_budget=plan.skipped_for_budget,
stopped_at_capacity=statuses[STATUS_AT_CAPACITY] > 0,
+ run_train=sum(pick.run == ScoringRun.TRAIN for pick in started_picks),
+ run_score=sum(pick.run == ScoringRun.SCORE for pick in started_picks),
)
await workflow.execute_activity(
capture_anomaly_scoring_tick_activity,
diff --git a/products/signals/backend/migrations/0199_anomalyscoringstate.py b/products/signals/backend/migrations/0199_anomalyscoringstate.py
new file mode 100644
index 000000000000..98a7597c3e81
--- /dev/null
+++ b/products/signals/backend/migrations/0199_anomalyscoringstate.py
@@ -0,0 +1,63 @@
+# Generated by Django 5.2.18 on 2026-10-24 19:12
+
+import django.db.models.manager
+import django.db.models.deletion
+from django.db import migrations, models
+
+import posthog.uuidt
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("posthog", "1412_drop_retired_desktop_file_system_tables"),
+ ("signals", "0198_useractivitysummary_computed"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="AnomalyScoringState",
+ fields=[
+ (
+ "id",
+ models.UUIDField(
+ default=posthog.uuidt.uuid7,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ ("insight_id", models.BigIntegerField()),
+ ("profile", models.CharField(max_length=64)),
+ ("interval", models.CharField(max_length=16)),
+ ("detector_version", models.CharField(max_length=32)),
+ ("query_hash", models.CharField(max_length=32)),
+ ("trained_at", models.DateTimeField()),
+ ("scored_through", models.DateTimeField(blank=True, null=True)),
+ ("missing_series", models.PositiveIntegerField(default=0)),
+ ("series", models.JSONField(default=dict)),
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("updated_at", models.DateTimeField(auto_now=True)),
+ (
+ "team",
+ models.ForeignKey(
+ db_constraint=False,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="+",
+ to="posthog.team",
+ ),
+ ),
+ ],
+ options={
+ "default_manager_name": "all_teams",
+ "constraints": [
+ models.UniqueConstraint(
+ fields=("team", "insight_id", "profile"),
+ name="signals_anomaly_scoring_state_unique",
+ )
+ ],
+ },
+ managers=[
+ ("all_teams", django.db.models.manager.Manager()),
+ ],
+ ),
+ ]
diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txt
index ad1b5ceff741..5e45e928fd34 100644
--- a/products/signals/backend/migrations/max_migration.txt
+++ b/products/signals/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0198_useractivitysummary_computed
+0199_anomalyscoringstate
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index 6cf3a6f76b6f..430f7bda2b58 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -4013,3 +4013,42 @@ class UserActivitySummaryQueueCursor(models.Model):
source = models.CharField(max_length=40, primary_key=True)
position = models.DateTimeField()
updated_at = models.DateTimeField(auto_now=True)
+
+
+class AnomalyScoringState(TeamScopedRootMixin, UUIDModel):
+ """The stored series tail of one insight and scoring profile of the anomaly scoring tick.
+
+ A train step fetches the full detector history and writes the tail. A score step fetches only the
+ newest buckets, merges them into the tail, and scores the new buckets on it. See
+ `anomaly_scores/state.py`. `$anomaly_score` events stay output only.
+ """
+
+ # See SignalReportRefund.all_teams for rationale.
+ all_teams = models.Manager() # noqa: DJ012
+
+ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, db_constraint=False, related_name="+")
+ # Not a foreign key: insights belong to another product. A deleted insight leaves a small row behind.
+ insight_id = models.BigIntegerField()
+ profile = models.CharField(max_length=64)
+ # The insight as the state was built for it. A change to any of them makes the next visit train.
+ interval = models.CharField(max_length=16)
+ detector_version = models.CharField(max_length=32)
+ query_hash = models.CharField(max_length=32)
+ trained_at = models.DateTimeField()
+ # The newest scored bucket start over all series. Null when the insight returned no completed bucket.
+ scored_through = models.DateTimeField(null=True, blank=True)
+ # Stored series that the last score step did not get back. More than half of them makes the next visit train.
+ missing_series = models.PositiveIntegerField(default=0)
+ # Series key to `{"first_bucket": ISO bucket start, "values": [...]}`, with the newest values the detector reads.
+ series = models.JSONField(default=dict)
+
+ created_at = models.DateTimeField(auto_now_add=True)
+ updated_at = models.DateTimeField(auto_now=True)
+
+ class Meta:
+ default_manager_name = "all_teams"
+ constraints = [
+ models.UniqueConstraint(
+ fields=["team", "insight_id", "profile"], name="signals_anomaly_scoring_state_unique"
+ ),
+ ]
diff --git a/products/signals/backend/test/test_anomaly_scores.py b/products/signals/backend/test/test_anomaly_scores.py
index 9b9dcc60337d..f9f6930b5cd8 100644
--- a/products/signals/backend/test/test_anomaly_scores.py
+++ b/products/signals/backend/test/test_anomaly_scores.py
@@ -27,16 +27,25 @@ from posthog.interval_specs import interval_spec
from posthog.models import Team
from products.product_analytics.backend.facade.models import Insight
-from products.signals.backend.anomaly_scores.config import DEFAULT_LIMITS, DEFAULT_PROFILE, IntervalScoring
+from products.signals.backend.anomaly_scores.config import DEFAULT_LIMITS, DEFAULT_PROFILE, IntervalScoring, ScoringRun
from products.signals.backend.anomaly_scores.scoring import (
+ SeriesWindow,
+ SplitRun,
+ StateIdentity,
+ TrainedState,
UnsupportedInsightError,
+ bucket_end,
buckets_to_score,
fetch_buckets,
+ history_buckets,
query_cost,
score_insight,
score_series,
+ score_window,
+ train_window,
trends_query_for,
)
+from products.signals.backend.anomaly_scores.state import needs_training
from products.signals.backend.anomaly_scores.tick import _INSIGHT_TIMEOUT
NEW_YORK = ZoneInfo("America/New_York")
@@ -178,6 +187,68 @@ class TestScoreSeries(SimpleTestCase):
assert all(point.score is None and point.flag is None for point in points)
+class TestScoreWindow(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("day", IntervalType.DAY, False),
+ ("week", IntervalType.WEEK, False),
+ ("hour", IntervalType.HOUR, False),
+ ("day_late_events", IntervalType.DAY, True),
+ ("hour_late_events", IntervalType.HOUR, True),
+ ]
+ )
+ def test_a_score_step_on_the_stored_window_scores_like_the_full_history(
+ self, _name: str, interval: IntervalType, late_events: bool
+ ) -> None:
+ scoring = DEFAULT_PROFILE.intervals[interval]
+ history = history_buckets(scoring)
+ count = history + 40
+ values = list(np.random.default_rng(2).normal(100, 10, size=count))
+ values[-1] = 400.0
+ buckets = _buckets(datetime(2025, 1, 6, tzinfo=UTC), interval, count)
+ trained_at = bucket_end(buckets[-3], interval) + timedelta(minutes=5)
+ scored_at = bucket_end(buckets[-1], interval) + timedelta(minutes=5)
+ window = train_window(values[:-1], buckets[:-1], history=history, interval=interval, now=trained_at)
+ assert window is not None
+ later = list(values)
+ if late_events:
+ # Events that arrive after the train step change a bucket that the score step fetches again.
+ later[-4] += 50.0
+ refresh = 3
+ # The score step fetches the refresh buckets, one guard bucket, and the two new buckets.
+ fetched = slice(count - (refresh + 1 + 2), count)
+
+ scored = score_window(
+ window, later[fetched], buckets[fetched], refresh=refresh, scoring=scoring, interval=interval, now=scored_at
+ )
+
+ assert scored is not None
+ points, updated = scored
+ expected = score_series(later, buckets, scoring=scoring, interval=interval, now=scored_at, after=buckets[-3])
+ assert [(point.bucket_start, point.score, point.flag) for point in points] == [
+ (point.bucket_start, point.score, point.flag) for point in expected
+ ]
+ assert [point.bucket_start for point in points] == buckets[-2:]
+ assert points[-1].flag is True
+ assert updated == SeriesWindow(first_bucket=buckets[-history], values=later[-history:])
+
+ def test_fresh_buckets_after_a_gap_leave_the_window_unscored(self) -> None:
+ buckets = _buckets(datetime(2026, 1, 1, tzinfo=UTC), IntervalType.DAY, 70)
+ window = SeriesWindow(first_bucket=buckets[0], values=[1.0] * 60)
+
+ scored = score_window(
+ window,
+ [1.0] * 5,
+ buckets[62:67],
+ refresh=3,
+ scoring=DAY_SCORING,
+ interval=IntervalType.DAY,
+ now=buckets[67],
+ )
+
+ assert scored is None
+
+
def _insight(series: list[EventsNode], **query: Any) -> Insight:
return Insight(id=7, query=TrendsQuery(series=series, **query).model_dump(exclude_none=True))
@@ -255,6 +326,130 @@ class TestScoreInsight(SimpleTestCase):
if waves == 1:
assert limits[0] == 150
+ @parameterized.expand([("no_smoothing", None, "-6d"), ("smoothing", 7, "-12d")])
+ def test_a_score_step_fetches_only_new_buckets_and_keeps_series_it_did_not_get_back(
+ self, _name: str, smoothing: int | None, date_from: str
+ ) -> None:
+ insight = _insight([EventsNode(event="$pageview")], trendsFilter=TrendsFilter(smoothingIntervals=smoothing))
+ team = Team(timezone="UTC")
+ rng = np.random.default_rng(3)
+
+ def result(breakdown: str, first_day: datetime, days: int) -> dict[str, Any]:
+ return {
+ "action": {"order": 0},
+ "label": "pageview",
+ "breakdown_value": breakdown,
+ "days": [(first_day + timedelta(days=day)).strftime("%Y-%m-%d") for day in range(days)],
+ "data": list(rng.normal(100, 10, size=days)),
+ }
+
+ def run(now: datetime, results: list[dict[str, Any]], split: SplitRun) -> tuple[Any, TrendsQuery]:
+ with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:
+ runner.return_value.run.return_value = MagicMock(results=results)
+ scores = score_insight(insight, team, now=now, split=split)
+ return scores, runner.call_args.kwargs["query"]
+
+ trained, _query = run(
+ datetime(2026, 1, 10, 0, 30, tzinfo=UTC),
+ [
+ result("kept", datetime(2025, 10, 1, tzinfo=UTC), 102),
+ result("gone", datetime(2025, 10, 1, tzinfo=UTC), 102),
+ ],
+ SplitRun(refresh_buckets=3),
+ )
+ assert trained.run == ScoringRun.TRAIN and trained.state is not None
+ keys = {series.breakdown_value: series.series_key for series in trained.series}
+
+ scores, query = run(
+ datetime(2026, 1, 12, 0, 30, tzinfo=UTC),
+ [result("kept", datetime(2026, 1, 6, tzinfo=UTC), 7), result("new", datetime(2026, 1, 6, tzinfo=UTC), 7)],
+ SplitRun(refresh_buckets=3, state=trained.state),
+ )
+
+ assert query.dateRange is not None and query.dateRange.date_from == date_from
+ assert scores.run == ScoringRun.SCORE
+ assert [
+ (series.breakdown_value, [point.bucket_start.day for point in series.points]) for series in scores.series
+ ] == [("kept", [10, 11])]
+ assert all(point.score is not None for point in scores.series[0].points)
+ assert scores.unmatched_series == 1
+ assert scores.state is not None
+ assert scores.state.missing_series == 1
+ assert scores.state.windows[keys["gone"]] == trained.state.windows[keys["gone"]]
+ assert scores.state.scored_through == datetime(2026, 1, 11, tzinfo=UTC)
+
+ def test_a_score_pick_whose_state_no_longer_matches_the_insight_trains(self) -> None:
+ stale = TrainedState(
+ identity=StateIdentity(interval=IntervalType.DAY, detector_version="old", query_hash="old"),
+ trained_at=datetime(2026, 1, 9, tzinfo=UTC),
+ scored_through=datetime(2026, 1, 9, tzinfo=UTC),
+ windows={},
+ )
+ with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:
+ runner.return_value.run.return_value = MagicMock(results=[])
+ scores = score_insight(
+ _insight([EventsNode(event="$pageview")]),
+ Team(timezone="UTC"),
+ now=datetime(2026, 1, 10, tzinfo=UTC),
+ split=SplitRun(refresh_buckets=3, state=stale),
+ )
+
+ assert scores.run == ScoringRun.TRAIN
+ assert runner.call_args.kwargs["query"].dateRange.date_from == "-90d"
+
+
+_IDENTITY = StateIdentity(interval=IntervalType.DAY, detector_version="v1", query_hash="q1")
+_SCORED_THROUGH = datetime(2026, 1, 9, tzinfo=UTC)
+_WINDOW = SeriesWindow(first_bucket=datetime(2025, 11, 11, tzinfo=UTC), values=[1.0] * 60)
+
+
+def _state(**changes: Any) -> TrainedState:
+ fields: dict[str, Any] = {
+ "identity": _IDENTITY,
+ "trained_at": datetime(2026, 1, 5, tzinfo=UTC),
+ "scored_through": _SCORED_THROUGH,
+ "windows": {"a": _WINDOW, "b": _WINDOW},
+ **changes,
+ }
+ return TrainedState(**fields)
+
+
+class TestNeedsTraining(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("fresh_state", _state(), False),
+ ("one_of_two_series_missing", _state(missing_series=1), False),
+ ("no_state", None, True),
+ (
+ "query_changed",
+ _state(identity=StateIdentity(interval=IntervalType.DAY, detector_version="v1", query_hash="q0")),
+ True,
+ ),
+ (
+ "detector_changed",
+ _state(identity=StateIdentity(interval=IntervalType.DAY, detector_version="v0", query_hash="q1")),
+ True,
+ ),
+ ("trained_too_long_ago", _state(trained_at=datetime(2026, 1, 2, tzinfo=UTC)), True),
+ ("gap_over_the_backfill_cap", _state(scored_through=datetime(2025, 12, 1, tzinfo=UTC)), True),
+ ("most_series_missing", _state(missing_series=2), True),
+ ]
+ )
+ def test_a_visit_trains_when_the_stored_state_cannot_score_it(
+ self, _name: str, state: TrainedState | None, trains: bool
+ ) -> None:
+ assert (
+ needs_training(
+ state,
+ identity=_IDENTITY,
+ now=datetime(2026, 1, 10, 1, tzinfo=UTC),
+ timezone=ZoneInfo("UTC"),
+ train_every=7,
+ max_backfill=30,
+ )
+ is trains
+ )
+
class TestMetricScoringQuery(SimpleTestCase):
@parameterized.expand([("without_compare", None), ("with_saved_compare", CompareFilter(compare=True))])
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index f7697e168203..0daeededb270 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -11,6 +11,7 @@ from unittest.mock import MagicMock, patch
from django.core.cache import cache
from django.test import SimpleTestCase
+import numpy as np
from parameterized import parameterized
from temporalio import activity
from temporalio.testing import WorkflowEnvironment
@@ -27,7 +28,7 @@ from products.dashboards.backend.models.dashboard import Dashboard
from products.dashboards.backend.models.dashboard_tile import DashboardTile
from products.product_analytics.backend.facade.api import record_insight_view
from products.product_analytics.backend.facade.models import Insight
-from products.signals.backend.anomaly_scores.config import DISABLED, AnomalyScoringConfig, parse_config
+from products.signals.backend.anomaly_scores.config import DISABLED, AnomalyScoringConfig, ScoringRun, parse_config
from products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_events
from products.signals.backend.anomaly_scores.queue import (
SKIP_COOLDOWN,
@@ -53,6 +54,7 @@ from products.signals.backend.anomaly_scores.scoring import (
UnsupportedInsightError,
)
from products.signals.backend.anomaly_scores.selection import SelectionConfig, Weighting
+from products.signals.backend.anomaly_scores.state import RunPlan
from products.signals.backend.anomaly_scores.tick import (
AnomalyScoringTickInput,
AnomalyScoringTickPlan,
@@ -63,6 +65,7 @@ from products.signals.backend.anomaly_scores.tick import (
plan_tick,
run_insight,
)
+from products.signals.backend.models import AnomalyScoringState
NEW_YORK = ZoneInfo("America/New_York")
ORGANIZATION_ID = "4dc8564d-bd82-1065-2f40-97f7c50f67cf"
@@ -121,6 +124,9 @@ class TestTickConfig(SimpleTestCase):
"intervals": ["minute", "week"],
"max_insights_per_tick": 0,
"limits": {"max_backfill_buckets": 100_000, "max_series": -1},
+ "train_score_split": "true",
+ "train_every_buckets": {"hour": 100_000, "day": 0, "week": 2},
+ "refresh_buckets": ["day"],
}
)
@@ -129,6 +135,14 @@ class TestTickConfig(SimpleTestCase):
assert config.intervals == (IntervalType.WEEK,)
assert config.max_insights_per_tick == DISABLED.max_insights_per_tick
assert (config.limits.max_backfill_buckets, config.limits.max_series) == (90, 20)
+ assert config.train_score_split is False
+ assert config.train_every_buckets == {
+ IntervalType.HOUR: 168,
+ IntervalType.DAY: 7,
+ IntervalType.WEEK: 2,
+ IntervalType.MONTH: 3,
+ }
+ assert config.refresh_buckets == DISABLED.refresh_buckets
@parameterized.expand(
[
@@ -136,6 +150,8 @@ class TestTickConfig(SimpleTestCase):
("max_concurrency", 100, 8),
("max_queries_per_hour", 1_000_000, 2000),
("max_queries_per_project_per_hour", 501, 500),
+ ("max_score_queries_per_hour", 1_000_000, 5000),
+ ("max_score_queries_per_project_per_hour", 1001, 1000),
]
)
def test_tick_levers_above_their_maximum_parse_to_the_maximum(self, field: str, value: int, expected: int) -> None:
@@ -170,6 +186,28 @@ class TestQueryBudget(SimpleTestCase):
("tick_cap", {1: 1, 2: 1, 3: 1}, QueryBudget(fleet=100, projects={1: 30}), 1, [1], 0, 0),
# Another tick reserved the queries after the snapshot.
("reservation_fails", {1: 1, 2: 1, 3: 1}, QueryBudget(fleet=100, projects={1: 30}), 10, [2, 3], 1, 1),
+ # The train budget is spent, and the score step of the second insight charges its own budget.
+ (
+ "score_step_uses_its_own_budget",
+ {1: 1, 2: 1, 3: 1},
+ QueryBudget(fleet=0, projects={1: 0}),
+ 10,
+ [2],
+ 2,
+ 0,
+ QueryBudget(fleet=5, projects={1: 5}),
+ ),
+ # A score step does not fit a spent score budget, even with train budget left.
+ (
+ "score_budget_spent",
+ {1: 1, 2: 1, 3: 1},
+ QueryBudget(fleet=100, projects={1: 30}),
+ 10,
+ [1, 3],
+ 1,
+ 0,
+ QueryBudget(fleet=0, projects={1: 0}),
+ ),
]
)
def test_the_walk_skips_picks_over_budget_and_continues(
@@ -181,20 +219,28 @@ class TestQueryBudget(SimpleTestCase):
expected: list[int],
skipped: int,
refused_reservations: int,
+ score_budget: QueryBudget | None = None,
) -> None:
- costs = {1: 3, 2: 3, 3: 2}
- calls: list[int] = []
+ plans = {
+ 1: RunPlan(queries=3),
+ 2: RunPlan(queries=3, run=ScoringRun.SCORE if score_budget else None),
+ 3: RunPlan(queries=2),
+ }
+ budgets = {ScoringRun.TRAIN: budget}
+ if score_budget is not None:
+ budgets[ScoringRun.SCORE] = score_budget
+ calls: list[RunPlan] = []
- def reserve(team_id: int, cost: int) -> bool:
- calls.append(cost)
+ def reserve(team_id: int, plan: RunPlan) -> bool:
+ calls.append(plan)
return len(calls) > refused_reservations
picked, skipped_count = within_budget(
- [1, 2, 3], team_ids, budget, limit, lambda ids: {i: costs[i] for i in ids}, reserve
+ [1, 2, 3], team_ids, budgets, limit, lambda ids: {i: plans[i] for i in ids}, reserve
)
assert (list(picked), skipped_count) == (expected, skipped)
- assert picked == {insight_id: costs[insight_id] for insight_id in expected}
+ assert picked == {insight_id: plans[insight_id] for insight_id in expected}
@parameterized.expand(
[
@@ -207,28 +253,52 @@ class TestQueryBudget(SimpleTestCase):
) -> None:
batches: list[list[int]] = []
- def costs_for(insight_ids: list[int]) -> dict[int, int]:
+ def plans_for(insight_ids: list[int]) -> dict[int, RunPlan]:
batches.append(insight_ids)
- return dict.fromkeys(insight_ids, 1)
-
- within_budget([1, 2, 3, 4], {1: 1, 2: 2, 3: 1, 4: 2}, budget, 10, costs_for, lambda _team, _cost: True)
+ return dict.fromkeys(insight_ids, RunPlan(queries=1))
+
+ within_budget(
+ [1, 2, 3, 4],
+ {1: 1, 2: 2, 3: 1, 4: 2},
+ {ScoringRun.TRAIN: budget},
+ 10,
+ plans_for,
+ lambda _team, _plan: True,
+ )
assert batches == loaded
- def test_a_reservation_takes_both_budgets_or_neither_and_resets_each_hour(self) -> None:
- config = AnomalyScoringConfig(enabled=True, max_queries_per_hour=10, max_queries_per_project_per_hour=6)
+ @parameterized.expand([(ScoringRun.TRAIN, ScoringRun.SCORE), (ScoringRun.SCORE, ScoringRun.TRAIN)])
+ def test_a_reservation_takes_both_budgets_or_neither_and_resets_each_hour(
+ self, kind: ScoringRun, other: ScoringRun
+ ) -> None:
+ config = AnomalyScoringConfig(
+ enabled=True,
+ max_queries_per_hour=10,
+ max_queries_per_project_per_hour=6,
+ max_score_queries_per_hour=10,
+ max_score_queries_per_project_per_hour=6,
+ )
next_hour = NOW + timedelta(hours=1)
for now in (NOW, next_hour):
- cache.delete_many([_fleet_budget_key(now), _project_budget_key(1, now), _project_budget_key(2, now)])
+ for budget in ScoringRun:
+ cache.delete_many(
+ [
+ _fleet_budget_key(now, budget),
+ _project_budget_key(1, now, budget),
+ _project_budget_key(2, now, budget),
+ ]
+ )
reserved = [
- reserve_queries(config, team_id, queries, NOW + timedelta(minutes=minute))
+ reserve_queries(config, team_id, queries, NOW + timedelta(minutes=minute), kind)
for team_id, queries, minute in [(1, 3, 0), (1, 3, 10), (1, 1, 20), (2, 4, 30), (2, 1, 40)]
]
assert reserved == [True, True, False, True, False]
- assert remaining_budget(config, [1, 2], NOW) == QueryBudget(fleet=0, projects={1: 0, 2: 2})
- assert remaining_budget(config, [1, 2], next_hour) == QueryBudget(fleet=10, projects={1: 6, 2: 6})
+ assert remaining_budget(config, [1, 2], NOW, kind) == QueryBudget(fleet=0, projects={1: 0, 2: 2})
+ assert remaining_budget(config, [1, 2], NOW, other) == QueryBudget(fleet=10, projects={1: 6, 2: 6})
+ assert remaining_budget(config, [1, 2], next_hour, kind) == QueryBudget(fleet=10, projects={1: 6, 2: 6})
class TestTickSelection(SimpleTestCase):
@@ -416,6 +486,61 @@ class TestRunInsight(SimpleTestCase):
assert SKIP_COOLDOWN not in cooldowns
+class TestTrainScoreSplit(BaseTest):
+ def _run(self, insight: Insight, pick: InsightPick, now: datetime, data: list[float]) -> list[dict[str, Any]]:
+ days = [(now - timedelta(days=len(data) - 1 - day)).strftime("%Y-%m-%d") for day in range(len(data))]
+ results = [{"action": {"order": 0}, "label": "pageview", "days": days, "data": data, "breakdown_value": None}]
+ with (
+ patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner,
+ patch("products.signals.backend.anomaly_scores.tick.emit") as emit,
+ patch("products.signals.backend.anomaly_scores.tick.mark_attempted"),
+ ):
+ runner.return_value.run.return_value = MagicMock(results=results)
+ run_insight(ScoreInsightInput(pick=pick), now)
+ return emit.call_args.args[1]
+
+ def test_a_train_pick_stores_the_tail_that_the_next_score_pick_scores_on(self) -> None:
+ self.team.timezone = "UTC"
+ self.team.save()
+ insight = Insight.objects.create(team=self.team, saved=True, query=_trends(interval="day"))
+ values = [float(value) for value in np.random.default_rng(4).normal(100, 5, size=100)]
+ # JSON has no NaN, so the oldest stored bucket is stored as null. The detector no longer reads it.
+ values[39] = float("nan")
+ trained_at = datetime(2026, 10, 20, 0, 30, tzinfo=UTC)
+
+ trained = self._run(
+ insight,
+ InsightPick(team_id=self.team.id, insight_id=insight.id, scored_through={}, run=ScoringRun.TRAIN),
+ trained_at,
+ values,
+ )
+ scored = self._run(
+ insight,
+ InsightPick(
+ team_id=self.team.id,
+ insight_id=insight.id,
+ scored_through={trained[-1]["properties"]["$anomaly_series_key"]: datetime(2026, 10, 19, tzinfo=UTC)},
+ run=ScoringRun.SCORE,
+ refresh_buckets=3,
+ ),
+ trained_at + timedelta(days=2),
+ [*values[-4:], 500.0, 100.0],
+ )
+
+ assert {event["properties"]["$anomaly_run"] for event in trained} == {"train"}
+ assert [
+ (
+ event["properties"]["$anomaly_run"],
+ event["properties"]["$anomaly_bucket_start"],
+ event["properties"]["$anomaly_flag"],
+ )
+ for event in scored
+ ] == [("score", "2026-10-20T00:00:00+00:00", 0), ("score", "2026-10-21T00:00:00+00:00", 1)]
+ state = AnomalyScoringState.objects.for_team(self.team.id).get(insight_id=insight.id)
+ assert state.scored_through == datetime(2026, 10, 21, tzinfo=UTC)
+ assert state.trained_at == trained_at
+
+
class TestEligibleTeams(BaseTest):
@parameterized.expand(
[