Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): limit anomaly scoring by cpu per project and fleet (#127033)

posthog[bot]8255cf2ae1d0
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.py
index 9fe62d152f78..f0f76e6beeed 100644
--- a/products/signals/backend/anomaly_scores/config.py
+++ b/products/signals/backend/anomaly_scores/config.py
@@ -156,6 +156,10 @@ class AnomalyScoringConfig:
due_list: bool = False
# Each enrolled team's due list refreshes about this often.
due_list_refresh_hours: int = 6
+ # ClickHouse CPU seconds of scoring queries per UTC day, read from the query log, see `spend.py`. Zero is off.
+ # A project at its limit gets no picks until the next UTC day. Near the fleet limit, train picks stop first.
+ max_cpu_seconds_per_project_per_day: int = 0
+ max_fleet_cpu_seconds_per_day: int = 0
DISABLED = AnomalyScoringConfig()
@@ -303,6 +307,12 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:
_positive_int(payload.get("due_list_refresh_hours"), defaults.due_list_refresh_hours),
MAX_DUE_LIST_REFRESH_HOURS,
),
+ max_cpu_seconds_per_project_per_day=_positive_int(
+ payload.get("max_cpu_seconds_per_project_per_day"), defaults.max_cpu_seconds_per_project_per_day
+ ),
+ max_fleet_cpu_seconds_per_day=_positive_int(
+ payload.get("max_fleet_cpu_seconds_per_day"), defaults.max_fleet_cpu_seconds_per_day
+ ),
)
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index 32049c1f029c..9ecb3a5a553b 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -11,6 +11,9 @@ first: it takes the trained insights from their stored state, longest overdue fi
keep them current. The train pass then takes the other due insights in the `pick_order` order. An insight
whose model is due for a soft retrain goes to the train pass first. When the train pass does not pick it,
the score pass scores it with the old model, so the train budget does not decide the score coverage.
+
+The CPU limits of `spend.py` apply before the query budgets: a walk leaves out the insights of a team over its
+daily CPU limit, and every insight of a pass that the fleet limit stops.
"""
import time
@@ -67,6 +70,7 @@ from products.signals.backend.anomaly_scores.selection import (
pick_order,
score_order,
)
+from products.signals.backend.anomaly_scores.spend import CpuSpendReport, cpu_gate
from products.signals.backend.anomaly_scores.state import RunPlan, load_states, plan_run, score_ready_by_insight
from products.signals.backend.models import AnomalyScoringDueInsight, SignalTeamConfig
from products.subscriptions.backend.facade.api import insight_ids_with_active_subscriptions
@@ -203,6 +207,7 @@ class Selection:
due_insights: int | None = None
oldest_due_seconds: int | None = None
oldest_refresh_seconds: int | None = None
+ cpu: CpuSpendReport | None = None
@frozen
@@ -682,6 +687,8 @@ def select_insights(
With the due list on, the plan reads the stored due list instead, and ``deadline`` has no effect.
"""
teams = {team.id: team for team in eligible_teams(config)}
+ gate = cpu_gate(config, now)
+ cpu_skipped: Counter[ScoringRun] = Counter()
due_list: DueList | None = None
if config.due_list:
due_list = load_due_list(teams, config.intervals, now)
@@ -724,6 +731,9 @@ def select_insights(
cap: int,
plans: Callable[[list[int]], dict[int, RunPlan]] = plans_for,
) -> tuple[dict[int, RunPlan], int]:
+ allowed = [insight_id for insight_id in order if not gate.blocks(team_ids[insight_id], kind)]
+ cpu_skipped[kind] += len(order) - len(allowed)
+ order = allowed
# Read the budget before any cost, so a spent hour loads no insight.
return within_budget(
order,
@@ -849,6 +859,14 @@ def select_insights(
due_insights=None if due_list is None else len(due_list.insights),
oldest_due_seconds=None if due_list is None else due_list.oldest_due_seconds,
oldest_refresh_seconds=None if due_list is None else due_list.oldest_refresh_seconds,
+ cpu=CpuSpendReport(
+ status=gate.status,
+ fleet_cpu_seconds=gate.fleet_cpu_seconds,
+ fleet_read_bytes=gate.fleet_read_bytes,
+ teams_over_limit=len(gate.blocked_teams & teams.keys()),
+ train_skipped=cpu_skipped[ScoringRun.TRAIN],
+ score_skipped=cpu_skipped[ScoringRun.SCORE],
+ ),
)
diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.py
index 6b96eecd35a7..c612b7a18108 100644
--- a/products/signals/backend/anomaly_scores/scoring.py
+++ b/products/signals/backend/anomaly_scores/scoring.py
@@ -629,9 +629,9 @@ def _fetch(
tag_queries(**get_team_query_tags(team), insight_id=insight.pk)
tag_queries(product=Product.SIGNALS, feature=Feature.ALERTING)
- if run is not None:
- # Splits the scoring queries in `query_log`, so train and score cost can be compared.
- tag_queries(name=f"anomaly_scoring_{run.value}")
+ # Splits the scoring queries in `query_log`, so train and score cost can be compared, and lets `spend.py`
+ # find them. A visit with the split off fetches the full history, as a train step does.
+ tag_queries(name=f"anomaly_scoring_{(run or ScoringRun.TRAIN).value}")
runner = TrendsQueryRunner(
query=scoring_query,
team=team,
diff --git a/products/signals/backend/anomaly_scores/spend.py b/products/signals/backend/anomaly_scores/spend.py
new file mode 100644
index 000000000000..7812d7986d3a
--- /dev/null
+++ b/products/signals/backend/anomaly_scores/spend.py
@@ -0,0 +1,192 @@
+"""The ClickHouse CPU that anomaly scoring spends, read from the query log, for the CPU limits of the tick.
+
+The query budgets of `queue.py` count queries, but the CPU of one query can be far more than the CPU of another.
+One read of `query_log_archive` sums the CPU and the read bytes of the scoring queries of the current UTC day,
+per team. The cache keeps the sum, and the tick reads the query log again only when the sum is older than
+`_REFRESH_EVERY`. When the sum is missing or stale, the tick does not apply the CPU limits, and the query
+budgets still apply.
+"""
+
+import datetime
+from enum import StrEnum
+
+from django.core.cache import cache
+
+import structlog
+
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
+from posthog.clickhouse.workload import Workload
+from posthog.dataclasses import frozen
+
+from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig, ScoringRun
+
+logger = structlog.get_logger(__name__)
+
+# The `name` tag of each scoring query, set in `scoring.fetch_series`.
+SCORING_QUERY_NAMES = tuple(f"anomaly_scoring_{run.value}" for run in ScoringRun)
+# The query log reaches the archive through a buffer, so a few minutes of spend can be missing from a read.
+_REFRESH_EVERY = datetime.timedelta(minutes=10)
+# Older than this, a sum does not limit the tick.
+_STALE_AFTER = datetime.timedelta(minutes=30)
+_CACHE_KEY = "signals:anomaly_scores:cpu_spend"
+_CACHE_TTL = datetime.timedelta(hours=2)
+# Train picks stop at this share of the fleet limit, so the cheaper score picks still run up to the full limit.
+TRAIN_STOP_SHARE = 0.8
+
+# `query_log_archive` keeps one row for each node of a distributed query, until a merge sums them. The `lc_*`
+# columns hold the tags on every row, `OSCPUVirtualTimeMicroseconds` holds the CPU of the node, and `read_bytes`
+# is zero except on the initial row. So a plain sum over the rows gives the CPU of all nodes and the bytes once.
+_SPEND_QUERY = """
+SELECT
+ team_id,
+ sum(ProfileEvents_OSCPUVirtualTimeMicroseconds) AS cpu_microseconds,
+ sum(read_bytes) AS read_bytes
+FROM query_log_archive
+WHERE initial_query_start_time >= %(since)s
+ AND lc_product = %(product)s
+ AND lc_feature = %(feature)s
+ AND lc_name IN %(names)s
+GROUP BY team_id
+SETTINGS skip_unavailable_shards = 1
+"""
+
+
+class SpendStatus(StrEnum):
+ FRESH = "fresh"
+ # No sum for the current UTC day: the read failed, or no read ran yet.
+ MISSING = "missing"
+ STALE = "stale"
+
+
+@frozen
+class TeamSpend:
+ cpu_seconds: float
+ read_bytes: int
+
+
+@frozen
+class SpendSnapshot:
+ """The scoring spend of each team from the start of ``day`` (UTC) to ``read_at``."""
+
+ day: datetime.date
+ read_at: datetime.datetime
+ teams: dict[int, TeamSpend]
+
+ @property
+ def fleet_cpu_seconds(self) -> float:
+ return sum(spend.cpu_seconds for spend in self.teams.values())
+
+ @property
+ def fleet_read_bytes(self) -> int:
+ return sum(spend.read_bytes for spend in self.teams.values())
+
+
+@frozen
+class CpuGate:
+ """What the CPU limits block in one plan. Without fresh spend, nothing is blocked."""
+
+ status: SpendStatus
+ # The teams at or over the per-project limit.
+ blocked_teams: frozenset[int] = frozenset()
+ # The plan kinds that the fleet limit stops.
+ blocked_runs: frozenset[ScoringRun] = frozenset()
+ fleet_cpu_seconds: float | None = None
+ fleet_read_bytes: int | None = None
+
+ def blocks(self, team_id: int, run: ScoringRun) -> bool:
+ return run in self.blocked_runs or team_id in self.blocked_teams
+
+
+@frozen
+class CpuSpendReport:
+ """What the CPU limits saw and did in one plan, for the tick event."""
+
+ status: SpendStatus
+ fleet_cpu_seconds: float | None = None
+ fleet_read_bytes: int | None = None
+ # Enrolled teams at or over the per-project limit.
+ teams_over_limit: int = 0
+ # Due insights that the walk of each pass left out because of a CPU limit.
+ train_skipped: int = 0
+ score_skipped: int = 0
+
+
+def _utc_day(now: datetime.datetime) -> datetime.date:
+ return now.astimezone(datetime.UTC).date()
+
+
+def _day_start(day: datetime.date) -> datetime.datetime:
+ return datetime.datetime.combine(day, datetime.time.min, tzinfo=datetime.UTC)
+
+
+def read_spend(now: datetime.datetime) -> SpendSnapshot:
+ """The scoring spend of each team in the UTC day of ``now``, from the query log."""
+ day = _utc_day(now)
+ tag_queries(product=Product.SIGNALS, feature=Feature.ALERTING, name="anomaly_scoring_spend")
+ rows = sync_execute(
+ _SPEND_QUERY,
+ {
+ "since": _day_start(day),
+ "product": Product.SIGNALS.value,
+ "feature": Feature.ALERTING.value,
+ "names": SCORING_QUERY_NAMES,
+ },
+ workload=Workload.OFFLINE,
+ )
+ return SpendSnapshot(
+ day=day,
+ read_at=now,
+ teams={
+ int(team_id): TeamSpend(cpu_seconds=cpu_microseconds / 1_000_000, read_bytes=int(read_bytes))
+ for team_id, cpu_microseconds, read_bytes in rows
+ },
+ )
+
+
+def current_spend(now: datetime.datetime) -> SpendSnapshot | None:
+ """The cached spend of the current UTC day. A cached spend older than `_REFRESH_EVERY` or of an earlier day is read again.
+
+ A failed read keeps the cached spend, so the caller can tell a stale spend from a missing one.
+ """
+ cached = cache.get(_CACHE_KEY)
+ snapshot = cached if isinstance(cached, SpendSnapshot) else None
+ if snapshot is not None and snapshot.day == _utc_day(now) and now - snapshot.read_at < _REFRESH_EVERY:
+ return snapshot
+ try:
+ snapshot = read_spend(now)
+ except Exception as error:
+ logger.warning("anomaly_scoring_spend_read_failed", error_class=type(error).__name__)
+ return snapshot
+ cache.set(_CACHE_KEY, snapshot, timeout=int(_CACHE_TTL.total_seconds()))
+ return snapshot
+
+
+def cpu_gate(config: AnomalyScoringConfig, now: datetime.datetime) -> CpuGate:
+ """The teams and plan kinds that the CPU limits of ``config`` block now.
+
+ The per-project limit blocks a team until the next UTC day. The fleet limit stops train picks at
+ `TRAIN_STOP_SHARE` of it, and score picks at all of it.
+ """
+ snapshot = current_spend(now)
+ if snapshot is None or snapshot.day != _utc_day(now):
+ return CpuGate(status=SpendStatus.MISSING)
+ if now - snapshot.read_at > _STALE_AFTER:
+ return CpuGate(status=SpendStatus.STALE)
+ fleet = snapshot.fleet_cpu_seconds
+ blocked_runs: set[ScoringRun] = set()
+ if limit := config.max_fleet_cpu_seconds_per_day:
+ if fleet >= limit * TRAIN_STOP_SHARE:
+ blocked_runs.add(ScoringRun.TRAIN)
+ if fleet >= limit:
+ blocked_runs.add(ScoringRun.SCORE)
+ project_limit = config.max_cpu_seconds_per_project_per_day
+ return CpuGate(
+ status=SpendStatus.FRESH,
+ blocked_teams=frozenset(
+ team_id for team_id, spend in snapshot.teams.items() if project_limit and spend.cpu_seconds >= project_limit
+ ),
+ blocked_runs=frozenset(blocked_runs),
+ fleet_cpu_seconds=fleet,
+ fleet_read_bytes=snapshot.fleet_read_bytes,
+ )
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index 0a8541b31a1b..2355886637c3 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -65,6 +65,7 @@ from products.signals.backend.anomaly_scores.scoring import (
score_insight,
)
from products.signals.backend.anomaly_scores.selection import parse_selection_config
+from products.signals.backend.anomaly_scores.spend import CpuSpendReport
from products.signals.backend.anomaly_scores.state import invalidate_state, load_state, save_state
logger = structlog.get_logger(__name__)
@@ -134,6 +135,7 @@ class AnomalyScoringTickPlan:
due_insights: int | None = None
oldest_due_seconds: int | None = None
oldest_refresh_seconds: int | None = None
+ cpu: CpuSpendReport | None = None
@frozen
@@ -197,6 +199,8 @@ class AnomalyScoringTickResult:
due_insights: int | None = None
oldest_due_seconds: int | None = None
oldest_refresh_seconds: int | None = None
+ # The CPU spend that the plan read, and what its CPU limits left out. None when the plan did not run.
+ cpu: CpuSpendReport | None = None
def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:
@@ -231,6 +235,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
due_insights=selection.due_insights,
oldest_due_seconds=selection.oldest_due_seconds,
oldest_refresh_seconds=selection.oldest_refresh_seconds,
+ cpu=selection.cpu,
)
@@ -493,7 +498,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
due_insights=result.due_insights,
oldest_due_seconds=result.oldest_due_seconds,
oldest_refresh_seconds=result.oldest_refresh_seconds,
+ cpu=result.cpu,
)
+ cpu = result.cpu
ph_background_capture()(
distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,
event="anomaly scoring tick",
@@ -528,6 +535,12 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
"due_insights": result.due_insights,
"oldest_due_seconds": result.oldest_due_seconds,
"oldest_refresh_seconds": result.oldest_refresh_seconds,
+ "cpu_spend_status": cpu.status if cpu else None,
+ "fleet_cpu_seconds_today": cpu.fleet_cpu_seconds if cpu else None,
+ "fleet_read_bytes_today": cpu.fleet_read_bytes if cpu else None,
+ "teams_over_cpu_limit": cpu.teams_over_limit if cpu else None,
+ "train_skipped_for_cpu": cpu.train_skipped if cpu else None,
+ "score_skipped_for_cpu": cpu.score_skipped if cpu else None,
},
)
@@ -648,6 +661,7 @@ class AnomalyScoringTickWorkflow:
due_insights=plan.due_insights,
oldest_due_seconds=plan.oldest_due_seconds,
oldest_refresh_seconds=plan.oldest_refresh_seconds,
+ cpu=plan.cpu,
)
await self._capture(result)
return result
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index 28b41af2de6d..8e7fa61a6716 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -24,6 +24,7 @@ from posthog.schema import IntervalType
from posthog.hogql.errors import QueryError, ResolutionError, TableAccessDeniedError
+from posthog.clickhouse.client import sync_execute
from posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimitExceeded, ClickHouseQueryTimeOut
from posthog.models import Organization, Team, User
from posthog.tasks.alerts.detectors.base import ARTIFACT_SCHEMA_VERSION
@@ -77,6 +78,14 @@ from products.signals.backend.anomaly_scores.scoring import (
trends_query_for,
)
from products.signals.backend.anomaly_scores.selection import SelectionConfig, Weighting
+from products.signals.backend.anomaly_scores.spend import (
+ _CACHE_KEY,
+ SpendSnapshot,
+ SpendStatus,
+ TeamSpend,
+ cpu_gate,
+ read_spend,
+)
from products.signals.backend.anomaly_scores.state import RunPlan, load_state, plan_run, save_state
from products.signals.backend.anomaly_scores.tick import (
AnomalyScoringTickInput,
@@ -217,6 +226,72 @@ class TestTickConfig(SimpleTestCase):
assert (config.max_score_insights_per_tick, config.max_train_insights_per_tick) == (score_cap, train_cap)
+class TestCpuGate(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("limits_off", 0, 0, None, False, SpendStatus.FRESH, set(), set()),
+ ("team_over_its_limit", 10, 0, None, False, SpendStatus.FRESH, {1}, set()),
+ ("fleet_near_its_limit_stops_train", 0, 20, None, False, SpendStatus.FRESH, set(), {ScoringRun.TRAIN}),
+ (
+ "fleet_at_its_limit_stops_both",
+ 0,
+ 17,
+ None,
+ False,
+ SpendStatus.FRESH,
+ set(),
+ {ScoringRun.TRAIN, ScoringRun.SCORE},
+ ),
+ (
+ "recent_cache_skips_the_read",
+ 10,
+ 17,
+ 5,
+ True,
+ SpendStatus.FRESH,
+ {1},
+ {ScoringRun.TRAIN, ScoringRun.SCORE},
+ ),
+ ("stale_spend_blocks_nothing", 10, 17, 40, True, SpendStatus.STALE, set(), set()),
+ ("missing_spend_blocks_nothing", 10, 17, None, True, SpendStatus.MISSING, set(), set()),
+ ]
+ )
+ def test_cpu_limits_block_only_with_fresh_spend(
+ self,
+ _name: str,
+ project_limit: int,
+ fleet_limit: int,
+ cached_minutes_ago: int | None,
+ read_fails: bool,
+ status: SpendStatus,
+ blocked_teams: set[int],
+ blocked_runs: set[ScoringRun],
+ ) -> None:
+ def snapshot(read_at: datetime) -> SpendSnapshot:
+ return SpendSnapshot(
+ day=read_at.date(),
+ read_at=read_at,
+ teams={1: TeamSpend(cpu_seconds=12.0, read_bytes=100), 2: TeamSpend(cpu_seconds=5.0, read_bytes=50)},
+ )
+
+ cache.delete(_CACHE_KEY)
+ if cached_minutes_ago is not None:
+ cache.set(_CACHE_KEY, snapshot(NOW - timedelta(minutes=cached_minutes_ago)))
+ config = AnomalyScoringConfig(
+ enabled=True,
+ max_cpu_seconds_per_project_per_day=project_limit,
+ max_fleet_cpu_seconds_per_day=fleet_limit,
+ )
+
+ with patch(
+ "products.signals.backend.anomaly_scores.spend.read_spend",
+ side_effect=ClickHouseQueryTimeOut() if read_fails else snapshot,
+ ):
+ gate = cpu_gate(config, NOW)
+
+ assert (gate.status, set(gate.blocked_teams), set(gate.blocked_runs)) == (status, blocked_teams, blocked_runs)
+
+
class TestQueryBudget(SimpleTestCase):
@parameterized.expand(
[
@@ -1028,6 +1103,47 @@ class TestDueList(ClickhouseTestMixin, BaseTest):
assert sorted(rows) == sorted((team_id, insight.id) for team_id, insight in active.items())
+class TestReadSpend(ClickhouseTestMixin, BaseTest):
+ def test_sums_the_scoring_queries_of_the_day_per_team(self) -> None:
+ today = NOW.replace(hour=1)
+ other_team_id = self.team.id + 1_000_000
+ rows = [
+ # One distributed train query: the initial row holds the bytes, and every node row holds its CPU.
+ (today, self.team.id, "train", 1, 2_000_000, 300, "signals", "alerting", "anomaly_scoring_train"),
+ (today, self.team.id, "train", 0, 1_000_000, 0, "signals", "alerting", "anomaly_scoring_train"),
+ (today, self.team.id, "score", 1, 500_000, 40, "signals", "alerting", "anomaly_scoring_score"),
+ (today, other_team_id, "other_team", 1, 4_000_000, 7, "signals", "alerting", "anomaly_scoring_score"),
+ # Not a scoring query: the scored-buckets read, and a query of another product with the same name.
+ (today, self.team.id, "buckets", 1, 9_000_000, 9, "signals", "alerting", ""),
+ (today, self.team.id, "foreign", 1, 9_000_000, 9, "insights", "alerting", "anomaly_scoring_train"),
+ (
+ today - timedelta(days=1),
+ self.team.id,
+ "yesterday",
+ 1,
+ 9_000_000,
+ 9,
+ "signals",
+ "alerting",
+ "anomaly_scoring_train",
+ ),
+ ]
+ sync_execute(
+ """INSERT INTO sharded_query_log_archive_v3
+ (initial_query_start_time, team_id, initial_query_id, is_initial_query,
+ ProfileEvents_OSCPUVirtualTimeMicroseconds, read_bytes, lc_product, lc_feature, lc_name) VALUES""",
+ rows,
+ )
+
+ snapshot = read_spend(NOW)
+
+ # The query log also holds the real queries of other tests.
+ assert {team_id: snapshot.teams.get(team_id) for team_id in (self.team.id, other_team_id)} == {
+ self.team.id: TeamSpend(cpu_seconds=3.5, read_bytes=340),
+ other_team_id: TeamSpend(cpu_seconds=4.0, read_bytes=7),
+ }
+
+
class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
def test_reads_the_newest_scored_bucket_of_each_series(self) -> None:
def score(insight_id: int, series_key: str, bucket: datetime, source: str = "insight") -> None:
@@ -1185,6 +1301,47 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
assert [pick.insight_id for pick in selection.picks] == [insights[self.team.id].id]
assert (selection.teams_planned, selection.teams_skipped) == (1, 1)
+ @time_machine.travel(NOW, tick=False)
+ def test_select_insights_skips_a_project_over_its_cpu_limit(self) -> None:
+ heavy_team = Team.objects.create(organization=self.organization, name="Heavy")
+ insights = {}
+ for team in (self.team, heavy_team):
+ insight = Insight.objects.create(team=team, name=team.name, query=_trends(interval="day"))
+ DashboardTile.objects.create(dashboard=Dashboard.objects.create(team=team, name="Main"), insight=insight)
+ cache.delete_many([_cooldown_key(insight.id), _team_loaded_key(team.id)])
+ insights[team.id] = insight.id
+ cache.delete_many(
+ [_CACHE_KEY, _fleet_budget_key(NOW), *(_project_budget_key(team_id, NOW) for team_id in insights)]
+ )
+ config = AnomalyScoringConfig(
+ enabled=True,
+ organization_ids=(str(self.organization.id),),
+ allow_team_ids=(self.team.id, heavy_team.id),
+ max_cpu_seconds_per_project_per_day=60,
+ )
+ spend = SpendSnapshot(
+ day=NOW.date(),
+ read_at=NOW,
+ teams={
+ heavy_team.id: TeamSpend(cpu_seconds=61.0, read_bytes=1),
+ self.team.id: TeamSpend(cpu_seconds=1.0, read_bytes=1),
+ },
+ )
+
+ with (
+ patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]),
+ patch("products.signals.backend.anomaly_scores.spend.read_spend", return_value=spend),
+ ):
+ selection = select_insights(config, SelectionConfig(weighting=Weighting.UNIFORM), NOW, limit=10)
+
+ assert [pick.insight_id for pick in selection.picks] == [insights[self.team.id]]
+ assert selection.cpu is not None
+ assert (selection.cpu.status, selection.cpu.teams_over_limit, selection.cpu.train_skipped) == (
+ SpendStatus.FRESH,
+ 1,
+ 1,
+ )
+
@time_machine.travel(NOW, tick=False)
def test_a_plan_past_its_deadline_loads_one_team_and_the_next_plan_starts_from_another(self) -> None:
other_team = Team.objects.create(organization=self.organization, name="Other")