:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): bound anomaly scoring with hourly query budgets (#122429)
posthog[bot]1500e3f68fb0
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.pyindex 0e97e4690af6..5d4631dee843 100644--- a/products/signals/backend/anomaly_scores/config.py+++ b/products/signals/backend/anomaly_scores/config.py@@ -93,13 +93,21 @@ class AnomalyScoringConfig:allow_team_ids: tuple[int, ...] = ()skip_team_ids: tuple[int, ...] = ()intervals: tuple[IntervalType, ...] = DEFAULT_TICK_INTERVALS- # The budget is scoring queries, not events. One insight is one query.+ # Smooths the spend over the hour, so one tick cannot use the whole query budget.max_insights_per_tick: int = 8max_concurrency: int = 2+ # Scoring queries per clock hour. One pick sends one query per series, see `scoring.query_cost`.+ max_queries_per_hour: int = 120+ max_queries_per_project_per_hour: int = 30limits: ScoringLimits = DEFAULT_LIMITSDISABLED = AnomalyScoringConfig()+# The largest value a flag payload can set for each lever of the tick.+MAX_INSIGHTS_PER_TICK = 50+MAX_CONCURRENCY = 8+MAX_QUERIES_PER_HOUR = 2000+MAX_QUERIES_PER_PROJECT_PER_HOUR = 500def _positive_int(value: Any, default: int) -> int:@@ -161,8 +169,17 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:allow_team_ids=_team_ids(payload.get("allow_team_ids")),skip_team_ids=_team_ids(payload.get("skip_team_ids")),intervals=_intervals(payload.get("intervals")),- max_insights_per_tick=_positive_int(payload.get("max_insights_per_tick"), defaults.max_insights_per_tick),- max_concurrency=_positive_int(payload.get("max_concurrency"), defaults.max_concurrency),+ max_insights_per_tick=min(+ _positive_int(payload.get("max_insights_per_tick"), defaults.max_insights_per_tick), MAX_INSIGHTS_PER_TICK+ ),+ max_concurrency=min(_positive_int(payload.get("max_concurrency"), defaults.max_concurrency), MAX_CONCURRENCY),+ max_queries_per_hour=min(+ _positive_int(payload.get("max_queries_per_hour"), defaults.max_queries_per_hour), MAX_QUERIES_PER_HOUR+ ),+ max_queries_per_project_per_hour=min(+ _positive_int(payload.get("max_queries_per_project_per_hour"), defaults.max_queries_per_project_per_hour),+ MAX_QUERIES_PER_PROJECT_PER_HOUR,+ ),limits=parse_limits(payload.get("limits")),)diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.pyindex 782fdf0d1f5a..468150a5e115 100644--- a/products/signals/backend/anomaly_scores/queue.py+++ b/products/signals/backend/anomaly_scores/queue.py@@ -1,12 +1,13 @@"""Pick the watched insights that the next anomaly scoring tick scores.There is no stored state. One query per team reads the newest scored bucket of each series back-from the `$anomaly_score` events. `selection.pick_order` then orders the due insights.+from the `$anomaly_score` events. `selection.pick_order` then orders the due insights, and the tick+picks from that order the insights whose queries fit the hourly budgets."""import datetimeimport itertools-from collections.abc import Collection+from collections.abc import Callable, Collectionfrom typing import Any, castfrom django.core.cache import cache@@ -25,9 +26,12 @@ from posthog.models import Teamfrom posthog.tasks.alerts.utils import NON_TIME_SERIES_DISPLAY_TYPESfrom products.alerts_platform.backend.facade.scheduling import CalendarInterval, alert_check_offset-from products.product_analytics.backend.facade.api import watched_trends_insights+from products.product_analytics.backend.facade.api import (+ insights_including_soft_deleted_for_team,+ watched_trends_insights,+)from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig-from products.signals.backend.anomaly_scores.scoring import bucket_end+from products.signals.backend.anomaly_scores.scoring import bucket_end, query_costfrom products.signals.backend.anomaly_scores.selection import (Candidate,InsightUsage,@@ -49,6 +53,9 @@ _MAX_SCORED_SERIES = 50_000ATTEMPT_COOLDOWN = datetime.timedelta(hours=1)# For a failure that the next hour does not fix: an unsupported shape, a query that is too expensive, or no access.SKIP_COOLDOWN = datetime.timedelta(days=1)+# A query counter lives a little longer than its clock hour, so a tick at the end of the hour still reads it.+# A lost counter lets the scorer spend at most one more hour of budget.+_BUDGET_TTL = datetime.timedelta(minutes=70)_CALENDAR_INTERVALS = {IntervalType.HOUR: CalendarInterval.HOURLY,@@ -91,13 +98,26 @@ class InsightPick:# The newest scored bucket start of each series, in UTC. Empty when the insight was never scored.scored_through: dict[str, datetime.datetime]interval: IntervalType | None = None+ # The ClickHouse queries this pick sends. Selection charges them to the hourly budgets, so two ticks+ # that overlap cannot spend the same budget.+ query_cost: int = 1@frozenclass Selection:picks: list[InsightPick]- # Due insights, picked or not. More than the picks means the tick cap cut the queue.+ # Due insights, picked or not. More than the picks means the tick cap or a budget cut the queue.candidates: int+ # Due insights that the walk reached but skipped, because their queries did not fit a budget.+ skipped_for_budget: int = 0+++@frozen+class QueryBudget:+ """Scoring queries left in this clock hour, for the fleet and for each project."""++ fleet: int+ projects: dict[int, int]def eligible_teams(config: AnomalyScoringConfig) -> list[Team]:@@ -199,6 +219,97 @@ def _cooling_down(insight_ids: list[int]) -> set[int]:return {insight_id for insight_id in insight_ids if _cooldown_key(insight_id) in found}+def _hour(now: datetime.datetime) -> str:+ return _utc(now).strftime("%Y%m%d%H")+++def _fleet_budget_key(now: datetime.datetime) -> str:+ return f"signals:anomaly_scores:queries:{_hour(now)}"+++def _project_budget_key(team_id: int, now: datetime.datetime) -> str:+ return f"signals:anomaly_scores:queries:{_hour(now)}:team:{team_id}"+++def remaining_budget(config: AnomalyScoringConfig, team_ids: Collection[int], now: datetime.datetime) -> QueryBudget:+ project_keys = {team_id: _project_budget_key(team_id, now) for team_id in team_ids}+ spent = cache.get_many([_fleet_budget_key(now), *project_keys.values()])+ return QueryBudget(+ fleet=config.max_queries_per_hour - spent.get(_fleet_budget_key(now), 0),+ projects={+ team_id: config.max_queries_per_project_per_hour - spent.get(key, 0)+ for team_id, key in project_keys.items()+ },+ )+++def _reserve(key: str, queries: int, cap: int) -> bool:+ # `incr` is atomic, so two planners that run at the same time cannot both take the last queries.+ # `incr` fails on a missing key, and `add` does nothing when the key exists.+ cache.add(key, 0, timeout=int(_BUDGET_TTL.total_seconds()))+ if cache.incr(key, queries) <= cap:+ return True+ cache.decr(key, queries)+ return False+++def reserve_queries(config: AnomalyScoringConfig, team_id: int, queries: int, now: datetime.datetime) -> bool:+ """Charge ``queries`` to both hourly budgets, or to neither when one of them has too few left."""+ fleet_key = _fleet_budget_key(now)+ if not _reserve(fleet_key, queries, config.max_queries_per_hour):+ return False+ if _reserve(_project_budget_key(team_id, now), queries, config.max_queries_per_project_per_hour):+ return True+ cache.decr(fleet_key, queries)+ return False+++def within_budget(+ order: list[int],+ team_ids: dict[int, int],+ budget: QueryBudget,+ limit: int,+ costs_for: Callable[[list[int]], dict[int, int]],+ reserve: Callable[[int, int], bool],+) -> tuple[dict[int, int], int]:+ """The insights of ``order`` whose cost fits both budgets, up to ``limit``, with their costs, and the skip count.++ A pick that does not fit is skipped and the walk continues, so a large insight does not block smaller ones.+ ``budget`` is a snapshot that skips most misses without a cache call. ``reserve`` charges a pick for real.+ ``costs_for`` loads costs in batches of ``limit``, and only for insights that the snapshot can still fit.+ """+ fleet = budget.fleet+ projects = dict(budget.projects)+ picked: dict[int, int] = {}+ skipped = 0+ batch_size = max(1, limit)+ for start in range(0, len(order), batch_size):+ if len(picked) >= limit:+ break+ batch = order[start : start + batch_size]+ affordable = [insight_id for insight_id in batch if fleet > 0 and projects.get(team_ids[insight_id], 0) > 0]+ costs = costs_for(affordable) if affordable else {}+ for insight_id in batch:+ if len(picked) >= limit:+ break+ team_id = team_ids[insight_id]+ cost = costs.get(insight_id)+ if cost is None or cost > fleet or cost > projects.get(team_id, 0) or not reserve(team_id, cost):+ skipped += 1+ continue+ picked[insight_id] = cost+ fleet -= cost+ projects[team_id] -= cost+ return picked, skipped+++def _query_costs(team: Team, insight_ids: list[int]) -> dict[int, int]:+ insights = insights_including_soft_deleted_for_team(team_id=team.id, insight_ids=insight_ids)+ costs = {insight.id: query_cost(insight, team) for insight in insights}+ # An insight that no longer exists sends no query, but one query is the safe charge.+ return {insight_id: costs.get(insight_id, 1) for insight_id in insight_ids}++def select_insights(config: AnomalyScoringConfig, selection_config: SelectionConfig, now: datetime.datetime, limit: int) -> Selection:@@ -253,13 +364,32 @@ def select_insights(order = [insight_id for turn in itertools.zip_longest(*team_orders) for insight_id in turn if insight_id is not None]+ team_ids = {insight_id: due_by_id[insight_id].team_id for insight_id in order}++ def costs_for(insight_ids: list[int]) -> dict[int, int]:+ costs: dict[int, int] = {}+ for team_id in sorted({team_ids[insight_id] for insight_id in insight_ids}):+ team_batch = [insight_id for insight_id in insight_ids if team_ids[insight_id] == team_id]+ costs.update(_query_costs(teams[team_id], team_batch))+ return costs++ # Read the budget before any cost, so a spent hour loads no insight.+ picked, skipped = within_budget(+ order,+ team_ids,+ remaining_budget(config, set(team_ids.values()), now),+ limit,+ costs_for,+ reserve=lambda team_id, cost: reserve_queries(config, team_id, cost, now),+ )picks = [InsightPick(- team_id=due_by_id[insight_id].team_id,+ team_id=team_ids[insight_id],insight_id=insight_id,scored_through=state.get(insight_id, {}),interval=due_by_id[insight_id].interval,+ query_cost=cost,)- for insight_id in order[:limit]+ for insight_id, cost in picked.items()]- return Selection(picks=picks, candidates=len(order))+ return Selection(picks=picks, candidates=len(order), skipped_for_budget=skipped)diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.pyindex 489fe36d4c47..25f9badba1d1 100644--- a/products/signals/backend/anomaly_scores/scoring.py+++ b/products/signals/backend/anomaly_scores/scoring.py@@ -20,6 +20,7 @@ from posthog.clickhouse.workload import Workloadfrom posthog.dataclasses import frozenfrom posthog.event_usage import EventSourcefrom posthog.hogql_queries.utils.breakdowns import has_breakdown_filter+from posthog.hogql_queries.validation.rules import expanded_series_countfrom posthog.interval_specs import interval_specfrom posthog.models import Teamfrom posthog.schema_migrations.upgrade_manager import upgrade_insight@@ -106,6 +107,8 @@ def trends_query_for(insight: Insight, profile: ScoringProfile) -> TrendsQuery:if query.get("kind") != "TrendsQuery":raise UnsupportedInsightError(f"Only trends insights are scored, not {query.get('kind')}")trends = TrendsQuery.model_validate(query)+ if not trends.series:+ raise UnsupportedInsightError("Insight has no series")if is_non_time_series_trend(trends):raise UnsupportedInsightError("Insight shows a single value, not a time series")if trends.trendsFilter and trends.trendsFilter.display == ChartDisplayType.ACTIONS_LINE_GRAPH_CUMULATIVE:@@ -218,6 +221,23 @@ def scoring_query_settings(queries: int) -> HogQLGlobalSettings:)+def query_cost(insight: Insight, team: Team) -> int:+ """The ClickHouse queries that scoring this insight sends, so the tick can budget them before it runs.++ It counts the scoring query the same way as `TrendsQueryRunner`: one query per series, and one per+ cohort for each series of a cohort breakdown.+ """+ try:+ query = build_scoring_query(trends_query_for(insight, resolve_profile(team, insight)), buckets=1)+ except Exception:+ # Scoring builds the same query, so it fails before it sends a query and reports the error itself.+ # One query is the safe charge, and one bad insight cannot stop the tick for every project.+ return 1+ # A team can resolve a cohort breakdown in one query per series. The larger count is the safe charge.+ cohort_breakdown = query.breakdownFilter is not None and query.breakdownFilter.breakdown_type == "cohort"+ return max(1, expanded_series_count(query, cohort_breakdown_expands=cohort_breakdown))++def score_series(values: Sequence[float],buckets: Sequence[datetime],diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.pyindex 312f9706c904..81ee7638aefd 100644--- a/products/signals/backend/anomaly_scores/tick.py+++ b/products/signals/backend/anomaly_scores/tick.py@@ -20,6 +20,7 @@ from posthog.hogql.errors import QueryError, TableAccessDeniedErrorfrom posthog.dataclasses import frozenfrom posthog.exceptions import (+ ClickHouseAtCapacity,ClickHouseClusterMemoryLimitExceeded,ClickHouseEstimatedQueryExecutionTimeTooLong,ClickHouseQueryMemoryLimitExceeded,@@ -71,6 +72,8 @@ STATUS_TOO_EXPENSIVE = "too_expensive"STATUS_ACCESS_DENIED = "access_denied"# The saved query is invalid, for example it reads a warehouse table that no longer exists.STATUS_BROKEN_QUERY = "broken_query"+# ClickHouse rejected the query because it is full. The tick starts no more picks.+STATUS_AT_CAPACITY = "at_capacity"# Any other failure. The activity raises it, and the workflow counts it as `errored`.STATUS_ERRORED = "errored"@@ -99,6 +102,7 @@ class AnomalyScoringTickPlan:candidates: intmax_concurrency: intlimits: ScoringLimits = DEFAULT_LIMITS+ skipped_for_budget: int = 0@frozen@@ -125,6 +129,10 @@ class AnomalyScoringTickResult:duration_ms: int = 0# Picks left for a later tick because the tick budget ran out.deferred: int = 0+ # The query cost of the picks that started.+ queries: int = 0+ skipped_for_budget: int = 0+ stopped_at_capacity: bool = Falsedef plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:@@ -141,13 +149,16 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlcandidates=selection.candidates,max_concurrency=config.max_concurrency,limits=config.limits,+ skipped_for_budget=selection.skipped_for_budget,)def failure_status(error: Exception) -> str | None:- """The status of a failure that the next tick does not fix, or None for any other failure."""+ """The status of a known failure, or None for any other failure."""if isinstance(error, UnsupportedInsightError):return STATUS_UNSUPPORTED+ if isinstance(error, ClickHouseAtCapacity):+ return STATUS_AT_CAPACITY# The cluster limit is a subclass of the query limit, but it is load from other queries, so a later tick can pass.if isinstance(error, ClickHouseClusterMemoryLimitExceeded):return None@@ -189,7 +200,9 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsighrecord_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0)if status is None:raise- mark_attempted(pick.insight_id, SKIP_COOLDOWN)+ # A full cluster is not a fault of the insight, so it keeps the attempt cooldown.+ if status != STATUS_AT_CAPACITY:+ mark_attempted(pick.insight_id, SKIP_COOLDOWN)return ScoreInsightResult(status=status, events=0)duration_ms = round((time.monotonic() - started) * 1000)logger.info(@@ -248,6 +261,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:events=result.events,duration_ms=result.duration_ms,deferred=result.deferred,+ queries=result.queries,+ skipped_for_budget=result.skipped_for_budget,+ stopped_at_capacity=result.stopped_at_capacity,)ph_background_capture()(distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,@@ -261,6 +277,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:"events": result.events,"duration_ms": result.duration_ms,"deferred": result.deferred,+ "queries": result.queries,+ "skipped_for_budget": result.skipped_for_budget,+ "stopped_at_capacity": result.stopped_at_capacity,},)@@ -305,11 +324,14 @@ class AnomalyScoringTickWorkflow:errored = 0events = 0deferred = 0+ started_picks: list[InsightPick] = []concurrency = max(1, plan.max_concurrency)for index in range(0, len(plan.picks), concurrency):if workflow.now() - started >= TICK_BUDGET:deferred = len(plan.picks) - indexbreak+ batch = plan.picks[index : index + concurrency]+ started_picks.extend(batch)results = await asyncio.gather(*(workflow.execute_activity(@@ -320,7 +342,7 @@ class AnomalyScoringTickWorkflow:# A failed insight waits for its cooldown and a later tick, not a retry in this one.retry_policy=RetryPolicy(maximum_attempts=1),)- for pick in plan.picks[index : index + concurrency]+ for pick in batch),return_exceptions=True,)@@ -330,16 +352,22 @@ class AnomalyScoringTickWorkflow:continuestatuses[outcome.status] += 1events += outcome.events+ # More queries add load to a full cluster. The picks in flight finish, and no new pick starts.+ if statuses[STATUS_AT_CAPACITY]:+ breakresult = AnomalyScoringTickResult(enabled=True,candidates=plan.candidates,- picked=len(plan.picks),+ picked=len(started_picks),statuses=dict(statuses),errored=errored,events=events,duration_ms=round((workflow.now() - started).total_seconds() * 1000),deferred=deferred,+ queries=sum(pick.query_cost for pick in started_picks),+ skipped_for_budget=plan.skipped_for_budget,+ stopped_at_capacity=statuses[STATUS_AT_CAPACITY] > 0,)await workflow.execute_activity(capture_anomaly_scoring_tick_activity,diff --git a/products/signals/backend/test/test_anomaly_scores.py b/products/signals/backend/test/test_anomaly_scores.pyindex b915b1cf6637..8a12923a13a6 100644--- a/products/signals/backend/test/test_anomaly_scores.py+++ b/products/signals/backend/test/test_anomaly_scores.py@@ -31,6 +31,7 @@ from products.signals.backend.anomaly_scores.scoring import (UnsupportedInsightError,buckets_to_score,fetch_buckets,+ query_cost,score_insight,score_series,trends_query_for,@@ -287,3 +288,53 @@ class TestTrendsQueryFor(SimpleTestCase):def test_unsupported_shapes_raise(self, _name: str, query: dict[str, Any]) -> None:with self.assertRaises(UnsupportedInsightError):trends_query_for(Insight(id=1, query=query), DEFAULT_PROFILE)+++def _events(count: int) -> list[dict[str, Any]]:+ return [{"kind": "EventsNode", "event": f"event_{i}"} for i in range(count)]+++class TestQueryCost(SimpleTestCase):+ @parameterized.expand(+ [+ ("one_per_series", {"kind": "TrendsQuery", "series": _events(3)}, 3),+ ("wrapped", {"kind": "InsightVizNode", "source": {"kind": "TrendsQuery", "series": _events(2)}}, 2),+ (+ "breakdown_and_formula_add_none",+ {+ "kind": "TrendsQuery",+ "series": _events(2),+ "breakdownFilter": {"breakdown": "$browser", "breakdown_type": "event"},+ "trendsFilter": {"formulas": ["A/B", "A+B"]},+ },+ 2,+ ),+ # The scoring query keeps every series, so the charge has no series cap.+ ("more_series_than_max_series", {"kind": "TrendsQuery", "series": _events(30)}, 30),+ (+ "one_per_cohort",+ {+ "kind": "TrendsQuery",+ "series": _events(2),+ "breakdownFilter": {"breakdown": ["all", 1, 2], "breakdown_type": "cohort"},+ },+ 6,+ ),+ # The scoring query drops the previous period.+ ("compare_adds_none", {"kind": "TrendsQuery", "series": _events(2), "compareFilter": {"compare": True}}, 2),+ ("not_scored", {"kind": "FunnelsQuery", "series": _events(4)}, 1),+ (+ "empty_series_with_breakdown",+ {+ "kind": "TrendsQuery",+ "series": [],+ "breakdownFilter": {"breakdown": "$browser", "breakdown_type": "event"},+ },+ 1,+ ),+ ]+ )+ def test_a_pick_costs_the_queries_the_scoring_runner_sends(+ self, _name: str, query: dict[str, Any], expected: int+ ) -> None:+ assert query_cost(Insight(id=1, query=query), MagicMock(spec=Team)) == expecteddiff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.pyindex 22317a029d3e..f7697e168203 100644--- a/products/signals/backend/test/test_anomaly_scoring_tick.py+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py@@ -1,7 +1,9 @@+import uuidfrom datetime import UTC, datetime, timedeltafrom typing import Anyfrom zoneinfo import ZoneInfo+import pytestimport time_machinefrom posthog.test.base import BaseTest, ClickhouseTestMixin, _create_eventfrom unittest.mock import MagicMock, patch@@ -10,12 +12,15 @@ from django.core.cache import cachefrom django.test import SimpleTestCasefrom parameterized import parameterized+from temporalio import activity+from temporalio.testing import WorkflowEnvironment+from temporalio.worker import UnsandboxedWorkflowRunner, Workerfrom posthog.schema import IntervalTypefrom posthog.hogql.errors import QueryError, ResolutionError, TableAccessDeniedError-from posthog.exceptions import ClickHouseClusterMemoryLimitExceeded, ClickHouseQueryTimeOut+from posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimitExceeded, ClickHouseQueryTimeOutfrom posthog.models import Organization, Team, Userfrom products.dashboards.backend.models.dashboard import Dashboard@@ -27,13 +32,19 @@ from products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID,from products.signals.backend.anomaly_scores.queue import (SKIP_COOLDOWN,InsightPick,+ QueryBudget,WatchedInsight,_cooldown_key,+ _fleet_budget_key,+ _project_budget_key,eligible_teams,is_due,+ remaining_budget,+ reserve_queries,scored_buckets,select_insights,watched_insights,+ within_budget,)from products.signals.backend.anomaly_scores.scoring import (InsightScores,@@ -42,7 +53,16 @@ from products.signals.backend.anomaly_scores.scoring import (UnsupportedInsightError,)from products.signals.backend.anomaly_scores.selection import SelectionConfig, Weighting-from products.signals.backend.anomaly_scores.tick import ScoreInsightInput, plan_tick, run_insight+from products.signals.backend.anomaly_scores.tick import (+ AnomalyScoringTickInput,+ AnomalyScoringTickPlan,+ AnomalyScoringTickResult,+ AnomalyScoringTickWorkflow,+ ScoreInsightInput,+ ScoreInsightResult,+ plan_tick,+ run_insight,+)NEW_YORK = ZoneInfo("America/New_York")ORGANIZATION_ID = "4dc8564d-bd82-1065-2f40-97f7c50f67cf"@@ -110,6 +130,106 @@ class TestTickConfig(SimpleTestCase):assert config.max_insights_per_tick == DISABLED.max_insights_per_tickassert (config.limits.max_backfill_buckets, config.limits.max_series) == (90, 20)+ @parameterized.expand(+ [+ ("max_insights_per_tick", 51, 50),+ ("max_concurrency", 100, 8),+ ("max_queries_per_hour", 1_000_000, 2000),+ ("max_queries_per_project_per_hour", 501, 500),+ ]+ )+ def test_tick_levers_above_their_maximum_parse_to_the_maximum(self, field: str, value: int, expected: int) -> None:+ config = parse_config({"enabled": True, "organization_ids": [ORGANIZATION_ID], field: value})++ assert getattr(config, field) == expected+++class TestQueryBudget(SimpleTestCase):+ @parameterized.expand(+ [+ # The second insight does not fit the 2 queries left, and the cheaper third one does.+ (+ "project_budget_skips_to_cheaper",+ {1: 1, 2: 1, 3: 1},+ QueryBudget(fleet=100, projects={1: 5}),+ 10,+ [1, 3],+ 1,+ 0,+ ),+ (+ "fleet_budget_spans_projects",+ {1: 1, 2: 2, 3: 3},+ QueryBudget(fleet=4, projects={1: 30, 2: 30, 3: 30}),+ 10,+ [1],+ 2,+ 0,+ ),+ ("project_spent", {1: 1, 2: 2, 3: 2}, QueryBudget(fleet=100, projects={1: 30, 2: 0}), 10, [1], 2, 0),+ ("tick_cap", {1: 1, 2: 1, 3: 1}, QueryBudget(fleet=100, projects={1: 30}), 1, [1], 0, 0),+ # Another tick reserved the queries after the snapshot.+ ("reservation_fails", {1: 1, 2: 1, 3: 1}, QueryBudget(fleet=100, projects={1: 30}), 10, [2, 3], 1, 1),+ ]+ )+ def test_the_walk_skips_picks_over_budget_and_continues(+ self,+ _name: str,+ team_ids: dict[int, int],+ budget: QueryBudget,+ limit: int,+ expected: list[int],+ skipped: int,+ refused_reservations: int,+ ) -> None:+ costs = {1: 3, 2: 3, 3: 2}+ calls: list[int] = []++ def reserve(team_id: int, cost: int) -> bool:+ calls.append(cost)+ return len(calls) > refused_reservations++ picked, skipped_count = within_budget(+ [1, 2, 3], team_ids, budget, limit, lambda ids: {i: costs[i] for i in ids}, reserve+ )++ assert (list(picked), skipped_count) == (expected, skipped)+ assert picked == {insight_id: costs[insight_id] for insight_id in expected}++ @parameterized.expand(+ [+ ("fleet_spent", QueryBudget(fleet=0, projects={1: 30, 2: 30}), []),+ ("one_project_spent", QueryBudget(fleet=100, projects={1: 0, 2: 30}), [[2, 4]]),+ ]+ )+ def test_costs_load_only_for_insights_a_budget_can_still_fit(+ self, _name: str, budget: QueryBudget, loaded: list[list[int]]+ ) -> None:+ batches: list[list[int]] = []++ def costs_for(insight_ids: list[int]) -> dict[int, int]:+ batches.append(insight_ids)+ return dict.fromkeys(insight_ids, 1)++ within_budget([1, 2, 3, 4], {1: 1, 2: 2, 3: 1, 4: 2}, budget, 10, costs_for, lambda _team, _cost: True)++ assert batches == loaded++ def test_a_reservation_takes_both_budgets_or_neither_and_resets_each_hour(self) -> None:+ config = AnomalyScoringConfig(enabled=True, max_queries_per_hour=10, max_queries_per_project_per_hour=6)+ next_hour = NOW + timedelta(hours=1)+ for now in (NOW, next_hour):+ cache.delete_many([_fleet_budget_key(now), _project_budget_key(1, now), _project_budget_key(2, now)])++ reserved = [+ reserve_queries(config, team_id, queries, NOW + timedelta(minutes=minute))+ for team_id, queries, minute in [(1, 3, 0), (1, 3, 10), (1, 1, 20), (2, 4, 30), (2, 1, 40)]+ ]++ assert reserved == [True, True, False, True, False]+ assert remaining_budget(config, [1, 2], NOW) == QueryBudget(fleet=0, projects={1: 0, 2: 2})+ assert remaining_budget(config, [1, 2], next_hour) == QueryBudget(fleet=10, projects={1: 6, 2: 6})+class TestTickSelection(SimpleTestCase):@parameterized.expand(@@ -276,6 +396,12 @@ class TestRunInsight(SimpleTestCase):"anomaly_scoring.picks", 1, attributes={"status": expected, "interval": IntervalType.DAY.value})+ def test_a_full_cluster_returns_at_capacity_and_waits_an_hour(self) -> None:+ status, cooldowns, _metrics = self._run(ClickHouseAtCapacity())++ assert status == "at_capacity"+ assert SKIP_COOLDOWN not in cooldowns+@parameterized.expand([("cluster_memory", ClickHouseClusterMemoryLimitExceeded()),@@ -399,6 +525,7 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):for insight in insights.values():DashboardTile.objects.create(dashboard=dashboard, insight=insight)cache.delete(_cooldown_key(insight.id))+ cache.delete_many([_fleet_budget_key(NOW), _project_budget_key(self.team.id, NOW)])today = NOW.replace(hour=0)for name, bucket in (("stale", today - timedelta(days=3)), ("fresh", today - timedelta(days=1))):_create_event(@@ -453,3 +580,69 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):assert selection.picks[0].insight_id in {insight.id for insight in large_insights}assert selection.picks[1].insight_id == small_insight.idassert selection.candidates == 4+++async def _run_tick(+ plan: AnomalyScoringTickPlan, statuses: dict[int, str]+) -> tuple[AnomalyScoringTickResult, list[int]]:+ started: list[int] = []++ @activity.defn(name="plan_anomaly_scoring_tick_activity")+ async def fake_plan(input: AnomalyScoringTickInput) -> AnomalyScoringTickPlan:+ return plan++ @activity.defn(name="score_anomaly_insight_activity")+ async def fake_score(input: ScoreInsightInput) -> ScoreInsightResult:+ started.append(input.pick.insight_id)+ return ScoreInsightResult(status=statuses.get(input.pick.insight_id, "scored"), events=1)++ @activity.defn(name="capture_anomaly_scoring_tick_activity")+ async def fake_capture(result: AnomalyScoringTickResult) -> None:+ pass++ async with await WorkflowEnvironment.start_time_skipping() as env:+ async with Worker(+ env.client,+ task_queue="anomaly-scoring-tick-test",+ workflows=[AnomalyScoringTickWorkflow],+ activities=[fake_plan, fake_score, fake_capture],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ result = await env.client.execute_workflow(+ AnomalyScoringTickWorkflow.run,+ AnomalyScoringTickInput(),+ id=f"anomaly-scoring-tick-{uuid.uuid4()}",+ task_queue="anomaly-scoring-tick-test",+ )+ return result, started+++ "statuses,started,queries,stopped",+ [+ ({}, [1, 2, 3, 4, 5], 15, False),+ # The second pick of the first wave finishes, and the tick starts no new pick.+ ({1: "at_capacity"}, [1, 2], 6, True),+ ],+)+async def test_the_tick_stops_starting_picks_when_clickhouse_is_at_capacity(+ statuses: dict[int, str], started: list[int], queries: int, stopped: bool+) -> None:+ plan = AnomalyScoringTickPlan(+ enabled=True,+ picks=[InsightPick(team_id=1, insight_id=i, scored_through={}, query_cost=3) for i in range(1, 6)],+ candidates=8,+ max_concurrency=2,+ skipped_for_budget=3,+ )++ result, ran = await _run_tick(plan, statuses)++ assert sorted(ran) == started+ assert (result.picked, result.queries, result.skipped_for_budget, result.stopped_at_capacity) == (+ len(started),+ queries,+ 3,+ stopped,+ )