Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): bound anomaly scoring with hourly query budgets (#122429)

posthog[bot]1500e3f68fb0
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.py
index 0e97e4690af6..5d4631dee843 100644
--- a/products/signals/backend/anomaly_scores/config.py
+++ b/products/signals/backend/anomaly_scores/config.py
@@ -93,13 +93,21 @@ class AnomalyScoringConfig:
allow_team_ids: tuple[int, ...] = ()
skip_team_ids: tuple[int, ...] = ()
intervals: tuple[IntervalType, ...] = DEFAULT_TICK_INTERVALS
- # The budget is scoring queries, not events. One insight is one query.
+ # Smooths the spend over the hour, so one tick cannot use the whole query budget.
max_insights_per_tick: int = 8
max_concurrency: int = 2
+ # Scoring queries per clock hour. One pick sends one query per series, see `scoring.query_cost`.
+ max_queries_per_hour: int = 120
+ max_queries_per_project_per_hour: int = 30
limits: ScoringLimits = DEFAULT_LIMITS
DISABLED = AnomalyScoringConfig()
+# The largest value a flag payload can set for each lever of the tick.
+MAX_INSIGHTS_PER_TICK = 50
+MAX_CONCURRENCY = 8
+MAX_QUERIES_PER_HOUR = 2000
+MAX_QUERIES_PER_PROJECT_PER_HOUR = 500
def _positive_int(value: Any, default: int) -> int:
@@ -161,8 +169,17 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:
allow_team_ids=_team_ids(payload.get("allow_team_ids")),
skip_team_ids=_team_ids(payload.get("skip_team_ids")),
intervals=_intervals(payload.get("intervals")),
- max_insights_per_tick=_positive_int(payload.get("max_insights_per_tick"), defaults.max_insights_per_tick),
- max_concurrency=_positive_int(payload.get("max_concurrency"), defaults.max_concurrency),
+ max_insights_per_tick=min(
+ _positive_int(payload.get("max_insights_per_tick"), defaults.max_insights_per_tick), MAX_INSIGHTS_PER_TICK
+ ),
+ max_concurrency=min(_positive_int(payload.get("max_concurrency"), defaults.max_concurrency), MAX_CONCURRENCY),
+ max_queries_per_hour=min(
+ _positive_int(payload.get("max_queries_per_hour"), defaults.max_queries_per_hour), MAX_QUERIES_PER_HOUR
+ ),
+ max_queries_per_project_per_hour=min(
+ _positive_int(payload.get("max_queries_per_project_per_hour"), defaults.max_queries_per_project_per_hour),
+ MAX_QUERIES_PER_PROJECT_PER_HOUR,
+ ),
limits=parse_limits(payload.get("limits")),
)
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index 782fdf0d1f5a..468150a5e115 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -1,12 +1,13 @@
"""Pick the watched insights that the next anomaly scoring tick scores.
There is no stored state. One query per team reads the newest scored bucket of each series back
-from the `$anomaly_score` events. `selection.pick_order` then orders the due insights.
+from the `$anomaly_score` events. `selection.pick_order` then orders the due insights, and the tick
+picks from that order the insights whose queries fit the hourly budgets.
"""
import datetime
import itertools
-from collections.abc import Collection
+from collections.abc import Callable, Collection
from typing import Any, cast
from django.core.cache import cache
@@ -25,9 +26,12 @@ from posthog.models import Team
from posthog.tasks.alerts.utils import NON_TIME_SERIES_DISPLAY_TYPES
from products.alerts_platform.backend.facade.scheduling import CalendarInterval, alert_check_offset
-from products.product_analytics.backend.facade.api import watched_trends_insights
+from products.product_analytics.backend.facade.api import (
+ insights_including_soft_deleted_for_team,
+ watched_trends_insights,
+)
from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig
-from products.signals.backend.anomaly_scores.scoring import bucket_end
+from products.signals.backend.anomaly_scores.scoring import bucket_end, query_cost
from products.signals.backend.anomaly_scores.selection import (
Candidate,
InsightUsage,
@@ -49,6 +53,9 @@ _MAX_SCORED_SERIES = 50_000
ATTEMPT_COOLDOWN = datetime.timedelta(hours=1)
# For a failure that the next hour does not fix: an unsupported shape, a query that is too expensive, or no access.
SKIP_COOLDOWN = datetime.timedelta(days=1)
+# A query counter lives a little longer than its clock hour, so a tick at the end of the hour still reads it.
+# A lost counter lets the scorer spend at most one more hour of budget.
+_BUDGET_TTL = datetime.timedelta(minutes=70)
_CALENDAR_INTERVALS = {
IntervalType.HOUR: CalendarInterval.HOURLY,
@@ -91,13 +98,26 @@ class InsightPick:
# The newest scored bucket start of each series, in UTC. Empty when the insight was never scored.
scored_through: dict[str, datetime.datetime]
interval: IntervalType | None = None
+ # The ClickHouse queries this pick sends. Selection charges them to the hourly budgets, so two ticks
+ # that overlap cannot spend the same budget.
+ query_cost: int = 1
@frozen
class Selection:
picks: list[InsightPick]
- # Due insights, picked or not. More than the picks means the tick cap cut the queue.
+ # Due insights, picked or not. More than the picks means the tick cap or a budget cut the queue.
candidates: int
+ # Due insights that the walk reached but skipped, because their queries did not fit a budget.
+ skipped_for_budget: int = 0
+
+
+@frozen
+class QueryBudget:
+ """Scoring queries left in this clock hour, for the fleet and for each project."""
+
+ fleet: int
+ projects: dict[int, int]
def eligible_teams(config: AnomalyScoringConfig) -> list[Team]:
@@ -199,6 +219,97 @@ def _cooling_down(insight_ids: list[int]) -> set[int]:
return {insight_id for insight_id in insight_ids if _cooldown_key(insight_id) in found}
+def _hour(now: datetime.datetime) -> str:
+ return _utc(now).strftime("%Y%m%d%H")
+
+
+def _fleet_budget_key(now: datetime.datetime) -> str:
+ return f"signals:anomaly_scores:queries:{_hour(now)}"
+
+
+def _project_budget_key(team_id: int, now: datetime.datetime) -> str:
+ return f"signals:anomaly_scores:queries:{_hour(now)}:team:{team_id}"
+
+
+def remaining_budget(config: AnomalyScoringConfig, team_ids: Collection[int], now: datetime.datetime) -> QueryBudget:
+ project_keys = {team_id: _project_budget_key(team_id, now) for team_id in team_ids}
+ spent = cache.get_many([_fleet_budget_key(now), *project_keys.values()])
+ return QueryBudget(
+ fleet=config.max_queries_per_hour - spent.get(_fleet_budget_key(now), 0),
+ projects={
+ team_id: config.max_queries_per_project_per_hour - spent.get(key, 0)
+ for team_id, key in project_keys.items()
+ },
+ )
+
+
+def _reserve(key: str, queries: int, cap: int) -> bool:
+ # `incr` is atomic, so two planners that run at the same time cannot both take the last queries.
+ # `incr` fails on a missing key, and `add` does nothing when the key exists.
+ cache.add(key, 0, timeout=int(_BUDGET_TTL.total_seconds()))
+ if cache.incr(key, queries) <= cap:
+ return True
+ cache.decr(key, queries)
+ return False
+
+
+def reserve_queries(config: AnomalyScoringConfig, team_id: int, queries: int, now: datetime.datetime) -> bool:
+ """Charge ``queries`` to both hourly budgets, or to neither when one of them has too few left."""
+ fleet_key = _fleet_budget_key(now)
+ if not _reserve(fleet_key, queries, config.max_queries_per_hour):
+ return False
+ if _reserve(_project_budget_key(team_id, now), queries, config.max_queries_per_project_per_hour):
+ return True
+ cache.decr(fleet_key, queries)
+ return False
+
+
+def within_budget(
+ order: list[int],
+ team_ids: dict[int, int],
+ budget: QueryBudget,
+ limit: int,
+ costs_for: Callable[[list[int]], dict[int, int]],
+ reserve: Callable[[int, int], bool],
+) -> tuple[dict[int, int], int]:
+ """The insights of ``order`` whose cost fits both budgets, up to ``limit``, with their costs, and the skip count.
+
+ A pick that does not fit is skipped and the walk continues, so a large insight does not block smaller ones.
+ ``budget`` is a snapshot that skips most misses without a cache call. ``reserve`` charges a pick for real.
+ ``costs_for`` loads costs in batches of ``limit``, and only for insights that the snapshot can still fit.
+ """
+ fleet = budget.fleet
+ projects = dict(budget.projects)
+ picked: dict[int, int] = {}
+ skipped = 0
+ batch_size = max(1, limit)
+ for start in range(0, len(order), batch_size):
+ if len(picked) >= limit:
+ break
+ batch = order[start : start + batch_size]
+ affordable = [insight_id for insight_id in batch if fleet > 0 and projects.get(team_ids[insight_id], 0) > 0]
+ costs = costs_for(affordable) if affordable else {}
+ for insight_id in batch:
+ if len(picked) >= limit:
+ break
+ team_id = team_ids[insight_id]
+ cost = costs.get(insight_id)
+ if cost is None or cost > fleet or cost > projects.get(team_id, 0) or not reserve(team_id, cost):
+ skipped += 1
+ continue
+ picked[insight_id] = cost
+ fleet -= cost
+ projects[team_id] -= cost
+ return picked, skipped
+
+
+def _query_costs(team: Team, insight_ids: list[int]) -> dict[int, int]:
+ insights = insights_including_soft_deleted_for_team(team_id=team.id, insight_ids=insight_ids)
+ costs = {insight.id: query_cost(insight, team) for insight in insights}
+ # An insight that no longer exists sends no query, but one query is the safe charge.
+ return {insight_id: costs.get(insight_id, 1) for insight_id in insight_ids}
+
+
def select_insights(
config: AnomalyScoringConfig, selection_config: SelectionConfig, now: datetime.datetime, limit: int
) -> Selection:
@@ -253,13 +364,32 @@ def select_insights(
order = [
insight_id for turn in itertools.zip_longest(*team_orders) for insight_id in turn if insight_id is not None
]
+ team_ids = {insight_id: due_by_id[insight_id].team_id for insight_id in order}
+
+ def costs_for(insight_ids: list[int]) -> dict[int, int]:
+ costs: dict[int, int] = {}
+ for team_id in sorted({team_ids[insight_id] for insight_id in insight_ids}):
+ team_batch = [insight_id for insight_id in insight_ids if team_ids[insight_id] == team_id]
+ costs.update(_query_costs(teams[team_id], team_batch))
+ return costs
+
+ # Read the budget before any cost, so a spent hour loads no insight.
+ picked, skipped = within_budget(
+ order,
+ team_ids,
+ remaining_budget(config, set(team_ids.values()), now),
+ limit,
+ costs_for,
+ reserve=lambda team_id, cost: reserve_queries(config, team_id, cost, now),
+ )
picks = [
InsightPick(
- team_id=due_by_id[insight_id].team_id,
+ team_id=team_ids[insight_id],
insight_id=insight_id,
scored_through=state.get(insight_id, {}),
interval=due_by_id[insight_id].interval,
+ query_cost=cost,
)
- for insight_id in order[:limit]
+ for insight_id, cost in picked.items()
]
- return Selection(picks=picks, candidates=len(order))
+ return Selection(picks=picks, candidates=len(order), skipped_for_budget=skipped)
diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.py
index 489fe36d4c47..25f9badba1d1 100644
--- a/products/signals/backend/anomaly_scores/scoring.py
+++ b/products/signals/backend/anomaly_scores/scoring.py
@@ -20,6 +20,7 @@ from posthog.clickhouse.workload import Workload
from posthog.dataclasses import frozen
from posthog.event_usage import EventSource
from posthog.hogql_queries.utils.breakdowns import has_breakdown_filter
+from posthog.hogql_queries.validation.rules import expanded_series_count
from posthog.interval_specs import interval_spec
from posthog.models import Team
from posthog.schema_migrations.upgrade_manager import upgrade_insight
@@ -106,6 +107,8 @@ def trends_query_for(insight: Insight, profile: ScoringProfile) -> TrendsQuery:
if query.get("kind") != "TrendsQuery":
raise UnsupportedInsightError(f"Only trends insights are scored, not {query.get('kind')}")
trends = TrendsQuery.model_validate(query)
+ if not trends.series:
+ raise UnsupportedInsightError("Insight has no series")
if is_non_time_series_trend(trends):
raise UnsupportedInsightError("Insight shows a single value, not a time series")
if trends.trendsFilter and trends.trendsFilter.display == ChartDisplayType.ACTIONS_LINE_GRAPH_CUMULATIVE:
@@ -218,6 +221,23 @@ def scoring_query_settings(queries: int) -> HogQLGlobalSettings:
)
+def query_cost(insight: Insight, team: Team) -> int:
+ """The ClickHouse queries that scoring this insight sends, so the tick can budget them before it runs.
+
+ It counts the scoring query the same way as `TrendsQueryRunner`: one query per series, and one per
+ cohort for each series of a cohort breakdown.
+ """
+ try:
+ query = build_scoring_query(trends_query_for(insight, resolve_profile(team, insight)), buckets=1)
+ except Exception:
+ # Scoring builds the same query, so it fails before it sends a query and reports the error itself.
+ # One query is the safe charge, and one bad insight cannot stop the tick for every project.
+ return 1
+ # A team can resolve a cohort breakdown in one query per series. The larger count is the safe charge.
+ cohort_breakdown = query.breakdownFilter is not None and query.breakdownFilter.breakdown_type == "cohort"
+ return max(1, expanded_series_count(query, cohort_breakdown_expands=cohort_breakdown))
+
+
def score_series(
values: Sequence[float],
buckets: Sequence[datetime],
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index 312f9706c904..81ee7638aefd 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -20,6 +20,7 @@ from posthog.hogql.errors import QueryError, TableAccessDeniedError
from posthog.dataclasses import frozen
from posthog.exceptions import (
+ ClickHouseAtCapacity,
ClickHouseClusterMemoryLimitExceeded,
ClickHouseEstimatedQueryExecutionTimeTooLong,
ClickHouseQueryMemoryLimitExceeded,
@@ -71,6 +72,8 @@ STATUS_TOO_EXPENSIVE = "too_expensive"
STATUS_ACCESS_DENIED = "access_denied"
# The saved query is invalid, for example it reads a warehouse table that no longer exists.
STATUS_BROKEN_QUERY = "broken_query"
+# ClickHouse rejected the query because it is full. The tick starts no more picks.
+STATUS_AT_CAPACITY = "at_capacity"
# Any other failure. The activity raises it, and the workflow counts it as `errored`.
STATUS_ERRORED = "errored"
@@ -99,6 +102,7 @@ class AnomalyScoringTickPlan:
candidates: int
max_concurrency: int
limits: ScoringLimits = DEFAULT_LIMITS
+ skipped_for_budget: int = 0
@frozen
@@ -125,6 +129,10 @@ class AnomalyScoringTickResult:
duration_ms: int = 0
# Picks left for a later tick because the tick budget ran out.
deferred: int = 0
+ # The query cost of the picks that started.
+ queries: int = 0
+ skipped_for_budget: int = 0
+ stopped_at_capacity: bool = False
def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:
@@ -141,13 +149,16 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
candidates=selection.candidates,
max_concurrency=config.max_concurrency,
limits=config.limits,
+ skipped_for_budget=selection.skipped_for_budget,
)
def failure_status(error: Exception) -> str | None:
- """The status of a failure that the next tick does not fix, or None for any other failure."""
+ """The status of a known failure, or None for any other failure."""
if isinstance(error, UnsupportedInsightError):
return STATUS_UNSUPPORTED
+ if isinstance(error, ClickHouseAtCapacity):
+ return STATUS_AT_CAPACITY
# The cluster limit is a subclass of the query limit, but it is load from other queries, so a later tick can pass.
if isinstance(error, ClickHouseClusterMemoryLimitExceeded):
return None
@@ -189,7 +200,9 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0)
if status is None:
raise
- mark_attempted(pick.insight_id, SKIP_COOLDOWN)
+ # A full cluster is not a fault of the insight, so it keeps the attempt cooldown.
+ if status != STATUS_AT_CAPACITY:
+ mark_attempted(pick.insight_id, SKIP_COOLDOWN)
return ScoreInsightResult(status=status, events=0)
duration_ms = round((time.monotonic() - started) * 1000)
logger.info(
@@ -248,6 +261,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
events=result.events,
duration_ms=result.duration_ms,
deferred=result.deferred,
+ queries=result.queries,
+ skipped_for_budget=result.skipped_for_budget,
+ stopped_at_capacity=result.stopped_at_capacity,
)
ph_background_capture()(
distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,
@@ -261,6 +277,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
"events": result.events,
"duration_ms": result.duration_ms,
"deferred": result.deferred,
+ "queries": result.queries,
+ "skipped_for_budget": result.skipped_for_budget,
+ "stopped_at_capacity": result.stopped_at_capacity,
},
)
@@ -305,11 +324,14 @@ class AnomalyScoringTickWorkflow:
errored = 0
events = 0
deferred = 0
+ started_picks: list[InsightPick] = []
concurrency = max(1, plan.max_concurrency)
for index in range(0, len(plan.picks), concurrency):
if workflow.now() - started >= TICK_BUDGET:
deferred = len(plan.picks) - index
break
+ batch = plan.picks[index : index + concurrency]
+ started_picks.extend(batch)
results = await asyncio.gather(
*(
workflow.execute_activity(
@@ -320,7 +342,7 @@ class AnomalyScoringTickWorkflow:
# A failed insight waits for its cooldown and a later tick, not a retry in this one.
retry_policy=RetryPolicy(maximum_attempts=1),
)
- for pick in plan.picks[index : index + concurrency]
+ for pick in batch
),
return_exceptions=True,
)
@@ -330,16 +352,22 @@ class AnomalyScoringTickWorkflow:
continue
statuses[outcome.status] += 1
events += outcome.events
+ # More queries add load to a full cluster. The picks in flight finish, and no new pick starts.
+ if statuses[STATUS_AT_CAPACITY]:
+ break
result = AnomalyScoringTickResult(
enabled=True,
candidates=plan.candidates,
- picked=len(plan.picks),
+ picked=len(started_picks),
statuses=dict(statuses),
errored=errored,
events=events,
duration_ms=round((workflow.now() - started).total_seconds() * 1000),
deferred=deferred,
+ queries=sum(pick.query_cost for pick in started_picks),
+ skipped_for_budget=plan.skipped_for_budget,
+ stopped_at_capacity=statuses[STATUS_AT_CAPACITY] > 0,
)
await workflow.execute_activity(
capture_anomaly_scoring_tick_activity,
diff --git a/products/signals/backend/test/test_anomaly_scores.py b/products/signals/backend/test/test_anomaly_scores.py
index b915b1cf6637..8a12923a13a6 100644
--- a/products/signals/backend/test/test_anomaly_scores.py
+++ b/products/signals/backend/test/test_anomaly_scores.py
@@ -31,6 +31,7 @@ from products.signals.backend.anomaly_scores.scoring import (
UnsupportedInsightError,
buckets_to_score,
fetch_buckets,
+ query_cost,
score_insight,
score_series,
trends_query_for,
@@ -287,3 +288,53 @@ class TestTrendsQueryFor(SimpleTestCase):
def test_unsupported_shapes_raise(self, _name: str, query: dict[str, Any]) -> None:
with self.assertRaises(UnsupportedInsightError):
trends_query_for(Insight(id=1, query=query), DEFAULT_PROFILE)
+
+
+def _events(count: int) -> list[dict[str, Any]]:
+ return [{"kind": "EventsNode", "event": f"event_{i}"} for i in range(count)]
+
+
+class TestQueryCost(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("one_per_series", {"kind": "TrendsQuery", "series": _events(3)}, 3),
+ ("wrapped", {"kind": "InsightVizNode", "source": {"kind": "TrendsQuery", "series": _events(2)}}, 2),
+ (
+ "breakdown_and_formula_add_none",
+ {
+ "kind": "TrendsQuery",
+ "series": _events(2),
+ "breakdownFilter": {"breakdown": "$browser", "breakdown_type": "event"},
+ "trendsFilter": {"formulas": ["A/B", "A+B"]},
+ },
+ 2,
+ ),
+ # The scoring query keeps every series, so the charge has no series cap.
+ ("more_series_than_max_series", {"kind": "TrendsQuery", "series": _events(30)}, 30),
+ (
+ "one_per_cohort",
+ {
+ "kind": "TrendsQuery",
+ "series": _events(2),
+ "breakdownFilter": {"breakdown": ["all", 1, 2], "breakdown_type": "cohort"},
+ },
+ 6,
+ ),
+ # The scoring query drops the previous period.
+ ("compare_adds_none", {"kind": "TrendsQuery", "series": _events(2), "compareFilter": {"compare": True}}, 2),
+ ("not_scored", {"kind": "FunnelsQuery", "series": _events(4)}, 1),
+ (
+ "empty_series_with_breakdown",
+ {
+ "kind": "TrendsQuery",
+ "series": [],
+ "breakdownFilter": {"breakdown": "$browser", "breakdown_type": "event"},
+ },
+ 1,
+ ),
+ ]
+ )
+ def test_a_pick_costs_the_queries_the_scoring_runner_sends(
+ self, _name: str, query: dict[str, Any], expected: int
+ ) -> None:
+ assert query_cost(Insight(id=1, query=query), MagicMock(spec=Team)) == expected
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index 22317a029d3e..f7697e168203 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -1,7 +1,9 @@
+import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
from zoneinfo import ZoneInfo
+import pytest
import time_machine
from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event
from unittest.mock import MagicMock, patch
@@ -10,12 +12,15 @@ from django.core.cache import cache
from django.test import SimpleTestCase
from parameterized import parameterized
+from temporalio import activity
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import UnsandboxedWorkflowRunner, Worker
from posthog.schema import IntervalType
from posthog.hogql.errors import QueryError, ResolutionError, TableAccessDeniedError
-from posthog.exceptions import ClickHouseClusterMemoryLimitExceeded, ClickHouseQueryTimeOut
+from posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimitExceeded, ClickHouseQueryTimeOut
from posthog.models import Organization, Team, User
from products.dashboards.backend.models.dashboard import Dashboard
@@ -27,13 +32,19 @@ from products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID,
from products.signals.backend.anomaly_scores.queue import (
SKIP_COOLDOWN,
InsightPick,
+ QueryBudget,
WatchedInsight,
_cooldown_key,
+ _fleet_budget_key,
+ _project_budget_key,
eligible_teams,
is_due,
+ remaining_budget,
+ reserve_queries,
scored_buckets,
select_insights,
watched_insights,
+ within_budget,
)
from products.signals.backend.anomaly_scores.scoring import (
InsightScores,
@@ -42,7 +53,16 @@ from products.signals.backend.anomaly_scores.scoring import (
UnsupportedInsightError,
)
from products.signals.backend.anomaly_scores.selection import SelectionConfig, Weighting
-from products.signals.backend.anomaly_scores.tick import ScoreInsightInput, plan_tick, run_insight
+from products.signals.backend.anomaly_scores.tick import (
+ AnomalyScoringTickInput,
+ AnomalyScoringTickPlan,
+ AnomalyScoringTickResult,
+ AnomalyScoringTickWorkflow,
+ ScoreInsightInput,
+ ScoreInsightResult,
+ plan_tick,
+ run_insight,
+)
NEW_YORK = ZoneInfo("America/New_York")
ORGANIZATION_ID = "4dc8564d-bd82-1065-2f40-97f7c50f67cf"
@@ -110,6 +130,106 @@ class TestTickConfig(SimpleTestCase):
assert config.max_insights_per_tick == DISABLED.max_insights_per_tick
assert (config.limits.max_backfill_buckets, config.limits.max_series) == (90, 20)
+ @parameterized.expand(
+ [
+ ("max_insights_per_tick", 51, 50),
+ ("max_concurrency", 100, 8),
+ ("max_queries_per_hour", 1_000_000, 2000),
+ ("max_queries_per_project_per_hour", 501, 500),
+ ]
+ )
+ def test_tick_levers_above_their_maximum_parse_to_the_maximum(self, field: str, value: int, expected: int) -> None:
+ config = parse_config({"enabled": True, "organization_ids": [ORGANIZATION_ID], field: value})
+
+ assert getattr(config, field) == expected
+
+
+class TestQueryBudget(SimpleTestCase):
+ @parameterized.expand(
+ [
+ # The second insight does not fit the 2 queries left, and the cheaper third one does.
+ (
+ "project_budget_skips_to_cheaper",
+ {1: 1, 2: 1, 3: 1},
+ QueryBudget(fleet=100, projects={1: 5}),
+ 10,
+ [1, 3],
+ 1,
+ 0,
+ ),
+ (
+ "fleet_budget_spans_projects",
+ {1: 1, 2: 2, 3: 3},
+ QueryBudget(fleet=4, projects={1: 30, 2: 30, 3: 30}),
+ 10,
+ [1],
+ 2,
+ 0,
+ ),
+ ("project_spent", {1: 1, 2: 2, 3: 2}, QueryBudget(fleet=100, projects={1: 30, 2: 0}), 10, [1], 2, 0),
+ ("tick_cap", {1: 1, 2: 1, 3: 1}, QueryBudget(fleet=100, projects={1: 30}), 1, [1], 0, 0),
+ # Another tick reserved the queries after the snapshot.
+ ("reservation_fails", {1: 1, 2: 1, 3: 1}, QueryBudget(fleet=100, projects={1: 30}), 10, [2, 3], 1, 1),
+ ]
+ )
+ def test_the_walk_skips_picks_over_budget_and_continues(
+ self,
+ _name: str,
+ team_ids: dict[int, int],
+ budget: QueryBudget,
+ limit: int,
+ expected: list[int],
+ skipped: int,
+ refused_reservations: int,
+ ) -> None:
+ costs = {1: 3, 2: 3, 3: 2}
+ calls: list[int] = []
+
+ def reserve(team_id: int, cost: int) -> bool:
+ calls.append(cost)
+ return len(calls) > refused_reservations
+
+ picked, skipped_count = within_budget(
+ [1, 2, 3], team_ids, budget, limit, lambda ids: {i: costs[i] for i in ids}, reserve
+ )
+
+ assert (list(picked), skipped_count) == (expected, skipped)
+ assert picked == {insight_id: costs[insight_id] for insight_id in expected}
+
+ @parameterized.expand(
+ [
+ ("fleet_spent", QueryBudget(fleet=0, projects={1: 30, 2: 30}), []),
+ ("one_project_spent", QueryBudget(fleet=100, projects={1: 0, 2: 30}), [[2, 4]]),
+ ]
+ )
+ def test_costs_load_only_for_insights_a_budget_can_still_fit(
+ self, _name: str, budget: QueryBudget, loaded: list[list[int]]
+ ) -> None:
+ batches: list[list[int]] = []
+
+ def costs_for(insight_ids: list[int]) -> dict[int, int]:
+ batches.append(insight_ids)
+ return dict.fromkeys(insight_ids, 1)
+
+ within_budget([1, 2, 3, 4], {1: 1, 2: 2, 3: 1, 4: 2}, budget, 10, costs_for, lambda _team, _cost: True)
+
+ assert batches == loaded
+
+ def test_a_reservation_takes_both_budgets_or_neither_and_resets_each_hour(self) -> None:
+ config = AnomalyScoringConfig(enabled=True, max_queries_per_hour=10, max_queries_per_project_per_hour=6)
+ next_hour = NOW + timedelta(hours=1)
+ for now in (NOW, next_hour):
+ cache.delete_many([_fleet_budget_key(now), _project_budget_key(1, now), _project_budget_key(2, now)])
+
+ reserved = [
+ reserve_queries(config, team_id, queries, NOW + timedelta(minutes=minute))
+ for team_id, queries, minute in [(1, 3, 0), (1, 3, 10), (1, 1, 20), (2, 4, 30), (2, 1, 40)]
+ ]
+
+ assert reserved == [True, True, False, True, False]
+ assert remaining_budget(config, [1, 2], NOW) == QueryBudget(fleet=0, projects={1: 0, 2: 2})
+ assert remaining_budget(config, [1, 2], next_hour) == QueryBudget(fleet=10, projects={1: 6, 2: 6})
+
class TestTickSelection(SimpleTestCase):
@parameterized.expand(
@@ -276,6 +396,12 @@ class TestRunInsight(SimpleTestCase):
"anomaly_scoring.picks", 1, attributes={"status": expected, "interval": IntervalType.DAY.value}
)
+ def test_a_full_cluster_returns_at_capacity_and_waits_an_hour(self) -> None:
+ status, cooldowns, _metrics = self._run(ClickHouseAtCapacity())
+
+ assert status == "at_capacity"
+ assert SKIP_COOLDOWN not in cooldowns
+
@parameterized.expand(
[
("cluster_memory", ClickHouseClusterMemoryLimitExceeded()),
@@ -399,6 +525,7 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
for insight in insights.values():
DashboardTile.objects.create(dashboard=dashboard, insight=insight)
cache.delete(_cooldown_key(insight.id))
+ cache.delete_many([_fleet_budget_key(NOW), _project_budget_key(self.team.id, NOW)])
today = NOW.replace(hour=0)
for name, bucket in (("stale", today - timedelta(days=3)), ("fresh", today - timedelta(days=1))):
_create_event(
@@ -453,3 +580,69 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
assert selection.picks[0].insight_id in {insight.id for insight in large_insights}
assert selection.picks[1].insight_id == small_insight.id
assert selection.candidates == 4
+
+
+async def _run_tick(
+ plan: AnomalyScoringTickPlan, statuses: dict[int, str]
+) -> tuple[AnomalyScoringTickResult, list[int]]:
+ started: list[int] = []
+
+ @activity.defn(name="plan_anomaly_scoring_tick_activity")
+ async def fake_plan(input: AnomalyScoringTickInput) -> AnomalyScoringTickPlan:
+ return plan
+
+ @activity.defn(name="score_anomaly_insight_activity")
+ async def fake_score(input: ScoreInsightInput) -> ScoreInsightResult:
+ started.append(input.pick.insight_id)
+ return ScoreInsightResult(status=statuses.get(input.pick.insight_id, "scored"), events=1)
+
+ @activity.defn(name="capture_anomaly_scoring_tick_activity")
+ async def fake_capture(result: AnomalyScoringTickResult) -> None:
+ pass
+
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with Worker(
+ env.client,
+ task_queue="anomaly-scoring-tick-test",
+ workflows=[AnomalyScoringTickWorkflow],
+ activities=[fake_plan, fake_score, fake_capture],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ result = await env.client.execute_workflow(
+ AnomalyScoringTickWorkflow.run,
+ AnomalyScoringTickInput(),
+ id=f"anomaly-scoring-tick-{uuid.uuid4()}",
+ task_queue="anomaly-scoring-tick-test",
+ )
+ return result, started
+
+
+ "statuses,started,queries,stopped",
+ [
+ ({}, [1, 2, 3, 4, 5], 15, False),
+ # The second pick of the first wave finishes, and the tick starts no new pick.
+ ({1: "at_capacity"}, [1, 2], 6, True),
+ ],
+)
+async def test_the_tick_stops_starting_picks_when_clickhouse_is_at_capacity(
+ statuses: dict[int, str], started: list[int], queries: int, stopped: bool
+) -> None:
+ plan = AnomalyScoringTickPlan(
+ enabled=True,
+ picks=[InsightPick(team_id=1, insight_id=i, scored_through={}, query_cost=3) for i in range(1, 6)],
+ candidates=8,
+ max_concurrency=2,
+ skipped_for_budget=3,
+ )
+
+ result, ran = await _run_tick(plan, statuses)
+
+ assert sorted(ran) == started
+ assert (result.picked, result.queries, result.skipped_for_budget, result.stopped_at_capacity) == (
+ len(started),
+ queries,
+ 3,
+ stopped,
+ )