Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): fan out the anomaly scoring tick into parallel shards (#127048)

posthog[bot]f6ff4baea4ee
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.py
index 3d8b1b961de4..e5e06ca3e3ef 100644
--- a/products/signals/backend/anomaly_scores/config.py
+++ b/products/signals/backend/anomaly_scores/config.py
@@ -120,6 +120,14 @@ MAX_MODEL_AGE_BUCKETS_LIMIT = {
DEFAULT_TICK_INTERVALS = (IntervalType.DAY, IntervalType.WEEK, IntervalType.MONTH)
+@frozen
+class TickFanOut:
+ """How the tick runs its picks: this many shards side by side, each with this many picks in flight."""
+
+ shards: int
+ concurrency: int
+
+
@frozen
class AnomalyScoringConfig:
enabled: bool = False
@@ -134,9 +142,14 @@ class AnomalyScoringConfig:
skip_team_ids: tuple[int, ...] = ()
intervals: tuple[IntervalType, ...] = DEFAULT_TICK_INTERVALS
# Smooths the spend over the hour, so one tick cannot use the whole query budget. With the split on,
- # the two passes use the per-pass caps below instead.
+ # the two passes use the per-pass caps below instead. Each pick cap is per shard, see `fan_out`.
max_insights_per_tick: int = 8
+ # Picks in flight in each shard.
max_concurrency: int = 2
+ # Shards of the tick. Each shard runs its own picks, and all the picks of one team go to one shard.
+ shards: int = 1
+ # Picks in flight over all the shards. It lowers the shard count, so the cluster load has one upper limit.
+ max_total_concurrency: int = 8
# Scoring queries per clock hour. One pick sends one query per series, see `scoring.query_cost`.
max_queries_per_hour: int = 120
max_queries_per_project_per_hour: int = 30
@@ -166,11 +179,19 @@ class AnomalyScoringConfig:
max_cpu_seconds_per_project_per_day: int = 0
max_fleet_cpu_seconds_per_day: int = 0
+ def fan_out(self) -> TickFanOut:
+ concurrency = min(self.max_concurrency, self.max_total_concurrency)
+ return TickFanOut(
+ shards=max(1, min(self.shards, self.max_total_concurrency // concurrency)), concurrency=concurrency
+ )
+
DISABLED = AnomalyScoringConfig()
# The largest value a flag payload can set for each lever of the tick.
MAX_INSIGHTS_PER_TICK = 50
MAX_CONCURRENCY = 8
+MAX_SHARDS = 8
+MAX_TOTAL_CONCURRENCY = 32
MAX_QUERIES_PER_HOUR = 2000
MAX_QUERIES_PER_PROJECT_PER_HOUR = 500
MAX_SCORE_QUERIES_PER_HOUR = 5000
@@ -279,6 +300,10 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:
_positive_int(payload.get("max_insights_per_tick"), defaults.max_insights_per_tick), MAX_INSIGHTS_PER_TICK
),
max_concurrency=min(_positive_int(payload.get("max_concurrency"), defaults.max_concurrency), MAX_CONCURRENCY),
+ shards=min(_positive_int(payload.get("shards"), defaults.shards), MAX_SHARDS),
+ max_total_concurrency=min(
+ _positive_int(payload.get("max_total_concurrency"), defaults.max_total_concurrency), MAX_TOTAL_CONCURRENCY
+ ),
max_queries_per_hour=min(
_positive_int(payload.get("max_queries_per_hour"), defaults.max_queries_per_hour), MAX_QUERIES_PER_HOUR
),
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index 9ecb3a5a553b..b45c1da801c6 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -683,6 +683,8 @@ def select_insights(
) -> Selection:
"""The picks of the tick. ``limit`` replaces the pick cap of each pass, for a manual run.
+ Each pick cap of the config is per shard, so the tick picks up to the cap once for each shard.
+
``deadline`` is a ``time.monotonic`` value. After it, no more teams load, and the plan uses the teams it has.
With the due list on, the plan reads the stored due list instead, and ``deadline`` has no effect.
"""
@@ -707,9 +709,9 @@ def select_insights(
# Without its scored state, every insight of the team looks never scored, so the team waits for a later tick.
insights = [insight for insight in insights if insight.team_id in loaded_teams]
+ # A due row of a never scored insight has no watermarks, so its newest bucket is None.
newest_buckets = {
- insight.insight_id: max(state[insight.insight_id].values()) if insight.insight_id in state else None
- for insight in insights
+ insight.insight_id: max(state.get(insight.insight_id, {}).values(), default=None) for insight in insights
}
cooling = _cooling_down([insight.insight_id for insight in insights])
active_by_id = {insight.insight_id: insight for insight in insights if insight.insight_id not in cooling}
@@ -749,7 +751,8 @@ def select_insights(
score_ready: set[int] = set()
score_picked: dict[int, RunPlan] = {}
score_skipped = 0
- score_cap = config.max_score_insights_per_tick if limit is None else limit
+ shards = config.fan_out().shards
+ score_cap = config.max_score_insights_per_tick * shards if limit is None else limit
if config.train_score_split:
score_pass = _score_order(active_by_id, teams, config, now)
score_order_ids, stale_order_ids, score_ready = score_pass.order, score_pass.stale_order, score_pass.ready
@@ -807,9 +810,9 @@ def select_insights(
]
)
if config.train_score_split:
- train_cap = config.max_train_insights_per_tick
+ train_cap = config.max_train_insights_per_tick * shards
else:
- train_cap = config.max_insights_per_tick
+ train_cap = config.max_insights_per_tick * shards
# A stale model plans as a train step here. With the split off, every pick charges the train budgets.
train_picked, train_skipped = walk(train_order, ScoringRun.TRAIN, train_cap if limit is None else limit)
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index a7fc181c1965..c6b79046a762 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -2,6 +2,8 @@
The `anomaly-scoring` flag payload holds every lever, and a missing or malformed payload turns the
tick off. Each insight activity scores one insight and writes its new buckets as `$anomaly_score` events.
+The workflow splits the picks of one plan into shards that run side by side. One plan picks each insight once,
+so two shards never score the same insight, and the plan charges the hourly budgets before any shard starts.
"""
import time
@@ -9,6 +11,7 @@ import asyncio
import datetime
import dataclasses
from collections import Counter
+from collections.abc import Sequence
from django.conf import settings
from django.core.cache import cache
@@ -99,7 +102,7 @@ _PLAN_LOAD_BUDGET = _PLAN_TIMEOUT / 2
_INSIGHT_TIMEOUT = datetime.timedelta(seconds=SCORING_QUERY_BUDGET_SECONDS) + datetime.timedelta(seconds=90)
_INSIGHT_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)
_CAPTURE_TIMEOUT = datetime.timedelta(minutes=1)
-# No batch of insights starts after this time. The picks that are left wait for a later tick.
+# No batch of insights starts after this time, in any shard. The picks that are left wait for a later tick.
TICK_BUDGET = datetime.timedelta(minutes=10)
# A census key lives past its UTC day, so a tick at the end of the day still reads it.
_CENSUS_TTL = datetime.timedelta(days=2)
@@ -118,7 +121,9 @@ class AnomalyScoringTickPlan:
enabled: bool
picks: list[InsightPick]
candidates: int
+ # Picks in flight in each shard.
max_concurrency: int
+ shards: int = 1
limits: ScoringLimits = DEFAULT_LIMITS
skipped_for_budget: int = 0
picked_score: int = 0
@@ -204,6 +209,20 @@ class AnomalyScoringTickResult:
oldest_refresh_seconds: int | None = None
# The CPU spend that the plan read, and what its CPU limits left out. None when the plan did not run.
cpu: CpuSpendReport | None = None
+ # Shards that started picks, the most picks that one shard started, and the time of the slowest shard.
+ shards: int = 0
+ max_shard_picks: int = 0
+ longest_shard_ms: int = 0
+
+
+@frozen(frozen=False)
+class _ShardRun:
+ started: list[InsightPick]
+ statuses: Counter[str]
+ errored: int = 0
+ events: int = 0
+ deferred: int = 0
+ duration_ms: int = 0
def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:
@@ -217,11 +236,13 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
record_candidates(selection.candidates)
# With the due list on, the census is empty, and the refresh job sends it.
capture_census(selection.census, now)
+ fan_out = config.fan_out()
return AnomalyScoringTickPlan(
enabled=True,
picks=selection.picks,
candidates=selection.candidates,
- max_concurrency=config.max_concurrency,
+ max_concurrency=fan_out.concurrency,
+ shards=fan_out.shards,
limits=config.limits,
skipped_for_budget=selection.skipped_for_budget,
picked_score=selection.picked_score,
@@ -243,6 +264,22 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
)
+def shard_picks(picks: Sequence[InsightPick], shards: int) -> list[list[InsightPick]]:
+ """Split the picks into at most ``shards`` non-empty lists, each in the order of the plan.
+
+ All the picks of one team go to one list, so the turns between teams stay as the plan made them. The team
+ with the most picks goes first, to the list with the fewest picks, so the shards end at about the same time.
+ """
+ loads = [0] * max(1, shards)
+ shard_of: dict[int, int] = {}
+ for team_id, count in sorted(Counter(pick.team_id for pick in picks).items(), key=lambda item: -item[1]):
+ shard = loads.index(min(loads))
+ shard_of[team_id] = shard
+ loads[shard] += count
+ lists = [[pick for pick in picks if shard_of[pick.team_id] == shard] for shard in range(len(loads))]
+ return [shard_list for shard_list in lists if shard_list]
+
+
def plan_error_class(error: ActivityError) -> str:
"""The class of the error that failed the plan activity. An activity timeout is a Temporal `TimeoutError`."""
cause = error.cause
@@ -503,6 +540,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
due_insights=result.due_insights,
oldest_due_seconds=result.oldest_due_seconds,
oldest_refresh_seconds=result.oldest_refresh_seconds,
+ shards=result.shards,
+ max_shard_picks=result.max_shard_picks,
+ longest_shard_ms=result.longest_shard_ms,
cpu=result.cpu,
)
cpu = result.cpu
@@ -541,6 +581,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
"due_insights": result.due_insights,
"oldest_due_seconds": result.oldest_due_seconds,
"oldest_refresh_seconds": result.oldest_refresh_seconds,
+ "shards": result.shards,
+ "max_shard_picks": result.max_shard_picks,
+ "longest_shard_ms": result.longest_shard_ms,
"cpu_spend_status": cpu.status if cpu else None,
"fleet_cpu_seconds_today": cpu.fleet_cpu_seconds if cpu else None,
"fleet_read_bytes_today": cpu.fleet_read_bytes if cpu else None,
@@ -575,6 +618,10 @@ async def capture_anomaly_scoring_tick_activity(result: AnomalyScoringTickResult
@workflow.defn(name=TICK_WORKFLOW_NAME)
class AnomalyScoringTickWorkflow:
+ def __init__(self) -> None:
+ # Set by the first shard that gets an at capacity result. Every shard reads it before it starts a batch.
+ self._at_capacity = False
+
@workflow.run
async def run(self, inputs: AnomalyScoringTickInput) -> AnomalyScoringTickResult:
started = workflow.now()
@@ -602,51 +649,23 @@ class AnomalyScoringTickWorkflow:
if not plan.enabled:
return AnomalyScoringTickResult()
+ shards = await asyncio.gather(
+ *(self._run_shard(picks, plan, started) for picks in shard_picks(plan.picks, plan.shards))
+ )
statuses: Counter[str] = Counter()
- errored = 0
- events = 0
- deferred = 0
- started_picks: list[InsightPick] = []
- concurrency = max(1, plan.max_concurrency)
- for index in range(0, len(plan.picks), concurrency):
- if workflow.now() - started >= TICK_BUDGET:
- deferred = len(plan.picks) - index
- break
- batch = plan.picks[index : index + concurrency]
- started_picks.extend(batch)
- results = await asyncio.gather(
- *(
- workflow.execute_activity(
- score_anomaly_insight_activity,
- ScoreInsightInput(pick=pick, limits=plan.limits),
- start_to_close_timeout=_INSIGHT_TIMEOUT,
- heartbeat_timeout=_INSIGHT_HEARTBEAT_TIMEOUT,
- # A failed insight waits for its cooldown and a later tick, not a retry in this one.
- retry_policy=RetryPolicy(maximum_attempts=1),
- )
- for pick in batch
- ),
- return_exceptions=True,
- )
- for outcome in results:
- if isinstance(outcome, BaseException):
- errored += 1
- continue
- statuses[outcome.status] += 1
- events += outcome.events
- # More queries add load to a full cluster. The picks in flight finish, and no new pick starts.
- if statuses[STATUS_AT_CAPACITY]:
- break
+ for shard in shards:
+ statuses.update(shard.statuses)
+ started_picks = [pick for shard in shards for pick in shard.started]
result = AnomalyScoringTickResult(
enabled=True,
candidates=plan.candidates,
picked=len(started_picks),
statuses=dict(statuses),
- errored=errored,
- events=events,
+ errored=sum(shard.errored for shard in shards),
+ events=sum(shard.events for shard in shards),
duration_ms=round((workflow.now() - started).total_seconds() * 1000),
- deferred=deferred,
+ deferred=sum(shard.deferred for shard in shards),
queries=sum(pick.query_cost for pick in started_picks),
skipped_for_budget=plan.skipped_for_budget,
stopped_at_capacity=statuses[STATUS_AT_CAPACITY] > 0,
@@ -668,11 +687,54 @@ class AnomalyScoringTickWorkflow:
due_insights=plan.due_insights,
oldest_due_seconds=plan.oldest_due_seconds,
oldest_refresh_seconds=plan.oldest_refresh_seconds,
+ shards=len(shards),
+ max_shard_picks=max((len(shard.started) for shard in shards), default=0),
+ longest_shard_ms=max((shard.duration_ms for shard in shards), default=0),
cpu=plan.cpu,
)
await self._capture(result)
return result
+ async def _run_shard(
+ self, picks: list[InsightPick], plan: AnomalyScoringTickPlan, tick_started: datetime.datetime
+ ) -> _ShardRun:
+ shard = _ShardRun(started=[], statuses=Counter())
+ shard_started = workflow.now()
+ concurrency = max(1, plan.max_concurrency)
+ for index in range(0, len(picks), concurrency):
+ # More queries add load to a full cluster. The picks in flight in every shard finish, and no new pick starts.
+ if self._at_capacity:
+ break
+ if workflow.now() - tick_started >= TICK_BUDGET:
+ shard.deferred = len(picks) - index
+ break
+ batch = picks[index : index + concurrency]
+ shard.started.extend(batch)
+ results = await asyncio.gather(
+ *(
+ workflow.execute_activity(
+ score_anomaly_insight_activity,
+ ScoreInsightInput(pick=pick, limits=plan.limits),
+ start_to_close_timeout=_INSIGHT_TIMEOUT,
+ heartbeat_timeout=_INSIGHT_HEARTBEAT_TIMEOUT,
+ # A failed insight waits for its cooldown and a later tick, not a retry in this one.
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ for pick in batch
+ ),
+ return_exceptions=True,
+ )
+ for outcome in results:
+ if isinstance(outcome, BaseException):
+ shard.errored += 1
+ continue
+ shard.statuses[outcome.status] += 1
+ shard.events += outcome.events
+ if shard.statuses[STATUS_AT_CAPACITY]:
+ self._at_capacity = True
+ shard.duration_ms = round((workflow.now() - shard_started).total_seconds() * 1000)
+ return shard
+
async def _capture(self, result: AnomalyScoringTickResult) -> None:
await workflow.execute_activity(
capture_anomaly_scoring_tick_activity,
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index 8d2a2ab458bb..a4dfb80e7874 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -1,6 +1,8 @@
import json
import uuid
+import asyncio
import hashlib
+import dataclasses
from datetime import UTC, datetime, timedelta
from typing import Any
from zoneinfo import ZoneInfo
@@ -98,6 +100,7 @@ from products.signals.backend.anomaly_scores.tick import (
capture_census,
plan_tick,
run_insight,
+ shard_picks,
)
from products.signals.backend.models import (
AnomalyScoringDueInsight,
@@ -249,6 +252,8 @@ class TestTickConfig(SimpleTestCase):
[
("max_insights_per_tick", 51, 50),
("max_concurrency", 100, 8),
+ ("shards", 100, 8),
+ ("max_total_concurrency", 100, 32),
("max_queries_per_hour", 1_000_000, 2000),
("max_queries_per_project_per_hour", 501, 500),
("max_score_queries_per_hour", 1_000_000, 5000),
@@ -263,6 +268,21 @@ class TestTickConfig(SimpleTestCase):
assert getattr(config, field) == expected
+ @parameterized.expand(
+ [
+ ("defaults", {}, 1, 2),
+ ("shards_within_the_total", {"shards": 4}, 4, 2),
+ ("total_lowers_the_shards", {"shards": 8, "max_concurrency": 4, "max_total_concurrency": 12}, 3, 4),
+ ("total_below_one_shard", {"shards": 4, "max_concurrency": 4, "max_total_concurrency": 2}, 1, 2),
+ ]
+ )
+ def test_the_total_concurrency_bounds_the_shards(
+ self, _name: str, levers: dict[str, int], shards: int, concurrency: int
+ ) -> None:
+ fan_out = parse_config({"enabled": True, "organization_ids": [ORGANIZATION_ID], **levers}).fan_out()
+
+ assert (fan_out.shards, fan_out.concurrency) == (shards, concurrency)
+
@parameterized.expand(
[
("defaults", {}, 20, 2),
@@ -284,6 +304,27 @@ class TestTickConfig(SimpleTestCase):
assert (config.max_score_insights_per_tick, config.max_train_insights_per_tick) == (score_cap, train_cap)
+class TestShardPicks(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("one_shard_keeps_the_plan_order", 1, [[(1, 1), (2, 2), (1, 3), (3, 4), (1, 5), (2, 6)]]),
+ ("the_largest_team_goes_alone", 2, [[(1, 1), (1, 3), (1, 5)], [(2, 2), (3, 4), (2, 6)]]),
+ ("no_empty_shard", 5, [[(1, 1), (1, 3), (1, 5)], [(2, 2), (2, 6)], [(3, 4)]]),
+ ]
+ )
+ def test_each_team_goes_to_one_shard_in_plan_order(
+ self, _name: str, shards: int, expected: list[list[tuple[int, int]]]
+ ) -> None:
+ picks = [
+ InsightPick(team_id=team_id, insight_id=insight_id, scored_through={})
+ for team_id, insight_id in [(1, 1), (2, 2), (1, 3), (3, 4), (1, 5), (2, 6)]
+ ]
+
+ lists = shard_picks(picks, shards)
+
+ assert [[(pick.team_id, pick.insight_id) for pick in shard] for shard in lists] == expected
+
+
class TestCpuGate(SimpleTestCase):
@parameterized.expand(
[
@@ -1097,6 +1138,29 @@ class TestDueList(ClickhouseTestMixin, BaseTest):
assert [(pick.insight_id, pick.scored_through) for pick in selection.picks] == [(due.id, watermarks)]
assert (selection.active, selection.due_insights, selection.oldest_due_seconds) == (2, 1, 3600)
+ @parameterized.expand([("one_shard", 1, 1), ("three_shards", 3, 3)])
+ @time_machine.travel(NOW, tick=False)
+ def test_the_pick_cap_applies_to_each_shard(self, _name: str, shards: int, picked: int) -> None:
+ insights = [
+ Insight.objects.create(team=self.team, name=f"due_{index}", query=_trends(interval="day"))
+ for index in range(4)
+ ]
+ for insight in insights:
+ self._row(insight, NOW - timedelta(hours=1), {})
+ cache.delete_many(
+ [
+ *(_cooldown_key(insight.id) for insight in insights),
+ _fleet_budget_key(NOW),
+ _project_budget_key(self.team.id, NOW),
+ ]
+ )
+ config = dataclasses.replace(self._config(), max_insights_per_tick=1, shards=shards)
+
+ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):
+ selection = select_insights(config, SelectionConfig(weighting=Weighting.UNIFORM), NOW)
+
+ assert len(selection.picks) == picked
+
@time_machine.travel(NOW, tick=False)
def test_a_scored_pick_keeps_the_watermark_of_a_series_that_it_did_not_return(self) -> None:
today = NOW.replace(hour=0)
@@ -1627,8 +1691,10 @@ async def _run_tick(
plan: AnomalyScoringTickPlan | Exception,
statuses: dict[int, str],
captured: list[AnomalyScoringTickResult] | None = None,
+ waits_for: dict[int, int] | None = None,
) -> tuple[AnomalyScoringTickResult, list[int]]:
started: list[int] = []
+ starts: dict[int, asyncio.Event] = {}
@activity.defn(name="plan_anomaly_scoring_tick_activity")
async def fake_plan(input: AnomalyScoringTickInput) -> AnomalyScoringTickPlan:
@@ -1639,6 +1705,10 @@ async def _run_tick(
@activity.defn(name="score_anomaly_insight_activity")
async def fake_score(input: ScoreInsightInput) -> ScoreInsightResult:
started.append(input.pick.insight_id)
+ starts.setdefault(input.pick.insight_id, asyncio.Event()).set()
+ if waits_for and input.pick.insight_id in waits_for:
+ # Bounded, so a pick whose other pick never starts fails the test instead of hanging it.
+ await asyncio.wait_for(starts.setdefault(waits_for[input.pick.insight_id], asyncio.Event()).wait(), 10)
return ScoreInsightResult(status=statuses.get(input.pick.insight_id, "scored"), events=1)
@activity.defn(name="capture_anomaly_scoring_tick_activity")
@@ -1696,6 +1766,26 @@ async def test_the_tick_stops_starting_picks_when_clickhouse_is_at_capacity(
) == (len(started), queries, 3, stopped, 5.0)
+async def test_shards_run_side_by_side_with_each_team_in_one_shard() -> None:
+ plan = AnomalyScoringTickPlan(
+ enabled=True,
+ picks=[
+ InsightPick(team_id=team_id, insight_id=insight_id, scored_through={})
+ for team_id, insight_id in [(1, 1), (2, 3), (1, 2), (2, 4)]
+ ],
+ candidates=4,
+ max_concurrency=1,
+ shards=2,
+ )
+
+ # Pick 1 finishes only after pick 3 starts, so shards that run one after the other fail pick 1.
+ result, ran = await _run_tick(plan, {}, waits_for={1: 3})
+
+ assert sorted(ran) == [1, 2, 3, 4]
+ assert (result.picked, result.errored, result.shards, result.max_shard_picks) == (4, 0, 2, 2)
+
+
@pytest.mark.asyncio
async def test_a_failed_plan_sends_a_tick_event_and_still_fails_the_workflow() -> None:
captured: list[AnomalyScoringTickResult] = []