:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): fan out the anomaly scoring tick into parallel shards (#127048)
posthog[bot]f6ff4baea4ee
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.pyindex 3d8b1b961de4..e5e06ca3e3ef 100644--- a/products/signals/backend/anomaly_scores/config.py+++ b/products/signals/backend/anomaly_scores/config.py@@ -120,6 +120,14 @@ MAX_MODEL_AGE_BUCKETS_LIMIT = {DEFAULT_TICK_INTERVALS = (IntervalType.DAY, IntervalType.WEEK, IntervalType.MONTH)+@frozen+class TickFanOut:+ """How the tick runs its picks: this many shards side by side, each with this many picks in flight."""++ shards: int+ concurrency: int++@frozenclass AnomalyScoringConfig:enabled: bool = False@@ -134,9 +142,14 @@ class AnomalyScoringConfig:skip_team_ids: tuple[int, ...] = ()intervals: tuple[IntervalType, ...] = DEFAULT_TICK_INTERVALS# Smooths the spend over the hour, so one tick cannot use the whole query budget. With the split on,- # the two passes use the per-pass caps below instead.+ # the two passes use the per-pass caps below instead. Each pick cap is per shard, see `fan_out`.max_insights_per_tick: int = 8+ # Picks in flight in each shard.max_concurrency: int = 2+ # Shards of the tick. Each shard runs its own picks, and all the picks of one team go to one shard.+ shards: int = 1+ # Picks in flight over all the shards. It lowers the shard count, so the cluster load has one upper limit.+ max_total_concurrency: int = 8# Scoring queries per clock hour. One pick sends one query per series, see `scoring.query_cost`.max_queries_per_hour: int = 120max_queries_per_project_per_hour: int = 30@@ -166,11 +179,19 @@ class AnomalyScoringConfig:max_cpu_seconds_per_project_per_day: int = 0max_fleet_cpu_seconds_per_day: int = 0+ def fan_out(self) -> TickFanOut:+ concurrency = min(self.max_concurrency, self.max_total_concurrency)+ return TickFanOut(+ shards=max(1, min(self.shards, self.max_total_concurrency // concurrency)), concurrency=concurrency+ )+DISABLED = AnomalyScoringConfig()# The largest value a flag payload can set for each lever of the tick.MAX_INSIGHTS_PER_TICK = 50MAX_CONCURRENCY = 8+MAX_SHARDS = 8+MAX_TOTAL_CONCURRENCY = 32MAX_QUERIES_PER_HOUR = 2000MAX_QUERIES_PER_PROJECT_PER_HOUR = 500MAX_SCORE_QUERIES_PER_HOUR = 5000@@ -279,6 +300,10 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:_positive_int(payload.get("max_insights_per_tick"), defaults.max_insights_per_tick), MAX_INSIGHTS_PER_TICK),max_concurrency=min(_positive_int(payload.get("max_concurrency"), defaults.max_concurrency), MAX_CONCURRENCY),+ shards=min(_positive_int(payload.get("shards"), defaults.shards), MAX_SHARDS),+ max_total_concurrency=min(+ _positive_int(payload.get("max_total_concurrency"), defaults.max_total_concurrency), MAX_TOTAL_CONCURRENCY+ ),max_queries_per_hour=min(_positive_int(payload.get("max_queries_per_hour"), defaults.max_queries_per_hour), MAX_QUERIES_PER_HOUR),diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.pyindex 9ecb3a5a553b..b45c1da801c6 100644--- a/products/signals/backend/anomaly_scores/queue.py+++ b/products/signals/backend/anomaly_scores/queue.py@@ -683,6 +683,8 @@ def select_insights() -> Selection:"""The picks of the tick. ``limit`` replaces the pick cap of each pass, for a manual run.+ Each pick cap of the config is per shard, so the tick picks up to the cap once for each shard.+``deadline`` is a ``time.monotonic`` value. After it, no more teams load, and the plan uses the teams it has.With the due list on, the plan reads the stored due list instead, and ``deadline`` has no effect."""@@ -707,9 +709,9 @@ def select_insights(# Without its scored state, every insight of the team looks never scored, so the team waits for a later tick.insights = [insight for insight in insights if insight.team_id in loaded_teams]+ # A due row of a never scored insight has no watermarks, so its newest bucket is None.newest_buckets = {- insight.insight_id: max(state[insight.insight_id].values()) if insight.insight_id in state else None- for insight in insights+ insight.insight_id: max(state.get(insight.insight_id, {}).values(), default=None) for insight in insights}cooling = _cooling_down([insight.insight_id for insight in insights])active_by_id = {insight.insight_id: insight for insight in insights if insight.insight_id not in cooling}@@ -749,7 +751,8 @@ def select_insights(score_ready: set[int] = set()score_picked: dict[int, RunPlan] = {}score_skipped = 0- score_cap = config.max_score_insights_per_tick if limit is None else limit+ shards = config.fan_out().shards+ score_cap = config.max_score_insights_per_tick * shards if limit is None else limitif config.train_score_split:score_pass = _score_order(active_by_id, teams, config, now)score_order_ids, stale_order_ids, score_ready = score_pass.order, score_pass.stale_order, score_pass.ready@@ -807,9 +810,9 @@ def select_insights(])if config.train_score_split:- train_cap = config.max_train_insights_per_tick+ train_cap = config.max_train_insights_per_tick * shardselse:- train_cap = config.max_insights_per_tick+ train_cap = config.max_insights_per_tick * shards# A stale model plans as a train step here. With the split off, every pick charges the train budgets.train_picked, train_skipped = walk(train_order, ScoringRun.TRAIN, train_cap if limit is None else limit)diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.pyindex a7fc181c1965..c6b79046a762 100644--- a/products/signals/backend/anomaly_scores/tick.py+++ b/products/signals/backend/anomaly_scores/tick.py@@ -2,6 +2,8 @@The `anomaly-scoring` flag payload holds every lever, and a missing or malformed payload turns thetick off. Each insight activity scores one insight and writes its new buckets as `$anomaly_score` events.+The workflow splits the picks of one plan into shards that run side by side. One plan picks each insight once,+so two shards never score the same insight, and the plan charges the hourly budgets before any shard starts."""import time@@ -9,6 +11,7 @@ import asyncioimport datetimeimport dataclassesfrom collections import Counter+from collections.abc import Sequencefrom django.conf import settingsfrom django.core.cache import cache@@ -99,7 +102,7 @@ _PLAN_LOAD_BUDGET = _PLAN_TIMEOUT / 2_INSIGHT_TIMEOUT = datetime.timedelta(seconds=SCORING_QUERY_BUDGET_SECONDS) + datetime.timedelta(seconds=90)_INSIGHT_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)_CAPTURE_TIMEOUT = datetime.timedelta(minutes=1)-# No batch of insights starts after this time. The picks that are left wait for a later tick.+# No batch of insights starts after this time, in any shard. The picks that are left wait for a later tick.TICK_BUDGET = datetime.timedelta(minutes=10)# A census key lives past its UTC day, so a tick at the end of the day still reads it._CENSUS_TTL = datetime.timedelta(days=2)@@ -118,7 +121,9 @@ class AnomalyScoringTickPlan:enabled: boolpicks: list[InsightPick]candidates: int+ # Picks in flight in each shard.max_concurrency: int+ shards: int = 1limits: ScoringLimits = DEFAULT_LIMITSskipped_for_budget: int = 0picked_score: int = 0@@ -204,6 +209,20 @@ class AnomalyScoringTickResult:oldest_refresh_seconds: int | None = None# The CPU spend that the plan read, and what its CPU limits left out. None when the plan did not run.cpu: CpuSpendReport | None = None+ # Shards that started picks, the most picks that one shard started, and the time of the slowest shard.+ shards: int = 0+ max_shard_picks: int = 0+ longest_shard_ms: int = 0+++@frozen(frozen=False)+class _ShardRun:+ started: list[InsightPick]+ statuses: Counter[str]+ errored: int = 0+ events: int = 0+ deferred: int = 0+ duration_ms: int = 0def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:@@ -217,11 +236,13 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlrecord_candidates(selection.candidates)# With the due list on, the census is empty, and the refresh job sends it.capture_census(selection.census, now)+ fan_out = config.fan_out()return AnomalyScoringTickPlan(enabled=True,picks=selection.picks,candidates=selection.candidates,- max_concurrency=config.max_concurrency,+ max_concurrency=fan_out.concurrency,+ shards=fan_out.shards,limits=config.limits,skipped_for_budget=selection.skipped_for_budget,picked_score=selection.picked_score,@@ -243,6 +264,22 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl)+def shard_picks(picks: Sequence[InsightPick], shards: int) -> list[list[InsightPick]]:+ """Split the picks into at most ``shards`` non-empty lists, each in the order of the plan.++ All the picks of one team go to one list, so the turns between teams stay as the plan made them. The team+ with the most picks goes first, to the list with the fewest picks, so the shards end at about the same time.+ """+ loads = [0] * max(1, shards)+ shard_of: dict[int, int] = {}+ for team_id, count in sorted(Counter(pick.team_id for pick in picks).items(), key=lambda item: -item[1]):+ shard = loads.index(min(loads))+ shard_of[team_id] = shard+ loads[shard] += count+ lists = [[pick for pick in picks if shard_of[pick.team_id] == shard] for shard in range(len(loads))]+ return [shard_list for shard_list in lists if shard_list]++def plan_error_class(error: ActivityError) -> str:"""The class of the error that failed the plan activity. An activity timeout is a Temporal `TimeoutError`."""cause = error.cause@@ -503,6 +540,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:due_insights=result.due_insights,oldest_due_seconds=result.oldest_due_seconds,oldest_refresh_seconds=result.oldest_refresh_seconds,+ shards=result.shards,+ max_shard_picks=result.max_shard_picks,+ longest_shard_ms=result.longest_shard_ms,cpu=result.cpu,)cpu = result.cpu@@ -541,6 +581,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:"due_insights": result.due_insights,"oldest_due_seconds": result.oldest_due_seconds,"oldest_refresh_seconds": result.oldest_refresh_seconds,+ "shards": result.shards,+ "max_shard_picks": result.max_shard_picks,+ "longest_shard_ms": result.longest_shard_ms,"cpu_spend_status": cpu.status if cpu else None,"fleet_cpu_seconds_today": cpu.fleet_cpu_seconds if cpu else None,"fleet_read_bytes_today": cpu.fleet_read_bytes if cpu else None,@@ -575,6 +618,10 @@ async def capture_anomaly_scoring_tick_activity(result: AnomalyScoringTickResult@workflow.defn(name=TICK_WORKFLOW_NAME)class AnomalyScoringTickWorkflow:+ def __init__(self) -> None:+ # Set by the first shard that gets an at capacity result. Every shard reads it before it starts a batch.+ self._at_capacity = False+@workflow.runasync def run(self, inputs: AnomalyScoringTickInput) -> AnomalyScoringTickResult:started = workflow.now()@@ -602,51 +649,23 @@ class AnomalyScoringTickWorkflow:if not plan.enabled:return AnomalyScoringTickResult()+ shards = await asyncio.gather(+ *(self._run_shard(picks, plan, started) for picks in shard_picks(plan.picks, plan.shards))+ )statuses: Counter[str] = Counter()- errored = 0- events = 0- deferred = 0- started_picks: list[InsightPick] = []- concurrency = max(1, plan.max_concurrency)- for index in range(0, len(plan.picks), concurrency):- if workflow.now() - started >= TICK_BUDGET:- deferred = len(plan.picks) - index- break- batch = plan.picks[index : index + concurrency]- started_picks.extend(batch)- results = await asyncio.gather(- *(- workflow.execute_activity(- score_anomaly_insight_activity,- ScoreInsightInput(pick=pick, limits=plan.limits),- start_to_close_timeout=_INSIGHT_TIMEOUT,- heartbeat_timeout=_INSIGHT_HEARTBEAT_TIMEOUT,- # A failed insight waits for its cooldown and a later tick, not a retry in this one.- retry_policy=RetryPolicy(maximum_attempts=1),- )- for pick in batch- ),- return_exceptions=True,- )- for outcome in results:- if isinstance(outcome, BaseException):- errored += 1- continue- statuses[outcome.status] += 1- events += outcome.events- # More queries add load to a full cluster. The picks in flight finish, and no new pick starts.- if statuses[STATUS_AT_CAPACITY]:- break+ for shard in shards:+ statuses.update(shard.statuses)+ started_picks = [pick for shard in shards for pick in shard.started]result = AnomalyScoringTickResult(enabled=True,candidates=plan.candidates,picked=len(started_picks),statuses=dict(statuses),- errored=errored,- events=events,+ errored=sum(shard.errored for shard in shards),+ events=sum(shard.events for shard in shards),duration_ms=round((workflow.now() - started).total_seconds() * 1000),- deferred=deferred,+ deferred=sum(shard.deferred for shard in shards),queries=sum(pick.query_cost for pick in started_picks),skipped_for_budget=plan.skipped_for_budget,stopped_at_capacity=statuses[STATUS_AT_CAPACITY] > 0,@@ -668,11 +687,54 @@ class AnomalyScoringTickWorkflow:due_insights=plan.due_insights,oldest_due_seconds=plan.oldest_due_seconds,oldest_refresh_seconds=plan.oldest_refresh_seconds,+ shards=len(shards),+ max_shard_picks=max((len(shard.started) for shard in shards), default=0),+ longest_shard_ms=max((shard.duration_ms for shard in shards), default=0),cpu=plan.cpu,)await self._capture(result)return result+ async def _run_shard(+ self, picks: list[InsightPick], plan: AnomalyScoringTickPlan, tick_started: datetime.datetime+ ) -> _ShardRun:+ shard = _ShardRun(started=[], statuses=Counter())+ shard_started = workflow.now()+ concurrency = max(1, plan.max_concurrency)+ for index in range(0, len(picks), concurrency):+ # More queries add load to a full cluster. The picks in flight in every shard finish, and no new pick starts.+ if self._at_capacity:+ break+ if workflow.now() - tick_started >= TICK_BUDGET:+ shard.deferred = len(picks) - index+ break+ batch = picks[index : index + concurrency]+ shard.started.extend(batch)+ results = await asyncio.gather(+ *(+ workflow.execute_activity(+ score_anomaly_insight_activity,+ ScoreInsightInput(pick=pick, limits=plan.limits),+ start_to_close_timeout=_INSIGHT_TIMEOUT,+ heartbeat_timeout=_INSIGHT_HEARTBEAT_TIMEOUT,+ # A failed insight waits for its cooldown and a later tick, not a retry in this one.+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+ for pick in batch+ ),+ return_exceptions=True,+ )+ for outcome in results:+ if isinstance(outcome, BaseException):+ shard.errored += 1+ continue+ shard.statuses[outcome.status] += 1+ shard.events += outcome.events+ if shard.statuses[STATUS_AT_CAPACITY]:+ self._at_capacity = True+ shard.duration_ms = round((workflow.now() - shard_started).total_seconds() * 1000)+ return shard+async def _capture(self, result: AnomalyScoringTickResult) -> None:await workflow.execute_activity(capture_anomaly_scoring_tick_activity,diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.pyindex 8d2a2ab458bb..a4dfb80e7874 100644--- a/products/signals/backend/test/test_anomaly_scoring_tick.py+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py@@ -1,6 +1,8 @@import jsonimport uuid+import asyncioimport hashlib+import dataclassesfrom datetime import UTC, datetime, timedeltafrom typing import Anyfrom zoneinfo import ZoneInfo@@ -98,6 +100,7 @@ from products.signals.backend.anomaly_scores.tick import (capture_census,plan_tick,run_insight,+ shard_picks,)from products.signals.backend.models import (AnomalyScoringDueInsight,@@ -249,6 +252,8 @@ class TestTickConfig(SimpleTestCase):[("max_insights_per_tick", 51, 50),("max_concurrency", 100, 8),+ ("shards", 100, 8),+ ("max_total_concurrency", 100, 32),("max_queries_per_hour", 1_000_000, 2000),("max_queries_per_project_per_hour", 501, 500),("max_score_queries_per_hour", 1_000_000, 5000),@@ -263,6 +268,21 @@ class TestTickConfig(SimpleTestCase):assert getattr(config, field) == expected+ @parameterized.expand(+ [+ ("defaults", {}, 1, 2),+ ("shards_within_the_total", {"shards": 4}, 4, 2),+ ("total_lowers_the_shards", {"shards": 8, "max_concurrency": 4, "max_total_concurrency": 12}, 3, 4),+ ("total_below_one_shard", {"shards": 4, "max_concurrency": 4, "max_total_concurrency": 2}, 1, 2),+ ]+ )+ def test_the_total_concurrency_bounds_the_shards(+ self, _name: str, levers: dict[str, int], shards: int, concurrency: int+ ) -> None:+ fan_out = parse_config({"enabled": True, "organization_ids": [ORGANIZATION_ID], **levers}).fan_out()++ assert (fan_out.shards, fan_out.concurrency) == (shards, concurrency)+@parameterized.expand([("defaults", {}, 20, 2),@@ -284,6 +304,27 @@ class TestTickConfig(SimpleTestCase):assert (config.max_score_insights_per_tick, config.max_train_insights_per_tick) == (score_cap, train_cap)+class TestShardPicks(SimpleTestCase):+ @parameterized.expand(+ [+ ("one_shard_keeps_the_plan_order", 1, [[(1, 1), (2, 2), (1, 3), (3, 4), (1, 5), (2, 6)]]),+ ("the_largest_team_goes_alone", 2, [[(1, 1), (1, 3), (1, 5)], [(2, 2), (3, 4), (2, 6)]]),+ ("no_empty_shard", 5, [[(1, 1), (1, 3), (1, 5)], [(2, 2), (2, 6)], [(3, 4)]]),+ ]+ )+ def test_each_team_goes_to_one_shard_in_plan_order(+ self, _name: str, shards: int, expected: list[list[tuple[int, int]]]+ ) -> None:+ picks = [+ InsightPick(team_id=team_id, insight_id=insight_id, scored_through={})+ for team_id, insight_id in [(1, 1), (2, 2), (1, 3), (3, 4), (1, 5), (2, 6)]+ ]++ lists = shard_picks(picks, shards)++ assert [[(pick.team_id, pick.insight_id) for pick in shard] for shard in lists] == expected++class TestCpuGate(SimpleTestCase):@parameterized.expand([@@ -1097,6 +1138,29 @@ class TestDueList(ClickhouseTestMixin, BaseTest):assert [(pick.insight_id, pick.scored_through) for pick in selection.picks] == [(due.id, watermarks)]assert (selection.active, selection.due_insights, selection.oldest_due_seconds) == (2, 1, 3600)+ @parameterized.expand([("one_shard", 1, 1), ("three_shards", 3, 3)])+ @time_machine.travel(NOW, tick=False)+ def test_the_pick_cap_applies_to_each_shard(self, _name: str, shards: int, picked: int) -> None:+ insights = [+ Insight.objects.create(team=self.team, name=f"due_{index}", query=_trends(interval="day"))+ for index in range(4)+ ]+ for insight in insights:+ self._row(insight, NOW - timedelta(hours=1), {})+ cache.delete_many(+ [+ *(_cooldown_key(insight.id) for insight in insights),+ _fleet_budget_key(NOW),+ _project_budget_key(self.team.id, NOW),+ ]+ )+ config = dataclasses.replace(self._config(), max_insights_per_tick=1, shards=shards)++ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):+ selection = select_insights(config, SelectionConfig(weighting=Weighting.UNIFORM), NOW)++ assert len(selection.picks) == picked+@time_machine.travel(NOW, tick=False)def test_a_scored_pick_keeps_the_watermark_of_a_series_that_it_did_not_return(self) -> None:today = NOW.replace(hour=0)@@ -1627,8 +1691,10 @@ async def _run_tick(plan: AnomalyScoringTickPlan | Exception,statuses: dict[int, str],captured: list[AnomalyScoringTickResult] | None = None,+ waits_for: dict[int, int] | None = None,) -> tuple[AnomalyScoringTickResult, list[int]]:started: list[int] = []+ starts: dict[int, asyncio.Event] = {}@activity.defn(name="plan_anomaly_scoring_tick_activity")async def fake_plan(input: AnomalyScoringTickInput) -> AnomalyScoringTickPlan:@@ -1639,6 +1705,10 @@ async def _run_tick(@activity.defn(name="score_anomaly_insight_activity")async def fake_score(input: ScoreInsightInput) -> ScoreInsightResult:started.append(input.pick.insight_id)+ starts.setdefault(input.pick.insight_id, asyncio.Event()).set()+ if waits_for and input.pick.insight_id in waits_for:+ # Bounded, so a pick whose other pick never starts fails the test instead of hanging it.+ await asyncio.wait_for(starts.setdefault(waits_for[input.pick.insight_id], asyncio.Event()).wait(), 10)return ScoreInsightResult(status=statuses.get(input.pick.insight_id, "scored"), events=1)@activity.defn(name="capture_anomaly_scoring_tick_activity")@@ -1696,6 +1766,26 @@ async def test_the_tick_stops_starting_picks_when_clickhouse_is_at_capacity() == (len(started), queries, 3, stopped, 5.0)+async def test_shards_run_side_by_side_with_each_team_in_one_shard() -> None:+ plan = AnomalyScoringTickPlan(+ enabled=True,+ picks=[+ InsightPick(team_id=team_id, insight_id=insight_id, scored_through={})+ for team_id, insight_id in [(1, 1), (2, 3), (1, 2), (2, 4)]+ ],+ candidates=4,+ max_concurrency=1,+ shards=2,+ )++ # Pick 1 finishes only after pick 3 starts, so shards that run one after the other fail pick 1.+ result, ran = await _run_tick(plan, {}, waits_for={1: 3})++ assert sorted(ran) == [1, 2, 3, 4]+ assert (result.picked, result.errored, result.shards, result.max_shard_picks) == (4, 0, 2, 2)++@pytest.mark.asyncioasync def test_a_failed_plan_sends_a_tick_event_and_still_fails_the_workflow() -> None:captured: list[AnomalyScoringTickResult] = []