:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(signals): shuffle the team order of anomaly scoring each tick (#128161)
posthog[bot]cbf81a96acc2
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.pyindex 1ef85b41bfe0..8e756642bc34 100644--- a/products/signals/backend/anomaly_scores/queue.py+++ b/products/signals/backend/anomaly_scores/queue.py@@ -66,6 +66,7 @@ from products.signals.backend.anomaly_scores.selection import (InsightUsage,ScoreCandidate,SelectionConfig,+ _unit_interval,insight_usage,pick_order,score_order,@@ -672,9 +673,12 @@ def _run_plans(return {insight_id: plans.get(insight_id, missing) for insight_id in insight_ids}-def _take_turns(team_orders: list[list[int]]) -> list[int]:- # Projects take turns, so one large project cannot take every slot of the tick budget.- return [insight_id for turn in itertools.zip_longest(*team_orders) for insight_id in turn if insight_id is not None]+def _take_turns(team_orders: Mapping[int, list[int]], tick_start: datetime.datetime) -> list[int]:+ # Projects take turns, so one large project cannot take every slot of the tick budget. A hashed team+ # order for each tick, so the pick cap does not always cut off the same projects.+ team_ids = sorted(team_orders, key=lambda team_id: _unit_interval("team", team_id, tick_start.isoformat()))+ turns = itertools.zip_longest(*(team_orders[team_id] for team_id in team_ids))+ return [insight_id for turn in turns for insight_id in turn if insight_id is not None]def load_scored_state(@@ -850,15 +854,16 @@ def select_insights())train_order = _take_turns(- [- pick_order(- candidates_by_team[team_id],+ {+ team_id: pick_order(+ candidates,config=selection_config,tick_start=now,max_staleness=config.limits.max_backfill_buckets,)- for team_id in sorted(candidates_by_team)- ]+ for team_id, candidates in candidates_by_team.items()+ },+ now,)if config.train_score_split:train_cap = config.max_train_insights_per_tick * shards@@ -942,8 +947,8 @@ def _score_order(A stale model is in `stale_order` only, because the train pass gets it first. `ready` leaves out thestale models, so they reach the train pass."""- team_orders: list[list[int]] = []- stale_orders: list[list[int]] = []+ team_orders: dict[int, list[int]] = {}+ stale_orders: dict[int, list[int]] = {}ready: set[int] = set()for team_id in sorted({insight.team_id for insight in active_by_id.values()}):team = teams[team_id]@@ -964,11 +969,9 @@ def _score_order(and is_due(insight, scored[insight.insight_id].scored_through, team.timezone_info, now)]for stale, orders in ((False, team_orders), (True, stale_orders)):- orders.append(- score_order(- [candidate for is_stale, candidate in candidates if is_stale == stale],- now=now,- max_staleness=config.limits.max_backfill_buckets,- )+ orders[team_id] = score_order(+ [candidate for is_stale, candidate in candidates if is_stale == stale],+ now=now,+ max_staleness=config.limits.max_backfill_buckets,)- return ScorePassOrder(order=_take_turns(team_orders), stale_order=_take_turns(stale_orders), ready=ready)+ return ScorePassOrder(order=_take_turns(team_orders, now), stale_order=_take_turns(stale_orders, now), ready=ready)diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.pyindex 418e53a75557..bda09efa978d 100644--- a/products/signals/backend/test/test_anomaly_scoring_tick.py+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py@@ -65,6 +65,7 @@ from products.signals.backend.anomaly_scores.queue import (_cooldown_key,_fleet_budget_key,_project_budget_key,+ _take_turns,_team_loaded_key,_too_expensive_key,active_insights,@@ -605,6 +606,16 @@ class TestTickSelection(SimpleTestCase):assert is_due(insight, newest_bucket, timezone, now) is due+ def test_teams_take_turns_in_an_order_that_changes_with_the_tick(self) -> None:+ team_orders = {1: [11, 12], 1_000_000: [21, 22]}+ ticks = [NOW + timedelta(minutes=10 * index) for index in range(50)]++ orders = [_take_turns(team_orders, tick) for tick in ticks]++ assert {order[0] for order in orders} == {11, 21}+ assert all(order in ([11, 21, 12, 22], [21, 11, 22, 12]) for order in orders)+ assert all(_take_turns(team_orders, tick) == order for tick, order in zip(ticks, orders))+class TestScoreEvents(SimpleTestCase):def test_a_revisit_emits_every_bucket_after_the_newest_scored_one_of_each_series(self) -> None:@@ -1438,10 +1449,11 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):selection = select_insights(config, SelectionConfig(weighting=Weighting.WEIGHTED), NOW, limit=2)- assert [pick.team_id for pick in selection.picks] == [self.team.id, small_team.id]- assert selection.picks[0].insight_id in {insight.id for insight in large_insights}- assert selection.picks[1].insight_id == small_insight.id- assert [pick.too_expensive for pick in selection.picks] == [0, 2]+ picks = {pick.team_id: pick for pick in selection.picks}+ assert len(selection.picks) == 2 and set(picks) == {self.team.id, small_team.id}+ assert picks[self.team.id].insight_id in {insight.id for insight in large_insights}+ assert picks[small_team.id].insight_id == small_insight.id+ assert [picks[self.team.id].too_expensive, picks[small_team.id].too_expensive] == [0, 2]assert selection.candidates == 4assert [(team.team_id, team.active, team.active_intervals) for team in selection.census] == [(self.team.id, 3, {"day": 3}),