Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): shuffle the team order of anomaly scoring each tick (#128161)

posthog[bot]cbf81a96acc2
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index 1ef85b41bfe0..8e756642bc34 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -66,6 +66,7 @@ from products.signals.backend.anomaly_scores.selection import (
InsightUsage,
ScoreCandidate,
SelectionConfig,
+ _unit_interval,
insight_usage,
pick_order,
score_order,
@@ -672,9 +673,12 @@ def _run_plans(
return {insight_id: plans.get(insight_id, missing) for insight_id in insight_ids}
-def _take_turns(team_orders: list[list[int]]) -> list[int]:
- # Projects take turns, so one large project cannot take every slot of the tick budget.
- return [insight_id for turn in itertools.zip_longest(*team_orders) for insight_id in turn if insight_id is not None]
+def _take_turns(team_orders: Mapping[int, list[int]], tick_start: datetime.datetime) -> list[int]:
+ # Projects take turns, so one large project cannot take every slot of the tick budget. A hashed team
+ # order for each tick, so the pick cap does not always cut off the same projects.
+ team_ids = sorted(team_orders, key=lambda team_id: _unit_interval("team", team_id, tick_start.isoformat()))
+ turns = itertools.zip_longest(*(team_orders[team_id] for team_id in team_ids))
+ return [insight_id for turn in turns for insight_id in turn if insight_id is not None]
def load_scored_state(
@@ -850,15 +854,16 @@ def select_insights(
)
)
train_order = _take_turns(
- [
- pick_order(
- candidates_by_team[team_id],
+ {
+ team_id: pick_order(
+ candidates,
config=selection_config,
tick_start=now,
max_staleness=config.limits.max_backfill_buckets,
)
- for team_id in sorted(candidates_by_team)
- ]
+ for team_id, candidates in candidates_by_team.items()
+ },
+ now,
)
if config.train_score_split:
train_cap = config.max_train_insights_per_tick * shards
@@ -942,8 +947,8 @@ def _score_order(
A stale model is in `stale_order` only, because the train pass gets it first. `ready` leaves out the
stale models, so they reach the train pass.
"""
- team_orders: list[list[int]] = []
- stale_orders: list[list[int]] = []
+ team_orders: dict[int, list[int]] = {}
+ stale_orders: dict[int, list[int]] = {}
ready: set[int] = set()
for team_id in sorted({insight.team_id for insight in active_by_id.values()}):
team = teams[team_id]
@@ -964,11 +969,9 @@ def _score_order(
and is_due(insight, scored[insight.insight_id].scored_through, team.timezone_info, now)
]
for stale, orders in ((False, team_orders), (True, stale_orders)):
- orders.append(
- score_order(
- [candidate for is_stale, candidate in candidates if is_stale == stale],
- now=now,
- max_staleness=config.limits.max_backfill_buckets,
- )
+ orders[team_id] = score_order(
+ [candidate for is_stale, candidate in candidates if is_stale == stale],
+ now=now,
+ max_staleness=config.limits.max_backfill_buckets,
)
- return ScorePassOrder(order=_take_turns(team_orders), stale_order=_take_turns(stale_orders), ready=ready)
+ return ScorePassOrder(order=_take_turns(team_orders, now), stale_order=_take_turns(stale_orders, now), ready=ready)
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index 418e53a75557..bda09efa978d 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -65,6 +65,7 @@ from products.signals.backend.anomaly_scores.queue import (
_cooldown_key,
_fleet_budget_key,
_project_budget_key,
+ _take_turns,
_team_loaded_key,
_too_expensive_key,
active_insights,
@@ -605,6 +606,16 @@ class TestTickSelection(SimpleTestCase):
assert is_due(insight, newest_bucket, timezone, now) is due
+ def test_teams_take_turns_in_an_order_that_changes_with_the_tick(self) -> None:
+ team_orders = {1: [11, 12], 1_000_000: [21, 22]}
+ ticks = [NOW + timedelta(minutes=10 * index) for index in range(50)]
+
+ orders = [_take_turns(team_orders, tick) for tick in ticks]
+
+ assert {order[0] for order in orders} == {11, 21}
+ assert all(order in ([11, 21, 12, 22], [21, 11, 22, 12]) for order in orders)
+ assert all(_take_turns(team_orders, tick) == order for tick, order in zip(ticks, orders))
+
class TestScoreEvents(SimpleTestCase):
def test_a_revisit_emits_every_bucket_after_the_newest_scored_one_of_each_series(self) -> None:
@@ -1438,10 +1449,11 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):
selection = select_insights(config, SelectionConfig(weighting=Weighting.WEIGHTED), NOW, limit=2)
- assert [pick.team_id for pick in selection.picks] == [self.team.id, small_team.id]
- assert selection.picks[0].insight_id in {insight.id for insight in large_insights}
- assert selection.picks[1].insight_id == small_insight.id
- assert [pick.too_expensive for pick in selection.picks] == [0, 2]
+ picks = {pick.team_id: pick for pick in selection.picks}
+ assert len(selection.picks) == 2 and set(picks) == {self.team.id, small_team.id}
+ assert picks[self.team.id].insight_id in {insight.id for insight in large_insights}
+ assert picks[small_team.id].insight_id == small_insight.id
+ assert [picks[self.team.id].too_expensive, picks[small_team.id].too_expensive] == [0, 2]
assert selection.candidates == 4
assert [(team.team_id, team.active, team.active_intervals) for team in selection.census] == [
(self.team.id, 3, {"day": 3}),