:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): plan anomaly scoring from a stored due list (#127031)
posthog[bot]ab580b44aa91
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yamlindex fd9519f2b38d..0d531456c39d 100644--- a/.semgrep/rules/security/idor-team-scoped-models.yaml+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml@@ -428,6 +428,7 @@ rules:|LegacySignalSpend|AnomalyScoringState+ |AnomalyScoringDueInsight|SignalScratchpad|SignalProjectProfile|SingleSessionSummary@@ -923,6 +924,7 @@ rules:|LegacySignalSpend|AnomalyScoringState+ |AnomalyScoringDueInsight|SignalScratchpad|SignalProjectProfile|SingleSessionSummarydiff --git a/posthog/temporal/schedule.py b/posthog/temporal/schedule.pyindex 929d7d4c4e6c..9dee010db1e7 100644--- a/posthog/temporal/schedule.py+++ b/posthog/temporal/schedule.py@@ -140,7 +140,10 @@ from products.replay_vision.backend.temporal.vision_alerts.schedule import creatfrom products.review_hog.backend.facade.temporal import create_review_hog_finding_outcomes_schedulefrom products.security.backend.facade.temporal import create_sync_access_rules_schedulefrom products.signals.backend.activity_summary.schedule import create_activity_summary_tick_schedule-from products.signals.backend.anomaly_scores.schedule import create_anomaly_scoring_tick_schedule+from products.signals.backend.anomaly_scores.schedule import (+ create_anomaly_scoring_due_list_refresh_schedule,+ create_anomaly_scoring_tick_schedule,+)from products.signals.backend.emission.conversations_schedule import create_conversations_signals_coordinator_schedulefrom products.signals.backend.ranking.schedule import create_inbox_ranking_scoring_schedulefrom products.signals.backend.temporal.agentic.schedule import (@@ -1021,6 +1024,7 @@ schedules = [create_inbox_ranking_scoring_schedule,create_activity_summary_tick_schedule,create_anomaly_scoring_tick_schedule,+ create_anomaly_scoring_due_list_refresh_schedule,create_scout_suggestions_coordinator_schedule,create_support_reply_coordinator_schedule,create_channel_summary_coordinator_schedule,diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.pyindex d91b06987e6e..9fe62d152f78 100644--- a/products/signals/backend/anomaly_scores/config.py+++ b/products/signals/backend/anomaly_scores/config.py@@ -152,6 +152,10 @@ class AnomalyScoringConfig:max_train_insights_per_tick: int = 2# An insight is scored only while one of its activity signals falls in this many days, see `queue.active_insights`.active_window_days: int = 90+ # On, the tick plans from the stored due list, and a slow job refreshes the list. See `due_list.py`.+ due_list: bool = False+ # Each enrolled team's due list refreshes about this often.+ due_list_refresh_hours: int = 6DISABLED = AnomalyScoringConfig()@@ -165,6 +169,7 @@ MAX_SCORE_QUERIES_PER_PROJECT_PER_HOUR = 1000MAX_SCORE_INSIGHTS_PER_TICK = 200MAX_TRAIN_INSIGHTS_PER_TICK = 50MAX_ACTIVE_WINDOW_DAYS = 365+MAX_DUE_LIST_REFRESH_HOURS = 48def _positive_int(value: Any, default: int) -> int:@@ -293,6 +298,11 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:active_window_days=min(_positive_int(payload.get("active_window_days"), defaults.active_window_days), MAX_ACTIVE_WINDOW_DAYS),+ due_list=payload.get("due_list") is True,+ due_list_refresh_hours=min(+ _positive_int(payload.get("due_list_refresh_hours"), defaults.due_list_refresh_hours),+ MAX_DUE_LIST_REFRESH_HOURS,+ ),)diff --git a/products/signals/backend/anomaly_scores/due_list.py b/products/signals/backend/anomaly_scores/due_list.pynew file mode 100644index 000000000000..9d46563dbb29--- /dev/null+++ b/products/signals/backend/anomaly_scores/due_list.py@@ -0,0 +1,238 @@+"""The stored due list of the anomaly scoring tick, and the slow refresh that keeps it.++With `due_list` on, the tick plans from the due rows and sends no scored state query to ClickHouse. The list+can be stale. A pick of an insight that is already done waits for the attempt cooldown, and the backfill+scores the buckets of an insight that the list shows as due late.++The refresh adds the active insights of each enrolled team, removes the insights that left, and sets the+usage. Each pick then moves its own row, so the list stays close to right between two refreshes.+"""++import time+import datetime+from collections.abc import Collection++from django.core.cache import cache++import structlog++from posthog.schema import IntervalType++from posthog.dataclasses import frozen+from posthog.interval_specs import interval_spec+from posthog.models import Team++from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig, ScoringLimits+from products.signals.backend.anomaly_scores.queue import (+ ATTEMPT_COOLDOWN,+ ActiveInsight,+ TeamCensus,+ active_insights,+ due_at,+ eligible_teams,+ parse_watermarks,+ reason_counts,+ scored_buckets,+ scored_state_since,+ team_census,+)+from products.signals.backend.anomaly_scores.selection import insight_usage+from products.signals.backend.models import AnomalyScoringDueInsight, AnomalyScoringState++logger = structlog.get_logger(__name__)++# The last refresh of a team. It lives past the longest refresh window, so a team waits for its turn.+_TEAM_REFRESHED_TTL = datetime.timedelta(days=3)+++@frozen+class TeamRefresh:+ census: TeamCensus+ added: int+ removed: int+++@frozen+class DueListRefresh:+ enabled: bool = False+ teams_refreshed: int = 0+ teams_failed: int = 0+ # Teams past their refresh window that wait for a later run.+ teams_waiting: int = 0+ rows_added: int = 0+ rows_removed: int = 0+ # Active insights of the refreshed teams, and how many of them have each activity reason.+ active: int = 0+ active_reasons: dict[str, int] | None = None+ census: list[TeamCensus] | None = None+++def _team_refreshed_key(team_id: int) -> str:+ return f"signals:anomaly_scores:due_list_refreshed:{team_id}"+++def dump_watermarks(watermarks: dict[str, datetime.datetime]) -> dict[str, str]:+ return {key: bucket.astimezone(datetime.UTC).isoformat() for key, bucket in watermarks.items()}+++def seed_watermarks(+ config: AnomalyScoringConfig, team: Team, insights: Collection[ActiveInsight], now: datetime.datetime+) -> dict[int, dict[str, datetime.datetime]]:+ """The scored state of insights new to the list.++ A stored model gives each of its series the newest scored bucket of the model, so the switch causes no+ retrain. Only insights with no stored model read their state from ClickHouse, once, when they join the list.+ """+ if not insights:+ return {}+ seeded: dict[int, dict[str, datetime.datetime]] = {}+ rows = (+ AnomalyScoringState.objects.for_team(team.id)+ .filter(insight_id__in=[insight.insight_id for insight in insights], scored_through__isnull=False)+ .values_list("insight_id", "scored_through", "series_keys")+ )+ for insight_id, scored_through, series_keys in rows:+ if scored_through is None or not isinstance(series_keys, list):+ continue+ watermarks = seeded.setdefault(insight_id, {})+ for series_key in series_keys:+ watermarks[str(series_key)] = max(watermarks.get(str(series_key), scored_through), scored_through)+ windows: dict[datetime.datetime, set[int]] = {}+ for insight in insights:+ if not seeded.get(insight.insight_id):+ windows.setdefault(scored_state_since(config, insight.interval, now), set()).add(insight.insight_id)+ if windows:+ seeded.update(scored_buckets(team, windows, now))+ return seeded+++def refresh_team(config: AnomalyScoringConfig, team: Team, now: datetime.datetime) -> TeamRefresh:+ insights = active_insights([team.id], config.intervals, now, config.active_window_days)+ by_id = {insight.insight_id: insight for insight in insights}+ scope = AnomalyScoringDueInsight.all_teams.filter(team_id=team.id)+ removed, _ = scope.exclude(insight_id__in=list(by_id)).delete()+ existing = {row.insight_id: row for row in scope.filter(insight_id__in=list(by_id))}+ seeded = seed_watermarks(config, team, [insight for insight in insights if insight.insight_id not in existing], now)+ usage = insight_usage(team_id=team.id, insight_ids=list(by_id), now=now)++ created: list[AnomalyScoringDueInsight] = []+ updated: list[AnomalyScoringDueInsight] = []+ for insight in insights:+ row = existing.get(insight.insight_id)+ watermarks = parse_watermarks(row.series_scored_through) if row else seeded.get(insight.insight_id, {})+ next_due_at = (+ due_at(insight.insight_id, insight.interval, max(watermarks.values(), default=None), team.timezone_info)+ or now+ )+ if row is not None and row.interval == insight.interval.value:+ # A failed pick moved the row past its cooldown.+ next_due_at = max(next_due_at, row.next_due_at)+ fields = {+ "interval": insight.interval.value,+ "next_due_at": next_due_at,+ "refreshed_at": now,+ "viewers": usage[insight.insight_id].viewers,+ "has_enabled_alert": usage[insight.insight_id].has_enabled_alert,+ }+ if row is None:+ created.append(+ AnomalyScoringDueInsight(+ team_id=team.id,+ insight_id=insight.insight_id,+ series_scored_through=dump_watermarks(watermarks),+ **fields,+ )+ )+ continue+ for name, value in fields.items():+ setattr(row, name, value)+ row.updated_at = now+ updated.append(row)+ # A pick writes the watermarks of its own row, so the refresh never writes them for an existing row.+ AnomalyScoringDueInsight.all_teams.bulk_create(created, ignore_conflicts=True)+ AnomalyScoringDueInsight.all_teams.bulk_update(+ updated, ["interval", "next_due_at", "refreshed_at", "viewers", "has_enabled_alert", "updated_at"]+ )+ [census] = team_census([team.id], insights)+ return TeamRefresh(census=census, added=len(created), removed=removed)+++def refresh_due_list(config: AnomalyScoringConfig, now: datetime.datetime, deadline: float) -> DueListRefresh:+ """Refresh the teams past their refresh window, the longest waiting first, until ``deadline``.++ ``deadline`` is a ``time.monotonic`` value. At least one team refreshes per run. A failed team also goes to+ the back, so a team that always fails does not block the others.+ """+ if not config.enabled or not config.due_list:+ return DueListRefresh()+ teams = {team.id: team for team in eligible_teams(config)}+ stamps = cache.get_many([_team_refreshed_key(team_id) for team_id in teams])+ window_start = (now - datetime.timedelta(hours=config.due_list_refresh_hours)).timestamp()+ waiting = sorted(+ (team_id for team_id in teams if stamps.get(_team_refreshed_key(team_id), 0.0) <= window_start),+ key=lambda team_id: (stamps.get(_team_refreshed_key(team_id), 0.0), team_id),+ )+ census: list[TeamCensus] = []+ attempted = failed = added = removed = 0+ for team_id in waiting:+ if attempted and time.monotonic() >= deadline:+ break+ attempted += 1+ try:+ refresh = refresh_team(config, teams[team_id], now)+ except Exception as error:+ failed += 1+ logger.warning(+ "anomaly_scoring_due_list_refresh_failed",+ team_id=team_id,+ error_class=type(error).__name__,+ exc_info=True,+ )+ else:+ census.append(refresh.census)+ added += refresh.added+ removed += refresh.removed+ cache.set(_team_refreshed_key(team_id), now.timestamp(), timeout=int(_TEAM_REFRESHED_TTL.total_seconds()))+ return DueListRefresh(+ enabled=True,+ teams_refreshed=attempted - failed,+ teams_failed=failed,+ teams_waiting=len(waiting) - attempted,+ rows_added=added,+ rows_removed=removed,+ active=sum(team.active for team in census),+ active_reasons={reason: sum(team.active_reasons[reason] for team in census) for reason in reason_counts([])},+ census=census,+ )+++def record_scored(+ team: Team,+ insight_id: int,+ watermarks: dict[str, datetime.datetime],+ now: datetime.datetime,+ limits: ScoringLimits,+) -> None:+ """Merge the newest scored bucket of each series into the insight's row, and set when it is due next."""+ row = AnomalyScoringDueInsight.all_teams.filter(team_id=team.id, insight_id=insight_id).first()+ if row is None:+ return+ interval = IntervalType(row.interval)+ merged = parse_watermarks(row.series_scored_through)+ for series_key, bucket in watermarks.items():+ merged[series_key] = max(merged.get(series_key, bucket), bucket)+ # A series older than the backfill cap scores as a new series, so its entry has no effect.+ since = now - interval_spec(interval).period * (limits.max_backfill_buckets + 1)+ kept = {series_key: bucket for series_key, bucket in merged.items() if bucket >= since}+ due = due_at(insight_id, interval, max(kept.values(), default=None), team.timezone_info)+ # A visit with no new bucket must not make the row due again before the cooldown ends.+ row.next_due_at = max(due or now, now + ATTEMPT_COOLDOWN)+ row.series_scored_through = dump_watermarks(kept)+ row.save(update_fields=["next_due_at", "series_scored_through", "updated_at"])+++def record_failed(team_id: int, insight_id: int, now: datetime.datetime, cooldown: datetime.timedelta) -> None:+ """Move the insight's row past the cooldown of a failed pick."""+ AnomalyScoringDueInsight.all_teams.filter(team_id=team_id, insight_id=insight_id).update(+ next_due_at=now + cooldown, updated_at=now+ )diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.pyindex ebcd7ca654d2..32049c1f029c 100644--- a/products/signals/backend/anomaly_scores/queue.py+++ b/products/signals/backend/anomaly_scores/queue.py@@ -1,7 +1,8 @@"""Pick the active insights that the next anomaly scoring tick scores.One query per team reads the newest scored bucket of each series back from the `$anomaly_score`-events. `selection.pick_order` then orders the due insights, and the tick picks from that order the+events. With `due_list` on, the plan reads the stored due list of `due_list.py` instead, and sends no+ClickHouse query. `selection.pick_order` then orders the due insights, and the tick picks from that order theinsights whose queries fit the hourly budgets. With the train and score split on, `state.plan_run`makes each pick a train step or a score step, and each kind has its own budgets.@@ -22,6 +23,7 @@ from enum import StrEnumfrom typing import Any, castfrom django.core.cache import cache+from django.db.models import Count, Minimport structlog@@ -66,7 +68,7 @@ from products.signals.backend.anomaly_scores.selection import (score_order,)from products.signals.backend.anomaly_scores.state import RunPlan, load_states, plan_run, score_ready_by_insight-from products.signals.backend.models import SignalTeamConfig+from products.signals.backend.models import AnomalyScoringDueInsight, SignalTeamConfigfrom products.subscriptions.backend.facade.api import insight_ids_with_active_subscriptionsfrom ee.billing.quota_limiting import QuotaLimitingCaches, QuotaResource, list_limited_team_attributes@@ -105,6 +107,8 @@ _MAX_AT_CAPACITY_TEAMS = 2# The last scored state load of a team. The team that waited longest loads first, so a plan that stops early# does not drop the same teams on every tick._TEAM_LOADED_TTL = datetime.timedelta(days=1)+# The most due rows that one plan reads. Far more than the picks of one tick.+_MAX_DUE_ROWS = 5000# The scorer rejects these, so selection leaves them out instead of picking them on every tick._UNSCORED_DISPLAYS = sorted(display.value for display in {*NON_TIME_SERIES_DISPLAY_TYPES, ChartDisplayType.ACTIONS_LINE_GRAPH_CUMULATIVE}@@ -194,6 +198,24 @@ class Selection:# Eligible teams, and their organizations.enrolled_teams: int = 0enrolled_organizations: int = 0+ # Due rows of the stored due list, the time since the oldest of them became due, and the time since the+ # oldest refresh of a row. None when the tick does not plan from the due list.+ due_insights: int | None = None+ oldest_due_seconds: int | None = None+ oldest_refresh_seconds: int | None = None+++@frozen+class DueList:+ """The due rows of the stored due list, for the eligible teams."""++ insights: list[ActiveInsight]+ scored_through: dict[int, dict[str, datetime.datetime]]+ usage: dict[int, InsightUsage]+ # Rows of the eligible teams, due or not.+ rows: int+ oldest_due_seconds: int | None+ oldest_refresh_seconds: int | None@frozen@@ -366,17 +388,64 @@ def scored_buckets(return state+def due_at(+ insight_id: int, interval: IntervalType, newest_bucket: datetime.datetime | None, timezone: datetime.tzinfo+) -> datetime.datetime | None:+ """When the bucket after the newest scored one is complete and this insight's slot after its end starts.++ None when the insight was never scored, because it is due at once.+ """+ if newest_bucket is None:+ return None+ next_start = bucket_end(newest_bucket.astimezone(timezone), interval)+ next_end = bucket_end(next_start, interval)+ # The same offset as alert checks, so insights spread over the window after each boundary.+ return next_end + alert_check_offset(_CALENDAR_INTERVALS[interval], f"anomaly:{insight_id}")++def is_due(insight: ActiveInsight, newest_bucket: datetime.datetime | None, timezone: datetime.tzinfo, now: datetime.datetime) -> bool:- """The bucket after the newest scored one is complete, and this insight's slot after its end has passed."""- if newest_bucket is None:- return True- next_start = bucket_end(newest_bucket.astimezone(timezone), insight.interval)- next_end = bucket_end(next_start, insight.interval)- # The same offset as alert checks, so insights spread over the window after each boundary.- offset = alert_check_offset(_CALENDAR_INTERVALS[insight.interval], f"anomaly:{insight.insight_id}")- return now >= next_end + offset+ due = due_at(insight.insight_id, insight.interval, newest_bucket, timezone)+ return due is None or now >= due+++def parse_watermarks(value: Any) -> dict[str, datetime.datetime]:+ """The stored map of series key to newest scored bucket. An entry that does not parse is left out."""+ if not isinstance(value, dict):+ return {}+ watermarks: dict[str, datetime.datetime] = {}+ for series_key, stamp in value.items():+ try:+ watermarks[str(series_key)] = _utc(datetime.datetime.fromisoformat(stamp))+ except (TypeError, ValueError):+ continue+ return watermarks+++def load_due_list(team_ids: Collection[int], intervals: tuple[IntervalType, ...], now: datetime.datetime) -> DueList:+ """The due rows of the teams, longest overdue first, in a fixed number of queries."""+ scope = AnomalyScoringDueInsight.all_teams.filter(team_id__in=list(team_ids))+ summary = scope.aggregate(rows=Count("id"), oldest_refresh=Min("refreshed_at"))+ rows = list(+ scope.filter(next_due_at__lte=now, interval__in=[interval.value for interval in intervals]).order_by(+ "next_due_at", "insight_id"+ )[:_MAX_DUE_ROWS]+ )+ oldest_refresh = summary["oldest_refresh"]+ return DueList(+ insights=[+ ActiveInsight(team_id=row.team_id, insight_id=row.insight_id, interval=IntervalType(row.interval))+ for row in rows+ ],+ scored_through={row.insight_id: parse_watermarks(row.series_scored_through) for row in rows},+ usage={+ row.insight_id: InsightUsage(viewers=row.viewers, has_enabled_alert=row.has_enabled_alert) for row in rows+ },+ rows=summary["rows"],+ oldest_due_seconds=round((now - rows[0].next_due_at).total_seconds()) if rows else None,+ oldest_refresh_seconds=None if oldest_refresh is None else round((now - oldest_refresh).total_seconds()),+ )def _cooldown_key(insight_id: int) -> str:@@ -610,16 +679,26 @@ def select_insights("""The picks of the tick. ``limit`` replaces the pick cap of each pass, for a manual run.``deadline`` is a ``time.monotonic`` value. After it, no more teams load, and the plan uses the teams it has.+ With the due list on, the plan reads the stored due list instead, and ``deadline`` has no effect."""teams = {team.id: team for team in eligible_teams(config)}- insights = active_insights(list(teams), config.intervals, now, config.active_window_days)- active = len(insights)- active_reasons = reason_counts(insights)- census = team_census(teams, insights)- teams_with_insights = {insight.team_id for insight in insights}- state, loaded_teams = load_scored_state(config, teams, insights, now, deadline)- # Without its scored state, every insight of the team looks never scored, so the team waits for a later tick.- insights = [insight for insight in insights if insight.team_id in loaded_teams]+ due_list: DueList | None = None+ if config.due_list:+ due_list = load_due_list(teams, config.intervals, now)+ insights = due_list.insights+ state = due_list.scored_through+ # The refresh job sends the activity reasons and the census.+ active, active_reasons, census = due_list.rows, {}, []+ loaded_teams = teams_with_insights = {insight.team_id for insight in insights}+ else:+ insights = active_insights(list(teams), config.intervals, now, config.active_window_days)+ active = len(insights)+ active_reasons = reason_counts(insights)+ census = team_census(teams, insights)+ teams_with_insights = {insight.team_id for insight in insights}+ state, loaded_teams = load_scored_state(config, teams, insights, now, deadline)+ # Without its scored state, every insight of the team looks never scored, so the team waits for a later tick.+ insights = [insight for insight in insights if insight.team_id in loaded_teams]newest_buckets = {insight.insight_id: max(state[insight.insight_id].values()) if insight.insight_id in state else None@@ -686,9 +765,12 @@ def select_insights(and is_due(insight, newest_buckets[insight_id], teams[insight.team_id].timezone_info, now)}usage: dict[int, InsightUsage] = {}- for team_id in sorted({insight.team_id for insight in due_by_id.values()}):- due_ids = [insight.insight_id for insight in due_by_id.values() if insight.team_id == team_id]- usage.update(insight_usage(team_id=team_id, insight_ids=due_ids, now=now))+ if due_list is not None:+ usage = due_list.usage+ else:+ for team_id in sorted({insight.team_id for insight in due_by_id.values()}):+ due_ids = [insight.insight_id for insight in due_by_id.values() if insight.team_id == team_id]+ usage.update(insight_usage(team_id=team_id, insight_ids=due_ids, now=now))candidates_by_team: dict[int, list[Candidate]] = {}for insight in due_by_id.values():newest = newest_buckets[insight.insight_id]@@ -764,6 +846,9 @@ def select_insights(teams_skipped=len(teams_with_insights) - len(loaded_teams),enrolled_teams=len(teams),enrolled_organizations=len({team.organization_id for team in teams.values()}),+ due_insights=None if due_list is None else len(due_list.insights),+ oldest_due_seconds=None if due_list is None else due_list.oldest_due_seconds,+ oldest_refresh_seconds=None if due_list is None else due_list.oldest_refresh_seconds,)diff --git a/products/signals/backend/anomaly_scores/refresh.py b/products/signals/backend/anomaly_scores/refresh.pynew file mode 100644index 000000000000..6bdd2285af54--- /dev/null+++ b/products/signals/backend/anomaly_scores/refresh.py@@ -0,0 +1,100 @@+"""The scheduled refresh of the anomaly scoring due list: a Temporal workflow apart from the tick.++The `due_list` field of the `anomaly-scoring` flag payload gates the work. See `due_list.py`.+"""++import time+import datetime++from django.conf import settings+from django.utils import timezone++from temporalio import activity, workflow+from temporalio.common import RetryPolicy++from posthog.dataclasses import frozen+from posthog.ph_client import ph_background_capture+from posthog.sync import database_sync_to_async+from posthog.temporal.common.scoped import scoped_temporal+from posthog.temporal.common.utils import close_db_connections++from products.signals.backend.anomaly_scores.config import ANOMALY_SCORING_FLAG_DISTINCT_ID, read_config, read_payload+from products.signals.backend.anomaly_scores.due_list import refresh_due_list+from products.signals.backend.anomaly_scores.tick import capture_census++REFRESH_WORKFLOW_NAME = "anomaly-scoring-due-list-refresh"+# No team starts to refresh after this time. The teams that are left go first on the next run.+_REFRESH_BUDGET = datetime.timedelta(minutes=5)+# One team can still run after the budget, and its census and summary captures run after it.+_REFRESH_ACTIVITY_TIMEOUT = datetime.timedelta(minutes=10)+REFRESH_TIMEOUT = _REFRESH_ACTIVITY_TIMEOUT * 2+++@frozen+class DueListRefreshInput:+ pass+++@frozen+class DueListRefreshResult:+ enabled: bool = False+ teams_refreshed: int = 0+ teams_failed: int = 0+ teams_waiting: int = 0+ rows_added: int = 0+ rows_removed: int = 0+ duration_ms: int = 0+++def run_refresh(now: datetime.datetime) -> DueListRefreshResult:+ started = time.monotonic()+ config = read_config(read_payload())+ refresh = refresh_due_list(config, now, deadline=started + _REFRESH_BUDGET.total_seconds())+ if not refresh.enabled:+ return DueListRefreshResult()+ capture_census(refresh.census or [], now)+ result = DueListRefreshResult(+ enabled=True,+ teams_refreshed=refresh.teams_refreshed,+ teams_failed=refresh.teams_failed,+ teams_waiting=refresh.teams_waiting,+ rows_added=refresh.rows_added,+ rows_removed=refresh.rows_removed,+ duration_ms=round((time.monotonic() - started) * 1000),+ )+ ph_background_capture()(+ distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,+ event="anomaly scoring due list refresh",+ properties={+ "deployment": settings.CLOUD_DEPLOYMENT,+ "teams_refreshed": result.teams_refreshed,+ "teams_failed": result.teams_failed,+ "teams_waiting": result.teams_waiting,+ "rows_added": result.rows_added,+ "rows_removed": result.rows_removed,+ "duration_ms": result.duration_ms,+ "active_insights": refresh.active,+ **{f"active_{reason}": count for reason, count in (refresh.active_reasons or {}).items()},+ },+ )+ return result+++@scoped_temporal()+@close_db_connections+async def refresh_anomaly_scoring_due_list_activity(input: DueListRefreshInput) -> DueListRefreshResult:+ return await database_sync_to_async(run_refresh, thread_sensitive=False)(timezone.now())++[email protected](name=REFRESH_WORKFLOW_NAME)+class AnomalyScoringDueListRefreshWorkflow:+ @workflow.run+ async def run(self, inputs: DueListRefreshInput) -> DueListRefreshResult:+ return await workflow.execute_activity(+ refresh_anomaly_scoring_due_list_activity,+ inputs,+ start_to_close_timeout=_REFRESH_ACTIVITY_TIMEOUT,+ # A failed run leaves its teams waiting, and the next run starts with them.+ retry_policy=RetryPolicy(maximum_attempts=1),+ )diff --git a/products/signals/backend/anomaly_scores/schedule.py b/products/signals/backend/anomaly_scores/schedule.pyindex a613d140936d..2fc6fa809c18 100644--- a/products/signals/backend/anomaly_scores/schedule.py+++ b/products/signals/backend/anomaly_scores/schedule.py@@ -1,4 +1,4 @@-"""Schedule registration for the anomaly scoring tick."""+"""Schedule registration for the anomaly scoring tick and the due list refresh."""from dataclasses import asdictfrom datetime import timedelta@@ -18,10 +18,24 @@ from temporalio.client import (from posthog.scheduling.jitter import deterministic_offsetfrom posthog.temporal.common.schedule import a_create_schedule, a_schedule_exists, a_update_schedule+from products.signals.backend.anomaly_scores.refresh import REFRESH_TIMEOUT, REFRESH_WORKFLOW_NAME, DueListRefreshInputfrom products.signals.backend.anomaly_scores.tick import TICK_TIMEOUT, TICK_WORKFLOW_NAME, AnomalyScoringTickInputANOMALY_SCORING_TICK_SCHEDULE_ID = "anomaly-scoring-tick-schedule"ANOMALY_SCORING_TICK_INTERVAL = timedelta(minutes=5)+ANOMALY_SCORING_DUE_LIST_REFRESH_SCHEDULE_ID = "anomaly-scoring-due-list-refresh-schedule"+# Each run refreshes only the teams past their refresh window, so most runs end at once.+ANOMALY_SCORING_DUE_LIST_REFRESH_INTERVAL = timedelta(minutes=15)+++async def _create_or_update(client: Client, schedule_id: str, schedule: Schedule) -> None:+ if await a_schedule_exists(client, schedule_id):+ # Keep the live state, so a deploy does not resume a schedule that an operator paused.+ description = await client.get_schedule_handle(schedule_id).describe()+ schedule.state = description.schedule.state+ await a_update_schedule(client, schedule_id, schedule)+ else:+ await a_create_schedule(client, schedule_id, schedule, trigger_immediately=False)async def create_anomaly_scoring_tick_schedule(client: Client) -> None:@@ -49,10 +63,33 @@ async def create_anomaly_scoring_tick_schedule(client: Client) -> None:),policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.SKIP),)- if await a_schedule_exists(client, ANOMALY_SCORING_TICK_SCHEDULE_ID):- # Keep the live state, so a deploy does not resume a schedule that an operator paused.- description = await client.get_schedule_handle(ANOMALY_SCORING_TICK_SCHEDULE_ID).describe()- schedule.state = description.schedule.state- await a_update_schedule(client, ANOMALY_SCORING_TICK_SCHEDULE_ID, schedule)- else:- await a_create_schedule(client, ANOMALY_SCORING_TICK_SCHEDULE_ID, schedule, trigger_immediately=False)+ await _create_or_update(client, ANOMALY_SCORING_TICK_SCHEDULE_ID, schedule)+++async def create_anomaly_scoring_due_list_refresh_schedule(client: Client) -> None:+ """Create or update the due list refresh schedule on the self-driving task queue.++ The `due_list` field of the `anomaly-scoring` flag payload gates the work. SKIP on overlap: two runs+ must not refresh the same team at the same time.+ """+ schedule = Schedule(+ action=ScheduleActionStartWorkflow(+ REFRESH_WORKFLOW_NAME,+ asdict(DueListRefreshInput()),+ id=ANOMALY_SCORING_DUE_LIST_REFRESH_SCHEDULE_ID,+ task_queue=settings.SELF_DRIVING_TASK_QUEUE,+ execution_timeout=REFRESH_TIMEOUT,+ ),+ spec=ScheduleSpec(+ intervals=[+ ScheduleIntervalSpec(+ every=ANOMALY_SCORING_DUE_LIST_REFRESH_INTERVAL,+ offset=deterministic_offset(+ ANOMALY_SCORING_DUE_LIST_REFRESH_SCHEDULE_ID, ANOMALY_SCORING_DUE_LIST_REFRESH_INTERVAL+ ),+ )+ ]+ ),+ policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.SKIP),+ )+ await _create_or_update(client, ANOMALY_SCORING_DUE_LIST_REFRESH_SCHEDULE_ID, schedule)diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.pyindex d54f46600181..9d9857c9b9c5 100644--- a/products/signals/backend/anomaly_scores/tick.py+++ b/products/signals/backend/anomaly_scores/tick.py@@ -47,6 +47,7 @@ from products.signals.backend.anomaly_scores.config import (read_payload,resolve_profile,)+from products.signals.backend.anomaly_scores.due_list import record_failed, record_scoredfrom products.signals.backend.anomaly_scores.emission import emit, score_eventsfrom products.signals.backend.anomaly_scores.metrics import record_candidates, record_pickfrom products.signals.backend.anomaly_scores.queue import (@@ -130,6 +131,9 @@ class AnomalyScoringTickPlan:enrolled_organizations: int = 0teams_planned: int = 0teams_skipped: int = 0+ due_insights: int | None = None+ oldest_due_seconds: int | None = None+ oldest_refresh_seconds: int | None = None@frozen@@ -188,6 +192,11 @@ class AnomalyScoringTickResult:# The plan activity failed, and the tick scored nothing. `plan_error` is the error class.plan_failed: bool = Falseplan_error: str | None = None+ # Due rows that the plan read from the stored due list, the time since the oldest of them became due, and+ # the time since the oldest row refresh. None when the plan does not read the due list.+ due_insights: int | None = None+ oldest_due_seconds: int | None = None+ oldest_refresh_seconds: int | None = Nonedef plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:@@ -199,6 +208,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPldeadline = time.monotonic() + _PLAN_LOAD_BUDGET.total_seconds()selection = select_insights(config, parse_selection_config(payload), now, limit, deadline=deadline)record_candidates(selection.candidates)+ # With the due list on, the census is empty, and the refresh job sends it.capture_census(selection.census, now)return AnomalyScoringTickPlan(enabled=True,@@ -218,6 +228,9 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlenrolled_organizations=selection.enrolled_organizations,teams_planned=selection.teams_planned,teams_skipped=selection.teams_skipped,+ due_insights=selection.due_insights,+ oldest_due_seconds=selection.oldest_due_seconds,+ oldest_refresh_seconds=selection.oldest_refresh_seconds,)@@ -275,10 +288,12 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh)record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0, run=pick.run)capture_pick(pick, status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, run=pick.run)+ # A full cluster is not a fault of the insight, so it keeps the attempt cooldown.+ cooldown = SKIP_COOLDOWN if status not in (None, STATUS_AT_CAPACITY) else ATTEMPT_COOLDOWN+ record_failed(pick.team_id, pick.insight_id, now, cooldown)if status is None:raise- # A full cluster is not a fault of the insight, so it keeps the attempt cooldown.- if status != STATUS_AT_CAPACITY:+ if cooldown == SKIP_COOLDOWN:mark_attempted(pick.insight_id, SKIP_COOLDOWN)return ScoreInsightResult(status=status, events=0)duration_ms = round((time.monotonic() - started) * 1000)@@ -376,6 +391,8 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsighNone,)if insight is None:+ # The next refresh removes the row of a deleted insight.+ record_failed(pick.team_id, pick.insight_id, now, SKIP_COOLDOWN)return ScoreInsightResult(status=STATUS_MISSING, events=0)team = Team.objects.get(id=pick.team_id)# Set before scoring, so a failure also waits for the cooldown instead of taking a slot on every tick.@@ -396,6 +413,7 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigherror=str(error)[:200],)invalidate_state(team, insight.id, profile)+ record_failed(team.id, insight.id, now, ATTEMPT_COOLDOWN)return ScoreInsightResult(status=STATUS_ARTIFACT_UNAVAILABLE, events=0, run=ScoringRun.SCORE)scores = score_insight(insight,@@ -422,6 +440,17 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh# A state write before a failed emit would skip those buckets for good.if scores.state is not None:save_state(team, insight.id, profile, scores.state)+ record_scored(+ team,+ insight.id,+ {+ series.series_key: max(point.bucket_start for point in series.points)+ for series in scores.series+ if series.points+ },+ now,+ input.limits,+ )return ScoreInsightResult(status=STATUS_SCORED if events else STATUS_UP_TO_DATE,events=len(events),@@ -461,6 +490,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:teams_skipped=result.teams_skipped,plan_failed=result.plan_failed,plan_error=result.plan_error,+ due_insights=result.due_insights,+ oldest_due_seconds=result.oldest_due_seconds,+ oldest_refresh_seconds=result.oldest_refresh_seconds,)ph_background_capture()(distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,@@ -493,6 +525,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:"teams_skipped": result.teams_skipped,"plan_failed": result.plan_failed,"plan_error": result.plan_error,+ "due_insights": result.due_insights,+ "oldest_due_seconds": result.oldest_due_seconds,+ "oldest_refresh_seconds": result.oldest_refresh_seconds,},)@@ -610,6 +645,9 @@ class AnomalyScoringTickWorkflow:plan_ms=plan_ms,teams_planned=plan.teams_planned,teams_skipped=plan.teams_skipped,+ due_insights=plan.due_insights,+ oldest_due_seconds=plan.oldest_due_seconds,+ oldest_refresh_seconds=plan.oldest_refresh_seconds,)await self._capture(result)return resultdiff --git a/products/signals/backend/migrations/0225_anomalyscoringdueinsight.py b/products/signals/backend/migrations/0225_anomalyscoringdueinsight.pynew file mode 100644index 000000000000..cedd352e33f2--- /dev/null+++ b/products/signals/backend/migrations/0225_anomalyscoringdueinsight.py@@ -0,0 +1,65 @@+# Generated by Django 6.1.2 on 2026-11-01 16:43++import django.db.models.manager+import django.db.models.deletion+from django.db import migrations, models++import posthog.uuidt+++class Migration(migrations.Migration):+ dependencies = [+ ("posthog", "1416_uploadedmedia_serving_disabled_at"),+ ("signals", "0224_signalscoutbackgroundband_audience"),+ ]++ operations = [+ migrations.CreateModel(+ name="AnomalyScoringDueInsight",+ fields=[+ (+ "id",+ models.UUIDField(+ default=posthog.uuidt.uuid7,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("insight_id", models.BigIntegerField()),+ ("interval", models.CharField(max_length=16)),+ ("next_due_at", models.DateTimeField()),+ ("refreshed_at", models.DateTimeField()),+ (+ "series_scored_through",+ models.JSONField(db_default={}, default=dict),+ ),+ ("viewers", models.PositiveIntegerField(default=0)),+ ("has_enabled_alert", models.BooleanField(default=False)),+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("updated_at", models.DateTimeField(auto_now=True)),+ (+ "team",+ models.ForeignKey(+ db_constraint=False,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="+",+ to="posthog.team",+ ),+ ),+ ],+ options={+ "default_manager_name": "all_teams",+ "indexes": [models.Index(fields=["next_due_at"], name="signals_anomaly_due_next_idx")],+ "constraints": [+ models.UniqueConstraint(+ fields=("team", "insight_id"),+ name="signals_anomaly_due_insight_unique",+ )+ ],+ },+ managers=[+ ("all_teams", django.db.models.manager.Manager()),+ ],+ ),+ ]diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txtindex d205921321cb..7a5ec262518e 100644--- a/products/signals/backend/migrations/max_migration.txt+++ b/products/signals/backend/migrations/max_migration.txt@@ -1 +1 @@-0224_signalscoutbackgroundband_audience+0225_anomalyscoringdueinsightdiff --git a/products/signals/backend/models.py b/products/signals/backend/models.pyindex 37e0bd0c3538..c72d0d82197c 100644--- a/products/signals/backend/models.py+++ b/products/signals/backend/models.py@@ -4384,3 +4384,38 @@ class AnomalyScoringState(TeamScopedRootMixin, UUIDModel):fields=["team", "insight_id", "profile"], name="signals_anomaly_scoring_state_unique"),]+++class AnomalyScoringDueInsight(TeamScopedRootMixin, UUIDModel):+ """One active insight on the stored due list of the anomaly scoring tick.++ A slow refresh job adds the active insights, removes the ones that left, and sets the usage. The tick+ reads the due rows, and each pick moves its row's `next_due_at`. See `anomaly_scores/due_list.py`.+ """++ # See SignalReportRefund.all_teams for rationale.+ all_teams = models.Manager() # noqa: DJ012++ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, db_constraint=False, related_name="+")+ # Not a foreign key: insights belong to another product.+ insight_id = models.BigIntegerField()+ interval = models.CharField(max_length=16)+ # The tick picks the row at or after this time. A never-scored insight is due at once.+ next_due_at = models.DateTimeField()+ refreshed_at = models.DateTimeField()+ # Series key to the newest scored bucket start of the series, as an ISO time in UTC.+ series_scored_through = models.JSONField(default=dict, db_default={})+ viewers = models.PositiveIntegerField(default=0)+ has_enabled_alert = models.BooleanField(default=False)++ created_at = models.DateTimeField(auto_now_add=True)+ updated_at = models.DateTimeField(auto_now=True)++ class Meta:+ default_manager_name = "all_teams"+ constraints = [+ models.UniqueConstraint(fields=["team", "insight_id"], name="signals_anomaly_due_insight_unique"),+ ]+ indexes = [+ models.Index(fields=["next_due_at"], name="signals_anomaly_due_next_idx"),+ ]diff --git a/products/signals/backend/temporal/__init__.py b/products/signals/backend/temporal/__init__.pyindex cef5192e596a..3bdaae305790 100644--- a/products/signals/backend/temporal/__init__.py+++ b/products/signals/backend/temporal/__init__.py@@ -4,6 +4,10 @@ from products.signals.backend.activity_summary.tick import (plan_activity_summary_tick_activity,update_activity_summary_pair_activity,)+from products.signals.backend.anomaly_scores.refresh import (+ AnomalyScoringDueListRefreshWorkflow,+ refresh_anomaly_scoring_due_list_activity,+)from products.signals.backend.anomaly_scores.tick import (AnomalyScoringTickWorkflow,capture_anomaly_scoring_tick_activity,@@ -244,7 +248,12 @@ ACTIVITIES = []# The temporal-worker-self-driving fleet runs these, not video-export.-SELF_DRIVING_WORKFLOWS = [InboxRankingScoringWorkflow, ActivitySummaryTickWorkflow, AnomalyScoringTickWorkflow]+SELF_DRIVING_WORKFLOWS = [+ InboxRankingScoringWorkflow,+ ActivitySummaryTickWorkflow,+ AnomalyScoringTickWorkflow,+ AnomalyScoringDueListRefreshWorkflow,+]SELF_DRIVING_ACTIVITIES = [score_inbox_reports_activity,plan_activity_summary_tick_activity,@@ -253,4 +262,5 @@ SELF_DRIVING_ACTIVITIES = [plan_anomaly_scoring_tick_activity,score_anomaly_insight_activity,capture_anomaly_scoring_tick_activity,+ refresh_anomaly_scoring_due_list_activity,]diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.pyindex 9b0043303ca7..0d3ffda6f7c4 100644--- a/products/signals/backend/test/test_anomaly_scoring_tick.py+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py@@ -35,6 +35,7 @@ from products.product_analytics.backend.facade.api import record_insight_viewfrom products.product_analytics.backend.facade.models import Insightfrom products.signals.backend.anomaly_scores.artifacts import clear_artifact_cachefrom products.signals.backend.anomaly_scores.config import (+ DEFAULT_LIMITS,DEFAULT_PROFILE,DISABLED,AnomalyScoringConfig,@@ -42,6 +43,7 @@ from products.signals.backend.anomaly_scores.config import (covers_team,parse_config,)+from products.signals.backend.anomaly_scores.due_list import record_scored, refresh_due_listfrom products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_eventsfrom products.signals.backend.anomaly_scores.queue import (SKIP_COOLDOWN,@@ -87,7 +89,12 @@ from products.signals.backend.anomaly_scores.tick import (plan_tick,run_insight,)-from products.signals.backend.models import AnomalyScoringState, SignalSourceConfig, SignalTeamConfig+from products.signals.backend.models import (+ AnomalyScoringDueInsight,+ AnomalyScoringState,+ SignalSourceConfig,+ SignalTeamConfig,+)from products.subscriptions.backend.facade.testing import create_subscriptionNEW_YORK = ZoneInfo("America/New_York")@@ -502,6 +509,7 @@ class TestRunInsight(SimpleTestCase):patch(f"{tick}.Team"),patch(f"{tick}.score_insight", side_effect=error),patch(f"{tick}.mark_attempted") as mark_attempted,+ patch(f"{tick}.record_failed") as record_failed,patch("products.signals.backend.anomaly_scores.metrics.posthoganalytics.default_client", client),):pick = InsightPick(team_id=1, insight_id=7, scored_through={}, interval=IntervalType.DAY)@@ -510,6 +518,9 @@ class TestRunInsight(SimpleTestCase):except Exception:status = "raised"cooldowns = [call.args[1] for call in mark_attempted.call_args_list]+ # The due list row waits as long as the cache cooldown, so a failed pick does not stay due.+ [row_cooldown] = [call.args[3] for call in record_failed.call_args_list]+ assert row_cooldown == cooldowns[-1][pick_event] = [call.kwargs for call in capture.call_args_list if call.kwargs["event"] == "anomaly scoring pick"]@@ -907,6 +918,115 @@ class TestActiveInsights(BaseTest):]+class TestDueList(ClickhouseTestMixin, BaseTest):+ def _config(self, *team_ids: int) -> AnomalyScoringConfig:+ return AnomalyScoringConfig(+ enabled=True,+ organization_ids=(str(self.organization.id),),+ allow_team_ids=team_ids or (self.team.id,),+ due_list=True,+ )++ def _row(self, insight: Insight, next_due_at: datetime, watermarks: dict[str, datetime]) -> None:+ AnomalyScoringDueInsight.all_teams.create(+ team_id=insight.team_id,+ insight_id=insight.id,+ interval="day",+ next_due_at=next_due_at,+ refreshed_at=NOW,+ series_scored_through={key: bucket.isoformat() for key, bucket in watermarks.items()},+ )++ @time_machine.travel(NOW, tick=False)+ def test_the_tick_picks_due_rows_only_and_sends_no_clickhouse_query(self) -> None:+ today = NOW.replace(hour=0)+ due = Insight.objects.create(team=self.team, name="due", query=_trends(interval="day"))+ waiting = Insight.objects.create(team=self.team, name="waiting", query=_trends(interval="day"))+ watermarks = {"a": today - timedelta(days=6), "b": today - timedelta(days=3)}+ self._row(due, NOW - timedelta(hours=1), watermarks)+ self._row(waiting, NOW + timedelta(hours=1), {})+ cache.delete_many(+ [+ _cooldown_key(due.id),+ _cooldown_key(waiting.id),+ _fleet_budget_key(NOW),+ _project_budget_key(self.team.id, NOW),+ ]+ )++ with (+ patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]),+ patch("products.signals.backend.anomaly_scores.queue.scored_buckets", side_effect=AssertionError),+ ):+ selection = select_insights(self._config(), SelectionConfig(weighting=Weighting.UNIFORM), NOW, limit=10)++ assert [(pick.insight_id, pick.scored_through) for pick in selection.picks] == [(due.id, watermarks)]+ assert (selection.active, selection.due_insights, selection.oldest_due_seconds) == (2, 1, 3600)++ @time_machine.travel(NOW, tick=False)+ def test_a_scored_pick_keeps_the_watermark_of_a_series_that_it_did_not_return(self) -> None:+ today = NOW.replace(hour=0)+ insight = Insight.objects.create(team=self.team, name="breakdown", query=_trends(interval="day"))+ self._row(insight, NOW, {"gone": today - timedelta(days=5), "kept": today - timedelta(days=2)})++ record_scored(self.team, insight.id, {"kept": today - timedelta(days=1)}, NOW, DEFAULT_LIMITS)++ row = AnomalyScoringDueInsight.all_teams.get(team_id=self.team.id, insight_id=insight.id)+ assert row.series_scored_through == {+ "gone": (today - timedelta(days=5)).isoformat(),+ "kept": (today - timedelta(days=1)).isoformat(),+ }+ # Today's bucket completes at midnight, and the insight is due in its slot after that.+ assert row.next_due_at >= today + timedelta(days=1)++ @time_machine.travel(NOW, tick=False)+ def test_a_failed_pick_moves_its_row_past_the_cooldown(self) -> None:+ insight = Insight.objects.create(team=self.team, name="slow", query=_trends(interval="day"))+ self._row(insight, NOW, {})+ tick = "products.signals.backend.anomaly_scores.tick"++ with (+ patch(f"{tick}.ph_background_capture"),+ patch(f"{tick}.score_insight", side_effect=ClickHouseQueryTimeOut()),+ ):+ run_insight(+ ScoreInsightInput(+ pick=InsightPick(+ team_id=self.team.id, insight_id=insight.id, scored_through={}, interval=IntervalType.DAY+ )+ ),+ NOW,+ )++ row = AnomalyScoringDueInsight.all_teams.get(team_id=self.team.id, insight_id=insight.id)+ assert row.next_due_at == NOW + SKIP_COOLDOWN++ @time_machine.travel(NOW, tick=False)+ def test_the_refresh_removes_inactive_insights_and_resumes_with_the_teams_that_waited(self) -> None:+ other_team = Team.objects.create(organization=self.organization, name="Other")+ active = {+ team.id: Insight.objects.create(team=team, name=team.name, query=_trends(interval="day"))+ for team in (self.team, other_team)+ }+ inactive = Insight.objects.create(team=self.team, name="inactive", query=_trends(interval="day"))+ Insight.objects.filter(id=inactive.id).update(last_modified_at=_LONG_AGO)+ self._row(inactive, NOW, {})+ cache.delete_many(+ [f"signals:anomaly_scores:due_list_refreshed:{team_id}" for team_id in (self.team.id, other_team.id)]+ )+ config = self._config(self.team.id, other_team.id)++ refreshes = []+ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):+ for later in (NOW, NOW + timedelta(minutes=15), NOW + timedelta(minutes=30)):+ refresh = refresh_due_list(config, later, deadline=0.0)+ refreshes.append((refresh.teams_refreshed, refresh.teams_waiting))++ assert refreshes == [(1, 1), (1, 0), (0, 0)]+ rows = AnomalyScoringDueInsight.all_teams.filter(team_id__in=list(active)).values_list("team_id", "insight_id")+ assert sorted(rows) == sorted((team_id, insight.id) for team_id, insight in active.items())++class TestScoredBuckets(ClickhouseTestMixin, BaseTest):def test_reads_the_newest_scored_bucket_of_each_series(self) -> None:def score(insight_id: int, series_key: str, bucket: datetime, source: str = "insight") -> None: