Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): plan anomaly scoring from a stored due list (#127031)

posthog[bot]ab580b44aa91
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yaml
index fd9519f2b38d..0d531456c39d 100644
--- a/.semgrep/rules/security/idor-team-scoped-models.yaml
+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml
@@ -428,6 +428,7 @@ rules:
|LegacySignalSpend
|AnomalyScoringState
+ |AnomalyScoringDueInsight
|SignalScratchpad
|SignalProjectProfile
|SingleSessionSummary
@@ -923,6 +924,7 @@ rules:
|LegacySignalSpend
|AnomalyScoringState
+ |AnomalyScoringDueInsight
|SignalScratchpad
|SignalProjectProfile
|SingleSessionSummary
diff --git a/posthog/temporal/schedule.py b/posthog/temporal/schedule.py
index 929d7d4c4e6c..9dee010db1e7 100644
--- a/posthog/temporal/schedule.py
+++ b/posthog/temporal/schedule.py
@@ -140,7 +140,10 @@ from products.replay_vision.backend.temporal.vision_alerts.schedule import creat
from products.review_hog.backend.facade.temporal import create_review_hog_finding_outcomes_schedule
from products.security.backend.facade.temporal import create_sync_access_rules_schedule
from products.signals.backend.activity_summary.schedule import create_activity_summary_tick_schedule
-from products.signals.backend.anomaly_scores.schedule import create_anomaly_scoring_tick_schedule
+from products.signals.backend.anomaly_scores.schedule import (
+ create_anomaly_scoring_due_list_refresh_schedule,
+ create_anomaly_scoring_tick_schedule,
+)
from products.signals.backend.emission.conversations_schedule import create_conversations_signals_coordinator_schedule
from products.signals.backend.ranking.schedule import create_inbox_ranking_scoring_schedule
from products.signals.backend.temporal.agentic.schedule import (
@@ -1021,6 +1024,7 @@ schedules = [
create_inbox_ranking_scoring_schedule,
create_activity_summary_tick_schedule,
create_anomaly_scoring_tick_schedule,
+ create_anomaly_scoring_due_list_refresh_schedule,
create_scout_suggestions_coordinator_schedule,
create_support_reply_coordinator_schedule,
create_channel_summary_coordinator_schedule,
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.py
index d91b06987e6e..9fe62d152f78 100644
--- a/products/signals/backend/anomaly_scores/config.py
+++ b/products/signals/backend/anomaly_scores/config.py
@@ -152,6 +152,10 @@ class AnomalyScoringConfig:
max_train_insights_per_tick: int = 2
# An insight is scored only while one of its activity signals falls in this many days, see `queue.active_insights`.
active_window_days: int = 90
+ # On, the tick plans from the stored due list, and a slow job refreshes the list. See `due_list.py`.
+ due_list: bool = False
+ # Each enrolled team's due list refreshes about this often.
+ due_list_refresh_hours: int = 6
DISABLED = AnomalyScoringConfig()
@@ -165,6 +169,7 @@ MAX_SCORE_QUERIES_PER_PROJECT_PER_HOUR = 1000
MAX_SCORE_INSIGHTS_PER_TICK = 200
MAX_TRAIN_INSIGHTS_PER_TICK = 50
MAX_ACTIVE_WINDOW_DAYS = 365
+MAX_DUE_LIST_REFRESH_HOURS = 48
def _positive_int(value: Any, default: int) -> int:
@@ -293,6 +298,11 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:
active_window_days=min(
_positive_int(payload.get("active_window_days"), defaults.active_window_days), MAX_ACTIVE_WINDOW_DAYS
),
+ due_list=payload.get("due_list") is True,
+ due_list_refresh_hours=min(
+ _positive_int(payload.get("due_list_refresh_hours"), defaults.due_list_refresh_hours),
+ MAX_DUE_LIST_REFRESH_HOURS,
+ ),
)
diff --git a/products/signals/backend/anomaly_scores/due_list.py b/products/signals/backend/anomaly_scores/due_list.py
new file mode 100644
index 000000000000..9d46563dbb29
--- /dev/null
+++ b/products/signals/backend/anomaly_scores/due_list.py
@@ -0,0 +1,238 @@
+"""The stored due list of the anomaly scoring tick, and the slow refresh that keeps it.
+
+With `due_list` on, the tick plans from the due rows and sends no scored state query to ClickHouse. The list
+can be stale. A pick of an insight that is already done waits for the attempt cooldown, and the backfill
+scores the buckets of an insight that the list shows as due late.
+
+The refresh adds the active insights of each enrolled team, removes the insights that left, and sets the
+usage. Each pick then moves its own row, so the list stays close to right between two refreshes.
+"""
+
+import time
+import datetime
+from collections.abc import Collection
+
+from django.core.cache import cache
+
+import structlog
+
+from posthog.schema import IntervalType
+
+from posthog.dataclasses import frozen
+from posthog.interval_specs import interval_spec
+from posthog.models import Team
+
+from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig, ScoringLimits
+from products.signals.backend.anomaly_scores.queue import (
+ ATTEMPT_COOLDOWN,
+ ActiveInsight,
+ TeamCensus,
+ active_insights,
+ due_at,
+ eligible_teams,
+ parse_watermarks,
+ reason_counts,
+ scored_buckets,
+ scored_state_since,
+ team_census,
+)
+from products.signals.backend.anomaly_scores.selection import insight_usage
+from products.signals.backend.models import AnomalyScoringDueInsight, AnomalyScoringState
+
+logger = structlog.get_logger(__name__)
+
+# The last refresh of a team. It lives past the longest refresh window, so a team waits for its turn.
+_TEAM_REFRESHED_TTL = datetime.timedelta(days=3)
+
+
+@frozen
+class TeamRefresh:
+ census: TeamCensus
+ added: int
+ removed: int
+
+
+@frozen
+class DueListRefresh:
+ enabled: bool = False
+ teams_refreshed: int = 0
+ teams_failed: int = 0
+ # Teams past their refresh window that wait for a later run.
+ teams_waiting: int = 0
+ rows_added: int = 0
+ rows_removed: int = 0
+ # Active insights of the refreshed teams, and how many of them have each activity reason.
+ active: int = 0
+ active_reasons: dict[str, int] | None = None
+ census: list[TeamCensus] | None = None
+
+
+def _team_refreshed_key(team_id: int) -> str:
+ return f"signals:anomaly_scores:due_list_refreshed:{team_id}"
+
+
+def dump_watermarks(watermarks: dict[str, datetime.datetime]) -> dict[str, str]:
+ return {key: bucket.astimezone(datetime.UTC).isoformat() for key, bucket in watermarks.items()}
+
+
+def seed_watermarks(
+ config: AnomalyScoringConfig, team: Team, insights: Collection[ActiveInsight], now: datetime.datetime
+) -> dict[int, dict[str, datetime.datetime]]:
+ """The scored state of insights new to the list.
+
+ A stored model gives each of its series the newest scored bucket of the model, so the switch causes no
+ retrain. Only insights with no stored model read their state from ClickHouse, once, when they join the list.
+ """
+ if not insights:
+ return {}
+ seeded: dict[int, dict[str, datetime.datetime]] = {}
+ rows = (
+ AnomalyScoringState.objects.for_team(team.id)
+ .filter(insight_id__in=[insight.insight_id for insight in insights], scored_through__isnull=False)
+ .values_list("insight_id", "scored_through", "series_keys")
+ )
+ for insight_id, scored_through, series_keys in rows:
+ if scored_through is None or not isinstance(series_keys, list):
+ continue
+ watermarks = seeded.setdefault(insight_id, {})
+ for series_key in series_keys:
+ watermarks[str(series_key)] = max(watermarks.get(str(series_key), scored_through), scored_through)
+ windows: dict[datetime.datetime, set[int]] = {}
+ for insight in insights:
+ if not seeded.get(insight.insight_id):
+ windows.setdefault(scored_state_since(config, insight.interval, now), set()).add(insight.insight_id)
+ if windows:
+ seeded.update(scored_buckets(team, windows, now))
+ return seeded
+
+
+def refresh_team(config: AnomalyScoringConfig, team: Team, now: datetime.datetime) -> TeamRefresh:
+ insights = active_insights([team.id], config.intervals, now, config.active_window_days)
+ by_id = {insight.insight_id: insight for insight in insights}
+ scope = AnomalyScoringDueInsight.all_teams.filter(team_id=team.id)
+ removed, _ = scope.exclude(insight_id__in=list(by_id)).delete()
+ existing = {row.insight_id: row for row in scope.filter(insight_id__in=list(by_id))}
+ seeded = seed_watermarks(config, team, [insight for insight in insights if insight.insight_id not in existing], now)
+ usage = insight_usage(team_id=team.id, insight_ids=list(by_id), now=now)
+
+ created: list[AnomalyScoringDueInsight] = []
+ updated: list[AnomalyScoringDueInsight] = []
+ for insight in insights:
+ row = existing.get(insight.insight_id)
+ watermarks = parse_watermarks(row.series_scored_through) if row else seeded.get(insight.insight_id, {})
+ next_due_at = (
+ due_at(insight.insight_id, insight.interval, max(watermarks.values(), default=None), team.timezone_info)
+ or now
+ )
+ if row is not None and row.interval == insight.interval.value:
+ # A failed pick moved the row past its cooldown.
+ next_due_at = max(next_due_at, row.next_due_at)
+ fields = {
+ "interval": insight.interval.value,
+ "next_due_at": next_due_at,
+ "refreshed_at": now,
+ "viewers": usage[insight.insight_id].viewers,
+ "has_enabled_alert": usage[insight.insight_id].has_enabled_alert,
+ }
+ if row is None:
+ created.append(
+ AnomalyScoringDueInsight(
+ team_id=team.id,
+ insight_id=insight.insight_id,
+ series_scored_through=dump_watermarks(watermarks),
+ **fields,
+ )
+ )
+ continue
+ for name, value in fields.items():
+ setattr(row, name, value)
+ row.updated_at = now
+ updated.append(row)
+ # A pick writes the watermarks of its own row, so the refresh never writes them for an existing row.
+ AnomalyScoringDueInsight.all_teams.bulk_create(created, ignore_conflicts=True)
+ AnomalyScoringDueInsight.all_teams.bulk_update(
+ updated, ["interval", "next_due_at", "refreshed_at", "viewers", "has_enabled_alert", "updated_at"]
+ )
+ [census] = team_census([team.id], insights)
+ return TeamRefresh(census=census, added=len(created), removed=removed)
+
+
+def refresh_due_list(config: AnomalyScoringConfig, now: datetime.datetime, deadline: float) -> DueListRefresh:
+ """Refresh the teams past their refresh window, the longest waiting first, until ``deadline``.
+
+ ``deadline`` is a ``time.monotonic`` value. At least one team refreshes per run. A failed team also goes to
+ the back, so a team that always fails does not block the others.
+ """
+ if not config.enabled or not config.due_list:
+ return DueListRefresh()
+ teams = {team.id: team for team in eligible_teams(config)}
+ stamps = cache.get_many([_team_refreshed_key(team_id) for team_id in teams])
+ window_start = (now - datetime.timedelta(hours=config.due_list_refresh_hours)).timestamp()
+ waiting = sorted(
+ (team_id for team_id in teams if stamps.get(_team_refreshed_key(team_id), 0.0) <= window_start),
+ key=lambda team_id: (stamps.get(_team_refreshed_key(team_id), 0.0), team_id),
+ )
+ census: list[TeamCensus] = []
+ attempted = failed = added = removed = 0
+ for team_id in waiting:
+ if attempted and time.monotonic() >= deadline:
+ break
+ attempted += 1
+ try:
+ refresh = refresh_team(config, teams[team_id], now)
+ except Exception as error:
+ failed += 1
+ logger.warning(
+ "anomaly_scoring_due_list_refresh_failed",
+ team_id=team_id,
+ error_class=type(error).__name__,
+ exc_info=True,
+ )
+ else:
+ census.append(refresh.census)
+ added += refresh.added
+ removed += refresh.removed
+ cache.set(_team_refreshed_key(team_id), now.timestamp(), timeout=int(_TEAM_REFRESHED_TTL.total_seconds()))
+ return DueListRefresh(
+ enabled=True,
+ teams_refreshed=attempted - failed,
+ teams_failed=failed,
+ teams_waiting=len(waiting) - attempted,
+ rows_added=added,
+ rows_removed=removed,
+ active=sum(team.active for team in census),
+ active_reasons={reason: sum(team.active_reasons[reason] for team in census) for reason in reason_counts([])},
+ census=census,
+ )
+
+
+def record_scored(
+ team: Team,
+ insight_id: int,
+ watermarks: dict[str, datetime.datetime],
+ now: datetime.datetime,
+ limits: ScoringLimits,
+) -> None:
+ """Merge the newest scored bucket of each series into the insight's row, and set when it is due next."""
+ row = AnomalyScoringDueInsight.all_teams.filter(team_id=team.id, insight_id=insight_id).first()
+ if row is None:
+ return
+ interval = IntervalType(row.interval)
+ merged = parse_watermarks(row.series_scored_through)
+ for series_key, bucket in watermarks.items():
+ merged[series_key] = max(merged.get(series_key, bucket), bucket)
+ # A series older than the backfill cap scores as a new series, so its entry has no effect.
+ since = now - interval_spec(interval).period * (limits.max_backfill_buckets + 1)
+ kept = {series_key: bucket for series_key, bucket in merged.items() if bucket >= since}
+ due = due_at(insight_id, interval, max(kept.values(), default=None), team.timezone_info)
+ # A visit with no new bucket must not make the row due again before the cooldown ends.
+ row.next_due_at = max(due or now, now + ATTEMPT_COOLDOWN)
+ row.series_scored_through = dump_watermarks(kept)
+ row.save(update_fields=["next_due_at", "series_scored_through", "updated_at"])
+
+
+def record_failed(team_id: int, insight_id: int, now: datetime.datetime, cooldown: datetime.timedelta) -> None:
+ """Move the insight's row past the cooldown of a failed pick."""
+ AnomalyScoringDueInsight.all_teams.filter(team_id=team_id, insight_id=insight_id).update(
+ next_due_at=now + cooldown, updated_at=now
+ )
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index ebcd7ca654d2..32049c1f029c 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -1,7 +1,8 @@
"""Pick the active insights that the next anomaly scoring tick scores.
One query per team reads the newest scored bucket of each series back from the `$anomaly_score`
-events. `selection.pick_order` then orders the due insights, and the tick picks from that order the
+events. With `due_list` on, the plan reads the stored due list of `due_list.py` instead, and sends no
+ClickHouse query. `selection.pick_order` then orders the due insights, and the tick picks from that order the
insights whose queries fit the hourly budgets. With the train and score split on, `state.plan_run`
makes each pick a train step or a score step, and each kind has its own budgets.
@@ -22,6 +23,7 @@ from enum import StrEnum
from typing import Any, cast
from django.core.cache import cache
+from django.db.models import Count, Min
import structlog
@@ -66,7 +68,7 @@ from products.signals.backend.anomaly_scores.selection import (
score_order,
)
from products.signals.backend.anomaly_scores.state import RunPlan, load_states, plan_run, score_ready_by_insight
-from products.signals.backend.models import SignalTeamConfig
+from products.signals.backend.models import AnomalyScoringDueInsight, SignalTeamConfig
from products.subscriptions.backend.facade.api import insight_ids_with_active_subscriptions
from ee.billing.quota_limiting import QuotaLimitingCaches, QuotaResource, list_limited_team_attributes
@@ -105,6 +107,8 @@ _MAX_AT_CAPACITY_TEAMS = 2
# The last scored state load of a team. The team that waited longest loads first, so a plan that stops early
# does not drop the same teams on every tick.
_TEAM_LOADED_TTL = datetime.timedelta(days=1)
+# The most due rows that one plan reads. Far more than the picks of one tick.
+_MAX_DUE_ROWS = 5000
# The scorer rejects these, so selection leaves them out instead of picking them on every tick.
_UNSCORED_DISPLAYS = sorted(
display.value for display in {*NON_TIME_SERIES_DISPLAY_TYPES, ChartDisplayType.ACTIONS_LINE_GRAPH_CUMULATIVE}
@@ -194,6 +198,24 @@ class Selection:
# Eligible teams, and their organizations.
enrolled_teams: int = 0
enrolled_organizations: int = 0
+ # Due rows of the stored due list, the time since the oldest of them became due, and the time since the
+ # oldest refresh of a row. None when the tick does not plan from the due list.
+ due_insights: int | None = None
+ oldest_due_seconds: int | None = None
+ oldest_refresh_seconds: int | None = None
+
+
+@frozen
+class DueList:
+ """The due rows of the stored due list, for the eligible teams."""
+
+ insights: list[ActiveInsight]
+ scored_through: dict[int, dict[str, datetime.datetime]]
+ usage: dict[int, InsightUsage]
+ # Rows of the eligible teams, due or not.
+ rows: int
+ oldest_due_seconds: int | None
+ oldest_refresh_seconds: int | None
@frozen
@@ -366,17 +388,64 @@ def scored_buckets(
return state
+def due_at(
+ insight_id: int, interval: IntervalType, newest_bucket: datetime.datetime | None, timezone: datetime.tzinfo
+) -> datetime.datetime | None:
+ """When the bucket after the newest scored one is complete and this insight's slot after its end starts.
+
+ None when the insight was never scored, because it is due at once.
+ """
+ if newest_bucket is None:
+ return None
+ next_start = bucket_end(newest_bucket.astimezone(timezone), interval)
+ next_end = bucket_end(next_start, interval)
+ # The same offset as alert checks, so insights spread over the window after each boundary.
+ return next_end + alert_check_offset(_CALENDAR_INTERVALS[interval], f"anomaly:{insight_id}")
+
+
def is_due(
insight: ActiveInsight, newest_bucket: datetime.datetime | None, timezone: datetime.tzinfo, now: datetime.datetime
) -> bool:
- """The bucket after the newest scored one is complete, and this insight's slot after its end has passed."""
- if newest_bucket is None:
- return True
- next_start = bucket_end(newest_bucket.astimezone(timezone), insight.interval)
- next_end = bucket_end(next_start, insight.interval)
- # The same offset as alert checks, so insights spread over the window after each boundary.
- offset = alert_check_offset(_CALENDAR_INTERVALS[insight.interval], f"anomaly:{insight.insight_id}")
- return now >= next_end + offset
+ due = due_at(insight.insight_id, insight.interval, newest_bucket, timezone)
+ return due is None or now >= due
+
+
+def parse_watermarks(value: Any) -> dict[str, datetime.datetime]:
+ """The stored map of series key to newest scored bucket. An entry that does not parse is left out."""
+ if not isinstance(value, dict):
+ return {}
+ watermarks: dict[str, datetime.datetime] = {}
+ for series_key, stamp in value.items():
+ try:
+ watermarks[str(series_key)] = _utc(datetime.datetime.fromisoformat(stamp))
+ except (TypeError, ValueError):
+ continue
+ return watermarks
+
+
+def load_due_list(team_ids: Collection[int], intervals: tuple[IntervalType, ...], now: datetime.datetime) -> DueList:
+ """The due rows of the teams, longest overdue first, in a fixed number of queries."""
+ scope = AnomalyScoringDueInsight.all_teams.filter(team_id__in=list(team_ids))
+ summary = scope.aggregate(rows=Count("id"), oldest_refresh=Min("refreshed_at"))
+ rows = list(
+ scope.filter(next_due_at__lte=now, interval__in=[interval.value for interval in intervals]).order_by(
+ "next_due_at", "insight_id"
+ )[:_MAX_DUE_ROWS]
+ )
+ oldest_refresh = summary["oldest_refresh"]
+ return DueList(
+ insights=[
+ ActiveInsight(team_id=row.team_id, insight_id=row.insight_id, interval=IntervalType(row.interval))
+ for row in rows
+ ],
+ scored_through={row.insight_id: parse_watermarks(row.series_scored_through) for row in rows},
+ usage={
+ row.insight_id: InsightUsage(viewers=row.viewers, has_enabled_alert=row.has_enabled_alert) for row in rows
+ },
+ rows=summary["rows"],
+ oldest_due_seconds=round((now - rows[0].next_due_at).total_seconds()) if rows else None,
+ oldest_refresh_seconds=None if oldest_refresh is None else round((now - oldest_refresh).total_seconds()),
+ )
def _cooldown_key(insight_id: int) -> str:
@@ -610,16 +679,26 @@ def select_insights(
"""The picks of the tick. ``limit`` replaces the pick cap of each pass, for a manual run.
``deadline`` is a ``time.monotonic`` value. After it, no more teams load, and the plan uses the teams it has.
+ With the due list on, the plan reads the stored due list instead, and ``deadline`` has no effect.
"""
teams = {team.id: team for team in eligible_teams(config)}
- insights = active_insights(list(teams), config.intervals, now, config.active_window_days)
- active = len(insights)
- active_reasons = reason_counts(insights)
- census = team_census(teams, insights)
- teams_with_insights = {insight.team_id for insight in insights}
- state, loaded_teams = load_scored_state(config, teams, insights, now, deadline)
- # Without its scored state, every insight of the team looks never scored, so the team waits for a later tick.
- insights = [insight for insight in insights if insight.team_id in loaded_teams]
+ due_list: DueList | None = None
+ if config.due_list:
+ due_list = load_due_list(teams, config.intervals, now)
+ insights = due_list.insights
+ state = due_list.scored_through
+ # The refresh job sends the activity reasons and the census.
+ active, active_reasons, census = due_list.rows, {}, []
+ loaded_teams = teams_with_insights = {insight.team_id for insight in insights}
+ else:
+ insights = active_insights(list(teams), config.intervals, now, config.active_window_days)
+ active = len(insights)
+ active_reasons = reason_counts(insights)
+ census = team_census(teams, insights)
+ teams_with_insights = {insight.team_id for insight in insights}
+ state, loaded_teams = load_scored_state(config, teams, insights, now, deadline)
+ # Without its scored state, every insight of the team looks never scored, so the team waits for a later tick.
+ insights = [insight for insight in insights if insight.team_id in loaded_teams]
newest_buckets = {
insight.insight_id: max(state[insight.insight_id].values()) if insight.insight_id in state else None
@@ -686,9 +765,12 @@ def select_insights(
and is_due(insight, newest_buckets[insight_id], teams[insight.team_id].timezone_info, now)
}
usage: dict[int, InsightUsage] = {}
- for team_id in sorted({insight.team_id for insight in due_by_id.values()}):
- due_ids = [insight.insight_id for insight in due_by_id.values() if insight.team_id == team_id]
- usage.update(insight_usage(team_id=team_id, insight_ids=due_ids, now=now))
+ if due_list is not None:
+ usage = due_list.usage
+ else:
+ for team_id in sorted({insight.team_id for insight in due_by_id.values()}):
+ due_ids = [insight.insight_id for insight in due_by_id.values() if insight.team_id == team_id]
+ usage.update(insight_usage(team_id=team_id, insight_ids=due_ids, now=now))
candidates_by_team: dict[int, list[Candidate]] = {}
for insight in due_by_id.values():
newest = newest_buckets[insight.insight_id]
@@ -764,6 +846,9 @@ def select_insights(
teams_skipped=len(teams_with_insights) - len(loaded_teams),
enrolled_teams=len(teams),
enrolled_organizations=len({team.organization_id for team in teams.values()}),
+ due_insights=None if due_list is None else len(due_list.insights),
+ oldest_due_seconds=None if due_list is None else due_list.oldest_due_seconds,
+ oldest_refresh_seconds=None if due_list is None else due_list.oldest_refresh_seconds,
)
diff --git a/products/signals/backend/anomaly_scores/refresh.py b/products/signals/backend/anomaly_scores/refresh.py
new file mode 100644
index 000000000000..6bdd2285af54
--- /dev/null
+++ b/products/signals/backend/anomaly_scores/refresh.py
@@ -0,0 +1,100 @@
+"""The scheduled refresh of the anomaly scoring due list: a Temporal workflow apart from the tick.
+
+The `due_list` field of the `anomaly-scoring` flag payload gates the work. See `due_list.py`.
+"""
+
+import time
+import datetime
+
+from django.conf import settings
+from django.utils import timezone
+
+from temporalio import activity, workflow
+from temporalio.common import RetryPolicy
+
+from posthog.dataclasses import frozen
+from posthog.ph_client import ph_background_capture
+from posthog.sync import database_sync_to_async
+from posthog.temporal.common.scoped import scoped_temporal
+from posthog.temporal.common.utils import close_db_connections
+
+from products.signals.backend.anomaly_scores.config import ANOMALY_SCORING_FLAG_DISTINCT_ID, read_config, read_payload
+from products.signals.backend.anomaly_scores.due_list import refresh_due_list
+from products.signals.backend.anomaly_scores.tick import capture_census
+
+REFRESH_WORKFLOW_NAME = "anomaly-scoring-due-list-refresh"
+# No team starts to refresh after this time. The teams that are left go first on the next run.
+_REFRESH_BUDGET = datetime.timedelta(minutes=5)
+# One team can still run after the budget, and its census and summary captures run after it.
+_REFRESH_ACTIVITY_TIMEOUT = datetime.timedelta(minutes=10)
+REFRESH_TIMEOUT = _REFRESH_ACTIVITY_TIMEOUT * 2
+
+
+@frozen
+class DueListRefreshInput:
+ pass
+
+
+@frozen
+class DueListRefreshResult:
+ enabled: bool = False
+ teams_refreshed: int = 0
+ teams_failed: int = 0
+ teams_waiting: int = 0
+ rows_added: int = 0
+ rows_removed: int = 0
+ duration_ms: int = 0
+
+
+def run_refresh(now: datetime.datetime) -> DueListRefreshResult:
+ started = time.monotonic()
+ config = read_config(read_payload())
+ refresh = refresh_due_list(config, now, deadline=started + _REFRESH_BUDGET.total_seconds())
+ if not refresh.enabled:
+ return DueListRefreshResult()
+ capture_census(refresh.census or [], now)
+ result = DueListRefreshResult(
+ enabled=True,
+ teams_refreshed=refresh.teams_refreshed,
+ teams_failed=refresh.teams_failed,
+ teams_waiting=refresh.teams_waiting,
+ rows_added=refresh.rows_added,
+ rows_removed=refresh.rows_removed,
+ duration_ms=round((time.monotonic() - started) * 1000),
+ )
+ ph_background_capture()(
+ distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,
+ event="anomaly scoring due list refresh",
+ properties={
+ "deployment": settings.CLOUD_DEPLOYMENT,
+ "teams_refreshed": result.teams_refreshed,
+ "teams_failed": result.teams_failed,
+ "teams_waiting": result.teams_waiting,
+ "rows_added": result.rows_added,
+ "rows_removed": result.rows_removed,
+ "duration_ms": result.duration_ms,
+ "active_insights": refresh.active,
+ **{f"active_{reason}": count for reason, count in (refresh.active_reasons or {}).items()},
+ },
+ )
+ return result
+
+
+@scoped_temporal()
+@close_db_connections
+async def refresh_anomaly_scoring_due_list_activity(input: DueListRefreshInput) -> DueListRefreshResult:
+ return await database_sync_to_async(run_refresh, thread_sensitive=False)(timezone.now())
+
+
[email protected](name=REFRESH_WORKFLOW_NAME)
+class AnomalyScoringDueListRefreshWorkflow:
+ @workflow.run
+ async def run(self, inputs: DueListRefreshInput) -> DueListRefreshResult:
+ return await workflow.execute_activity(
+ refresh_anomaly_scoring_due_list_activity,
+ inputs,
+ start_to_close_timeout=_REFRESH_ACTIVITY_TIMEOUT,
+ # A failed run leaves its teams waiting, and the next run starts with them.
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
diff --git a/products/signals/backend/anomaly_scores/schedule.py b/products/signals/backend/anomaly_scores/schedule.py
index a613d140936d..2fc6fa809c18 100644
--- a/products/signals/backend/anomaly_scores/schedule.py
+++ b/products/signals/backend/anomaly_scores/schedule.py
@@ -1,4 +1,4 @@
-"""Schedule registration for the anomaly scoring tick."""
+"""Schedule registration for the anomaly scoring tick and the due list refresh."""
from dataclasses import asdict
from datetime import timedelta
@@ -18,10 +18,24 @@ from temporalio.client import (
from posthog.scheduling.jitter import deterministic_offset
from posthog.temporal.common.schedule import a_create_schedule, a_schedule_exists, a_update_schedule
+from products.signals.backend.anomaly_scores.refresh import REFRESH_TIMEOUT, REFRESH_WORKFLOW_NAME, DueListRefreshInput
from products.signals.backend.anomaly_scores.tick import TICK_TIMEOUT, TICK_WORKFLOW_NAME, AnomalyScoringTickInput
ANOMALY_SCORING_TICK_SCHEDULE_ID = "anomaly-scoring-tick-schedule"
ANOMALY_SCORING_TICK_INTERVAL = timedelta(minutes=5)
+ANOMALY_SCORING_DUE_LIST_REFRESH_SCHEDULE_ID = "anomaly-scoring-due-list-refresh-schedule"
+# Each run refreshes only the teams past their refresh window, so most runs end at once.
+ANOMALY_SCORING_DUE_LIST_REFRESH_INTERVAL = timedelta(minutes=15)
+
+
+async def _create_or_update(client: Client, schedule_id: str, schedule: Schedule) -> None:
+ if await a_schedule_exists(client, schedule_id):
+ # Keep the live state, so a deploy does not resume a schedule that an operator paused.
+ description = await client.get_schedule_handle(schedule_id).describe()
+ schedule.state = description.schedule.state
+ await a_update_schedule(client, schedule_id, schedule)
+ else:
+ await a_create_schedule(client, schedule_id, schedule, trigger_immediately=False)
async def create_anomaly_scoring_tick_schedule(client: Client) -> None:
@@ -49,10 +63,33 @@ async def create_anomaly_scoring_tick_schedule(client: Client) -> None:
),
policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.SKIP),
)
- if await a_schedule_exists(client, ANOMALY_SCORING_TICK_SCHEDULE_ID):
- # Keep the live state, so a deploy does not resume a schedule that an operator paused.
- description = await client.get_schedule_handle(ANOMALY_SCORING_TICK_SCHEDULE_ID).describe()
- schedule.state = description.schedule.state
- await a_update_schedule(client, ANOMALY_SCORING_TICK_SCHEDULE_ID, schedule)
- else:
- await a_create_schedule(client, ANOMALY_SCORING_TICK_SCHEDULE_ID, schedule, trigger_immediately=False)
+ await _create_or_update(client, ANOMALY_SCORING_TICK_SCHEDULE_ID, schedule)
+
+
+async def create_anomaly_scoring_due_list_refresh_schedule(client: Client) -> None:
+ """Create or update the due list refresh schedule on the self-driving task queue.
+
+ The `due_list` field of the `anomaly-scoring` flag payload gates the work. SKIP on overlap: two runs
+ must not refresh the same team at the same time.
+ """
+ schedule = Schedule(
+ action=ScheduleActionStartWorkflow(
+ REFRESH_WORKFLOW_NAME,
+ asdict(DueListRefreshInput()),
+ id=ANOMALY_SCORING_DUE_LIST_REFRESH_SCHEDULE_ID,
+ task_queue=settings.SELF_DRIVING_TASK_QUEUE,
+ execution_timeout=REFRESH_TIMEOUT,
+ ),
+ spec=ScheduleSpec(
+ intervals=[
+ ScheduleIntervalSpec(
+ every=ANOMALY_SCORING_DUE_LIST_REFRESH_INTERVAL,
+ offset=deterministic_offset(
+ ANOMALY_SCORING_DUE_LIST_REFRESH_SCHEDULE_ID, ANOMALY_SCORING_DUE_LIST_REFRESH_INTERVAL
+ ),
+ )
+ ]
+ ),
+ policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.SKIP),
+ )
+ await _create_or_update(client, ANOMALY_SCORING_DUE_LIST_REFRESH_SCHEDULE_ID, schedule)
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index d54f46600181..9d9857c9b9c5 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -47,6 +47,7 @@ from products.signals.backend.anomaly_scores.config import (
read_payload,
resolve_profile,
)
+from products.signals.backend.anomaly_scores.due_list import record_failed, record_scored
from products.signals.backend.anomaly_scores.emission import emit, score_events
from products.signals.backend.anomaly_scores.metrics import record_candidates, record_pick
from products.signals.backend.anomaly_scores.queue import (
@@ -130,6 +131,9 @@ class AnomalyScoringTickPlan:
enrolled_organizations: int = 0
teams_planned: int = 0
teams_skipped: int = 0
+ due_insights: int | None = None
+ oldest_due_seconds: int | None = None
+ oldest_refresh_seconds: int | None = None
@frozen
@@ -188,6 +192,11 @@ class AnomalyScoringTickResult:
# The plan activity failed, and the tick scored nothing. `plan_error` is the error class.
plan_failed: bool = False
plan_error: str | None = None
+ # Due rows that the plan read from the stored due list, the time since the oldest of them became due, and
+ # the time since the oldest row refresh. None when the plan does not read the due list.
+ due_insights: int | None = None
+ oldest_due_seconds: int | None = None
+ oldest_refresh_seconds: int | None = None
def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:
@@ -199,6 +208,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
deadline = time.monotonic() + _PLAN_LOAD_BUDGET.total_seconds()
selection = select_insights(config, parse_selection_config(payload), now, limit, deadline=deadline)
record_candidates(selection.candidates)
+ # With the due list on, the census is empty, and the refresh job sends it.
capture_census(selection.census, now)
return AnomalyScoringTickPlan(
enabled=True,
@@ -218,6 +228,9 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
enrolled_organizations=selection.enrolled_organizations,
teams_planned=selection.teams_planned,
teams_skipped=selection.teams_skipped,
+ due_insights=selection.due_insights,
+ oldest_due_seconds=selection.oldest_due_seconds,
+ oldest_refresh_seconds=selection.oldest_refresh_seconds,
)
@@ -275,10 +288,12 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
)
record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0, run=pick.run)
capture_pick(pick, status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, run=pick.run)
+ # A full cluster is not a fault of the insight, so it keeps the attempt cooldown.
+ cooldown = SKIP_COOLDOWN if status not in (None, STATUS_AT_CAPACITY) else ATTEMPT_COOLDOWN
+ record_failed(pick.team_id, pick.insight_id, now, cooldown)
if status is None:
raise
- # A full cluster is not a fault of the insight, so it keeps the attempt cooldown.
- if status != STATUS_AT_CAPACITY:
+ if cooldown == SKIP_COOLDOWN:
mark_attempted(pick.insight_id, SKIP_COOLDOWN)
return ScoreInsightResult(status=status, events=0)
duration_ms = round((time.monotonic() - started) * 1000)
@@ -376,6 +391,8 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
None,
)
if insight is None:
+ # The next refresh removes the row of a deleted insight.
+ record_failed(pick.team_id, pick.insight_id, now, SKIP_COOLDOWN)
return ScoreInsightResult(status=STATUS_MISSING, events=0)
team = Team.objects.get(id=pick.team_id)
# Set before scoring, so a failure also waits for the cooldown instead of taking a slot on every tick.
@@ -396,6 +413,7 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
error=str(error)[:200],
)
invalidate_state(team, insight.id, profile)
+ record_failed(team.id, insight.id, now, ATTEMPT_COOLDOWN)
return ScoreInsightResult(status=STATUS_ARTIFACT_UNAVAILABLE, events=0, run=ScoringRun.SCORE)
scores = score_insight(
insight,
@@ -422,6 +440,17 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
# A state write before a failed emit would skip those buckets for good.
if scores.state is not None:
save_state(team, insight.id, profile, scores.state)
+ record_scored(
+ team,
+ insight.id,
+ {
+ series.series_key: max(point.bucket_start for point in series.points)
+ for series in scores.series
+ if series.points
+ },
+ now,
+ input.limits,
+ )
return ScoreInsightResult(
status=STATUS_SCORED if events else STATUS_UP_TO_DATE,
events=len(events),
@@ -461,6 +490,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
teams_skipped=result.teams_skipped,
plan_failed=result.plan_failed,
plan_error=result.plan_error,
+ due_insights=result.due_insights,
+ oldest_due_seconds=result.oldest_due_seconds,
+ oldest_refresh_seconds=result.oldest_refresh_seconds,
)
ph_background_capture()(
distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,
@@ -493,6 +525,9 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
"teams_skipped": result.teams_skipped,
"plan_failed": result.plan_failed,
"plan_error": result.plan_error,
+ "due_insights": result.due_insights,
+ "oldest_due_seconds": result.oldest_due_seconds,
+ "oldest_refresh_seconds": result.oldest_refresh_seconds,
},
)
@@ -610,6 +645,9 @@ class AnomalyScoringTickWorkflow:
plan_ms=plan_ms,
teams_planned=plan.teams_planned,
teams_skipped=plan.teams_skipped,
+ due_insights=plan.due_insights,
+ oldest_due_seconds=plan.oldest_due_seconds,
+ oldest_refresh_seconds=plan.oldest_refresh_seconds,
)
await self._capture(result)
return result
diff --git a/products/signals/backend/migrations/0225_anomalyscoringdueinsight.py b/products/signals/backend/migrations/0225_anomalyscoringdueinsight.py
new file mode 100644
index 000000000000..cedd352e33f2
--- /dev/null
+++ b/products/signals/backend/migrations/0225_anomalyscoringdueinsight.py
@@ -0,0 +1,65 @@
+# Generated by Django 6.1.2 on 2026-11-01 16:43
+
+import django.db.models.manager
+import django.db.models.deletion
+from django.db import migrations, models
+
+import posthog.uuidt
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("posthog", "1416_uploadedmedia_serving_disabled_at"),
+ ("signals", "0224_signalscoutbackgroundband_audience"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="AnomalyScoringDueInsight",
+ fields=[
+ (
+ "id",
+ models.UUIDField(
+ default=posthog.uuidt.uuid7,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ ("insight_id", models.BigIntegerField()),
+ ("interval", models.CharField(max_length=16)),
+ ("next_due_at", models.DateTimeField()),
+ ("refreshed_at", models.DateTimeField()),
+ (
+ "series_scored_through",
+ models.JSONField(db_default={}, default=dict),
+ ),
+ ("viewers", models.PositiveIntegerField(default=0)),
+ ("has_enabled_alert", models.BooleanField(default=False)),
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("updated_at", models.DateTimeField(auto_now=True)),
+ (
+ "team",
+ models.ForeignKey(
+ db_constraint=False,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="+",
+ to="posthog.team",
+ ),
+ ),
+ ],
+ options={
+ "default_manager_name": "all_teams",
+ "indexes": [models.Index(fields=["next_due_at"], name="signals_anomaly_due_next_idx")],
+ "constraints": [
+ models.UniqueConstraint(
+ fields=("team", "insight_id"),
+ name="signals_anomaly_due_insight_unique",
+ )
+ ],
+ },
+ managers=[
+ ("all_teams", django.db.models.manager.Manager()),
+ ],
+ ),
+ ]
diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txt
index d205921321cb..7a5ec262518e 100644
--- a/products/signals/backend/migrations/max_migration.txt
+++ b/products/signals/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0224_signalscoutbackgroundband_audience
+0225_anomalyscoringdueinsight
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index 37e0bd0c3538..c72d0d82197c 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -4384,3 +4384,38 @@ class AnomalyScoringState(TeamScopedRootMixin, UUIDModel):
fields=["team", "insight_id", "profile"], name="signals_anomaly_scoring_state_unique"
),
]
+
+
+class AnomalyScoringDueInsight(TeamScopedRootMixin, UUIDModel):
+ """One active insight on the stored due list of the anomaly scoring tick.
+
+ A slow refresh job adds the active insights, removes the ones that left, and sets the usage. The tick
+ reads the due rows, and each pick moves its row's `next_due_at`. See `anomaly_scores/due_list.py`.
+ """
+
+ # See SignalReportRefund.all_teams for rationale.
+ all_teams = models.Manager() # noqa: DJ012
+
+ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, db_constraint=False, related_name="+")
+ # Not a foreign key: insights belong to another product.
+ insight_id = models.BigIntegerField()
+ interval = models.CharField(max_length=16)
+ # The tick picks the row at or after this time. A never-scored insight is due at once.
+ next_due_at = models.DateTimeField()
+ refreshed_at = models.DateTimeField()
+ # Series key to the newest scored bucket start of the series, as an ISO time in UTC.
+ series_scored_through = models.JSONField(default=dict, db_default={})
+ viewers = models.PositiveIntegerField(default=0)
+ has_enabled_alert = models.BooleanField(default=False)
+
+ created_at = models.DateTimeField(auto_now_add=True)
+ updated_at = models.DateTimeField(auto_now=True)
+
+ class Meta:
+ default_manager_name = "all_teams"
+ constraints = [
+ models.UniqueConstraint(fields=["team", "insight_id"], name="signals_anomaly_due_insight_unique"),
+ ]
+ indexes = [
+ models.Index(fields=["next_due_at"], name="signals_anomaly_due_next_idx"),
+ ]
diff --git a/products/signals/backend/temporal/__init__.py b/products/signals/backend/temporal/__init__.py
index cef5192e596a..3bdaae305790 100644
--- a/products/signals/backend/temporal/__init__.py
+++ b/products/signals/backend/temporal/__init__.py
@@ -4,6 +4,10 @@ from products.signals.backend.activity_summary.tick import (
plan_activity_summary_tick_activity,
update_activity_summary_pair_activity,
)
+from products.signals.backend.anomaly_scores.refresh import (
+ AnomalyScoringDueListRefreshWorkflow,
+ refresh_anomaly_scoring_due_list_activity,
+)
from products.signals.backend.anomaly_scores.tick import (
AnomalyScoringTickWorkflow,
capture_anomaly_scoring_tick_activity,
@@ -244,7 +248,12 @@ ACTIVITIES = [
]
# The temporal-worker-self-driving fleet runs these, not video-export.
-SELF_DRIVING_WORKFLOWS = [InboxRankingScoringWorkflow, ActivitySummaryTickWorkflow, AnomalyScoringTickWorkflow]
+SELF_DRIVING_WORKFLOWS = [
+ InboxRankingScoringWorkflow,
+ ActivitySummaryTickWorkflow,
+ AnomalyScoringTickWorkflow,
+ AnomalyScoringDueListRefreshWorkflow,
+]
SELF_DRIVING_ACTIVITIES = [
score_inbox_reports_activity,
plan_activity_summary_tick_activity,
@@ -253,4 +262,5 @@ SELF_DRIVING_ACTIVITIES = [
plan_anomaly_scoring_tick_activity,
score_anomaly_insight_activity,
capture_anomaly_scoring_tick_activity,
+ refresh_anomaly_scoring_due_list_activity,
]
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index 9b0043303ca7..0d3ffda6f7c4 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -35,6 +35,7 @@ from products.product_analytics.backend.facade.api import record_insight_view
from products.product_analytics.backend.facade.models import Insight
from products.signals.backend.anomaly_scores.artifacts import clear_artifact_cache
from products.signals.backend.anomaly_scores.config import (
+ DEFAULT_LIMITS,
DEFAULT_PROFILE,
DISABLED,
AnomalyScoringConfig,
@@ -42,6 +43,7 @@ from products.signals.backend.anomaly_scores.config import (
covers_team,
parse_config,
)
+from products.signals.backend.anomaly_scores.due_list import record_scored, refresh_due_list
from products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_events
from products.signals.backend.anomaly_scores.queue import (
SKIP_COOLDOWN,
@@ -87,7 +89,12 @@ from products.signals.backend.anomaly_scores.tick import (
plan_tick,
run_insight,
)
-from products.signals.backend.models import AnomalyScoringState, SignalSourceConfig, SignalTeamConfig
+from products.signals.backend.models import (
+ AnomalyScoringDueInsight,
+ AnomalyScoringState,
+ SignalSourceConfig,
+ SignalTeamConfig,
+)
from products.subscriptions.backend.facade.testing import create_subscription
NEW_YORK = ZoneInfo("America/New_York")
@@ -502,6 +509,7 @@ class TestRunInsight(SimpleTestCase):
patch(f"{tick}.Team"),
patch(f"{tick}.score_insight", side_effect=error),
patch(f"{tick}.mark_attempted") as mark_attempted,
+ patch(f"{tick}.record_failed") as record_failed,
patch("products.signals.backend.anomaly_scores.metrics.posthoganalytics.default_client", client),
):
pick = InsightPick(team_id=1, insight_id=7, scored_through={}, interval=IntervalType.DAY)
@@ -510,6 +518,9 @@ class TestRunInsight(SimpleTestCase):
except Exception:
status = "raised"
cooldowns = [call.args[1] for call in mark_attempted.call_args_list]
+ # The due list row waits as long as the cache cooldown, so a failed pick does not stay due.
+ [row_cooldown] = [call.args[3] for call in record_failed.call_args_list]
+ assert row_cooldown == cooldowns[-1]
[pick_event] = [
call.kwargs for call in capture.call_args_list if call.kwargs["event"] == "anomaly scoring pick"
]
@@ -907,6 +918,115 @@ class TestActiveInsights(BaseTest):
]
+class TestDueList(ClickhouseTestMixin, BaseTest):
+ def _config(self, *team_ids: int) -> AnomalyScoringConfig:
+ return AnomalyScoringConfig(
+ enabled=True,
+ organization_ids=(str(self.organization.id),),
+ allow_team_ids=team_ids or (self.team.id,),
+ due_list=True,
+ )
+
+ def _row(self, insight: Insight, next_due_at: datetime, watermarks: dict[str, datetime]) -> None:
+ AnomalyScoringDueInsight.all_teams.create(
+ team_id=insight.team_id,
+ insight_id=insight.id,
+ interval="day",
+ next_due_at=next_due_at,
+ refreshed_at=NOW,
+ series_scored_through={key: bucket.isoformat() for key, bucket in watermarks.items()},
+ )
+
+ @time_machine.travel(NOW, tick=False)
+ def test_the_tick_picks_due_rows_only_and_sends_no_clickhouse_query(self) -> None:
+ today = NOW.replace(hour=0)
+ due = Insight.objects.create(team=self.team, name="due", query=_trends(interval="day"))
+ waiting = Insight.objects.create(team=self.team, name="waiting", query=_trends(interval="day"))
+ watermarks = {"a": today - timedelta(days=6), "b": today - timedelta(days=3)}
+ self._row(due, NOW - timedelta(hours=1), watermarks)
+ self._row(waiting, NOW + timedelta(hours=1), {})
+ cache.delete_many(
+ [
+ _cooldown_key(due.id),
+ _cooldown_key(waiting.id),
+ _fleet_budget_key(NOW),
+ _project_budget_key(self.team.id, NOW),
+ ]
+ )
+
+ with (
+ patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]),
+ patch("products.signals.backend.anomaly_scores.queue.scored_buckets", side_effect=AssertionError),
+ ):
+ selection = select_insights(self._config(), SelectionConfig(weighting=Weighting.UNIFORM), NOW, limit=10)
+
+ assert [(pick.insight_id, pick.scored_through) for pick in selection.picks] == [(due.id, watermarks)]
+ assert (selection.active, selection.due_insights, selection.oldest_due_seconds) == (2, 1, 3600)
+
+ @time_machine.travel(NOW, tick=False)
+ def test_a_scored_pick_keeps_the_watermark_of_a_series_that_it_did_not_return(self) -> None:
+ today = NOW.replace(hour=0)
+ insight = Insight.objects.create(team=self.team, name="breakdown", query=_trends(interval="day"))
+ self._row(insight, NOW, {"gone": today - timedelta(days=5), "kept": today - timedelta(days=2)})
+
+ record_scored(self.team, insight.id, {"kept": today - timedelta(days=1)}, NOW, DEFAULT_LIMITS)
+
+ row = AnomalyScoringDueInsight.all_teams.get(team_id=self.team.id, insight_id=insight.id)
+ assert row.series_scored_through == {
+ "gone": (today - timedelta(days=5)).isoformat(),
+ "kept": (today - timedelta(days=1)).isoformat(),
+ }
+ # Today's bucket completes at midnight, and the insight is due in its slot after that.
+ assert row.next_due_at >= today + timedelta(days=1)
+
+ @time_machine.travel(NOW, tick=False)
+ def test_a_failed_pick_moves_its_row_past_the_cooldown(self) -> None:
+ insight = Insight.objects.create(team=self.team, name="slow", query=_trends(interval="day"))
+ self._row(insight, NOW, {})
+ tick = "products.signals.backend.anomaly_scores.tick"
+
+ with (
+ patch(f"{tick}.ph_background_capture"),
+ patch(f"{tick}.score_insight", side_effect=ClickHouseQueryTimeOut()),
+ ):
+ run_insight(
+ ScoreInsightInput(
+ pick=InsightPick(
+ team_id=self.team.id, insight_id=insight.id, scored_through={}, interval=IntervalType.DAY
+ )
+ ),
+ NOW,
+ )
+
+ row = AnomalyScoringDueInsight.all_teams.get(team_id=self.team.id, insight_id=insight.id)
+ assert row.next_due_at == NOW + SKIP_COOLDOWN
+
+ @time_machine.travel(NOW, tick=False)
+ def test_the_refresh_removes_inactive_insights_and_resumes_with_the_teams_that_waited(self) -> None:
+ other_team = Team.objects.create(organization=self.organization, name="Other")
+ active = {
+ team.id: Insight.objects.create(team=team, name=team.name, query=_trends(interval="day"))
+ for team in (self.team, other_team)
+ }
+ inactive = Insight.objects.create(team=self.team, name="inactive", query=_trends(interval="day"))
+ Insight.objects.filter(id=inactive.id).update(last_modified_at=_LONG_AGO)
+ self._row(inactive, NOW, {})
+ cache.delete_many(
+ [f"signals:anomaly_scores:due_list_refreshed:{team_id}" for team_id in (self.team.id, other_team.id)]
+ )
+ config = self._config(self.team.id, other_team.id)
+
+ refreshes = []
+ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):
+ for later in (NOW, NOW + timedelta(minutes=15), NOW + timedelta(minutes=30)):
+ refresh = refresh_due_list(config, later, deadline=0.0)
+ refreshes.append((refresh.teams_refreshed, refresh.teams_waiting))
+
+ assert refreshes == [(1, 1), (1, 0), (0, 0)]
+ rows = AnomalyScoringDueInsight.all_teams.filter(team_id__in=list(active)).values_list("team_id", "insight_id")
+ assert sorted(rows) == sorted((team_id, insight.id) for team_id, insight in active.items())
+
+
class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
def test_reads_the_newest_scored_bucket_of_each_series(self) -> None:
def score(insight_id: int, series_key: str, bucket: datetime, source: str = "insight") -> None: