Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): classify anomaly scoring failures and bound query load (#122424)

posthog[bot]a0462fc1012d
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.py
index 14ea7ff644eb..0e97e4690af6 100644
--- a/products/signals/backend/anomaly_scores/config.py
+++ b/products/signals/backend/anomaly_scores/config.py
@@ -32,7 +32,6 @@ ANOMALY_SCORING_FLAG_DISTINCT_ID = "signals_anomaly_scoring"
class IntervalScoring:
"""How one interval is scored. The detector is any registered detector config, so swapping it needs no code."""
- lookback: str
min_points: int
detector: dict[str, Any]
@@ -64,10 +63,10 @@ def _zscore(window: int) -> dict[str, Any]:
DEFAULT_PROFILE = ScoringProfile(
name="default-v1",
intervals={
- IntervalType.HOUR: IntervalScoring(lookback="-28d", min_points=336, detector=_zscore(336)),
- IntervalType.DAY: IntervalScoring(lookback="-365d", min_points=56, detector=_zscore(56)),
- IntervalType.WEEK: IntervalScoring(lookback="-156w", min_points=52, detector=_zscore(52)),
- IntervalType.MONTH: IntervalScoring(lookback="-60m", min_points=24, detector=_zscore(24)),
+ IntervalType.HOUR: IntervalScoring(min_points=336, detector=_zscore(336)),
+ IntervalType.DAY: IntervalScoring(min_points=56, detector=_zscore(56)),
+ IntervalType.WEEK: IntervalScoring(min_points=52, detector=_zscore(52)),
+ IntervalType.MONTH: IntervalScoring(min_points=24, detector=_zscore(24)),
},
)
diff --git a/products/signals/backend/anomaly_scores/metrics.py b/products/signals/backend/anomaly_scores/metrics.py
new file mode 100644
index 000000000000..06b3ed442a1e
--- /dev/null
+++ b/products/signals/backend/anomaly_scores/metrics.py
@@ -0,0 +1,30 @@
+"""SDK metrics of the anomaly scoring tick. Attributes stay low cardinality, so ids go to the logs only."""
+
+import posthoganalytics
+from posthoganalytics.metrics_capture import PostHogMetrics
+from temporalio import workflow
+
+
+def _metrics() -> PostHogMetrics | None:
+ # A replayed workflow runs its code again, and the values were already recorded the first time.
+ if workflow.in_workflow() and workflow.unsafe.is_replaying():
+ return None
+ client = posthoganalytics.default_client
+ return client.metrics if client is not None else None
+
+
+def record_pick(*, status: str, interval: str, duration_ms: float, events: int) -> None:
+ metrics = _metrics()
+ if metrics is None:
+ return
+ attributes = {"status": status, "interval": interval}
+ metrics.count("anomaly_scoring.picks", 1, attributes=attributes)
+ metrics.histogram("anomaly_scoring.query.duration", duration_ms, unit="ms", attributes=attributes)
+ if events:
+ metrics.count("anomaly_scoring.events", events, attributes={"interval": interval})
+
+
+def record_candidates(candidates: int) -> None:
+ metrics = _metrics()
+ if metrics is not None:
+ metrics.gauge("anomaly_scoring.candidates", candidates, attributes={})
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index dad7b1dc851c..782fdf0d1f5a 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -47,7 +47,8 @@ _MAX_SCORED_SERIES = 50_000
# An insight is not picked again in this time after an attempt. Events take a moment to reach
# ClickHouse, and an insight that writes nothing would otherwise stay at the head of the queue.
ATTEMPT_COOLDOWN = datetime.timedelta(hours=1)
-UNSUPPORTED_COOLDOWN = datetime.timedelta(days=1)
+# For a failure that the next hour does not fix: an unsupported shape, a query that is too expensive, or no access.
+SKIP_COOLDOWN = datetime.timedelta(days=1)
_CALENDAR_INTERVALS = {
IntervalType.HOUR: CalendarInterval.HOURLY,
@@ -89,6 +90,7 @@ class InsightPick:
insight_id: int
# The newest scored bucket start of each series, in UTC. Empty when the insight was never scored.
scored_through: dict[str, datetime.datetime]
+ interval: IntervalType | None = None
@frozen
@@ -159,7 +161,7 @@ def scored_buckets(
"limit": ast.Constant(value=_MAX_SCORED_SERIES),
},
workload=Workload.OFFLINE,
- ch_user=ClickHouseUser.SIGNALS,
+ ch_user=ClickHouseUser.DEFAULT,
)
state: dict[int, dict[str, datetime.datetime]] = {}
for insight_id, series_key, newest_bucket in cast(list[tuple[Any, ...]], response.results or []):
@@ -253,7 +255,10 @@ def select_insights(
]
picks = [
InsightPick(
- team_id=due_by_id[insight_id].team_id, insight_id=insight_id, scored_through=state.get(insight_id, {})
+ team_id=due_by_id[insight_id].team_id,
+ insight_id=insight_id,
+ scored_through=state.get(insight_id, {}),
+ interval=due_by_id[insight_id].interval,
)
for insight_id in order[:limit]
]
diff --git a/products/signals/backend/anomaly_scores/schedule.py b/products/signals/backend/anomaly_scores/schedule.py
index 49365286ba56..a613d140936d 100644
--- a/products/signals/backend/anomaly_scores/schedule.py
+++ b/products/signals/backend/anomaly_scores/schedule.py
@@ -18,7 +18,7 @@ from temporalio.client import (
from posthog.scheduling.jitter import deterministic_offset
from posthog.temporal.common.schedule import a_create_schedule, a_schedule_exists, a_update_schedule
-from products.signals.backend.anomaly_scores.tick import TICK_WORKFLOW_NAME, AnomalyScoringTickInput
+from products.signals.backend.anomaly_scores.tick import TICK_TIMEOUT, TICK_WORKFLOW_NAME, AnomalyScoringTickInput
ANOMALY_SCORING_TICK_SCHEDULE_ID = "anomaly-scoring-tick-schedule"
ANOMALY_SCORING_TICK_INTERVAL = timedelta(minutes=5)
@@ -37,7 +37,7 @@ async def create_anomaly_scoring_tick_schedule(client: Client) -> None:
asdict(AnomalyScoringTickInput()),
id=ANOMALY_SCORING_TICK_SCHEDULE_ID,
task_queue=settings.SELF_DRIVING_TASK_QUEUE,
- execution_timeout=ANOMALY_SCORING_TICK_INTERVAL,
+ execution_timeout=TICK_TIMEOUT,
),
spec=ScheduleSpec(
intervals=[
diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.py
index c35a7d13dc34..489fe36d4c47 100644
--- a/products/signals/backend/anomaly_scores/scoring.py
+++ b/products/signals/backend/anomaly_scores/scoring.py
@@ -1,4 +1,5 @@
import json
+import math
import hashlib
import dataclasses
from collections.abc import Sequence
@@ -10,11 +11,12 @@ import numpy as np
from posthog.schema import BreakdownFilter, ChartDisplayType, DateRange, IntervalType, TrendsQuery
-from posthog.hogql.constants import LimitContext
+from posthog.hogql.constants import INSIGHT_QUERY_FANOUT_CONCURRENCY, HogQLGlobalSettings, LimitContext
from posthog.api.services.query import ExecutionMode
-from posthog.caching.calculate_results import calculate_for_query_based_insight
-from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
+from posthog.clickhouse.client.connection import ClickHouseUser
+from posthog.clickhouse.query_tagging import Feature, Product, get_team_query_tags, tag_queries
+from posthog.clickhouse.workload import Workload
from posthog.dataclasses import frozen
from posthog.event_usage import EventSource
from posthog.hogql_queries.utils.breakdowns import has_breakdown_filter
@@ -26,6 +28,7 @@ from posthog.tasks.alerts.detectors.registry import get_detector
from posthog.tasks.alerts.utils import WRAPPER_NODE_KINDS, is_non_time_series_trend
from products.product_analytics.backend.facade.models import Insight
+from products.product_analytics.backend.facade.queries import TrendsQueryRunner
from products.signals.backend.anomaly_scores.config import (
DEFAULT_LIMITS,
IntervalScoring,
@@ -37,6 +40,16 @@ from products.signals.backend.anomaly_scores.config import (
# Naming fields do not change what a series counts, so a rename keeps its key.
_SERIES_NAMING_FIELDS = {"custom_name", "name"}
+_MAX_SMOOTHING_BUCKETS = 90
+
+_DATE_UNITS = {IntervalType.HOUR: "h", IntervalType.DAY: "d", IntervalType.WEEK: "w", IntervalType.MONTH: "m"}
+
+# The scoring query runs as background work, so it gets more time than a chart query and fewer threads.
+_MAX_QUERY_SECONDS = 150
+# Trends runs one query per series in a pool of INSIGHT_QUERY_FANOUT_CONCURRENCY threads, so more series run in waves.
+# All the waves of one insight share this budget. The insight activity then has time to classify a timeout.
+SCORING_QUERY_BUDGET_SECONDS = 210
+
class UnsupportedInsightError(Exception):
"""The insight is not a shape we score. The caller skips it; it is not an error."""
@@ -97,6 +110,9 @@ def trends_query_for(insight: Insight, profile: ScoringProfile) -> TrendsQuery:
raise UnsupportedInsightError("Insight shows a single value, not a time series")
if trends.trendsFilter and trends.trendsFilter.display == ChartDisplayType.ACTIONS_LINE_GRAPH_CUMULATIVE:
raise UnsupportedInsightError("Cumulative insights are not scored")
+ if trends.trendsFilter and trends.trendsFilter.display == ChartDisplayType.CALENDAR_HEATMAP:
+ # The chart has its own runner and shows hours of the week, not a time series.
+ raise UnsupportedInsightError("Calendar heatmaps are not scored")
interval_scoring(profile, _interval(trends))
return trends
@@ -108,16 +124,74 @@ def interval_scoring(profile: ScoringProfile, interval: IntervalType) -> Interva
return scoring
-def build_scoring_query(
- query: TrendsQuery, scoring: IntervalScoring, limits: ScoringLimits = DEFAULT_LIMITS
-) -> TrendsQuery:
- """Fetch the scorer's lookback instead of the chart's date range.
+def _detector_history(detector: dict[str, Any]) -> int:
+ """Buckets the detector reads to score its newest bucket. A detector without a window reads all it gets."""
+ if isinstance(detector.get("detectors"), list):
+ return max((_detector_history(sub) for sub in detector["detectors"]), default=0)
+ window = detector.get("window")
+ if not isinstance(window, int):
+ return 0
+ preprocessing = detector.get("preprocessing") or {}
+ # Differencing and trailing smoothing each read buckets before the first one they output.
+ consumed = (
+ (1 if preprocessing.get("diffs_n") else 0)
+ + max((preprocessing.get("smooth_n") or 0) - 1, 0)
+ + (preprocessing.get("lags_n") or 0)
+ )
+ return window + max(detector.get("training_offset_n", 1), 1) + consumed
+
+
+def history_buckets(scoring: IntervalScoring) -> int:
+ """Completed buckets up to and including a scored bucket that its score depends on."""
+ return max(scoring.min_points, _detector_history(scoring.detector))
+
+
+def buckets_to_score(
+ interval: IntervalType,
+ *,
+ now: datetime,
+ after: datetime | None,
+ timezone: ZoneInfo,
+ limits: ScoringLimits = DEFAULT_LIMITS,
+) -> int:
+ if after is None:
+ return limits.max_backfill_buckets
+ count = 0
+ start = bucket_end(after.astimezone(timezone), interval)
+ while count < limits.max_backfill_buckets and bucket_end(start, interval) <= now:
+ count += 1
+ start = bucket_end(start, interval)
+ return max(count, 1)
+
+
+def fetch_buckets(query: TrendsQuery, scoring: IntervalScoring, to_score: int) -> int:
+ """Completed buckets the scoring query fetches: the history of the oldest bucket to score, then the rest.
+
+ The count has one bucket more than the scores read, so a first bucket that the date range cuts short is never read.
+ """
+ count = history_buckets(scoring) + to_score
+ smoothing = query.trendsFilter.smoothingIntervals if query.trendsFilter else None
+ if smoothing and smoothing > _MAX_SMOOTHING_BUCKETS:
+ # The field has no upper bound, and the fetch grows with it.
+ raise UnsupportedInsightError(f"Smoothing over {smoothing} buckets is not scored")
+ if smoothing and smoothing > 1:
+ # Trends smooths over the fetched buckets only, so the oldest values need their own trailing window.
+ count += smoothing - 1
+ days_of_week = query.dateRange.daysOfWeek if query.dateRange else None
+ if days_of_week and _interval(query) == IntervalType.DAY:
+ # Trends drops the buckets of the days that the filter leaves out.
+ count = math.ceil(count * 7 / len(set(days_of_week)))
+ return count
+
+
+def build_scoring_query(query: TrendsQuery, buckets: int, limits: ScoringLimits = DEFAULT_LIMITS) -> TrendsQuery:
+ """Fetch the newest ``buckets`` completed buckets instead of the chart's date range.
Series, filters and formulas stay as they are, so the scores describe the series the chart shows.
"""
days_of_week = query.dateRange.daysOfWeek if query.dateRange else None
update: dict[str, Any] = {
- "dateRange": DateRange(date_from=scoring.lookback, daysOfWeek=days_of_week),
+ "dateRange": DateRange(date_from=f"-{buckets}{_DATE_UNITS[_interval(query)]}", daysOfWeek=days_of_week),
"compareFilter": None,
}
if has_breakdown_filter(query.breakdownFilter):
@@ -133,6 +207,17 @@ def build_scoring_query(
return query.model_copy(update=update)
+def scoring_query_settings(queries: int) -> HogQLGlobalSettings:
+ """Settings for each of the ``queries`` queries of one insight, so that all of them stop within the shared budget."""
+ waves = max(1, math.ceil(queries / INSIGHT_QUERY_FANOUT_CONCURRENCY))
+ return HogQLGlobalSettings(
+ max_execution_time=min(_MAX_QUERY_SECONDS, SCORING_QUERY_BUDGET_SECONDS // waves),
+ max_threads=8,
+ # A timeout must throw: a partial result cuts off the end of the series, and that end scores as a drop.
+ timeout_overflow_mode="throw",
+ )
+
+
def score_series(
values: Sequence[float],
buckets: Sequence[datetime],
@@ -184,27 +269,36 @@ def score_insight(
query = trends_query_for(insight, profile)
interval = _interval(query)
scoring = interval_scoring(profile, interval)
- scoring_query = build_scoring_query(query, scoring, limits)
+ timezone = team.timezone_info
+ to_score = buckets_to_score(interval, now=now, after=after, timezone=timezone, limits=limits)
+ scoring_query = build_scoring_query(query, fetch_buckets(query, scoring, to_score), limits)
+ tag_queries(**get_team_query_tags(team), insight_id=insight.pk)
tag_queries(product=Product.SIGNALS, feature=Feature.ALERTING)
- calculation = calculate_for_query_based_insight(
- insight,
+ runner = TrendsQueryRunner(
+ query=scoring_query,
team=team,
- execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS,
# No request user. Run as the insight's creator so warehouse access control resolves for them.
user=insight.created_by,
- query_override=scoring_query.model_dump(exclude_none=True),
- analytics_props={"source": EventSource.SELF_DRIVING},
- # The async context raises the ClickHouse timeout to minutes. One slow insight must not hold a tick.
+ # The async context raises the timeout above scoring_query_settings. One slow insight must not hold a tick.
limit_context=LimitContext.QUERY,
+ workload=Workload.OFFLINE,
+ ch_user=ClickHouseUser.DEFAULT,
)
- if calculation.result is None:
+ # The runner expands breakdown cohorts and compare periods into the series, so its count is the query count.
+ runner.hogql_settings = scoring_query_settings(len(runner.series))
+ response = runner.run(
+ execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS,
+ insight_id=insight.pk,
+ analytics_props={"source": EventSource.SELF_DRIVING},
+ )
+ results = getattr(response, "results", None)
+ if not isinstance(results, list):
raise RuntimeError(f"No results for insight {insight.pk}")
- timezone = team.timezone_info
series: list[ScoredSeries] = []
key_counts: dict[str, int] = {}
- for result in cast(list[dict[str, Any]], calculation.result)[: limits.max_series]:
+ for result in cast(list[dict[str, Any]], results)[: limits.max_series]:
buckets = [_parse_bucket(day, timezone) for day in result.get("days") or []]
breakdown_value = _breakdown_label(result.get("breakdown_value"))
key = series_key(insight.pk, _series_definition(scoring_query, result), breakdown_value)
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index 67f5dc219f96..9a51f3616e22 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -4,6 +4,7 @@ The `anomaly-scoring` flag payload holds every lever, and a missing or malformed
tick off. Each insight activity scores one insight and writes its new buckets as `$anomaly_score` events.
"""
+import time
import asyncio
import datetime
from collections import Counter
@@ -15,7 +16,15 @@ import structlog
from temporalio import activity, workflow
from temporalio.common import RetryPolicy
+from posthog.hogql.errors import TableAccessDeniedError
+
from posthog.dataclasses import frozen
+from posthog.exceptions import (
+ ClickHouseClusterMemoryLimitExceeded,
+ ClickHouseEstimatedQueryExecutionTimeTooLong,
+ ClickHouseQueryMemoryLimitExceeded,
+ ClickHouseQueryTimeOut,
+)
from posthog.models import Team
from posthog.ph_client import ph_background_capture
from posthog.sync import database_sync_to_async
@@ -32,14 +41,19 @@ from products.signals.backend.anomaly_scores.config import (
read_payload,
)
from products.signals.backend.anomaly_scores.emission import emit, score_events
+from products.signals.backend.anomaly_scores.metrics import record_candidates, record_pick
from products.signals.backend.anomaly_scores.queue import (
ATTEMPT_COOLDOWN,
- UNSUPPORTED_COOLDOWN,
+ SKIP_COOLDOWN,
InsightPick,
mark_attempted,
select_insights,
)
-from products.signals.backend.anomaly_scores.scoring import UnsupportedInsightError, score_insight
+from products.signals.backend.anomaly_scores.scoring import (
+ SCORING_QUERY_BUDGET_SECONDS,
+ UnsupportedInsightError,
+ score_insight,
+)
from products.signals.backend.anomaly_scores.selection import parse_selection_config
logger = structlog.get_logger(__name__)
@@ -51,11 +65,23 @@ STATUS_SCORED = "scored"
STATUS_UP_TO_DATE = "up_to_date"
STATUS_UNSUPPORTED = "unsupported"
STATUS_MISSING = "missing"
+# The scoring query hit a ClickHouse time or memory limit. A retry in an hour hits it again.
+STATUS_TOO_EXPENSIVE = "too_expensive"
+# The insight's creator cannot read a table of the query.
+STATUS_ACCESS_DENIED = "access_denied"
+# Any other failure. The activity raises it, and the workflow counts it as `errored`.
+STATUS_ERRORED = "errored"
_PLAN_TIMEOUT = datetime.timedelta(minutes=2)
-# The scoring query times out at 60 seconds. The rest is detection and the capture request.
-_INSIGHT_TIMEOUT = datetime.timedelta(minutes=3)
+# All the scoring queries of one insight stop within their budget. The rest is detection, the capture request,
+# and the failure classification. A query timeout must reach the classification, so that it gets the 1 day cooldown.
+_INSIGHT_TIMEOUT = datetime.timedelta(seconds=SCORING_QUERY_BUDGET_SECONDS) + datetime.timedelta(seconds=90)
_INSIGHT_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)
+_CAPTURE_TIMEOUT = datetime.timedelta(minutes=1)
+# No batch of insights starts after this time. The picks that are left wait for a later tick.
+TICK_BUDGET = datetime.timedelta(minutes=10)
+# The last batch can start at the budget, and the summary capture runs after it.
+TICK_TIMEOUT = _PLAN_TIMEOUT + TICK_BUDGET + _INSIGHT_TIMEOUT + _CAPTURE_TIMEOUT
@frozen
@@ -83,6 +109,7 @@ class ScoreInsightInput:
class ScoreInsightResult:
status: str
events: int
+ series: int = 0
@frozen
@@ -93,6 +120,9 @@ class AnomalyScoringTickResult:
statuses: dict[str, int] | None = None
errored: int = 0
events: int = 0
+ duration_ms: int = 0
+ # Picks left for a later tick because the tick budget ran out.
+ deferred: int = 0
def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:
@@ -102,6 +132,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
if not config.enabled:
return AnomalyScoringTickPlan(enabled=False, picks=[], candidates=0, max_concurrency=config.max_concurrency)
selection = select_insights(config, parse_selection_config(payload), now, limit or config.max_insights_per_tick)
+ record_candidates(selection.candidates)
return AnomalyScoringTickPlan(
enabled=True,
picks=selection.picks,
@@ -111,7 +142,65 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
)
+def failure_status(error: Exception) -> str | None:
+ """The status of a failure that the next tick does not fix, or None for any other failure."""
+ if isinstance(error, UnsupportedInsightError):
+ return STATUS_UNSUPPORTED
+ # The cluster limit is a subclass of the query limit, but it is load from other queries, so a later tick can pass.
+ if isinstance(error, ClickHouseClusterMemoryLimitExceeded):
+ return None
+ if isinstance(
+ error,
+ ClickHouseQueryTimeOut | ClickHouseQueryMemoryLimitExceeded | ClickHouseEstimatedQueryExecutionTimeTooLong,
+ ):
+ return STATUS_TOO_EXPENSIVE
+ if isinstance(error, TableAccessDeniedError):
+ return STATUS_ACCESS_DENIED
+ return None
+
+
def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsightResult:
+ """Score one pick, and log and measure its outcome. A known failure returns its status, and any other one raises."""
+ pick = input.pick
+ interval = pick.interval.value if pick.interval else "unknown"
+ started = time.monotonic()
+ try:
+ result = _score_pick(input, now)
+ except Exception as error:
+ status = failure_status(error)
+ duration_ms = round((time.monotonic() - started) * 1000)
+ log = logger.info if status else logger.exception
+ log(
+ "anomaly_scoring_insight_failed",
+ team_id=pick.team_id,
+ insight_id=pick.insight_id,
+ interval=interval,
+ status=status or STATUS_ERRORED,
+ error_class=type(error).__name__,
+ error=str(error)[:200],
+ duration_ms=duration_ms,
+ )
+ record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0)
+ if status is None:
+ raise
+ mark_attempted(pick.insight_id, SKIP_COOLDOWN)
+ return ScoreInsightResult(status=status, events=0)
+ duration_ms = round((time.monotonic() - started) * 1000)
+ logger.info(
+ "anomaly_scoring_insight_scored",
+ team_id=pick.team_id,
+ insight_id=pick.insight_id,
+ interval=interval,
+ status=result.status,
+ series=result.series,
+ events=result.events,
+ duration_ms=duration_ms,
+ )
+ record_pick(status=result.status, interval=interval, duration_ms=duration_ms, events=result.events)
+ return result
+
+
+def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsightResult:
pick = input.pick
insight = next(
(
@@ -126,24 +215,21 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
team = Team.objects.get(id=pick.team_id)
# Set before scoring, so a failure also waits for the cooldown instead of taking a slot on every tick.
mark_attempted(pick.insight_id, ATTEMPT_COOLDOWN)
- try:
- scores = score_insight(
- insight,
- team,
- now=now,
- # Every series is filtered again on its own newest bucket below. A new series gets up to the backfill cap.
- after=min(pick.scored_through.values()) if pick.scored_through else None,
- limits=input.limits,
- )
- except UnsupportedInsightError as error:
- logger.info("anomaly_scoring_insight_unsupported", insight_id=pick.insight_id, reason=str(error))
- mark_attempted(pick.insight_id, UNSUPPORTED_COOLDOWN)
- return ScoreInsightResult(status=STATUS_UNSUPPORTED, events=0)
+ scores = score_insight(
+ insight,
+ team,
+ now=now,
+ # Every series is filtered again on its own newest bucket below. A new series gets up to the backfill cap.
+ after=min(pick.scored_through.values()) if pick.scored_through else None,
+ limits=input.limits,
+ )
events = score_events(
team_id=team.id, insight_id=insight.id, scores=scores, scored_through=pick.scored_through, scored_at=now
)
emit(team, events)
- return ScoreInsightResult(status=STATUS_SCORED if events else STATUS_UP_TO_DATE, events=len(events))
+ return ScoreInsightResult(
+ status=STATUS_SCORED if events else STATUS_UP_TO_DATE, events=len(events), series=len(scores.series)
+ )
def capture_tick(result: AnomalyScoringTickResult) -> None:
@@ -154,6 +240,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
statuses=result.statuses,
errored=result.errored,
events=result.events,
+ duration_ms=result.duration_ms,
+ deferred=result.deferred,
)
ph_background_capture()(
distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,
@@ -165,6 +253,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
**{f"status_{status}": count for status, count in (result.statuses or {}).items()},
"errored": result.errored,
"events": result.events,
+ "duration_ms": result.duration_ms,
+ "deferred": result.deferred,
},
)
@@ -195,6 +285,7 @@ async def capture_anomaly_scoring_tick_activity(result: AnomalyScoringTickResult
class AnomalyScoringTickWorkflow:
@workflow.run
async def run(self, inputs: AnomalyScoringTickInput) -> AnomalyScoringTickResult:
+ started = workflow.now()
plan = await workflow.execute_activity(
plan_anomaly_scoring_tick_activity,
inputs,
@@ -207,8 +298,12 @@ class AnomalyScoringTickWorkflow:
statuses: Counter[str] = Counter()
errored = 0
events = 0
+ deferred = 0
concurrency = max(1, plan.max_concurrency)
for index in range(0, len(plan.picks), concurrency):
+ if workflow.now() - started >= TICK_BUDGET:
+ deferred = len(plan.picks) - index
+ break
results = await asyncio.gather(
*(
workflow.execute_activity(
@@ -237,11 +332,13 @@ class AnomalyScoringTickWorkflow:
statuses=dict(statuses),
errored=errored,
events=events,
+ duration_ms=round((workflow.now() - started).total_seconds() * 1000),
+ deferred=deferred,
)
await workflow.execute_activity(
capture_anomaly_scoring_tick_activity,
result,
- start_to_close_timeout=datetime.timedelta(minutes=1),
+ start_to_close_timeout=_CAPTURE_TIMEOUT,
retry_policy=RetryPolicy(maximum_attempts=2),
)
return result
diff --git a/products/signals/backend/test/test_anomaly_scores.py b/products/signals/backend/test/test_anomaly_scores.py
index aa7e9bec54d8..b915b1cf6637 100644
--- a/products/signals/backend/test/test_anomaly_scores.py
+++ b/products/signals/backend/test/test_anomaly_scores.py
@@ -20,6 +20,8 @@ from posthog.schema import (
TrendsQuery,
)
+from posthog.hogql.constants import INSIGHT_QUERY_FANOUT_CONCURRENCY
+
from posthog.interval_specs import interval_spec
from posthog.models import Team
@@ -27,15 +29,17 @@ from products.product_analytics.backend.facade.models import Insight
from products.signals.backend.anomaly_scores.config import DEFAULT_LIMITS, DEFAULT_PROFILE, IntervalScoring
from products.signals.backend.anomaly_scores.scoring import (
UnsupportedInsightError,
+ buckets_to_score,
+ fetch_buckets,
score_insight,
score_series,
trends_query_for,
)
+from products.signals.backend.anomaly_scores.tick import _INSIGHT_TIMEOUT
NEW_YORK = ZoneInfo("America/New_York")
DAY_SCORING = DEFAULT_PROFILE.intervals[IntervalType.DAY]
ZSCORE_OR_KNN = IntervalScoring(
- lookback="-365d",
min_points=56,
detector={
"type": "ensemble",
@@ -114,6 +118,49 @@ class TestScoreSeries(SimpleTestCase):
assert points[-1].bucket_start == buckets[last_completed]
+ @parameterized.expand(
+ [
+ ("first_score", None, DEFAULT_LIMITS.max_backfill_buckets, None),
+ ("revisit", 3, 3, None),
+ ("chart_smoothing", 3, 3, 7),
+ ]
+ )
+ def test_the_fetched_buckets_score_like_the_full_history(
+ self, _name: str, scored_days_ago: int | None, to_score: int, smoothing: int | None
+ ) -> None:
+ values = list(np.random.default_rng(1).normal(100, 20, size=365))
+ buckets = _buckets(datetime(2025, 10, 1, tzinfo=UTC), IntervalType.DAY, 365)
+ now = buckets[-1] + timedelta(days=1, hours=1)
+ after = None if scored_days_ago is None else buckets[-1 - scored_days_ago]
+ query = TrendsQuery(
+ series=[EventsNode(event="$pageview")], trendsFilter=TrendsFilter(smoothingIntervals=smoothing)
+ )
+
+ assert buckets_to_score(IntervalType.DAY, now=now, after=after, timezone=ZoneInfo("UTC")) == to_score
+ fetched = fetch_buckets(query, DAY_SCORING, to_score)
+
+ def score(start: int) -> list[float | None]:
+ fetched_values = values[start:]
+ if smoothing:
+ # Trends averages the trailing buckets that the query fetched, so the fetch start shifts the average.
+ fetched_values = [
+ float(np.floor(np.mean(fetched_values[max(0, i - smoothing + 1) : i + 1])))
+ for i in range(len(fetched_values))
+ ]
+ points = score_series(
+ fetched_values,
+ buckets[start:],
+ scoring=DAY_SCORING,
+ interval=IntervalType.DAY,
+ now=now,
+ after=after,
+ )
+ return [point.score for point in points]
+
+ assert fetched < 100
+ assert score(-fetched) == score(0)
+ assert None not in score(-fetched)
+
def test_warm_up_points_have_no_score_and_no_flag(self) -> None:
buckets = _buckets(datetime(2026, 1, 1, tzinfo=UTC), IntervalType.DAY, 10)
@@ -140,10 +187,8 @@ def _result(order: int, label: str) -> dict[str, Any]:
class TestScoreInsight(SimpleTestCase):
def _keys(self, insight: Insight, results: list[dict[str, Any]]) -> dict[str, str]:
- with patch(
- "products.signals.backend.anomaly_scores.scoring.calculate_for_query_based_insight",
- return_value=MagicMock(result=results),
- ):
+ with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:
+ runner.return_value.run.return_value = MagicMock(results=results)
scores = score_insight(insight, Team(timezone="UTC"), now=datetime(2026, 1, 10, tzinfo=UTC))
return {series.label: series.series_key for series in scores.series}
@@ -175,6 +220,36 @@ class TestScoreInsight(SimpleTestCase):
assert named["web"] == single["web"]
assert named["app"] != named["web"]
+ @parameterized.expand([("one_query", 1), ("one_full_wave", 10), ("two_waves", 11), ("many_waves", 120)])
+ def test_every_query_wave_of_an_insight_stops_before_the_activity_timeout(self, _name: str, queries: int) -> None:
+ limits: list[int] = []
+ insight = _insight([EventsNode(event=f"event_{index}") for index in range(queries)])
+ with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:
+ runner.return_value.series = [object()] * queries
+
+ def run(**_kwargs: Any) -> MagicMock:
+ limits.append(runner.return_value.hogql_settings.max_execution_time)
+ return MagicMock(results=[])
+
+ runner.return_value.run.side_effect = run
+ score_insight(insight, Team(timezone="UTC"), now=datetime(2026, 1, 10, tzinfo=UTC))
+
+ waves = -(-queries // INSIGHT_QUERY_FANOUT_CONCURRENCY)
+ # Each wave can run to the time limit, and the activity still needs time to classify the timeout.
+ assert timedelta(seconds=waves * limits[0]) <= _INSIGHT_TIMEOUT - timedelta(minutes=1)
+ if waves == 1:
+ assert limits[0] == 150
+
+
+class TestFetchBuckets(SimpleTestCase):
+ def test_unbounded_smoothing_is_not_scored(self) -> None:
+ query = TrendsQuery(
+ series=[EventsNode(event="$pageview")], trendsFilter=TrendsFilter(smoothingIntervals=10_000_000)
+ )
+
+ with self.assertRaises(UnsupportedInsightError):
+ fetch_buckets(query, DAY_SCORING, 1)
+
class TestTrendsQueryFor(SimpleTestCase):
@parameterized.expand(
@@ -194,6 +269,13 @@ class TestTrendsQueryFor(SimpleTestCase):
trendsFilter=TrendsFilter(display=ChartDisplayType.ACTIONS_LINE_GRAPH_CUMULATIVE),
).model_dump(exclude_none=True),
),
+ (
+ "calendar_heatmap",
+ TrendsQuery(
+ series=[EventsNode(event="$pageview")],
+ trendsFilter=TrendsFilter(display=ChartDisplayType.CALENDAR_HEATMAP),
+ ).model_dump(exclude_none=True),
+ ),
(
"minute_interval",
TrendsQuery(series=[EventsNode(event="$pageview")], interval=IntervalType.MINUTE).model_dump(
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index 76d7561939c2..61773af34882 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -4,7 +4,7 @@ from zoneinfo import ZoneInfo
import time_machine
from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
from django.core.cache import cache
from django.test import SimpleTestCase
@@ -13,6 +13,9 @@ from parameterized import parameterized
from posthog.schema import IntervalType
+from posthog.hogql.errors import TableAccessDeniedError
+
+from posthog.exceptions import ClickHouseClusterMemoryLimitExceeded, ClickHouseQueryTimeOut
from posthog.models import Organization, Team, User
from products.dashboards.backend.models.dashboard import Dashboard
@@ -22,6 +25,8 @@ from products.product_analytics.backend.facade.models import Insight
from products.signals.backend.anomaly_scores.config import DISABLED, AnomalyScoringConfig, parse_config
from products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_events
from products.signals.backend.anomaly_scores.queue import (
+ SKIP_COOLDOWN,
+ InsightPick,
WatchedInsight,
_cooldown_key,
eligible_teams,
@@ -30,9 +35,14 @@ from products.signals.backend.anomaly_scores.queue import (
select_insights,
watched_insights,
)
-from products.signals.backend.anomaly_scores.scoring import InsightScores, ScoredPoint, ScoredSeries
+from products.signals.backend.anomaly_scores.scoring import (
+ InsightScores,
+ ScoredPoint,
+ ScoredSeries,
+ UnsupportedInsightError,
+)
from products.signals.backend.anomaly_scores.selection import SelectionConfig, Weighting
-from products.signals.backend.anomaly_scores.tick import plan_tick
+from products.signals.backend.anomaly_scores.tick import ScoreInsightInput, plan_tick, run_insight
NEW_YORK = ZoneInfo("America/New_York")
ORGANIZATION_ID = "4dc8564d-bd82-1065-2f40-97f7c50f67cf"
@@ -225,6 +235,59 @@ class TestScoreEvents(SimpleTestCase):
]
+class TestRunInsight(SimpleTestCase):
+ def _run(self, error: Exception) -> tuple[str, list[Any], MagicMock]:
+ tick = "products.signals.backend.anomaly_scores.tick"
+ client = MagicMock()
+ with (
+ patch(
+ f"{tick}.insights_including_soft_deleted_for_team",
+ return_value=[Insight(id=7, saved=True, deleted=False)],
+ ),
+ patch(f"{tick}.Team"),
+ patch(f"{tick}.score_insight", side_effect=error),
+ patch(f"{tick}.mark_attempted") as mark_attempted,
+ patch("products.signals.backend.anomaly_scores.metrics.posthoganalytics.default_client", client),
+ ):
+ pick = InsightPick(team_id=1, insight_id=7, scored_through={}, interval=IntervalType.DAY)
+ try:
+ status = run_insight(ScoreInsightInput(pick=pick), NOW).status
+ except Exception:
+ status = "raised"
+ cooldowns = [call.args[1] for call in mark_attempted.call_args_list]
+ return status, cooldowns, client.metrics
+
+ @parameterized.expand(
+ [
+ ("timeout", ClickHouseQueryTimeOut(), "too_expensive"),
+ ("access_denied", TableAccessDeniedError("no access"), "access_denied"),
+ ("unsupported", UnsupportedInsightError("not trends"), "unsupported"),
+ ]
+ )
+ def test_a_known_failure_returns_its_status_and_waits_a_day(
+ self, _name: str, error: Exception, expected: str
+ ) -> None:
+ status, cooldowns, metrics = self._run(error)
+
+ assert status == expected
+ assert cooldowns[-1] == SKIP_COOLDOWN
+ metrics.count.assert_any_call(
+ "anomaly_scoring.picks", 1, attributes={"status": expected, "interval": IntervalType.DAY.value}
+ )
+
+ @parameterized.expand(
+ [
+ ("cluster_memory", ClickHouseClusterMemoryLimitExceeded()),
+ ("unknown", RuntimeError("boom")),
+ ]
+ )
+ def test_any_other_failure_raises_and_waits_an_hour(self, _name: str, error: Exception) -> None:
+ status, cooldowns, _metrics = self._run(error)
+
+ assert status == "raised"
+ assert SKIP_COOLDOWN not in cooldowns
+
+
class TestEligibleTeams(BaseTest):
@parameterized.expand(
[