:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): classify anomaly scoring failures and bound query load (#122424)
posthog[bot]a0462fc1012d
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.pyindex 14ea7ff644eb..0e97e4690af6 100644--- a/products/signals/backend/anomaly_scores/config.py+++ b/products/signals/backend/anomaly_scores/config.py@@ -32,7 +32,6 @@ ANOMALY_SCORING_FLAG_DISTINCT_ID = "signals_anomaly_scoring"class IntervalScoring:"""How one interval is scored. The detector is any registered detector config, so swapping it needs no code."""- lookback: strmin_points: intdetector: dict[str, Any]@@ -64,10 +63,10 @@ def _zscore(window: int) -> dict[str, Any]:DEFAULT_PROFILE = ScoringProfile(name="default-v1",intervals={- IntervalType.HOUR: IntervalScoring(lookback="-28d", min_points=336, detector=_zscore(336)),- IntervalType.DAY: IntervalScoring(lookback="-365d", min_points=56, detector=_zscore(56)),- IntervalType.WEEK: IntervalScoring(lookback="-156w", min_points=52, detector=_zscore(52)),- IntervalType.MONTH: IntervalScoring(lookback="-60m", min_points=24, detector=_zscore(24)),+ IntervalType.HOUR: IntervalScoring(min_points=336, detector=_zscore(336)),+ IntervalType.DAY: IntervalScoring(min_points=56, detector=_zscore(56)),+ IntervalType.WEEK: IntervalScoring(min_points=52, detector=_zscore(52)),+ IntervalType.MONTH: IntervalScoring(min_points=24, detector=_zscore(24)),},)diff --git a/products/signals/backend/anomaly_scores/metrics.py b/products/signals/backend/anomaly_scores/metrics.pynew file mode 100644index 000000000000..06b3ed442a1e--- /dev/null+++ b/products/signals/backend/anomaly_scores/metrics.py@@ -0,0 +1,30 @@+"""SDK metrics of the anomaly scoring tick. Attributes stay low cardinality, so ids go to the logs only."""++import posthoganalytics+from posthoganalytics.metrics_capture import PostHogMetrics+from temporalio import workflow+++def _metrics() -> PostHogMetrics | None:+ # A replayed workflow runs its code again, and the values were already recorded the first time.+ if workflow.in_workflow() and workflow.unsafe.is_replaying():+ return None+ client = posthoganalytics.default_client+ return client.metrics if client is not None else None+++def record_pick(*, status: str, interval: str, duration_ms: float, events: int) -> None:+ metrics = _metrics()+ if metrics is None:+ return+ attributes = {"status": status, "interval": interval}+ metrics.count("anomaly_scoring.picks", 1, attributes=attributes)+ metrics.histogram("anomaly_scoring.query.duration", duration_ms, unit="ms", attributes=attributes)+ if events:+ metrics.count("anomaly_scoring.events", events, attributes={"interval": interval})+++def record_candidates(candidates: int) -> None:+ metrics = _metrics()+ if metrics is not None:+ metrics.gauge("anomaly_scoring.candidates", candidates, attributes={})diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.pyindex dad7b1dc851c..782fdf0d1f5a 100644--- a/products/signals/backend/anomaly_scores/queue.py+++ b/products/signals/backend/anomaly_scores/queue.py@@ -47,7 +47,8 @@ _MAX_SCORED_SERIES = 50_000# An insight is not picked again in this time after an attempt. Events take a moment to reach# ClickHouse, and an insight that writes nothing would otherwise stay at the head of the queue.ATTEMPT_COOLDOWN = datetime.timedelta(hours=1)-UNSUPPORTED_COOLDOWN = datetime.timedelta(days=1)+# For a failure that the next hour does not fix: an unsupported shape, a query that is too expensive, or no access.+SKIP_COOLDOWN = datetime.timedelta(days=1)_CALENDAR_INTERVALS = {IntervalType.HOUR: CalendarInterval.HOURLY,@@ -89,6 +90,7 @@ class InsightPick:insight_id: int# The newest scored bucket start of each series, in UTC. Empty when the insight was never scored.scored_through: dict[str, datetime.datetime]+ interval: IntervalType | None = None@frozen@@ -159,7 +161,7 @@ def scored_buckets("limit": ast.Constant(value=_MAX_SCORED_SERIES),},workload=Workload.OFFLINE,- ch_user=ClickHouseUser.SIGNALS,+ ch_user=ClickHouseUser.DEFAULT,)state: dict[int, dict[str, datetime.datetime]] = {}for insight_id, series_key, newest_bucket in cast(list[tuple[Any, ...]], response.results or []):@@ -253,7 +255,10 @@ def select_insights(]picks = [InsightPick(- team_id=due_by_id[insight_id].team_id, insight_id=insight_id, scored_through=state.get(insight_id, {})+ team_id=due_by_id[insight_id].team_id,+ insight_id=insight_id,+ scored_through=state.get(insight_id, {}),+ interval=due_by_id[insight_id].interval,)for insight_id in order[:limit]]diff --git a/products/signals/backend/anomaly_scores/schedule.py b/products/signals/backend/anomaly_scores/schedule.pyindex 49365286ba56..a613d140936d 100644--- a/products/signals/backend/anomaly_scores/schedule.py+++ b/products/signals/backend/anomaly_scores/schedule.py@@ -18,7 +18,7 @@ from temporalio.client import (from posthog.scheduling.jitter import deterministic_offsetfrom posthog.temporal.common.schedule import a_create_schedule, a_schedule_exists, a_update_schedule-from products.signals.backend.anomaly_scores.tick import TICK_WORKFLOW_NAME, AnomalyScoringTickInput+from products.signals.backend.anomaly_scores.tick import TICK_TIMEOUT, TICK_WORKFLOW_NAME, AnomalyScoringTickInputANOMALY_SCORING_TICK_SCHEDULE_ID = "anomaly-scoring-tick-schedule"ANOMALY_SCORING_TICK_INTERVAL = timedelta(minutes=5)@@ -37,7 +37,7 @@ async def create_anomaly_scoring_tick_schedule(client: Client) -> None:asdict(AnomalyScoringTickInput()),id=ANOMALY_SCORING_TICK_SCHEDULE_ID,task_queue=settings.SELF_DRIVING_TASK_QUEUE,- execution_timeout=ANOMALY_SCORING_TICK_INTERVAL,+ execution_timeout=TICK_TIMEOUT,),spec=ScheduleSpec(intervals=[diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.pyindex c35a7d13dc34..489fe36d4c47 100644--- a/products/signals/backend/anomaly_scores/scoring.py+++ b/products/signals/backend/anomaly_scores/scoring.py@@ -1,4 +1,5 @@import json+import mathimport hashlibimport dataclassesfrom collections.abc import Sequence@@ -10,11 +11,12 @@ import numpy as npfrom posthog.schema import BreakdownFilter, ChartDisplayType, DateRange, IntervalType, TrendsQuery-from posthog.hogql.constants import LimitContext+from posthog.hogql.constants import INSIGHT_QUERY_FANOUT_CONCURRENCY, HogQLGlobalSettings, LimitContextfrom posthog.api.services.query import ExecutionMode-from posthog.caching.calculate_results import calculate_for_query_based_insight-from posthog.clickhouse.query_tagging import Feature, Product, tag_queries+from posthog.clickhouse.client.connection import ClickHouseUser+from posthog.clickhouse.query_tagging import Feature, Product, get_team_query_tags, tag_queries+from posthog.clickhouse.workload import Workloadfrom posthog.dataclasses import frozenfrom posthog.event_usage import EventSourcefrom posthog.hogql_queries.utils.breakdowns import has_breakdown_filter@@ -26,6 +28,7 @@ from posthog.tasks.alerts.detectors.registry import get_detectorfrom posthog.tasks.alerts.utils import WRAPPER_NODE_KINDS, is_non_time_series_trendfrom products.product_analytics.backend.facade.models import Insight+from products.product_analytics.backend.facade.queries import TrendsQueryRunnerfrom products.signals.backend.anomaly_scores.config import (DEFAULT_LIMITS,IntervalScoring,@@ -37,6 +40,16 @@ from products.signals.backend.anomaly_scores.config import (# Naming fields do not change what a series counts, so a rename keeps its key._SERIES_NAMING_FIELDS = {"custom_name", "name"}+_MAX_SMOOTHING_BUCKETS = 90++_DATE_UNITS = {IntervalType.HOUR: "h", IntervalType.DAY: "d", IntervalType.WEEK: "w", IntervalType.MONTH: "m"}++# The scoring query runs as background work, so it gets more time than a chart query and fewer threads.+_MAX_QUERY_SECONDS = 150+# Trends runs one query per series in a pool of INSIGHT_QUERY_FANOUT_CONCURRENCY threads, so more series run in waves.+# All the waves of one insight share this budget. The insight activity then has time to classify a timeout.+SCORING_QUERY_BUDGET_SECONDS = 210+class UnsupportedInsightError(Exception):"""The insight is not a shape we score. The caller skips it; it is not an error."""@@ -97,6 +110,9 @@ def trends_query_for(insight: Insight, profile: ScoringProfile) -> TrendsQuery:raise UnsupportedInsightError("Insight shows a single value, not a time series")if trends.trendsFilter and trends.trendsFilter.display == ChartDisplayType.ACTIONS_LINE_GRAPH_CUMULATIVE:raise UnsupportedInsightError("Cumulative insights are not scored")+ if trends.trendsFilter and trends.trendsFilter.display == ChartDisplayType.CALENDAR_HEATMAP:+ # The chart has its own runner and shows hours of the week, not a time series.+ raise UnsupportedInsightError("Calendar heatmaps are not scored")interval_scoring(profile, _interval(trends))return trends@@ -108,16 +124,74 @@ def interval_scoring(profile: ScoringProfile, interval: IntervalType) -> Intervareturn scoring-def build_scoring_query(- query: TrendsQuery, scoring: IntervalScoring, limits: ScoringLimits = DEFAULT_LIMITS-) -> TrendsQuery:- """Fetch the scorer's lookback instead of the chart's date range.+def _detector_history(detector: dict[str, Any]) -> int:+ """Buckets the detector reads to score its newest bucket. A detector without a window reads all it gets."""+ if isinstance(detector.get("detectors"), list):+ return max((_detector_history(sub) for sub in detector["detectors"]), default=0)+ window = detector.get("window")+ if not isinstance(window, int):+ return 0+ preprocessing = detector.get("preprocessing") or {}+ # Differencing and trailing smoothing each read buckets before the first one they output.+ consumed = (+ (1 if preprocessing.get("diffs_n") else 0)+ + max((preprocessing.get("smooth_n") or 0) - 1, 0)+ + (preprocessing.get("lags_n") or 0)+ )+ return window + max(detector.get("training_offset_n", 1), 1) + consumed+++def history_buckets(scoring: IntervalScoring) -> int:+ """Completed buckets up to and including a scored bucket that its score depends on."""+ return max(scoring.min_points, _detector_history(scoring.detector))+++def buckets_to_score(+ interval: IntervalType,+ *,+ now: datetime,+ after: datetime | None,+ timezone: ZoneInfo,+ limits: ScoringLimits = DEFAULT_LIMITS,+) -> int:+ if after is None:+ return limits.max_backfill_buckets+ count = 0+ start = bucket_end(after.astimezone(timezone), interval)+ while count < limits.max_backfill_buckets and bucket_end(start, interval) <= now:+ count += 1+ start = bucket_end(start, interval)+ return max(count, 1)+++def fetch_buckets(query: TrendsQuery, scoring: IntervalScoring, to_score: int) -> int:+ """Completed buckets the scoring query fetches: the history of the oldest bucket to score, then the rest.++ The count has one bucket more than the scores read, so a first bucket that the date range cuts short is never read.+ """+ count = history_buckets(scoring) + to_score+ smoothing = query.trendsFilter.smoothingIntervals if query.trendsFilter else None+ if smoothing and smoothing > _MAX_SMOOTHING_BUCKETS:+ # The field has no upper bound, and the fetch grows with it.+ raise UnsupportedInsightError(f"Smoothing over {smoothing} buckets is not scored")+ if smoothing and smoothing > 1:+ # Trends smooths over the fetched buckets only, so the oldest values need their own trailing window.+ count += smoothing - 1+ days_of_week = query.dateRange.daysOfWeek if query.dateRange else None+ if days_of_week and _interval(query) == IntervalType.DAY:+ # Trends drops the buckets of the days that the filter leaves out.+ count = math.ceil(count * 7 / len(set(days_of_week)))+ return count+++def build_scoring_query(query: TrendsQuery, buckets: int, limits: ScoringLimits = DEFAULT_LIMITS) -> TrendsQuery:+ """Fetch the newest ``buckets`` completed buckets instead of the chart's date range.Series, filters and formulas stay as they are, so the scores describe the series the chart shows."""days_of_week = query.dateRange.daysOfWeek if query.dateRange else Noneupdate: dict[str, Any] = {- "dateRange": DateRange(date_from=scoring.lookback, daysOfWeek=days_of_week),+ "dateRange": DateRange(date_from=f"-{buckets}{_DATE_UNITS[_interval(query)]}", daysOfWeek=days_of_week),"compareFilter": None,}if has_breakdown_filter(query.breakdownFilter):@@ -133,6 +207,17 @@ def build_scoring_query(return query.model_copy(update=update)+def scoring_query_settings(queries: int) -> HogQLGlobalSettings:+ """Settings for each of the ``queries`` queries of one insight, so that all of them stop within the shared budget."""+ waves = max(1, math.ceil(queries / INSIGHT_QUERY_FANOUT_CONCURRENCY))+ return HogQLGlobalSettings(+ max_execution_time=min(_MAX_QUERY_SECONDS, SCORING_QUERY_BUDGET_SECONDS // waves),+ max_threads=8,+ # A timeout must throw: a partial result cuts off the end of the series, and that end scores as a drop.+ timeout_overflow_mode="throw",+ )++def score_series(values: Sequence[float],buckets: Sequence[datetime],@@ -184,27 +269,36 @@ def score_insight(query = trends_query_for(insight, profile)interval = _interval(query)scoring = interval_scoring(profile, interval)- scoring_query = build_scoring_query(query, scoring, limits)+ timezone = team.timezone_info+ to_score = buckets_to_score(interval, now=now, after=after, timezone=timezone, limits=limits)+ scoring_query = build_scoring_query(query, fetch_buckets(query, scoring, to_score), limits)+ tag_queries(**get_team_query_tags(team), insight_id=insight.pk)tag_queries(product=Product.SIGNALS, feature=Feature.ALERTING)- calculation = calculate_for_query_based_insight(- insight,+ runner = TrendsQueryRunner(+ query=scoring_query,team=team,- execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS,# No request user. Run as the insight's creator so warehouse access control resolves for them.user=insight.created_by,- query_override=scoring_query.model_dump(exclude_none=True),- analytics_props={"source": EventSource.SELF_DRIVING},- # The async context raises the ClickHouse timeout to minutes. One slow insight must not hold a tick.+ # The async context raises the timeout above scoring_query_settings. One slow insight must not hold a tick.limit_context=LimitContext.QUERY,+ workload=Workload.OFFLINE,+ ch_user=ClickHouseUser.DEFAULT,)- if calculation.result is None:+ # The runner expands breakdown cohorts and compare periods into the series, so its count is the query count.+ runner.hogql_settings = scoring_query_settings(len(runner.series))+ response = runner.run(+ execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS,+ insight_id=insight.pk,+ analytics_props={"source": EventSource.SELF_DRIVING},+ )+ results = getattr(response, "results", None)+ if not isinstance(results, list):raise RuntimeError(f"No results for insight {insight.pk}")- timezone = team.timezone_infoseries: list[ScoredSeries] = []key_counts: dict[str, int] = {}- for result in cast(list[dict[str, Any]], calculation.result)[: limits.max_series]:+ for result in cast(list[dict[str, Any]], results)[: limits.max_series]:buckets = [_parse_bucket(day, timezone) for day in result.get("days") or []]breakdown_value = _breakdown_label(result.get("breakdown_value"))key = series_key(insight.pk, _series_definition(scoring_query, result), breakdown_value)diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.pyindex 67f5dc219f96..9a51f3616e22 100644--- a/products/signals/backend/anomaly_scores/tick.py+++ b/products/signals/backend/anomaly_scores/tick.py@@ -4,6 +4,7 @@ The `anomaly-scoring` flag payload holds every lever, and a missing or malformedtick off. Each insight activity scores one insight and writes its new buckets as `$anomaly_score` events."""+import timeimport asyncioimport datetimefrom collections import Counter@@ -15,7 +16,15 @@ import structlogfrom temporalio import activity, workflowfrom temporalio.common import RetryPolicy+from posthog.hogql.errors import TableAccessDeniedError+from posthog.dataclasses import frozen+from posthog.exceptions import (+ ClickHouseClusterMemoryLimitExceeded,+ ClickHouseEstimatedQueryExecutionTimeTooLong,+ ClickHouseQueryMemoryLimitExceeded,+ ClickHouseQueryTimeOut,+)from posthog.models import Teamfrom posthog.ph_client import ph_background_capturefrom posthog.sync import database_sync_to_async@@ -32,14 +41,19 @@ from products.signals.backend.anomaly_scores.config import (read_payload,)from products.signals.backend.anomaly_scores.emission import emit, score_events+from products.signals.backend.anomaly_scores.metrics import record_candidates, record_pickfrom products.signals.backend.anomaly_scores.queue import (ATTEMPT_COOLDOWN,- UNSUPPORTED_COOLDOWN,+ SKIP_COOLDOWN,InsightPick,mark_attempted,select_insights,)-from products.signals.backend.anomaly_scores.scoring import UnsupportedInsightError, score_insight+from products.signals.backend.anomaly_scores.scoring import (+ SCORING_QUERY_BUDGET_SECONDS,+ UnsupportedInsightError,+ score_insight,+)from products.signals.backend.anomaly_scores.selection import parse_selection_configlogger = structlog.get_logger(__name__)@@ -51,11 +65,23 @@ STATUS_SCORED = "scored"STATUS_UP_TO_DATE = "up_to_date"STATUS_UNSUPPORTED = "unsupported"STATUS_MISSING = "missing"+# The scoring query hit a ClickHouse time or memory limit. A retry in an hour hits it again.+STATUS_TOO_EXPENSIVE = "too_expensive"+# The insight's creator cannot read a table of the query.+STATUS_ACCESS_DENIED = "access_denied"+# Any other failure. The activity raises it, and the workflow counts it as `errored`.+STATUS_ERRORED = "errored"_PLAN_TIMEOUT = datetime.timedelta(minutes=2)-# The scoring query times out at 60 seconds. The rest is detection and the capture request.-_INSIGHT_TIMEOUT = datetime.timedelta(minutes=3)+# All the scoring queries of one insight stop within their budget. The rest is detection, the capture request,+# and the failure classification. A query timeout must reach the classification, so that it gets the 1 day cooldown.+_INSIGHT_TIMEOUT = datetime.timedelta(seconds=SCORING_QUERY_BUDGET_SECONDS) + datetime.timedelta(seconds=90)_INSIGHT_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)+_CAPTURE_TIMEOUT = datetime.timedelta(minutes=1)+# No batch of insights starts after this time. The picks that are left wait for a later tick.+TICK_BUDGET = datetime.timedelta(minutes=10)+# The last batch can start at the budget, and the summary capture runs after it.+TICK_TIMEOUT = _PLAN_TIMEOUT + TICK_BUDGET + _INSIGHT_TIMEOUT + _CAPTURE_TIMEOUT@frozen@@ -83,6 +109,7 @@ class ScoreInsightInput:class ScoreInsightResult:status: strevents: int+ series: int = 0@frozen@@ -93,6 +120,9 @@ class AnomalyScoringTickResult:statuses: dict[str, int] | None = Noneerrored: int = 0events: int = 0+ duration_ms: int = 0+ # Picks left for a later tick because the tick budget ran out.+ deferred: int = 0def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:@@ -102,6 +132,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlif not config.enabled:return AnomalyScoringTickPlan(enabled=False, picks=[], candidates=0, max_concurrency=config.max_concurrency)selection = select_insights(config, parse_selection_config(payload), now, limit or config.max_insights_per_tick)+ record_candidates(selection.candidates)return AnomalyScoringTickPlan(enabled=True,picks=selection.picks,@@ -111,7 +142,65 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl)+def failure_status(error: Exception) -> str | None:+ """The status of a failure that the next tick does not fix, or None for any other failure."""+ if isinstance(error, UnsupportedInsightError):+ return STATUS_UNSUPPORTED+ # The cluster limit is a subclass of the query limit, but it is load from other queries, so a later tick can pass.+ if isinstance(error, ClickHouseClusterMemoryLimitExceeded):+ return None+ if isinstance(+ error,+ ClickHouseQueryTimeOut | ClickHouseQueryMemoryLimitExceeded | ClickHouseEstimatedQueryExecutionTimeTooLong,+ ):+ return STATUS_TOO_EXPENSIVE+ if isinstance(error, TableAccessDeniedError):+ return STATUS_ACCESS_DENIED+ return None++def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsightResult:+ """Score one pick, and log and measure its outcome. A known failure returns its status, and any other one raises."""+ pick = input.pick+ interval = pick.interval.value if pick.interval else "unknown"+ started = time.monotonic()+ try:+ result = _score_pick(input, now)+ except Exception as error:+ status = failure_status(error)+ duration_ms = round((time.monotonic() - started) * 1000)+ log = logger.info if status else logger.exception+ log(+ "anomaly_scoring_insight_failed",+ team_id=pick.team_id,+ insight_id=pick.insight_id,+ interval=interval,+ status=status or STATUS_ERRORED,+ error_class=type(error).__name__,+ error=str(error)[:200],+ duration_ms=duration_ms,+ )+ record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0)+ if status is None:+ raise+ mark_attempted(pick.insight_id, SKIP_COOLDOWN)+ return ScoreInsightResult(status=status, events=0)+ duration_ms = round((time.monotonic() - started) * 1000)+ logger.info(+ "anomaly_scoring_insight_scored",+ team_id=pick.team_id,+ insight_id=pick.insight_id,+ interval=interval,+ status=result.status,+ series=result.series,+ events=result.events,+ duration_ms=duration_ms,+ )+ record_pick(status=result.status, interval=interval, duration_ms=duration_ms, events=result.events)+ return result+++def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsightResult:pick = input.pickinsight = next((@@ -126,24 +215,21 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsighteam = Team.objects.get(id=pick.team_id)# Set before scoring, so a failure also waits for the cooldown instead of taking a slot on every tick.mark_attempted(pick.insight_id, ATTEMPT_COOLDOWN)- try:- scores = score_insight(- insight,- team,- now=now,- # Every series is filtered again on its own newest bucket below. A new series gets up to the backfill cap.- after=min(pick.scored_through.values()) if pick.scored_through else None,- limits=input.limits,- )- except UnsupportedInsightError as error:- logger.info("anomaly_scoring_insight_unsupported", insight_id=pick.insight_id, reason=str(error))- mark_attempted(pick.insight_id, UNSUPPORTED_COOLDOWN)- return ScoreInsightResult(status=STATUS_UNSUPPORTED, events=0)+ scores = score_insight(+ insight,+ team,+ now=now,+ # Every series is filtered again on its own newest bucket below. A new series gets up to the backfill cap.+ after=min(pick.scored_through.values()) if pick.scored_through else None,+ limits=input.limits,+ )events = score_events(team_id=team.id, insight_id=insight.id, scores=scores, scored_through=pick.scored_through, scored_at=now)emit(team, events)- return ScoreInsightResult(status=STATUS_SCORED if events else STATUS_UP_TO_DATE, events=len(events))+ return ScoreInsightResult(+ status=STATUS_SCORED if events else STATUS_UP_TO_DATE, events=len(events), series=len(scores.series)+ )def capture_tick(result: AnomalyScoringTickResult) -> None:@@ -154,6 +240,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:statuses=result.statuses,errored=result.errored,events=result.events,+ duration_ms=result.duration_ms,+ deferred=result.deferred,)ph_background_capture()(distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,@@ -165,6 +253,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:**{f"status_{status}": count for status, count in (result.statuses or {}).items()},"errored": result.errored,"events": result.events,+ "duration_ms": result.duration_ms,+ "deferred": result.deferred,},)@@ -195,6 +285,7 @@ async def capture_anomaly_scoring_tick_activity(result: AnomalyScoringTickResultclass AnomalyScoringTickWorkflow:@workflow.runasync def run(self, inputs: AnomalyScoringTickInput) -> AnomalyScoringTickResult:+ started = workflow.now()plan = await workflow.execute_activity(plan_anomaly_scoring_tick_activity,inputs,@@ -207,8 +298,12 @@ class AnomalyScoringTickWorkflow:statuses: Counter[str] = Counter()errored = 0events = 0+ deferred = 0concurrency = max(1, plan.max_concurrency)for index in range(0, len(plan.picks), concurrency):+ if workflow.now() - started >= TICK_BUDGET:+ deferred = len(plan.picks) - index+ breakresults = await asyncio.gather(*(workflow.execute_activity(@@ -237,11 +332,13 @@ class AnomalyScoringTickWorkflow:statuses=dict(statuses),errored=errored,events=events,+ duration_ms=round((workflow.now() - started).total_seconds() * 1000),+ deferred=deferred,)await workflow.execute_activity(capture_anomaly_scoring_tick_activity,result,- start_to_close_timeout=datetime.timedelta(minutes=1),+ start_to_close_timeout=_CAPTURE_TIMEOUT,retry_policy=RetryPolicy(maximum_attempts=2),)return resultdiff --git a/products/signals/backend/test/test_anomaly_scores.py b/products/signals/backend/test/test_anomaly_scores.pyindex aa7e9bec54d8..b915b1cf6637 100644--- a/products/signals/backend/test/test_anomaly_scores.py+++ b/products/signals/backend/test/test_anomaly_scores.py@@ -20,6 +20,8 @@ from posthog.schema import (TrendsQuery,)+from posthog.hogql.constants import INSIGHT_QUERY_FANOUT_CONCURRENCY+from posthog.interval_specs import interval_specfrom posthog.models import Team@@ -27,15 +29,17 @@ from products.product_analytics.backend.facade.models import Insightfrom products.signals.backend.anomaly_scores.config import DEFAULT_LIMITS, DEFAULT_PROFILE, IntervalScoringfrom products.signals.backend.anomaly_scores.scoring import (UnsupportedInsightError,+ buckets_to_score,+ fetch_buckets,score_insight,score_series,trends_query_for,)+from products.signals.backend.anomaly_scores.tick import _INSIGHT_TIMEOUTNEW_YORK = ZoneInfo("America/New_York")DAY_SCORING = DEFAULT_PROFILE.intervals[IntervalType.DAY]ZSCORE_OR_KNN = IntervalScoring(- lookback="-365d",min_points=56,detector={"type": "ensemble",@@ -114,6 +118,49 @@ class TestScoreSeries(SimpleTestCase):assert points[-1].bucket_start == buckets[last_completed]+ @parameterized.expand(+ [+ ("first_score", None, DEFAULT_LIMITS.max_backfill_buckets, None),+ ("revisit", 3, 3, None),+ ("chart_smoothing", 3, 3, 7),+ ]+ )+ def test_the_fetched_buckets_score_like_the_full_history(+ self, _name: str, scored_days_ago: int | None, to_score: int, smoothing: int | None+ ) -> None:+ values = list(np.random.default_rng(1).normal(100, 20, size=365))+ buckets = _buckets(datetime(2025, 10, 1, tzinfo=UTC), IntervalType.DAY, 365)+ now = buckets[-1] + timedelta(days=1, hours=1)+ after = None if scored_days_ago is None else buckets[-1 - scored_days_ago]+ query = TrendsQuery(+ series=[EventsNode(event="$pageview")], trendsFilter=TrendsFilter(smoothingIntervals=smoothing)+ )++ assert buckets_to_score(IntervalType.DAY, now=now, after=after, timezone=ZoneInfo("UTC")) == to_score+ fetched = fetch_buckets(query, DAY_SCORING, to_score)++ def score(start: int) -> list[float | None]:+ fetched_values = values[start:]+ if smoothing:+ # Trends averages the trailing buckets that the query fetched, so the fetch start shifts the average.+ fetched_values = [+ float(np.floor(np.mean(fetched_values[max(0, i - smoothing + 1) : i + 1])))+ for i in range(len(fetched_values))+ ]+ points = score_series(+ fetched_values,+ buckets[start:],+ scoring=DAY_SCORING,+ interval=IntervalType.DAY,+ now=now,+ after=after,+ )+ return [point.score for point in points]++ assert fetched < 100+ assert score(-fetched) == score(0)+ assert None not in score(-fetched)+def test_warm_up_points_have_no_score_and_no_flag(self) -> None:buckets = _buckets(datetime(2026, 1, 1, tzinfo=UTC), IntervalType.DAY, 10)@@ -140,10 +187,8 @@ def _result(order: int, label: str) -> dict[str, Any]:class TestScoreInsight(SimpleTestCase):def _keys(self, insight: Insight, results: list[dict[str, Any]]) -> dict[str, str]:- with patch(- "products.signals.backend.anomaly_scores.scoring.calculate_for_query_based_insight",- return_value=MagicMock(result=results),- ):+ with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:+ runner.return_value.run.return_value = MagicMock(results=results)scores = score_insight(insight, Team(timezone="UTC"), now=datetime(2026, 1, 10, tzinfo=UTC))return {series.label: series.series_key for series in scores.series}@@ -175,6 +220,36 @@ class TestScoreInsight(SimpleTestCase):assert named["web"] == single["web"]assert named["app"] != named["web"]+ @parameterized.expand([("one_query", 1), ("one_full_wave", 10), ("two_waves", 11), ("many_waves", 120)])+ def test_every_query_wave_of_an_insight_stops_before_the_activity_timeout(self, _name: str, queries: int) -> None:+ limits: list[int] = []+ insight = _insight([EventsNode(event=f"event_{index}") for index in range(queries)])+ with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:+ runner.return_value.series = [object()] * queries++ def run(**_kwargs: Any) -> MagicMock:+ limits.append(runner.return_value.hogql_settings.max_execution_time)+ return MagicMock(results=[])++ runner.return_value.run.side_effect = run+ score_insight(insight, Team(timezone="UTC"), now=datetime(2026, 1, 10, tzinfo=UTC))++ waves = -(-queries // INSIGHT_QUERY_FANOUT_CONCURRENCY)+ # Each wave can run to the time limit, and the activity still needs time to classify the timeout.+ assert timedelta(seconds=waves * limits[0]) <= _INSIGHT_TIMEOUT - timedelta(minutes=1)+ if waves == 1:+ assert limits[0] == 150+++class TestFetchBuckets(SimpleTestCase):+ def test_unbounded_smoothing_is_not_scored(self) -> None:+ query = TrendsQuery(+ series=[EventsNode(event="$pageview")], trendsFilter=TrendsFilter(smoothingIntervals=10_000_000)+ )++ with self.assertRaises(UnsupportedInsightError):+ fetch_buckets(query, DAY_SCORING, 1)+class TestTrendsQueryFor(SimpleTestCase):@parameterized.expand(@@ -194,6 +269,13 @@ class TestTrendsQueryFor(SimpleTestCase):trendsFilter=TrendsFilter(display=ChartDisplayType.ACTIONS_LINE_GRAPH_CUMULATIVE),).model_dump(exclude_none=True),),+ (+ "calendar_heatmap",+ TrendsQuery(+ series=[EventsNode(event="$pageview")],+ trendsFilter=TrendsFilter(display=ChartDisplayType.CALENDAR_HEATMAP),+ ).model_dump(exclude_none=True),+ ),("minute_interval",TrendsQuery(series=[EventsNode(event="$pageview")], interval=IntervalType.MINUTE).model_dump(diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.pyindex 76d7561939c2..61773af34882 100644--- a/products/signals/backend/test/test_anomaly_scoring_tick.py+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py@@ -4,7 +4,7 @@ from zoneinfo import ZoneInfoimport time_machinefrom posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event-from unittest.mock import patch+from unittest.mock import MagicMock, patchfrom django.core.cache import cachefrom django.test import SimpleTestCase@@ -13,6 +13,9 @@ from parameterized import parameterizedfrom posthog.schema import IntervalType+from posthog.hogql.errors import TableAccessDeniedError++from posthog.exceptions import ClickHouseClusterMemoryLimitExceeded, ClickHouseQueryTimeOutfrom posthog.models import Organization, Team, Userfrom products.dashboards.backend.models.dashboard import Dashboard@@ -22,6 +25,8 @@ from products.product_analytics.backend.facade.models import Insightfrom products.signals.backend.anomaly_scores.config import DISABLED, AnomalyScoringConfig, parse_configfrom products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_eventsfrom products.signals.backend.anomaly_scores.queue import (+ SKIP_COOLDOWN,+ InsightPick,WatchedInsight,_cooldown_key,eligible_teams,@@ -30,9 +35,14 @@ from products.signals.backend.anomaly_scores.queue import (select_insights,watched_insights,)-from products.signals.backend.anomaly_scores.scoring import InsightScores, ScoredPoint, ScoredSeries+from products.signals.backend.anomaly_scores.scoring import (+ InsightScores,+ ScoredPoint,+ ScoredSeries,+ UnsupportedInsightError,+)from products.signals.backend.anomaly_scores.selection import SelectionConfig, Weighting-from products.signals.backend.anomaly_scores.tick import plan_tick+from products.signals.backend.anomaly_scores.tick import ScoreInsightInput, plan_tick, run_insightNEW_YORK = ZoneInfo("America/New_York")ORGANIZATION_ID = "4dc8564d-bd82-1065-2f40-97f7c50f67cf"@@ -225,6 +235,59 @@ class TestScoreEvents(SimpleTestCase):]+class TestRunInsight(SimpleTestCase):+ def _run(self, error: Exception) -> tuple[str, list[Any], MagicMock]:+ tick = "products.signals.backend.anomaly_scores.tick"+ client = MagicMock()+ with (+ patch(+ f"{tick}.insights_including_soft_deleted_for_team",+ return_value=[Insight(id=7, saved=True, deleted=False)],+ ),+ patch(f"{tick}.Team"),+ patch(f"{tick}.score_insight", side_effect=error),+ patch(f"{tick}.mark_attempted") as mark_attempted,+ patch("products.signals.backend.anomaly_scores.metrics.posthoganalytics.default_client", client),+ ):+ pick = InsightPick(team_id=1, insight_id=7, scored_through={}, interval=IntervalType.DAY)+ try:+ status = run_insight(ScoreInsightInput(pick=pick), NOW).status+ except Exception:+ status = "raised"+ cooldowns = [call.args[1] for call in mark_attempted.call_args_list]+ return status, cooldowns, client.metrics++ @parameterized.expand(+ [+ ("timeout", ClickHouseQueryTimeOut(), "too_expensive"),+ ("access_denied", TableAccessDeniedError("no access"), "access_denied"),+ ("unsupported", UnsupportedInsightError("not trends"), "unsupported"),+ ]+ )+ def test_a_known_failure_returns_its_status_and_waits_a_day(+ self, _name: str, error: Exception, expected: str+ ) -> None:+ status, cooldowns, metrics = self._run(error)++ assert status == expected+ assert cooldowns[-1] == SKIP_COOLDOWN+ metrics.count.assert_any_call(+ "anomaly_scoring.picks", 1, attributes={"status": expected, "interval": IntervalType.DAY.value}+ )++ @parameterized.expand(+ [+ ("cluster_memory", ClickHouseClusterMemoryLimitExceeded()),+ ("unknown", RuntimeError("boom")),+ ]+ )+ def test_any_other_failure_raises_and_waits_an_hour(self, _name: str, error: Exception) -> None:+ status, cooldowns, _metrics = self._run(error)++ assert status == "raised"+ assert SKIP_COOLDOWN not in cooldowns++class TestEligibleTeams(BaseTest):@parameterized.expand([