:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): score trends insight series with existing anomaly detectors (#121942)
posthog[bot]44f20a9dd2bf
diff --git a/products/signals/backend/anomaly_scores/__init__.py b/products/signals/backend/anomaly_scores/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.pynew file mode 100644index 000000000000..1ccfb1511f7b--- /dev/null+++ b/products/signals/backend/anomaly_scores/config.py@@ -0,0 +1,61 @@+from typing import TYPE_CHECKING, Any++from posthog.schema import IntervalType++from posthog.dataclasses import frozen++if TYPE_CHECKING:+ from posthog.models import Team++ from products.product_analytics.backend.facade.models import Insight+++@frozen+class IntervalScoring:+ """How one interval is scored. The detector is any registered detector config, so swapping it needs no code."""++ lookback: str+ min_points: int+ detector: dict[str, Any]+++@frozen+class ScoringProfile:+ name: str+ intervals: dict[IntervalType, IntervalScoring]++ def for_interval(self, interval: IntervalType) -> IntervalScoring | None:+ return self.intervals.get(interval)+++@frozen+class ScoringLimits:+ """Caps that bound the cost of scoring one insight. Every value is a plain field, so a flag payload can override it."""++ max_breakdown_values: int = 10+ max_series: int = 20+ max_backfill_buckets: int = 30+++def _zscore(window: int) -> dict[str, Any]:+ # diffs_n is set explicitly: when preprocessing is given without it, the detector scores raw values,+ # and on sparse series raw values score almost every non-zero bucket near 1.0.+ return {"type": "zscore", "threshold": 0.95, "window": window, "preprocessing": {"diffs_n": 1, "smooth_n": 3}}+++DEFAULT_PROFILE = ScoringProfile(+ name="default-v1",+ intervals={+ IntervalType.HOUR: IntervalScoring(lookback="-28d", min_points=336, detector=_zscore(336)),+ IntervalType.DAY: IntervalScoring(lookback="-365d", min_points=56, detector=_zscore(56)),+ IntervalType.WEEK: IntervalScoring(lookback="-156w", min_points=52, detector=_zscore(52)),+ IntervalType.MONTH: IntervalScoring(lookback="-60m", min_points=24, detector=_zscore(24)),+ },+)++DEFAULT_LIMITS = ScoringLimits()+++def resolve_profile(team: "Team", insight: "Insight") -> ScoringProfile:+ """The one place that picks scoring settings for an insight, so project and per-metric overrides only change this."""+ return DEFAULT_PROFILEdiff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.pynew file mode 100644index 000000000000..a6a1b667aef3--- /dev/null+++ b/products/signals/backend/anomaly_scores/scoring.py@@ -0,0 +1,298 @@+import json+import hashlib+import dataclasses+from collections.abc import Sequence+from datetime import UTC, datetime, timedelta+from typing import Any, cast+from zoneinfo import ZoneInfo++import numpy as np++from posthog.schema import BreakdownFilter, ChartDisplayType, DateRange, IntervalType, TrendsQuery++from posthog.api.services.query import ExecutionMode+from posthog.caching.calculate_results import calculate_for_query_based_insight+from posthog.clickhouse.query_tagging import Feature, Product, tag_queries+from posthog.dataclasses import frozen+from posthog.event_usage import EventSource+from posthog.hogql_queries.utils.breakdowns import has_breakdown_filter+from posthog.interval_specs import interval_spec+from posthog.models import Team+from posthog.schema_migrations.upgrade_manager import upgrade_insight+from posthog.tasks.alerts.detectors.base import DetectionResult+from posthog.tasks.alerts.detectors.registry import get_detector+from posthog.tasks.alerts.utils import WRAPPER_NODE_KINDS, is_non_time_series_trend++from products.product_analytics.backend.facade.models import Insight+from products.signals.backend.anomaly_scores.config import (+ DEFAULT_LIMITS,+ IntervalScoring,+ ScoringLimits,+ ScoringProfile,+ resolve_profile,+)++# Naming fields do not change what a series counts, so a rename keeps its key.+_SERIES_NAMING_FIELDS = {"custom_name", "name"}+++class UnsupportedInsightError(Exception):+ """The insight is not a shape we score. The caller skips it; it is not an error."""+++@frozen+class ScoredPoint:+ bucket_start: datetime+ value: float+ # Both are None during warm-up, so an average of the flag stays a true anomaly rate.+ score: float | None+ flag: bool | None+ details: dict[str, Any]+++@frozen+class ScoredSeries:+ series_key: str+ label: str+ breakdown_value: str | None+ points: list[ScoredPoint]+++@frozen+class InsightScores:+ interval: IntervalType+ profile: str+ detector: str+ detector_version: str+ series: list[ScoredSeries]+++def _short_hash(payload: Any) -> str:+ canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)+ return hashlib.sha256(canonical.encode()).hexdigest()[:16]+++def detector_version(scoring: IntervalScoring) -> str:+ """Identifies the scoring behavior, so scores from different settings are never mixed in one series."""+ return _short_hash(dataclasses.asdict(scoring))+++def series_key(insight_id: int, series_definition: dict[str, Any], breakdown_value: str | None) -> str:+ return _short_hash({"insight": insight_id, "series": series_definition, "breakdown": breakdown_value})+++def trends_query_for(insight: Insight, profile: ScoringProfile) -> TrendsQuery:+ if insight.query is None:+ raise UnsupportedInsightError("Insight has no query")+ with upgrade_insight(insight):+ query = insight.query+ if query.get("kind") in WRAPPER_NODE_KINDS:+ query = query.get("source") or {}+ if query.get("kind") != "TrendsQuery":+ raise UnsupportedInsightError(f"Only trends insights are scored, not {query.get('kind')}")+ trends = TrendsQuery.model_validate(query)+ if is_non_time_series_trend(trends):+ raise UnsupportedInsightError("Insight shows a single value, not a time series")+ if trends.trendsFilter and trends.trendsFilter.display == ChartDisplayType.ACTIONS_LINE_GRAPH_CUMULATIVE:+ raise UnsupportedInsightError("Cumulative insights are not scored")+ interval_scoring(profile, _interval(trends))+ return trends+++def interval_scoring(profile: ScoringProfile, interval: IntervalType) -> IntervalScoring:+ scoring = profile.for_interval(interval)+ if scoring is None:+ raise UnsupportedInsightError(f"Interval {interval.value} is not scored")+ return scoring+++def build_scoring_query(+ query: TrendsQuery, scoring: IntervalScoring, limits: ScoringLimits = DEFAULT_LIMITS+) -> TrendsQuery:+ """Fetch the scorer's lookback instead of the chart's date range.++ Series, filters and formulas stay as they are, so the scores describe the series the chart shows.+ """+ days_of_week = query.dateRange.daysOfWeek if query.dateRange else None+ update: dict[str, Any] = {+ "dateRange": DateRange(date_from=scoring.lookback, daysOfWeek=days_of_week),+ "compareFilter": None,+ }+ if has_breakdown_filter(query.breakdownFilter):+ lines_per_value = len(_formulas(query)) or len(query.series)+ breakdown_filter = query.breakdownFilter or BreakdownFilter()+ # "Other" mixes whichever values fall outside the top N on each run, so it is not one series.+ update["breakdownFilter"] = breakdown_filter.model_copy(+ update={+ "breakdown_limit": max(1, min(limits.max_breakdown_values, limits.max_series // lines_per_value)),+ "breakdown_hide_other_aggregation": True,+ }+ )+ return query.model_copy(update=update)+++def score_series(+ values: Sequence[float],+ buckets: Sequence[datetime],+ *,+ scoring: IntervalScoring,+ interval: IntervalType,+ now: datetime,+ after: datetime | None = None,+ limits: ScoringLimits = DEFAULT_LIMITS,+) -> list[ScoredPoint]:+ """Score one series and return its newest completed buckets later than ``after``.++ ``buckets`` are aware bucket starts in the project timezone.+ """+ if len(values) != len(buckets):+ raise ValueError("Each value needs a bucket")+ completed = [i for i, start in enumerate(buckets) if _bucket_end(start, interval) <= now]+ returned = [i for i in completed if after is None or buckets[i] > after][-limits.max_backfill_buckets :]+ detector = get_detector(scoring.detector)+ data = np.array(values, dtype=float)++ points: list[ScoredPoint] = []+ for i in returned:+ # Score each bucket on the history up to it, so the ongoing bucket and later buckets never shape the score.+ # detect() fits once per call; detect_batch() refits at every point of the lookback for model-based detectors.+ result = detector.detect(data[: i + 1]) if i + 1 >= scoring.min_points else None+ score = None if result is None or result.score is None else float(result.score)+ points.append(+ ScoredPoint(+ bucket_start=buckets[i],+ value=float(values[i]),+ score=score,+ flag=None if result is None or score is None else result.is_anomaly,+ details={} if result is None or score is None else _details(result),+ )+ )+ return points+++def score_insight(+ insight: Insight,+ team: Team,+ *,+ now: datetime,+ after: datetime | None = None,+ limits: ScoringLimits = DEFAULT_LIMITS,+) -> InsightScores:+ profile = resolve_profile(team, insight)+ query = trends_query_for(insight, profile)+ interval = _interval(query)+ scoring = interval_scoring(profile, interval)+ scoring_query = build_scoring_query(query, scoring, limits)++ tag_queries(product=Product.SIGNALS, feature=Feature.ALERTING)+ calculation = calculate_for_query_based_insight(+ insight,+ team=team,+ execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS,+ # No request user. Run as the insight's creator so warehouse access control resolves for them.+ user=insight.created_by,+ query_override=scoring_query.model_dump(exclude_none=True),+ analytics_props={"source": EventSource.SELF_DRIVING},+ )+ if calculation.result is None:+ raise RuntimeError(f"No results for insight {insight.pk}")++ timezone = team.timezone_info+ series: list[ScoredSeries] = []+ for result in cast(list[dict[str, Any]], calculation.result)[: limits.max_series]:+ buckets = [_parse_bucket(day, timezone) for day in result.get("days") or []]+ breakdown_value = _breakdown_label(result.get("breakdown_value"))+ series.append(+ ScoredSeries(+ series_key=series_key(insight.pk, _series_definition(scoring_query, result), breakdown_value),+ label=str(result.get("label") or ""),+ breakdown_value=breakdown_value,+ points=score_series(+ result.get("data") or [],+ buckets,+ scoring=scoring,+ interval=interval,+ now=now,+ after=after,+ limits=limits,+ ),+ )+ )+ return InsightScores(+ interval=interval,+ profile=profile.name,+ detector=str(scoring.detector["type"]),+ detector_version=detector_version(scoring),+ series=series,+ )+++def _formulas(query: TrendsQuery) -> list[str]:+ trends_filter = query.trendsFilter+ if trends_filter is None:+ return []+ if trends_filter.formulaNodes:+ return [node.formula for node in trends_filter.formulaNodes]+ if trends_filter.formulas:+ return list(trends_filter.formulas)+ return [trends_filter.formula] if trends_filter.formula else []+++def _series_node_definition(node: Any) -> dict[str, Any]:+ return node.model_dump(exclude_none=True, exclude=_SERIES_NAMING_FIELDS)+++def _series_definition(query: TrendsQuery, result: dict[str, Any]) -> dict[str, Any]:+ """What a result line counts, independent of where its series sits in the query."""+ formulas = _formulas(query)+ if formulas:+ # A formula refers to series by letter, so its meaning includes every series in order.+ order = result.get("order") or 0+ return {+ "formula": formulas[order] if order < len(formulas) else None,+ "series": [_series_node_definition(node) for node in query.series],+ }+ order = (result.get("action") or {}).get("order")+ if order is None or order >= len(query.series):+ return {"label": result.get("label")}+ return _series_node_definition(query.series[order])+++def _interval(query: TrendsQuery) -> IntervalType:+ return query.interval or IntervalType.DAY+++def _bucket_end(start: datetime, interval: IntervalType) -> datetime:+ if interval == IntervalType.HOUR:+ # An hour is elapsed time, so step in UTC to stay one hour long across DST changes.+ return (start.astimezone(UTC) + timedelta(hours=1)).astimezone(start.tzinfo)+ # Wall-clock arithmetic on an aware datetime, so a day stays one calendar day across DST changes.+ return start + interval_spec(interval).period+++def _details(result: DetectionResult) -> dict[str, Any]:+ """Detector-specific values for one bucket. Only scalars are kept, so the event property stays small."""+ sub_results = result.metadata.get("sub_results")+ if isinstance(sub_results, list):+ return {+ "sub_scores": {+ f"{sub.get('type')}_{position}": sub.get("score") for position, sub in enumerate(sub_results)+ }+ }+ return {+ key: value for key, value in result.metadata.items() if value is None or isinstance(value, int | float | str)+ }+++def _parse_bucket(day: str, timezone: ZoneInfo) -> datetime:+ # Trends returns bucket starts in the project's timezone, usually as naive strings.+ parsed = datetime.fromisoformat(day)+ return parsed.astimezone(timezone) if parsed.tzinfo else parsed.replace(tzinfo=timezone)+++def _breakdown_label(value: Any) -> str | None:+ if value is None:+ return None+ if isinstance(value, list):+ return "::".join(str(part) for part in value)+ return str(value)diff --git a/products/signals/backend/management/commands/score_insight_anomalies.py b/products/signals/backend/management/commands/score_insight_anomalies.pynew file mode 100644index 000000000000..8ce509ec9aee--- /dev/null+++ b/products/signals/backend/management/commands/score_insight_anomalies.py@@ -0,0 +1,50 @@+"""Score a trends insight's series for anomalies and print the result. Writes nothing.++python manage.py score_insight_anomalies --team 1 --insight 123+"""++from typing import Any++from django.core.management.base import BaseCommand, CommandError, CommandParser+from django.utils import timezone++from posthog.models import Team++from products.product_analytics.backend.facade.api import insights_including_soft_deleted_for_team+from products.signals.backend.anomaly_scores.scoring import InsightScores, UnsupportedInsightError, score_insight+++class Command(BaseCommand):+ help = "Score a trends insight's series for anomalies and print value, score, flag and details per bucket"++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument("--team", type=int, required=True, help="Team id")+ parser.add_argument("--insight", type=int, required=True, help="Insight id")++ def handle(self, *args: Any, **options: Any) -> None:+ team = Team.objects.filter(pk=options["team"]).first()+ if team is None:+ raise CommandError(f"Team {options['team']} not found")+ insights = insights_including_soft_deleted_for_team(team_id=team.pk, insight_ids=[options["insight"]])+ insight = next((insight for insight in insights if not insight.deleted), None)+ if insight is None:+ raise CommandError(f"Insight {options['insight']} not found in team {team.pk}")+ try:+ scores = score_insight(insight, team, now=timezone.now())+ except UnsupportedInsightError as err:+ raise CommandError(f"Not scored: {err}")+ self._print(scores)++ def _print(self, scores: InsightScores) -> None:+ self.stdout.write(+ f"interval={scores.interval.value} profile={scores.profile} detector={scores.detector} version={scores.detector_version}"+ )+ for series in scores.series:+ name = series.label if series.breakdown_value is None else f"{series.label} [{series.breakdown_value}]"+ self.stdout.write(f"\nseries {series.series_key} {name}")+ for point in series.points:+ score = "-" if point.score is None else f"{point.score:.3f}"+ flag = "-" if point.flag is None else str(int(point.flag))+ self.stdout.write(+ f" {point.bucket_start.isoformat()} value={point.value:g} score={score} flag={flag} details={point.details}"+ )diff --git a/products/signals/backend/test/test_anomaly_scores.py b/products/signals/backend/test/test_anomaly_scores.pynew file mode 100644index 000000000000..1cb07f09c7cf--- /dev/null+++ b/products/signals/backend/test/test_anomaly_scores.py@@ -0,0 +1,195 @@+from datetime import UTC, datetime, timedelta+from typing import Any+from zoneinfo import ZoneInfo++from unittest.mock import MagicMock, patch++from django.test import SimpleTestCase++import numpy as np+from parameterized import parameterized++from posthog.schema import (+ ChartDisplayType,+ EventPropertyFilter,+ EventsNode,+ FunnelsQuery,+ IntervalType,+ PropertyOperator,+ TrendsFilter,+ TrendsQuery,+)++from posthog.interval_specs import interval_spec+from posthog.models import Team++from products.product_analytics.backend.facade.models import Insight+from products.signals.backend.anomaly_scores.config import DEFAULT_LIMITS, DEFAULT_PROFILE, IntervalScoring+from products.signals.backend.anomaly_scores.scoring import (+ UnsupportedInsightError,+ score_insight,+ score_series,+ trends_query_for,+)++NEW_YORK = ZoneInfo("America/New_York")+DAY_SCORING = DEFAULT_PROFILE.intervals[IntervalType.DAY]+ZSCORE_OR_KNN = IntervalScoring(+ lookback="-365d",+ min_points=56,+ detector={+ "type": "ensemble",+ "operator": "or",+ "detectors": [+ {"type": "zscore", "threshold": 0.95, "window": 56, "preprocessing": {"diffs_n": 1}},+ {"type": "knn", "threshold": 0.95},+ ],+ },+)+++def _buckets(start: datetime, interval: IntervalType, count: int) -> list[datetime]:+ return [start + interval_spec(interval).period * i for i in range(count)]+++class TestScoreSeries(SimpleTestCase):+ @parameterized.expand(+ [+ ("default_profile", DAY_SCORING, {"mean", "std", "value", "raw_zscore"}),+ ("ensemble", ZSCORE_OR_KNN, {"sub_scores"}),+ ]+ )+ def test_flags_the_spike_and_not_its_flat_neighbor(+ self, _name: str, scoring: IntervalScoring, detail_keys: set[str]+ ) -> None:+ values = list(np.random.default_rng(0).normal(100, 5, size=120))+ values[110] = 400.0+ buckets = _buckets(datetime(2026, 1, 1, tzinfo=NEW_YORK), IntervalType.DAY, 120)++ points = score_series(+ values,+ buckets,+ scoring=scoring,+ interval=IntervalType.DAY,+ now=buckets[-1] + timedelta(hours=12),+ )++ by_bucket = {point.bucket_start: point for point in points}+ assert by_bucket[buckets[110]].flag is True+ assert by_bucket[buckets[109]].flag is False+ assert set(by_bucket[buckets[110]].details) == detail_keys+ assert len(points) == DEFAULT_LIMITS.max_backfill_buckets++ @parameterized.expand(+ [+ ("hour", IntervalType.HOUR, datetime(2026, 3, 1, tzinfo=UTC), datetime(2026, 3, 1, 4, 30, tzinfo=UTC), 3),+ ("day", IntervalType.DAY, datetime(2026, 3, 1, tzinfo=UTC), datetime(2026, 3, 5, 23, tzinfo=UTC), 3),+ ("week", IntervalType.WEEK, datetime(2026, 3, 1, tzinfo=UTC), datetime(2026, 3, 29, 1, tzinfo=UTC), 3),+ # 03:00 UTC on Feb 1 is still January in New York, so January is ongoing.+ (+ "month_boundary_new_york",+ IntervalType.MONTH,+ datetime(2025, 9, 1, tzinfo=NEW_YORK),+ datetime(2026, 2, 1, 3, tzinfo=UTC),+ 3,+ ),+ (+ "month_after_boundary_new_york",+ IntervalType.MONTH,+ datetime(2025, 9, 1, tzinfo=NEW_YORK),+ datetime(2026, 2, 1, 6, tzinfo=UTC),+ 4,+ ),+ ]+ )+ def test_never_returns_the_ongoing_bucket(+ self, _name: str, interval: IntervalType, start: datetime, now: datetime, last_completed: int+ ) -> None:+ buckets = _buckets(start, interval, 6)+ buckets = [bucket for bucket in buckets if bucket <= now]++ points = score_series(+ [1.0] * len(buckets), buckets, scoring=DEFAULT_PROFILE.intervals[interval], interval=interval, now=now+ )++ assert points[-1].bucket_start == buckets[last_completed]++ def test_warm_up_points_have_no_score_and_no_flag(self) -> None:+ buckets = _buckets(datetime(2026, 1, 1, tzinfo=UTC), IntervalType.DAY, 10)++ points = score_series(+ [1.0, 2.0, 1.0, 50.0, 1.0, 2.0, 1.0, 2.0, 1.0, 2.0],+ buckets,+ scoring=DAY_SCORING,+ interval=IntervalType.DAY,+ now=buckets[-1] + timedelta(days=1),+ )++ assert len(points) == 10+ assert all(point.score is None and point.flag is None for point in points)+++def _insight(series: list[EventsNode], **query: Any) -> Insight:+ return Insight(id=7, query=TrendsQuery(series=series, **query).model_dump(exclude_none=True))+++def _result(order: int, label: str) -> dict[str, Any]:+ days = [f"2026-01-{day:02d}" for day in range(1, 6)]+ return {"action": {"order": order}, "label": label, "days": days, "data": [1.0] * 5, "breakdown_value": None}+++class TestScoreInsight(SimpleTestCase):+ def _keys(self, insight: Insight, results: list[dict[str, Any]]) -> dict[str, str]:+ with patch(+ "products.signals.backend.anomaly_scores.scoring.calculate_for_query_based_insight",+ return_value=MagicMock(result=results),+ ):+ scores = score_insight(insight, Team(timezone="UTC"), now=datetime(2026, 1, 10, tzinfo=UTC))+ return {series.label: series.series_key for series in scores.series}++ def test_series_key_follows_the_series_definition_not_its_position(self) -> None:+ pageview = EventsNode(event="$pageview")+ signup = EventsNode(event="signed_up")+ chrome_pageview = EventsNode(+ event="$pageview",+ properties=[EventPropertyFilter(key="$browser", value="Chrome", operator=PropertyOperator.EXACT)],+ )++ original = self._keys(_insight([pageview, signup]), [_result(0, "pageview"), _result(1, "signup")])+ swapped = self._keys(_insight([signup, pageview]), [_result(0, "signup"), _result(1, "pageview")])+ filtered = self._keys(_insight([chrome_pageview, signup]), [_result(0, "pageview"), _result(1, "signup")])++ assert swapped == original+ assert filtered["signup"] == original["signup"]+ assert filtered["pageview"] != original["pageview"]+++class TestTrendsQueryFor(SimpleTestCase):+ @parameterized.expand(+ [+ ("funnel", FunnelsQuery(series=[EventsNode(event="$pageview")]).model_dump(exclude_none=True)),+ (+ "single_value",+ TrendsQuery(+ series=[EventsNode(event="$pageview")],+ trendsFilter=TrendsFilter(display=ChartDisplayType.BOLD_NUMBER),+ ).model_dump(exclude_none=True),+ ),+ (+ "cumulative",+ TrendsQuery(+ series=[EventsNode(event="$pageview")],+ trendsFilter=TrendsFilter(display=ChartDisplayType.ACTIONS_LINE_GRAPH_CUMULATIVE),+ ).model_dump(exclude_none=True),+ ),+ (+ "minute_interval",+ TrendsQuery(series=[EventsNode(event="$pageview")], interval=IntervalType.MINUTE).model_dump(+ exclude_none=True+ ),+ ),+ ]+ )+ def test_unsupported_shapes_raise(self, _name: str, query: dict[str, Any]) -> None:+ with self.assertRaises(UnsupportedInsightError):+ trends_query_for(Insight(id=1, query=query), DEFAULT_PROFILE)