:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): train anomaly detectors into stored model artifacts (#123112)
posthog[bot]053a4f952205
diff --git a/frontend/src/taxonomy/core-filter-definitions-by-group.json b/frontend/src/taxonomy/core-filter-definitions-by-group.jsonindex 9dee9bfc3ed5..05f9cf13a10a 100644--- a/frontend/src/taxonomy/core-filter-definitions-by-group.json+++ b/frontend/src/taxonomy/core-filter-definitions-by-group.json@@ -797,6 +797,11 @@"label": "Anomaly interval","type": "String"},+ "$anomaly_model_id": {+ "description": "ID of the stored model that scored the bucket. Buckets with the same ID were scored with the same fit. Empty when scoring fits the detector again on every run.",+ "label": "Anomaly model ID",+ "type": "String"+ },"$anomaly_profile": {"description": "Name of the scoring profile that produced the score.","examples": ["default-v1"],@@ -804,7 +809,7 @@"type": "String"},"$anomaly_run": {- "description": "How the bucket was scored. `train` fetched the full history of the series, and `score` scored the bucket on stored history. Empty when scoring fetches the full history on every run.",+ "description": "How the bucket was scored. `train` fit a new model on the history of the series, and `score` scored the bucket with a stored model. Empty when scoring fits the detector again on every run.","examples": ["train", "score"],"label": "Anomaly scoring run","type": "String"@@ -4745,6 +4750,11 @@"label": "Anomaly interval","type": "String"},+ "$anomaly_model_id": {+ "description": "ID of the stored model that scored the bucket. Buckets with the same ID were scored with the same fit. Empty when scoring fits the detector again on every run.",+ "label": "Anomaly model ID",+ "type": "String"+ },"$anomaly_profile": {"description": "Name of the scoring profile that produced the score.","examples": ["default-v1"],@@ -4752,7 +4762,7 @@"type": "String"},"$anomaly_run": {- "description": "How the bucket was scored. `train` fetched the full history of the series, and `score` scored the bucket on stored history. Empty when scoring fetches the full history on every run.",+ "description": "How the bucket was scored. `train` fit a new model on the history of the series, and `score` scored the bucket with a stored model. Empty when scoring fits the detector again on every run.","examples": ["train", "score"],"label": "Anomaly scoring run","type": "String"diff --git a/posthog/tasks/alerts/detectors/base.py b/posthog/tasks/alerts/detectors/base.pyindex fe0f6543cfe4..26cd5c44801d 100644--- a/posthog/tasks/alerts/detectors/base.py+++ b/posthog/tasks/alerts/detectors/base.py@@ -4,10 +4,14 @@ from typing import Anyimport numpy as np+from posthog.dataclasses import frozenfrom posthog.tasks.alerts.detectors.preprocessing import preprocess_data+# A change to what `fit` stores or what `score` reads needs a new version, so an older artifact is never read.+ARTIFACT_SCHEMA_VERSION = 1-@dataclass++@dataclass(frozen=False)class DetectionResult:"""Result from anomaly detection."""@@ -16,6 +20,27 @@ class DetectionResult:triggered_indices: list[int] = field(default_factory=list)all_scores: list[float | None] = field(default_factory=list)metadata: dict[str, Any] = field(default_factory=dict)+ # One entry per scored bucket of `score()`, with the values that `metadata` holds for `detect()`.+ all_metadata: list[dict[str, Any]] = field(default_factory=list)+++@frozen+class DetectorArtifact:+ """What `fit` learned from a history. `score` reads it instead of fitting again."""++ detector_type: str+ schema_version: int+ # The detector config the artifact was fit with, preprocessing and threshold included.+ config: dict[str, Any]+ # JSON-safe values only.+ params: dict[str, Any]+ model_bytes: bytes | None = None+ # The artifacts of the sub-detectors of an ensemble, in config order.+ children: tuple["DetectorArtifact", ...] = ()+++class FitError(ValueError):+ """The history cannot fit this detector, for example because it is too short."""class BaseDetector(ABC):@@ -65,6 +90,82 @@ class BaseDetector(ABC):"""pass+ def fit(self, history: np.ndarray) -> DetectorArtifact:+ """Fit on ``history``, the buckets before the first bucket to score.++ The newest ``training_offset - 1`` buckets of ``history`` stay out of the fit, so+ ``score(fit(data[:-1]), ...)`` reads the same training data as ``detect(data)``.+ """+ raise NotImplementedError(f"{type(self).__name__} cannot fit an artifact")++ def score(self, artifact: DetectorArtifact, data: np.ndarray) -> DetectionResult:+ """Score buckets with a fitted artifact.++ ``data`` holds ``context_buckets()`` buckets, then the buckets to score. The result has one entry in+ ``all_scores`` and ``all_metadata`` per scored bucket, and ``triggered_indices`` count from the first scored bucket.+ """+ self._check_artifact(artifact)+ context = self.context_buckets()+ if len(data) <= context:+ return DetectionResult(is_anomaly=False)+ processed = self.preprocess(np.asarray(data, dtype=float))[context:]+ scores, metadata = self._score_rows(artifact, processed)+ return self._scored_result(scores, metadata)++ def context_buckets(self) -> int:+ """Buckets that preprocessing reads before the first scored bucket: diffs, lags and smoothing."""+ preprocessing = self.preprocessing_config+ return (+ max((preprocessing.get("smooth_n") or 0) - 1, 0)+ + (1 if preprocessing.get("diffs_n") else 0)+ + min(max(preprocessing.get("lags_n") or 0, 0), 10)+ )++ def _score_rows(+ self, artifact: DetectorArtifact, rows: np.ndarray+ ) -> tuple[list[float | None], list[dict[str, Any]]]:+ """The score and metadata of each preprocessed row."""+ raise NotImplementedError(f"{type(self).__name__} cannot score an artifact")++ def _artifact(+ self,+ params: dict[str, Any],+ *,+ model_bytes: bytes | None = None,+ children: tuple[DetectorArtifact, ...] = (),+ ) -> DetectorArtifact:+ return DetectorArtifact(+ detector_type=str(self.config.get("type")),+ schema_version=ARTIFACT_SCHEMA_VERSION,+ config=self.config,+ params=params,+ model_bytes=model_bytes,+ children=children,+ )++ def _check_artifact(self, artifact: DetectorArtifact) -> None:+ if artifact.schema_version != ARTIFACT_SCHEMA_VERSION or artifact.detector_type != self.config.get("type"):+ raise ValueError(f"Artifact {artifact.detector_type} v{artifact.schema_version} does not fit this detector")++ def _fit_rows(self, history: np.ndarray, min_rows: int) -> np.ndarray:+ """The preprocessed rows of ``history`` that a fit reads, without the newest ``training_offset - 1``."""+ rows = self.preprocess(np.asarray(history, dtype=float))+ rows = rows[: len(rows) - (max(self.training_offset, 1) - 1)]+ if len(rows) < min_rows:+ raise FitError(f"{len(rows)} buckets cannot fit, {min_rows} needed")+ return rows++ def _scored_result(self, scores: list[float | None], metadata: list[dict[str, Any]]) -> DetectionResult:+ threshold = self.config.get("threshold", self.DEFAULT_THRESHOLD)+ triggered = [i for i, score in enumerate(scores) if score is not None and score > threshold]+ return DetectionResult(+ is_anomaly=bool(triggered) and triggered[-1] == len(scores) - 1,+ score=scores[-1] if scores else None,+ triggered_indices=triggered,+ all_scores=scores,+ all_metadata=metadata,+ )+def train_test_split(self, data: np.ndarray) -> tuple[np.ndarray, np.ndarray]:"""Split data into training (historical) and test (recent) portions.diff --git a/posthog/tasks/alerts/detectors/ensemble.py b/posthog/tasks/alerts/detectors/ensemble.pyindex b7e15c1723fa..2249611f168a 100644--- a/posthog/tasks/alerts/detectors/ensemble.py+++ b/posthog/tasks/alerts/detectors/ensemble.py@@ -2,7 +2,7 @@ from typing import Anyimport numpy as np-from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifactfrom posthog.tasks.alerts.detectors.registry import get_detector, register_detector@@ -72,6 +72,54 @@ class EnsembleDetector(BaseDetector):},)+ def fit(self, history: np.ndarray) -> DetectorArtifact:+ return self._artifact(+ {"operator": self.operator}, children=tuple(detector.fit(history) for detector in self.sub_detectors)+ )++ def context_buckets(self) -> int:+ return max(detector.context_buckets() for detector in self.sub_detectors)++ def score(self, artifact: DetectorArtifact, data: np.ndarray) -> DetectionResult:+ self._check_artifact(artifact)+ if len(artifact.children) != len(self.sub_detectors):+ raise ValueError("The artifact has another number of sub-detectors")+ context = self.context_buckets()+ if len(data) <= context:+ return DetectionResult(is_anomaly=False)+ # Each sub-detector reads only the context it needs.+ results = [+ detector.score(child, data[context - detector.context_buckets() :])+ for detector, child in zip(self.sub_detectors, artifact.children)+ ]+ combine = all if self.operator == "and" else any+ pick = min if self.operator == "and" else max+ scores: list[float | None] = []+ flags: list[int] = []+ metadata: list[dict[str, Any]] = []+ for i in range(len(data) - context):+ sub_scores = [result.all_scores[i] for result in results]+ known = [score for score in sub_scores if score is not None]+ scores.append(pick(known) if known else None)+ if combine(i in result.triggered_indices for result in results):+ flags.append(i)+ metadata.append(+ {+ "operator": self.operator,+ "sub_results": [+ {"type": cfg.get("type"), "is_anomaly": i in result.triggered_indices, "score": score}+ for cfg, result, score in zip(self.config.get("detectors", []), results, sub_scores)+ ],+ }+ )+ return DetectionResult(+ is_anomaly=bool(flags) and flags[-1] == len(scores) - 1,+ score=scores[-1] if scores else None,+ triggered_indices=flags,+ all_scores=scores,+ all_metadata=metadata,+ )+def detect_batch(self, data: np.ndarray) -> DetectionResult:results = [d.detect_batch(data) for d in self.sub_detectors]diff --git a/posthog/tasks/alerts/detectors/pyod_detectors/base_pyod.py b/posthog/tasks/alerts/detectors/pyod_detectors/base_pyod.pyindex e8a59080182c..2a276473dbde 100644--- a/posthog/tasks/alerts/detectors/pyod_detectors/base_pyod.py+++ b/posthog/tasks/alerts/detectors/pyod_detectors/base_pyod.py@@ -1,9 +1,12 @@+import iofrom abc import abstractmethod+from typing import Anyimport numpy as np+import joblibfrom pyod.models.base import BaseDetector as PyODBaseDetector-from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifact, FitErrorclass BasePyODDetector(BaseDetector):@@ -63,6 +66,49 @@ class BasePyODDetector(BaseDetector):metadata={"raw_score": float(model.decision_function(last_point)[0])},)+ def fit(self, history: np.ndarray) -> DetectorArtifact:+ # detect() needs MIN_SAMPLES buckets, the scored one included.+ if len(history) + 1 < self.MIN_SAMPLES:+ raise FitError(f"{len(history)} buckets cannot fit, {self.MIN_SAMPLES - 1} needed")+ rows = self._fit_rows(history, min_rows=1)+ if rows.ndim == 1:+ rows = rows.reshape(-1, 1)+ if not np.all(np.isfinite(rows)):+ raise FitError("The history has values that are not finite")+ try:+ model = self._build_model(n_samples=len(rows))+ model.fit(rows)+ except (ValueError, np.linalg.LinAlgError) as error:+ raise FitError(str(error)) from error+ buffer = io.BytesIO()+ joblib.dump(model, buffer)+ return self._artifact({"n_samples": len(rows)}, model_bytes=buffer.getvalue())++ def _score_rows(+ self, artifact: DetectorArtifact, rows: np.ndarray+ ) -> tuple[list[float | None], list[dict[str, Any]]]:+ if artifact.model_bytes is None:+ raise ValueError("The artifact has no model")+ # nosemgrep: trailofbits.python.scikit-joblib-load.scikit-joblib-load (bytes come only from fit() via a blob whose SHA-256 the caller checked against Postgres)+ model: PyODBaseDetector = joblib.load(io.BytesIO(artifact.model_bytes))+ if rows.ndim == 1:+ rows = rows.reshape(-1, 1)+ scores: list[float | None] = []+ metadata: list[dict[str, Any]] = []+ for i in range(len(rows)):+ point = rows[i : i + 1]+ if not np.all(np.isfinite(point)):+ scores.append(None)+ metadata.append({})+ continue+ try:+ scores.append(float(model.predict_proba(point)[0, 1]))+ metadata.append({"raw_score": float(model.decision_function(point)[0])})+ except (ValueError, np.linalg.LinAlgError):+ scores.append(None)+ metadata.append({})+ return scores, metadata+def detect_batch(self, data: np.ndarray) -> DetectionResult:if not self._validate_data(data, min_length=self.MIN_SAMPLES):return DetectionResult(is_anomaly=False)diff --git a/posthog/tasks/alerts/detectors/statistical/iqr.py b/posthog/tasks/alerts/detectors/statistical/iqr.pyindex 1fa63e6323e0..a1fab3993915 100644--- a/posthog/tasks/alerts/detectors/statistical/iqr.py+++ b/posthog/tasks/alerts/detectors/statistical/iqr.py@@ -5,7 +5,7 @@ from scipy.special import erffrom posthog.schema import DetectorType-from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifactfrom posthog.tasks.alerts.detectors.registry import register_detector@@ -26,8 +26,10 @@ def _iqr_distance_to_probability(distance: float, window_distances: np.ndarray)Uses pyod's 'unify' approach: standardize the distance against thedistribution of distances observed in the training window, then apply erf."""- mean_d = float(window_distances.mean())- std_d = float(window_distances.std())+ return _normalize_iqr_distance(distance, float(window_distances.mean()), float(window_distances.std()))+++def _normalize_iqr_distance(distance: float, mean_d: float, std_d: float) -> float:if std_d == 0:return 1.0 if distance > mean_d else 0.0standardized = (distance - mean_d) / std_d@@ -176,6 +178,63 @@ class IQRDetector(BaseDetector):all_scores=scores,)+ def fit(self, history: np.ndarray) -> DetectorArtifact:+ multiplier = self.config.get("multiplier", 1.5)+ window = self.config.get("window", 30)+ diffs_n = 1 if self.preprocessing_config.get("diffs_n") else 0+ rows = self._fit_rows(history, min_rows=window + diffs_n)+ values = rows if rows.ndim == 1 else rows[:, 0]+ window_data = values[diffs_n:][-window:]+ q1 = np.percentile(window_data, 25)+ q3 = np.percentile(window_data, 75)+ iqr = q3 - q1+ lower_fence = q1 - multiplier * iqr+ upper_fence = q3 + multiplier * iqr+ window_distances = _iqr_fence_distances(window_data, lower_fence, upper_fence, iqr)+ return self._artifact(+ {+ "q1": float(q1),+ "q3": float(q3),+ "iqr": float(iqr),+ "lower_fence": float(lower_fence),+ "upper_fence": float(upper_fence),+ "mean_distance": float(window_distances.mean()),+ "std_distance": float(window_distances.std()),+ }+ )++ def _score_rows(+ self, artifact: DetectorArtifact, rows: np.ndarray+ ) -> tuple[list[float | None], list[dict[str, Any]]]:+ params = artifact.params+ iqr = params["iqr"]+ lower_fence = params["lower_fence"]+ upper_fence = params["upper_fence"]+ scores: list[float | None] = []+ metadata: list[dict[str, Any]] = []+ for value in rows if rows.ndim == 1 else rows[:, 0]:+ if iqr == 0:+ raw_distance = 0.0+ elif value < lower_fence:+ raw_distance = (lower_fence - value) / iqr+ elif value > upper_fence:+ raw_distance = (value - upper_fence) / iqr+ else:+ raw_distance = 0.0+ scores.append(_normalize_iqr_distance(raw_distance, params["mean_distance"], params["std_distance"]))+ metadata.append(+ {+ "q1": params["q1"],+ "q3": params["q3"],+ "iqr": iqr,+ "lower_fence": lower_fence,+ "upper_fence": upper_fence,+ "value": float(value),+ "raw_distance": float(raw_distance),+ }+ )+ return scores, metadata+@classmethoddef get_default_config(cls) -> dict[str, Any]:return {diff --git a/posthog/tasks/alerts/detectors/statistical/mad.py b/posthog/tasks/alerts/detectors/statistical/mad.pyindex b50cd278b50e..b7e77300cf61 100644--- a/posthog/tasks/alerts/detectors/statistical/mad.py+++ b/posthog/tasks/alerts/detectors/statistical/mad.py@@ -1,9 +1,11 @@+from typing import Any+import numpy as npfrom pyod.models.mad import MADfrom posthog.schema import DetectorType-from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifactfrom posthog.tasks.alerts.detectors.registry import register_detector@@ -115,6 +117,48 @@ class MADDetector(BaseDetector):metadata={"threshold": threshold, "window": window},)+ def fit(self, history: np.ndarray) -> DetectorArtifact:+ window = self.config.get("window", 30)+ diffs_n = 1 if self.preprocessing_config.get("diffs_n") else 0+ rows = self._fit_rows(history, min_rows=window + diffs_n)+ values = rows if rows.ndim == 1 else rows[:, 0]+ clf = MAD()+ clf.fit(values[diffs_n:][-window:].reshape(-1, 1))+ # predict_proba scales a raw score by the range of the training scores.+ return self._artifact(+ {+ "median": float(clf.median_),+ "median_abs_deviation": float(clf.median_diff_),+ "min_score": float(clf.decision_scores_.min()),+ "max_score": float(clf.decision_scores_.max()),+ }+ )++ def _score_rows(+ self, artifact: DetectorArtifact, rows: np.ndarray+ ) -> tuple[list[float | None], list[dict[str, Any]]]:+ params = artifact.params+ values = rows if rows.ndim == 1 else rows[:, 0]+ clf = MAD()+ clf.median_ = params["median"]+ clf.median_diff_ = params["median_abs_deviation"]+ raw_scores = clf.decision_function(values.reshape(-1, 1))+ # MinMaxScaler keeps a zero range as one, so the scores only shift.+ score_range = params["max_score"] - params["min_score"] or 1.0+ scores: list[float | None] = []+ metadata: list[dict[str, Any]] = []+ for value, raw_score in zip(values, raw_scores):+ scores.append(float(np.clip((raw_score - params["min_score"]) / score_range, 0.0, 1.0)))+ metadata.append(+ {+ "median": params["median"],+ "median_abs_deviation": params["median_abs_deviation"],+ "value": float(value),+ "raw_score": float(raw_score),+ }+ )+ return scores, metadata+@classmethoddef get_default_config(cls) -> dict:return {diff --git a/posthog/tasks/alerts/detectors/statistical/zscore.py b/posthog/tasks/alerts/detectors/statistical/zscore.pyindex 2a5e1cfdfd04..31dd1014cec9 100644--- a/posthog/tasks/alerts/detectors/statistical/zscore.py+++ b/posthog/tasks/alerts/detectors/statistical/zscore.py@@ -1,8 +1,10 @@+from typing import Any+import numpy as npfrom posthog.schema import DetectorType-from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifactfrom posthog.tasks.alerts.detectors.registry import register_detector@@ -15,8 +17,10 @@ def _zscore_to_probability(z_score: float, window_zscores: np.ndarray) -> float:training window: 0 means at or below the minimum, 1 means at or abovethe maximum."""- min_z = float(window_zscores.min())- max_z = float(window_zscores.max())+ return _normalize_zscore(z_score, float(window_zscores.min()), float(window_zscores.max()))+++def _normalize_zscore(z_score: float, min_z: float, max_z: float) -> float:if max_z == min_z:return 1.0 if z_score > max_z else 0.0return float(np.clip((z_score - min_z) / (max_z - min_z), 0.0, 1.0))@@ -149,6 +153,37 @@ class ZScoreDetector(BaseDetector):metadata={"threshold": threshold, "window": window},)+ def fit(self, history: np.ndarray) -> DetectorArtifact:+ window = self.config.get("window", 30)+ diffs_n = 1 if self.preprocessing_config.get("diffs_n") else 0+ rows = self._fit_rows(history, min_rows=window + diffs_n)+ values = rows if rows.ndim == 1 else rows[:, 0]+ window_data = values[diffs_n:][-window:]+ mean = np.mean(window_data)+ std = np.std(window_data)+ params: dict[str, Any] = {"mean": float(mean), "std": float(std)}+ if std != 0:+ window_zscores = np.abs((window_data - mean) / std)+ params.update(min_z=float(window_zscores.min()), max_z=float(window_zscores.max()))+ return self._artifact(params)++ def _score_rows(+ self, artifact: DetectorArtifact, rows: np.ndarray+ ) -> tuple[list[float | None], list[dict[str, Any]]]:+ mean = artifact.params["mean"]+ std = artifact.params["std"]+ scores: list[float | None] = []+ metadata: list[dict[str, Any]] = []+ for value in rows if rows.ndim == 1 else rows[:, 0]:+ if std == 0:+ scores.append(1.0 if abs(value - mean) > 0 else 0.0)+ metadata.append({"mean": mean, "std": 0.0, "value": float(value), "raw_zscore": None})+ continue+ z_score = abs((value - mean) / std)+ scores.append(_normalize_zscore(z_score, artifact.params["min_z"], artifact.params["max_z"]))+ metadata.append({"mean": mean, "std": std, "value": float(value), "raw_zscore": float(z_score)})+ return scores, metadata+@classmethoddef get_default_config(cls) -> dict:return {diff --git a/posthog/tasks/alerts/detectors/test/test_detectors.py b/posthog/tasks/alerts/detectors/test/test_detectors.pyindex cb3003c819c4..5e090c20abf1 100644--- a/posthog/tasks/alerts/detectors/test/test_detectors.py+++ b/posthog/tasks/alerts/detectors/test/test_detectors.py@@ -820,3 +820,64 @@ class TestRealisticScoreBehavior:f"Ensemble OR fired on stable data (score={result.score:.4f}, "f"sub_results={result.metadata.get('sub_results')})")+++_PARITY_PREPROCESSING = [+ ("raw", {}),+ ("diffs", {"diffs_n": 1}),+ ("lags", {"lags_n": 3}),+ ("smooth", {"smooth_n": 3}),+ ("diffs_lags_smooth", {"diffs_n": 1, "lags_n": 5, "smooth_n": 3}),+]+++def _parity_config(detector_type: str, preprocessing: dict[str, Any], training_offset: int) -> dict[str, Any]:+ shared = {"preprocessing": preprocessing, "training_offset_n": training_offset}+ if detector_type == "ensemble":+ return {+ "type": "ensemble",+ "operator": "or",+ "detectors": [{"type": "zscore", "window": 30, **shared}, {"type": "knn", **shared}],+ }+ if detector_type == "threshold":+ return {"type": "threshold", "upper_bound": 150, **shared}+ return {"type": detector_type, "window": 30, **shared}+++class TestFitAndScore:+ @parameterized.expand(+ [+ (f"{detector_type}_{name}_offset_{offset}", detector_type, preprocessing, offset)+ for detector_type in get_available_detectors()+ for name, preprocessing in _PARITY_PREPROCESSING+ for offset in (1, 3)+ ]+ )+ def test_scoring_a_fitted_artifact_matches_detect(+ self, _name: str, detector_type: str, preprocessing: dict[str, Any], offset: int+ ) -> None:+ data = np.random.default_rng(0).normal(100, 10, size=80)+ data[-1] = 200.0+ detector = get_detector(_parity_config(detector_type, preprocessing, offset))++ expected = detector.detect(data)+ scored = detector.score(detector.fit(data[:-1]), data[-1 - detector.context_buckets() :])++ assert len(scored.all_scores) == 1+ assert scored.score == pytest.approx(expected.score)+ assert scored.is_anomaly is expected.is_anomaly++ def test_ensemble_scores_each_bucket_through_its_sub_artifacts(self) -> None:+ detector = get_detector(_parity_config("ensemble", {"diffs_n": 1, "lags_n": 5, "smooth_n": 3}, 1))+ data = np.random.default_rng(1).normal(100, 10, size=80)+ data[-2] = 400.0++ artifact = detector.fit(data[:60])+ scored = detector.score(artifact, data[60 - detector.context_buckets() :])++ assert [child.detector_type for child in artifact.children] == ["zscore", "knn"]+ assert artifact.children[1].model_bytes+ assert detector.context_buckets() == 8+ assert len(scored.all_scores) == len(scored.all_metadata) == 20+ assert 18 in scored.triggered_indices+ assert [sub["type"] for sub in scored.all_metadata[18]["sub_results"]] == ["zscore", "knn"]diff --git a/posthog/tasks/alerts/detectors/threshold.py b/posthog/tasks/alerts/detectors/threshold.pyindex 64583390e8f5..e1f28cdad01d 100644--- a/posthog/tasks/alerts/detectors/threshold.py+++ b/posthog/tasks/alerts/detectors/threshold.py@@ -1,8 +1,10 @@+from typing import Any+import numpy as npfrom posthog.schema import DetectorType-from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifactfrom posthog.tasks.alerts.detectors.registry import register_detector@@ -81,6 +83,23 @@ class ThresholdDetector(BaseDetector):metadata={"lower_bound": lower, "upper_bound": upper},)+ def fit(self, history: np.ndarray) -> DetectorArtifact:+ # The bounds come from the config, so nothing is fitted.+ return self._artifact({})++ def _score_rows(+ self, artifact: DetectorArtifact, rows: np.ndarray+ ) -> tuple[list[float | None], list[dict[str, Any]]]:+ lower = self.config.get("lower_bound")+ upper = self.config.get("upper_bound")+ scores: list[float | None] = []+ metadata: list[dict[str, Any]] = []+ for value in rows if rows.ndim == 1 else rows[:, 0]:+ is_breach = (lower is not None and value < lower) or (upper is not None and value > upper)+ scores.append(1.0 if is_breach else 0.0)+ metadata.append({"lower_bound": lower, "upper_bound": upper, "value": float(value)})+ return scores, metadata+@classmethoddef get_default_config(cls) -> dict:return {diff --git a/posthog/taxonomy/taxonomy.py b/posthog/taxonomy/taxonomy.pyindex dbcba028435e..bdad515cd8f4 100644--- a/posthog/taxonomy/taxonomy.py+++ b/posthog/taxonomy/taxonomy.py@@ -3378,10 +3378,15 @@ CORE_FILTER_DEFINITIONS_BY_GROUP: dict[str, dict[str, CoreFilterDefinition]] = {},"$anomaly_run": {"label": "Anomaly scoring run",- "description": "How the bucket was scored. `train` fetched the full history of the series, and `score` scored the bucket on stored history. Empty when scoring fetches the full history on every run.",+ "description": "How the bucket was scored. `train` fit a new model on the history of the series, and `score` scored the bucket with a stored model. Empty when scoring fits the detector again on every run.","examples": ["train", "score"],"type": "String",},+ "$anomaly_model_id": {+ "label": "Anomaly model ID",+ "description": "ID of the stored model that scored the bucket. Buckets with the same ID were scored with the same fit. Empty when scoring fits the detector again on every run.",+ "type": "String",+ },"$csp_document_url": {"label": "Document URL","description": "The URL of the document where the violation occurred.",diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 35729ea25b4f..3f1d570bc974 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -48,7 +48,6 @@1 posthog/storage/auth_token_cache_verifier.py1 posthog/storage/hypercache_verifier.py1 posthog/tasks/alerts/detector.py-1 posthog/tasks/alerts/detectors/base.py1 posthog/temporal/ai/anomaly_investigation/notebook.py1 posthog/temporal/ai/anomaly_investigation/runner.py1 posthog/temporal/ai/anomaly_investigation/tools.pydiff --git a/products/signals/backend/anomaly_scores/artifacts.py b/products/signals/backend/anomaly_scores/artifacts.pynew file mode 100644index 000000000000..cb9dcc4c21ae--- /dev/null+++ b/products/signals/backend/anomaly_scores/artifacts.py@@ -0,0 +1,126 @@+"""Detector artifact blobs in object storage: one blob per insight and scoring profile.++A blob holds one `DetectorArtifact` per series key, as JSON. PyOD models are joblib bytes inside it,+so a blob is read only from a key this code wrote, and only after its bytes match the SHA-256 that+the state row stores. A key is never written twice, so a loaded blob is cached in the process by key.+"""++import json+import uuid+import base64+import hashlib+import threading+from collections import OrderedDict+from typing import Any++from posthog.dataclasses import frozen+from posthog.storage import object_storage+from posthog.tasks.alerts.detectors.base import ARTIFACT_SCHEMA_VERSION, DetectorArtifact++KEY_PREFIX = "anomaly_models"+# A blob holds the artifacts of at most `MAX_LIMITS.max_series` series, so the cache stays small.+_MAX_CACHED_BLOBS = 256+++class ArtifactUnavailableError(Exception):+ """The blob is missing, its bytes do not match the stored hash, or a new release cannot read it. The next visit trains."""+++@frozen+class StoredArtifacts:+ key: str+ sha256: str+ # The uuid of the blob, on every `$anomaly_score` event that its artifacts scored.+ model_id: str+++def artifact_key(team_id: int, insight_id: int, profile: str, model_id: str) -> str:+ return f"{KEY_PREFIX}/{team_id}/{insight_id}/{profile}/{model_id}"+++def model_id_of(key: str) -> str:+ return key.rsplit("/", 1)[-1]+++def _artifact_json(artifact: DetectorArtifact) -> dict[str, Any]:+ return {+ "detector_type": artifact.detector_type,+ "schema_version": artifact.schema_version,+ "config": artifact.config,+ "params": artifact.params,+ "model_bytes": None if artifact.model_bytes is None else base64.b64encode(artifact.model_bytes).decode(),+ "children": [_artifact_json(child) for child in artifact.children],+ }+++def _artifact_from_json(stored: dict[str, Any]) -> DetectorArtifact:+ model_bytes = stored.get("model_bytes")+ return DetectorArtifact(+ detector_type=stored["detector_type"],+ schema_version=stored["schema_version"],+ config=stored["config"],+ params=stored["params"],+ model_bytes=None if model_bytes is None else base64.b64decode(model_bytes),+ children=tuple(_artifact_from_json(child) for child in stored.get("children") or []),+ )+++def serialize(artifacts: dict[str, DetectorArtifact]) -> bytes:+ return json.dumps(+ {"series": {key: _artifact_json(artifact) for key, artifact in artifacts.items()}}, sort_keys=True+ ).encode()+++def deserialize(blob: bytes) -> dict[str, DetectorArtifact]:+ return {key: _artifact_from_json(stored) for key, stored in json.loads(blob)["series"].items()}+++_cache: OrderedDict[str, tuple[str, dict[str, DetectorArtifact]]] = OrderedDict()+_cache_lock = threading.Lock()+++def write_artifacts(+ team_id: int, insight_id: int, profile: str, artifacts: dict[str, DetectorArtifact]+) -> StoredArtifacts:+ model_id = str(uuid.uuid4())+ key = artifact_key(team_id, insight_id, profile, model_id)+ blob = serialize(artifacts)+ object_storage.write(key, blob)+ return StoredArtifacts(key=key, sha256=hashlib.sha256(blob).hexdigest(), model_id=model_id)+++def load_artifacts(key: str, sha256: str) -> dict[str, DetectorArtifact]:+ with _cache_lock:+ cached = _cache.get(key)+ if cached is not None:+ _cache.move_to_end(key)+ if cached is not None and cached[0] == sha256:+ return cached[1]+ blob = object_storage.read_bytes(key, missing_ok=True) if key else None+ if blob is None:+ raise ArtifactUnavailableError(f"No artifact blob at {key!r}")+ if hashlib.sha256(blob).hexdigest() != sha256:+ raise ArtifactUnavailableError(f"The artifact blob at {key!r} does not match its hash")+ artifacts = deserialize(blob)+ if not all(_readable(artifact) for artifact in artifacts.values()):+ raise ArtifactUnavailableError(f"The artifact blob at {key!r} has an older schema version")+ with _cache_lock:+ _cache[key] = (sha256, artifacts)+ while len(_cache) > _MAX_CACHED_BLOBS:+ _cache.popitem(last=False)+ return artifacts+++def _readable(artifact: DetectorArtifact) -> bool:+ return artifact.schema_version == ARTIFACT_SCHEMA_VERSION and all(_readable(child) for child in artifact.children)+++def delete_artifacts(key: str) -> None:+ with _cache_lock:+ _cache.pop(key, None)+ object_storage.delete(key)+++def clear_artifact_cache() -> None:+ with _cache_lock:+ _cache.clear()diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.pyindex dbecc1450d98..c0e2a74a1622 100644--- a/products/signals/backend/anomaly_scores/config.py+++ b/products/signals/backend/anomaly_scores/config.py@@ -84,18 +84,15 @@ def resolve_profile(team: "Team", insight: "Insight") -> ScoringProfile:class ScoringRun(StrEnum):"""The kind of visit when the train and score split is on."""- # Fetch the full detector history, score the new buckets, and store the series tail.+ # Fetch the training history, fit and store one artifact per series, and score the new buckets with them.TRAIN = "train"- # Fetch only the refresh and new buckets, merge them into the stored tail, and score the new buckets.+ # Fetch only the new buckets and the context that preprocessing reads, and score them with the stored artifacts.SCORE = "score"# Retrain hourly insights once a day, daily once a week, weekly once a month, and monthly once a quarter.DEFAULT_TRAIN_EVERY_BUCKETS = {IntervalType.HOUR: 24, IntervalType.DAY: 7, IntervalType.WEEK: 4, IntervalType.MONTH: 3}MAX_TRAIN_EVERY_BUCKETS = {IntervalType.HOUR: 168, IntervalType.DAY: 30, IntervalType.WEEK: 13, IntervalType.MONTH: 12}-# Stored buckets that a score step fetches again, so late events reach the stored values.-DEFAULT_REFRESH_BUCKETS = {IntervalType.HOUR: 3, IntervalType.DAY: 3, IntervalType.WEEK: 2, IntervalType.MONTH: 1}-MAX_REFRESH_BUCKETS = {IntervalType.HOUR: 24, IntervalType.DAY: 14, IntervalType.WEEK: 4, IntervalType.MONTH: 3}# Hourly insights are left out by default: with many eligible insights, the time between two visits of@@ -123,7 +120,6 @@ class AnomalyScoringConfig:train_every_buckets: dict[IntervalType, int] = dataclasses.field(default_factory=lambda: dict(DEFAULT_TRAIN_EVERY_BUCKETS))- refresh_buckets: dict[IntervalType, int] = dataclasses.field(default_factory=lambda: dict(DEFAULT_REFRESH_BUCKETS))# Score step queries per clock hour. Train steps use the budgets above, so a cheap score step never waits for them.max_score_queries_per_hour: int = 600max_score_queries_per_project_per_hour: int = 150@@ -226,9 +222,6 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:train_every_buckets=_buckets_per_interval(payload.get("train_every_buckets"), DEFAULT_TRAIN_EVERY_BUCKETS, MAX_TRAIN_EVERY_BUCKETS),- refresh_buckets=_buckets_per_interval(- payload.get("refresh_buckets"), DEFAULT_REFRESH_BUCKETS, MAX_REFRESH_BUCKETS- ),max_score_queries_per_hour=min(_positive_int(payload.get("max_score_queries_per_hour"), defaults.max_score_queries_per_hour),MAX_SCORE_QUERIES_PER_HOUR,diff --git a/products/signals/backend/anomaly_scores/emission.py b/products/signals/backend/anomaly_scores/emission.pyindex 57b1ba67853d..dc534d22b66f 100644--- a/products/signals/backend/anomaly_scores/emission.py+++ b/products/signals/backend/anomaly_scores/emission.py@@ -58,6 +58,7 @@ def _event("$anomaly_scored_at": scored_at.astimezone(datetime.UTC).isoformat(),"$anomaly_details": json.dumps(point.details, sort_keys=True),**({"$anomaly_run": scores.run.value} if scores.run else {}),+ **({"$anomaly_model_id": scores.model_id} if scores.model_id else {}),},}diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.pyindex 21121ed3a2d4..5b6fafae5983 100644--- a/products/signals/backend/anomaly_scores/queue.py+++ b/products/signals/backend/anomaly_scores/queue.py@@ -105,8 +105,6 @@ class InsightPick:query_cost: int = 1# None when the train and score split is off.run: ScoringRun | None = None- # Stored buckets that a score step fetches again.- refresh_buckets: int = 0@frozen@@ -441,7 +439,6 @@ def select_insights(interval=due_by_id[insight_id].interval,query_cost=plan.queries,run=plan.run,- refresh_buckets=plan.refresh_buckets,)for insight_id, plan in picked.items()]diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.pyindex 15b6f8053bad..3b1bc560192c 100644--- a/products/signals/backend/anomaly_scores/scoring.py+++ b/products/signals/backend/anomaly_scores/scoring.py@@ -25,12 +25,13 @@ from posthog.hogql_queries.validation.rules import expanded_series_countfrom posthog.interval_specs import interval_specfrom posthog.models import Teamfrom posthog.schema_migrations.upgrade_manager import upgrade_insight-from posthog.tasks.alerts.detectors.base import DetectionResult+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifact, FitErrorfrom posthog.tasks.alerts.detectors.registry import get_detectorfrom posthog.tasks.alerts.utils import WRAPPER_NODE_KINDS, is_non_time_series_trendfrom products.product_analytics.backend.facade.models import Insightfrom products.product_analytics.backend.facade.queries import TrendsQueryRunner+from products.signals.backend.anomaly_scores.artifacts import model_id_offrom products.signals.backend.anomaly_scores.config import (DEFAULT_LIMITS,IntervalScoring,@@ -78,15 +79,6 @@ class ScoredSeries:points: list[ScoredPoint]-@frozen-class SeriesWindow:- """The newest completed values of one series, oldest first. A score step reads them as the history of its new buckets."""-- # Aware bucket start of the first value. Trends zero-fills buckets, so each next value is one interval later.- first_bucket: datetime- values: list[float]--@frozenclass StateIdentity:"""What a stored state was built from. A score step needs the same identity, or the visit trains."""@@ -102,17 +94,22 @@ class TrainedState:trained_at: datetime# The newest scored bucket start over all series, or None when no series had a completed bucket.scored_through: datetime | None- windows: dict[str, SeriesWindow]+ # The series that the stored artifact blob covers.+ series_keys: tuple[str, ...]+ # Empty until the blob of a train step is written.+ artifact_key: str = ""+ artifact_sha256: str = ""# Stored series that the last score step did not get back.missing_series: int = 0@frozenclass SplitRun:- """A visit of the train and score split. A score step needs ``state``; with no matching state, the visit trains."""+ """A visit of the train and score split. A score step needs ``state`` and its ``artifacts``, or the visit trains."""- refresh_buckets: intstate: TrainedState | None = None+ # Series key to the artifact of the stored blob of ``state``.+ artifacts: dict[str, DetectorArtifact] | None = None@frozen@@ -126,8 +123,12 @@ class InsightScores:run: ScoringRun | None = None# The state for the next visit to store.state: TrainedState | None = None- # Series of a score step with no stored window. A later train step scores them.+ # The artifacts that a train step fit, for the caller to store. None on a score step.+ artifacts: dict[str, DetectorArtifact] | None = None+ # Series of a score step with no stored artifact. A later train step fits them.unmatched_series: int = 0+ # The artifact blob that scored the buckets. None when the split is off.+ model_id: str | None = None@frozen@@ -144,9 +145,15 @@ def _short_hash(payload: Any) -> str:return hashlib.sha256(canonical.encode()).hexdigest()[:16]-def detector_version(scoring: IntervalScoring) -> str:- """Identifies the scoring behavior, so scores from different settings are never mixed in one series."""- return _short_hash(dataclasses.asdict(scoring))+def detector_version(scoring: IntervalScoring, *, artifacts: bool = False) -> str:+ """Identifies the scoring behavior, so scores from different settings are never mixed in one series.++ Scores from stored artifacts are fit once per train step, so they get their own version.+ """+ behavior: dict[str, Any] = {"min_points": scoring.min_points, "detector": scoring.detector}+ if artifacts:+ behavior["fit"] = "artifact"+ return _short_hash(behavior)def query_hash(query: TrendsQuery, timezone: ZoneInfo, limits: ScoringLimits = DEFAULT_LIMITS) -> str:@@ -154,7 +161,7 @@ def query_hash(query: TrendsQuery, timezone: ZoneInfo, limits: ScoringLimits = Ddumped = build_scoring_query(query, buckets=1, limits=limits).model_dump(mode="json", exclude_none=True)# The fetch size changes with every visit. `daysOfWeek` stays, because it changes the buckets.(dumped.get("dateRange") or {}).pop("date_from", None)- # Stored bucket starts step in wall-clock time, so a timezone change needs new windows.+ # Bucket starts step in wall-clock time, so a timezone change needs new artifacts.return _short_hash({"query": dumped, "timezone": timezone.key})@@ -236,7 +243,7 @@ def fetch_buckets(query: TrendsQuery, scoring: IntervalScoring, to_score: int, *"""Completed buckets the scoring query fetches: the history of the oldest bucket to score, then the rest.The count has one bucket more than the scores read, so a first bucket that the date range cuts short is never read.- ``history`` replaces the detector history, for a score step that reads its history from the stored state.+ ``history`` replaces the detector history, for the train and score steps of the split."""count = (history_buckets(scoring) if history is None else history) + to_scoresmoothing = query.trendsFilter.smoothingIntervals if query.trendsFilter else None@@ -291,7 +298,7 @@ def scoring_query_settings(queries: int) -> HogQLGlobalSettings:def supports_score_step(query: TrendsQuery) -> bool:- """A score step steps over stored buckets one interval at a time, so the buckets must have no gaps."""+ """A score step reads the buckets before a scored bucket as its context, so the buckets must have no gaps."""# Trends drops the buckets of the days that the filter leaves out.return not (query.dateRange and query.dateRange.daysOfWeek)@@ -299,7 +306,7 @@ def supports_score_step(query: TrendsQuery) -> bool:def state_identity(query: TrendsQuery, scoring: IntervalScoring, team: Team, limits: ScoringLimits) -> StateIdentity:return StateIdentity(interval=scoring_interval(query),- detector_version=detector_version(scoring),+ detector_version=detector_version(scoring, artifacts=True),query_hash=query_hash(query, team.timezone_info, limits),)@@ -360,65 +367,63 @@ def score_series(return points-def window_buckets(window: SeriesWindow, interval: IntervalType) -> list[datetime]:- buckets: list[datetime] = []- start = window.first_bucket- for _ in window.values:- buckets.append(start)- start = bucket_end(start, interval)- return buckets+@frozen+class ScoredIndices:+ completed: list[int]+ # The newest completed buckets later than `after`, capped at `max_backfill_buckets`.+ returned: list[int]-def train_window(- values: Sequence[float], buckets: Sequence[datetime], *, history: int, interval: IntervalType, now: datetime-) -> SeriesWindow | None:- """The newest ``history`` completed values of a fetched series, or None when they do not step one interval apart."""- completed = [i for i, start in enumerate(buckets) if bucket_end(start, interval) <= now][-history:]- if not completed:- return None- window = SeriesWindow(first_bucket=buckets[completed[0]], values=[float(values[i]) for i in completed])- # A repeated hour at a DST fall-back breaks the steps. The series waits for a train step after the change.- if window_buckets(window, interval) != [buckets[i] for i in completed]:- return None- return window+def _scored_indices(+ buckets: Sequence[datetime], *, interval: IntervalType, now: datetime, after: datetime | None, limits: ScoringLimits+) -> ScoredIndices:+ """The indices of the completed buckets, and of the newest of them later than ``after``."""+ completed = [i for i, start in enumerate(buckets) if bucket_end(start, interval) <= now]+ returned = [i for i in completed if after is None or buckets[i] > after][-limits.max_backfill_buckets :]+ return ScoredIndices(completed=completed, returned=returned)-def score_window(- window: SeriesWindow,+def score_with_artifact(+ detector: BaseDetector,+ artifact: DetectorArtifact | None,values: Sequence[float],buckets: Sequence[datetime],- *,- refresh: int,- scoring: IntervalScoring,- interval: IntervalType,- now: datetime,- limits: ScoringLimits = DEFAULT_LIMITS,-) -> tuple[list[ScoredPoint], SeriesWindow] | None:- """Merge freshly fetched buckets into a stored window, score the buckets after it, and trim it to its history.+ indices: list[int],+) -> list[ScoredPoint]:+ """Score the buckets at ``indices`` with a fitted artifact. A bucket without enough context before it gets no score."""+ context = detector.context_buckets()+ scored = [i for i in indices if i >= context]+ result = None+ if artifact is not None and scored:+ result = detector.score(artifact, np.array(values[scored[0] - context : scored[-1] + 1], dtype=float))+ positions = {i: position for position, i in enumerate(scored)}+ points: list[ScoredPoint] = []+ for i in indices:+ position = positions.get(i)+ score = flag = None+ details: dict[str, Any] = {}+ if result is not None and position is not None and result.all_scores[position] is not None:+ score = float(cast(float, result.all_scores[position]))+ flag = position in result.triggered_indices+ details = _details_of(result.all_metadata[position])+ points.append(+ ScoredPoint(bucket_start=buckets[i], value=float(values[i]), score=score, flag=flag, details=details)+ )+ return points- A fresh value replaces the stored value of its bucket for the newest ``refresh`` stored buckets, so late events- reach the window. Older fresh buckets can be cut short or smoothed over too few buckets, so they are not read.- Returns None when the fresh buckets leave a gap after the window. The series then keeps its window.- """- stored = window_buckets(window, interval)- merged = dict(zip(stored, window.values))- refresh_from = [*stored, bucket_end(stored[-1], interval)][max(0, len(stored) - refresh)]- fresh = [(start, value) for start, value in zip(buckets, values) if bucket_end(start, interval) <= now]- for start, value in fresh:- if start >= refresh_from:- merged[start] = float(value)- starts = list(stored)- next_start = bucket_end(stored[-1], interval)- while next_start in merged:- starts.append(next_start)- next_start = bucket_end(next_start, interval)- if any(start >= next_start for start, _ in fresh):- return None- data = [merged[start] for start in starts]- points = score_series(data, starts, scoring=scoring, interval=interval, now=now, after=stored[-1], limits=limits)- keep = history_buckets(scoring)- return points, SeriesWindow(first_bucket=starts[-keep:][0], values=data[-keep:])+def _fit(+ detector: BaseDetector, scoring: IntervalScoring, values: Sequence[float], end: int+) -> DetectorArtifact | None:+ """Fit on the buckets before ``end`` that ``detect()`` reads to score ``end``, or None when they are too few."""+ history = np.array(values[max(0, end - (history_buckets(scoring) - 1)) : end], dtype=float)+ # The scored bucket counts toward `min_points`, as in `score_series`.+ if len(history) + 1 < scoring.min_points:+ return None+ try:+ return detector.fit(history)+ except FitError:+ return Nonedef score_insight(@@ -432,121 +437,162 @@ def score_insight() -> InsightScores:"""Score the new buckets of an insight.- With ``split``, a stored state with the insight's identity makes this a score step, and anything else a- train step. Both return the state for the next visit.+ With ``split``, a stored state with the insight's identity and its artifacts makes this a score step, and+ anything else a train step. Both return the state for the next visit, and a train step returns the+ artifacts to store."""profile = resolve_profile(team, insight)query = trends_query_for(insight, profile)interval = scoring_interval(query)scoring = interval_scoring(profile, interval)timezone = team.timezone_info- identity = state_identity(query, scoring, team, limits) if split is not None else None- state = split.state if split is not None else None- if state is not None and (state.identity != identity or not supports_score_step(query)):- state = Nonescores = InsightScores(interval=interval,profile=profile.name,detector=str(scoring.detector["type"]),- detector_version=detector_version(scoring),+ detector_version=detector_version(scoring, artifacts=split is not None),series=[],)- if split is not None and state is not None and identity is not None:- to_score = buckets_to_score(interval, now=now, after=state.scored_through, timezone=timezone, limits=limits)- # One bucket more than the score step reads, as in the full fetch.- buckets = fetch_buckets(query, scoring, to_score, history=split.refresh_buckets + 1)- fetched = _fetch(insight, team, query, buckets, limits, ScoringRun.SCORE)+ if split is None:+ to_score = buckets_to_score(interval, now=now, after=after, timezone=timezone, limits=limits)+ fetched = _fetch(insight, team, query, fetch_buckets(query, scoring, to_score), limits, None)+ series = [+ ScoredSeries(+ series_key=line.series_key,+ label=line.label,+ breakdown_value=line.breakdown_value,+ points=score_series(+ line.values, line.buckets, scoring=scoring, interval=interval, now=now, after=after, limits=limits+ ),+ )+ for line in fetched+ ]+ return dataclasses.replace(scores, series=series)++ identity = state_identity(query, scoring, team, limits)+ state = split.state+ detector = get_detector(scoring.detector)+ if (+ state is not None+ and split.artifacts is not None+ and state.identity == identity+ and state.scored_through is not None+ and supports_score_step(query)+ ):return _score_step(+ insight,+ team,+ query,scores,- fetched,state,- refresh=split.refresh_buckets,+ split.artifacts,+ detector,scoring=scoring,- interval=interval,now=now,+ after=after,limits=limits,)+ return _train_step(+ insight, team, query, scores, identity, detector, scoring=scoring, now=now, after=after, limits=limits+ )- to_score = buckets_to_score(interval, now=now, after=after, timezone=timezone, limits=limits)- run = ScoringRun.TRAIN if split is not None else None- fetched = _fetch(insight, team, query, fetch_buckets(query, scoring, to_score), limits, run)- series = [- ScoredSeries(- series_key=line.series_key,- label=line.label,- breakdown_value=line.breakdown_value,- points=score_series(- line.values,- line.buckets,- scoring=scoring,- interval=interval,- now=now,- after=after,- limits=limits,- ),- )- for line in fetched- ]- if identity is None:- return dataclasses.replace(scores, series=series)- windows = {- line.series_key: window- for line in fetched- if (- window := train_window(- line.values, line.buckets, history=history_buckets(scoring), interval=interval, now=now++def _train_step(+ insight: Insight,+ team: Team,+ query: TrendsQuery,+ scores: InsightScores,+ identity: StateIdentity,+ detector: BaseDetector,+ *,+ scoring: IntervalScoring,+ now: datetime,+ after: datetime | None,+ limits: ScoringLimits,+) -> InsightScores:+ """Fit each series on the history before its due buckets, and score the due buckets with the new artifact."""+ interval = scores.interval+ to_score = buckets_to_score(interval, now=now, after=after, timezone=team.timezone_info, limits=limits)+ buckets = fetch_buckets(query, scoring, to_score)+ fetched = _fetch(insight, team, query, buckets, limits, ScoringRun.TRAIN)+ series: list[ScoredSeries] = []+ artifacts: dict[str, DetectorArtifact] = {}+ newest: list[datetime] = []+ for line in fetched:+ indices = _scored_indices(line.buckets, interval=interval, now=now, after=after, limits=limits)+ completed, returned = indices.completed, indices.returned+ if completed:+ newest.append(line.buckets[completed[-1]])+ # With no due bucket, the fit reads the newest completed buckets.+ end = returned[0] if returned else (completed[-1] + 1 if completed else 0)+ artifact = _fit(detector, scoring, line.values, end)+ if artifact is not None:+ artifacts[line.series_key] = artifact+ series.append(+ ScoredSeries(+ series_key=line.series_key,+ label=line.label,+ breakdown_value=line.breakdown_value,+ points=score_with_artifact(detector, artifact, line.values, line.buckets, returned),))- is not None- }return dataclasses.replace(scores,series=series,run=ScoringRun.TRAIN,+ artifacts=artifacts,state=TrainedState(- identity=identity, trained_at=now, scored_through=_scored_through(windows, interval), windows=windows+ identity=identity,+ trained_at=now,+ scored_through=max(newest).astimezone(UTC) if newest else None,+ series_keys=tuple(artifacts),),)def _score_step(+ insight: Insight,+ team: Team,+ query: TrendsQuery,scores: InsightScores,- fetched: list[_FetchedSeries],state: TrainedState,+ artifacts: dict[str, DetectorArtifact],+ detector: BaseDetector,*,- refresh: int,scoring: IntervalScoring,- interval: IntervalType,now: datetime,+ after: datetime | None,limits: ScoringLimits,) -> InsightScores:+ """Score the buckets after the older of ``after`` and ``state.scored_through`` with the stored artifacts.++ ``after`` is the oldest newest-scored bucket over the series. A series that missed a visit, for example a+ breakdown value out of the top N, then still gets its missed buckets, and the events drop each series' buckets+ up to its own newest scored bucket. Nothing is fitted.+ """+ interval = scores.interval+ since = min((through for through in (after, state.scored_through) if through is not None), default=None)+ to_score = buckets_to_score(interval, now=now, after=since, timezone=team.timezone_info, limits=limits)+ # One bucket more than the context that preprocessing reads, as in the full fetch.+ buckets = fetch_buckets(query, scoring, to_score, history=detector.context_buckets() + 1)+ fetched = _fetch(insight, team, query, buckets, limits, ScoringRun.SCORE)series: list[ScoredSeries] = []- windows = dict(state.windows)unmatched = 0- returned: set[str] = set()+ returned_keys: set[str] = set()+ scored_through = state.scored_throughfor line in fetched:- returned.add(line.series_key)- window = state.windows.get(line.series_key)+ returned_keys.add(line.series_key)+ artifact = artifacts.get(line.series_key)# The short range can rank a different top N of breakdown values. A new value waits for a train step.- scored = (- None- if window is None- else score_window(- window,- line.values,- line.buckets,- refresh=refresh,- scoring=scoring,- interval=interval,- now=now,- limits=limits,- )- )- if scored is None:+ if artifact is None:unmatched += 1continue- points, windows[line.series_key] = scored+ returned = _scored_indices(line.buckets, interval=interval, now=now, after=since, limits=limits).returned+ points = score_with_artifact(detector, artifact, line.values, line.buckets, returned)+ if points:+ newest = points[-1].bucket_start.astimezone(UTC)+ scored_through = newest if scored_through is None else max(scored_through, newest)series.append(ScoredSeries(series_key=line.series_key, label=line.label, breakdown_value=line.breakdown_value, points=points@@ -557,21 +603,13 @@ def _score_step(series=series,run=ScoringRun.SCORE,unmatched_series=unmatched,- # A stored series that is missing from the result keeps its window.state=dataclasses.replace(- state,- scored_through=_scored_through(windows, interval),- windows=windows,- missing_series=len(set(state.windows) - returned),+ state, scored_through=scored_through, missing_series=len(set(state.series_keys) - returned_keys)),+ model_id=model_id_of(state.artifact_key),)-def _scored_through(windows: dict[str, SeriesWindow], interval: IntervalType) -> datetime | None:- ends = [window_buckets(window, interval)[-1] for window in windows.values() if window.values]- return max(ends).astimezone(UTC) if ends else None--def _fetch(insight: Insight,team: Team,@@ -684,17 +722,19 @@ def bucket_end(start: datetime, interval: IntervalType) -> datetime:def _details(result: DetectionResult) -> dict[str, Any]:+ return _details_of(result.metadata)+++def _details_of(metadata: dict[str, Any]) -> dict[str, Any]:"""Detector-specific values for one bucket. Only scalars are kept, so the event property stays small."""- sub_results = result.metadata.get("sub_results")+ sub_results = metadata.get("sub_results")if isinstance(sub_results, list):return {"sub_scores": {f"{sub.get('type')}_{position}": sub.get("score") for position, sub in enumerate(sub_results)}}- return {- key: value for key, value in result.metadata.items() if value is None or isinstance(value, int | float | str)- }+ return {key: value for key, value in metadata.items() if value is None or isinstance(value, int | float | str)}def _parse_bucket(day: str, timezone: ZoneInfo) -> datetime:diff --git a/products/signals/backend/anomaly_scores/state.py b/products/signals/backend/anomaly_scores/state.pyindex 48c870bc3d33..c46f117169f7 100644--- a/products/signals/backend/anomaly_scores/state.py+++ b/products/signals/backend/anomaly_scores/state.py@@ -1,16 +1,17 @@"""The stored state of the train and score split, and the choice between a train step and a score step.-A train step fetches the full detector history and stores the newest values of each series. A score-step fetches only the newest buckets and scores them on the stored values. See `AnomalyScoringState`.+A train step fetches the training history, fits one detector artifact per series, and stores them as+one blob. A score step fetches only the newest buckets and scores them with the stored artifacts. See+`AnomalyScoringState` and `artifacts.py`."""-import mathimport datetimeimport dataclassesfrom collections.abc import Collection-from typing import Anyfrom zoneinfo import ZoneInfo+import structlog+from posthog.schema import IntervalTypefrom posthog.dataclasses import frozen@@ -18,6 +19,7 @@ from posthog.interval_specs import interval_specfrom posthog.models import Teamfrom products.product_analytics.backend.facade.models import Insight+from products.signals.backend.anomaly_scores.artifacts import delete_artifactsfrom products.signals.backend.anomaly_scores.config import (DEFAULT_LIMITS,AnomalyScoringConfig,@@ -25,7 +27,6 @@ from products.signals.backend.anomaly_scores.config import (resolve_profile,)from products.signals.backend.anomaly_scores.scoring import (- SeriesWindow,StateIdentity,TrainedState,buckets_to_score,@@ -38,6 +39,8 @@ from products.signals.backend.anomaly_scores.scoring import ()from products.signals.backend.models import AnomalyScoringState+logger = structlog.get_logger(__name__)+@frozenclass RunPlan:@@ -46,7 +49,6 @@ class RunPlan:queries: int# None when the split is off.run: ScoringRun | None = None- refresh_buckets: int = 0def needs_training(@@ -58,7 +60,13 @@ def needs_training(train_every: int,max_backfill: int,) -> bool:- if state is None or state.identity != identity or not state.windows or state.scored_through is None:+ if (+ state is None+ or state.identity != identity+ or not state.series_keys+ or not state.artifact_key+ or state.scored_through is None+ ):return Trueinterval = identity.intervalif state.trained_at + interval_spec(interval).period * train_every <= now:@@ -73,7 +81,7 @@ def needs_training()if due > max_backfill:return True- return state.missing_series * 2 > len(state.windows)+ return state.missing_series * 2 > len(state.series_keys)def plan_run(@@ -91,7 +99,6 @@ def plan_run(except Exception:# The scorer fails on the same query and reports the error. A train step charges the larger budget.return RunPlan(queries=queries, run=ScoringRun.TRAIN)- refresh = config.refresh_buckets.get(interval, 0)trains = not supports_score_step(query) or needs_training(state,identity=identity,@@ -100,7 +107,7 @@ def plan_run(train_every=config.train_every_buckets[interval],max_backfill=config.limits.max_backfill_buckets,)- return RunPlan(queries=queries, run=ScoringRun.TRAIN if trains else ScoringRun.SCORE, refresh_buckets=refresh)+ return RunPlan(queries=queries, run=ScoringRun.TRAIN if trains else ScoringRun.SCORE)def load_states(team: Team, insights: Collection[Insight]) -> dict[int, TrainedState]:@@ -109,11 +116,7 @@ def load_states(team: Team, insights: Collection[Insight]) -> dict[int, TrainedSif not profiles:return {}rows = AnomalyScoringState.objects.for_team(team.id).filter(insight_id__in=list(profiles))- return {- row.insight_id: _from_row(row, team.timezone_info)- for row in rows- if profiles.get(row.insight_id) == row.profile- }+ return {row.insight_id: _from_row(row) for row in rows if profiles.get(row.insight_id) == row.profile}def load_state(team: Team, insight: Insight) -> TrainedState | None:@@ -121,6 +124,9 @@ def load_state(team: Team, insight: Insight) -> TrainedState | None:def save_state(team: Team, insight_id: int, profile: str, state: TrainedState) -> None:+ """Write the state row. A new artifact blob replaces the previous one, which is deleted after the write."""+ scope = AnomalyScoringState.objects.for_team(team.id)+ previous_key = scope.filter(insight_id=insight_id, profile=profile).values_list("artifact_key", flat=True).first()fields = {"interval": state.identity.interval.value,"detector_version": state.identity.detector_version,@@ -128,41 +134,37 @@ def save_state(team: Team, insight_id: int, profile: str, state: TrainedState) -"trained_at": state.trained_at,"scored_through": state.scored_through,"missing_series": state.missing_series,- "series": {key: _window_json(window) for key, window in state.windows.items()},+ "artifact_key": state.artifact_key,+ "artifact_sha256": state.artifact_sha256,+ "series_keys": list(state.series_keys),}# The scope filters on the project's canonical team, and save() writes that team for a new row.- AnomalyScoringState.objects.for_team(team.id).update_or_create(+ scope.update_or_create(insight_id=insight_id, profile=profile, defaults=fields, create_defaults={**fields, "team_id": team.id})+ if previous_key and previous_key != state.artifact_key:+ try:+ delete_artifacts(previous_key)+ except Exception:+ # The row no longer names the blob, so a failed delete leaves an unread blob and nothing else.+ logger.warning("anomaly_scoring_artifact_delete_failed", insight_id=insight_id, exc_info=True)-def _window_json(window: SeriesWindow) -> dict[str, Any]:- return {- "first_bucket": window.first_bucket.isoformat(),- # JSON has no NaN.- "values": [value if math.isfinite(value) else None for value in window.values],- }+def invalidate_state(team: Team, insight_id: int, profile: str) -> None:+ """Make the next visit of the insight train. The row keeps its blob key, so that train step deletes the blob."""+ AnomalyScoringState.objects.for_team(team.id).filter(insight_id=insight_id, profile=profile).update(series_keys=[])-def _from_row(row: AnomalyScoringState, timezone: datetime.tzinfo) -> TrainedState:- windows: dict[str, SeriesWindow] = {}- for key, stored in (row.series or {}).items():- try:- window = SeriesWindow(- # Bucket starts step in wall-clock time of the project timezone.- first_bucket=datetime.datetime.fromisoformat(stored["first_bucket"]).astimezone(timezone),- values=[math.nan if value is None else float(value) for value in stored["values"]],- )- except (KeyError, TypeError, ValueError):- continue- if window.values:- windows[key] = window+def _from_row(row: AnomalyScoringState) -> TrainedState:+ series_keys = row.series_keys if isinstance(row.series_keys, list) else []return TrainedState(identity=StateIdentity(interval=IntervalType(row.interval), detector_version=row.detector_version, query_hash=row.query_hash),trained_at=row.trained_at,scored_through=row.scored_through,- windows=windows,+ series_keys=tuple(str(key) for key in series_keys),+ artifact_key=row.artifact_key,+ artifact_sha256=row.artifact_sha256,missing_series=row.missing_series,)diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.pyindex 104194333bf0..62079d017cb2 100644--- a/products/signals/backend/anomaly_scores/tick.py+++ b/products/signals/backend/anomaly_scores/tick.py@@ -7,6 +7,7 @@ tick off. Each insight activity scores one insight and writes its new buckets asimport timeimport asyncioimport datetime+import dataclassesfrom collections import Counterfrom django.conf import settings@@ -34,6 +35,7 @@ from posthog.temporal.common.scoped import scoped_temporalfrom posthog.temporal.common.utils import close_db_connectionsfrom products.product_analytics.backend.facade.api import insights_including_soft_deleted_for_team+from products.signals.backend.anomaly_scores.artifacts import ArtifactUnavailableError, load_artifacts, write_artifactsfrom products.signals.backend.anomaly_scores.config import (ANOMALY_SCORING_FLAG_DISTINCT_ID,DEFAULT_LIMITS,@@ -59,7 +61,7 @@ from products.signals.backend.anomaly_scores.scoring import (score_insight,)from products.signals.backend.anomaly_scores.selection import parse_selection_config-from products.signals.backend.anomaly_scores.state import load_state, save_state+from products.signals.backend.anomaly_scores.state import invalidate_state, load_state, save_statelogger = structlog.get_logger(__name__)@@ -78,6 +80,8 @@ STATUS_ACCESS_DENIED = "access_denied"STATUS_BROKEN_QUERY = "broken_query"# ClickHouse rejected the query because it is full. The tick starts no more picks.STATUS_AT_CAPACITY = "at_capacity"+# The stored artifact blob is missing or does not match its hash. The next visit trains.+STATUS_ARTIFACT_UNAVAILABLE = "artifact_unavailable"# Any other failure. The activity raises it, and the workflow counts it as `errored`.STATUS_ERRORED = "errored"@@ -122,7 +126,7 @@ class ScoreInsightResult:series: int = 0# What the visit ran. A score pick with no matching state trains. None when the split is off.run: ScoringRun | None = None- # Series of a score step with no stored window, and stored series missing from its result.+ # Series of a score step with no stored artifact, and stored series missing from its result.unmatched_series: int = 0missing_series: int = 0@@ -249,12 +253,23 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsighteam = Team.objects.get(id=pick.team_id)# Set before scoring, so a failure also waits for the cooldown instead of taking a slot on every tick.mark_attempted(pick.insight_id, ATTEMPT_COOLDOWN)+ profile = resolve_profile(team, insight).namesplit = Noneif pick.run is not None:- split = SplitRun(- refresh_buckets=pick.refresh_buckets,- state=load_state(team, insight) if pick.run == ScoringRun.SCORE else None,- )+ split = SplitRun()+ state = load_state(team, insight) if pick.run == ScoringRun.SCORE else None+ if state is not None:+ try:+ split = SplitRun(state=state, artifacts=load_artifacts(state.artifact_key, state.artifact_sha256))+ except ArtifactUnavailableError as error:+ logger.warning(+ "anomaly_scoring_artifact_unavailable",+ team_id=team.id,+ insight_id=insight.id,+ error=str(error)[:200],+ )+ invalidate_state(team, insight.id, profile)+ return ScoreInsightResult(status=STATUS_ARTIFACT_UNAVAILABLE, events=0, run=ScoringRun.SCORE)scores = score_insight(insight,team,@@ -264,6 +279,14 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsighlimits=input.limits,split=split,)+ if scores.artifacts and scores.state is not None:+ # The blob goes first, so every event of this visit names a model that exists.+ stored = write_artifacts(team.id, insight.id, profile, scores.artifacts)+ scores = dataclasses.replace(+ scores,+ model_id=stored.model_id,+ state=dataclasses.replace(scores.state, artifact_key=stored.key, artifact_sha256=stored.sha256),+ )events = score_events(team_id=team.id, insight_id=insight.id, scores=scores, scored_through=pick.scored_through, scored_at=now)@@ -271,7 +294,7 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh# After the emit: a lost state write only scores buckets again, and their events collapse by uuid.# A state write before a failed emit would skip those buckets for good.if scores.state is not None:- save_state(team, insight.id, resolve_profile(team, insight).name, scores.state)+ save_state(team, insight.id, profile, scores.state)return ScoreInsightResult(status=STATUS_SCORED if events else STATUS_UP_TO_DATE,events=len(events),diff --git a/products/signals/backend/migrations/0200_anomalyscoringstate_untrack_series.py b/products/signals/backend/migrations/0200_anomalyscoringstate_untrack_series.pynew file mode 100644index 000000000000..e6d87422c5dc--- /dev/null+++ b/products/signals/backend/migrations/0200_anomalyscoringstate_untrack_series.py@@ -0,0 +1,23 @@+from django.db import migrations++from posthog.migration_helpers import untrack_field+++class Migration(migrations.Migration):+ dependencies = [+ ("signals", "0199_anomalyscoringstate"),+ ]++ operations = [+ # The column stays until a later drop. New rows do not name it, so it needs a database default.+ untrack_field(+ "anomalyscoringstate",+ "series",+ database_operations=[+ migrations.RunSQL(+ sql="ALTER TABLE signals_anomalyscoringstate ALTER COLUMN series SET DEFAULT '{}'::jsonb",+ reverse_sql="ALTER TABLE signals_anomalyscoringstate ALTER COLUMN series DROP DEFAULT",+ )+ ],+ ),+ ]diff --git a/products/signals/backend/migrations/0201_anomalyscoringstate_artifacts.py b/products/signals/backend/migrations/0201_anomalyscoringstate_artifacts.pynew file mode 100644index 000000000000..a22b39d83dd7--- /dev/null+++ b/products/signals/backend/migrations/0201_anomalyscoringstate_artifacts.py@@ -0,0 +1,25 @@+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("signals", "0200_anomalyscoringstate_untrack_series"),+ ]++ operations = [+ migrations.AddField(+ model_name="anomalyscoringstate",+ name="artifact_key",+ field=models.CharField(db_default="", default="", max_length=255),+ ),+ migrations.AddField(+ model_name="anomalyscoringstate",+ name="artifact_sha256",+ field=models.CharField(db_default="", default="", max_length=64),+ ),+ migrations.AddField(+ model_name="anomalyscoringstate",+ name="series_keys",+ field=models.JSONField(db_default=[], default=list),+ ),+ ]diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txtindex 5e45e928fd34..023db632f653 100644--- a/products/signals/backend/migrations/max_migration.txt+++ b/products/signals/backend/migrations/max_migration.txt@@ -1 +1 @@-0199_anomalyscoringstate+0201_anomalyscoringstate_artifactsdiff --git a/products/signals/backend/models.py b/products/signals/backend/models.pyindex 430f7bda2b58..d8166f0151ab 100644--- a/products/signals/backend/models.py+++ b/products/signals/backend/models.py@@ -4016,10 +4016,10 @@ class UserActivitySummaryQueueCursor(models.Model):class AnomalyScoringState(TeamScopedRootMixin, UUIDModel):- """The stored series tail of one insight and scoring profile of the anomaly scoring tick.+ """The trained detector artifacts of one insight and scoring profile of the anomaly scoring tick.- A train step fetches the full detector history and writes the tail. A score step fetches only the- newest buckets, merges them into the tail, and scores the new buckets on it. See+ A train step fits one artifact per series and writes them as one blob to object storage. A score+ step fetches only the newest buckets and scores them with the stored artifacts. See`anomaly_scores/state.py`. `$anomaly_score` events stay output only."""@@ -4039,8 +4039,12 @@ class AnomalyScoringState(TeamScopedRootMixin, UUIDModel):scored_through = models.DateTimeField(null=True, blank=True)# Stored series that the last score step did not get back. More than half of them makes the next visit train.missing_series = models.PositiveIntegerField(default=0)- # Series key to `{"first_bucket": ISO bucket start, "values": [...]}`, with the newest values the detector reads.- series = models.JSONField(default=dict)+ # Object storage key of the artifact blob. Keys are never reused, so a loaded blob can be cached by key.+ artifact_key = models.CharField(max_length=255, default="", db_default="")+ # Checked before every load. A mismatch or a missing blob makes the next visit train.+ artifact_sha256 = models.CharField(max_length=64, default="", db_default="")+ # The series keys that the blob has an artifact for.+ series_keys = models.JSONField(default=list, db_default=[])created_at = models.DateTimeField(auto_now_add=True)updated_at = models.DateTimeField(auto_now=True)diff --git a/products/signals/backend/test/test_anomaly_scores.py b/products/signals/backend/test/test_anomaly_scores.pyindex f9f6930b5cd8..50461c364717 100644--- a/products/signals/backend/test/test_anomaly_scores.py+++ b/products/signals/backend/test/test_anomaly_scores.py@@ -1,3 +1,4 @@+import dataclassesfrom datetime import UTC, datetime, timedeltafrom typing import Anyfrom zoneinfo import ZoneInfo@@ -25,11 +26,13 @@ from posthog.hogql.constants import INSIGHT_QUERY_FANOUT_CONCURRENCYfrom posthog.interval_specs import interval_specfrom posthog.models import Team+from posthog.tasks.alerts.detectors.registry import get_detectorfrom products.product_analytics.backend.facade.models import Insight+from products.signals.backend.anomaly_scores.artifacts import deserialize, serializefrom products.signals.backend.anomaly_scores.config import DEFAULT_LIMITS, DEFAULT_PROFILE, IntervalScoring, ScoringRunfrom products.signals.backend.anomaly_scores.scoring import (- SeriesWindow,+ InsightScores,SplitRun,StateIdentity,TrainedState,@@ -37,12 +40,9 @@ from products.signals.backend.anomaly_scores.scoring import (bucket_end,buckets_to_score,fetch_buckets,- history_buckets,query_cost,score_insight,score_series,- score_window,- train_window,trends_query_for,)from products.signals.backend.anomaly_scores.state import needs_training@@ -187,66 +187,24 @@ class TestScoreSeries(SimpleTestCase):assert all(point.score is None and point.flag is None for point in points)-class TestScoreWindow(SimpleTestCase):+class TestArtifactRoundTrip(SimpleTestCase):@parameterized.expand([- ("day", IntervalType.DAY, False),- ("week", IntervalType.WEEK, False),- ("hour", IntervalType.HOUR, False),- ("day_late_events", IntervalType.DAY, True),- ("hour_late_events", IntervalType.HOUR, True),+ ("json_params", DAY_SCORING.detector),+ ("pyod_model_bytes", {"type": "knn", "threshold": 0.95, "preprocessing": {"diffs_n": 1, "lags_n": 5}}),])- def test_a_score_step_on_the_stored_window_scores_like_the_full_history(- self, _name: str, interval: IntervalType, late_events: bool- ) -> None:- scoring = DEFAULT_PROFILE.intervals[interval]- history = history_buckets(scoring)- count = history + 40- values = list(np.random.default_rng(2).normal(100, 10, size=count))- values[-1] = 400.0- buckets = _buckets(datetime(2025, 1, 6, tzinfo=UTC), interval, count)- trained_at = bucket_end(buckets[-3], interval) + timedelta(minutes=5)- scored_at = bucket_end(buckets[-1], interval) + timedelta(minutes=5)- window = train_window(values[:-1], buckets[:-1], history=history, interval=interval, now=trained_at)- assert window is not None- later = list(values)- if late_events:- # Events that arrive after the train step change a bucket that the score step fetches again.- later[-4] += 50.0- refresh = 3- # The score step fetches the refresh buckets, one guard bucket, and the two new buckets.- fetched = slice(count - (refresh + 1 + 2), count)-- scored = score_window(- window, later[fetched], buckets[fetched], refresh=refresh, scoring=scoring, interval=interval, now=scored_at- )+ def test_a_stored_artifact_scores_like_the_fitted_one(self, _name: str, config: dict[str, Any]) -> None:+ detector = get_detector(config)+ data = np.random.default_rng(2).normal(100, 10, size=100)+ data[-1] = 400.0+ artifact = detector.fit(data[:90])+ scored = data[90 - detector.context_buckets() :]- assert scored is not None- points, updated = scored- expected = score_series(later, buckets, scoring=scoring, interval=interval, now=scored_at, after=buckets[-3])- assert [(point.bucket_start, point.score, point.flag) for point in points] == [- (point.bucket_start, point.score, point.flag) for point in expected- ]- assert [point.bucket_start for point in points] == buckets[-2:]- assert points[-1].flag is True- assert updated == SeriesWindow(first_bucket=buckets[-history], values=later[-history:])-- def test_fresh_buckets_after_a_gap_leave_the_window_unscored(self) -> None:- buckets = _buckets(datetime(2026, 1, 1, tzinfo=UTC), IntervalType.DAY, 70)- window = SeriesWindow(first_bucket=buckets[0], values=[1.0] * 60)-- scored = score_window(- window,- [1.0] * 5,- buckets[62:67],- refresh=3,- scoring=DAY_SCORING,- interval=IntervalType.DAY,- now=buckets[67],- )+ loaded = deserialize(serialize({"series": artifact}))["series"]- assert scored is None+ assert loaded == artifact+ assert detector.score(loaded, scored).all_scores == detector.score(artifact, scored).all_scoresdef _insight(series: list[EventsNode], **query: Any) -> Insight:@@ -326,8 +284,9 @@ class TestScoreInsight(SimpleTestCase):if waves == 1:assert limits[0] == 150+ # The context of the day detector is 3 buckets, then one guard bucket and the 2 new buckets.@parameterized.expand([("no_smoothing", None, "-6d"), ("smoothing", 7, "-12d")])- def test_a_score_step_fetches_only_new_buckets_and_keeps_series_it_did_not_get_back(+ def test_a_score_step_fetches_only_the_context_and_new_buckets(self, _name: str, smoothing: int | None, date_from: str) -> None:insight = _insight([EventsNode(event="$pageview")], trendsFilter=TrendsFilter(smoothingIntervals=smoothing))@@ -343,7 +302,7 @@ class TestScoreInsight(SimpleTestCase):"data": list(rng.normal(100, 10, size=days)),}- def run(now: datetime, results: list[dict[str, Any]], split: SplitRun) -> tuple[Any, TrendsQuery]:+ def run(now: datetime, results: list[dict[str, Any]], split: SplitRun) -> tuple[InsightScores, TrendsQuery]:with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:runner.return_value.run.return_value = MagicMock(results=results)scores = score_insight(insight, team, now=now, split=split)@@ -355,15 +314,16 @@ class TestScoreInsight(SimpleTestCase):result("kept", datetime(2025, 10, 1, tzinfo=UTC), 102),result("gone", datetime(2025, 10, 1, tzinfo=UTC), 102),],- SplitRun(refresh_buckets=3),+ SplitRun(),)- assert trained.run == ScoringRun.TRAIN and trained.state is not None- keys = {series.breakdown_value: series.series_key for series in trained.series}+ assert trained.run == ScoringRun.TRAIN and trained.state is not None and trained.artifacts is not None+ assert set(trained.state.series_keys) == {series.series_key for series in trained.series}+ state = dataclasses.replace(trained.state, artifact_key="anomaly_models/1/7/default-v1/model")scores, query = run(datetime(2026, 1, 12, 0, 30, tzinfo=UTC),[result("kept", datetime(2026, 1, 6, tzinfo=UTC), 7), result("new", datetime(2026, 1, 6, tzinfo=UTC), 7)],- SplitRun(refresh_buckets=3, state=trained.state),+ SplitRun(state=state, artifacts=trained.artifacts),)assert query.dateRange is not None and query.dateRange.date_from == date_from@@ -375,15 +335,32 @@ class TestScoreInsight(SimpleTestCase):assert scores.unmatched_series == 1assert scores.state is not Noneassert scores.state.missing_series == 1- assert scores.state.windows[keys["gone"]] == trained.state.windows[keys["gone"]]+ assert scores.state.series_keys == trained.state.series_keysassert scores.state.scored_through == datetime(2026, 1, 11, tzinfo=UTC)+ assert scores.model_id == "model"++ # "gone" comes back. Its own newest scored bucket is still the one from the train step.+ with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:+ runner.return_value.run.return_value = MagicMock(+ results=[result("gone", datetime(2026, 1, 6, tzinfo=UTC), 7)]+ )+ returned = score_insight(+ insight,+ team,+ now=datetime(2026, 1, 13, 0, 30, tzinfo=UTC),+ after=trained.state.scored_through,+ split=SplitRun(state=scores.state, artifacts=trained.artifacts),+ )++ assert [point.bucket_start.day for point in returned.series[0].points] == [10, 11, 12]def test_a_score_pick_whose_state_no_longer_matches_the_insight_trains(self) -> None:stale = TrainedState(identity=StateIdentity(interval=IntervalType.DAY, detector_version="old", query_hash="old"),trained_at=datetime(2026, 1, 9, tzinfo=UTC),scored_through=datetime(2026, 1, 9, tzinfo=UTC),- windows={},+ series_keys=("a",),+ artifact_key="anomaly_models/1/7/default-v1/old",)with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:runner.return_value.run.return_value = MagicMock(results=[])@@ -391,16 +368,41 @@ class TestScoreInsight(SimpleTestCase):_insight([EventsNode(event="$pageview")]),Team(timezone="UTC"),now=datetime(2026, 1, 10, tzinfo=UTC),- split=SplitRun(refresh_buckets=3, state=stale),+ split=SplitRun(state=stale, artifacts={}),)assert scores.run == ScoringRun.TRAIN+ # The detector history and the backfill cap of a first visit, the same fetch as without the split.assert runner.call_args.kwargs["query"].dateRange.date_from == "-90d"+ @parameterized.expand([(interval.value, interval) for interval in DEFAULT_PROFILE.intervals])+ def test_a_train_step_fits_every_series_at_each_default_interval(self, _name: str, interval: IntervalType) -> None:+ buckets = [datetime(2024, 1, 1, tzinfo=UTC)]+ while len(buckets) < 400:+ buckets.append(bucket_end(buckets[-1], interval))+ result = {+ "action": {"order": 0},+ "label": "pageview",+ "breakdown_value": None,+ "days": [bucket.strftime("%Y-%m-%d %H:%M:%S") for bucket in buckets],+ "data": list(np.random.default_rng(5).normal(100, 10, size=len(buckets))),+ }+ with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:+ runner.return_value.run.return_value = MagicMock(results=[result])+ scores = score_insight(+ _insight([EventsNode(event="$pageview")], interval=interval),+ Team(timezone="UTC"),+ now=bucket_end(buckets[-1], interval),+ split=SplitRun(),+ )++ assert scores.run == ScoringRun.TRAIN and scores.artifacts is not None+ assert set(scores.artifacts) == {series.series_key for series in scores.series} != set()+ assert all(point.score is not None for point in scores.series[0].points)+_IDENTITY = StateIdentity(interval=IntervalType.DAY, detector_version="v1", query_hash="q1")_SCORED_THROUGH = datetime(2026, 1, 9, tzinfo=UTC)-_WINDOW = SeriesWindow(first_bucket=datetime(2025, 11, 11, tzinfo=UTC), values=[1.0] * 60)def _state(**changes: Any) -> TrainedState:@@ -408,7 +410,8 @@ def _state(**changes: Any) -> TrainedState:"identity": _IDENTITY,"trained_at": datetime(2026, 1, 5, tzinfo=UTC),"scored_through": _SCORED_THROUGH,- "windows": {"a": _WINDOW, "b": _WINDOW},+ "series_keys": ("a", "b"),+ "artifact_key": "anomaly_models/1/7/default-v1/model",**changes,}return TrainedState(**fields)@@ -420,6 +423,7 @@ class TestNeedsTraining(SimpleTestCase):("fresh_state", _state(), False),("one_of_two_series_missing", _state(missing_series=1), False),("no_state", None, True),+ ("no_artifact", _state(artifact_key=""), True),("query_changed",_state(identity=StateIdentity(interval=IntervalType.DAY, detector_version="v1", query_hash="q0")),diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.pyindex 0daeededb270..7dc48843538e 100644--- a/products/signals/backend/test/test_anomaly_scoring_tick.py+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py@@ -1,4 +1,6 @@+import jsonimport uuid+import hashlibfrom datetime import UTC, datetime, timedeltafrom typing import Anyfrom zoneinfo import ZoneInfo@@ -23,11 +25,13 @@ from posthog.hogql.errors import QueryError, ResolutionError, TableAccessDeniedEfrom posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimitExceeded, ClickHouseQueryTimeOutfrom posthog.models import Organization, Team, User+from posthog.tasks.alerts.detectors.base import ARTIFACT_SCHEMA_VERSIONfrom products.dashboards.backend.models.dashboard import Dashboardfrom products.dashboards.backend.models.dashboard_tile import DashboardTilefrom products.product_analytics.backend.facade.api import record_insight_viewfrom products.product_analytics.backend.facade.models import Insight+from products.signals.backend.anomaly_scores.artifacts import clear_artifact_cachefrom products.signals.backend.anomaly_scores.config import DISABLED, AnomalyScoringConfig, ScoringRun, parse_configfrom products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_eventsfrom products.signals.backend.anomaly_scores.queue import (@@ -54,7 +58,7 @@ from products.signals.backend.anomaly_scores.scoring import (UnsupportedInsightError,)from products.signals.backend.anomaly_scores.selection import SelectionConfig, Weighting-from products.signals.backend.anomaly_scores.state import RunPlan+from products.signals.backend.anomaly_scores.state import RunPlan, load_state, plan_runfrom products.signals.backend.anomaly_scores.tick import (AnomalyScoringTickInput,AnomalyScoringTickPlan,@@ -126,7 +130,6 @@ class TestTickConfig(SimpleTestCase):"limits": {"max_backfill_buckets": 100_000, "max_series": -1},"train_score_split": "true","train_every_buckets": {"hour": 100_000, "day": 0, "week": 2},- "refresh_buckets": ["day"],})@@ -142,7 +145,6 @@ class TestTickConfig(SimpleTestCase):IntervalType.WEEK: 2,IntervalType.MONTH: 3,}- assert config.refresh_buckets == DISABLED.refresh_buckets@parameterized.expand([@@ -487,7 +489,26 @@ class TestRunInsight(SimpleTestCase):class TestTrainScoreSplit(BaseTest):- def _run(self, insight: Insight, pick: InsightPick, now: datetime, data: list[float]) -> list[dict[str, Any]]:+ def setUp(self) -> None:+ super().setUp()+ self.team.timezone = "UTC"+ self.team.save()+ self.blobs: dict[str, bytes] = {}+ storage = "products.signals.backend.anomaly_scores.artifacts.object_storage"+ for name, fake in [+ ("write", lambda key, blob: self.blobs.__setitem__(key, blob)),+ ("read_bytes", lambda key, missing_ok: self.blobs.get(key)),+ ("delete", lambda key: self.blobs.pop(key, None)),+ ]:+ patcher = patch(f"{storage}.{name}", side_effect=fake)+ patcher.start()+ self.addCleanup(patcher.stop)+ clear_artifact_cache()+ self.addCleanup(clear_artifact_cache)++ def _run(+ self, insight: Insight, pick: InsightPick, now: datetime, data: list[float]+ ) -> tuple[ScoreInsightResult, list[dict[str, Any]]]:days = [(now - timedelta(days=len(data) - 1 - day)).strftime("%Y-%m-%d") for day in range(len(data))]results = [{"action": {"order": 0}, "label": "pageview", "days": days, "data": data, "breakdown_value": None}]with (@@ -496,36 +517,40 @@ class TestTrainScoreSplit(BaseTest):patch("products.signals.backend.anomaly_scores.tick.mark_attempted"),):runner.return_value.run.return_value = MagicMock(results=results)- run_insight(ScoreInsightInput(pick=pick), now)- return emit.call_args.args[1]+ result = run_insight(ScoreInsightInput(pick=pick), now)+ return result, emit.call_args.args[1] if emit.called else []- def test_a_train_pick_stores_the_tail_that_the_next_score_pick_scores_on(self) -> None:- self.team.timezone = "UTC"- self.team.save()- insight = Insight.objects.create(team=self.team, saved=True, query=_trends(interval="day"))+ def _train(self, insight: Insight, trained_at: datetime) -> list[dict[str, Any]]:values = [float(value) for value in np.random.default_rng(4).normal(100, 5, size=100)]- # JSON has no NaN, so the oldest stored bucket is stored as null. The detector no longer reads it.- values[39] = float("nan")- trained_at = datetime(2026, 10, 20, 0, 30, tzinfo=UTC)-- trained = self._run(+ _result, events = self._run(insight,InsightPick(team_id=self.team.id, insight_id=insight.id, scored_through={}, run=ScoringRun.TRAIN),trained_at,values,)- scored = self._run(+ self.values = values+ return events++ def _score_pick(self, insight: Insight, trained: list[dict[str, Any]]) -> InsightPick:+ return InsightPick(+ team_id=self.team.id,+ insight_id=insight.id,+ scored_through={trained[-1]["properties"]["$anomaly_series_key"]: datetime(2026, 10, 19, tzinfo=UTC)},+ run=ScoringRun.SCORE,+ )++ def test_a_train_pick_stores_the_artifact_that_the_next_score_pick_scores_with(self) -> None:+ insight = Insight.objects.create(team=self.team, saved=True, query=_trends(interval="day"))+ trained_at = datetime(2026, 10, 20, 0, 30, tzinfo=UTC)+ trained = self._train(insight, trained_at)+ first_key = AnomalyScoringState.objects.for_team(self.team.id).get(insight_id=insight.id).artifact_key+ _result, scored = self._run(insight,- InsightPick(- team_id=self.team.id,- insight_id=insight.id,- scored_through={trained[-1]["properties"]["$anomaly_series_key"]: datetime(2026, 10, 19, tzinfo=UTC)},- run=ScoringRun.SCORE,- refresh_buckets=3,- ),+ self._score_pick(insight, trained),trained_at + timedelta(days=2),- [*values[-4:], 500.0, 100.0],+ [*self.values[-4:], 500.0, 100.0],)+ self._train(insight, trained_at + timedelta(days=7))assert {event["properties"]["$anomaly_run"] for event in trained} == {"train"}assert [@@ -536,9 +561,42 @@ class TestTrainScoreSplit(BaseTest):)for event in scored] == [("score", "2026-10-20T00:00:00+00:00", 0), ("score", "2026-10-21T00:00:00+00:00", 1)]+ model_ids = {event["properties"]["$anomaly_model_id"] for event in [*trained, *scored]}+ assert model_ids == {first_key.rsplit("/", 1)[-1]}+ state = AnomalyScoringState.objects.for_team(self.team.id).get(insight_id=insight.id)+ assert state.trained_at == trained_at + timedelta(days=7)+ # A retrain replaces the blob and deletes the previous one.+ assert list(self.blobs) == [state.artifact_key] != [first_key]++ @parameterized.expand([("corrupt",), ("missing",), ("older_schema",)])+ def test_an_artifact_blob_that_fails_its_check_makes_the_next_visit_train(self, case: str) -> None:+ insight = Insight.objects.create(team=self.team, saved=True, query=_trends(interval="day"))+ trained_at = datetime(2026, 10, 20, 0, 30, tzinfo=UTC)+ trained = self._train(insight, trained_at)state = AnomalyScoringState.objects.for_team(self.team.id).get(insight_id=insight.id)- assert state.scored_through == datetime(2026, 10, 21, tzinfo=UTC)- assert state.trained_at == trained_at+ key = state.artifact_key+ clear_artifact_cache()+ if case == "missing":+ del self.blobs[key]+ elif case == "corrupt":+ self.blobs[key] = b"{}"+ else:+ stored = json.loads(self.blobs[key])+ for artifact in stored["series"].values():+ artifact["schema_version"] = ARTIFACT_SCHEMA_VERSION - 1+ self.blobs[key] = json.dumps(stored).encode()+ state.artifact_sha256 = hashlib.sha256(self.blobs[key]).hexdigest()+ state.save()+ scored_at = trained_at + timedelta(days=2)++ result, events = self._run(insight, self._score_pick(insight, trained), scored_at, self.values[-6:])++ assert (result.status, events) == ("artifact_unavailable", [])+ plan = plan_run(insight, self.team, load_state(self.team, insight), parse_config(_SPLIT_PAYLOAD), scored_at)+ assert plan.run == ScoringRun.TRAIN+++_SPLIT_PAYLOAD = {"enabled": True, "organization_ids": [ORGANIZATION_ID], "train_score_split": True}class TestEligibleTeams(BaseTest):