Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): train anomaly detectors into stored model artifacts (#123112)

posthog[bot]053a4f952205
diff --git a/frontend/src/taxonomy/core-filter-definitions-by-group.json b/frontend/src/taxonomy/core-filter-definitions-by-group.json
index 9dee9bfc3ed5..05f9cf13a10a 100644
--- a/frontend/src/taxonomy/core-filter-definitions-by-group.json
+++ b/frontend/src/taxonomy/core-filter-definitions-by-group.json
@@ -797,6 +797,11 @@
"label": "Anomaly interval",
"type": "String"
},
+ "$anomaly_model_id": {
+ "description": "ID of the stored model that scored the bucket. Buckets with the same ID were scored with the same fit. Empty when scoring fits the detector again on every run.",
+ "label": "Anomaly model ID",
+ "type": "String"
+ },
"$anomaly_profile": {
"description": "Name of the scoring profile that produced the score.",
"examples": ["default-v1"],
@@ -804,7 +809,7 @@
"type": "String"
},
"$anomaly_run": {
- "description": "How the bucket was scored. `train` fetched the full history of the series, and `score` scored the bucket on stored history. Empty when scoring fetches the full history on every run.",
+ "description": "How the bucket was scored. `train` fit a new model on the history of the series, and `score` scored the bucket with a stored model. Empty when scoring fits the detector again on every run.",
"examples": ["train", "score"],
"label": "Anomaly scoring run",
"type": "String"
@@ -4745,6 +4750,11 @@
"label": "Anomaly interval",
"type": "String"
},
+ "$anomaly_model_id": {
+ "description": "ID of the stored model that scored the bucket. Buckets with the same ID were scored with the same fit. Empty when scoring fits the detector again on every run.",
+ "label": "Anomaly model ID",
+ "type": "String"
+ },
"$anomaly_profile": {
"description": "Name of the scoring profile that produced the score.",
"examples": ["default-v1"],
@@ -4752,7 +4762,7 @@
"type": "String"
},
"$anomaly_run": {
- "description": "How the bucket was scored. `train` fetched the full history of the series, and `score` scored the bucket on stored history. Empty when scoring fetches the full history on every run.",
+ "description": "How the bucket was scored. `train` fit a new model on the history of the series, and `score` scored the bucket with a stored model. Empty when scoring fits the detector again on every run.",
"examples": ["train", "score"],
"label": "Anomaly scoring run",
"type": "String"
diff --git a/posthog/tasks/alerts/detectors/base.py b/posthog/tasks/alerts/detectors/base.py
index fe0f6543cfe4..26cd5c44801d 100644
--- a/posthog/tasks/alerts/detectors/base.py
+++ b/posthog/tasks/alerts/detectors/base.py
@@ -4,10 +4,14 @@ from typing import Any
import numpy as np
+from posthog.dataclasses import frozen
from posthog.tasks.alerts.detectors.preprocessing import preprocess_data
+# A change to what `fit` stores or what `score` reads needs a new version, so an older artifact is never read.
+ARTIFACT_SCHEMA_VERSION = 1
-@dataclass
+
+@dataclass(frozen=False)
class DetectionResult:
"""Result from anomaly detection."""
@@ -16,6 +20,27 @@ class DetectionResult:
triggered_indices: list[int] = field(default_factory=list)
all_scores: list[float | None] = field(default_factory=list)
metadata: dict[str, Any] = field(default_factory=dict)
+ # One entry per scored bucket of `score()`, with the values that `metadata` holds for `detect()`.
+ all_metadata: list[dict[str, Any]] = field(default_factory=list)
+
+
+@frozen
+class DetectorArtifact:
+ """What `fit` learned from a history. `score` reads it instead of fitting again."""
+
+ detector_type: str
+ schema_version: int
+ # The detector config the artifact was fit with, preprocessing and threshold included.
+ config: dict[str, Any]
+ # JSON-safe values only.
+ params: dict[str, Any]
+ model_bytes: bytes | None = None
+ # The artifacts of the sub-detectors of an ensemble, in config order.
+ children: tuple["DetectorArtifact", ...] = ()
+
+
+class FitError(ValueError):
+ """The history cannot fit this detector, for example because it is too short."""
class BaseDetector(ABC):
@@ -65,6 +90,82 @@ class BaseDetector(ABC):
"""
pass
+ def fit(self, history: np.ndarray) -> DetectorArtifact:
+ """Fit on ``history``, the buckets before the first bucket to score.
+
+ The newest ``training_offset - 1`` buckets of ``history`` stay out of the fit, so
+ ``score(fit(data[:-1]), ...)`` reads the same training data as ``detect(data)``.
+ """
+ raise NotImplementedError(f"{type(self).__name__} cannot fit an artifact")
+
+ def score(self, artifact: DetectorArtifact, data: np.ndarray) -> DetectionResult:
+ """Score buckets with a fitted artifact.
+
+ ``data`` holds ``context_buckets()`` buckets, then the buckets to score. The result has one entry in
+ ``all_scores`` and ``all_metadata`` per scored bucket, and ``triggered_indices`` count from the first scored bucket.
+ """
+ self._check_artifact(artifact)
+ context = self.context_buckets()
+ if len(data) <= context:
+ return DetectionResult(is_anomaly=False)
+ processed = self.preprocess(np.asarray(data, dtype=float))[context:]
+ scores, metadata = self._score_rows(artifact, processed)
+ return self._scored_result(scores, metadata)
+
+ def context_buckets(self) -> int:
+ """Buckets that preprocessing reads before the first scored bucket: diffs, lags and smoothing."""
+ preprocessing = self.preprocessing_config
+ return (
+ max((preprocessing.get("smooth_n") or 0) - 1, 0)
+ + (1 if preprocessing.get("diffs_n") else 0)
+ + min(max(preprocessing.get("lags_n") or 0, 0), 10)
+ )
+
+ def _score_rows(
+ self, artifact: DetectorArtifact, rows: np.ndarray
+ ) -> tuple[list[float | None], list[dict[str, Any]]]:
+ """The score and metadata of each preprocessed row."""
+ raise NotImplementedError(f"{type(self).__name__} cannot score an artifact")
+
+ def _artifact(
+ self,
+ params: dict[str, Any],
+ *,
+ model_bytes: bytes | None = None,
+ children: tuple[DetectorArtifact, ...] = (),
+ ) -> DetectorArtifact:
+ return DetectorArtifact(
+ detector_type=str(self.config.get("type")),
+ schema_version=ARTIFACT_SCHEMA_VERSION,
+ config=self.config,
+ params=params,
+ model_bytes=model_bytes,
+ children=children,
+ )
+
+ def _check_artifact(self, artifact: DetectorArtifact) -> None:
+ if artifact.schema_version != ARTIFACT_SCHEMA_VERSION or artifact.detector_type != self.config.get("type"):
+ raise ValueError(f"Artifact {artifact.detector_type} v{artifact.schema_version} does not fit this detector")
+
+ def _fit_rows(self, history: np.ndarray, min_rows: int) -> np.ndarray:
+ """The preprocessed rows of ``history`` that a fit reads, without the newest ``training_offset - 1``."""
+ rows = self.preprocess(np.asarray(history, dtype=float))
+ rows = rows[: len(rows) - (max(self.training_offset, 1) - 1)]
+ if len(rows) < min_rows:
+ raise FitError(f"{len(rows)} buckets cannot fit, {min_rows} needed")
+ return rows
+
+ def _scored_result(self, scores: list[float | None], metadata: list[dict[str, Any]]) -> DetectionResult:
+ threshold = self.config.get("threshold", self.DEFAULT_THRESHOLD)
+ triggered = [i for i, score in enumerate(scores) if score is not None and score > threshold]
+ return DetectionResult(
+ is_anomaly=bool(triggered) and triggered[-1] == len(scores) - 1,
+ score=scores[-1] if scores else None,
+ triggered_indices=triggered,
+ all_scores=scores,
+ all_metadata=metadata,
+ )
+
def train_test_split(self, data: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
"""Split data into training (historical) and test (recent) portions.
diff --git a/posthog/tasks/alerts/detectors/ensemble.py b/posthog/tasks/alerts/detectors/ensemble.py
index b7e15c1723fa..2249611f168a 100644
--- a/posthog/tasks/alerts/detectors/ensemble.py
+++ b/posthog/tasks/alerts/detectors/ensemble.py
@@ -2,7 +2,7 @@ from typing import Any
import numpy as np
-from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult
+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifact
from posthog.tasks.alerts.detectors.registry import get_detector, register_detector
@@ -72,6 +72,54 @@ class EnsembleDetector(BaseDetector):
},
)
+ def fit(self, history: np.ndarray) -> DetectorArtifact:
+ return self._artifact(
+ {"operator": self.operator}, children=tuple(detector.fit(history) for detector in self.sub_detectors)
+ )
+
+ def context_buckets(self) -> int:
+ return max(detector.context_buckets() for detector in self.sub_detectors)
+
+ def score(self, artifact: DetectorArtifact, data: np.ndarray) -> DetectionResult:
+ self._check_artifact(artifact)
+ if len(artifact.children) != len(self.sub_detectors):
+ raise ValueError("The artifact has another number of sub-detectors")
+ context = self.context_buckets()
+ if len(data) <= context:
+ return DetectionResult(is_anomaly=False)
+ # Each sub-detector reads only the context it needs.
+ results = [
+ detector.score(child, data[context - detector.context_buckets() :])
+ for detector, child in zip(self.sub_detectors, artifact.children)
+ ]
+ combine = all if self.operator == "and" else any
+ pick = min if self.operator == "and" else max
+ scores: list[float | None] = []
+ flags: list[int] = []
+ metadata: list[dict[str, Any]] = []
+ for i in range(len(data) - context):
+ sub_scores = [result.all_scores[i] for result in results]
+ known = [score for score in sub_scores if score is not None]
+ scores.append(pick(known) if known else None)
+ if combine(i in result.triggered_indices for result in results):
+ flags.append(i)
+ metadata.append(
+ {
+ "operator": self.operator,
+ "sub_results": [
+ {"type": cfg.get("type"), "is_anomaly": i in result.triggered_indices, "score": score}
+ for cfg, result, score in zip(self.config.get("detectors", []), results, sub_scores)
+ ],
+ }
+ )
+ return DetectionResult(
+ is_anomaly=bool(flags) and flags[-1] == len(scores) - 1,
+ score=scores[-1] if scores else None,
+ triggered_indices=flags,
+ all_scores=scores,
+ all_metadata=metadata,
+ )
+
def detect_batch(self, data: np.ndarray) -> DetectionResult:
results = [d.detect_batch(data) for d in self.sub_detectors]
diff --git a/posthog/tasks/alerts/detectors/pyod_detectors/base_pyod.py b/posthog/tasks/alerts/detectors/pyod_detectors/base_pyod.py
index e8a59080182c..2a276473dbde 100644
--- a/posthog/tasks/alerts/detectors/pyod_detectors/base_pyod.py
+++ b/posthog/tasks/alerts/detectors/pyod_detectors/base_pyod.py
@@ -1,9 +1,12 @@
+import io
from abc import abstractmethod
+from typing import Any
import numpy as np
+import joblib
from pyod.models.base import BaseDetector as PyODBaseDetector
-from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult
+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifact, FitError
class BasePyODDetector(BaseDetector):
@@ -63,6 +66,49 @@ class BasePyODDetector(BaseDetector):
metadata={"raw_score": float(model.decision_function(last_point)[0])},
)
+ def fit(self, history: np.ndarray) -> DetectorArtifact:
+ # detect() needs MIN_SAMPLES buckets, the scored one included.
+ if len(history) + 1 < self.MIN_SAMPLES:
+ raise FitError(f"{len(history)} buckets cannot fit, {self.MIN_SAMPLES - 1} needed")
+ rows = self._fit_rows(history, min_rows=1)
+ if rows.ndim == 1:
+ rows = rows.reshape(-1, 1)
+ if not np.all(np.isfinite(rows)):
+ raise FitError("The history has values that are not finite")
+ try:
+ model = self._build_model(n_samples=len(rows))
+ model.fit(rows)
+ except (ValueError, np.linalg.LinAlgError) as error:
+ raise FitError(str(error)) from error
+ buffer = io.BytesIO()
+ joblib.dump(model, buffer)
+ return self._artifact({"n_samples": len(rows)}, model_bytes=buffer.getvalue())
+
+ def _score_rows(
+ self, artifact: DetectorArtifact, rows: np.ndarray
+ ) -> tuple[list[float | None], list[dict[str, Any]]]:
+ if artifact.model_bytes is None:
+ raise ValueError("The artifact has no model")
+ # nosemgrep: trailofbits.python.scikit-joblib-load.scikit-joblib-load (bytes come only from fit() via a blob whose SHA-256 the caller checked against Postgres)
+ model: PyODBaseDetector = joblib.load(io.BytesIO(artifact.model_bytes))
+ if rows.ndim == 1:
+ rows = rows.reshape(-1, 1)
+ scores: list[float | None] = []
+ metadata: list[dict[str, Any]] = []
+ for i in range(len(rows)):
+ point = rows[i : i + 1]
+ if not np.all(np.isfinite(point)):
+ scores.append(None)
+ metadata.append({})
+ continue
+ try:
+ scores.append(float(model.predict_proba(point)[0, 1]))
+ metadata.append({"raw_score": float(model.decision_function(point)[0])})
+ except (ValueError, np.linalg.LinAlgError):
+ scores.append(None)
+ metadata.append({})
+ return scores, metadata
+
def detect_batch(self, data: np.ndarray) -> DetectionResult:
if not self._validate_data(data, min_length=self.MIN_SAMPLES):
return DetectionResult(is_anomaly=False)
diff --git a/posthog/tasks/alerts/detectors/statistical/iqr.py b/posthog/tasks/alerts/detectors/statistical/iqr.py
index 1fa63e6323e0..a1fab3993915 100644
--- a/posthog/tasks/alerts/detectors/statistical/iqr.py
+++ b/posthog/tasks/alerts/detectors/statistical/iqr.py
@@ -5,7 +5,7 @@ from scipy.special import erf
from posthog.schema import DetectorType
-from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult
+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifact
from posthog.tasks.alerts.detectors.registry import register_detector
@@ -26,8 +26,10 @@ def _iqr_distance_to_probability(distance: float, window_distances: np.ndarray)
Uses pyod's 'unify' approach: standardize the distance against the
distribution of distances observed in the training window, then apply erf.
"""
- mean_d = float(window_distances.mean())
- std_d = float(window_distances.std())
+ return _normalize_iqr_distance(distance, float(window_distances.mean()), float(window_distances.std()))
+
+
+def _normalize_iqr_distance(distance: float, mean_d: float, std_d: float) -> float:
if std_d == 0:
return 1.0 if distance > mean_d else 0.0
standardized = (distance - mean_d) / std_d
@@ -176,6 +178,63 @@ class IQRDetector(BaseDetector):
all_scores=scores,
)
+ def fit(self, history: np.ndarray) -> DetectorArtifact:
+ multiplier = self.config.get("multiplier", 1.5)
+ window = self.config.get("window", 30)
+ diffs_n = 1 if self.preprocessing_config.get("diffs_n") else 0
+ rows = self._fit_rows(history, min_rows=window + diffs_n)
+ values = rows if rows.ndim == 1 else rows[:, 0]
+ window_data = values[diffs_n:][-window:]
+ q1 = np.percentile(window_data, 25)
+ q3 = np.percentile(window_data, 75)
+ iqr = q3 - q1
+ lower_fence = q1 - multiplier * iqr
+ upper_fence = q3 + multiplier * iqr
+ window_distances = _iqr_fence_distances(window_data, lower_fence, upper_fence, iqr)
+ return self._artifact(
+ {
+ "q1": float(q1),
+ "q3": float(q3),
+ "iqr": float(iqr),
+ "lower_fence": float(lower_fence),
+ "upper_fence": float(upper_fence),
+ "mean_distance": float(window_distances.mean()),
+ "std_distance": float(window_distances.std()),
+ }
+ )
+
+ def _score_rows(
+ self, artifact: DetectorArtifact, rows: np.ndarray
+ ) -> tuple[list[float | None], list[dict[str, Any]]]:
+ params = artifact.params
+ iqr = params["iqr"]
+ lower_fence = params["lower_fence"]
+ upper_fence = params["upper_fence"]
+ scores: list[float | None] = []
+ metadata: list[dict[str, Any]] = []
+ for value in rows if rows.ndim == 1 else rows[:, 0]:
+ if iqr == 0:
+ raw_distance = 0.0
+ elif value < lower_fence:
+ raw_distance = (lower_fence - value) / iqr
+ elif value > upper_fence:
+ raw_distance = (value - upper_fence) / iqr
+ else:
+ raw_distance = 0.0
+ scores.append(_normalize_iqr_distance(raw_distance, params["mean_distance"], params["std_distance"]))
+ metadata.append(
+ {
+ "q1": params["q1"],
+ "q3": params["q3"],
+ "iqr": iqr,
+ "lower_fence": lower_fence,
+ "upper_fence": upper_fence,
+ "value": float(value),
+ "raw_distance": float(raw_distance),
+ }
+ )
+ return scores, metadata
+
@classmethod
def get_default_config(cls) -> dict[str, Any]:
return {
diff --git a/posthog/tasks/alerts/detectors/statistical/mad.py b/posthog/tasks/alerts/detectors/statistical/mad.py
index b50cd278b50e..b7e77300cf61 100644
--- a/posthog/tasks/alerts/detectors/statistical/mad.py
+++ b/posthog/tasks/alerts/detectors/statistical/mad.py
@@ -1,9 +1,11 @@
+from typing import Any
+
import numpy as np
from pyod.models.mad import MAD
from posthog.schema import DetectorType
-from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult
+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifact
from posthog.tasks.alerts.detectors.registry import register_detector
@@ -115,6 +117,48 @@ class MADDetector(BaseDetector):
metadata={"threshold": threshold, "window": window},
)
+ def fit(self, history: np.ndarray) -> DetectorArtifact:
+ window = self.config.get("window", 30)
+ diffs_n = 1 if self.preprocessing_config.get("diffs_n") else 0
+ rows = self._fit_rows(history, min_rows=window + diffs_n)
+ values = rows if rows.ndim == 1 else rows[:, 0]
+ clf = MAD()
+ clf.fit(values[diffs_n:][-window:].reshape(-1, 1))
+ # predict_proba scales a raw score by the range of the training scores.
+ return self._artifact(
+ {
+ "median": float(clf.median_),
+ "median_abs_deviation": float(clf.median_diff_),
+ "min_score": float(clf.decision_scores_.min()),
+ "max_score": float(clf.decision_scores_.max()),
+ }
+ )
+
+ def _score_rows(
+ self, artifact: DetectorArtifact, rows: np.ndarray
+ ) -> tuple[list[float | None], list[dict[str, Any]]]:
+ params = artifact.params
+ values = rows if rows.ndim == 1 else rows[:, 0]
+ clf = MAD()
+ clf.median_ = params["median"]
+ clf.median_diff_ = params["median_abs_deviation"]
+ raw_scores = clf.decision_function(values.reshape(-1, 1))
+ # MinMaxScaler keeps a zero range as one, so the scores only shift.
+ score_range = params["max_score"] - params["min_score"] or 1.0
+ scores: list[float | None] = []
+ metadata: list[dict[str, Any]] = []
+ for value, raw_score in zip(values, raw_scores):
+ scores.append(float(np.clip((raw_score - params["min_score"]) / score_range, 0.0, 1.0)))
+ metadata.append(
+ {
+ "median": params["median"],
+ "median_abs_deviation": params["median_abs_deviation"],
+ "value": float(value),
+ "raw_score": float(raw_score),
+ }
+ )
+ return scores, metadata
+
@classmethod
def get_default_config(cls) -> dict:
return {
diff --git a/posthog/tasks/alerts/detectors/statistical/zscore.py b/posthog/tasks/alerts/detectors/statistical/zscore.py
index 2a5e1cfdfd04..31dd1014cec9 100644
--- a/posthog/tasks/alerts/detectors/statistical/zscore.py
+++ b/posthog/tasks/alerts/detectors/statistical/zscore.py
@@ -1,8 +1,10 @@
+from typing import Any
+
import numpy as np
from posthog.schema import DetectorType
-from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult
+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifact
from posthog.tasks.alerts.detectors.registry import register_detector
@@ -15,8 +17,10 @@ def _zscore_to_probability(z_score: float, window_zscores: np.ndarray) -> float:
training window: 0 means at or below the minimum, 1 means at or above
the maximum.
"""
- min_z = float(window_zscores.min())
- max_z = float(window_zscores.max())
+ return _normalize_zscore(z_score, float(window_zscores.min()), float(window_zscores.max()))
+
+
+def _normalize_zscore(z_score: float, min_z: float, max_z: float) -> float:
if max_z == min_z:
return 1.0 if z_score > max_z else 0.0
return float(np.clip((z_score - min_z) / (max_z - min_z), 0.0, 1.0))
@@ -149,6 +153,37 @@ class ZScoreDetector(BaseDetector):
metadata={"threshold": threshold, "window": window},
)
+ def fit(self, history: np.ndarray) -> DetectorArtifact:
+ window = self.config.get("window", 30)
+ diffs_n = 1 if self.preprocessing_config.get("diffs_n") else 0
+ rows = self._fit_rows(history, min_rows=window + diffs_n)
+ values = rows if rows.ndim == 1 else rows[:, 0]
+ window_data = values[diffs_n:][-window:]
+ mean = np.mean(window_data)
+ std = np.std(window_data)
+ params: dict[str, Any] = {"mean": float(mean), "std": float(std)}
+ if std != 0:
+ window_zscores = np.abs((window_data - mean) / std)
+ params.update(min_z=float(window_zscores.min()), max_z=float(window_zscores.max()))
+ return self._artifact(params)
+
+ def _score_rows(
+ self, artifact: DetectorArtifact, rows: np.ndarray
+ ) -> tuple[list[float | None], list[dict[str, Any]]]:
+ mean = artifact.params["mean"]
+ std = artifact.params["std"]
+ scores: list[float | None] = []
+ metadata: list[dict[str, Any]] = []
+ for value in rows if rows.ndim == 1 else rows[:, 0]:
+ if std == 0:
+ scores.append(1.0 if abs(value - mean) > 0 else 0.0)
+ metadata.append({"mean": mean, "std": 0.0, "value": float(value), "raw_zscore": None})
+ continue
+ z_score = abs((value - mean) / std)
+ scores.append(_normalize_zscore(z_score, artifact.params["min_z"], artifact.params["max_z"]))
+ metadata.append({"mean": mean, "std": std, "value": float(value), "raw_zscore": float(z_score)})
+ return scores, metadata
+
@classmethod
def get_default_config(cls) -> dict:
return {
diff --git a/posthog/tasks/alerts/detectors/test/test_detectors.py b/posthog/tasks/alerts/detectors/test/test_detectors.py
index cb3003c819c4..5e090c20abf1 100644
--- a/posthog/tasks/alerts/detectors/test/test_detectors.py
+++ b/posthog/tasks/alerts/detectors/test/test_detectors.py
@@ -820,3 +820,64 @@ class TestRealisticScoreBehavior:
f"Ensemble OR fired on stable data (score={result.score:.4f}, "
f"sub_results={result.metadata.get('sub_results')})"
)
+
+
+_PARITY_PREPROCESSING = [
+ ("raw", {}),
+ ("diffs", {"diffs_n": 1}),
+ ("lags", {"lags_n": 3}),
+ ("smooth", {"smooth_n": 3}),
+ ("diffs_lags_smooth", {"diffs_n": 1, "lags_n": 5, "smooth_n": 3}),
+]
+
+
+def _parity_config(detector_type: str, preprocessing: dict[str, Any], training_offset: int) -> dict[str, Any]:
+ shared = {"preprocessing": preprocessing, "training_offset_n": training_offset}
+ if detector_type == "ensemble":
+ return {
+ "type": "ensemble",
+ "operator": "or",
+ "detectors": [{"type": "zscore", "window": 30, **shared}, {"type": "knn", **shared}],
+ }
+ if detector_type == "threshold":
+ return {"type": "threshold", "upper_bound": 150, **shared}
+ return {"type": detector_type, "window": 30, **shared}
+
+
+class TestFitAndScore:
+ @parameterized.expand(
+ [
+ (f"{detector_type}_{name}_offset_{offset}", detector_type, preprocessing, offset)
+ for detector_type in get_available_detectors()
+ for name, preprocessing in _PARITY_PREPROCESSING
+ for offset in (1, 3)
+ ]
+ )
+ def test_scoring_a_fitted_artifact_matches_detect(
+ self, _name: str, detector_type: str, preprocessing: dict[str, Any], offset: int
+ ) -> None:
+ data = np.random.default_rng(0).normal(100, 10, size=80)
+ data[-1] = 200.0
+ detector = get_detector(_parity_config(detector_type, preprocessing, offset))
+
+ expected = detector.detect(data)
+ scored = detector.score(detector.fit(data[:-1]), data[-1 - detector.context_buckets() :])
+
+ assert len(scored.all_scores) == 1
+ assert scored.score == pytest.approx(expected.score)
+ assert scored.is_anomaly is expected.is_anomaly
+
+ def test_ensemble_scores_each_bucket_through_its_sub_artifacts(self) -> None:
+ detector = get_detector(_parity_config("ensemble", {"diffs_n": 1, "lags_n": 5, "smooth_n": 3}, 1))
+ data = np.random.default_rng(1).normal(100, 10, size=80)
+ data[-2] = 400.0
+
+ artifact = detector.fit(data[:60])
+ scored = detector.score(artifact, data[60 - detector.context_buckets() :])
+
+ assert [child.detector_type for child in artifact.children] == ["zscore", "knn"]
+ assert artifact.children[1].model_bytes
+ assert detector.context_buckets() == 8
+ assert len(scored.all_scores) == len(scored.all_metadata) == 20
+ assert 18 in scored.triggered_indices
+ assert [sub["type"] for sub in scored.all_metadata[18]["sub_results"]] == ["zscore", "knn"]
diff --git a/posthog/tasks/alerts/detectors/threshold.py b/posthog/tasks/alerts/detectors/threshold.py
index 64583390e8f5..e1f28cdad01d 100644
--- a/posthog/tasks/alerts/detectors/threshold.py
+++ b/posthog/tasks/alerts/detectors/threshold.py
@@ -1,8 +1,10 @@
+from typing import Any
+
import numpy as np
from posthog.schema import DetectorType
-from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult
+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifact
from posthog.tasks.alerts.detectors.registry import register_detector
@@ -81,6 +83,23 @@ class ThresholdDetector(BaseDetector):
metadata={"lower_bound": lower, "upper_bound": upper},
)
+ def fit(self, history: np.ndarray) -> DetectorArtifact:
+ # The bounds come from the config, so nothing is fitted.
+ return self._artifact({})
+
+ def _score_rows(
+ self, artifact: DetectorArtifact, rows: np.ndarray
+ ) -> tuple[list[float | None], list[dict[str, Any]]]:
+ lower = self.config.get("lower_bound")
+ upper = self.config.get("upper_bound")
+ scores: list[float | None] = []
+ metadata: list[dict[str, Any]] = []
+ for value in rows if rows.ndim == 1 else rows[:, 0]:
+ is_breach = (lower is not None and value < lower) or (upper is not None and value > upper)
+ scores.append(1.0 if is_breach else 0.0)
+ metadata.append({"lower_bound": lower, "upper_bound": upper, "value": float(value)})
+ return scores, metadata
+
@classmethod
def get_default_config(cls) -> dict:
return {
diff --git a/posthog/taxonomy/taxonomy.py b/posthog/taxonomy/taxonomy.py
index dbcba028435e..bdad515cd8f4 100644
--- a/posthog/taxonomy/taxonomy.py
+++ b/posthog/taxonomy/taxonomy.py
@@ -3378,10 +3378,15 @@ CORE_FILTER_DEFINITIONS_BY_GROUP: dict[str, dict[str, CoreFilterDefinition]] = {
},
"$anomaly_run": {
"label": "Anomaly scoring run",
- "description": "How the bucket was scored. `train` fetched the full history of the series, and `score` scored the bucket on stored history. Empty when scoring fetches the full history on every run.",
+ "description": "How the bucket was scored. `train` fit a new model on the history of the series, and `score` scored the bucket with a stored model. Empty when scoring fits the detector again on every run.",
"examples": ["train", "score"],
"type": "String",
},
+ "$anomaly_model_id": {
+ "label": "Anomaly model ID",
+ "description": "ID of the stored model that scored the bucket. Buckets with the same ID were scored with the same fit. Empty when scoring fits the detector again on every run.",
+ "type": "String",
+ },
"$csp_document_url": {
"label": "Document URL",
"description": "The URL of the document where the violation occurred.",
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 35729ea25b4f..3f1d570bc974 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -48,7 +48,6 @@
1 posthog/storage/auth_token_cache_verifier.py
1 posthog/storage/hypercache_verifier.py
1 posthog/tasks/alerts/detector.py
-1 posthog/tasks/alerts/detectors/base.py
1 posthog/temporal/ai/anomaly_investigation/notebook.py
1 posthog/temporal/ai/anomaly_investigation/runner.py
1 posthog/temporal/ai/anomaly_investigation/tools.py
diff --git a/products/signals/backend/anomaly_scores/artifacts.py b/products/signals/backend/anomaly_scores/artifacts.py
new file mode 100644
index 000000000000..cb9dcc4c21ae
--- /dev/null
+++ b/products/signals/backend/anomaly_scores/artifacts.py
@@ -0,0 +1,126 @@
+"""Detector artifact blobs in object storage: one blob per insight and scoring profile.
+
+A blob holds one `DetectorArtifact` per series key, as JSON. PyOD models are joblib bytes inside it,
+so a blob is read only from a key this code wrote, and only after its bytes match the SHA-256 that
+the state row stores. A key is never written twice, so a loaded blob is cached in the process by key.
+"""
+
+import json
+import uuid
+import base64
+import hashlib
+import threading
+from collections import OrderedDict
+from typing import Any
+
+from posthog.dataclasses import frozen
+from posthog.storage import object_storage
+from posthog.tasks.alerts.detectors.base import ARTIFACT_SCHEMA_VERSION, DetectorArtifact
+
+KEY_PREFIX = "anomaly_models"
+# A blob holds the artifacts of at most `MAX_LIMITS.max_series` series, so the cache stays small.
+_MAX_CACHED_BLOBS = 256
+
+
+class ArtifactUnavailableError(Exception):
+ """The blob is missing, its bytes do not match the stored hash, or a new release cannot read it. The next visit trains."""
+
+
+@frozen
+class StoredArtifacts:
+ key: str
+ sha256: str
+ # The uuid of the blob, on every `$anomaly_score` event that its artifacts scored.
+ model_id: str
+
+
+def artifact_key(team_id: int, insight_id: int, profile: str, model_id: str) -> str:
+ return f"{KEY_PREFIX}/{team_id}/{insight_id}/{profile}/{model_id}"
+
+
+def model_id_of(key: str) -> str:
+ return key.rsplit("/", 1)[-1]
+
+
+def _artifact_json(artifact: DetectorArtifact) -> dict[str, Any]:
+ return {
+ "detector_type": artifact.detector_type,
+ "schema_version": artifact.schema_version,
+ "config": artifact.config,
+ "params": artifact.params,
+ "model_bytes": None if artifact.model_bytes is None else base64.b64encode(artifact.model_bytes).decode(),
+ "children": [_artifact_json(child) for child in artifact.children],
+ }
+
+
+def _artifact_from_json(stored: dict[str, Any]) -> DetectorArtifact:
+ model_bytes = stored.get("model_bytes")
+ return DetectorArtifact(
+ detector_type=stored["detector_type"],
+ schema_version=stored["schema_version"],
+ config=stored["config"],
+ params=stored["params"],
+ model_bytes=None if model_bytes is None else base64.b64decode(model_bytes),
+ children=tuple(_artifact_from_json(child) for child in stored.get("children") or []),
+ )
+
+
+def serialize(artifacts: dict[str, DetectorArtifact]) -> bytes:
+ return json.dumps(
+ {"series": {key: _artifact_json(artifact) for key, artifact in artifacts.items()}}, sort_keys=True
+ ).encode()
+
+
+def deserialize(blob: bytes) -> dict[str, DetectorArtifact]:
+ return {key: _artifact_from_json(stored) for key, stored in json.loads(blob)["series"].items()}
+
+
+_cache: OrderedDict[str, tuple[str, dict[str, DetectorArtifact]]] = OrderedDict()
+_cache_lock = threading.Lock()
+
+
+def write_artifacts(
+ team_id: int, insight_id: int, profile: str, artifacts: dict[str, DetectorArtifact]
+) -> StoredArtifacts:
+ model_id = str(uuid.uuid4())
+ key = artifact_key(team_id, insight_id, profile, model_id)
+ blob = serialize(artifacts)
+ object_storage.write(key, blob)
+ return StoredArtifacts(key=key, sha256=hashlib.sha256(blob).hexdigest(), model_id=model_id)
+
+
+def load_artifacts(key: str, sha256: str) -> dict[str, DetectorArtifact]:
+ with _cache_lock:
+ cached = _cache.get(key)
+ if cached is not None:
+ _cache.move_to_end(key)
+ if cached is not None and cached[0] == sha256:
+ return cached[1]
+ blob = object_storage.read_bytes(key, missing_ok=True) if key else None
+ if blob is None:
+ raise ArtifactUnavailableError(f"No artifact blob at {key!r}")
+ if hashlib.sha256(blob).hexdigest() != sha256:
+ raise ArtifactUnavailableError(f"The artifact blob at {key!r} does not match its hash")
+ artifacts = deserialize(blob)
+ if not all(_readable(artifact) for artifact in artifacts.values()):
+ raise ArtifactUnavailableError(f"The artifact blob at {key!r} has an older schema version")
+ with _cache_lock:
+ _cache[key] = (sha256, artifacts)
+ while len(_cache) > _MAX_CACHED_BLOBS:
+ _cache.popitem(last=False)
+ return artifacts
+
+
+def _readable(artifact: DetectorArtifact) -> bool:
+ return artifact.schema_version == ARTIFACT_SCHEMA_VERSION and all(_readable(child) for child in artifact.children)
+
+
+def delete_artifacts(key: str) -> None:
+ with _cache_lock:
+ _cache.pop(key, None)
+ object_storage.delete(key)
+
+
+def clear_artifact_cache() -> None:
+ with _cache_lock:
+ _cache.clear()
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.py
index dbecc1450d98..c0e2a74a1622 100644
--- a/products/signals/backend/anomaly_scores/config.py
+++ b/products/signals/backend/anomaly_scores/config.py
@@ -84,18 +84,15 @@ def resolve_profile(team: "Team", insight: "Insight") -> ScoringProfile:
class ScoringRun(StrEnum):
"""The kind of visit when the train and score split is on."""
- # Fetch the full detector history, score the new buckets, and store the series tail.
+ # Fetch the training history, fit and store one artifact per series, and score the new buckets with them.
TRAIN = "train"
- # Fetch only the refresh and new buckets, merge them into the stored tail, and score the new buckets.
+ # Fetch only the new buckets and the context that preprocessing reads, and score them with the stored artifacts.
SCORE = "score"
# Retrain hourly insights once a day, daily once a week, weekly once a month, and monthly once a quarter.
DEFAULT_TRAIN_EVERY_BUCKETS = {IntervalType.HOUR: 24, IntervalType.DAY: 7, IntervalType.WEEK: 4, IntervalType.MONTH: 3}
MAX_TRAIN_EVERY_BUCKETS = {IntervalType.HOUR: 168, IntervalType.DAY: 30, IntervalType.WEEK: 13, IntervalType.MONTH: 12}
-# Stored buckets that a score step fetches again, so late events reach the stored values.
-DEFAULT_REFRESH_BUCKETS = {IntervalType.HOUR: 3, IntervalType.DAY: 3, IntervalType.WEEK: 2, IntervalType.MONTH: 1}
-MAX_REFRESH_BUCKETS = {IntervalType.HOUR: 24, IntervalType.DAY: 14, IntervalType.WEEK: 4, IntervalType.MONTH: 3}
# Hourly insights are left out by default: with many eligible insights, the time between two visits of
@@ -123,7 +120,6 @@ class AnomalyScoringConfig:
train_every_buckets: dict[IntervalType, int] = dataclasses.field(
default_factory=lambda: dict(DEFAULT_TRAIN_EVERY_BUCKETS)
)
- refresh_buckets: dict[IntervalType, int] = dataclasses.field(default_factory=lambda: dict(DEFAULT_REFRESH_BUCKETS))
# Score step queries per clock hour. Train steps use the budgets above, so a cheap score step never waits for them.
max_score_queries_per_hour: int = 600
max_score_queries_per_project_per_hour: int = 150
@@ -226,9 +222,6 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:
train_every_buckets=_buckets_per_interval(
payload.get("train_every_buckets"), DEFAULT_TRAIN_EVERY_BUCKETS, MAX_TRAIN_EVERY_BUCKETS
),
- refresh_buckets=_buckets_per_interval(
- payload.get("refresh_buckets"), DEFAULT_REFRESH_BUCKETS, MAX_REFRESH_BUCKETS
- ),
max_score_queries_per_hour=min(
_positive_int(payload.get("max_score_queries_per_hour"), defaults.max_score_queries_per_hour),
MAX_SCORE_QUERIES_PER_HOUR,
diff --git a/products/signals/backend/anomaly_scores/emission.py b/products/signals/backend/anomaly_scores/emission.py
index 57b1ba67853d..dc534d22b66f 100644
--- a/products/signals/backend/anomaly_scores/emission.py
+++ b/products/signals/backend/anomaly_scores/emission.py
@@ -58,6 +58,7 @@ def _event(
"$anomaly_scored_at": scored_at.astimezone(datetime.UTC).isoformat(),
"$anomaly_details": json.dumps(point.details, sort_keys=True),
**({"$anomaly_run": scores.run.value} if scores.run else {}),
+ **({"$anomaly_model_id": scores.model_id} if scores.model_id else {}),
},
}
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index 21121ed3a2d4..5b6fafae5983 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -105,8 +105,6 @@ class InsightPick:
query_cost: int = 1
# None when the train and score split is off.
run: ScoringRun | None = None
- # Stored buckets that a score step fetches again.
- refresh_buckets: int = 0
@frozen
@@ -441,7 +439,6 @@ def select_insights(
interval=due_by_id[insight_id].interval,
query_cost=plan.queries,
run=plan.run,
- refresh_buckets=plan.refresh_buckets,
)
for insight_id, plan in picked.items()
]
diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.py
index 15b6f8053bad..3b1bc560192c 100644
--- a/products/signals/backend/anomaly_scores/scoring.py
+++ b/products/signals/backend/anomaly_scores/scoring.py
@@ -25,12 +25,13 @@ from posthog.hogql_queries.validation.rules import expanded_series_count
from posthog.interval_specs import interval_spec
from posthog.models import Team
from posthog.schema_migrations.upgrade_manager import upgrade_insight
-from posthog.tasks.alerts.detectors.base import DetectionResult
+from posthog.tasks.alerts.detectors.base import BaseDetector, DetectionResult, DetectorArtifact, FitError
from posthog.tasks.alerts.detectors.registry import get_detector
from posthog.tasks.alerts.utils import WRAPPER_NODE_KINDS, is_non_time_series_trend
from products.product_analytics.backend.facade.models import Insight
from products.product_analytics.backend.facade.queries import TrendsQueryRunner
+from products.signals.backend.anomaly_scores.artifacts import model_id_of
from products.signals.backend.anomaly_scores.config import (
DEFAULT_LIMITS,
IntervalScoring,
@@ -78,15 +79,6 @@ class ScoredSeries:
points: list[ScoredPoint]
-@frozen
-class SeriesWindow:
- """The newest completed values of one series, oldest first. A score step reads them as the history of its new buckets."""
-
- # Aware bucket start of the first value. Trends zero-fills buckets, so each next value is one interval later.
- first_bucket: datetime
- values: list[float]
-
-
@frozen
class StateIdentity:
"""What a stored state was built from. A score step needs the same identity, or the visit trains."""
@@ -102,17 +94,22 @@ class TrainedState:
trained_at: datetime
# The newest scored bucket start over all series, or None when no series had a completed bucket.
scored_through: datetime | None
- windows: dict[str, SeriesWindow]
+ # The series that the stored artifact blob covers.
+ series_keys: tuple[str, ...]
+ # Empty until the blob of a train step is written.
+ artifact_key: str = ""
+ artifact_sha256: str = ""
# Stored series that the last score step did not get back.
missing_series: int = 0
@frozen
class SplitRun:
- """A visit of the train and score split. A score step needs ``state``; with no matching state, the visit trains."""
+ """A visit of the train and score split. A score step needs ``state`` and its ``artifacts``, or the visit trains."""
- refresh_buckets: int
state: TrainedState | None = None
+ # Series key to the artifact of the stored blob of ``state``.
+ artifacts: dict[str, DetectorArtifact] | None = None
@frozen
@@ -126,8 +123,12 @@ class InsightScores:
run: ScoringRun | None = None
# The state for the next visit to store.
state: TrainedState | None = None
- # Series of a score step with no stored window. A later train step scores them.
+ # The artifacts that a train step fit, for the caller to store. None on a score step.
+ artifacts: dict[str, DetectorArtifact] | None = None
+ # Series of a score step with no stored artifact. A later train step fits them.
unmatched_series: int = 0
+ # The artifact blob that scored the buckets. None when the split is off.
+ model_id: str | None = None
@frozen
@@ -144,9 +145,15 @@ def _short_hash(payload: Any) -> str:
return hashlib.sha256(canonical.encode()).hexdigest()[:16]
-def detector_version(scoring: IntervalScoring) -> str:
- """Identifies the scoring behavior, so scores from different settings are never mixed in one series."""
- return _short_hash(dataclasses.asdict(scoring))
+def detector_version(scoring: IntervalScoring, *, artifacts: bool = False) -> str:
+ """Identifies the scoring behavior, so scores from different settings are never mixed in one series.
+
+ Scores from stored artifacts are fit once per train step, so they get their own version.
+ """
+ behavior: dict[str, Any] = {"min_points": scoring.min_points, "detector": scoring.detector}
+ if artifacts:
+ behavior["fit"] = "artifact"
+ return _short_hash(behavior)
def query_hash(query: TrendsQuery, timezone: ZoneInfo, limits: ScoringLimits = DEFAULT_LIMITS) -> str:
@@ -154,7 +161,7 @@ def query_hash(query: TrendsQuery, timezone: ZoneInfo, limits: ScoringLimits = D
dumped = build_scoring_query(query, buckets=1, limits=limits).model_dump(mode="json", exclude_none=True)
# The fetch size changes with every visit. `daysOfWeek` stays, because it changes the buckets.
(dumped.get("dateRange") or {}).pop("date_from", None)
- # Stored bucket starts step in wall-clock time, so a timezone change needs new windows.
+ # Bucket starts step in wall-clock time, so a timezone change needs new artifacts.
return _short_hash({"query": dumped, "timezone": timezone.key})
@@ -236,7 +243,7 @@ def fetch_buckets(query: TrendsQuery, scoring: IntervalScoring, to_score: int, *
"""Completed buckets the scoring query fetches: the history of the oldest bucket to score, then the rest.
The count has one bucket more than the scores read, so a first bucket that the date range cuts short is never read.
- ``history`` replaces the detector history, for a score step that reads its history from the stored state.
+ ``history`` replaces the detector history, for the train and score steps of the split.
"""
count = (history_buckets(scoring) if history is None else history) + to_score
smoothing = query.trendsFilter.smoothingIntervals if query.trendsFilter else None
@@ -291,7 +298,7 @@ def scoring_query_settings(queries: int) -> HogQLGlobalSettings:
def supports_score_step(query: TrendsQuery) -> bool:
- """A score step steps over stored buckets one interval at a time, so the buckets must have no gaps."""
+ """A score step reads the buckets before a scored bucket as its context, so the buckets must have no gaps."""
# Trends drops the buckets of the days that the filter leaves out.
return not (query.dateRange and query.dateRange.daysOfWeek)
@@ -299,7 +306,7 @@ def supports_score_step(query: TrendsQuery) -> bool:
def state_identity(query: TrendsQuery, scoring: IntervalScoring, team: Team, limits: ScoringLimits) -> StateIdentity:
return StateIdentity(
interval=scoring_interval(query),
- detector_version=detector_version(scoring),
+ detector_version=detector_version(scoring, artifacts=True),
query_hash=query_hash(query, team.timezone_info, limits),
)
@@ -360,65 +367,63 @@ def score_series(
return points
-def window_buckets(window: SeriesWindow, interval: IntervalType) -> list[datetime]:
- buckets: list[datetime] = []
- start = window.first_bucket
- for _ in window.values:
- buckets.append(start)
- start = bucket_end(start, interval)
- return buckets
+@frozen
+class ScoredIndices:
+ completed: list[int]
+ # The newest completed buckets later than `after`, capped at `max_backfill_buckets`.
+ returned: list[int]
-def train_window(
- values: Sequence[float], buckets: Sequence[datetime], *, history: int, interval: IntervalType, now: datetime
-) -> SeriesWindow | None:
- """The newest ``history`` completed values of a fetched series, or None when they do not step one interval apart."""
- completed = [i for i, start in enumerate(buckets) if bucket_end(start, interval) <= now][-history:]
- if not completed:
- return None
- window = SeriesWindow(first_bucket=buckets[completed[0]], values=[float(values[i]) for i in completed])
- # A repeated hour at a DST fall-back breaks the steps. The series waits for a train step after the change.
- if window_buckets(window, interval) != [buckets[i] for i in completed]:
- return None
- return window
+def _scored_indices(
+ buckets: Sequence[datetime], *, interval: IntervalType, now: datetime, after: datetime | None, limits: ScoringLimits
+) -> ScoredIndices:
+ """The indices of the completed buckets, and of the newest of them later than ``after``."""
+ completed = [i for i, start in enumerate(buckets) if bucket_end(start, interval) <= now]
+ returned = [i for i in completed if after is None or buckets[i] > after][-limits.max_backfill_buckets :]
+ return ScoredIndices(completed=completed, returned=returned)
-def score_window(
- window: SeriesWindow,
+def score_with_artifact(
+ detector: BaseDetector,
+ artifact: DetectorArtifact | None,
values: Sequence[float],
buckets: Sequence[datetime],
- *,
- refresh: int,
- scoring: IntervalScoring,
- interval: IntervalType,
- now: datetime,
- limits: ScoringLimits = DEFAULT_LIMITS,
-) -> tuple[list[ScoredPoint], SeriesWindow] | None:
- """Merge freshly fetched buckets into a stored window, score the buckets after it, and trim it to its history.
+ indices: list[int],
+) -> list[ScoredPoint]:
+ """Score the buckets at ``indices`` with a fitted artifact. A bucket without enough context before it gets no score."""
+ context = detector.context_buckets()
+ scored = [i for i in indices if i >= context]
+ result = None
+ if artifact is not None and scored:
+ result = detector.score(artifact, np.array(values[scored[0] - context : scored[-1] + 1], dtype=float))
+ positions = {i: position for position, i in enumerate(scored)}
+ points: list[ScoredPoint] = []
+ for i in indices:
+ position = positions.get(i)
+ score = flag = None
+ details: dict[str, Any] = {}
+ if result is not None and position is not None and result.all_scores[position] is not None:
+ score = float(cast(float, result.all_scores[position]))
+ flag = position in result.triggered_indices
+ details = _details_of(result.all_metadata[position])
+ points.append(
+ ScoredPoint(bucket_start=buckets[i], value=float(values[i]), score=score, flag=flag, details=details)
+ )
+ return points
- A fresh value replaces the stored value of its bucket for the newest ``refresh`` stored buckets, so late events
- reach the window. Older fresh buckets can be cut short or smoothed over too few buckets, so they are not read.
- Returns None when the fresh buckets leave a gap after the window. The series then keeps its window.
- """
- stored = window_buckets(window, interval)
- merged = dict(zip(stored, window.values))
- refresh_from = [*stored, bucket_end(stored[-1], interval)][max(0, len(stored) - refresh)]
- fresh = [(start, value) for start, value in zip(buckets, values) if bucket_end(start, interval) <= now]
- for start, value in fresh:
- if start >= refresh_from:
- merged[start] = float(value)
- starts = list(stored)
- next_start = bucket_end(stored[-1], interval)
- while next_start in merged:
- starts.append(next_start)
- next_start = bucket_end(next_start, interval)
- if any(start >= next_start for start, _ in fresh):
- return None
- data = [merged[start] for start in starts]
- points = score_series(data, starts, scoring=scoring, interval=interval, now=now, after=stored[-1], limits=limits)
- keep = history_buckets(scoring)
- return points, SeriesWindow(first_bucket=starts[-keep:][0], values=data[-keep:])
+def _fit(
+ detector: BaseDetector, scoring: IntervalScoring, values: Sequence[float], end: int
+) -> DetectorArtifact | None:
+ """Fit on the buckets before ``end`` that ``detect()`` reads to score ``end``, or None when they are too few."""
+ history = np.array(values[max(0, end - (history_buckets(scoring) - 1)) : end], dtype=float)
+ # The scored bucket counts toward `min_points`, as in `score_series`.
+ if len(history) + 1 < scoring.min_points:
+ return None
+ try:
+ return detector.fit(history)
+ except FitError:
+ return None
def score_insight(
@@ -432,121 +437,162 @@ def score_insight(
) -> InsightScores:
"""Score the new buckets of an insight.
- With ``split``, a stored state with the insight's identity makes this a score step, and anything else a
- train step. Both return the state for the next visit.
+ With ``split``, a stored state with the insight's identity and its artifacts makes this a score step, and
+ anything else a train step. Both return the state for the next visit, and a train step returns the
+ artifacts to store.
"""
profile = resolve_profile(team, insight)
query = trends_query_for(insight, profile)
interval = scoring_interval(query)
scoring = interval_scoring(profile, interval)
timezone = team.timezone_info
- identity = state_identity(query, scoring, team, limits) if split is not None else None
- state = split.state if split is not None else None
- if state is not None and (state.identity != identity or not supports_score_step(query)):
- state = None
scores = InsightScores(
interval=interval,
profile=profile.name,
detector=str(scoring.detector["type"]),
- detector_version=detector_version(scoring),
+ detector_version=detector_version(scoring, artifacts=split is not None),
series=[],
)
- if split is not None and state is not None and identity is not None:
- to_score = buckets_to_score(interval, now=now, after=state.scored_through, timezone=timezone, limits=limits)
- # One bucket more than the score step reads, as in the full fetch.
- buckets = fetch_buckets(query, scoring, to_score, history=split.refresh_buckets + 1)
- fetched = _fetch(insight, team, query, buckets, limits, ScoringRun.SCORE)
+ if split is None:
+ to_score = buckets_to_score(interval, now=now, after=after, timezone=timezone, limits=limits)
+ fetched = _fetch(insight, team, query, fetch_buckets(query, scoring, to_score), limits, None)
+ series = [
+ ScoredSeries(
+ series_key=line.series_key,
+ label=line.label,
+ breakdown_value=line.breakdown_value,
+ points=score_series(
+ line.values, line.buckets, scoring=scoring, interval=interval, now=now, after=after, limits=limits
+ ),
+ )
+ for line in fetched
+ ]
+ return dataclasses.replace(scores, series=series)
+
+ identity = state_identity(query, scoring, team, limits)
+ state = split.state
+ detector = get_detector(scoring.detector)
+ if (
+ state is not None
+ and split.artifacts is not None
+ and state.identity == identity
+ and state.scored_through is not None
+ and supports_score_step(query)
+ ):
return _score_step(
+ insight,
+ team,
+ query,
scores,
- fetched,
state,
- refresh=split.refresh_buckets,
+ split.artifacts,
+ detector,
scoring=scoring,
- interval=interval,
now=now,
+ after=after,
limits=limits,
)
+ return _train_step(
+ insight, team, query, scores, identity, detector, scoring=scoring, now=now, after=after, limits=limits
+ )
- to_score = buckets_to_score(interval, now=now, after=after, timezone=timezone, limits=limits)
- run = ScoringRun.TRAIN if split is not None else None
- fetched = _fetch(insight, team, query, fetch_buckets(query, scoring, to_score), limits, run)
- series = [
- ScoredSeries(
- series_key=line.series_key,
- label=line.label,
- breakdown_value=line.breakdown_value,
- points=score_series(
- line.values,
- line.buckets,
- scoring=scoring,
- interval=interval,
- now=now,
- after=after,
- limits=limits,
- ),
- )
- for line in fetched
- ]
- if identity is None:
- return dataclasses.replace(scores, series=series)
- windows = {
- line.series_key: window
- for line in fetched
- if (
- window := train_window(
- line.values, line.buckets, history=history_buckets(scoring), interval=interval, now=now
+
+def _train_step(
+ insight: Insight,
+ team: Team,
+ query: TrendsQuery,
+ scores: InsightScores,
+ identity: StateIdentity,
+ detector: BaseDetector,
+ *,
+ scoring: IntervalScoring,
+ now: datetime,
+ after: datetime | None,
+ limits: ScoringLimits,
+) -> InsightScores:
+ """Fit each series on the history before its due buckets, and score the due buckets with the new artifact."""
+ interval = scores.interval
+ to_score = buckets_to_score(interval, now=now, after=after, timezone=team.timezone_info, limits=limits)
+ buckets = fetch_buckets(query, scoring, to_score)
+ fetched = _fetch(insight, team, query, buckets, limits, ScoringRun.TRAIN)
+ series: list[ScoredSeries] = []
+ artifacts: dict[str, DetectorArtifact] = {}
+ newest: list[datetime] = []
+ for line in fetched:
+ indices = _scored_indices(line.buckets, interval=interval, now=now, after=after, limits=limits)
+ completed, returned = indices.completed, indices.returned
+ if completed:
+ newest.append(line.buckets[completed[-1]])
+ # With no due bucket, the fit reads the newest completed buckets.
+ end = returned[0] if returned else (completed[-1] + 1 if completed else 0)
+ artifact = _fit(detector, scoring, line.values, end)
+ if artifact is not None:
+ artifacts[line.series_key] = artifact
+ series.append(
+ ScoredSeries(
+ series_key=line.series_key,
+ label=line.label,
+ breakdown_value=line.breakdown_value,
+ points=score_with_artifact(detector, artifact, line.values, line.buckets, returned),
)
)
- is not None
- }
return dataclasses.replace(
scores,
series=series,
run=ScoringRun.TRAIN,
+ artifacts=artifacts,
state=TrainedState(
- identity=identity, trained_at=now, scored_through=_scored_through(windows, interval), windows=windows
+ identity=identity,
+ trained_at=now,
+ scored_through=max(newest).astimezone(UTC) if newest else None,
+ series_keys=tuple(artifacts),
),
)
def _score_step(
+ insight: Insight,
+ team: Team,
+ query: TrendsQuery,
scores: InsightScores,
- fetched: list[_FetchedSeries],
state: TrainedState,
+ artifacts: dict[str, DetectorArtifact],
+ detector: BaseDetector,
*,
- refresh: int,
scoring: IntervalScoring,
- interval: IntervalType,
now: datetime,
+ after: datetime | None,
limits: ScoringLimits,
) -> InsightScores:
+ """Score the buckets after the older of ``after`` and ``state.scored_through`` with the stored artifacts.
+
+ ``after`` is the oldest newest-scored bucket over the series. A series that missed a visit, for example a
+ breakdown value out of the top N, then still gets its missed buckets, and the events drop each series' buckets
+ up to its own newest scored bucket. Nothing is fitted.
+ """
+ interval = scores.interval
+ since = min((through for through in (after, state.scored_through) if through is not None), default=None)
+ to_score = buckets_to_score(interval, now=now, after=since, timezone=team.timezone_info, limits=limits)
+ # One bucket more than the context that preprocessing reads, as in the full fetch.
+ buckets = fetch_buckets(query, scoring, to_score, history=detector.context_buckets() + 1)
+ fetched = _fetch(insight, team, query, buckets, limits, ScoringRun.SCORE)
series: list[ScoredSeries] = []
- windows = dict(state.windows)
unmatched = 0
- returned: set[str] = set()
+ returned_keys: set[str] = set()
+ scored_through = state.scored_through
for line in fetched:
- returned.add(line.series_key)
- window = state.windows.get(line.series_key)
+ returned_keys.add(line.series_key)
+ artifact = artifacts.get(line.series_key)
# The short range can rank a different top N of breakdown values. A new value waits for a train step.
- scored = (
- None
- if window is None
- else score_window(
- window,
- line.values,
- line.buckets,
- refresh=refresh,
- scoring=scoring,
- interval=interval,
- now=now,
- limits=limits,
- )
- )
- if scored is None:
+ if artifact is None:
unmatched += 1
continue
- points, windows[line.series_key] = scored
+ returned = _scored_indices(line.buckets, interval=interval, now=now, after=since, limits=limits).returned
+ points = score_with_artifact(detector, artifact, line.values, line.buckets, returned)
+ if points:
+ newest = points[-1].bucket_start.astimezone(UTC)
+ scored_through = newest if scored_through is None else max(scored_through, newest)
series.append(
ScoredSeries(
series_key=line.series_key, label=line.label, breakdown_value=line.breakdown_value, points=points
@@ -557,21 +603,13 @@ def _score_step(
series=series,
run=ScoringRun.SCORE,
unmatched_series=unmatched,
- # A stored series that is missing from the result keeps its window.
state=dataclasses.replace(
- state,
- scored_through=_scored_through(windows, interval),
- windows=windows,
- missing_series=len(set(state.windows) - returned),
+ state, scored_through=scored_through, missing_series=len(set(state.series_keys) - returned_keys)
),
+ model_id=model_id_of(state.artifact_key),
)
-def _scored_through(windows: dict[str, SeriesWindow], interval: IntervalType) -> datetime | None:
- ends = [window_buckets(window, interval)[-1] for window in windows.values() if window.values]
- return max(ends).astimezone(UTC) if ends else None
-
-
def _fetch(
insight: Insight,
team: Team,
@@ -684,17 +722,19 @@ def bucket_end(start: datetime, interval: IntervalType) -> datetime:
def _details(result: DetectionResult) -> dict[str, Any]:
+ return _details_of(result.metadata)
+
+
+def _details_of(metadata: dict[str, Any]) -> dict[str, Any]:
"""Detector-specific values for one bucket. Only scalars are kept, so the event property stays small."""
- sub_results = result.metadata.get("sub_results")
+ sub_results = metadata.get("sub_results")
if isinstance(sub_results, list):
return {
"sub_scores": {
f"{sub.get('type')}_{position}": sub.get("score") for position, sub in enumerate(sub_results)
}
}
- return {
- key: value for key, value in result.metadata.items() if value is None or isinstance(value, int | float | str)
- }
+ return {key: value for key, value in metadata.items() if value is None or isinstance(value, int | float | str)}
def _parse_bucket(day: str, timezone: ZoneInfo) -> datetime:
diff --git a/products/signals/backend/anomaly_scores/state.py b/products/signals/backend/anomaly_scores/state.py
index 48c870bc3d33..c46f117169f7 100644
--- a/products/signals/backend/anomaly_scores/state.py
+++ b/products/signals/backend/anomaly_scores/state.py
@@ -1,16 +1,17 @@
"""The stored state of the train and score split, and the choice between a train step and a score step.
-A train step fetches the full detector history and stores the newest values of each series. A score
-step fetches only the newest buckets and scores them on the stored values. See `AnomalyScoringState`.
+A train step fetches the training history, fits one detector artifact per series, and stores them as
+one blob. A score step fetches only the newest buckets and scores them with the stored artifacts. See
+`AnomalyScoringState` and `artifacts.py`.
"""
-import math
import datetime
import dataclasses
from collections.abc import Collection
-from typing import Any
from zoneinfo import ZoneInfo
+import structlog
+
from posthog.schema import IntervalType
from posthog.dataclasses import frozen
@@ -18,6 +19,7 @@ from posthog.interval_specs import interval_spec
from posthog.models import Team
from products.product_analytics.backend.facade.models import Insight
+from products.signals.backend.anomaly_scores.artifacts import delete_artifacts
from products.signals.backend.anomaly_scores.config import (
DEFAULT_LIMITS,
AnomalyScoringConfig,
@@ -25,7 +27,6 @@ from products.signals.backend.anomaly_scores.config import (
resolve_profile,
)
from products.signals.backend.anomaly_scores.scoring import (
- SeriesWindow,
StateIdentity,
TrainedState,
buckets_to_score,
@@ -38,6 +39,8 @@ from products.signals.backend.anomaly_scores.scoring import (
)
from products.signals.backend.models import AnomalyScoringState
+logger = structlog.get_logger(__name__)
+
@frozen
class RunPlan:
@@ -46,7 +49,6 @@ class RunPlan:
queries: int
# None when the split is off.
run: ScoringRun | None = None
- refresh_buckets: int = 0
def needs_training(
@@ -58,7 +60,13 @@ def needs_training(
train_every: int,
max_backfill: int,
) -> bool:
- if state is None or state.identity != identity or not state.windows or state.scored_through is None:
+ if (
+ state is None
+ or state.identity != identity
+ or not state.series_keys
+ or not state.artifact_key
+ or state.scored_through is None
+ ):
return True
interval = identity.interval
if state.trained_at + interval_spec(interval).period * train_every <= now:
@@ -73,7 +81,7 @@ def needs_training(
)
if due > max_backfill:
return True
- return state.missing_series * 2 > len(state.windows)
+ return state.missing_series * 2 > len(state.series_keys)
def plan_run(
@@ -91,7 +99,6 @@ def plan_run(
except Exception:
# The scorer fails on the same query and reports the error. A train step charges the larger budget.
return RunPlan(queries=queries, run=ScoringRun.TRAIN)
- refresh = config.refresh_buckets.get(interval, 0)
trains = not supports_score_step(query) or needs_training(
state,
identity=identity,
@@ -100,7 +107,7 @@ def plan_run(
train_every=config.train_every_buckets[interval],
max_backfill=config.limits.max_backfill_buckets,
)
- return RunPlan(queries=queries, run=ScoringRun.TRAIN if trains else ScoringRun.SCORE, refresh_buckets=refresh)
+ return RunPlan(queries=queries, run=ScoringRun.TRAIN if trains else ScoringRun.SCORE)
def load_states(team: Team, insights: Collection[Insight]) -> dict[int, TrainedState]:
@@ -109,11 +116,7 @@ def load_states(team: Team, insights: Collection[Insight]) -> dict[int, TrainedS
if not profiles:
return {}
rows = AnomalyScoringState.objects.for_team(team.id).filter(insight_id__in=list(profiles))
- return {
- row.insight_id: _from_row(row, team.timezone_info)
- for row in rows
- if profiles.get(row.insight_id) == row.profile
- }
+ return {row.insight_id: _from_row(row) for row in rows if profiles.get(row.insight_id) == row.profile}
def load_state(team: Team, insight: Insight) -> TrainedState | None:
@@ -121,6 +124,9 @@ def load_state(team: Team, insight: Insight) -> TrainedState | None:
def save_state(team: Team, insight_id: int, profile: str, state: TrainedState) -> None:
+ """Write the state row. A new artifact blob replaces the previous one, which is deleted after the write."""
+ scope = AnomalyScoringState.objects.for_team(team.id)
+ previous_key = scope.filter(insight_id=insight_id, profile=profile).values_list("artifact_key", flat=True).first()
fields = {
"interval": state.identity.interval.value,
"detector_version": state.identity.detector_version,
@@ -128,41 +134,37 @@ def save_state(team: Team, insight_id: int, profile: str, state: TrainedState) -
"trained_at": state.trained_at,
"scored_through": state.scored_through,
"missing_series": state.missing_series,
- "series": {key: _window_json(window) for key, window in state.windows.items()},
+ "artifact_key": state.artifact_key,
+ "artifact_sha256": state.artifact_sha256,
+ "series_keys": list(state.series_keys),
}
# The scope filters on the project's canonical team, and save() writes that team for a new row.
- AnomalyScoringState.objects.for_team(team.id).update_or_create(
+ scope.update_or_create(
insight_id=insight_id, profile=profile, defaults=fields, create_defaults={**fields, "team_id": team.id}
)
+ if previous_key and previous_key != state.artifact_key:
+ try:
+ delete_artifacts(previous_key)
+ except Exception:
+ # The row no longer names the blob, so a failed delete leaves an unread blob and nothing else.
+ logger.warning("anomaly_scoring_artifact_delete_failed", insight_id=insight_id, exc_info=True)
-def _window_json(window: SeriesWindow) -> dict[str, Any]:
- return {
- "first_bucket": window.first_bucket.isoformat(),
- # JSON has no NaN.
- "values": [value if math.isfinite(value) else None for value in window.values],
- }
+def invalidate_state(team: Team, insight_id: int, profile: str) -> None:
+ """Make the next visit of the insight train. The row keeps its blob key, so that train step deletes the blob."""
+ AnomalyScoringState.objects.for_team(team.id).filter(insight_id=insight_id, profile=profile).update(series_keys=[])
-def _from_row(row: AnomalyScoringState, timezone: datetime.tzinfo) -> TrainedState:
- windows: dict[str, SeriesWindow] = {}
- for key, stored in (row.series or {}).items():
- try:
- window = SeriesWindow(
- # Bucket starts step in wall-clock time of the project timezone.
- first_bucket=datetime.datetime.fromisoformat(stored["first_bucket"]).astimezone(timezone),
- values=[math.nan if value is None else float(value) for value in stored["values"]],
- )
- except (KeyError, TypeError, ValueError):
- continue
- if window.values:
- windows[key] = window
+def _from_row(row: AnomalyScoringState) -> TrainedState:
+ series_keys = row.series_keys if isinstance(row.series_keys, list) else []
return TrainedState(
identity=StateIdentity(
interval=IntervalType(row.interval), detector_version=row.detector_version, query_hash=row.query_hash
),
trained_at=row.trained_at,
scored_through=row.scored_through,
- windows=windows,
+ series_keys=tuple(str(key) for key in series_keys),
+ artifact_key=row.artifact_key,
+ artifact_sha256=row.artifact_sha256,
missing_series=row.missing_series,
)
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index 104194333bf0..62079d017cb2 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -7,6 +7,7 @@ tick off. Each insight activity scores one insight and writes its new buckets as
import time
import asyncio
import datetime
+import dataclasses
from collections import Counter
from django.conf import settings
@@ -34,6 +35,7 @@ from posthog.temporal.common.scoped import scoped_temporal
from posthog.temporal.common.utils import close_db_connections
from products.product_analytics.backend.facade.api import insights_including_soft_deleted_for_team
+from products.signals.backend.anomaly_scores.artifacts import ArtifactUnavailableError, load_artifacts, write_artifacts
from products.signals.backend.anomaly_scores.config import (
ANOMALY_SCORING_FLAG_DISTINCT_ID,
DEFAULT_LIMITS,
@@ -59,7 +61,7 @@ from products.signals.backend.anomaly_scores.scoring import (
score_insight,
)
from products.signals.backend.anomaly_scores.selection import parse_selection_config
-from products.signals.backend.anomaly_scores.state import load_state, save_state
+from products.signals.backend.anomaly_scores.state import invalidate_state, load_state, save_state
logger = structlog.get_logger(__name__)
@@ -78,6 +80,8 @@ STATUS_ACCESS_DENIED = "access_denied"
STATUS_BROKEN_QUERY = "broken_query"
# ClickHouse rejected the query because it is full. The tick starts no more picks.
STATUS_AT_CAPACITY = "at_capacity"
+# The stored artifact blob is missing or does not match its hash. The next visit trains.
+STATUS_ARTIFACT_UNAVAILABLE = "artifact_unavailable"
# Any other failure. The activity raises it, and the workflow counts it as `errored`.
STATUS_ERRORED = "errored"
@@ -122,7 +126,7 @@ class ScoreInsightResult:
series: int = 0
# What the visit ran. A score pick with no matching state trains. None when the split is off.
run: ScoringRun | None = None
- # Series of a score step with no stored window, and stored series missing from its result.
+ # Series of a score step with no stored artifact, and stored series missing from its result.
unmatched_series: int = 0
missing_series: int = 0
@@ -249,12 +253,23 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
team = Team.objects.get(id=pick.team_id)
# Set before scoring, so a failure also waits for the cooldown instead of taking a slot on every tick.
mark_attempted(pick.insight_id, ATTEMPT_COOLDOWN)
+ profile = resolve_profile(team, insight).name
split = None
if pick.run is not None:
- split = SplitRun(
- refresh_buckets=pick.refresh_buckets,
- state=load_state(team, insight) if pick.run == ScoringRun.SCORE else None,
- )
+ split = SplitRun()
+ state = load_state(team, insight) if pick.run == ScoringRun.SCORE else None
+ if state is not None:
+ try:
+ split = SplitRun(state=state, artifacts=load_artifacts(state.artifact_key, state.artifact_sha256))
+ except ArtifactUnavailableError as error:
+ logger.warning(
+ "anomaly_scoring_artifact_unavailable",
+ team_id=team.id,
+ insight_id=insight.id,
+ error=str(error)[:200],
+ )
+ invalidate_state(team, insight.id, profile)
+ return ScoreInsightResult(status=STATUS_ARTIFACT_UNAVAILABLE, events=0, run=ScoringRun.SCORE)
scores = score_insight(
insight,
team,
@@ -264,6 +279,14 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
limits=input.limits,
split=split,
)
+ if scores.artifacts and scores.state is not None:
+ # The blob goes first, so every event of this visit names a model that exists.
+ stored = write_artifacts(team.id, insight.id, profile, scores.artifacts)
+ scores = dataclasses.replace(
+ scores,
+ model_id=stored.model_id,
+ state=dataclasses.replace(scores.state, artifact_key=stored.key, artifact_sha256=stored.sha256),
+ )
events = score_events(
team_id=team.id, insight_id=insight.id, scores=scores, scored_through=pick.scored_through, scored_at=now
)
@@ -271,7 +294,7 @@ def _score_pick(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
# After the emit: a lost state write only scores buckets again, and their events collapse by uuid.
# A state write before a failed emit would skip those buckets for good.
if scores.state is not None:
- save_state(team, insight.id, resolve_profile(team, insight).name, scores.state)
+ save_state(team, insight.id, profile, scores.state)
return ScoreInsightResult(
status=STATUS_SCORED if events else STATUS_UP_TO_DATE,
events=len(events),
diff --git a/products/signals/backend/migrations/0200_anomalyscoringstate_untrack_series.py b/products/signals/backend/migrations/0200_anomalyscoringstate_untrack_series.py
new file mode 100644
index 000000000000..e6d87422c5dc
--- /dev/null
+++ b/products/signals/backend/migrations/0200_anomalyscoringstate_untrack_series.py
@@ -0,0 +1,23 @@
+from django.db import migrations
+
+from posthog.migration_helpers import untrack_field
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("signals", "0199_anomalyscoringstate"),
+ ]
+
+ operations = [
+ # The column stays until a later drop. New rows do not name it, so it needs a database default.
+ untrack_field(
+ "anomalyscoringstate",
+ "series",
+ database_operations=[
+ migrations.RunSQL(
+ sql="ALTER TABLE signals_anomalyscoringstate ALTER COLUMN series SET DEFAULT '{}'::jsonb",
+ reverse_sql="ALTER TABLE signals_anomalyscoringstate ALTER COLUMN series DROP DEFAULT",
+ )
+ ],
+ ),
+ ]
diff --git a/products/signals/backend/migrations/0201_anomalyscoringstate_artifacts.py b/products/signals/backend/migrations/0201_anomalyscoringstate_artifacts.py
new file mode 100644
index 000000000000..a22b39d83dd7
--- /dev/null
+++ b/products/signals/backend/migrations/0201_anomalyscoringstate_artifacts.py
@@ -0,0 +1,25 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("signals", "0200_anomalyscoringstate_untrack_series"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="anomalyscoringstate",
+ name="artifact_key",
+ field=models.CharField(db_default="", default="", max_length=255),
+ ),
+ migrations.AddField(
+ model_name="anomalyscoringstate",
+ name="artifact_sha256",
+ field=models.CharField(db_default="", default="", max_length=64),
+ ),
+ migrations.AddField(
+ model_name="anomalyscoringstate",
+ name="series_keys",
+ field=models.JSONField(db_default=[], default=list),
+ ),
+ ]
diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txt
index 5e45e928fd34..023db632f653 100644
--- a/products/signals/backend/migrations/max_migration.txt
+++ b/products/signals/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0199_anomalyscoringstate
+0201_anomalyscoringstate_artifacts
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index 430f7bda2b58..d8166f0151ab 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -4016,10 +4016,10 @@ class UserActivitySummaryQueueCursor(models.Model):
class AnomalyScoringState(TeamScopedRootMixin, UUIDModel):
- """The stored series tail of one insight and scoring profile of the anomaly scoring tick.
+ """The trained detector artifacts of one insight and scoring profile of the anomaly scoring tick.
- A train step fetches the full detector history and writes the tail. A score step fetches only the
- newest buckets, merges them into the tail, and scores the new buckets on it. See
+ A train step fits one artifact per series and writes them as one blob to object storage. A score
+ step fetches only the newest buckets and scores them with the stored artifacts. See
`anomaly_scores/state.py`. `$anomaly_score` events stay output only.
"""
@@ -4039,8 +4039,12 @@ class AnomalyScoringState(TeamScopedRootMixin, UUIDModel):
scored_through = models.DateTimeField(null=True, blank=True)
# Stored series that the last score step did not get back. More than half of them makes the next visit train.
missing_series = models.PositiveIntegerField(default=0)
- # Series key to `{"first_bucket": ISO bucket start, "values": [...]}`, with the newest values the detector reads.
- series = models.JSONField(default=dict)
+ # Object storage key of the artifact blob. Keys are never reused, so a loaded blob can be cached by key.
+ artifact_key = models.CharField(max_length=255, default="", db_default="")
+ # Checked before every load. A mismatch or a missing blob makes the next visit train.
+ artifact_sha256 = models.CharField(max_length=64, default="", db_default="")
+ # The series keys that the blob has an artifact for.
+ series_keys = models.JSONField(default=list, db_default=[])
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
diff --git a/products/signals/backend/test/test_anomaly_scores.py b/products/signals/backend/test/test_anomaly_scores.py
index f9f6930b5cd8..50461c364717 100644
--- a/products/signals/backend/test/test_anomaly_scores.py
+++ b/products/signals/backend/test/test_anomaly_scores.py
@@ -1,3 +1,4 @@
+import dataclasses
from datetime import UTC, datetime, timedelta
from typing import Any
from zoneinfo import ZoneInfo
@@ -25,11 +26,13 @@ from posthog.hogql.constants import INSIGHT_QUERY_FANOUT_CONCURRENCY
from posthog.interval_specs import interval_spec
from posthog.models import Team
+from posthog.tasks.alerts.detectors.registry import get_detector
from products.product_analytics.backend.facade.models import Insight
+from products.signals.backend.anomaly_scores.artifacts import deserialize, serialize
from products.signals.backend.anomaly_scores.config import DEFAULT_LIMITS, DEFAULT_PROFILE, IntervalScoring, ScoringRun
from products.signals.backend.anomaly_scores.scoring import (
- SeriesWindow,
+ InsightScores,
SplitRun,
StateIdentity,
TrainedState,
@@ -37,12 +40,9 @@ from products.signals.backend.anomaly_scores.scoring import (
bucket_end,
buckets_to_score,
fetch_buckets,
- history_buckets,
query_cost,
score_insight,
score_series,
- score_window,
- train_window,
trends_query_for,
)
from products.signals.backend.anomaly_scores.state import needs_training
@@ -187,66 +187,24 @@ class TestScoreSeries(SimpleTestCase):
assert all(point.score is None and point.flag is None for point in points)
-class TestScoreWindow(SimpleTestCase):
+class TestArtifactRoundTrip(SimpleTestCase):
@parameterized.expand(
[
- ("day", IntervalType.DAY, False),
- ("week", IntervalType.WEEK, False),
- ("hour", IntervalType.HOUR, False),
- ("day_late_events", IntervalType.DAY, True),
- ("hour_late_events", IntervalType.HOUR, True),
+ ("json_params", DAY_SCORING.detector),
+ ("pyod_model_bytes", {"type": "knn", "threshold": 0.95, "preprocessing": {"diffs_n": 1, "lags_n": 5}}),
]
)
- def test_a_score_step_on_the_stored_window_scores_like_the_full_history(
- self, _name: str, interval: IntervalType, late_events: bool
- ) -> None:
- scoring = DEFAULT_PROFILE.intervals[interval]
- history = history_buckets(scoring)
- count = history + 40
- values = list(np.random.default_rng(2).normal(100, 10, size=count))
- values[-1] = 400.0
- buckets = _buckets(datetime(2025, 1, 6, tzinfo=UTC), interval, count)
- trained_at = bucket_end(buckets[-3], interval) + timedelta(minutes=5)
- scored_at = bucket_end(buckets[-1], interval) + timedelta(minutes=5)
- window = train_window(values[:-1], buckets[:-1], history=history, interval=interval, now=trained_at)
- assert window is not None
- later = list(values)
- if late_events:
- # Events that arrive after the train step change a bucket that the score step fetches again.
- later[-4] += 50.0
- refresh = 3
- # The score step fetches the refresh buckets, one guard bucket, and the two new buckets.
- fetched = slice(count - (refresh + 1 + 2), count)
-
- scored = score_window(
- window, later[fetched], buckets[fetched], refresh=refresh, scoring=scoring, interval=interval, now=scored_at
- )
+ def test_a_stored_artifact_scores_like_the_fitted_one(self, _name: str, config: dict[str, Any]) -> None:
+ detector = get_detector(config)
+ data = np.random.default_rng(2).normal(100, 10, size=100)
+ data[-1] = 400.0
+ artifact = detector.fit(data[:90])
+ scored = data[90 - detector.context_buckets() :]
- assert scored is not None
- points, updated = scored
- expected = score_series(later, buckets, scoring=scoring, interval=interval, now=scored_at, after=buckets[-3])
- assert [(point.bucket_start, point.score, point.flag) for point in points] == [
- (point.bucket_start, point.score, point.flag) for point in expected
- ]
- assert [point.bucket_start for point in points] == buckets[-2:]
- assert points[-1].flag is True
- assert updated == SeriesWindow(first_bucket=buckets[-history], values=later[-history:])
-
- def test_fresh_buckets_after_a_gap_leave_the_window_unscored(self) -> None:
- buckets = _buckets(datetime(2026, 1, 1, tzinfo=UTC), IntervalType.DAY, 70)
- window = SeriesWindow(first_bucket=buckets[0], values=[1.0] * 60)
-
- scored = score_window(
- window,
- [1.0] * 5,
- buckets[62:67],
- refresh=3,
- scoring=DAY_SCORING,
- interval=IntervalType.DAY,
- now=buckets[67],
- )
+ loaded = deserialize(serialize({"series": artifact}))["series"]
- assert scored is None
+ assert loaded == artifact
+ assert detector.score(loaded, scored).all_scores == detector.score(artifact, scored).all_scores
def _insight(series: list[EventsNode], **query: Any) -> Insight:
@@ -326,8 +284,9 @@ class TestScoreInsight(SimpleTestCase):
if waves == 1:
assert limits[0] == 150
+ # The context of the day detector is 3 buckets, then one guard bucket and the 2 new buckets.
@parameterized.expand([("no_smoothing", None, "-6d"), ("smoothing", 7, "-12d")])
- def test_a_score_step_fetches_only_new_buckets_and_keeps_series_it_did_not_get_back(
+ def test_a_score_step_fetches_only_the_context_and_new_buckets(
self, _name: str, smoothing: int | None, date_from: str
) -> None:
insight = _insight([EventsNode(event="$pageview")], trendsFilter=TrendsFilter(smoothingIntervals=smoothing))
@@ -343,7 +302,7 @@ class TestScoreInsight(SimpleTestCase):
"data": list(rng.normal(100, 10, size=days)),
}
- def run(now: datetime, results: list[dict[str, Any]], split: SplitRun) -> tuple[Any, TrendsQuery]:
+ def run(now: datetime, results: list[dict[str, Any]], split: SplitRun) -> tuple[InsightScores, TrendsQuery]:
with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:
runner.return_value.run.return_value = MagicMock(results=results)
scores = score_insight(insight, team, now=now, split=split)
@@ -355,15 +314,16 @@ class TestScoreInsight(SimpleTestCase):
result("kept", datetime(2025, 10, 1, tzinfo=UTC), 102),
result("gone", datetime(2025, 10, 1, tzinfo=UTC), 102),
],
- SplitRun(refresh_buckets=3),
+ SplitRun(),
)
- assert trained.run == ScoringRun.TRAIN and trained.state is not None
- keys = {series.breakdown_value: series.series_key for series in trained.series}
+ assert trained.run == ScoringRun.TRAIN and trained.state is not None and trained.artifacts is not None
+ assert set(trained.state.series_keys) == {series.series_key for series in trained.series}
+ state = dataclasses.replace(trained.state, artifact_key="anomaly_models/1/7/default-v1/model")
scores, query = run(
datetime(2026, 1, 12, 0, 30, tzinfo=UTC),
[result("kept", datetime(2026, 1, 6, tzinfo=UTC), 7), result("new", datetime(2026, 1, 6, tzinfo=UTC), 7)],
- SplitRun(refresh_buckets=3, state=trained.state),
+ SplitRun(state=state, artifacts=trained.artifacts),
)
assert query.dateRange is not None and query.dateRange.date_from == date_from
@@ -375,15 +335,32 @@ class TestScoreInsight(SimpleTestCase):
assert scores.unmatched_series == 1
assert scores.state is not None
assert scores.state.missing_series == 1
- assert scores.state.windows[keys["gone"]] == trained.state.windows[keys["gone"]]
+ assert scores.state.series_keys == trained.state.series_keys
assert scores.state.scored_through == datetime(2026, 1, 11, tzinfo=UTC)
+ assert scores.model_id == "model"
+
+ # "gone" comes back. Its own newest scored bucket is still the one from the train step.
+ with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:
+ runner.return_value.run.return_value = MagicMock(
+ results=[result("gone", datetime(2026, 1, 6, tzinfo=UTC), 7)]
+ )
+ returned = score_insight(
+ insight,
+ team,
+ now=datetime(2026, 1, 13, 0, 30, tzinfo=UTC),
+ after=trained.state.scored_through,
+ split=SplitRun(state=scores.state, artifacts=trained.artifacts),
+ )
+
+ assert [point.bucket_start.day for point in returned.series[0].points] == [10, 11, 12]
def test_a_score_pick_whose_state_no_longer_matches_the_insight_trains(self) -> None:
stale = TrainedState(
identity=StateIdentity(interval=IntervalType.DAY, detector_version="old", query_hash="old"),
trained_at=datetime(2026, 1, 9, tzinfo=UTC),
scored_through=datetime(2026, 1, 9, tzinfo=UTC),
- windows={},
+ series_keys=("a",),
+ artifact_key="anomaly_models/1/7/default-v1/old",
)
with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:
runner.return_value.run.return_value = MagicMock(results=[])
@@ -391,16 +368,41 @@ class TestScoreInsight(SimpleTestCase):
_insight([EventsNode(event="$pageview")]),
Team(timezone="UTC"),
now=datetime(2026, 1, 10, tzinfo=UTC),
- split=SplitRun(refresh_buckets=3, state=stale),
+ split=SplitRun(state=stale, artifacts={}),
)
assert scores.run == ScoringRun.TRAIN
+ # The detector history and the backfill cap of a first visit, the same fetch as without the split.
assert runner.call_args.kwargs["query"].dateRange.date_from == "-90d"
+ @parameterized.expand([(interval.value, interval) for interval in DEFAULT_PROFILE.intervals])
+ def test_a_train_step_fits_every_series_at_each_default_interval(self, _name: str, interval: IntervalType) -> None:
+ buckets = [datetime(2024, 1, 1, tzinfo=UTC)]
+ while len(buckets) < 400:
+ buckets.append(bucket_end(buckets[-1], interval))
+ result = {
+ "action": {"order": 0},
+ "label": "pageview",
+ "breakdown_value": None,
+ "days": [bucket.strftime("%Y-%m-%d %H:%M:%S") for bucket in buckets],
+ "data": list(np.random.default_rng(5).normal(100, 10, size=len(buckets))),
+ }
+ with patch("products.signals.backend.anomaly_scores.scoring.TrendsQueryRunner") as runner:
+ runner.return_value.run.return_value = MagicMock(results=[result])
+ scores = score_insight(
+ _insight([EventsNode(event="$pageview")], interval=interval),
+ Team(timezone="UTC"),
+ now=bucket_end(buckets[-1], interval),
+ split=SplitRun(),
+ )
+
+ assert scores.run == ScoringRun.TRAIN and scores.artifacts is not None
+ assert set(scores.artifacts) == {series.series_key for series in scores.series} != set()
+ assert all(point.score is not None for point in scores.series[0].points)
+
_IDENTITY = StateIdentity(interval=IntervalType.DAY, detector_version="v1", query_hash="q1")
_SCORED_THROUGH = datetime(2026, 1, 9, tzinfo=UTC)
-_WINDOW = SeriesWindow(first_bucket=datetime(2025, 11, 11, tzinfo=UTC), values=[1.0] * 60)
def _state(**changes: Any) -> TrainedState:
@@ -408,7 +410,8 @@ def _state(**changes: Any) -> TrainedState:
"identity": _IDENTITY,
"trained_at": datetime(2026, 1, 5, tzinfo=UTC),
"scored_through": _SCORED_THROUGH,
- "windows": {"a": _WINDOW, "b": _WINDOW},
+ "series_keys": ("a", "b"),
+ "artifact_key": "anomaly_models/1/7/default-v1/model",
**changes,
}
return TrainedState(**fields)
@@ -420,6 +423,7 @@ class TestNeedsTraining(SimpleTestCase):
("fresh_state", _state(), False),
("one_of_two_series_missing", _state(missing_series=1), False),
("no_state", None, True),
+ ("no_artifact", _state(artifact_key=""), True),
(
"query_changed",
_state(identity=StateIdentity(interval=IntervalType.DAY, detector_version="v1", query_hash="q0")),
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index 0daeededb270..7dc48843538e 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -1,4 +1,6 @@
+import json
import uuid
+import hashlib
from datetime import UTC, datetime, timedelta
from typing import Any
from zoneinfo import ZoneInfo
@@ -23,11 +25,13 @@ from posthog.hogql.errors import QueryError, ResolutionError, TableAccessDeniedE
from posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimitExceeded, ClickHouseQueryTimeOut
from posthog.models import Organization, Team, User
+from posthog.tasks.alerts.detectors.base import ARTIFACT_SCHEMA_VERSION
from products.dashboards.backend.models.dashboard import Dashboard
from products.dashboards.backend.models.dashboard_tile import DashboardTile
from products.product_analytics.backend.facade.api import record_insight_view
from products.product_analytics.backend.facade.models import Insight
+from products.signals.backend.anomaly_scores.artifacts import clear_artifact_cache
from products.signals.backend.anomaly_scores.config import DISABLED, AnomalyScoringConfig, ScoringRun, parse_config
from products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_events
from products.signals.backend.anomaly_scores.queue import (
@@ -54,7 +58,7 @@ from products.signals.backend.anomaly_scores.scoring import (
UnsupportedInsightError,
)
from products.signals.backend.anomaly_scores.selection import SelectionConfig, Weighting
-from products.signals.backend.anomaly_scores.state import RunPlan
+from products.signals.backend.anomaly_scores.state import RunPlan, load_state, plan_run
from products.signals.backend.anomaly_scores.tick import (
AnomalyScoringTickInput,
AnomalyScoringTickPlan,
@@ -126,7 +130,6 @@ class TestTickConfig(SimpleTestCase):
"limits": {"max_backfill_buckets": 100_000, "max_series": -1},
"train_score_split": "true",
"train_every_buckets": {"hour": 100_000, "day": 0, "week": 2},
- "refresh_buckets": ["day"],
}
)
@@ -142,7 +145,6 @@ class TestTickConfig(SimpleTestCase):
IntervalType.WEEK: 2,
IntervalType.MONTH: 3,
}
- assert config.refresh_buckets == DISABLED.refresh_buckets
@parameterized.expand(
[
@@ -487,7 +489,26 @@ class TestRunInsight(SimpleTestCase):
class TestTrainScoreSplit(BaseTest):
- def _run(self, insight: Insight, pick: InsightPick, now: datetime, data: list[float]) -> list[dict[str, Any]]:
+ def setUp(self) -> None:
+ super().setUp()
+ self.team.timezone = "UTC"
+ self.team.save()
+ self.blobs: dict[str, bytes] = {}
+ storage = "products.signals.backend.anomaly_scores.artifacts.object_storage"
+ for name, fake in [
+ ("write", lambda key, blob: self.blobs.__setitem__(key, blob)),
+ ("read_bytes", lambda key, missing_ok: self.blobs.get(key)),
+ ("delete", lambda key: self.blobs.pop(key, None)),
+ ]:
+ patcher = patch(f"{storage}.{name}", side_effect=fake)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+ clear_artifact_cache()
+ self.addCleanup(clear_artifact_cache)
+
+ def _run(
+ self, insight: Insight, pick: InsightPick, now: datetime, data: list[float]
+ ) -> tuple[ScoreInsightResult, list[dict[str, Any]]]:
days = [(now - timedelta(days=len(data) - 1 - day)).strftime("%Y-%m-%d") for day in range(len(data))]
results = [{"action": {"order": 0}, "label": "pageview", "days": days, "data": data, "breakdown_value": None}]
with (
@@ -496,36 +517,40 @@ class TestTrainScoreSplit(BaseTest):
patch("products.signals.backend.anomaly_scores.tick.mark_attempted"),
):
runner.return_value.run.return_value = MagicMock(results=results)
- run_insight(ScoreInsightInput(pick=pick), now)
- return emit.call_args.args[1]
+ result = run_insight(ScoreInsightInput(pick=pick), now)
+ return result, emit.call_args.args[1] if emit.called else []
- def test_a_train_pick_stores_the_tail_that_the_next_score_pick_scores_on(self) -> None:
- self.team.timezone = "UTC"
- self.team.save()
- insight = Insight.objects.create(team=self.team, saved=True, query=_trends(interval="day"))
+ def _train(self, insight: Insight, trained_at: datetime) -> list[dict[str, Any]]:
values = [float(value) for value in np.random.default_rng(4).normal(100, 5, size=100)]
- # JSON has no NaN, so the oldest stored bucket is stored as null. The detector no longer reads it.
- values[39] = float("nan")
- trained_at = datetime(2026, 10, 20, 0, 30, tzinfo=UTC)
-
- trained = self._run(
+ _result, events = self._run(
insight,
InsightPick(team_id=self.team.id, insight_id=insight.id, scored_through={}, run=ScoringRun.TRAIN),
trained_at,
values,
)
- scored = self._run(
+ self.values = values
+ return events
+
+ def _score_pick(self, insight: Insight, trained: list[dict[str, Any]]) -> InsightPick:
+ return InsightPick(
+ team_id=self.team.id,
+ insight_id=insight.id,
+ scored_through={trained[-1]["properties"]["$anomaly_series_key"]: datetime(2026, 10, 19, tzinfo=UTC)},
+ run=ScoringRun.SCORE,
+ )
+
+ def test_a_train_pick_stores_the_artifact_that_the_next_score_pick_scores_with(self) -> None:
+ insight = Insight.objects.create(team=self.team, saved=True, query=_trends(interval="day"))
+ trained_at = datetime(2026, 10, 20, 0, 30, tzinfo=UTC)
+ trained = self._train(insight, trained_at)
+ first_key = AnomalyScoringState.objects.for_team(self.team.id).get(insight_id=insight.id).artifact_key
+ _result, scored = self._run(
insight,
- InsightPick(
- team_id=self.team.id,
- insight_id=insight.id,
- scored_through={trained[-1]["properties"]["$anomaly_series_key"]: datetime(2026, 10, 19, tzinfo=UTC)},
- run=ScoringRun.SCORE,
- refresh_buckets=3,
- ),
+ self._score_pick(insight, trained),
trained_at + timedelta(days=2),
- [*values[-4:], 500.0, 100.0],
+ [*self.values[-4:], 500.0, 100.0],
)
+ self._train(insight, trained_at + timedelta(days=7))
assert {event["properties"]["$anomaly_run"] for event in trained} == {"train"}
assert [
@@ -536,9 +561,42 @@ class TestTrainScoreSplit(BaseTest):
)
for event in scored
] == [("score", "2026-10-20T00:00:00+00:00", 0), ("score", "2026-10-21T00:00:00+00:00", 1)]
+ model_ids = {event["properties"]["$anomaly_model_id"] for event in [*trained, *scored]}
+ assert model_ids == {first_key.rsplit("/", 1)[-1]}
+ state = AnomalyScoringState.objects.for_team(self.team.id).get(insight_id=insight.id)
+ assert state.trained_at == trained_at + timedelta(days=7)
+ # A retrain replaces the blob and deletes the previous one.
+ assert list(self.blobs) == [state.artifact_key] != [first_key]
+
+ @parameterized.expand([("corrupt",), ("missing",), ("older_schema",)])
+ def test_an_artifact_blob_that_fails_its_check_makes_the_next_visit_train(self, case: str) -> None:
+ insight = Insight.objects.create(team=self.team, saved=True, query=_trends(interval="day"))
+ trained_at = datetime(2026, 10, 20, 0, 30, tzinfo=UTC)
+ trained = self._train(insight, trained_at)
state = AnomalyScoringState.objects.for_team(self.team.id).get(insight_id=insight.id)
- assert state.scored_through == datetime(2026, 10, 21, tzinfo=UTC)
- assert state.trained_at == trained_at
+ key = state.artifact_key
+ clear_artifact_cache()
+ if case == "missing":
+ del self.blobs[key]
+ elif case == "corrupt":
+ self.blobs[key] = b"{}"
+ else:
+ stored = json.loads(self.blobs[key])
+ for artifact in stored["series"].values():
+ artifact["schema_version"] = ARTIFACT_SCHEMA_VERSION - 1
+ self.blobs[key] = json.dumps(stored).encode()
+ state.artifact_sha256 = hashlib.sha256(self.blobs[key]).hexdigest()
+ state.save()
+ scored_at = trained_at + timedelta(days=2)
+
+ result, events = self._run(insight, self._score_pick(insight, trained), scored_at, self.values[-6:])
+
+ assert (result.status, events) == ("artifact_unavailable", [])
+ plan = plan_run(insight, self.team, load_state(self.team, insight), parse_config(_SPLIT_PAYLOAD), scored_at)
+ assert plan.run == ScoringRun.TRAIN
+
+
+_SPLIT_PAYLOAD = {"enabled": True, "organization_ids": [ORGANIZATION_ID], "train_score_split": True}
class TestEligibleTeams(BaseTest):