Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(autoresearch): drop models that cannot score from the shadow set (#119214)

posthog[bot]de4ec68c7c22
diff --git a/products/autoresearch/backend/inference/AGENTS.md b/products/autoresearch/backend/inference/AGENTS.md
index e43de498297e..e1e6c5024e74 100644
--- a/products/autoresearch/backend/inference/AGENTS.md
+++ b/products/autoresearch/backend/inference/AGENTS.md
@@ -70,7 +70,7 @@ After a live champion run completes, `_score_shadow_set()` in `scoring.py` score
- **Same people.** Each shadow model scores exactly the persons the champion's run scored, including a rolling subset. Its anchors bind the same cutoff, and a feature result that keys a different set of persons fails that model, because unpaired scores do not compare.
- **One feature query per distinct `features.sql`.** The digest of the bundle's SQL (`features_sql_digest()`) keys the materialized rows. The champion's rows serve a shadow model with the same SQL, and each other SQL runs once. A failed materialization is cached too. A result is released after the last model that uses it, so the phase does not hold one result per distinct SQL.
- **Person-less events.** A shadow event has `$autoresearch_model_role` `shadow`, no `$set`, and no person processing, and its `distinct_id` is unique per model and person (`_shadow_distinct_id()`), because ingestion deduplicates on timestamp, distinct_id, token, and event name, not on the UUID. Validation keys on `$autoresearch_person_id`. A shadow score never reaches the output person property. Right before its batch goes out, a shadow model checks that it is still in the set (`_require_still_in_shadow_set()`), in place of `_require_still_champion()`.
-- **Isolated runs.** Each shadow model records its own inference run, with `metrics["shadow"]` set and `scheduled` false, so `find_unscorable_champion()` never reads it, even after the model becomes the champion. A shadow failure fails only that run. The champion's run records `metrics["shadow_models"]`: the models that completed, failed, or were skipped.
+- **Isolated runs.** Each shadow model records its own inference run, with `metrics["shadow"]` set and `scheduled` false, so `find_unscorable_champion()` never reads it, even after the model becomes the champion. A shadow failure fails only that run. Repeatable shadow failures on `UNSCORABLE_AFTER_FAILED_DAYS` scoring days remove the model from the shadow set (`models_failing_repeatably_in_shadow()`). The champion's run records `metrics["shadow_models"]`: the models that completed, failed, or were skipped.
- **The rolling ranking ignores shadow events.** `build_inference_anchors_sql()` counts only non-shadow predictions as scores, so shadow scoring does not change who the champion scores next.
- **Cost bound.** No new shadow model starts after `SHADOW_TIME_BUDGET_S` (20 minutes). The rest are skipped for that cadence. The inference activity timeout covers the champion's worst case, the budget, and one more model.
diff --git a/products/autoresearch/backend/inference/failures.py b/products/autoresearch/backend/inference/failures.py
index ce68a9ec8777..82e1c5700ce0 100644
--- a/products/autoresearch/backend/inference/failures.py
+++ b/products/autoresearch/backend/inference/failures.py
@@ -1,9 +1,13 @@
-"""Classify a failed scoring run, and tell when the champion's scheduled runs show it cannot score."""
+"""Classify a failed scoring run, and tell when a model's runs show it cannot score."""
-from collections.abc import Iterator
+from collections import defaultdict
+from collections.abc import Iterable, Iterator
from datetime import date, timedelta
from enum import StrEnum
+from uuid import UUID
+from django.db.models import F, Q, QuerySet, Window
+from django.db.models.functions import RowNumber
from django.utils import timezone as django_timezone
import structlog
@@ -12,7 +16,7 @@ from posthog.dataclasses import frozen
from posthog.errors import QueryErrorCategory, classify_query_error
from products.autoresearch.backend.inference.sandbox import ModelLoadError
-from products.autoresearch.backend.models import AutoresearchModel, AutoresearchRun
+from products.autoresearch.backend.models import AutoresearchModel, AutoresearchPipeline, AutoresearchRun
logger = structlog.get_logger(__name__)
@@ -89,29 +93,61 @@ def find_unscorable_champion(champion: AutoresearchModel | None) -> UnscorableCh
"""
if champion is None:
return None
+ runs = _scoring_runs(champion.team_id, champion.pipeline_id, Q(scheduled=True)).filter(model=champion)
+ failed_days = _repeatable_failure_days(runs.order_by("-created_at")[:_RUNS_TO_SCAN])
+ if len(failed_days) < UNSCORABLE_AFTER_FAILED_DAYS:
+ return None
+ newest_day = max(failed_days)
+ return UnscorableChampion(failure_kind=failed_days[newest_day], onset=min(failed_days))
+
+
+def models_failing_repeatably_in_shadow(pipeline: AutoresearchPipeline, model_ids: Iterable[UUID]) -> set[UUID]:
+ """
+ Return the shadow candidates that cannot score. The rule is the one ``find_unscorable_champion``
+ applies, read over each model's shadow runs and the scheduled runs it had as a champion.
+ One query reads the newest ``_RUNS_TO_SCAN`` runs of every model.
+ """
+ model_ids = set(model_ids)
+ if not model_ids:
+ return set()
runs = (
- AutoresearchRun.objects.for_team(champion.team_id)
- .filter(
- pipeline_id=champion.pipeline_id,
- model=champion,
- run_type=AutoresearchRun.RunType.INFERENCE,
- scheduled=True,
- status__in=(AutoresearchRun.Status.COMPLETED, AutoresearchRun.Status.FAILED),
- )
- .order_by("-created_at")[:_RUNS_TO_SCAN]
+ _scoring_runs(pipeline.team_id, pipeline.pk, Q(scheduled=True) | Q(metrics__shadow=True))
+ .filter(model_id__in=model_ids)
+ .annotate(recency=Window(RowNumber(), partition_by=F("model_id"), order_by=F("created_at").desc()))
+ .filter(recency__lte=_RUNS_TO_SCAN)
+ .order_by("model_id", "-created_at")
)
- failed_days: dict[date, str] = {}
+ runs_by_model: dict[UUID, list[AutoresearchRun]] = defaultdict(list)
for run in runs:
+ if run.model_id is not None:
+ runs_by_model[run.model_id].append(run)
+ return {
+ model_id
+ for model_id, model_runs in runs_by_model.items()
+ if len(_repeatable_failure_days(model_runs)) >= UNSCORABLE_AFTER_FAILED_DAYS
+ }
+
+
+def _scoring_runs(team_id: int, pipeline_id: UUID, runs_filter: Q) -> QuerySet[AutoresearchRun]:
+ return AutoresearchRun.objects.for_team(team_id).filter(
+ runs_filter,
+ pipeline_id=pipeline_id,
+ run_type=AutoresearchRun.RunType.INFERENCE,
+ status__in=(AutoresearchRun.Status.COMPLETED, AutoresearchRun.Status.FAILED),
+ )
+
+
+def _repeatable_failure_days(newest_first: Iterable[AutoresearchRun]) -> dict[date, str]:
+ """The newest repeatable failure kind of each scoring day after the last completed run."""
+ failed_days: dict[date, str] = {}
+ for run in newest_first:
if run.status == AutoresearchRun.Status.COMPLETED:
break
kind = (run.metrics or {}).get("failure_kind")
if kind in REPEATABLE_FAILURE_KINDS:
# The newest failure of a day wins, so the kind reported is the latest one.
failed_days.setdefault(_run_date(run), kind)
- if len(failed_days) < UNSCORABLE_AFTER_FAILED_DAYS:
- return None
- newest_day = max(failed_days)
- return UnscorableChampion(failure_kind=failed_days[newest_day], onset=min(failed_days))
+ return failed_days
def fail_stale_inference_runs() -> int:
diff --git a/products/autoresearch/backend/training/AGENTS.md b/products/autoresearch/backend/training/AGENTS.md
index 5d1850607136..c6b799489133 100644
--- a/products/autoresearch/backend/training/AGENTS.md
+++ b/products/autoresearch/backend/training/AGENTS.md
@@ -51,6 +51,7 @@ The other half is `../inference/`, which consumes what this package produces and
`shadow_set(pipeline)` computes the models worth scoring side by side. Nothing stores the set.
It holds the champion, the previous champion (the newest archived bundle-backed row with `promoted_at` set), and up to `SHADOW_CHALLENGER_LIMIT` (3) fitted bundle-backed challengers. No two members share a `recipe_hash`.
A challenger younger than `horizon_days + SHADOW_MIN_MATURED_DATES` days keeps its place, so a new challenger cannot displace it. Past that age, a newer challenger displaces it. A challenger that does not enter at completion is never fitted, so it cannot enter later.
+ A model that cannot score never enters the set, and the next eligible model takes its slot. A previous-champion candidate cannot score when promotion archived it with `metrics.archived_reason` `unscorable` (set on a `replaced_unscorable` promotion, and cleared when a rollback restores the champion). Any candidate cannot score when `models_failing_repeatably_in_shadow()` in `../inference/failures.py` finds repeatable failures on `UNSCORABLE_AFTER_FAILED_DAYS` scoring days since its last completed run, read over its shadow runs and its scheduled champion runs. Kind `other` never counts. Nothing deletes the model row or its artifacts.
The models API exposes membership as `in_shadow_set`. Every live scoring cadence scores the set (see `../inference/AGENTS.md`). Nothing promotes from the set yet.
- `realized_context.py`
`build_realized_context(pipeline)` reads the validation history through `latest_validation_runs()` in `../evaluation/history.py`, the same read as the `online_performance` API.
diff --git a/products/autoresearch/backend/training/promotion.py b/products/autoresearch/backend/training/promotion.py
index 8880cbd093fe..a1468c4d237f 100644
--- a/products/autoresearch/backend/training/promotion.py
+++ b/products/autoresearch/backend/training/promotion.py
@@ -44,7 +44,12 @@ from products.autoresearch.backend.models import (
from products.autoresearch.backend.query import QueryCost
from products.autoresearch.backend.training import artifacts
from products.autoresearch.backend.training.recipe_validation import RecipeValidationError, validate_model_class
-from products.autoresearch.backend.training.shadow_set import FITTED_METRIC_KEY, shadow_set
+from products.autoresearch.backend.training.shadow_set import (
+ ARCHIVED_REASON_METRIC_KEY,
+ ARCHIVED_UNSCORABLE,
+ FITTED_METRIC_KEY,
+ shadow_set,
+)
from products.notebooks.backend.facade import api as notebooks_facade
logger = structlog.get_logger(__name__)
@@ -526,7 +531,8 @@ def _roll_back_promotion(
if restored is not None:
restored.role = AutoresearchModel.Role.CHAMPION
restored.archived_at = None
- restored.save(update_fields=["role", "archived_at"])
+ restored.metrics = {k: v for k, v in (restored.metrics or {}).items() if k != ARCHIVED_REASON_METRIC_KEY}
+ restored.save(update_fields=["role", "archived_at", "metrics"])
elif previous_status is not None and locked.status == AutoresearchPipeline.Status.RUNNING:
# The candidate was the first champion and made the pipeline live. With no champion
# the daily sweep fails every cadence, so the pipeline goes back to waiting for one.
@@ -641,8 +647,11 @@ def _finalize_under_lock(
role: str
if promotion_reason:
if current is not None:
+ archived_metrics = {k: v for k, v in (current.metrics or {}).items() if k != ARCHIVED_REASON_METRIC_KEY}
+ if promotion_reason == "replaced_unscorable":
+ archived_metrics[ARCHIVED_REASON_METRIC_KEY] = ARCHIVED_UNSCORABLE
AutoresearchModel.objects.filter(pk=current.pk).update(
- role=AutoresearchModel.Role.ARCHIVED, archived_at=now
+ role=AutoresearchModel.Role.ARCHIVED, archived_at=now, metrics=archived_metrics
)
role = AutoresearchModel.Role.CHAMPION
promoted = True
diff --git a/products/autoresearch/backend/training/shadow_set.py b/products/autoresearch/backend/training/shadow_set.py
index 70c2b5799663..e163b7927150 100644
--- a/products/autoresearch/backend/training/shadow_set.py
+++ b/products/autoresearch/backend/training/shadow_set.py
@@ -12,6 +12,7 @@ from uuid import UUID
from django.db.models import F
from django.utils import timezone as django_timezone
+from products.autoresearch.backend.inference.failures import models_failing_repeatably_in_shadow
from products.autoresearch.backend.models import AutoresearchModel, AutoresearchPipeline
SHADOW_CHALLENGER_LIMIT = 3
@@ -21,6 +22,9 @@ SHADOW_MIN_MATURED_DATES = 7
# Set in a model's metrics when its model.pkl is fitted. A challenger without it has no
# artifact that scoring could load.
FITTED_METRIC_KEY = "model_fitted"
+# Set in an archived champion's metrics when promotion replaced it because it could not score.
+ARCHIVED_REASON_METRIC_KEY = "archived_reason"
+ARCHIVED_UNSCORABLE = "unscorable"
def shadow_set(
@@ -30,7 +34,9 @@ def shadow_set(
Return the champion, the previous champion, and up to ``SHADOW_CHALLENGER_LIMIT``
challengers, newest first. No two members share a ``recipe_hash``.
- Only a bundle-backed model with a fitted ``model.pkl`` qualifies as a challenger.
+ Only a bundle-backed model with a fitted ``model.pkl`` qualifies as a challenger. A model
+ that cannot score (archived as unscorable, or a streak of repeatable failures) is skipped,
+ and the next eligible model takes its place.
``pending_fit`` names a challenger whose fit is not scheduled yet, so completion can ask
whether the fit would put it in the set.
"""
@@ -41,15 +47,13 @@ def shadow_set(
champion = models.filter(role=AutoresearchModel.Role.CHAMPION).first()
if champion is not None:
members.append(champion)
- previous = (
- models.filter(role=AutoresearchModel.Role.ARCHIVED, promoted_at__isnull=False)
+ former_champions = [
+ model
+ for model in models.filter(role=AutoresearchModel.Role.ARCHIVED, promoted_at__isnull=False)
.exclude(artifact_prefix="")
.order_by(F("archived_at").desc(nulls_last=True), "-promoted_at")
- .first()
- )
- if previous is not None and previous.recipe_hash not in {m.recipe_hash for m in members}:
- members.append(previous)
-
+ if (model.metrics or {}).get(ARCHIVED_REASON_METRIC_KEY) != ARCHIVED_UNSCORABLE
+ ]
challengers = [
model
for model in models.filter(role=AutoresearchModel.Role.CHALLENGER)
@@ -57,6 +61,13 @@ def shadow_set(
.order_by("-created_at")
if model.pk == pending_fit or (model.metrics or {}).get(FITTED_METRIC_KEY)
]
+ unscorable = models_failing_repeatably_in_shadow(pipeline, (m.pk for m in [*former_champions, *challengers]))
+
+ previous = next((model for model in former_champions if model.pk not in unscorable), None)
+ if previous is not None and previous.recipe_hash not in {m.recipe_hash for m in members}:
+ members.append(previous)
+
+ challengers = [model for model in challengers if model.pk not in unscorable]
members.extend(_select_challengers(challengers, pipeline=pipeline, now=now, taken={m.recipe_hash for m in members}))
return members
diff --git a/products/autoresearch/backend/training/test_promotion.py b/products/autoresearch/backend/training/test_promotion.py
index d43cc11c06ef..7a267aaa5f40 100644
--- a/products/autoresearch/backend/training/test_promotion.py
+++ b/products/autoresearch/backend/training/test_promotion.py
@@ -357,6 +357,9 @@ class TestCompleteTrainingRun(TeamScopedTestMixin, BaseTest):
assert self._champion().holdout_score == (0.7 if replaced else 0.8)
if replaced:
assert self._champion().metrics["promotion_reason"] == "replaced_unscorable"
+ champion.refresh_from_db()
+ assert champion.role == AutoresearchModel.Role.ARCHIVED
+ assert champion.metrics["archived_reason"] == "unscorable"
@parameterized.expand([("part_day_anchors", None, True), ("utc_day_anchors", "utc_day", False)])
def test_a_champion_trained_on_part_day_anchors_is_replaced_below_the_margin(self, _name, alignment, replaced):
diff --git a/products/autoresearch/backend/training/test_shadow_set.py b/products/autoresearch/backend/training/test_shadow_set.py
index 0f5f8bf05f21..72111ce3e5fe 100644
--- a/products/autoresearch/backend/training/test_shadow_set.py
+++ b/products/autoresearch/backend/training/test_shadow_set.py
@@ -6,7 +6,7 @@ from django.utils import timezone as django_timezone
from parameterized import parameterized
-from products.autoresearch.backend.models import AutoresearchModel, AutoresearchPipeline
+from products.autoresearch.backend.models import AutoresearchModel, AutoresearchPipeline, AutoresearchRun
from products.autoresearch.backend.testing import TeamScopedTestMixin
from products.autoresearch.backend.training.shadow_set import FITTED_METRIC_KEY, shadow_set
@@ -33,6 +33,8 @@ class TestShadowSet(TeamScopedTestMixin, BaseTest):
bundle: bool = True,
fitted: bool = False,
promoted: bool = False,
+ metrics: dict | None = None,
+ archived_days_ago: int = 0,
) -> AutoresearchModel:
created_at = self.now - timedelta(days=age_days)
model = AutoresearchModel.objects.create(
@@ -41,10 +43,12 @@ class TestShadowSet(TeamScopedTestMixin, BaseTest):
recipe_hash=recipe_hash,
model_recipe={},
artifact_prefix=f"bundles/{name}" if bundle else "",
- metrics={FITTED_METRIC_KEY: True} if fitted else {},
+ metrics={**({FITTED_METRIC_KEY: True} if fitted else {}), **(metrics or {})},
agent_description=name,
promoted_at=created_at if promoted else None,
- archived_at=self.now if role == AutoresearchModel.Role.ARCHIVED else None,
+ archived_at=self.now - timedelta(days=archived_days_ago)
+ if role == AutoresearchModel.Role.ARCHIVED
+ else None,
)
AutoresearchModel.objects.filter(pk=model.pk).update(created_at=created_at)
return model
@@ -123,3 +127,80 @@ class TestShadowSet(TeamScopedTestMixin, BaseTest):
)
assert [m.agent_description for m in shadow_set(self.pipeline, now=self.now)] == expected
+
+ def _shadow_runs(self, model: AutoresearchModel, failure_kinds: list[str]) -> None:
+ for days_ago, failure_kind in enumerate(reversed(failure_kinds), start=1):
+ run = AutoresearchRun.objects.create(
+ pipeline=self.pipeline,
+ model=model,
+ run_type=AutoresearchRun.RunType.INFERENCE,
+ status=AutoresearchRun.Status.FAILED,
+ metrics={
+ "shadow": True,
+ "prediction_date": (self.now - timedelta(days=days_ago)).date().isoformat(),
+ "failure_kind": failure_kind,
+ },
+ )
+ AutoresearchRun.objects.filter(pk=run.pk).update(created_at=self.now - timedelta(days=days_ago))
+
+ @parameterized.expand(
+ [
+ (
+ "archived_as_unscorable_is_skipped",
+ {"archived_reason": "unscorable"},
+ [],
+ [],
+ ["champion", "older", "c1"],
+ ),
+ (
+ "two_repeatable_failures_skip_it",
+ {},
+ ["limit_exceeded", "query_failed"],
+ [],
+ ["champion", "older", "c1"],
+ ),
+ ("one_repeatable_failure_keeps_it", {}, ["limit_exceeded"], [], ["champion", "previous", "c1"]),
+ ("transient_failures_keep_it", {}, ["other", "other"], [], ["champion", "previous", "c1"]),
+ (
+ "a_challenger_with_two_repeatable_failures_leaves",
+ {},
+ [],
+ ["model_load_failed", "model_load_failed"],
+ ["champion", "previous"],
+ ),
+ ]
+ )
+ def test_models_that_cannot_score_leave_the_set(
+ self,
+ _name,
+ previous_metrics: dict,
+ previous_failures: list[str],
+ challenger_failures: list[str],
+ expected: list[str],
+ ):
+ self._model("champion", role=AutoresearchModel.Role.CHAMPION, age_days=1, recipe_hash="champ", promoted=True)
+ previous = self._model(
+ "previous",
+ role=AutoresearchModel.Role.ARCHIVED,
+ age_days=30,
+ recipe_hash="prev",
+ promoted=True,
+ metrics=previous_metrics,
+ )
+ self._model(
+ "older",
+ role=AutoresearchModel.Role.ARCHIVED,
+ age_days=60,
+ recipe_hash="older",
+ promoted=True,
+ archived_days_ago=30,
+ )
+ challenger = self._model("c1", role=AutoresearchModel.Role.CHALLENGER, age_days=1, recipe_hash="c", fitted=True)
+ self._shadow_runs(previous, previous_failures)
+ self._shadow_runs(challenger, challenger_failures)
+
+ # One run-history query covers every candidate, however many failed.
+ with self.assertNumQueries(6):
+ members = shadow_set(self.pipeline, now=self.now)
+ assert [m.agent_description for m in members] == expected
+ assert AutoresearchModel.objects.filter(pipeline=self.pipeline).count() == 4