Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(autoresearch): keep training after a first champion rolls back (#126646)

posthog[bot]55223ce8a410
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 69703c2f78c6..f6fa324f981c 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -6718,6 +6718,10 @@ snapshots:
hash: v1.k794b7964.073ed0d4859386827dd0e45de3cf5eb857867e74e1b44f2fa4ede4a108742cca.hVQJteiYVN-fx70VzB55DOgKQVvHoDNtPEqzxqho5Ko
products-autoresearch-model-detail-scene--coverage-with-failed-runs--wide--light:
hash: v1.k794b7964.e50fd4ee2847fe0504c0aebc10c7eedb34d1ed4d2eff05b68848bc6f45b9f0d9.3zl510wgOjWMfOqmOycvoGw-M4D9VGx-OBni0PdNeok
+ products-autoresearch-model-detail-scene--first-champion-rolled-back--wide--dark:
+ hash: v1.k794b7964.bffc646c9c852bd652ba735564e0763d355979163c210368ea23e05554ae8fca.Be4kPj6BVaiNFWsNnZVL5rKyjBfc0WcEs_SVE8hbBx0
+ products-autoresearch-model-detail-scene--first-champion-rolled-back--wide--light:
+ hash: v1.k794b7964.86e7e37b4a768fb1d0d5b3e126581d16314fe29d9bd24e5d25ddaa505276ed0f.avvhk9BUjvIwcJ6q47-uiSktBYBomO64C9wF23tR2co
products-autoresearch-model-detail-scene--narrow--narrow--dark:
hash: v1.k794b7964.8d62dc0e26c677229f657685e9a91ff35456e8ea8a947afa92782e2b3359eb4e.YtQHUt83vFWkDF3SKK0iSQnyLikqzTYog0_46H_jOEg
products-autoresearch-model-detail-scene--narrow--narrow--light:
diff --git a/products/autoresearch/backend/facade/api.py b/products/autoresearch/backend/facade/api.py
index 3fd341095c1e..1ec658bd2351 100644
--- a/products/autoresearch/backend/facade/api.py
+++ b/products/autoresearch/backend/facade/api.py
@@ -204,6 +204,7 @@ class _PipelineActivity:
experiment_count: int
live_training_run: LiveTrainingRun | None
schedule_runs: list[ScheduleRun]
+ not_promoted_reason: str | None
def _best_so_far_trend(scores: Iterable[tuple[int, float | None]]) -> list[TrainingTrendPoint]:
@@ -311,6 +312,29 @@ def _first_checks_expected(team_id: int, champions: dict[UUID, AutoresearchModel
return first_checks
+def _not_promoted_reasons(team_id: int, pipeline_ids: list[UUID]) -> dict[UUID, str]:
+ """Why the best model of each pipeline's newest completed training run did not go live, when it did not."""
+ if not pipeline_ids:
+ return {}
+ newest_completed_runs = (
+ AutoresearchTrainingRun.objects.for_team(team_id)
+ .filter(pipeline_id__in=pipeline_ids, status=AutoresearchTrainingRun.Status.COMPLETED)
+ .order_by("pipeline_id", "-created_at")
+ .distinct("pipeline_id")
+ .values_list("id", flat=True)
+ )
+ reasons: dict[UUID, str] = {}
+ for pipeline_id, reason in (
+ AutoresearchModel.objects.for_team(team_id)
+ .filter(source_training_run_id__in=list(newest_completed_runs), metrics__has_key="not_promoted_reason")
+ .order_by("created_at")
+ .values_list("pipeline_id", "metrics__not_promoted_reason")
+ ):
+ if isinstance(reason, str) and reason:
+ reasons[pipeline_id] = reason
+ return reasons
+
+
def _pipeline_activity(
team_id: int, pipeline_ids: list[UUID], champions: dict[UUID, AutoresearchModel]
) -> dict[UUID, _PipelineActivity]:
@@ -360,6 +384,7 @@ def _pipeline_activity(
likely_counts = _champion_likely_counts(team_id, champions)
first_checks = _first_checks_expected(team_id, champions)
schedule_runs = schedule_runs_for(pipeline_ids, team_id)
+ not_promoted_reasons = _not_promoted_reasons(team_id, [pid for pid in pipeline_ids if pid not in champions])
return {
pipeline_id: _PipelineActivity(
champion_realized_auc_trend=[
@@ -379,6 +404,7 @@ def _pipeline_activity(
else None
),
schedule_runs=schedule_runs.get(pipeline_id, []),
+ not_promoted_reason=not_promoted_reasons.get(pipeline_id),
)
for pipeline_id in pipeline_ids
}
@@ -421,6 +447,11 @@ def _pipeline_to_contract(
effective_agent_reasoning_effort=agent_config.reasoning_effort,
training_phase=schedule.phase,
next_research_at=schedule.next_research_at,
+ not_promoted_reason=(
+ activity.not_promoted_reason
+ if row.status == AutoresearchPipeline.Status.BOOTSTRAPPING and champion is None
+ else None
+ ),
success_auc=row.success_auc,
plateau_iterations=row.plateau_iterations,
output_person_property=row.output_person_property,
diff --git a/products/autoresearch/backend/facade/contracts.py b/products/autoresearch/backend/facade/contracts.py
index 415835747144..33ccb8f7f535 100644
--- a/products/autoresearch/backend/facade/contracts.py
+++ b/products/autoresearch/backend/facade/contracts.py
@@ -159,6 +159,7 @@ class Pipeline:
effective_agent_reasoning_effort: str | None
training_phase: str
next_research_at: datetime | None
+ not_promoted_reason: str | None
success_auc: float | None
plateau_iterations: int
output_person_property: str
diff --git a/products/autoresearch/backend/presentation/views/serializers.py b/products/autoresearch/backend/presentation/views/serializers.py
index 895b77aad2fd..bdb38776af14 100644
--- a/products/autoresearch/backend/presentation/views/serializers.py
+++ b/products/autoresearch/backend/presentation/views/serializers.py
@@ -729,6 +729,14 @@ class AutoresearchPipelineSerializer(DataclassSerializer):
"Null when research_interval_days is null."
),
)
+ not_promoted_reason = serializers.CharField(
+ read_only=True,
+ allow_null=True,
+ help_text=(
+ "Why the best model of the newest completed training run did not go live, for example a failed fit "
+ "or a failed scorability check. Set only while the pipeline is bootstrapping with no champion."
+ ),
+ )
success_auc = serializers.FloatField(
required=False,
allow_null=True,
@@ -865,6 +873,7 @@ class AutoresearchPipelineSerializer(DataclassSerializer):
"effective_agent_reasoning_effort",
"training_phase",
"next_research_at",
+ "not_promoted_reason",
"success_auc",
"plateau_iterations",
"output_person_property",
diff --git a/products/autoresearch/backend/query.py b/products/autoresearch/backend/query.py
index d910552db713..b28eefa3b12f 100644
--- a/products/autoresearch/backend/query.py
+++ b/products/autoresearch/backend/query.py
@@ -1,13 +1,13 @@
import random
from collections.abc import Callable
from time import perf_counter, sleep
-from typing import Any, TypeVar
+from typing import Any, TypeVar, cast
import structlog
from posthog.schema import CacheMissResponse, HogQLQuery, QueryStatusResponse
-from posthog.hogql.constants import LimitContext
+from posthog.hogql.constants import HogQLGlobalSettings, LimitContext
from posthog.hogql.query_stats import query_stats_scope
from posthog.clickhouse.workload import Workload
@@ -40,6 +40,7 @@ class QueryContext:
# A query inside a web request keeps the 60 s interactive limit, because a longer query
# would outlive the request.
INTERACTIVE_QUERY = QueryContext(limit_context=LimitContext.QUERY, workload=Workload.DEFAULT)
+INTERACTIVE_QUERY_TIME_LIMIT_S = cast(int, HogQLGlobalSettings().max_execution_time)
# Scoring and online validation run in a Temporal activity that nobody waits on, so they
# get the increased batch limit and run on the offline workload.
BATCH_QUERY = QueryContext(limit_context=LimitContext.QUERY_ASYNC, workload=Workload.OFFLINE)
diff --git a/products/autoresearch/backend/temporal/AGENTS.md b/products/autoresearch/backend/temporal/AGENTS.md
index de8d13438016..c079b3448111 100644
--- a/products/autoresearch/backend/temporal/AGENTS.md
+++ b/products/autoresearch/backend/temporal/AGENTS.md
@@ -63,6 +63,8 @@ Kickoff then follows the research schedule in `../training/schedule.py`, which i
In explore, it starts a run (`explore`) until a completed run does not promote its best model, or until `EXPLORE_RUN_LIMIT` research runs have run. Rescue runs do not count.
In maintain, it starts a run (`scheduled`) once `research_interval_days` calendar days have passed since the latest training run of any kind, and reports `not_due` before that. A null interval reports `manual_only`.
Each run takes the pipeline's `iterations_per_run`, which `run_training()` lowers to the project's `max_iterations_per_run`. Kickoff does not read or spend `iteration_budget_remaining`.
+Kickoff accepts a `running` pipeline, and a `bootstrapping` pipeline whose first champion rolled back (`awaiting_first_champion()` in `../training/schedule.py`: no champion, no pending or running training run, and at least one completed run). Otherwise it reports `not_eligible`.
+The daily sweep lists such a pipeline last, with `ActivePipeline.kickoff_only`. The coordinator then only kicks off its training, because it has no champion to score or validate. This is behind `workflow.patched("autoresearch-kickoff-only-2026-10")`. A history without the marker also starts scoring and validation for it, and those activities skip a pipeline that is not live. The catch-up sweep does not list it.
When no run is due and `find_unscorable_champion()` finds that the champion cannot score, kickoff launches a rescue run of `RESCUE_ITERATION_BUDGET` iterations, at most once in each window of `UNSCORABLE_AFTER_FAILED_DAYS` UTC days, and reports `rescue_unscorable`.
Before it launches, kickoff checks the project's daily training cap (`check_training_run_cap()` in `../limits.py`), the same check `start_training` makes, and reports `team_daily_cap` when the project has no slot left. The cap counts every sandbox run of the team in the last 24 hours, so it blocks rescue runs too. The check takes a transaction-scoped advisory lock on the team inside the pipeline row lock, so two pipelines of one team cannot both take the last slot.
Each run records why it started in `kickoff_reason`. The coordinator body never branches on the kickoff result, because that would break replay.
diff --git a/products/autoresearch/backend/temporal/test_workflows.py b/products/autoresearch/backend/temporal/test_workflows.py
index 4425a0b36c54..610b0c43502e 100644
--- a/products/autoresearch/backend/temporal/test_workflows.py
+++ b/products/autoresearch/backend/temporal/test_workflows.py
@@ -776,6 +776,57 @@ class TestCoordinatorActivities(TeamScopedTestMixin, BaseTest):
assert not result.kicked_off
mock_run_training.assert_not_called()
+ @parameterized.expand(
+ [
+ ("first_champion_rolled_back", [AutoresearchTrainingRun.Status.COMPLETED], True, "scheduled"),
+ (
+ "training_run_live",
+ [AutoresearchTrainingRun.Status.COMPLETED, AutoresearchTrainingRun.Status.RUNNING],
+ False,
+ "not_eligible",
+ ),
+ ("no_completed_run", [AutoresearchTrainingRun.Status.FAILED], False, "not_eligible"),
+ ]
+ )
+ @patch("products.autoresearch.backend.temporal.workflows.run_training")
+ def test_bootstrapping_pipeline_trains_again_after_its_first_champion_rolled_back(
+ self,
+ _name: str,
+ run_statuses: list[str],
+ expected_listed: bool,
+ expected_reason: str,
+ mock_run_training: MagicMock,
+ ) -> None:
+ with time_machine.travel(datetime(2026, 8, 1, tzinfo=UTC), tick=False):
+ pipeline = self._create_pipeline(status=AutoresearchPipeline.Status.BOOTSTRAPPING, research_interval_days=7)
+ for run_status in run_statuses:
+ with time_machine.travel(datetime(2026, 9, 1, 2, tzinfo=UTC), tick=False):
+ run = AutoresearchTrainingRun.objects.create(
+ pipeline=pipeline, status=run_status, iteration_budget=3, summary={"champion_promoted": False}
+ )
+ AutoresearchModel.objects.create(
+ pipeline=pipeline,
+ source_training_run=run,
+ role=AutoresearchModel.Role.CHALLENGER,
+ model_recipe={"stub": True},
+ recipe_hash="rolled-back",
+ metrics={"not_promoted_reason": "The champion fit failed"},
+ )
+
+ daily = activity_load_active_pipelines(LoadActivePipelinesInput())
+ catch_up = activity_load_active_pipelines(LoadActivePipelinesInput(catch_up_date="2026-09-11"))
+ result = activity_kickoff_training(KickoffTrainingInput(pipeline_id=str(pipeline.id), team_id=self.team.id))
+
+ listed = [p for p in daily.pipelines if p.pipeline_id == str(pipeline.id)]
+ assert listed == (
+ [ActivePipeline(pipeline_id=str(pipeline.id), team_id=self.team.id, score_due=True, kickoff_only=True)]
+ if expected_listed
+ else []
+ )
+ assert str(pipeline.id) not in [p.pipeline_id for p in catch_up.pipelines]
+ assert result.reason == expected_reason
+ assert mock_run_training.called is expected_listed
+
@parameterized.expand([("transient_failure_raises", RuntimeError("boom")), ("refused_launch", ValueError("left"))])
@patch("products.autoresearch.backend.temporal.workflows.run_training")
def test_failed_launch(self, _name: str, error: Exception, mock_run_training: MagicMock) -> None:
@@ -924,11 +975,15 @@ async def test_coordinator_fails_the_runs_of_a_failed_inference_child(catch_up:
async def load(inp: LoadActivePipelinesInput) -> LoadActivePipelinesResult:
load_inputs.append(inp)
return LoadActivePipelinesResult(
- pipelines=[ActivePipeline(pipeline_id=pid, team_id=1, score_due=True) for pid in ("lost", "scored")]
+ pipelines=[
+ *[ActivePipeline(pipeline_id=pid, team_id=1, score_due=True) for pid in ("lost", "scored")],
+ ActivePipeline(pipeline_id="waiting", team_id=1, score_due=True, kickoff_only=True),
+ ]
)
@activity.defn(name="autoresearch-inference.run_inference")
async def infer(inp: RunInferenceInput) -> RunInferenceResult:
+ steps.append(f"inference-{inp.pipeline_id}")
if inp.pipeline_id == "lost":
raise ApplicationError("worker lost", non_retryable=True)
return RunInferenceResult(run_id=f"run-{inp.pipeline_id}", rows_scored=1, status="completed")
@@ -944,12 +999,12 @@ async def test_coordinator_fails_the_runs_of_a_failed_inference_child(catch_up:
@activity.defn(name="autoresearch-validation.run_validation")
async def validate(inp: RunValidationInput) -> RunValidationResult:
- steps.append("validation")
+ steps.append(f"validation-{inp.pipeline_id}")
return RunValidationResult(dates_validated=1, total_rows=1, status="completed")
@activity.defn(name="autoresearch-coordinator.kickoff_training")
async def kickoff(inp: KickoffTrainingInput) -> KickoffTrainingResult:
- steps.append("kickoff")
+ steps.append(f"kickoff-{inp.pipeline_id}")
return KickoffTrainingResult(kicked_off=False, reason="not_due")
@activity.defn(name="autoresearch-coordinator.run_shadow_scoring")
@@ -978,8 +1033,10 @@ async def test_coordinator_fails_the_runs_of_a_failed_inference_child(catch_up:
]
# Cleanup is scoped to the failed child's execution, so a catch-up child's live row stays open.
assert failed_runs[0].workflow_run_id
- assert (result.pipelines_processed, result.pipelines_errored) == (1, 1)
+ assert (result.pipelines_processed, result.pipelines_errored) == (2, 1)
assert "shadow-scored" in steps
# The daily sweep already kicked off training and validated, so a catch-up only scores.
- assert ("kickoff" in steps, "validation" in steps) == (not catch_up, not catch_up)
+ assert ("kickoff-scored" in steps, "validation-scored" in steps) == (not catch_up, not catch_up)
+ # A pipeline with no champion is never scored or validated, and only the daily sweep kicks off its training.
+ assert [step for step in steps if step.endswith("-waiting")] == ([] if catch_up else ["kickoff-waiting"])
assert [step for step in steps if step.startswith("mark-")] == ([] if catch_up else ["mark-running", "mark-done"])
diff --git a/products/autoresearch/backend/temporal/workflows.py b/products/autoresearch/backend/temporal/workflows.py
index 9a7bfbadfe72..3033a4ac36d4 100644
--- a/products/autoresearch/backend/temporal/workflows.py
+++ b/products/autoresearch/backend/temporal/workflows.py
@@ -27,7 +27,7 @@ with workflow.unsafe.imports_passed_through():
import asyncio as _asyncio
from django.db import transaction
- from django.db.models import QuerySet
+ from django.db.models import Q, QuerySet
from django.utils import timezone as django_timezone
from posthog.models.team.team import Team
@@ -62,7 +62,11 @@ with workflow.unsafe.imports_passed_through():
)
from products.autoresearch.backend.query import BATCH_QUERY, SCORING_QUERY
from products.autoresearch.backend.training.runner import run_training
- from products.autoresearch.backend.training.schedule import research_schedule, schedule_runs_for
+ from products.autoresearch.backend.training.schedule import (
+ awaiting_first_champion,
+ research_schedule,
+ schedule_runs_for,
+ )
from products.autoresearch.backend.training.stale import fail_stale_training_runs
from products.tasks.backend.facade.access import get_desktop_access_decision
from products.tasks.backend.facade.usage import task_run_usage_limited
@@ -671,6 +675,8 @@ class ActivePipeline:
team_id: int
# Validation runs on every sweep; scoring and training kickoff only on a cadence day.
score_due: bool
+ # A bootstrapping pipeline whose first champion rolled back. The sweep only kicks off its training.
+ kickoff_only: bool = False
@frozen
@@ -770,6 +776,10 @@ def activity_load_active_pipelines(inp: LoadActivePipelinesInput) -> LoadActiveP
A catch-up lists nothing while the daily sweep of that date still runs (``activity_mark_daily_sweep``),
including its kickoff, validation and shadow phases, so the two sweeps never exceed one bound
together. The next catch-up tick tries again.
+
+ The daily sweep also lists each ``bootstrapping`` pipeline whose first champion rolled back
+ (``awaiting_first_champion()``), last and with ``kickoff_only``, so its research schedule
+ continues. It has no champion, so the sweep does not score or validate it.
"""
fail_stale_inference_runs()
fail_stale_training_runs()
@@ -780,9 +790,12 @@ def activity_load_active_pipelines(inp: LoadActivePipelinesInput) -> LoadActiveP
yesterday = today - timedelta(days=1)
# unscoped() because the coordinator's job is exactly to sweep every team. The
# pipelines it returns carry their team so the downstream activities can scope.
+ sweep_filter = Q(status__in=_LIVE_STATUSES)
+ if inp.catch_up_date is None:
+ sweep_filter |= awaiting_first_champion()
candidates = (
AutoresearchPipeline.objects.unscoped()
- .filter(status__in=_LIVE_STATUSES)
+ .filter(sweep_filter)
# A null is_active counts as deactivated, as ActiveOrganizationPermission reads it.
.filter(team__organization__is_active=True)
.exclude(team__organization__is_pending_deletion=True)
@@ -801,6 +814,18 @@ def activity_load_active_pipelines(inp: LoadActivePipelinesInput) -> LoadActiveP
creator, team_id=pipeline.team_id, organization_id=str(pipeline.team.organization_id)
):
continue
+ if pipeline.status == AutoresearchPipeline.Status.BOOTSTRAPPING:
+ # score_due stays True for a coordinator that started before the kickoff-only patch: it
+ # kicks off training, and the scoring and validation activities skip a pipeline that is not live.
+ pipelines.append(
+ (
+ True,
+ ActivePipeline(
+ pipeline_id=str(pipeline.pk), team_id=pipeline.team_id, score_due=True, kickoff_only=True
+ ),
+ )
+ )
+ continue
# By calendar day, because a run finishes some time after the 02:00 UTC tick and an
# elapsed-time comparison would skip the next day's tick.
last_scored = pipeline.last_scored_at.date() if pipeline.last_scored_at else None
@@ -869,6 +894,9 @@ def _pause_orphaned_pipeline(pipeline: AutoresearchPipeline, *, reason: str) ->
def activity_kickoff_training(inp: KickoffTrainingInput) -> KickoffTrainingResult:
"""Start a scheduled agent training run, if the pipeline's research schedule says one is due.
+ The pipeline must be ``running``, or ``bootstrapping`` after its first champion rolled back
+ (``awaiting_first_champion()``).
+
The run row and its task binding commit together under the pipeline row lock, as in the
facade's ``start_training``, so a manual start cannot interleave. A launch failure raises,
so the activity retry applies.
@@ -886,7 +914,9 @@ def activity_kickoff_training(inp: KickoffTrainingInput) -> KickoffTrainingResul
with team_scope(inp.team_id), transaction.atomic():
pipeline = AutoresearchPipeline.objects.select_for_update().get(pk=inp.pipeline_id)
- if pipeline.status != AutoresearchPipeline.Status.RUNNING:
+ if pipeline.status != AutoresearchPipeline.Status.RUNNING and not (
+ AutoresearchPipeline.objects.filter(pk=pipeline.pk).filter(awaiting_first_champion()).exists()
+ ):
return KickoffTrainingResult(kicked_off=False, reason="not_eligible")
if AutoresearchTrainingRun.objects.filter(
@@ -1028,6 +1058,8 @@ _BOUNDED_SWEEP_PATCH = "autoresearch-bounded-sweep-2026-10"
_CLOSE_LOST_RUNS_PATCH = "autoresearch-close-lost-runs-2026-10"
# Histories that started before this marker do not tell a catch-up that the daily sweep is running.
_DAILY_SWEEP_MARKER_PATCH = "autoresearch-daily-sweep-marker-2026-10"
+# Histories that started before this marker also start scoring and validation for a kickoff-only pipeline.
+_KICKOFF_ONLY_PATCH = "autoresearch-kickoff-only-2026-10"
@workflow.defn(name="autoresearch-coordinator")
@@ -1158,12 +1190,19 @@ class AutoresearchCoordinatorWorkflow(PostHogWorkflow):
) -> _PipelineSweep:
"""On a cadence day score, then kick off training. Then validate. One step at a time, so a pipeline holds one slot.
- A catch-up sweep only scores.
+ A catch-up sweep only scores. A kickoff-only pipeline only kicks off training.
"""
pipeline_id = pipeline.pipeline_id
inference_result: InferenceWorkflowResult | BaseException | None = None
validation_result: ValidationWorkflowResult | BaseException | None = None
kickoff_result: KickoffTrainingResult | BaseException | None = None
+ if workflow.patched(_KICKOFF_ONLY_PATCH) and pipeline.kickoff_only:
+ if not catch_up:
+ kickoff_result = await _settle(self._kickoff(pipeline))
+ return _PipelineSweep(
+ outcome=self._evaluate(pipeline_id, inference_result, validation_result, kickoff_result),
+ inference=None,
+ )
if pipeline.score_due:
inference_result = await _settle(
workflow.execute_child_workflow(
diff --git a/products/autoresearch/backend/tests/test_api.py b/products/autoresearch/backend/tests/test_api.py
index 8150ad1965c3..b79f9ccf12d2 100644
--- a/products/autoresearch/backend/tests/test_api.py
+++ b/products/autoresearch/backend/tests/test_api.py
@@ -428,6 +428,36 @@ class TestAutoresearchPipelineAPI(TeamScopedTestMixin, APIBaseTest):
assert resp.json()["id"] == str(pipeline.id)
assert resp.json()["target_definition"] == {"type": "event"}
+ @parameterized.expand(
+ [
+ ("first_champion_rolled_back", AutoresearchPipeline.Status.BOOTSTRAPPING, False, "The champion fit failed"),
+ ("champion_live", AutoresearchPipeline.Status.RUNNING, True, None),
+ ]
+ )
+ def test_retrieve_explains_why_no_model_is_live(
+ self, _name: str, pipeline_status: str, has_champion: bool, expected_reason: str | None
+ ) -> None:
+ pipeline = self._make_pipeline(status=pipeline_status)
+ run = AutoresearchTrainingRun.objects.create(
+ pipeline=pipeline, status=AutoresearchTrainingRun.Status.COMPLETED, iteration_budget=5
+ )
+ AutoresearchModel.objects.create(
+ pipeline=pipeline,
+ source_training_run=run,
+ role=AutoresearchModel.Role.CHALLENGER,
+ model_recipe={"stub": True},
+ recipe_hash="rolled-back",
+ metrics={"not_promoted_reason": "The champion fit failed"},
+ )
+ if has_champion:
+ AutoresearchModel.objects.create(
+ pipeline=pipeline, role=AutoresearchModel.Role.CHAMPION, model_recipe={"stub": True}, recipe_hash="live"
+ )
+
+ resp = self.client.get(f"{self.base_url}/{pipeline.id}/")
+
+ assert resp.json()["not_promoted_reason"] == expected_reason
+
def test_other_team_cannot_access_pipeline(self):
other_org = Organization.objects.create(name="Other Org")
other_team = Team.objects.create(organization=other_org, name="Other Team")
diff --git a/products/autoresearch/backend/training/AGENTS.md b/products/autoresearch/backend/training/AGENTS.md
index b19126f50496..3e15a18aaf2f 100644
--- a/products/autoresearch/backend/training/AGENTS.md
+++ b/products/autoresearch/backend/training/AGENTS.md
@@ -17,6 +17,7 @@ The other half is `../inference/`, which consumes what this package produces and
`build_agent_description()` assembles the agent's brief — the target, the horizon, the population, and the contract for the bundle it must author.
When the `autoresearch-report-notebook` flag is on for the launching user, the token also holds `REPORT_NOTEBOOK_MCP_SCOPES` and the brief adds two Finalize steps: before complete, the agent creates one report notebook and passes its `short_id` to complete. After complete, it adds the SQL and chart cells, because the run's model row (and so its `model_explanation`) exists only from then on. `report.md` stays required either way.
The brief's worked `features.sql` reads events through a pre-filtered subquery (the event names the features use, the anchor persons, and the anchors' widest window) before it joins, because ClickHouse builds the hash table from the right side of a join and a direct events join reads the whole team's events.
+ The brief states the two limits of the fit at completion, from the constants: the fitted `model.pkl` must stay under `MAX_ARTIFACT_BYTES` (the readback cap of `fit_champion_model()`), and `features.sql` must finish inside `INTERACTIVE_QUERY_TIME_LIMIT_S` (`../query.py`), because the fit runs it under `INTERACTIVE_QUERY`. A model that breaks either limit rolls back.
The brief also states the inference cutoff: the start of the prediction date in UTC (`ScoringWindow` in `../inference/scoring.py`), not `now()`.
The brief's cost guidance is advice only. Cost does not enter champion selection, so the brief does not tell the agent to trade AUC for a cheaper query. A rescue run is the one exception, see the unscorable champion below.
For a person column that events do not carry, the brief recommends `raw_persons` in a subquery filtered with `id IN (SELECT person_id FROM {anchors})`, never `LEFT JOIN persons`, because the persons table dedupes every person of the team before any filter applies.
@@ -53,7 +54,7 @@ The other half is `../inference/`, which consumes what this package produces and
A promoted bundle-backed model is fitted. A bundle-backed challenger is fitted only when it enters the shadow set (see `shadow_set.py`). The fit runs after commit and never changes the completion result.
A successful challenger fit sets `metrics.model_fitted`, and a failed one only logs. A challenger without `model_fitted` stays out of the shadow set.
After a champion fit, `check_scorability()` runs the bundle's `features.sql` against today's inference anchors under `BATCH_QUERY`. A passing check records `scorability_elapsed_s` and `scorability_rows_read` in the model metrics and sets `model_fitted`.
- A failed fit, a failed check, or a check above `SCORABILITY_TIME_BUDGET_S` (half the batch query limit) rolls the promotion back under the pipeline lock: the candidate becomes a challenger with `not_promoted_reason` in its metrics, and the champion it archived comes back. A first champion that rolls back puts the pipeline back in the status it had before promotion. The rolled-back candidate has no `model_fitted`, so it never enters the shadow set.
+ A failed fit, a failed check, or a check above `SCORABILITY_TIME_BUDGET_S` (half the batch query limit) rolls the promotion back under the pipeline lock: the candidate becomes a challenger with `not_promoted_reason` in its metrics, and the champion it archived comes back. A first champion that rolls back puts the pipeline back in the status it had before promotion. The run summary then records `champion_promoted: false`, so the research schedule ends explore for it. A `bootstrapping` pipeline in this state (`awaiting_first_champion()` in `schedule.py`) keeps its research schedule, see `../temporal/AGENTS.md`, and the pipeline API returns the candidate's reason as `not_promoted_reason`. The rolled-back candidate has no `model_fitted`, so it never enters the shadow set.
The rollback changes nothing when the candidate is no longer the champion. A scoring run that started with the candidate fails in `_require_still_champion()` before it emits.
A recipe-only champion is not checked.
A champion whose scheduled scoring runs failed with a repeatable `failure_kind` on the last two prediction dates, with no success after the first of them, is unscorable (`find_unscorable_champion()` in `../inference/failures.py`). Any candidate replaces it whatever the margin, with `promotion_reason` `replaced_unscorable` in the model metrics, and the fit and the scorability check still roll back a candidate that cannot score either. The brief tells the agent the failure kind and the onset date. A rescue run (see `../temporal/AGENTS.md`) also tells the agent to prefer a cheap model that scores over a higher holdout score.
diff --git a/products/autoresearch/backend/training/runner.py b/products/autoresearch/backend/training/runner.py
index 6d3cff1eddf6..91fb420af0c0 100644
--- a/products/autoresearch/backend/training/runner.py
+++ b/products/autoresearch/backend/training/runner.py
@@ -48,7 +48,9 @@ from products.autoresearch.backend.models import (
AutoresearchSuggestion,
AutoresearchTrainingRun,
)
+from products.autoresearch.backend.query import INTERACTIVE_QUERY_TIME_LIMIT_S
from products.autoresearch.backend.training.agent_model import agent_run_config_for
+from products.autoresearch.backend.training.artifacts import MAX_ARTIFACT_BYTES
from products.autoresearch.backend.training.explanation import MAX_TOP_FEATURES
from products.autoresearch.backend.training.realized_context import (
RealizedContext,
@@ -476,6 +478,16 @@ def build_agent_description(
is not promoted and the previous champion keeps serving.
- Cost does not change which iteration wins, so keep each hypothesis cheap from the start.
+ **Fit limits at completion.** When you complete the run, the backend fits the winning bundle
+ once more. Two limits apply to that fit, and a model that fails either one is not promoted:
+ - `features.sql` runs against the full training population, and must finish inside the
+ {INTERACTIVE_QUERY_TIME_LIMIT_S} s interactive query limit. Check `elapsed_s` in the materialize
+ response, and keep the query well under that limit. Use the pre-filtered, rolled-up shapes above.
+ - The fitted `model.pkl` must stay under {MAX_ARTIFACT_BYTES // (1024 * 1024)} MiB
+ ({MAX_ARTIFACT_BYTES} bytes). Tree ensembles grow fast: bound `n_estimators` (for example 200 or
+ fewer) and `max_depth` (for example 8 or less), or set `min_samples_leaf`, and check the pickle size
+ in your sandbox before you upload.
+
### Step 3 — Materialize features, then fit and evaluate (in your sandbox)
**Your sandbox is already equipped — do NOT `pip install` anything.** numpy, pandas,
diff --git a/products/autoresearch/backend/training/schedule.py b/products/autoresearch/backend/training/schedule.py
index 4a5e4caae18a..0753821def29 100644
--- a/products/autoresearch/backend/training/schedule.py
+++ b/products/autoresearch/backend/training/schedule.py
@@ -4,6 +4,9 @@ Research has two phases, derived from the pipeline's training runs and never sto
Explore starts a run on each sweep until a completed run does not promote its best model,
or until ``EXPLORE_RUN_LIMIT`` research runs have run. Maintain starts one run
``research_interval_days`` after the latest training run of any kind, so a manual run resets the clock.
+
+A ``bootstrapping`` pipeline whose first champion rolled back follows the same schedule
+(``awaiting_first_champion()``).
"""
from collections.abc import Iterable
@@ -11,9 +14,11 @@ from datetime import UTC, datetime, time, timedelta
from typing import Literal
from uuid import UUID
+from django.db.models import Exists, OuterRef, Q
+
from posthog.dataclasses import frozen
-from ..models import AutoresearchTrainingRun
+from ..models import AutoresearchModel, AutoresearchPipeline, AutoresearchTrainingRun
# Research runs before explore ends, if no run stopped it first. Rescue runs do not count.
EXPLORE_RUN_LIMIT = 5
@@ -91,3 +96,27 @@ def schedule_runs_for(pipeline_ids: Iterable[UUID], team_id: int) -> dict[UUID,
)
)
return runs
+
+
+def awaiting_first_champion() -> Q:
+ """Filter pipelines to the ``bootstrapping`` ones that wait for a scheduled run to find a first champion.
+
+ A first champion that fails its fit or its scorability check rolls back, and the pipeline goes
+ back to ``bootstrapping`` with a completed run. Such a pipeline has no champion and no pending
+ or running training run. Without scheduled research it stays in this state.
+ """
+ run_of_pipeline = AutoresearchTrainingRun.all_teams.filter(team_id=OuterRef("team_id"), pipeline_id=OuterRef("pk"))
+ return Q(
+ Exists(run_of_pipeline.filter(status=AutoresearchTrainingRun.Status.COMPLETED)),
+ ~Exists(
+ run_of_pipeline.filter(
+ status__in=[AutoresearchTrainingRun.Status.PENDING, AutoresearchTrainingRun.Status.RUNNING]
+ )
+ ),
+ ~Exists(
+ AutoresearchModel.all_teams.filter(
+ team_id=OuterRef("team_id"), pipeline_id=OuterRef("pk"), role=AutoresearchModel.Role.CHAMPION
+ )
+ ),
+ status=AutoresearchPipeline.Status.BOOTSTRAPPING,
+ )
diff --git a/products/autoresearch/backend/training/test_training.py b/products/autoresearch/backend/training/test_training.py
index 12fa980d6d6f..e9e396850072 100644
--- a/products/autoresearch/backend/training/test_training.py
+++ b/products/autoresearch/backend/training/test_training.py
@@ -97,6 +97,9 @@ class TestBuildAgentDescription(TeamScopedTestMixin, BaseTest):
# Step 3 is the sandbox fit/eval loop.
assert "fit and evaluate" in prompt
assert "roc_auc_score" in prompt
+ # A first champion that breaks either fit limit rolls back.
+ assert "60 s interactive query limit" in prompt
+ assert "must stay under 10 MiB" in prompt
@parameterized.expand(
[
diff --git a/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx b/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx
index 7da94448591a..b1400e0b9c88 100644
--- a/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx
+++ b/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx
@@ -40,6 +40,7 @@ const pipeline = {
effective_agent_reasoning_effort: 'medium',
training_phase: 'maintain',
next_research_at: '2026-11-12T00:00:00Z',
+ not_promoted_reason: null,
success_auc: null,
plateau_iterations: 5,
output_person_property: 'p_file_shared_7d',
@@ -437,6 +438,27 @@ export const AgentResearchWithoutReaderNotes: Story = {
},
}
+// The first champion failed its fit, so the model went back to bootstrapping with no champion.
+export const FirstChampionRolledBack: Story = {
+ decorators: [
+ mswDecorator({
+ get: {
+ [`/api/projects/:team_id/autoresearch/${PIPELINE_ID}/`]: {
+ ...pipeline,
+ status: 'bootstrapping',
+ training_phase: 'maintain',
+ champion_holdout_auc: null,
+ champion_realized_auc: null,
+ not_promoted_reason:
+ 'The champion fit failed: model.pkl is 14830476 bytes, over the 10485760-byte cap',
+ },
+ [`/api/projects/:team_id/autoresearch/${PIPELINE_ID}/models/`]: toPaginatedResponse([challenger]),
+ },
+ }),
+ ],
+ parameters: { testOptions: { viewportWidths: ['wide'] } },
+}
+
export const Accuracy: Story = {
parameters: {
pageUrl: `${urls.autoresearchPipeline(PIPELINE_ID)}?tab=accuracy`,
diff --git a/products/autoresearch/frontend/AutoresearchPipelineScene.tsx b/products/autoresearch/frontend/AutoresearchPipelineScene.tsx
index d3787736e0e5..aecfd6aa70e2 100644
--- a/products/autoresearch/frontend/AutoresearchPipelineScene.tsx
+++ b/products/autoresearch/frontend/AutoresearchPipelineScene.tsx
@@ -25,6 +25,7 @@ import { QuillScoreNowButton } from './pipeline/QuillScoreNowButton'
import { ScoreNowButton } from './pipeline/ScoreNowButton'
import { SetupTab } from './pipeline/SetupTab'
import { pipelineQuestion, pipelineSubtitle } from './pipelineQuestion'
+import { notLiveExplanation } from './trainingScheduleLogic'
export const scene: SceneExport = {
component: AutoresearchPipelineScene,
@@ -46,6 +47,7 @@ export function AutoresearchPipelineScene(): JSX.Element {
]
const heading = pipeline ? pipelineQuestion(pipeline) : pipelineLoading ? '' : 'Model'
+ const notLive = pipeline ? notLiveExplanation(pipeline) : null
if (!isEnabled) {
return <NotFound object="Autoresearch" caption="This feature is not enabled for your project." />
@@ -85,6 +87,14 @@ export function AutoresearchPipelineScene(): JSX.Element {
<>
<PipelineSummary />
{lifecycleSteps && <LifecycleStrip steps={lifecycleSteps} />}
+ {notLive && (
+ <LemonBanner type="warning">
+ <div>{notLive}</div>
+ <div className="text-xs font-mono mt-1 break-words">
+ Reason: {pipeline?.not_promoted_reason}
+ </div>
+ </LemonBanner>
+ )}
<LemonTabs
activeKey={activeTab}
onChange={(key) => setActiveTab(key as AutoresearchPipelineTab)}
diff --git a/products/autoresearch/frontend/generated/api.schemas.ts b/products/autoresearch/frontend/generated/api.schemas.ts
index 229f7a69a151..eda072ac86db 100644
--- a/products/autoresearch/frontend/generated/api.schemas.ts
+++ b/products/autoresearch/frontend/generated/api.schemas.ts
@@ -290,6 +290,11 @@ export interface AutoresearchPipelineApi {
* @nullable
*/
readonly next_research_at: string | null
+ /**
+ * Why the best model of the newest completed training run did not go live, for example a failed fit or a failed scorability check. Set only while the pipeline is bootstrapping with no champion.
+ * @nullable
+ */
+ readonly not_promoted_reason: string | null
/**
* Target AUC threshold. Training stops early if this score is reached.
* @nullable
diff --git a/products/autoresearch/frontend/trainingScheduleLogic.ts b/products/autoresearch/frontend/trainingScheduleLogic.ts
index a1ce313dc358..710454142fb2 100644
--- a/products/autoresearch/frontend/trainingScheduleLogic.ts
+++ b/products/autoresearch/frontend/trainingScheduleLogic.ts
@@ -41,6 +41,18 @@ export function trainingScheduleStatus(pipeline: AutoresearchPipelineApi, now: d
return `Next research run on ${next.format('MMM D')}.`
}
+/** Why the model has no live version after its best model failed the final check, or null. */
+export function notLiveExplanation(pipeline: AutoresearchPipelineApi): string | null {
+ if (!pipeline.not_promoted_reason || pipeline.live_training_run) {
+ return null
+ }
+ const next =
+ pipeline.research_interval_days === null
+ ? 'Start a training run to try again.'
+ : 'The next research run tries again.'
+ return `The best model from the last training run couldn't go live, so nothing is scored yet. ${next}`
+}
+
export interface trainingScheduleLogicValues {
currentTeamId: number | null
pipeline: AutoresearchPipelineApi | null
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 9e1d20ca9067..93336ad42b5e 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -15884,6 +15884,11 @@ export namespace Schemas {
* @nullable
*/
readonly next_research_at: string | null;
+ /**
+ * Why the best model of the newest completed training run did not go live, for example a failed fit or a failed scorability check. Set only while the pipeline is bootstrapping with no champion.
+ * @nullable
+ */
+ readonly not_promoted_reason: string | null;
/**
* Target AUC threshold. Training stops early if this score is reached.
* @nullable