:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(autoresearch): keep training after a first champion rolls back (#126646)
posthog[bot]55223ce8a410
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex 69703c2f78c6..f6fa324f981c 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -6718,6 +6718,10 @@ snapshots:hash: v1.k794b7964.073ed0d4859386827dd0e45de3cf5eb857867e74e1b44f2fa4ede4a108742cca.hVQJteiYVN-fx70VzB55DOgKQVvHoDNtPEqzxqho5Koproducts-autoresearch-model-detail-scene--coverage-with-failed-runs--wide--light:hash: v1.k794b7964.e50fd4ee2847fe0504c0aebc10c7eedb34d1ed4d2eff05b68848bc6f45b9f0d9.3zl510wgOjWMfOqmOycvoGw-M4D9VGx-OBni0PdNeok+ products-autoresearch-model-detail-scene--first-champion-rolled-back--wide--dark:+ hash: v1.k794b7964.bffc646c9c852bd652ba735564e0763d355979163c210368ea23e05554ae8fca.Be4kPj6BVaiNFWsNnZVL5rKyjBfc0WcEs_SVE8hbBx0+ products-autoresearch-model-detail-scene--first-champion-rolled-back--wide--light:+ hash: v1.k794b7964.86e7e37b4a768fb1d0d5b3e126581d16314fe29d9bd24e5d25ddaa505276ed0f.avvhk9BUjvIwcJ6q47-uiSktBYBomO64C9wF23tR2coproducts-autoresearch-model-detail-scene--narrow--narrow--dark:hash: v1.k794b7964.8d62dc0e26c677229f657685e9a91ff35456e8ea8a947afa92782e2b3359eb4e.YtQHUt83vFWkDF3SKK0iSQnyLikqzTYog0_46H_jOEgproducts-autoresearch-model-detail-scene--narrow--narrow--light:diff --git a/products/autoresearch/backend/facade/api.py b/products/autoresearch/backend/facade/api.pyindex 3fd341095c1e..1ec658bd2351 100644--- a/products/autoresearch/backend/facade/api.py+++ b/products/autoresearch/backend/facade/api.py@@ -204,6 +204,7 @@ class _PipelineActivity:experiment_count: intlive_training_run: LiveTrainingRun | Noneschedule_runs: list[ScheduleRun]+ not_promoted_reason: str | Nonedef _best_so_far_trend(scores: Iterable[tuple[int, float | None]]) -> list[TrainingTrendPoint]:@@ -311,6 +312,29 @@ def _first_checks_expected(team_id: int, champions: dict[UUID, AutoresearchModelreturn first_checks+def _not_promoted_reasons(team_id: int, pipeline_ids: list[UUID]) -> dict[UUID, str]:+ """Why the best model of each pipeline's newest completed training run did not go live, when it did not."""+ if not pipeline_ids:+ return {}+ newest_completed_runs = (+ AutoresearchTrainingRun.objects.for_team(team_id)+ .filter(pipeline_id__in=pipeline_ids, status=AutoresearchTrainingRun.Status.COMPLETED)+ .order_by("pipeline_id", "-created_at")+ .distinct("pipeline_id")+ .values_list("id", flat=True)+ )+ reasons: dict[UUID, str] = {}+ for pipeline_id, reason in (+ AutoresearchModel.objects.for_team(team_id)+ .filter(source_training_run_id__in=list(newest_completed_runs), metrics__has_key="not_promoted_reason")+ .order_by("created_at")+ .values_list("pipeline_id", "metrics__not_promoted_reason")+ ):+ if isinstance(reason, str) and reason:+ reasons[pipeline_id] = reason+ return reasons++def _pipeline_activity(team_id: int, pipeline_ids: list[UUID], champions: dict[UUID, AutoresearchModel]) -> dict[UUID, _PipelineActivity]:@@ -360,6 +384,7 @@ def _pipeline_activity(likely_counts = _champion_likely_counts(team_id, champions)first_checks = _first_checks_expected(team_id, champions)schedule_runs = schedule_runs_for(pipeline_ids, team_id)+ not_promoted_reasons = _not_promoted_reasons(team_id, [pid for pid in pipeline_ids if pid not in champions])return {pipeline_id: _PipelineActivity(champion_realized_auc_trend=[@@ -379,6 +404,7 @@ def _pipeline_activity(else None),schedule_runs=schedule_runs.get(pipeline_id, []),+ not_promoted_reason=not_promoted_reasons.get(pipeline_id),)for pipeline_id in pipeline_ids}@@ -421,6 +447,11 @@ def _pipeline_to_contract(effective_agent_reasoning_effort=agent_config.reasoning_effort,training_phase=schedule.phase,next_research_at=schedule.next_research_at,+ not_promoted_reason=(+ activity.not_promoted_reason+ if row.status == AutoresearchPipeline.Status.BOOTSTRAPPING and champion is None+ else None+ ),success_auc=row.success_auc,plateau_iterations=row.plateau_iterations,output_person_property=row.output_person_property,diff --git a/products/autoresearch/backend/facade/contracts.py b/products/autoresearch/backend/facade/contracts.pyindex 415835747144..33ccb8f7f535 100644--- a/products/autoresearch/backend/facade/contracts.py+++ b/products/autoresearch/backend/facade/contracts.py@@ -159,6 +159,7 @@ class Pipeline:effective_agent_reasoning_effort: str | Nonetraining_phase: strnext_research_at: datetime | None+ not_promoted_reason: str | Nonesuccess_auc: float | Noneplateau_iterations: intoutput_person_property: strdiff --git a/products/autoresearch/backend/presentation/views/serializers.py b/products/autoresearch/backend/presentation/views/serializers.pyindex 895b77aad2fd..bdb38776af14 100644--- a/products/autoresearch/backend/presentation/views/serializers.py+++ b/products/autoresearch/backend/presentation/views/serializers.py@@ -729,6 +729,14 @@ class AutoresearchPipelineSerializer(DataclassSerializer):"Null when research_interval_days is null."),)+ not_promoted_reason = serializers.CharField(+ read_only=True,+ allow_null=True,+ help_text=(+ "Why the best model of the newest completed training run did not go live, for example a failed fit "+ "or a failed scorability check. Set only while the pipeline is bootstrapping with no champion."+ ),+ )success_auc = serializers.FloatField(required=False,allow_null=True,@@ -865,6 +873,7 @@ class AutoresearchPipelineSerializer(DataclassSerializer):"effective_agent_reasoning_effort","training_phase","next_research_at",+ "not_promoted_reason","success_auc","plateau_iterations","output_person_property",diff --git a/products/autoresearch/backend/query.py b/products/autoresearch/backend/query.pyindex d910552db713..b28eefa3b12f 100644--- a/products/autoresearch/backend/query.py+++ b/products/autoresearch/backend/query.py@@ -1,13 +1,13 @@import randomfrom collections.abc import Callablefrom time import perf_counter, sleep-from typing import Any, TypeVar+from typing import Any, TypeVar, castimport structlogfrom posthog.schema import CacheMissResponse, HogQLQuery, QueryStatusResponse-from posthog.hogql.constants import LimitContext+from posthog.hogql.constants import HogQLGlobalSettings, LimitContextfrom posthog.hogql.query_stats import query_stats_scopefrom posthog.clickhouse.workload import Workload@@ -40,6 +40,7 @@ class QueryContext:# A query inside a web request keeps the 60 s interactive limit, because a longer query# would outlive the request.INTERACTIVE_QUERY = QueryContext(limit_context=LimitContext.QUERY, workload=Workload.DEFAULT)+INTERACTIVE_QUERY_TIME_LIMIT_S = cast(int, HogQLGlobalSettings().max_execution_time)# Scoring and online validation run in a Temporal activity that nobody waits on, so they# get the increased batch limit and run on the offline workload.BATCH_QUERY = QueryContext(limit_context=LimitContext.QUERY_ASYNC, workload=Workload.OFFLINE)diff --git a/products/autoresearch/backend/temporal/AGENTS.md b/products/autoresearch/backend/temporal/AGENTS.mdindex de8d13438016..c079b3448111 100644--- a/products/autoresearch/backend/temporal/AGENTS.md+++ b/products/autoresearch/backend/temporal/AGENTS.md@@ -63,6 +63,8 @@ Kickoff then follows the research schedule in `../training/schedule.py`, which iIn explore, it starts a run (`explore`) until a completed run does not promote its best model, or until `EXPLORE_RUN_LIMIT` research runs have run. Rescue runs do not count.In maintain, it starts a run (`scheduled`) once `research_interval_days` calendar days have passed since the latest training run of any kind, and reports `not_due` before that. A null interval reports `manual_only`.Each run takes the pipeline's `iterations_per_run`, which `run_training()` lowers to the project's `max_iterations_per_run`. Kickoff does not read or spend `iteration_budget_remaining`.+Kickoff accepts a `running` pipeline, and a `bootstrapping` pipeline whose first champion rolled back (`awaiting_first_champion()` in `../training/schedule.py`: no champion, no pending or running training run, and at least one completed run). Otherwise it reports `not_eligible`.+The daily sweep lists such a pipeline last, with `ActivePipeline.kickoff_only`. The coordinator then only kicks off its training, because it has no champion to score or validate. This is behind `workflow.patched("autoresearch-kickoff-only-2026-10")`. A history without the marker also starts scoring and validation for it, and those activities skip a pipeline that is not live. The catch-up sweep does not list it.When no run is due and `find_unscorable_champion()` finds that the champion cannot score, kickoff launches a rescue run of `RESCUE_ITERATION_BUDGET` iterations, at most once in each window of `UNSCORABLE_AFTER_FAILED_DAYS` UTC days, and reports `rescue_unscorable`.Before it launches, kickoff checks the project's daily training cap (`check_training_run_cap()` in `../limits.py`), the same check `start_training` makes, and reports `team_daily_cap` when the project has no slot left. The cap counts every sandbox run of the team in the last 24 hours, so it blocks rescue runs too. The check takes a transaction-scoped advisory lock on the team inside the pipeline row lock, so two pipelines of one team cannot both take the last slot.Each run records why it started in `kickoff_reason`. The coordinator body never branches on the kickoff result, because that would break replay.diff --git a/products/autoresearch/backend/temporal/test_workflows.py b/products/autoresearch/backend/temporal/test_workflows.pyindex 4425a0b36c54..610b0c43502e 100644--- a/products/autoresearch/backend/temporal/test_workflows.py+++ b/products/autoresearch/backend/temporal/test_workflows.py@@ -776,6 +776,57 @@ class TestCoordinatorActivities(TeamScopedTestMixin, BaseTest):assert not result.kicked_offmock_run_training.assert_not_called()+ @parameterized.expand(+ [+ ("first_champion_rolled_back", [AutoresearchTrainingRun.Status.COMPLETED], True, "scheduled"),+ (+ "training_run_live",+ [AutoresearchTrainingRun.Status.COMPLETED, AutoresearchTrainingRun.Status.RUNNING],+ False,+ "not_eligible",+ ),+ ("no_completed_run", [AutoresearchTrainingRun.Status.FAILED], False, "not_eligible"),+ ]+ )+ @patch("products.autoresearch.backend.temporal.workflows.run_training")+ def test_bootstrapping_pipeline_trains_again_after_its_first_champion_rolled_back(+ self,+ _name: str,+ run_statuses: list[str],+ expected_listed: bool,+ expected_reason: str,+ mock_run_training: MagicMock,+ ) -> None:+ with time_machine.travel(datetime(2026, 8, 1, tzinfo=UTC), tick=False):+ pipeline = self._create_pipeline(status=AutoresearchPipeline.Status.BOOTSTRAPPING, research_interval_days=7)+ for run_status in run_statuses:+ with time_machine.travel(datetime(2026, 9, 1, 2, tzinfo=UTC), tick=False):+ run = AutoresearchTrainingRun.objects.create(+ pipeline=pipeline, status=run_status, iteration_budget=3, summary={"champion_promoted": False}+ )+ AutoresearchModel.objects.create(+ pipeline=pipeline,+ source_training_run=run,+ role=AutoresearchModel.Role.CHALLENGER,+ model_recipe={"stub": True},+ recipe_hash="rolled-back",+ metrics={"not_promoted_reason": "The champion fit failed"},+ )++ daily = activity_load_active_pipelines(LoadActivePipelinesInput())+ catch_up = activity_load_active_pipelines(LoadActivePipelinesInput(catch_up_date="2026-09-11"))+ result = activity_kickoff_training(KickoffTrainingInput(pipeline_id=str(pipeline.id), team_id=self.team.id))++ listed = [p for p in daily.pipelines if p.pipeline_id == str(pipeline.id)]+ assert listed == (+ [ActivePipeline(pipeline_id=str(pipeline.id), team_id=self.team.id, score_due=True, kickoff_only=True)]+ if expected_listed+ else []+ )+ assert str(pipeline.id) not in [p.pipeline_id for p in catch_up.pipelines]+ assert result.reason == expected_reason+ assert mock_run_training.called is expected_listed+@parameterized.expand([("transient_failure_raises", RuntimeError("boom")), ("refused_launch", ValueError("left"))])@patch("products.autoresearch.backend.temporal.workflows.run_training")def test_failed_launch(self, _name: str, error: Exception, mock_run_training: MagicMock) -> None:@@ -924,11 +975,15 @@ async def test_coordinator_fails_the_runs_of_a_failed_inference_child(catch_up:async def load(inp: LoadActivePipelinesInput) -> LoadActivePipelinesResult:load_inputs.append(inp)return LoadActivePipelinesResult(- pipelines=[ActivePipeline(pipeline_id=pid, team_id=1, score_due=True) for pid in ("lost", "scored")]+ pipelines=[+ *[ActivePipeline(pipeline_id=pid, team_id=1, score_due=True) for pid in ("lost", "scored")],+ ActivePipeline(pipeline_id="waiting", team_id=1, score_due=True, kickoff_only=True),+ ])@activity.defn(name="autoresearch-inference.run_inference")async def infer(inp: RunInferenceInput) -> RunInferenceResult:+ steps.append(f"inference-{inp.pipeline_id}")if inp.pipeline_id == "lost":raise ApplicationError("worker lost", non_retryable=True)return RunInferenceResult(run_id=f"run-{inp.pipeline_id}", rows_scored=1, status="completed")@@ -944,12 +999,12 @@ async def test_coordinator_fails_the_runs_of_a_failed_inference_child(catch_up:@activity.defn(name="autoresearch-validation.run_validation")async def validate(inp: RunValidationInput) -> RunValidationResult:- steps.append("validation")+ steps.append(f"validation-{inp.pipeline_id}")return RunValidationResult(dates_validated=1, total_rows=1, status="completed")@activity.defn(name="autoresearch-coordinator.kickoff_training")async def kickoff(inp: KickoffTrainingInput) -> KickoffTrainingResult:- steps.append("kickoff")+ steps.append(f"kickoff-{inp.pipeline_id}")return KickoffTrainingResult(kicked_off=False, reason="not_due")@activity.defn(name="autoresearch-coordinator.run_shadow_scoring")@@ -978,8 +1033,10 @@ async def test_coordinator_fails_the_runs_of_a_failed_inference_child(catch_up:]# Cleanup is scoped to the failed child's execution, so a catch-up child's live row stays open.assert failed_runs[0].workflow_run_id- assert (result.pipelines_processed, result.pipelines_errored) == (1, 1)+ assert (result.pipelines_processed, result.pipelines_errored) == (2, 1)assert "shadow-scored" in steps# The daily sweep already kicked off training and validated, so a catch-up only scores.- assert ("kickoff" in steps, "validation" in steps) == (not catch_up, not catch_up)+ assert ("kickoff-scored" in steps, "validation-scored" in steps) == (not catch_up, not catch_up)+ # A pipeline with no champion is never scored or validated, and only the daily sweep kicks off its training.+ assert [step for step in steps if step.endswith("-waiting")] == ([] if catch_up else ["kickoff-waiting"])assert [step for step in steps if step.startswith("mark-")] == ([] if catch_up else ["mark-running", "mark-done"])diff --git a/products/autoresearch/backend/temporal/workflows.py b/products/autoresearch/backend/temporal/workflows.pyindex 9a7bfbadfe72..3033a4ac36d4 100644--- a/products/autoresearch/backend/temporal/workflows.py+++ b/products/autoresearch/backend/temporal/workflows.py@@ -27,7 +27,7 @@ with workflow.unsafe.imports_passed_through():import asyncio as _asynciofrom django.db import transaction- from django.db.models import QuerySet+ from django.db.models import Q, QuerySetfrom django.utils import timezone as django_timezonefrom posthog.models.team.team import Team@@ -62,7 +62,11 @@ with workflow.unsafe.imports_passed_through():)from products.autoresearch.backend.query import BATCH_QUERY, SCORING_QUERYfrom products.autoresearch.backend.training.runner import run_training- from products.autoresearch.backend.training.schedule import research_schedule, schedule_runs_for+ from products.autoresearch.backend.training.schedule import (+ awaiting_first_champion,+ research_schedule,+ schedule_runs_for,+ )from products.autoresearch.backend.training.stale import fail_stale_training_runsfrom products.tasks.backend.facade.access import get_desktop_access_decisionfrom products.tasks.backend.facade.usage import task_run_usage_limited@@ -671,6 +675,8 @@ class ActivePipeline:team_id: int# Validation runs on every sweep; scoring and training kickoff only on a cadence day.score_due: bool+ # A bootstrapping pipeline whose first champion rolled back. The sweep only kicks off its training.+ kickoff_only: bool = False@frozen@@ -770,6 +776,10 @@ def activity_load_active_pipelines(inp: LoadActivePipelinesInput) -> LoadActivePA catch-up lists nothing while the daily sweep of that date still runs (``activity_mark_daily_sweep``),including its kickoff, validation and shadow phases, so the two sweeps never exceed one boundtogether. The next catch-up tick tries again.++ The daily sweep also lists each ``bootstrapping`` pipeline whose first champion rolled back+ (``awaiting_first_champion()``), last and with ``kickoff_only``, so its research schedule+ continues. It has no champion, so the sweep does not score or validate it."""fail_stale_inference_runs()fail_stale_training_runs()@@ -780,9 +790,12 @@ def activity_load_active_pipelines(inp: LoadActivePipelinesInput) -> LoadActivePyesterday = today - timedelta(days=1)# unscoped() because the coordinator's job is exactly to sweep every team. The# pipelines it returns carry their team so the downstream activities can scope.+ sweep_filter = Q(status__in=_LIVE_STATUSES)+ if inp.catch_up_date is None:+ sweep_filter |= awaiting_first_champion()candidates = (AutoresearchPipeline.objects.unscoped()- .filter(status__in=_LIVE_STATUSES)+ .filter(sweep_filter)# A null is_active counts as deactivated, as ActiveOrganizationPermission reads it..filter(team__organization__is_active=True).exclude(team__organization__is_pending_deletion=True)@@ -801,6 +814,18 @@ def activity_load_active_pipelines(inp: LoadActivePipelinesInput) -> LoadActivePcreator, team_id=pipeline.team_id, organization_id=str(pipeline.team.organization_id)):continue+ if pipeline.status == AutoresearchPipeline.Status.BOOTSTRAPPING:+ # score_due stays True for a coordinator that started before the kickoff-only patch: it+ # kicks off training, and the scoring and validation activities skip a pipeline that is not live.+ pipelines.append(+ (+ True,+ ActivePipeline(+ pipeline_id=str(pipeline.pk), team_id=pipeline.team_id, score_due=True, kickoff_only=True+ ),+ )+ )+ continue# By calendar day, because a run finishes some time after the 02:00 UTC tick and an# elapsed-time comparison would skip the next day's tick.last_scored = pipeline.last_scored_at.date() if pipeline.last_scored_at else None@@ -869,6 +894,9 @@ def _pause_orphaned_pipeline(pipeline: AutoresearchPipeline, *, reason: str) ->def activity_kickoff_training(inp: KickoffTrainingInput) -> KickoffTrainingResult:"""Start a scheduled agent training run, if the pipeline's research schedule says one is due.+ The pipeline must be ``running``, or ``bootstrapping`` after its first champion rolled back+ (``awaiting_first_champion()``).+The run row and its task binding commit together under the pipeline row lock, as in thefacade's ``start_training``, so a manual start cannot interleave. A launch failure raises,so the activity retry applies.@@ -886,7 +914,9 @@ def activity_kickoff_training(inp: KickoffTrainingInput) -> KickoffTrainingResulwith team_scope(inp.team_id), transaction.atomic():pipeline = AutoresearchPipeline.objects.select_for_update().get(pk=inp.pipeline_id)- if pipeline.status != AutoresearchPipeline.Status.RUNNING:+ if pipeline.status != AutoresearchPipeline.Status.RUNNING and not (+ AutoresearchPipeline.objects.filter(pk=pipeline.pk).filter(awaiting_first_champion()).exists()+ ):return KickoffTrainingResult(kicked_off=False, reason="not_eligible")if AutoresearchTrainingRun.objects.filter(@@ -1028,6 +1058,8 @@ _BOUNDED_SWEEP_PATCH = "autoresearch-bounded-sweep-2026-10"_CLOSE_LOST_RUNS_PATCH = "autoresearch-close-lost-runs-2026-10"# Histories that started before this marker do not tell a catch-up that the daily sweep is running._DAILY_SWEEP_MARKER_PATCH = "autoresearch-daily-sweep-marker-2026-10"+# Histories that started before this marker also start scoring and validation for a kickoff-only pipeline.+_KICKOFF_ONLY_PATCH = "autoresearch-kickoff-only-2026-10"@workflow.defn(name="autoresearch-coordinator")@@ -1158,12 +1190,19 @@ class AutoresearchCoordinatorWorkflow(PostHogWorkflow):) -> _PipelineSweep:"""On a cadence day score, then kick off training. Then validate. One step at a time, so a pipeline holds one slot.- A catch-up sweep only scores.+ A catch-up sweep only scores. A kickoff-only pipeline only kicks off training."""pipeline_id = pipeline.pipeline_idinference_result: InferenceWorkflowResult | BaseException | None = Nonevalidation_result: ValidationWorkflowResult | BaseException | None = Nonekickoff_result: KickoffTrainingResult | BaseException | None = None+ if workflow.patched(_KICKOFF_ONLY_PATCH) and pipeline.kickoff_only:+ if not catch_up:+ kickoff_result = await _settle(self._kickoff(pipeline))+ return _PipelineSweep(+ outcome=self._evaluate(pipeline_id, inference_result, validation_result, kickoff_result),+ inference=None,+ )if pipeline.score_due:inference_result = await _settle(workflow.execute_child_workflow(diff --git a/products/autoresearch/backend/tests/test_api.py b/products/autoresearch/backend/tests/test_api.pyindex 8150ad1965c3..b79f9ccf12d2 100644--- a/products/autoresearch/backend/tests/test_api.py+++ b/products/autoresearch/backend/tests/test_api.py@@ -428,6 +428,36 @@ class TestAutoresearchPipelineAPI(TeamScopedTestMixin, APIBaseTest):assert resp.json()["id"] == str(pipeline.id)assert resp.json()["target_definition"] == {"type": "event"}+ @parameterized.expand(+ [+ ("first_champion_rolled_back", AutoresearchPipeline.Status.BOOTSTRAPPING, False, "The champion fit failed"),+ ("champion_live", AutoresearchPipeline.Status.RUNNING, True, None),+ ]+ )+ def test_retrieve_explains_why_no_model_is_live(+ self, _name: str, pipeline_status: str, has_champion: bool, expected_reason: str | None+ ) -> None:+ pipeline = self._make_pipeline(status=pipeline_status)+ run = AutoresearchTrainingRun.objects.create(+ pipeline=pipeline, status=AutoresearchTrainingRun.Status.COMPLETED, iteration_budget=5+ )+ AutoresearchModel.objects.create(+ pipeline=pipeline,+ source_training_run=run,+ role=AutoresearchModel.Role.CHALLENGER,+ model_recipe={"stub": True},+ recipe_hash="rolled-back",+ metrics={"not_promoted_reason": "The champion fit failed"},+ )+ if has_champion:+ AutoresearchModel.objects.create(+ pipeline=pipeline, role=AutoresearchModel.Role.CHAMPION, model_recipe={"stub": True}, recipe_hash="live"+ )++ resp = self.client.get(f"{self.base_url}/{pipeline.id}/")++ assert resp.json()["not_promoted_reason"] == expected_reason+def test_other_team_cannot_access_pipeline(self):other_org = Organization.objects.create(name="Other Org")other_team = Team.objects.create(organization=other_org, name="Other Team")diff --git a/products/autoresearch/backend/training/AGENTS.md b/products/autoresearch/backend/training/AGENTS.mdindex b19126f50496..3e15a18aaf2f 100644--- a/products/autoresearch/backend/training/AGENTS.md+++ b/products/autoresearch/backend/training/AGENTS.md@@ -17,6 +17,7 @@ The other half is `../inference/`, which consumes what this package produces and`build_agent_description()` assembles the agent's brief — the target, the horizon, the population, and the contract for the bundle it must author.When the `autoresearch-report-notebook` flag is on for the launching user, the token also holds `REPORT_NOTEBOOK_MCP_SCOPES` and the brief adds two Finalize steps: before complete, the agent creates one report notebook and passes its `short_id` to complete. After complete, it adds the SQL and chart cells, because the run's model row (and so its `model_explanation`) exists only from then on. `report.md` stays required either way.The brief's worked `features.sql` reads events through a pre-filtered subquery (the event names the features use, the anchor persons, and the anchors' widest window) before it joins, because ClickHouse builds the hash table from the right side of a join and a direct events join reads the whole team's events.+ The brief states the two limits of the fit at completion, from the constants: the fitted `model.pkl` must stay under `MAX_ARTIFACT_BYTES` (the readback cap of `fit_champion_model()`), and `features.sql` must finish inside `INTERACTIVE_QUERY_TIME_LIMIT_S` (`../query.py`), because the fit runs it under `INTERACTIVE_QUERY`. A model that breaks either limit rolls back.The brief also states the inference cutoff: the start of the prediction date in UTC (`ScoringWindow` in `../inference/scoring.py`), not `now()`.The brief's cost guidance is advice only. Cost does not enter champion selection, so the brief does not tell the agent to trade AUC for a cheaper query. A rescue run is the one exception, see the unscorable champion below.For a person column that events do not carry, the brief recommends `raw_persons` in a subquery filtered with `id IN (SELECT person_id FROM {anchors})`, never `LEFT JOIN persons`, because the persons table dedupes every person of the team before any filter applies.@@ -53,7 +54,7 @@ The other half is `../inference/`, which consumes what this package produces andA promoted bundle-backed model is fitted. A bundle-backed challenger is fitted only when it enters the shadow set (see `shadow_set.py`). The fit runs after commit and never changes the completion result.A successful challenger fit sets `metrics.model_fitted`, and a failed one only logs. A challenger without `model_fitted` stays out of the shadow set.After a champion fit, `check_scorability()` runs the bundle's `features.sql` against today's inference anchors under `BATCH_QUERY`. A passing check records `scorability_elapsed_s` and `scorability_rows_read` in the model metrics and sets `model_fitted`.- A failed fit, a failed check, or a check above `SCORABILITY_TIME_BUDGET_S` (half the batch query limit) rolls the promotion back under the pipeline lock: the candidate becomes a challenger with `not_promoted_reason` in its metrics, and the champion it archived comes back. A first champion that rolls back puts the pipeline back in the status it had before promotion. The rolled-back candidate has no `model_fitted`, so it never enters the shadow set.+ A failed fit, a failed check, or a check above `SCORABILITY_TIME_BUDGET_S` (half the batch query limit) rolls the promotion back under the pipeline lock: the candidate becomes a challenger with `not_promoted_reason` in its metrics, and the champion it archived comes back. A first champion that rolls back puts the pipeline back in the status it had before promotion. The run summary then records `champion_promoted: false`, so the research schedule ends explore for it. A `bootstrapping` pipeline in this state (`awaiting_first_champion()` in `schedule.py`) keeps its research schedule, see `../temporal/AGENTS.md`, and the pipeline API returns the candidate's reason as `not_promoted_reason`. The rolled-back candidate has no `model_fitted`, so it never enters the shadow set.The rollback changes nothing when the candidate is no longer the champion. A scoring run that started with the candidate fails in `_require_still_champion()` before it emits.A recipe-only champion is not checked.A champion whose scheduled scoring runs failed with a repeatable `failure_kind` on the last two prediction dates, with no success after the first of them, is unscorable (`find_unscorable_champion()` in `../inference/failures.py`). Any candidate replaces it whatever the margin, with `promotion_reason` `replaced_unscorable` in the model metrics, and the fit and the scorability check still roll back a candidate that cannot score either. The brief tells the agent the failure kind and the onset date. A rescue run (see `../temporal/AGENTS.md`) also tells the agent to prefer a cheap model that scores over a higher holdout score.diff --git a/products/autoresearch/backend/training/runner.py b/products/autoresearch/backend/training/runner.pyindex 6d3cff1eddf6..91fb420af0c0 100644--- a/products/autoresearch/backend/training/runner.py+++ b/products/autoresearch/backend/training/runner.py@@ -48,7 +48,9 @@ from products.autoresearch.backend.models import (AutoresearchSuggestion,AutoresearchTrainingRun,)+from products.autoresearch.backend.query import INTERACTIVE_QUERY_TIME_LIMIT_Sfrom products.autoresearch.backend.training.agent_model import agent_run_config_for+from products.autoresearch.backend.training.artifacts import MAX_ARTIFACT_BYTESfrom products.autoresearch.backend.training.explanation import MAX_TOP_FEATURESfrom products.autoresearch.backend.training.realized_context import (RealizedContext,@@ -476,6 +478,16 @@ def build_agent_description(is not promoted and the previous champion keeps serving.- Cost does not change which iteration wins, so keep each hypothesis cheap from the start.+ **Fit limits at completion.** When you complete the run, the backend fits the winning bundle+ once more. Two limits apply to that fit, and a model that fails either one is not promoted:+ - `features.sql` runs against the full training population, and must finish inside the+ {INTERACTIVE_QUERY_TIME_LIMIT_S} s interactive query limit. Check `elapsed_s` in the materialize+ response, and keep the query well under that limit. Use the pre-filtered, rolled-up shapes above.+ - The fitted `model.pkl` must stay under {MAX_ARTIFACT_BYTES // (1024 * 1024)} MiB+ ({MAX_ARTIFACT_BYTES} bytes). Tree ensembles grow fast: bound `n_estimators` (for example 200 or+ fewer) and `max_depth` (for example 8 or less), or set `min_samples_leaf`, and check the pickle size+ in your sandbox before you upload.+### Step 3 — Materialize features, then fit and evaluate (in your sandbox)**Your sandbox is already equipped — do NOT `pip install` anything.** numpy, pandas,diff --git a/products/autoresearch/backend/training/schedule.py b/products/autoresearch/backend/training/schedule.pyindex 4a5e4caae18a..0753821def29 100644--- a/products/autoresearch/backend/training/schedule.py+++ b/products/autoresearch/backend/training/schedule.py@@ -4,6 +4,9 @@ Research has two phases, derived from the pipeline's training runs and never stoExplore starts a run on each sweep until a completed run does not promote its best model,or until ``EXPLORE_RUN_LIMIT`` research runs have run. Maintain starts one run``research_interval_days`` after the latest training run of any kind, so a manual run resets the clock.++A ``bootstrapping`` pipeline whose first champion rolled back follows the same schedule+(``awaiting_first_champion()``)."""from collections.abc import Iterable@@ -11,9 +14,11 @@ from datetime import UTC, datetime, time, timedeltafrom typing import Literalfrom uuid import UUID+from django.db.models import Exists, OuterRef, Q+from posthog.dataclasses import frozen-from ..models import AutoresearchTrainingRun+from ..models import AutoresearchModel, AutoresearchPipeline, AutoresearchTrainingRun# Research runs before explore ends, if no run stopped it first. Rescue runs do not count.EXPLORE_RUN_LIMIT = 5@@ -91,3 +96,27 @@ def schedule_runs_for(pipeline_ids: Iterable[UUID], team_id: int) -> dict[UUID,))return runs+++def awaiting_first_champion() -> Q:+ """Filter pipelines to the ``bootstrapping`` ones that wait for a scheduled run to find a first champion.++ A first champion that fails its fit or its scorability check rolls back, and the pipeline goes+ back to ``bootstrapping`` with a completed run. Such a pipeline has no champion and no pending+ or running training run. Without scheduled research it stays in this state.+ """+ run_of_pipeline = AutoresearchTrainingRun.all_teams.filter(team_id=OuterRef("team_id"), pipeline_id=OuterRef("pk"))+ return Q(+ Exists(run_of_pipeline.filter(status=AutoresearchTrainingRun.Status.COMPLETED)),+ ~Exists(+ run_of_pipeline.filter(+ status__in=[AutoresearchTrainingRun.Status.PENDING, AutoresearchTrainingRun.Status.RUNNING]+ )+ ),+ ~Exists(+ AutoresearchModel.all_teams.filter(+ team_id=OuterRef("team_id"), pipeline_id=OuterRef("pk"), role=AutoresearchModel.Role.CHAMPION+ )+ ),+ status=AutoresearchPipeline.Status.BOOTSTRAPPING,+ )diff --git a/products/autoresearch/backend/training/test_training.py b/products/autoresearch/backend/training/test_training.pyindex 12fa980d6d6f..e9e396850072 100644--- a/products/autoresearch/backend/training/test_training.py+++ b/products/autoresearch/backend/training/test_training.py@@ -97,6 +97,9 @@ class TestBuildAgentDescription(TeamScopedTestMixin, BaseTest):# Step 3 is the sandbox fit/eval loop.assert "fit and evaluate" in promptassert "roc_auc_score" in prompt+ # A first champion that breaks either fit limit rolls back.+ assert "60 s interactive query limit" in prompt+ assert "must stay under 10 MiB" in prompt@parameterized.expand([diff --git a/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx b/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsxindex 7da94448591a..b1400e0b9c88 100644--- a/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx+++ b/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx@@ -40,6 +40,7 @@ const pipeline = {effective_agent_reasoning_effort: 'medium',training_phase: 'maintain',next_research_at: '2026-11-12T00:00:00Z',+ not_promoted_reason: null,success_auc: null,plateau_iterations: 5,output_person_property: 'p_file_shared_7d',@@ -437,6 +438,27 @@ export const AgentResearchWithoutReaderNotes: Story = {},}+// The first champion failed its fit, so the model went back to bootstrapping with no champion.+export const FirstChampionRolledBack: Story = {+ decorators: [+ mswDecorator({+ get: {+ [`/api/projects/:team_id/autoresearch/${PIPELINE_ID}/`]: {+ ...pipeline,+ status: 'bootstrapping',+ training_phase: 'maintain',+ champion_holdout_auc: null,+ champion_realized_auc: null,+ not_promoted_reason:+ 'The champion fit failed: model.pkl is 14830476 bytes, over the 10485760-byte cap',+ },+ [`/api/projects/:team_id/autoresearch/${PIPELINE_ID}/models/`]: toPaginatedResponse([challenger]),+ },+ }),+ ],+ parameters: { testOptions: { viewportWidths: ['wide'] } },+}+export const Accuracy: Story = {parameters: {pageUrl: `${urls.autoresearchPipeline(PIPELINE_ID)}?tab=accuracy`,diff --git a/products/autoresearch/frontend/AutoresearchPipelineScene.tsx b/products/autoresearch/frontend/AutoresearchPipelineScene.tsxindex d3787736e0e5..aecfd6aa70e2 100644--- a/products/autoresearch/frontend/AutoresearchPipelineScene.tsx+++ b/products/autoresearch/frontend/AutoresearchPipelineScene.tsx@@ -25,6 +25,7 @@ import { QuillScoreNowButton } from './pipeline/QuillScoreNowButton'import { ScoreNowButton } from './pipeline/ScoreNowButton'import { SetupTab } from './pipeline/SetupTab'import { pipelineQuestion, pipelineSubtitle } from './pipelineQuestion'+import { notLiveExplanation } from './trainingScheduleLogic'export const scene: SceneExport = {component: AutoresearchPipelineScene,@@ -46,6 +47,7 @@ export function AutoresearchPipelineScene(): JSX.Element {]const heading = pipeline ? pipelineQuestion(pipeline) : pipelineLoading ? '' : 'Model'+ const notLive = pipeline ? notLiveExplanation(pipeline) : nullif (!isEnabled) {return <NotFound object="Autoresearch" caption="This feature is not enabled for your project." />@@ -85,6 +87,14 @@ export function AutoresearchPipelineScene(): JSX.Element {<><PipelineSummary />{lifecycleSteps && <LifecycleStrip steps={lifecycleSteps} />}+ {notLive && (+ <LemonBanner type="warning">+ <div>{notLive}</div>+ <div className="text-xs font-mono mt-1 break-words">+ Reason: {pipeline?.not_promoted_reason}+ </div>+ </LemonBanner>+ )}<LemonTabsactiveKey={activeTab}onChange={(key) => setActiveTab(key as AutoresearchPipelineTab)}diff --git a/products/autoresearch/frontend/generated/api.schemas.ts b/products/autoresearch/frontend/generated/api.schemas.tsindex 229f7a69a151..eda072ac86db 100644--- a/products/autoresearch/frontend/generated/api.schemas.ts+++ b/products/autoresearch/frontend/generated/api.schemas.ts@@ -290,6 +290,11 @@ export interface AutoresearchPipelineApi {* @nullable*/readonly next_research_at: string | null+ /**+ * Why the best model of the newest completed training run did not go live, for example a failed fit or a failed scorability check. Set only while the pipeline is bootstrapping with no champion.+ * @nullable+ */+ readonly not_promoted_reason: string | null/*** Target AUC threshold. Training stops early if this score is reached.* @nullablediff --git a/products/autoresearch/frontend/trainingScheduleLogic.ts b/products/autoresearch/frontend/trainingScheduleLogic.tsindex a1ce313dc358..710454142fb2 100644--- a/products/autoresearch/frontend/trainingScheduleLogic.ts+++ b/products/autoresearch/frontend/trainingScheduleLogic.ts@@ -41,6 +41,18 @@ export function trainingScheduleStatus(pipeline: AutoresearchPipelineApi, now: dreturn `Next research run on ${next.format('MMM D')}.`}+/** Why the model has no live version after its best model failed the final check, or null. */+export function notLiveExplanation(pipeline: AutoresearchPipelineApi): string | null {+ if (!pipeline.not_promoted_reason || pipeline.live_training_run) {+ return null+ }+ const next =+ pipeline.research_interval_days === null+ ? 'Start a training run to try again.'+ : 'The next research run tries again.'+ return `The best model from the last training run couldn't go live, so nothing is scored yet. ${next}`+}+export interface trainingScheduleLogicValues {currentTeamId: number | nullpipeline: AutoresearchPipelineApi | nulldiff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 9e1d20ca9067..93336ad42b5e 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -15884,6 +15884,11 @@ export namespace Schemas {* @nullable*/readonly next_research_at: string | null;+ /**+ * Why the best model of the newest completed training run did not go live, for example a failed fit or a failed scorability check. Set only while the pipeline is bootstrapping with no champion.+ * @nullable+ */+ readonly not_promoted_reason: string | null;/*** Target AUC threshold. Training stops early if this score is reached.* @nullable