:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(autoresearch): reap training runs stranded by a lost sandbox (#126751)
posthog[bot]66cba7f2e904
diff --git a/products/autoresearch/backend/facade/api.py b/products/autoresearch/backend/facade/api.pyindex 7783924ba958..760a310275fc 100644--- a/products/autoresearch/backend/facade/api.py+++ b/products/autoresearch/backend/facade/api.py@@ -1486,7 +1486,11 @@ def start_training(# The runner and the sandbox import pandas and pyarrow, so the router path loads them only here.from ..inference.sandbox import SandboxInferenceError # noqa: PLC0415from ..training.runner import run_training # noqa: PLC0415+ from ..training.stale import fail_stale_training_runs # noqa: PLC0415+ # A run whose sandbox died would otherwise refuse this start for good.+ if (pipeline_uuid := _as_uuid(pipeline_id)) is not None:+ fail_stale_training_runs(team_id=team_id, pipeline_id=pipeline_uuid)with transaction.atomic():pipeline = _claim_pipeline_for_training(team_id, pipeline_id)if not allow_action_target and pipeline.target_definition.get("type") == "action":diff --git a/products/autoresearch/backend/temporal/AGENTS.md b/products/autoresearch/backend/temporal/AGENTS.mdindex 8c8b8cb2e682..6f16fee0818c 100644--- a/products/autoresearch/backend/temporal/AGENTS.md+++ b/products/autoresearch/backend/temporal/AGENTS.md@@ -55,6 +55,7 @@ Scoring and training kickoff run on a cadence day only, compared by calendar dayDiscovery pauses a pipeline whose creator has lost access to the team, and skips one outside the flag rollout.When an inference child ends in an exception, the coordinator fails the runs it left RUNNING, with `failure_kind` `other`, because a lost worker never records an outcome. This is behind `workflow.patched("autoresearch-close-lost-runs-2026-10")`.Discovery also fails every inference run still RUNNING after `INFERENCE_WORKFLOW_TIMEOUT` (`fail_stale_inference_runs()` in `../inference/failures.py`). This is the backstop for a run that the coordinator did not close.+Discovery also closes every training run that its `TaskRun` can no longer close (`fail_stale_training_runs()` in `../training/stale.py`), across all teams. Kickoff calls it for its own pipeline before the live-run check, outside the row lock, so a run whose sandbox died does not block the pipeline's training.The catch-up sweep lists only the due pipelines whose latest scheduled run of the day failed with a kind a retry can pass (`scheduled_run_can_retry()`). A pipeline with no run for the day is skipped, because the daily sweep may not have reached it yet, and a second start would score it twice. Its inference children use `ALLOW_DUPLICATE_FAILED_ONLY`, so a catch-up starts only after the daily child of the date failed or timed out. It skips training kickoff and validation, because the daily sweep already ran them.A retry of a scheduled scoring attempt fills the RUNNING row of the attempt before it, so one date keeps one row.Kickoff applies the Tasks entitlement and usage gates `/train` applies, launches at most one run per UTC day, and holds the pipeline row lock through the launch, as `start_training` does.diff --git a/products/autoresearch/backend/temporal/test_workflows.py b/products/autoresearch/backend/temporal/test_workflows.pyindex e7997de5d255..b37615a05b32 100644--- a/products/autoresearch/backend/temporal/test_workflows.py+++ b/products/autoresearch/backend/temporal/test_workflows.py@@ -577,6 +577,29 @@ class TestCoordinatorActivities(TeamScopedTestMixin, BaseTest):assert result.errormock_run_training.assert_not_called()+ @patch("products.autoresearch.backend.training.stale.tasks_cancellation")+ @patch("products.autoresearch.backend.training.stale.tasks_facade.get_task_run", return_value=None)+ @patch("products.autoresearch.backend.temporal.workflows.run_training")+ def test_kickoff_reaps_a_run_whose_sandbox_died(+ self, mock_run_training: MagicMock, _get_task_run: MagicMock, _cancellation: MagicMock+ ) -> None:+ pipeline = self._create_pipeline()+ with time_machine.travel(django_timezone.now() - timedelta(days=1), tick=False):+ stranded = AutoresearchTrainingRun.objects.create(+ pipeline=pipeline,+ status=AutoresearchTrainingRun.Status.RUNNING,+ iteration_budget=10,+ started_at=django_timezone.now(),+ task_run_id=uuid.uuid4(),+ )++ result = activity_kickoff_training(KickoffTrainingInput(pipeline_id=str(pipeline.id), team_id=self.team.id))++ assert result.kicked_off+ mock_run_training.assert_called_once()+ stranded.refresh_from_db()+ assert stranded.status == AutoresearchTrainingRun.Status.FAILED+@parameterized.expand([("pending_run", AutoresearchTrainingRun.Status.PENDING, False, True, True, "already_running"),diff --git a/products/autoresearch/backend/temporal/workflows.py b/products/autoresearch/backend/temporal/workflows.pyindex 55ae1ba4390f..6a26416ab00b 100644--- a/products/autoresearch/backend/temporal/workflows.py+++ b/products/autoresearch/backend/temporal/workflows.py@@ -62,6 +62,7 @@ with workflow.unsafe.imports_passed_through():from products.autoresearch.backend.query import BATCH_QUERY, SCORING_QUERYfrom products.autoresearch.backend.training.runner import run_trainingfrom products.autoresearch.backend.training.schedule import research_schedule, schedule_runs_for+ from products.autoresearch.backend.training.stale import fail_stale_training_runsfrom products.tasks.backend.facade.access import get_desktop_access_decisionfrom products.tasks.backend.facade.usage import task_run_usage_limited@@ -755,7 +756,8 @@ def activity_load_active_pipelines(inp: LoadActivePipelinesInput) -> LoadActivePoutside the ``autoresearch`` rollout is skipped but stays live, so it resumes when the rolloutreaches it again.- It also fails every inference run that its lost worker left RUNNING.+ It also fails every inference run that its lost worker left RUNNING, and closes every training+ run that its TaskRun can no longer close (``fail_stale_training_runs()``).The result lists the pipelines that are due and did not score yesterday first, so a boundedsweep starts the pipelines that missed a day before the others.@@ -768,6 +770,7 @@ def activity_load_active_pipelines(inp: LoadActivePipelinesInput) -> LoadActivePtogether. The next catch-up tick tries again."""fail_stale_inference_runs()+ fail_stale_training_runs()if inp.catch_up_date is not None and cache.get(daily_sweep_key(inp.catch_up_date)) == DAILY_SWEEP_RUNNING:logger.info("autoresearch_catch_up_deferred", catch_up_date=inp.catch_up_date)return LoadActivePipelinesResult(pipelines=[])@@ -866,7 +869,10 @@ def activity_kickoff_training(inp: KickoffTrainingInput) -> KickoffTrainingResulThe run row and its task binding commit together under the pipeline row lock, as in thefacade's ``start_training``, so a manual start cannot interleave. A launch failure raises,so the activity retry applies.++ A stale training run of the pipeline is closed first, outside the lock, so it does not block the start."""+ fail_stale_training_runs(team_id=inp.team_id, pipeline_id=inp.pipeline_id)# The gates can call out to billing and flags, so they run before the row lock is taken.with team_scope(inp.team_id):loaded = AutoresearchPipeline.objects.select_related("created_by", "team__organization").get(pk=inp.pipeline_id)diff --git a/products/autoresearch/backend/training/AGENTS.md b/products/autoresearch/backend/training/AGENTS.mdindex 6044bef0a5fa..f314f1272a85 100644--- a/products/autoresearch/backend/training/AGENTS.md+++ b/products/autoresearch/backend/training/AGENTS.md@@ -35,6 +35,12 @@ The other half is `../inference/`, which consumes what this package produces andThe safety net. `handle_task_run_completed()` is called from the `TaskRun` `post_save` signal registered in `../apps.py`, and runs synchronously in the Temporal worker thread.If the agent recorded iterations but never called complete, this finalizes through the same promotion path. If it recorded nothing, the run is marked failed — which is what produces `"Agent recorded no iterations before the run ended."``run_training()` writes the `autoresearch_training_run_id` marker into `TaskRun.state` when it creates the task, and tasks refuses a patch to it. The marker names a run; ownership is proved by the run's server-stamped `task_run_id`, so a `TaskRun` may only finalize the run whose `task_run_id` is its own id.+- `stale.py`+ The backstop for the safety net. `fail_stale_training_runs()` closes every `pending` or `running` run with a `task_run_id` that is older than `STALE_TRAINING_RUN_AFTER` (twice the Tasks `MAX_SANDBOX_TTL_SECONDS`, read through the tasks facade).+ When the `TaskRun` is terminal (`missed_handoff`), it calls `handle_task_run_completed()`, so the run ends with the outcome the receiver would have given it. When the `TaskRun` is still live or no longer exists (`lost_driver`), it fails the run and cancels the `TaskRun` best-effort. A cancel failure only logs.+ Every failure re-reads the run under its row lock and changes only an open run, so a run that completes at the same time keeps its outcome. Each closed run logs `autoresearch_stale_training_run_reaped` and increments `autoresearch_stale_training_runs_reaped_total`, both with the case.+ A run with no `task_run_id` (an external run from `open_training_run()`, or a stub) is not reaped.+ Coordinator discovery calls it for all teams. `activity_kickoff_training` and the facade's `start_training` call it for their pipeline before the live-run check.- `promotion.py`Champion selection. `complete_training_run()` is the single entry point, used both by the training-run `complete` API action and by `ingestion.py`.A challenger must beat the incumbent by `CHAMPION_PROMOTION_MARGIN` (0.005 holdout AUC) to be promoted — near-ties keep the incumbent rather than churning the champion on noise.@@ -94,7 +100,7 @@ Two things routinely surprise people:## Where the rest of the system meets this package- **Launched by** — the `train` API action in `../presentation/views/views.py`, the `autoresearch_train` management command, and `activity_kickoff_training` in `../temporal/workflows.py`.-- **Finalized by** — the `complete` action on the training-run viewset, or `ingestion.py` via the `TaskRun` `post_save` signal wired in `../apps.py`.+- **Finalized by** — the `complete` action on the training-run viewset, or `ingestion.py` via the `TaskRun` `post_save` signal wired in `../apps.py`, or `stale.py` when neither ran.- **Consumed by** — `../inference/`, which reads the champion's `artifact_prefix` and runs its bundle. `fit_champion_model()` in `../inference/sandbox.py` is what actually fits and persists `model.pkl` at completion time, for the champion and for a challenger that enters the shadow set. Shadow scoring loads that `model.pkl` the same way the champion's scoring does.- **Agent-facing surface** — the `autoresearch-training-runs-*` MCP tools in `../../mcp/tools.yaml`, backed by the viewsets in `../presentation/views/views.py`. The sandbox agent has no other way to write.- **Labels and features** — `../dataset/labeling.py` builds the training population the bundle is fitted against.diff --git a/products/autoresearch/backend/training/stale.py b/products/autoresearch/backend/training/stale.pynew file mode 100644index 000000000000..bb87355c9304--- /dev/null+++ b/products/autoresearch/backend/training/stale.py@@ -0,0 +1,149 @@+"""+Close training runs that their TaskRun can no longer close.++A training run reaches a terminal status only through the agent's complete call or the+``TaskRun`` post_save receiver in ``ingestion.py``. When the sandbox dies hard, or the receiver+never runs, the run stays open, and every training start of its pipeline refuses to run.++Two cases:+ - ``missed_handoff``: the TaskRun is terminal, but the training run is still open. The reaper+ calls the receiver's handler, so the run ends with the outcome it would have had.+ - ``lost_driver``: the TaskRun is still live, or it no longer exists. The reaper fails the run+ and cancels the TaskRun best-effort.++A run with no TaskRun (an external run from ``open_training_run()``, or a stub) is not reaped.+"""++from __future__ import annotations++from datetime import timedelta+from enum import StrEnum+from uuid import UUID++from django.db import transaction+from django.db.models import Q, QuerySet+from django.utils import timezone as django_timezone++import structlog+from prometheus_client import Counter++from posthog.models.scoping import team_scope++from products.autoresearch.backend.models import AutoresearchTrainingRun+from products.autoresearch.backend.training.ingestion import _apply_failure, handle_task_run_completed+from products.tasks.backend.facade import (+ api as tasks_facade,+ cancellation as tasks_cancellation,+)++logger = structlog.get_logger(__name__)++# A sandbox stops at its TTL, so a run older than twice the TTL has no live agent behind it.+# The margin keeps a run that is near its wall clock safe from the reaper.+STALE_TRAINING_RUN_AFTER = timedelta(seconds=2 * tasks_facade.MAX_SANDBOX_TTL_SECONDS)++_OPEN_STATUSES = (AutoresearchTrainingRun.Status.PENDING, AutoresearchTrainingRun.Status.RUNNING)++STALE_TRAINING_RUNS_REAPED = Counter(+ "autoresearch_stale_training_runs_reaped_total",+ "Training runs that the stale-run reaper closed, by case.",+ labelnames=["case"],+)+++class ReapCase(StrEnum):+ LOST_DRIVER = "lost_driver"+ MISSED_HANDOFF = "missed_handoff"+++def fail_stale_training_runs(*, team_id: int | None = None, pipeline_id: str | UUID | None = None) -> int:+ """+ Close every open training run older than ``STALE_TRAINING_RUN_AFTER`` and return how many it closed.++ With no ``team_id`` it reads every team. Call it outside a transaction, because the finalize+ path reads object storage and takes its own locks.+ """+ reaped = 0+ for run in _stale_runs(team_id=team_id, pipeline_id=pipeline_id):+ try:+ with team_scope(run.team_id):+ case = _reap(run)+ except Exception:+ logger.exception("autoresearch_stale_training_run_reap_failed", training_run_id=str(run.pk))+ continue+ if case is None:+ continue+ reaped += 1+ STALE_TRAINING_RUNS_REAPED.labels(case=case).inc()+ logger.warning(+ "autoresearch_stale_training_run_reaped",+ training_run_id=str(run.pk),+ pipeline_id=str(run.pipeline_id),+ team_id=run.team_id,+ task_run_id=str(run.task_run_id),+ case=case,+ )+ return reaped+++def _stale_runs(*, team_id: int | None, pipeline_id: str | UUID | None) -> QuerySet[AutoresearchTrainingRun]:+ cutoff = django_timezone.now() - STALE_TRAINING_RUN_AFTER+ runs = (+ AutoresearchTrainingRun.objects.unscoped()+ if team_id is None+ else AutoresearchTrainingRun.objects.for_team(team_id)+ )+ runs = runs.filter(+ Q(started_at__lt=cutoff) | Q(started_at__isnull=True, created_at__lt=cutoff),+ status__in=_OPEN_STATUSES,+ task_run_id__isnull=False,+ )+ if pipeline_id is not None:+ runs = runs.filter(pipeline__pk=pipeline_id)+ return runs.order_by("created_at")+++def _reap(run: AutoresearchTrainingRun) -> ReapCase | None:+ if run.task_run_id is None:+ return None+ task_run = tasks_facade.get_task_run(run.task_run_id, team_id=run.team_id)+ if task_run is not None and task_run.is_terminal:+ handle_task_run_completed(task_run)+ run.refresh_from_db(fields=["status"])+ if run.status not in _OPEN_STATUSES:+ return ReapCase.MISSED_HANDOFF+ # The handler skips a PENDING run and a TaskRun without the state marker.+ failed = _fail_if_open(run, error="The training task ended, but the run was not finalized.")+ return ReapCase.MISSED_HANDOFF if failed else None++ failed = _fail_if_open(+ run,+ error="The training sandbox stopped without a result. The run was open longer than the sandbox can live.",+ )+ if not failed:+ return None+ if task_run is not None:+ _cancel_task_run(task_run.id, task_run.task_id, team_id=run.team_id)+ return ReapCase.LOST_DRIVER+++def _fail_if_open(run: AutoresearchTrainingRun, error: str) -> bool:+ with transaction.atomic():+ # Re-read under the lock completion takes, so a run that completed in the meantime keeps its outcome.+ locked = AutoresearchTrainingRun.objects.select_for_update().get(pk=run.pk)+ if locked.status not in _OPEN_STATUSES:+ return False+ _apply_failure(locked, error)+ return True+++def _cancel_task_run(task_run_id: UUID, task_id: UUID, *, team_id: int) -> None:+ try:+ outcome, _ = tasks_cancellation.cancel_task_run(+ task_run_id, task_id, team_id, reason="Autoresearch training run was stale", source="autoresearch"+ )+ except Exception:+ logger.exception("autoresearch_stale_training_cancel_failed", task_run_id=str(task_run_id))+ return+ if outcome not in ("accepted", "already_terminal", "not_found"):+ logger.warning("autoresearch_stale_training_cancel_not_accepted", task_run_id=str(task_run_id), outcome=outcome)diff --git a/products/autoresearch/backend/training/test_training_ingestion.py b/products/autoresearch/backend/training/test_training_ingestion.pyindex a512660f5eaf..2c5170bcf48b 100644--- a/products/autoresearch/backend/training/test_training_ingestion.py+++ b/products/autoresearch/backend/training/test_training_ingestion.py@@ -1,6 +1,11 @@+from datetime import timedelta+from types import SimpleNamespace+from typing import Optionalfrom uuid import UUID, uuid4+import time_machinefrom posthog.test.base import BaseTest+from unittest.mock import MagicMock, patchfrom django.utils import timezone as django_timezone@@ -13,7 +18,9 @@ from products.autoresearch.backend.models import (AutoresearchTrainingRun,)from products.autoresearch.backend.testing import TeamScopedTestMixin+from products.autoresearch.backend.training.artifacts import BundleNotFoundfrom products.autoresearch.backend.training.ingestion import FinishedTaskRun, handle_task_run_completed+from products.autoresearch.backend.training.stale import STALE_TRAINING_RUN_AFTER, fail_stale_training_runs_TASK_RUN_ID = UUID("00000000-0000-0000-0000-000000000001")@@ -186,3 +193,85 @@ class TestHandleTaskRunCompleted(TeamScopedTestMixin, BaseTest):pipeline.refresh_from_db()assert training_run.status == AutoresearchTrainingRun.Status.FAILEDassert pipeline.status == expected_status++ @parameterized.expand(+ [+ ("lost_driver_live_task_run", "in_progress", True, True, AutoresearchTrainingRun.Status.FAILED, True),+ ("lost_driver_missing_task_run", None, True, True, AutoresearchTrainingRun.Status.FAILED, False),+ ("missed_handoff_finalizes", "completed", True, True, AutoresearchTrainingRun.Status.COMPLETED, False),+ ("inside_the_cutoff", "in_progress", False, True, AutoresearchTrainingRun.Status.RUNNING, False),+ ("external_run_without_task", None, True, False, AutoresearchTrainingRun.Status.RUNNING, False),+ ]+ )+ @time_machine.travel("2026-09-11T12:00:00Z", tick=False)+ @patch("products.autoresearch.backend.training.stale.tasks_cancellation")+ @patch("products.autoresearch.backend.training.stale.tasks_facade.get_task_run")+ def test_fail_stale_training_runs(+ self,+ _name: str,+ task_run_status: Optional[str],+ stale: bool,+ bound: bool,+ expected_status: str,+ expect_cancel: bool,+ get_task_run: MagicMock,+ cancellation: MagicMock,+ ) -> None:+ pipeline = self._make_pipeline(status=AutoresearchPipeline.Status.BOOTSTRAPPING)+ age = STALE_TRAINING_RUN_AFTER + timedelta(minutes=1) if stale else STALE_TRAINING_RUN_AFTER / 2+ training_run = self._make_training_run(+ pipeline,+ started_at=django_timezone.now() - age,+ task_run_id=_TASK_RUN_ID if bound else None,+ )+ self._record_iteration(training_run, number=0, status="kept", holdout=0.82)+ get_task_run.return_value = (+ None+ if task_run_status is None+ else SimpleNamespace(+ id=_TASK_RUN_ID,+ task_id=uuid4(),+ team_id=self.team.id,+ status=task_run_status,+ is_terminal=task_run_status == "completed",+ state={"autoresearch_training_run_id": str(training_run.id)},+ error_message=None,+ )+ )+ cancellation.cancel_task_run.return_value = ("accepted", None)++ with patch("products.autoresearch.backend.training.artifacts.read_bundle", side_effect=BundleNotFound):+ fail_stale_training_runs()++ training_run.refresh_from_db()+ assert training_run.status == expected_status+ assert cancellation.cancel_task_run.called == expect_cancel+ if expected_status == AutoresearchTrainingRun.Status.FAILED:+ assert training_run.error+ pipeline.refresh_from_db()+ assert pipeline.status == AutoresearchPipeline.Status.DRAFT++ @time_machine.travel("2026-09-11T12:00:00Z", tick=False)+ @patch("products.autoresearch.backend.training.stale.tasks_cancellation")+ @patch("products.autoresearch.backend.training.stale.tasks_facade.get_task_run")+ def test_fail_stale_training_runs_keeps_a_run_that_completed_meanwhile(+ self, get_task_run: MagicMock, cancellation: MagicMock+ ) -> None:+ pipeline = self._make_pipeline()+ training_run = self._make_training_run(+ pipeline, started_at=django_timezone.now() - STALE_TRAINING_RUN_AFTER - timedelta(minutes=1)+ )++ def completed_by_agent(*_args: object, **_kwargs: object) -> None:+ AutoresearchTrainingRun.objects.filter(pk=training_run.pk).update(+ status=AutoresearchTrainingRun.Status.COMPLETED+ )+ return None++ get_task_run.side_effect = completed_by_agent++ assert fail_stale_training_runs(team_id=self.team.id) == 0++ training_run.refresh_from_db()+ assert training_run.status == AutoresearchTrainingRun.Status.COMPLETED+ cancellation.cancel_task_run.assert_not_called()diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.pyindex 2056ae81e45c..3e124cf6be55 100644--- a/products/tasks/backend/facade/api.py+++ b/products/tasks/backend/facade/api.py@@ -137,6 +137,7 @@ from products.tasks.backend.logic.services.network_policy import (normalize_requested_domains,)from products.tasks.backend.logic.services.sandbox import get_sandbox_class_for_sandbox_id, is_public_sandbox_repo+from products.tasks.backend.logic.services.sandbox_config import MAX_SANDBOX_TTL_SECONDSfrom products.tasks.backend.logic.services.space_setup import (SPACE_SETUP_FEED_EVENT,SPACE_SETUP_MODEL,@@ -275,6 +276,7 @@ __all__ = ["uses_own_subscription","beacon_task_presence","bootstrap_task_run",+ "MAX_SANDBOX_TTL_SECONDS","SANDBOX_REPOSITORIES_ROOT","can_mint_readonly_github_token","is_public_sandbox_repo",