Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(autoresearch): cap sandbox training runs per project per day (#126759)

posthog[bot]5543119dbb42
diff --git a/products/autoresearch/AGENTS.md b/products/autoresearch/AGENTS.md
index 43154d9b89e2..3eacddd73c1b 100644
--- a/products/autoresearch/AGENTS.md
+++ b/products/autoresearch/AGENTS.md
@@ -44,6 +44,10 @@ The real-agent training path additionally needs Tasks access, because it runs in
Training never ends, but exploration comes first.
The daily sweep starts research runs on scoring days until a run fails to promote a better model or 5 research runs have run, then one run every `research_interval_days` after the latest run (`backend/training/schedule.py`).
Each run takes `iterations_per_run` iterations, and records why it started in `kickoff_reason`.
+Per-project limits in `backend/limits.py` bound every sandbox run, however it starts (sweep, UI, API or MCP).
+A project starts at most `max_training_runs_per_day` sandbox runs (default 3) in any rolling 24 hours, across all its pipelines, and no run gets more than `max_iterations_per_run` iterations (default 10).
+Both resolve from the `autoresearch` flag payload: the project's `team_configs["<team_id>"]` entry, then `default_team_config`, then the code constants. Edit the payload to change a limit without a deploy.
+Runs that `open_training_run()` opens for an external agent do not count, because the caller's own agent does the work.
Each run also records the runtime adapter, model and reasoning effort it was dispatched with (see `backend/training/AGENTS.md`).
`iteration_budget` and `iteration_budget_remaining` are deprecated: nothing scheduled reads them.
diff --git a/products/autoresearch/backend/facade/api.py b/products/autoresearch/backend/facade/api.py
index 69a01bb994ff..3fd341095c1e 100644
--- a/products/autoresearch/backend/facade/api.py
+++ b/products/autoresearch/backend/facade/api.py
@@ -58,6 +58,7 @@ from ..evaluation.segment_thresholds import (
training_base_rate,
)
from ..inference.failures import INFERENCE_WORKFLOW_TIMEOUT
+from ..limits import check_training_run_cap, limits_for_team
from ..models import (
RESEARCH_INTERVAL_DAYS_CHOICES as _RESEARCH_INTERVAL_DAYS_CHOICES,
AutoresearchIteration,
@@ -120,6 +121,7 @@ from .contracts import (
SuggestionNotFound as SuggestionNotFound,
TemplateInfo,
TrainingRun,
+ TrainingRunCapReached,
TrainingRunHistory,
TrainingRunHistoryEntry,
TrainingRunNotFound,
@@ -1475,6 +1477,9 @@ def start_training(
``allow_action_target=False`` refuses an action target with ``InvalidTarget``. The check
reads the locked row, so a concurrent edit of the target cannot slip past it.
+
+ Raises ``TrainingRunCapReached`` when the project has used its sandbox training runs for the
+ last 24 hours, across all its pipelines.
"""
# The runner and the sandbox import pandas and pyarrow, so the router path loads them only here.
from ..inference.sandbox import SandboxInferenceError # noqa: PLC0415
@@ -1484,10 +1489,14 @@ def start_training(
# A run whose sandbox died would otherwise refuse this start for good.
if (pipeline_uuid := _as_uuid(pipeline_id)) is not None:
fail_stale_training_runs(team_id=team_id, pipeline_id=pipeline_uuid)
+ # The flag read can call out, so it runs before the row lock is taken.
+ limits = limits_for_team(team_id)
with transaction.atomic():
pipeline = _claim_pipeline_for_training(team_id, pipeline_id)
if not allow_action_target and pipeline.target_definition.get("type") == "action":
raise InvalidTarget("An action target needs the action:read scope.")
+ if cap := check_training_run_cap(team_id, limits):
+ raise TrainingRunCapReached(used=cap.used, limit=cap.limit, next_slot_at=cap.next_slot_at)
budget = iteration_budget or pipeline.iterations_per_run
try:
training_run = run_training(
@@ -1495,6 +1504,7 @@ def start_training(
iteration_budget=budget,
user_id=user_id,
kickoff_reason=AutoresearchTrainingRun.KickoffReason.MANUAL,
+ limits=limits,
)
except ActionNotFound:
raise AutoresearchConflict("The pipeline's target action no longer exists.")
diff --git a/products/autoresearch/backend/facade/contracts.py b/products/autoresearch/backend/facade/contracts.py
index 3d1752bba06d..415835747144 100644
--- a/products/autoresearch/backend/facade/contracts.py
+++ b/products/autoresearch/backend/facade/contracts.py
@@ -45,6 +45,26 @@ class AutoresearchConflict(ValueError):
"""
+class TrainingRunCapReached(Exception):
+ """The project has started its limit of sandbox training runs in the last 24 hours.
+
+ The viewset maps it to a 429, so the message is user-facing copy.
+ """
+
+ def __init__(self, *, used: int, limit: int, next_slot_at: datetime | None) -> None:
+ self.used = used
+ self.limit = limit
+ self.next_slot_at = next_slot_at
+ if limit == 0 or next_slot_at is None:
+ message = "Training runs are turned off for this project. Contact support to turn them on."
+ else:
+ message = (
+ f"This project started {used} training runs in the last 24 hours, and its limit is {limit}. "
+ f"The next run can start after {next_slot_at:%H:%M} UTC."
+ )
+ super().__init__(message)
+
+
class InvalidTarget(ValueError):
"""The target names something the labeler cannot honor.
diff --git a/products/autoresearch/backend/limits.py b/products/autoresearch/backend/limits.py
new file mode 100644
index 000000000000..693ac88256b4
--- /dev/null
+++ b/products/autoresearch/backend/limits.py
@@ -0,0 +1,145 @@
+"""Per-project autoresearch limits, resolved from the `autoresearch` flag payload.
+
+A training run is a paid Tasks sandbox whose LLM traffic the project does not pay for, so these
+limits are the only bound on what one project can spend. Each limit resolves through three
+layers, most specific first: the project's own `team_configs["<team_id>"]` entry, then the
+fleet-wide `default_team_config`, then the code constant here. Edit the flag payload to change a
+limit without a deploy:
+
+ {
+ "default_team_config": {"max_training_runs_per_day": 3, "max_iterations_per_run": 10},
+ "team_configs": {"2": {"max_training_runs_per_day": 20}}
+ }
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timedelta
+
+from django.db import connection
+from django.db.models import Q, QuerySet
+from django.utils import timezone
+
+import structlog
+import posthoganalytics
+
+from posthog.dataclasses import frozen
+from posthog.exceptions_capture import capture_exception
+
+from .access import AUTORESEARCH_FLAG
+from .models import AutoresearchTrainingRun
+
+logger = structlog.get_logger(__name__)
+
+# Fixed distinct id for the payload read, because the limits are per project, not per person.
+_PAYLOAD_DISTINCT_ID = "internal_autoresearch_team_limits"
+
+DEFAULT_MAX_TRAINING_RUNS_PER_DAY = 3
+DEFAULT_MAX_ITERATIONS_PER_RUN = 10
+
+TEAM_CONFIG_MAX_TRAINING_RUNS_PER_DAY = "max_training_runs_per_day"
+TEAM_CONFIG_MAX_ITERATIONS_PER_RUN = "max_iterations_per_run"
+
+# The daily cap counts runs over a rolling window, so a burst just after midnight UTC cannot
+# spend two days of runs in one sitting.
+TRAINING_RUN_WINDOW = timedelta(hours=24)
+
+# Namespace for the per-team advisory lock, so it cannot collide with another product's key.
+_TRAINING_CAP_LOCK_NAMESPACE = 0x41525452 # "ARTR"
+
+
+@frozen
+class TeamLimits:
+ # 0 stops every sandbox training run for the project.
+ max_training_runs_per_day: int
+ max_iterations_per_run: int
+
+
+@frozen
+class TrainingCapReached:
+ used: int
+ limit: int
+ # When the oldest counted run leaves the window. None when the limit is 0.
+ next_slot_at: datetime | None
+
+
+def limits_for_team(team_id: int) -> TeamLimits:
+ """The project's effective limits. A missing or unreadable payload resolves to the code constants."""
+ payload = _read_flag_payload()
+ return TeamLimits(
+ max_training_runs_per_day=_resolve_int(
+ payload, team_id, TEAM_CONFIG_MAX_TRAINING_RUNS_PER_DAY, DEFAULT_MAX_TRAINING_RUNS_PER_DAY, minimum=0
+ ),
+ max_iterations_per_run=_resolve_int(
+ payload, team_id, TEAM_CONFIG_MAX_ITERATIONS_PER_RUN, DEFAULT_MAX_ITERATIONS_PER_RUN, minimum=1
+ ),
+ )
+
+
+def check_training_run_cap(team_id: int, limits: TeamLimits) -> TrainingCapReached | None:
+ """Whether the project has used its sandbox training runs for the window, or None when a slot is free.
+
+ Call inside the ``transaction.atomic()`` that creates the run. The advisory lock is held until
+ that transaction ends, so two pipelines of one team cannot both take the last slot.
+ """
+ with connection.cursor() as cursor:
+ cursor.execute("SELECT pg_advisory_xact_lock(%s, %s)", [_TRAINING_CAP_LOCK_NAMESPACE, team_id])
+ limit = limits.max_training_runs_per_day
+ window_start = timezone.now() - TRAINING_RUN_WINDOW
+ created = list(
+ counted_training_runs(team_id)
+ .filter(created_at__gte=window_start)
+ .order_by("created_at")
+ .values_list("created_at", flat=True)
+ )
+ if len(created) < limit:
+ return None
+ next_slot_at = created[len(created) - limit] + TRAINING_RUN_WINDOW if limit > 0 else None
+ logger.info("autoresearch_training_cap_reached", team_id=team_id, used=len(created), limit=limit)
+ return TrainingCapReached(used=len(created), limit=limit, next_slot_at=next_slot_at)
+
+
+def counted_training_runs(team_id: int) -> QuerySet[AutoresearchTrainingRun]:
+ """The project's training runs that ran, or are about to run, in a paid sandbox.
+
+ A run that ``open_training_run()`` opened records no agent settings and no task, because the
+ caller's own agent does the work, so it does not count. Neither does a stub run.
+ """
+ live = [AutoresearchTrainingRun.Status.PENDING, AutoresearchTrainingRun.Status.RUNNING]
+ return AutoresearchTrainingRun.objects.for_team(team_id).filter(
+ Q(task_id__isnull=False) | Q(status__in=live, agent_runtime_adapter__isnull=False)
+ )
+
+
+def _read_flag_payload() -> dict | None:
+ try:
+ payload = posthoganalytics.get_feature_flag_payload(AUTORESEARCH_FLAG, _PAYLOAD_DISTINCT_ID, match_value=True)
+ if isinstance(payload, str):
+ payload = json.loads(payload)
+ return payload if isinstance(payload, dict) else None
+ except Exception as error:
+ capture_exception(error)
+ return None
+
+
+def _resolve_int(payload: dict | None, team_id: int, key: str, default: int, *, minimum: int) -> int:
+ """The first valid value from the team's entry, then the fleet default, then ``default``.
+
+ A value that is not an integer at least ``minimum`` is skipped, so a typo in one layer falls
+ through to the next rather than lifting the limit.
+ """
+ if payload is None:
+ return default
+ team_configs = payload.get("team_configs")
+ layers = [
+ team_configs.get(str(team_id)) if isinstance(team_configs, dict) else None,
+ payload.get("default_team_config"),
+ ]
+ for layer in layers:
+ if not isinstance(layer, dict):
+ continue
+ value = layer.get(key)
+ if isinstance(value, int) and not isinstance(value, bool) and value >= minimum:
+ return value
+ return default
diff --git a/products/autoresearch/backend/presentation/views/serializers.py b/products/autoresearch/backend/presentation/views/serializers.py
index f14c80716b4a..895b77aad2fd 100644
--- a/products/autoresearch/backend/presentation/views/serializers.py
+++ b/products/autoresearch/backend/presentation/views/serializers.py
@@ -2020,8 +2020,23 @@ class StartTrainingRequestSerializer(serializers.Serializer):
iteration_budget = serializers.IntegerField(
required=False,
min_value=1,
- max_value=500,
- help_text="Iterations for this training run. Defaults to the pipeline's iterations_per_run.",
+ max_value=50,
+ help_text=(
+ "Iterations for this training run. Defaults to the pipeline's iterations_per_run. "
+ "The project's iteration limit per run can lower it."
+ ),
+ )
+
+
+class TrainingRunCapReachedSerializer(serializers.Serializer):
+ detail = serializers.CharField(help_text="Why the run did not start.")
+ used = serializers.IntegerField(
+ help_text="Sandbox training runs the project started in the last 24 hours, across all its pipelines."
+ )
+ limit = serializers.IntegerField(help_text="Sandbox training runs the project may start in any 24 hours.")
+ next_slot_at = serializers.DateTimeField(
+ allow_null=True,
+ help_text="When the next training run can start. Null when training runs are turned off for the project.",
)
diff --git a/products/autoresearch/backend/presentation/views/views.py b/products/autoresearch/backend/presentation/views/views.py
index 516067d50c31..1cc816658ea9 100644
--- a/products/autoresearch/backend/presentation/views/views.py
+++ b/products/autoresearch/backend/presentation/views/views.py
@@ -45,6 +45,7 @@ from products.autoresearch.backend.facade.contracts import (
InvalidTarget,
PipelineNotFound,
SuggestionNotFound,
+ TrainingRunCapReached,
TrainingRunNotFound,
)
from products.tasks.backend.facade import api as tasks_facade
@@ -78,6 +79,7 @@ from .serializers import (
StartTrainingRequestSerializer,
StoredArtifactSerializer,
TemplateInfoSerializer,
+ TrainingRunCapReachedSerializer,
TrainingRunHistoryQuerySerializer,
TrainingRunHistorySerializer,
ValidatePipelineRequestSerializer,
@@ -442,7 +444,13 @@ class AutoresearchPipelineViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet)
)
),
404: OpenApiResponse(description="The pipeline does not exist or is archived."),
- 429: OpenApiResponse(description="The team is over its PostHog Desktop usage limit."),
+ 429: OpenApiResponse(
+ response=TrainingRunCapReachedSerializer,
+ description=(
+ "The project started its limit of training runs in the last 24 hours (this body), "
+ "or the team is over its PostHog Desktop usage limit."
+ ),
+ ),
503: OpenApiResponse(description="PostHog Desktop access could not be checked. Try again."),
},
summary="Start a training run",
@@ -482,6 +490,9 @@ class AutoresearchPipelineViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet)
raise ValidationError({"target_definition": str(exc)}) from exc
except AutoresearchConflict as exc:
raise ValidationError(str(exc)) from exc
+ except TrainingRunCapReached as exc:
+ body = {"detail": str(exc), "used": exc.used, "limit": exc.limit, "next_slot_at": exc.next_slot_at}
+ return Response(TrainingRunCapReachedSerializer(instance=body).data, status=429)
return Response(AutoresearchTrainingRunSerializer(instance=training_run).data)
@validated_request(
diff --git a/products/autoresearch/backend/temporal/AGENTS.md b/products/autoresearch/backend/temporal/AGENTS.md
index 6f16fee0818c..de8d13438016 100644
--- a/products/autoresearch/backend/temporal/AGENTS.md
+++ b/products/autoresearch/backend/temporal/AGENTS.md
@@ -62,8 +62,9 @@ Kickoff applies the Tasks entitlement and usage gates `/train` applies, launches
Kickoff then follows the research schedule in `../training/schedule.py`, which it derives from the pipeline's training runs and never stores.
In explore, it starts a run (`explore`) until a completed run does not promote its best model, or until `EXPLORE_RUN_LIMIT` research runs have run. Rescue runs do not count.
In maintain, it starts a run (`scheduled`) once `research_interval_days` calendar days have passed since the latest training run of any kind, and reports `not_due` before that. A null interval reports `manual_only`.
-Each run takes the pipeline's `iterations_per_run`. Kickoff does not read or spend `iteration_budget_remaining`.
+Each run takes the pipeline's `iterations_per_run`, which `run_training()` lowers to the project's `max_iterations_per_run`. Kickoff does not read or spend `iteration_budget_remaining`.
When no run is due and `find_unscorable_champion()` finds that the champion cannot score, kickoff launches a rescue run of `RESCUE_ITERATION_BUDGET` iterations, at most once in each window of `UNSCORABLE_AFTER_FAILED_DAYS` UTC days, and reports `rescue_unscorable`.
+Before it launches, kickoff checks the project's daily training cap (`check_training_run_cap()` in `../limits.py`), the same check `start_training` makes, and reports `team_daily_cap` when the project has no slot left. The cap counts every sandbox run of the team in the last 24 hours, so it blocks rescue runs too. The check takes a transaction-scoped advisory lock on the team inside the pipeline row lock, so two pipelines of one team cannot both take the last slot.
Each run records why it started in `kickoff_reason`. The coordinator body never branches on the kickoff result, because that would break replay.
Note that training is launched by an _activity_, not a child workflow, because the actual agent run happens in a Tasks sandbox with its own lifecycle. Autoresearch does not own that workflow; it fires it and the `TaskRun` `post_save` signal (`../training/ingestion.py`) picks the result back up.
diff --git a/products/autoresearch/backend/temporal/test_workflows.py b/products/autoresearch/backend/temporal/test_workflows.py
index b37615a05b32..4425a0b36c54 100644
--- a/products/autoresearch/backend/temporal/test_workflows.py
+++ b/products/autoresearch/backend/temporal/test_workflows.py
@@ -633,6 +633,53 @@ class TestCoordinatorActivities(TeamScopedTestMixin, BaseTest):
assert result.reason == expected_reason
mock_run_training.assert_not_called()
+ @parameterized.expand(
+ [
+ ("sandbox_run_on_another_pipeline_takes_the_slot", "sandbox", 2, None, "team_daily_cap"),
+ ("live_launch_without_a_task_yet_takes_the_slot", "launching", 2, None, "team_daily_cap"),
+ ("run_older_than_the_window_does_not_count", "sandbox", 25, None, "explore"),
+ ("external_agent_run_does_not_count", "external", 2, None, "explore"),
+ ("team_override_raises_the_cap", "sandbox", 2, "this_team", "explore"),
+ ("override_for_another_team_does_not_apply", "sandbox", 2, "other_team", "team_daily_cap"),
+ ]
+ )
+ @patch("products.autoresearch.backend.limits.posthoganalytics.get_feature_flag_payload")
+ @patch("products.autoresearch.backend.temporal.workflows.run_training")
+ def test_kickoff_daily_cap_counts_every_pipeline_of_the_team(
+ self,
+ _name: str,
+ prior_run: str,
+ hours_ago: int,
+ override_for: Optional[str],
+ expected_reason: str,
+ mock_run_training: MagicMock,
+ mock_payload: MagicMock,
+ ) -> None:
+ override_team_id = {"this_team": self.team.id, "other_team": self.team.id + 1}.get(override_for or "")
+ mock_payload.return_value = {
+ "default_team_config": {"max_training_runs_per_day": 1},
+ "team_configs": {str(override_team_id): {"max_training_runs_per_day": 2}} if override_team_id else {},
+ }
+ other_pipeline = self._create_pipeline(name="other pipeline")
+ with time_machine.travel(django_timezone.now() - timedelta(hours=hours_ago), tick=False):
+ AutoresearchTrainingRun.objects.create(
+ pipeline=other_pipeline,
+ status=AutoresearchTrainingRun.Status.COMPLETED
+ if prior_run == "sandbox"
+ else AutoresearchTrainingRun.Status.RUNNING,
+ kickoff_reason="explore" if prior_run != "external" else "manual",
+ iteration_budget=3,
+ task_id=uuid.uuid4() if prior_run == "sandbox" else None,
+ agent_runtime_adapter=None if prior_run == "external" else "claude",
+ )
+ pipeline = self._create_pipeline()
+
+ result = activity_kickoff_training(KickoffTrainingInput(pipeline_id=str(pipeline.id), team_id=self.team.id))
+
+ assert result.reason == expected_reason
+ assert result.error is None
+ assert mock_run_training.called is (expected_reason == "explore")
+
@parameterized.expand(
[
("new_pipeline_explores", 7, [], False, "explore"),
diff --git a/products/autoresearch/backend/temporal/workflows.py b/products/autoresearch/backend/temporal/workflows.py
index fc08ecb48b5b..9a7bfbadfe72 100644
--- a/products/autoresearch/backend/temporal/workflows.py
+++ b/products/autoresearch/backend/temporal/workflows.py
@@ -53,6 +53,7 @@ with workflow.unsafe.imports_passed_through():
run_inference_for_pipeline,
score_shadow_set_for_run,
)
+ from products.autoresearch.backend.limits import check_training_run_cap, limits_for_team
from products.autoresearch.backend.models import (
AutoresearchModel,
AutoresearchPipeline,
@@ -732,6 +733,7 @@ class KickoffTrainingResult:
kicked_off: bool
# "explore" | "scheduled" | "not_due" | "manual_only" | "already_running" | "already_ran_today"
# | "not_eligible" | "tasks_gated" | "no_creator" | "not_launchable" | "rescue_unscorable" | "rescue_throttled"
+ # | "team_daily_cap"
reason: str
error: Optional[str] = None
# Why an expected skip happened, without counting it as a failure.
@@ -879,6 +881,7 @@ def activity_kickoff_training(inp: KickoffTrainingInput) -> KickoffTrainingResul
loaded = AutoresearchPipeline.objects.select_related("created_by", "team__organization").get(pk=inp.pipeline_id)
if loaded.created_by is not None and (gate := _tasks_gate(loaded)):
return KickoffTrainingResult(kicked_off=False, reason="tasks_gated", detail=gate)
+ limits = limits_for_team(inp.team_id)
with team_scope(inp.team_id), transaction.atomic():
pipeline = AutoresearchPipeline.objects.select_for_update().get(pk=inp.pipeline_id)
@@ -937,6 +940,11 @@ def activity_kickoff_training(inp: KickoffTrainingInput) -> KickoffTrainingResul
)
return KickoffTrainingResult(kicked_off=False, reason="no_creator", error=error)
+ # Checked last, so a pipeline with no run due does not wait on the team's advisory lock.
+ if cap := check_training_run_cap(inp.team_id, limits):
+ detail = f"{cap.used} of {cap.limit} training runs in the last 24 hours"
+ return KickoffTrainingResult(kicked_off=False, reason="team_daily_cap", detail=detail)
+
rescue = kickoff_reason == AutoresearchTrainingRun.KickoffReason.RESCUE
try:
with transaction.atomic():
@@ -946,6 +954,7 @@ def activity_kickoff_training(inp: KickoffTrainingInput) -> KickoffTrainingResul
user_id=pipeline.created_by_id,
rescue=rescue,
kickoff_reason=kickoff_reason,
+ limits=limits,
)
except (ValueError, SandboxInferenceError, ActionNotFound) as exc:
# run_training refuses a departed creator or an unresolvable target before it writes
diff --git a/products/autoresearch/backend/tests/test_api.py b/products/autoresearch/backend/tests/test_api.py
index 4ebaebcc2286..8150ad1965c3 100644
--- a/products/autoresearch/backend/tests/test_api.py
+++ b/products/autoresearch/backend/tests/test_api.py
@@ -916,7 +916,7 @@ class TestAutoresearchPipelineAPI(TeamScopedTestMixin, APIBaseTest):
# The run must not exist before the call: the endpoint rejects pipelines that
# already have a live run, so the mock creates it the way run_training would.
def _fake_run_training(
- *, pipeline: AutoresearchPipeline, iteration_budget: int, user_id: Any, kickoff_reason: str
+ *, pipeline: AutoresearchPipeline, iteration_budget: int, user_id: Any, kickoff_reason: str, limits: Any
):
return AutoresearchTrainingRun.objects.create(
pipeline=pipeline,
@@ -1079,6 +1079,36 @@ class TestAutoresearchPipelineAPI(TeamScopedTestMixin, APIBaseTest):
assert resp.status_code == status.HTTP_400_BAD_REQUEST
mock_run_training.assert_not_called()
+ @parameterized.expand([("limit_reached", 1), ("training_turned_off", 0)])
+ def test_start_training_at_the_project_daily_cap_returns_429(self, _name: str, limit: int):
+ other_pipeline = self._make_pipeline(name="Other pipeline")
+ started_at = django_timezone.now() - timedelta(hours=3)
+ AutoresearchTrainingRun.objects.create(
+ pipeline=other_pipeline,
+ status=AutoresearchTrainingRun.Status.COMPLETED,
+ iteration_budget=5,
+ task_id=uuid.uuid4(),
+ )
+ AutoresearchTrainingRun.objects.filter(pipeline=other_pipeline).update(created_at=started_at)
+ pipeline = self._make_pipeline()
+ with (
+ patch(
+ "products.autoresearch.backend.limits.posthoganalytics.get_feature_flag_payload",
+ return_value={"default_team_config": {"max_training_runs_per_day": limit}},
+ ),
+ patch("products.autoresearch.backend.training.runner.run_training") as mock_run_training,
+ ):
+ resp = self.client.post(f"{self.base_url}/{pipeline.id}/train/")
+
+ assert resp.status_code == status.HTTP_429_TOO_MANY_REQUESTS, resp.content
+ body = resp.json()
+ assert (body["used"], body["limit"]) == (1, limit)
+ if limit:
+ assert body["next_slot_at"] == (started_at + timedelta(hours=24)).isoformat().replace("+00:00", "Z")
+ else:
+ assert body["next_slot_at"] is None
+ mock_run_training.assert_not_called()
+
# ──────────────────────────────────── update restrictions ─────────────────────────────────────
def _make_trained_pipeline(self) -> AutoresearchPipeline:
diff --git a/products/autoresearch/backend/training/AGENTS.md b/products/autoresearch/backend/training/AGENTS.md
index f314f1272a85..b19126f50496 100644
--- a/products/autoresearch/backend/training/AGENTS.md
+++ b/products/autoresearch/backend/training/AGENTS.md
@@ -12,6 +12,7 @@ The other half is `../inference/`, which consumes what this package produces and
- `runner.py`
The real path. `run_training()` creates the `AutoresearchTrainingRun` (status `RUNNING`) and fires `Task.create_and_run()` with `internal=True` and no repository, so the run shows up as an internal Task rather than in the normal Tasks list.
The brief carries user-authored text, so the sandbox token holds only `TRAINING_MCP_SCOPES` (the `execute-sql` reads, the autoresearch scopes, and `user:read`, which the PostHog MCP server needs to start a session), and an empty connector allowlist keeps the team's shared MCP connectors out of the sandbox.
+ `run_training()` lowers the requested iterations to the project's `max_iterations_per_run` (`../limits.py`), so scheduled and manual runs share one ceiling. The daily run cap is checked by its callers, inside the pipeline row lock.
The task runs on the model and reasoning effort that `agent_run_config_for()` in `agent_model.py` resolves, and the run records the runtime adapter, model and effort in `agent_runtime_adapter`, `agent_model` and `agent_reasoning_effort`.
`build_agent_description()` assembles the agent's brief — the target, the horizon, the population, and the contract for the bundle it must author.
When the `autoresearch-report-notebook` flag is on for the launching user, the token also holds `REPORT_NOTEBOOK_MCP_SCOPES` and the brief adds two Finalize steps: before complete, the agent creates one report notebook and passes its `short_id` to complete. After complete, it adds the SQL and chart cells, because the run's model row (and so its `model_explanation`) exists only from then on. `report.md` stays required either way.
diff --git a/products/autoresearch/backend/training/runner.py b/products/autoresearch/backend/training/runner.py
index 9c63e230aa81..6d3cff1eddf6 100644
--- a/products/autoresearch/backend/training/runner.py
+++ b/products/autoresearch/backend/training/runner.py
@@ -41,6 +41,7 @@ from products.autoresearch.backend.access import has_report_notebook_access
from products.autoresearch.backend.dataset.labeling import TrainingSample, build_target_condition
from products.autoresearch.backend.inference.failures import UnscorableChampion, find_unscorable_champion
from products.autoresearch.backend.inference.sandbox import _resolve_acting_user, measure_training_sample
+from products.autoresearch.backend.limits import TeamLimits, limits_for_team
from products.autoresearch.backend.models import (
AutoresearchModel,
AutoresearchPipeline,
@@ -944,6 +945,7 @@ def run_training(
user_id: int | None,
rescue: bool = False,
kickoff_reason: str = AutoresearchTrainingRun.KickoffReason.MANUAL,
+ limits: TeamLimits | None = None,
) -> AutoresearchTrainingRun:
"""
Launch a real agent sandbox training run.
@@ -952,6 +954,10 @@ def run_training(
to start the autoresearch agent. Returns the training run immediately; recipe
ingestion happens asynchronously when the TaskRun completes.
+ The run takes at most the project's ``max_iterations_per_run`` (``../limits.py``), whatever
+ the caller asked for. A caller that holds a lock passes the ``limits`` it read before taking
+ it, because the flag read can call out.
+
Raises if Temporal is unreachable or the task cannot be created.
"""
if user_id is None:
@@ -970,6 +976,8 @@ def run_training(
target_event=pipeline.target_event, target_definition=pipeline.target_definition, team=pipeline.team
)
agent_config = agent_run_config_for(pipeline)
+ limits = limits or limits_for_team(pipeline.team_id)
+ iteration_budget = min(iteration_budget, limits.max_iterations_per_run)
now = django_timezone.now()
with transaction.atomic():
diff --git a/products/autoresearch/backend/training/test_training.py b/products/autoresearch/backend/training/test_training.py
index f9ddb75b7e0c..12fa980d6d6f 100644
--- a/products/autoresearch/backend/training/test_training.py
+++ b/products/autoresearch/backend/training/test_training.py
@@ -15,6 +15,7 @@ from products.actions.backend.facade.testing import create_action
from products.autoresearch.backend.dataset.labeling import TrainingSample
from products.autoresearch.backend.inference.failures import UnscorableChampion
from products.autoresearch.backend.inference.sandbox import SandboxInferenceError
+from products.autoresearch.backend.limits import TeamLimits
from products.autoresearch.backend.models import (
AutoresearchModel,
AutoresearchPipeline,
@@ -380,6 +381,40 @@ class TestRunTraining(TeamScopedTestMixin, BaseTest):
self.pipeline.refresh_from_db()
assert self.pipeline.status == AutoresearchPipeline.Status.BOOTSTRAPPING
+ @parameterized.expand(
+ [
+ ("default_ceiling", None, None, 25, 10),
+ ("under_the_ceiling", None, None, 5, 5),
+ ("team_override_raises_the_ceiling", 30, None, 25, 25),
+ ("caller_limits_skip_the_flag_read", 30, 7, 25, 7),
+ ]
+ )
+ def test_dispatch_caps_iterations_at_the_project_limit(
+ self,
+ facade: MagicMock,
+ _name: str,
+ team_ceiling: int | None,
+ caller_ceiling: int | None,
+ requested: int,
+ expected: int,
+ ) -> None:
+ self._dispatched(facade)
+ payload = (
+ {"team_configs": {str(self.team.id): {"max_iterations_per_run": team_ceiling}}} if team_ceiling else None
+ )
+ limits = (
+ TeamLimits(max_training_runs_per_day=3, max_iterations_per_run=caller_ceiling) if caller_ceiling else None
+ )
+
+ with patch(
+ "products.autoresearch.backend.limits.posthoganalytics.get_feature_flag_payload", return_value=payload
+ ) as flag_read:
+ training_run = run_training(self.pipeline, iteration_budget=requested, user_id=self.user.id, limits=limits)
+
+ assert training_run.iteration_budget == expected
+ assert flag_read.called is (limits is None)
+ assert f"**Iteration budget**: {expected}" in facade.create_and_run_task.call_args.kwargs["description"]
+
@parameterized.expand(
[
("blank_settings", None, None, ("claude", "claude-opus-5-5", "medium")),
diff --git a/products/autoresearch/frontend/autoresearchLogic.ts b/products/autoresearch/frontend/autoresearchLogic.ts
index 5e13cfb81905..7f9b083baebf 100644
--- a/products/autoresearch/frontend/autoresearchLogic.ts
+++ b/products/autoresearch/frontend/autoresearchLogic.ts
@@ -290,7 +290,11 @@ export const autoresearchLogic = kea<autoresearchLogicType>([
lemonToast.success(`Started training "${pipeline.name}"`)
actions.loadPipelines()
} catch (error: any) {
- posthog.capture('autoresearch model action failed', { action: 'train', pipeline_id: pipeline.id })
+ posthog.capture('autoresearch model action failed', {
+ action: 'train',
+ pipeline_id: pipeline.id,
+ status: error?.status,
+ })
lemonToast.error(error?.detail ?? error?.data?.detail ?? 'Failed to start training')
} finally {
actions.setPipelineMutating(pipeline.id, false)
diff --git a/products/autoresearch/frontend/autoresearchPipelineLogic.ts b/products/autoresearch/frontend/autoresearchPipelineLogic.ts
index 104fc29b3175..8f58d797ab04 100644
--- a/products/autoresearch/frontend/autoresearchPipelineLogic.ts
+++ b/products/autoresearch/frontend/autoresearchPipelineLogic.ts
@@ -1695,9 +1695,14 @@ export const autoresearchPipelineLogic = kea<autoresearchPipelineLogicType>([
actions.loadPipeline()
lemonToast.success('Training run started')
},
- startTrainingFailure: () => {
- posthog.capture('autoresearch model action failed', { action: 'train', pipeline_id: props.id })
- lemonToast.error('Could not start training run')
+ startTrainingFailure: ({ errorObject }) => {
+ posthog.capture('autoresearch model action failed', {
+ action: 'train',
+ pipeline_id: props.id,
+ status: errorObject?.status,
+ })
+ // A project at its daily training limit explains when the next run can start in `detail`.
+ lemonToast.error(errorObject?.detail ?? errorObject?.data?.detail ?? 'Could not start training run')
},
pausePipelineSuccess: () => {
posthog.capture('autoresearch model paused', { pipeline_id: props.id })
diff --git a/products/autoresearch/frontend/generated/api.schemas.ts b/products/autoresearch/frontend/generated/api.schemas.ts
index fdc97a14be5c..229f7a69a151 100644
--- a/products/autoresearch/frontend/generated/api.schemas.ts
+++ b/products/autoresearch/frontend/generated/api.schemas.ts
@@ -1909,13 +1909,27 @@ export interface OnlinePerformanceApi {
export interface StartTrainingRequestApi {
/**
- * Iterations for this training run. Defaults to the pipeline's iterations_per_run.
+ * Iterations for this training run. Defaults to the pipeline's iterations_per_run. The project's iteration limit per run can lower it.
* @minimum 1
- * @maximum 500
+ * @maximum 50
*/
iteration_budget?: number
}
+export interface TrainingRunCapReachedApi {
+ /** Why the run did not start. */
+ detail: string
+ /** Sandbox training runs the project started in the last 24 hours, across all its pipelines. */
+ used: number
+ /** Sandbox training runs the project may start in any 24 hours. */
+ limit: number
+ /**
+ * When the next training run can start. Null when training runs are turned off for the project.
+ * @nullable
+ */
+ next_slot_at: string | null
+}
+
export interface AgentReasoningEffortOptionApi {
/** Effort id to pass as agent_reasoning_effort, e.g. 'medium'. */
value: string
diff --git a/products/autoresearch/frontend/generated/api.zod.ts b/products/autoresearch/frontend/generated/api.zod.ts
index 4728b6ac8fab..e81c01cd067d 100644
--- a/products/autoresearch/frontend/generated/api.zod.ts
+++ b/products/autoresearch/frontend/generated/api.zod.ts
@@ -843,7 +843,7 @@ export const AutoresearchPartialUpdateBody = /* @__PURE__ */ zod.object({
* Start an asynchronous training run for this pipeline. Creates a Task/TaskRun sandbox where the autoresearch agent iterates on features and models, and returns the run immediately with status 'running'. Poll the training run until it reaches a terminal status (completed or failed). A pipeline's first run has no champion until it completes and promotion runs; on a retrain the existing champion stays live and keeps scoring until a new one is promoted.
* @summary Start a training run
*/
-export const autoresearchTrainCreateBodyIterationBudgetMax = 500
+export const autoresearchTrainCreateBodyIterationBudgetMax = 50
export const AutoresearchTrainCreateBody = /* @__PURE__ */ zod.object({
iteration_budget: zod
@@ -851,7 +851,9 @@ export const AutoresearchTrainCreateBody = /* @__PURE__ */ zod.object({
.min(1)
.max(autoresearchTrainCreateBodyIterationBudgetMax)
.optional()
- .describe("Iterations for this training run. Defaults to the pipeline's iterations_per_run."),
+ .describe(
+ "Iterations for this training run. Defaults to the pipeline's iterations_per_run. The project's iteration limit per run can lower it."
+ ),
})
/**
diff --git a/products/autoresearch/mcp/tools.yaml b/products/autoresearch/mcp/tools.yaml
index 0536e0432fc1..082a6175c5a2 100644
--- a/products/autoresearch/mcp/tools.yaml
+++ b/products/autoresearch/mcp/tools.yaml
@@ -496,8 +496,11 @@ tools:
because the run is a cloud sandbox. After a run promotes a champion, the daily 02:00 UTC scoring job scores
the inference population and emits autoresearch_prediction events. To score now instead, call
autoresearch-score-create once. The run uses the pipeline's iterations_per_run unless you override
- iteration_budget for this run. A manual run resets the clock for the next scheduled research run. A pipeline
- with an action target also needs the action:read scope.
+ iteration_budget for this run, and the project's iteration limit per run can lower either. Returns 429 when
+ the project has started its limit of training runs in the last 24 hours, across all its pipelines: do not
+ retry before next_slot_at in the response, and tell the user when the next run can start. A manual run
+ resets the clock for the next scheduled research run. A pipeline with an action target also needs the
+ action:read scope.
feature_flag: autoresearch
autoresearch-training-runs-artifacts-delete-create:
operation: autoresearch_training_runs_artifacts_delete_create
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index 3eb7dba86288..1f09dce9fb24 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -1642,7 +1642,7 @@
"feature_flag": "autoresearch"
},
"autoresearch-train-create": {
- "description": "Start an asynchronous training run for a pipeline. Launches the autoresearch agent in a sandbox and returns the training run immediately with status 'running'. Poll autoresearch-training-runs-list until the run reaches 'completed' or 'failed'. A pipeline's first run has no champion until it completes and server-side promotion runs; on a retrain the existing champion keeps scoring until a new one is promoted. Returns 400 if a training run is already in progress, the pipeline is paused, or its creator or target is no longer valid (recreate the pipeline). Returns 403 from inside a sandbox, such as a PostHog AI cloud task: do not retry, and ask the user to start training from the pipeline's page in PostHog. Needs PostHog Desktop access, because the run is a cloud sandbox. After a run promotes a champion, the daily 02:00 UTC scoring job scores the inference population and emits autoresearch_prediction events. To score now instead, call autoresearch-score-create once. The run uses the pipeline's iterations_per_run unless you override iteration_budget for this run. A manual run resets the clock for the next scheduled research run. A pipeline with an action target also needs the action:read scope.",
+ "description": "Start an asynchronous training run for a pipeline. Launches the autoresearch agent in a sandbox and returns the training run immediately with status 'running'. Poll autoresearch-training-runs-list until the run reaches 'completed' or 'failed'. A pipeline's first run has no champion until it completes and server-side promotion runs; on a retrain the existing champion keeps scoring until a new one is promoted. Returns 400 if a training run is already in progress, the pipeline is paused, or its creator or target is no longer valid (recreate the pipeline). Returns 403 from inside a sandbox, such as a PostHog AI cloud task: do not retry, and ask the user to start training from the pipeline's page in PostHog. Needs PostHog Desktop access, because the run is a cloud sandbox. After a run promotes a champion, the daily 02:00 UTC scoring job scores the inference population and emits autoresearch_prediction events. To score now instead, call autoresearch-score-create once. The run uses the pipeline's iterations_per_run unless you override iteration_budget for this run, and the project's iteration limit per run can lower either. Returns 429 when the project has started its limit of training runs in the last 24 hours, across all its pipelines: do not retry before next_slot_at in the response, and tell the user when the next run can start. A manual run resets the clock for the next scheduled research run. A pipeline with an action target also needs the action:read scope.",
"category": "Autoresearch",
"feature": "autoresearch",
"summary": "Start training",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index bfd0e41def4a..c49c2644a2ed 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -1657,7 +1657,7 @@
"feature_flag": "autoresearch"
},
"autoresearch-train-create": {
- "description": "Start an asynchronous training run for a pipeline. Launches the autoresearch agent in a sandbox and returns the training run immediately with status 'running'. Poll autoresearch-training-runs-list until the run reaches 'completed' or 'failed'. A pipeline's first run has no champion until it completes and server-side promotion runs; on a retrain the existing champion keeps scoring until a new one is promoted. Returns 400 if a training run is already in progress, the pipeline is paused, or its creator or target is no longer valid (recreate the pipeline). Returns 403 from inside a sandbox, such as a PostHog AI cloud task: do not retry, and ask the user to start training from the pipeline's page in PostHog. Needs PostHog Desktop access, because the run is a cloud sandbox. After a run promotes a champion, the daily 02:00 UTC scoring job scores the inference population and emits autoresearch_prediction events. To score now instead, call autoresearch-score-create once. The run uses the pipeline's iterations_per_run unless you override iteration_budget for this run. A manual run resets the clock for the next scheduled research run. A pipeline with an action target also needs the action:read scope.",
+ "description": "Start an asynchronous training run for a pipeline. Launches the autoresearch agent in a sandbox and returns the training run immediately with status 'running'. Poll autoresearch-training-runs-list until the run reaches 'completed' or 'failed'. A pipeline's first run has no champion until it completes and server-side promotion runs; on a retrain the existing champion keeps scoring until a new one is promoted. Returns 400 if a training run is already in progress, the pipeline is paused, or its creator or target is no longer valid (recreate the pipeline). Returns 403 from inside a sandbox, such as a PostHog AI cloud task: do not retry, and ask the user to start training from the pipeline's page in PostHog. Needs PostHog Desktop access, because the run is a cloud sandbox. After a run promotes a champion, the daily 02:00 UTC scoring job scores the inference population and emits autoresearch_prediction events. To score now instead, call autoresearch-score-create once. The run uses the pipeline's iterations_per_run unless you override iteration_budget for this run, and the project's iteration limit per run can lower either. Returns 429 when the project has started its limit of training runs in the last 24 hours, across all its pipelines: do not retry before next_slot_at in the response, and tell the user when the next run can start. A manual run resets the clock for the next scheduled research run. A pipeline with an action target also needs the action:read scope.",
"category": "Autoresearch",
"feature": "autoresearch",
"summary": "Start training",
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index acdeed5ad65c..02c530c6c8b6 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -108973,9 +108973,9 @@ export namespace Schemas {
export interface StartTrainingRequest {
/**
- * Iterations for this training run. Defaults to the pipeline's iterations_per_run.
+ * Iterations for this training run. Defaults to the pipeline's iterations_per_run. The project's iteration limit per run can lower it.
* @minimum 1
- * @maximum 500
+ * @maximum 50
*/
iteration_budget?: number;
}
@@ -113824,6 +113824,20 @@ export namespace Schemas {
regenerate?: boolean;
}
+ export interface TrainingRunCapReached {
+ /** Why the run did not start. */
+ detail: string;
+ /** Sandbox training runs the project started in the last 24 hours, across all its pipelines. */
+ used: number;
+ /** Sandbox training runs the project may start in any 24 hours. */
+ limit: number;
+ /**
+ * When the next training run can start. Null when training runs are turned off for the project.
+ * @nullable
+ */
+ next_slot_at: string | null;
+ }
+
/**
* One prior completed training run plus its full iteration trail.
*/
diff --git a/services/mcp/src/generated/autoresearch/api.ts b/services/mcp/src/generated/autoresearch/api.ts
index c1b316a0e118..004a3a7fd290 100644
--- a/services/mcp/src/generated/autoresearch/api.ts
+++ b/services/mcp/src/generated/autoresearch/api.ts
@@ -1008,7 +1008,7 @@ export const AutoresearchTrainCreateParams = () => zod.object({
),
})
-export const autoresearchTrainCreateBodyIterationBudgetMax = 500
+export const autoresearchTrainCreateBodyIterationBudgetMax = 50
export const AutoresearchTrainCreateBody = () => zod.object({
iteration_budget: zod
@@ -1016,7 +1016,9 @@ export const AutoresearchTrainCreateBody = () => zod.object({
.min(1)
.max(autoresearchTrainCreateBodyIterationBudgetMax)
.optional()
- .describe("Iterations for this training run. Defaults to the pipeline's iterations_per_run."),
+ .describe(
+ "Iterations for this training run. Defaults to the pipeline's iterations_per_run. The project's iteration limit per run can lower it."
+ ),
})
/**