:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(autoresearch): close lost scoring runs and retry them the same day (#119224)
posthog[bot]08c3ef1caee8
diff --git a/products/autoresearch/backend/inference/failures.py b/products/autoresearch/backend/inference/failures.pyindex 82e1c5700ce0..2ff2beb9bbce 100644--- a/products/autoresearch/backend/inference/failures.py+++ b/products/autoresearch/backend/inference/failures.py@@ -150,27 +150,76 @@ def _repeatable_failure_days(newest_first: Iterable[AutoresearchRun]) -> dict[dareturn failed_days-def fail_stale_inference_runs() -> int:- """- Fail every inference run, across all teams, that is still RUNNING after ``INFERENCE_WORKFLOW_TIMEOUT``.-- A lost worker, a heartbeat timeout, or a cancellation ends the attempt before the scoring code- records an outcome. The kind is ``other``, so ``find_unscorable_champion`` does not count the run.- """+def _fail_lost_runs(runs: QuerySet[AutoresearchRun]) -> int:now = django_timezone.now()- stale = AutoresearchRun.objects.unscoped().filter(- run_type=AutoresearchRun.RunType.INFERENCE,- status=AutoresearchRun.Status.RUNNING,- started_at__lt=now - INFERENCE_WORKFLOW_TIMEOUT,- )failed = 0- for run in stale:+ for run in runs:run.status = AutoresearchRun.Status.FAILEDrun.error = "Scoring stopped without a result. The worker was lost or the attempt timed out."run.metrics["failure_kind"] = FailureKind.OTHERrun.completed_at = nowrun.save(update_fields=["status", "error", "metrics", "completed_at"])failed += 1+ return failed+++def fail_stale_inference_runs() -> int:+ """+ Fail every inference run, across all teams, that is still RUNNING after ``INFERENCE_WORKFLOW_TIMEOUT``.++ A lost worker, a heartbeat timeout, or a cancellation ends the attempt before the scoring code+ records an outcome. The kind is ``other``, so ``find_unscorable_champion`` does not count the run.+ This is the backstop for ``fail_lost_scheduled_runs``.+ """+ failed = _fail_lost_runs(+ AutoresearchRun.objects.unscoped().filter(+ run_type=AutoresearchRun.RunType.INFERENCE,+ status=AutoresearchRun.Status.RUNNING,+ started_at__lt=django_timezone.now() - INFERENCE_WORKFLOW_TIMEOUT,+ )+ )if failed:logger.warning("autoresearch_stale_inference_runs_failed", count=failed)return failed+++def _scheduled_runs(team_id: int, pipeline_id: str, prediction_date: str) -> QuerySet[AutoresearchRun]:+ return AutoresearchRun.objects.for_team(team_id).filter(+ pipeline__pk=pipeline_id,+ run_type=AutoresearchRun.RunType.INFERENCE,+ scheduled=True,+ metrics__prediction_date=prediction_date,+ )+++def fail_lost_scheduled_runs(+ *, team_id: int, pipeline_id: str, prediction_date: str, workflow_run_id: str | None = None+) -> int:+ """+ Fail the scheduled inference runs of one pipeline and date that are still RUNNING.++ Call it only after the inference workflow of that date has ended, because then no attempt can close them.+ With ``workflow_run_id`` it fails only the rows of that child execution, so a catch-up child that+ started for the same date in the meantime keeps its live row.+ """+ runs = _scheduled_runs(team_id, pipeline_id, prediction_date).filter(status=AutoresearchRun.Status.RUNNING)+ if workflow_run_id is not None:+ runs = runs.filter(metrics__workflow_run_id=workflow_run_id)+ failed = _fail_lost_runs(runs)+ if failed:+ logger.warning("autoresearch_lost_inference_runs_failed", pipeline_id=pipeline_id, count=failed)+ return failed+++def scheduled_run_can_retry(*, team_id: int, pipeline_id: str, prediction_date: str) -> bool:+ """+ True when the latest scheduled run of the date failed with a kind that can pass on a retry.++ A pipeline with no scheduled run for the date returns False, because the daily sweep may not have reached it yet.+ """+ latest = _scheduled_runs(team_id, pipeline_id, prediction_date).order_by("-created_at").first()+ return (+ latest is not None+ and latest.status == AutoresearchRun.Status.FAILED+ and (latest.metrics or {}).get("failure_kind") not in REPEATABLE_FAILURE_KINDS+ )diff --git a/products/autoresearch/backend/temporal/AGENTS.md b/products/autoresearch/backend/temporal/AGENTS.mdindex 49de8787cdb8..6e84b246df3a 100644--- a/products/autoresearch/backend/temporal/AGENTS.md+++ b/products/autoresearch/backend/temporal/AGENTS.md@@ -7,7 +7,8 @@ Scoring, validation and training live in `../inference/`, `../evaluation/`, and## What lives here- `workflows.py`- - `AutoresearchCoordinatorWorkflow` (`autoresearch-coordinator`) — the daily tick. Loads active pipelines (`activity_load_active_pipelines`) and runs inference, training kickoff, and validation per pipeline, with at most `CoordinatorWorkflowInput.max_concurrency` pipelines in flight (`DEFAULT_COORDINATOR_CONCURRENCY`, 4). After every champion has scored, it runs `activity_run_shadow_scoring` per completed champion run, under the same bound.+ - `AutoresearchCoordinatorWorkflow` (`autoresearch-coordinator`) — the daily tick. Loads active pipelines (`activity_load_active_pipelines`) and runs inference, training kickoff, and validation per pipeline, with at most `CoordinatorWorkflowInput.max_concurrency` pipelines in flight (`DEFAULT_COORDINATOR_CONCURRENCY`, 4). After every champion has scored, it runs `activity_run_shadow_scoring` per completed champion run, under the same bound. With `CoordinatorWorkflowInput.catch_up` it runs a catch-up sweep, which scores only.+ - `activity_fail_lost_inference_runs` — fails the scheduled runs of one pipeline and date that are still `running` (`fail_lost_scheduled_runs()` in `../inference/failures.py`). The coordinator starts it when an inference child ends in a `ChildWorkflowError`, which covers a failed last attempt and the workflow timeout. A child that did not start (its id is already running) does not trigger it, so a live run stays open. `activity_run_inference` records `metrics["workflow_run_id"]` on a scheduled run, and the cleanup fails only the rows of the failed child's run id, so a catch-up child that started for the same date keeps its live row.- `AutoresearchInferenceWorkflow` (`autoresearch-inference`) — `activity_run_inference` for one pipeline. The activity reads the current champion itself, so a retry after a mid-scoring promotion scores with the new one. The sweep sets `defer_shadows`, so the activity scores the champion only.- `activity_run_shadow_scoring` — scores the shadow set against the people a champion's completed run scored (`score_shadow_set_for_run()` in `../inference/scoring.py`). It takes only the run id. It has one attempt, because a retry would open a second shadow run per model, and its failure never fails the champion's run or counts as a pipeline error.- `AutoresearchValidationWorkflow` (`autoresearch-validation`) — `activity_run_validation` for one pipeline.@@ -17,7 +18,9 @@ Scoring, validation and training live in `../inference/`, `../evaluation/`, andEach workflow has a typed input/result dataclass pair (`InferenceWorkflowInput` / `InferenceWorkflowResult`, and so on). Activities are named with a `autoresearch-<workflow>.<activity>` convention.- `schedule.py`- `create_autoresearch_daily_schedule()` registers schedule `autoresearch-daily-coordinator` driving workflow id `autoresearch-coordinator`.+ `create_autoresearch_daily_schedule()` registers schedule `autoresearch-daily-coordinator` driving workflow id `autoresearch-coordinator` at 02:00 UTC, and schedule `autoresearch-catch-up-coordinator` driving workflow id `autoresearch-coordinator-catch-up` at 08:00, 14:00 and 20:00 UTC with `catch_up=True`.+ The catch-up fires more than once, because an inference child can fail up to `INFERENCE_WORKFLOW_TIMEOUT` after it starts, and the daily sweep starts children until its own timeout. Each tick retries the runs lost since the tick before it, so a pipeline gets at most three retries a day.+ The catch-up has its own workflow id. The daily coordinator marks its date `running` in the cache before discovery and `done` after its last scoring, kickoff, validation and shadow step (`activity_mark_daily_sweep`, key `daily_sweep_key()`). A catch-up tick lists no pipelines while the marker of its date reads `running`, so the two sweeps never run past one concurrency bound together. The next tick tries again. The `running` marker expires with `COORDINATOR_TIMEOUT`, so a coordinator that dies without marking `done` holds the catch-up back no longer than it could still run.Overlap policy is **SKIP** — a tick that is still running causes the next one to be dropped rather than queued, so a slow day cannot pile up a backlog of duplicate scoring runs.An update keeps the schedule's live state, so a deploy does not resume a schedule an operator paused.The queue defaults to `self-driving-task-queue`, which the temporal-worker-self-driving fleet polls. Deploy a worker that polls a new queue before you set `AUTORESEARCH_TASK_QUEUE` to it, or the schedule strands its runs.@@ -32,10 +35,15 @@ daily schedule├─ activity_load_active_pipelines├─ per pipeline, at most max_concurrency at once, one step at a time:│ ├─ AutoresearchInferenceWorkflow → ../inference/ (champion only)+ │ │ └─ on ChildWorkflowError: activity_fail_lost_inference_runs│ ├─ activity_kickoff_training → ../training/│ └─ AutoresearchValidationWorkflow → ../evaluation/└─ after every pipeline above, per completed champion run, under the same bound:└─ activity_run_shadow_scoring → ../inference/++catch-up schedule (08:00, 14:00, 20:00 UTC, catch_up=True)+ └─ AutoresearchCoordinatorWorkflow+ └─ the same tree, for lost pipelines only, without kickoff and validation```The bound exists because each scoring activity holds a feature matrix in worker memory and runs batch queries. An unbounded sweep started every pipeline at the same moment, which overloaded ClickHouse and got the shared worker OOM-killed. A pipeline holds its slot through all its steps, so validation never runs next to its own scoring. Discovery lists the due pipelines that did not score yesterday first, so they get the first slots. The ordering and the bound come from recorded history (the activity result and the workflow input), so they are deterministic.@@ -45,7 +53,9 @@ The coordinator decides _whether_ each pipeline is due; the child workflows do oValidation runs on every daily sweep, because a matured date should not wait for the next scoring day.Scoring and training kickoff run on a cadence day only, compared by calendar day, and kickoff starts after scoring ends. Validation starts after kickoff.Discovery pauses a pipeline whose creator has lost access to the team, and skips one outside the flag rollout.-Discovery also fails every inference run still RUNNING after `INFERENCE_WORKFLOW_TIMEOUT` (`fail_stale_inference_runs()` in `../inference/failures.py`), with `failure_kind` `other`, because a lost worker never records an outcome.+When an inference child ends in an exception, the coordinator fails the runs it left RUNNING, with `failure_kind` `other`, because a lost worker never records an outcome. This is behind `workflow.patched("autoresearch-close-lost-runs-2026-10")`.+Discovery also fails every inference run still RUNNING after `INFERENCE_WORKFLOW_TIMEOUT` (`fail_stale_inference_runs()` in `../inference/failures.py`). This is the backstop for a run that the coordinator did not close.+The catch-up sweep lists only the due pipelines whose latest scheduled run of the day failed with a kind a retry can pass (`scheduled_run_can_retry()`). A pipeline with no run for the day is skipped, because the daily sweep may not have reached it yet, and a second start would score it twice. Its inference children use `ALLOW_DUPLICATE_FAILED_ONLY`, so a catch-up starts only after the daily child of the date failed or timed out. It skips training kickoff and validation, because the daily sweep already ran them.A retry of a scheduled scoring attempt fills the RUNNING row of the attempt before it, so one date keeps one row.Kickoff applies the Tasks entitlement and usage gates `/train` applies, launches at most one run per UTC day, and holds the pipeline row lock through the launch, as `start_training` does.Kickoff then follows the research schedule in `../training/schedule.py`, which it derives from the pipeline's training runs and never stores.diff --git a/products/autoresearch/backend/temporal/__init__.py b/products/autoresearch/backend/temporal/__init__.pyindex 17bda4978e7e..8fc4e2aa72e5 100644--- a/products/autoresearch/backend/temporal/__init__.py+++ b/products/autoresearch/backend/temporal/__init__.py@@ -11,8 +11,10 @@ from .workflows import (AutoresearchPropertyCleanupWorkflow,AutoresearchValidationWorkflow,activity_fail_cleanup_run,+ activity_fail_lost_inference_runs,activity_kickoff_training,activity_load_active_pipelines,+ activity_mark_daily_sweep,activity_remove_person_property,activity_run_inference,activity_run_shadow_scoring,@@ -28,8 +30,10 @@ WORKFLOWS = [ACTIVITIES = [activity_fail_cleanup_run,+ activity_fail_lost_inference_runs,activity_kickoff_training,activity_load_active_pipelines,+ activity_mark_daily_sweep,activity_remove_person_property,activity_run_inference,activity_run_shadow_scoring,diff --git a/products/autoresearch/backend/temporal/schedule.py b/products/autoresearch/backend/temporal/schedule.pyindex af215bfe50cb..b75c922e7e4e 100644--- a/products/autoresearch/backend/temporal/schedule.py+++ b/products/autoresearch/backend/temporal/schedule.py@@ -2,8 +2,6 @@from __future__ import annotations-from datetime import timedelta-from django.conf import settingsfrom temporalio.client import (@@ -19,44 +17,73 @@ from temporalio.client import (from posthog.temporal.common.schedule import a_create_schedule, a_schedule_exists, a_update_schedule-from products.autoresearch.backend.temporal.workflows import CoordinatorWorkflowInput+from products.autoresearch.backend.temporal.workflows import COORDINATOR_TIMEOUT, CoordinatorWorkflowInput_SCHEDULE_ID = "autoresearch-daily-coordinator"_WORKFLOW_ID = "autoresearch-coordinator"+_CATCH_UP_SCHEDULE_ID = "autoresearch-catch-up-coordinator"+_CATCH_UP_WORKFLOW_ID = "autoresearch-coordinator-catch-up"+# An inference child can fail up to INFERENCE_WORKFLOW_TIMEOUT after it starts, and the daily sweep starts+# children until its own timeout. Each tick retries the runs lost since the tick before it.+_CATCH_UP_HOURS = [8, 14, 20]-async def create_autoresearch_daily_schedule(client: Client) -> None:- """Create or update the daily schedule for the autoresearch coordinator workflow.-- Fires once per day at 2 AM UTC. The coordinator fans out inference, validation,- and training-kickoff for every active pipeline. SKIP overlap prevents a new run- from starting if the previous day's coordinator is still executing.- """- schedule = Schedule(+def _coordinator_schedule(+ *, workflow_id: str, inp: CoordinatorWorkflowInput, hours: list[int], comment: str+) -> Schedule:+ return Schedule(action=ScheduleActionStartWorkflow("autoresearch-coordinator",- CoordinatorWorkflowInput(),- id=_WORKFLOW_ID,+ inp,+ id=workflow_id,task_queue=settings.AUTORESEARCH_TASK_QUEUE,- # Longer than one pipeline's children (inference, then kickoff), shorter than a day, so a- # stuck coordinator ends before the next tick instead of making SKIP drop every later one.- execution_timeout=timedelta(hours=12),+ execution_timeout=COORDINATOR_TIMEOUT,),spec=ScheduleSpec(calendars=[- ScheduleCalendarSpec(- comment="Daily at 2 AM UTC",- hour=[ScheduleRange(start=2, end=2)],- )+ ScheduleCalendarSpec(comment=comment, hour=[ScheduleRange(start=hour, end=hour) for hour in hours])]),policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.SKIP),)- if await a_schedule_exists(client, _SCHEDULE_ID):++async def _create_or_update(client: Client, schedule_id: str, schedule: Schedule) -> None:+ if await a_schedule_exists(client, schedule_id):# Keep the live state, so a deploy does not resume a schedule an operator paused to stop spend.- description = await client.get_schedule_handle(_SCHEDULE_ID).describe()+ description = await client.get_schedule_handle(schedule_id).describe()schedule.state = description.schedule.state- await a_update_schedule(client, _SCHEDULE_ID, schedule)+ await a_update_schedule(client, schedule_id, schedule)else:- await a_create_schedule(client, _SCHEDULE_ID, schedule, trigger_immediately=False)+ await a_create_schedule(client, schedule_id, schedule, trigger_immediately=False)+++async def create_autoresearch_daily_schedule(client: Client) -> None:+ """Create or update the two daily schedules for the autoresearch coordinator workflow.++ The daily sweep fires at 2 AM UTC. The coordinator fans out inference, validation,+ and training-kickoff for every active pipeline. SKIP overlap prevents a new run+ from starting if the previous day's coordinator is still executing.++ The catch-up sweep fires at 8 AM, 2 PM and 8 PM UTC. Each tick scores again only the pipelines+ whose latest scheduled run of the day was lost, so a child that times out after one tick gets+ the next. It has its own workflow id, so it can run while the daily sweep is still working+ through its backlog.+ """+ await _create_or_update(+ client,+ _SCHEDULE_ID,+ _coordinator_schedule(+ workflow_id=_WORKFLOW_ID, inp=CoordinatorWorkflowInput(), hours=[2], comment="Daily at 2 AM UTC"+ ),+ )+ await _create_or_update(+ client,+ _CATCH_UP_SCHEDULE_ID,+ _coordinator_schedule(+ workflow_id=_CATCH_UP_WORKFLOW_ID,+ inp=CoordinatorWorkflowInput(catch_up=True),+ hours=_CATCH_UP_HOURS,+ comment="Catch-up at 8 AM, 2 PM and 8 PM UTC",+ ),+ )diff --git a/products/autoresearch/backend/temporal/test_workflows.py b/products/autoresearch/backend/temporal/test_workflows.pyindex ddcfe59ed94a..5f793e1e43c9 100644--- a/products/autoresearch/backend/temporal/test_workflows.py+++ b/products/autoresearch/backend/temporal/test_workflows.py@@ -1,5 +1,6 @@import uuidimport asyncio+from dataclasses import replacefrom datetime import UTC, datetime, timedeltafrom typing import Any, Optional@@ -8,6 +9,7 @@ import time_machinefrom posthog.test.base import BaseTestfrom unittest.mock import MagicMock, patch+from django.core.cache import cachefrom django.test import SimpleTestCasefrom django.utils import timezone as django_timezone@@ -38,11 +40,13 @@ from products.autoresearch.backend.temporal.workflows import (AutoresearchValidationWorkflow,CoordinatorWorkflowInput,FailCleanupRunInput,+ FailLostInferenceRunsInput,InferenceWorkflowResult,KickoffTrainingInput,KickoffTrainingResult,LoadActivePipelinesInput,LoadActivePipelinesResult,+ MarkDailySweepInput,PropertyCleanupWorkflowInput,PropertyCleanupWorkflowResult,RunInferenceInput,@@ -52,11 +56,14 @@ from products.autoresearch.backend.temporal.workflows import (RunValidationInput,RunValidationResult,ValidationWorkflowResult,+ activity_fail_lost_inference_runs,activity_kickoff_training,activity_load_active_pipelines,+ activity_mark_daily_sweep,activity_remove_person_property,activity_run_inference,activity_run_validation,+ daily_sweep_key,evaluate_pipeline_outcome,)from products.autoresearch.backend.testing import TeamScopedTestMixin@@ -350,13 +357,15 @@ class TestCoordinatorActivities(TeamScopedTestMixin, BaseTest):metrics={"prediction_date": row.get("prediction_date", "2026-09-11"), "horizon_days": 30},)mock_inference.return_value = MagicMock(pk="run-1", rows_scored=0, status="completed", error="")+ env = ActivityEnvironment()- ActivityEnvironment().run(+ env.run(activity_run_inference,RunInferenceInput(pipeline_id=str(pipeline.id), team_id=self.team.id, prediction_date="2026-09-11"),)assert mock_inference.call_args.kwargs["run"] == (earlier if reused else None)+ assert mock_inference.call_args.kwargs["extra_metrics"] == {"workflow_run_id": env.info.workflow_run_id}def test_load_active_pipelines_fails_inference_runs_left_running(self) -> None:pipeline = self._create_pipeline()@@ -383,6 +392,93 @@ class TestCoordinatorActivities(TeamScopedTestMixin, BaseTest):assert lost.metrics["failure_kind"] == "other"assert live.status == AutoresearchRun.Status.RUNNING+ def test_fail_lost_inference_runs_closes_only_the_scheduled_runs_of_the_date(self) -> None:+ pipeline = self._create_pipeline()++ def _running(+ prediction_date: str, scheduled: bool = True, workflow_run_id: str = "daily-run"+ ) -> AutoresearchRun:+ return AutoresearchRun.objects.create(+ pipeline=pipeline,+ run_type=AutoresearchRun.RunType.INFERENCE,+ scheduled=scheduled,+ status=AutoresearchRun.Status.RUNNING,+ started_at=django_timezone.now() - timedelta(minutes=10),+ metrics={"prediction_date": prediction_date, "horizon_days": 30, "workflow_run_id": workflow_run_id},+ )++ lost = _running("2026-09-11")+ manual = _running("2026-09-11", scheduled=False)+ other_date = _running("2026-09-12")+ catch_up = _running("2026-09-11", workflow_run_id="catch-up-run")++ ActivityEnvironment().run(+ activity_fail_lost_inference_runs,+ FailLostInferenceRunsInput(+ pipeline_id=str(pipeline.id),+ team_id=self.team.id,+ prediction_date="2026-09-11",+ workflow_run_id="daily-run",+ ),+ )++ for run in (lost, manual, other_date, catch_up):+ run.refresh_from_db()+ assert (lost.status, lost.metrics["failure_kind"]) == (AutoresearchRun.Status.FAILED, "other")+ assert lost.completed_at is not None+ assert manual.status == AutoresearchRun.Status.RUNNING+ assert other_date.status == AutoresearchRun.Status.RUNNING+ assert catch_up.status == AutoresearchRun.Status.RUNNING++ @parameterized.expand(+ [+ ("lost_run_is_retried", [("failed", "other")], None, True, None),+ ("not_reached_by_the_daily_sweep_is_skipped", [], None, False, None),+ (+ "scored_today_is_skipped",+ [("failed", "other"), ("completed", None)],+ datetime(2026, 9, 11, 3),+ False,+ None,+ ),+ ("still_running_is_skipped", [("failed", "other"), ("running", None)], None, False, None),+ ("repeatable_failure_is_skipped", [("failed", "query_failed")], None, False, None),+ ("not_due_is_skipped", [("failed", "other")], datetime(2026, 9, 11, 1), False, None),+ ("deferred_while_the_daily_sweep_runs", [("failed", "other")], None, False, "running"),+ ("runs_once_the_daily_sweep_is_done", [("failed", "other")], None, True, "done"),+ ]+ )+ def test_catch_up_discovery_lists_only_pipelines_whose_run_was_lost(+ self,+ _name: str,+ runs: list[tuple[str, Optional[str]]],+ scored_at: Optional[datetime],+ listed: bool,+ daily_sweep: Optional[str],+ ) -> None:+ pipeline = self._create_pipeline(last_scored_at=scored_at.replace(tzinfo=UTC) if scored_at else None)+ if daily_sweep is not None:+ ActivityEnvironment().run(+ activity_mark_daily_sweep, MarkDailySweepInput(run_date="2026-09-11", state=daily_sweep)+ )+ self.addCleanup(cache.delete, daily_sweep_key("2026-09-11"))+ for index, (status, failure_kind) in enumerate(runs):+ run = AutoresearchRun.objects.create(+ pipeline=pipeline,+ run_type=AutoresearchRun.RunType.INFERENCE,+ scheduled=True,+ status=status,+ started_at=django_timezone.now() - timedelta(hours=2),+ metrics={"prediction_date": "2026-09-11", "horizon_days": 30, "failure_kind": failure_kind},+ )+ AutoresearchRun.objects.filter(pk=run.pk).update(+ created_at=django_timezone.now() - timedelta(hours=3) + timedelta(minutes=index)+ )++ result = activity_load_active_pipelines(LoadActivePipelinesInput(catch_up_date="2026-09-11"))++ assert [p.pipeline_id for p in result.pipelines] == ([str(pipeline.id)] if listed else [])+@parameterized.expand([("scheduled", False), ("manual", True)])@patch("products.autoresearch.backend.temporal.workflows.run_inference_for_pipeline")def test_inference_refuses_a_deleted_action_target(@@ -673,6 +769,10 @@ async def test_coordinator_bounds_the_sweep_and_scores_shadows_after_every_champasync def kickoff(inp: KickoffTrainingInput) -> KickoffTrainingResult:return KickoffTrainingResult(kicked_off=False, reason="not_due")+ @activity.defn(name="autoresearch-coordinator.mark_daily_sweep")+ async def mark(inp: MarkDailySweepInput) -> None:+ events.append(("mark", inp.state))+@activity.defn(name="autoresearch-coordinator.run_shadow_scoring")async def shadows(inp: RunShadowScoringInput) -> RunShadowScoringResult:assert inp.run_id == f"run-{inp.pipeline_id}"@@ -686,7 +786,7 @@ async def test_coordinator_bounds_the_sweep_and_scores_shadows_after_every_champenv.client,task_queue="coordinator-test",workflows=[AutoresearchCoordinatorWorkflow, AutoresearchInferenceWorkflow, AutoresearchValidationWorkflow],- activities=[load, infer, validate, kickoff, shadows],+ activities=[load, infer, validate, kickoff, shadows, mark],workflow_runner=UnsandboxedWorkflowRunner(),):result = await env.client.execute_workflow(@@ -697,9 +797,82 @@ async def test_coordinator_bounds_the_sweep_and_scores_shadows_after_every_champ)assert max_in_flight == 2+ # The daily sweep stays marked running until its last validation and shadow phase ends.+ assert (events[0], events[-1]) == (("mark", "running"), ("mark", "done"))first_shadow = next(index for index, (name, _) in enumerate(events) if name == "shadow")assert {pid for name, pid in events[:first_shadow] if name == "champion"} == set(pipeline_ids)assert sorted(pid for name, pid in events if name == "shadow") == pipeline_idsassert all(inp.defer_shadows and inp.coordinator_concurrency == 2 for inp in inference_inputs)# A shadow failure leaves the champion's pipeline counted as processed.assert (result.pipelines_processed, result.pipelines_errored, result.status) == (5, 0, "completed")++[email protected]("catch_up", [False, True])+async def test_coordinator_fails_the_runs_of_a_failed_inference_child(catch_up: bool) -> None:+ load_inputs: list[LoadActivePipelinesInput] = []+ failed_runs: list[FailLostInferenceRunsInput] = []+ steps: list[str] = []++ @activity.defn(name="autoresearch-coordinator.load_active_pipelines")+ async def load(inp: LoadActivePipelinesInput) -> LoadActivePipelinesResult:+ load_inputs.append(inp)+ return LoadActivePipelinesResult(+ pipelines=[ActivePipeline(pipeline_id=pid, team_id=1, score_due=True) for pid in ("lost", "scored")]+ )++ @activity.defn(name="autoresearch-inference.run_inference")+ async def infer(inp: RunInferenceInput) -> RunInferenceResult:+ if inp.pipeline_id == "lost":+ raise ApplicationError("worker lost", non_retryable=True)+ return RunInferenceResult(run_id=f"run-{inp.pipeline_id}", rows_scored=1, status="completed")++ @activity.defn(name="autoresearch-coordinator.fail_lost_inference_runs")+ async def fail_lost(inp: FailLostInferenceRunsInput) -> int:+ failed_runs.append(inp)+ return 1++ @activity.defn(name="autoresearch-coordinator.mark_daily_sweep")+ async def mark(inp: MarkDailySweepInput) -> None:+ steps.append(f"mark-{inp.state}")++ @activity.defn(name="autoresearch-validation.run_validation")+ async def validate(inp: RunValidationInput) -> RunValidationResult:+ steps.append("validation")+ return RunValidationResult(dates_validated=1, total_rows=1, status="completed")++ @activity.defn(name="autoresearch-coordinator.kickoff_training")+ async def kickoff(inp: KickoffTrainingInput) -> KickoffTrainingResult:+ steps.append("kickoff")+ return KickoffTrainingResult(kicked_off=False, reason="not_due")++ @activity.defn(name="autoresearch-coordinator.run_shadow_scoring")+ async def shadows(inp: RunShadowScoringInput) -> RunShadowScoringResult:+ steps.append(f"shadow-{inp.pipeline_id}")+ return RunShadowScoringResult(completed=1, failed=0, skipped=0, status="completed")++ async with await WorkflowEnvironment.start_time_skipping() as env:+ async with Worker(+ env.client,+ task_queue="coordinator-test",+ workflows=[AutoresearchCoordinatorWorkflow, AutoresearchInferenceWorkflow, AutoresearchValidationWorkflow],+ activities=[load, infer, fail_lost, validate, kickoff, shadows, mark],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ result = await env.client.execute_workflow(+ AutoresearchCoordinatorWorkflow.run,+ CoordinatorWorkflowInput(run_date="2026-10-18", catch_up=catch_up),+ id=str(uuid.uuid4()),+ task_queue="coordinator-test",+ )++ assert load_inputs == [LoadActivePipelinesInput(catch_up_date="2026-10-18" if catch_up else None)]+ assert [replace(inp, workflow_run_id=None) for inp in failed_runs] == [+ FailLostInferenceRunsInput(pipeline_id="lost", team_id=1, prediction_date="2026-10-18")+ ]+ # Cleanup is scoped to the failed child's execution, so a catch-up child's live row stays open.+ assert failed_runs[0].workflow_run_id+ assert (result.pipelines_processed, result.pipelines_errored) == (1, 1)+ assert "shadow-scored" in steps+ # The daily sweep already kicked off training and validated, so a catch-up only scores.+ assert ("kickoff" in steps, "validation" in steps) == (not catch_up, not catch_up)+ assert [step for step in steps if step.startswith("mark-")] == ([] if catch_up else ["mark-running", "mark-done"])diff --git a/products/autoresearch/backend/temporal/workflows.py b/products/autoresearch/backend/temporal/workflows.pyindex 1a728236dd0a..35aec7bd3380 100644--- a/products/autoresearch/backend/temporal/workflows.py+++ b/products/autoresearch/backend/temporal/workflows.py@@ -16,10 +16,12 @@ from datetime import date, timedeltafrom functools import partialfrom typing import Optional, TypeVar, cast+from django.core.cache import cache+import structlogfrom temporalio import activity, workflow-from temporalio.common import RetryPolicy-from temporalio.exceptions import ActivityError, ApplicationError+from temporalio.common import RetryPolicy, WorkflowIDReusePolicy+from temporalio.exceptions import ActivityError, ApplicationError, ChildWorkflowErrorwith workflow.unsafe.imports_passed_through():import asyncio as _asyncio@@ -38,8 +40,10 @@ with workflow.unsafe.imports_passed_through():from products.autoresearch.backend.inference.failures import (INFERENCE_WORKFLOW_TIMEOUT,UNSCORABLE_AFTER_FAILED_DAYS,+ fail_lost_scheduled_runs,fail_stale_inference_runs,find_unscorable_champion,+ scheduled_run_can_retry,)from products.autoresearch.backend.inference.person_property import remove_output_person_propertyfrom products.autoresearch.backend.inference.sandbox import SandboxInferenceError, _resolve_acting_user@@ -158,11 +162,7 @@ def activity_run_inference(inp: RunInferenceInput) -> RunInferenceResult:query_context=SCORING_QUERY,scheduled=manual_run is None,defer_shadows=inp.defer_shadows,- extra_metrics=(- {"coordinator_concurrency": inp.coordinator_concurrency}- if inp.coordinator_concurrency is not None- else None- ),+ extra_metrics=_scheduled_run_metrics(inp) if manual_run is None else None,)return RunInferenceResult(run_id=str(run.pk),@@ -172,6 +172,14 @@ def activity_run_inference(inp: RunInferenceInput) -> RunInferenceResult:)+def _scheduled_run_metrics(inp: RunInferenceInput) -> dict[str, object]:+ # The workflow run id lets the coordinator close only the rows of the child execution that failed.+ metrics: dict[str, object] = {"workflow_run_id": activity.info().workflow_run_id}+ if inp.coordinator_concurrency is not None:+ metrics["coordinator_concurrency"] = inp.coordinator_concurrency+ return metrics++def _open_scheduled_run(pipeline: AutoresearchPipeline, prediction_date: str) -> AutoresearchRun | None:"""The RUNNING row an earlier attempt of this scheduled run created, or None on the first attempt."""return (@@ -628,6 +636,8 @@ class CoordinatorWorkflowInput:run_date: Optional[str] = None# Read from the input, so an operator can change it on the schedule without a deploy.max_concurrency: int = DEFAULT_COORDINATOR_CONCURRENCY+ # Set by the catch-up schedule: score again only the pipelines whose scheduled run of the day was lost.+ catch_up: bool = False@frozen@@ -639,7 +649,8 @@ class CoordinatorWorkflowResult:@frozenclass LoadActivePipelinesInput:- pass+ # Set in catch-up mode. Discovery then lists only the due pipelines whose scheduled run of this date can retry.+ catch_up_date: Optional[str] = None@frozen@@ -671,12 +682,13 @@ class PipelineRunOutcome:def evaluate_pipeline_outcome(pipeline_id: str,inference: InferenceWorkflowResult | BaseException | None,- validation: ValidationWorkflowResult | BaseException,+ validation: ValidationWorkflowResult | BaseException | None,kickoff: KickoffTrainingResult | BaseException | None,) -> PipelineRunOutcome:"""Classify one pipeline's step results, treating soft-failed activity results as failures too.- ``None`` is a step the sweep did not start, because the pipeline was not due for scoring.+ ``None`` is a step the sweep did not start, because the pipeline was not due for scoring, or+ because a catch-up sweep only scores."""errors: list[str] = []@@ -687,7 +699,9 @@ def evaluate_pipeline_outcome(elif inference.status == "failed":errors.append(f"inference: {inference.error or 'failed'}")- if isinstance(validation, BaseException):+ if validation is None:+ pass+ elif isinstance(validation, BaseException):errors.append(f"validation: {validation}")elif validation.status == "failed":errors.append(f"validation: {validation.error or 'failed'}")@@ -735,8 +749,18 @@ def activity_load_active_pipelines(inp: LoadActivePipelinesInput) -> LoadActivePThe result lists the pipelines that are due and did not score yesterday first, so a boundedsweep starts the pipelines that missed a day before the others.++ With ``catch_up_date`` it lists only the due pipelines whose latest scheduled run of that date+ failed with a kind a retry can pass (``scheduled_run_can_retry()``). A pipeline that the daily+ sweep has not reached yet has no run for the date, so the catch-up does not score it twice.+ A catch-up lists nothing while the daily sweep of that date still runs (``activity_mark_daily_sweep``),+ including its kickoff, validation and shadow phases, so the two sweeps never exceed one bound+ together. The next catch-up tick tries again."""fail_stale_inference_runs()+ if inp.catch_up_date is not None and cache.get(daily_sweep_key(inp.catch_up_date)) == DAILY_SWEEP_RUNNING:+ logger.info("autoresearch_catch_up_deferred", catch_up_date=inp.catch_up_date)+ return LoadActivePipelinesResult(pipelines=[])today = django_timezone.now().date()yesterday = today - timedelta(days=1)# unscoped() because the coordinator's job is exactly to sweep every team. The@@ -765,6 +789,13 @@ def activity_load_active_pipelines(inp: LoadActivePipelinesInput) -> LoadActiveP# elapsed-time comparison would skip the next day's tick.last_scored = pipeline.last_scored_at.date() if pipeline.last_scored_at else Nonescore_due = last_scored is None or (today - last_scored).days >= pipeline.cadence_days+ if inp.catch_up_date is not None and not (+ score_due+ and scheduled_run_can_retry(+ team_id=pipeline.team_id, pipeline_id=str(pipeline.pk), prediction_date=inp.catch_up_date+ )+ ):+ continuemissed_yesterday = score_due and (last_scored is None or last_scored < yesterday)pipelines.append((@@ -776,6 +807,37 @@ def activity_load_active_pipelines(inp: LoadActivePipelinesInput) -> LoadActivePreturn LoadActivePipelinesResult(pipelines=[pipeline for _, pipeline in pipelines])+# The coordinator's execution timeout, which the schedules set. Longer than one pipeline's children+# (inference, then kickoff), shorter than a day, so a stuck coordinator ends before the next tick+# instead of making SKIP drop every later one. The daily sweep marker expires with it.+COORDINATOR_TIMEOUT = timedelta(hours=12)+DAILY_SWEEP_RUNNING = "running"+DAILY_SWEEP_DONE = "done"+# A finished sweep stays marked past the last catch-up tick of its date.+_DAILY_SWEEP_DONE_TTL = timedelta(days=2)+++def daily_sweep_key(run_date: str) -> str:+ return f"autoresearch:daily-sweep:{run_date}"+++@frozen+class MarkDailySweepInput:+ run_date: str+ state: str++[email protected](name="autoresearch-coordinator.mark_daily_sweep")+def activity_mark_daily_sweep(inp: MarkDailySweepInput) -> None:+ """Record that the daily sweep of a date is running or done, so a catch-up waits for it.++ The running marker expires with ``COORDINATOR_TIMEOUT``, so a coordinator that dies without+ marking done holds the catch-up back no longer than it could still run.+ """+ ttl = COORDINATOR_TIMEOUT if inp.state == DAILY_SWEEP_RUNNING else _DAILY_SWEEP_DONE_TTL+ cache.set(daily_sweep_key(inp.run_date), inp.state, timeout=int(ttl.total_seconds()))++def _pause_orphaned_pipeline(pipeline: AutoresearchPipeline, *, reason: str) -> None:# Conditional on the status read above, so a concurrent archive or pause keeps its outcome.paused = (@@ -878,6 +940,26 @@ def activity_kickoff_training(inp: KickoffTrainingInput) -> KickoffTrainingResulreturn KickoffTrainingResult(kicked_off=True, reason=kickoff_reason.value)+@frozen+class FailLostInferenceRunsInput:+ pipeline_id: str+ team_id: int+ prediction_date: str+ # The failed child's run id. A catch-up child for the same date has another one, so its live row stays open.+ workflow_run_id: Optional[str] = None++[email protected](name="autoresearch-coordinator.fail_lost_inference_runs")+def activity_fail_lost_inference_runs(inp: FailLostInferenceRunsInput) -> int:+ """Fail the scheduled runs that an inference workflow left RUNNING when it ended in an exception."""+ return fail_lost_scheduled_runs(+ team_id=inp.team_id,+ pipeline_id=inp.pipeline_id,+ prediction_date=inp.prediction_date,+ workflow_run_id=inp.workflow_run_id,+ )++def _sweep_access_block(pipeline: AutoresearchPipeline) -> str | None:"""Why the sweep may no longer act on this pipeline, or None.@@ -916,6 +998,10 @@ def _tasks_gate(pipeline: AutoresearchPipeline) -> str | None:# Histories that started before this marker replay the unbounded sweep._BOUNDED_SWEEP_PATCH = "autoresearch-bounded-sweep-2026-10"+# Histories that started before this marker do not close the runs of a failed inference child.+_CLOSE_LOST_RUNS_PATCH = "autoresearch-close-lost-runs-2026-10"+# Histories that started before this marker do not tell a catch-up that the daily sweep is running.+_DAILY_SWEEP_MARKER_PATCH = "autoresearch-daily-sweep-marker-2026-10"@workflow.defn(name="autoresearch-coordinator")@@ -936,6 +1022,10 @@ class AutoresearchCoordinatorWorkflow(PostHogWorkflow):training, then validates. Once every champion has scored, the coordinator scores eachpipeline's shadow set (``activity_run_shadow_scoring``) under the same bound.+ When an inference child ends in an exception, the coordinator fails the runs it left RUNNING.+ A catch-up sweep (``catch_up``) scores again only the pipelines whose scheduled run of the day+ was lost, and skips training kickoff and validation, because the daily sweep already ran them.+Per-pipeline failures — raised exceptions andactivity results that report a soft failure — are logged and counted inpipelines_errored, but do not prevent other pipelines from running.@@ -949,9 +1039,28 @@ class AutoresearchCoordinatorWorkflow(PostHogWorkflow):workflow.logger.info("autoresearch_coordinator_start", extra={"run_date": run_date})+ tracks_daily_sweep = not inp.catch_up and workflow.patched(_DAILY_SWEEP_MARKER_PATCH)+ if tracks_daily_sweep:+ await _settle(self._mark_daily_sweep(run_date, DAILY_SWEEP_RUNNING))+ try:+ return await self._sweep(inp, run_date)+ finally:+ if tracks_daily_sweep:+ await _settle(self._mark_daily_sweep(run_date, DAILY_SWEEP_DONE))++ def _mark_daily_sweep(self, run_date: str, state: str) -> Awaitable[None]:+ return workflow.execute_activity(+ activity_mark_daily_sweep,+ MarkDailySweepInput(run_date=run_date, state=state),+ start_to_close_timeout=timedelta(minutes=1),+ retry_policy=RetryPolicy(maximum_attempts=5),+ )++ async def _sweep(self, inp: CoordinatorWorkflowInput, run_date: str) -> CoordinatorWorkflowResult:+active = await workflow.execute_activity(activity_load_active_pipelines,- LoadActivePipelinesInput(),+ LoadActivePipelinesInput(catch_up_date=run_date if inp.catch_up else None),start_to_close_timeout=timedelta(minutes=2),retry_policy=_COORDINATOR_RETRY,)@@ -962,7 +1071,7 @@ class AutoresearchCoordinatorWorkflow(PostHogWorkflow):outcomes: list[PipelineRunOutcome | BaseException]if workflow.patched(_BOUNDED_SWEEP_PATCH):- outcomes = await self._run_bounded(active.pipelines, run_date, max(1, inp.max_concurrency))+ outcomes = await self._run_bounded(active.pipelines, run_date, max(1, inp.max_concurrency), inp.catch_up)else:outcomes = await _asyncio.gather(*[self._run_pipeline(pipeline, run_date) for pipeline in active.pipelines],@@ -984,7 +1093,7 @@ class AutoresearchCoordinatorWorkflow(PostHogWorkflow):)async def _run_bounded(- self, pipelines: list[ActivePipeline], run_date: str, concurrency: int+ self, pipelines: list[ActivePipeline], run_date: str, concurrency: int, catch_up: bool) -> list[PipelineRunOutcome | BaseException]:"""Run every pipeline with at most ``concurrency`` in flight, then every shadow set under the same bound."""slots = _asyncio.Semaphore(concurrency)@@ -994,7 +1103,7 @@ class AutoresearchCoordinatorWorkflow(PostHogWorkflow):return await step()sweeps = await _asyncio.gather(- *[bounded(partial(self._run_pipeline_in_sequence, p, run_date, concurrency)) for p in pipelines],+ *[bounded(partial(self._run_pipeline_in_sequence, p, run_date, concurrency, catch_up)) for p in pipelines],return_exceptions=True,)# Shadow scoring starts only after every champion has finished, so champions free their memory first.@@ -1019,11 +1128,15 @@ class AutoresearchCoordinatorWorkflow(PostHogWorkflow):return [sweep if isinstance(sweep, BaseException) else sweep.outcome for sweep in sweeps]async def _run_pipeline_in_sequence(- self, pipeline: ActivePipeline, run_date: str, concurrency: int+ self, pipeline: ActivePipeline, run_date: str, concurrency: int, catch_up: bool) -> _PipelineSweep:- """On a cadence day score, then kick off training. Then validate. One step at a time, so a pipeline holds one slot."""+ """On a cadence day score, then kick off training. Then validate. One step at a time, so a pipeline holds one slot.++ A catch-up sweep only scores.+ """pipeline_id = pipeline.pipeline_idinference_result: InferenceWorkflowResult | BaseException | None = None+ validation_result: ValidationWorkflowResult | BaseException | None = Nonekickoff_result: KickoffTrainingResult | BaseException | None = Noneif pipeline.score_due:inference_result = await _settle(@@ -1038,10 +1151,21 @@ class AutoresearchCoordinatorWorkflow(PostHogWorkflow):),id=inference_workflow_id(pipeline_id, run_date),execution_timeout=INFERENCE_WORKFLOW_TIMEOUT,+ # A catch-up starts only after the daily sweep's child for the date failed or timed out.+ id_reuse_policy=(+ WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY+ if catch_up+ else WorkflowIDReusePolicy.ALLOW_DUPLICATE+ ),))- kickoff_result = await _settle(self._kickoff(pipeline))- validation_result = await _settle(self._validate(pipeline, run_date))+ # A child that failed to start (it is already running) is not a ChildWorkflowError, so its runs stay open.+ if isinstance(inference_result, ChildWorkflowError) and workflow.patched(_CLOSE_LOST_RUNS_PATCH):+ await _settle(self._fail_lost_runs(pipeline, run_date, inference_result.run_id))+ if not catch_up:+ kickoff_result = await _settle(self._kickoff(pipeline))+ if not catch_up:+ validation_result = await _settle(self._validate(pipeline, run_date))outcome = self._evaluate(pipeline_id, inference_result, validation_result, kickoff_result)return _PipelineSweep(outcome=outcome,@@ -1057,6 +1181,19 @@ class AutoresearchCoordinatorWorkflow(PostHogWorkflow):retry_policy=_SHADOW_RETRY,)+ def _fail_lost_runs(self, pipeline: ActivePipeline, run_date: str, workflow_run_id: str) -> Awaitable[int]:+ return workflow.execute_activity(+ activity_fail_lost_inference_runs,+ FailLostInferenceRunsInput(+ pipeline_id=pipeline.pipeline_id,+ team_id=pipeline.team_id,+ prediction_date=run_date,+ workflow_run_id=workflow_run_id,+ ),+ start_to_close_timeout=timedelta(minutes=1),+ retry_policy=RetryPolicy(maximum_attempts=5),+ )+def _validate(self, pipeline: ActivePipeline, run_date: str) -> Awaitable[ValidationWorkflowResult]:return workflow.execute_child_workflow(AutoresearchValidationWorkflow.run,@@ -1077,7 +1214,7 @@ class AutoresearchCoordinatorWorkflow(PostHogWorkflow):self,pipeline_id: str,inference: InferenceWorkflowResult | BaseException | None,- validation: ValidationWorkflowResult | BaseException,+ validation: ValidationWorkflowResult | BaseException | None,kickoff: KickoffTrainingResult | BaseException | None,) -> PipelineRunOutcome:outcome = evaluate_pipeline_outcome(