:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): bound backfill dispatch and pause or stop on repeated failures (#113431)
Bernat Torresbedbeda875e9
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.mdindex e8ceee308d75..b7d089f7825f 100644--- a/docs/internal/ai-observability-judge-inputs.md+++ b/docs/internal/ai-observability-judge-inputs.md@@ -184,8 +184,16 @@ The playground keeps the provider's explanation so users can correct the settingA backfill that stops on its own shows as **Interrupted**, with a stop reason.A backfill a user stops shows as **Cancelled**.+Backfills read a page of units (500 by default, bounded between 1 and 1,000) and wait for every outcome before advancing progress.+A backfill runs at most 16 evaluations at once (`LLMA_EVAL_BACKFILL_MAX_IN_FLIGHT`).+A run holds one of the shared ClickHouse slots only while it reads its unit, and spends most of its time waiting on the judge.+Lower the setting if backfills crowd out live evaluations.Backfill activities retry temporary database, DNS, connection, and rate-limit failures up to 5 times within 10 minutes, or within the live time limit when that is longer, with backoff from 10 seconds to one minute.A backfill run that still hits a DNS failure on its last attempt fails instead of recording a skip, so **Retry remaining** can run it again.+After 10 failed runs in a row, the backfill pauses for 1, 2, 5, 10, then 30 minutes, and a success resets that sequence.+The failure count and the pauses carry over between pages.+When the pauses run out, or a run reports an error that disables the evaluation, the backfill stops as **Interrupted** and keeps its cursor.+Its stop reason says whether the evaluation was disabled or the runs kept failing.Live evaluation retry budgets remain unchanged.Authentication, quota, blocked-endpoint, and permanent request errors retain their existing handling.Result emission has a separate retry budget, so retrying emission does not repeat a completed judge call.@@ -194,6 +202,8 @@ An interrupted backfill preserves completed results.**Retry remaining** creates a new backfill with the original date range and filters, using the current evaluation settings.It preserves usable existing results and retries units without usable results.It is not available for a backfill that re-ran existing results.+A cancelled backfill still counts the runs it started before the cancel, because they finish and are billed.+Persistent failures can still interrupt a backfill after its recovery budget; recovery does not guarantee a verdict for invalid or permanently rejected inputs.## Result encodingdiff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.pyindex f89cd60f24d7..3d2bcc13fd4f 100644--- a/posthog/settings/temporal.py+++ b/posthog/settings/temporal.py@@ -408,6 +408,9 @@ LLMA_TASK_QUEUE = _set_temporal_task_queue("llm-analytics-task-queue")# A tick costs one candidate query whatever it dispatches, so a larger batch means fewer ticks and# less ClickHouse work for the same backfill.LLMA_EVAL_BACKFILL_BATCH_SIZE: int = get_from_env("LLMA_EVAL_BACKFILL_BATCH_SIZE", 500, type_cast=int)+# Evaluations one backfill runs at once. A run holds one of the shared CLICKHOUSE_LLM_ANALYTICS_MAX_CONCURRENT_QUERIES+# slots only while it reads its unit, so most runs wait on the judge. Lower this if backfills crowd out live work.+LLMA_EVAL_BACKFILL_MAX_IN_FLIGHT: int = get_from_env("LLMA_EVAL_BACKFILL_MAX_IN_FLIGHT", 16, type_cast=int)# Defaults to the general-purpose fleet so dispatch always has a live worker; set the env to# "mcp-analytics-task-queue" to route MCP analytics clustering to a dedicated, separately-scalable# worker once one is deployed.diff --git a/posthog/temporal/ai_observability/evaluation_backfill.py b/posthog/temporal/ai_observability/evaluation_backfill.pyindex 26f75160a7d5..1affe9654837 100644--- a/posthog/temporal/ai_observability/evaluation_backfill.py+++ b/posthog/temporal/ai_observability/evaluation_backfill.py@@ -6,6 +6,8 @@ own and continues as new after each batch, however long the window takes to draiimport asyncioimport hashlib+import contextlib+from collections import Counterfrom dataclasses import replacefrom datetime import datetime, timedeltafrom enum import StrEnum@@ -13,13 +15,14 @@ from typing import Anyfrom uuid import UUIDfrom django.conf import settings-from django.db.models import F+from django.db.models import Case, F, Q, When+from django.db.models.functions import Coalescefrom django.utils import timezoneimport structlogimport temporaliofrom temporalio.common import RetryPolicy, WorkflowIDReusePolicy-from temporalio.exceptions import WorkflowAlreadyStartedError, is_cancelled_exception+from temporalio.exceptions import ChildWorkflowError, WorkflowAlreadyStartedError, is_cancelled_exceptionfrom temporalio.workflow import ParentClosePolicyfrom posthog.dataclasses import frozen@@ -27,6 +30,7 @@ from posthog.event_usage import groupsfrom posthog.models.team import Teamfrom posthog.ph_client import ph_background_capturefrom posthog.sync import database_sync_to_async+from posthog.temporal.ai_observability.evaluation_errors import USER_ERROR_SPECSfrom posthog.temporal.ai_observability.evaluation_event_io import as_utc_datetimefrom posthog.temporal.ai_observability.evaluation_types import EVALUATION_WORKFLOW_PREFIXESfrom posthog.temporal.ai_observability.evaluation_workflow_activities import RunEvaluationInputs@@ -36,6 +40,7 @@ from posthog.temporal.ai_observability.run_aggregate_evaluation import (RunAggregateEvaluationInputs,resolve_settle_plan,)+from posthog.temporal.ai_observability.run_evaluation import WorkflowResultfrom posthog.temporal.common.base import PostHogWorkflowfrom products.ai_observability.backend.backfill_candidates import count_backfill_candidates, fetch_backfill_candidates@@ -50,9 +55,9 @@ logger = structlog.get_logger(__name__)BACKFILL_WORKFLOW_NAME = "llma-evaluation-backfill"BACKFILL_TICK_INTERVAL = timedelta(seconds=60)-# Long enough for the slowest child: a trace evaluation can settle for its 2 hour maximum age-# before the judge even starts.-CHILD_EXECUTION_TIMEOUT = timedelta(hours=3)+# A backfill child skips the settle wait, so its activity budgets add up to under an hour. A page waits+# on every child, so a stuck child holds it this long before it counts as a failed run.+CHILD_EXECUTION_TIMEOUT = timedelta(hours=1)ACTIVITY_TIMEOUT = timedelta(seconds=30)# A tick that cannot reach a worker must fail rather than sit in the queue, because the loop only@@ -69,6 +74,17 @@ MIN_BACKFILL_BATCH_SIZE = 1MAX_BACKFILL_BATCH_SIZE = 1000# A tick that keeps failing would otherwise leave the row RUNNING and the loop spinning forever.BACKFILL_MAX_CONSECUTIVE_FAILURES = 5+BACKFILL_DEFAULT_MAX_IN_FLIGHT = 16+BACKFILL_FAILURES_BEFORE_PAUSE = 10+# Each pause in a row waits longer, and a success starts the sequence over. Once the pauses run out,+# the backfill stops and the units it did not reach stay for Retry remaining.+BACKFILL_FAILURE_PAUSES = (+ timedelta(minutes=1),+ timedelta(minutes=2),+ timedelta(minutes=5),+ timedelta(minutes=10),+ timedelta(minutes=30),+)def settle_horizon(target: str, settle: dict[str, Any] | None) -> timedelta:@@ -95,6 +111,9 @@ class EvaluationBackfillInputs:team_id: int# Reset by any tick that succeeds.consecutive_failures: int = 0+ # Carried across pages, so a backfill smaller than one pause cycle can still stop.+ failed_runs_in_a_row: int = 0+ pauses_in_a_row: int = 0class TickAction(StrEnum):@@ -112,6 +131,8 @@ class PrepareTickOutput:settle: dict[str, Any] | None = Nonererun_existing: bool = Falsebatch_size: int = 0+ # A tick prepared by a worker that predates this field carries 0, which the workflow reads as the default.+ max_in_flight: int = 0@frozen@@ -157,6 +178,11 @@ class AdvanceCursorInputs:exhausted: bool# Defaulted so an advance recorded before this field existed still deserializes.failed_delta: int = 0+ completed_delta: int | None = None+ evaluation_skipped_delta: int = 0+ stop_reason: str | None = None+ # Records a page a user cancelled midway, whose started runs still finish and bill.+ cancelled: bool = False@frozen@@ -183,6 +209,84 @@ class ChildWorkflow:workflow_id: str+class ChildOutcome(StrEnum):+ EVALUATED = "evaluated"+ SKIPPED = "skipped"+ FAILED = "failed"+ EXISTING = "existing"+ NOT_STARTED = "not_started"+++@frozen+class BackfillChildOutcome:+ outcome: ChildOutcome+ stop_reason: str | None = None+++def _stop_reason(result: WorkflowResult) -> str | None:+ reason = result.get("skip_reason", "")+ if reason == "evaluation_deleted":+ return reason+ spec = USER_ERROR_SPECS.get(reason)+ if reason == "evaluation_disabled" or (spec is not None and spec.disables_evaluation):+ return "evaluation_disabled"+ return None+++class _BackfillPacing:+ """Decides, as runs finish, whether a backfill keeps starting runs, pauses, or stops."""++ def __init__(self, failures: int = 0, pauses: int = 0) -> None:+ self.failures = failures+ self.pauses = pauses+ self.stop_reason: str | None = None+ self._pausing = asyncio.Lock()++ @property+ def stopped(self) -> bool:+ return self.stop_reason is not None++ async def ready(self) -> bool:+ async with self._pausing:+ while not self.stopped and self.failures >= BACKFILL_FAILURES_BEFORE_PAUSE:+ if self.pauses == len(BACKFILL_FAILURE_PAUSES):+ self.stop_reason = "execution_failed"+ else:+ # Count runs still in flight toward the next streak, and let a success during+ # the pause restart the sequence.+ pause = BACKFILL_FAILURE_PAUSES[self.pauses]+ self.pauses += 1+ self.failures = 0+ temporalio.workflow.logger.info(+ "llma.evaluation_backfill_paused",+ extra={"pause_seconds": pause.total_seconds(), "pauses_in_a_row": self.pauses},+ )+ # A run that stops the backfill mid-pause ends the pause too.+ with contextlib.suppress(TimeoutError):+ await temporalio.workflow.wait_condition(lambda: self.stopped, timeout=pause)+ return not self.stopped++ def finish_page(self) -> None:+ # The last page starts no run after its final failure, so ready() never sees a spent budget.+ if self.failures >= BACKFILL_FAILURES_BEFORE_PAUSE and self.pauses == len(BACKFILL_FAILURE_PAUSES):+ self.stop_reason = self.stop_reason or "execution_failed"++ def record(self, result: BackfillChildOutcome) -> None:+ if result.stop_reason:+ self.stop_reason = self.stop_reason or result.stop_reason+ elif result.outcome == ChildOutcome.FAILED:+ self.failures += 1+ elif result.outcome == ChildOutcome.EVALUATED:+ self.failures = 0+ self.pauses = 0+++def _child_outcome(result: WorkflowResult) -> ChildOutcome:+ # A run that returned a skip completed, so Retry remaining could not start it again. Only runs+ # that failed count toward the remaining units.+ return ChildOutcome.SKIPPED if result["skipped"] else ChildOutcome.EVALUATED++def child_workflow_name_and_id(*,evaluation_id: str,@@ -314,6 +418,7 @@ def _prepare_backfill_tick(inputs: EvaluationBackfillInputs) -> PrepareTickOutpusettle=evaluation.target_config,rerun_existing=row.rerun_existing,batch_size=max(MIN_BACKFILL_BATCH_SIZE, min(settings.LLMA_EVAL_BACKFILL_BATCH_SIZE, MAX_BACKFILL_BATCH_SIZE)),+ max_in_flight=max(1, settings.LLMA_EVAL_BACKFILL_MAX_IN_FLIGHT),)@@ -376,17 +481,36 @@ async def find_evaluation_backfill_candidates_activity(inputs: FindCandidatesInpdef _advance_backfill_cursor(inputs: AdvanceCursorInputs) -> AdvanceCursorOutput:- updates: dict[str, Any] = {+ counts: dict[str, Any] = {"dispatched_count": F("dispatched_count") + inputs.dispatched_delta,"skipped_count": F("skipped_count") + inputs.skipped_delta,"failed_count": F("failed_count") + inputs.failed_delta,}+ if inputs.completed_delta is not None:+ # Runs that already dispatched without tracking outcomes must retain unknown totals.+ tracks_outcomes = Q(cursor_timestamp__isnull=True) | Q(completed_count__isnull=False)+ counts["completed_count"] = Case(+ When(tracks_outcomes, then=Coalesce(F("completed_count"), 0) + inputs.completed_delta),+ default=None,+ )+ counts["evaluation_skipped_count"] = F("evaluation_skipped_count") + inputs.evaluation_skipped_deltaif inputs.new_cursor_timestamp is not None:- updates["cursor_timestamp"] = datetime.fromisoformat(inputs.new_cursor_timestamp)- updates["cursor_unit_id"] = inputs.new_cursor_unit_id- if inputs.exhausted:- updates["status"] = EvaluationBackfillStatus.COMPLETED- updates["finished_at"] = timezone.now()+ counts["cursor_timestamp"] = datetime.fromisoformat(inputs.new_cursor_timestamp)+ counts["cursor_unit_id"] = inputs.new_cursor_unit_id+ terminal: dict[str, Any] = {}+ if inputs.stop_reason:+ terminal = {+ "status": EvaluationBackfillStatus.INTERRUPTED,+ "status_reason": inputs.stop_reason,+ "finished_at": timezone.now(),+ }+ elif inputs.exhausted:+ terminal = {"status": EvaluationBackfillStatus.COMPLETED, "finished_at": timezone.now()}+ if inputs.completed_delta is not None:+ terminal["remaining_count"] = Case(+ When(tracks_outcomes, then=F("failed_count") + inputs.failed_delta), default=None+ )+ updates = {**counts, **terminal}expected_timestamp = (datetime.fromisoformat(inputs.expected_cursor_timestamp) if inputs.expected_cursor_timestamp else None@@ -396,7 +520,7 @@ def _advance_backfill_cursor(inputs: AdvanceCursorInputs) -> AdvanceCursorOutput# result was lost after it committed, and a second blind increment would inflate progress.# Filtering on RUNNING lets a concurrent cancel win over the advance.updated = row.filter(- status=EvaluationBackfillStatus.RUNNING,+ status=EvaluationBackfillStatus.CANCELLED if inputs.cancelled else EvaluationBackfillStatus.RUNNING,cursor_timestamp=expected_timestamp,cursor_unit_id=inputs.expected_cursor_unit_id,).update(**updates)@@ -406,18 +530,34 @@ def _advance_backfill_cursor(inputs: AdvanceCursorInputs) -> AdvanceCursorOutputteam_id=inputs.team_id,dispatched=inputs.dispatched_delta,skipped=inputs.skipped_delta,+ completed=inputs.completed_delta,+ evaluation_skipped=inputs.evaluation_skipped_delta,failed=inputs.failed_delta,exhausted=inputs.exhausted,+ stop_reason=inputs.stop_reason,+ cancelled=inputs.cancelled,applied=bool(updated),)if updated:- return AdvanceCursorOutput(finished=inputs.exhausted)+ if inputs.stop_reason:+ status = "failed" if inputs.stop_reason == "execution_failed" else "stopped"+ report_backfill_finished(inputs.team_id, inputs.backfill_id, status=status, stop_reason=inputs.stop_reason)+ return AdvanceCursorOutput(finished=inputs.exhausted or inputs.stop_reason is not None or inputs.cancelled)# Zero rows has two causes that end differently. If the row is no longer RUNNING, someone# cancelled or completed it and the loop stops. If it is still RUNNING, an earlier attempt of# this same advance already committed and only its result was lost, so the loop must carry on# from the stored cursor instead of stalling with the window half walked.- status = row.values_list("status", flat=True).first()- return AdvanceCursorOutput(finished=status != EvaluationBackfillStatus.RUNNING)+ if not inputs.cancelled and inputs.new_cursor_timestamp is not None:+ # A cancel can land after this page's runs finished. They still billed, so the cancelled+ # row keeps their counts. Moving the cursor keeps a retried advance from counting twice.+ if row.filter(+ status=EvaluationBackfillStatus.CANCELLED,+ cursor_timestamp=expected_timestamp,+ cursor_unit_id=inputs.expected_cursor_unit_id,+ ).update(**counts):+ return AdvanceCursorOutput(finished=True)+ current_status = row.values_list("status", flat=True).first()+ return AdvanceCursorOutput(finished=current_status != EvaluationBackfillStatus.RUNNING)@temporalio.activity.defn@@ -443,6 +583,8 @@ def _measure_backfill_remainder(inputs: MeasureRemainderInputs) -> None:count taken after the settle horizon separates the two."""row = EvaluationBackfill.objects.for_team(inputs.team_id).select_related("evaluation").get(pk=inputs.backfill_id)+ if row.completed_count is not None:+ returnteam = Team.objects.get(pk=inputs.team_id)scope = count_backfill_candidates(team=team,@@ -508,6 +650,9 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):)if tick.action == TickAction.FINISHED:return+ if temporalio.workflow.patched("evaluation-backfill-await-children"):+ await self._dispatch_bounded_batch(inputs, tick)+ returnfinished = await self._dispatch_batch(inputs, tick)except Exception as error:# A cancel reaches the workflow as a cancelled activity, which is not a tick that@@ -523,15 +668,107 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):await temporalio.workflow.sleep(BACKFILL_TICK_INTERVAL)temporalio.workflow.continue_as_new(replace(inputs, consecutive_failures=0))+ async def _dispatch_bounded_batch(self, inputs: EvaluationBackfillInputs, tick: PrepareTickOutput) -> None:+ found = await self._find_candidates(inputs, tick)+ # A tick prepared by a worker that predates max_in_flight carries 0.+ slots = asyncio.Semaphore(tick.max_in_flight or BACKFILL_DEFAULT_MAX_IN_FLIGHT)+ pacing = _BackfillPacing(inputs.failed_runs_in_a_row, inputs.pauses_in_a_row)+ started: list[str] = []++ async def run_in_slot(candidate: CandidatePayload) -> BackfillChildOutcome:+ async with slots:+ if not await pacing.ready():+ return BackfillChildOutcome(outcome=ChildOutcome.NOT_STARTED)+ result = await self._run_child(inputs, tick, candidate, started)+ pacing.record(result)+ return result++ async def record(results: list[BackfillChildOutcome], *, cancelled: bool = False) -> AdvanceCursorOutput:+ outcomes = Counter(result.outcome for result in results)+ # A stopped page keeps its cursor, because it never reached some of its units.+ keep_cursor = pacing.stopped and not cancelled+ return await temporalio.workflow.execute_activity(+ advance_evaluation_backfill_cursor_activity,+ AdvanceCursorInputs(+ backfill_id=inputs.backfill_id,+ team_id=inputs.team_id,+ expected_cursor_timestamp=found.started_from_cursor_timestamp,+ expected_cursor_unit_id=found.started_from_cursor_unit_id,+ new_cursor_timestamp=None if keep_cursor else found.next_cursor_timestamp,+ new_cursor_unit_id="" if keep_cursor else found.next_cursor_unit_id,+ dispatched_delta=len(started),+ skipped_delta=outcomes[ChildOutcome.EXISTING],+ completed_delta=outcomes[ChildOutcome.EVALUATED],+ evaluation_skipped_delta=outcomes[ChildOutcome.SKIPPED],+ failed_delta=outcomes[ChildOutcome.FAILED],+ exhausted=found.exhausted and not pacing.stopped and not cancelled,+ stop_reason=None if cancelled else pacing.stop_reason,+ cancelled=cancelled,+ ),+ start_to_close_timeout=ACTIVITY_TIMEOUT,+ schedule_to_close_timeout=ACTIVITY_SCHEDULE_TO_CLOSE,+ retry_policy=ACTIVITY_RETRY_POLICY,+ )++ tasks = [asyncio.create_task(run_in_slot(candidate)) for candidate in found.candidates]+ try:+ results = await asyncio.gather(*tasks)+ except asyncio.CancelledError:+ # The children already started keep running and bill, so the cancelled row still counts them.+ await record(+ [task.result() for task in tasks if task.done() and not task.cancelled() and not task.exception()],+ cancelled=True,+ )+ raise+ pacing.finish_page()+ advance = await record(results)+ if advance.finished:+ await self._measure_remainder(inputs)+ return+ temporalio.workflow.continue_as_new(+ replace(+ inputs,+ consecutive_failures=0,+ failed_runs_in_a_row=pacing.failures,+ pauses_in_a_row=pacing.pauses,+ )+ )++ async def _run_child(+ self,+ inputs: EvaluationBackfillInputs,+ tick: PrepareTickOutput,+ candidate: CandidatePayload,+ started_units: list[str],+ ) -> BackfillChildOutcome:+ child = child_workflow_name_and_id(+ evaluation_id=tick.evaluation_id,+ evaluation_type=tick.evaluation_type,+ target=tick.target,+ unit_id=candidate.unit_id,+ backfill_id=inputs.backfill_id,+ rerun_existing=tick.rerun_existing,+ )+ try:+ try:+ handle = await self._start_child_handle(inputs, tick, candidate)+ except WorkflowAlreadyStartedError:+ return BackfillChildOutcome(outcome=ChildOutcome.EXISTING)+ started_units.append(candidate.unit_id)+ result = await asyncio.shield(handle)+ return BackfillChildOutcome(outcome=_child_outcome(result), stop_reason=_stop_reason(result))+ except Exception as error:+ # A cancelled backfill raises here; a run cancelled on its own is only a failed run.+ if is_cancelled_exception(error) and not isinstance(error, ChildWorkflowError):+ raise+ temporalio.workflow.logger.warning(+ "llma.evaluation_backfill_child_failed", extra={"workflow_id": child.workflow_id}, exc_info=True+ )+ return BackfillChildOutcome(outcome=ChildOutcome.FAILED)+async def _dispatch_batch(self, inputs: EvaluationBackfillInputs, tick: PrepareTickOutput) -> bool:"""Walk one page, start a child per unit, and advance the cursor. True ends the loop."""- found = await temporalio.workflow.execute_activity(- find_evaluation_backfill_candidates_activity,- FindCandidatesInputs(backfill_id=inputs.backfill_id, team_id=inputs.team_id, limit=tick.batch_size),- start_to_close_timeout=timedelta(seconds=120),- schedule_to_close_timeout=ACTIVITY_SCHEDULE_TO_CLOSE,- retry_policy=ACTIVITY_RETRY_POLICY,- )+ found = await self._find_candidates(inputs, tick)# A page whose children mostly went out must not be re-dispatched because one start# raised: the retry would collide with every child already running. The unit that failed# is left to a later backfill and counted as failed, not skipped, because skipped means@@ -570,26 +807,38 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):# After the advance, never before it: inserting an activity ahead of one an in-flight# history already recorded breaks replay. The row therefore completes with no count# for a moment, and a null count states that rather than claiming full coverage.- try:- await temporalio.workflow.execute_activity(- measure_evaluation_backfill_remainder_activity,- MeasureRemainderInputs(backfill_id=inputs.backfill_id, team_id=inputs.team_id),- start_to_close_timeout=timedelta(seconds=120),- schedule_to_close_timeout=ACTIVITY_SCHEDULE_TO_CLOSE,- retry_policy=ACTIVITY_RETRY_POLICY,- )- increment_backfill_remainder_outcome("success")- except Exception as error:- # The walk is done either way, so failing the tick here would spend a consecutive- # failure and log at exception level over a number the row can live without.- if is_cancelled_exception(error):- raise- increment_backfill_remainder_outcome("failed")- temporalio.workflow.logger.warning(- "llma.evaluation_backfill_remainder_failed", extra={"backfill_id": inputs.backfill_id}- )+ await self._measure_remainder(inputs)return advance.finished+ async def _find_candidates(self, inputs: EvaluationBackfillInputs, tick: PrepareTickOutput) -> FindCandidatesOutput:+ return await temporalio.workflow.execute_activity(+ find_evaluation_backfill_candidates_activity,+ FindCandidatesInputs(backfill_id=inputs.backfill_id, team_id=inputs.team_id, limit=tick.batch_size),+ start_to_close_timeout=timedelta(seconds=120),+ schedule_to_close_timeout=ACTIVITY_SCHEDULE_TO_CLOSE,+ retry_policy=ACTIVITY_RETRY_POLICY,+ )++ async def _measure_remainder(self, inputs: EvaluationBackfillInputs) -> None:+ try:+ await temporalio.workflow.execute_activity(+ measure_evaluation_backfill_remainder_activity,+ MeasureRemainderInputs(backfill_id=inputs.backfill_id, team_id=inputs.team_id),+ start_to_close_timeout=timedelta(seconds=120),+ schedule_to_close_timeout=ACTIVITY_SCHEDULE_TO_CLOSE,+ retry_policy=ACTIVITY_RETRY_POLICY,+ )+ increment_backfill_remainder_outcome("success")+ except Exception as error:+ # The walk is done either way, so failing the tick here would spend a consecutive+ # failure and log at exception level over a number the row can live without.+ if is_cancelled_exception(error):+ raise+ increment_backfill_remainder_outcome("failed")+ temporalio.workflow.logger.warning(+ "llma.evaluation_backfill_remainder_failed", extra={"backfill_id": inputs.backfill_id}+ )+async def _handle_failed_tick(self, inputs: EvaluationBackfillInputs) -> None:failures = inputs.consecutive_failures + 1temporalio.workflow.logger.exception(@@ -615,6 +864,18 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):self, inputs: EvaluationBackfillInputs, tick: PrepareTickOutput, candidate: CandidatePayload) -> bool:"""Start one evaluation child. False means the unit already had a run, so it was skipped."""+ try:+ await self._start_child_handle(inputs, tick, candidate)+ return True+ except WorkflowAlreadyStartedError:+ return False++ async def _start_child_handle(+ self,+ inputs: EvaluationBackfillInputs,+ tick: PrepareTickOutput,+ candidate: CandidatePayload,+ ) -> temporalio.workflow.ChildWorkflowHandle[Any, WorkflowResult]:unit_id = candidate.unit_idchild = child_workflow_name_and_id(evaluation_id=tick.evaluation_id,@@ -653,16 +914,12 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):anchor_timestamp=candidate.unit_timestamp,backfill_id=inputs.backfill_id,)- try:- await temporalio.workflow.start_child_workflow(- child.name,- args,- id=child.workflow_id,- task_queue=settings.LLMA_EVALS_TASK_QUEUE,- id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,- parent_close_policy=ParentClosePolicy.ABANDON,- execution_timeout=CHILD_EXECUTION_TIMEOUT,- )- return True- except WorkflowAlreadyStartedError:- return False+ return await temporalio.workflow.start_child_workflow(+ child.name,+ args,+ id=child.workflow_id,+ task_queue=settings.LLMA_EVALS_TASK_QUEUE,+ id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,+ parent_close_policy=ParentClosePolicy.ABANDON,+ execution_timeout=CHILD_EXECUTION_TIMEOUT,+ )diff --git a/posthog/temporal/ai_observability/evaluation_workflow_activities.py b/posthog/temporal/ai_observability/evaluation_workflow_activities.pyindex 15979d596c02..56daa41f7e87 100644--- a/posthog/temporal/ai_observability/evaluation_workflow_activities.py+++ b/posthog/temporal/ai_observability/evaluation_workflow_activities.py@@ -542,8 +542,10 @@ async def run_local_evaluation_activity(inputs: RunLocalEvaluationInputs) -> Locevaluation = await database_sync_to_async_pool(fetch_evaluation)(inputs.evaluation_id, inputs.event_data["team_id"])evaluation_type = evaluation.get("evaluation_type", "llm_judge")- if evaluation_type not in ("hog", "sentiment"):- # An llm_judge returns before the read: its own activity hydrates what it grades.+ stale_backfill = inputs.backfill_id is not None and (evaluation["deleted"] or not evaluation["enabled"])+ if evaluation_type not in ("hog", "sentiment") or stale_backfill:+ # An llm_judge returns before the read: its own activity hydrates what it grades. A backfill+ # of an evaluation turned off since it started returns unrun, so the workflow reports the skip.return LocalEvaluationOutcome(evaluation=evaluation, result=None, emitted=False)event_data = await database_sync_to_async(hydrate_event_reference, thread_sensitive=False)(inputs.event_data)diff --git a/posthog/temporal/ai_observability/run_aggregate_evaluation.py b/posthog/temporal/ai_observability/run_aggregate_evaluation.pyindex 0a3d1002c7b2..d745e49471e2 100644--- a/posthog/temporal/ai_observability/run_aggregate_evaluation.py+++ b/posthog/temporal/ai_observability/run_aggregate_evaluation.py@@ -48,6 +48,7 @@ from posthog.temporal.ai_observability.evaluation_workflow_activities import (from posthog.temporal.ai_observability.metrics import increment_errors, increment_settle_pollfrom posthog.temporal.ai_observability.run_evaluation import (WorkflowResult,+ compact_backfill_result,handle_llm_judge_activity_error,handle_terminal_user_error_result,)@@ -487,6 +488,9 @@ class RunAggregateEvaluationWorkflow(PostHogWorkflow):@temporalio.workflow.runasync def run(self, inputs: RunAggregateEvaluationInputs) -> WorkflowResult:+ return compact_backfill_result(await self._run(inputs), inputs.backfill_id)++ async def _run(self, inputs: RunAggregateEvaluationInputs) -> WorkflowResult:# A historical unit is settled by definition, so a backfill skips the settle wait and# aggregates from the anchor, its first matching generation. Live starts carry no anchor,# so old histories never take this branch and it needs no patch marker.diff --git a/posthog/temporal/ai_observability/run_evaluation.py b/posthog/temporal/ai_observability/run_evaluation.pyindex 57ed366011f7..5af97c6a5741 100644--- a/posthog/temporal/ai_observability/run_evaluation.py+++ b/posthog/temporal/ai_observability/run_evaluation.py@@ -255,6 +255,17 @@ async def handle_terminal_user_error_result(return workflow_result+def compact_backfill_result(result: WorkflowResult, backfill_id: str | None) -> WorkflowResult:+ # A backfill waits on its children, so each result lands in the backfill's own history. Hog+ # output has no size cap, and a page of large results can pass Temporal's history limit.+ if backfill_id is None:+ return result+ compact = result.copy()+ compact.pop("reasoning", None)+ compact.pop("message", None)+ return compact++@temporalio.workflow.defn(name="run-evaluation")class RunEvaluationWorkflow(PostHogWorkflow):@staticmethod@@ -267,6 +278,9 @@ class RunEvaluationWorkflow(PostHogWorkflow):@temporalio.workflow.runasync def run(self, inputs: RunEvaluationInputs) -> WorkflowResult:+ return compact_backfill_result(await self._run(inputs), inputs.backfill_id)++ async def _run(self, inputs: RunEvaluationInputs) -> WorkflowResult:# Every #71518-era run recorded the "remove-trial-evals" patch marker; accept those# histories for one release, then delete this call (temporal-workflow-versioning rule).temporalio.workflow.deprecate_patch("remove-trial-evals")@@ -326,6 +340,24 @@ class RunEvaluationWorkflow(PostHogWorkflow):evaluation_type = evaluation.get("evaluation_type", "llm_judge")+ # A backfill can outlive the evaluation it runs, so it checks before the billed judge call.+ # The skip also tells the backfill to stop the rest of its page.+ if (+ result is None+ and inputs.backfill_id is not None+ and (evaluation["deleted"] or not evaluation["enabled"])+ and temporalio.workflow.patched("evaluation-backfill-skips-disabled-evaluation")+ ):+ disabled_result: WorkflowResult = {+ "skipped": True,+ "skip_reason": "evaluation_deleted" if evaluation["deleted"] else "evaluation_disabled",+ "evaluation_id": inputs.evaluation_id,+ "evaluation_type": evaluation_type,+ }+ if evaluation.get("output_type") not in ("numeric", "categorical"):+ disabled_result["verdict"] = None+ return disabled_result+if result is None:if evaluation_type == "hog":result = await temporalio.workflow.execute_activity(diff --git a/posthog/temporal/ai_observability/test_evaluation_backfill.py b/posthog/temporal/ai_observability/test_evaluation_backfill.pyindex d4dd0e6ee2bc..a6946596f7a3 100644--- a/posthog/temporal/ai_observability/test_evaluation_backfill.py+++ b/posthog/temporal/ai_observability/test_evaluation_backfill.py@@ -1,5 +1,7 @@import uuid+import asyncioimport dataclasses+from collections.abc import Mappingfrom datetime import UTC, datetime, timedeltafrom typing import Any, cast@@ -11,9 +13,16 @@ from django.test import override_settingsimport temporaliofrom asgiref.sync import async_to_sync+from temporalio.api.enums.v1 import EventTypefrom temporalio.client import WorkflowFailureErrorfrom temporalio.common import WorkflowIDReusePolicy-from temporalio.exceptions import ActivityError, ApplicationError, CancelledError, WorkflowAlreadyStartedError+from temporalio.exceptions import (+ ActivityError,+ ApplicationError,+ CancelledError,+ ChildWorkflowError,+ WorkflowAlreadyStartedError,+)from temporalio.testing import ActivityEnvironment, WorkflowEnvironmentfrom temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Workerfrom temporalio.workflow import ParentClosePolicy@@ -21,6 +30,7 @@ from temporalio.workflow import ParentClosePolicyfrom posthog.models import Organization, Teamfrom posthog.temporal.ai_observability.evaluation_backfill import (ACTIVITY_RETRY_POLICY,+ BACKFILL_FAILURE_PAUSES,BACKFILL_MAX_CONSECUTIVE_FAILURES,MAX_BACKFILL_BATCH_SIZE,AdvanceCursorInputs,@@ -52,7 +62,7 @@ from posthog.temporal.ai_observability.run_aggregate_evaluation import (RunAggregateEvaluationInputs,RunAggregateEvaluationWorkflow,)-from posthog.temporal.ai_observability.run_evaluation import RunEvaluationWorkflow+from posthog.temporal.ai_observability.run_evaluation import RunEvaluationWorkflow, WorkflowResultfrom products.ai_observability.backend.backfill_candidates import BackfillCandidate, BackfillScope, CandidatePagefrom products.ai_observability.backend.models.evaluation_backfill import EvaluationBackfill, EvaluationBackfillStatus@@ -70,29 +80,62 @@ class _BackfillMocks:*,activity_results: dict | None = None,child_errors_for_ids: dict[str, Exception] | None = None,+ child_results_for_ids: Mapping[str, WorkflowResult | Exception | asyncio.Future] | None = None,) -> None:self.activity_results = activity_results or {}self.child_errors_for_ids = child_errors_for_ids or {}+ self.child_results_for_ids = child_results_for_ids or {}+ self.active_children = 0+ self.peak_children = 0+ self.active_at_advance: list[int] = []self.activity_calls: list[tuple] = []+ self.sleep = AsyncMock()+ self.waits: list[tuple[timedelta, bool]] = []+ self.waiting = Falseself.child_calls: list[dict] = []+ async def wait_condition(self, fn, *, timeout: timedelta) -> None:+ self.waiting = True+ for _ in range(100):+ if fn():+ self.waits.append((timeout, True))+ return+ await asyncio.sleep(0)+ self.waits.append((timeout, False))+ raise TimeoutError+async def execute_activity(self, activity_fn, activity_input, **_) -> object:self.activity_calls.append((activity_fn, activity_input))result = self.activity_results.get(activity_fn)if isinstance(result, Exception):raise resultif activity_fn is advance_evaluation_backfill_cursor_activity and activity_fn not in self.activity_results:- return AdvanceCursorOutput(finished=False)+ self.active_at_advance.append(self.active_children)+ return AdvanceCursorOutput(finished=activity_input.exhausted or activity_input.stop_reason is not None)return resultasync def start_child_workflow(self, *args, **kwargs) -> object:- wid = kwargs.get("id")+ wid = str(kwargs["id"])self.child_calls.append({"name": args[0], "id": wid, "inputs": args[1] if len(args) > 1 else None, "kwargs": kwargs})if wid is not None and wid in self.child_errors_for_ids:raise self.child_errors_for_ids[wid]- return object()++ async def complete() -> WorkflowResult:+ self.active_children += 1+ self.peak_children = max(self.peak_children, self.active_children)+ await asyncio.sleep(0)+ self.active_children -= 1+ default_result: WorkflowResult = {"evaluation_id": "E", "evaluation_type": "hog", "skipped": False}+ result = self.child_results_for_ids.get(wid, default_result)+ if isinstance(result, asyncio.Future):+ return await result+ if isinstance(result, Exception):+ raise result+ return result++ return asyncio.create_task(complete())def _inputs() -> EvaluationBackfillInputs:@@ -124,16 +167,18 @@ def _found(candidates: list[CandidatePayload], *, exhausted: bool = False) -> Fi)-async def _run(mocks: _BackfillMocks, inputs: EvaluationBackfillInputs | None = None):+async def _run(mocks: _BackfillMocks, inputs: EvaluationBackfillInputs | None = None, *, bounded: bool = False):# `workflow.logger` reaches into the workflow runtime, which isn't set up here.noop = staticmethod(lambda *_a, **_kw: None)- fake_logger = type("Logger", (), {"exception": noop, "warning": noop})()+ fake_logger = type("Logger", (), {"exception": noop, "warning": noop, "info": noop})()with (patch("temporalio.workflow.logger", fake_logger),patch("temporalio.workflow.execute_activity", side_effect=mocks.execute_activity),patch("temporalio.workflow.start_child_workflow", side_effect=mocks.start_child_workflow),patch("temporalio.workflow.continue_as_new") as continue_as_new,- patch("temporalio.workflow.sleep", new=AsyncMock()),+ patch("temporalio.workflow.sleep", new=mocks.sleep),+ patch("temporalio.workflow.wait_condition", new=mocks.wait_condition),+ patch("temporalio.workflow.patched", return_value=bounded),):await EvaluationBackfillWorkflow().run(inputs or _inputs())return continue_as_new@@ -147,6 +192,22 @@ def _advance_input(mocks: _BackfillMocks) -> AdvanceCursorInputs:return next(call for fn, call in mocks.activity_calls if fn is advance_evaluation_backfill_cursor_activity)[email protected](name="run-evaluation")+class _StubEvaluationRun:+ """Fails every run, or finishes u0 and u1 and leaves the rest running until cancelled."""++ fail = True++ @temporalio.workflow.run+ async def run(self, _: Any) -> WorkflowResult:+ if self.fail:+ raise ApplicationError("judge unavailable", non_retryable=True)+ if temporalio.workflow.info().workflow_id.endswith(("-u0-ingestion", "-u1-ingestion")):+ return {"evaluation_id": "E", "evaluation_type": "hog", "skipped": False}+ await temporalio.workflow.wait_condition(lambda: False)+ raise AssertionError("unreachable")++class TestEvaluationBackfillWorkflow:@pytest.mark.asyncio@pytest.mark.parametrize("target", ["generation", "trace", "session"])@@ -282,6 +343,303 @@ class TestEvaluationBackfillWorkflow:workflow_runner=UnsandboxedWorkflowRunner(),).replay_workflow(history)+ @pytest.mark.asyncio+ @pytest.mark.parametrize("cancel", [False, True], ids=["persistent_outage_across_pages", "cancel_mid_page"])+ async def test_bounded_backfill_on_a_real_worker(self, cancel: bool, caplog: pytest.LogCaptureFixture) -> None:+ caplog.set_level("INFO", logger="temporalio.workflow")+ caplog.set_level("INFO", logger="temporalio.activity")+ units = [_candidate(f"u{i}") for i in range(5 if cancel else 70)]+ page_size = 5 if cancel else 35+ advances: list[AdvanceCursorInputs] = []+ measured: list[MeasureRemainderInputs] = []++ @temporalio.activity.defn(name="prepare_evaluation_backfill_tick_activity")+ async def prepare(_: EvaluationBackfillInputs) -> PrepareTickOutput:+ return _tick(max_in_flight=page_size if cancel else 1)++ @temporalio.activity.defn(name="find_evaluation_backfill_candidates_activity")+ async def find(_: FindCandidatesInputs) -> FindCandidatesOutput:+ first = page_size * sum(advance.new_cursor_timestamp is not None for advance in advances)+ return _found(units[first : first + page_size], exhausted=first + page_size >= len(units))++ @temporalio.activity.defn(name="advance_evaluation_backfill_cursor_activity")+ async def advance(inputs: AdvanceCursorInputs) -> AdvanceCursorOutput:+ advances.append(inputs)+ return AdvanceCursorOutput(finished=inputs.exhausted or inputs.stop_reason is not None or inputs.cancelled)++ @temporalio.activity.defn(name="measure_evaluation_backfill_remainder_activity")+ async def measure(inputs: MeasureRemainderInputs) -> None:+ measured.append(inputs)++ _StubEvaluationRun.fail = not cancel+ task_queue = settings.LLMA_EVALS_TASK_QUEUE+ async with await WorkflowEnvironment.start_time_skipping() as env:+ async with Worker(+ env.client,+ task_queue=task_queue,+ workflows=[EvaluationBackfillWorkflow, _StubEvaluationRun],+ activities=[prepare, find, advance, measure],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ handle = await env.client.start_workflow(+ EvaluationBackfillWorkflow.run,+ _inputs(),+ id=str(uuid.uuid4()),+ task_queue=task_queue,+ execution_timeout=timedelta(hours=6),+ )+ if cancel:+ # Cancel once the parent has taken in both finished runs and every start.+ for _ in range(200):+ events = [event.event_type for event in (await handle.fetch_history()).events]+ if (+ events.count(EventType.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED) == 2+ and events.count(EventType.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED) == 5+ and events[-1] == EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED+ ):+ break+ await asyncio.sleep(0.05)+ await handle.cancel()+ with pytest.raises(WorkflowFailureError):+ await handle.result()+ else:+ await handle.result()+ histories = [+ await env.client.get_workflow_handle(handle.id, run_id=handle.run_id).fetch_history(),+ await env.client.get_workflow_handle(handle.id).fetch_history(),+ ]++ last = advances[-1]+ if cancel:+ assert last.cancelled+ assert (last.dispatched_delta, last.completed_delta, last.failed_delta) == (5, 2, 0)+ assert measured == []+ else:+ assert len(advances) == 2+ assert sum(page.dispatched_delta for page in advances) == 60+ assert last.stop_reason == "execution_failed"+ assert len(measured) == 1+ replayer = Replayer(workflows=[EvaluationBackfillWorkflow], workflow_runner=UnsandboxedWorkflowRunner())+ for history in histories:+ await replayer.replay_workflow(history)++ @pytest.mark.asyncio+ @pytest.mark.parametrize("max_in_flight,peak", [(0, 16), (4, 4)])+ async def test_batch_dispatch_waits_for_outcomes_before_advancing(self, max_in_flight: int, peak: int) -> None:+ found = dataclasses.replace(+ _found([_candidate(f"u{i}") for i in range(19)], exhausted=True),+ next_cursor_timestamp=(UNIT_TIMESTAMP - timedelta(hours=1)).isoformat(),+ next_cursor_unit_id="filtered-out-unit",+ )+ mocks = _BackfillMocks(+ activity_results={+ prepare_evaluation_backfill_tick_activity: _tick(max_in_flight=max_in_flight),+ find_evaluation_backfill_candidates_activity: found,+ }+ )++ await _run(mocks, bounded=True)++ advances = [value for fn, value in mocks.activity_calls if fn is advance_evaluation_backfill_cursor_activity]+ assert mocks.peak_children == peak+ assert mocks.active_at_advance == [0]+ assert len(advances) == 1+ assert advances[0].completed_delta == 19+ assert advances[0].exhausted+ assert (advances[0].expected_cursor_timestamp, advances[0].expected_cursor_unit_id) == (+ found.started_from_cursor_timestamp,+ found.started_from_cursor_unit_id,+ )+ assert (advances[0].new_cursor_timestamp, advances[0].new_cursor_unit_id) == (+ found.next_cursor_timestamp,+ found.next_cursor_unit_id,+ )++ @pytest.mark.asyncio+ @pytest.mark.parametrize("cancelled_run", [False, True])+ async def test_bounded_dispatch_records_execution_failures_and_skips(self, cancelled_run: bool) -> None:+ failure: Exception = RuntimeError("worker unavailable")+ if cancelled_run:+ failure = ChildWorkflowError(+ "Child workflow cancelled",+ namespace="default",+ workflow_id="llma-hog-eval-E-u1-ingestion",+ run_id="run",+ workflow_type="run-evaluation",+ initiated_event_id=1,+ started_event_id=2,+ retry_state=None,+ )+ failure.__cause__ = CancelledError("cancelled")+ mocks = _BackfillMocks(+ activity_results={+ prepare_evaluation_backfill_tick_activity: _tick(),+ find_evaluation_backfill_candidates_activity: _found(+ [_candidate(f"u{i}") for i in range(4)], exhausted=True+ ),+ },+ child_results_for_ids={+ "llma-hog-eval-E-u1-ingestion": failure,+ "llma-hog-eval-E-u2-ingestion": {+ "evaluation_id": "E",+ "evaluation_type": "hog",+ "skipped": True,+ "skip_reason": "request_rejected",+ },+ "llma-hog-eval-E-u3-ingestion": {+ "evaluation_id": "E",+ "evaluation_type": "hog",+ "skipped": True,+ "skip_reason": "key_invalid",+ },+ },+ )++ await _run(mocks, bounded=True)++ result = _advance_input(mocks)+ assert (+ result.dispatched_delta,+ result.completed_delta,+ result.evaluation_skipped_delta,+ result.failed_delta,+ ) == (4, 1, 2, 1)++ @pytest.mark.asyncio+ async def test_a_cancel_mid_page_records_the_runs_already_started(self) -> None:+ mocks = _BackfillMocks(+ activity_results={+ prepare_evaluation_backfill_tick_activity: _tick(max_in_flight=2),+ find_evaluation_backfill_candidates_activity: _found([_candidate(f"u{i}") for i in range(4)]),+ },+ child_results_for_ids={"llma-hog-eval-E-u1-ingestion": asyncio.get_running_loop().create_future()},+ )+ run = asyncio.create_task(_run(mocks, bounded=True))+ while len(mocks.child_calls) < 3:+ await asyncio.sleep(0)++ run.cancel()+ with pytest.raises(asyncio.CancelledError):+ await run++ result = _advance_input(mocks)+ assert result.cancelled+ assert (result.dispatched_delta, result.completed_delta, result.failed_delta) == (3, 1, 0)+ assert result.new_cursor_unit_id == "u3"++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "failing,stopping,page_size,started,pauses,stop_reason",+ [+ pytest.param(+ set(range(70)), None, 70, 60, list(BACKFILL_FAILURE_PAUSES), "execution_failed", id="persistent_outage"+ ),+ pytest.param(+ set(range(70)),+ None,+ 20,+ 60,+ list(BACKFILL_FAILURE_PAUSES),+ "execution_failed",+ id="persistent_outage_across_pages",+ ),+ pytest.param(+ set(range(10, 70)),+ None,+ 70,+ 70,+ list(BACKFILL_FAILURE_PAUSES),+ "execution_failed",+ id="outage_spends_budget_on_last_unit",+ ),+ pytest.param({i for i in range(70) if i % 10}, None, 70, 70, [], None, id="scattered_failures"),+ pytest.param(+ set(range(10)) | set(range(11, 21)),+ None,+ 70,+ 70,+ [BACKFILL_FAILURE_PAUSES[0]] * 2,+ None,+ id="success_resets_pauses",+ ),+ pytest.param(set(), (1, "key_invalid"), 70, 2, [], "evaluation_disabled", id="evaluation_disabled"),+ pytest.param(set(), (1, "evaluation_deleted"), 70, 2, [], "evaluation_deleted", id="evaluation_deleted"),+ ],+ )+ async def test_failures_pause_and_stop_the_backfill(+ self,+ failing: set[int],+ stopping: tuple[int, str] | None,+ page_size: int,+ started: int,+ pauses: list[timedelta],+ stop_reason: str | None,+ ) -> None:+ results: dict[str, WorkflowResult | Exception] = {+ f"llma-hog-eval-E-u{i}-ingestion": RuntimeError("judge unavailable") for i in failing+ }+ if stopping is not None:+ unit, reason = stopping+ results[f"llma-hog-eval-E-u{unit}-ingestion"] = {+ "evaluation_id": "E",+ "evaluation_type": "hog",+ "skipped": True,+ "skip_reason": reason,+ }+ units = [_candidate(f"u{i}") for i in range(70)]+ inputs, advances, sleeps, waits = _inputs(), [], [], []+ for first in range(0, len(units), page_size):+ mocks = _BackfillMocks(+ activity_results={+ prepare_evaluation_backfill_tick_activity: _tick(max_in_flight=1),+ find_evaluation_backfill_candidates_activity: _found(+ units[first : first + page_size], exhausted=first + page_size >= len(units)+ ),+ },+ child_results_for_ids=results,+ )+ continue_as_new = await _run(mocks, inputs, bounded=True)+ advances.append(_advance_input(mocks))+ sleeps += [call.args[0] for call in mocks.sleep.call_args_list]+ waits += [timeout for timeout, _ in mocks.waits]+ if not continue_as_new.called:+ break+ inputs = continue_as_new.call_args.args[0]++ advance = advances[-1]+ assert sum(page.dispatched_delta for page in advances) == started+ assert sleeps == []+ assert waits == pauses+ assert advance.stop_reason == stop_reason+ assert advance.exhausted == (stop_reason is None)+ assert (advance.new_cursor_timestamp is None) == (stop_reason is not None)++ @pytest.mark.asyncio+ async def test_a_stop_during_a_pause_ends_the_pause(self) -> None:+ disabling = asyncio.get_running_loop().create_future()+ results: dict[str, WorkflowResult | Exception | asyncio.Future] = {+ f"llma-hog-eval-E-u{i}-ingestion": RuntimeError("judge unavailable") for i in range(1, 20)+ }+ results["llma-hog-eval-E-u0-ingestion"] = disabling+ mocks = _BackfillMocks(+ activity_results={+ prepare_evaluation_backfill_tick_activity: _tick(max_in_flight=2),+ find_evaluation_backfill_candidates_activity: _found([_candidate(f"u{i}") for i in range(20)]),+ },+ child_results_for_ids=results,+ )+ run = asyncio.create_task(_run(mocks, bounded=True))+ while not mocks.waiting:+ await asyncio.sleep(0)++ disabling.set_result(+ {"evaluation_id": "E", "evaluation_type": "hog", "skipped": True, "skip_reason": "key_invalid"}+ )+ await run++ assert mocks.waits == [(BACKFILL_FAILURE_PAUSES[0], True)]+ assert _advance_input(mocks).stop_reason == "evaluation_disabled"+@pytest.mark.asyncioasync def test_finished_tick_returns_without_dispatch(self) -> None:mocks = _BackfillMocks(@@ -609,6 +967,8 @@ def _advance(dispatched_delta: int = 1,skipped_delta: int = 0,failed_delta: int = 0,+ completed_delta: int | None = None,+ evaluation_skipped_delta: int = 0,exhausted: bool = False,) -> AdvanceCursorInputs:return AdvanceCursorInputs(@@ -621,6 +981,8 @@ def _advance(dispatched_delta=dispatched_delta,skipped_delta=skipped_delta,failed_delta=failed_delta,+ completed_delta=completed_delta,+ evaluation_skipped_delta=evaluation_skipped_delta,exhausted=exhausted,)@@ -792,11 +1154,11 @@ class TestEvaluationBackfillActivities:]@pytest.mark.parametrize(- "configured,expected",- [(7, 7), (0, 1), (-5, 1), (MAX_BACKFILL_BATCH_SIZE * 10, MAX_BACKFILL_BATCH_SIZE)],+ "configured,expected,in_flight",+ [(7, 7, 7), (0, 1, 1), (-5, 1, 1), (MAX_BACKFILL_BATCH_SIZE * 10, MAX_BACKFILL_BATCH_SIZE, 10000)],)- def test_prepare_dispatches_with_a_clamped_batch_size(self, backfill_data, configured, expected) -> None:- with override_settings(LLMA_EVAL_BACKFILL_BATCH_SIZE=configured):+ def test_prepare_dispatches_with_a_clamped_batch_size(self, backfill_data, configured, expected, in_flight) -> None:+ with override_settings(LLMA_EVAL_BACKFILL_BATCH_SIZE=configured, LLMA_EVAL_BACKFILL_MAX_IN_FLIGHT=configured):result = async_to_sync(prepare_evaluation_backfill_tick_activity)(_activity_inputs(backfill_data))assert result == PrepareTickOutput(@@ -808,6 +1170,7 @@ class TestEvaluationBackfillActivities:settle={},rerun_existing=False,batch_size=expected,+ max_in_flight=in_flight,)def test_find_serializes_candidates_and_cursor(self, backfill_data) -> None:@@ -855,20 +1218,37 @@ class TestEvaluationBackfillActivities:)assert result.exhausted- def test_advance_is_idempotent_on_retry(self, backfill_data) -> None:- advance = _advance(backfill_data, dispatched_delta=3, skipped_delta=1, failed_delta=2)+ @pytest.mark.parametrize("stop_reason", [None, "execution_failed"])+ def test_advance_is_idempotent_on_retry(self, backfill_data, stop_reason: str | None) -> None:+ advance = _advance(+ backfill_data,+ dispatched_delta=5,+ skipped_delta=1,+ failed_delta=2,+ completed_delta=2,+ evaluation_skipped_delta=1,+ )+ if stop_reason:+ advance = dataclasses.replace(+ advance, new_cursor_timestamp=None, new_cursor_unit_id="", stop_reason=stop_reason+ )first = async_to_sync(advance_evaluation_backfill_cursor_activity)(advance)second = async_to_sync(advance_evaluation_backfill_cursor_activity)(advance)backfill_data["backfill"].refresh_from_db()row = backfill_data["backfill"]- assert (row.dispatched_count, row.skipped_count, row.failed_count) == (3, 1, 2)- assert backfill_data["backfill"].cursor_unit_id == "u3"- assert not first.finished- # The second call matched nothing because the first already moved the cursor. Reading that- # as finished would end the loop with the row still RUNNING and the window half walked.- assert not second.finished+ assert (row.dispatched_count, row.skipped_count, row.failed_count) == (5, 1, 2)+ assert (row.completed_count, row.evaluation_skipped_count) == (2, 1)+ assert (row.status, row.status_reason) == (+ (EvaluationBackfillStatus.INTERRUPTED, stop_reason)+ if stop_reason+ else (EvaluationBackfillStatus.RUNNING, "")+ )+ assert row.cursor_unit_id == ("" if stop_reason else "u3")+ # The second call matched nothing because the first already moved the cursor or ended the+ # row. Reading a moved cursor as finished would end the loop with the window half walked.+ assert first.finished == second.finished == bool(stop_reason)def test_advance_marks_completed_when_exhausted(self, backfill_data) -> None:result = async_to_sync(advance_evaluation_backfill_cursor_activity)(_advance(backfill_data, exhausted=True))@@ -878,12 +1258,31 @@ class TestEvaluationBackfillActivities:assert backfill_data["backfill"].status == EvaluationBackfillStatus.COMPLETEDassert backfill_data["backfill"].finished_at is not None- def test_advance_loses_to_concurrent_cancel(self, backfill_data) -> None:+ def test_resuming_legacy_backfill_does_not_claim_partial_outcomes_are_totals(self, backfill_data) -> None:+ _update_backfill(backfill_data, dispatched_count=3, cursor_timestamp=UNIT_TIMESTAMP, cursor_unit_id="u0")+ advance = dataclasses.replace(+ _advance(backfill_data, dispatched_delta=2, completed_delta=2, exhausted=True),+ expected_cursor_timestamp=UNIT_TIMESTAMP.isoformat(),+ expected_cursor_unit_id="u0",+ )++ async_to_sync(advance_evaluation_backfill_cursor_activity)(advance)++ row = backfill_data["backfill"]+ row.refresh_from_db()+ assert row.dispatched_count == 5+ assert row.completed_count is None+ assert row.remaining_count is None++ @pytest.mark.parametrize("cancelled", [False, True])+ def test_advance_after_a_cancel_counts_the_page_once(self, backfill_data, cancelled: bool) -> None:_update_backfill(backfill_data, status=EvaluationBackfillStatus.CANCELLED)+ advance = dataclasses.replace(_advance(backfill_data), cancelled=cancelled)- result = async_to_sync(advance_evaluation_backfill_cursor_activity)(_advance(backfill_data))+ first = async_to_sync(advance_evaluation_backfill_cursor_activity)(advance)+ second = async_to_sync(advance_evaluation_backfill_cursor_activity)(advance)- assert result.finished+ assert first.finished and second.finishedbackfill_data["backfill"].refresh_from_db()assert backfill_data["backfill"].status == EvaluationBackfillStatus.CANCELLED- assert backfill_data["backfill"].dispatched_count == 0+ assert backfill_data["backfill"].dispatched_count == 1diff --git a/posthog/temporal/ai_observability/test_run_aggregate_evaluation.py b/posthog/temporal/ai_observability/test_run_aggregate_evaluation.pyindex e138e63470e5..c09e4415418c 100644--- a/posthog/temporal/ai_observability/test_run_aggregate_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_aggregate_evaluation.py@@ -416,6 +416,7 @@ class TestRunAggregateEvaluationWorkflow:assert calls == ["fetch", "execute", "emit", "telemetry"]assert result["verdict"] is True+ assert "reasoning" not in resultassert window_starts == [datetime.fromisoformat(anchor).isoformat()]# Bounded to the span the live path would have covered from the anchor, so the grade does# not sweep in everything that arrived between the anchor and now.diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.pyindex d4ff271ba8f6..28e5b957d4aa 100644--- a/posthog/temporal/ai_observability/test_run_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_evaluation.py@@ -95,6 +95,7 @@ from .run_evaluation import (RunLocalEvaluationInputs,SendEvaluationDisabledEmailInputs,WorkflowResult,+ compact_backfill_result,disable_evaluation_activity,emit_evaluation_event_activity,execute_hog_eval_activity,@@ -2184,6 +2185,36 @@ class TestRunEvaluationWorkflow:mock_emit.assert_awaited_once()assert mock_emit.await_args_list[0].args[0].event_data == full_event+ @pytest.mark.asyncio+ @pytest.mark.parametrize("evaluation_type", ["hog", "sentiment"])+ @pytest.mark.parametrize("change", [{"enabled": False}, {"deleted": True}], ids=["disabled", "deleted"])+ async def test_backfill_does_not_run_a_local_evaluation_turned_off(+ self, evaluation_type: str, change: dict[str, bool]+ ) -> None:+ evaluation = _local_evaluation(+ **{"evaluation_type": evaluation_type, "enabled": True, "deleted": False, **change}+ )+ module = "posthog.temporal.ai_observability.evaluation_workflow_activities"+ with (+ patch(f"{module}.fetch_evaluation", return_value=evaluation),+ patch(f"{module}.run_hog_eval_for_event", new_callable=AsyncMock) as run_hog,+ patch(f"{module}.run_sentiment_eval", new_callable=AsyncMock) as run_sentiment,+ patch(f"{module}.emit_generation_evaluation_event", new_callable=AsyncMock) as emit,+ ):+ outcome = await run_local_evaluation_activity(+ RunLocalEvaluationInputs(+ evaluation_id="eval-1",+ event_data=dict(THIN_REFERENCE),+ start_time=datetime(2024, 1, 1, 12, 0, tzinfo=UTC),+ backfill_id="B",+ )+ )++ assert (outcome.result, outcome.emitted) == (None, False)+ run_hog.assert_not_awaited()+ run_sentiment.assert_not_awaited()+ emit.assert_not_awaited()+@pytest.mark.asyncioasync def test_the_generation_emit_activity_hydrates_a_thin_reference(self):full_event = create_mock_event_data(team_id=1, uuid="g1")@@ -2532,6 +2563,51 @@ class TestRunEvaluationWorkflow:assert result["verdict"] is Trueassert result["skipped"] is False+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "change,skip_reason", [({"enabled": False}, "evaluation_disabled"), ({"deleted": True}, "evaluation_deleted")]+ )+ async def test_backfill_run_skips_the_judge_for_a_disabled_evaluation(+ self, change: dict[str, bool], skip_reason: str+ ) -> None:+ @activity.defn(name="run_local_evaluation_activity")+ async def mock_run_local_evaluation(inputs: RunLocalEvaluationInputs) -> LocalEvaluationOutcome:+ evaluation = {+ "id": inputs.evaluation_id,+ "evaluation_type": "llm_judge",+ "output_type": "boolean",+ "team_id": 1,+ "enabled": True,+ "deleted": False,+ }+ return LocalEvaluationOutcome(evaluation={**evaluation, **change}, result=None, emitted=False)++ @activity.defn(name="execute_llm_judge_activity")+ async def mock_judge(inputs: ExecuteLLMJudgeInputs) -> EvaluationActivityResult:+ raise AssertionError("a backfill must not bill a judge call for a disabled evaluation")++ task_queue = str(uuid.uuid4())+ async with await WorkflowEnvironment.start_time_skipping() as env:+ async with Worker(+ env.client,+ task_queue=task_queue,+ workflows=[RunEvaluationWorkflow],+ activities=[mock_run_local_evaluation, mock_judge],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ result: WorkflowResult = await env.client.execute_workflow(+ RunEvaluationWorkflow.run,+ RunEvaluationInputs(+ evaluation_id=str(uuid.uuid4()), event_data=create_mock_event_data(team_id=1), backfill_id="B"+ ),+ id=str(uuid.uuid4()),+ task_queue=task_queue,+ execution_timeout=timedelta(minutes=5),+ )++ assert result["skipped"] is True+ assert result["skip_reason"] == skip_reason+@pytest.mark.asyncioasync def test_terminal_user_error_does_not_email_when_evaluation_was_already_disabled(self):calls: list[str] = []@@ -4682,3 +4758,24 @@ class TestRunLocalEvaluationActivity:second_uuid = mock_capture.call_args_list[1].kwargs["event_uuid"]assert first_uuid is not Noneassert first_uuid == second_uuid+++ "backfill_id,expected",+ [+ pytest.param(+ None, {"skipped": True, "skip_reason": "parse_error", "reasoning": "x", "message": "y"}, id="live"+ ),+ pytest.param("bf-1", {"skipped": True, "skip_reason": "parse_error"}, id="backfill"),+ ],+)+def test_backfill_results_drop_free_text(backfill_id: str | None, expected: dict) -> None:+ result: WorkflowResult = {+ "evaluation_id": "e",+ "evaluation_type": "hog",+ "skipped": True,+ "skip_reason": "parse_error",+ "reasoning": "x",+ "message": "y",+ }+ assert compact_backfill_result(result, backfill_id) == {"evaluation_id": "e", "evaluation_type": "hog", **expected}diff --git a/products/ai_observability/backend/api/evaluation_backfills.py b/products/ai_observability/backend/api/evaluation_backfills.pyindex 26f803e0a733..148dffe5c59f 100644--- a/products/ai_observability/backend/api/evaluation_backfills.py+++ b/products/ai_observability/backend/api/evaluation_backfills.py@@ -668,7 +668,7 @@ class EvaluationBackfillViewSet(asyncio.run(client.get_workflow_handle(backfill_workflow_id(str(backfill.pk))).cancel())except Exception as error:# The workflow reads the row's status at every tick and stops on a terminal one, so- # a failed handle costs nothing and does not need a stack trace.+ # a failed handle only lets the current page finish and does not need a stack trace.logger.warning("llma.evaluation_backfill_cancel_failed", backfill_id=str(backfill.pk), error=str(error))# The status and finished_at were written by the update, not on this instance.backfill.refresh_from_db()