Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): bound backfill dispatch and pause or stop on repeated failures (#113431)

Bernat Torresbedbeda875e9
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index e8ceee308d75..b7d089f7825f 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -184,8 +184,16 @@ The playground keeps the provider's explanation so users can correct the setting
A backfill that stops on its own shows as **Interrupted**, with a stop reason.
A backfill a user stops shows as **Cancelled**.
+Backfills read a page of units (500 by default, bounded between 1 and 1,000) and wait for every outcome before advancing progress.
+A backfill runs at most 16 evaluations at once (`LLMA_EVAL_BACKFILL_MAX_IN_FLIGHT`).
+A run holds one of the shared ClickHouse slots only while it reads its unit, and spends most of its time waiting on the judge.
+Lower the setting if backfills crowd out live evaluations.
Backfill activities retry temporary database, DNS, connection, and rate-limit failures up to 5 times within 10 minutes, or within the live time limit when that is longer, with backoff from 10 seconds to one minute.
A backfill run that still hits a DNS failure on its last attempt fails instead of recording a skip, so **Retry remaining** can run it again.
+After 10 failed runs in a row, the backfill pauses for 1, 2, 5, 10, then 30 minutes, and a success resets that sequence.
+The failure count and the pauses carry over between pages.
+When the pauses run out, or a run reports an error that disables the evaluation, the backfill stops as **Interrupted** and keeps its cursor.
+Its stop reason says whether the evaluation was disabled or the runs kept failing.
Live evaluation retry budgets remain unchanged.
Authentication, quota, blocked-endpoint, and permanent request errors retain their existing handling.
Result emission has a separate retry budget, so retrying emission does not repeat a completed judge call.
@@ -194,6 +202,8 @@ An interrupted backfill preserves completed results.
**Retry remaining** creates a new backfill with the original date range and filters, using the current evaluation settings.
It preserves usable existing results and retries units without usable results.
It is not available for a backfill that re-ran existing results.
+A cancelled backfill still counts the runs it started before the cancel, because they finish and are billed.
+Persistent failures can still interrupt a backfill after its recovery budget; recovery does not guarantee a verdict for invalid or permanently rejected inputs.
## Result encoding
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py
index f89cd60f24d7..3d2bcc13fd4f 100644
--- a/posthog/settings/temporal.py
+++ b/posthog/settings/temporal.py
@@ -408,6 +408,9 @@ LLMA_TASK_QUEUE = _set_temporal_task_queue("llm-analytics-task-queue")
# A tick costs one candidate query whatever it dispatches, so a larger batch means fewer ticks and
# less ClickHouse work for the same backfill.
LLMA_EVAL_BACKFILL_BATCH_SIZE: int = get_from_env("LLMA_EVAL_BACKFILL_BATCH_SIZE", 500, type_cast=int)
+# Evaluations one backfill runs at once. A run holds one of the shared CLICKHOUSE_LLM_ANALYTICS_MAX_CONCURRENT_QUERIES
+# slots only while it reads its unit, so most runs wait on the judge. Lower this if backfills crowd out live work.
+LLMA_EVAL_BACKFILL_MAX_IN_FLIGHT: int = get_from_env("LLMA_EVAL_BACKFILL_MAX_IN_FLIGHT", 16, type_cast=int)
# Defaults to the general-purpose fleet so dispatch always has a live worker; set the env to
# "mcp-analytics-task-queue" to route MCP analytics clustering to a dedicated, separately-scalable
# worker once one is deployed.
diff --git a/posthog/temporal/ai_observability/evaluation_backfill.py b/posthog/temporal/ai_observability/evaluation_backfill.py
index 26f75160a7d5..1affe9654837 100644
--- a/posthog/temporal/ai_observability/evaluation_backfill.py
+++ b/posthog/temporal/ai_observability/evaluation_backfill.py
@@ -6,6 +6,8 @@ own and continues as new after each batch, however long the window takes to drai
import asyncio
import hashlib
+import contextlib
+from collections import Counter
from dataclasses import replace
from datetime import datetime, timedelta
from enum import StrEnum
@@ -13,13 +15,14 @@ from typing import Any
from uuid import UUID
from django.conf import settings
-from django.db.models import F
+from django.db.models import Case, F, Q, When
+from django.db.models.functions import Coalesce
from django.utils import timezone
import structlog
import temporalio
from temporalio.common import RetryPolicy, WorkflowIDReusePolicy
-from temporalio.exceptions import WorkflowAlreadyStartedError, is_cancelled_exception
+from temporalio.exceptions import ChildWorkflowError, WorkflowAlreadyStartedError, is_cancelled_exception
from temporalio.workflow import ParentClosePolicy
from posthog.dataclasses import frozen
@@ -27,6 +30,7 @@ from posthog.event_usage import groups
from posthog.models.team import Team
from posthog.ph_client import ph_background_capture
from posthog.sync import database_sync_to_async
+from posthog.temporal.ai_observability.evaluation_errors import USER_ERROR_SPECS
from posthog.temporal.ai_observability.evaluation_event_io import as_utc_datetime
from posthog.temporal.ai_observability.evaluation_types import EVALUATION_WORKFLOW_PREFIXES
from posthog.temporal.ai_observability.evaluation_workflow_activities import RunEvaluationInputs
@@ -36,6 +40,7 @@ from posthog.temporal.ai_observability.run_aggregate_evaluation import (
RunAggregateEvaluationInputs,
resolve_settle_plan,
)
+from posthog.temporal.ai_observability.run_evaluation import WorkflowResult
from posthog.temporal.common.base import PostHogWorkflow
from products.ai_observability.backend.backfill_candidates import count_backfill_candidates, fetch_backfill_candidates
@@ -50,9 +55,9 @@ logger = structlog.get_logger(__name__)
BACKFILL_WORKFLOW_NAME = "llma-evaluation-backfill"
BACKFILL_TICK_INTERVAL = timedelta(seconds=60)
-# Long enough for the slowest child: a trace evaluation can settle for its 2 hour maximum age
-# before the judge even starts.
-CHILD_EXECUTION_TIMEOUT = timedelta(hours=3)
+# A backfill child skips the settle wait, so its activity budgets add up to under an hour. A page waits
+# on every child, so a stuck child holds it this long before it counts as a failed run.
+CHILD_EXECUTION_TIMEOUT = timedelta(hours=1)
ACTIVITY_TIMEOUT = timedelta(seconds=30)
# A tick that cannot reach a worker must fail rather than sit in the queue, because the loop only
@@ -69,6 +74,17 @@ MIN_BACKFILL_BATCH_SIZE = 1
MAX_BACKFILL_BATCH_SIZE = 1000
# A tick that keeps failing would otherwise leave the row RUNNING and the loop spinning forever.
BACKFILL_MAX_CONSECUTIVE_FAILURES = 5
+BACKFILL_DEFAULT_MAX_IN_FLIGHT = 16
+BACKFILL_FAILURES_BEFORE_PAUSE = 10
+# Each pause in a row waits longer, and a success starts the sequence over. Once the pauses run out,
+# the backfill stops and the units it did not reach stay for Retry remaining.
+BACKFILL_FAILURE_PAUSES = (
+ timedelta(minutes=1),
+ timedelta(minutes=2),
+ timedelta(minutes=5),
+ timedelta(minutes=10),
+ timedelta(minutes=30),
+)
def settle_horizon(target: str, settle: dict[str, Any] | None) -> timedelta:
@@ -95,6 +111,9 @@ class EvaluationBackfillInputs:
team_id: int
# Reset by any tick that succeeds.
consecutive_failures: int = 0
+ # Carried across pages, so a backfill smaller than one pause cycle can still stop.
+ failed_runs_in_a_row: int = 0
+ pauses_in_a_row: int = 0
class TickAction(StrEnum):
@@ -112,6 +131,8 @@ class PrepareTickOutput:
settle: dict[str, Any] | None = None
rerun_existing: bool = False
batch_size: int = 0
+ # A tick prepared by a worker that predates this field carries 0, which the workflow reads as the default.
+ max_in_flight: int = 0
@frozen
@@ -157,6 +178,11 @@ class AdvanceCursorInputs:
exhausted: bool
# Defaulted so an advance recorded before this field existed still deserializes.
failed_delta: int = 0
+ completed_delta: int | None = None
+ evaluation_skipped_delta: int = 0
+ stop_reason: str | None = None
+ # Records a page a user cancelled midway, whose started runs still finish and bill.
+ cancelled: bool = False
@frozen
@@ -183,6 +209,84 @@ class ChildWorkflow:
workflow_id: str
+class ChildOutcome(StrEnum):
+ EVALUATED = "evaluated"
+ SKIPPED = "skipped"
+ FAILED = "failed"
+ EXISTING = "existing"
+ NOT_STARTED = "not_started"
+
+
+@frozen
+class BackfillChildOutcome:
+ outcome: ChildOutcome
+ stop_reason: str | None = None
+
+
+def _stop_reason(result: WorkflowResult) -> str | None:
+ reason = result.get("skip_reason", "")
+ if reason == "evaluation_deleted":
+ return reason
+ spec = USER_ERROR_SPECS.get(reason)
+ if reason == "evaluation_disabled" or (spec is not None and spec.disables_evaluation):
+ return "evaluation_disabled"
+ return None
+
+
+class _BackfillPacing:
+ """Decides, as runs finish, whether a backfill keeps starting runs, pauses, or stops."""
+
+ def __init__(self, failures: int = 0, pauses: int = 0) -> None:
+ self.failures = failures
+ self.pauses = pauses
+ self.stop_reason: str | None = None
+ self._pausing = asyncio.Lock()
+
+ @property
+ def stopped(self) -> bool:
+ return self.stop_reason is not None
+
+ async def ready(self) -> bool:
+ async with self._pausing:
+ while not self.stopped and self.failures >= BACKFILL_FAILURES_BEFORE_PAUSE:
+ if self.pauses == len(BACKFILL_FAILURE_PAUSES):
+ self.stop_reason = "execution_failed"
+ else:
+ # Count runs still in flight toward the next streak, and let a success during
+ # the pause restart the sequence.
+ pause = BACKFILL_FAILURE_PAUSES[self.pauses]
+ self.pauses += 1
+ self.failures = 0
+ temporalio.workflow.logger.info(
+ "llma.evaluation_backfill_paused",
+ extra={"pause_seconds": pause.total_seconds(), "pauses_in_a_row": self.pauses},
+ )
+ # A run that stops the backfill mid-pause ends the pause too.
+ with contextlib.suppress(TimeoutError):
+ await temporalio.workflow.wait_condition(lambda: self.stopped, timeout=pause)
+ return not self.stopped
+
+ def finish_page(self) -> None:
+ # The last page starts no run after its final failure, so ready() never sees a spent budget.
+ if self.failures >= BACKFILL_FAILURES_BEFORE_PAUSE and self.pauses == len(BACKFILL_FAILURE_PAUSES):
+ self.stop_reason = self.stop_reason or "execution_failed"
+
+ def record(self, result: BackfillChildOutcome) -> None:
+ if result.stop_reason:
+ self.stop_reason = self.stop_reason or result.stop_reason
+ elif result.outcome == ChildOutcome.FAILED:
+ self.failures += 1
+ elif result.outcome == ChildOutcome.EVALUATED:
+ self.failures = 0
+ self.pauses = 0
+
+
+def _child_outcome(result: WorkflowResult) -> ChildOutcome:
+ # A run that returned a skip completed, so Retry remaining could not start it again. Only runs
+ # that failed count toward the remaining units.
+ return ChildOutcome.SKIPPED if result["skipped"] else ChildOutcome.EVALUATED
+
+
def child_workflow_name_and_id(
*,
evaluation_id: str,
@@ -314,6 +418,7 @@ def _prepare_backfill_tick(inputs: EvaluationBackfillInputs) -> PrepareTickOutpu
settle=evaluation.target_config,
rerun_existing=row.rerun_existing,
batch_size=max(MIN_BACKFILL_BATCH_SIZE, min(settings.LLMA_EVAL_BACKFILL_BATCH_SIZE, MAX_BACKFILL_BATCH_SIZE)),
+ max_in_flight=max(1, settings.LLMA_EVAL_BACKFILL_MAX_IN_FLIGHT),
)
@@ -376,17 +481,36 @@ async def find_evaluation_backfill_candidates_activity(inputs: FindCandidatesInp
def _advance_backfill_cursor(inputs: AdvanceCursorInputs) -> AdvanceCursorOutput:
- updates: dict[str, Any] = {
+ counts: dict[str, Any] = {
"dispatched_count": F("dispatched_count") + inputs.dispatched_delta,
"skipped_count": F("skipped_count") + inputs.skipped_delta,
"failed_count": F("failed_count") + inputs.failed_delta,
}
+ if inputs.completed_delta is not None:
+ # Runs that already dispatched without tracking outcomes must retain unknown totals.
+ tracks_outcomes = Q(cursor_timestamp__isnull=True) | Q(completed_count__isnull=False)
+ counts["completed_count"] = Case(
+ When(tracks_outcomes, then=Coalesce(F("completed_count"), 0) + inputs.completed_delta),
+ default=None,
+ )
+ counts["evaluation_skipped_count"] = F("evaluation_skipped_count") + inputs.evaluation_skipped_delta
if inputs.new_cursor_timestamp is not None:
- updates["cursor_timestamp"] = datetime.fromisoformat(inputs.new_cursor_timestamp)
- updates["cursor_unit_id"] = inputs.new_cursor_unit_id
- if inputs.exhausted:
- updates["status"] = EvaluationBackfillStatus.COMPLETED
- updates["finished_at"] = timezone.now()
+ counts["cursor_timestamp"] = datetime.fromisoformat(inputs.new_cursor_timestamp)
+ counts["cursor_unit_id"] = inputs.new_cursor_unit_id
+ terminal: dict[str, Any] = {}
+ if inputs.stop_reason:
+ terminal = {
+ "status": EvaluationBackfillStatus.INTERRUPTED,
+ "status_reason": inputs.stop_reason,
+ "finished_at": timezone.now(),
+ }
+ elif inputs.exhausted:
+ terminal = {"status": EvaluationBackfillStatus.COMPLETED, "finished_at": timezone.now()}
+ if inputs.completed_delta is not None:
+ terminal["remaining_count"] = Case(
+ When(tracks_outcomes, then=F("failed_count") + inputs.failed_delta), default=None
+ )
+ updates = {**counts, **terminal}
expected_timestamp = (
datetime.fromisoformat(inputs.expected_cursor_timestamp) if inputs.expected_cursor_timestamp else None
@@ -396,7 +520,7 @@ def _advance_backfill_cursor(inputs: AdvanceCursorInputs) -> AdvanceCursorOutput
# result was lost after it committed, and a second blind increment would inflate progress.
# Filtering on RUNNING lets a concurrent cancel win over the advance.
updated = row.filter(
- status=EvaluationBackfillStatus.RUNNING,
+ status=EvaluationBackfillStatus.CANCELLED if inputs.cancelled else EvaluationBackfillStatus.RUNNING,
cursor_timestamp=expected_timestamp,
cursor_unit_id=inputs.expected_cursor_unit_id,
).update(**updates)
@@ -406,18 +530,34 @@ def _advance_backfill_cursor(inputs: AdvanceCursorInputs) -> AdvanceCursorOutput
team_id=inputs.team_id,
dispatched=inputs.dispatched_delta,
skipped=inputs.skipped_delta,
+ completed=inputs.completed_delta,
+ evaluation_skipped=inputs.evaluation_skipped_delta,
failed=inputs.failed_delta,
exhausted=inputs.exhausted,
+ stop_reason=inputs.stop_reason,
+ cancelled=inputs.cancelled,
applied=bool(updated),
)
if updated:
- return AdvanceCursorOutput(finished=inputs.exhausted)
+ if inputs.stop_reason:
+ status = "failed" if inputs.stop_reason == "execution_failed" else "stopped"
+ report_backfill_finished(inputs.team_id, inputs.backfill_id, status=status, stop_reason=inputs.stop_reason)
+ return AdvanceCursorOutput(finished=inputs.exhausted or inputs.stop_reason is not None or inputs.cancelled)
# Zero rows has two causes that end differently. If the row is no longer RUNNING, someone
# cancelled or completed it and the loop stops. If it is still RUNNING, an earlier attempt of
# this same advance already committed and only its result was lost, so the loop must carry on
# from the stored cursor instead of stalling with the window half walked.
- status = row.values_list("status", flat=True).first()
- return AdvanceCursorOutput(finished=status != EvaluationBackfillStatus.RUNNING)
+ if not inputs.cancelled and inputs.new_cursor_timestamp is not None:
+ # A cancel can land after this page's runs finished. They still billed, so the cancelled
+ # row keeps their counts. Moving the cursor keeps a retried advance from counting twice.
+ if row.filter(
+ status=EvaluationBackfillStatus.CANCELLED,
+ cursor_timestamp=expected_timestamp,
+ cursor_unit_id=inputs.expected_cursor_unit_id,
+ ).update(**counts):
+ return AdvanceCursorOutput(finished=True)
+ current_status = row.values_list("status", flat=True).first()
+ return AdvanceCursorOutput(finished=current_status != EvaluationBackfillStatus.RUNNING)
@temporalio.activity.defn
@@ -443,6 +583,8 @@ def _measure_backfill_remainder(inputs: MeasureRemainderInputs) -> None:
count taken after the settle horizon separates the two.
"""
row = EvaluationBackfill.objects.for_team(inputs.team_id).select_related("evaluation").get(pk=inputs.backfill_id)
+ if row.completed_count is not None:
+ return
team = Team.objects.get(pk=inputs.team_id)
scope = count_backfill_candidates(
team=team,
@@ -508,6 +650,9 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):
)
if tick.action == TickAction.FINISHED:
return
+ if temporalio.workflow.patched("evaluation-backfill-await-children"):
+ await self._dispatch_bounded_batch(inputs, tick)
+ return
finished = await self._dispatch_batch(inputs, tick)
except Exception as error:
# A cancel reaches the workflow as a cancelled activity, which is not a tick that
@@ -523,15 +668,107 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):
await temporalio.workflow.sleep(BACKFILL_TICK_INTERVAL)
temporalio.workflow.continue_as_new(replace(inputs, consecutive_failures=0))
+ async def _dispatch_bounded_batch(self, inputs: EvaluationBackfillInputs, tick: PrepareTickOutput) -> None:
+ found = await self._find_candidates(inputs, tick)
+ # A tick prepared by a worker that predates max_in_flight carries 0.
+ slots = asyncio.Semaphore(tick.max_in_flight or BACKFILL_DEFAULT_MAX_IN_FLIGHT)
+ pacing = _BackfillPacing(inputs.failed_runs_in_a_row, inputs.pauses_in_a_row)
+ started: list[str] = []
+
+ async def run_in_slot(candidate: CandidatePayload) -> BackfillChildOutcome:
+ async with slots:
+ if not await pacing.ready():
+ return BackfillChildOutcome(outcome=ChildOutcome.NOT_STARTED)
+ result = await self._run_child(inputs, tick, candidate, started)
+ pacing.record(result)
+ return result
+
+ async def record(results: list[BackfillChildOutcome], *, cancelled: bool = False) -> AdvanceCursorOutput:
+ outcomes = Counter(result.outcome for result in results)
+ # A stopped page keeps its cursor, because it never reached some of its units.
+ keep_cursor = pacing.stopped and not cancelled
+ return await temporalio.workflow.execute_activity(
+ advance_evaluation_backfill_cursor_activity,
+ AdvanceCursorInputs(
+ backfill_id=inputs.backfill_id,
+ team_id=inputs.team_id,
+ expected_cursor_timestamp=found.started_from_cursor_timestamp,
+ expected_cursor_unit_id=found.started_from_cursor_unit_id,
+ new_cursor_timestamp=None if keep_cursor else found.next_cursor_timestamp,
+ new_cursor_unit_id="" if keep_cursor else found.next_cursor_unit_id,
+ dispatched_delta=len(started),
+ skipped_delta=outcomes[ChildOutcome.EXISTING],
+ completed_delta=outcomes[ChildOutcome.EVALUATED],
+ evaluation_skipped_delta=outcomes[ChildOutcome.SKIPPED],
+ failed_delta=outcomes[ChildOutcome.FAILED],
+ exhausted=found.exhausted and not pacing.stopped and not cancelled,
+ stop_reason=None if cancelled else pacing.stop_reason,
+ cancelled=cancelled,
+ ),
+ start_to_close_timeout=ACTIVITY_TIMEOUT,
+ schedule_to_close_timeout=ACTIVITY_SCHEDULE_TO_CLOSE,
+ retry_policy=ACTIVITY_RETRY_POLICY,
+ )
+
+ tasks = [asyncio.create_task(run_in_slot(candidate)) for candidate in found.candidates]
+ try:
+ results = await asyncio.gather(*tasks)
+ except asyncio.CancelledError:
+ # The children already started keep running and bill, so the cancelled row still counts them.
+ await record(
+ [task.result() for task in tasks if task.done() and not task.cancelled() and not task.exception()],
+ cancelled=True,
+ )
+ raise
+ pacing.finish_page()
+ advance = await record(results)
+ if advance.finished:
+ await self._measure_remainder(inputs)
+ return
+ temporalio.workflow.continue_as_new(
+ replace(
+ inputs,
+ consecutive_failures=0,
+ failed_runs_in_a_row=pacing.failures,
+ pauses_in_a_row=pacing.pauses,
+ )
+ )
+
+ async def _run_child(
+ self,
+ inputs: EvaluationBackfillInputs,
+ tick: PrepareTickOutput,
+ candidate: CandidatePayload,
+ started_units: list[str],
+ ) -> BackfillChildOutcome:
+ child = child_workflow_name_and_id(
+ evaluation_id=tick.evaluation_id,
+ evaluation_type=tick.evaluation_type,
+ target=tick.target,
+ unit_id=candidate.unit_id,
+ backfill_id=inputs.backfill_id,
+ rerun_existing=tick.rerun_existing,
+ )
+ try:
+ try:
+ handle = await self._start_child_handle(inputs, tick, candidate)
+ except WorkflowAlreadyStartedError:
+ return BackfillChildOutcome(outcome=ChildOutcome.EXISTING)
+ started_units.append(candidate.unit_id)
+ result = await asyncio.shield(handle)
+ return BackfillChildOutcome(outcome=_child_outcome(result), stop_reason=_stop_reason(result))
+ except Exception as error:
+ # A cancelled backfill raises here; a run cancelled on its own is only a failed run.
+ if is_cancelled_exception(error) and not isinstance(error, ChildWorkflowError):
+ raise
+ temporalio.workflow.logger.warning(
+ "llma.evaluation_backfill_child_failed", extra={"workflow_id": child.workflow_id}, exc_info=True
+ )
+ return BackfillChildOutcome(outcome=ChildOutcome.FAILED)
+
async def _dispatch_batch(self, inputs: EvaluationBackfillInputs, tick: PrepareTickOutput) -> bool:
"""Walk one page, start a child per unit, and advance the cursor. True ends the loop."""
- found = await temporalio.workflow.execute_activity(
- find_evaluation_backfill_candidates_activity,
- FindCandidatesInputs(backfill_id=inputs.backfill_id, team_id=inputs.team_id, limit=tick.batch_size),
- start_to_close_timeout=timedelta(seconds=120),
- schedule_to_close_timeout=ACTIVITY_SCHEDULE_TO_CLOSE,
- retry_policy=ACTIVITY_RETRY_POLICY,
- )
+ found = await self._find_candidates(inputs, tick)
# A page whose children mostly went out must not be re-dispatched because one start
# raised: the retry would collide with every child already running. The unit that failed
# is left to a later backfill and counted as failed, not skipped, because skipped means
@@ -570,26 +807,38 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):
# After the advance, never before it: inserting an activity ahead of one an in-flight
# history already recorded breaks replay. The row therefore completes with no count
# for a moment, and a null count states that rather than claiming full coverage.
- try:
- await temporalio.workflow.execute_activity(
- measure_evaluation_backfill_remainder_activity,
- MeasureRemainderInputs(backfill_id=inputs.backfill_id, team_id=inputs.team_id),
- start_to_close_timeout=timedelta(seconds=120),
- schedule_to_close_timeout=ACTIVITY_SCHEDULE_TO_CLOSE,
- retry_policy=ACTIVITY_RETRY_POLICY,
- )
- increment_backfill_remainder_outcome("success")
- except Exception as error:
- # The walk is done either way, so failing the tick here would spend a consecutive
- # failure and log at exception level over a number the row can live without.
- if is_cancelled_exception(error):
- raise
- increment_backfill_remainder_outcome("failed")
- temporalio.workflow.logger.warning(
- "llma.evaluation_backfill_remainder_failed", extra={"backfill_id": inputs.backfill_id}
- )
+ await self._measure_remainder(inputs)
return advance.finished
+ async def _find_candidates(self, inputs: EvaluationBackfillInputs, tick: PrepareTickOutput) -> FindCandidatesOutput:
+ return await temporalio.workflow.execute_activity(
+ find_evaluation_backfill_candidates_activity,
+ FindCandidatesInputs(backfill_id=inputs.backfill_id, team_id=inputs.team_id, limit=tick.batch_size),
+ start_to_close_timeout=timedelta(seconds=120),
+ schedule_to_close_timeout=ACTIVITY_SCHEDULE_TO_CLOSE,
+ retry_policy=ACTIVITY_RETRY_POLICY,
+ )
+
+ async def _measure_remainder(self, inputs: EvaluationBackfillInputs) -> None:
+ try:
+ await temporalio.workflow.execute_activity(
+ measure_evaluation_backfill_remainder_activity,
+ MeasureRemainderInputs(backfill_id=inputs.backfill_id, team_id=inputs.team_id),
+ start_to_close_timeout=timedelta(seconds=120),
+ schedule_to_close_timeout=ACTIVITY_SCHEDULE_TO_CLOSE,
+ retry_policy=ACTIVITY_RETRY_POLICY,
+ )
+ increment_backfill_remainder_outcome("success")
+ except Exception as error:
+ # The walk is done either way, so failing the tick here would spend a consecutive
+ # failure and log at exception level over a number the row can live without.
+ if is_cancelled_exception(error):
+ raise
+ increment_backfill_remainder_outcome("failed")
+ temporalio.workflow.logger.warning(
+ "llma.evaluation_backfill_remainder_failed", extra={"backfill_id": inputs.backfill_id}
+ )
+
async def _handle_failed_tick(self, inputs: EvaluationBackfillInputs) -> None:
failures = inputs.consecutive_failures + 1
temporalio.workflow.logger.exception(
@@ -615,6 +864,18 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):
self, inputs: EvaluationBackfillInputs, tick: PrepareTickOutput, candidate: CandidatePayload
) -> bool:
"""Start one evaluation child. False means the unit already had a run, so it was skipped."""
+ try:
+ await self._start_child_handle(inputs, tick, candidate)
+ return True
+ except WorkflowAlreadyStartedError:
+ return False
+
+ async def _start_child_handle(
+ self,
+ inputs: EvaluationBackfillInputs,
+ tick: PrepareTickOutput,
+ candidate: CandidatePayload,
+ ) -> temporalio.workflow.ChildWorkflowHandle[Any, WorkflowResult]:
unit_id = candidate.unit_id
child = child_workflow_name_and_id(
evaluation_id=tick.evaluation_id,
@@ -653,16 +914,12 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):
anchor_timestamp=candidate.unit_timestamp,
backfill_id=inputs.backfill_id,
)
- try:
- await temporalio.workflow.start_child_workflow(
- child.name,
- args,
- id=child.workflow_id,
- task_queue=settings.LLMA_EVALS_TASK_QUEUE,
- id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,
- parent_close_policy=ParentClosePolicy.ABANDON,
- execution_timeout=CHILD_EXECUTION_TIMEOUT,
- )
- return True
- except WorkflowAlreadyStartedError:
- return False
+ return await temporalio.workflow.start_child_workflow(
+ child.name,
+ args,
+ id=child.workflow_id,
+ task_queue=settings.LLMA_EVALS_TASK_QUEUE,
+ id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,
+ parent_close_policy=ParentClosePolicy.ABANDON,
+ execution_timeout=CHILD_EXECUTION_TIMEOUT,
+ )
diff --git a/posthog/temporal/ai_observability/evaluation_workflow_activities.py b/posthog/temporal/ai_observability/evaluation_workflow_activities.py
index 15979d596c02..56daa41f7e87 100644
--- a/posthog/temporal/ai_observability/evaluation_workflow_activities.py
+++ b/posthog/temporal/ai_observability/evaluation_workflow_activities.py
@@ -542,8 +542,10 @@ async def run_local_evaluation_activity(inputs: RunLocalEvaluationInputs) -> Loc
evaluation = await database_sync_to_async_pool(fetch_evaluation)(inputs.evaluation_id, inputs.event_data["team_id"])
evaluation_type = evaluation.get("evaluation_type", "llm_judge")
- if evaluation_type not in ("hog", "sentiment"):
- # An llm_judge returns before the read: its own activity hydrates what it grades.
+ stale_backfill = inputs.backfill_id is not None and (evaluation["deleted"] or not evaluation["enabled"])
+ if evaluation_type not in ("hog", "sentiment") or stale_backfill:
+ # An llm_judge returns before the read: its own activity hydrates what it grades. A backfill
+ # of an evaluation turned off since it started returns unrun, so the workflow reports the skip.
return LocalEvaluationOutcome(evaluation=evaluation, result=None, emitted=False)
event_data = await database_sync_to_async(hydrate_event_reference, thread_sensitive=False)(inputs.event_data)
diff --git a/posthog/temporal/ai_observability/run_aggregate_evaluation.py b/posthog/temporal/ai_observability/run_aggregate_evaluation.py
index 0a3d1002c7b2..d745e49471e2 100644
--- a/posthog/temporal/ai_observability/run_aggregate_evaluation.py
+++ b/posthog/temporal/ai_observability/run_aggregate_evaluation.py
@@ -48,6 +48,7 @@ from posthog.temporal.ai_observability.evaluation_workflow_activities import (
from posthog.temporal.ai_observability.metrics import increment_errors, increment_settle_poll
from posthog.temporal.ai_observability.run_evaluation import (
WorkflowResult,
+ compact_backfill_result,
handle_llm_judge_activity_error,
handle_terminal_user_error_result,
)
@@ -487,6 +488,9 @@ class RunAggregateEvaluationWorkflow(PostHogWorkflow):
@temporalio.workflow.run
async def run(self, inputs: RunAggregateEvaluationInputs) -> WorkflowResult:
+ return compact_backfill_result(await self._run(inputs), inputs.backfill_id)
+
+ async def _run(self, inputs: RunAggregateEvaluationInputs) -> WorkflowResult:
# A historical unit is settled by definition, so a backfill skips the settle wait and
# aggregates from the anchor, its first matching generation. Live starts carry no anchor,
# so old histories never take this branch and it needs no patch marker.
diff --git a/posthog/temporal/ai_observability/run_evaluation.py b/posthog/temporal/ai_observability/run_evaluation.py
index 57ed366011f7..5af97c6a5741 100644
--- a/posthog/temporal/ai_observability/run_evaluation.py
+++ b/posthog/temporal/ai_observability/run_evaluation.py
@@ -255,6 +255,17 @@ async def handle_terminal_user_error_result(
return workflow_result
+def compact_backfill_result(result: WorkflowResult, backfill_id: str | None) -> WorkflowResult:
+ # A backfill waits on its children, so each result lands in the backfill's own history. Hog
+ # output has no size cap, and a page of large results can pass Temporal's history limit.
+ if backfill_id is None:
+ return result
+ compact = result.copy()
+ compact.pop("reasoning", None)
+ compact.pop("message", None)
+ return compact
+
+
@temporalio.workflow.defn(name="run-evaluation")
class RunEvaluationWorkflow(PostHogWorkflow):
@staticmethod
@@ -267,6 +278,9 @@ class RunEvaluationWorkflow(PostHogWorkflow):
@temporalio.workflow.run
async def run(self, inputs: RunEvaluationInputs) -> WorkflowResult:
+ return compact_backfill_result(await self._run(inputs), inputs.backfill_id)
+
+ async def _run(self, inputs: RunEvaluationInputs) -> WorkflowResult:
# Every #71518-era run recorded the "remove-trial-evals" patch marker; accept those
# histories for one release, then delete this call (temporal-workflow-versioning rule).
temporalio.workflow.deprecate_patch("remove-trial-evals")
@@ -326,6 +340,24 @@ class RunEvaluationWorkflow(PostHogWorkflow):
evaluation_type = evaluation.get("evaluation_type", "llm_judge")
+ # A backfill can outlive the evaluation it runs, so it checks before the billed judge call.
+ # The skip also tells the backfill to stop the rest of its page.
+ if (
+ result is None
+ and inputs.backfill_id is not None
+ and (evaluation["deleted"] or not evaluation["enabled"])
+ and temporalio.workflow.patched("evaluation-backfill-skips-disabled-evaluation")
+ ):
+ disabled_result: WorkflowResult = {
+ "skipped": True,
+ "skip_reason": "evaluation_deleted" if evaluation["deleted"] else "evaluation_disabled",
+ "evaluation_id": inputs.evaluation_id,
+ "evaluation_type": evaluation_type,
+ }
+ if evaluation.get("output_type") not in ("numeric", "categorical"):
+ disabled_result["verdict"] = None
+ return disabled_result
+
if result is None:
if evaluation_type == "hog":
result = await temporalio.workflow.execute_activity(
diff --git a/posthog/temporal/ai_observability/test_evaluation_backfill.py b/posthog/temporal/ai_observability/test_evaluation_backfill.py
index d4dd0e6ee2bc..a6946596f7a3 100644
--- a/posthog/temporal/ai_observability/test_evaluation_backfill.py
+++ b/posthog/temporal/ai_observability/test_evaluation_backfill.py
@@ -1,5 +1,7 @@
import uuid
+import asyncio
import dataclasses
+from collections.abc import Mapping
from datetime import UTC, datetime, timedelta
from typing import Any, cast
@@ -11,9 +13,16 @@ from django.test import override_settings
import temporalio
from asgiref.sync import async_to_sync
+from temporalio.api.enums.v1 import EventType
from temporalio.client import WorkflowFailureError
from temporalio.common import WorkflowIDReusePolicy
-from temporalio.exceptions import ActivityError, ApplicationError, CancelledError, WorkflowAlreadyStartedError
+from temporalio.exceptions import (
+ ActivityError,
+ ApplicationError,
+ CancelledError,
+ ChildWorkflowError,
+ WorkflowAlreadyStartedError,
+)
from temporalio.testing import ActivityEnvironment, WorkflowEnvironment
from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Worker
from temporalio.workflow import ParentClosePolicy
@@ -21,6 +30,7 @@ from temporalio.workflow import ParentClosePolicy
from posthog.models import Organization, Team
from posthog.temporal.ai_observability.evaluation_backfill import (
ACTIVITY_RETRY_POLICY,
+ BACKFILL_FAILURE_PAUSES,
BACKFILL_MAX_CONSECUTIVE_FAILURES,
MAX_BACKFILL_BATCH_SIZE,
AdvanceCursorInputs,
@@ -52,7 +62,7 @@ from posthog.temporal.ai_observability.run_aggregate_evaluation import (
RunAggregateEvaluationInputs,
RunAggregateEvaluationWorkflow,
)
-from posthog.temporal.ai_observability.run_evaluation import RunEvaluationWorkflow
+from posthog.temporal.ai_observability.run_evaluation import RunEvaluationWorkflow, WorkflowResult
from products.ai_observability.backend.backfill_candidates import BackfillCandidate, BackfillScope, CandidatePage
from products.ai_observability.backend.models.evaluation_backfill import EvaluationBackfill, EvaluationBackfillStatus
@@ -70,29 +80,62 @@ class _BackfillMocks:
*,
activity_results: dict | None = None,
child_errors_for_ids: dict[str, Exception] | None = None,
+ child_results_for_ids: Mapping[str, WorkflowResult | Exception | asyncio.Future] | None = None,
) -> None:
self.activity_results = activity_results or {}
self.child_errors_for_ids = child_errors_for_ids or {}
+ self.child_results_for_ids = child_results_for_ids or {}
+ self.active_children = 0
+ self.peak_children = 0
+ self.active_at_advance: list[int] = []
self.activity_calls: list[tuple] = []
+ self.sleep = AsyncMock()
+ self.waits: list[tuple[timedelta, bool]] = []
+ self.waiting = False
self.child_calls: list[dict] = []
+ async def wait_condition(self, fn, *, timeout: timedelta) -> None:
+ self.waiting = True
+ for _ in range(100):
+ if fn():
+ self.waits.append((timeout, True))
+ return
+ await asyncio.sleep(0)
+ self.waits.append((timeout, False))
+ raise TimeoutError
+
async def execute_activity(self, activity_fn, activity_input, **_) -> object:
self.activity_calls.append((activity_fn, activity_input))
result = self.activity_results.get(activity_fn)
if isinstance(result, Exception):
raise result
if activity_fn is advance_evaluation_backfill_cursor_activity and activity_fn not in self.activity_results:
- return AdvanceCursorOutput(finished=False)
+ self.active_at_advance.append(self.active_children)
+ return AdvanceCursorOutput(finished=activity_input.exhausted or activity_input.stop_reason is not None)
return result
async def start_child_workflow(self, *args, **kwargs) -> object:
- wid = kwargs.get("id")
+ wid = str(kwargs["id"])
self.child_calls.append(
{"name": args[0], "id": wid, "inputs": args[1] if len(args) > 1 else None, "kwargs": kwargs}
)
if wid is not None and wid in self.child_errors_for_ids:
raise self.child_errors_for_ids[wid]
- return object()
+
+ async def complete() -> WorkflowResult:
+ self.active_children += 1
+ self.peak_children = max(self.peak_children, self.active_children)
+ await asyncio.sleep(0)
+ self.active_children -= 1
+ default_result: WorkflowResult = {"evaluation_id": "E", "evaluation_type": "hog", "skipped": False}
+ result = self.child_results_for_ids.get(wid, default_result)
+ if isinstance(result, asyncio.Future):
+ return await result
+ if isinstance(result, Exception):
+ raise result
+ return result
+
+ return asyncio.create_task(complete())
def _inputs() -> EvaluationBackfillInputs:
@@ -124,16 +167,18 @@ def _found(candidates: list[CandidatePayload], *, exhausted: bool = False) -> Fi
)
-async def _run(mocks: _BackfillMocks, inputs: EvaluationBackfillInputs | None = None):
+async def _run(mocks: _BackfillMocks, inputs: EvaluationBackfillInputs | None = None, *, bounded: bool = False):
# `workflow.logger` reaches into the workflow runtime, which isn't set up here.
noop = staticmethod(lambda *_a, **_kw: None)
- fake_logger = type("Logger", (), {"exception": noop, "warning": noop})()
+ fake_logger = type("Logger", (), {"exception": noop, "warning": noop, "info": noop})()
with (
patch("temporalio.workflow.logger", fake_logger),
patch("temporalio.workflow.execute_activity", side_effect=mocks.execute_activity),
patch("temporalio.workflow.start_child_workflow", side_effect=mocks.start_child_workflow),
patch("temporalio.workflow.continue_as_new") as continue_as_new,
- patch("temporalio.workflow.sleep", new=AsyncMock()),
+ patch("temporalio.workflow.sleep", new=mocks.sleep),
+ patch("temporalio.workflow.wait_condition", new=mocks.wait_condition),
+ patch("temporalio.workflow.patched", return_value=bounded),
):
await EvaluationBackfillWorkflow().run(inputs or _inputs())
return continue_as_new
@@ -147,6 +192,22 @@ def _advance_input(mocks: _BackfillMocks) -> AdvanceCursorInputs:
return next(call for fn, call in mocks.activity_calls if fn is advance_evaluation_backfill_cursor_activity)
[email protected](name="run-evaluation")
+class _StubEvaluationRun:
+ """Fails every run, or finishes u0 and u1 and leaves the rest running until cancelled."""
+
+ fail = True
+
+ @temporalio.workflow.run
+ async def run(self, _: Any) -> WorkflowResult:
+ if self.fail:
+ raise ApplicationError("judge unavailable", non_retryable=True)
+ if temporalio.workflow.info().workflow_id.endswith(("-u0-ingestion", "-u1-ingestion")):
+ return {"evaluation_id": "E", "evaluation_type": "hog", "skipped": False}
+ await temporalio.workflow.wait_condition(lambda: False)
+ raise AssertionError("unreachable")
+
+
class TestEvaluationBackfillWorkflow:
@pytest.mark.asyncio
@pytest.mark.parametrize("target", ["generation", "trace", "session"])
@@ -282,6 +343,303 @@ class TestEvaluationBackfillWorkflow:
workflow_runner=UnsandboxedWorkflowRunner(),
).replay_workflow(history)
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("cancel", [False, True], ids=["persistent_outage_across_pages", "cancel_mid_page"])
+ async def test_bounded_backfill_on_a_real_worker(self, cancel: bool, caplog: pytest.LogCaptureFixture) -> None:
+ caplog.set_level("INFO", logger="temporalio.workflow")
+ caplog.set_level("INFO", logger="temporalio.activity")
+ units = [_candidate(f"u{i}") for i in range(5 if cancel else 70)]
+ page_size = 5 if cancel else 35
+ advances: list[AdvanceCursorInputs] = []
+ measured: list[MeasureRemainderInputs] = []
+
+ @temporalio.activity.defn(name="prepare_evaluation_backfill_tick_activity")
+ async def prepare(_: EvaluationBackfillInputs) -> PrepareTickOutput:
+ return _tick(max_in_flight=page_size if cancel else 1)
+
+ @temporalio.activity.defn(name="find_evaluation_backfill_candidates_activity")
+ async def find(_: FindCandidatesInputs) -> FindCandidatesOutput:
+ first = page_size * sum(advance.new_cursor_timestamp is not None for advance in advances)
+ return _found(units[first : first + page_size], exhausted=first + page_size >= len(units))
+
+ @temporalio.activity.defn(name="advance_evaluation_backfill_cursor_activity")
+ async def advance(inputs: AdvanceCursorInputs) -> AdvanceCursorOutput:
+ advances.append(inputs)
+ return AdvanceCursorOutput(finished=inputs.exhausted or inputs.stop_reason is not None or inputs.cancelled)
+
+ @temporalio.activity.defn(name="measure_evaluation_backfill_remainder_activity")
+ async def measure(inputs: MeasureRemainderInputs) -> None:
+ measured.append(inputs)
+
+ _StubEvaluationRun.fail = not cancel
+ task_queue = settings.LLMA_EVALS_TASK_QUEUE
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with Worker(
+ env.client,
+ task_queue=task_queue,
+ workflows=[EvaluationBackfillWorkflow, _StubEvaluationRun],
+ activities=[prepare, find, advance, measure],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ handle = await env.client.start_workflow(
+ EvaluationBackfillWorkflow.run,
+ _inputs(),
+ id=str(uuid.uuid4()),
+ task_queue=task_queue,
+ execution_timeout=timedelta(hours=6),
+ )
+ if cancel:
+ # Cancel once the parent has taken in both finished runs and every start.
+ for _ in range(200):
+ events = [event.event_type for event in (await handle.fetch_history()).events]
+ if (
+ events.count(EventType.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED) == 2
+ and events.count(EventType.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED) == 5
+ and events[-1] == EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED
+ ):
+ break
+ await asyncio.sleep(0.05)
+ await handle.cancel()
+ with pytest.raises(WorkflowFailureError):
+ await handle.result()
+ else:
+ await handle.result()
+ histories = [
+ await env.client.get_workflow_handle(handle.id, run_id=handle.run_id).fetch_history(),
+ await env.client.get_workflow_handle(handle.id).fetch_history(),
+ ]
+
+ last = advances[-1]
+ if cancel:
+ assert last.cancelled
+ assert (last.dispatched_delta, last.completed_delta, last.failed_delta) == (5, 2, 0)
+ assert measured == []
+ else:
+ assert len(advances) == 2
+ assert sum(page.dispatched_delta for page in advances) == 60
+ assert last.stop_reason == "execution_failed"
+ assert len(measured) == 1
+ replayer = Replayer(workflows=[EvaluationBackfillWorkflow], workflow_runner=UnsandboxedWorkflowRunner())
+ for history in histories:
+ await replayer.replay_workflow(history)
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("max_in_flight,peak", [(0, 16), (4, 4)])
+ async def test_batch_dispatch_waits_for_outcomes_before_advancing(self, max_in_flight: int, peak: int) -> None:
+ found = dataclasses.replace(
+ _found([_candidate(f"u{i}") for i in range(19)], exhausted=True),
+ next_cursor_timestamp=(UNIT_TIMESTAMP - timedelta(hours=1)).isoformat(),
+ next_cursor_unit_id="filtered-out-unit",
+ )
+ mocks = _BackfillMocks(
+ activity_results={
+ prepare_evaluation_backfill_tick_activity: _tick(max_in_flight=max_in_flight),
+ find_evaluation_backfill_candidates_activity: found,
+ }
+ )
+
+ await _run(mocks, bounded=True)
+
+ advances = [value for fn, value in mocks.activity_calls if fn is advance_evaluation_backfill_cursor_activity]
+ assert mocks.peak_children == peak
+ assert mocks.active_at_advance == [0]
+ assert len(advances) == 1
+ assert advances[0].completed_delta == 19
+ assert advances[0].exhausted
+ assert (advances[0].expected_cursor_timestamp, advances[0].expected_cursor_unit_id) == (
+ found.started_from_cursor_timestamp,
+ found.started_from_cursor_unit_id,
+ )
+ assert (advances[0].new_cursor_timestamp, advances[0].new_cursor_unit_id) == (
+ found.next_cursor_timestamp,
+ found.next_cursor_unit_id,
+ )
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("cancelled_run", [False, True])
+ async def test_bounded_dispatch_records_execution_failures_and_skips(self, cancelled_run: bool) -> None:
+ failure: Exception = RuntimeError("worker unavailable")
+ if cancelled_run:
+ failure = ChildWorkflowError(
+ "Child workflow cancelled",
+ namespace="default",
+ workflow_id="llma-hog-eval-E-u1-ingestion",
+ run_id="run",
+ workflow_type="run-evaluation",
+ initiated_event_id=1,
+ started_event_id=2,
+ retry_state=None,
+ )
+ failure.__cause__ = CancelledError("cancelled")
+ mocks = _BackfillMocks(
+ activity_results={
+ prepare_evaluation_backfill_tick_activity: _tick(),
+ find_evaluation_backfill_candidates_activity: _found(
+ [_candidate(f"u{i}") for i in range(4)], exhausted=True
+ ),
+ },
+ child_results_for_ids={
+ "llma-hog-eval-E-u1-ingestion": failure,
+ "llma-hog-eval-E-u2-ingestion": {
+ "evaluation_id": "E",
+ "evaluation_type": "hog",
+ "skipped": True,
+ "skip_reason": "request_rejected",
+ },
+ "llma-hog-eval-E-u3-ingestion": {
+ "evaluation_id": "E",
+ "evaluation_type": "hog",
+ "skipped": True,
+ "skip_reason": "key_invalid",
+ },
+ },
+ )
+
+ await _run(mocks, bounded=True)
+
+ result = _advance_input(mocks)
+ assert (
+ result.dispatched_delta,
+ result.completed_delta,
+ result.evaluation_skipped_delta,
+ result.failed_delta,
+ ) == (4, 1, 2, 1)
+
+ @pytest.mark.asyncio
+ async def test_a_cancel_mid_page_records_the_runs_already_started(self) -> None:
+ mocks = _BackfillMocks(
+ activity_results={
+ prepare_evaluation_backfill_tick_activity: _tick(max_in_flight=2),
+ find_evaluation_backfill_candidates_activity: _found([_candidate(f"u{i}") for i in range(4)]),
+ },
+ child_results_for_ids={"llma-hog-eval-E-u1-ingestion": asyncio.get_running_loop().create_future()},
+ )
+ run = asyncio.create_task(_run(mocks, bounded=True))
+ while len(mocks.child_calls) < 3:
+ await asyncio.sleep(0)
+
+ run.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await run
+
+ result = _advance_input(mocks)
+ assert result.cancelled
+ assert (result.dispatched_delta, result.completed_delta, result.failed_delta) == (3, 1, 0)
+ assert result.new_cursor_unit_id == "u3"
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "failing,stopping,page_size,started,pauses,stop_reason",
+ [
+ pytest.param(
+ set(range(70)), None, 70, 60, list(BACKFILL_FAILURE_PAUSES), "execution_failed", id="persistent_outage"
+ ),
+ pytest.param(
+ set(range(70)),
+ None,
+ 20,
+ 60,
+ list(BACKFILL_FAILURE_PAUSES),
+ "execution_failed",
+ id="persistent_outage_across_pages",
+ ),
+ pytest.param(
+ set(range(10, 70)),
+ None,
+ 70,
+ 70,
+ list(BACKFILL_FAILURE_PAUSES),
+ "execution_failed",
+ id="outage_spends_budget_on_last_unit",
+ ),
+ pytest.param({i for i in range(70) if i % 10}, None, 70, 70, [], None, id="scattered_failures"),
+ pytest.param(
+ set(range(10)) | set(range(11, 21)),
+ None,
+ 70,
+ 70,
+ [BACKFILL_FAILURE_PAUSES[0]] * 2,
+ None,
+ id="success_resets_pauses",
+ ),
+ pytest.param(set(), (1, "key_invalid"), 70, 2, [], "evaluation_disabled", id="evaluation_disabled"),
+ pytest.param(set(), (1, "evaluation_deleted"), 70, 2, [], "evaluation_deleted", id="evaluation_deleted"),
+ ],
+ )
+ async def test_failures_pause_and_stop_the_backfill(
+ self,
+ failing: set[int],
+ stopping: tuple[int, str] | None,
+ page_size: int,
+ started: int,
+ pauses: list[timedelta],
+ stop_reason: str | None,
+ ) -> None:
+ results: dict[str, WorkflowResult | Exception] = {
+ f"llma-hog-eval-E-u{i}-ingestion": RuntimeError("judge unavailable") for i in failing
+ }
+ if stopping is not None:
+ unit, reason = stopping
+ results[f"llma-hog-eval-E-u{unit}-ingestion"] = {
+ "evaluation_id": "E",
+ "evaluation_type": "hog",
+ "skipped": True,
+ "skip_reason": reason,
+ }
+ units = [_candidate(f"u{i}") for i in range(70)]
+ inputs, advances, sleeps, waits = _inputs(), [], [], []
+ for first in range(0, len(units), page_size):
+ mocks = _BackfillMocks(
+ activity_results={
+ prepare_evaluation_backfill_tick_activity: _tick(max_in_flight=1),
+ find_evaluation_backfill_candidates_activity: _found(
+ units[first : first + page_size], exhausted=first + page_size >= len(units)
+ ),
+ },
+ child_results_for_ids=results,
+ )
+ continue_as_new = await _run(mocks, inputs, bounded=True)
+ advances.append(_advance_input(mocks))
+ sleeps += [call.args[0] for call in mocks.sleep.call_args_list]
+ waits += [timeout for timeout, _ in mocks.waits]
+ if not continue_as_new.called:
+ break
+ inputs = continue_as_new.call_args.args[0]
+
+ advance = advances[-1]
+ assert sum(page.dispatched_delta for page in advances) == started
+ assert sleeps == []
+ assert waits == pauses
+ assert advance.stop_reason == stop_reason
+ assert advance.exhausted == (stop_reason is None)
+ assert (advance.new_cursor_timestamp is None) == (stop_reason is not None)
+
+ @pytest.mark.asyncio
+ async def test_a_stop_during_a_pause_ends_the_pause(self) -> None:
+ disabling = asyncio.get_running_loop().create_future()
+ results: dict[str, WorkflowResult | Exception | asyncio.Future] = {
+ f"llma-hog-eval-E-u{i}-ingestion": RuntimeError("judge unavailable") for i in range(1, 20)
+ }
+ results["llma-hog-eval-E-u0-ingestion"] = disabling
+ mocks = _BackfillMocks(
+ activity_results={
+ prepare_evaluation_backfill_tick_activity: _tick(max_in_flight=2),
+ find_evaluation_backfill_candidates_activity: _found([_candidate(f"u{i}") for i in range(20)]),
+ },
+ child_results_for_ids=results,
+ )
+ run = asyncio.create_task(_run(mocks, bounded=True))
+ while not mocks.waiting:
+ await asyncio.sleep(0)
+
+ disabling.set_result(
+ {"evaluation_id": "E", "evaluation_type": "hog", "skipped": True, "skip_reason": "key_invalid"}
+ )
+ await run
+
+ assert mocks.waits == [(BACKFILL_FAILURE_PAUSES[0], True)]
+ assert _advance_input(mocks).stop_reason == "evaluation_disabled"
+
@pytest.mark.asyncio
async def test_finished_tick_returns_without_dispatch(self) -> None:
mocks = _BackfillMocks(
@@ -609,6 +967,8 @@ def _advance(
dispatched_delta: int = 1,
skipped_delta: int = 0,
failed_delta: int = 0,
+ completed_delta: int | None = None,
+ evaluation_skipped_delta: int = 0,
exhausted: bool = False,
) -> AdvanceCursorInputs:
return AdvanceCursorInputs(
@@ -621,6 +981,8 @@ def _advance(
dispatched_delta=dispatched_delta,
skipped_delta=skipped_delta,
failed_delta=failed_delta,
+ completed_delta=completed_delta,
+ evaluation_skipped_delta=evaluation_skipped_delta,
exhausted=exhausted,
)
@@ -792,11 +1154,11 @@ class TestEvaluationBackfillActivities:
]
@pytest.mark.parametrize(
- "configured,expected",
- [(7, 7), (0, 1), (-5, 1), (MAX_BACKFILL_BATCH_SIZE * 10, MAX_BACKFILL_BATCH_SIZE)],
+ "configured,expected,in_flight",
+ [(7, 7, 7), (0, 1, 1), (-5, 1, 1), (MAX_BACKFILL_BATCH_SIZE * 10, MAX_BACKFILL_BATCH_SIZE, 10000)],
)
- def test_prepare_dispatches_with_a_clamped_batch_size(self, backfill_data, configured, expected) -> None:
- with override_settings(LLMA_EVAL_BACKFILL_BATCH_SIZE=configured):
+ def test_prepare_dispatches_with_a_clamped_batch_size(self, backfill_data, configured, expected, in_flight) -> None:
+ with override_settings(LLMA_EVAL_BACKFILL_BATCH_SIZE=configured, LLMA_EVAL_BACKFILL_MAX_IN_FLIGHT=configured):
result = async_to_sync(prepare_evaluation_backfill_tick_activity)(_activity_inputs(backfill_data))
assert result == PrepareTickOutput(
@@ -808,6 +1170,7 @@ class TestEvaluationBackfillActivities:
settle={},
rerun_existing=False,
batch_size=expected,
+ max_in_flight=in_flight,
)
def test_find_serializes_candidates_and_cursor(self, backfill_data) -> None:
@@ -855,20 +1218,37 @@ class TestEvaluationBackfillActivities:
)
assert result.exhausted
- def test_advance_is_idempotent_on_retry(self, backfill_data) -> None:
- advance = _advance(backfill_data, dispatched_delta=3, skipped_delta=1, failed_delta=2)
+ @pytest.mark.parametrize("stop_reason", [None, "execution_failed"])
+ def test_advance_is_idempotent_on_retry(self, backfill_data, stop_reason: str | None) -> None:
+ advance = _advance(
+ backfill_data,
+ dispatched_delta=5,
+ skipped_delta=1,
+ failed_delta=2,
+ completed_delta=2,
+ evaluation_skipped_delta=1,
+ )
+ if stop_reason:
+ advance = dataclasses.replace(
+ advance, new_cursor_timestamp=None, new_cursor_unit_id="", stop_reason=stop_reason
+ )
first = async_to_sync(advance_evaluation_backfill_cursor_activity)(advance)
second = async_to_sync(advance_evaluation_backfill_cursor_activity)(advance)
backfill_data["backfill"].refresh_from_db()
row = backfill_data["backfill"]
- assert (row.dispatched_count, row.skipped_count, row.failed_count) == (3, 1, 2)
- assert backfill_data["backfill"].cursor_unit_id == "u3"
- assert not first.finished
- # The second call matched nothing because the first already moved the cursor. Reading that
- # as finished would end the loop with the row still RUNNING and the window half walked.
- assert not second.finished
+ assert (row.dispatched_count, row.skipped_count, row.failed_count) == (5, 1, 2)
+ assert (row.completed_count, row.evaluation_skipped_count) == (2, 1)
+ assert (row.status, row.status_reason) == (
+ (EvaluationBackfillStatus.INTERRUPTED, stop_reason)
+ if stop_reason
+ else (EvaluationBackfillStatus.RUNNING, "")
+ )
+ assert row.cursor_unit_id == ("" if stop_reason else "u3")
+ # The second call matched nothing because the first already moved the cursor or ended the
+ # row. Reading a moved cursor as finished would end the loop with the window half walked.
+ assert first.finished == second.finished == bool(stop_reason)
def test_advance_marks_completed_when_exhausted(self, backfill_data) -> None:
result = async_to_sync(advance_evaluation_backfill_cursor_activity)(_advance(backfill_data, exhausted=True))
@@ -878,12 +1258,31 @@ class TestEvaluationBackfillActivities:
assert backfill_data["backfill"].status == EvaluationBackfillStatus.COMPLETED
assert backfill_data["backfill"].finished_at is not None
- def test_advance_loses_to_concurrent_cancel(self, backfill_data) -> None:
+ def test_resuming_legacy_backfill_does_not_claim_partial_outcomes_are_totals(self, backfill_data) -> None:
+ _update_backfill(backfill_data, dispatched_count=3, cursor_timestamp=UNIT_TIMESTAMP, cursor_unit_id="u0")
+ advance = dataclasses.replace(
+ _advance(backfill_data, dispatched_delta=2, completed_delta=2, exhausted=True),
+ expected_cursor_timestamp=UNIT_TIMESTAMP.isoformat(),
+ expected_cursor_unit_id="u0",
+ )
+
+ async_to_sync(advance_evaluation_backfill_cursor_activity)(advance)
+
+ row = backfill_data["backfill"]
+ row.refresh_from_db()
+ assert row.dispatched_count == 5
+ assert row.completed_count is None
+ assert row.remaining_count is None
+
+ @pytest.mark.parametrize("cancelled", [False, True])
+ def test_advance_after_a_cancel_counts_the_page_once(self, backfill_data, cancelled: bool) -> None:
_update_backfill(backfill_data, status=EvaluationBackfillStatus.CANCELLED)
+ advance = dataclasses.replace(_advance(backfill_data), cancelled=cancelled)
- result = async_to_sync(advance_evaluation_backfill_cursor_activity)(_advance(backfill_data))
+ first = async_to_sync(advance_evaluation_backfill_cursor_activity)(advance)
+ second = async_to_sync(advance_evaluation_backfill_cursor_activity)(advance)
- assert result.finished
+ assert first.finished and second.finished
backfill_data["backfill"].refresh_from_db()
assert backfill_data["backfill"].status == EvaluationBackfillStatus.CANCELLED
- assert backfill_data["backfill"].dispatched_count == 0
+ assert backfill_data["backfill"].dispatched_count == 1
diff --git a/posthog/temporal/ai_observability/test_run_aggregate_evaluation.py b/posthog/temporal/ai_observability/test_run_aggregate_evaluation.py
index e138e63470e5..c09e4415418c 100644
--- a/posthog/temporal/ai_observability/test_run_aggregate_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_aggregate_evaluation.py
@@ -416,6 +416,7 @@ class TestRunAggregateEvaluationWorkflow:
assert calls == ["fetch", "execute", "emit", "telemetry"]
assert result["verdict"] is True
+ assert "reasoning" not in result
assert window_starts == [datetime.fromisoformat(anchor).isoformat()]
# Bounded to the span the live path would have covered from the anchor, so the grade does
# not sweep in everything that arrived between the anchor and now.
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index d4ff271ba8f6..28e5b957d4aa 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -95,6 +95,7 @@ from .run_evaluation import (
RunLocalEvaluationInputs,
SendEvaluationDisabledEmailInputs,
WorkflowResult,
+ compact_backfill_result,
disable_evaluation_activity,
emit_evaluation_event_activity,
execute_hog_eval_activity,
@@ -2184,6 +2185,36 @@ class TestRunEvaluationWorkflow:
mock_emit.assert_awaited_once()
assert mock_emit.await_args_list[0].args[0].event_data == full_event
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("evaluation_type", ["hog", "sentiment"])
+ @pytest.mark.parametrize("change", [{"enabled": False}, {"deleted": True}], ids=["disabled", "deleted"])
+ async def test_backfill_does_not_run_a_local_evaluation_turned_off(
+ self, evaluation_type: str, change: dict[str, bool]
+ ) -> None:
+ evaluation = _local_evaluation(
+ **{"evaluation_type": evaluation_type, "enabled": True, "deleted": False, **change}
+ )
+ module = "posthog.temporal.ai_observability.evaluation_workflow_activities"
+ with (
+ patch(f"{module}.fetch_evaluation", return_value=evaluation),
+ patch(f"{module}.run_hog_eval_for_event", new_callable=AsyncMock) as run_hog,
+ patch(f"{module}.run_sentiment_eval", new_callable=AsyncMock) as run_sentiment,
+ patch(f"{module}.emit_generation_evaluation_event", new_callable=AsyncMock) as emit,
+ ):
+ outcome = await run_local_evaluation_activity(
+ RunLocalEvaluationInputs(
+ evaluation_id="eval-1",
+ event_data=dict(THIN_REFERENCE),
+ start_time=datetime(2024, 1, 1, 12, 0, tzinfo=UTC),
+ backfill_id="B",
+ )
+ )
+
+ assert (outcome.result, outcome.emitted) == (None, False)
+ run_hog.assert_not_awaited()
+ run_sentiment.assert_not_awaited()
+ emit.assert_not_awaited()
+
@pytest.mark.asyncio
async def test_the_generation_emit_activity_hydrates_a_thin_reference(self):
full_event = create_mock_event_data(team_id=1, uuid="g1")
@@ -2532,6 +2563,51 @@ class TestRunEvaluationWorkflow:
assert result["verdict"] is True
assert result["skipped"] is False
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "change,skip_reason", [({"enabled": False}, "evaluation_disabled"), ({"deleted": True}, "evaluation_deleted")]
+ )
+ async def test_backfill_run_skips_the_judge_for_a_disabled_evaluation(
+ self, change: dict[str, bool], skip_reason: str
+ ) -> None:
+ @activity.defn(name="run_local_evaluation_activity")
+ async def mock_run_local_evaluation(inputs: RunLocalEvaluationInputs) -> LocalEvaluationOutcome:
+ evaluation = {
+ "id": inputs.evaluation_id,
+ "evaluation_type": "llm_judge",
+ "output_type": "boolean",
+ "team_id": 1,
+ "enabled": True,
+ "deleted": False,
+ }
+ return LocalEvaluationOutcome(evaluation={**evaluation, **change}, result=None, emitted=False)
+
+ @activity.defn(name="execute_llm_judge_activity")
+ async def mock_judge(inputs: ExecuteLLMJudgeInputs) -> EvaluationActivityResult:
+ raise AssertionError("a backfill must not bill a judge call for a disabled evaluation")
+
+ task_queue = str(uuid.uuid4())
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with Worker(
+ env.client,
+ task_queue=task_queue,
+ workflows=[RunEvaluationWorkflow],
+ activities=[mock_run_local_evaluation, mock_judge],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ result: WorkflowResult = await env.client.execute_workflow(
+ RunEvaluationWorkflow.run,
+ RunEvaluationInputs(
+ evaluation_id=str(uuid.uuid4()), event_data=create_mock_event_data(team_id=1), backfill_id="B"
+ ),
+ id=str(uuid.uuid4()),
+ task_queue=task_queue,
+ execution_timeout=timedelta(minutes=5),
+ )
+
+ assert result["skipped"] is True
+ assert result["skip_reason"] == skip_reason
+
@pytest.mark.asyncio
async def test_terminal_user_error_does_not_email_when_evaluation_was_already_disabled(self):
calls: list[str] = []
@@ -4682,3 +4758,24 @@ class TestRunLocalEvaluationActivity:
second_uuid = mock_capture.call_args_list[1].kwargs["event_uuid"]
assert first_uuid is not None
assert first_uuid == second_uuid
+
+
+ "backfill_id,expected",
+ [
+ pytest.param(
+ None, {"skipped": True, "skip_reason": "parse_error", "reasoning": "x", "message": "y"}, id="live"
+ ),
+ pytest.param("bf-1", {"skipped": True, "skip_reason": "parse_error"}, id="backfill"),
+ ],
+)
+def test_backfill_results_drop_free_text(backfill_id: str | None, expected: dict) -> None:
+ result: WorkflowResult = {
+ "evaluation_id": "e",
+ "evaluation_type": "hog",
+ "skipped": True,
+ "skip_reason": "parse_error",
+ "reasoning": "x",
+ "message": "y",
+ }
+ assert compact_backfill_result(result, backfill_id) == {"evaluation_id": "e", "evaluation_type": "hog", **expected}
diff --git a/products/ai_observability/backend/api/evaluation_backfills.py b/products/ai_observability/backend/api/evaluation_backfills.py
index 26f803e0a733..148dffe5c59f 100644
--- a/products/ai_observability/backend/api/evaluation_backfills.py
+++ b/products/ai_observability/backend/api/evaluation_backfills.py
@@ -668,7 +668,7 @@ class EvaluationBackfillViewSet(
asyncio.run(client.get_workflow_handle(backfill_workflow_id(str(backfill.pk))).cancel())
except Exception as error:
# The workflow reads the row's status at every tick and stops on a terminal one, so
- # a failed handle costs nothing and does not need a stack trace.
+ # a failed handle only lets the current page finish and does not need a stack trace.
logger.warning("llma.evaluation_backfill_cancel_failed", backfill_id=str(backfill.pk), error=str(error))
# The status and finished_at were written by the update, not on this instance.
backfill.refresh_from_db()