:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): skip evaluations whose generation aged out of ai_events (#122224)
Carlos Marchal1034635b4620
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.mdindex 0894ce0c6b07..680193dd40eb 100644--- a/docs/internal/ai-observability-judge-inputs.md+++ b/docs/internal/ai-observability-judge-inputs.md@@ -196,7 +196,7 @@ Ranges ending at or before that boundary are rejected.Existing backfills keep their stored windows; **Retry remaining** applies the boundary when it creates a new backfill.Generation references get up to three lookup attempts.-A final miss for a backfill reference fails the run without reporting an exception to error tracking.+A final miss for a backfill reference records the evaluation as skipped with `generation_not_found`, emits no `$ai_evaluation` event, and reports no exception to error tracking.Live references marked `awaiting_ingestion` wait 15 seconds between misses and still report the final miss.A backfill that stops on its own shows as **Interrupted**, with a stop reason.diff --git a/posthog/temporal/ai_observability/evaluation_event_io.py b/posthog/temporal/ai_observability/evaluation_event_io.pyindex 49eb498dd181..df63f1009670 100644--- a/posthog/temporal/ai_observability/evaluation_event_io.py+++ b/posthog/temporal/ai_observability/evaluation_event_io.py@@ -5,6 +5,7 @@ import temporalio.activityfrom temporalio.exceptions import ApplicationErrorfrom posthog.dataclasses import frozen+from posthog.temporal.ai_observability.evaluation_types import GENERATION_NOT_FOUNDfrom posthog.temporal.common.errors import NonReportableApplicationErrorfrom posthog.utils import ensure_utc@@ -17,6 +18,16 @@ GENERATION_NOT_FOUND_MAX_ATTEMPTS = 3GENERATION_NOT_FOUND_RETRY_DELAY = timedelta(seconds=15)+class GenerationNotFoundError(NonReportableApplicationError):+ """The last miss on a reference that has no ingestion to wait for.++ Only a backfill sends such a reference, and it selects candidates from `events`, which keeps+ rows longer than `ai_events`. So the row most likely aged out of `ai_events`. An evaluation+ catches this and records a skip. A caller that does not catch it fails the run, and the error+ stays out of error tracking.+ """++def as_utc_datetime(value: str | datetime) -> datetime:"""Read a ClickHouse event timestamp, which reaches us as a naive datetime on a direct calland as an ISO string once Temporal has serialized it through a payload."""@@ -93,9 +104,11 @@ def hydrate_event_reference(event_data: dict[str, Any]) -> dict[str, Any]:if temporalio.activity.in_activity() and temporalio.activity.info().attempt < GENERATION_NOT_FOUND_MAX_ATTEMPTS:raise NonReportableApplicationError("Generation not found",- type="generation_not_found",+ type=GENERATION_NOT_FOUND,next_retry_delay=GENERATION_NOT_FOUND_RETRY_DELAY if event_data.get("awaiting_ingestion") else None,)- error_class = ApplicationError if event_data.get("awaiting_ingestion") else NonReportableApplicationError- raise error_class("Generation not found", type="generation_not_found", non_retryable=True)+ # A live reference still fails the run and reaches error tracking: the scheduler saw the+ # event, so a miss after the retries means ingestion lost or delayed it.+ error_class = ApplicationError if event_data.get("awaiting_ingestion") else GenerationNotFoundError+ raise error_class("Generation not found", type=GENERATION_NOT_FOUND, non_retryable=True)return eventdiff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.pyindex d7cfbfd6be72..e891e5823bb1 100644--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py@@ -30,11 +30,16 @@ from posthog.temporal.ai_observability.evaluation_errors import (truncate_error_detail,)from posthog.temporal.ai_observability.evaluation_event_io import (+ GenerationNotFoundError,extract_event_io,extract_event_tools,hydrate_event_reference,)-from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult, build_skipped_evaluation_result+from posthog.temporal.ai_observability.evaluation_types import (+ EvaluationActivityResult,+ build_generation_not_found_result,+ build_skipped_evaluation_result,+)from posthog.temporal.ai_observability.message_utils import extract_text_from_messages, format_tool_definitionsfrom posthog.temporal.ai_observability.metrics import (increment_errors,@@ -502,7 +507,10 @@ def execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActivdef _execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActivityResult:evaluation = inputs.evaluation- event_data = hydrate_event_reference(inputs.event_data)+ try:+ event_data = hydrate_event_reference(inputs.event_data)+ except GenerationNotFoundError:+ return build_generation_not_found_result(evaluation)if evaluation["evaluation_type"] != "llm_judge":raise ApplicationError(diff --git a/posthog/temporal/ai_observability/evaluation_sentiment.py b/posthog/temporal/ai_observability/evaluation_sentiment.pyindex 80a0a5b1f7e8..d37aabeb2dee 100644--- a/posthog/temporal/ai_observability/evaluation_sentiment.py+++ b/posthog/temporal/ai_observability/evaluation_sentiment.py@@ -6,21 +6,12 @@ from temporalio import activityfrom temporalio.exceptions import ApplicationErrorfrom posthog.temporal.ai_observability.evaluation_event_io import extract_event_io-from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult+from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult, build_skipped_sentiment_resultfrom posthog.temporal.ai_observability.sentiment.extraction import extract_sentiment_eval_messagesfrom posthog.temporal.ai_observability.sentiment.schema import PendingClassification, SentimentResultfrom posthog.temporal.ai_observability.sentiment.utils import build_generation_result-def _skipped_sentiment_activity_result(skip_reason: str, reasoning: str) -> EvaluationActivityResult:- return {- "result_type": "sentiment",- "reasoning": reasoning,- "skipped": True,- "skip_reason": skip_reason,- }--def _build_sentiment_activity_result(event_uuid: str,trace_id: str,@@ -28,12 +19,13 @@ def _build_sentiment_activity_result(classification_results: list[SentimentResult],) -> EvaluationActivityResult:if not user_messages:- return _skipped_sentiment_activity_result(- "no_user_messages", "No user messages found; sentiment evaluation skipped."+ return build_skipped_sentiment_result(+ skip_reason="no_user_messages", reasoning="No user messages found; sentiment evaluation skipped.")if not classification_results:- return _skipped_sentiment_activity_result(- "no_classifications", "No sentiment classifications produced; sentiment evaluation skipped."+ return build_skipped_sentiment_result(+ skip_reason="no_classifications",+ reasoning="No sentiment classifications produced; sentiment evaluation skipped.",)pending = [@@ -92,8 +84,8 @@ async def run_sentiment_eval(evaluation: dict[str, Any], event_data: dict[str, Atrace_id = properties.get("$ai_trace_id", event_uuid)user_messages = extract_sentiment_eval_messages(io.input_raw)if not user_messages:- return _skipped_sentiment_activity_result(- "no_user_messages", "No user messages found; sentiment evaluation skipped."+ return build_skipped_sentiment_result(+ skip_reason="no_user_messages", reasoning="No user messages found; sentiment evaluation skipped.")texts = [text for _message_index, text in user_messages]diff --git a/posthog/temporal/ai_observability/evaluation_types.py b/posthog/temporal/ai_observability/evaluation_types.pyindex 9df3f23d55cb..627121044d88 100644--- a/posthog/temporal/ai_observability/evaluation_types.py+++ b/posthog/temporal/ai_observability/evaluation_types.py@@ -9,6 +9,8 @@ EVALUATION_WORKFLOW_PREFIXES = {"sentiment": "llma-sentiment-eval",}+GENERATION_NOT_FOUND = "generation_not_found"+class EvaluationActivityResult(TypedDict, total=False):"""Result produced by evaluation execution activities.@@ -84,3 +86,30 @@ def build_skipped_evaluation_result(if allows_na:result["applicable"] = Falsereturn result+++def build_skipped_sentiment_result(*, reasoning: str, skip_reason: str) -> EvaluationActivityResult:+ return {+ "result_type": "sentiment",+ "reasoning": reasoning,+ "skipped": True,+ "skip_reason": skip_reason,+ }+++def build_generation_not_found_result(evaluation: dict[str, Any]) -> EvaluationActivityResult:+ """Skip for a generation that left `ai_events` before its evaluation ran.++ The workflow returns this skip without an emit, because an $ai_evaluation event needs the+ generation that it grades.+ """+ reasoning = "Generation not found; evaluation skipped."+ output_type = evaluation.get("output_type", "boolean")+ if output_type == "sentiment":+ return build_skipped_sentiment_result(reasoning=reasoning, skip_reason=GENERATION_NOT_FOUND)+ return build_skipped_evaluation_result(+ output_type=output_type,+ allows_na=(evaluation.get("output_config") or {}).get("allows_na", False),+ reasoning=reasoning,+ skip_reason=GENERATION_NOT_FOUND,+ )diff --git a/posthog/temporal/ai_observability/evaluation_workflow_activities.py b/posthog/temporal/ai_observability/evaluation_workflow_activities.pyindex 56daa41f7e87..91e82a62e143 100644--- a/posthog/temporal/ai_observability/evaluation_workflow_activities.py+++ b/posthog/temporal/ai_observability/evaluation_workflow_activities.py@@ -20,11 +20,18 @@ from posthog.dataclasses import frozenfrom posthog.models.team import Teamfrom posthog.sync import database_sync_to_async, database_sync_to_async_poolfrom posthog.temporal.ai_observability.evaluation_errors import is_terminal_user_error_result-from posthog.temporal.ai_observability.evaluation_event_io import as_utc_datetime, hydrate_event_reference+from posthog.temporal.ai_observability.evaluation_event_io import (+ GenerationNotFoundError,+ as_utc_datetime,+ hydrate_event_reference,+)from posthog.temporal.ai_observability.evaluation_hog import run_hog_eval_for_eventfrom posthog.temporal.ai_observability.evaluation_llm_judge import DEFAULT_JUDGE_MODELfrom posthog.temporal.ai_observability.evaluation_sentiment import run_sentiment_eval-from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult+from posthog.temporal.ai_observability.evaluation_types import (+ EvaluationActivityResult,+ build_generation_not_found_result,+)from posthog.temporal.ai_observability.metrics import increment_emit_event_outcomefrom posthog.temporal.ai_observability.retries import is_last_attemptfrom posthog.temporal.ai_observability.team_capture import capture_ai_internal_for_team, raise_emit_failure@@ -548,7 +555,12 @@ async def run_local_evaluation_activity(inputs: RunLocalEvaluationInputs) -> Loc# of an evaluation turned off since it started returns unrun, so the workflow reports the skip.return LocalEvaluationOutcome(evaluation=evaluation, result=None, emitted=False)- event_data = await database_sync_to_async(hydrate_event_reference, thread_sensitive=False)(inputs.event_data)+ try:+ event_data = await database_sync_to_async(hydrate_event_reference, thread_sensitive=False)(inputs.event_data)+ except GenerationNotFoundError:+ return LocalEvaluationOutcome(+ evaluation=evaluation, result=build_generation_not_found_result(evaluation), emitted=False+ )if evaluation_type == "hog":result = await run_hog_eval_for_event(evaluation, event_data)else:diff --git a/posthog/temporal/ai_observability/run_evaluation.py b/posthog/temporal/ai_observability/run_evaluation.pyindex 5af97c6a5741..e61fb893e609 100644--- a/posthog/temporal/ai_observability/run_evaluation.py+++ b/posthog/temporal/ai_observability/run_evaluation.py@@ -31,7 +31,7 @@ from posthog.temporal.ai_observability.evaluation_llm_judge import (get_output_type_config,)from posthog.temporal.ai_observability.evaluation_sentiment import execute_sentiment_eval_activity-from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult+from posthog.temporal.ai_observability.evaluation_types import GENERATION_NOT_FOUND, EvaluationActivityResultfrom posthog.temporal.ai_observability.evaluation_workflow_activities import (EMIT_EVALUATION_EVENT_FAILED_ERROR_TYPE,EmitEvaluationEventInputs,@@ -112,6 +112,22 @@ class WorkflowResult(TypedDict, total=False):message: NotRequired[str]+def build_skipped_workflow_result(+ *, evaluation: dict[str, Any], evaluation_type: str, skip_reason: str, message: str | None = None+) -> WorkflowResult:+ result: WorkflowResult = {+ "skipped": True,+ "skip_reason": skip_reason,+ "evaluation_id": evaluation["id"],+ "evaluation_type": evaluation_type,+ }+ if message is not None:+ result["message"] = message+ if evaluation.get("output_type") not in ("numeric", "categorical"):+ result["verdict"] = None+ return result++async def handle_llm_judge_activity_error(e: temporalio.exceptions.ActivityError, evaluation: dict[str, Any], evaluation_type: str) -> WorkflowResult | None:@@ -142,16 +158,9 @@ async def handle_llm_judge_activity_error()if error_type == "parse_error":- skip_result: WorkflowResult = {- "skipped": True,- "skip_reason": error_type,- "message": e.cause.message,- "evaluation_id": evaluation["id"],- "evaluation_type": evaluation_type,- }- if evaluation.get("output_type") not in ("numeric", "categorical"):- skip_result["verdict"] = None- return skip_result+ return build_skipped_workflow_result(+ evaluation=evaluation, evaluation_type=evaluation_type, skip_reason=error_type, message=e.cause.message+ )key_id = details.get("key_id")if key_id and error_type in ("auth_error", "permission_error", "quota_error", "rate_limit"):@@ -243,16 +252,9 @@ async def handle_terminal_user_error_result(team_id=evaluation["team_id"],)- workflow_result: WorkflowResult = {- "skipped": True,- "skip_reason": skip_reason,- "message": result["reasoning"],- "evaluation_id": evaluation["id"],- "evaluation_type": evaluation_type,- }- if evaluation.get("output_type") not in ("numeric", "categorical"):- workflow_result["verdict"] = None- return workflow_result+ return build_skipped_workflow_result(+ evaluation=evaluation, evaluation_type=evaluation_type, skip_reason=skip_reason, message=result["reasoning"]+ )def compact_backfill_result(result: WorkflowResult, backfill_id: str | None) -> WorkflowResult:@@ -348,15 +350,11 @@ class RunEvaluationWorkflow(PostHogWorkflow):and (evaluation["deleted"] or not evaluation["enabled"])and temporalio.workflow.patched("evaluation-backfill-skips-disabled-evaluation")):- disabled_result: WorkflowResult = {- "skipped": True,- "skip_reason": "evaluation_deleted" if evaluation["deleted"] else "evaluation_disabled",- "evaluation_id": inputs.evaluation_id,- "evaluation_type": evaluation_type,- }- if evaluation.get("output_type") not in ("numeric", "categorical"):- disabled_result["verdict"] = None- return disabled_result+ return build_skipped_workflow_result(+ evaluation=evaluation,+ evaluation_type=evaluation_type,+ skip_reason="evaluation_deleted" if evaluation["deleted"] else "evaluation_disabled",+ )if result is None:if evaluation_type == "hog":@@ -404,6 +402,20 @@ class RunEvaluationWorkflow(PostHogWorkflow):non_retryable=True,)+ # The generation left ai_events, so no $ai_evaluation event can describe it and the emit+ # activity would fail to read it. The backfill counts the run as skipped, not failed.+ # During a rolling deploy an old workflow worker can receive this result and schedule+ # the emit, so the patch keeps those histories replaying through the emit command.+ if result.get("skip_reason") == GENERATION_NOT_FOUND and temporalio.workflow.patched(+ "skip-emit-for-missing-generation-2026-10"+ ):+ return build_skipped_workflow_result(+ evaluation=evaluation,+ evaluation_type=evaluation_type,+ skip_reason=GENERATION_NOT_FOUND,+ message=result["reasoning"],+ )+if is_terminal_user_error_result(result):return await handle_terminal_user_error_result(evaluation=evaluation,diff --git a/posthog/temporal/ai_observability/run_trace_evaluation.py b/posthog/temporal/ai_observability/run_trace_evaluation.pyindex 735b542d5434..f645eeae5ae1 100644--- a/posthog/temporal/ai_observability/run_trace_evaluation.py+++ b/posthog/temporal/ai_observability/run_trace_evaluation.py@@ -67,6 +67,7 @@ from posthog.temporal.ai_observability.message_utils import extract_text_from_mefrom posthog.temporal.ai_observability.metrics import increment_emit_event_outcome, increment_errorsfrom posthog.temporal.ai_observability.run_evaluation import (WorkflowResult,+ build_skipped_workflow_result,handle_llm_judge_activity_error,handle_terminal_user_error_result,)@@ -886,15 +887,11 @@ class RunTraceEvaluationWorkflow(PostHogWorkflow):# The evaluation may have been paused or deleted during the aggregation window —# bail out instead of running against config the user just turned off.if evaluation["deleted"] or not evaluation["enabled"]:- disabled_result: WorkflowResult = {- "skipped": True,- "skip_reason": "evaluation_deleted" if evaluation["deleted"] else "evaluation_disabled",- "evaluation_id": inputs.evaluation_id,- "evaluation_type": evaluation_type,- }- if evaluation.get("output_type") not in ("numeric", "categorical"):- disabled_result["verdict"] = None- return disabled_result+ return build_skipped_workflow_result(+ evaluation=evaluation,+ evaluation_type=evaluation_type,+ skip_reason="evaluation_deleted" if evaluation["deleted"] else "evaluation_disabled",+ )execute_inputs = ExecuteTraceEvaluationInputs(evaluation=evaluation,diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.pyindex 4fb7c7ec3d8e..26ac0f2ddad6 100644--- a/posthog/temporal/ai_observability/test_run_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_evaluation.py@@ -66,7 +66,7 @@ from .evaluation_errors import (status_reason_detail_for_terminal_user_error,terminal_user_error_result_from_application_error,)-from .evaluation_event_io import GENERATION_NOT_FOUND_RETRY_DELAY, hydrate_event_reference+from .evaluation_event_io import GENERATION_NOT_FOUND_RETRY_DELAY, GenerationNotFoundError, hydrate_event_referencefrom .evaluation_llm_judge import (BACKFILL_ACTIVITY_RETRY_POLICY,NumericWithNAEvalResult,@@ -77,7 +77,7 @@ from .evaluation_llm_judge import (call_llm_judge,get_output_type_config,)-from .evaluation_types import build_skipped_evaluation_result+from .evaluation_types import GENERATION_NOT_FOUND, build_generation_not_found_result, build_skipped_evaluation_resultfrom .evaluation_workflow_activities import (LocalEvaluationOutcome,backfill_verdict_timestamp,@@ -2308,9 +2308,64 @@ class TestRunEvaluationWorkflow:assert raised.value.type == "generation_not_found"assert raised.value.non_retryable is not retryableassert is_expected_activity_failure(raised.value) is (retryable or not live)+ assert isinstance(raised.value, GenerationNotFoundError) is (not retryable and not live)expected_delay = GENERATION_NOT_FOUND_RETRY_DELAY if retryable and live else Noneassert raised.value.next_retry_delay == expected_delay+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "evaluation_type,output_type,expected_result_type",+ [+ ("hog", "boolean", "boolean"),+ ("hog", "numeric", "numeric"),+ ("sentiment", "sentiment", "sentiment"),+ ("llm_judge", "boolean", "boolean"),+ ],+ )+ async def test_an_expired_generation_is_skipped_without_grading_or_emitting(+ self, evaluation_type: str, output_type: str, expected_result_type: str+ ):+ evaluation = _local_evaluation(+ evaluation_type=evaluation_type,+ output_type=output_type,+ evaluation_config={"prompt": "grade it"},+ enabled=True,+ deleted=False,+ )+ module = "posthog.temporal.ai_observability.evaluation_workflow_activities"+ with (+ patch(HYDRATE_FETCH, return_value=None),+ patch(f"{module}.fetch_evaluation", return_value=evaluation),+ patch(f"{module}.run_hog_eval_for_event", new_callable=AsyncMock) as run_hog,+ patch(f"{module}.run_sentiment_eval", new_callable=AsyncMock) as run_sentiment,+ patch(f"{module}.emit_generation_evaluation_event", new_callable=AsyncMock) as emit,+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.call_llm_judge") as call_judge,+ ):+ if evaluation_type == "llm_judge":+ result = _execute_llm_judge_activity(+ ExecuteLLMJudgeInputs(evaluation=evaluation, event_data=dict(THIN_REFERENCE))+ )+ else:+ outcome = await run_local_evaluation_activity(+ RunLocalEvaluationInputs(+ evaluation_id="eval-1",+ event_data=dict(THIN_REFERENCE),+ start_time=datetime(2024, 1, 1, 12, 0, tzinfo=UTC),+ backfill_id="B",+ )+ )+ assert outcome.emitted is False+ assert outcome.result is not None+ result = outcome.result++ assert result["skipped"] is True+ assert result["skip_reason"] == GENERATION_NOT_FOUND+ assert result["result_type"] == expected_result_type+ run_hog.assert_not_awaited()+ run_sentiment.assert_not_awaited()+ emit.assert_not_awaited()+ call_judge.assert_not_called()+def test_parse_inputs(self):"""Test that parse_inputs correctly parses workflow inputs"""event_data = create_mock_event_data(team_id=1)@@ -2637,6 +2692,51 @@ class TestRunEvaluationWorkflow:assert result["skipped"] is Trueassert result["skip_reason"] == skip_reason+ @pytest.mark.asyncio+ @pytest.mark.parametrize("evaluation_type", ["hog", "llm_judge"])+ async def test_backfill_run_reports_an_expired_generation_as_skipped_without_emitting(+ self, evaluation_type: str+ ) -> None:+ evaluation = _local_evaluation(evaluation_type=evaluation_type, enabled=True, deleted=False)++ @activity.defn(name="run_local_evaluation_activity")+ async def mock_run_local_evaluation(inputs: RunLocalEvaluationInputs) -> LocalEvaluationOutcome:+ local_result = build_generation_not_found_result(evaluation) if evaluation_type == "hog" else None+ return LocalEvaluationOutcome(evaluation=evaluation, result=local_result, emitted=False)++ @activity.defn(name="execute_llm_judge_activity")+ async def mock_judge(inputs: ExecuteLLMJudgeInputs) -> EvaluationActivityResult:+ return build_generation_not_found_result(inputs.evaluation)++ @activity.defn(name="emit_evaluation_event_activity")+ async def mock_emit(inputs: EmitEvaluationEventInputs) -> None:+ raise AssertionError("an expired generation has nothing for an $ai_evaluation event to describe")++ task_queue = str(uuid.uuid4())+ async with await WorkflowEnvironment.start_time_skipping() as env:+ async with Worker(+ env.client,+ task_queue=task_queue,+ workflows=[RunEvaluationWorkflow],+ activities=[mock_run_local_evaluation, mock_judge, mock_emit],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ result: WorkflowResult = await env.client.execute_workflow(+ RunEvaluationWorkflow.run,+ RunEvaluationInputs(evaluation_id="eval-1", event_data=dict(THIN_REFERENCE), backfill_id="B"),+ id=str(uuid.uuid4()),+ task_queue=task_queue,+ execution_timeout=timedelta(minutes=5),+ )++ assert result == {+ "skipped": True,+ "skip_reason": GENERATION_NOT_FOUND,+ "evaluation_id": "eval-1",+ "evaluation_type": evaluation_type,+ "verdict": None,+ }+@pytest.mark.asyncioasync def test_terminal_user_error_does_not_email_when_evaluation_was_already_disabled(self):calls: list[str] = []