Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): skip evaluations whose generation aged out of ai_events (#122224)

Carlos Marchal1034635b4620
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index 0894ce0c6b07..680193dd40eb 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -196,7 +196,7 @@ Ranges ending at or before that boundary are rejected.
Existing backfills keep their stored windows; **Retry remaining** applies the boundary when it creates a new backfill.
Generation references get up to three lookup attempts.
-A final miss for a backfill reference fails the run without reporting an exception to error tracking.
+A final miss for a backfill reference records the evaluation as skipped with `generation_not_found`, emits no `$ai_evaluation` event, and reports no exception to error tracking.
Live references marked `awaiting_ingestion` wait 15 seconds between misses and still report the final miss.
A backfill that stops on its own shows as **Interrupted**, with a stop reason.
diff --git a/posthog/temporal/ai_observability/evaluation_event_io.py b/posthog/temporal/ai_observability/evaluation_event_io.py
index 49eb498dd181..df63f1009670 100644
--- a/posthog/temporal/ai_observability/evaluation_event_io.py
+++ b/posthog/temporal/ai_observability/evaluation_event_io.py
@@ -5,6 +5,7 @@ import temporalio.activity
from temporalio.exceptions import ApplicationError
from posthog.dataclasses import frozen
+from posthog.temporal.ai_observability.evaluation_types import GENERATION_NOT_FOUND
from posthog.temporal.common.errors import NonReportableApplicationError
from posthog.utils import ensure_utc
@@ -17,6 +18,16 @@ GENERATION_NOT_FOUND_MAX_ATTEMPTS = 3
GENERATION_NOT_FOUND_RETRY_DELAY = timedelta(seconds=15)
+class GenerationNotFoundError(NonReportableApplicationError):
+ """The last miss on a reference that has no ingestion to wait for.
+
+ Only a backfill sends such a reference, and it selects candidates from `events`, which keeps
+ rows longer than `ai_events`. So the row most likely aged out of `ai_events`. An evaluation
+ catches this and records a skip. A caller that does not catch it fails the run, and the error
+ stays out of error tracking.
+ """
+
+
def as_utc_datetime(value: str | datetime) -> datetime:
"""Read a ClickHouse event timestamp, which reaches us as a naive datetime on a direct call
and as an ISO string once Temporal has serialized it through a payload."""
@@ -93,9 +104,11 @@ def hydrate_event_reference(event_data: dict[str, Any]) -> dict[str, Any]:
if temporalio.activity.in_activity() and temporalio.activity.info().attempt < GENERATION_NOT_FOUND_MAX_ATTEMPTS:
raise NonReportableApplicationError(
"Generation not found",
- type="generation_not_found",
+ type=GENERATION_NOT_FOUND,
next_retry_delay=GENERATION_NOT_FOUND_RETRY_DELAY if event_data.get("awaiting_ingestion") else None,
)
- error_class = ApplicationError if event_data.get("awaiting_ingestion") else NonReportableApplicationError
- raise error_class("Generation not found", type="generation_not_found", non_retryable=True)
+ # A live reference still fails the run and reaches error tracking: the scheduler saw the
+ # event, so a miss after the retries means ingestion lost or delayed it.
+ error_class = ApplicationError if event_data.get("awaiting_ingestion") else GenerationNotFoundError
+ raise error_class("Generation not found", type=GENERATION_NOT_FOUND, non_retryable=True)
return event
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index d7cfbfd6be72..e891e5823bb1 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -30,11 +30,16 @@ from posthog.temporal.ai_observability.evaluation_errors import (
truncate_error_detail,
)
from posthog.temporal.ai_observability.evaluation_event_io import (
+ GenerationNotFoundError,
extract_event_io,
extract_event_tools,
hydrate_event_reference,
)
-from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult, build_skipped_evaluation_result
+from posthog.temporal.ai_observability.evaluation_types import (
+ EvaluationActivityResult,
+ build_generation_not_found_result,
+ build_skipped_evaluation_result,
+)
from posthog.temporal.ai_observability.message_utils import extract_text_from_messages, format_tool_definitions
from posthog.temporal.ai_observability.metrics import (
increment_errors,
@@ -502,7 +507,10 @@ def execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActiv
def _execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActivityResult:
evaluation = inputs.evaluation
- event_data = hydrate_event_reference(inputs.event_data)
+ try:
+ event_data = hydrate_event_reference(inputs.event_data)
+ except GenerationNotFoundError:
+ return build_generation_not_found_result(evaluation)
if evaluation["evaluation_type"] != "llm_judge":
raise ApplicationError(
diff --git a/posthog/temporal/ai_observability/evaluation_sentiment.py b/posthog/temporal/ai_observability/evaluation_sentiment.py
index 80a0a5b1f7e8..d37aabeb2dee 100644
--- a/posthog/temporal/ai_observability/evaluation_sentiment.py
+++ b/posthog/temporal/ai_observability/evaluation_sentiment.py
@@ -6,21 +6,12 @@ from temporalio import activity
from temporalio.exceptions import ApplicationError
from posthog.temporal.ai_observability.evaluation_event_io import extract_event_io
-from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult
+from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult, build_skipped_sentiment_result
from posthog.temporal.ai_observability.sentiment.extraction import extract_sentiment_eval_messages
from posthog.temporal.ai_observability.sentiment.schema import PendingClassification, SentimentResult
from posthog.temporal.ai_observability.sentiment.utils import build_generation_result
-def _skipped_sentiment_activity_result(skip_reason: str, reasoning: str) -> EvaluationActivityResult:
- return {
- "result_type": "sentiment",
- "reasoning": reasoning,
- "skipped": True,
- "skip_reason": skip_reason,
- }
-
-
def _build_sentiment_activity_result(
event_uuid: str,
trace_id: str,
@@ -28,12 +19,13 @@ def _build_sentiment_activity_result(
classification_results: list[SentimentResult],
) -> EvaluationActivityResult:
if not user_messages:
- return _skipped_sentiment_activity_result(
- "no_user_messages", "No user messages found; sentiment evaluation skipped."
+ return build_skipped_sentiment_result(
+ skip_reason="no_user_messages", reasoning="No user messages found; sentiment evaluation skipped."
)
if not classification_results:
- return _skipped_sentiment_activity_result(
- "no_classifications", "No sentiment classifications produced; sentiment evaluation skipped."
+ return build_skipped_sentiment_result(
+ skip_reason="no_classifications",
+ reasoning="No sentiment classifications produced; sentiment evaluation skipped.",
)
pending = [
@@ -92,8 +84,8 @@ async def run_sentiment_eval(evaluation: dict[str, Any], event_data: dict[str, A
trace_id = properties.get("$ai_trace_id", event_uuid)
user_messages = extract_sentiment_eval_messages(io.input_raw)
if not user_messages:
- return _skipped_sentiment_activity_result(
- "no_user_messages", "No user messages found; sentiment evaluation skipped."
+ return build_skipped_sentiment_result(
+ skip_reason="no_user_messages", reasoning="No user messages found; sentiment evaluation skipped."
)
texts = [text for _message_index, text in user_messages]
diff --git a/posthog/temporal/ai_observability/evaluation_types.py b/posthog/temporal/ai_observability/evaluation_types.py
index 9df3f23d55cb..627121044d88 100644
--- a/posthog/temporal/ai_observability/evaluation_types.py
+++ b/posthog/temporal/ai_observability/evaluation_types.py
@@ -9,6 +9,8 @@ EVALUATION_WORKFLOW_PREFIXES = {
"sentiment": "llma-sentiment-eval",
}
+GENERATION_NOT_FOUND = "generation_not_found"
+
class EvaluationActivityResult(TypedDict, total=False):
"""Result produced by evaluation execution activities.
@@ -84,3 +86,30 @@ def build_skipped_evaluation_result(
if allows_na:
result["applicable"] = False
return result
+
+
+def build_skipped_sentiment_result(*, reasoning: str, skip_reason: str) -> EvaluationActivityResult:
+ return {
+ "result_type": "sentiment",
+ "reasoning": reasoning,
+ "skipped": True,
+ "skip_reason": skip_reason,
+ }
+
+
+def build_generation_not_found_result(evaluation: dict[str, Any]) -> EvaluationActivityResult:
+ """Skip for a generation that left `ai_events` before its evaluation ran.
+
+ The workflow returns this skip without an emit, because an $ai_evaluation event needs the
+ generation that it grades.
+ """
+ reasoning = "Generation not found; evaluation skipped."
+ output_type = evaluation.get("output_type", "boolean")
+ if output_type == "sentiment":
+ return build_skipped_sentiment_result(reasoning=reasoning, skip_reason=GENERATION_NOT_FOUND)
+ return build_skipped_evaluation_result(
+ output_type=output_type,
+ allows_na=(evaluation.get("output_config") or {}).get("allows_na", False),
+ reasoning=reasoning,
+ skip_reason=GENERATION_NOT_FOUND,
+ )
diff --git a/posthog/temporal/ai_observability/evaluation_workflow_activities.py b/posthog/temporal/ai_observability/evaluation_workflow_activities.py
index 56daa41f7e87..91e82a62e143 100644
--- a/posthog/temporal/ai_observability/evaluation_workflow_activities.py
+++ b/posthog/temporal/ai_observability/evaluation_workflow_activities.py
@@ -20,11 +20,18 @@ from posthog.dataclasses import frozen
from posthog.models.team import Team
from posthog.sync import database_sync_to_async, database_sync_to_async_pool
from posthog.temporal.ai_observability.evaluation_errors import is_terminal_user_error_result
-from posthog.temporal.ai_observability.evaluation_event_io import as_utc_datetime, hydrate_event_reference
+from posthog.temporal.ai_observability.evaluation_event_io import (
+ GenerationNotFoundError,
+ as_utc_datetime,
+ hydrate_event_reference,
+)
from posthog.temporal.ai_observability.evaluation_hog import run_hog_eval_for_event
from posthog.temporal.ai_observability.evaluation_llm_judge import DEFAULT_JUDGE_MODEL
from posthog.temporal.ai_observability.evaluation_sentiment import run_sentiment_eval
-from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult
+from posthog.temporal.ai_observability.evaluation_types import (
+ EvaluationActivityResult,
+ build_generation_not_found_result,
+)
from posthog.temporal.ai_observability.metrics import increment_emit_event_outcome
from posthog.temporal.ai_observability.retries import is_last_attempt
from posthog.temporal.ai_observability.team_capture import capture_ai_internal_for_team, raise_emit_failure
@@ -548,7 +555,12 @@ async def run_local_evaluation_activity(inputs: RunLocalEvaluationInputs) -> Loc
# of an evaluation turned off since it started returns unrun, so the workflow reports the skip.
return LocalEvaluationOutcome(evaluation=evaluation, result=None, emitted=False)
- event_data = await database_sync_to_async(hydrate_event_reference, thread_sensitive=False)(inputs.event_data)
+ try:
+ event_data = await database_sync_to_async(hydrate_event_reference, thread_sensitive=False)(inputs.event_data)
+ except GenerationNotFoundError:
+ return LocalEvaluationOutcome(
+ evaluation=evaluation, result=build_generation_not_found_result(evaluation), emitted=False
+ )
if evaluation_type == "hog":
result = await run_hog_eval_for_event(evaluation, event_data)
else:
diff --git a/posthog/temporal/ai_observability/run_evaluation.py b/posthog/temporal/ai_observability/run_evaluation.py
index 5af97c6a5741..e61fb893e609 100644
--- a/posthog/temporal/ai_observability/run_evaluation.py
+++ b/posthog/temporal/ai_observability/run_evaluation.py
@@ -31,7 +31,7 @@ from posthog.temporal.ai_observability.evaluation_llm_judge import (
get_output_type_config,
)
from posthog.temporal.ai_observability.evaluation_sentiment import execute_sentiment_eval_activity
-from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult
+from posthog.temporal.ai_observability.evaluation_types import GENERATION_NOT_FOUND, EvaluationActivityResult
from posthog.temporal.ai_observability.evaluation_workflow_activities import (
EMIT_EVALUATION_EVENT_FAILED_ERROR_TYPE,
EmitEvaluationEventInputs,
@@ -112,6 +112,22 @@ class WorkflowResult(TypedDict, total=False):
message: NotRequired[str]
+def build_skipped_workflow_result(
+ *, evaluation: dict[str, Any], evaluation_type: str, skip_reason: str, message: str | None = None
+) -> WorkflowResult:
+ result: WorkflowResult = {
+ "skipped": True,
+ "skip_reason": skip_reason,
+ "evaluation_id": evaluation["id"],
+ "evaluation_type": evaluation_type,
+ }
+ if message is not None:
+ result["message"] = message
+ if evaluation.get("output_type") not in ("numeric", "categorical"):
+ result["verdict"] = None
+ return result
+
+
async def handle_llm_judge_activity_error(
e: temporalio.exceptions.ActivityError, evaluation: dict[str, Any], evaluation_type: str
) -> WorkflowResult | None:
@@ -142,16 +158,9 @@ async def handle_llm_judge_activity_error(
)
if error_type == "parse_error":
- skip_result: WorkflowResult = {
- "skipped": True,
- "skip_reason": error_type,
- "message": e.cause.message,
- "evaluation_id": evaluation["id"],
- "evaluation_type": evaluation_type,
- }
- if evaluation.get("output_type") not in ("numeric", "categorical"):
- skip_result["verdict"] = None
- return skip_result
+ return build_skipped_workflow_result(
+ evaluation=evaluation, evaluation_type=evaluation_type, skip_reason=error_type, message=e.cause.message
+ )
key_id = details.get("key_id")
if key_id and error_type in ("auth_error", "permission_error", "quota_error", "rate_limit"):
@@ -243,16 +252,9 @@ async def handle_terminal_user_error_result(
team_id=evaluation["team_id"],
)
- workflow_result: WorkflowResult = {
- "skipped": True,
- "skip_reason": skip_reason,
- "message": result["reasoning"],
- "evaluation_id": evaluation["id"],
- "evaluation_type": evaluation_type,
- }
- if evaluation.get("output_type") not in ("numeric", "categorical"):
- workflow_result["verdict"] = None
- return workflow_result
+ return build_skipped_workflow_result(
+ evaluation=evaluation, evaluation_type=evaluation_type, skip_reason=skip_reason, message=result["reasoning"]
+ )
def compact_backfill_result(result: WorkflowResult, backfill_id: str | None) -> WorkflowResult:
@@ -348,15 +350,11 @@ class RunEvaluationWorkflow(PostHogWorkflow):
and (evaluation["deleted"] or not evaluation["enabled"])
and temporalio.workflow.patched("evaluation-backfill-skips-disabled-evaluation")
):
- disabled_result: WorkflowResult = {
- "skipped": True,
- "skip_reason": "evaluation_deleted" if evaluation["deleted"] else "evaluation_disabled",
- "evaluation_id": inputs.evaluation_id,
- "evaluation_type": evaluation_type,
- }
- if evaluation.get("output_type") not in ("numeric", "categorical"):
- disabled_result["verdict"] = None
- return disabled_result
+ return build_skipped_workflow_result(
+ evaluation=evaluation,
+ evaluation_type=evaluation_type,
+ skip_reason="evaluation_deleted" if evaluation["deleted"] else "evaluation_disabled",
+ )
if result is None:
if evaluation_type == "hog":
@@ -404,6 +402,20 @@ class RunEvaluationWorkflow(PostHogWorkflow):
non_retryable=True,
)
+ # The generation left ai_events, so no $ai_evaluation event can describe it and the emit
+ # activity would fail to read it. The backfill counts the run as skipped, not failed.
+ # During a rolling deploy an old workflow worker can receive this result and schedule
+ # the emit, so the patch keeps those histories replaying through the emit command.
+ if result.get("skip_reason") == GENERATION_NOT_FOUND and temporalio.workflow.patched(
+ "skip-emit-for-missing-generation-2026-10"
+ ):
+ return build_skipped_workflow_result(
+ evaluation=evaluation,
+ evaluation_type=evaluation_type,
+ skip_reason=GENERATION_NOT_FOUND,
+ message=result["reasoning"],
+ )
+
if is_terminal_user_error_result(result):
return await handle_terminal_user_error_result(
evaluation=evaluation,
diff --git a/posthog/temporal/ai_observability/run_trace_evaluation.py b/posthog/temporal/ai_observability/run_trace_evaluation.py
index 735b542d5434..f645eeae5ae1 100644
--- a/posthog/temporal/ai_observability/run_trace_evaluation.py
+++ b/posthog/temporal/ai_observability/run_trace_evaluation.py
@@ -67,6 +67,7 @@ from posthog.temporal.ai_observability.message_utils import extract_text_from_me
from posthog.temporal.ai_observability.metrics import increment_emit_event_outcome, increment_errors
from posthog.temporal.ai_observability.run_evaluation import (
WorkflowResult,
+ build_skipped_workflow_result,
handle_llm_judge_activity_error,
handle_terminal_user_error_result,
)
@@ -886,15 +887,11 @@ class RunTraceEvaluationWorkflow(PostHogWorkflow):
# The evaluation may have been paused or deleted during the aggregation window —
# bail out instead of running against config the user just turned off.
if evaluation["deleted"] or not evaluation["enabled"]:
- disabled_result: WorkflowResult = {
- "skipped": True,
- "skip_reason": "evaluation_deleted" if evaluation["deleted"] else "evaluation_disabled",
- "evaluation_id": inputs.evaluation_id,
- "evaluation_type": evaluation_type,
- }
- if evaluation.get("output_type") not in ("numeric", "categorical"):
- disabled_result["verdict"] = None
- return disabled_result
+ return build_skipped_workflow_result(
+ evaluation=evaluation,
+ evaluation_type=evaluation_type,
+ skip_reason="evaluation_deleted" if evaluation["deleted"] else "evaluation_disabled",
+ )
execute_inputs = ExecuteTraceEvaluationInputs(
evaluation=evaluation,
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index 4fb7c7ec3d8e..26ac0f2ddad6 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -66,7 +66,7 @@ from .evaluation_errors import (
status_reason_detail_for_terminal_user_error,
terminal_user_error_result_from_application_error,
)
-from .evaluation_event_io import GENERATION_NOT_FOUND_RETRY_DELAY, hydrate_event_reference
+from .evaluation_event_io import GENERATION_NOT_FOUND_RETRY_DELAY, GenerationNotFoundError, hydrate_event_reference
from .evaluation_llm_judge import (
BACKFILL_ACTIVITY_RETRY_POLICY,
NumericWithNAEvalResult,
@@ -77,7 +77,7 @@ from .evaluation_llm_judge import (
call_llm_judge,
get_output_type_config,
)
-from .evaluation_types import build_skipped_evaluation_result
+from .evaluation_types import GENERATION_NOT_FOUND, build_generation_not_found_result, build_skipped_evaluation_result
from .evaluation_workflow_activities import (
LocalEvaluationOutcome,
backfill_verdict_timestamp,
@@ -2308,9 +2308,64 @@ class TestRunEvaluationWorkflow:
assert raised.value.type == "generation_not_found"
assert raised.value.non_retryable is not retryable
assert is_expected_activity_failure(raised.value) is (retryable or not live)
+ assert isinstance(raised.value, GenerationNotFoundError) is (not retryable and not live)
expected_delay = GENERATION_NOT_FOUND_RETRY_DELAY if retryable and live else None
assert raised.value.next_retry_delay == expected_delay
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "evaluation_type,output_type,expected_result_type",
+ [
+ ("hog", "boolean", "boolean"),
+ ("hog", "numeric", "numeric"),
+ ("sentiment", "sentiment", "sentiment"),
+ ("llm_judge", "boolean", "boolean"),
+ ],
+ )
+ async def test_an_expired_generation_is_skipped_without_grading_or_emitting(
+ self, evaluation_type: str, output_type: str, expected_result_type: str
+ ):
+ evaluation = _local_evaluation(
+ evaluation_type=evaluation_type,
+ output_type=output_type,
+ evaluation_config={"prompt": "grade it"},
+ enabled=True,
+ deleted=False,
+ )
+ module = "posthog.temporal.ai_observability.evaluation_workflow_activities"
+ with (
+ patch(HYDRATE_FETCH, return_value=None),
+ patch(f"{module}.fetch_evaluation", return_value=evaluation),
+ patch(f"{module}.run_hog_eval_for_event", new_callable=AsyncMock) as run_hog,
+ patch(f"{module}.run_sentiment_eval", new_callable=AsyncMock) as run_sentiment,
+ patch(f"{module}.emit_generation_evaluation_event", new_callable=AsyncMock) as emit,
+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.call_llm_judge") as call_judge,
+ ):
+ if evaluation_type == "llm_judge":
+ result = _execute_llm_judge_activity(
+ ExecuteLLMJudgeInputs(evaluation=evaluation, event_data=dict(THIN_REFERENCE))
+ )
+ else:
+ outcome = await run_local_evaluation_activity(
+ RunLocalEvaluationInputs(
+ evaluation_id="eval-1",
+ event_data=dict(THIN_REFERENCE),
+ start_time=datetime(2024, 1, 1, 12, 0, tzinfo=UTC),
+ backfill_id="B",
+ )
+ )
+ assert outcome.emitted is False
+ assert outcome.result is not None
+ result = outcome.result
+
+ assert result["skipped"] is True
+ assert result["skip_reason"] == GENERATION_NOT_FOUND
+ assert result["result_type"] == expected_result_type
+ run_hog.assert_not_awaited()
+ run_sentiment.assert_not_awaited()
+ emit.assert_not_awaited()
+ call_judge.assert_not_called()
+
def test_parse_inputs(self):
"""Test that parse_inputs correctly parses workflow inputs"""
event_data = create_mock_event_data(team_id=1)
@@ -2637,6 +2692,51 @@ class TestRunEvaluationWorkflow:
assert result["skipped"] is True
assert result["skip_reason"] == skip_reason
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("evaluation_type", ["hog", "llm_judge"])
+ async def test_backfill_run_reports_an_expired_generation_as_skipped_without_emitting(
+ self, evaluation_type: str
+ ) -> None:
+ evaluation = _local_evaluation(evaluation_type=evaluation_type, enabled=True, deleted=False)
+
+ @activity.defn(name="run_local_evaluation_activity")
+ async def mock_run_local_evaluation(inputs: RunLocalEvaluationInputs) -> LocalEvaluationOutcome:
+ local_result = build_generation_not_found_result(evaluation) if evaluation_type == "hog" else None
+ return LocalEvaluationOutcome(evaluation=evaluation, result=local_result, emitted=False)
+
+ @activity.defn(name="execute_llm_judge_activity")
+ async def mock_judge(inputs: ExecuteLLMJudgeInputs) -> EvaluationActivityResult:
+ return build_generation_not_found_result(inputs.evaluation)
+
+ @activity.defn(name="emit_evaluation_event_activity")
+ async def mock_emit(inputs: EmitEvaluationEventInputs) -> None:
+ raise AssertionError("an expired generation has nothing for an $ai_evaluation event to describe")
+
+ task_queue = str(uuid.uuid4())
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with Worker(
+ env.client,
+ task_queue=task_queue,
+ workflows=[RunEvaluationWorkflow],
+ activities=[mock_run_local_evaluation, mock_judge, mock_emit],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ result: WorkflowResult = await env.client.execute_workflow(
+ RunEvaluationWorkflow.run,
+ RunEvaluationInputs(evaluation_id="eval-1", event_data=dict(THIN_REFERENCE), backfill_id="B"),
+ id=str(uuid.uuid4()),
+ task_queue=task_queue,
+ execution_timeout=timedelta(minutes=5),
+ )
+
+ assert result == {
+ "skipped": True,
+ "skip_reason": GENERATION_NOT_FOUND,
+ "evaluation_id": "eval-1",
+ "evaluation_type": evaluation_type,
+ "verdict": None,
+ }
+
@pytest.mark.asyncio
async def test_terminal_user_error_does_not_email_when_evaluation_was_already_disabled(self):
calls: list[str] = []