:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
"""Trace-level evaluation workflow.
The evaluation scheduler starts this workflow when the first $ai_generation matching a
trace-target evaluation arrives. The workflow id (`llma-trace-eval-{evaluation_id}-{trace_id}`)
plus the USE_EXISTING conflict policy make every later matching event of the same trace a
no-op while a run is pending or in flight, and ALLOW_DUPLICATE_FAILED_ONLY prevents
re-evaluating a trace once a run completed. The workflow sleeps for an aggregation window so
the rest of the trace can arrive, then pulls the whole trace from ClickHouse and evaluates it
in one shot.
Temporal payloads are capped at ~2 MiB, so trace content never flows through the workflow:
each execute activity fetches the trace itself and returns only the small verdict dict.
"""
import json
import asyncio
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import TYPE_CHECKING, Any
import structlog
import temporalio
import posthoganalytics
from temporalio.common import RetryPolicy
from temporalio.exceptions import ApplicationError
from posthog.schema import DateRange, LLMTrace, QueryLogTags, TraceQuery
from posthog.hogql import ast
from posthog.hogql.parser import parse_select
from posthog.api.capture import CaptureInternalError
from posthog.dataclasses import frozen
from posthog.hogql_queries.ai.ai_table_resolver import query_ai_events
from posthog.hogql_queries.ai.trace_query_runner import TraceQueryRunner
from posthog.models.team import Team
from posthog.sync import database_sync_to_async
from posthog.temporal.ai_observability.evaluation_errors import is_terminal_user_error_result
from posthog.temporal.ai_observability.evaluation_event_io import as_utc_datetime, extract_event_io
from posthog.temporal.ai_observability.evaluation_hog import (
build_hog_event_global,
execute_hog_eval_bytecode,
finalize_hog_eval_result,
hog_bytecode_references_global,
)
from posthog.temporal.ai_observability.evaluation_llm_judge import (
LLM_JUDGE_RETRY_POLICY,
call_llm_judge,
get_output_type_config,
)
from posthog.temporal.ai_observability.evaluation_payload import (
PAYLOAD_BYTES_EXPR,
payload_budget_bytes,
should_skip_for_payload,
)
from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult, build_skipped_evaluation_result
from posthog.temporal.ai_observability.evaluation_workflow_activities import (
EmitInternalTelemetryInputs,
RunEvaluationInputs,
backfill_verdict_timestamp,
build_evaluation_event_properties,
capture_evaluation_run_usage,
emit_internal_telemetry_activity,
fetch_evaluation_activity,
)
from posthog.temporal.ai_observability.judge_attachments import build_attachment_prompt
from posthog.temporal.ai_observability.message_utils import extract_text_from_messages
from posthog.temporal.ai_observability.metrics import increment_emit_event_outcome, increment_errors
from posthog.temporal.ai_observability.run_evaluation import (
WorkflowResult,
build_skipped_workflow_result,
handle_llm_judge_activity_error,
handle_terminal_user_error_result,
)
from posthog.temporal.ai_observability.team_capture import (
EMIT_EVENT_RETRY_POLICY,
EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT,
EMIT_EVENT_START_TO_CLOSE_TIMEOUT,
capture_ai_internal_for_team,
raise_emit_failure,
)
from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.utils import close_db_connections
from products.ai_observability.backend.models.evaluation_configs import (
EVALUATION_TEST_LOOKBACK_DAYS,
TRACE_EVAL_DEFAULT_WINDOW_SECONDS,
TRACE_EVAL_MAX_WINDOW_SECONDS,
)
from products.ai_observability.backend.text_repr.formatters import (
FormatterOptions,
RenderBudgetExceeded,
format_trace_text_repr,
llm_trace_to_formatter_format,
)
from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaMarker
from products.ai_observability.backend.text_repr.formatters.message_formatter import has_message_content
if TYPE_CHECKING:
from posthog.models import User
logger = structlog.get_logger(__name__)
# How far back from the workflow start to look for trace events. The trigger is the first
# *matching* generation, so earlier events of the same trace (spans, prior generations) are
# expected; client-set timestamps far in the past fall outside this window by design.
TRACE_EVENTS_LOOKBACK = timedelta(hours=24)
# Guard against degenerate traces (e.g. a bug attaching everything to trace id "0"): traces
# above this event count are skipped without fetching their payload.
MAX_TRACE_EVAL_EVENTS = 500
# Char budget for the judge transcript. The shared formatter defaults to ~2M chars (sized to
# fill an LLM context window for summarization); a per-trace judge verdict doesn't need that
# much, so we cap lower to bound cost. Over budget, the formatter uniformly samples lines.
JUDGE_TRACE_MAX_CHARS = 150_000
_TRACE_EVENT_NAMES = ("$ai_span", "$ai_generation", "$ai_embedding", "$ai_metric", "$ai_feedback", "$ai_trace")
# The runner drops the `$ai_trace` root row from `LLMTrace.events`, so it can never contribute a
# line to the judge transcript.
_RENDERABLE_TRACE_EVENT_NAMES = tuple(name for name in _TRACE_EVENT_NAMES if name != "$ai_trace")
def _event_names_clause(event_names: tuple[str, ...]) -> str:
return ", ".join(f"'{name}'" for name in event_names)
def _trace_event_count_sql(event_names: tuple[str, ...]) -> str:
"""Count query written against ai_events; query_ai_events rewrites it for the events table when
ai_events returns nothing. HAVING makes a zero count return no rows, which both triggers the
events-table fallback and keeps "no events" distinguishable without a second query."""
return f"""
SELECT count() AS event_count
FROM posthog.ai_events AS ai_events
WHERE event IN ({_event_names_clause(event_names)})
AND trace_id = {{trace_id}}
AND timestamp >= {{date_from}}
AND timestamp <= {{date_to}}
HAVING event_count > 0
"""
_SKIP_REASONING = {
"trace_not_found": "No trace events were found within the evaluation window; evaluation skipped.",
"trace_too_large": (
f"Trace exceeds {MAX_TRACE_EVAL_EVENTS} events — likely a shared or runaway trace id; evaluation skipped."
),
}
@dataclass
class RunTraceEvaluationInputs:
evaluation_id: str
team_id: int
trace_id: str
distinct_id: str
session_id: str | None = None
window_seconds: int = TRACE_EVAL_DEFAULT_WINDOW_SECONDS
@property
def properties_to_log(self) -> dict[str, Any]:
"""Properties for PostHogClientInterceptor error capture."""
return {
"evaluation_id": self.evaluation_id,
"team_id": self.team_id,
"trace_id": self.trace_id,
}
@frozen
class ExecuteTraceEvaluationInputs:
evaluation: dict[str, Any]
team_id: int
trace_id: str
window_start: str
# Upper bound of the fetch, ISO. Unset on a live run, which reads up to now; a backfilled run
# sets it so an old unit is graded over the same span the live path would have covered.
window_end: str | None = None
backfill_id: str | None = None
@property
def window_end_datetime(self) -> datetime | None:
return datetime.fromisoformat(self.window_end) if self.window_end else None
@property
def properties_to_log(self) -> dict[str, Any]:
return {
"team_id": self.team_id,
"evaluation_id": self.evaluation.get("id"),
"trace_id": self.trace_id,
}
@dataclass
class TraceFetchOutcome:
trace: LLMTrace | None
skip_reason: str | None
event_count: int
def _count_trace_events(
team: Team,
trace_id: str,
date_from: datetime,
date_to: datetime,
*,
event_names: tuple[str, ...] = _TRACE_EVENT_NAMES,
user: "User | None" = None,
) -> int:
result = query_ai_events(
query=parse_select(_trace_event_count_sql(event_names)),
placeholders={
"trace_id": ast.Constant(value=trace_id),
"date_from": ast.Constant(value=date_from),
"date_to": ast.Constant(value=date_to),
},
team=team,
user=user,
query_type="TraceEvaluationEventCount",
bypass_property_access_control=user is None,
posthog_tables_only=True,
fall_back_to_events=True,
)
if not result.results:
return 0
return int(result.results[0][0])
_TRACE_PAYLOAD_BYTES_SQL = f"""
SELECT {PAYLOAD_BYTES_EXPR} AS payload_bytes
FROM posthog.ai_events AS ai_events
WHERE event IN ({_event_names_clause(_TRACE_EVENT_NAMES)})
AND trace_id = {{trace_id}}
AND timestamp >= {{date_from}}
AND timestamp <= {{date_to}}
"""
def _sum_trace_payload_bytes(
team: Team, trace_id: str, date_from: datetime, date_to: datetime, *, user: "User | None" = None
) -> int:
result = query_ai_events(
query=parse_select(_TRACE_PAYLOAD_BYTES_SQL),
placeholders={
"trace_id": ast.Constant(value=trace_id),
"date_from": ast.Constant(value=date_from),
"date_to": ast.Constant(value=date_to),
},
team=team,
user=user,
query_type="TraceEvaluationPayloadBytes",
bypass_property_access_control=user is None,
posthog_tables_only=True,
fall_back_to_events=False,
)
if not result.results:
return 0
return int(result.results[0][0] or 0)
def _read_missed_events(
team: Team, trace: LLMTrace, date_from: datetime, date_to: datetime, *, user: "User | None" = None
) -> bool:
"""Decide whether a trace that came back with no events was read in full.
A renderable event the fetch did not return means `ai_events` served a partial trace. This count
falls back to the shared events table when `ai_events` holds no renderable row, so it finds the
events the fetch did not see. Every evaluation would grade the wrong unit without them.
"""
return (
_count_trace_events(team, trace.id, date_from, date_to, event_names=_RENDERABLE_TRACE_EVENT_NAMES, user=user)
> 0
)
def _fetch_trace(
team: Team,
trace_id: str,
date_from: datetime,
date_to: datetime,
*,
bound_to_date_to: bool = False,
user: "User | None" = None,
) -> TraceFetchOutcome:
"""Fetch a single full trace from ClickHouse over an explicit window, with a cheap count
preflight so degenerate traces are skipped before pulling their payload.
`bound_to_date_to` grades the trace as of `date_to`. A live run leaves it off and reads the
whole trace, which is what it graded before backfills existed."""
event_count = _count_trace_events(team, trace_id, date_from, date_to, user=user)
if event_count == 0:
return TraceFetchOutcome(trace=None, skip_reason="trace_not_found", event_count=0)
if event_count > MAX_TRACE_EVAL_EVENTS:
return TraceFetchOutcome(trace=None, skip_reason="trace_too_large", event_count=event_count)
# Log-only for trace. `should_skip_for_payload` returns False for this target by design: trace
# evaluations already run in production, and enforcing a brand-new dimension on them would start
# skipping units that grade fine today. The metric supplies the distribution needed to set a
# trace budget with evidence, and flipping it to enforcing is a deliberate follow-up.
should_skip_for_payload(
target="trace",
payload_bytes=_sum_trace_payload_bytes(team, trace_id, date_from, date_to, user=user),
budget_bytes=payload_budget_bytes(JUDGE_TRACE_MAX_CHARS),
)
runner = TraceQueryRunner(
team=team,
user=user,
query=TraceQuery(
traceId=trace_id,
dateRange=DateRange(date_from=date_from.isoformat(), date_to=date_to.isoformat()),
tags=QueryLogTags(productKey="AIObservability"),
),
# Without this the runner returns the whole trace whatever `dateRange` says, so a backfilled
# run would grade events that the live run never saw, and its totals would report cost and
# latency from them. The runner applies the upper bound only. A lower bound would cut off the
# early events of the trace, which live runs do grade.
bound_events_to_date_range=bound_to_date_to,
posthog_tables_only=True,
bypass_property_access_control=user is None,
)
response = runner.calculate()
if not response.results:
return TraceFetchOutcome(trace=None, skip_reason="trace_not_found", event_count=event_count)
trace = response.results[0]
# The count preflight above includes the `$ai_trace` root row, which the runner drops from
# `events`, so a non-zero count does not promise that the fetch saw the whole trace.
if not trace.events and _read_missed_events(team, trace, date_from, date_to, user=user):
return TraceFetchOutcome(trace=None, skip_reason="trace_not_found", event_count=event_count)
return TraceFetchOutcome(trace=trace, skip_reason=None, event_count=event_count)
def fetch_trace_for_evaluation(
team_id: int, trace_id: str, window_start: datetime, window_end: datetime | None = None
) -> TraceFetchOutcome:
"""Fetch the full trace for an online evaluation run, looking back from the workflow start."""
team = Team.objects.get(id=team_id)
date_from = window_start - TRACE_EVENTS_LOOKBACK
date_to = window_end or datetime.now(UTC)
return _fetch_trace(team, trace_id, date_from, date_to, bound_to_date_to=window_end is not None)
@frozen
class TraceHogTestResult:
"""One trace's outcome from `run_hog_eval_over_recent_traces`, shaped for the editor test
endpoint and Max's authoring tool rather than for online emission."""
trace_id: str
verdict: bool | None
reasoning: str
error: str | None
input_preview: str
output_preview: str
score: float | None = None
categories: list[str] | None = None
applicable: bool | None = None
@dataclass(frozen=True)
class TraceHogTestSample:
trace_id: str
trigger_timestamp: datetime
def _coerce_trace_test_timestamp(value: Any) -> datetime:
if isinstance(value, datetime):
timestamp = value
elif isinstance(value, str):
timestamp = datetime.fromisoformat(value.replace("Z", "+00:00"))
else:
raise TypeError(f"Unexpected trace sample timestamp type: {type(value).__name__}")
return timestamp if timestamp.tzinfo is not None else timestamp.replace(tzinfo=UTC)
def _sample_recent_traces(
team: Team,
condition_filter: ast.Expr | None,
sample_count: int,
date_from: datetime,
date_to: datetime,
*,
user: "User | None" = None,
) -> list[TraceHogTestSample]:
"""Pick the most recent distinct trace ids whose *triggering* generation matches the
evaluation's conditions — mirroring the online scheduler, which starts a trace-eval run
from the first matching `$ai_generation`. Historical previews use that event timestamp as
the closest available proxy for the workflow start time."""
where_exprs: list[ast.Expr] = [
ast.CompareOperation(
op=ast.CompareOperationOp.In,
left=ast.Field(chain=["event"]),
right=ast.Constant(value=["$ai_generation"]),
),
ast.CompareOperation(
op=ast.CompareOperationOp.NotEq,
left=ast.Field(chain=["trace_id"]),
right=ast.Constant(value=""),
),
ast.CompareOperation(
op=ast.CompareOperationOp.GtEq,
left=ast.Field(chain=["timestamp"]),
right=ast.Constant(value=date_from),
),
ast.CompareOperation(
op=ast.CompareOperationOp.LtEq,
left=ast.Field(chain=["timestamp"]),
right=ast.Constant(value=date_to),
),
]
if condition_filter is not None:
where_exprs.append(condition_filter)
query = ast.SelectQuery(
select=[
ast.Field(chain=["trace_id"]),
ast.Alias(alias="trigger_timestamp", expr=ast.Call(name="min", args=[ast.Field(chain=["timestamp"])])),
ast.Alias(alias="latest", expr=ast.Call(name="max", args=[ast.Field(chain=["timestamp"])])),
],
select_from=ast.JoinExpr(table=ast.Field(chain=["posthog", "ai_events"]), alias="ai_events"),
where=ast.Placeholder(expr=ast.Field(chain=["where_clause"])),
group_by=[ast.Field(chain=["trace_id"])],
order_by=[ast.OrderExpr(expr=ast.Field(chain=["latest"]), order="DESC")],
limit=ast.Constant(value=sample_count),
)
response = query_ai_events(
query=query,
placeholders={"where_clause": ast.And(exprs=where_exprs)},
team=team,
user=user,
query_type="EvaluationTestHogTraceSample",
fall_back_to_events=True,
)
return [
TraceHogTestSample(trace_id=str(row[0]), trigger_timestamp=_coerce_trace_test_timestamp(row[1]))
for row in (response.results or [])
]
def _trace_io_preview(trace: LLMTrace) -> tuple[str, str]:
"""First non-empty input and last non-empty output across the trace's events, for a compact
preview. The Hog body still sees the full trace globals; this is only for display."""
input_preview = ""
output_preview = ""
for event in trace.events or []:
io = extract_event_io(event.event, event.properties)
if not input_preview:
input_preview = extract_text_from_messages(io.input_raw)[:200]
output_text = extract_text_from_messages(io.output_raw)[:200]
if output_text:
output_preview = output_text
return input_preview, output_preview
def run_hog_eval_over_recent_traces(
*,
team: Team,
bytecode: list[Any],
condition_filter: ast.Expr | None,
sample_count: int,
allows_na: bool,
window_seconds: int = TRACE_EVAL_DEFAULT_WINDOW_SECONDS,
lookback_days: int = EVALUATION_TEST_LOOKBACK_DAYS,
output_type: str = "boolean",
output_config: dict[str, Any] | None = None,
user: "User | None" = None,
) -> list[TraceHogTestResult]:
"""Sample recent traces matching the conditions and run trace-level Hog bytecode against each.
Shared by the editor `test_hog` endpoint and Max's authoring tool so a trace-target eval is
previewed the same way it runs online — whole trace, trace globals — instead of against a
single generation.
"""
now = datetime.now(UTC)
window = timedelta(seconds=window_seconds)
# Only sample triggers whose full aggregation window has elapsed.
sample_date_to = now - window
sample_date_from = sample_date_to - timedelta(days=lookback_days)
trace_samples = _sample_recent_traces(
team,
condition_filter,
sample_count,
sample_date_from,
sample_date_to,
user=user,
)
results: list[TraceHogTestResult] = []
for sample in trace_samples:
trace_date_from = sample.trigger_timestamp - TRACE_EVENTS_LOOKBACK
trace_date_to = sample.trigger_timestamp + window
outcome = _fetch_trace(team, sample.trace_id, trace_date_from, trace_date_to, user=user)
if outcome.skip_reason or outcome.trace is None:
results.append(
TraceHogTestResult(
trace_id=sample.trace_id,
verdict=None,
reasoning="",
error=_SKIP_REASONING.get(outcome.skip_reason or "trace_not_found", "Evaluation skipped."),
input_preview="",
output_preview="",
)
)
continue
globals_dict = build_trace_hog_globals(outcome.trace, sample.trace_id, bytecode=bytecode)
result = execute_hog_eval_bytecode(
bytecode, globals_dict, allows_na=allows_na, output_type=output_type, output_config=output_config
)
input_preview, output_preview = _trace_io_preview(outcome.trace)
results.append(
TraceHogTestResult(
trace_id=sample.trace_id,
verdict=result.get("verdict"),
score=result.get("score"),
categories=result.get("categories"),
applicable=result.get("applicable"),
reasoning=result["reasoning"],
error=result["error"],
input_preview=input_preview,
output_preview=output_preview,
)
)
return results
def _build_trace_skip_result(
allows_na: bool, skip_reason: str, *, output_type: str = "boolean"
) -> EvaluationActivityResult:
"""Mirror of `_build_errored_trace_result` for trace-level skips — no LLM call is made,
so model/provider are omitted and downstream cost attribution stays clean."""
return build_skipped_evaluation_result(
output_type=output_type,
allows_na=allows_na,
reasoning=_SKIP_REASONING.get(skip_reason, "Evaluation skipped."),
skip_reason=skip_reason,
)
def build_trace_system_prompt(
prompt: str, allows_na: bool, *, output_type: str = "boolean", output_config: dict[str, Any] | None = None
) -> str:
"""Trace-level variant of `build_system_prompt` — frames the unit under evaluation as the
whole trace rather than a single generation."""
config = get_output_type_config(allows_na, output_type=output_type, output_config=output_config)
return f"""You are an evaluator. Evaluate the following AI trace — the full sequence of LLM calls and operations from one execution — according to this criteria:
{prompt}
{config.instructions}"""
def format_trace_for_judge(trace: LLMTrace, media_marker: MediaMarker | None = None) -> str:
"""Serialize a trace into the canonical text representation for the LLM judge.
Preserve complete generation outputs before truncating them so the judge does not grade
formatting artifacts as model mistakes. Truncate input history first; use message
truncation and sampling only if the outputs still cannot fit the transcript budget.
"""
trace_dict, hierarchy = llm_trace_to_formatter_format(trace)
options: FormatterOptions = {
"include_markers": False,
"collapsed": False,
"preserve_generation_output": True,
"include_line_numbers": True,
"max_length": None,
"max_render_length": JUDGE_TRACE_MAX_CHARS,
}
if media_marker is not None:
options["media_marker"] = media_marker
for truncated in (False, True):
options["truncated"] = truncated
try:
text, _ = format_trace_text_repr(trace_dict, hierarchy, options)
return text
except RenderBudgetExceeded:
pass
del options["max_render_length"]
options["preserve_generation_output"] = False
options["max_length"] = JUDGE_TRACE_MAX_CHARS
text, _ = format_trace_text_repr(trace_dict, hierarchy, options)
return text
def _has_state_content(state: object, *, is_output: bool = False) -> bool:
# Message headers and whitespace can render without any content for the judge to grade.
if isinstance(state, list) and state and isinstance(state[0], dict):
if "role" in state[0] or "content" in state[0]:
return has_message_content(state, is_output=is_output)
return bool(state.strip()) if isinstance(state, str) else bool(state)
def _has_judge_transcript(trace: LLMTrace) -> bool:
"""Whether the trace formats into something the LLM judge can read.
`format_trace_text_repr` renders the trace-level input and output only when the event hierarchy
is empty, so a trace with neither formats down to its name alone and the judge grades nothing.
A Hog eval has no such requirement: it reads trace-level cost and latency straight off the root
event, so this gate belongs to the judge rather than to the fetch.
"""
return (
bool(trace.events)
or _has_state_content(trace.inputState)
or _has_state_content(trace.outputState, is_output=True)
)
def build_trace_hog_globals(trace: LLMTrace, trace_id: str, *, bytecode: list[Any] | None = None) -> dict[str, Any]:
"""Build Hog globals for a trace-level eval.
`events` and `trace` keep their original shapes for saved trace Hog source. New target-independent
source uses `evaluation_events`, which adds readable text projections, and `target`.
Existing bytecode receives the original globals. Bytecode that uses a new shared global only
builds the event collections it references, avoiding duplicate HogVM and worker memory cost.
"""
trace_events = trace.events or []
references_target = bytecode is not None and hog_bytecode_references_global(bytecode, "target")
references_events = bytecode is not None and hog_bytecode_references_global(bytecode, "events")
references_evaluation_events = bytecode is not None and hog_bytecode_references_global(
bytecode, "evaluation_events"
)
can_skip_compatibility_events = (references_target or references_evaluation_events) and not references_events
globals_dict: dict[str, Any] = {
"trace": {"id": trace_id, "event_count": len(trace_events)},
}
if bytecode is None or references_target:
globals_dict["target"] = {
"type": "trace",
"id": trace_id,
"total_cost_usd": trace.totalCost,
"total_latency_seconds": trace.totalLatency,
}
if bytecode is None or not can_skip_compatibility_events:
globals_dict["events"] = [
build_hog_event_global(
event.event,
event.properties,
event_uuid=event.id,
timestamp=event.createdAt,
include_text=False,
)
for event in trace_events
]
if bytecode is None or references_evaluation_events:
globals_dict["evaluation_events"] = [
build_hog_event_global(
event.event,
event.properties,
event_uuid=event.id,
timestamp=event.createdAt,
)
for event in trace_events
]
return globals_dict
@temporalio.activity.defn
@close_db_connections
# capture_exceptions=False: the worker interceptor reports judge failures, and its capture carries
# the team and evaluation ids. A capture inside the activity wins the SDK's dedupe and loses them.
@posthoganalytics.scoped(capture_exceptions=False)
def execute_trace_llm_judge_activity(inputs: ExecuteTraceEvaluationInputs) -> EvaluationActivityResult:
"""Fetch the whole trace and run the LLM judge over its transcript.
Fetch and judge happen in one activity on purpose: returning the trace through the
workflow would hit Temporal's ~2 MiB payload limit on large traces.
"""
evaluation = inputs.evaluation
if evaluation["evaluation_type"] != "llm_judge":
raise ApplicationError(
f"Unsupported evaluation type: {evaluation['evaluation_type']}",
non_retryable=True,
)
prompt = evaluation.get("evaluation_config", {}).get("prompt")
if not prompt:
raise ApplicationError("Missing prompt in evaluation_config", non_retryable=True)
if evaluation["output_type"] not in ("boolean", "numeric", "categorical"):
raise ApplicationError(
f"Unsupported output type: {evaluation['output_type']}. Supported types: 'boolean', 'numeric', 'categorical'.",
non_retryable=True,
)
output_type = evaluation.get("output_type", "boolean")
output_config = evaluation.get("output_config") or {}
allows_na = output_config.get("allows_na", False)
outcome = fetch_trace_for_evaluation(
inputs.team_id, inputs.trace_id, datetime.fromisoformat(inputs.window_start), inputs.window_end_datetime
)
if outcome.skip_reason or outcome.trace is None:
return _build_trace_skip_result(allows_na, outcome.skip_reason or "trace_not_found", output_type=output_type)
if not _has_judge_transcript(outcome.trace):
return _build_trace_skip_result(allows_na, "trace_not_found", output_type=output_type)
trace = outcome.trace
user_prompt = format_trace_for_judge(trace)
return call_llm_judge(
evaluation=evaluation,
system_prompt=build_trace_system_prompt(
prompt, allows_na, output_type=output_type, output_config=output_config
),
user_prompt=user_prompt,
allows_na=allows_na,
backfill=inputs.backfill_id is not None,
attachment_prompt=build_attachment_prompt(
evaluation, user_prompt, lambda media_marker: format_trace_for_judge(trace, media_marker)
),
)
@temporalio.activity.defn
async def execute_trace_hog_eval_activity(inputs: ExecuteTraceEvaluationInputs) -> EvaluationActivityResult:
"""Fetch the whole trace and run Hog bytecode against trace-level globals."""
evaluation = inputs.evaluation
if evaluation["evaluation_type"] != "hog":
raise ApplicationError(
f"Unsupported evaluation type: {evaluation['evaluation_type']}",
non_retryable=True,
)
bytecode = evaluation.get("evaluation_config", {}).get("bytecode")
if not bytecode:
raise ApplicationError("Missing bytecode in evaluation_config", non_retryable=True)
output_type = evaluation.get("output_type", "boolean")
output_config = evaluation.get("output_config") or {}
allows_na = output_config.get("allows_na", False)
def _execute() -> tuple[dict[str, Any] | None, str | None]:
outcome = fetch_trace_for_evaluation(
inputs.team_id, inputs.trace_id, datetime.fromisoformat(inputs.window_start), inputs.window_end_datetime
)
if outcome.skip_reason or outcome.trace is None:
return None, outcome.skip_reason or "trace_not_found"
globals_dict = build_trace_hog_globals(outcome.trace, inputs.trace_id, bytecode=bytecode)
return execute_hog_eval_bytecode(
bytecode, globals_dict, allows_na=allows_na, output_type=output_type, output_config=output_config
), None
result, skip_reason = await database_sync_to_async(_execute, thread_sensitive=False)()
if skip_reason or result is None:
return _build_trace_skip_result(allows_na, skip_reason or "trace_not_found", output_type=output_type)
return finalize_hog_eval_result(result, evaluation=evaluation, allows_na=allows_na, unit_label="trace")
@frozen
class EmitTraceEvaluationEventInputs:
evaluation: dict[str, Any]
team_id: int
trace_id: str
distinct_id: str
session_id: str | None
result: EvaluationActivityResult
start_time: datetime
target: str = "trace"
ai_session_id: str | None = None
backfill_id: str | None = None
event_timestamp: str | None = None
@property
def properties_to_log(self) -> dict[str, Any]:
return {
"team_id": self.team_id,
"evaluation_id": self.evaluation.get("id"),
"trace_id": self.trace_id,
"target": self.target,
}
@temporalio.activity.defn
async def emit_trace_evaluation_event_activity(inputs: EmitTraceEvaluationEventInputs) -> None:
"""Emit the $ai_evaluation event for a trace-level run, targeting the trace id."""
def _emit():
# No single source event to inherit from, so SOURCE_AI_PROPERTIES_TO_COPY (span/parent
# linkage copied in the generation path) intentionally does not apply here.
properties = build_evaluation_event_properties(
inputs.evaluation, inputs.result, inputs.start_time, inputs.backfill_id
)
if inputs.target == "session" and inputs.ai_session_id:
properties.update(
{
"$ai_target_id": inputs.ai_session_id,
"$ai_target_type": "session_id",
# Makes the verdict session-scoped so the session view can read it. No
# $ai_trace_id: the verdict belongs to the session, not to any one trace, and
# inventing a trace id would make it render as that trace's verdict.
"$ai_session_id": inputs.ai_session_id,
"$session_id": inputs.session_id,
}
)
else:
properties.update(
{
"$ai_target_id": inputs.trace_id,
"$ai_target_type": "trace_id",
# The eval event carries the trace id itself so it shows up inside the trace view.
"$ai_trace_id": inputs.trace_id,
"$session_id": inputs.session_id,
}
)
# A backfilled verdict sits at its unit's own time so time-bucketed views line it up with
# the trace it grades instead of with the day the backfill ran. The dispatcher always
# sends the anchor and the backfill id together, so there is no third case.
if inputs.event_timestamp and inputs.backfill_id:
timestamp = backfill_verdict_timestamp(
as_utc_datetime(inputs.event_timestamp),
str(inputs.evaluation["id"]),
inputs.backfill_id,
str(properties["$ai_target_id"]),
)
else:
timestamp = datetime.now(UTC)
capture_ai_internal_for_team(
team_id=inputs.team_id,
event_name="$ai_evaluation",
event_source="llm_analytics_evaluation",
distinct_id=inputs.distinct_id,
timestamp=timestamp,
properties=properties,
)
# Completed runs emit telemetry in the workflow's separate activity.
if inputs.result.get("skipped"):
try:
capture_evaluation_run_usage(inputs.evaluation, inputs.result, team_id=inputs.team_id)
except Exception:
# Telemetry failures must not retry an already emitted evaluation.
logger.warning("evaluation_usage_capture_failed", team_id=inputs.team_id, exc_info=True)
try:
await database_sync_to_async(_emit, thread_sensitive=False)()
increment_emit_event_outcome("success")
except CaptureInternalError as e:
if e.is_billing_limit_exceeded:
increment_emit_event_outcome("dropped_billing_limited")
logger.info("Skipping eval event emission; team over billing quota", team_id=inputs.team_id)
return
increment_emit_event_outcome("failed")
raise_emit_failure(e)
except Exception as e:
increment_emit_event_outcome("failed")
raise_emit_failure(e)
@temporalio.workflow.defn(name="run-trace-evaluation")
class RunTraceEvaluationWorkflow(PostHogWorkflow):
@staticmethod
def parse_inputs(inputs: list[str]) -> RunTraceEvaluationInputs:
return RunTraceEvaluationInputs(**json.loads(inputs[0]))
@temporalio.workflow.run
async def run(self, inputs: RunTraceEvaluationInputs) -> WorkflowResult:
# Every #71518-era run recorded the "remove-trial-evals" patch marker; accept those
# histories for one release, then delete this call (temporal-workflow-versioning rule).
temporalio.workflow.deprecate_patch("remove-trial-evals")
window_start = temporalio.workflow.now()
# Wait for the rest of the trace to arrive. Timers are server-side, so sleeping
# workflows don't hold worker slots.
window_seconds = min(max(inputs.window_seconds, 0), TRACE_EVAL_MAX_WINDOW_SECONDS)
if window_seconds:
await asyncio.sleep(window_seconds)
eval_start = temporalio.workflow.now()
evaluation = await temporalio.workflow.execute_activity(
fetch_evaluation_activity,
RunEvaluationInputs(evaluation_id=inputs.evaluation_id, event_data={"team_id": inputs.team_id}),
schedule_to_close_timeout=timedelta(seconds=30),
retry_policy=RetryPolicy(maximum_attempts=3),
)
evaluation_type = evaluation.get("evaluation_type", "llm_judge")
# The evaluation may have been paused or deleted during the aggregation window —
# bail out instead of running against config the user just turned off.
if evaluation["deleted"] or not evaluation["enabled"]:
return build_skipped_workflow_result(
evaluation=evaluation,
evaluation_type=evaluation_type,
skip_reason="evaluation_deleted" if evaluation["deleted"] else "evaluation_disabled",
)
execute_inputs = ExecuteTraceEvaluationInputs(
evaluation=evaluation,
team_id=inputs.team_id,
trace_id=inputs.trace_id,
window_start=window_start.isoformat(),
)
if evaluation_type == "hog":
# Unlike single-event hog evals, this activity includes a ClickHouse fetch, so
# allow one retry for transient query failures (the bytecode is deterministic).
result = await temporalio.workflow.execute_activity(
execute_trace_hog_eval_activity,
execute_inputs,
schedule_to_close_timeout=timedelta(minutes=2),
retry_policy=RetryPolicy(maximum_attempts=2),
)
else:
try:
result = await temporalio.workflow.execute_activity(
execute_trace_llm_judge_activity,
execute_inputs,
# > single-event judge timeout: the activity also fetches the trace from ClickHouse
schedule_to_close_timeout=timedelta(minutes=8),
retry_policy=LLM_JUDGE_RETRY_POLICY,
)
except temporalio.exceptions.ActivityError as e:
handled = await handle_llm_judge_activity_error(e, evaluation, evaluation_type)
if handled is not None:
return handled
raise
if is_terminal_user_error_result(result):
return await handle_terminal_user_error_result(
evaluation=evaluation,
evaluation_type=evaluation_type,
result=result,
)
try:
await temporalio.workflow.execute_activity(
emit_trace_evaluation_event_activity,
EmitTraceEvaluationEventInputs(
evaluation=evaluation,
team_id=inputs.team_id,
trace_id=inputs.trace_id,
distinct_id=inputs.distinct_id,
session_id=inputs.session_id,
result=result,
start_time=eval_start,
),
start_to_close_timeout=EMIT_EVENT_START_TO_CLOSE_TIMEOUT,
schedule_to_close_timeout=EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT,
retry_policy=EMIT_EVENT_RETRY_POLICY,
)
except Exception:
increment_errors("emit_evaluation_event_failed", provider=result.get("provider"))
raise
if not result.get("skipped"):
await temporalio.workflow.execute_activity(
emit_internal_telemetry_activity,
EmitInternalTelemetryInputs(
evaluation=evaluation,
team_id=inputs.team_id,
result=result,
),
schedule_to_close_timeout=timedelta(seconds=30),
)
workflow_result: WorkflowResult = {
"reasoning": result["reasoning"],
"evaluation_id": evaluation["id"],
"evaluation_type": evaluation_type,
"is_byok": result.get("is_byok", False),
"skipped": result.get("skipped", False),
}
if "verdict" in result:
workflow_result["verdict"] = result["verdict"]
if "score" in result:
workflow_result["score"] = result["score"]
if "categories" in result:
workflow_result["categories"] = result["categories"]
if result.get("skipped"):
skip_reason = result.get("skip_reason")
if skip_reason is not None:
workflow_result["skip_reason"] = skip_reason
return workflow_result