Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): serve trace evaluation results from an evaluations resource (#121264)

Carlos Marchal44caa527ed39
diff --git a/.github/new-events-schema-targets.txt b/.github/new-events-schema-targets.txt
index 21fc710e4fba..f995fa0285d1 100644
--- a/.github/new-events-schema-targets.txt
+++ b/.github/new-events-schema-targets.txt
@@ -25,6 +25,8 @@ posthog/clickhouse/test
ee/clickhouse
products/ai_observability/backend/tests/logic/evaluations
products/ai_observability/backend/tests/logic/traces
+products/ai_observability/backend/tests/test_facade_api.py
+products/ai_observability/backend/tests/test_trace_views.py
products/cohorts
products/customer_analytics/backend/hogql_queries
products/error_tracking/backend/hogql_queries
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 6c7c9c047751..ac6bdf1d4429 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -9155,17 +9155,17 @@ snapshots:
scenes-app-ai-observability-trace-view-eval-results--loading--light:
hash: v1.k794b7964.c2193c953b9f6410a018135730f03a16effbae102e843e7cb86d31a585ff25a0.ZnwmN5ZhvryitB3fPLZmzGRm31kQtra85ZjR7rR2Wdg
scenes-app-ai-observability-trace-view-eval-results--narrow--dark:
- hash: v1.k794b7964.0d9f5ea29085fdce3b5416a6cd7629cf7487718b3287170e35a0444ce84f7130.wjDaCjEydB47et5XF4RnA6PcbBO28LdcXTXUdgJkqOg
+ hash: v1.k794b7964.c180c03264d4b4dd70fc1d8ffe79d4793914955f7c3ff4b2b0baea6800789935.DeEtCZVUgqNNRgneYw2hJ9aC_SKsli-b515v_sX1eJ8
scenes-app-ai-observability-trace-view-eval-results--narrow--light:
- hash: v1.k794b7964.066e10797c7eeef745043b8e1c5bd2f6f726db2dc927ffabef178c65f92ae47c.lcwsAGwJT5CDGqDhrFvHUGPF8K-C_VpfqDlMnf1v8yM
+ hash: v1.k794b7964.aff05e292bff56b5de4d3375e7d3029cdf436ff67671fc31302cc7b969f08322.DChS6K6n3sd7HCl9EcQ7pRcWh65hAY4zhAt2q5bD9pw
scenes-app-ai-observability-trace-view-eval-results--results--dark:
- hash: v1.k794b7964.6878e01169aa5df1d0a472e54502445cb4dac02f049402c42bd77405678202ad.joUtOO1BtYJsiG77d2Oor5OLT2b0ojMFPXxBUcvMV4w
+ hash: v1.k794b7964.7d4e5284a5dd4eaaf6ba53e25fb91fc10df27d98d044a8707fed755869fac300.tXQXVBQUsx6yiC3RlpEWlOAxWQXoys_i3TtiI9BmGaE
scenes-app-ai-observability-trace-view-eval-results--results--light:
- hash: v1.k794b7964.28feede2cc8e1490cacc3434a878109a1b3e891c5d335fa4cfd5e8b2c12a7d4c.1hm9PJZxIAjDh8vetWyBPYQds9ck3Vwu0nJdGafQNnQ
+ hash: v1.k794b7964.dbe4f1d49b4c48fbd58d59105298f3543459aaa4ebef54ffa342b9d2e97302ed.InRPlKGDDdjouYufMdxD5brZbWnsjQFrXvaiHLoQHlg
scenes-app-ai-observability-trace-view-eval-results--with-targets--dark:
- hash: v1.k794b7964.3668c4cab9a050d2d21e9cd1274b5e348da24fedf3264ede6cc9a19c59be0eae.8Zc5q6EnyO2a8nC1KKbD86vECGtDU-mLzxroEIqaQks
+ hash: v1.k794b7964.bb8fdbede3c8fc1006a9a8f5d3be808341f36237c56c38f3244e11cd248d47ef.4RE1N5YlNm27-UwPmtDgtDifGqEhzT89XUXQrWSC59w
scenes-app-ai-observability-trace-view-eval-results--with-targets--light:
- hash: v1.k794b7964.5b99a44a4fe9414c181d52304bee27dc0411488565499a069379edfca998ac53.xqNk1JI7RXzSiB36VQL2Tx8jNZjLqTB3V3y-7UVMqLw
+ hash: v1.k794b7964.2561e19f756b8b0c9c54e3926d2740a17edefed875f822f5c40db6e9350d0e99.qy2zcYWPbI9yn-QMrXugQISB2l1hJcsLAIkCdcjm_A0
scenes-app-ai-observability-trace-view-input-and-output--empty-object--dark:
hash: v1.k794b7964.6ed3f97001d8b595545a78f37627d3e09c188a3dcd782c8ab3389c65cbd4210f.2U9W7fXNr-OZRXfkmXy1ICFu5-dSKZUsMMWOtbf4PWw
scenes-app-ai-observability-trace-view-input-and-output--empty-object--light:
@@ -9287,9 +9287,9 @@ snapshots:
scenes-app-ai-observability-trace-view-node-detail--evals-load-error--light:
hash: v1.k794b7964.50777b9d2294d5daefde05454f4d88c77100bc9cf1bb00cba7417e45659441de.zNzJetAUSwx_LsQxfGzn5GiP0drok5qlGDjtdBA1f74
scenes-app-ai-observability-trace-view-node-detail--evals-tab--dark:
- hash: v1.k794b7964.b32052d233df672ac33d635e9e60682b279b0a85b8197d5c037923f1060ecb67.pjxZIIGAyjrunK5zd2J1auEwmwS2okFOGDJHtPnCBS0
+ hash: v1.k794b7964.f6d7647543214d678b4ba05b43ed8d4e6fa2d2ba949845252c5e357831b6f8a1.QXLvkyt9b16kwXPe_oYPlGWeHWI-dnm2zsRyLmlyH9Y
scenes-app-ai-observability-trace-view-node-detail--evals-tab--light:
- hash: v1.k794b7964.56c0cd429a52b1707971a0d892f22346a5757ed5eb5bd2e576305ab85d5daaf5.HHwjHccgpak73NKgv5Gfmd5RhUWYNgNe1x4qE6iJxDw
+ hash: v1.k794b7964.83491c8cf61cb4d613f16ad9ef759f959b4771365e890f956d5e15c10ee9c98d.UKEhF6lfSpzMVVnf3fEqWcoThCHvNqV8hWjmsC92TpM
scenes-app-ai-observability-trace-view-node-detail--failed-generation-with-image--dark:
hash: v1.k794b7964.c161e39c0753e90108d0026cbc0bdc6963e8540346685edf024fd8dedf32ec4c.A2Eid0pxBKQqtQ4wohIT_9dHMqvB3fQB5UigMmuDLDA
scenes-app-ai-observability-trace-view-node-detail--failed-generation-with-image--light:
diff --git a/frontend/src/initKea.ts b/frontend/src/initKea.ts
index a4190a8ea0ce..22afad8e652f 100644
--- a/frontend/src/initKea.ts
+++ b/frontend/src/initKea.ts
@@ -127,6 +127,7 @@ const ERROR_FILTER_ALLOW_LIST_BY_LOGIC_PATH: Record<string, string> = {
createProviderKey: 'products.ai_observability.settings.llmProviderKeysLogic',
updateProviderKey: 'products.ai_observability.settings.llmProviderKeysLogic',
deleteProviderKey: 'products.ai_observability.settings.llmProviderKeysLogic',
+ loadEvaluationResults: 'products.ai_observability.frontend.redesign.trace.traceViewLogic', // The trace view's Evals tab shows the error
}
// A keyed logic's path ends with its key, so match the owner path as a prefix.
diff --git a/products/ai_observability/backend/facade/api.py b/products/ai_observability/backend/facade/api.py
index 6c012cfd5202..faa371330d0a 100644
--- a/products/ai_observability/backend/facade/api.py
+++ b/products/ai_observability/backend/facade/api.py
@@ -3,6 +3,8 @@ from datetime import datetime
from posthog.models import Team, User
from ..logic.evaluations.pass_filters import build_evaluation_pass_filters
+from ..logic.traces.evaluation_result import EvaluationResult
+from ..logic.traces.evaluation_result_queries import load_evaluation_results, load_output_configs
from ..logic.traces.events_fallback import fallback_window
from ..logic.traces.node import NodeEvent
from ..logic.traces.timeline import TimelineRow
@@ -155,3 +157,32 @@ def get_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilters
or categorical passing rule, and no skipped, N/A or sentiment runs. The conditions change when an evaluation's
output config changes, so a caller that stores them rebuilds them on `evaluation_output_changed`."""
return build_evaluation_pass_filters(team_id)
+
+
+def get_trace_evaluation_results(team: Team, user: User | None, trace_id: str) -> contracts.TraceEvaluationResults:
+ rows = load_evaluation_results(team, user, trace_id)
+ configs = load_output_configs(team, user, {row.evaluation_id for row in rows if row.evaluation_id})
+ return contracts.TraceEvaluationResults(
+ results=[
+ _to_evaluation_result(
+ EvaluationResult.from_row(
+ row, trace_id=trace_id, output_config=configs.get(row.evaluation_id or "", {})
+ )
+ )
+ for row in rows
+ ]
+ )
+
+
+def _to_evaluation_result(result: EvaluationResult) -> contracts.TraceEvaluationResult:
+ return contracts.TraceEvaluationResult(
+ id=result.id,
+ evaluation_id=result.evaluation_id,
+ evaluation_name=result.name,
+ target_node_id=result.target_node_id,
+ outcome=result.outcome,
+ label=result.label,
+ reasoning=result.reasoning,
+ timestamp=result.timestamp,
+ is_backfill=result.is_backfill,
+ )
diff --git a/products/ai_observability/backend/facade/contracts.py b/products/ai_observability/backend/facade/contracts.py
index 9f1669cad03e..d7eac52720d5 100644
--- a/products/ai_observability/backend/facade/contracts.py
+++ b/products/ai_observability/backend/facade/contracts.py
@@ -109,3 +109,24 @@ class EvaluationPassFilters(Contract):
graded: str # The run counts toward a pass rate.
passed: str # The run passed. Implies `graded`.
+
+
+EvaluationOutcome = Literal["pass", "fail", "inconclusive", "unrated"]
+
+
+class TraceEvaluationResult(Contract):
+ id: str
+ evaluation_id: str | None
+ evaluation_name: str
+ target_node_id: str = Field(
+ description="The tree node the result grades: the generation's event uuid, or the trace id for a trace-level result."
+ )
+ outcome: EvaluationOutcome
+ label: str = Field(description='What the result says, such as "True", "0.82", "Positive" or "Skipped".')
+ reasoning: str | None
+ timestamp: datetime
+ is_backfill: bool
+
+
+class TraceEvaluationResults(Contract):
+ results: list[TraceEvaluationResult]
diff --git a/products/ai_observability/backend/logic/traces/evaluation_result.py b/products/ai_observability/backend/logic/traces/evaluation_result.py
new file mode 100644
index 000000000000..3e940ef859c7
--- /dev/null
+++ b/products/ai_observability/backend/logic/traces/evaluation_result.py
@@ -0,0 +1,168 @@
+import json
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Self
+
+from posthog.dataclasses import frozen
+
+from ...facade.contracts import EvaluationOutcome
+from ...models.evaluation_configs import OutputType
+from ...models.evaluation_outcomes import EvaluationReportOutcomeDefinition, get_outcome_definition
+from .evaluation_result_queries import EvaluationResultRow
+from .trace_queries import canonical_uuid, read_flag, read_number, read_text
+
+_SENTIMENT_OUTCOMES: dict[str, EvaluationOutcome] = {"positive": "pass", "neutral": "unrated", "negative": "fail"}
+UNKNOWN_EVALUATION_NAME = "Unknown evaluation"
+
+
+def _outcome_definition(
+ result_type: OutputType, output_config: Mapping[str, object]
+) -> EvaluationReportOutcomeDefinition | None:
+ # Stored configs are not revalidated on read, so a config that does not parse grades nothing.
+ try:
+ return get_outcome_definition(
+ result_type, true_is_failure=output_config.get("true_is_failure") is True, output_config=dict(output_config)
+ )
+ except ValueError:
+ return None
+
+
+def _passes(
+ definition: EvaluationReportOutcomeDefinition | None, result: object, applicable: bool | None
+) -> bool | None:
+ outcome = definition.label_for(result, applicable) if definition else None
+ return outcome == "pass" if outcome in ("pass", "fail") else None
+
+
+@frozen
+class _Display:
+ outcome: EvaluationOutcome
+ label: str
+
+
+_NOT_APPLICABLE = _Display(outcome="inconclusive", label="N/A")
+
+
+def _graded(passed: bool | None, label: str) -> _Display:
+ return _Display(outcome="unrated" if passed is None else "pass" if passed else "fail", label=label)
+
+
+def _format_score(score: float) -> str:
+ rounded = float(f"{score:.2f}") if abs(score) >= 1 else float(f"{score:.2g}")
+ return str(int(rounded)) if rounded.is_integer() else repr(rounded)
+
+
+def _result_type(raw: str | None) -> OutputType | None:
+ return OutputType(raw) if raw in OutputType.values else None
+
+
+def _parse_categories(raw: str | None, applicable: str | None) -> list[str] | None:
+ if raw is None:
+ return [] if read_flag(applicable) else None
+ try:
+ categories = json.loads(raw)
+ except ValueError:
+ return None
+ if isinstance(categories, list) and all(isinstance(category, str) for category in categories):
+ return categories
+ return None
+
+
+@frozen
+class _RawResult:
+ result_type: OutputType
+ is_sentiment: bool
+ sentiment_label: str | None
+ result: bool | None
+ applicable: bool | None
+ skipped: bool
+ score: float | None
+ categories: list[str] | None
+
+ @classmethod
+ def from_row(cls, row: EvaluationResultRow) -> Self:
+ sentiment_label = read_text(row.sentiment_label)
+ result_type = _result_type(row.result_type) or (
+ OutputType.SENTIMENT if row.evaluation_type == "sentiment" or sentiment_label else OutputType.BOOLEAN
+ )
+ not_applicable = row.applicable == "false"
+ return cls(
+ result_type=result_type,
+ is_sentiment=result_type == OutputType.SENTIMENT
+ or row.evaluation_type == "sentiment"
+ or sentiment_label is not None,
+ sentiment_label=sentiment_label,
+ result=(
+ None
+ if result_type != OutputType.BOOLEAN or not_applicable or row.result is None
+ else read_flag(row.result)
+ ),
+ applicable=None if row.applicable is None else read_flag(row.applicable),
+ skipped=read_flag(row.skipped),
+ score=read_number(row.score) if result_type == OutputType.NUMERIC and not not_applicable else None,
+ categories=(
+ _parse_categories(row.categories, row.applicable)
+ if result_type == OutputType.CATEGORICAL and not not_applicable
+ else None
+ ),
+ )
+
+ def display(self, definition: EvaluationReportOutcomeDefinition | None) -> _Display:
+ if self.skipped:
+ return _Display(outcome="inconclusive", label="Skipped")
+ if self.result_type == OutputType.CATEGORICAL:
+ if self.applicable is False:
+ return _NOT_APPLICABLE
+ if self.categories is None:
+ return _Display(outcome="inconclusive", label="No result")
+ config = definition.categorical_config if definition else None
+ names = {option.key: option.label for option in config.options} if config else {}
+ label = ", ".join(names.get(key, key) for key in self.categories)
+ return _graded(_passes(definition, self.categories, self.applicable), label or "No categories")
+ if self.result_type == OutputType.NUMERIC:
+ if self.applicable is False:
+ return _NOT_APPLICABLE
+ if self.score is None:
+ return _Display(outcome="inconclusive", label="No score")
+ passed = _passes(definition, self.score, self.applicable)
+ label = _format_score(self.score)
+ # A rounded label that would pass where the score fails, or the reverse, shows the score unrounded.
+ rounded_passes = _passes(definition, float(label), self.applicable)
+ return _graded(passed, label if rounded_passes == passed else str(self.score))
+ if self.is_sentiment:
+ sentiment = (self.sentiment_label or "unknown").lower()
+ return _Display(
+ outcome=_SENTIMENT_OUTCOMES.get(sentiment, "inconclusive"), label=sentiment[0].upper() + sentiment[1:]
+ )
+ if self.result is None:
+ return _NOT_APPLICABLE
+ return _graded(_passes(definition, self.result, self.applicable), "True" if self.result else "False")
+
+
+@frozen
+class EvaluationResult:
+ id: str
+ evaluation_id: str | None
+ name: str
+ target_node_id: str
+ outcome: EvaluationOutcome
+ label: str
+ reasoning: str | None
+ timestamp: datetime
+ is_backfill: bool
+
+ @classmethod
+ def from_row(cls, row: EvaluationResultRow, *, trace_id: str, output_config: Mapping[str, object]) -> Self:
+ raw = _RawResult.from_row(row)
+ display = raw.display(_outcome_definition(raw.result_type, output_config))
+ return cls(
+ id=row.uuid,
+ evaluation_id=row.evaluation_id if row.evaluation_id and canonical_uuid(row.evaluation_id) else None,
+ name=read_text(row.evaluation_name) or UNKNOWN_EVALUATION_NAME,
+ target_node_id=read_text(row.target_event_id) or trace_id,
+ outcome=display.outcome,
+ label=display.label,
+ reasoning=read_text(row.reasoning),
+ timestamp=row.timestamp,
+ is_backfill=bool(read_text(row.backfill_id)),
+ )
diff --git a/products/ai_observability/backend/logic/traces/evaluation_result_queries.py b/products/ai_observability/backend/logic/traces/evaluation_result_queries.py
new file mode 100644
index 000000000000..5201b4379fa5
--- /dev/null
+++ b/products/ai_observability/backend/logic/traces/evaluation_result_queries.py
@@ -0,0 +1,136 @@
+from collections.abc import Iterable, Mapping, Sequence
+from datetime import datetime
+from typing import Any
+
+from django.db.models import QuerySet
+
+from posthog.hogql import ast
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.database import Database
+from posthog.hogql.query import execute_hogql_query
+
+from posthog.clickhouse.query_tagging import Product, tag_queries, tags_context
+from posthog.dataclasses import frozen
+from posthog.models import Team, User
+
+from products.access_control.backend.facade.user_access_control import UserAccessControl
+
+from ...models.evaluations import Evaluation
+from .trace_queries import canonical_uuid
+
+EVALUATION_RESULTS_LIMIT = 250
+
+
+@frozen
+class EvaluationResultRow:
+ uuid: str
+ timestamp: datetime
+ evaluation_id: str | None
+ evaluation_name: str | None
+ target_event_id: str | None
+ result: str | None
+ reasoning: str | None
+ applicable: str | None
+ evaluation_type: str | None
+ result_type: str | None
+ sentiment_label: str | None
+ skipped: str | None
+ score: str | None
+ categories: str | None
+ backfill_id: str | None
+
+
+_PROPERTY_COLUMNS = {
+ "evaluation_id": "$ai_evaluation_id",
+ "evaluation_name": "$ai_evaluation_name",
+ "target_event_id": "$ai_target_event_id",
+ "result": "$ai_evaluation_result",
+ "reasoning": "$ai_evaluation_reasoning",
+ "applicable": "$ai_evaluation_applicable",
+ "evaluation_type": "$ai_evaluation_runtime",
+ "result_type": "$ai_evaluation_result_type",
+ "sentiment_label": "$ai_sentiment_label",
+ "skipped": "$ai_evaluation_skipped",
+ "score": "$ai_evaluation_numeric_result",
+ "categories": "$ai_evaluation_categorical_result",
+ "backfill_id": "$ai_evaluation_backfill_id",
+}
+
+
+def _equals(left: ast.Expr, value: str) -> ast.Expr:
+ return ast.CompareOperation(op=ast.CompareOperationOp.Eq, left=left, right=ast.Constant(value=value))
+
+
+def _results_query(trace_id: str) -> ast.SelectQuery:
+ return ast.SelectQuery(
+ select=[
+ ast.Field(chain=["uuid"]),
+ ast.Field(chain=["timestamp"]),
+ *(ast.Field(chain=["properties", name]) for name in _PROPERTY_COLUMNS.values()),
+ ],
+ select_from=ast.JoinExpr(table=ast.Field(chain=["events"])),
+ where=ast.And(
+ exprs=[
+ _equals(ast.Field(chain=["event"]), "$ai_evaluation"),
+ _equals(ast.Field(chain=["properties", "$ai_trace_id"]), trace_id),
+ ]
+ ),
+ order_by=[ast.OrderExpr(expr=ast.Field(chain=["timestamp"]), order="DESC")],
+ limit_by=ast.LimitByExpr(n=ast.Constant(value=1), exprs=[ast.Field(chain=["uuid"])]),
+ limit=ast.Constant(value=EVALUATION_RESULTS_LIMIT),
+ )
+
+
+def _raw(value: Any) -> str | None:
+ # A property with a Boolean definition reads as a Python bool, so it is written the way untyped JSON reads.
+ if isinstance(value, bool):
+ return "true" if value else "false"
+ return None if value is None else str(value)
+
+
+def _result_row(values: Sequence[Any]) -> EvaluationResultRow:
+ uuid, timestamp, *properties = values
+ return EvaluationResultRow(
+ uuid=str(uuid),
+ timestamp=timestamp,
+ **{field: _raw(value) for field, value in zip(_PROPERTY_COLUMNS, properties, strict=True)},
+ )
+
+
+def load_evaluation_results(team: Team, user: User | None, trace_id: str) -> tuple[EvaluationResultRow, ...]:
+ name = "ai_trace_evaluation_results"
+ with tags_context(product=Product.LLM_ANALYTICS):
+ tag_queries(name=name)
+ response = execute_hogql_query(
+ query=_results_query(trace_id),
+ team=team,
+ user=user,
+ query_type=name,
+ context=HogQLContext(
+ team_id=team.pk, team=team, user=user, database=Database.create_for_posthog_tables(team)
+ ),
+ )
+ return tuple(_result_row(values) for values in response.results or [])
+
+
+def load_output_configs(
+ team: Team, user: User | None, evaluation_ids: Iterable[str]
+) -> dict[str, Mapping[str, object]]:
+ """Keyed by each id as given, not by its canonical UUID form."""
+ # Results can come from outside PostHog with any id, and a malformed UUID in the lookup raises.
+ ids_by_uuid = {uuid: evaluation_id for evaluation_id in evaluation_ids if (uuid := canonical_uuid(evaluation_id))}
+ if not ids_by_uuid:
+ return {}
+ # A deleted detector's results still read as a detector's, so deleted evaluations stay in.
+ evaluations: QuerySet[Evaluation] = Evaluation.objects.filter(team_id=team.pk, id__in=ids_by_uuid)
+ if user is not None:
+ # Evaluations have their own access control, apart from the llm_analytics access that opens the trace.
+ evaluations = UserAccessControl(user=user, team=team).filter_queryset_by_access_level(
+ evaluations, resource="evaluation"
+ )
+ rows = evaluations.values_list("id", "output_config")
+ return {
+ ids_by_uuid[str(evaluation_id)]: output_config
+ for evaluation_id, output_config in rows
+ if isinstance(output_config, Mapping)
+ }
diff --git a/products/ai_observability/backend/logic/traces/trace_queries.py b/products/ai_observability/backend/logic/traces/trace_queries.py
index e6e77c1dfe4b..fb87916d6559 100644
--- a/products/ai_observability/backend/logic/traces/trace_queries.py
+++ b/products/ai_observability/backend/logic/traces/trace_queries.py
@@ -373,7 +373,7 @@ def load_trace(team: Team, user: User | None, trace_id: str, timestamp_hint: dat
return LoadedTrace(rows=rows, person=person)
-def _canonical_uuid(value: str) -> str | None:
+def canonical_uuid(value: str) -> str | None:
try:
return str(UUID(value))
except ValueError:
@@ -394,7 +394,7 @@ def _trace_node_row(values: Sequence[Any]) -> TraceNodeRow:
def load_trace_node(
team: Team, user: User | None, trace_id: str, node_id: str, timestamp_hint: datetime | None
) -> TraceNodeRow | None:
- event_uuid = _canonical_uuid(node_id)
+ event_uuid = canonical_uuid(node_id)
if event_uuid is None:
return None
results = _read_trace(
diff --git a/products/ai_observability/backend/presentation/views.py b/products/ai_observability/backend/presentation/views.py
index 2d0dcd3f9129..e8c27e757109 100644
--- a/products/ai_observability/backend/presentation/views.py
+++ b/products/ai_observability/backend/presentation/views.py
@@ -48,7 +48,7 @@ THROTTLED_RESPONSE = OpenApiResponse(description="Too many queries are running f
class TraceViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):
scope_object = "llm_analytics"
- scope_object_read_actions = ["retrieve", "node"]
+ scope_object_read_actions = ["retrieve", "node", "evaluations"]
# Traces are project-wide data, so a grant on a single llm_analytics object must not open them.
requires_resource_level_access = True
serializer_class = _FallbackSerializer
@@ -108,3 +108,22 @@ class TraceViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):
except api.TraceNodeNotFoundError:
return Response({"detail": "Node not found."}, status=status.HTTP_404_NOT_FOUND)
return Response(node.model_dump(mode="json", by_alias=True))
+
+ @validated_request(
+ parameters=[TRACE_ID_PARAMETER],
+ responses={
+ 200: OpenApiResponse(response=contracts.TraceEvaluationResults),
+ 400: OpenApiResponse(description="The encoded trace id is malformed."),
+ 429: THROTTLED_RESPONSE,
+ },
+ operation_id="ai_observability_traces_evaluations_retrieve",
+ )
+ @action(detail=True, methods=["GET"], url_path="evaluations")
+ def evaluations(self, request: ValidatedRequest, pk: str, **kwargs: Any) -> Response:
+ """A trace's evaluation results, newest first and at most 250, each with its outcome and label ready to render."""
+ try:
+ trace_id = decode_trace_id_segment(pk)
+ except MalformedTraceIdSegmentError:
+ return Response({"detail": "Malformed encoded trace id."}, status=status.HTTP_400_BAD_REQUEST)
+ results = self._read(request, lambda user, _hint: api.get_trace_evaluation_results(self.team, user, trace_id))
+ return Response(results.model_dump(mode="json", by_alias=True))
diff --git a/products/ai_observability/backend/tests/logic/traces/rows.py b/products/ai_observability/backend/tests/logic/traces/rows.py
index e1eb34bac822..422a7f071b30 100644
--- a/products/ai_observability/backend/tests/logic/traces/rows.py
+++ b/products/ai_observability/backend/tests/logic/traces/rows.py
@@ -1,9 +1,14 @@
+from collections.abc import Mapping
from datetime import UTC, datetime
from typing import Any
+from posthog.models import Team
+
+from products.ai_observability.backend.logic.traces.evaluation_result_queries import EvaluationResultRow
from products.ai_observability.backend.logic.traces.trace_queries import TraceEventRow
T0 = datetime(2026, 9, 1, 10, 0, 0, tzinfo=UTC)
+EVALUATION_ID = "0192f2a4-5c1e-7000-8000-000000000e01"
def make_row(**overrides: Any) -> TraceEventRow:
@@ -31,3 +36,43 @@ def make_row(**overrides: Any) -> TraceEventRow:
"ingestion_source": None,
}
return TraceEventRow(**{**fields, **overrides})
+
+
+def make_evaluation_result_row(**overrides: Any) -> EvaluationResultRow:
+ fields: dict[str, Any] = {
+ "uuid": "result-1",
+ "timestamp": T0,
+ "evaluation_id": EVALUATION_ID,
+ "evaluation_name": "Answers the question",
+ "target_event_id": None,
+ "result": "true",
+ "reasoning": "Direct answer.",
+ "applicable": None,
+ "evaluation_type": "llm_judge",
+ "result_type": None,
+ "sentiment_label": None,
+ "skipped": None,
+ "score": None,
+ "categories": None,
+ "backfill_id": None,
+ }
+ return EvaluationResultRow(**{**fields, **overrides})
+
+
+def make_evaluation_result_event(
+ *,
+ uuid: str,
+ team: Team,
+ timestamp: datetime,
+ trace_id: str = "trace-1",
+ event: str = "$ai_evaluation",
+ properties: Mapping[str, object] | None = None,
+) -> dict[str, object]:
+ return {
+ "event": event,
+ "event_uuid": uuid,
+ "distinct_id": "user-1",
+ "team": team,
+ "timestamp": timestamp,
+ "properties": {"$ai_trace_id": trace_id, "$ai_evaluation_result": True, **(properties or {})},
+ }
diff --git a/products/ai_observability/backend/tests/logic/traces/test_evaluation_result.py b/products/ai_observability/backend/tests/logic/traces/test_evaluation_result.py
new file mode 100644
index 000000000000..f82aa7942f81
--- /dev/null
+++ b/products/ai_observability/backend/tests/logic/traces/test_evaluation_result.py
@@ -0,0 +1,177 @@
+from typing import Any
+
+from parameterized import parameterized
+
+from products.ai_observability.backend.logic.traces.evaluation_result import EvaluationResult
+
+from .rows import EVALUATION_ID, make_evaluation_result_row
+
+DETECTOR = {"true_is_failure": True}
+AT_LEAST_0_7 = {"passing_rule": {"operator": "gte", "threshold": 0.7}}
+BILLING_ONLY = {
+ "options": [{"key": "billing", "label": "Billing"}, {"key": "refund", "label": "Refund"}],
+ "passing_rule": {"categories": ["billing"]},
+}
+
+
+def evaluation_result(overrides: dict[str, Any], output_config: dict[str, Any] | None = None) -> EvaluationResult:
+ return EvaluationResult.from_row(
+ make_evaluation_result_row(**overrides), trace_id="trace-1", output_config=output_config or {}
+ )
+
+
+ [
+ ("a true result", {}, {}, "pass", "True"),
+ ("a false result", {"result": "false"}, {}, "fail", "False"),
+ ("a result written as True", {"result": "True"}, {}, "pass", "True"),
+ ("a true result from a detector", {}, DETECTOR, "fail", "True"),
+ ("a false result from a detector", {"result": "false"}, DETECTOR, "pass", "False"),
+ (
+ "a skipped run whose result is still false",
+ {"skipped": "true", "result": "false"},
+ {},
+ "inconclusive",
+ "Skipped",
+ ),
+ ("a missing result", {"result": None}, {}, "inconclusive", "N/A"),
+ ("a result marked not applicable", {"applicable": "false"}, {}, "inconclusive", "N/A"),
+ ("a score with no passing rule", {"result_type": "numeric", "score": "0.82"}, {}, "unrated", "0.82"),
+ ("a score under the threshold", {"result_type": "numeric", "score": "0.4"}, AT_LEAST_0_7, "fail", "0.4"),
+ ("a score over the threshold", {"result_type": "numeric", "score": "0.9"}, AT_LEAST_0_7, "pass", "0.9"),
+ (
+ "a long score rounded to two places",
+ {"result_type": "numeric", "score": "12.3456"},
+ {},
+ "unrated",
+ "12.35",
+ ),
+ ("a whole score", {"result_type": "numeric", "score": "3"}, {}, "unrated", "3"),
+ (
+ "a score whose rounding would flip the verdict",
+ {"result_type": "numeric", "score": "0.6999"},
+ AT_LEAST_0_7,
+ "fail",
+ "0.6999",
+ ),
+ ("a numeric run with no score", {"result_type": "numeric"}, {}, "inconclusive", "No score"),
+ ("a numeric run with an empty score", {"result_type": "numeric", "score": ""}, {}, "inconclusive", "No score"),
+ (
+ "a numeric run marked not applicable",
+ {"result_type": "numeric", "score": "0.5", "applicable": "false"},
+ {},
+ "inconclusive",
+ "N/A",
+ ),
+ (
+ "a stored threshold that is not a number",
+ {"result_type": "numeric", "score": "0.4"},
+ {"passing_rule": {"operator": "gte", "threshold": "0.7"}},
+ "unrated",
+ "0.4",
+ ),
+ (
+ "a stored rule with an unknown operator",
+ {"result_type": "numeric", "score": "0.4"},
+ {"passing_rule": {"operator": "gt", "threshold": 0.7}},
+ "unrated",
+ "0.4",
+ ),
+ (
+ "categories inside the passing rule",
+ {"result_type": "categorical", "categories": '["billing"]'},
+ BILLING_ONLY,
+ "pass",
+ "Billing",
+ ),
+ (
+ "categories outside the passing rule",
+ {"result_type": "categorical", "categories": '["billing", "refund"]'},
+ BILLING_ONLY,
+ "fail",
+ "Billing, Refund",
+ ),
+ (
+ "a category with no option keeps its key",
+ {"result_type": "categorical", "categories": '["other"]'},
+ {},
+ "unrated",
+ "other",
+ ),
+ (
+ "an applicable run with no categories",
+ {"result_type": "categorical", "applicable": "true"},
+ BILLING_ONLY,
+ "fail",
+ "No categories",
+ ),
+ (
+ "categories that are not JSON",
+ {"result_type": "categorical", "categories": "billing"},
+ BILLING_ONLY,
+ "inconclusive",
+ "No result",
+ ),
+ (
+ "categories marked not applicable",
+ {"result_type": "categorical", "categories": '["billing"]', "applicable": "false"},
+ BILLING_ONLY,
+ "inconclusive",
+ "N/A",
+ ),
+ ("a positive sentiment", {"result_type": "sentiment", "sentiment_label": "positive"}, {}, "pass", "Positive"),
+ ("a neutral sentiment", {"result_type": "sentiment", "sentiment_label": "neutral"}, {}, "unrated", "Neutral"),
+ ("a negative sentiment", {"result_type": "sentiment", "sentiment_label": "negative"}, {}, "fail", "Negative"),
+ ("an unlabelled sentiment", {"result_type": "sentiment"}, {}, "inconclusive", "Unknown"),
+ (
+ "a sentiment run with no result type",
+ {"evaluation_type": "sentiment", "sentiment_label": "positive"},
+ {},
+ "pass",
+ "Positive",
+ ),
+ ]
+)
+def test_outcome_and_label(
+ _name: str, overrides: dict[str, Any], output_config: dict[str, Any], outcome: str, label: str
+) -> None:
+ result = evaluation_result(overrides, output_config)
+
+ assert (result.outcome, result.label) == (outcome, label)
+
+
+ [
+ (
+ "a generation result",
+ {"target_event_id": "gen-1"},
+ ("gen-1", False, EVALUATION_ID, "Answers the question", "Direct answer."),
+ ),
+ ("a trace-level result", {}, ("trace-1", False, EVALUATION_ID, "Answers the question", "Direct answer.")),
+ (
+ "a backfilled result",
+ {"backfill_id": "backfill-1"},
+ ("trace-1", True, EVALUATION_ID, "Answers the question", "Direct answer."),
+ ),
+ (
+ "a result from outside whose evaluation id is not a UUID",
+ {"evaluation_id": "not-a-uuid"},
+ ("trace-1", False, None, "Answers the question", "Direct answer."),
+ ),
+ (
+ "a result from outside with no id, name or reasoning",
+ {"evaluation_id": None, "evaluation_name": None, "reasoning": ""},
+ ("trace-1", False, None, "Unknown evaluation", None),
+ ),
+ ]
+)
+def test_target_name_and_backfill(_name: str, overrides: dict[str, Any], expected: tuple[Any, ...]) -> None:
+ result = evaluation_result(overrides)
+
+ assert (
+ result.target_node_id,
+ result.is_backfill,
+ result.evaluation_id,
+ result.name,
+ result.reasoning,
+ ) == expected
diff --git a/products/ai_observability/backend/tests/logic/traces/test_evaluation_result_queries.py b/products/ai_observability/backend/tests/logic/traces/test_evaluation_result_queries.py
new file mode 100644
index 000000000000..2a13c0d12bee
--- /dev/null
+++ b/products/ai_observability/backend/tests/logic/traces/test_evaluation_result_queries.py
@@ -0,0 +1,38 @@
+from datetime import timedelta
+
+from posthog.test.base import BaseTest, ClickhouseTestMixin
+
+from posthog.models import Team
+from posthog.models.event.util import bulk_create_events
+
+from products.ai_observability.backend.logic.traces.evaluation_result_queries import load_evaluation_results
+
+from .rows import T0, make_evaluation_result_event
+
+LATE = "00000000-0000-0000-0000-00000000000b"
+EARLY = "00000000-0000-0000-0000-00000000000a"
+
+
+class TestLoadEvaluationResults(ClickhouseTestMixin, BaseTest):
+ def test_reads_every_result_of_the_trace_however_late_it_was_stamped(self) -> None:
+ other_team = Team.objects.create(organization=self.organization, name="other")
+ bulk_create_events(
+ [
+ make_evaluation_result_event(uuid=EARLY, team=self.team, timestamp=T0 + timedelta(seconds=1)),
+ make_evaluation_result_event(uuid=LATE, team=self.team, timestamp=T0 + timedelta(days=90)),
+ make_evaluation_result_event(
+ uuid="00000000-0000-0000-0000-00000000000c",
+ team=self.team,
+ timestamp=T0 + timedelta(seconds=1),
+ trace_id="trace-2",
+ ),
+ make_evaluation_result_event(
+ uuid="00000000-0000-0000-0000-00000000000d", team=other_team, timestamp=T0 + timedelta(seconds=1)
+ ),
+ make_evaluation_result_event(
+ uuid="00000000-0000-0000-0000-00000000000e", team=self.team, timestamp=T0, event="$ai_generation"
+ ),
+ ]
+ )
+
+ assert [row.uuid for row in load_evaluation_results(self.team, None, "trace-1")] == [LATE, EARLY]
diff --git a/products/ai_observability/backend/tests/test_facade_api.py b/products/ai_observability/backend/tests/test_facade_api.py
index 939b0da5995f..c59e846293c4 100644
--- a/products/ai_observability/backend/tests/test_facade_api.py
+++ b/products/ai_observability/backend/tests/test_facade_api.py
@@ -1,12 +1,23 @@
-from datetime import UTC, datetime
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
import pytest
+from posthog.test.base import BaseTest, ClickhouseTestMixin
from unittest.mock import patch
+from parameterized import parameterized
+
+from posthog.constants import AvailableFeature
+from posthog.models import Team
+from posthog.models.event.util import bulk_create_events
+
+from products.access_control.backend.models.access_control import AccessControl
from products.ai_observability.backend.facade import api
from products.ai_observability.backend.logic.traces.trace_queries import LoadedTrace
+from products.ai_observability.backend.models.evaluations import Evaluation
+from products.event_definitions.backend.models.property_definition import PropertyDefinition
-from .logic.traces.rows import make_row
+from .logic.traces.rows import make_evaluation_result_event, make_row
T0 = datetime(2026, 9, 1, 10, 0, tzinfo=UTC)
@@ -29,3 +40,108 @@ def test_missing_trace_raises_not_found() -> None:
with patch.object(api, "load_trace", return_value=LoadedTrace(rows=(), person=None)):
with pytest.raises(api.TraceNotFoundError):
api.get_trace(team=None, user=None, trace_id="missing", timestamp_hint=None) # type: ignore[arg-type]
+
+
+class TestGetTraceEvaluationResults(ClickhouseTestMixin, BaseTest):
+ def _detector(self, team: Team | None = None, *, deleted: bool = False, stored_config: object = None) -> str:
+ evaluation = Evaluation.objects.create(
+ team=team or self.team,
+ name="Mentions a competitor",
+ evaluation_type="llm_judge",
+ evaluation_config={"prompt": "Does the answer mention a competitor?"},
+ output_type="boolean",
+ output_config={"true_is_failure": True},
+ deleted=deleted,
+ )
+ if stored_config is not None:
+ Evaluation.objects.filter(id=evaluation.id).update(output_config=stored_config)
+ return str(evaluation.id)
+
+ def _other_team(self) -> Team:
+ return Team.objects.create(organization=self.organization, name="other")
+
+ def _detector_hidden_from_the_viewer(self) -> str:
+ self.organization.available_product_features = [
+ {"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}
+ ]
+ self.organization.save()
+ AccessControl.objects.create(
+ team=self.team, resource="evaluation", access_level="none", organization_member=self.organization_membership
+ )
+ return self._detector()
+
+ @parameterized.expand(
+ [
+ ("an active detector", lambda test: test._detector(), "fail"),
+ ("a deleted detector", lambda test: test._detector(deleted=True), "fail"),
+ ("another team's detector", lambda test: test._detector(test._other_team()), "pass"),
+ ("a detector the viewer cannot read", lambda test: test._detector_hidden_from_the_viewer(), "pass"),
+ ("an id that is not a UUID", lambda test: "not-a-uuid", "pass"),
+ (
+ "a detector whose stored config is not an object",
+ lambda test: test._detector(stored_config=["true_is_failure"]),
+ "pass",
+ ),
+ ]
+ )
+ def test_applies_the_results_stored_output_config(
+ self, _name: str, evaluation_id_for: Callable[["TestGetTraceEvaluationResults"], str], outcome: str
+ ) -> None:
+ bulk_create_events(
+ [
+ make_evaluation_result_event(
+ uuid="00000000-0000-0000-0000-000000000001",
+ team=self.team,
+ timestamp=T0 + timedelta(seconds=1),
+ properties={"$ai_evaluation_id": evaluation_id_for(self)},
+ )
+ ]
+ )
+
+ [result] = api.get_trace_evaluation_results(self.team, self.user, "trace-1").results
+
+ assert result.outcome == outcome
+
+ def test_reads_a_registered_boolean_property_like_its_ingested_value(self) -> None:
+ PropertyDefinition.objects.create(
+ team=self.team,
+ name="$ai_evaluation_applicable",
+ property_type="Boolean",
+ type=PropertyDefinition.Type.EVENT,
+ )
+ bulk_create_events(
+ [
+ make_evaluation_result_event(
+ uuid="00000000-0000-0000-0000-000000000001",
+ team=self.team,
+ timestamp=T0 + timedelta(seconds=1),
+ properties={"$ai_evaluation_applicable": False},
+ )
+ ]
+ )
+
+ [result] = api.get_trace_evaluation_results(self.team, None, "trace-1").results
+
+ assert (result.outcome, result.label) == ("inconclusive", "N/A")
+
+ def test_lists_results_by_their_event_time_newest_first(self) -> None:
+ live = "00000000-0000-0000-0000-000000000001"
+ backfill = "00000000-0000-0000-0000-000000000002"
+ bulk_create_events(
+ [
+ make_evaluation_result_event(uuid=live, team=self.team, timestamp=T0 + timedelta(minutes=5)),
+ make_evaluation_result_event(
+ uuid=backfill,
+ team=self.team,
+ timestamp=T0 + timedelta(seconds=1),
+ properties={
+ "$ai_evaluation_start_time": (T0 + timedelta(days=20)).isoformat(),
+ "$ai_evaluation_backfill_id": "backfill-1",
+ },
+ ),
+ ]
+ )
+
+ results = api.get_trace_evaluation_results(self.team, None, "trace-1").results
+
+ assert [(result.id, result.is_backfill) for result in results] == [(live, False), (backfill, True)]
diff --git a/products/ai_observability/backend/tests/test_trace_views.py b/products/ai_observability/backend/tests/test_trace_views.py
index 0a5bf3c091a7..2c0a54cba76c 100644
--- a/products/ai_observability/backend/tests/test_trace_views.py
+++ b/products/ai_observability/backend/tests/test_trace_views.py
@@ -1,5 +1,5 @@
import base64
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from posthog.test.base import APIBaseTest, ClickhouseTestMixin
from unittest.mock import patch
@@ -10,6 +10,7 @@ from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, Concurrenc
from posthog.constants import AvailableFeature
from posthog.models import Team
from posthog.models.ai_events.test_util import bulk_create_ai_events
+from posthog.models.event.util import bulk_create_events
from posthog.models.organization import OrganizationMembership
from posthog.models.user import User
@@ -19,9 +20,12 @@ from products.access_control.backend.models.property_access_control import Prope
from products.ai_observability.backend.models.review_queues import ReviewQueue
from products.event_definitions.backend.models.property_definition import PropertyDefinition
+from .logic.traces.rows import make_evaluation_result_event
+
T0 = datetime(2026, 9, 1, 10, 0, tzinfo=UTC)
GENERATION = "00000000-0000-0000-0000-000000000001"
TRACE_EVENT = "00000000-0000-0000-0000-000000000002"
+EVALUATION_RESULT = "00000000-0000-0000-0000-000000000003"
QUESTION = [{"role": "user", "content": "Why did my invoice go up?"}]
ANSWER = [{"role": "assistant", "content": "Two seats were added."}]
@@ -80,6 +84,23 @@ def _create_node_trace(
bulk_create_ai_events(events)
+def _create_evaluation_result(team: Team) -> None:
+ bulk_create_events(
+ [
+ make_evaluation_result_event(
+ uuid=EVALUATION_RESULT,
+ team=team,
+ timestamp=T0 + timedelta(seconds=30),
+ properties={
+ "$ai_target_event_id": GENERATION,
+ "$ai_evaluation_name": "Answers the question",
+ "$ai_evaluation_reasoning": "Direct answer.",
+ },
+ )
+ ]
+ )
+
+
def _encode(trace_id: str) -> str:
return base64.urlsafe_b64encode(trace_id.encode()).decode().rstrip("=")
@@ -130,7 +151,11 @@ class TestTraceViewSet(ClickhouseTestMixin, APIBaseTest):
("browser session", False, "app_per_org"),
("personal api key", True, "api_per_team"),
]
- for read, suffix in [("trace", ""), ("node", f"nodes/{_encode('trace-1')}/")]
+ for read, suffix in [
+ ("trace", ""),
+ ("node", f"nodes/{_encode('trace-1')}/"),
+ ("evaluations", "evaluations/"),
+ ]
]
)
def test_throttles_when_the_query_concurrency_limit_is_exhausted(
@@ -235,6 +260,60 @@ class TestTraceNodeView(ClickhouseTestMixin, APIBaseTest):
assert properties["$ai_provider"] == "openai"
+class TestTraceEvaluationsView(ClickhouseTestMixin, APIBaseTest):
+ def _url(self, segment: str) -> str:
+ return f"/api/projects/{self.team.id}/ai_observability/traces/{segment}/evaluations/"
+
+ def test_returns_the_results_in_camel_case(self) -> None:
+ _create_evaluation_result(self.team)
+
+ response = self.client.get(self._url(_encode("trace-1")))
+
+ assert response.status_code == 200
+ assert response.json()["results"][0] | {"timestamp": None} == {
+ "id": EVALUATION_RESULT,
+ "evaluationId": None,
+ "evaluationName": "Answers the question",
+ "targetNodeId": GENERATION,
+ "outcome": "pass",
+ "label": "True",
+ "reasoning": "Direct answer.",
+ "timestamp": None,
+ "isBackfill": False,
+ }
+
+ def test_an_unknown_trace_has_no_results(self) -> None:
+ response = self.client.get(self._url(_encode("trace-2")))
+
+ assert (response.status_code, response.json()) == (200, {"results": []})
+
+ def test_rejects_an_unencoded_trace_id(self) -> None:
+ assert self.client.get(self._url("trace-1")).status_code == 400
+
+ def test_masks_properties_the_viewer_may_not_read(self) -> None:
+ self.organization.available_product_features = [
+ {"key": AvailableFeature.PROPERTY_ACCESS_CONTROL, "name": AvailableFeature.PROPERTY_ACCESS_CONTROL}
+ ]
+ self.organization.save()
+ definition = PropertyDefinition.objects.create(
+ team=self.team,
+ name="$ai_evaluation_reasoning",
+ property_type="String",
+ type=PropertyDefinition.Type.EVENT,
+ )
+ PropertyAccessControl.objects.create(
+ team=self.team,
+ property_definition=definition,
+ organization_member=self.organization_membership,
+ access_level=PropertyAccessLevel.NONE.value,
+ )
+ _create_evaluation_result(self.team)
+
+ result = self.client.get(self._url(_encode("trace-1"))).json()["results"][0]
+
+ assert (result["reasoning"], result["outcome"]) == (None, "pass")
+
+
class TestTraceViewSetResourceLevelAccess(ClickhouseTestMixin, APIBaseTest):
def test_a_grant_on_one_object_does_not_open_traces(self) -> None:
self.organization.available_product_features = [
diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.ts
index 1288a51216b2..5265a0cb9272 100644
--- a/products/ai_observability/frontend/generated/api.schemas.ts
+++ b/products/ai_observability/frontend/generated/api.schemas.ts
@@ -1404,6 +1404,34 @@ export interface TraceApi {
nodeIdsByAlias: TraceApiNodeIdsByAlias
}
+export type TraceEvaluationResultOutcomeEnumApi =
+ (typeof TraceEvaluationResultOutcomeEnumApi)[keyof typeof TraceEvaluationResultOutcomeEnumApi]
+
+export const TraceEvaluationResultOutcomeEnumApi = {
+ Pass: 'pass',
+ Fail: 'fail',
+ Inconclusive: 'inconclusive',
+ Unrated: 'unrated',
+} as const
+
+export interface TraceEvaluationResultApi {
+ id: string
+ evaluationId: string | null
+ evaluationName: string
+ /** The tree node the result grades: the generation's event uuid, or the trace id for a trace-level result. */
+ targetNodeId: string
+ outcome: TraceEvaluationResultOutcomeEnumApi
+ /** What the result says, such as "True", "0.82", "Positive" or "Skipped". */
+ label: string
+ reasoning: string | null
+ timestamp: string
+ isBackfill: boolean
+}
+
+export interface TraceEvaluationResultsApi {
+ results: TraceEvaluationResultApi[]
+}
+
export interface TraceNodeMetadataApi {
provider: string | null
temperature: number | null
diff --git a/products/ai_observability/frontend/generated/api.ts b/products/ai_observability/frontend/generated/api.ts
index 3d452f40baf0..bf3d0381929c 100644
--- a/products/ai_observability/frontend/generated/api.ts
+++ b/products/ai_observability/frontend/generated/api.ts
@@ -143,6 +143,7 @@ import type {
TextReprRequestApi,
TextReprResponseApi,
TraceApi,
+ TraceEvaluationResultsApi,
TraceNodeDetailApi,
TraceReviewApi,
TraceReviewCreateApi,
@@ -675,6 +676,24 @@ export const aiObservabilityTracesRetrieve = async (
})
}
+export const getAiObservabilityTracesEvaluationsRetrieveUrl = (projectId: string, id: string) => {
+ return `/api/projects/${projectId}/ai_observability/traces/${id}/evaluations/`
+}
+
+/**
+ * A trace's evaluation results, newest first and at most 250, each with its outcome and label ready to render.
+ */
+export const aiObservabilityTracesEvaluationsRetrieve = async (
+ projectId: string,
+ id: string,
+ options?: RequestInit
+): Promise<TraceEvaluationResultsApi> => {
+ return apiMutator<TraceEvaluationResultsApi>(getAiObservabilityTracesEvaluationsRetrieveUrl(projectId, id), {
+ ...options,
+ method: 'GET',
+ })
+}
+
export const getAiObservabilityTracesNodesRetrieveUrl = (
projectId: string,
id: string,
diff --git a/products/ai_observability/frontend/redesign/trace/adapter/testFixtures.ts b/products/ai_observability/frontend/redesign/trace/adapter/testFixtures.ts
index 226e0d6a76de..0b91e903def2 100644
--- a/products/ai_observability/frontend/redesign/trace/adapter/testFixtures.ts
+++ b/products/ai_observability/frontend/redesign/trace/adapter/testFixtures.ts
@@ -2,6 +2,7 @@ import { LLMTrace, LLMTraceEvent } from '~/queries/schema/schema-general'
import type {
TraceApi,
+ TraceEvaluationResultApi,
TraceNodeApi,
TraceNodeDetailApi,
TraceNodeIOKeysApi,
@@ -123,3 +124,18 @@ export function makeTraceResource(overrides: Partial<TraceApi> = {}): TraceApi {
...overrides,
}
}
+
+export function makeEvaluationResult(overrides: Partial<TraceEvaluationResultApi> = {}): TraceEvaluationResultApi {
+ return {
+ id: 'r1',
+ evaluationId: 'ev-helpful',
+ evaluationName: 'Answers the question',
+ targetNodeId: 'trace-1',
+ outcome: 'pass',
+ label: 'True',
+ reasoning: 'Direct answer.',
+ timestamp: '2026-09-01T10:16:00Z',
+ isBackfill: false,
+ ...overrides,
+ }
+}
diff --git a/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.test.ts b/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.test.ts
index ddf14238bedf..028219ed32ab 100644
--- a/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.test.ts
+++ b/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.test.ts
@@ -1,8 +1,8 @@
import { urls } from 'scenes/urls'
-import { EvaluationOutputConfig, EvaluationRun, LLMJudgeEvaluation } from '../../../evaluations/types'
import type { TraceNodeApi } from '../../../generated/api.schemas'
-import { EvalResultsContext, toEvalResults } from './toEvalResults'
+import { makeEvaluationResult } from './testFixtures'
+import { toEvalResults } from './toEvalResults'
const node = (id: string, kind: TraceNodeApi['kind'], name: string, children: TraceNodeApi[] = []): TraceNodeApi => ({
id,
@@ -29,110 +29,32 @@ const tree = [
]),
]
-const run = (overrides: Partial<EvaluationRun> = {}): EvaluationRun => ({
- id: 'r1',
- evaluation_id: 'ev-helpful',
- evaluation_name: 'Answers the question',
- generation_id: null,
- trace_id: 'trace-1',
- timestamp: '2026-09-01T10:16:00Z',
- result: true,
- reasoning: 'Direct answer.',
- status: 'completed',
- ...overrides,
-})
-
-const evaluation = (id: string, outputConfig: EvaluationOutputConfig): LLMJudgeEvaluation => ({
- id,
- name: id,
- enabled: true,
- status: 'active',
- status_reason: null,
- status_reason_detail: null,
- evaluation_type: 'llm_judge',
- evaluation_config: { prompt: 'Grade the answer.' },
- output_type: 'numeric',
- output_config: outputConfig,
- conditions: [],
- target: 'generation',
- target_config: {},
- model_configuration: null,
- created_at: '2026-08-01T00:00:00Z',
- updated_at: '2026-08-01T00:00:00Z',
-})
-
-const context = (overrides: Partial<EvalResultsContext> = {}): EvalResultsContext => ({
- tree,
- viewedNodeId: 'trace-1',
- evaluations: [evaluation('ev-quality', { passing_rule: { operator: 'gte', threshold: 0.7 } })],
- detectorEvaluationIds: ['ev-detector'],
- ...overrides,
-})
-
describe('toEvalResults', () => {
- it.each<[string, Partial<EvaluationRun>, string, string]>([
- ['a failed run', { status: 'failed' }, 'error', 'Error'],
- ['a running run', { status: 'running' }, 'pending', 'Running'],
- ['a true result', { result: true }, 'pass', 'True'],
- ['a false result', { result: false }, 'fail', 'False'],
- ['a true result from a detector', { evaluation_id: 'ev-detector', result: true }, 'fail', 'True'],
- ['a false result from a detector', { evaluation_id: 'ev-detector', result: false }, 'pass', 'False'],
- ['a skipped run, whose result is still false', { skipped: true, result: false }, 'inconclusive', 'Skipped'],
- ['a missing result', { result: null }, 'inconclusive', 'N/A'],
- ['a score with no passing rule', { result_type: 'numeric', score: 0.82 }, 'unrated', '0.82'],
- [
- 'a score under the passing threshold',
- { evaluation_id: 'ev-quality', result_type: 'numeric', score: 0.4 },
- 'fail',
- '0.4',
- ],
- ['a numeric run with no score', { result_type: 'numeric', score: null }, 'inconclusive', 'No score'],
- ['a positive sentiment', { result_type: 'sentiment', sentiment_label: 'positive' }, 'pass', 'Positive'],
- ['a neutral sentiment', { result_type: 'sentiment', sentiment_label: 'neutral' }, 'unrated', 'Neutral'],
- ['an unknown sentiment', { result_type: 'sentiment', sentiment_label: null }, 'inconclusive', 'Unknown'],
- ])('maps %s to %s', (_name, overrides, outcome, label) => {
- expect(toEvalResults([run(overrides)], context())).toMatchObject([{ outcome, label }])
- })
-
- it('lists the newest verdict first, dating a backfill by when it ran, and marks backfills', () => {
- const results = toEvalResults(
- [
- run({ id: 'live', timestamp: '2026-09-01T10:16:00Z' }),
- run({
- id: 'backfill',
- timestamp: '2026-09-01T10:15:30Z',
- start_time: '2026-09-20T08:00:00Z',
- backfill_id: 'backfill-1',
- }),
- run({ id: 'older', timestamp: '2026-09-01T10:15:45Z' }),
- ],
- context()
- )
- expect(results.map((result) => [result.id, result.isBackfill, result.timestamp])).toEqual([
- ['backfill', true, '2026-09-01T10:15:30Z'],
- ['live', false, '2026-09-01T10:16:00Z'],
- ['older', false, '2026-09-01T10:15:45Z'],
- ])
- expect(results[0].href).toBe(urls.aiObservabilityEvaluation('ev-helpful'))
- })
-
it.each([
- ['a trace-level run seen on the root targets nothing', 'trace-1', null, null],
+ ['a trace-level result seen on the root targets nothing', 'trace-1', 'trace-1', null],
[
- 'a run on a generation seen on the root names that generation apart from its namesake',
+ 'a result on a generation seen on the root names that generation apart from its namesake',
'trace-1',
'gen-2',
{ nodeId: 'gen-2', label: 'gpt-4.1-mini (openai) #2' },
],
- ['a run on the viewed generation targets nothing', 'gen-1', 'gen-1', null],
+ ['a result on the viewed generation targets nothing', 'gen-1', 'gen-1', null],
[
- 'a run on a generation missing from the tree falls back to a short id',
+ 'a result on a node missing from the tree falls back to a short id',
'trace-1',
'gen-0123456789abcdef',
{ nodeId: 'gen-0123456789abcdef', label: 'gen-01234567...' },
],
- ])('%s', (_name, viewedNodeId, generationId, expected) => {
- const [result] = toEvalResults([run({ generation_id: generationId })], context({ viewedNodeId }))
+ ])('%s', (_name, viewedNodeId, targetNodeId, expected) => {
+ const [result] = toEvalResults([makeEvaluationResult({ targetNodeId })], { tree, viewedNodeId })
expect(result.target).toEqual(expected)
})
+
+ it.each([
+ ['an evaluation id links to its evaluation', 'ev-helpful', urls.aiObservabilityEvaluation('ev-helpful')],
+ ['a result with no evaluation id has no link', null, null],
+ ])('%s', (_name, evaluationId, href) => {
+ const [result] = toEvalResults([makeEvaluationResult({ evaluationId })], { tree, viewedNodeId: 'trace-1' })
+ expect(result.href).toBe(href)
+ })
})
diff --git a/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.ts b/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.ts
index 60e474340709..50b610405d5e 100644
--- a/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.ts
+++ b/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.ts
@@ -1,41 +1,12 @@
import { urls } from 'scenes/urls'
-import { EvaluationResultDisplayOptions, getEvaluationResultDisplay } from '../../../components/EvaluationResultTag'
-import { EvaluationConfig, EvaluationRun } from '../../../evaluations/types'
-import type { TraceNodeApi } from '../../../generated/api.schemas'
-import { EvalOutcome, EvalResult, EvalTarget } from '../types'
+import type { TraceEvaluationResultApi, TraceNodeApi } from '../../../generated/api.schemas'
+import { EvalResult, EvalTarget } from '../types'
export interface EvalResultsContext {
- /** Every node of the trace, root first, to resolve and name what each run targets. */
+ /** Every node of the trace, root first, to name what each result targets. */
tree: TraceNodeApi[]
viewedNodeId: string
- evaluations: EvaluationConfig[]
- detectorEvaluationIds: string[]
-}
-
-function displayOptions(run: EvaluationRun, context: EvalResultsContext): EvaluationResultDisplayOptions {
- const outputConfig = context.evaluations.find((evaluation) => evaluation.id === run.evaluation_id)?.output_config
- return {
- trueIsFailure: context.detectorEvaluationIds.includes(run.evaluation_id),
- passingRule: outputConfig?.passing_rule,
- categoryOptions: outputConfig?.options,
- }
-}
-
-function outcomeOf(run: EvaluationRun, tagType: string | undefined): EvalOutcome {
- if (run.status === 'failed') {
- return 'error'
- }
- if (run.status === 'running') {
- return 'pending'
- }
- if (tagType === 'success') {
- return 'pass'
- }
- if (tagType === 'danger') {
- return 'fail'
- }
- return tagType === 'none' ? 'unrated' : 'inconclusive'
}
function flatten(nodes: TraceNodeApi[]): TraceNodeApi[] {
@@ -57,34 +28,29 @@ function nodeLabels(tree: TraceNodeApi[]): Map<string, string> {
)
}
-function targetOf(run: EvaluationRun, labels: Map<string, string>, context: EvalResultsContext): EvalTarget | null {
- const nodeId = run.generation_id ?? context.tree[0]?.id
- if (!nodeId || nodeId === context.viewedNodeId) {
+function targetOf(
+ result: TraceEvaluationResultApi,
+ labels: Map<string, string>,
+ viewedNodeId: string
+): EvalTarget | null {
+ const nodeId = result.targetNodeId
+ if (nodeId === viewedNodeId) {
return null
}
return { nodeId, label: labels.get(nodeId) ?? `${nodeId.slice(0, 12)}...` }
}
-function producedAt(run: EvaluationRun): number {
- return new Date(run.start_time ?? run.timestamp).getTime()
-}
-
-export function toEvalResults(runs: EvaluationRun[], context: EvalResultsContext): EvalResult[] {
+export function toEvalResults(results: TraceEvaluationResultApi[], context: EvalResultsContext): EvalResult[] {
const labels = nodeLabels(context.tree)
- return [...runs]
- .sort((a, b) => producedAt(b) - producedAt(a))
- .map((run) => {
- const display = getEvaluationResultDisplay(run, displayOptions(run, context))
- return {
- id: run.id,
- name: run.evaluation_name,
- href: urls.aiObservabilityEvaluation(run.evaluation_id),
- outcome: outcomeOf(run, display.type),
- label: display.label,
- reasoning: run.reasoning || null,
- timestamp: run.timestamp,
- isBackfill: !!run.backfill_id,
- target: targetOf(run, labels, context),
- }
- })
+ return results.map((result) => ({
+ id: result.id,
+ name: result.evaluationName,
+ href: result.evaluationId ? urls.aiObservabilityEvaluation(result.evaluationId) : null,
+ outcome: result.outcome,
+ label: result.label,
+ reasoning: result.reasoning,
+ timestamp: result.timestamp,
+ isBackfill: result.isBackfill,
+ target: targetOf(result, labels, context.viewedNodeId),
+ }))
}
diff --git a/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.test.ts b/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.test.ts
index 44f7c40e9914..0b606e25e7a3 100644
--- a/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.test.ts
+++ b/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.test.ts
@@ -1,7 +1,6 @@
-import { EvaluationRun } from '../../../evaluations/types'
import type { TraceNodeApi } from '../../../generated/api.schemas'
import { EvalResult, EvalsState, NodeLoad } from '../types'
-import { makeNodeDetail } from './testFixtures'
+import { makeEvaluationResult, makeNodeDetail } from './testFixtures'
import { toNodeDetail } from './toNodeDetail'
const node = (id: string, kind: 'trace' | 'span' | 'generation'): TraceNodeApi => ({
@@ -21,18 +20,8 @@ const node = (id: string, kind: 'trace' | 'span' | 'generation'): TraceNodeApi =
},
})
-const run = (overrides: Partial<EvaluationRun> = {}): EvaluationRun => ({
- id: 'r1',
- evaluation_id: 'ev1',
- evaluation_name: 'Answers the question',
- generation_id: 'gen-1',
- trace_id: 'trace-1',
- timestamp: '2026-09-01T10:16:00Z',
- result: true,
- reasoning: 'Direct answer.',
- status: 'completed',
- ...overrides,
-})
+const result = (overrides: Parameters<typeof makeEvaluationResult>[0] = {}): ReturnType<typeof makeEvaluationResult> =>
+ makeEvaluationResult({ targetNodeId: 'gen-1', ...overrides })
function readyResults(evals: EvalsState): EvalResult[] {
if (evals.status !== 'ready') {
@@ -46,11 +35,8 @@ const ready = (detail: ReturnType<typeof makeNodeDetail>): NodeLoad => ({ status
describe('toNodeDetail', () => {
const base = {
tree: [],
- evalRuns: [],
- evalRunsLoading: false,
- evaluations: [],
- detectorEvaluationIds: [],
- evaluationsSettled: true,
+ evaluationResults: [],
+ evaluationResultsFailed: false,
teamId: 7,
person: null,
trace: { createdAt: '2026-09-01T10:15:00Z', sessionId: 'sess-1' },
@@ -92,7 +78,7 @@ describe('toNodeDetail', () => {
metadata: { provider: 'openai', temperature: 0.2, sessionId: 'sess-9', promptVersion: 6 },
})
),
- evalRuns: [run(), run({ id: 'r2', generation_id: 'gen-other' })],
+ evaluationResults: [result(), result({ id: 'r2', targetNodeId: 'gen-other' })],
})
expect(detail.content).toMatchObject({
kind: 'messages',
@@ -110,18 +96,25 @@ describe('toNodeDetail', () => {
expect(readyResults(detail.evals).map((result) => result.id)).toEqual(['r1'])
})
- it.each([
- ['eval runs are loading', { evalRunsLoading: true }],
- ['the evaluation configs that decide detector polarity have not loaded', { evaluationsSettled: false }],
- ])('reports evals as loading while %s', (_name, overrides) => {
+ it.each<[string, { evaluationResults: null; evaluationResultsFailed: boolean }, EvalsState]>([
+ [
+ 'loading until the results arrive',
+ { evaluationResults: null, evaluationResultsFailed: false },
+ { status: 'loading' },
+ ],
+ [
+ 'an error when the read fails',
+ { evaluationResults: null, evaluationResultsFailed: true },
+ { status: 'error', errorMessage: 'Evaluation results could not be loaded. Reload the page to try again.' },
+ ],
+ ])('reports evals as %s', (_name, overrides, evals) => {
const detail = toNodeDetail({
...base,
...overrides,
node: node('gen-1', 'generation'),
load: { status: 'loading' },
- evalRuns: [run()],
})
- expect(detail.evals).toEqual({ status: 'loading' })
+ expect(detail.evals).toEqual(evals)
})
it('span shows its state as input and output, surfaces its error and labels the person', () => {
@@ -147,30 +140,33 @@ describe('toNodeDetail', () => {
it.each([
['with a trace event', true],
['without a trace event', false],
- ])('trace root %s takes its time and session from the trace and shows every eval run', (_name, withEvent) => {
- const detail = toNodeDetail({
- ...base,
- node: node('trace-1', 'trace'),
- load: ready(
- makeNodeDetail({
- id: 'trace-1',
- kind: 'trace',
- withEvent,
- properties: { $ai_input_state: { question: 'Hi' } },
- ioKeys: { input: '$ai_input_state' },
- })
- ),
- evalRuns: [run(), run({ id: 'r2' })],
- })
- expect(detail.properties).toMatchObject({ timestamp: '2026-09-01T10:15:00Z', sessionId: 'sess-1' })
- expect(readyResults(detail.evals)).toHaveLength(2)
- expect(detail.content).toEqual({
- kind: 'io',
- input: withEvent ? { question: 'Hi' } : undefined,
- output: undefined,
- })
- expect(detail.raw === null).toBe(!withEvent)
- })
+ ])(
+ 'trace root %s takes its time and session from the trace and shows every evaluation result',
+ (_name, withEvent) => {
+ const detail = toNodeDetail({
+ ...base,
+ node: node('trace-1', 'trace'),
+ load: ready(
+ makeNodeDetail({
+ id: 'trace-1',
+ kind: 'trace',
+ withEvent,
+ properties: { $ai_input_state: { question: 'Hi' } },
+ ioKeys: { input: '$ai_input_state' },
+ })
+ ),
+ evaluationResults: [result(), result({ id: 'r2' })],
+ })
+ expect(detail.properties).toMatchObject({ timestamp: '2026-09-01T10:15:00Z', sessionId: 'sess-1' })
+ expect(readyResults(detail.evals)).toHaveLength(2)
+ expect(detail.content).toEqual({
+ kind: 'io',
+ input: withEvent ? { question: 'Hi' } : undefined,
+ output: undefined,
+ })
+ expect(detail.raw === null).toBe(!withEvent)
+ }
+ )
it('renders an OTel span with a message array input and a text output as messages', () => {
const detail = toNodeDetail({
diff --git a/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.ts b/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.ts
index 27401e540e02..c1e965799e80 100644
--- a/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.ts
+++ b/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.ts
@@ -1,5 +1,4 @@
-import { EvaluationConfig, EvaluationRun } from '../../../evaluations/types'
-import type { TraceNodeApi, TraceNodeDetailApi } from '../../../generated/api.schemas'
+import type { TraceEvaluationResultApi, TraceNodeApi, TraceNodeDetailApi } from '../../../generated/api.schemas'
import { NodeDetailProps } from '../components/NodeDetail'
import { EvalsState, LoadFailure, NodeContent, NodeLoad, NodeProperties } from '../types'
import { NodeIO, readNodeIO } from './nodeIO'
@@ -19,11 +18,8 @@ interface NodeDetailArgs {
node: TraceNodeApi
load: NodeLoad
tree: TraceNodeApi[]
- evalRuns: EvaluationRun[]
- evalRunsLoading: boolean
- evaluations: EvaluationConfig[]
- detectorEvaluationIds: string[]
- evaluationsSettled: boolean
+ evaluationResults: TraceEvaluationResultApi[] | null
+ evaluationResultsFailed: boolean
teamId: TeamId
person: string | null
trace: { createdAt: string; sessionId: string | null }
@@ -78,31 +74,30 @@ function rawEvent(detail: TraceNodeDetailApi): Record<string, unknown> | null {
: null
}
-// Detector polarity comes from the evaluation configs, so results wait for them to avoid showing a flipped verdict.
-function evalsState(runs: EvaluationRun[], node: TraceNodeApi, args: NodeDetailArgs): EvalsState {
- if (args.evalRunsLoading || !args.evaluationsSettled) {
- return { status: 'loading' }
+const EVALS_LOAD_ERROR = 'Evaluation results could not be loaded. Reload the page to try again.'
+
+function evalsState(node: TraceNodeApi, args: NodeDetailArgs): EvalsState {
+ if (args.evaluationResultsFailed) {
+ return { status: 'error', errorMessage: EVALS_LOAD_ERROR }
}
- return {
- status: 'ready',
- results: toEvalResults(runs, {
- tree: args.tree,
- viewedNodeId: node.id,
- evaluations: args.evaluations,
- detectorEvaluationIds: args.detectorEvaluationIds,
- }),
+ if (!args.evaluationResults) {
+ return { status: 'loading' }
}
+ const results =
+ node.kind === 'trace'
+ ? args.evaluationResults
+ : args.evaluationResults.filter((result) => result.targetNodeId === node.id)
+ return { status: 'ready', results: toEvalResults(results, { tree: args.tree, viewedNodeId: node.id }) }
}
export function toNodeDetail(args: NodeDetailArgs): NodeDetailData {
- const { node, load, evalRuns, teamId } = args
- const runs = node.kind === 'trace' ? evalRuns : evalRuns.filter((run) => run.generation_id === node.id)
+ const { node, load, teamId } = args
return {
node,
content: loadedContent(load, node, teamId),
error: load.status === 'ready' ? load.node.error : null,
properties: nodeProperties(load, node, args),
- evals: evalsState(runs, node, args),
+ evals: evalsState(node, args),
raw: load.status === 'ready' ? rawEvent(load.node) : null,
}
}
diff --git a/products/ai_observability/frontend/redesign/trace/adapter/traceSceneWiring.test.tsx b/products/ai_observability/frontend/redesign/trace/adapter/traceSceneWiring.test.tsx
index 35072fd85e89..12ddf7a12f54 100644
--- a/products/ai_observability/frontend/redesign/trace/adapter/traceSceneWiring.test.tsx
+++ b/products/ai_observability/frontend/redesign/trace/adapter/traceSceneWiring.test.tsx
@@ -23,6 +23,7 @@ jest.mock('../../../utils', () => ({
const TRACE_URL = '/api/projects/:team_id/ai_observability/traces/:id/'
const NODE_URL = '/api/projects/:team_id/ai_observability/traces/:id/nodes/:node_id/'
+const EVALUATIONS_URL = '/api/projects/:team_id/ai_observability/traces/:id/evaluations/'
const answeredGeneration = makeNodeDetail({
id: 'gen-2',
properties: {
@@ -39,6 +40,7 @@ const nodeMock = {
return [200, nodeId === 'gen-2' ? answeredGeneration : makeNodeDetail({ id: nodeId })]
},
}
+const evaluationsMock = { [EVALUATIONS_URL]: () => [200, { results: [] }] }
const generation = makeEvent({
id: 'gen-1',
@@ -67,7 +69,13 @@ describe('TraceScene', () => {
describe('the new view', () => {
beforeEach(() => {
- useMocks({ get: { [TRACE_URL]: () => [200, makeTraceResource({ defaultNodeId: 'gen-2' })], ...nodeMock } })
+ useMocks({
+ get: {
+ [TRACE_URL]: () => [200, makeTraceResource({ defaultNodeId: 'gen-2' })],
+ ...nodeMock,
+ ...evaluationsMock,
+ },
+ })
})
it('opens on the default node without the legacy trace query', async () => {
@@ -94,7 +102,7 @@ describe('TraceScene', () => {
describe('behind the flag gate', () => {
beforeEach(() => {
- useMocks({ get: { [TRACE_URL]: () => [200, makeTraceResource()], ...nodeMock } })
+ useMocks({ get: { [TRACE_URL]: () => [200, makeTraceResource()], ...nodeMock, ...evaluationsMock } })
})
async function renderTraceRoute(flagEnabled: boolean): Promise<ReturnType<typeof render>> {
diff --git a/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.test.ts b/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.test.ts
index 39b16099d8f5..7dec0e91c45a 100644
--- a/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.test.ts
+++ b/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.test.ts
@@ -15,13 +15,10 @@ import { decodeIdSegment, makeNodeDetail, makeTraceResource } from './testFixtur
import { traceViewAdapterLogic } from './traceViewAdapterLogic'
jest.mock('~/queries/query', () => ({ ...jest.requireActual('~/queries/query'), performQuery: jest.fn() }))
-jest.mock('../../../utils', () => ({
- ...jest.requireActual('../../../utils'),
- queryEvaluationRuns: jest.fn().mockResolvedValue([]),
-}))
const TRACE_URL = '/api/projects/:team_id/ai_observability/traces/:id/'
const NODE_URL = '/api/projects/:team_id/ai_observability/traces/:id/nodes/:node_id/'
+const EVALUATIONS_URL = '/api/projects/:team_id/ai_observability/traces/:id/evaluations/'
const answeredGeneration = makeNodeDetail({
id: 'gen-2',
@@ -56,6 +53,7 @@ describe('traceViewAdapterLogic', () => {
await traceGate
return [200, traceResource]
},
+ [EVALUATIONS_URL]: () => [200, { results: [] }],
[NODE_URL]: ({ params }) => {
const nodeId = decodeIdSegment(String(params.node_id))
nodeRequests.push(nodeId)
diff --git a/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.ts b/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.ts
index 43eb6fdb93ba..19efefb44bb8 100644
--- a/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.ts
+++ b/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.ts
@@ -5,10 +5,7 @@ import { subscriptions } from 'kea-subscriptions'
import { teamLogic } from 'scenes/teamLogic'
import { aiObservabilityTraceLogic } from '../../../aiObservabilityTraceLogic'
-import { llmEvaluationsLogic } from '../../../evaluations/llmEvaluationsLogic'
-import { EvaluationConfig, EvaluationRun } from '../../../evaluations/types'
-import type { TraceNodeApi, TraceTimelineRowApi } from '../../../generated/api.schemas'
-import { generationEvaluationRunsLogic } from '../../../generationEvaluationRunsLogic'
+import type { TraceEvaluationResultApi, TraceNodeApi, TraceTimelineRowApi } from '../../../generated/api.schemas'
import { TraceHeaderProps } from '../components/TraceHeader'
import { TraceSummaryBarProps } from '../components/TraceSummaryBar'
import { TraceViewStatus, traceViewLogic } from '../traceViewLogic'
@@ -31,16 +28,13 @@ export interface traceViewAdapterLogicValues {
newerTraceId: string | null // aiObservabilityTraceLogic
olderTimestamp: string | null // aiObservabilityTraceLogic
olderTraceId: string | null // aiObservabilityTraceLogic
- generationEvaluationRuns: EvaluationRun[] // generationEvaluationRunsLogic
- generationEvaluationRunsLoading: boolean // generationEvaluationRunsLogic
- detectorEvaluationIds: string[] // llmEvaluationsLogic
- evaluations: EvaluationConfig[] // llmEvaluationsLogic
- evaluationsSettled: boolean // llmEvaluationsLogic
searchParams: Record<string, any> // router
currentTeamId: number | null // teamLogic
createdAt: string | null // traceViewLogic
defaultNodeId: string // traceViewLogic
errorMessage: string // traceViewLogic
+ evaluationResults: TraceEvaluationResultApi[] | null // traceViewLogic
+ evaluationResultsFailed: boolean // traceViewLogic
nodeIdsByAlias: Record<string, string> // traceViewLogic
nodes: Record<string, NodeLoad> // traceViewLogic
personLabel: string | null // traceViewLogic
@@ -144,11 +138,8 @@ export interface traceViewAdapterLogicMeta {
selectedTreeNode: TraceNodeApi | null,
selectedNodeLoad: NodeLoad,
tree: TraceNodeApi[],
- generationEvaluationRuns: EvaluationRun[],
- generationEvaluationRunsLoading: boolean,
- evaluations: EvaluationConfig[],
- detectorEvaluationIds: string[],
- evaluationsSettled: boolean,
+ evaluationResults: TraceEvaluationResultApi[] | null,
+ evaluationResultsFailed: boolean,
currentTeamId: number | null,
personLabel: string | null,
createdAt: string | null,
@@ -198,15 +189,13 @@ export const traceViewAdapterLogic = kea<traceViewAdapterLogicType>([
'sessionId',
'defaultNodeId',
'nodeIdsByAlias',
+ 'evaluationResults',
+ 'evaluationResultsFailed',
],
aiObservabilityTraceLogic,
['eventId', 'initialTab', 'olderTraceId', 'olderTimestamp', 'newerTraceId', 'newerTimestamp'],
- generationEvaluationRunsLogic({ traceId: props.traceId }),
- ['generationEvaluationRuns', 'generationEvaluationRunsLoading'],
teamLogic,
['currentTeamId'],
- llmEvaluationsLogic,
- ['evaluations', 'detectorEvaluationIds', 'evaluationsSettled'],
router,
['searchParams'],
],
@@ -309,11 +298,8 @@ export const traceViewAdapterLogic = kea<traceViewAdapterLogicType>([
s.selectedTreeNode,
s.selectedNodeLoad,
s.tree,
- s.generationEvaluationRuns,
- s.generationEvaluationRunsLoading,
- s.evaluations,
- s.detectorEvaluationIds,
- s.evaluationsSettled,
+ s.evaluationResults,
+ s.evaluationResultsFailed,
s.currentTeamId,
s.personLabel,
s.createdAt,
@@ -323,11 +309,8 @@ export const traceViewAdapterLogic = kea<traceViewAdapterLogicType>([
node: TraceNodeApi | null,
load: NodeLoad,
tree: TraceNodeApi[],
- evalRuns: EvaluationRun[],
- evalRunsLoading: boolean,
- evaluations: EvaluationConfig[],
- detectorEvaluationIds: string[],
- evaluationsSettled: boolean,
+ evaluationResults: TraceEvaluationResultApi[] | null,
+ evaluationResultsFailed: boolean,
teamId: number | null,
person: string | null,
createdAt: string | null,
@@ -338,11 +321,8 @@ export const traceViewAdapterLogic = kea<traceViewAdapterLogicType>([
node,
load,
tree,
- evalRuns,
- evalRunsLoading,
- evaluations,
- detectorEvaluationIds,
- evaluationsSettled,
+ evaluationResults,
+ evaluationResultsFailed,
teamId,
person,
trace: { createdAt, sessionId },
diff --git a/products/ai_observability/frontend/redesign/trace/components/EvalOutcomeTag.tsx b/products/ai_observability/frontend/redesign/trace/components/EvalOutcomeTag.tsx
index 7a54425a7fa5..8cbef21130fa 100644
--- a/products/ai_observability/frontend/redesign/trace/components/EvalOutcomeTag.tsx
+++ b/products/ai_observability/frontend/redesign/trace/components/EvalOutcomeTag.tsx
@@ -1,4 +1,4 @@
-import { IconCheck, IconClock, IconMinus, IconWarning, IconX } from '@posthog/icons'
+import { IconCheck, IconMinus, IconX } from '@posthog/icons'
import { LemonTag, LemonTagType } from '@posthog/lemon-ui'
import { EvalOutcome } from '../types'
@@ -6,8 +6,6 @@ import { EvalOutcome } from '../types'
const OUTCOME_STYLES: Record<EvalOutcome, { type: LemonTagType; icon: JSX.Element | undefined }> = {
pass: { type: 'success', icon: <IconCheck /> },
fail: { type: 'danger', icon: <IconX /> },
- error: { type: 'danger', icon: <IconWarning /> },
- pending: { type: 'default', icon: <IconClock /> },
inconclusive: { type: 'muted', icon: <IconMinus /> },
unrated: { type: 'default', icon: undefined },
}
diff --git a/products/ai_observability/frontend/redesign/trace/storyFixtures.tsx b/products/ai_observability/frontend/redesign/trace/storyFixtures.tsx
index 0219778bfdf4..7add40e950a3 100644
--- a/products/ai_observability/frontend/redesign/trace/storyFixtures.tsx
+++ b/products/ai_observability/frontend/redesign/trace/storyFixtures.tsx
@@ -339,28 +339,6 @@ export const FIXTURE_EVALS: EvalResult[] = [
isBackfill: false,
target: null,
},
- {
- id: 'e-tone',
- name: 'Tone',
- href: '/evaluations/tone',
- outcome: 'error',
- label: 'Error',
- reasoning: 'The judge model returned an empty response.',
- timestamp: EVAL_TIMESTAMP,
- isBackfill: false,
- target: null,
- },
- {
- id: 'e-policy',
- name: 'Follows billing policy',
- href: '/evaluations/follows-billing-policy',
- outcome: 'pending',
- label: 'Running',
- reasoning: null,
- timestamp: EVAL_TIMESTAMP,
- isBackfill: false,
- target: null,
- },
]
export const FIXTURE_TRACE_EVALS: EvalResult[] = FIXTURE_EVALS.map((result) => ({
diff --git a/products/ai_observability/frontend/redesign/trace/traceViewLogic.test.ts b/products/ai_observability/frontend/redesign/trace/traceViewLogic.test.ts
index ceb79aa19cd1..11f89295ea48 100644
--- a/products/ai_observability/frontend/redesign/trace/traceViewLogic.test.ts
+++ b/products/ai_observability/frontend/redesign/trace/traceViewLogic.test.ts
@@ -1,17 +1,21 @@
+import { waitFor } from '@testing-library/react'
import { expectLogic } from 'kea-test-utils'
import posthog from 'posthog-js'
+import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
+
import { resumeKeaLoadersErrors, silenceKeaLoadersErrors } from '~/initKea'
import { panelLayoutLogic } from '~/layout/panel-layout/panelLayoutLogic'
import { useMocks } from '~/mocks/jest'
import { initKeaTests } from '~/test/init'
import type { TraceTimelineRowApi } from '../../generated/api.schemas'
-import { decodeIdSegment, makeNodeDetail, makeTraceResource } from './adapter/testFixtures'
+import { decodeIdSegment, makeEvaluationResult, makeNodeDetail, makeTraceResource } from './adapter/testFixtures'
import { traceViewLogic } from './traceViewLogic'
const TRACE_URL = '/api/projects/:team_id/ai_observability/traces/:id/'
const NODE_URL = '/api/projects/:team_id/ai_observability/traces/:id/nodes/:node_id/'
+const EVALUATIONS_URL = '/api/projects/:team_id/ai_observability/traces/:id/evaluations/'
const EXPIRED_ROW: TraceTimelineRowApi = {
id: 'expired-1',
kind: 'span',
@@ -69,6 +73,7 @@ describe('traceViewLogic', () => {
requestedPath = url.pathname
return [200, makeTraceResource({ name: null })]
},
+ [EVALUATIONS_URL]: () => [200, { results: [] }],
},
})
initKeaTests()
@@ -128,6 +133,47 @@ describe('traceViewLogic', () => {
.toMatchValues({ status: 'error', errorMessage: expectedMessage })
})
+ it.each([
+ ['answers', 200, { evaluationResults: [makeEvaluationResult()], evaluationResultsFailed: false }, false],
+ ['fails', 500, { evaluationResults: null, evaluationResultsFailed: true }, true],
+ ] as const)(
+ 'reads the evaluation results alongside the trace and keeps the trace ready, without a toast, when that read %s',
+ async (_description, statusCode, expected, reported) => {
+ const toast = jest.spyOn(lemonToast, 'error')
+ const report = jest.spyOn(posthog, 'captureException')
+ const traceAnswer = deferred()
+ let traceAnswered = false
+ let evaluationsReadBeforeTrace = false
+ useMocks({
+ get: {
+ [TRACE_URL]: async () => {
+ await traceAnswer.promise
+ traceAnswered = true
+ return [200, makeTraceResource()]
+ },
+ [EVALUATIONS_URL]: () => {
+ evaluationsReadBeforeTrace = !traceAnswered
+ return statusCode === 200
+ ? [200, { results: [makeEvaluationResult()] }]
+ : [statusCode, { detail: 'nope' }]
+ },
+ },
+ })
+ const logic = traceViewLogic({ traceId: 'trace-1', timestampHint: null })
+ logic.mount()
+
+ await waitFor(() => expect(logic.values.evaluationResultsLoading).toBe(false))
+ expect(evaluationsReadBeforeTrace).toBe(true)
+ traceAnswer.resolve()
+
+ await expectLogic(logic)
+ .toFinishAllListeners()
+ .toMatchValues({ ...expected, status: 'ready' })
+ expect(toast).not.toHaveBeenCalled()
+ expect(report).toHaveBeenCalledTimes(reported ? 1 : 0)
+ }
+ )
+
it('loads the thread nodes once the trace arrives', async () => {
const logic = await mountedLogic()
diff --git a/products/ai_observability/frontend/redesign/trace/traceViewLogic.ts b/products/ai_observability/frontend/redesign/trace/traceViewLogic.ts
index 92282d95f313..253915f1b9d3 100644
--- a/products/ai_observability/frontend/redesign/trace/traceViewLogic.ts
+++ b/products/ai_observability/frontend/redesign/trace/traceViewLogic.ts
@@ -20,8 +20,18 @@ import { urls } from 'scenes/urls'
import { panelLayoutLogic } from '~/layout/panel-layout/panelLayoutLogic'
-import { aiObservabilityTracesNodesRetrieve, aiObservabilityTracesRetrieve } from '../../generated/api'
-import type { TraceApi, TraceNodeApi, TraceNodeDetailApi, TraceTimelineRowApi } from '../../generated/api.schemas'
+import {
+ aiObservabilityTracesEvaluationsRetrieve,
+ aiObservabilityTracesNodesRetrieve,
+ aiObservabilityTracesRetrieve,
+} from '../../generated/api'
+import type {
+ TraceApi,
+ TraceEvaluationResultApi,
+ TraceNodeApi,
+ TraceNodeDetailApi,
+ TraceTimelineRowApi,
+} from '../../generated/api.schemas'
import { TraceSummaryBarProps } from './components/TraceSummaryBar'
import { encodeTraceIdSegment } from './traceIdSegment'
import type { LoadFailure, NodeLoad, NodeLoadTrigger } from './types'
@@ -38,6 +48,9 @@ export interface traceViewLogicValues {
createdAt: string | null
defaultNodeId: string
errorMessage: string
+ evaluationResults: TraceEvaluationResultApi[] | null
+ evaluationResultsFailed: boolean
+ evaluationResultsLoading: boolean
loadFailure: LoadFailure | null
nodeIdsByAlias: Record<string, string>
nodes: Record<string, NodeLoad>
@@ -67,6 +80,21 @@ export interface traceViewLogicActions {
collapsed: boolean | null
owner: string
} // panelLayoutLogic
+ loadEvaluationResults: () => any
+ loadEvaluationResultsFailure: (
+ error: string,
+ errorObject?: any
+ ) => {
+ error: string
+ errorObject?: any
+ }
+ loadEvaluationResultsSuccess: (
+ evaluationResults: TraceEvaluationResultApi[],
+ payload?: any
+ ) => {
+ evaluationResults: TraceEvaluationResultApi[]
+ payload?: any
+ }
loadNode: (
nodeId: string,
trigger: NodeLoadTrigger
@@ -185,8 +213,27 @@ export const traceViewLogic = kea<traceViewLogicType>([
},
},
],
+ evaluationResults: [
+ null as TraceEvaluationResultApi[] | null,
+ {
+ loadEvaluationResults: async (): Promise<TraceEvaluationResultApi[]> => {
+ const { results } = await aiObservabilityTracesEvaluationsRetrieve(
+ String(ApiConfig.getCurrentProjectId()),
+ encodeTraceIdSegment(props.traceId)
+ )
+ return results
+ },
+ },
+ ],
})),
reducers({
+ evaluationResultsFailed: [
+ false,
+ {
+ loadEvaluationResults: () => false,
+ loadEvaluationResultsFailure: () => true,
+ },
+ ],
loadFailure: [
null as LoadFailure | null,
{
@@ -341,6 +388,7 @@ export const traceViewLogic = kea<traceViewLogicType>([
}),
afterMount(({ actions, props }) => {
actions.loadTrace()
+ actions.loadEvaluationResults()
actions.setLayoutNavCollapsedOverride(layoutNavOverrideOwner(props), true)
}),
beforeUnmount(({ actions, props }) => {
diff --git a/products/ai_observability/frontend/redesign/trace/types.ts b/products/ai_observability/frontend/redesign/trace/types.ts
index 864a6a6271f5..ee40660dc84d 100644
--- a/products/ai_observability/frontend/redesign/trace/types.ts
+++ b/products/ai_observability/frontend/redesign/trace/types.ts
@@ -1,4 +1,4 @@
-import type { TraceNodeApi, TraceNodeDetailApi } from '../../generated/api.schemas'
+import type { TraceEvaluationResultOutcomeEnumApi, TraceNodeApi, TraceNodeDetailApi } from '../../generated/api.schemas'
export type TraceNodeKind = TraceNodeApi['kind']
@@ -77,7 +77,7 @@ export interface NodeProperties {
* `unrated` is a result with a value but no verdict, such as a numeric score without a passing rule.
* `inconclusive` is a run that produced no usable value: skipped, not applicable, no score, unknown sentiment.
*/
-export type EvalOutcome = 'pass' | 'fail' | 'error' | 'pending' | 'inconclusive' | 'unrated'
+export type EvalOutcome = TraceEvaluationResultOutcomeEnumApi
export interface EvalTarget {
label: string
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index e282de2dfdd4..73403f56343b 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -116069,6 +116069,34 @@ export namespace Schemas {
nodeIdsByAlias: TraceNodeIdsByAlias;
}
+ export type TraceEvaluationResultOutcomeEnum = typeof TraceEvaluationResultOutcomeEnum[keyof typeof TraceEvaluationResultOutcomeEnum];
+
+
+ export const TraceEvaluationResultOutcomeEnum = {
+ Pass: 'pass',
+ Fail: 'fail',
+ Inconclusive: 'inconclusive',
+ Unrated: 'unrated',
+ } as const;
+
+ export interface TraceEvaluationResult {
+ id: string;
+ evaluationId: string | null;
+ evaluationName: string;
+ /** The tree node the result grades: the generation's event uuid, or the trace id for a trace-level result. */
+ targetNodeId: string;
+ outcome: TraceEvaluationResultOutcomeEnum;
+ /** What the result says, such as "True", "0.82", "Positive" or "Skipped". */
+ label: string;
+ reasoning: string | null;
+ timestamp: string;
+ isBackfill: boolean;
+ }
+
+ export interface TraceEvaluationResults {
+ results: TraceEvaluationResult[];
+ }
+
export interface TraceNodeMetadata {
provider: string | null;
temperature: number | null;