:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): serve trace evaluation results from an evaluations resource (#121264)
Carlos Marchal44caa527ed39
diff --git a/.github/new-events-schema-targets.txt b/.github/new-events-schema-targets.txtindex 21fc710e4fba..f995fa0285d1 100644--- a/.github/new-events-schema-targets.txt+++ b/.github/new-events-schema-targets.txt@@ -25,6 +25,8 @@ posthog/clickhouse/testee/clickhouseproducts/ai_observability/backend/tests/logic/evaluationsproducts/ai_observability/backend/tests/logic/traces+products/ai_observability/backend/tests/test_facade_api.py+products/ai_observability/backend/tests/test_trace_views.pyproducts/cohortsproducts/customer_analytics/backend/hogql_queriesproducts/error_tracking/backend/hogql_queriesdiff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex 6c7c9c047751..ac6bdf1d4429 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -9155,17 +9155,17 @@ snapshots:scenes-app-ai-observability-trace-view-eval-results--loading--light:hash: v1.k794b7964.c2193c953b9f6410a018135730f03a16effbae102e843e7cb86d31a585ff25a0.ZnwmN5ZhvryitB3fPLZmzGRm31kQtra85ZjR7rR2Wdgscenes-app-ai-observability-trace-view-eval-results--narrow--dark:- hash: v1.k794b7964.0d9f5ea29085fdce3b5416a6cd7629cf7487718b3287170e35a0444ce84f7130.wjDaCjEydB47et5XF4RnA6PcbBO28LdcXTXUdgJkqOg+ hash: v1.k794b7964.c180c03264d4b4dd70fc1d8ffe79d4793914955f7c3ff4b2b0baea6800789935.DeEtCZVUgqNNRgneYw2hJ9aC_SKsli-b515v_sX1eJ8scenes-app-ai-observability-trace-view-eval-results--narrow--light:- hash: v1.k794b7964.066e10797c7eeef745043b8e1c5bd2f6f726db2dc927ffabef178c65f92ae47c.lcwsAGwJT5CDGqDhrFvHUGPF8K-C_VpfqDlMnf1v8yM+ hash: v1.k794b7964.aff05e292bff56b5de4d3375e7d3029cdf436ff67671fc31302cc7b969f08322.DChS6K6n3sd7HCl9EcQ7pRcWh65hAY4zhAt2q5bD9pwscenes-app-ai-observability-trace-view-eval-results--results--dark:- hash: v1.k794b7964.6878e01169aa5df1d0a472e54502445cb4dac02f049402c42bd77405678202ad.joUtOO1BtYJsiG77d2Oor5OLT2b0ojMFPXxBUcvMV4w+ hash: v1.k794b7964.7d4e5284a5dd4eaaf6ba53e25fb91fc10df27d98d044a8707fed755869fac300.tXQXVBQUsx6yiC3RlpEWlOAxWQXoys_i3TtiI9BmGaEscenes-app-ai-observability-trace-view-eval-results--results--light:- hash: v1.k794b7964.28feede2cc8e1490cacc3434a878109a1b3e891c5d335fa4cfd5e8b2c12a7d4c.1hm9PJZxIAjDh8vetWyBPYQds9ck3Vwu0nJdGafQNnQ+ hash: v1.k794b7964.dbe4f1d49b4c48fbd58d59105298f3543459aaa4ebef54ffa342b9d2e97302ed.InRPlKGDDdjouYufMdxD5brZbWnsjQFrXvaiHLoQHlgscenes-app-ai-observability-trace-view-eval-results--with-targets--dark:- hash: v1.k794b7964.3668c4cab9a050d2d21e9cd1274b5e348da24fedf3264ede6cc9a19c59be0eae.8Zc5q6EnyO2a8nC1KKbD86vECGtDU-mLzxroEIqaQks+ hash: v1.k794b7964.bb8fdbede3c8fc1006a9a8f5d3be808341f36237c56c38f3244e11cd248d47ef.4RE1N5YlNm27-UwPmtDgtDifGqEhzT89XUXQrWSC59wscenes-app-ai-observability-trace-view-eval-results--with-targets--light:- hash: v1.k794b7964.5b99a44a4fe9414c181d52304bee27dc0411488565499a069379edfca998ac53.xqNk1JI7RXzSiB36VQL2Tx8jNZjLqTB3V3y-7UVMqLw+ hash: v1.k794b7964.2561e19f756b8b0c9c54e3926d2740a17edefed875f822f5c40db6e9350d0e99.qy2zcYWPbI9yn-QMrXugQISB2l1hJcsLAIkCdcjm_A0scenes-app-ai-observability-trace-view-input-and-output--empty-object--dark:hash: v1.k794b7964.6ed3f97001d8b595545a78f37627d3e09c188a3dcd782c8ab3389c65cbd4210f.2U9W7fXNr-OZRXfkmXy1ICFu5-dSKZUsMMWOtbf4PWwscenes-app-ai-observability-trace-view-input-and-output--empty-object--light:@@ -9287,9 +9287,9 @@ snapshots:scenes-app-ai-observability-trace-view-node-detail--evals-load-error--light:hash: v1.k794b7964.50777b9d2294d5daefde05454f4d88c77100bc9cf1bb00cba7417e45659441de.zNzJetAUSwx_LsQxfGzn5GiP0drok5qlGDjtdBA1f74scenes-app-ai-observability-trace-view-node-detail--evals-tab--dark:- hash: v1.k794b7964.b32052d233df672ac33d635e9e60682b279b0a85b8197d5c037923f1060ecb67.pjxZIIGAyjrunK5zd2J1auEwmwS2okFOGDJHtPnCBS0+ hash: v1.k794b7964.f6d7647543214d678b4ba05b43ed8d4e6fa2d2ba949845252c5e357831b6f8a1.QXLvkyt9b16kwXPe_oYPlGWeHWI-dnm2zsRyLmlyH9Yscenes-app-ai-observability-trace-view-node-detail--evals-tab--light:- hash: v1.k794b7964.56c0cd429a52b1707971a0d892f22346a5757ed5eb5bd2e576305ab85d5daaf5.HHwjHccgpak73NKgv5Gfmd5RhUWYNgNe1x4qE6iJxDw+ hash: v1.k794b7964.83491c8cf61cb4d613f16ad9ef759f959b4771365e890f956d5e15c10ee9c98d.UKEhF6lfSpzMVVnf3fEqWcoThCHvNqV8hWjmsC92TpMscenes-app-ai-observability-trace-view-node-detail--failed-generation-with-image--dark:hash: v1.k794b7964.c161e39c0753e90108d0026cbc0bdc6963e8540346685edf024fd8dedf32ec4c.A2Eid0pxBKQqtQ4wohIT_9dHMqvB3fQB5UigMmuDLDAscenes-app-ai-observability-trace-view-node-detail--failed-generation-with-image--light:diff --git a/frontend/src/initKea.ts b/frontend/src/initKea.tsindex a4190a8ea0ce..22afad8e652f 100644--- a/frontend/src/initKea.ts+++ b/frontend/src/initKea.ts@@ -127,6 +127,7 @@ const ERROR_FILTER_ALLOW_LIST_BY_LOGIC_PATH: Record<string, string> = {createProviderKey: 'products.ai_observability.settings.llmProviderKeysLogic',updateProviderKey: 'products.ai_observability.settings.llmProviderKeysLogic',deleteProviderKey: 'products.ai_observability.settings.llmProviderKeysLogic',+ loadEvaluationResults: 'products.ai_observability.frontend.redesign.trace.traceViewLogic', // The trace view's Evals tab shows the error}// A keyed logic's path ends with its key, so match the owner path as a prefix.diff --git a/products/ai_observability/backend/facade/api.py b/products/ai_observability/backend/facade/api.pyindex 6c012cfd5202..faa371330d0a 100644--- a/products/ai_observability/backend/facade/api.py+++ b/products/ai_observability/backend/facade/api.py@@ -3,6 +3,8 @@ from datetime import datetimefrom posthog.models import Team, Userfrom ..logic.evaluations.pass_filters import build_evaluation_pass_filters+from ..logic.traces.evaluation_result import EvaluationResult+from ..logic.traces.evaluation_result_queries import load_evaluation_results, load_output_configsfrom ..logic.traces.events_fallback import fallback_windowfrom ..logic.traces.node import NodeEventfrom ..logic.traces.timeline import TimelineRow@@ -155,3 +157,32 @@ def get_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFiltersor categorical passing rule, and no skipped, N/A or sentiment runs. The conditions change when an evaluation'soutput config changes, so a caller that stores them rebuilds them on `evaluation_output_changed`."""return build_evaluation_pass_filters(team_id)+++def get_trace_evaluation_results(team: Team, user: User | None, trace_id: str) -> contracts.TraceEvaluationResults:+ rows = load_evaluation_results(team, user, trace_id)+ configs = load_output_configs(team, user, {row.evaluation_id for row in rows if row.evaluation_id})+ return contracts.TraceEvaluationResults(+ results=[+ _to_evaluation_result(+ EvaluationResult.from_row(+ row, trace_id=trace_id, output_config=configs.get(row.evaluation_id or "", {})+ )+ )+ for row in rows+ ]+ )+++def _to_evaluation_result(result: EvaluationResult) -> contracts.TraceEvaluationResult:+ return contracts.TraceEvaluationResult(+ id=result.id,+ evaluation_id=result.evaluation_id,+ evaluation_name=result.name,+ target_node_id=result.target_node_id,+ outcome=result.outcome,+ label=result.label,+ reasoning=result.reasoning,+ timestamp=result.timestamp,+ is_backfill=result.is_backfill,+ )diff --git a/products/ai_observability/backend/facade/contracts.py b/products/ai_observability/backend/facade/contracts.pyindex 9f1669cad03e..d7eac52720d5 100644--- a/products/ai_observability/backend/facade/contracts.py+++ b/products/ai_observability/backend/facade/contracts.py@@ -109,3 +109,24 @@ class EvaluationPassFilters(Contract):graded: str # The run counts toward a pass rate.passed: str # The run passed. Implies `graded`.+++EvaluationOutcome = Literal["pass", "fail", "inconclusive", "unrated"]+++class TraceEvaluationResult(Contract):+ id: str+ evaluation_id: str | None+ evaluation_name: str+ target_node_id: str = Field(+ description="The tree node the result grades: the generation's event uuid, or the trace id for a trace-level result."+ )+ outcome: EvaluationOutcome+ label: str = Field(description='What the result says, such as "True", "0.82", "Positive" or "Skipped".')+ reasoning: str | None+ timestamp: datetime+ is_backfill: bool+++class TraceEvaluationResults(Contract):+ results: list[TraceEvaluationResult]diff --git a/products/ai_observability/backend/logic/traces/evaluation_result.py b/products/ai_observability/backend/logic/traces/evaluation_result.pynew file mode 100644index 000000000000..3e940ef859c7--- /dev/null+++ b/products/ai_observability/backend/logic/traces/evaluation_result.py@@ -0,0 +1,168 @@+import json+from collections.abc import Mapping+from datetime import datetime+from typing import Self++from posthog.dataclasses import frozen++from ...facade.contracts import EvaluationOutcome+from ...models.evaluation_configs import OutputType+from ...models.evaluation_outcomes import EvaluationReportOutcomeDefinition, get_outcome_definition+from .evaluation_result_queries import EvaluationResultRow+from .trace_queries import canonical_uuid, read_flag, read_number, read_text++_SENTIMENT_OUTCOMES: dict[str, EvaluationOutcome] = {"positive": "pass", "neutral": "unrated", "negative": "fail"}+UNKNOWN_EVALUATION_NAME = "Unknown evaluation"+++def _outcome_definition(+ result_type: OutputType, output_config: Mapping[str, object]+) -> EvaluationReportOutcomeDefinition | None:+ # Stored configs are not revalidated on read, so a config that does not parse grades nothing.+ try:+ return get_outcome_definition(+ result_type, true_is_failure=output_config.get("true_is_failure") is True, output_config=dict(output_config)+ )+ except ValueError:+ return None+++def _passes(+ definition: EvaluationReportOutcomeDefinition | None, result: object, applicable: bool | None+) -> bool | None:+ outcome = definition.label_for(result, applicable) if definition else None+ return outcome == "pass" if outcome in ("pass", "fail") else None+++@frozen+class _Display:+ outcome: EvaluationOutcome+ label: str+++_NOT_APPLICABLE = _Display(outcome="inconclusive", label="N/A")+++def _graded(passed: bool | None, label: str) -> _Display:+ return _Display(outcome="unrated" if passed is None else "pass" if passed else "fail", label=label)+++def _format_score(score: float) -> str:+ rounded = float(f"{score:.2f}") if abs(score) >= 1 else float(f"{score:.2g}")+ return str(int(rounded)) if rounded.is_integer() else repr(rounded)+++def _result_type(raw: str | None) -> OutputType | None:+ return OutputType(raw) if raw in OutputType.values else None+++def _parse_categories(raw: str | None, applicable: str | None) -> list[str] | None:+ if raw is None:+ return [] if read_flag(applicable) else None+ try:+ categories = json.loads(raw)+ except ValueError:+ return None+ if isinstance(categories, list) and all(isinstance(category, str) for category in categories):+ return categories+ return None+++@frozen+class _RawResult:+ result_type: OutputType+ is_sentiment: bool+ sentiment_label: str | None+ result: bool | None+ applicable: bool | None+ skipped: bool+ score: float | None+ categories: list[str] | None++ @classmethod+ def from_row(cls, row: EvaluationResultRow) -> Self:+ sentiment_label = read_text(row.sentiment_label)+ result_type = _result_type(row.result_type) or (+ OutputType.SENTIMENT if row.evaluation_type == "sentiment" or sentiment_label else OutputType.BOOLEAN+ )+ not_applicable = row.applicable == "false"+ return cls(+ result_type=result_type,+ is_sentiment=result_type == OutputType.SENTIMENT+ or row.evaluation_type == "sentiment"+ or sentiment_label is not None,+ sentiment_label=sentiment_label,+ result=(+ None+ if result_type != OutputType.BOOLEAN or not_applicable or row.result is None+ else read_flag(row.result)+ ),+ applicable=None if row.applicable is None else read_flag(row.applicable),+ skipped=read_flag(row.skipped),+ score=read_number(row.score) if result_type == OutputType.NUMERIC and not not_applicable else None,+ categories=(+ _parse_categories(row.categories, row.applicable)+ if result_type == OutputType.CATEGORICAL and not not_applicable+ else None+ ),+ )++ def display(self, definition: EvaluationReportOutcomeDefinition | None) -> _Display:+ if self.skipped:+ return _Display(outcome="inconclusive", label="Skipped")+ if self.result_type == OutputType.CATEGORICAL:+ if self.applicable is False:+ return _NOT_APPLICABLE+ if self.categories is None:+ return _Display(outcome="inconclusive", label="No result")+ config = definition.categorical_config if definition else None+ names = {option.key: option.label for option in config.options} if config else {}+ label = ", ".join(names.get(key, key) for key in self.categories)+ return _graded(_passes(definition, self.categories, self.applicable), label or "No categories")+ if self.result_type == OutputType.NUMERIC:+ if self.applicable is False:+ return _NOT_APPLICABLE+ if self.score is None:+ return _Display(outcome="inconclusive", label="No score")+ passed = _passes(definition, self.score, self.applicable)+ label = _format_score(self.score)+ # A rounded label that would pass where the score fails, or the reverse, shows the score unrounded.+ rounded_passes = _passes(definition, float(label), self.applicable)+ return _graded(passed, label if rounded_passes == passed else str(self.score))+ if self.is_sentiment:+ sentiment = (self.sentiment_label or "unknown").lower()+ return _Display(+ outcome=_SENTIMENT_OUTCOMES.get(sentiment, "inconclusive"), label=sentiment[0].upper() + sentiment[1:]+ )+ if self.result is None:+ return _NOT_APPLICABLE+ return _graded(_passes(definition, self.result, self.applicable), "True" if self.result else "False")+++@frozen+class EvaluationResult:+ id: str+ evaluation_id: str | None+ name: str+ target_node_id: str+ outcome: EvaluationOutcome+ label: str+ reasoning: str | None+ timestamp: datetime+ is_backfill: bool++ @classmethod+ def from_row(cls, row: EvaluationResultRow, *, trace_id: str, output_config: Mapping[str, object]) -> Self:+ raw = _RawResult.from_row(row)+ display = raw.display(_outcome_definition(raw.result_type, output_config))+ return cls(+ id=row.uuid,+ evaluation_id=row.evaluation_id if row.evaluation_id and canonical_uuid(row.evaluation_id) else None,+ name=read_text(row.evaluation_name) or UNKNOWN_EVALUATION_NAME,+ target_node_id=read_text(row.target_event_id) or trace_id,+ outcome=display.outcome,+ label=display.label,+ reasoning=read_text(row.reasoning),+ timestamp=row.timestamp,+ is_backfill=bool(read_text(row.backfill_id)),+ )diff --git a/products/ai_observability/backend/logic/traces/evaluation_result_queries.py b/products/ai_observability/backend/logic/traces/evaluation_result_queries.pynew file mode 100644index 000000000000..5201b4379fa5--- /dev/null+++ b/products/ai_observability/backend/logic/traces/evaluation_result_queries.py@@ -0,0 +1,136 @@+from collections.abc import Iterable, Mapping, Sequence+from datetime import datetime+from typing import Any++from django.db.models import QuerySet++from posthog.hogql import ast+from posthog.hogql.context import HogQLContext+from posthog.hogql.database.database import Database+from posthog.hogql.query import execute_hogql_query++from posthog.clickhouse.query_tagging import Product, tag_queries, tags_context+from posthog.dataclasses import frozen+from posthog.models import Team, User++from products.access_control.backend.facade.user_access_control import UserAccessControl++from ...models.evaluations import Evaluation+from .trace_queries import canonical_uuid++EVALUATION_RESULTS_LIMIT = 250+++@frozen+class EvaluationResultRow:+ uuid: str+ timestamp: datetime+ evaluation_id: str | None+ evaluation_name: str | None+ target_event_id: str | None+ result: str | None+ reasoning: str | None+ applicable: str | None+ evaluation_type: str | None+ result_type: str | None+ sentiment_label: str | None+ skipped: str | None+ score: str | None+ categories: str | None+ backfill_id: str | None+++_PROPERTY_COLUMNS = {+ "evaluation_id": "$ai_evaluation_id",+ "evaluation_name": "$ai_evaluation_name",+ "target_event_id": "$ai_target_event_id",+ "result": "$ai_evaluation_result",+ "reasoning": "$ai_evaluation_reasoning",+ "applicable": "$ai_evaluation_applicable",+ "evaluation_type": "$ai_evaluation_runtime",+ "result_type": "$ai_evaluation_result_type",+ "sentiment_label": "$ai_sentiment_label",+ "skipped": "$ai_evaluation_skipped",+ "score": "$ai_evaluation_numeric_result",+ "categories": "$ai_evaluation_categorical_result",+ "backfill_id": "$ai_evaluation_backfill_id",+}+++def _equals(left: ast.Expr, value: str) -> ast.Expr:+ return ast.CompareOperation(op=ast.CompareOperationOp.Eq, left=left, right=ast.Constant(value=value))+++def _results_query(trace_id: str) -> ast.SelectQuery:+ return ast.SelectQuery(+ select=[+ ast.Field(chain=["uuid"]),+ ast.Field(chain=["timestamp"]),+ *(ast.Field(chain=["properties", name]) for name in _PROPERTY_COLUMNS.values()),+ ],+ select_from=ast.JoinExpr(table=ast.Field(chain=["events"])),+ where=ast.And(+ exprs=[+ _equals(ast.Field(chain=["event"]), "$ai_evaluation"),+ _equals(ast.Field(chain=["properties", "$ai_trace_id"]), trace_id),+ ]+ ),+ order_by=[ast.OrderExpr(expr=ast.Field(chain=["timestamp"]), order="DESC")],+ limit_by=ast.LimitByExpr(n=ast.Constant(value=1), exprs=[ast.Field(chain=["uuid"])]),+ limit=ast.Constant(value=EVALUATION_RESULTS_LIMIT),+ )+++def _raw(value: Any) -> str | None:+ # A property with a Boolean definition reads as a Python bool, so it is written the way untyped JSON reads.+ if isinstance(value, bool):+ return "true" if value else "false"+ return None if value is None else str(value)+++def _result_row(values: Sequence[Any]) -> EvaluationResultRow:+ uuid, timestamp, *properties = values+ return EvaluationResultRow(+ uuid=str(uuid),+ timestamp=timestamp,+ **{field: _raw(value) for field, value in zip(_PROPERTY_COLUMNS, properties, strict=True)},+ )+++def load_evaluation_results(team: Team, user: User | None, trace_id: str) -> tuple[EvaluationResultRow, ...]:+ name = "ai_trace_evaluation_results"+ with tags_context(product=Product.LLM_ANALYTICS):+ tag_queries(name=name)+ response = execute_hogql_query(+ query=_results_query(trace_id),+ team=team,+ user=user,+ query_type=name,+ context=HogQLContext(+ team_id=team.pk, team=team, user=user, database=Database.create_for_posthog_tables(team)+ ),+ )+ return tuple(_result_row(values) for values in response.results or [])+++def load_output_configs(+ team: Team, user: User | None, evaluation_ids: Iterable[str]+) -> dict[str, Mapping[str, object]]:+ """Keyed by each id as given, not by its canonical UUID form."""+ # Results can come from outside PostHog with any id, and a malformed UUID in the lookup raises.+ ids_by_uuid = {uuid: evaluation_id for evaluation_id in evaluation_ids if (uuid := canonical_uuid(evaluation_id))}+ if not ids_by_uuid:+ return {}+ # A deleted detector's results still read as a detector's, so deleted evaluations stay in.+ evaluations: QuerySet[Evaluation] = Evaluation.objects.filter(team_id=team.pk, id__in=ids_by_uuid)+ if user is not None:+ # Evaluations have their own access control, apart from the llm_analytics access that opens the trace.+ evaluations = UserAccessControl(user=user, team=team).filter_queryset_by_access_level(+ evaluations, resource="evaluation"+ )+ rows = evaluations.values_list("id", "output_config")+ return {+ ids_by_uuid[str(evaluation_id)]: output_config+ for evaluation_id, output_config in rows+ if isinstance(output_config, Mapping)+ }diff --git a/products/ai_observability/backend/logic/traces/trace_queries.py b/products/ai_observability/backend/logic/traces/trace_queries.pyindex e6e77c1dfe4b..fb87916d6559 100644--- a/products/ai_observability/backend/logic/traces/trace_queries.py+++ b/products/ai_observability/backend/logic/traces/trace_queries.py@@ -373,7 +373,7 @@ def load_trace(team: Team, user: User | None, trace_id: str, timestamp_hint: datreturn LoadedTrace(rows=rows, person=person)-def _canonical_uuid(value: str) -> str | None:+def canonical_uuid(value: str) -> str | None:try:return str(UUID(value))except ValueError:@@ -394,7 +394,7 @@ def _trace_node_row(values: Sequence[Any]) -> TraceNodeRow:def load_trace_node(team: Team, user: User | None, trace_id: str, node_id: str, timestamp_hint: datetime | None) -> TraceNodeRow | None:- event_uuid = _canonical_uuid(node_id)+ event_uuid = canonical_uuid(node_id)if event_uuid is None:return Noneresults = _read_trace(diff --git a/products/ai_observability/backend/presentation/views.py b/products/ai_observability/backend/presentation/views.pyindex 2d0dcd3f9129..e8c27e757109 100644--- a/products/ai_observability/backend/presentation/views.py+++ b/products/ai_observability/backend/presentation/views.py@@ -48,7 +48,7 @@ THROTTLED_RESPONSE = OpenApiResponse(description="Too many queries are running fclass TraceViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):scope_object = "llm_analytics"- scope_object_read_actions = ["retrieve", "node"]+ scope_object_read_actions = ["retrieve", "node", "evaluations"]# Traces are project-wide data, so a grant on a single llm_analytics object must not open them.requires_resource_level_access = Trueserializer_class = _FallbackSerializer@@ -108,3 +108,22 @@ class TraceViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):except api.TraceNodeNotFoundError:return Response({"detail": "Node not found."}, status=status.HTTP_404_NOT_FOUND)return Response(node.model_dump(mode="json", by_alias=True))++ @validated_request(+ parameters=[TRACE_ID_PARAMETER],+ responses={+ 200: OpenApiResponse(response=contracts.TraceEvaluationResults),+ 400: OpenApiResponse(description="The encoded trace id is malformed."),+ 429: THROTTLED_RESPONSE,+ },+ operation_id="ai_observability_traces_evaluations_retrieve",+ )+ @action(detail=True, methods=["GET"], url_path="evaluations")+ def evaluations(self, request: ValidatedRequest, pk: str, **kwargs: Any) -> Response:+ """A trace's evaluation results, newest first and at most 250, each with its outcome and label ready to render."""+ try:+ trace_id = decode_trace_id_segment(pk)+ except MalformedTraceIdSegmentError:+ return Response({"detail": "Malformed encoded trace id."}, status=status.HTTP_400_BAD_REQUEST)+ results = self._read(request, lambda user, _hint: api.get_trace_evaluation_results(self.team, user, trace_id))+ return Response(results.model_dump(mode="json", by_alias=True))diff --git a/products/ai_observability/backend/tests/logic/traces/rows.py b/products/ai_observability/backend/tests/logic/traces/rows.pyindex e1eb34bac822..422a7f071b30 100644--- a/products/ai_observability/backend/tests/logic/traces/rows.py+++ b/products/ai_observability/backend/tests/logic/traces/rows.py@@ -1,9 +1,14 @@+from collections.abc import Mappingfrom datetime import UTC, datetimefrom typing import Any+from posthog.models import Team++from products.ai_observability.backend.logic.traces.evaluation_result_queries import EvaluationResultRowfrom products.ai_observability.backend.logic.traces.trace_queries import TraceEventRowT0 = datetime(2026, 9, 1, 10, 0, 0, tzinfo=UTC)+EVALUATION_ID = "0192f2a4-5c1e-7000-8000-000000000e01"def make_row(**overrides: Any) -> TraceEventRow:@@ -31,3 +36,43 @@ def make_row(**overrides: Any) -> TraceEventRow:"ingestion_source": None,}return TraceEventRow(**{**fields, **overrides})+++def make_evaluation_result_row(**overrides: Any) -> EvaluationResultRow:+ fields: dict[str, Any] = {+ "uuid": "result-1",+ "timestamp": T0,+ "evaluation_id": EVALUATION_ID,+ "evaluation_name": "Answers the question",+ "target_event_id": None,+ "result": "true",+ "reasoning": "Direct answer.",+ "applicable": None,+ "evaluation_type": "llm_judge",+ "result_type": None,+ "sentiment_label": None,+ "skipped": None,+ "score": None,+ "categories": None,+ "backfill_id": None,+ }+ return EvaluationResultRow(**{**fields, **overrides})+++def make_evaluation_result_event(+ *,+ uuid: str,+ team: Team,+ timestamp: datetime,+ trace_id: str = "trace-1",+ event: str = "$ai_evaluation",+ properties: Mapping[str, object] | None = None,+) -> dict[str, object]:+ return {+ "event": event,+ "event_uuid": uuid,+ "distinct_id": "user-1",+ "team": team,+ "timestamp": timestamp,+ "properties": {"$ai_trace_id": trace_id, "$ai_evaluation_result": True, **(properties or {})},+ }diff --git a/products/ai_observability/backend/tests/logic/traces/test_evaluation_result.py b/products/ai_observability/backend/tests/logic/traces/test_evaluation_result.pynew file mode 100644index 000000000000..f82aa7942f81--- /dev/null+++ b/products/ai_observability/backend/tests/logic/traces/test_evaluation_result.py@@ -0,0 +1,177 @@+from typing import Any++from parameterized import parameterized++from products.ai_observability.backend.logic.traces.evaluation_result import EvaluationResult++from .rows import EVALUATION_ID, make_evaluation_result_row++DETECTOR = {"true_is_failure": True}+AT_LEAST_0_7 = {"passing_rule": {"operator": "gte", "threshold": 0.7}}+BILLING_ONLY = {+ "options": [{"key": "billing", "label": "Billing"}, {"key": "refund", "label": "Refund"}],+ "passing_rule": {"categories": ["billing"]},+}+++def evaluation_result(overrides: dict[str, Any], output_config: dict[str, Any] | None = None) -> EvaluationResult:+ return EvaluationResult.from_row(+ make_evaluation_result_row(**overrides), trace_id="trace-1", output_config=output_config or {}+ )+++ [+ ("a true result", {}, {}, "pass", "True"),+ ("a false result", {"result": "false"}, {}, "fail", "False"),+ ("a result written as True", {"result": "True"}, {}, "pass", "True"),+ ("a true result from a detector", {}, DETECTOR, "fail", "True"),+ ("a false result from a detector", {"result": "false"}, DETECTOR, "pass", "False"),+ (+ "a skipped run whose result is still false",+ {"skipped": "true", "result": "false"},+ {},+ "inconclusive",+ "Skipped",+ ),+ ("a missing result", {"result": None}, {}, "inconclusive", "N/A"),+ ("a result marked not applicable", {"applicable": "false"}, {}, "inconclusive", "N/A"),+ ("a score with no passing rule", {"result_type": "numeric", "score": "0.82"}, {}, "unrated", "0.82"),+ ("a score under the threshold", {"result_type": "numeric", "score": "0.4"}, AT_LEAST_0_7, "fail", "0.4"),+ ("a score over the threshold", {"result_type": "numeric", "score": "0.9"}, AT_LEAST_0_7, "pass", "0.9"),+ (+ "a long score rounded to two places",+ {"result_type": "numeric", "score": "12.3456"},+ {},+ "unrated",+ "12.35",+ ),+ ("a whole score", {"result_type": "numeric", "score": "3"}, {}, "unrated", "3"),+ (+ "a score whose rounding would flip the verdict",+ {"result_type": "numeric", "score": "0.6999"},+ AT_LEAST_0_7,+ "fail",+ "0.6999",+ ),+ ("a numeric run with no score", {"result_type": "numeric"}, {}, "inconclusive", "No score"),+ ("a numeric run with an empty score", {"result_type": "numeric", "score": ""}, {}, "inconclusive", "No score"),+ (+ "a numeric run marked not applicable",+ {"result_type": "numeric", "score": "0.5", "applicable": "false"},+ {},+ "inconclusive",+ "N/A",+ ),+ (+ "a stored threshold that is not a number",+ {"result_type": "numeric", "score": "0.4"},+ {"passing_rule": {"operator": "gte", "threshold": "0.7"}},+ "unrated",+ "0.4",+ ),+ (+ "a stored rule with an unknown operator",+ {"result_type": "numeric", "score": "0.4"},+ {"passing_rule": {"operator": "gt", "threshold": 0.7}},+ "unrated",+ "0.4",+ ),+ (+ "categories inside the passing rule",+ {"result_type": "categorical", "categories": '["billing"]'},+ BILLING_ONLY,+ "pass",+ "Billing",+ ),+ (+ "categories outside the passing rule",+ {"result_type": "categorical", "categories": '["billing", "refund"]'},+ BILLING_ONLY,+ "fail",+ "Billing, Refund",+ ),+ (+ "a category with no option keeps its key",+ {"result_type": "categorical", "categories": '["other"]'},+ {},+ "unrated",+ "other",+ ),+ (+ "an applicable run with no categories",+ {"result_type": "categorical", "applicable": "true"},+ BILLING_ONLY,+ "fail",+ "No categories",+ ),+ (+ "categories that are not JSON",+ {"result_type": "categorical", "categories": "billing"},+ BILLING_ONLY,+ "inconclusive",+ "No result",+ ),+ (+ "categories marked not applicable",+ {"result_type": "categorical", "categories": '["billing"]', "applicable": "false"},+ BILLING_ONLY,+ "inconclusive",+ "N/A",+ ),+ ("a positive sentiment", {"result_type": "sentiment", "sentiment_label": "positive"}, {}, "pass", "Positive"),+ ("a neutral sentiment", {"result_type": "sentiment", "sentiment_label": "neutral"}, {}, "unrated", "Neutral"),+ ("a negative sentiment", {"result_type": "sentiment", "sentiment_label": "negative"}, {}, "fail", "Negative"),+ ("an unlabelled sentiment", {"result_type": "sentiment"}, {}, "inconclusive", "Unknown"),+ (+ "a sentiment run with no result type",+ {"evaluation_type": "sentiment", "sentiment_label": "positive"},+ {},+ "pass",+ "Positive",+ ),+ ]+)+def test_outcome_and_label(+ _name: str, overrides: dict[str, Any], output_config: dict[str, Any], outcome: str, label: str+) -> None:+ result = evaluation_result(overrides, output_config)++ assert (result.outcome, result.label) == (outcome, label)+++ [+ (+ "a generation result",+ {"target_event_id": "gen-1"},+ ("gen-1", False, EVALUATION_ID, "Answers the question", "Direct answer."),+ ),+ ("a trace-level result", {}, ("trace-1", False, EVALUATION_ID, "Answers the question", "Direct answer.")),+ (+ "a backfilled result",+ {"backfill_id": "backfill-1"},+ ("trace-1", True, EVALUATION_ID, "Answers the question", "Direct answer."),+ ),+ (+ "a result from outside whose evaluation id is not a UUID",+ {"evaluation_id": "not-a-uuid"},+ ("trace-1", False, None, "Answers the question", "Direct answer."),+ ),+ (+ "a result from outside with no id, name or reasoning",+ {"evaluation_id": None, "evaluation_name": None, "reasoning": ""},+ ("trace-1", False, None, "Unknown evaluation", None),+ ),+ ]+)+def test_target_name_and_backfill(_name: str, overrides: dict[str, Any], expected: tuple[Any, ...]) -> None:+ result = evaluation_result(overrides)++ assert (+ result.target_node_id,+ result.is_backfill,+ result.evaluation_id,+ result.name,+ result.reasoning,+ ) == expecteddiff --git a/products/ai_observability/backend/tests/logic/traces/test_evaluation_result_queries.py b/products/ai_observability/backend/tests/logic/traces/test_evaluation_result_queries.pynew file mode 100644index 000000000000..2a13c0d12bee--- /dev/null+++ b/products/ai_observability/backend/tests/logic/traces/test_evaluation_result_queries.py@@ -0,0 +1,38 @@+from datetime import timedelta++from posthog.test.base import BaseTest, ClickhouseTestMixin++from posthog.models import Team+from posthog.models.event.util import bulk_create_events++from products.ai_observability.backend.logic.traces.evaluation_result_queries import load_evaluation_results++from .rows import T0, make_evaluation_result_event++LATE = "00000000-0000-0000-0000-00000000000b"+EARLY = "00000000-0000-0000-0000-00000000000a"+++class TestLoadEvaluationResults(ClickhouseTestMixin, BaseTest):+ def test_reads_every_result_of_the_trace_however_late_it_was_stamped(self) -> None:+ other_team = Team.objects.create(organization=self.organization, name="other")+ bulk_create_events(+ [+ make_evaluation_result_event(uuid=EARLY, team=self.team, timestamp=T0 + timedelta(seconds=1)),+ make_evaluation_result_event(uuid=LATE, team=self.team, timestamp=T0 + timedelta(days=90)),+ make_evaluation_result_event(+ uuid="00000000-0000-0000-0000-00000000000c",+ team=self.team,+ timestamp=T0 + timedelta(seconds=1),+ trace_id="trace-2",+ ),+ make_evaluation_result_event(+ uuid="00000000-0000-0000-0000-00000000000d", team=other_team, timestamp=T0 + timedelta(seconds=1)+ ),+ make_evaluation_result_event(+ uuid="00000000-0000-0000-0000-00000000000e", team=self.team, timestamp=T0, event="$ai_generation"+ ),+ ]+ )++ assert [row.uuid for row in load_evaluation_results(self.team, None, "trace-1")] == [LATE, EARLY]diff --git a/products/ai_observability/backend/tests/test_facade_api.py b/products/ai_observability/backend/tests/test_facade_api.pyindex 939b0da5995f..c59e846293c4 100644--- a/products/ai_observability/backend/tests/test_facade_api.py+++ b/products/ai_observability/backend/tests/test_facade_api.py@@ -1,12 +1,23 @@-from datetime import UTC, datetime+from collections.abc import Callable+from datetime import UTC, datetime, timedeltaimport pytest+from posthog.test.base import BaseTest, ClickhouseTestMixinfrom unittest.mock import patch+from parameterized import parameterized++from posthog.constants import AvailableFeature+from posthog.models import Team+from posthog.models.event.util import bulk_create_events++from products.access_control.backend.models.access_control import AccessControlfrom products.ai_observability.backend.facade import apifrom products.ai_observability.backend.logic.traces.trace_queries import LoadedTrace+from products.ai_observability.backend.models.evaluations import Evaluation+from products.event_definitions.backend.models.property_definition import PropertyDefinition-from .logic.traces.rows import make_row+from .logic.traces.rows import make_evaluation_result_event, make_rowT0 = datetime(2026, 9, 1, 10, 0, tzinfo=UTC)@@ -29,3 +40,108 @@ def test_missing_trace_raises_not_found() -> None:with patch.object(api, "load_trace", return_value=LoadedTrace(rows=(), person=None)):with pytest.raises(api.TraceNotFoundError):api.get_trace(team=None, user=None, trace_id="missing", timestamp_hint=None) # type: ignore[arg-type]+++class TestGetTraceEvaluationResults(ClickhouseTestMixin, BaseTest):+ def _detector(self, team: Team | None = None, *, deleted: bool = False, stored_config: object = None) -> str:+ evaluation = Evaluation.objects.create(+ team=team or self.team,+ name="Mentions a competitor",+ evaluation_type="llm_judge",+ evaluation_config={"prompt": "Does the answer mention a competitor?"},+ output_type="boolean",+ output_config={"true_is_failure": True},+ deleted=deleted,+ )+ if stored_config is not None:+ Evaluation.objects.filter(id=evaluation.id).update(output_config=stored_config)+ return str(evaluation.id)++ def _other_team(self) -> Team:+ return Team.objects.create(organization=self.organization, name="other")++ def _detector_hidden_from_the_viewer(self) -> str:+ self.organization.available_product_features = [+ {"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}+ ]+ self.organization.save()+ AccessControl.objects.create(+ team=self.team, resource="evaluation", access_level="none", organization_member=self.organization_membership+ )+ return self._detector()++ @parameterized.expand(+ [+ ("an active detector", lambda test: test._detector(), "fail"),+ ("a deleted detector", lambda test: test._detector(deleted=True), "fail"),+ ("another team's detector", lambda test: test._detector(test._other_team()), "pass"),+ ("a detector the viewer cannot read", lambda test: test._detector_hidden_from_the_viewer(), "pass"),+ ("an id that is not a UUID", lambda test: "not-a-uuid", "pass"),+ (+ "a detector whose stored config is not an object",+ lambda test: test._detector(stored_config=["true_is_failure"]),+ "pass",+ ),+ ]+ )+ def test_applies_the_results_stored_output_config(+ self, _name: str, evaluation_id_for: Callable[["TestGetTraceEvaluationResults"], str], outcome: str+ ) -> None:+ bulk_create_events(+ [+ make_evaluation_result_event(+ uuid="00000000-0000-0000-0000-000000000001",+ team=self.team,+ timestamp=T0 + timedelta(seconds=1),+ properties={"$ai_evaluation_id": evaluation_id_for(self)},+ )+ ]+ )++ [result] = api.get_trace_evaluation_results(self.team, self.user, "trace-1").results++ assert result.outcome == outcome++ def test_reads_a_registered_boolean_property_like_its_ingested_value(self) -> None:+ PropertyDefinition.objects.create(+ team=self.team,+ name="$ai_evaluation_applicable",+ property_type="Boolean",+ type=PropertyDefinition.Type.EVENT,+ )+ bulk_create_events(+ [+ make_evaluation_result_event(+ uuid="00000000-0000-0000-0000-000000000001",+ team=self.team,+ timestamp=T0 + timedelta(seconds=1),+ properties={"$ai_evaluation_applicable": False},+ )+ ]+ )++ [result] = api.get_trace_evaluation_results(self.team, None, "trace-1").results++ assert (result.outcome, result.label) == ("inconclusive", "N/A")++ def test_lists_results_by_their_event_time_newest_first(self) -> None:+ live = "00000000-0000-0000-0000-000000000001"+ backfill = "00000000-0000-0000-0000-000000000002"+ bulk_create_events(+ [+ make_evaluation_result_event(uuid=live, team=self.team, timestamp=T0 + timedelta(minutes=5)),+ make_evaluation_result_event(+ uuid=backfill,+ team=self.team,+ timestamp=T0 + timedelta(seconds=1),+ properties={+ "$ai_evaluation_start_time": (T0 + timedelta(days=20)).isoformat(),+ "$ai_evaluation_backfill_id": "backfill-1",+ },+ ),+ ]+ )++ results = api.get_trace_evaluation_results(self.team, None, "trace-1").results++ assert [(result.id, result.is_backfill) for result in results] == [(live, False), (backfill, True)]diff --git a/products/ai_observability/backend/tests/test_trace_views.py b/products/ai_observability/backend/tests/test_trace_views.pyindex 0a5bf3c091a7..2c0a54cba76c 100644--- a/products/ai_observability/backend/tests/test_trace_views.py+++ b/products/ai_observability/backend/tests/test_trace_views.py@@ -1,5 +1,5 @@import base64-from datetime import UTC, datetime+from datetime import UTC, datetime, timedeltafrom posthog.test.base import APIBaseTest, ClickhouseTestMixinfrom unittest.mock import patch@@ -10,6 +10,7 @@ from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, Concurrencfrom posthog.constants import AvailableFeaturefrom posthog.models import Teamfrom posthog.models.ai_events.test_util import bulk_create_ai_events+from posthog.models.event.util import bulk_create_eventsfrom posthog.models.organization import OrganizationMembershipfrom posthog.models.user import User@@ -19,9 +20,12 @@ from products.access_control.backend.models.property_access_control import Propefrom products.ai_observability.backend.models.review_queues import ReviewQueuefrom products.event_definitions.backend.models.property_definition import PropertyDefinition+from .logic.traces.rows import make_evaluation_result_event+T0 = datetime(2026, 9, 1, 10, 0, tzinfo=UTC)GENERATION = "00000000-0000-0000-0000-000000000001"TRACE_EVENT = "00000000-0000-0000-0000-000000000002"+EVALUATION_RESULT = "00000000-0000-0000-0000-000000000003"QUESTION = [{"role": "user", "content": "Why did my invoice go up?"}]ANSWER = [{"role": "assistant", "content": "Two seats were added."}]@@ -80,6 +84,23 @@ def _create_node_trace(bulk_create_ai_events(events)+def _create_evaluation_result(team: Team) -> None:+ bulk_create_events(+ [+ make_evaluation_result_event(+ uuid=EVALUATION_RESULT,+ team=team,+ timestamp=T0 + timedelta(seconds=30),+ properties={+ "$ai_target_event_id": GENERATION,+ "$ai_evaluation_name": "Answers the question",+ "$ai_evaluation_reasoning": "Direct answer.",+ },+ )+ ]+ )++def _encode(trace_id: str) -> str:return base64.urlsafe_b64encode(trace_id.encode()).decode().rstrip("=")@@ -130,7 +151,11 @@ class TestTraceViewSet(ClickhouseTestMixin, APIBaseTest):("browser session", False, "app_per_org"),("personal api key", True, "api_per_team"),]- for read, suffix in [("trace", ""), ("node", f"nodes/{_encode('trace-1')}/")]+ for read, suffix in [+ ("trace", ""),+ ("node", f"nodes/{_encode('trace-1')}/"),+ ("evaluations", "evaluations/"),+ ]])def test_throttles_when_the_query_concurrency_limit_is_exhausted(@@ -235,6 +260,60 @@ class TestTraceNodeView(ClickhouseTestMixin, APIBaseTest):assert properties["$ai_provider"] == "openai"+class TestTraceEvaluationsView(ClickhouseTestMixin, APIBaseTest):+ def _url(self, segment: str) -> str:+ return f"/api/projects/{self.team.id}/ai_observability/traces/{segment}/evaluations/"++ def test_returns_the_results_in_camel_case(self) -> None:+ _create_evaluation_result(self.team)++ response = self.client.get(self._url(_encode("trace-1")))++ assert response.status_code == 200+ assert response.json()["results"][0] | {"timestamp": None} == {+ "id": EVALUATION_RESULT,+ "evaluationId": None,+ "evaluationName": "Answers the question",+ "targetNodeId": GENERATION,+ "outcome": "pass",+ "label": "True",+ "reasoning": "Direct answer.",+ "timestamp": None,+ "isBackfill": False,+ }++ def test_an_unknown_trace_has_no_results(self) -> None:+ response = self.client.get(self._url(_encode("trace-2")))++ assert (response.status_code, response.json()) == (200, {"results": []})++ def test_rejects_an_unencoded_trace_id(self) -> None:+ assert self.client.get(self._url("trace-1")).status_code == 400++ def test_masks_properties_the_viewer_may_not_read(self) -> None:+ self.organization.available_product_features = [+ {"key": AvailableFeature.PROPERTY_ACCESS_CONTROL, "name": AvailableFeature.PROPERTY_ACCESS_CONTROL}+ ]+ self.organization.save()+ definition = PropertyDefinition.objects.create(+ team=self.team,+ name="$ai_evaluation_reasoning",+ property_type="String",+ type=PropertyDefinition.Type.EVENT,+ )+ PropertyAccessControl.objects.create(+ team=self.team,+ property_definition=definition,+ organization_member=self.organization_membership,+ access_level=PropertyAccessLevel.NONE.value,+ )+ _create_evaluation_result(self.team)++ result = self.client.get(self._url(_encode("trace-1"))).json()["results"][0]++ assert (result["reasoning"], result["outcome"]) == (None, "pass")++class TestTraceViewSetResourceLevelAccess(ClickhouseTestMixin, APIBaseTest):def test_a_grant_on_one_object_does_not_open_traces(self) -> None:self.organization.available_product_features = [diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.tsindex 1288a51216b2..5265a0cb9272 100644--- a/products/ai_observability/frontend/generated/api.schemas.ts+++ b/products/ai_observability/frontend/generated/api.schemas.ts@@ -1404,6 +1404,34 @@ export interface TraceApi {nodeIdsByAlias: TraceApiNodeIdsByAlias}+export type TraceEvaluationResultOutcomeEnumApi =+ (typeof TraceEvaluationResultOutcomeEnumApi)[keyof typeof TraceEvaluationResultOutcomeEnumApi]++export const TraceEvaluationResultOutcomeEnumApi = {+ Pass: 'pass',+ Fail: 'fail',+ Inconclusive: 'inconclusive',+ Unrated: 'unrated',+} as const++export interface TraceEvaluationResultApi {+ id: string+ evaluationId: string | null+ evaluationName: string+ /** The tree node the result grades: the generation's event uuid, or the trace id for a trace-level result. */+ targetNodeId: string+ outcome: TraceEvaluationResultOutcomeEnumApi+ /** What the result says, such as "True", "0.82", "Positive" or "Skipped". */+ label: string+ reasoning: string | null+ timestamp: string+ isBackfill: boolean+}++export interface TraceEvaluationResultsApi {+ results: TraceEvaluationResultApi[]+}+export interface TraceNodeMetadataApi {provider: string | nulltemperature: number | nulldiff --git a/products/ai_observability/frontend/generated/api.ts b/products/ai_observability/frontend/generated/api.tsindex 3d452f40baf0..bf3d0381929c 100644--- a/products/ai_observability/frontend/generated/api.ts+++ b/products/ai_observability/frontend/generated/api.ts@@ -143,6 +143,7 @@ import type {TextReprRequestApi,TextReprResponseApi,TraceApi,+ TraceEvaluationResultsApi,TraceNodeDetailApi,TraceReviewApi,TraceReviewCreateApi,@@ -675,6 +676,24 @@ export const aiObservabilityTracesRetrieve = async (})}+export const getAiObservabilityTracesEvaluationsRetrieveUrl = (projectId: string, id: string) => {+ return `/api/projects/${projectId}/ai_observability/traces/${id}/evaluations/`+}++/**+ * A trace's evaluation results, newest first and at most 250, each with its outcome and label ready to render.+ */+export const aiObservabilityTracesEvaluationsRetrieve = async (+ projectId: string,+ id: string,+ options?: RequestInit+): Promise<TraceEvaluationResultsApi> => {+ return apiMutator<TraceEvaluationResultsApi>(getAiObservabilityTracesEvaluationsRetrieveUrl(projectId, id), {+ ...options,+ method: 'GET',+ })+}+export const getAiObservabilityTracesNodesRetrieveUrl = (projectId: string,id: string,diff --git a/products/ai_observability/frontend/redesign/trace/adapter/testFixtures.ts b/products/ai_observability/frontend/redesign/trace/adapter/testFixtures.tsindex 226e0d6a76de..0b91e903def2 100644--- a/products/ai_observability/frontend/redesign/trace/adapter/testFixtures.ts+++ b/products/ai_observability/frontend/redesign/trace/adapter/testFixtures.ts@@ -2,6 +2,7 @@ import { LLMTrace, LLMTraceEvent } from '~/queries/schema/schema-general'import type {TraceApi,+ TraceEvaluationResultApi,TraceNodeApi,TraceNodeDetailApi,TraceNodeIOKeysApi,@@ -123,3 +124,18 @@ export function makeTraceResource(overrides: Partial<TraceApi> = {}): TraceApi {...overrides,}}++export function makeEvaluationResult(overrides: Partial<TraceEvaluationResultApi> = {}): TraceEvaluationResultApi {+ return {+ id: 'r1',+ evaluationId: 'ev-helpful',+ evaluationName: 'Answers the question',+ targetNodeId: 'trace-1',+ outcome: 'pass',+ label: 'True',+ reasoning: 'Direct answer.',+ timestamp: '2026-09-01T10:16:00Z',+ isBackfill: false,+ ...overrides,+ }+}diff --git a/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.test.ts b/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.test.tsindex ddf14238bedf..028219ed32ab 100644--- a/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.test.ts+++ b/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.test.ts@@ -1,8 +1,8 @@import { urls } from 'scenes/urls'-import { EvaluationOutputConfig, EvaluationRun, LLMJudgeEvaluation } from '../../../evaluations/types'import type { TraceNodeApi } from '../../../generated/api.schemas'-import { EvalResultsContext, toEvalResults } from './toEvalResults'+import { makeEvaluationResult } from './testFixtures'+import { toEvalResults } from './toEvalResults'const node = (id: string, kind: TraceNodeApi['kind'], name: string, children: TraceNodeApi[] = []): TraceNodeApi => ({id,@@ -29,110 +29,32 @@ const tree = []),]-const run = (overrides: Partial<EvaluationRun> = {}): EvaluationRun => ({- id: 'r1',- evaluation_id: 'ev-helpful',- evaluation_name: 'Answers the question',- generation_id: null,- trace_id: 'trace-1',- timestamp: '2026-09-01T10:16:00Z',- result: true,- reasoning: 'Direct answer.',- status: 'completed',- ...overrides,-})--const evaluation = (id: string, outputConfig: EvaluationOutputConfig): LLMJudgeEvaluation => ({- id,- name: id,- enabled: true,- status: 'active',- status_reason: null,- status_reason_detail: null,- evaluation_type: 'llm_judge',- evaluation_config: { prompt: 'Grade the answer.' },- output_type: 'numeric',- output_config: outputConfig,- conditions: [],- target: 'generation',- target_config: {},- model_configuration: null,- created_at: '2026-08-01T00:00:00Z',- updated_at: '2026-08-01T00:00:00Z',-})--const context = (overrides: Partial<EvalResultsContext> = {}): EvalResultsContext => ({- tree,- viewedNodeId: 'trace-1',- evaluations: [evaluation('ev-quality', { passing_rule: { operator: 'gte', threshold: 0.7 } })],- detectorEvaluationIds: ['ev-detector'],- ...overrides,-})-describe('toEvalResults', () => {- it.each<[string, Partial<EvaluationRun>, string, string]>([- ['a failed run', { status: 'failed' }, 'error', 'Error'],- ['a running run', { status: 'running' }, 'pending', 'Running'],- ['a true result', { result: true }, 'pass', 'True'],- ['a false result', { result: false }, 'fail', 'False'],- ['a true result from a detector', { evaluation_id: 'ev-detector', result: true }, 'fail', 'True'],- ['a false result from a detector', { evaluation_id: 'ev-detector', result: false }, 'pass', 'False'],- ['a skipped run, whose result is still false', { skipped: true, result: false }, 'inconclusive', 'Skipped'],- ['a missing result', { result: null }, 'inconclusive', 'N/A'],- ['a score with no passing rule', { result_type: 'numeric', score: 0.82 }, 'unrated', '0.82'],- [- 'a score under the passing threshold',- { evaluation_id: 'ev-quality', result_type: 'numeric', score: 0.4 },- 'fail',- '0.4',- ],- ['a numeric run with no score', { result_type: 'numeric', score: null }, 'inconclusive', 'No score'],- ['a positive sentiment', { result_type: 'sentiment', sentiment_label: 'positive' }, 'pass', 'Positive'],- ['a neutral sentiment', { result_type: 'sentiment', sentiment_label: 'neutral' }, 'unrated', 'Neutral'],- ['an unknown sentiment', { result_type: 'sentiment', sentiment_label: null }, 'inconclusive', 'Unknown'],- ])('maps %s to %s', (_name, overrides, outcome, label) => {- expect(toEvalResults([run(overrides)], context())).toMatchObject([{ outcome, label }])- })-- it('lists the newest verdict first, dating a backfill by when it ran, and marks backfills', () => {- const results = toEvalResults(- [- run({ id: 'live', timestamp: '2026-09-01T10:16:00Z' }),- run({- id: 'backfill',- timestamp: '2026-09-01T10:15:30Z',- start_time: '2026-09-20T08:00:00Z',- backfill_id: 'backfill-1',- }),- run({ id: 'older', timestamp: '2026-09-01T10:15:45Z' }),- ],- context()- )- expect(results.map((result) => [result.id, result.isBackfill, result.timestamp])).toEqual([- ['backfill', true, '2026-09-01T10:15:30Z'],- ['live', false, '2026-09-01T10:16:00Z'],- ['older', false, '2026-09-01T10:15:45Z'],- ])- expect(results[0].href).toBe(urls.aiObservabilityEvaluation('ev-helpful'))- })-it.each([- ['a trace-level run seen on the root targets nothing', 'trace-1', null, null],+ ['a trace-level result seen on the root targets nothing', 'trace-1', 'trace-1', null],[- 'a run on a generation seen on the root names that generation apart from its namesake',+ 'a result on a generation seen on the root names that generation apart from its namesake','trace-1','gen-2',{ nodeId: 'gen-2', label: 'gpt-4.1-mini (openai) #2' },],- ['a run on the viewed generation targets nothing', 'gen-1', 'gen-1', null],+ ['a result on the viewed generation targets nothing', 'gen-1', 'gen-1', null],[- 'a run on a generation missing from the tree falls back to a short id',+ 'a result on a node missing from the tree falls back to a short id','trace-1','gen-0123456789abcdef',{ nodeId: 'gen-0123456789abcdef', label: 'gen-01234567...' },],- ])('%s', (_name, viewedNodeId, generationId, expected) => {- const [result] = toEvalResults([run({ generation_id: generationId })], context({ viewedNodeId }))+ ])('%s', (_name, viewedNodeId, targetNodeId, expected) => {+ const [result] = toEvalResults([makeEvaluationResult({ targetNodeId })], { tree, viewedNodeId })expect(result.target).toEqual(expected)})++ it.each([+ ['an evaluation id links to its evaluation', 'ev-helpful', urls.aiObservabilityEvaluation('ev-helpful')],+ ['a result with no evaluation id has no link', null, null],+ ])('%s', (_name, evaluationId, href) => {+ const [result] = toEvalResults([makeEvaluationResult({ evaluationId })], { tree, viewedNodeId: 'trace-1' })+ expect(result.href).toBe(href)+ })})diff --git a/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.ts b/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.tsindex 60e474340709..50b610405d5e 100644--- a/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.ts+++ b/products/ai_observability/frontend/redesign/trace/adapter/toEvalResults.ts@@ -1,41 +1,12 @@import { urls } from 'scenes/urls'-import { EvaluationResultDisplayOptions, getEvaluationResultDisplay } from '../../../components/EvaluationResultTag'-import { EvaluationConfig, EvaluationRun } from '../../../evaluations/types'-import type { TraceNodeApi } from '../../../generated/api.schemas'-import { EvalOutcome, EvalResult, EvalTarget } from '../types'+import type { TraceEvaluationResultApi, TraceNodeApi } from '../../../generated/api.schemas'+import { EvalResult, EvalTarget } from '../types'export interface EvalResultsContext {- /** Every node of the trace, root first, to resolve and name what each run targets. */+ /** Every node of the trace, root first, to name what each result targets. */tree: TraceNodeApi[]viewedNodeId: string- evaluations: EvaluationConfig[]- detectorEvaluationIds: string[]-}--function displayOptions(run: EvaluationRun, context: EvalResultsContext): EvaluationResultDisplayOptions {- const outputConfig = context.evaluations.find((evaluation) => evaluation.id === run.evaluation_id)?.output_config- return {- trueIsFailure: context.detectorEvaluationIds.includes(run.evaluation_id),- passingRule: outputConfig?.passing_rule,- categoryOptions: outputConfig?.options,- }-}--function outcomeOf(run: EvaluationRun, tagType: string | undefined): EvalOutcome {- if (run.status === 'failed') {- return 'error'- }- if (run.status === 'running') {- return 'pending'- }- if (tagType === 'success') {- return 'pass'- }- if (tagType === 'danger') {- return 'fail'- }- return tagType === 'none' ? 'unrated' : 'inconclusive'}function flatten(nodes: TraceNodeApi[]): TraceNodeApi[] {@@ -57,34 +28,29 @@ function nodeLabels(tree: TraceNodeApi[]): Map<string, string> {)}-function targetOf(run: EvaluationRun, labels: Map<string, string>, context: EvalResultsContext): EvalTarget | null {- const nodeId = run.generation_id ?? context.tree[0]?.id- if (!nodeId || nodeId === context.viewedNodeId) {+function targetOf(+ result: TraceEvaluationResultApi,+ labels: Map<string, string>,+ viewedNodeId: string+): EvalTarget | null {+ const nodeId = result.targetNodeId+ if (nodeId === viewedNodeId) {return null}return { nodeId, label: labels.get(nodeId) ?? `${nodeId.slice(0, 12)}...` }}-function producedAt(run: EvaluationRun): number {- return new Date(run.start_time ?? run.timestamp).getTime()-}--export function toEvalResults(runs: EvaluationRun[], context: EvalResultsContext): EvalResult[] {+export function toEvalResults(results: TraceEvaluationResultApi[], context: EvalResultsContext): EvalResult[] {const labels = nodeLabels(context.tree)- return [...runs]- .sort((a, b) => producedAt(b) - producedAt(a))- .map((run) => {- const display = getEvaluationResultDisplay(run, displayOptions(run, context))- return {- id: run.id,- name: run.evaluation_name,- href: urls.aiObservabilityEvaluation(run.evaluation_id),- outcome: outcomeOf(run, display.type),- label: display.label,- reasoning: run.reasoning || null,- timestamp: run.timestamp,- isBackfill: !!run.backfill_id,- target: targetOf(run, labels, context),- }- })+ return results.map((result) => ({+ id: result.id,+ name: result.evaluationName,+ href: result.evaluationId ? urls.aiObservabilityEvaluation(result.evaluationId) : null,+ outcome: result.outcome,+ label: result.label,+ reasoning: result.reasoning,+ timestamp: result.timestamp,+ isBackfill: result.isBackfill,+ target: targetOf(result, labels, context.viewedNodeId),+ }))}diff --git a/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.test.ts b/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.test.tsindex 44f7c40e9914..0b606e25e7a3 100644--- a/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.test.ts+++ b/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.test.ts@@ -1,7 +1,6 @@-import { EvaluationRun } from '../../../evaluations/types'import type { TraceNodeApi } from '../../../generated/api.schemas'import { EvalResult, EvalsState, NodeLoad } from '../types'-import { makeNodeDetail } from './testFixtures'+import { makeEvaluationResult, makeNodeDetail } from './testFixtures'import { toNodeDetail } from './toNodeDetail'const node = (id: string, kind: 'trace' | 'span' | 'generation'): TraceNodeApi => ({@@ -21,18 +20,8 @@ const node = (id: string, kind: 'trace' | 'span' | 'generation'): TraceNodeApi =},})-const run = (overrides: Partial<EvaluationRun> = {}): EvaluationRun => ({- id: 'r1',- evaluation_id: 'ev1',- evaluation_name: 'Answers the question',- generation_id: 'gen-1',- trace_id: 'trace-1',- timestamp: '2026-09-01T10:16:00Z',- result: true,- reasoning: 'Direct answer.',- status: 'completed',- ...overrides,-})+const result = (overrides: Parameters<typeof makeEvaluationResult>[0] = {}): ReturnType<typeof makeEvaluationResult> =>+ makeEvaluationResult({ targetNodeId: 'gen-1', ...overrides })function readyResults(evals: EvalsState): EvalResult[] {if (evals.status !== 'ready') {@@ -46,11 +35,8 @@ const ready = (detail: ReturnType<typeof makeNodeDetail>): NodeLoad => ({ statusdescribe('toNodeDetail', () => {const base = {tree: [],- evalRuns: [],- evalRunsLoading: false,- evaluations: [],- detectorEvaluationIds: [],- evaluationsSettled: true,+ evaluationResults: [],+ evaluationResultsFailed: false,teamId: 7,person: null,trace: { createdAt: '2026-09-01T10:15:00Z', sessionId: 'sess-1' },@@ -92,7 +78,7 @@ describe('toNodeDetail', () => {metadata: { provider: 'openai', temperature: 0.2, sessionId: 'sess-9', promptVersion: 6 },})),- evalRuns: [run(), run({ id: 'r2', generation_id: 'gen-other' })],+ evaluationResults: [result(), result({ id: 'r2', targetNodeId: 'gen-other' })],})expect(detail.content).toMatchObject({kind: 'messages',@@ -110,18 +96,25 @@ describe('toNodeDetail', () => {expect(readyResults(detail.evals).map((result) => result.id)).toEqual(['r1'])})- it.each([- ['eval runs are loading', { evalRunsLoading: true }],- ['the evaluation configs that decide detector polarity have not loaded', { evaluationsSettled: false }],- ])('reports evals as loading while %s', (_name, overrides) => {+ it.each<[string, { evaluationResults: null; evaluationResultsFailed: boolean }, EvalsState]>([+ [+ 'loading until the results arrive',+ { evaluationResults: null, evaluationResultsFailed: false },+ { status: 'loading' },+ ],+ [+ 'an error when the read fails',+ { evaluationResults: null, evaluationResultsFailed: true },+ { status: 'error', errorMessage: 'Evaluation results could not be loaded. Reload the page to try again.' },+ ],+ ])('reports evals as %s', (_name, overrides, evals) => {const detail = toNodeDetail({...base,...overrides,node: node('gen-1', 'generation'),load: { status: 'loading' },- evalRuns: [run()],})- expect(detail.evals).toEqual({ status: 'loading' })+ expect(detail.evals).toEqual(evals)})it('span shows its state as input and output, surfaces its error and labels the person', () => {@@ -147,30 +140,33 @@ describe('toNodeDetail', () => {it.each([['with a trace event', true],['without a trace event', false],- ])('trace root %s takes its time and session from the trace and shows every eval run', (_name, withEvent) => {- const detail = toNodeDetail({- ...base,- node: node('trace-1', 'trace'),- load: ready(- makeNodeDetail({- id: 'trace-1',- kind: 'trace',- withEvent,- properties: { $ai_input_state: { question: 'Hi' } },- ioKeys: { input: '$ai_input_state' },- })- ),- evalRuns: [run(), run({ id: 'r2' })],- })- expect(detail.properties).toMatchObject({ timestamp: '2026-09-01T10:15:00Z', sessionId: 'sess-1' })- expect(readyResults(detail.evals)).toHaveLength(2)- expect(detail.content).toEqual({- kind: 'io',- input: withEvent ? { question: 'Hi' } : undefined,- output: undefined,- })- expect(detail.raw === null).toBe(!withEvent)- })+ ])(+ 'trace root %s takes its time and session from the trace and shows every evaluation result',+ (_name, withEvent) => {+ const detail = toNodeDetail({+ ...base,+ node: node('trace-1', 'trace'),+ load: ready(+ makeNodeDetail({+ id: 'trace-1',+ kind: 'trace',+ withEvent,+ properties: { $ai_input_state: { question: 'Hi' } },+ ioKeys: { input: '$ai_input_state' },+ })+ ),+ evaluationResults: [result(), result({ id: 'r2' })],+ })+ expect(detail.properties).toMatchObject({ timestamp: '2026-09-01T10:15:00Z', sessionId: 'sess-1' })+ expect(readyResults(detail.evals)).toHaveLength(2)+ expect(detail.content).toEqual({+ kind: 'io',+ input: withEvent ? { question: 'Hi' } : undefined,+ output: undefined,+ })+ expect(detail.raw === null).toBe(!withEvent)+ }+ )it('renders an OTel span with a message array input and a text output as messages', () => {const detail = toNodeDetail({diff --git a/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.ts b/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.tsindex 27401e540e02..c1e965799e80 100644--- a/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.ts+++ b/products/ai_observability/frontend/redesign/trace/adapter/toNodeDetail.ts@@ -1,5 +1,4 @@-import { EvaluationConfig, EvaluationRun } from '../../../evaluations/types'-import type { TraceNodeApi, TraceNodeDetailApi } from '../../../generated/api.schemas'+import type { TraceEvaluationResultApi, TraceNodeApi, TraceNodeDetailApi } from '../../../generated/api.schemas'import { NodeDetailProps } from '../components/NodeDetail'import { EvalsState, LoadFailure, NodeContent, NodeLoad, NodeProperties } from '../types'import { NodeIO, readNodeIO } from './nodeIO'@@ -19,11 +18,8 @@ interface NodeDetailArgs {node: TraceNodeApiload: NodeLoadtree: TraceNodeApi[]- evalRuns: EvaluationRun[]- evalRunsLoading: boolean- evaluations: EvaluationConfig[]- detectorEvaluationIds: string[]- evaluationsSettled: boolean+ evaluationResults: TraceEvaluationResultApi[] | null+ evaluationResultsFailed: booleanteamId: TeamIdperson: string | nulltrace: { createdAt: string; sessionId: string | null }@@ -78,31 +74,30 @@ function rawEvent(detail: TraceNodeDetailApi): Record<string, unknown> | null {: null}-// Detector polarity comes from the evaluation configs, so results wait for them to avoid showing a flipped verdict.-function evalsState(runs: EvaluationRun[], node: TraceNodeApi, args: NodeDetailArgs): EvalsState {- if (args.evalRunsLoading || !args.evaluationsSettled) {- return { status: 'loading' }+const EVALS_LOAD_ERROR = 'Evaluation results could not be loaded. Reload the page to try again.'++function evalsState(node: TraceNodeApi, args: NodeDetailArgs): EvalsState {+ if (args.evaluationResultsFailed) {+ return { status: 'error', errorMessage: EVALS_LOAD_ERROR }}- return {- status: 'ready',- results: toEvalResults(runs, {- tree: args.tree,- viewedNodeId: node.id,- evaluations: args.evaluations,- detectorEvaluationIds: args.detectorEvaluationIds,- }),+ if (!args.evaluationResults) {+ return { status: 'loading' }}+ const results =+ node.kind === 'trace'+ ? args.evaluationResults+ : args.evaluationResults.filter((result) => result.targetNodeId === node.id)+ return { status: 'ready', results: toEvalResults(results, { tree: args.tree, viewedNodeId: node.id }) }}export function toNodeDetail(args: NodeDetailArgs): NodeDetailData {- const { node, load, evalRuns, teamId } = args- const runs = node.kind === 'trace' ? evalRuns : evalRuns.filter((run) => run.generation_id === node.id)+ const { node, load, teamId } = argsreturn {node,content: loadedContent(load, node, teamId),error: load.status === 'ready' ? load.node.error : null,properties: nodeProperties(load, node, args),- evals: evalsState(runs, node, args),+ evals: evalsState(node, args),raw: load.status === 'ready' ? rawEvent(load.node) : null,}}diff --git a/products/ai_observability/frontend/redesign/trace/adapter/traceSceneWiring.test.tsx b/products/ai_observability/frontend/redesign/trace/adapter/traceSceneWiring.test.tsxindex 35072fd85e89..12ddf7a12f54 100644--- a/products/ai_observability/frontend/redesign/trace/adapter/traceSceneWiring.test.tsx+++ b/products/ai_observability/frontend/redesign/trace/adapter/traceSceneWiring.test.tsx@@ -23,6 +23,7 @@ jest.mock('../../../utils', () => ({const TRACE_URL = '/api/projects/:team_id/ai_observability/traces/:id/'const NODE_URL = '/api/projects/:team_id/ai_observability/traces/:id/nodes/:node_id/'+const EVALUATIONS_URL = '/api/projects/:team_id/ai_observability/traces/:id/evaluations/'const answeredGeneration = makeNodeDetail({id: 'gen-2',properties: {@@ -39,6 +40,7 @@ const nodeMock = {return [200, nodeId === 'gen-2' ? answeredGeneration : makeNodeDetail({ id: nodeId })]},}+const evaluationsMock = { [EVALUATIONS_URL]: () => [200, { results: [] }] }const generation = makeEvent({id: 'gen-1',@@ -67,7 +69,13 @@ describe('TraceScene', () => {describe('the new view', () => {beforeEach(() => {- useMocks({ get: { [TRACE_URL]: () => [200, makeTraceResource({ defaultNodeId: 'gen-2' })], ...nodeMock } })+ useMocks({+ get: {+ [TRACE_URL]: () => [200, makeTraceResource({ defaultNodeId: 'gen-2' })],+ ...nodeMock,+ ...evaluationsMock,+ },+ })})it('opens on the default node without the legacy trace query', async () => {@@ -94,7 +102,7 @@ describe('TraceScene', () => {describe('behind the flag gate', () => {beforeEach(() => {- useMocks({ get: { [TRACE_URL]: () => [200, makeTraceResource()], ...nodeMock } })+ useMocks({ get: { [TRACE_URL]: () => [200, makeTraceResource()], ...nodeMock, ...evaluationsMock } })})async function renderTraceRoute(flagEnabled: boolean): Promise<ReturnType<typeof render>> {diff --git a/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.test.ts b/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.test.tsindex 39b16099d8f5..7dec0e91c45a 100644--- a/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.test.ts+++ b/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.test.ts@@ -15,13 +15,10 @@ import { decodeIdSegment, makeNodeDetail, makeTraceResource } from './testFixturimport { traceViewAdapterLogic } from './traceViewAdapterLogic'jest.mock('~/queries/query', () => ({ ...jest.requireActual('~/queries/query'), performQuery: jest.fn() }))-jest.mock('../../../utils', () => ({- ...jest.requireActual('../../../utils'),- queryEvaluationRuns: jest.fn().mockResolvedValue([]),-}))const TRACE_URL = '/api/projects/:team_id/ai_observability/traces/:id/'const NODE_URL = '/api/projects/:team_id/ai_observability/traces/:id/nodes/:node_id/'+const EVALUATIONS_URL = '/api/projects/:team_id/ai_observability/traces/:id/evaluations/'const answeredGeneration = makeNodeDetail({id: 'gen-2',@@ -56,6 +53,7 @@ describe('traceViewAdapterLogic', () => {await traceGatereturn [200, traceResource]},+ [EVALUATIONS_URL]: () => [200, { results: [] }],[NODE_URL]: ({ params }) => {const nodeId = decodeIdSegment(String(params.node_id))nodeRequests.push(nodeId)diff --git a/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.ts b/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.tsindex 43eb6fdb93ba..19efefb44bb8 100644--- a/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.ts+++ b/products/ai_observability/frontend/redesign/trace/adapter/traceViewAdapterLogic.ts@@ -5,10 +5,7 @@ import { subscriptions } from 'kea-subscriptions'import { teamLogic } from 'scenes/teamLogic'import { aiObservabilityTraceLogic } from '../../../aiObservabilityTraceLogic'-import { llmEvaluationsLogic } from '../../../evaluations/llmEvaluationsLogic'-import { EvaluationConfig, EvaluationRun } from '../../../evaluations/types'-import type { TraceNodeApi, TraceTimelineRowApi } from '../../../generated/api.schemas'-import { generationEvaluationRunsLogic } from '../../../generationEvaluationRunsLogic'+import type { TraceEvaluationResultApi, TraceNodeApi, TraceTimelineRowApi } from '../../../generated/api.schemas'import { TraceHeaderProps } from '../components/TraceHeader'import { TraceSummaryBarProps } from '../components/TraceSummaryBar'import { TraceViewStatus, traceViewLogic } from '../traceViewLogic'@@ -31,16 +28,13 @@ export interface traceViewAdapterLogicValues {newerTraceId: string | null // aiObservabilityTraceLogicolderTimestamp: string | null // aiObservabilityTraceLogicolderTraceId: string | null // aiObservabilityTraceLogic- generationEvaluationRuns: EvaluationRun[] // generationEvaluationRunsLogic- generationEvaluationRunsLoading: boolean // generationEvaluationRunsLogic- detectorEvaluationIds: string[] // llmEvaluationsLogic- evaluations: EvaluationConfig[] // llmEvaluationsLogic- evaluationsSettled: boolean // llmEvaluationsLogicsearchParams: Record<string, any> // routercurrentTeamId: number | null // teamLogiccreatedAt: string | null // traceViewLogicdefaultNodeId: string // traceViewLogicerrorMessage: string // traceViewLogic+ evaluationResults: TraceEvaluationResultApi[] | null // traceViewLogic+ evaluationResultsFailed: boolean // traceViewLogicnodeIdsByAlias: Record<string, string> // traceViewLogicnodes: Record<string, NodeLoad> // traceViewLogicpersonLabel: string | null // traceViewLogic@@ -144,11 +138,8 @@ export interface traceViewAdapterLogicMeta {selectedTreeNode: TraceNodeApi | null,selectedNodeLoad: NodeLoad,tree: TraceNodeApi[],- generationEvaluationRuns: EvaluationRun[],- generationEvaluationRunsLoading: boolean,- evaluations: EvaluationConfig[],- detectorEvaluationIds: string[],- evaluationsSettled: boolean,+ evaluationResults: TraceEvaluationResultApi[] | null,+ evaluationResultsFailed: boolean,currentTeamId: number | null,personLabel: string | null,createdAt: string | null,@@ -198,15 +189,13 @@ export const traceViewAdapterLogic = kea<traceViewAdapterLogicType>(['sessionId','defaultNodeId','nodeIdsByAlias',+ 'evaluationResults',+ 'evaluationResultsFailed',],aiObservabilityTraceLogic,['eventId', 'initialTab', 'olderTraceId', 'olderTimestamp', 'newerTraceId', 'newerTimestamp'],- generationEvaluationRunsLogic({ traceId: props.traceId }),- ['generationEvaluationRuns', 'generationEvaluationRunsLoading'],teamLogic,['currentTeamId'],- llmEvaluationsLogic,- ['evaluations', 'detectorEvaluationIds', 'evaluationsSettled'],router,['searchParams'],],@@ -309,11 +298,8 @@ export const traceViewAdapterLogic = kea<traceViewAdapterLogicType>([s.selectedTreeNode,s.selectedNodeLoad,s.tree,- s.generationEvaluationRuns,- s.generationEvaluationRunsLoading,- s.evaluations,- s.detectorEvaluationIds,- s.evaluationsSettled,+ s.evaluationResults,+ s.evaluationResultsFailed,s.currentTeamId,s.personLabel,s.createdAt,@@ -323,11 +309,8 @@ export const traceViewAdapterLogic = kea<traceViewAdapterLogicType>([node: TraceNodeApi | null,load: NodeLoad,tree: TraceNodeApi[],- evalRuns: EvaluationRun[],- evalRunsLoading: boolean,- evaluations: EvaluationConfig[],- detectorEvaluationIds: string[],- evaluationsSettled: boolean,+ evaluationResults: TraceEvaluationResultApi[] | null,+ evaluationResultsFailed: boolean,teamId: number | null,person: string | null,createdAt: string | null,@@ -338,11 +321,8 @@ export const traceViewAdapterLogic = kea<traceViewAdapterLogicType>([node,load,tree,- evalRuns,- evalRunsLoading,- evaluations,- detectorEvaluationIds,- evaluationsSettled,+ evaluationResults,+ evaluationResultsFailed,teamId,person,trace: { createdAt, sessionId },diff --git a/products/ai_observability/frontend/redesign/trace/components/EvalOutcomeTag.tsx b/products/ai_observability/frontend/redesign/trace/components/EvalOutcomeTag.tsxindex 7a54425a7fa5..8cbef21130fa 100644--- a/products/ai_observability/frontend/redesign/trace/components/EvalOutcomeTag.tsx+++ b/products/ai_observability/frontend/redesign/trace/components/EvalOutcomeTag.tsx@@ -1,4 +1,4 @@-import { IconCheck, IconClock, IconMinus, IconWarning, IconX } from '@posthog/icons'+import { IconCheck, IconMinus, IconX } from '@posthog/icons'import { LemonTag, LemonTagType } from '@posthog/lemon-ui'import { EvalOutcome } from '../types'@@ -6,8 +6,6 @@ import { EvalOutcome } from '../types'const OUTCOME_STYLES: Record<EvalOutcome, { type: LemonTagType; icon: JSX.Element | undefined }> = {pass: { type: 'success', icon: <IconCheck /> },fail: { type: 'danger', icon: <IconX /> },- error: { type: 'danger', icon: <IconWarning /> },- pending: { type: 'default', icon: <IconClock /> },inconclusive: { type: 'muted', icon: <IconMinus /> },unrated: { type: 'default', icon: undefined },}diff --git a/products/ai_observability/frontend/redesign/trace/storyFixtures.tsx b/products/ai_observability/frontend/redesign/trace/storyFixtures.tsxindex 0219778bfdf4..7add40e950a3 100644--- a/products/ai_observability/frontend/redesign/trace/storyFixtures.tsx+++ b/products/ai_observability/frontend/redesign/trace/storyFixtures.tsx@@ -339,28 +339,6 @@ export const FIXTURE_EVALS: EvalResult[] = [isBackfill: false,target: null,},- {- id: 'e-tone',- name: 'Tone',- href: '/evaluations/tone',- outcome: 'error',- label: 'Error',- reasoning: 'The judge model returned an empty response.',- timestamp: EVAL_TIMESTAMP,- isBackfill: false,- target: null,- },- {- id: 'e-policy',- name: 'Follows billing policy',- href: '/evaluations/follows-billing-policy',- outcome: 'pending',- label: 'Running',- reasoning: null,- timestamp: EVAL_TIMESTAMP,- isBackfill: false,- target: null,- },]export const FIXTURE_TRACE_EVALS: EvalResult[] = FIXTURE_EVALS.map((result) => ({diff --git a/products/ai_observability/frontend/redesign/trace/traceViewLogic.test.ts b/products/ai_observability/frontend/redesign/trace/traceViewLogic.test.tsindex ceb79aa19cd1..11f89295ea48 100644--- a/products/ai_observability/frontend/redesign/trace/traceViewLogic.test.ts+++ b/products/ai_observability/frontend/redesign/trace/traceViewLogic.test.ts@@ -1,17 +1,21 @@+import { waitFor } from '@testing-library/react'import { expectLogic } from 'kea-test-utils'import posthog from 'posthog-js'+import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'+import { resumeKeaLoadersErrors, silenceKeaLoadersErrors } from '~/initKea'import { panelLayoutLogic } from '~/layout/panel-layout/panelLayoutLogic'import { useMocks } from '~/mocks/jest'import { initKeaTests } from '~/test/init'import type { TraceTimelineRowApi } from '../../generated/api.schemas'-import { decodeIdSegment, makeNodeDetail, makeTraceResource } from './adapter/testFixtures'+import { decodeIdSegment, makeEvaluationResult, makeNodeDetail, makeTraceResource } from './adapter/testFixtures'import { traceViewLogic } from './traceViewLogic'const TRACE_URL = '/api/projects/:team_id/ai_observability/traces/:id/'const NODE_URL = '/api/projects/:team_id/ai_observability/traces/:id/nodes/:node_id/'+const EVALUATIONS_URL = '/api/projects/:team_id/ai_observability/traces/:id/evaluations/'const EXPIRED_ROW: TraceTimelineRowApi = {id: 'expired-1',kind: 'span',@@ -69,6 +73,7 @@ describe('traceViewLogic', () => {requestedPath = url.pathnamereturn [200, makeTraceResource({ name: null })]},+ [EVALUATIONS_URL]: () => [200, { results: [] }],},})initKeaTests()@@ -128,6 +133,47 @@ describe('traceViewLogic', () => {.toMatchValues({ status: 'error', errorMessage: expectedMessage })})+ it.each([+ ['answers', 200, { evaluationResults: [makeEvaluationResult()], evaluationResultsFailed: false }, false],+ ['fails', 500, { evaluationResults: null, evaluationResultsFailed: true }, true],+ ] as const)(+ 'reads the evaluation results alongside the trace and keeps the trace ready, without a toast, when that read %s',+ async (_description, statusCode, expected, reported) => {+ const toast = jest.spyOn(lemonToast, 'error')+ const report = jest.spyOn(posthog, 'captureException')+ const traceAnswer = deferred()+ let traceAnswered = false+ let evaluationsReadBeforeTrace = false+ useMocks({+ get: {+ [TRACE_URL]: async () => {+ await traceAnswer.promise+ traceAnswered = true+ return [200, makeTraceResource()]+ },+ [EVALUATIONS_URL]: () => {+ evaluationsReadBeforeTrace = !traceAnswered+ return statusCode === 200+ ? [200, { results: [makeEvaluationResult()] }]+ : [statusCode, { detail: 'nope' }]+ },+ },+ })+ const logic = traceViewLogic({ traceId: 'trace-1', timestampHint: null })+ logic.mount()++ await waitFor(() => expect(logic.values.evaluationResultsLoading).toBe(false))+ expect(evaluationsReadBeforeTrace).toBe(true)+ traceAnswer.resolve()++ await expectLogic(logic)+ .toFinishAllListeners()+ .toMatchValues({ ...expected, status: 'ready' })+ expect(toast).not.toHaveBeenCalled()+ expect(report).toHaveBeenCalledTimes(reported ? 1 : 0)+ }+ )+it('loads the thread nodes once the trace arrives', async () => {const logic = await mountedLogic()diff --git a/products/ai_observability/frontend/redesign/trace/traceViewLogic.ts b/products/ai_observability/frontend/redesign/trace/traceViewLogic.tsindex 92282d95f313..253915f1b9d3 100644--- a/products/ai_observability/frontend/redesign/trace/traceViewLogic.ts+++ b/products/ai_observability/frontend/redesign/trace/traceViewLogic.ts@@ -20,8 +20,18 @@ import { urls } from 'scenes/urls'import { panelLayoutLogic } from '~/layout/panel-layout/panelLayoutLogic'-import { aiObservabilityTracesNodesRetrieve, aiObservabilityTracesRetrieve } from '../../generated/api'-import type { TraceApi, TraceNodeApi, TraceNodeDetailApi, TraceTimelineRowApi } from '../../generated/api.schemas'+import {+ aiObservabilityTracesEvaluationsRetrieve,+ aiObservabilityTracesNodesRetrieve,+ aiObservabilityTracesRetrieve,+} from '../../generated/api'+import type {+ TraceApi,+ TraceEvaluationResultApi,+ TraceNodeApi,+ TraceNodeDetailApi,+ TraceTimelineRowApi,+} from '../../generated/api.schemas'import { TraceSummaryBarProps } from './components/TraceSummaryBar'import { encodeTraceIdSegment } from './traceIdSegment'import type { LoadFailure, NodeLoad, NodeLoadTrigger } from './types'@@ -38,6 +48,9 @@ export interface traceViewLogicValues {createdAt: string | nulldefaultNodeId: stringerrorMessage: string+ evaluationResults: TraceEvaluationResultApi[] | null+ evaluationResultsFailed: boolean+ evaluationResultsLoading: booleanloadFailure: LoadFailure | nullnodeIdsByAlias: Record<string, string>nodes: Record<string, NodeLoad>@@ -67,6 +80,21 @@ export interface traceViewLogicActions {collapsed: boolean | nullowner: string} // panelLayoutLogic+ loadEvaluationResults: () => any+ loadEvaluationResultsFailure: (+ error: string,+ errorObject?: any+ ) => {+ error: string+ errorObject?: any+ }+ loadEvaluationResultsSuccess: (+ evaluationResults: TraceEvaluationResultApi[],+ payload?: any+ ) => {+ evaluationResults: TraceEvaluationResultApi[]+ payload?: any+ }loadNode: (nodeId: string,trigger: NodeLoadTrigger@@ -185,8 +213,27 @@ export const traceViewLogic = kea<traceViewLogicType>([},},],+ evaluationResults: [+ null as TraceEvaluationResultApi[] | null,+ {+ loadEvaluationResults: async (): Promise<TraceEvaluationResultApi[]> => {+ const { results } = await aiObservabilityTracesEvaluationsRetrieve(+ String(ApiConfig.getCurrentProjectId()),+ encodeTraceIdSegment(props.traceId)+ )+ return results+ },+ },+ ],})),reducers({+ evaluationResultsFailed: [+ false,+ {+ loadEvaluationResults: () => false,+ loadEvaluationResultsFailure: () => true,+ },+ ],loadFailure: [null as LoadFailure | null,{@@ -341,6 +388,7 @@ export const traceViewLogic = kea<traceViewLogicType>([}),afterMount(({ actions, props }) => {actions.loadTrace()+ actions.loadEvaluationResults()actions.setLayoutNavCollapsedOverride(layoutNavOverrideOwner(props), true)}),beforeUnmount(({ actions, props }) => {diff --git a/products/ai_observability/frontend/redesign/trace/types.ts b/products/ai_observability/frontend/redesign/trace/types.tsindex 864a6a6271f5..ee40660dc84d 100644--- a/products/ai_observability/frontend/redesign/trace/types.ts+++ b/products/ai_observability/frontend/redesign/trace/types.ts@@ -1,4 +1,4 @@-import type { TraceNodeApi, TraceNodeDetailApi } from '../../generated/api.schemas'+import type { TraceEvaluationResultOutcomeEnumApi, TraceNodeApi, TraceNodeDetailApi } from '../../generated/api.schemas'export type TraceNodeKind = TraceNodeApi['kind']@@ -77,7 +77,7 @@ export interface NodeProperties {* `unrated` is a result with a value but no verdict, such as a numeric score without a passing rule.* `inconclusive` is a run that produced no usable value: skipped, not applicable, no score, unknown sentiment.*/-export type EvalOutcome = 'pass' | 'fail' | 'error' | 'pending' | 'inconclusive' | 'unrated'+export type EvalOutcome = TraceEvaluationResultOutcomeEnumApiexport interface EvalTarget {label: stringdiff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex e282de2dfdd4..73403f56343b 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -116069,6 +116069,34 @@ export namespace Schemas {nodeIdsByAlias: TraceNodeIdsByAlias;}+ export type TraceEvaluationResultOutcomeEnum = typeof TraceEvaluationResultOutcomeEnum[keyof typeof TraceEvaluationResultOutcomeEnum];+++ export const TraceEvaluationResultOutcomeEnum = {+ Pass: 'pass',+ Fail: 'fail',+ Inconclusive: 'inconclusive',+ Unrated: 'unrated',+ } as const;++ export interface TraceEvaluationResult {+ id: string;+ evaluationId: string | null;+ evaluationName: string;+ /** The tree node the result grades: the generation's event uuid, or the trace id for a trace-level result. */+ targetNodeId: string;+ outcome: TraceEvaluationResultOutcomeEnum;+ /** What the result says, such as "True", "0.82", "Positive" or "Skipped". */+ label: string;+ reasoning: string | null;+ timestamp: string;+ isBackfill: boolean;+ }++ export interface TraceEvaluationResults {+ results: TraceEvaluationResult[];+ }+export interface TraceNodeMetadata {provider: string | null;temperature: number | null;