Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): serve the online evals overview from python (#122069)

Bernat Torresdd50c85ae481
diff --git a/.github/new-events-schema-targets.txt b/.github/new-events-schema-targets.txt
index d1ab46afd897..8a532f1bf37a 100644
--- a/.github/new-events-schema-targets.txt
+++ b/.github/new-events-schema-targets.txt
@@ -75,6 +75,7 @@ posthog/api/test/test_query.py
posthog/models/test/test_async_deletion_model.py
posthog/models/web_preaggregated/test_web_pre_aggregated_timezones.py
posthog/tasks/test/test_stop_surveys_reached_target.py
+products/ai_observability/backend/api/test/test_evaluations.py
products/ai_observability/backend/api/test/test_personal_spend.py
products/revenue_analytics/backend/views/test/test_mrr_views.py
products/surveys/backend/api/test/test_survey.py
diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.py
index d8b8742743fa..e6a11e200495 100644
--- a/products/ai_observability/backend/api/evaluations.py
+++ b/products/ai_observability/backend/api/evaluations.py
@@ -1,7 +1,10 @@
+import hashlib
from collections import Counter
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
+from datetime import UTC, date, datetime
from typing import TYPE_CHECKING, Any, cast
+from django.core.cache import cache
from django.db import models, transaction
from django.db.models import Q, QuerySet
from django.http import Http404
@@ -9,7 +12,8 @@ from django.http import Http404
import structlog
import django_filters
from django_filters.rest_framework import DjangoFilterBackend
-from drf_spectacular.utils import extend_schema, extend_schema_field
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_field
from rest_framework import exceptions, serializers, viewsets
from rest_framework.decorators import action
from rest_framework.exceptions import NotFound
@@ -20,7 +24,9 @@ from rest_framework.response import Response
from posthog.hogql import ast
+from posthog.api.fields import RepeatedOrCommaSeparatedListField
from posthog.api.forbid_destroy_model import ForbidDestroyModel
+from posthog.api.mixins import ValidatedRequest, validated_request
from posthog.api.monitoring import monitor
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.scoped_related_fields import TeamScopedPrimaryKeyRelatedField
@@ -30,13 +36,15 @@ from posthog.event_usage import report_user_action
from posthog.hogql_queries.ai.ai_table_resolver import AIEventsUnavailableError, query_ai_events
from posthog.hogql_queries.ai.utils import HEAVY_COLUMN_NAMES, merge_heavy_properties
from posthog.models.team import Team
-from posthog.permissions import AccessControlPermission
+from posthog.permissions import AccessControlPermission, is_service_auth
from posthog.temporal.ai_observability.message_utils import extract_text_from_messages
from posthog.temporal.ai_observability.model_resolution import active_key_fallback
from posthog.temporal.ai_observability.run_evaluation import extract_event_io, run_hog_eval
from posthog.temporal.ai_observability.run_session_evaluation import run_hog_eval_over_recent_sessions
from posthog.temporal.ai_observability.run_trace_evaluation import run_hog_eval_over_recent_traces
+from products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_team
+from products.access_control.backend.facade.property_access import restriction_fingerprint
from products.access_control.backend.presentation.access_control import (
AccessControlViewSetMixin,
UserAccessControlSerializerMixin,
@@ -44,6 +52,11 @@ from products.access_control.backend.presentation.access_control import (
from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definition
from ..evaluation_conditions import build_condition_filter
+from ..facade import (
+ api as facade,
+ contracts,
+)
+from ..facade.contracts import Contract
from ..hog import compile_ai_observability_hog
from ..llm import DEFAULT_MODEL_BY_PROVIDER
from ..llm.decisions import decision_evaluations_enabled, is_decision_model
@@ -82,12 +95,15 @@ from ..models.evaluation_reports import EvaluationReport
from ..models.evaluations import Evaluation, EvaluationTarget
from ..models.model_configuration import LLMModelConfiguration
from ..models.provider_keys import LLMProvider, LLMProviderKey
+from ..presentation.query_concurrency import query_concurrency_slots
from .condition_sets import ConditionSetSerializer
from .metrics import llma_track_latency
if TYPE_CHECKING:
from posthog.models import User
+ from ..logic.evaluations.period import PeriodBounds
+
logger = structlog.get_logger(__name__)
@@ -998,6 +1014,71 @@ class HogEvaluationOutputType(models.TextChoices):
CATEGORICAL = OutputType.CATEGORICAL.value, OutputType.CATEGORICAL.label
+OVERVIEW_CACHE_SECONDS = 300
+
+
+@extend_schema_field(OpenApiTypes.STR)
+class _DateOrDateTimeField(serializers.Field):
+ """A `YYYY-MM-DD` date for whole days, or an ISO 8601 datetime for an exact instant."""
+
+ def to_internal_value(self, data: Any) -> date | datetime:
+ if not isinstance(data, str):
+ self.fail("invalid")
+ try:
+ return date.fromisoformat(data) if len(data) == 10 else datetime.fromisoformat(data)
+ except ValueError:
+ self.fail("invalid")
+
+ def to_representation(self, value: date | datetime) -> str:
+ return value.isoformat()
+
+ default_error_messages = {"invalid": "Enter a date (YYYY-MM-DD) or an ISO 8601 datetime."}
+
+
+class EvaluationsOverviewPeriodSerializer(serializers.Serializer):
+ date_from = _DateOrDateTimeField(
+ required=False,
+ help_text="Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. "
+ "Defaults to 13 days before `date_to`.",
+ )
+ date_to = _DateOrDateTimeField(
+ required=False,
+ help_text="End of the period: a date includes that whole day, a datetime is an exclusive instant. "
+ "Defaults to today.",
+ )
+ refresh = serializers.BooleanField(
+ required=False,
+ default=False,
+ help_text="Skip the result cached for the last five minutes and compute it again.",
+ )
+
+
+class EvaluationsOverviewQuerySerializer(EvaluationsOverviewPeriodSerializer):
+ evaluation_ids = RepeatedOrCommaSeparatedListField(
+ child=serializers.UUIDField(),
+ required=False,
+ help_text="Only these evaluations. Defaults to every evaluation the user can see.",
+ )
+
+
+class EvaluationRunOutcome(models.TextChoices):
+ PASSED = "passed", "Passed"
+ FAILED = "failed", "Failed"
+
+
+class EvaluationRunsQuerySerializer(EvaluationsOverviewPeriodSerializer):
+ outcome = serializers.ChoiceField(
+ choices=EvaluationRunOutcome.choices,
+ required=False,
+ help_text="Only runs with this outcome under the evaluation's passing rule. Without it, every graded run, "
+ "including scored, categorized and sentiment runs. An evaluation without a passing rule rejects this filter "
+ "with a 400.",
+ )
+ limit = serializers.IntegerField(
+ required=False, default=5, min_value=1, max_value=50, help_text="How many runs to return, newest first."
+ )
+
+
class TestHogRequestSerializer(serializers.Serializer):
output_type = serializers.ChoiceField(
choices=HogEvaluationOutputType.choices,
@@ -1507,6 +1588,133 @@ class EvaluationViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, Forbi
def partial_update(self, request, *args, **kwargs):
return super().partial_update(request, *args, **kwargs)
+ def _overview_period(self, request: ValidatedRequest) -> "PeriodBounds":
+ query = request.validated_query_data
+ try:
+ return facade.evaluations_period(self.team, query.get("date_from"), query.get("date_to"))
+ except facade.InvalidPeriodError as error:
+ raise serializers.ValidationError({error.field: str(error)} if error.field else str(error))
+
+ def _cached_overview_read(
+ self,
+ request: ValidatedRequest,
+ name: str,
+ evaluation_ids: Sequence[str],
+ period: "PeriodBounds",
+ read: Callable[[], Contract],
+ ) -> Response:
+ # Keyed by the evaluations the user can see and by their current property restrictions, because a hit skips
+ # the masking a fresh read applies, and a restriction can change while an entry is still cached.
+ visible = hashlib.sha256(",".join(sorted(evaluation_ids)).encode()).hexdigest()
+ restrictions = restriction_fingerprint(
+ get_restricted_properties_with_group_type_index_for_team(
+ user=cast("User", request.user), team_id=self.team.pk
+ )
+ )
+ cache_key = (
+ f"ai_evaluations_{name}/{self.team.pk}/{request.user.pk}/{period.start.astimezone(UTC).isoformat()}/{period.end.astimezone(UTC).isoformat()}"
+ f"/{visible}/{restrictions}"
+ )
+ if not request.validated_query_data["refresh"]:
+ cached = cache.get(cache_key)
+ if cached is not None:
+ return Response(cached)
+ tag_queries(product=Product.LLM_ANALYTICS, feature=Feature.QUERY)
+ with query_concurrency_slots(self.team):
+ payload = read().model_dump(mode="json", by_alias=True)
+ cache.set(cache_key, payload, timeout=OVERVIEW_CACHE_SECONDS)
+ return Response(payload)
+
+ @validated_request(
+ query_serializer=EvaluationsOverviewQuerySerializer,
+ responses={
+ 200: OpenApiResponse(response=contracts.EvaluationsOverview),
+ 400: OpenApiResponse(description="The period is malformed, empty, longer than 90 days or out of range."),
+ 429: OpenApiResponse(description="Too many queries are running for the project or organization."),
+ },
+ )
+ @action(detail=False, methods=["get"], url_path="overview", required_scopes=["evaluation:read"])
+ def overview(self, request: ValidatedRequest, **kwargs) -> Response:
+ """Each evaluation's results over a period, compared with the period of equal length before it."""
+ period = self._overview_period(request)
+ queryset = self.get_queryset()
+ # The base viewset narrows by access level on `list` only, so this read applies it itself.
+ if not is_service_auth(request):
+ queryset = self.user_access_control.filter_queryset_by_access_level(queryset)
+ query = request.validated_query_data
+ if "evaluation_ids" in query:
+ queryset = queryset.filter(id__in=query["evaluation_ids"])
+ evaluation_ids = [str(evaluation_id) for evaluation_id in queryset.values_list("id", flat=True)]
+ return self._cached_overview_read(
+ request,
+ "overview",
+ evaluation_ids,
+ period,
+ lambda: facade.get_evaluations_overview(self.team, cast("User", request.user), evaluation_ids, period),
+ )
+
+ @validated_request(
+ query_serializer=EvaluationRunsQuerySerializer,
+ responses={
+ 200: OpenApiResponse(response=contracts.EvaluationRuns),
+ 400: OpenApiResponse(
+ description="The period is invalid, or an outcome filter is set on an evaluation without a passing rule."
+ ),
+ 429: OpenApiResponse(description="Too many queries are running for the project or organization."),
+ },
+ )
+ @action(detail=True, methods=["get"], url_path="runs", required_scopes=["evaluation:read"])
+ def runs(self, request: ValidatedRequest, **kwargs) -> Response:
+ """One evaluation's latest graded runs in a period, newest first, optionally only the passed or failed ones."""
+ evaluation_id = str(self.get_object().id)
+ period = self._overview_period(request)
+ query = request.validated_query_data
+ outcome, limit = query.get("outcome"), query["limit"]
+ try:
+ return self._cached_overview_read(
+ request,
+ f"runs/{outcome}/{limit}",
+ [evaluation_id],
+ period,
+ lambda: facade.get_evaluation_runs(
+ self.team, cast("User", request.user), evaluation_id, period, outcome, limit
+ ),
+ )
+ except facade.NoPassingRuleError:
+ raise serializers.ValidationError(
+ {"outcome": "This evaluation has no passing rule, so its runs are neither passed nor failed."}
+ )
+
+ @validated_request(
+ query_serializer=EvaluationsOverviewPeriodSerializer,
+ responses={
+ 200: OpenApiResponse(response=contracts.ScoreHistogram),
+ 400: OpenApiResponse(
+ description="The period is invalid, or the evaluation is not numeric with a minimum and maximum score."
+ ),
+ 429: OpenApiResponse(description="Too many queries are running for the project or organization."),
+ },
+ )
+ @action(detail=True, methods=["get"], url_path="score_distribution", required_scopes=["evaluation:read"])
+ def score_distribution(self, request: ValidatedRequest, **kwargs) -> Response:
+ """How one numeric evaluation's scores in a period spread across its configured scale."""
+ evaluation_id = str(self.get_object().id)
+ period = self._overview_period(request)
+ try:
+ return self._cached_overview_read(
+ request,
+ "score_distribution",
+ [evaluation_id],
+ period,
+ lambda: facade.get_evaluation_score_distribution(
+ self.team, cast("User", request.user), evaluation_id, period
+ ),
+ )
+ except facade.NoScoreScaleError:
+ raise serializers.ValidationError(
+ "Only a numeric evaluation with a minimum and a maximum score has a score distribution."
+ )
+
@extend_schema(request=TestHogRequestSerializer, responses=TestHogResponseSerializer)
@action(detail=False, methods=["post"], url_path="test_hog", required_scopes=["evaluation:read"])
def test_hog(self, request: Request, **kwargs) -> Response:
diff --git a/products/ai_observability/backend/api/test/test_evaluations.py b/products/ai_observability/backend/api/test/test_evaluations.py
index 2464809e4cd5..8f0504a2cb45 100644
--- a/products/ai_observability/backend/api/test/test_evaluations.py
+++ b/products/ai_observability/backend/api/test/test_evaluations.py
@@ -2,7 +2,7 @@ import json
from datetime import timedelta
from uuid import UUID, uuid4
-from posthog.test.base import APIBaseTest
+from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
from unittest.mock import Mock, patch
from django.db import IntegrityError, connection, transaction
@@ -21,6 +21,7 @@ from posthog.models import Organization, OrganizationMembership, Project, Team,
from products.access_control.backend.models.access_control import AccessControl
from products.ai_observability.backend.api.evaluations import (
EvaluationSerializer,
+ EvaluationsOverviewPeriodSerializer,
ModelConfigurationSerializer,
TestHogRequestSerializer as HogRequestSerializer,
_TargetConfigField,
@@ -2550,7 +2551,7 @@ class TestReEnableValidatesRootCauseResolved(APIBaseTest):
self.assertIsNone(eval_obj.status_reason)
-class TestEvaluationsAccessControl(APIBaseTest):
+class _EvaluationAccessControlCase(APIBaseTest):
# Evaluations have their own access control resource (see ACCESS_CONTROL_RESOURCES in
# products/access_control/backend/facade/user_access_control.py). They used to inherit from `llm_analytics`.
def setUp(self) -> None:
@@ -2560,13 +2561,7 @@ class TestEvaluationsAccessControl(APIBaseTest):
{"key": AvailableFeature.ROLE_BASED_ACCESS, "name": AvailableFeature.ROLE_BASED_ACCESS},
]
self.organization.save()
-
- self.viewer_user = User.objects.create_and_join(self.organization, "[email protected]", "testtest")
- self.editor_user = User.objects.create_and_join(self.organization, "[email protected]", "testtest")
self.no_access_user = User.objects.create_and_join(self.organization, "[email protected]", "testtest")
-
- self._grant(self.viewer_user, "viewer")
- self._grant(self.editor_user, "editor")
self._grant(self.no_access_user, "none")
def _grant(
@@ -2581,16 +2576,27 @@ class TestEvaluationsAccessControl(APIBaseTest):
organization_member=membership,
)
- def _create_evaluation(self, name: str = "Existing Evaluation") -> Evaluation:
+ def _create_evaluation(self, name: str = "Existing Evaluation", enabled: bool = False) -> Evaluation:
return Evaluation.objects.create(
team=self.team,
name=name,
+ enabled=enabled,
evaluation_type="hog",
evaluation_config={"source": "return true"},
output_type="boolean",
created_by=self.user,
)
+
+class TestEvaluationsAccessControl(_EvaluationAccessControlCase):
+ def setUp(self) -> None:
+ super().setUp()
+ self.viewer_user = User.objects.create_and_join(self.organization, "[email protected]", "testtest")
+ self.editor_user = User.objects.create_and_join(self.organization, "[email protected]", "testtest")
+
+ self._grant(self.viewer_user, "viewer")
+ self._grant(self.editor_user, "editor")
+
def _create_payload(self, name: str = "New Evaluation") -> dict:
return {
"name": name,
@@ -2720,3 +2726,91 @@ class TestEvaluationsAccessControl(APIBaseTest):
response = self.client.get(f"/api/projects/{self.team.id}/llm_analytics/models/?provider=openai")
assert response.status_code == status.HTTP_200_OK
+
+
+class TestEvaluationsOverviewAccessControl(ClickhouseTestMixin, _EvaluationAccessControlCase):
+ def test_overview_hides_evaluations_the_user_cannot_see(self) -> None:
+ visible = self._create_evaluation("Visible", enabled=True)
+ hidden = self._create_evaluation("Hidden", enabled=True)
+ self._grant(self.no_access_user, "viewer", resource_id=str(visible.id))
+ for evaluation in (visible, hidden, hidden):
+ _create_event(
+ team=self.team,
+ event="$ai_evaluation",
+ distinct_id="user-1",
+ timestamp="2026-03-09T10:00:00Z",
+ properties={"$ai_evaluation_id": str(evaluation.id), "$ai_evaluation_result": True},
+ )
+ flush_persons_and_events()
+ url = (
+ f"/api/projects/{self.team.id}/evaluations/overview/?date_from=2026-03-08&date_to=2026-03-10"
+ f"&evaluation_ids={visible.id},{hidden.id}"
+ )
+ # A user who sees both evaluations fills the cache first, so a shared entry would leak their totals.
+ assert self.client.get(url).json()["totals"]["graded"] == 3
+ self.client.force_login(self.no_access_user)
+
+ response = self.client.get(url)
+
+ assert response.status_code == status.HTTP_200_OK
+ assert [row["name"] for row in response.json()["evaluations"]] == ["Visible"]
+ assert response.json()["totals"]["graded"] == 1
+
+ @parameterized.expand([("runs",), ("score_distribution",)])
+ def test_evaluation_reads_need_access_to_the_evaluation(self, endpoint: str) -> None:
+ hidden = self._create_evaluation("Hidden")
+ self.client.force_login(self.no_access_user)
+
+ response = self.client.get(f"/api/projects/{self.team.id}/evaluations/{hidden.id}/{endpoint}/")
+
+ assert response.status_code in (status.HTTP_403_FORBIDDEN, status.HTTP_404_NOT_FOUND)
+
+ @parameterized.expand(
+ [
+ ("period", "{id}/runs/?date_from=2026-03-10&date_to=2026-03-09", "date_from", "Must be before date_to."),
+ (
+ "period out of range",
+ "overview/?date_to=9999-12-31",
+ None,
+ "The period is outside the supported date range.",
+ ),
+ (
+ "outcome without a passing rule",
+ "{id}/runs/?outcome=failed",
+ "outcome",
+ "This evaluation has no passing rule, so its runs are neither passed nor failed.",
+ ),
+ (
+ "score distribution without bounds",
+ "{id}/score_distribution/",
+ None,
+ "Only a numeric evaluation with a minimum and a maximum score has a score distribution.",
+ ),
+ ]
+ )
+ def test_invalid_reads_return_400(self, _name: str, path: str, attr: str | None, detail: str) -> None:
+ evaluation = Evaluation.objects.create(
+ team=self.team,
+ name="Score",
+ evaluation_type="hog",
+ evaluation_config={"source": "return true"},
+ output_type="numeric",
+ output_config={},
+ )
+
+ response = self.client.get(f"/api/projects/{self.team.id}/evaluations/{path.format(id=evaluation.id)}")
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert (response.json()["attr"], response.json()["detail"]) == (attr, detail)
+
+
+class TestEvaluationsOverviewPeriod(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("date", {"date_from": "2026-03-10"}, True),
+ ("instant with offset", {"date_to": "2026-03-10T05:30:00+05:30"}, True),
+ ("not a date", {"date_from": "2026-13-01"}, False),
+ ]
+ )
+ def test_period_parsing(self, _name: str, data: dict[str, str], valid: bool) -> None:
+ assert EvaluationsOverviewPeriodSerializer(data=data).is_valid() is valid
diff --git a/products/ai_observability/backend/facade/api.py b/products/ai_observability/backend/facade/api.py
index faa371330d0a..578b0a0aa9c7 100644
--- a/products/ai_observability/backend/facade/api.py
+++ b/products/ai_observability/backend/facade/api.py
@@ -1,8 +1,19 @@
-from datetime import datetime
+from collections.abc import Sequence
+from datetime import date, datetime
from posthog.models import Team, User
-from ..logic.evaluations.pass_filters import build_evaluation_pass_filters
+from ..logic.evaluations.evaluation_metas import load_evaluation_meta, load_evaluation_metas
+from ..logic.evaluations.overview import evaluation_result, group_by_evaluation, overview_totals, pass_rate_overview
+from ..logic.evaluations.overview_queries import load_bucket_counts, load_category_counts
+from ..logic.evaluations.pass_filters import build_evaluation_pass_filters, build_pass_rules
+from ..logic.evaluations.period import (
+ InvalidPeriodError as InvalidPeriodError,
+ PeriodBounds,
+ overview_period,
+)
+from ..logic.evaluations.runs import RunOutcome, load_runs
+from ..logic.evaluations.score_distribution import load_score_distribution, score_buckets
from ..logic.traces.evaluation_result import EvaluationResult
from ..logic.traces.evaluation_result_queries import load_evaluation_results, load_output_configs
from ..logic.traces.events_fallback import fallback_window
@@ -23,6 +34,14 @@ class TraceNodeNotFoundError(Exception):
pass
+class NoPassingRuleError(Exception):
+ pass
+
+
+class NoScoreScaleError(Exception):
+ pass
+
+
_NO_METADATA = contracts.TraceNodeMetadata(
provider=None, temperature=None, session_id=None, prompt_name=None, prompt_version=None
)
@@ -186,3 +205,70 @@ def _to_evaluation_result(result: EvaluationResult) -> contracts.TraceEvaluation
timestamp=result.timestamp,
is_backfill=result.is_backfill,
)
+
+
+def evaluations_period(team: Team, date_from: date | datetime | None, date_to: date | datetime | None) -> PeriodBounds:
+ """Raises `InvalidPeriodError` for a period that is empty, longer than 90 days or out of range."""
+ return PeriodBounds.from_request(date_from, date_to, team.timezone_info)
+
+
+def get_evaluations_overview(
+ team: Team, user: User | None, evaluation_ids: Sequence[str], period: PeriodBounds
+) -> contracts.EvaluationsOverview:
+ """Each evaluation's results in the period, compared with the period of equal length before it, plus the
+ totals the overview cards show. Two ClickHouse reads, unless the evaluations group into more rows than half a
+ page holds.
+
+ While the period still runs, the previous one stops at the same point, so a partial today is not compared with
+ a full day."""
+ overview = overview_period(period.start, period.end, datetime.now(team.timezone_info), team.timezone_info)
+ metas = load_evaluation_metas(team.pk, evaluation_ids)
+ ids = [meta.id for meta in metas]
+ counts = group_by_evaluation(load_bucket_counts(team, user, overview, ids, build_pass_rules(team.pk, ids)))
+ categories = group_by_evaluation(
+ load_category_counts(
+ team, user, overview, {meta.id: len(meta.categories) for meta in metas if meta.kind == "categories"}
+ )
+ )
+ rows = [
+ evaluation_result(meta, overview.buckets, counts.get(meta.id, []), categories.get(meta.id, []))
+ for meta in metas
+ ]
+ bucket_count = len(overview.buckets)
+ return contracts.EvaluationsOverview(
+ start=overview.start,
+ end=overview.end,
+ previous_start=overview.previous_start,
+ previous_end=overview.previous_end,
+ interval=overview.interval,
+ buckets=list(overview.buckets),
+ evaluations=rows,
+ totals=overview_totals(rows, bucket_count),
+ pass_rate_overview=pass_rate_overview(rows, bucket_count),
+ )
+
+
+def get_evaluation_runs(
+ team: Team,
+ user: User | None,
+ evaluation_id: str,
+ period: PeriodBounds,
+ outcome: RunOutcome | None,
+ limit: int,
+) -> contracts.EvaluationRuns:
+ """Raises `NoPassingRuleError` for an outcome filter on an evaluation without a passing rule."""
+ if outcome is not None and load_evaluation_meta(team.pk, evaluation_id).kind != "pass_rate":
+ raise NoPassingRuleError(evaluation_id)
+ pass_rules = build_pass_rules(team.pk, [evaluation_id])
+ return contracts.EvaluationRuns(results=load_runs(team, user, evaluation_id, period, pass_rules, outcome, limit))
+
+
+def get_evaluation_score_distribution(
+ team: Team, user: User | None, evaluation_id: str, period: PeriodBounds
+) -> contracts.ScoreHistogram:
+ """Raises `NoScoreScaleError` unless the evaluation is numeric with a minimum below its maximum."""
+ meta = load_evaluation_meta(team.pk, evaluation_id)
+ buckets = score_buckets(meta.scale, meta.score_step) if meta.scale is not None else None
+ if meta.scale is None or buckets is None:
+ raise NoScoreScaleError(evaluation_id)
+ return load_score_distribution(team, user, evaluation_id, period, meta.scale, buckets)
diff --git a/products/ai_observability/backend/facade/contracts.py b/products/ai_observability/backend/facade/contracts.py
index d7eac52720d5..3103e4335a35 100644
--- a/products/ai_observability/backend/facade/contracts.py
+++ b/products/ai_observability/backend/facade/contracts.py
@@ -1,5 +1,5 @@
from datetime import datetime
-from typing import Any, Literal, get_args
+from typing import Annotated, Any, Literal, get_args
from pydantic import BaseModel, ConfigDict, Field
from pydantic.alias_generators import to_camel
@@ -130,3 +130,195 @@ class TraceEvaluationResult(Contract):
class TraceEvaluationResults(Contract):
results: list[TraceEvaluationResult]
+
+
+AttentionReason = Literal["errored", "silent", "worse"]
+BucketStatus = Literal["none", "dip", "normal"]
+Interval = Literal["hour", "day"]
+ChangeUnit = Literal["pp", "points"]
+
+
+class PassFailCounts(Contract):
+ passed: int
+ failed: int
+
+
+class ScoreScale(Contract):
+ min: float
+ max: float
+
+
+class ScoreDistribution(Contract):
+ mean: float
+ scale: ScoreScale | None = Field(description="The configured score bounds, or null when the evaluation sets none.")
+
+
+class PassRateSummary(Contract):
+ kind: Literal["pass_rate"]
+ passed: int
+ failed: int
+ previous: PassFailCounts | None = Field(
+ description="Counts in the previous period, or null when it graded too few to compare."
+ )
+ score: ScoreDistribution | None = Field(description="The score of numeric evaluations with a passing rule.")
+
+
+class ScoreSummary(Contract):
+ kind: Literal["score"]
+ score: ScoreDistribution | None = Field(description="Null when the current period scored nothing.")
+ previous_mean: float | None
+
+
+class CategoryCount(Contract):
+ key: str
+ label: str
+ count: int
+ previous_share: float | None = Field(
+ description=(
+ "Percentage of the previous period's graded runs with this category, or null when it graded too few to compare."
+ )
+ )
+
+
+class CategoriesSummary(Contract):
+ kind: Literal["categories"]
+ multiple: bool
+ categories: list[CategoryCount] = Field(description="Sorted by count, most frequent first.")
+ no_categories_count: int = Field(description="Multiple-selection runs that returned no category.")
+
+
+class SentimentSummary(Contract):
+ kind: Literal["sentiment"]
+ positive: int
+ neutral: int
+ negative: int
+ previous_negative_share: float | None
+
+
+ResultSummary = Annotated[
+ PassRateSummary | ScoreSummary | CategoriesSummary | SentimentSummary, Field(discriminator="kind")
+]
+
+
+class ResultChange(Contract):
+ value: float = Field(description="Signed difference against the previous period, rounded for display.")
+ unit: ChangeUnit
+ subject: str | None = Field(description="What changed when it is not the headline, such as a category label.")
+
+
+class EvaluationResultTrend(Contract):
+ values: list[float | None] = Field(
+ description="Headline value per bucket, aligned with the overview's `buckets`. Null where nothing was graded."
+ )
+ volume: list[int] = Field(description="Graded runs per bucket.")
+
+
+class EvaluationResult(Contract):
+ id: str
+ name: str
+ directory_name: str | None
+ evaluation_type: str
+ output_type: str
+ enabled: bool
+ status_reason: str | None = Field(
+ description="Why the evaluation stopped with an error, as an evaluation status reason, or null when it has none."
+ )
+ created_by: str | None
+ graded_count: int
+ previous_graded_count: int | None = Field(description="Graded runs in the previous period, or null when none.")
+ not_applicable_count: int
+ skipped_count: int = Field(description="Runs skipped before scoring.")
+ last_run_at: datetime | None = Field(description="The latest graded run in the current or previous period.")
+ summary: ResultSummary
+ trend: EvaluationResultTrend
+ change: ResultChange | None = Field(description="Change against the previous period, or null when not comparable.")
+ low_evidence: bool
+ attention_reasons: list[AttentionReason]
+ bucket_status: list[BucketStatus] = Field(
+ description="Each bucket against the previous period. A bucket with too few graded runs to judge is never a dip."
+ )
+ tracked_category: str | None = Field(description="Key of the category a categorical evaluation's trend follows.")
+
+
+class PeriodComparison(Contract):
+ """The same figure for both periods, over only the evaluations with results in both."""
+
+ current: float
+ previous: float
+
+
+class PassRateOverviewEntry(Contract):
+ evaluation_id: str
+ rate: float
+ low_evidence: bool
+
+
+class PassRateOverview(Contract):
+ entries: list[PassRateOverviewEntry] = Field(description="Enabled pass rate evaluations, lowest rate first.")
+ median: float | None = Field(description="Median of the per-evaluation pass rates without low evidence.")
+ median_comparison: PeriodComparison | None = Field(
+ description="Median pass rate in each period, over evaluations with results in both. Null when none has."
+ )
+ median_trend: list[float | None] = Field(description="Median pass rate per bucket, aligned with `buckets`.")
+
+
+class OverviewSentiment(Contract):
+ evaluation_id: str
+ negative_share: float
+ previous_share: float | None
+
+
+class OverviewTotals(Contract):
+ graded: int
+ graded_comparison: PeriodComparison | None = Field(
+ description="Graded runs in each period, over evaluations with results in both. Null when none has."
+ )
+ graded_trend: list[int]
+ sentiment: OverviewSentiment | None
+
+
+class OverviewPeriodFields(Contract):
+ start: datetime = Field(description="Start of the period, in the project's timezone.")
+ end: datetime = Field(description="End of the period, exclusive.")
+
+
+class EvaluationsOverview(OverviewPeriodFields):
+ previous_start: datetime
+ previous_end: datetime = Field(
+ description="End of the previous period. While the current period runs, the previous one stops at the same "
+ "point, so both cover the same length of time."
+ )
+ interval: Interval = Field(description="`hour` for a period of two days or less, else `day`.")
+ buckets: list[datetime] = Field(
+ description="Start of each bucket, in the project's timezone. Every series is aligned with it. The first "
+ "bucket can start before `start`."
+ )
+ evaluations: list[EvaluationResult]
+ totals: OverviewTotals = Field(description="Totals over the enabled evaluations.")
+ pass_rate_overview: PassRateOverview
+
+
+class EvaluationRun(Contract):
+ timestamp: datetime
+ outcome: Literal["passed", "failed"] | None = Field(
+ description="Null when the evaluation has no passing rule, or the run was not graded by it."
+ )
+ reasoning: str | None = Field(description="The evaluator's reasoning, cut to about 1,000 characters.")
+ target_event_id: str | None = Field(description="The generation the run graded. Null for trace and session runs.")
+ trace_id: str | None
+
+
+class EvaluationRuns(Contract):
+ results: list[EvaluationRun] = Field(description="Newest first.")
+
+
+class ScoreBucket(Contract):
+ label: str
+ count: int
+
+
+class ScoreHistogram(Contract):
+ scale: ScoreScale
+ buckets: list[ScoreBucket] = Field(
+ description="One bucket per integer on a small integer scale, else ten equal buckets across the scale."
+ )
diff --git a/products/ai_observability/backend/logic/evaluations/evaluation_metas.py b/products/ai_observability/backend/logic/evaluations/evaluation_metas.py
new file mode 100644
index 000000000000..6658b7e47f7e
--- /dev/null
+++ b/products/ai_observability/backend/logic/evaluations/evaluation_metas.py
@@ -0,0 +1,81 @@
+from collections.abc import Sequence
+
+from products.ai_observability.backend.models.evaluation_configs import CategoricalOutputConfig, NumericOutputConfig
+from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definition
+from products.ai_observability.backend.models.evaluations import Evaluation, EvaluationStatus
+
+from ...facade import contracts
+from .overview import CategoryOption, EvaluationMeta, SummaryKind
+
+
+def load_evaluation_metas(team_id: int, evaluation_ids: Sequence[str]) -> list[EvaluationMeta]:
+ """Newest first. Deleted evaluations and ids outside the team are left out."""
+ evaluations = (
+ Evaluation.objects.filter(team_id=team_id, id__in=evaluation_ids, deleted=False)
+ .select_related("directory", "created_by")
+ .order_by("-created_at", "id")
+ )
+ return [_evaluation_meta(evaluation) for evaluation in evaluations]
+
+
+def load_evaluation_meta(team_id: int, evaluation_id: str) -> EvaluationMeta:
+ return _evaluation_meta(
+ Evaluation.objects.select_related("directory", "created_by").get(
+ team_id=team_id, id=evaluation_id, deleted=False
+ )
+ )
+
+
+def _evaluation_meta(evaluation: Evaluation) -> EvaluationMeta:
+ config = evaluation.output_config or {}
+ try:
+ has_pass_rule = get_outcome_definition(
+ evaluation.output_type, true_is_failure=config.get("true_is_failure") is True, output_config=config
+ ).has_pass_rule
+ except ValueError:
+ has_pass_rule = False
+ kind: SummaryKind
+ if evaluation.output_type == "sentiment":
+ kind = "sentiment"
+ elif has_pass_rule:
+ kind = "pass_rate"
+ elif evaluation.output_type == "numeric":
+ kind = "score"
+ else:
+ kind = "categories"
+ scale = None
+ score_step = None
+ categories: tuple[CategoryOption, ...] = ()
+ multiple = False
+ # A config that no longer validates still shows its counts, without the bounds or labels it would carry.
+ try:
+ if evaluation.output_type == "numeric":
+ numeric = NumericOutputConfig.model_validate(config)
+ score_step = numeric.step
+ if numeric.min is not None and numeric.max is not None:
+ scale = contracts.ScoreScale(min=numeric.min, max=numeric.max)
+ elif evaluation.output_type == "categorical":
+ categorical = CategoricalOutputConfig.model_validate(config)
+ categories = tuple(CategoryOption(key=option.key, label=option.label) for option in categorical.options)
+ multiple = categorical.selection_mode == "multiple"
+ except ValueError:
+ pass
+ creator = evaluation.created_by
+ return EvaluationMeta(
+ id=str(evaluation.id),
+ name=evaluation.name,
+ directory_name=evaluation.directory.name if evaluation.directory else None,
+ evaluation_type=evaluation.evaluation_type,
+ output_type=evaluation.output_type,
+ enabled=evaluation.enabled,
+ errored=evaluation.status == EvaluationStatus.ERROR,
+ status_reason=evaluation.status_reason if evaluation.status == EvaluationStatus.ERROR else None,
+ created_by=(" ".join(filter(None, [creator.first_name, creator.last_name])) or creator.email)
+ if creator
+ else None,
+ kind=kind,
+ scale=scale,
+ score_step=score_step,
+ categories=categories,
+ multiple=multiple,
+ )
diff --git a/products/ai_observability/backend/logic/evaluations/overview.py b/products/ai_observability/backend/logic/evaluations/overview.py
new file mode 100644
index 000000000000..4e24ab1131c6
--- /dev/null
+++ b/products/ai_observability/backend/logic/evaluations/overview.py
@@ -0,0 +1,498 @@
+"""Per-evaluation results for the online evaluations overview, and the totals across them.
+
+Pure functions over the grouped counts from `overview_queries`.
+"""
+
+import math
+from bisect import bisect_right
+from collections import defaultdict
+from collections.abc import Iterable, Sequence
+from datetime import UTC, datetime
+from typing import Literal, TypeVar
+
+from posthog.dataclasses import frozen
+
+from ...facade import contracts
+from .overview_queries import CategoryBucketCount, EvaluationBucketCounts
+
+# Below this many graded results a summary is shown greyed, and its change is not compared. A previous period with
+# fewer is not compared either.
+LOW_EVIDENCE_GRADED_COUNT = 30
+# A change smaller than this, in percentage points, is not called out.
+NOTABLE_CHANGE_PP = 2
+# A bucket this many points worse than an evaluation's previous period counts as unusual.
+DIP_PP = 3
+# A numeric score counts as unusual when it moves this share of its scale away from its usual value.
+SCORE_SHIFT_SHARE = 0.05
+
+SummaryKind = Literal["pass_rate", "score", "categories", "sentiment"]
+
+_Row = TypeVar("_Row", EvaluationBucketCounts, CategoryBucketCount)
+
+
+def group_by_evaluation(rows: Iterable[_Row]) -> dict[str, list[_Row]]:
+ grouped: dict[str, list[_Row]] = defaultdict(list)
+ for row in rows:
+ grouped[row.evaluation_id].append(row)
+ return grouped
+
+
+@frozen
+class CategoryOption:
+ key: str
+ label: str
+
+
+@frozen
+class EvaluationMeta:
+ id: str
+ name: str
+ directory_name: str | None
+ evaluation_type: str
+ output_type: str
+ enabled: bool
+ errored: bool
+ status_reason: str | None
+ created_by: str | None
+ kind: SummaryKind
+ scale: contracts.ScoreScale | None
+ score_step: float | None
+ categories: tuple[CategoryOption, ...]
+ multiple: bool
+
+
+@frozen
+class _Totals:
+ passed: int
+ failed: int
+ not_applicable: int
+ skipped: int
+ score_count: int
+ score_sum: float
+ categorized: int
+ positive: int
+ neutral: int
+ negative: int
+
+
+def _sum(rows: Sequence[EvaluationBucketCounts]) -> _Totals:
+ return _Totals(
+ passed=sum(row.passed for row in rows),
+ failed=sum(row.failed for row in rows),
+ not_applicable=sum(row.not_applicable for row in rows),
+ skipped=sum(row.skipped for row in rows),
+ score_count=sum(row.score_count for row in rows),
+ score_sum=sum(row.score_sum for row in rows),
+ categorized=sum(row.categorized for row in rows),
+ positive=sum(row.positive for row in rows),
+ neutral=sum(row.neutral for row in rows),
+ negative=sum(row.negative for row in rows),
+ )
+
+
+def _js_round(value: float, digits: int) -> float:
+ # Math.round rounds halves up, where Python's round() rounds them to even.
+ factor = 10**digits
+ return math.floor(value * factor + 0.5) / factor
+
+
+def _share(part: float, whole: float) -> float:
+ return part / whole * 100 if whole > 0 else 0
+
+
+def _median(values: Sequence[float]) -> float | None:
+ if not values:
+ return None
+ ordered = sorted(values)
+ middle = len(ordered) // 2
+ return ordered[middle] if len(ordered) % 2 else (ordered[middle - 1] + ordered[middle]) / 2
+
+
+def pass_rate(passed: int, failed: int) -> float | None:
+ graded = passed + failed
+ return passed / graded * 100 if graded > 0 else None
+
+
+def _graded(kind: SummaryKind, totals: _Totals) -> int:
+ match kind:
+ case "pass_rate":
+ return totals.passed + totals.failed
+ case "score":
+ return totals.score_count
+ case "categories":
+ return totals.categorized
+ case "sentiment":
+ return totals.positive + totals.neutral + totals.negative
+
+
+def _score(meta: EvaluationMeta, totals: _Totals) -> contracts.ScoreDistribution | None:
+ if totals.score_count == 0:
+ return None
+ return contracts.ScoreDistribution(mean=totals.score_sum / totals.score_count, scale=meta.scale)
+
+
+def _category_counts(rows: Sequence[CategoryBucketCount], current: bool) -> dict[str, int]:
+ counts: dict[str, int] = defaultdict(int)
+ for row in rows:
+ if (row.bucket is not None) == current:
+ counts[row.key] += row.runs
+ return counts
+
+
+def _summary(
+ meta: EvaluationMeta,
+ current: _Totals,
+ previous: _Totals,
+ categories: Sequence[CategoryBucketCount],
+) -> contracts.ResultSummary:
+ match meta.kind:
+ case "pass_rate":
+ return contracts.PassRateSummary(
+ kind="pass_rate",
+ passed=current.passed,
+ failed=current.failed,
+ previous=contracts.PassFailCounts(passed=previous.passed, failed=previous.failed)
+ if previous.passed + previous.failed > 0
+ else None,
+ score=_score(meta, current) if meta.output_type == "numeric" else None,
+ )
+ case "score":
+ return contracts.ScoreSummary(
+ kind="score",
+ score=_score(meta, current),
+ previous_mean=previous.score_sum / previous.score_count if previous.score_count else None,
+ )
+ case "categories":
+ counts = _category_counts(categories, current=True)
+ previous_counts = _category_counts(categories, current=False)
+ options = list(meta.categories)
+ known = {option.key for option in options}
+ # A category from the previous period only has dropped to zero, which can be the largest move.
+ unknown = dict.fromkeys(key for key in [*counts, *previous_counts] if key and key not in known)
+ options += [CategoryOption(key=key, label=key) for key in unknown]
+ entries = [
+ contracts.CategoryCount(
+ key=option.key,
+ label=option.label,
+ count=counts.get(option.key, 0),
+ previous_share=_share(previous_counts.get(option.key, 0), previous.categorized)
+ if previous.categorized
+ else None,
+ )
+ for option in options
+ ]
+ return contracts.CategoriesSummary(
+ kind="categories",
+ multiple=meta.multiple,
+ categories=sorted(entries, key=lambda entry: -entry.count),
+ no_categories_count=counts.get("", 0),
+ )
+ case "sentiment":
+ total = previous.positive + previous.neutral + previous.negative
+ return contracts.SentimentSummary(
+ kind="sentiment",
+ positive=current.positive,
+ neutral=current.neutral,
+ negative=current.negative,
+ previous_negative_share=_share(previous.negative, total) if total else None,
+ )
+
+
+def has_low_evidence(graded_count: int) -> bool:
+ return 0 < graded_count < LOW_EVIDENCE_GRADED_COUNT
+
+
+def result_change(graded_count: int, summary: contracts.ResultSummary) -> contracts.ResultChange | None:
+ """Change against the previous equal period, or None when there is nothing fair to compare."""
+ if graded_count == 0 or has_low_evidence(graded_count):
+ return None
+ match summary:
+ case contracts.PassRateSummary():
+ current = pass_rate(summary.passed, summary.failed)
+ previous = pass_rate(summary.previous.passed, summary.previous.failed) if summary.previous else None
+ if current is None or previous is None:
+ return None
+ return contracts.ResultChange(value=_js_round(current - previous, 1), unit="pp", subject=None)
+ case contracts.ScoreSummary():
+ if summary.score is None or summary.previous_mean is None:
+ return None
+ return contracts.ResultChange(
+ value=_js_round(summary.score.mean - summary.previous_mean, 2), unit="points", subject=None
+ )
+ case contracts.CategoriesSummary():
+ moved = _largest_category_move(summary, graded_count)
+ if moved is None:
+ return None
+ category, delta = moved
+ return contracts.ResultChange(value=delta, unit="pp", subject=category.label)
+ case contracts.SentimentSummary():
+ if summary.previous_negative_share is None:
+ return None
+ total = summary.positive + summary.neutral + summary.negative
+ return contracts.ResultChange(
+ value=_js_round(_share(summary.negative, total) - summary.previous_negative_share, 1),
+ unit="pp",
+ subject="negative",
+ )
+
+
+def _largest_category_move(
+ summary: contracts.CategoriesSummary, graded_count: int
+) -> tuple[contracts.CategoryCount, float] | None:
+ """The category that moved most, since the top category is often the stable one."""
+ largest: tuple[contracts.CategoryCount, float] | None = None
+ for category in summary.categories:
+ if category.previous_share is None:
+ continue
+ delta = _js_round(_share(category.count, graded_count) - category.previous_share, 1)
+ if largest is None or abs(delta) > abs(largest[1]):
+ largest = (category, delta)
+ return largest
+
+
+def _tracked_category(
+ summary: contracts.ResultSummary, change: contracts.ResultChange | None, graded_count: int
+) -> contracts.CategoryCount | None:
+ """The category a categorical evaluation's trend follows: the one that moved most, else the most common."""
+ if not isinstance(summary, contracts.CategoriesSummary):
+ return None
+ # Labels need not be unique, so the trend follows the category itself and not its label.
+ moved = _largest_category_move(summary, graded_count) if change is not None else None
+ if moved is not None:
+ return moved[0]
+ return summary.categories[0] if summary.categories else None
+
+
+def _previous_value(summary: contracts.ResultSummary, tracked: str | None) -> float | None:
+ match summary:
+ case contracts.PassRateSummary():
+ return pass_rate(summary.previous.passed, summary.previous.failed) if summary.previous else None
+ case contracts.ScoreSummary():
+ return summary.previous_mean
+ case contracts.SentimentSummary():
+ return summary.previous_negative_share
+ case contracts.CategoriesSummary():
+ return next((category.previous_share for category in summary.categories if category.key == tracked), None)
+
+
+def _bucket_index(instants: Sequence[datetime], bucket: datetime) -> int:
+ """The bucket that holds a ClickHouse bucket start: the last one that starts at or before it, compared by
+ instant."""
+ return max(bisect_right(instants, bucket.astimezone(UTC)) - 1, 0)
+
+
+def _bucket_value(
+ meta: EvaluationMeta, row: _Totals | None, tracked_key: str | None, category_runs: int
+) -> float | None:
+ if row is None:
+ return None
+ match meta.kind:
+ case "pass_rate":
+ return pass_rate(row.passed, row.failed)
+ case "score":
+ return row.score_sum / row.score_count if row.score_count else None
+ case "sentiment":
+ total = row.positive + row.neutral + row.negative
+ return _share(row.negative, total) if total else None
+ case "categories":
+ return _share(category_runs, row.categorized) if row.categorized and tracked_key is not None else None
+
+
+def bucket_status(
+ summary: contracts.ResultSummary, trend: contracts.EvaluationResultTrend, tracked: str | None
+) -> list[contracts.BucketStatus]:
+ """Status of each bucket against the previous period, or against the range's own median without one.
+
+ Pass rates and sentiment have a worse direction. Scores and label shares have none, so a large move either way
+ counts as unusual. A bucket with fewer graded runs than a summary needs is never flagged, because one run moves
+ a thin bucket by more than the dip threshold, which hourly buckets would turn into noise.
+ """
+ previous = _previous_value(summary, tracked)
+ typical = previous if previous is not None else _median([value for value in trend.values if value is not None])
+ statuses: list[contracts.BucketStatus] = []
+ for value, graded in zip(trend.values, trend.volume):
+ if value is None:
+ statuses.append("none")
+ elif typical is None or graded < LOW_EVIDENCE_GRADED_COUNT:
+ statuses.append("normal")
+ else:
+ statuses.append("dip" if _is_dip(summary, value, typical) else "normal")
+ return statuses
+
+
+def _is_dip(summary: contracts.ResultSummary, value: float, typical: float) -> bool:
+ match summary:
+ case contracts.PassRateSummary():
+ return value <= typical - DIP_PP
+ case contracts.SentimentSummary():
+ return value >= typical + DIP_PP
+ case contracts.CategoriesSummary():
+ return abs(value - typical) >= DIP_PP
+ case contracts.ScoreSummary():
+ # Without configured bounds, or with equal ones, there is no scale to measure a move against.
+ scale = summary.score.scale if summary.score else None
+ span = scale.max - scale.min if scale else 0
+ return span > 0 and abs(value - typical) >= span * SCORE_SHIFT_SHARE
+
+
+def attention_reasons(
+ *,
+ enabled: bool,
+ errored: bool,
+ graded_count: int,
+ summary: contracts.ResultSummary,
+ change: contracts.ResultChange | None,
+) -> list[contracts.AttentionReason]:
+ # The error status also disables the evaluation, and that pause was not the user's choice, so it still asks for
+ # attention. Its missing results follow from the error, so it is never also silent or worse.
+ if errored:
+ return ["errored"]
+ # Pausing an evaluation is a deliberate choice, so a paused one never asks for attention.
+ if not enabled:
+ return []
+ reasons: list[contracts.AttentionReason] = []
+ if graded_count == 0:
+ reasons.append("silent")
+ # Only results with a clear worse direction count: a falling pass rate or a rising negative share.
+ if change is not None and change.unit == "pp":
+ if (isinstance(summary, contracts.PassRateSummary) and change.value <= -NOTABLE_CHANGE_PP) or (
+ isinstance(summary, contracts.SentimentSummary) and change.value >= NOTABLE_CHANGE_PP
+ ):
+ reasons.append("worse")
+ return reasons
+
+
+def evaluation_result(
+ meta: EvaluationMeta,
+ buckets: Sequence[datetime],
+ counts: Sequence[EvaluationBucketCounts],
+ categories: Sequence[CategoryBucketCount],
+) -> contracts.EvaluationResult:
+ """`counts` and `categories` hold only this evaluation's rows, see `group_by_evaluation`."""
+ instants = [bucket.astimezone(UTC) for bucket in buckets]
+ by_bucket: dict[int, list[EvaluationBucketCounts]] = defaultdict(list)
+ for row in counts:
+ if row.bucket is not None:
+ by_bucket[_bucket_index(instants, row.bucket)].append(row)
+ current = _sum([row for row in counts if row.bucket is not None])
+ previous = _sum([row for row in counts if row.bucket is None])
+ previous_graded = _graded(meta.kind, previous)
+ # A thin previous period is no fairer a baseline than a thin current one, so the summary leaves it out.
+ baseline = previous if previous_graded >= LOW_EVIDENCE_GRADED_COUNT else _sum([])
+ summary = _summary(meta, current, baseline, categories)
+ graded_count = _graded(meta.kind, current)
+ change = result_change(graded_count, summary)
+ tracked = _tracked_category(summary, change, graded_count)
+ tracked_key = tracked.key if tracked else None
+ tracked_runs: dict[int, int] = defaultdict(int)
+ for category in categories:
+ if category.bucket is not None and category.key == tracked_key:
+ tracked_runs[_bucket_index(instants, category.bucket)] += category.runs
+ bucket_totals = [_sum(by_bucket[index]) if index in by_bucket else None for index in range(len(buckets))]
+ trend = contracts.EvaluationResultTrend(
+ values=[
+ _bucket_value(meta, totals, tracked_key, tracked_runs[index]) for index, totals in enumerate(bucket_totals)
+ ],
+ volume=[_graded(meta.kind, totals) if totals is not None else 0 for totals in bucket_totals],
+ )
+ last_run_at = max((row.last_run_at for row in counts if row.last_run_at is not None), default=None)
+ return contracts.EvaluationResult(
+ id=meta.id,
+ name=meta.name,
+ directory_name=meta.directory_name,
+ evaluation_type=meta.evaluation_type,
+ output_type=meta.output_type,
+ enabled=meta.enabled,
+ status_reason=meta.status_reason,
+ created_by=meta.created_by,
+ graded_count=graded_count,
+ previous_graded_count=previous_graded or None,
+ not_applicable_count=current.not_applicable,
+ skipped_count=current.skipped,
+ last_run_at=last_run_at,
+ summary=summary,
+ trend=trend,
+ change=change,
+ low_evidence=has_low_evidence(graded_count),
+ attention_reasons=attention_reasons(
+ enabled=meta.enabled,
+ errored=meta.errored,
+ graded_count=graded_count,
+ summary=summary,
+ change=change,
+ ),
+ bucket_status=bucket_status(summary, trend, tracked_key),
+ tracked_category=tracked_key,
+ )
+
+
+def pass_rate_overview(rows: Sequence[contracts.EvaluationResult], bucket_count: int) -> contracts.PassRateOverview:
+ entries: list[contracts.PassRateOverviewEntry] = []
+ measured: list[contracts.EvaluationResult] = []
+ compared: list[tuple[float, float]] = []
+ for row in rows:
+ summary = row.summary
+ if not isinstance(summary, contracts.PassRateSummary) or not row.enabled:
+ continue
+ rate = pass_rate(summary.passed, summary.failed)
+ if rate is None:
+ continue
+ entries.append(contracts.PassRateOverviewEntry(evaluation_id=row.id, rate=rate, low_evidence=row.low_evidence))
+ if row.low_evidence:
+ continue
+ measured.append(row)
+ previous = pass_rate(summary.previous.passed, summary.previous.failed) if summary.previous else None
+ if previous is not None:
+ compared.append((rate, previous))
+ median_trend = (
+ [
+ _median([value for row in measured if (value := row.trend.values[index]) is not None])
+ for index in range(bucket_count)
+ ]
+ if measured
+ else []
+ )
+ # Both medians cover the same evaluations, so a change between them is not a change in which ones ran.
+ current_median = _median([current for current, _ in compared])
+ previous_median = _median([previous for _, previous in compared])
+ median_comparison = (
+ contracts.PeriodComparison(current=current_median, previous=previous_median)
+ if current_median is not None and previous_median is not None
+ else None
+ )
+ return contracts.PassRateOverview(
+ entries=sorted(entries, key=lambda entry: entry.rate),
+ median=_median([entry.rate for entry in entries if not entry.low_evidence]),
+ median_comparison=median_comparison,
+ median_trend=median_trend,
+ )
+
+
+def overview_totals(rows: Sequence[contracts.EvaluationResult], bucket_count: int) -> contracts.OverviewTotals:
+ """Totals for the metric cards at the top of the overview."""
+ active = [row for row in rows if row.enabled]
+ # Only evaluations with results in both periods, so a new or newly silent evaluation does not read as a change.
+ both = [row for row in active if row.graded_count > 0 and row.previous_graded_count is not None]
+ sentiment = None
+ for row in active:
+ if isinstance(row.summary, contracts.SentimentSummary) and row.graded_count > 0:
+ summary = row.summary
+ sentiment = contracts.OverviewSentiment(
+ evaluation_id=row.id,
+ negative_share=_share(summary.negative, summary.positive + summary.neutral + summary.negative),
+ previous_share=summary.previous_negative_share,
+ )
+ break
+ return contracts.OverviewTotals(
+ graded=sum(row.graded_count for row in active),
+ graded_comparison=contracts.PeriodComparison(
+ current=sum(row.graded_count for row in both),
+ previous=sum(row.previous_graded_count or 0 for row in both),
+ )
+ if both
+ else None,
+ graded_trend=[sum(row.trend.volume[index] for row in active) for index in range(bucket_count)]
+ if active
+ else [],
+ sentiment=sentiment,
+ )
diff --git a/products/ai_observability/backend/logic/evaluations/overview_queries.py b/products/ai_observability/backend/logic/evaluations/overview_queries.py
new file mode 100644
index 000000000000..4a45eae34a5d
--- /dev/null
+++ b/products/ai_observability/backend/logic/evaluations/overview_queries.py
@@ -0,0 +1,294 @@
+from collections.abc import Iterator, Mapping, Sequence
+from datetime import UTC, datetime
+from typing import Any
+
+from posthog.hogql import ast
+from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.database import Database
+from posthog.hogql.parser import parse_select
+from posthog.hogql.query import execute_hogql_query
+
+from posthog.clickhouse.query_tagging import Product, tag_queries, tags_context
+from posthog.dataclasses import frozen
+from posthog.models import Team, User
+
+from products.ai_observability.backend.models.evaluation_outcomes import (
+ CATEGORIES_EXPRESSION,
+ CATEGORIZED_PREDICATE,
+ NOT_SKIPPED_PREDICATE,
+ NUMERIC_SCORED_PREDICATE,
+ SCORE_EXPRESSION,
+ SENTIMENT_LABELS,
+ get_outcome_definition,
+)
+
+from .pass_filters import PassRules
+from .period import OverviewPeriod, PeriodBounds
+
+_SKIPPED = "properties.$ai_evaluation_skipped = 'true'"
+_NOT_APPLICABLE = f"properties.$ai_evaluation_applicable = 'false' AND {NOT_SKIPPED_PREDICATE}"
+_SENTIMENT = get_outcome_definition("sentiment")
+# One column per label, in the order of SENTIMENT_LABELS, which `load_bucket_counts` reads them in.
+_SENTIMENT_COUNTS = ", ".join(
+ f"countIf({_SENTIMENT.outcome_predicates[label]} AND {NOT_SKIPPED_PREDICATE}) AS {label}"
+ for label in SENTIMENT_LABELS
+)
+_ANY_LABEL = ", ".join(f"'{label}'" for label in SENTIMENT_LABELS)
+
+# A run that a summary kind without a passing rule grades: scored, categorized or labeled.
+GRADED_WITHOUT_RULE = (
+ f"({NUMERIC_SCORED_PREDICATE}) OR ({CATEGORIZED_PREDICATE})"
+ f" OR ({_SENTIMENT.result_expression} IN ({_ANY_LABEL}) AND {NOT_SKIPPED_PREDICATE})"
+)
+# A run any summary kind grades, so an evaluation whose recent runs were all N/A or skipped still reads as silent
+# since its last real result.
+_GRADED_ANY = f"graded_run OR {GRADED_WITHOUT_RULE}"
+
+PAGE_SIZE = MAX_SELECT_RETURNED_ROWS
+# Many evaluations over a long period can group into more rows than one response returns, for example categorical
+# evaluations with many categories over 90 days. So a read splits the evaluations into batches that are expected to
+# fit in half a page, and splits a batch again if it still fills a page.
+BATCH_ROWS = PAGE_SIZE // 2
+# A null bucket marks the previous period, which needs totals only, so it costs one row per evaluation.
+_BUCKET = "if(timestamp < {current_start}, NULL, {bucket_start})"
+_WHERE = (
+ "event = '$ai_evaluation' AND timestamp >= {previous_start} AND timestamp < {end}"
+ " AND (timestamp >= {current_start} OR timestamp < {previous_end})"
+ " AND properties.$ai_evaluation_id IN {evaluation_ids}"
+)
+
+_COUNTS_QUERY = f"""
+SELECT
+ toString(properties.$ai_evaluation_id) AS evaluation_id,
+ {_BUCKET} AS bucket,
+ countIf(and(({{graded}}) AS graded_run, ({{passes}}) AS passing_run)) AS passed,
+ countIf(graded_run AND NOT passing_run) AS failed,
+ countIf({_NOT_APPLICABLE}) AS not_applicable,
+ countIf({_SKIPPED}) AS skipped,
+ countIf({NUMERIC_SCORED_PREDICATE}) AS score_count,
+ sumIf({SCORE_EXPRESSION}, {NUMERIC_SCORED_PREDICATE}) AS score_sum,
+ countIf({CATEGORIZED_PREDICATE}) AS categorized,
+ {_SENTIMENT_COUNTS},
+ if(countIf(({_GRADED_ANY}) AS graded_any) = 0, NULL, maxIf(timestamp, graded_any)) AS last_run_at
+FROM events
+WHERE {_WHERE}
+GROUP BY evaluation_id, bucket
+ORDER BY evaluation_id, bucket
+LIMIT {{limit}} OFFSET {{offset}}
+"""
+
+# An empty category list becomes one '' row, so multiple-selection runs that chose nothing are counted too.
+_CATEGORY_QUERY = f"""
+SELECT
+ toString(properties.$ai_evaluation_id) AS evaluation_id,
+ {_BUCKET} AS bucket,
+ arrayJoin(if(empty({CATEGORIES_EXPRESSION}), [''], {CATEGORIES_EXPRESSION})) AS category,
+ count() AS runs
+FROM events
+WHERE {_WHERE} AND {CATEGORIZED_PREDICATE}
+GROUP BY evaluation_id, bucket, category
+ORDER BY evaluation_id, bucket, category
+LIMIT {{limit}} OFFSET {{offset}}
+"""
+
+
+@frozen
+class EvaluationBucketCounts:
+ evaluation_id: str
+ bucket: datetime | None # None is the previous period.
+ passed: int
+ failed: int
+ not_applicable: int
+ skipped: int
+ score_count: int
+ score_sum: float
+ categorized: int
+ positive: int
+ neutral: int
+ negative: int
+ last_run_at: datetime | None
+
+
+@frozen
+class CategoryBucketCount:
+ evaluation_id: str
+ bucket: datetime | None # None is the previous period.
+ key: str # '' for a run that chose no category.
+ runs: int
+
+
+def _run(
+ name: str,
+ query: str,
+ team: Team,
+ user: User | None,
+ period: OverviewPeriod,
+ expected_rows: Mapping[str, int],
+ extra: dict[str, ast.Expr],
+) -> list[Sequence[Any]]:
+ """`expected_rows` maps each evaluation to read to the most rows it is expected to group into."""
+ placeholders: dict[str, ast.Expr] = {
+ "previous_start": instant(period.previous_start),
+ "previous_end": instant(period.previous_end),
+ "current_start": instant(period.start),
+ "end": instant(period.end),
+ "bucket_start": _bucket_start(period, team.timezone),
+ **extra,
+ }
+
+ def read(evaluation_ids: Sequence[str], offset: int) -> list[Sequence[Any]]:
+ # One placeholder for the whole list, so the query text stays the same size however many evaluations it reads.
+ ids = ast.Constant(value=tuple(evaluation_ids))
+ limits = {"limit": ast.Constant(value=PAGE_SIZE), "offset": ast.Constant(value=offset)}
+ return run_evaluation_query(name, query, team, user, {**placeholders, "evaluation_ids": ids, **limits})
+
+ def read_all(evaluation_ids: Sequence[str]) -> list[Sequence[Any]]:
+ rows = read(evaluation_ids, 0)
+ if len(rows) < PAGE_SIZE:
+ return rows
+ if len(evaluation_ids) > 1:
+ middle = len(evaluation_ids) // 2
+ return read_all(evaluation_ids[:middle]) + read_all(evaluation_ids[middle:])
+ # One evaluation cannot be split further, so it pages through its own rows.
+ while len(page := read(evaluation_ids, len(rows))) == PAGE_SIZE:
+ rows += page
+ return rows + page
+
+ return [row for batch in _batches(expected_rows) for row in read_all(batch)]
+
+
+def _bucket_start(period: OverviewPeriod, timezone: str) -> ast.Expr:
+ timestamp = ast.Field(chain=["timestamp"])
+ if period.interval == "hour":
+ # Whole hours from the first bucket, the way `overview_period` steps its buckets. Local hours would split a
+ # bucket where a daylight saving change moves the clocks by half an hour. ClickHouse needs an origin before
+ # every timestamp, including the previous period's, so the origin is the first bucket's minute in 1970. It
+ # counts in the timestamp's timezone, which is UTC so that past offset changes do not shift the hours.
+ first = (period.buckets[0] if period.buckets else period.start).astimezone(UTC)
+ origin = instant(datetime(1970, 1, 1, minute=first.minute, tzinfo=UTC))
+ utc = ast.Call(name="toTimeZone", args=[timestamp, ast.Constant(value="UTC")])
+ return ast.Call(
+ name="toStartOfInterval",
+ args=[utc, ast.Call(name="toIntervalHour", args=[ast.Constant(value=1)]), origin],
+ )
+ # The project's timezone is explicit, because the team's modifiers can turn off the conversion to it, and
+ # `overview_period` builds its buckets in that timezone.
+ local = ast.Call(name="toTimeZone", args=[timestamp, ast.Constant(value=timezone)])
+ return ast.Call(name="toStartOfDay", args=[local])
+
+
+def _groups(period: OverviewPeriod) -> int:
+ """The most rows one evaluation groups into per category: one per bucket, plus one for the previous period."""
+ return len(period.buckets) + 1
+
+
+def _batches(expected_rows: Mapping[str, int]) -> Iterator[list[str]]:
+ batch: list[str] = []
+ rows = 0
+ for evaluation_id, expected in expected_rows.items():
+ if batch and rows + expected > BATCH_ROWS:
+ yield batch
+ batch, rows = [], 0
+ batch.append(evaluation_id)
+ rows += expected
+ if batch:
+ yield batch
+
+
+def instant(value: datetime) -> ast.Expr:
+ """A datetime constant prints as wall time in the project's timezone, which names two instants in a repeated
+ daylight saving hour. UTC wall time names exactly one."""
+ return ast.Call(
+ name="toDateTime64",
+ args=[
+ ast.Constant(value=value.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S.%f")),
+ ast.Constant(value=6),
+ ast.Constant(value="UTC"),
+ ],
+ )
+
+
+# The runs of one evaluation in a period, for the reads that list or bucket them.
+EVALUATION_RUNS_WHERE = (
+ "event = '$ai_evaluation' AND properties.$ai_evaluation_id = {evaluation_id}"
+ " AND timestamp >= {start} AND timestamp < {end}"
+)
+
+
+def evaluation_runs_placeholders(evaluation_id: str, period: PeriodBounds) -> dict[str, ast.Expr]:
+ return {
+ "evaluation_id": ast.Constant(value=evaluation_id),
+ "start": instant(period.start),
+ "end": instant(period.end),
+ }
+
+
+def run_evaluation_query(
+ name: str, query: str, team: Team, user: User | None, placeholders: dict[str, ast.Expr]
+) -> list[Sequence[Any]]:
+ with tags_context(product=Product.LLM_ANALYTICS):
+ tag_queries(name=name)
+ response = execute_hogql_query(
+ query=parse_select(query, placeholders=placeholders),
+ team=team,
+ user=user,
+ query_type=name,
+ context=HogQLContext(
+ team_id=team.pk, team=team, user=user, database=Database.create_for_posthog_tables(team)
+ ),
+ )
+ return list(response.results or [])
+
+
+def load_bucket_counts(
+ team: Team,
+ user: User | None,
+ period: OverviewPeriod,
+ evaluation_ids: Sequence[str],
+ pass_rules: PassRules,
+) -> list[EvaluationBucketCounts]:
+ if not evaluation_ids:
+ return []
+ rows = _run(
+ "ai_evaluations_overview_counts",
+ _COUNTS_QUERY,
+ team,
+ user,
+ period,
+ dict.fromkeys(evaluation_ids, _groups(period)),
+ {"graded": pass_rules.graded, "passes": pass_rules.passes},
+ )
+ return [
+ EvaluationBucketCounts(
+ evaluation_id=row[0],
+ bucket=row[1],
+ passed=row[2],
+ failed=row[3],
+ not_applicable=row[4],
+ skipped=row[5],
+ score_count=row[6],
+ score_sum=float(row[7] or 0),
+ categorized=row[8],
+ positive=row[9],
+ neutral=row[10],
+ negative=row[11],
+ last_run_at=row[12],
+ )
+ for row in rows
+ ]
+
+
+def load_category_counts(
+ team: Team, user: User | None, period: OverviewPeriod, configured_categories: Mapping[str, int]
+) -> list[CategoryBucketCount]:
+ """`configured_categories` maps each evaluation to read to the number of categories its config lists."""
+ if not configured_categories:
+ return []
+ # Runs can hold keys the config no longer lists, so the estimate doubles the configured keys and the '' row.
+ expected_rows = {
+ evaluation_id: _groups(period) * 2 * (configured + 1)
+ for evaluation_id, configured in configured_categories.items()
+ }
+ rows = _run("ai_evaluations_overview_categories", _CATEGORY_QUERY, team, user, period, expected_rows, {})
+ return [CategoryBucketCount(evaluation_id=row[0], bucket=row[1], key=row[2], runs=row[3]) for row in rows]
diff --git a/products/ai_observability/backend/logic/evaluations/pass_filters.py b/products/ai_observability/backend/logic/evaluations/pass_filters.py
index ae44fce458f9..a23dbcc2ca59 100644
--- a/products/ai_observability/backend/logic/evaluations/pass_filters.py
+++ b/products/ai_observability/backend/logic/evaluations/pass_filters.py
@@ -1,42 +1,48 @@
+from collections import defaultdict
+from collections.abc import Iterator, Sequence
+from typing import Literal
+
from django.db.models import Q
+from posthog.hogql import ast
from posthog.hogql.escape_sql import escape_hogql_string
+from posthog.hogql.parser import parse_expr
+
+from posthog.dataclasses import frozen
from products.ai_observability.backend.models.evaluation_configs import OutputType
from products.ai_observability.backend.models.evaluation_outcomes import (
EvaluationReportOutcomeDefinition,
+ RulePredicates,
+ categorical_rule_predicates,
get_outcome_definition,
+ numeric_rule_predicates,
)
from products.ai_observability.backend.models.evaluations import Evaluation
from ...facade import contracts
_DEFAULT_DEFINITION = get_outcome_definition(OutputType.BOOLEAN)
+_EVALUATION_ID = "properties.$ai_evaluation_id"
+
+
+@frozen
+class PassRules:
+ """The pass filters as parsed HogQL, with each evaluation's rule looked up from array placeholders."""
+
+ graded: ast.Expr
+ # Only means pass on a graded run, so a passed run is `graded AND passes`.
+ passes: ast.Expr
-def build_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilters:
+def build_evaluation_pass_filters(
+ team_id: int, evaluation_ids: Sequence[str] | None = None
+) -> contracts.EvaluationPassFilters:
+ """`evaluation_ids` limits the rules to the runs of those evaluations, for a read that filters on them."""
graded_branches: list[str] = []
passed_branches: list[str] = []
- # Every other evaluation falls through to the default rule. Soft-deleted evaluations stay in, because their past
- # events still need their own rule. Reads the writer, because a receiver of evaluation_output_changed calls this
- # right after the commit, before a replica can catch up.
- evaluations_with_own_rule = (
- Evaluation.objects.using("default")
- .filter(team_id=team_id)
- .filter(
- Q(output_type__in=[OutputType.NUMERIC, OutputType.CATEGORICAL, OutputType.SENTIMENT])
- | Q(output_config__true_is_failure=True)
- )
- .values_list("id", "output_type", "output_config")
- )
- for evaluation_id, output_type, output_config in evaluations_with_own_rule:
- try:
- definition = get_outcome_definition(
- output_type, true_is_failure=output_config.get("true_is_failure") is True, output_config=output_config
- )
- except ValueError:
- definition = None
- matches_evaluation = f"properties.$ai_evaluation_id = {escape_hogql_string(str(evaluation_id))}"
+ for evaluation_id, definition in _own_rules(team_id, evaluation_ids):
+ matches_evaluation = f"{_EVALUATION_ID} = {escape_hogql_string(evaluation_id)}"
# An evaluation without a pass rule grades none of its runs, including boolean runs from before it
# switched to sentiment, which the default branch would otherwise grade.
if definition is None or not definition.has_pass_rule:
@@ -53,9 +59,112 @@ def build_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilte
)
+def build_pass_rules(team_id: int, evaluation_ids: Sequence[str]) -> PassRules:
+ """The same rules as `build_evaluation_pass_filters`, in one branch per kind of rule rather than per evaluation,
+ so the query stays the same size however many evaluations it reads."""
+ no_rule: list[str] = []
+ inverted: list[str] = []
+ no_category: list[str] = []
+ numeric: dict[Literal["gte", "lte"], list[tuple[str, float]]] = defaultdict(list)
+ categorical: list[tuple[str, list[str]]] = []
+ for evaluation_id, definition in _own_rules(team_id, evaluation_ids):
+ if definition is None or not definition.has_pass_rule:
+ no_rule.append(evaluation_id)
+ elif definition.numeric_config is not None and definition.numeric_config.passing_rule is not None:
+ rule = definition.numeric_config.passing_rule
+ numeric[rule.operator].append((evaluation_id, rule.threshold))
+ elif definition.categorical_config is not None and definition.categorical_config.passing_rule is not None:
+ categories = definition.categorical_config.passing_rule.categories
+ if categories:
+ categorical.append((evaluation_id, categories))
+ else:
+ no_category.append(evaluation_id)
+ elif definition.passing_result is False:
+ inverted.append(evaluation_id)
+
+ placeholders: dict[str, ast.Expr] = {}
+ graded_branches: list[str] = []
+ passed_branches: list[str] = []
+
+ # A kind without evaluations gets no branch, because HogQL cannot type an empty array constant.
+ def branch(name: str, ids: Sequence[str], graded: str, passed: str) -> None:
+ placeholders[f"{name}_ids"] = ast.Constant(value=list(ids))
+ matches = f"{_EVALUATION_ID} IN {{{name}_ids}}"
+ graded_branches.extend([matches, graded])
+ passed_branches.extend([matches, passed])
+
+ def rule_branch(name: str, ids: Sequence[str], event_predicate: str, predicates: RulePredicates) -> None:
+ branch(name, ids, _graded_from(event_predicate, predicates.passed, predicates.failed), predicates.passed)
+
+ if no_rule:
+ branch("no_rule", no_rule, "false", "false")
+ if inverted:
+ definition = get_outcome_definition(OutputType.BOOLEAN, true_is_failure=True)
+ outcome = definition.outcome_predicates
+ rule_branch(
+ "inverted",
+ inverted,
+ definition.event_predicate,
+ RulePredicates(passed=outcome["pass"], failed=outcome["fail"]),
+ )
+ numeric_event = get_outcome_definition(OutputType.NUMERIC).event_predicate
+ for operator, rules in numeric.items():
+ name = f"numeric_{operator}"
+ placeholders[f"{name}_thresholds"] = ast.Constant(value=[threshold for _, threshold in rules])
+ # `transform` looks up a constant array through a hash table, where `indexOf` scans it on every run.
+ threshold = f"transform({_EVALUATION_ID}, {{{name}_ids}}, {{{name}_thresholds}}, 0)"
+ ids = [evaluation_id for evaluation_id, _ in rules]
+ rule_branch(name, ids, numeric_event, numeric_rule_predicates(operator, threshold))
+ categorical_event = get_outcome_definition(OutputType.CATEGORICAL).event_predicate
+ if no_category:
+ rule_branch("no_category", no_category, categorical_event, categorical_rule_predicates(None))
+ if categorical:
+ placeholders["categorical_passing"] = ast.Constant(value=[categories for _, categories in categorical])
+ placeholders["categorical_positions"] = ast.Constant(value=list(range(1, len(categorical) + 1)))
+ # `transform` cannot return an array, so it returns the position of the evaluation's categories instead.
+ position = f"transform({_EVALUATION_ID}, {{categorical_ids}}, {{categorical_positions}}, 0)"
+ passing = f"arrayElement({{categorical_passing}}, {position})"
+ ids = [evaluation_id for evaluation_id, _ in categorical]
+ rule_branch("categorical", ids, categorical_event, categorical_rule_predicates(passing))
+
+ graded = _multi_if(graded_branches, _graded(_DEFAULT_DEFINITION))
+ passes = _multi_if(passed_branches, _DEFAULT_DEFINITION.outcome_predicates["pass"])
+ return PassRules(
+ graded=parse_expr(graded, placeholders=placeholders), passes=parse_expr(passes, placeholders=placeholders)
+ )
+
+
+def _own_rules(
+ team_id: int, evaluation_ids: Sequence[str] | None
+) -> Iterator[tuple[str, EvaluationReportOutcomeDefinition | None]]:
+ """Each evaluation whose runs the default rule does not grade, with its rule, or None for a config that no longer
+ loads. Soft-deleted evaluations stay in, because their past events still need their own rule. Reads the writer,
+ because a receiver of evaluation_output_changed calls this right after the commit, before a replica can catch
+ up."""
+ evaluations = Evaluation.objects.using("default").filter(team_id=team_id)
+ if evaluation_ids is not None:
+ evaluations = evaluations.filter(id__in=evaluation_ids)
+ evaluations_with_own_rule = evaluations.filter(
+ Q(output_type__in=[OutputType.NUMERIC, OutputType.CATEGORICAL, OutputType.SENTIMENT])
+ | Q(output_config__true_is_failure=True)
+ ).values_list("id", "output_type", "output_config")
+ for evaluation_id, output_type, output_config in evaluations_with_own_rule:
+ try:
+ definition = get_outcome_definition(
+ output_type, true_is_failure=output_config.get("true_is_failure") is True, output_config=output_config
+ )
+ except ValueError:
+ definition = None
+ yield str(evaluation_id), definition
+
+
def _graded(definition: EvaluationReportOutcomeDefinition) -> str:
predicates = definition.outcome_predicates
- return f"{definition.event_predicate} AND (({predicates['pass']}) OR ({predicates['fail']}))"
+ return _graded_from(definition.event_predicate, predicates["pass"], predicates["fail"])
+
+
+def _graded_from(event_predicate: str, passed: str, failed: str) -> str:
+ return f"{event_predicate} AND (({passed}) OR ({failed}))"
def _multi_if(branches: list[str], default: str) -> str:
diff --git a/products/ai_observability/backend/logic/evaluations/period.py b/products/ai_observability/backend/logic/evaluations/period.py
new file mode 100644
index 000000000000..831c880e02c2
--- /dev/null
+++ b/products/ai_observability/backend/logic/evaluations/period.py
@@ -0,0 +1,112 @@
+from datetime import UTC, date, datetime, time, timedelta, tzinfo
+from typing import Literal
+
+from posthog.dataclasses import frozen
+
+Interval = Literal["hour", "day"]
+
+# A period this short or shorter is bucketed by hour, so a day or two still draws a line rather than one or two dots.
+MAX_HOURLY_SPAN = timedelta(days=2)
+DEFAULT_DAYS = 14
+MAX_DAYS = 90
+
+
+def _resolve_bound(value: date | datetime, tz: tzinfo, *, is_end: bool) -> datetime:
+ """A date is a whole day in the project's timezone, so as an end it means the following midnight."""
+ if isinstance(value, datetime):
+ return value.astimezone(tz) if value.tzinfo else value.replace(tzinfo=tz)
+ return datetime.combine(value + timedelta(days=1) if is_end else value, time.min, tz)
+
+
+class InvalidPeriodError(ValueError):
+ def __init__(self, message: str, *, field: str | None = None) -> None:
+ super().__init__(message)
+ # The request parameter at fault, or None when the period as a whole is out of range.
+ self.field = field
+
+
+@frozen
+class PeriodBounds:
+ start: datetime
+ end: datetime # Exclusive.
+
+ @classmethod
+ def from_request(
+ cls, date_from: date | datetime | None, date_to: date | datetime | None, tz: tzinfo
+ ) -> "PeriodBounds":
+ """A date is a whole day in the project's timezone, a datetime is an exact instant. Without bounds, the period
+ is the last 14 days up to and including today."""
+ try:
+ date_to = date_to or datetime.now(tz).date()
+ if date_from is None:
+ date_from = (
+ date_to - timedelta(days=DEFAULT_DAYS)
+ if isinstance(date_to, datetime)
+ else date_to - timedelta(days=DEFAULT_DAYS - 1)
+ )
+ end = _resolve_bound(date_to, tz, is_end=True)
+ start = _resolve_bound(date_from, tz, is_end=False)
+ # Both bounds share the project's tzinfo, which compares wall time, so this compares instants instead.
+ length = end.astimezone(UTC) - start.astimezone(UTC)
+ # The overview also reads the period of equal length before this one, so its start must exist too.
+ start.astimezone(UTC) - length
+ except OverflowError as error:
+ raise InvalidPeriodError("The period is outside the supported date range.") from error
+ if length <= timedelta(0):
+ raise InvalidPeriodError("Must be before date_to.", field="date_from")
+ # Whole days count by the calendar, because a day across a daylight saving change is 23 or 25 hours long.
+ # An instant has no such change, so a period with one counts real time.
+ if not isinstance(date_from, datetime) and not isinstance(date_to, datetime):
+ too_long = (date_to - date_from).days + 1 > MAX_DAYS
+ else:
+ too_long = length > timedelta(days=MAX_DAYS)
+ if too_long:
+ raise InvalidPeriodError(f"The period can cover at most {MAX_DAYS} days.", field="date_from")
+ return cls(start=start, end=end)
+
+
+@frozen
+class OverviewPeriod:
+ start: datetime
+ end: datetime
+ previous_start: datetime
+ # Before `start` when the current period is still running, so both periods cover the same length of time.
+ previous_end: datetime
+ interval: Interval
+ # The start of each bucket in the project's timezone. The first one can start before `start`.
+ buckets: tuple[datetime, ...]
+
+
+def overview_period(start: datetime, end: datetime, now: datetime, tz: tzinfo) -> OverviewPeriod:
+ # Two datetimes that share a tzinfo subtract and compare by wall time, so a daylight saving change would alter
+ # the period's length. UTC keeps every length and comparison in real time.
+ start_utc, end_utc = start.astimezone(UTC), end.astimezone(UTC)
+ duration = end_utc - start_utc
+ previous_start = start_utc - duration
+ interval: Interval = "hour" if duration <= MAX_HOURLY_SPAN else "day"
+ return OverviewPeriod(
+ start=start,
+ end=end,
+ previous_start=previous_start.astimezone(tz),
+ previous_end=max(min(end_utc, now) - duration, previous_start).astimezone(tz),
+ interval=interval,
+ buckets=_buckets(start.astimezone(tz), end_utc, interval, tz),
+ )
+
+
+def _buckets(start: datetime, end: datetime, interval: Interval, tz: tzinfo) -> tuple[datetime, ...]:
+ """`end` is in UTC, so each comparison with a local bucket is by instant."""
+ buckets: list[datetime] = []
+ if interval == "day":
+ # Midnight by local date, because a day is 23 or 25 hours long across a daylight saving change.
+ day = start.date()
+ while (bucket := datetime.combine(day, time.min, tz)) < end:
+ buckets.append(bucket)
+ day += timedelta(days=1)
+ return tuple(buckets)
+ # Stepping in UTC keeps each step one real hour across a daylight saving change.
+ bucket = start.replace(minute=0, second=0, microsecond=0)
+ while bucket < end:
+ buckets.append(bucket)
+ bucket = (bucket.astimezone(UTC) + timedelta(hours=1)).astimezone(tz)
+ return tuple(buckets)
diff --git a/products/ai_observability/backend/logic/evaluations/runs.py b/products/ai_observability/backend/logic/evaluations/runs.py
new file mode 100644
index 000000000000..d3d8d061957d
--- /dev/null
+++ b/products/ai_observability/backend/logic/evaluations/runs.py
@@ -0,0 +1,89 @@
+from typing import Literal
+
+from posthog.hogql import ast
+from posthog.hogql.parser import parse_expr
+from posthog.hogql.visitor import clone_expr
+
+from posthog.models import Team, User
+
+from ...facade import contracts
+from .overview_queries import (
+ EVALUATION_RUNS_WHERE,
+ GRADED_WITHOUT_RULE,
+ evaluation_runs_placeholders,
+ run_evaluation_query,
+)
+from .pass_filters import PassRules
+from .period import PeriodBounds
+
+RunOutcome = Literal["passed", "failed"]
+
+REASONING_MAX_CHARS = 1000
+
+_RUNS_QUERY = f"""
+SELECT
+ timestamp,
+ if({{graded}}, if({{passes}}, 'passed', 'failed'), NULL),
+ properties.$ai_evaluation_reasoning,
+ properties.$ai_target_event_id,
+ properties.$ai_trace_id
+FROM events
+WHERE {EVALUATION_RUNS_WHERE} AND {{matches}}
+ORDER BY timestamp DESC
+LIMIT {{limit}}
+"""
+
+
+def _text(value: object) -> str | None:
+ return value if isinstance(value, str) and value else None
+
+
+def _truncate(value: object) -> str | None:
+ text = _text(value)
+ if text is None or len(text) <= REASONING_MAX_CHARS:
+ return text
+ return f"{text[: REASONING_MAX_CHARS - 1]}…"
+
+
+def _matches(pass_rules: PassRules, outcome: RunOutcome | None) -> ast.Expr:
+ # Copies, because the select reads the same rule expressions.
+ graded, passes = clone_expr(pass_rules.graded), clone_expr(pass_rules.passes)
+ if outcome == "passed":
+ return ast.And(exprs=[graded, passes])
+ if outcome == "failed":
+ return ast.And(exprs=[graded, ast.Not(expr=passes)])
+ return ast.Or(exprs=[graded, parse_expr(GRADED_WITHOUT_RULE)])
+
+
+def load_runs(
+ team: Team,
+ user: User | None,
+ evaluation_id: str,
+ period: PeriodBounds,
+ pass_rules: PassRules,
+ outcome: RunOutcome | None,
+ limit: int,
+) -> list[contracts.EvaluationRun]:
+ rows = run_evaluation_query(
+ "ai_evaluations_runs",
+ _RUNS_QUERY,
+ team,
+ user,
+ {
+ **evaluation_runs_placeholders(evaluation_id, period),
+ "graded": pass_rules.graded,
+ "passes": pass_rules.passes,
+ "matches": _matches(pass_rules, outcome),
+ "limit": ast.Constant(value=limit),
+ },
+ )
+ return [
+ contracts.EvaluationRun(
+ timestamp=row[0],
+ outcome=row[1],
+ reasoning=_truncate(row[2]),
+ target_event_id=_text(row[3]),
+ trace_id=_text(row[4]),
+ )
+ for row in rows
+ ]
diff --git a/products/ai_observability/backend/logic/evaluations/score_distribution.py b/products/ai_observability/backend/logic/evaluations/score_distribution.py
new file mode 100644
index 000000000000..66878b120fd4
--- /dev/null
+++ b/products/ai_observability/backend/logic/evaluations/score_distribution.py
@@ -0,0 +1,84 @@
+from posthog.hogql import ast
+
+from posthog.dataclasses import frozen
+from posthog.models import Team, User
+
+from products.ai_observability.backend.models.evaluation_outcomes import NUMERIC_SCORED_PREDICATE, SCORE_EXPRESSION
+
+from ...facade import contracts
+from .overview_queries import EVALUATION_RUNS_WHERE, evaluation_runs_placeholders, run_evaluation_query
+from .period import PeriodBounds
+
+# An integer scale of 2 to this many steps gets one bucket per integer, any other scale gets equal buckets.
+MAX_INTEGER_BUCKETS = 20
+EQUAL_BUCKETS = 10
+
+# Scores outside the configured scale land in the end buckets. The emitter skips such runs, so this only catches
+# runs scored before the scale changed. The epsilon keeps a score on a bucket edge in the bucket it starts, where
+# float division can land just under it: (0.3 - 0) / 0.1 is 2.9999999999999996.
+_HISTOGRAM_QUERY = f"""
+SELECT least(greatest(toInt(floor(({SCORE_EXPRESSION} - {{low}}) / {{width}} + 1e-9)), 0), {{last}}) AS bucket, count()
+FROM events
+WHERE {EVALUATION_RUNS_WHERE} AND {NUMERIC_SCORED_PREDICATE}
+GROUP BY bucket
+LIMIT {MAX_INTEGER_BUCKETS + 1}
+"""
+
+
+@frozen
+class ScoreBuckets:
+ low: float
+ width: float
+ labels: tuple[str, ...]
+
+
+def score_buckets(scale: contracts.ScoreScale, step: float | None) -> ScoreBuckets | None:
+ span = scale.max - scale.min
+ if span <= 0:
+ return None
+ # A 0 to 1 scale is a pair of integers too, but its scores are fractions unless a whole step says otherwise.
+ integer_steps = step.is_integer() if step is not None else span >= 2
+ if scale.min.is_integer() and scale.max.is_integer() and integer_steps and span <= MAX_INTEGER_BUCKETS:
+ # Centering each bucket on its integer keeps a score of 3 in the "3" bucket however it rounds.
+ return ScoreBuckets(
+ low=scale.min - 0.5,
+ width=1,
+ labels=tuple(str(int(scale.min) + offset) for offset in range(int(span) + 1)),
+ )
+ width = span / EQUAL_BUCKETS
+ return ScoreBuckets(
+ low=scale.min,
+ width=width,
+ labels=tuple(
+ f"{scale.min + index * width:g}-{scale.min + (index + 1) * width:g}" for index in range(EQUAL_BUCKETS)
+ ),
+ )
+
+
+def load_score_distribution(
+ team: Team,
+ user: User | None,
+ evaluation_id: str,
+ period: PeriodBounds,
+ scale: contracts.ScoreScale,
+ buckets: ScoreBuckets,
+) -> contracts.ScoreHistogram:
+ rows = run_evaluation_query(
+ "ai_evaluations_score_distribution",
+ _HISTOGRAM_QUERY,
+ team,
+ user,
+ {
+ **evaluation_runs_placeholders(evaluation_id, period),
+ "low": ast.Constant(value=buckets.low),
+ "width": ast.Constant(value=buckets.width),
+ "last": ast.Constant(value=len(buckets.labels) - 1),
+ },
+ )
+ counts = {int(bucket): count for bucket, count in rows}
+ return contracts.ScoreHistogram(
+ scale=scale,
+ buckets=[
+ contracts.ScoreBucket(label=label, count=counts.get(index, 0)) for index, label in enumerate(buckets.labels)
+ ],
+ )
diff --git a/products/ai_observability/backend/models/evaluation_outcomes.py b/products/ai_observability/backend/models/evaluation_outcomes.py
index 3067f197ea9e..0c11a6c97c5a 100644
--- a/products/ai_observability/backend/models/evaluation_outcomes.py
+++ b/products/ai_observability/backend/models/evaluation_outcomes.py
@@ -7,9 +7,12 @@ that reads only those is correct without passing `true_is_failure` or `output_co
from collections.abc import Callable, Mapping
from dataclasses import dataclass
+from typing import Literal
from posthog.hogql.escape_sql import escape_hogql_string
+from posthog.dataclasses import frozen
+
from products.ai_observability.backend.models.evaluation_configs import CategoricalOutputConfig, NumericOutputConfig
SENTIMENT_LABELS = ("positive", "neutral", "negative")
@@ -69,9 +72,9 @@ class EvaluationReportOutcomeDefinition:
# String comparison is deliberate: an unregistered JSON bool property extracts as the
# string 'true', which the bool literal cannot compare against, while a registered
# Boolean property coerces correctly against the string form.
-_NOT_SKIPPED_PREDICATE = "(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')"
+NOT_SKIPPED_PREDICATE = "(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')"
-_APPLICABLE_PREDICATE = (
+APPLICABLE_PREDICATE = (
"(isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false')"
)
@@ -83,13 +86,13 @@ def _boolean_definition(true_is_failure: bool) -> EvaluationReportOutcomeDefinit
return EvaluationReportOutcomeDefinition(
outcomes=("pass", "fail", "na"),
outcome_predicates={
- "pass": f"properties.$ai_evaluation_result = {pass_literal} AND {_APPLICABLE_PREDICATE}",
- "fail": f"properties.$ai_evaluation_result = {fail_literal} AND {_APPLICABLE_PREDICATE}",
+ "pass": f"properties.$ai_evaluation_result = {pass_literal} AND {APPLICABLE_PREDICATE}",
+ "fail": f"properties.$ai_evaluation_result = {fail_literal} AND {APPLICABLE_PREDICATE}",
"na": "properties.$ai_evaluation_applicable = 'false'",
},
event_predicate=(
"(properties.$ai_evaluation_result_type = 'boolean' OR isNull(properties.$ai_evaluation_result_type)) "
- f"AND {_NOT_SKIPPED_PREDICATE}"
+ f"AND {NOT_SKIPPED_PREDICATE}"
),
result_expression="properties.$ai_evaluation_result",
applicable_expression="properties.$ai_evaluation_applicable",
@@ -102,7 +105,7 @@ def _sentiment_definition(true_is_failure: bool) -> EvaluationReportOutcomeDefin
return EvaluationReportOutcomeDefinition(
outcomes=SENTIMENT_LABELS,
outcome_predicates={label: f"properties.$ai_sentiment_label = '{label}'" for label in SENTIMENT_LABELS},
- event_predicate=f"properties.$ai_evaluation_result_type = 'sentiment' AND {_NOT_SKIPPED_PREDICATE}",
+ event_predicate=f"properties.$ai_evaluation_result_type = 'sentiment' AND {NOT_SKIPPED_PREDICATE}",
result_expression="properties.$ai_sentiment_label",
applicable_expression="NULL",
score_expression="properties.$ai_sentiment_score",
@@ -118,21 +121,59 @@ _DEFINITION_BUILDERS: Mapping[str, Callable[[bool], EvaluationReportOutcomeDefin
SUPPORTED_EVAL_REPORT_OUTPUT_TYPES = (*_DEFINITION_BUILDERS, "numeric", "categorical")
+CATEGORIES_EXPRESSION = "JSONExtract(ifNull(properties.$ai_evaluation_categorical_result, '[]'), 'Array(String)')"
+SCORE_EXPRESSION = "toFloat(properties.$ai_evaluation_numeric_result)"
+_NUMERIC_EVENT_PREDICATE = f"properties.$ai_evaluation_result_type = 'numeric' AND {NOT_SKIPPED_PREDICATE}"
+_CATEGORICAL_EVENT_PREDICATE = f"properties.$ai_evaluation_result_type = 'categorical' AND {NOT_SKIPPED_PREDICATE}"
+# A categorical run is graded only when the judge said it applies, where a numeric run is graded unless it said not.
+_CATEGORICAL_APPLICABLE_PREDICATE = "properties.$ai_evaluation_applicable = 'true'"
+# A run that produced a score or chose its categories, with or without a passing rule. Same as
+# EVALUATION_NUMERIC_GRADED_HOGQL and EVALUATION_CATEGORICAL_GRADED_HOGQL in the frontend's constants.ts.
+NUMERIC_SCORED_PREDICATE = (
+ f"{_NUMERIC_EVENT_PREDICATE} AND properties.$ai_evaluation_numeric_result IS NOT NULL AND {APPLICABLE_PREDICATE}"
+)
+CATEGORIZED_PREDICATE = f"{_CATEGORICAL_EVENT_PREDICATE} AND {_CATEGORICAL_APPLICABLE_PREDICATE}"
+
+
+@frozen
+class RulePredicates:
+ passed: str
+ failed: str
+
+
+def categorical_rule_predicates(passing_categories: str | None) -> RulePredicates:
+ """The pass and fail predicates of a categorical passing rule. `passing_categories` is a HogQL array, or None
+ for a rule that passes only a run that chose no category."""
+ if passing_categories is None:
+ passing = f"empty({CATEGORIES_EXPRESSION})"
+ else:
+ passing = f"(notEmpty({CATEGORIES_EXPRESSION}) AND hasAll({passing_categories}, {CATEGORIES_EXPRESSION}))"
+ return RulePredicates(
+ passed=f"{passing} AND {_CATEGORICAL_APPLICABLE_PREDICATE}",
+ failed=f"NOT {passing} AND {_CATEGORICAL_APPLICABLE_PREDICATE}",
+ )
+
+
+def numeric_rule_predicates(operator: Literal["gte", "lte"], threshold: str) -> RulePredicates:
+ """The pass and fail predicates of a numeric passing rule. `threshold` is a HogQL expression."""
+ passed_op, failed_op = (">=", "<") if operator == "gte" else ("<=", ">")
+ return RulePredicates(
+ passed=f"{SCORE_EXPRESSION} {passed_op} {threshold} AND {APPLICABLE_PREDICATE}",
+ failed=f"{SCORE_EXPRESSION} {failed_op} {threshold} AND {APPLICABLE_PREDICATE}",
+ )
+
+
def _categorical_definition(output_config: dict | None) -> EvaluationReportOutcomeDefinition:
config = CategoricalOutputConfig.model_validate(output_config) if output_config else None
- categories_expression = "JSONExtract(ifNull(properties.$ai_evaluation_categorical_result, '[]'), 'Array(String)')"
- applicable_predicate = "properties.$ai_evaluation_applicable = 'true'"
passed_predicate = "false"
failed_predicate = "false"
if config is not None and config.passing_rule is not None:
- if config.passing_rule.categories:
- passing = ", ".join(escape_hogql_string(category) for category in config.passing_rule.categories)
- passing_condition = f"(notEmpty({categories_expression}) AND hasAll([{passing}], {categories_expression}))"
- else:
- passing_condition = f"empty({categories_expression})"
- passed_predicate = f"{passing_condition} AND {applicable_predicate}"
- failed_predicate = f"NOT {passing_condition} AND {applicable_predicate}"
+ categories = config.passing_rule.categories
+ predicates = categorical_rule_predicates(
+ f"[{', '.join(escape_hogql_string(category) for category in categories)}]" if categories else None
+ )
+ passed_predicate, failed_predicate = predicates.passed, predicates.failed
return EvaluationReportOutcomeDefinition(
outcomes=("pass", "fail", "na"),
@@ -141,8 +182,8 @@ def _categorical_definition(output_config: dict | None) -> EvaluationReportOutco
"fail": failed_predicate,
"na": "properties.$ai_evaluation_applicable = 'false'",
},
- event_predicate=f"properties.$ai_evaluation_result_type = 'categorical' AND {_NOT_SKIPPED_PREDICATE}",
- result_expression=categories_expression,
+ event_predicate=_CATEGORICAL_EVENT_PREDICATE,
+ result_expression=CATEGORIES_EXPRESSION,
applicable_expression="properties.$ai_evaluation_applicable",
score_expression="NULL",
passing_result=None,
@@ -152,21 +193,17 @@ def _categorical_definition(output_config: dict | None) -> EvaluationReportOutco
def _numeric_definition(output_config: dict | None) -> EvaluationReportOutcomeDefinition:
config = NumericOutputConfig.model_validate(output_config or {})
- score = "toFloat(properties.$ai_evaluation_numeric_result)"
- applicable = "(isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false')"
passed = failed = "false"
if config.passing_rule is not None:
- passed_op, failed_op = (">=", "<") if config.passing_rule.operator == "gte" else ("<=", ">")
- threshold = repr(config.passing_rule.threshold)
- passed = f"{score} {passed_op} {threshold} AND {applicable}"
- failed = f"{score} {failed_op} {threshold} AND {applicable}"
+ predicates = numeric_rule_predicates(config.passing_rule.operator, repr(config.passing_rule.threshold))
+ passed, failed = predicates.passed, predicates.failed
return EvaluationReportOutcomeDefinition(
outcomes=("pass", "fail", "na"),
outcome_predicates={"pass": passed, "fail": failed, "na": "properties.$ai_evaluation_applicable = 'false'"},
- event_predicate=f"properties.$ai_evaluation_result_type = 'numeric' AND {_NOT_SKIPPED_PREDICATE}",
- result_expression=score,
+ event_predicate=_NUMERIC_EVENT_PREDICATE,
+ result_expression=SCORE_EXPRESSION,
applicable_expression="properties.$ai_evaluation_applicable",
- score_expression=score,
+ score_expression=SCORE_EXPRESSION,
passing_result=None,
numeric_config=config,
)
diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_overview.py b/products/ai_observability/backend/tests/logic/evaluations/test_overview.py
new file mode 100644
index 000000000000..49ba16b0fffb
--- /dev/null
+++ b/products/ai_observability/backend/tests/logic/evaluations/test_overview.py
@@ -0,0 +1,242 @@
+from datetime import UTC, datetime
+from typing import Any
+
+from parameterized import parameterized
+
+from products.ai_observability.backend.facade import contracts
+from products.ai_observability.backend.logic.evaluations.overview import (
+ CategoryOption,
+ EvaluationMeta,
+ SummaryKind,
+ evaluation_result,
+ overview_totals,
+ pass_rate_overview,
+)
+from products.ai_observability.backend.logic.evaluations.overview_queries import (
+ CategoryBucketCount,
+ EvaluationBucketCounts,
+)
+
+BUCKETS = [datetime(2026, 3, day, tzinfo=UTC) for day in (1, 2, 3, 4)]
+
+
+def _meta(kind: SummaryKind = "pass_rate", **fields: Any) -> EvaluationMeta:
+ defaults: dict[str, Any] = {
+ "id": "eval-1",
+ "name": "Example",
+ "directory_name": None,
+ "evaluation_type": "llm_judge",
+ "output_type": {"pass_rate": "boolean", "score": "numeric", "categories": "categorical"}.get(kind, kind),
+ "enabled": True,
+ "errored": False,
+ "status_reason": None,
+ "created_by": None,
+ "kind": kind,
+ "scale": None,
+ "score_step": None,
+ "categories": (),
+ "multiple": False,
+ }
+ return EvaluationMeta(**{**defaults, **fields})
+
+
+def _bucket(bucket: datetime | None, **counts: int | float) -> EvaluationBucketCounts:
+ fields: dict[str, Any] = {
+ "passed": 0,
+ "failed": 0,
+ "not_applicable": 0,
+ "skipped": 0,
+ "score_count": 0,
+ "score_sum": 0.0,
+ "categorized": 0,
+ "positive": 0,
+ "neutral": 0,
+ "negative": 0,
+ **counts,
+ }
+ return EvaluationBucketCounts(evaluation_id="eval-1", bucket=bucket, last_run_at=None, **fields)
+
+
+def _result(
+ meta: EvaluationMeta, counts: list[EvaluationBucketCounts], categories: list[CategoryBucketCount] | None = None
+) -> contracts.EvaluationResult:
+ return evaluation_result(meta, BUCKETS, counts, categories or [])
+
+
+ [
+ ("healthy", {}, [_bucket(BUCKETS[0], passed=90, failed=10), _bucket(None, passed=90, failed=10)], []),
+ ("paused never asks", {"enabled": False}, [_bucket(None, passed=90, failed=10)], []),
+ ("silent", {}, [], ["silent"]),
+ # The error status disables the evaluation, so it has no results, but it is neither silent nor ignored.
+ ("errored", {"enabled": False, "errored": True}, [_bucket(None, passed=80, failed=20)], ["errored"]),
+ (
+ "pass rate fell 2pp",
+ {},
+ [_bucket(BUCKETS[0], passed=88, failed=12), _bucket(None, passed=90, failed=10)],
+ ["worse"],
+ ),
+ (
+ "pass rate fell under 2pp",
+ {},
+ [_bucket(BUCKETS[0], passed=89, failed=11), _bucket(None, passed=90, failed=10)],
+ [],
+ ),
+ (
+ "thin previous period is never worse",
+ {},
+ [_bucket(BUCKETS[0], passed=450, failed=50), _bucket(None, passed=2)],
+ [],
+ ),
+ ]
+)
+def test_attention_reasons(
+ _name: str, meta: dict[str, Any], counts: list[EvaluationBucketCounts], expected: list[str]
+) -> None:
+ assert _result(_meta(**meta), counts).attention_reasons == expected
+
+
+ [
+ ("compared", [_bucket(BUCKETS[0], passed=80, failed=20), _bucket(None, passed=85, failed=15)], -5.0, False),
+ (
+ "low evidence is not compared",
+ [_bucket(BUCKETS[0], passed=7, failed=5), _bucket(None, passed=9, failed=1)],
+ None,
+ True,
+ ),
+ ("nothing before", [_bucket(BUCKETS[0], passed=80, failed=20)], None, False),
+ (
+ "thin previous period is not compared",
+ [_bucket(BUCKETS[0], passed=450, failed=50), _bucket(None, passed=2)],
+ None,
+ False,
+ ),
+ # 50 - 49.75 rounds half up like Math.round, where Python's round() would give 0.2.
+ (
+ "rounds half up",
+ [_bucket(BUCKETS[0], passed=40, failed=40), _bucket(None, passed=199, failed=201)],
+ 0.3,
+ False,
+ ),
+ ]
+)
+def test_pass_rate_change_and_low_evidence(
+ _name: str, counts: list[EvaluationBucketCounts], change: float | None, low_evidence: bool
+) -> None:
+ result = _result(_meta(), counts)
+ assert (result.change.value if result.change else None) == change
+ assert result.low_evidence is low_evidence
+
+
+def test_bucket_status_against_previous_period() -> None:
+ result = _result(
+ _meta(),
+ [
+ _bucket(BUCKETS[0], passed=90, failed=10),
+ _bucket(BUCKETS[1], passed=86, failed=14),
+ # Further below the previous rate, but too thin to judge.
+ _bucket(BUCKETS[3], passed=20, failed=5),
+ _bucket(None, passed=90, failed=10),
+ ],
+ )
+ assert result.trend.values == [90.0, 86.0, None, 80.0]
+ assert result.bucket_status == ["normal", "dip", "none", "normal"]
+
+
+def test_bucket_status_falls_back_to_own_median_without_previous_period() -> None:
+ result = _result(
+ _meta(),
+ [
+ _bucket(BUCKETS[0], passed=90, failed=10),
+ _bucket(BUCKETS[1], passed=90, failed=10),
+ _bucket(BUCKETS[2], passed=80, failed=20),
+ ],
+ )
+ assert result.bucket_status == ["normal", "normal", "dip", "none"]
+
+
+def test_score_summary() -> None:
+ meta = _meta("score", scale=contracts.ScoreScale(min=0, max=1))
+ result = _result(
+ meta,
+ [
+ _bucket(BUCKETS[0], score_count=40, score_sum=32.0),
+ _bucket(BUCKETS[1], score_count=40, score_sum=32.0),
+ _bucket(None, score_count=40, score_sum=26.0),
+ ],
+ )
+ assert isinstance(result.summary, contracts.ScoreSummary)
+ assert result.summary.score == contracts.ScoreDistribution(mean=0.8, scale=meta.scale)
+ assert result.change == contracts.ResultChange(value=0.15, unit="points", subject=None)
+ # A score moving 0.05 of its scale counts as unusual either way, with no worse direction.
+ assert result.bucket_status == ["dip", "dip", "none", "none"]
+ assert result.attention_reasons == []
+
+
+def test_categories_summary_tracks_the_label_that_moved_most() -> None:
+ meta = _meta(
+ "categories",
+ multiple=True,
+ categories=(CategoryOption(key="billing", label="Billing"), CategoryOption(key="bug", label="Bug")),
+ )
+ categories = [
+ CategoryBucketCount(evaluation_id="eval-1", bucket=BUCKETS[0], key="billing", runs=30),
+ CategoryBucketCount(evaluation_id="eval-1", bucket=BUCKETS[0], key="bug", runs=12),
+ CategoryBucketCount(evaluation_id="eval-1", bucket=BUCKETS[0], key="", runs=8),
+ CategoryBucketCount(evaluation_id="eval-1", bucket=None, key="billing", runs=30),
+ CategoryBucketCount(evaluation_id="eval-1", bucket=None, key="bug", runs=2),
+ CategoryBucketCount(evaluation_id="eval-1", bucket=None, key="refund", runs=4),
+ ]
+ result = _result(meta, [_bucket(BUCKETS[0], categorized=50), _bucket(None, categorized=40)], categories)
+ assert result.summary == contracts.CategoriesSummary(
+ kind="categories",
+ multiple=True,
+ categories=[
+ contracts.CategoryCount(key="billing", label="Billing", count=30, previous_share=75.0),
+ contracts.CategoryCount(key="bug", label="Bug", count=12, previous_share=5.0),
+ contracts.CategoryCount(key="refund", label="refund", count=0, previous_share=10.0),
+ ],
+ no_categories_count=8,
+ )
+ assert result.change == contracts.ResultChange(value=19.0, unit="pp", subject="Bug")
+ assert result.tracked_category == "bug"
+ assert result.trend.values == [24.0, None, None, None]
+
+
+def test_rising_negative_sentiment_is_worse() -> None:
+ result = _result(
+ _meta("sentiment"),
+ [_bucket(BUCKETS[0], positive=20, neutral=20, negative=10), _bucket(None, positive=25, neutral=20, negative=5)],
+ )
+ assert result.change == contracts.ResultChange(value=10.0, unit="pp", subject="negative")
+ assert result.attention_reasons == ["worse"]
+
+
+def test_overview_totals_compare_only_evaluations_with_results_in_both_periods() -> None:
+ steady = _result(_meta(), [_bucket(BUCKETS[0], passed=60, failed=20), _bucket(None, passed=40, failed=10)])
+ new = _result(_meta(), [_bucket(BUCKETS[1], passed=30, failed=0)])
+ sentiment = _result(
+ _meta("sentiment"), [_bucket(BUCKETS[1], negative=5, neutral=15), _bucket(None, negative=3, neutral=27)]
+ )
+ paused = _result(_meta(enabled=False), [_bucket(BUCKETS[0], passed=1000), _bucket(None, passed=1000)])
+
+ totals = overview_totals([steady, new, sentiment, paused], len(BUCKETS))
+
+ assert totals.graded == 130
+ assert totals.graded_comparison == contracts.PeriodComparison(current=100, previous=80)
+ assert totals.graded_trend == [80, 50, 0, 0]
+ assert totals.sentiment == contracts.OverviewSentiment(
+ evaluation_id="eval-1", negative_share=25.0, previous_share=10.0
+ )
+ assert overview_totals([new], len(BUCKETS)).graded_comparison is None
+
+
+def test_pass_rate_median_change_uses_the_same_evaluations_in_both_periods() -> None:
+ steady = _result(_meta(), [_bucket(BUCKETS[0], passed=80, failed=20), _bucket(None, passed=90, failed=10)])
+ new = _result(_meta(), [_bucket(BUCKETS[0], passed=40, failed=60)])
+
+ overview = pass_rate_overview([steady, new], len(BUCKETS))
+
+ assert overview.median == 60.0
+ assert overview.median_comparison == contracts.PeriodComparison(current=80.0, previous=90.0)
diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.py b/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.py
new file mode 100644
index 000000000000..12a829bd3865
--- /dev/null
+++ b/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.py
@@ -0,0 +1,372 @@
+from datetime import UTC, date, datetime, timedelta
+from zoneinfo import ZoneInfo
+
+import pytest
+import time_machine
+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
+from unittest.mock import patch
+
+from parameterized import parameterized
+
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.database import Database
+from posthog.hogql.printer.utils import prepare_and_print_ast
+
+from products.ai_observability.backend.facade.api import (
+ evaluations_period,
+ get_evaluation_runs,
+ get_evaluations_overview,
+)
+from products.ai_observability.backend.logic.evaluations.overview_queries import (
+ CategoryBucketCount,
+ load_bucket_counts,
+ load_category_counts,
+ run_evaluation_query,
+)
+from products.ai_observability.backend.logic.evaluations.pass_filters import build_pass_rules
+from products.ai_observability.backend.logic.evaluations.period import (
+ InvalidPeriodError,
+ OverviewPeriod,
+ PeriodBounds,
+ overview_period,
+)
+from products.ai_observability.backend.models.evaluations import Evaluation
+
+PERIOD = OverviewPeriod(
+ start=datetime(2026, 3, 8, tzinfo=UTC),
+ end=datetime(2026, 3, 15, tzinfo=UTC),
+ previous_start=datetime(2026, 3, 1, tzinfo=UTC),
+ previous_end=datetime(2026, 3, 8, tzinfo=UTC),
+ interval="day",
+ buckets=tuple(datetime(2026, 3, day, tzinfo=UTC) for day in range(8, 15)),
+)
+
+
+NEW_YORK = ZoneInfo("America/New_York")
+
+
+def _midnight(day: int) -> datetime:
+ return datetime(2026, 3, day, tzinfo=UTC)
+
+
+ [
+ # Clocks spring forward on March 8, so 48 real hours span 49 hours on the wall.
+ ("spring_forward", datetime(2026, 3, 9, 12, tzinfo=UTC), 48),
+ # Clocks fall back on November 1, so 24 real hours span 23 hours on the wall.
+ ("fall_back", datetime(2026, 11, 1, 18, tzinfo=UTC), 24),
+ ]
+)
+def test_rolling_hours_keep_their_real_length_across_daylight_saving(_name: str, now: datetime, hours: int) -> None:
+ length = timedelta(hours=hours)
+ period = overview_period((now - length).astimezone(NEW_YORK), now.astimezone(NEW_YORK), now, NEW_YORK)
+
+ assert period.interval == "hour"
+ assert len(period.buckets) == hours
+ assert (period.previous_start, period.previous_end) == (now - 2 * length, now - length)
+
+
+ [
+ ("end before start", date(2026, 3, 10), date(2026, 3, 9), "date_from"),
+ ("91 days", date(2026, 8, 4), date(2026, 11, 2), "date_from"),
+ ("last day of year 9999", None, date(9999, 12, 31), "out of range"),
+ ("first day of year 1", None, date(1, 1, 1), "out of range"),
+ # Clocks fall back on November 1, so these 90 days are 2161 hours long.
+ ("90 days across daylight saving", date(2026, 8, 5), date(2026, 11, 2), None),
+ (
+ "90 days of instants",
+ datetime(2026, 8, 5, tzinfo=UTC),
+ datetime(2026, 11, 3, tzinfo=UTC),
+ None,
+ ),
+ (
+ "an instant past 90 days",
+ datetime(2026, 8, 5, tzinfo=UTC),
+ datetime(2026, 11, 3, 0, 0, 1, tzinfo=UTC),
+ "date_from",
+ ),
+ ]
+)
+def test_period_from_request(
+ _name: str, date_from: date | datetime | None, date_to: date | datetime, error: str | None
+) -> None:
+ if error is None:
+ PeriodBounds.from_request(date_from, date_to, NEW_YORK)
+ return
+ with pytest.raises(InvalidPeriodError) as raised:
+ PeriodBounds.from_request(date_from, date_to, NEW_YORK)
+ assert (raised.value.field or "out of range") == error
+
+
+class TestEvaluationsOverviewQueries(ClickhouseTestMixin, BaseTest):
+ def _judge(self) -> Evaluation:
+ return Evaluation.objects.create(
+ team=self.team,
+ name="Judge",
+ evaluation_type="hog",
+ evaluation_config={"source": "return true"},
+ output_type="boolean",
+ output_config={},
+ )
+
+ def _run(self, evaluation: Evaluation, timestamp: str, **properties: object) -> None:
+ _create_event(
+ team=self.team,
+ event="$ai_evaluation",
+ distinct_id="user-1",
+ timestamp=timestamp,
+ properties={"$ai_evaluation_id": str(evaluation.id), **properties},
+ )
+
+ @parameterized.expand(
+ [
+ ("one query per kind", 50_000, 25_000, 2),
+ ("a batch per evaluation", 50_000, 1, 3),
+ # A full page splits its batch, down to one evaluation, which pages through its own rows.
+ ("full pages", 2, 50_000, 8),
+ ]
+ )
+ def test_groups_runs_by_evaluation_day_and_period(
+ self, _name: str, page_size: int, batch_rows: int, queries: int
+ ) -> None:
+ judge = self._judge()
+ topic = Evaluation.objects.create(
+ team=self.team,
+ name="Topic",
+ evaluation_type="hog",
+ evaluation_config={"source": "return []"},
+ output_type="categorical",
+ output_config={
+ "options": [{"key": "billing", "label": "Billing"}, {"key": "bug", "label": "Bug"}],
+ "selection_mode": "multiple",
+ },
+ )
+ unrequested = Evaluation.objects.create(
+ team=self.team, name="Not asked for", evaluation_type="hog", output_type="boolean", output_config={}
+ )
+ categorical = {"$ai_evaluation_result_type": "categorical", "$ai_evaluation_applicable": True}
+ self._run(judge, "2026-03-09T10:00:00Z", **{"$ai_evaluation_result": True})
+ self._run(judge, "2026-03-09T11:00:00Z", **{"$ai_evaluation_result": False})
+ self._run(judge, "2026-03-09T12:00:00Z", **{"$ai_evaluation_applicable": False})
+ skipped = {"$ai_evaluation_result": False, "$ai_evaluation_skipped": True}
+ self._run(judge, "2026-03-10T10:00:00Z", **skipped, **{"$ai_evaluation_skip_reason": "hog_input_error"})
+ self._run(judge, "2026-03-10T11:00:00Z", **skipped, **{"$ai_evaluation_skip_reason": "trace_not_found"})
+ self._run(judge, "2026-03-02T10:00:00Z", **{"$ai_evaluation_result": True})
+ self._run(judge, "2026-03-07T23:00:00Z", **{"$ai_evaluation_result": True})
+ self._run(judge, "2026-02-28T23:00:00Z", **{"$ai_evaluation_result": True})
+ self._run(
+ topic, "2026-03-09T10:00:00Z", **categorical, **{"$ai_evaluation_categorical_result": ["billing", "bug"]}
+ )
+ self._run(topic, "2026-03-09T11:00:00Z", **categorical, **{"$ai_evaluation_categorical_result": []})
+ self._run(topic, "2026-03-03T10:00:00Z", **categorical, **{"$ai_evaluation_categorical_result": ["bug"]})
+ self._run(unrequested, "2026-03-09T10:00:00Z", **{"$ai_evaluation_result": True})
+ flush_persons_and_events()
+
+ ids = [str(judge.id), str(topic.id)]
+ module = "products.ai_observability.backend.logic.evaluations.overview_queries"
+ with (
+ patch(f"{module}.PAGE_SIZE", page_size),
+ patch(f"{module}.BATCH_ROWS", batch_rows),
+ patch(f"{module}.run_evaluation_query", wraps=run_evaluation_query) as run,
+ ):
+ bucket_counts = load_bucket_counts(self.team, None, PERIOD, ids, build_pass_rules(self.team.pk, ids))
+ category_counts = load_category_counts(self.team, None, PERIOD, {str(topic.id): 2})
+ counts = {
+ (row.evaluation_id, row.bucket): (row.passed, row.failed, row.not_applicable, row.skipped, row.categorized)
+ for row in bucket_counts
+ }
+ categories = sorted(category_counts, key=lambda row: (row.bucket or datetime.min.replace(tzinfo=UTC), row.key))
+
+ assert counts == {
+ (str(judge.id), _midnight(9)): (1, 1, 1, 0, 0),
+ (str(judge.id), _midnight(10)): (0, 0, 0, 2, 0),
+ (str(judge.id), None): (2, 0, 0, 0, 0),
+ (str(topic.id), _midnight(9)): (0, 0, 0, 0, 2),
+ (str(topic.id), None): (0, 0, 0, 0, 1),
+ }
+ assert categories == [
+ CategoryBucketCount(evaluation_id=str(topic.id), bucket=None, key="bug", runs=1),
+ CategoryBucketCount(evaluation_id=str(topic.id), bucket=_midnight(9), key="", runs=1),
+ CategoryBucketCount(evaluation_id=str(topic.id), bucket=_midnight(9), key="billing", runs=1),
+ CategoryBucketCount(evaluation_id=str(topic.id), bucket=_midnight(9), key="bug", runs=1),
+ ]
+ assert run.call_count == queries
+
+ @time_machine.travel(datetime(2026, 3, 10, 3, 0, tzinfo=UTC), tick=False)
+ def test_periods_follow_the_project_timezone_and_a_running_day(self) -> None:
+ self.team.timezone = "Asia/Tokyo"
+ self.team.save()
+ judge = self._judge()
+ # Now is 12:00 on March 10 in Tokyo, so the previous period runs from March 7 to 12:00 on March 8.
+ for timestamp in (
+ "2026-03-06T14:00:00Z", # 23:00 March 6, before the previous period
+ "2026-03-06T16:00:00Z", # 01:00 March 7
+ "2026-03-08T02:00:00Z", # 11:00 March 8
+ "2026-03-08T05:00:00Z", # 14:00 March 8, later in the day than now
+ "2026-03-08T16:00:00Z", # 01:00 March 9
+ "2026-03-09T16:00:00Z", # 01:00 March 10
+ ):
+ self._run(judge, timestamp, **{"$ai_evaluation_result": True})
+ flush_persons_and_events()
+
+ overview = get_evaluations_overview(
+ self.team,
+ None,
+ [str(judge.id)],
+ evaluations_period(self.team, date(2026, 3, 9), date(2026, 3, 10)),
+ )
+
+ [row] = overview.evaluations
+ # Two whole days are short enough for hourly buckets, which start on Tokyo's hours.
+ assert overview.interval == "hour"
+ assert overview.buckets[0] == datetime(2026, 3, 9, tzinfo=ZoneInfo("Asia/Tokyo"))
+ assert [index for index, volume in enumerate(row.trend.volume) if volume] == [1, 25]
+ assert row.previous_graded_count == 2
+
+ @time_machine.travel(datetime(2026, 3, 12, 12, 0, tzinfo=UTC), tick=False)
+ def test_days_follow_the_project_timezone_when_the_team_turns_off_converting_to_it(self) -> None:
+ self.team.timezone = "America/New_York"
+ self.team.modifiers = {"convertToProjectTimezone": False}
+ self.team.save()
+ judge = self._judge()
+ # 22:00 on March 9 in New York, already March 10 in UTC.
+ self._run(judge, "2026-03-10T02:00:00Z", **{"$ai_evaluation_result": True})
+ flush_persons_and_events()
+
+ overview = get_evaluations_overview(
+ self.team,
+ None,
+ [str(judge.id)],
+ evaluations_period(self.team, date(2026, 3, 8), date(2026, 3, 11)),
+ )
+
+ [row] = overview.evaluations
+ assert overview.interval == "day"
+ assert row.trend.volume == [0, 1, 0, 0]
+
+ @parameterized.expand(
+ [
+ # 01:30 on November 1 in New York, before and after the clocks fall back.
+ ("new_york_fall_back", "America/New_York", datetime(2026, 11, 1, 18, tzinfo=UTC), 24, ["05:30", "06:30"]),
+ # Lord Howe moves its clocks by 30 minutes on October 4, so from then on the buckets start at half past
+ # the local hour, and 01:10 and 01:40 on October 5 fall in different buckets.
+ (
+ "lord_howe_spring_forward",
+ "Australia/Lord_Howe",
+ datetime(2026, 10, 4, 20, tzinfo=UTC),
+ 48,
+ ["2026-10-03T10:00", "2026-10-04T14:10", "2026-10-04T14:40"],
+ ),
+ ("kolkata_half_hour_offset", "Asia/Kolkata", datetime(2026, 3, 10, 12, tzinfo=UTC), 24, ["05:15", "06:45"]),
+ # 09:59 and 10:01 in Kathmandu, either side of an hour that starts at a quarter past in UTC.
+ (
+ "kathmandu_quarter_hour_offset",
+ "Asia/Kathmandu",
+ datetime(2026, 3, 10, 12, tzinfo=UTC),
+ 24,
+ ["04:14", "04:16"],
+ ),
+ ]
+ )
+ def test_every_hourly_run_lands_in_the_trend(
+ self, _name: str, timezone: str, now: datetime, hours: int, times: list[str]
+ ) -> None:
+ self.team.timezone = timezone
+ self.team.save()
+ judge = self._judge()
+ for at in times:
+ timestamp = at if "T" in at else f"{now.date().isoformat()}T{at}"
+ self._run(judge, f"{timestamp}:00Z", **{"$ai_evaluation_result": True})
+ flush_persons_and_events()
+
+ with time_machine.travel(now, tick=False):
+ overview = get_evaluations_overview(
+ self.team, None, [str(judge.id)], PeriodBounds(start=now - timedelta(hours=hours), end=now)
+ )
+
+ [row] = overview.evaluations
+ assert overview.interval == "hour"
+ assert [volume for volume in row.trend.volume if volume] == [1] * len(times)
+ assert row.graded_count == len(times)
+
+ @time_machine.travel(datetime(2026, 11, 1, 18, 0, tzinfo=UTC), tick=False)
+ def test_a_window_inside_a_repeated_daylight_saving_hour_reads_its_runs(self) -> None:
+ self.team.timezone = "America/New_York"
+ self.team.save()
+ judge = self._judge()
+ # 01:00 on November 1 in New York after the clocks fall back, between 01:40 before and 01:20 after it.
+ self._run(judge, "2026-11-01T06:00:00Z", **{"$ai_evaluation_result": False})
+ flush_persons_and_events()
+ period = PeriodBounds(
+ start=datetime(2026, 11, 1, 5, 40, tzinfo=UTC), end=datetime(2026, 11, 1, 6, 20, tzinfo=UTC)
+ )
+
+ overview = get_evaluations_overview(self.team, None, [str(judge.id)], period)
+ runs = get_evaluation_runs(self.team, None, str(judge.id), period, "failed", 5)
+
+ assert overview.evaluations[0].graded_count == 1
+ assert len(runs.results) == 1
+
+ def test_query_size_does_not_grow_with_the_number_of_evaluations(self) -> None:
+ def printed_size(count: int) -> int:
+ Evaluation.objects.filter(team=self.team).delete()
+ evaluations = Evaluation.objects.bulk_create(
+ Evaluation(
+ team=self.team,
+ name=f"Score {index}",
+ evaluation_type="hog",
+ output_type="numeric",
+ output_config={"passing_rule": {"operator": "gte", "threshold": index}},
+ )
+ for index in range(count)
+ )
+ ids = [str(evaluation.id) for evaluation in evaluations]
+ with patch(
+ "products.ai_observability.backend.logic.evaluations.overview_queries.execute_hogql_query"
+ ) as execute:
+ execute.return_value.results = []
+ load_bucket_counts(self.team, None, PERIOD, ids, build_pass_rules(self.team.pk, ids))
+ context = HogQLContext(
+ team_id=self.team.pk,
+ team=self.team,
+ enable_select_queries=True,
+ database=Database.create_for_posthog_tables(self.team),
+ )
+ sql, _ = prepare_and_print_ast(execute.call_args.kwargs["query"], context, "clickhouse")
+ return len(sql)
+
+ assert printed_size(300) < printed_size(10) * 1.1
+
+ @time_machine.travel(datetime(2026, 3, 10, 12, 30, tzinfo=UTC), tick=False)
+ def test_a_day_or_shorter_buckets_by_hour_against_the_hours_before(self) -> None:
+ judge = self._judge()
+ for timestamp in (
+ "2026-03-08T12:00:00Z", # before the previous 24 hours
+ "2026-03-08T13:00:00Z", # previous 24 hours
+ "2026-03-09T12:00:00Z", # previous 24 hours, in the hour the current period starts
+ "2026-03-09T12:45:00Z",
+ "2026-03-09T13:10:00Z",
+ "2026-03-09T13:50:00Z",
+ "2026-03-10T12:10:00Z",
+ ):
+ self._run(judge, timestamp, **{"$ai_evaluation_result": True})
+ flush_persons_and_events()
+ now = datetime(2026, 3, 10, 12, 30, tzinfo=UTC)
+
+ overview = get_evaluations_overview(
+ self.team, None, [str(judge.id)], PeriodBounds(start=now - timedelta(hours=24), end=now)
+ )
+
+ [row] = overview.evaluations
+ assert overview.interval == "hour"
+ assert overview.buckets[0] == datetime(2026, 3, 9, 12, tzinfo=UTC)
+ assert len(overview.buckets) == 25
+ assert (overview.previous_start, overview.previous_end) == (
+ now - timedelta(hours=48),
+ now - timedelta(hours=24),
+ )
+ assert row.trend.volume[:2] == [1, 2]
+ assert row.trend.volume[-1] == 1
+ assert sum(row.trend.volume) == 4
+ assert row.previous_graded_count == 2
diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py b/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py
index 70726ceb13dc..f520f35d15cd 100644
--- a/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py
+++ b/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py
@@ -4,10 +4,12 @@ from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flus
from parameterized import parameterized
+from posthog.hogql.parser import parse_expr
from posthog.hogql.query import execute_hogql_query
from products.ai_observability.backend.facade.api import get_evaluation_pass_filters
from products.ai_observability.backend.facade.evaluation_signals import evaluation_output_changed
+from products.ai_observability.backend.logic.evaluations.pass_filters import PassRules, build_pass_rules
from products.ai_observability.backend.models.evaluations import Evaluation
CATEGORY_OPTIONS = [{"key": "ok", "label": "Ok"}, {"key": "bad", "label": "Bad"}]
@@ -20,18 +22,33 @@ def _evaluation(team, output_type: str = "boolean", output_config: dict | None =
)
+def _per_evaluation_rules(team_id: int, _evaluation_ids: list[str]) -> PassRules:
+ filters = get_evaluation_pass_filters(team_id)
+ return PassRules(graded=parse_expr(filters.graded), passes=parse_expr(filters.passed))
+
+
class TestEvaluationPassFilters(ClickhouseTestMixin, BaseTest):
- def test_grades_each_kind_of_run_by_its_evaluation_rule(self) -> None:
+ @parameterized.expand([("per_evaluation", _per_evaluation_rules), ("by_kind_of_rule", build_pass_rules)])
+ def test_grades_each_kind_of_run_by_its_evaluation_rule(
+ self, _name: str, build: Callable[[int, list[str]], PassRules]
+ ) -> None:
plain = _evaluation(self.team)
detector = _evaluation(self.team, output_config={"true_is_failure": True})
deleted_detector = _evaluation(self.team, output_config={"true_is_failure": True}, deleted=True)
numeric = _evaluation(self.team, "numeric", {"passing_rule": {"operator": "gte", "threshold": 4}})
+ numeric_lower = _evaluation(self.team, "numeric", {"passing_rule": {"operator": "gte", "threshold": 2}})
+ numeric_at_most = _evaluation(self.team, "numeric", {"passing_rule": {"operator": "lte", "threshold": 4}})
numeric_without_rule = _evaluation(self.team, "numeric", {})
categorical = _evaluation(
self.team,
"categorical",
{"options": CATEGORY_OPTIONS, "selection_mode": "single", "passing_rule": {"categories": ["ok"]}},
)
+ bad_passes = _evaluation(
+ self.team,
+ "categorical",
+ {"options": CATEGORY_OPTIONS, "selection_mode": "multiple", "passing_rule": {"categories": ["bad"]}},
+ )
empty_passes = _evaluation(
self.team,
"categorical",
@@ -49,6 +66,19 @@ class TestEvaluationPassFilters(ClickhouseTestMixin, BaseTest):
"deleted detector false": (deleted_detector, {"$ai_evaluation_result": False}, (True, True)),
"numeric at threshold": (numeric, {"$ai_evaluation_numeric_result": 4}, (True, True)),
"numeric below threshold": (numeric, {"$ai_evaluation_numeric_result": 3.5}, (True, False)),
+ "lower threshold below the other": (numeric_lower, {"$ai_evaluation_numeric_result": 3.5}, (True, True)),
+ "at most, above": (numeric_at_most, {"$ai_evaluation_numeric_result": 4.5}, (True, False)),
+ "at most, at threshold": (numeric_at_most, {"$ai_evaluation_numeric_result": 4}, (True, True)),
+ "numeric n/a": (
+ numeric,
+ {"$ai_evaluation_numeric_result": 5, "$ai_evaluation_applicable": False},
+ (False, False),
+ ),
+ "numeric skipped": (
+ numeric,
+ {"$ai_evaluation_numeric_result": 5, "$ai_evaluation_skipped": True},
+ (False, False),
+ ),
"numeric without rule": (numeric_without_rule, {"$ai_evaluation_numeric_result": 5}, (False, False)),
"categorical passing key": (
categorical,
@@ -60,6 +90,16 @@ class TestEvaluationPassFilters(ClickhouseTestMixin, BaseTest):
{"$ai_evaluation_categorical_result": ["bad"], "$ai_evaluation_applicable": True},
(True, False),
),
+ "other passing key": (
+ bad_passes,
+ {"$ai_evaluation_categorical_result": ["bad"], "$ai_evaluation_applicable": True},
+ (True, True),
+ ),
+ "other passing key, both chosen": (
+ bad_passes,
+ {"$ai_evaluation_categorical_result": ["ok", "bad"], "$ai_evaluation_applicable": True},
+ (True, False),
+ ),
"empty passes, empty": (
empty_passes,
{"$ai_evaluation_categorical_result": [], "$ai_evaluation_applicable": True},
@@ -91,19 +131,20 @@ class TestEvaluationPassFilters(ClickhouseTestMixin, BaseTest):
)
flush_persons_and_events()
- filters = get_evaluation_pass_filters(self.team.id)
+ rules = build(self.team.id, list({str(evaluation.id) for evaluation, _, _ in cases.values()}))
response = execute_hogql_query(
- f"""
+ """
SELECT
JSONExtractString(properties, 'case'),
- countIf({filters.graded}) > 0,
- countIf({filters.passed}) > 0,
- countIf({filters.graded} AND NOT {filters.passed}) > 0
+ countIf({graded}) > 0,
+ countIf({graded} AND {passes}) > 0,
+ countIf({graded} AND NOT {passes}) > 0
FROM events
WHERE event = '$ai_evaluation'
GROUP BY 1
""",
team=self.team,
+ placeholders={"graded": rules.graded, "passes": rules.passes},
)
graded_passed_failed = {
diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_runs_and_scores.py b/products/ai_observability/backend/tests/logic/evaluations/test_runs_and_scores.py
new file mode 100644
index 000000000000..7e69f8971d0b
--- /dev/null
+++ b/products/ai_observability/backend/tests/logic/evaluations/test_runs_and_scores.py
@@ -0,0 +1,114 @@
+from datetime import UTC, datetime
+
+import pytest
+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
+
+from parameterized import parameterized
+
+from products.ai_observability.backend.facade import contracts
+from products.ai_observability.backend.facade.api import (
+ NoPassingRuleError,
+ NoScoreScaleError,
+ get_evaluation_runs,
+ get_evaluation_score_distribution,
+)
+from products.ai_observability.backend.logic.evaluations.period import PeriodBounds
+from products.ai_observability.backend.logic.evaluations.score_distribution import score_buckets
+from products.ai_observability.backend.models.evaluations import Evaluation
+
+PERIOD = PeriodBounds(start=datetime(2026, 3, 8, tzinfo=UTC), end=datetime(2026, 3, 11, tzinfo=UTC))
+
+
+ [
+ ("small integer scale", 1, 5, None, ("1", "2", "3", "4", "5")),
+ ("zero to one", 0, 1, None, tuple(f"{index / 10:g}-{(index + 1) / 10:g}" for index in range(10))),
+ ("whole step on a short scale", 0, 1, 1, ("0", "1")),
+ ("fractional step", 1, 5, 0.5, tuple(f"{1 + index * 0.4:g}-{1 + (index + 1) * 0.4:g}" for index in range(10))),
+ ("wide integer scale", 0, 100, None, tuple(f"{index * 10}-{(index + 1) * 10}" for index in range(10))),
+ ]
+)
+def test_score_buckets(_name: str, low: float, high: float, step: float | None, labels: tuple[str, ...]) -> None:
+ buckets = score_buckets(contracts.ScoreScale(min=low, max=high), step)
+ assert buckets is not None and buckets.labels == labels
+
+
+class TestEvaluationRunsAndScores(ClickhouseTestMixin, BaseTest):
+ def _evaluation(self, output_type: str, output_config: dict) -> Evaluation:
+ return Evaluation.objects.create(
+ team=self.team,
+ name=output_type,
+ evaluation_type="hog",
+ evaluation_config={"source": "return true"},
+ output_type=output_type,
+ output_config=output_config,
+ )
+
+ def _run(self, evaluation: Evaluation, timestamp: str, **properties: object) -> None:
+ _create_event(
+ team=self.team,
+ event="$ai_evaluation",
+ distinct_id="user-1",
+ timestamp=timestamp,
+ properties={"$ai_evaluation_id": str(evaluation.id), **properties},
+ )
+
+ def test_runs_and_score_distribution(self) -> None:
+ judge = self._evaluation("boolean", {})
+ relevance = self._evaluation(
+ "numeric", {"min": 1, "max": 5, "passing_rule": {"operator": "gte", "threshold": 4}}
+ )
+ fraction = self._evaluation("numeric", {"min": 0, "max": 1})
+ for hour in range(7):
+ self._run(
+ judge,
+ f"2026-03-09T1{hour}:00:00Z",
+ **{
+ "$ai_evaluation_result": False,
+ "$ai_evaluation_reasoning": "x" * 2000 if hour == 6 else f"reason {hour}",
+ "$ai_target_event_id": f"generation-{hour}",
+ "$ai_trace_id": f"trace-{hour}",
+ },
+ )
+ self._run(judge, "2026-03-09T20:00:00Z", **{"$ai_evaluation_result": True, "$ai_trace_id": "trace-passed"})
+ self._run(judge, "2026-03-09T21:00:00Z", **{"$ai_evaluation_result": False, "$ai_evaluation_skipped": True})
+ self._run(judge, "2026-03-01T10:00:00Z", **{"$ai_evaluation_result": False})
+ numeric = {"$ai_evaluation_result_type": "numeric"}
+ for score in (1, 4, 4, 5, 4.4):
+ self._run(relevance, "2026-03-09T10:00:00Z", **numeric, **{"$ai_evaluation_numeric_result": score})
+ # Float division puts 0.3, 0.6 and 0.7 just under their bucket's lower edge.
+ for score in (0.0, 0.3, 0.6, 0.7, 1.0):
+ self._run(fraction, "2026-03-09T10:00:00Z", **numeric, **{"$ai_evaluation_numeric_result": score})
+ flush_persons_and_events()
+
+ failures = get_evaluation_runs(self.team, None, str(judge.id), PERIOD, "failed", 5).results
+ assert [run.trace_id for run in failures] == ["trace-6", "trace-5", "trace-4", "trace-3", "trace-2"]
+ assert {run.outcome for run in failures} == {"failed"}
+ assert failures[0].reasoning == "x" * 999 + "…"
+ assert failures[1].target_event_id == "generation-5"
+ passed = get_evaluation_runs(self.team, None, str(judge.id), PERIOD, "passed", 5).results
+ assert [(run.trace_id, run.outcome) for run in passed] == [("trace-passed", "passed")]
+ # Without an outcome, the skipped run at 21:00 is left out.
+ graded = get_evaluation_runs(self.team, None, str(judge.id), PERIOD, None, 2).results
+ assert [run.outcome for run in graded] == ["passed", "failed"]
+ relevance_failures = get_evaluation_runs(self.team, None, str(relevance.id), PERIOD, "failed", 5).results
+ assert [run.timestamp.hour for run in relevance_failures] == [10]
+ # An evaluation without a passing rule still lists its scored runs, with no outcome.
+ scored = get_evaluation_runs(self.team, None, str(fraction.id), PERIOD, None, 50).results
+ assert len(scored) == 5 and {run.outcome for run in scored} == {None}
+ with pytest.raises(NoPassingRuleError):
+ get_evaluation_runs(self.team, None, str(fraction.id), PERIOD, "failed", 5)
+
+ assert get_evaluation_score_distribution(
+ self.team, None, str(relevance.id), PERIOD
+ ) == contracts.ScoreHistogram(
+ scale=contracts.ScoreScale(min=1, max=5),
+ buckets=[
+ contracts.ScoreBucket(label=str(score), count=count)
+ for score, count in zip(range(1, 6), [1, 0, 0, 3, 1])
+ ],
+ )
+ histogram = get_evaluation_score_distribution(self.team, None, str(fraction.id), PERIOD)
+ assert [bucket.count for bucket in histogram.buckets] == [1, 0, 0, 1, 0, 0, 1, 1, 0, 1]
+ with pytest.raises(NoScoreScaleError):
+ get_evaluation_score_distribution(self.team, None, str(judge.id), PERIOD)
diff --git a/products/ai_observability/frontend/evaluations/constants.ts b/products/ai_observability/frontend/evaluations/constants.ts
index 2544eaeef2f9..227e31315595 100644
--- a/products/ai_observability/frontend/evaluations/constants.ts
+++ b/products/ai_observability/frontend/evaluations/constants.ts
@@ -16,7 +16,7 @@ const EVALUATION_RESULT_FALSE_HOGQL = "properties.$ai_evaluation_result = 'false
// A skipped run was never graded, but an evaluation that disallows N/A still emits result=false
// alongside skipped=true, so any count that doesn't exclude skips reads them as failures. Mirrors
-// _NOT_SKIPPED_PREDICATE in products/ai_observability/backend/models/evaluation_outcomes.py, which
+// NOT_SKIPPED_PREDICATE in products/ai_observability/backend/models/evaluation_outcomes.py, which
// the report metrics already apply — without this the two surfaces disagree about the same runs.
export const EVALUATION_NOT_SKIPPED_HOGQL =
"(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')"
diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.ts
index bd2e7ca5d821..5a6edcfd7226 100644
--- a/products/ai_observability/frontend/generated/api.schemas.ts
+++ b/products/ai_observability/frontend/generated/api.schemas.ts
@@ -2613,6 +2613,234 @@ export interface PatchedEvaluationApi {
readonly user_access_level?: string | null
}
+/**
+ * Null when the evaluation has no passing rule, or the run was not graded by it.
+ */
+export type EvaluationRunApiOutcome = (typeof EvaluationRunApiOutcome)[keyof typeof EvaluationRunApiOutcome] | null
+
+export const EvaluationRunApiOutcome = {
+ Passed: 'passed',
+ Failed: 'failed',
+} as const
+
+export interface EvaluationRunApi {
+ timestamp: string
+ /** Null when the evaluation has no passing rule, or the run was not graded by it. */
+ outcome: EvaluationRunApiOutcome
+ /** The evaluator's reasoning, cut to about 1,000 characters. */
+ reasoning: string | null
+ /** The generation the run graded. Null for trace and session runs. */
+ targetEventId: string | null
+ traceId: string | null
+}
+
+export interface EvaluationRunsApi {
+ /** Newest first. */
+ results: EvaluationRunApi[]
+}
+
+export interface ScoreScaleApi {
+ min: number
+ max: number
+}
+
+export interface ScoreBucketApi {
+ label: string
+ count: number
+}
+
+export interface ScoreHistogramApi {
+ scale: ScoreScaleApi
+ /** One bucket per integer on a small integer scale, else ten equal buckets across the scale. */
+ buckets: ScoreBucketApi[]
+}
+
+export type EvaluationsOverviewIntervalEnumApi =
+ (typeof EvaluationsOverviewIntervalEnumApi)[keyof typeof EvaluationsOverviewIntervalEnumApi]
+
+export const EvaluationsOverviewIntervalEnumApi = {
+ Hour: 'hour',
+ Day: 'day',
+} as const
+
+export interface PassFailCountsApi {
+ passed: number
+ failed: number
+}
+
+export interface ScoreDistributionApi {
+ mean: number
+ /** The configured score bounds, or null when the evaluation sets none. */
+ scale: ScoreScaleApi | null
+}
+
+export interface PassRateSummaryApi {
+ kind: 'pass_rate'
+ passed: number
+ failed: number
+ /** Counts in the previous period, or null when it graded too few to compare. */
+ previous: PassFailCountsApi | null
+ /** The score of numeric evaluations with a passing rule. */
+ score: ScoreDistributionApi | null
+}
+
+export interface ScoreSummaryApi {
+ kind: 'score'
+ /** Null when the current period scored nothing. */
+ score: ScoreDistributionApi | null
+ previousMean: number | null
+}
+
+export interface CategoryCountApi {
+ key: string
+ label: string
+ count: number
+ /** Percentage of the previous period's graded runs with this category, or null when it graded too few to compare. */
+ previousShare: number | null
+}
+
+export interface CategoriesSummaryApi {
+ kind: 'categories'
+ multiple: boolean
+ /** Sorted by count, most frequent first. */
+ categories: CategoryCountApi[]
+ /** Multiple-selection runs that returned no category. */
+ noCategoriesCount: number
+}
+
+export interface SentimentSummaryApi {
+ kind: 'sentiment'
+ positive: number
+ neutral: number
+ negative: number
+ previousNegativeShare: number | null
+}
+
+export interface EvaluationResultTrendApi {
+ /** Headline value per bucket, aligned with the overview's `buckets`. Null where nothing was graded. */
+ values: (number | null)[]
+ /** Graded runs per bucket. */
+ volume: number[]
+}
+
+export type ResultChangeUnitEnumApi = (typeof ResultChangeUnitEnumApi)[keyof typeof ResultChangeUnitEnumApi]
+
+export const ResultChangeUnitEnumApi = {
+ Pp: 'pp',
+ Points: 'points',
+} as const
+
+export interface ResultChangeApi {
+ /** Signed difference against the previous period, rounded for display. */
+ value: number
+ unit: ResultChangeUnitEnumApi
+ /** What changed when it is not the headline, such as a category label. */
+ subject: string | null
+}
+
+export type AttentionReasonsEnumApi = (typeof AttentionReasonsEnumApi)[keyof typeof AttentionReasonsEnumApi]
+
+export const AttentionReasonsEnumApi = {
+ Errored: 'errored',
+ Silent: 'silent',
+ Worse: 'worse',
+} as const
+
+export type BucketStatusEnumApi = (typeof BucketStatusEnumApi)[keyof typeof BucketStatusEnumApi]
+
+export const BucketStatusEnumApi = {
+ None: 'none',
+ Dip: 'dip',
+ Normal: 'normal',
+} as const
+
+export interface EvaluationResultApi {
+ id: string
+ name: string
+ directoryName: string | null
+ evaluationType: string
+ outputType: string
+ enabled: boolean
+ /** Why the evaluation stopped with an error, as an evaluation status reason, or null when it has none. */
+ statusReason: string | null
+ createdBy: string | null
+ gradedCount: number
+ /** Graded runs in the previous period, or null when none. */
+ previousGradedCount: number | null
+ notApplicableCount: number
+ /** Runs skipped before scoring. */
+ skippedCount: number
+ /** The latest graded run in the current or previous period. */
+ lastRunAt: string | null
+ summary: PassRateSummaryApi | ScoreSummaryApi | CategoriesSummaryApi | SentimentSummaryApi
+ trend: EvaluationResultTrendApi
+ /** Change against the previous period, or null when not comparable. */
+ change: ResultChangeApi | null
+ lowEvidence: boolean
+ attentionReasons: AttentionReasonsEnumApi[]
+ /** Each bucket against the previous period. A bucket with too few graded runs to judge is never a dip. */
+ bucketStatus: BucketStatusEnumApi[]
+ /** Key of the category a categorical evaluation's trend follows. */
+ trackedCategory: string | null
+}
+
+/**
+ * The same figure for both periods, over only the evaluations with results in both.
+ */
+export interface PeriodComparisonApi {
+ current: number
+ previous: number
+}
+
+export interface OverviewSentimentApi {
+ evaluationId: string
+ negativeShare: number
+ previousShare: number | null
+}
+
+export interface OverviewTotalsApi {
+ graded: number
+ /** Graded runs in each period, over evaluations with results in both. Null when none has. */
+ gradedComparison: PeriodComparisonApi | null
+ gradedTrend: number[]
+ sentiment: OverviewSentimentApi | null
+}
+
+export interface PassRateOverviewEntryApi {
+ evaluationId: string
+ rate: number
+ lowEvidence: boolean
+}
+
+export interface PassRateOverviewApi {
+ /** Enabled pass rate evaluations, lowest rate first. */
+ entries: PassRateOverviewEntryApi[]
+ /** Median of the per-evaluation pass rates without low evidence. */
+ median: number | null
+ /** Median pass rate in each period, over evaluations with results in both. Null when none has. */
+ medianComparison: PeriodComparisonApi | null
+ /** Median pass rate per bucket, aligned with `buckets`. */
+ medianTrend: (number | null)[]
+}
+
+export interface EvaluationsOverviewApi {
+ /** Start of the period, in the project's timezone. */
+ start: string
+ /** End of the period, exclusive. */
+ end: string
+ previousStart: string
+ /** End of the previous period. While the current period runs, the previous one stops at the same point, so both cover the same length of time. */
+ previousEnd: string
+ /** `hour` for a period of two days or less, else `day`. */
+ interval: EvaluationsOverviewIntervalEnumApi
+ /** Start of each bucket, in the project's timezone. Every series is aligned with it. The first bucket can start before `start`. */
+ buckets: string[]
+ evaluations: EvaluationResultApi[]
+ /** Totals over the enabled evaluations. */
+ totals: OverviewTotalsApi
+ passRateOverview: PassRateOverviewApi
+}
+
export type TestHogRequestApiOutputConfigOptionsItem = {
/**
* Stable category key.
@@ -5427,6 +5655,77 @@ export type EvaluationsBackfillsListParams = {
offset?: number
}
+export type EvaluationsRunsRetrieveParams = {
+ /**
+ * Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. Defaults to 13 days before `date_to`.
+ */
+ date_from?: string
+ /**
+ * End of the period: a date includes that whole day, a datetime is an exclusive instant. Defaults to today.
+ */
+ date_to?: string
+ /**
+ * How many runs to return, newest first.
+ * @minimum 1
+ * @maximum 50
+ */
+ limit?: number
+ /**
+ * Only runs with this outcome under the evaluation's passing rule. Without it, every graded run, including scored, categorized and sentiment runs. An evaluation without a passing rule rejects this filter with a 400.
+ *
+ * * `passed` - Passed
+ * * `failed` - Failed
+ * @minLength 1
+ */
+ outcome?: EvaluationsRunsRetrieveOutcome
+ /**
+ * Skip the result cached for the last five minutes and compute it again.
+ */
+ refresh?: boolean
+}
+
+export type EvaluationsRunsRetrieveOutcome =
+ (typeof EvaluationsRunsRetrieveOutcome)[keyof typeof EvaluationsRunsRetrieveOutcome]
+
+export const EvaluationsRunsRetrieveOutcome = {
+ Passed: 'passed',
+ Failed: 'failed',
+} as const
+
+export type EvaluationsScoreDistributionRetrieveParams = {
+ /**
+ * Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. Defaults to 13 days before `date_to`.
+ */
+ date_from?: string
+ /**
+ * End of the period: a date includes that whole day, a datetime is an exclusive instant. Defaults to today.
+ */
+ date_to?: string
+ /**
+ * Skip the result cached for the last five minutes and compute it again.
+ */
+ refresh?: boolean
+}
+
+export type EvaluationsOverviewRetrieveParams = {
+ /**
+ * Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. Defaults to 13 days before `date_to`.
+ */
+ date_from?: string
+ /**
+ * End of the period: a date includes that whole day, a datetime is an exclusive instant. Defaults to today.
+ */
+ date_to?: string
+ /**
+ * Only these evaluations. Defaults to every evaluation the user can see.
+ */
+ evaluation_ids?: string[]
+ /**
+ * Skip the result cached for the last five minutes and compute it again.
+ */
+ refresh?: boolean
+}
+
export type LlmAnalyticsClusteringJobsListParams = {
/**
* Number of results to return per page.
diff --git a/products/ai_observability/frontend/generated/api.ts b/products/ai_observability/frontend/generated/api.ts
index bf3d0381929c..1575dcf51300 100644
--- a/products/ai_observability/frontend/generated/api.ts
+++ b/products/ai_observability/frontend/generated/api.ts
@@ -49,8 +49,13 @@ import type {
EvaluationReportUpdateApi,
EvaluationRunRequestApi,
EvaluationRunResponseApi,
+ EvaluationRunsApi,
EvaluationsBackfillsListParams,
EvaluationsListParams,
+ EvaluationsOverviewApi,
+ EvaluationsOverviewRetrieveParams,
+ EvaluationsRunsRetrieveParams,
+ EvaluationsScoreDistributionRetrieveParams,
ExperimentReceiptApi,
ExperimentSubmissionApi,
InstrumentationCheckActionApi,
@@ -131,6 +136,7 @@ import type {
ScoreDefinitionNewVersionApi,
ScoreDefinitionVersionApi,
ScoreDefinitionVersionPageApi,
+ ScoreHistogramApi,
SummarizeRequestApi,
SummarizeResponseApi,
TaggerApi,
@@ -1464,6 +1470,106 @@ export const evaluationsDestroy = async (projectId: string, id: string, options?
})
}
+export const getEvaluationsRunsRetrieveUrl = (
+ projectId: string,
+ id: string,
+ params?: EvaluationsRunsRetrieveParams
+) => {
+ const normalizedParams = new URLSearchParams()
+
+ Object.entries(params || {}).forEach(([key, value]) => {
+ if (value !== undefined) {
+ normalizedParams.append(key, value === null ? 'null' : String(value))
+ }
+ })
+
+ const stringifiedParams = normalizedParams.toString()
+
+ return stringifiedParams.length > 0
+ ? `/api/projects/${projectId}/evaluations/${id}/runs/?${stringifiedParams}`
+ : `/api/projects/${projectId}/evaluations/${id}/runs/`
+}
+
+/**
+ * One evaluation's latest graded runs in a period, newest first, optionally only the passed or failed ones.
+ */
+export const evaluationsRunsRetrieve = async (
+ projectId: string,
+ id: string,
+ params?: EvaluationsRunsRetrieveParams,
+ options?: RequestInit
+): Promise<EvaluationRunsApi> => {
+ return apiMutator<EvaluationRunsApi>(getEvaluationsRunsRetrieveUrl(projectId, id, params), {
+ ...options,
+ method: 'GET',
+ })
+}
+
+export const getEvaluationsScoreDistributionRetrieveUrl = (
+ projectId: string,
+ id: string,
+ params?: EvaluationsScoreDistributionRetrieveParams
+) => {
+ const normalizedParams = new URLSearchParams()
+
+ Object.entries(params || {}).forEach(([key, value]) => {
+ if (value !== undefined) {
+ normalizedParams.append(key, value === null ? 'null' : String(value))
+ }
+ })
+
+ const stringifiedParams = normalizedParams.toString()
+
+ return stringifiedParams.length > 0
+ ? `/api/projects/${projectId}/evaluations/${id}/score_distribution/?${stringifiedParams}`
+ : `/api/projects/${projectId}/evaluations/${id}/score_distribution/`
+}
+
+/**
+ * How one numeric evaluation's scores in a period spread across its configured scale.
+ */
+export const evaluationsScoreDistributionRetrieve = async (
+ projectId: string,
+ id: string,
+ params?: EvaluationsScoreDistributionRetrieveParams,
+ options?: RequestInit
+): Promise<ScoreHistogramApi> => {
+ return apiMutator<ScoreHistogramApi>(getEvaluationsScoreDistributionRetrieveUrl(projectId, id, params), {
+ ...options,
+ method: 'GET',
+ })
+}
+
+export const getEvaluationsOverviewRetrieveUrl = (projectId: string, params?: EvaluationsOverviewRetrieveParams) => {
+ const normalizedParams = new URLSearchParams()
+
+ Object.entries(params || {}).forEach(([key, value]) => {
+ if (value !== undefined) {
+ normalizedParams.append(key, value === null ? 'null' : String(value))
+ }
+ })
+
+ const stringifiedParams = normalizedParams.toString()
+
+ return stringifiedParams.length > 0
+ ? `/api/projects/${projectId}/evaluations/overview/?${stringifiedParams}`
+ : `/api/projects/${projectId}/evaluations/overview/`
+}
+
+/**
+ * Each evaluation's results over a period, compared with the period of equal length before it.
+ */
+export const evaluationsOverviewRetrieve = async (
+ projectId: string,
+ params?: EvaluationsOverviewRetrieveParams,
+ options?: RequestInit
+): Promise<EvaluationsOverviewApi> => {
+ return apiMutator<EvaluationsOverviewApi>(getEvaluationsOverviewRetrieveUrl(projectId, params), {
+ ...options,
+ method: 'GET',
+ })
+}
+
export const getEvaluationsTestHogCreateUrl = (projectId: string) => {
return `/api/projects/${projectId}/evaluations/test_hog/`
}
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 14fb2b14d8eb..2f246e41adf8 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -13328,6 +13328,15 @@ export namespace Schemas {
limit: number;
}
+ export type AttentionReasonsEnum = typeof AttentionReasonsEnum[keyof typeof AttentionReasonsEnum];
+
+
+ export const AttentionReasonsEnum = {
+ Errored: 'errored',
+ Silent: 'silent',
+ Worse: 'worse',
+ } as const;
+
export interface AttributeBreakdownRow {
count: number;
error_count: number;
@@ -19188,6 +19197,15 @@ export namespace Schemas {
limit: number;
}
+ export type BucketStatusEnum = typeof BucketStatusEnum[keyof typeof BucketStatusEnum];
+
+
+ export const BucketStatusEnum = {
+ None: 'none',
+ Dip: 'dip',
+ Normal: 'normal',
+ } as const;
+
/**
* * `distinct_id` - User ID (default)
* * `device_id` - Device ID
@@ -22254,6 +22272,23 @@ export namespace Schemas {
passing_rule?: CategoricalScorePassingRule | null;
}
+ export interface CategoryCount {
+ key: string;
+ label: string;
+ count: number;
+ /** Percentage of the previous period's graded runs with this category, or null when it graded too few to compare. */
+ previousShare: number | null;
+ }
+
+ export interface CategoriesSummary {
+ kind: 'categories';
+ multiple: boolean;
+ /** Sorted by count, most frequent first. */
+ categories: CategoryCount[];
+ /** Multiple-selection runs that returned no category. */
+ noCategoriesCount: number;
+ }
+
export interface CdcEnableResponse {
/** Whether CDC was enabled on the source. */
success: boolean;
@@ -43995,6 +44030,122 @@ export namespace Schemas {
readonly created_at: string;
}
+ export interface PassFailCounts {
+ passed: number;
+ failed: number;
+ }
+
+ export interface ScoreScale {
+ min: number;
+ max: number;
+ }
+
+ export interface ScoreDistribution {
+ mean: number;
+ /** The configured score bounds, or null when the evaluation sets none. */
+ scale: ScoreScale | null;
+ }
+
+ export interface PassRateSummary {
+ kind: 'pass_rate';
+ passed: number;
+ failed: number;
+ /** Counts in the previous period, or null when it graded too few to compare. */
+ previous: PassFailCounts | null;
+ /** The score of numeric evaluations with a passing rule. */
+ score: ScoreDistribution | null;
+ }
+
+ export interface ScoreSummary {
+ kind: 'score';
+ /** Null when the current period scored nothing. */
+ score: ScoreDistribution | null;
+ previousMean: number | null;
+ }
+
+ export interface SentimentSummary {
+ kind: 'sentiment';
+ positive: number;
+ neutral: number;
+ negative: number;
+ previousNegativeShare: number | null;
+ }
+
+ export interface EvaluationResultTrend {
+ /** Headline value per bucket, aligned with the overview's `buckets`. Null where nothing was graded. */
+ values: (number | null)[];
+ /** Graded runs per bucket. */
+ volume: number[];
+ }
+
+ export type ResultChangeUnitEnum = typeof ResultChangeUnitEnum[keyof typeof ResultChangeUnitEnum];
+
+
+ export const ResultChangeUnitEnum = {
+ Pp: 'pp',
+ Points: 'points',
+ } as const;
+
+ export interface ResultChange {
+ /** Signed difference against the previous period, rounded for display. */
+ value: number;
+ unit: ResultChangeUnitEnum;
+ /** What changed when it is not the headline, such as a category label. */
+ subject: string | null;
+ }
+
+ export interface EvaluationResult {
+ id: string;
+ name: string;
+ directoryName: string | null;
+ evaluationType: string;
+ outputType: string;
+ enabled: boolean;
+ /** Why the evaluation stopped with an error, as an evaluation status reason, or null when it has none. */
+ statusReason: string | null;
+ createdBy: string | null;
+ gradedCount: number;
+ /** Graded runs in the previous period, or null when none. */
+ previousGradedCount: number | null;
+ notApplicableCount: number;
+ /** Runs skipped before scoring. */
+ skippedCount: number;
+ /** The latest graded run in the current or previous period. */
+ lastRunAt: string | null;
+ summary: PassRateSummary | ScoreSummary | CategoriesSummary | SentimentSummary;
+ trend: EvaluationResultTrend;
+ /** Change against the previous period, or null when not comparable. */
+ change: ResultChange | null;
+ lowEvidence: boolean;
+ attentionReasons: AttentionReasonsEnum[];
+ /** Each bucket against the previous period. A bucket with too few graded runs to judge is never a dip. */
+ bucketStatus: BucketStatusEnum[];
+ /** Key of the category a categorical evaluation's trend follows. */
+ trackedCategory: string | null;
+ }
+
+ /**
+ * Null when the evaluation has no passing rule, or the run was not graded by it.
+ */
+ export type EvaluationRunOutcome = typeof EvaluationRunOutcome[keyof typeof EvaluationRunOutcome] | null;
+
+
+ export const EvaluationRunOutcome = {
+ Passed: 'passed',
+ Failed: 'failed',
+ } as const;
+
+ export interface EvaluationRun {
+ timestamp: string;
+ /** Null when the evaluation has no passing rule, or the run was not graded by it. */
+ outcome: EvaluationRunOutcome;
+ /** The evaluator's reasoning, cut to about 1,000 characters. */
+ reasoning: string | null;
+ /** The generation the run graded. Null for trace and session runs. */
+ targetEventId: string | null;
+ traceId: string | null;
+ }
+
export interface EvaluationRunEvaluation {
/** UUID of the evaluation being run. */
id: string;
@@ -44029,6 +44180,76 @@ export namespace Schemas {
target_event_id: string;
}
+ export interface EvaluationRuns {
+ /** Newest first. */
+ results: EvaluationRun[];
+ }
+
+ export type EvaluationsOverviewIntervalEnum = typeof EvaluationsOverviewIntervalEnum[keyof typeof EvaluationsOverviewIntervalEnum];
+
+
+ export const EvaluationsOverviewIntervalEnum = {
+ Hour: 'hour',
+ Day: 'day',
+ } as const;
+
+ /**
+ * The same figure for both periods, over only the evaluations with results in both.
+ */
+ export interface PeriodComparison {
+ current: number;
+ previous: number;
+ }
+
+ export interface OverviewSentiment {
+ evaluationId: string;
+ negativeShare: number;
+ previousShare: number | null;
+ }
+
+ export interface OverviewTotals {
+ graded: number;
+ /** Graded runs in each period, over evaluations with results in both. Null when none has. */
+ gradedComparison: PeriodComparison | null;
+ gradedTrend: number[];
+ sentiment: OverviewSentiment | null;
+ }
+
+ export interface PassRateOverviewEntry {
+ evaluationId: string;
+ rate: number;
+ lowEvidence: boolean;
+ }
+
+ export interface PassRateOverview {
+ /** Enabled pass rate evaluations, lowest rate first. */
+ entries: PassRateOverviewEntry[];
+ /** Median of the per-evaluation pass rates without low evidence. */
+ median: number | null;
+ /** Median pass rate in each period, over evaluations with results in both. Null when none has. */
+ medianComparison: PeriodComparison | null;
+ /** Median pass rate per bucket, aligned with `buckets`. */
+ medianTrend: (number | null)[];
+ }
+
+ export interface EvaluationsOverview {
+ /** Start of the period, in the project's timezone. */
+ start: string;
+ /** End of the period, exclusive. */
+ end: string;
+ previousStart: string;
+ /** End of the previous period. While the current period runs, the previous one stops at the same point, so both cover the same length of time. */
+ previousEnd: string;
+ /** `hour` for a period of two days or less, else `day`. */
+ interval: EvaluationsOverviewIntervalEnum;
+ /** Start of each bucket, in the project's timezone. Every series is aligned with it. The first bucket can start before `start`. */
+ buckets: string[];
+ evaluations: EvaluationResult[];
+ /** Totals over the enabled evaluations. */
+ totals: OverviewTotals;
+ passRateOverview: PassRateOverview;
+ }
+
export interface EventDefinitionBasic {
id: string;
name: string;
@@ -103505,6 +103726,11 @@ export namespace Schemas {
effective_destination_ids?: string[];
}
+ export interface ScoreBucket {
+ label: string;
+ count: number;
+ }
+
export interface ScoreDefinitionCreate {
/**
* Tag names. Replaces existing tags; send [] to remove all tags. Names are trimmed and lowercased.
@@ -103597,6 +103823,12 @@ export namespace Schemas {
results: ScoreDefinitionVersion[];
}
+ export interface ScoreHistogram {
+ scale: ScoreScale;
+ /** One bucket per integer on a small integer scale, else ten equal buckets across the scale. */
+ buckets: ScoreBucket[];
+ }
+
export interface ScoringActivateRequest {
/** Saved scoring version to activate for subsequent evaluations. */
config_id: string;
@@ -129486,6 +129718,77 @@ export namespace Schemas {
offset?: number;
};
+ export type EvaluationsRunsRetrieveParams = {
+ /**
+ * Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. Defaults to 13 days before `date_to`.
+ */
+ date_from?: string;
+ /**
+ * End of the period: a date includes that whole day, a datetime is an exclusive instant. Defaults to today.
+ */
+ date_to?: string;
+ /**
+ * How many runs to return, newest first.
+ * @minimum 1
+ * @maximum 50
+ */
+ limit?: number;
+ /**
+ * Only runs with this outcome under the evaluation's passing rule. Without it, every graded run, including scored, categorized and sentiment runs. An evaluation without a passing rule rejects this filter with a 400.
+ *
+ * * `passed` - Passed
+ * * `failed` - Failed
+ * @minLength 1
+ */
+ outcome?: EvaluationsRunsRetrieveOutcome;
+ /**
+ * Skip the result cached for the last five minutes and compute it again.
+ */
+ refresh?: boolean;
+ };
+
+ export type EvaluationsRunsRetrieveOutcome = typeof EvaluationsRunsRetrieveOutcome[keyof typeof EvaluationsRunsRetrieveOutcome];
+
+
+ export const EvaluationsRunsRetrieveOutcome = {
+ Passed: 'passed',
+ Failed: 'failed',
+ } as const;
+
+ export type EvaluationsScoreDistributionRetrieveParams = {
+ /**
+ * Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. Defaults to 13 days before `date_to`.
+ */
+ date_from?: string;
+ /**
+ * End of the period: a date includes that whole day, a datetime is an exclusive instant. Defaults to today.
+ */
+ date_to?: string;
+ /**
+ * Skip the result cached for the last five minutes and compute it again.
+ */
+ refresh?: boolean;
+ };
+
+ export type EvaluationsOverviewRetrieveParams = {
+ /**
+ * Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. Defaults to 13 days before `date_to`.
+ */
+ date_from?: string;
+ /**
+ * End of the period: a date includes that whole day, a datetime is an exclusive instant. Defaults to today.
+ */
+ date_to?: string;
+ /**
+ * Only these evaluations. Defaults to every evaluation the user can see.
+ */
+ evaluation_ids?: string[];
+ /**
+ * Skip the result cached for the last five minutes and compute it again.
+ */
+ refresh?: boolean;
+ };
+
export type EventDefinitionsListParams = {
/**
* `event_custom` keeps only names without a `$` prefix and `event_posthog` only names with one. Default `event`.