:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): serve the online evals overview from python (#122069)
Bernat Torresdd50c85ae481
diff --git a/.github/new-events-schema-targets.txt b/.github/new-events-schema-targets.txtindex d1ab46afd897..8a532f1bf37a 100644--- a/.github/new-events-schema-targets.txt+++ b/.github/new-events-schema-targets.txt@@ -75,6 +75,7 @@ posthog/api/test/test_query.pyposthog/models/test/test_async_deletion_model.pyposthog/models/web_preaggregated/test_web_pre_aggregated_timezones.pyposthog/tasks/test/test_stop_surveys_reached_target.py+products/ai_observability/backend/api/test/test_evaluations.pyproducts/ai_observability/backend/api/test/test_personal_spend.pyproducts/revenue_analytics/backend/views/test/test_mrr_views.pyproducts/surveys/backend/api/test/test_survey.pydiff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.pyindex d8b8742743fa..e6a11e200495 100644--- a/products/ai_observability/backend/api/evaluations.py+++ b/products/ai_observability/backend/api/evaluations.py@@ -1,7 +1,10 @@+import hashlibfrom collections import Counter-from collections.abc import Mapping, Sequence+from collections.abc import Callable, Mapping, Sequence+from datetime import UTC, date, datetimefrom typing import TYPE_CHECKING, Any, cast+from django.core.cache import cachefrom django.db import models, transactionfrom django.db.models import Q, QuerySetfrom django.http import Http404@@ -9,7 +12,8 @@ from django.http import Http404import structlogimport django_filtersfrom django_filters.rest_framework import DjangoFilterBackend-from drf_spectacular.utils import extend_schema, extend_schema_field+from drf_spectacular.types import OpenApiTypes+from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_fieldfrom rest_framework import exceptions, serializers, viewsetsfrom rest_framework.decorators import actionfrom rest_framework.exceptions import NotFound@@ -20,7 +24,9 @@ from rest_framework.response import Responsefrom posthog.hogql import ast+from posthog.api.fields import RepeatedOrCommaSeparatedListFieldfrom posthog.api.forbid_destroy_model import ForbidDestroyModel+from posthog.api.mixins import ValidatedRequest, validated_requestfrom posthog.api.monitoring import monitorfrom posthog.api.routing import TeamAndOrgViewSetMixinfrom posthog.api.scoped_related_fields import TeamScopedPrimaryKeyRelatedField@@ -30,13 +36,15 @@ from posthog.event_usage import report_user_actionfrom posthog.hogql_queries.ai.ai_table_resolver import AIEventsUnavailableError, query_ai_eventsfrom posthog.hogql_queries.ai.utils import HEAVY_COLUMN_NAMES, merge_heavy_propertiesfrom posthog.models.team import Team-from posthog.permissions import AccessControlPermission+from posthog.permissions import AccessControlPermission, is_service_authfrom posthog.temporal.ai_observability.message_utils import extract_text_from_messagesfrom posthog.temporal.ai_observability.model_resolution import active_key_fallbackfrom posthog.temporal.ai_observability.run_evaluation import extract_event_io, run_hog_evalfrom posthog.temporal.ai_observability.run_session_evaluation import run_hog_eval_over_recent_sessionsfrom posthog.temporal.ai_observability.run_trace_evaluation import run_hog_eval_over_recent_traces+from products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_team+from products.access_control.backend.facade.property_access import restriction_fingerprintfrom products.access_control.backend.presentation.access_control import (AccessControlViewSetMixin,UserAccessControlSerializerMixin,@@ -44,6 +52,11 @@ from products.access_control.backend.presentation.access_control import (from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definitionfrom ..evaluation_conditions import build_condition_filter+from ..facade import (+ api as facade,+ contracts,+)+from ..facade.contracts import Contractfrom ..hog import compile_ai_observability_hogfrom ..llm import DEFAULT_MODEL_BY_PROVIDERfrom ..llm.decisions import decision_evaluations_enabled, is_decision_model@@ -82,12 +95,15 @@ from ..models.evaluation_reports import EvaluationReportfrom ..models.evaluations import Evaluation, EvaluationTargetfrom ..models.model_configuration import LLMModelConfigurationfrom ..models.provider_keys import LLMProvider, LLMProviderKey+from ..presentation.query_concurrency import query_concurrency_slotsfrom .condition_sets import ConditionSetSerializerfrom .metrics import llma_track_latencyif TYPE_CHECKING:from posthog.models import User+ from ..logic.evaluations.period import PeriodBounds+logger = structlog.get_logger(__name__)@@ -998,6 +1014,71 @@ class HogEvaluationOutputType(models.TextChoices):CATEGORICAL = OutputType.CATEGORICAL.value, OutputType.CATEGORICAL.label+OVERVIEW_CACHE_SECONDS = 300+++@extend_schema_field(OpenApiTypes.STR)+class _DateOrDateTimeField(serializers.Field):+ """A `YYYY-MM-DD` date for whole days, or an ISO 8601 datetime for an exact instant."""++ def to_internal_value(self, data: Any) -> date | datetime:+ if not isinstance(data, str):+ self.fail("invalid")+ try:+ return date.fromisoformat(data) if len(data) == 10 else datetime.fromisoformat(data)+ except ValueError:+ self.fail("invalid")++ def to_representation(self, value: date | datetime) -> str:+ return value.isoformat()++ default_error_messages = {"invalid": "Enter a date (YYYY-MM-DD) or an ISO 8601 datetime."}+++class EvaluationsOverviewPeriodSerializer(serializers.Serializer):+ date_from = _DateOrDateTimeField(+ required=False,+ help_text="Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. "+ "Defaults to 13 days before `date_to`.",+ )+ date_to = _DateOrDateTimeField(+ required=False,+ help_text="End of the period: a date includes that whole day, a datetime is an exclusive instant. "+ "Defaults to today.",+ )+ refresh = serializers.BooleanField(+ required=False,+ default=False,+ help_text="Skip the result cached for the last five minutes and compute it again.",+ )+++class EvaluationsOverviewQuerySerializer(EvaluationsOverviewPeriodSerializer):+ evaluation_ids = RepeatedOrCommaSeparatedListField(+ child=serializers.UUIDField(),+ required=False,+ help_text="Only these evaluations. Defaults to every evaluation the user can see.",+ )+++class EvaluationRunOutcome(models.TextChoices):+ PASSED = "passed", "Passed"+ FAILED = "failed", "Failed"+++class EvaluationRunsQuerySerializer(EvaluationsOverviewPeriodSerializer):+ outcome = serializers.ChoiceField(+ choices=EvaluationRunOutcome.choices,+ required=False,+ help_text="Only runs with this outcome under the evaluation's passing rule. Without it, every graded run, "+ "including scored, categorized and sentiment runs. An evaluation without a passing rule rejects this filter "+ "with a 400.",+ )+ limit = serializers.IntegerField(+ required=False, default=5, min_value=1, max_value=50, help_text="How many runs to return, newest first."+ )++class TestHogRequestSerializer(serializers.Serializer):output_type = serializers.ChoiceField(choices=HogEvaluationOutputType.choices,@@ -1507,6 +1588,133 @@ class EvaluationViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, Forbidef partial_update(self, request, *args, **kwargs):return super().partial_update(request, *args, **kwargs)+ def _overview_period(self, request: ValidatedRequest) -> "PeriodBounds":+ query = request.validated_query_data+ try:+ return facade.evaluations_period(self.team, query.get("date_from"), query.get("date_to"))+ except facade.InvalidPeriodError as error:+ raise serializers.ValidationError({error.field: str(error)} if error.field else str(error))++ def _cached_overview_read(+ self,+ request: ValidatedRequest,+ name: str,+ evaluation_ids: Sequence[str],+ period: "PeriodBounds",+ read: Callable[[], Contract],+ ) -> Response:+ # Keyed by the evaluations the user can see and by their current property restrictions, because a hit skips+ # the masking a fresh read applies, and a restriction can change while an entry is still cached.+ visible = hashlib.sha256(",".join(sorted(evaluation_ids)).encode()).hexdigest()+ restrictions = restriction_fingerprint(+ get_restricted_properties_with_group_type_index_for_team(+ user=cast("User", request.user), team_id=self.team.pk+ )+ )+ cache_key = (+ f"ai_evaluations_{name}/{self.team.pk}/{request.user.pk}/{period.start.astimezone(UTC).isoformat()}/{period.end.astimezone(UTC).isoformat()}"+ f"/{visible}/{restrictions}"+ )+ if not request.validated_query_data["refresh"]:+ cached = cache.get(cache_key)+ if cached is not None:+ return Response(cached)+ tag_queries(product=Product.LLM_ANALYTICS, feature=Feature.QUERY)+ with query_concurrency_slots(self.team):+ payload = read().model_dump(mode="json", by_alias=True)+ cache.set(cache_key, payload, timeout=OVERVIEW_CACHE_SECONDS)+ return Response(payload)++ @validated_request(+ query_serializer=EvaluationsOverviewQuerySerializer,+ responses={+ 200: OpenApiResponse(response=contracts.EvaluationsOverview),+ 400: OpenApiResponse(description="The period is malformed, empty, longer than 90 days or out of range."),+ 429: OpenApiResponse(description="Too many queries are running for the project or organization."),+ },+ )+ @action(detail=False, methods=["get"], url_path="overview", required_scopes=["evaluation:read"])+ def overview(self, request: ValidatedRequest, **kwargs) -> Response:+ """Each evaluation's results over a period, compared with the period of equal length before it."""+ period = self._overview_period(request)+ queryset = self.get_queryset()+ # The base viewset narrows by access level on `list` only, so this read applies it itself.+ if not is_service_auth(request):+ queryset = self.user_access_control.filter_queryset_by_access_level(queryset)+ query = request.validated_query_data+ if "evaluation_ids" in query:+ queryset = queryset.filter(id__in=query["evaluation_ids"])+ evaluation_ids = [str(evaluation_id) for evaluation_id in queryset.values_list("id", flat=True)]+ return self._cached_overview_read(+ request,+ "overview",+ evaluation_ids,+ period,+ lambda: facade.get_evaluations_overview(self.team, cast("User", request.user), evaluation_ids, period),+ )++ @validated_request(+ query_serializer=EvaluationRunsQuerySerializer,+ responses={+ 200: OpenApiResponse(response=contracts.EvaluationRuns),+ 400: OpenApiResponse(+ description="The period is invalid, or an outcome filter is set on an evaluation without a passing rule."+ ),+ 429: OpenApiResponse(description="Too many queries are running for the project or organization."),+ },+ )+ @action(detail=True, methods=["get"], url_path="runs", required_scopes=["evaluation:read"])+ def runs(self, request: ValidatedRequest, **kwargs) -> Response:+ """One evaluation's latest graded runs in a period, newest first, optionally only the passed or failed ones."""+ evaluation_id = str(self.get_object().id)+ period = self._overview_period(request)+ query = request.validated_query_data+ outcome, limit = query.get("outcome"), query["limit"]+ try:+ return self._cached_overview_read(+ request,+ f"runs/{outcome}/{limit}",+ [evaluation_id],+ period,+ lambda: facade.get_evaluation_runs(+ self.team, cast("User", request.user), evaluation_id, period, outcome, limit+ ),+ )+ except facade.NoPassingRuleError:+ raise serializers.ValidationError(+ {"outcome": "This evaluation has no passing rule, so its runs are neither passed nor failed."}+ )++ @validated_request(+ query_serializer=EvaluationsOverviewPeriodSerializer,+ responses={+ 200: OpenApiResponse(response=contracts.ScoreHistogram),+ 400: OpenApiResponse(+ description="The period is invalid, or the evaluation is not numeric with a minimum and maximum score."+ ),+ 429: OpenApiResponse(description="Too many queries are running for the project or organization."),+ },+ )+ @action(detail=True, methods=["get"], url_path="score_distribution", required_scopes=["evaluation:read"])+ def score_distribution(self, request: ValidatedRequest, **kwargs) -> Response:+ """How one numeric evaluation's scores in a period spread across its configured scale."""+ evaluation_id = str(self.get_object().id)+ period = self._overview_period(request)+ try:+ return self._cached_overview_read(+ request,+ "score_distribution",+ [evaluation_id],+ period,+ lambda: facade.get_evaluation_score_distribution(+ self.team, cast("User", request.user), evaluation_id, period+ ),+ )+ except facade.NoScoreScaleError:+ raise serializers.ValidationError(+ "Only a numeric evaluation with a minimum and a maximum score has a score distribution."+ )+@extend_schema(request=TestHogRequestSerializer, responses=TestHogResponseSerializer)@action(detail=False, methods=["post"], url_path="test_hog", required_scopes=["evaluation:read"])def test_hog(self, request: Request, **kwargs) -> Response:diff --git a/products/ai_observability/backend/api/test/test_evaluations.py b/products/ai_observability/backend/api/test/test_evaluations.pyindex 2464809e4cd5..8f0504a2cb45 100644--- a/products/ai_observability/backend/api/test/test_evaluations.py+++ b/products/ai_observability/backend/api/test/test_evaluations.py@@ -2,7 +2,7 @@ import jsonfrom datetime import timedeltafrom uuid import UUID, uuid4-from posthog.test.base import APIBaseTest+from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_eventsfrom unittest.mock import Mock, patchfrom django.db import IntegrityError, connection, transaction@@ -21,6 +21,7 @@ from posthog.models import Organization, OrganizationMembership, Project, Team,from products.access_control.backend.models.access_control import AccessControlfrom products.ai_observability.backend.api.evaluations import (EvaluationSerializer,+ EvaluationsOverviewPeriodSerializer,ModelConfigurationSerializer,TestHogRequestSerializer as HogRequestSerializer,_TargetConfigField,@@ -2550,7 +2551,7 @@ class TestReEnableValidatesRootCauseResolved(APIBaseTest):self.assertIsNone(eval_obj.status_reason)-class TestEvaluationsAccessControl(APIBaseTest):+class _EvaluationAccessControlCase(APIBaseTest):# Evaluations have their own access control resource (see ACCESS_CONTROL_RESOURCES in# products/access_control/backend/facade/user_access_control.py). They used to inherit from `llm_analytics`.def setUp(self) -> None:@@ -2560,13 +2561,7 @@ class TestEvaluationsAccessControl(APIBaseTest):{"key": AvailableFeature.ROLE_BASED_ACCESS, "name": AvailableFeature.ROLE_BASED_ACCESS},]self.organization.save()-- self.viewer_user = User.objects.create_and_join(self.organization, "[email protected]", "testtest")- self.editor_user = User.objects.create_and_join(self.organization, "[email protected]", "testtest")self.no_access_user = User.objects.create_and_join(self.organization, "[email protected]", "testtest")-- self._grant(self.viewer_user, "viewer")- self._grant(self.editor_user, "editor")self._grant(self.no_access_user, "none")def _grant(@@ -2581,16 +2576,27 @@ class TestEvaluationsAccessControl(APIBaseTest):organization_member=membership,)- def _create_evaluation(self, name: str = "Existing Evaluation") -> Evaluation:+ def _create_evaluation(self, name: str = "Existing Evaluation", enabled: bool = False) -> Evaluation:return Evaluation.objects.create(team=self.team,name=name,+ enabled=enabled,evaluation_type="hog",evaluation_config={"source": "return true"},output_type="boolean",created_by=self.user,)++class TestEvaluationsAccessControl(_EvaluationAccessControlCase):+ def setUp(self) -> None:+ super().setUp()+ self.viewer_user = User.objects.create_and_join(self.organization, "[email protected]", "testtest")+ self.editor_user = User.objects.create_and_join(self.organization, "[email protected]", "testtest")++ self._grant(self.viewer_user, "viewer")+ self._grant(self.editor_user, "editor")+def _create_payload(self, name: str = "New Evaluation") -> dict:return {"name": name,@@ -2720,3 +2726,91 @@ class TestEvaluationsAccessControl(APIBaseTest):response = self.client.get(f"/api/projects/{self.team.id}/llm_analytics/models/?provider=openai")assert response.status_code == status.HTTP_200_OK+++class TestEvaluationsOverviewAccessControl(ClickhouseTestMixin, _EvaluationAccessControlCase):+ def test_overview_hides_evaluations_the_user_cannot_see(self) -> None:+ visible = self._create_evaluation("Visible", enabled=True)+ hidden = self._create_evaluation("Hidden", enabled=True)+ self._grant(self.no_access_user, "viewer", resource_id=str(visible.id))+ for evaluation in (visible, hidden, hidden):+ _create_event(+ team=self.team,+ event="$ai_evaluation",+ distinct_id="user-1",+ timestamp="2026-03-09T10:00:00Z",+ properties={"$ai_evaluation_id": str(evaluation.id), "$ai_evaluation_result": True},+ )+ flush_persons_and_events()+ url = (+ f"/api/projects/{self.team.id}/evaluations/overview/?date_from=2026-03-08&date_to=2026-03-10"+ f"&evaluation_ids={visible.id},{hidden.id}"+ )+ # A user who sees both evaluations fills the cache first, so a shared entry would leak their totals.+ assert self.client.get(url).json()["totals"]["graded"] == 3+ self.client.force_login(self.no_access_user)++ response = self.client.get(url)++ assert response.status_code == status.HTTP_200_OK+ assert [row["name"] for row in response.json()["evaluations"]] == ["Visible"]+ assert response.json()["totals"]["graded"] == 1++ @parameterized.expand([("runs",), ("score_distribution",)])+ def test_evaluation_reads_need_access_to_the_evaluation(self, endpoint: str) -> None:+ hidden = self._create_evaluation("Hidden")+ self.client.force_login(self.no_access_user)++ response = self.client.get(f"/api/projects/{self.team.id}/evaluations/{hidden.id}/{endpoint}/")++ assert response.status_code in (status.HTTP_403_FORBIDDEN, status.HTTP_404_NOT_FOUND)++ @parameterized.expand(+ [+ ("period", "{id}/runs/?date_from=2026-03-10&date_to=2026-03-09", "date_from", "Must be before date_to."),+ (+ "period out of range",+ "overview/?date_to=9999-12-31",+ None,+ "The period is outside the supported date range.",+ ),+ (+ "outcome without a passing rule",+ "{id}/runs/?outcome=failed",+ "outcome",+ "This evaluation has no passing rule, so its runs are neither passed nor failed.",+ ),+ (+ "score distribution without bounds",+ "{id}/score_distribution/",+ None,+ "Only a numeric evaluation with a minimum and a maximum score has a score distribution.",+ ),+ ]+ )+ def test_invalid_reads_return_400(self, _name: str, path: str, attr: str | None, detail: str) -> None:+ evaluation = Evaluation.objects.create(+ team=self.team,+ name="Score",+ evaluation_type="hog",+ evaluation_config={"source": "return true"},+ output_type="numeric",+ output_config={},+ )++ response = self.client.get(f"/api/projects/{self.team.id}/evaluations/{path.format(id=evaluation.id)}")++ assert response.status_code == status.HTTP_400_BAD_REQUEST+ assert (response.json()["attr"], response.json()["detail"]) == (attr, detail)+++class TestEvaluationsOverviewPeriod(SimpleTestCase):+ @parameterized.expand(+ [+ ("date", {"date_from": "2026-03-10"}, True),+ ("instant with offset", {"date_to": "2026-03-10T05:30:00+05:30"}, True),+ ("not a date", {"date_from": "2026-13-01"}, False),+ ]+ )+ def test_period_parsing(self, _name: str, data: dict[str, str], valid: bool) -> None:+ assert EvaluationsOverviewPeriodSerializer(data=data).is_valid() is validdiff --git a/products/ai_observability/backend/facade/api.py b/products/ai_observability/backend/facade/api.pyindex faa371330d0a..578b0a0aa9c7 100644--- a/products/ai_observability/backend/facade/api.py+++ b/products/ai_observability/backend/facade/api.py@@ -1,8 +1,19 @@-from datetime import datetime+from collections.abc import Sequence+from datetime import date, datetimefrom posthog.models import Team, User-from ..logic.evaluations.pass_filters import build_evaluation_pass_filters+from ..logic.evaluations.evaluation_metas import load_evaluation_meta, load_evaluation_metas+from ..logic.evaluations.overview import evaluation_result, group_by_evaluation, overview_totals, pass_rate_overview+from ..logic.evaluations.overview_queries import load_bucket_counts, load_category_counts+from ..logic.evaluations.pass_filters import build_evaluation_pass_filters, build_pass_rules+from ..logic.evaluations.period import (+ InvalidPeriodError as InvalidPeriodError,+ PeriodBounds,+ overview_period,+)+from ..logic.evaluations.runs import RunOutcome, load_runs+from ..logic.evaluations.score_distribution import load_score_distribution, score_bucketsfrom ..logic.traces.evaluation_result import EvaluationResultfrom ..logic.traces.evaluation_result_queries import load_evaluation_results, load_output_configsfrom ..logic.traces.events_fallback import fallback_window@@ -23,6 +34,14 @@ class TraceNodeNotFoundError(Exception):pass+class NoPassingRuleError(Exception):+ pass+++class NoScoreScaleError(Exception):+ pass++_NO_METADATA = contracts.TraceNodeMetadata(provider=None, temperature=None, session_id=None, prompt_name=None, prompt_version=None)@@ -186,3 +205,70 @@ def _to_evaluation_result(result: EvaluationResult) -> contracts.TraceEvaluationtimestamp=result.timestamp,is_backfill=result.is_backfill,)+++def evaluations_period(team: Team, date_from: date | datetime | None, date_to: date | datetime | None) -> PeriodBounds:+ """Raises `InvalidPeriodError` for a period that is empty, longer than 90 days or out of range."""+ return PeriodBounds.from_request(date_from, date_to, team.timezone_info)+++def get_evaluations_overview(+ team: Team, user: User | None, evaluation_ids: Sequence[str], period: PeriodBounds+) -> contracts.EvaluationsOverview:+ """Each evaluation's results in the period, compared with the period of equal length before it, plus the+ totals the overview cards show. Two ClickHouse reads, unless the evaluations group into more rows than half a+ page holds.++ While the period still runs, the previous one stops at the same point, so a partial today is not compared with+ a full day."""+ overview = overview_period(period.start, period.end, datetime.now(team.timezone_info), team.timezone_info)+ metas = load_evaluation_metas(team.pk, evaluation_ids)+ ids = [meta.id for meta in metas]+ counts = group_by_evaluation(load_bucket_counts(team, user, overview, ids, build_pass_rules(team.pk, ids)))+ categories = group_by_evaluation(+ load_category_counts(+ team, user, overview, {meta.id: len(meta.categories) for meta in metas if meta.kind == "categories"}+ )+ )+ rows = [+ evaluation_result(meta, overview.buckets, counts.get(meta.id, []), categories.get(meta.id, []))+ for meta in metas+ ]+ bucket_count = len(overview.buckets)+ return contracts.EvaluationsOverview(+ start=overview.start,+ end=overview.end,+ previous_start=overview.previous_start,+ previous_end=overview.previous_end,+ interval=overview.interval,+ buckets=list(overview.buckets),+ evaluations=rows,+ totals=overview_totals(rows, bucket_count),+ pass_rate_overview=pass_rate_overview(rows, bucket_count),+ )+++def get_evaluation_runs(+ team: Team,+ user: User | None,+ evaluation_id: str,+ period: PeriodBounds,+ outcome: RunOutcome | None,+ limit: int,+) -> contracts.EvaluationRuns:+ """Raises `NoPassingRuleError` for an outcome filter on an evaluation without a passing rule."""+ if outcome is not None and load_evaluation_meta(team.pk, evaluation_id).kind != "pass_rate":+ raise NoPassingRuleError(evaluation_id)+ pass_rules = build_pass_rules(team.pk, [evaluation_id])+ return contracts.EvaluationRuns(results=load_runs(team, user, evaluation_id, period, pass_rules, outcome, limit))+++def get_evaluation_score_distribution(+ team: Team, user: User | None, evaluation_id: str, period: PeriodBounds+) -> contracts.ScoreHistogram:+ """Raises `NoScoreScaleError` unless the evaluation is numeric with a minimum below its maximum."""+ meta = load_evaluation_meta(team.pk, evaluation_id)+ buckets = score_buckets(meta.scale, meta.score_step) if meta.scale is not None else None+ if meta.scale is None or buckets is None:+ raise NoScoreScaleError(evaluation_id)+ return load_score_distribution(team, user, evaluation_id, period, meta.scale, buckets)diff --git a/products/ai_observability/backend/facade/contracts.py b/products/ai_observability/backend/facade/contracts.pyindex d7eac52720d5..3103e4335a35 100644--- a/products/ai_observability/backend/facade/contracts.py+++ b/products/ai_observability/backend/facade/contracts.py@@ -1,5 +1,5 @@from datetime import datetime-from typing import Any, Literal, get_args+from typing import Annotated, Any, Literal, get_argsfrom pydantic import BaseModel, ConfigDict, Fieldfrom pydantic.alias_generators import to_camel@@ -130,3 +130,195 @@ class TraceEvaluationResult(Contract):class TraceEvaluationResults(Contract):results: list[TraceEvaluationResult]+++AttentionReason = Literal["errored", "silent", "worse"]+BucketStatus = Literal["none", "dip", "normal"]+Interval = Literal["hour", "day"]+ChangeUnit = Literal["pp", "points"]+++class PassFailCounts(Contract):+ passed: int+ failed: int+++class ScoreScale(Contract):+ min: float+ max: float+++class ScoreDistribution(Contract):+ mean: float+ scale: ScoreScale | None = Field(description="The configured score bounds, or null when the evaluation sets none.")+++class PassRateSummary(Contract):+ kind: Literal["pass_rate"]+ passed: int+ failed: int+ previous: PassFailCounts | None = Field(+ description="Counts in the previous period, or null when it graded too few to compare."+ )+ score: ScoreDistribution | None = Field(description="The score of numeric evaluations with a passing rule.")+++class ScoreSummary(Contract):+ kind: Literal["score"]+ score: ScoreDistribution | None = Field(description="Null when the current period scored nothing.")+ previous_mean: float | None+++class CategoryCount(Contract):+ key: str+ label: str+ count: int+ previous_share: float | None = Field(+ description=(+ "Percentage of the previous period's graded runs with this category, or null when it graded too few to compare."+ )+ )+++class CategoriesSummary(Contract):+ kind: Literal["categories"]+ multiple: bool+ categories: list[CategoryCount] = Field(description="Sorted by count, most frequent first.")+ no_categories_count: int = Field(description="Multiple-selection runs that returned no category.")+++class SentimentSummary(Contract):+ kind: Literal["sentiment"]+ positive: int+ neutral: int+ negative: int+ previous_negative_share: float | None+++ResultSummary = Annotated[+ PassRateSummary | ScoreSummary | CategoriesSummary | SentimentSummary, Field(discriminator="kind")+]+++class ResultChange(Contract):+ value: float = Field(description="Signed difference against the previous period, rounded for display.")+ unit: ChangeUnit+ subject: str | None = Field(description="What changed when it is not the headline, such as a category label.")+++class EvaluationResultTrend(Contract):+ values: list[float | None] = Field(+ description="Headline value per bucket, aligned with the overview's `buckets`. Null where nothing was graded."+ )+ volume: list[int] = Field(description="Graded runs per bucket.")+++class EvaluationResult(Contract):+ id: str+ name: str+ directory_name: str | None+ evaluation_type: str+ output_type: str+ enabled: bool+ status_reason: str | None = Field(+ description="Why the evaluation stopped with an error, as an evaluation status reason, or null when it has none."+ )+ created_by: str | None+ graded_count: int+ previous_graded_count: int | None = Field(description="Graded runs in the previous period, or null when none.")+ not_applicable_count: int+ skipped_count: int = Field(description="Runs skipped before scoring.")+ last_run_at: datetime | None = Field(description="The latest graded run in the current or previous period.")+ summary: ResultSummary+ trend: EvaluationResultTrend+ change: ResultChange | None = Field(description="Change against the previous period, or null when not comparable.")+ low_evidence: bool+ attention_reasons: list[AttentionReason]+ bucket_status: list[BucketStatus] = Field(+ description="Each bucket against the previous period. A bucket with too few graded runs to judge is never a dip."+ )+ tracked_category: str | None = Field(description="Key of the category a categorical evaluation's trend follows.")+++class PeriodComparison(Contract):+ """The same figure for both periods, over only the evaluations with results in both."""++ current: float+ previous: float+++class PassRateOverviewEntry(Contract):+ evaluation_id: str+ rate: float+ low_evidence: bool+++class PassRateOverview(Contract):+ entries: list[PassRateOverviewEntry] = Field(description="Enabled pass rate evaluations, lowest rate first.")+ median: float | None = Field(description="Median of the per-evaluation pass rates without low evidence.")+ median_comparison: PeriodComparison | None = Field(+ description="Median pass rate in each period, over evaluations with results in both. Null when none has."+ )+ median_trend: list[float | None] = Field(description="Median pass rate per bucket, aligned with `buckets`.")+++class OverviewSentiment(Contract):+ evaluation_id: str+ negative_share: float+ previous_share: float | None+++class OverviewTotals(Contract):+ graded: int+ graded_comparison: PeriodComparison | None = Field(+ description="Graded runs in each period, over evaluations with results in both. Null when none has."+ )+ graded_trend: list[int]+ sentiment: OverviewSentiment | None+++class OverviewPeriodFields(Contract):+ start: datetime = Field(description="Start of the period, in the project's timezone.")+ end: datetime = Field(description="End of the period, exclusive.")+++class EvaluationsOverview(OverviewPeriodFields):+ previous_start: datetime+ previous_end: datetime = Field(+ description="End of the previous period. While the current period runs, the previous one stops at the same "+ "point, so both cover the same length of time."+ )+ interval: Interval = Field(description="`hour` for a period of two days or less, else `day`.")+ buckets: list[datetime] = Field(+ description="Start of each bucket, in the project's timezone. Every series is aligned with it. The first "+ "bucket can start before `start`."+ )+ evaluations: list[EvaluationResult]+ totals: OverviewTotals = Field(description="Totals over the enabled evaluations.")+ pass_rate_overview: PassRateOverview+++class EvaluationRun(Contract):+ timestamp: datetime+ outcome: Literal["passed", "failed"] | None = Field(+ description="Null when the evaluation has no passing rule, or the run was not graded by it."+ )+ reasoning: str | None = Field(description="The evaluator's reasoning, cut to about 1,000 characters.")+ target_event_id: str | None = Field(description="The generation the run graded. Null for trace and session runs.")+ trace_id: str | None+++class EvaluationRuns(Contract):+ results: list[EvaluationRun] = Field(description="Newest first.")+++class ScoreBucket(Contract):+ label: str+ count: int+++class ScoreHistogram(Contract):+ scale: ScoreScale+ buckets: list[ScoreBucket] = Field(+ description="One bucket per integer on a small integer scale, else ten equal buckets across the scale."+ )diff --git a/products/ai_observability/backend/logic/evaluations/evaluation_metas.py b/products/ai_observability/backend/logic/evaluations/evaluation_metas.pynew file mode 100644index 000000000000..6658b7e47f7e--- /dev/null+++ b/products/ai_observability/backend/logic/evaluations/evaluation_metas.py@@ -0,0 +1,81 @@+from collections.abc import Sequence++from products.ai_observability.backend.models.evaluation_configs import CategoricalOutputConfig, NumericOutputConfig+from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definition+from products.ai_observability.backend.models.evaluations import Evaluation, EvaluationStatus++from ...facade import contracts+from .overview import CategoryOption, EvaluationMeta, SummaryKind+++def load_evaluation_metas(team_id: int, evaluation_ids: Sequence[str]) -> list[EvaluationMeta]:+ """Newest first. Deleted evaluations and ids outside the team are left out."""+ evaluations = (+ Evaluation.objects.filter(team_id=team_id, id__in=evaluation_ids, deleted=False)+ .select_related("directory", "created_by")+ .order_by("-created_at", "id")+ )+ return [_evaluation_meta(evaluation) for evaluation in evaluations]+++def load_evaluation_meta(team_id: int, evaluation_id: str) -> EvaluationMeta:+ return _evaluation_meta(+ Evaluation.objects.select_related("directory", "created_by").get(+ team_id=team_id, id=evaluation_id, deleted=False+ )+ )+++def _evaluation_meta(evaluation: Evaluation) -> EvaluationMeta:+ config = evaluation.output_config or {}+ try:+ has_pass_rule = get_outcome_definition(+ evaluation.output_type, true_is_failure=config.get("true_is_failure") is True, output_config=config+ ).has_pass_rule+ except ValueError:+ has_pass_rule = False+ kind: SummaryKind+ if evaluation.output_type == "sentiment":+ kind = "sentiment"+ elif has_pass_rule:+ kind = "pass_rate"+ elif evaluation.output_type == "numeric":+ kind = "score"+ else:+ kind = "categories"+ scale = None+ score_step = None+ categories: tuple[CategoryOption, ...] = ()+ multiple = False+ # A config that no longer validates still shows its counts, without the bounds or labels it would carry.+ try:+ if evaluation.output_type == "numeric":+ numeric = NumericOutputConfig.model_validate(config)+ score_step = numeric.step+ if numeric.min is not None and numeric.max is not None:+ scale = contracts.ScoreScale(min=numeric.min, max=numeric.max)+ elif evaluation.output_type == "categorical":+ categorical = CategoricalOutputConfig.model_validate(config)+ categories = tuple(CategoryOption(key=option.key, label=option.label) for option in categorical.options)+ multiple = categorical.selection_mode == "multiple"+ except ValueError:+ pass+ creator = evaluation.created_by+ return EvaluationMeta(+ id=str(evaluation.id),+ name=evaluation.name,+ directory_name=evaluation.directory.name if evaluation.directory else None,+ evaluation_type=evaluation.evaluation_type,+ output_type=evaluation.output_type,+ enabled=evaluation.enabled,+ errored=evaluation.status == EvaluationStatus.ERROR,+ status_reason=evaluation.status_reason if evaluation.status == EvaluationStatus.ERROR else None,+ created_by=(" ".join(filter(None, [creator.first_name, creator.last_name])) or creator.email)+ if creator+ else None,+ kind=kind,+ scale=scale,+ score_step=score_step,+ categories=categories,+ multiple=multiple,+ )diff --git a/products/ai_observability/backend/logic/evaluations/overview.py b/products/ai_observability/backend/logic/evaluations/overview.pynew file mode 100644index 000000000000..4e24ab1131c6--- /dev/null+++ b/products/ai_observability/backend/logic/evaluations/overview.py@@ -0,0 +1,498 @@+"""Per-evaluation results for the online evaluations overview, and the totals across them.++Pure functions over the grouped counts from `overview_queries`.+"""++import math+from bisect import bisect_right+from collections import defaultdict+from collections.abc import Iterable, Sequence+from datetime import UTC, datetime+from typing import Literal, TypeVar++from posthog.dataclasses import frozen++from ...facade import contracts+from .overview_queries import CategoryBucketCount, EvaluationBucketCounts++# Below this many graded results a summary is shown greyed, and its change is not compared. A previous period with+# fewer is not compared either.+LOW_EVIDENCE_GRADED_COUNT = 30+# A change smaller than this, in percentage points, is not called out.+NOTABLE_CHANGE_PP = 2+# A bucket this many points worse than an evaluation's previous period counts as unusual.+DIP_PP = 3+# A numeric score counts as unusual when it moves this share of its scale away from its usual value.+SCORE_SHIFT_SHARE = 0.05++SummaryKind = Literal["pass_rate", "score", "categories", "sentiment"]++_Row = TypeVar("_Row", EvaluationBucketCounts, CategoryBucketCount)+++def group_by_evaluation(rows: Iterable[_Row]) -> dict[str, list[_Row]]:+ grouped: dict[str, list[_Row]] = defaultdict(list)+ for row in rows:+ grouped[row.evaluation_id].append(row)+ return grouped+++@frozen+class CategoryOption:+ key: str+ label: str+++@frozen+class EvaluationMeta:+ id: str+ name: str+ directory_name: str | None+ evaluation_type: str+ output_type: str+ enabled: bool+ errored: bool+ status_reason: str | None+ created_by: str | None+ kind: SummaryKind+ scale: contracts.ScoreScale | None+ score_step: float | None+ categories: tuple[CategoryOption, ...]+ multiple: bool+++@frozen+class _Totals:+ passed: int+ failed: int+ not_applicable: int+ skipped: int+ score_count: int+ score_sum: float+ categorized: int+ positive: int+ neutral: int+ negative: int+++def _sum(rows: Sequence[EvaluationBucketCounts]) -> _Totals:+ return _Totals(+ passed=sum(row.passed for row in rows),+ failed=sum(row.failed for row in rows),+ not_applicable=sum(row.not_applicable for row in rows),+ skipped=sum(row.skipped for row in rows),+ score_count=sum(row.score_count for row in rows),+ score_sum=sum(row.score_sum for row in rows),+ categorized=sum(row.categorized for row in rows),+ positive=sum(row.positive for row in rows),+ neutral=sum(row.neutral for row in rows),+ negative=sum(row.negative for row in rows),+ )+++def _js_round(value: float, digits: int) -> float:+ # Math.round rounds halves up, where Python's round() rounds them to even.+ factor = 10**digits+ return math.floor(value * factor + 0.5) / factor+++def _share(part: float, whole: float) -> float:+ return part / whole * 100 if whole > 0 else 0+++def _median(values: Sequence[float]) -> float | None:+ if not values:+ return None+ ordered = sorted(values)+ middle = len(ordered) // 2+ return ordered[middle] if len(ordered) % 2 else (ordered[middle - 1] + ordered[middle]) / 2+++def pass_rate(passed: int, failed: int) -> float | None:+ graded = passed + failed+ return passed / graded * 100 if graded > 0 else None+++def _graded(kind: SummaryKind, totals: _Totals) -> int:+ match kind:+ case "pass_rate":+ return totals.passed + totals.failed+ case "score":+ return totals.score_count+ case "categories":+ return totals.categorized+ case "sentiment":+ return totals.positive + totals.neutral + totals.negative+++def _score(meta: EvaluationMeta, totals: _Totals) -> contracts.ScoreDistribution | None:+ if totals.score_count == 0:+ return None+ return contracts.ScoreDistribution(mean=totals.score_sum / totals.score_count, scale=meta.scale)+++def _category_counts(rows: Sequence[CategoryBucketCount], current: bool) -> dict[str, int]:+ counts: dict[str, int] = defaultdict(int)+ for row in rows:+ if (row.bucket is not None) == current:+ counts[row.key] += row.runs+ return counts+++def _summary(+ meta: EvaluationMeta,+ current: _Totals,+ previous: _Totals,+ categories: Sequence[CategoryBucketCount],+) -> contracts.ResultSummary:+ match meta.kind:+ case "pass_rate":+ return contracts.PassRateSummary(+ kind="pass_rate",+ passed=current.passed,+ failed=current.failed,+ previous=contracts.PassFailCounts(passed=previous.passed, failed=previous.failed)+ if previous.passed + previous.failed > 0+ else None,+ score=_score(meta, current) if meta.output_type == "numeric" else None,+ )+ case "score":+ return contracts.ScoreSummary(+ kind="score",+ score=_score(meta, current),+ previous_mean=previous.score_sum / previous.score_count if previous.score_count else None,+ )+ case "categories":+ counts = _category_counts(categories, current=True)+ previous_counts = _category_counts(categories, current=False)+ options = list(meta.categories)+ known = {option.key for option in options}+ # A category from the previous period only has dropped to zero, which can be the largest move.+ unknown = dict.fromkeys(key for key in [*counts, *previous_counts] if key and key not in known)+ options += [CategoryOption(key=key, label=key) for key in unknown]+ entries = [+ contracts.CategoryCount(+ key=option.key,+ label=option.label,+ count=counts.get(option.key, 0),+ previous_share=_share(previous_counts.get(option.key, 0), previous.categorized)+ if previous.categorized+ else None,+ )+ for option in options+ ]+ return contracts.CategoriesSummary(+ kind="categories",+ multiple=meta.multiple,+ categories=sorted(entries, key=lambda entry: -entry.count),+ no_categories_count=counts.get("", 0),+ )+ case "sentiment":+ total = previous.positive + previous.neutral + previous.negative+ return contracts.SentimentSummary(+ kind="sentiment",+ positive=current.positive,+ neutral=current.neutral,+ negative=current.negative,+ previous_negative_share=_share(previous.negative, total) if total else None,+ )+++def has_low_evidence(graded_count: int) -> bool:+ return 0 < graded_count < LOW_EVIDENCE_GRADED_COUNT+++def result_change(graded_count: int, summary: contracts.ResultSummary) -> contracts.ResultChange | None:+ """Change against the previous equal period, or None when there is nothing fair to compare."""+ if graded_count == 0 or has_low_evidence(graded_count):+ return None+ match summary:+ case contracts.PassRateSummary():+ current = pass_rate(summary.passed, summary.failed)+ previous = pass_rate(summary.previous.passed, summary.previous.failed) if summary.previous else None+ if current is None or previous is None:+ return None+ return contracts.ResultChange(value=_js_round(current - previous, 1), unit="pp", subject=None)+ case contracts.ScoreSummary():+ if summary.score is None or summary.previous_mean is None:+ return None+ return contracts.ResultChange(+ value=_js_round(summary.score.mean - summary.previous_mean, 2), unit="points", subject=None+ )+ case contracts.CategoriesSummary():+ moved = _largest_category_move(summary, graded_count)+ if moved is None:+ return None+ category, delta = moved+ return contracts.ResultChange(value=delta, unit="pp", subject=category.label)+ case contracts.SentimentSummary():+ if summary.previous_negative_share is None:+ return None+ total = summary.positive + summary.neutral + summary.negative+ return contracts.ResultChange(+ value=_js_round(_share(summary.negative, total) - summary.previous_negative_share, 1),+ unit="pp",+ subject="negative",+ )+++def _largest_category_move(+ summary: contracts.CategoriesSummary, graded_count: int+) -> tuple[contracts.CategoryCount, float] | None:+ """The category that moved most, since the top category is often the stable one."""+ largest: tuple[contracts.CategoryCount, float] | None = None+ for category in summary.categories:+ if category.previous_share is None:+ continue+ delta = _js_round(_share(category.count, graded_count) - category.previous_share, 1)+ if largest is None or abs(delta) > abs(largest[1]):+ largest = (category, delta)+ return largest+++def _tracked_category(+ summary: contracts.ResultSummary, change: contracts.ResultChange | None, graded_count: int+) -> contracts.CategoryCount | None:+ """The category a categorical evaluation's trend follows: the one that moved most, else the most common."""+ if not isinstance(summary, contracts.CategoriesSummary):+ return None+ # Labels need not be unique, so the trend follows the category itself and not its label.+ moved = _largest_category_move(summary, graded_count) if change is not None else None+ if moved is not None:+ return moved[0]+ return summary.categories[0] if summary.categories else None+++def _previous_value(summary: contracts.ResultSummary, tracked: str | None) -> float | None:+ match summary:+ case contracts.PassRateSummary():+ return pass_rate(summary.previous.passed, summary.previous.failed) if summary.previous else None+ case contracts.ScoreSummary():+ return summary.previous_mean+ case contracts.SentimentSummary():+ return summary.previous_negative_share+ case contracts.CategoriesSummary():+ return next((category.previous_share for category in summary.categories if category.key == tracked), None)+++def _bucket_index(instants: Sequence[datetime], bucket: datetime) -> int:+ """The bucket that holds a ClickHouse bucket start: the last one that starts at or before it, compared by+ instant."""+ return max(bisect_right(instants, bucket.astimezone(UTC)) - 1, 0)+++def _bucket_value(+ meta: EvaluationMeta, row: _Totals | None, tracked_key: str | None, category_runs: int+) -> float | None:+ if row is None:+ return None+ match meta.kind:+ case "pass_rate":+ return pass_rate(row.passed, row.failed)+ case "score":+ return row.score_sum / row.score_count if row.score_count else None+ case "sentiment":+ total = row.positive + row.neutral + row.negative+ return _share(row.negative, total) if total else None+ case "categories":+ return _share(category_runs, row.categorized) if row.categorized and tracked_key is not None else None+++def bucket_status(+ summary: contracts.ResultSummary, trend: contracts.EvaluationResultTrend, tracked: str | None+) -> list[contracts.BucketStatus]:+ """Status of each bucket against the previous period, or against the range's own median without one.++ Pass rates and sentiment have a worse direction. Scores and label shares have none, so a large move either way+ counts as unusual. A bucket with fewer graded runs than a summary needs is never flagged, because one run moves+ a thin bucket by more than the dip threshold, which hourly buckets would turn into noise.+ """+ previous = _previous_value(summary, tracked)+ typical = previous if previous is not None else _median([value for value in trend.values if value is not None])+ statuses: list[contracts.BucketStatus] = []+ for value, graded in zip(trend.values, trend.volume):+ if value is None:+ statuses.append("none")+ elif typical is None or graded < LOW_EVIDENCE_GRADED_COUNT:+ statuses.append("normal")+ else:+ statuses.append("dip" if _is_dip(summary, value, typical) else "normal")+ return statuses+++def _is_dip(summary: contracts.ResultSummary, value: float, typical: float) -> bool:+ match summary:+ case contracts.PassRateSummary():+ return value <= typical - DIP_PP+ case contracts.SentimentSummary():+ return value >= typical + DIP_PP+ case contracts.CategoriesSummary():+ return abs(value - typical) >= DIP_PP+ case contracts.ScoreSummary():+ # Without configured bounds, or with equal ones, there is no scale to measure a move against.+ scale = summary.score.scale if summary.score else None+ span = scale.max - scale.min if scale else 0+ return span > 0 and abs(value - typical) >= span * SCORE_SHIFT_SHARE+++def attention_reasons(+ *,+ enabled: bool,+ errored: bool,+ graded_count: int,+ summary: contracts.ResultSummary,+ change: contracts.ResultChange | None,+) -> list[contracts.AttentionReason]:+ # The error status also disables the evaluation, and that pause was not the user's choice, so it still asks for+ # attention. Its missing results follow from the error, so it is never also silent or worse.+ if errored:+ return ["errored"]+ # Pausing an evaluation is a deliberate choice, so a paused one never asks for attention.+ if not enabled:+ return []+ reasons: list[contracts.AttentionReason] = []+ if graded_count == 0:+ reasons.append("silent")+ # Only results with a clear worse direction count: a falling pass rate or a rising negative share.+ if change is not None and change.unit == "pp":+ if (isinstance(summary, contracts.PassRateSummary) and change.value <= -NOTABLE_CHANGE_PP) or (+ isinstance(summary, contracts.SentimentSummary) and change.value >= NOTABLE_CHANGE_PP+ ):+ reasons.append("worse")+ return reasons+++def evaluation_result(+ meta: EvaluationMeta,+ buckets: Sequence[datetime],+ counts: Sequence[EvaluationBucketCounts],+ categories: Sequence[CategoryBucketCount],+) -> contracts.EvaluationResult:+ """`counts` and `categories` hold only this evaluation's rows, see `group_by_evaluation`."""+ instants = [bucket.astimezone(UTC) for bucket in buckets]+ by_bucket: dict[int, list[EvaluationBucketCounts]] = defaultdict(list)+ for row in counts:+ if row.bucket is not None:+ by_bucket[_bucket_index(instants, row.bucket)].append(row)+ current = _sum([row for row in counts if row.bucket is not None])+ previous = _sum([row for row in counts if row.bucket is None])+ previous_graded = _graded(meta.kind, previous)+ # A thin previous period is no fairer a baseline than a thin current one, so the summary leaves it out.+ baseline = previous if previous_graded >= LOW_EVIDENCE_GRADED_COUNT else _sum([])+ summary = _summary(meta, current, baseline, categories)+ graded_count = _graded(meta.kind, current)+ change = result_change(graded_count, summary)+ tracked = _tracked_category(summary, change, graded_count)+ tracked_key = tracked.key if tracked else None+ tracked_runs: dict[int, int] = defaultdict(int)+ for category in categories:+ if category.bucket is not None and category.key == tracked_key:+ tracked_runs[_bucket_index(instants, category.bucket)] += category.runs+ bucket_totals = [_sum(by_bucket[index]) if index in by_bucket else None for index in range(len(buckets))]+ trend = contracts.EvaluationResultTrend(+ values=[+ _bucket_value(meta, totals, tracked_key, tracked_runs[index]) for index, totals in enumerate(bucket_totals)+ ],+ volume=[_graded(meta.kind, totals) if totals is not None else 0 for totals in bucket_totals],+ )+ last_run_at = max((row.last_run_at for row in counts if row.last_run_at is not None), default=None)+ return contracts.EvaluationResult(+ id=meta.id,+ name=meta.name,+ directory_name=meta.directory_name,+ evaluation_type=meta.evaluation_type,+ output_type=meta.output_type,+ enabled=meta.enabled,+ status_reason=meta.status_reason,+ created_by=meta.created_by,+ graded_count=graded_count,+ previous_graded_count=previous_graded or None,+ not_applicable_count=current.not_applicable,+ skipped_count=current.skipped,+ last_run_at=last_run_at,+ summary=summary,+ trend=trend,+ change=change,+ low_evidence=has_low_evidence(graded_count),+ attention_reasons=attention_reasons(+ enabled=meta.enabled,+ errored=meta.errored,+ graded_count=graded_count,+ summary=summary,+ change=change,+ ),+ bucket_status=bucket_status(summary, trend, tracked_key),+ tracked_category=tracked_key,+ )+++def pass_rate_overview(rows: Sequence[contracts.EvaluationResult], bucket_count: int) -> contracts.PassRateOverview:+ entries: list[contracts.PassRateOverviewEntry] = []+ measured: list[contracts.EvaluationResult] = []+ compared: list[tuple[float, float]] = []+ for row in rows:+ summary = row.summary+ if not isinstance(summary, contracts.PassRateSummary) or not row.enabled:+ continue+ rate = pass_rate(summary.passed, summary.failed)+ if rate is None:+ continue+ entries.append(contracts.PassRateOverviewEntry(evaluation_id=row.id, rate=rate, low_evidence=row.low_evidence))+ if row.low_evidence:+ continue+ measured.append(row)+ previous = pass_rate(summary.previous.passed, summary.previous.failed) if summary.previous else None+ if previous is not None:+ compared.append((rate, previous))+ median_trend = (+ [+ _median([value for row in measured if (value := row.trend.values[index]) is not None])+ for index in range(bucket_count)+ ]+ if measured+ else []+ )+ # Both medians cover the same evaluations, so a change between them is not a change in which ones ran.+ current_median = _median([current for current, _ in compared])+ previous_median = _median([previous for _, previous in compared])+ median_comparison = (+ contracts.PeriodComparison(current=current_median, previous=previous_median)+ if current_median is not None and previous_median is not None+ else None+ )+ return contracts.PassRateOverview(+ entries=sorted(entries, key=lambda entry: entry.rate),+ median=_median([entry.rate for entry in entries if not entry.low_evidence]),+ median_comparison=median_comparison,+ median_trend=median_trend,+ )+++def overview_totals(rows: Sequence[contracts.EvaluationResult], bucket_count: int) -> contracts.OverviewTotals:+ """Totals for the metric cards at the top of the overview."""+ active = [row for row in rows if row.enabled]+ # Only evaluations with results in both periods, so a new or newly silent evaluation does not read as a change.+ both = [row for row in active if row.graded_count > 0 and row.previous_graded_count is not None]+ sentiment = None+ for row in active:+ if isinstance(row.summary, contracts.SentimentSummary) and row.graded_count > 0:+ summary = row.summary+ sentiment = contracts.OverviewSentiment(+ evaluation_id=row.id,+ negative_share=_share(summary.negative, summary.positive + summary.neutral + summary.negative),+ previous_share=summary.previous_negative_share,+ )+ break+ return contracts.OverviewTotals(+ graded=sum(row.graded_count for row in active),+ graded_comparison=contracts.PeriodComparison(+ current=sum(row.graded_count for row in both),+ previous=sum(row.previous_graded_count or 0 for row in both),+ )+ if both+ else None,+ graded_trend=[sum(row.trend.volume[index] for row in active) for index in range(bucket_count)]+ if active+ else [],+ sentiment=sentiment,+ )diff --git a/products/ai_observability/backend/logic/evaluations/overview_queries.py b/products/ai_observability/backend/logic/evaluations/overview_queries.pynew file mode 100644index 000000000000..4a45eae34a5d--- /dev/null+++ b/products/ai_observability/backend/logic/evaluations/overview_queries.py@@ -0,0 +1,294 @@+from collections.abc import Iterator, Mapping, Sequence+from datetime import UTC, datetime+from typing import Any++from posthog.hogql import ast+from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS+from posthog.hogql.context import HogQLContext+from posthog.hogql.database.database import Database+from posthog.hogql.parser import parse_select+from posthog.hogql.query import execute_hogql_query++from posthog.clickhouse.query_tagging import Product, tag_queries, tags_context+from posthog.dataclasses import frozen+from posthog.models import Team, User++from products.ai_observability.backend.models.evaluation_outcomes import (+ CATEGORIES_EXPRESSION,+ CATEGORIZED_PREDICATE,+ NOT_SKIPPED_PREDICATE,+ NUMERIC_SCORED_PREDICATE,+ SCORE_EXPRESSION,+ SENTIMENT_LABELS,+ get_outcome_definition,+)++from .pass_filters import PassRules+from .period import OverviewPeriod, PeriodBounds++_SKIPPED = "properties.$ai_evaluation_skipped = 'true'"+_NOT_APPLICABLE = f"properties.$ai_evaluation_applicable = 'false' AND {NOT_SKIPPED_PREDICATE}"+_SENTIMENT = get_outcome_definition("sentiment")+# One column per label, in the order of SENTIMENT_LABELS, which `load_bucket_counts` reads them in.+_SENTIMENT_COUNTS = ", ".join(+ f"countIf({_SENTIMENT.outcome_predicates[label]} AND {NOT_SKIPPED_PREDICATE}) AS {label}"+ for label in SENTIMENT_LABELS+)+_ANY_LABEL = ", ".join(f"'{label}'" for label in SENTIMENT_LABELS)++# A run that a summary kind without a passing rule grades: scored, categorized or labeled.+GRADED_WITHOUT_RULE = (+ f"({NUMERIC_SCORED_PREDICATE}) OR ({CATEGORIZED_PREDICATE})"+ f" OR ({_SENTIMENT.result_expression} IN ({_ANY_LABEL}) AND {NOT_SKIPPED_PREDICATE})"+)+# A run any summary kind grades, so an evaluation whose recent runs were all N/A or skipped still reads as silent+# since its last real result.+_GRADED_ANY = f"graded_run OR {GRADED_WITHOUT_RULE}"++PAGE_SIZE = MAX_SELECT_RETURNED_ROWS+# Many evaluations over a long period can group into more rows than one response returns, for example categorical+# evaluations with many categories over 90 days. So a read splits the evaluations into batches that are expected to+# fit in half a page, and splits a batch again if it still fills a page.+BATCH_ROWS = PAGE_SIZE // 2+# A null bucket marks the previous period, which needs totals only, so it costs one row per evaluation.+_BUCKET = "if(timestamp < {current_start}, NULL, {bucket_start})"+_WHERE = (+ "event = '$ai_evaluation' AND timestamp >= {previous_start} AND timestamp < {end}"+ " AND (timestamp >= {current_start} OR timestamp < {previous_end})"+ " AND properties.$ai_evaluation_id IN {evaluation_ids}"+)++_COUNTS_QUERY = f"""+SELECT+ toString(properties.$ai_evaluation_id) AS evaluation_id,+ {_BUCKET} AS bucket,+ countIf(and(({{graded}}) AS graded_run, ({{passes}}) AS passing_run)) AS passed,+ countIf(graded_run AND NOT passing_run) AS failed,+ countIf({_NOT_APPLICABLE}) AS not_applicable,+ countIf({_SKIPPED}) AS skipped,+ countIf({NUMERIC_SCORED_PREDICATE}) AS score_count,+ sumIf({SCORE_EXPRESSION}, {NUMERIC_SCORED_PREDICATE}) AS score_sum,+ countIf({CATEGORIZED_PREDICATE}) AS categorized,+ {_SENTIMENT_COUNTS},+ if(countIf(({_GRADED_ANY}) AS graded_any) = 0, NULL, maxIf(timestamp, graded_any)) AS last_run_at+FROM events+WHERE {_WHERE}+GROUP BY evaluation_id, bucket+ORDER BY evaluation_id, bucket+LIMIT {{limit}} OFFSET {{offset}}+"""++# An empty category list becomes one '' row, so multiple-selection runs that chose nothing are counted too.+_CATEGORY_QUERY = f"""+SELECT+ toString(properties.$ai_evaluation_id) AS evaluation_id,+ {_BUCKET} AS bucket,+ arrayJoin(if(empty({CATEGORIES_EXPRESSION}), [''], {CATEGORIES_EXPRESSION})) AS category,+ count() AS runs+FROM events+WHERE {_WHERE} AND {CATEGORIZED_PREDICATE}+GROUP BY evaluation_id, bucket, category+ORDER BY evaluation_id, bucket, category+LIMIT {{limit}} OFFSET {{offset}}+"""+++@frozen+class EvaluationBucketCounts:+ evaluation_id: str+ bucket: datetime | None # None is the previous period.+ passed: int+ failed: int+ not_applicable: int+ skipped: int+ score_count: int+ score_sum: float+ categorized: int+ positive: int+ neutral: int+ negative: int+ last_run_at: datetime | None+++@frozen+class CategoryBucketCount:+ evaluation_id: str+ bucket: datetime | None # None is the previous period.+ key: str # '' for a run that chose no category.+ runs: int+++def _run(+ name: str,+ query: str,+ team: Team,+ user: User | None,+ period: OverviewPeriod,+ expected_rows: Mapping[str, int],+ extra: dict[str, ast.Expr],+) -> list[Sequence[Any]]:+ """`expected_rows` maps each evaluation to read to the most rows it is expected to group into."""+ placeholders: dict[str, ast.Expr] = {+ "previous_start": instant(period.previous_start),+ "previous_end": instant(period.previous_end),+ "current_start": instant(period.start),+ "end": instant(period.end),+ "bucket_start": _bucket_start(period, team.timezone),+ **extra,+ }++ def read(evaluation_ids: Sequence[str], offset: int) -> list[Sequence[Any]]:+ # One placeholder for the whole list, so the query text stays the same size however many evaluations it reads.+ ids = ast.Constant(value=tuple(evaluation_ids))+ limits = {"limit": ast.Constant(value=PAGE_SIZE), "offset": ast.Constant(value=offset)}+ return run_evaluation_query(name, query, team, user, {**placeholders, "evaluation_ids": ids, **limits})++ def read_all(evaluation_ids: Sequence[str]) -> list[Sequence[Any]]:+ rows = read(evaluation_ids, 0)+ if len(rows) < PAGE_SIZE:+ return rows+ if len(evaluation_ids) > 1:+ middle = len(evaluation_ids) // 2+ return read_all(evaluation_ids[:middle]) + read_all(evaluation_ids[middle:])+ # One evaluation cannot be split further, so it pages through its own rows.+ while len(page := read(evaluation_ids, len(rows))) == PAGE_SIZE:+ rows += page+ return rows + page++ return [row for batch in _batches(expected_rows) for row in read_all(batch)]+++def _bucket_start(period: OverviewPeriod, timezone: str) -> ast.Expr:+ timestamp = ast.Field(chain=["timestamp"])+ if period.interval == "hour":+ # Whole hours from the first bucket, the way `overview_period` steps its buckets. Local hours would split a+ # bucket where a daylight saving change moves the clocks by half an hour. ClickHouse needs an origin before+ # every timestamp, including the previous period's, so the origin is the first bucket's minute in 1970. It+ # counts in the timestamp's timezone, which is UTC so that past offset changes do not shift the hours.+ first = (period.buckets[0] if period.buckets else period.start).astimezone(UTC)+ origin = instant(datetime(1970, 1, 1, minute=first.minute, tzinfo=UTC))+ utc = ast.Call(name="toTimeZone", args=[timestamp, ast.Constant(value="UTC")])+ return ast.Call(+ name="toStartOfInterval",+ args=[utc, ast.Call(name="toIntervalHour", args=[ast.Constant(value=1)]), origin],+ )+ # The project's timezone is explicit, because the team's modifiers can turn off the conversion to it, and+ # `overview_period` builds its buckets in that timezone.+ local = ast.Call(name="toTimeZone", args=[timestamp, ast.Constant(value=timezone)])+ return ast.Call(name="toStartOfDay", args=[local])+++def _groups(period: OverviewPeriod) -> int:+ """The most rows one evaluation groups into per category: one per bucket, plus one for the previous period."""+ return len(period.buckets) + 1+++def _batches(expected_rows: Mapping[str, int]) -> Iterator[list[str]]:+ batch: list[str] = []+ rows = 0+ for evaluation_id, expected in expected_rows.items():+ if batch and rows + expected > BATCH_ROWS:+ yield batch+ batch, rows = [], 0+ batch.append(evaluation_id)+ rows += expected+ if batch:+ yield batch+++def instant(value: datetime) -> ast.Expr:+ """A datetime constant prints as wall time in the project's timezone, which names two instants in a repeated+ daylight saving hour. UTC wall time names exactly one."""+ return ast.Call(+ name="toDateTime64",+ args=[+ ast.Constant(value=value.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S.%f")),+ ast.Constant(value=6),+ ast.Constant(value="UTC"),+ ],+ )+++# The runs of one evaluation in a period, for the reads that list or bucket them.+EVALUATION_RUNS_WHERE = (+ "event = '$ai_evaluation' AND properties.$ai_evaluation_id = {evaluation_id}"+ " AND timestamp >= {start} AND timestamp < {end}"+)+++def evaluation_runs_placeholders(evaluation_id: str, period: PeriodBounds) -> dict[str, ast.Expr]:+ return {+ "evaluation_id": ast.Constant(value=evaluation_id),+ "start": instant(period.start),+ "end": instant(period.end),+ }+++def run_evaluation_query(+ name: str, query: str, team: Team, user: User | None, placeholders: dict[str, ast.Expr]+) -> list[Sequence[Any]]:+ with tags_context(product=Product.LLM_ANALYTICS):+ tag_queries(name=name)+ response = execute_hogql_query(+ query=parse_select(query, placeholders=placeholders),+ team=team,+ user=user,+ query_type=name,+ context=HogQLContext(+ team_id=team.pk, team=team, user=user, database=Database.create_for_posthog_tables(team)+ ),+ )+ return list(response.results or [])+++def load_bucket_counts(+ team: Team,+ user: User | None,+ period: OverviewPeriod,+ evaluation_ids: Sequence[str],+ pass_rules: PassRules,+) -> list[EvaluationBucketCounts]:+ if not evaluation_ids:+ return []+ rows = _run(+ "ai_evaluations_overview_counts",+ _COUNTS_QUERY,+ team,+ user,+ period,+ dict.fromkeys(evaluation_ids, _groups(period)),+ {"graded": pass_rules.graded, "passes": pass_rules.passes},+ )+ return [+ EvaluationBucketCounts(+ evaluation_id=row[0],+ bucket=row[1],+ passed=row[2],+ failed=row[3],+ not_applicable=row[4],+ skipped=row[5],+ score_count=row[6],+ score_sum=float(row[7] or 0),+ categorized=row[8],+ positive=row[9],+ neutral=row[10],+ negative=row[11],+ last_run_at=row[12],+ )+ for row in rows+ ]+++def load_category_counts(+ team: Team, user: User | None, period: OverviewPeriod, configured_categories: Mapping[str, int]+) -> list[CategoryBucketCount]:+ """`configured_categories` maps each evaluation to read to the number of categories its config lists."""+ if not configured_categories:+ return []+ # Runs can hold keys the config no longer lists, so the estimate doubles the configured keys and the '' row.+ expected_rows = {+ evaluation_id: _groups(period) * 2 * (configured + 1)+ for evaluation_id, configured in configured_categories.items()+ }+ rows = _run("ai_evaluations_overview_categories", _CATEGORY_QUERY, team, user, period, expected_rows, {})+ return [CategoryBucketCount(evaluation_id=row[0], bucket=row[1], key=row[2], runs=row[3]) for row in rows]diff --git a/products/ai_observability/backend/logic/evaluations/pass_filters.py b/products/ai_observability/backend/logic/evaluations/pass_filters.pyindex ae44fce458f9..a23dbcc2ca59 100644--- a/products/ai_observability/backend/logic/evaluations/pass_filters.py+++ b/products/ai_observability/backend/logic/evaluations/pass_filters.py@@ -1,42 +1,48 @@+from collections import defaultdict+from collections.abc import Iterator, Sequence+from typing import Literal+from django.db.models import Q+from posthog.hogql import astfrom posthog.hogql.escape_sql import escape_hogql_string+from posthog.hogql.parser import parse_expr++from posthog.dataclasses import frozenfrom products.ai_observability.backend.models.evaluation_configs import OutputTypefrom products.ai_observability.backend.models.evaluation_outcomes import (EvaluationReportOutcomeDefinition,+ RulePredicates,+ categorical_rule_predicates,get_outcome_definition,+ numeric_rule_predicates,)from products.ai_observability.backend.models.evaluations import Evaluationfrom ...facade import contracts_DEFAULT_DEFINITION = get_outcome_definition(OutputType.BOOLEAN)+_EVALUATION_ID = "properties.$ai_evaluation_id"+++@frozen+class PassRules:+ """The pass filters as parsed HogQL, with each evaluation's rule looked up from array placeholders."""++ graded: ast.Expr+ # Only means pass on a graded run, so a passed run is `graded AND passes`.+ passes: ast.Expr-def build_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilters:+def build_evaluation_pass_filters(+ team_id: int, evaluation_ids: Sequence[str] | None = None+) -> contracts.EvaluationPassFilters:+ """`evaluation_ids` limits the rules to the runs of those evaluations, for a read that filters on them."""graded_branches: list[str] = []passed_branches: list[str] = []- # Every other evaluation falls through to the default rule. Soft-deleted evaluations stay in, because their past- # events still need their own rule. Reads the writer, because a receiver of evaluation_output_changed calls this- # right after the commit, before a replica can catch up.- evaluations_with_own_rule = (- Evaluation.objects.using("default")- .filter(team_id=team_id)- .filter(- Q(output_type__in=[OutputType.NUMERIC, OutputType.CATEGORICAL, OutputType.SENTIMENT])- | Q(output_config__true_is_failure=True)- )- .values_list("id", "output_type", "output_config")- )- for evaluation_id, output_type, output_config in evaluations_with_own_rule:- try:- definition = get_outcome_definition(- output_type, true_is_failure=output_config.get("true_is_failure") is True, output_config=output_config- )- except ValueError:- definition = None- matches_evaluation = f"properties.$ai_evaluation_id = {escape_hogql_string(str(evaluation_id))}"+ for evaluation_id, definition in _own_rules(team_id, evaluation_ids):+ matches_evaluation = f"{_EVALUATION_ID} = {escape_hogql_string(evaluation_id)}"# An evaluation without a pass rule grades none of its runs, including boolean runs from before it# switched to sentiment, which the default branch would otherwise grade.if definition is None or not definition.has_pass_rule:@@ -53,9 +59,112 @@ def build_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilte)+def build_pass_rules(team_id: int, evaluation_ids: Sequence[str]) -> PassRules:+ """The same rules as `build_evaluation_pass_filters`, in one branch per kind of rule rather than per evaluation,+ so the query stays the same size however many evaluations it reads."""+ no_rule: list[str] = []+ inverted: list[str] = []+ no_category: list[str] = []+ numeric: dict[Literal["gte", "lte"], list[tuple[str, float]]] = defaultdict(list)+ categorical: list[tuple[str, list[str]]] = []+ for evaluation_id, definition in _own_rules(team_id, evaluation_ids):+ if definition is None or not definition.has_pass_rule:+ no_rule.append(evaluation_id)+ elif definition.numeric_config is not None and definition.numeric_config.passing_rule is not None:+ rule = definition.numeric_config.passing_rule+ numeric[rule.operator].append((evaluation_id, rule.threshold))+ elif definition.categorical_config is not None and definition.categorical_config.passing_rule is not None:+ categories = definition.categorical_config.passing_rule.categories+ if categories:+ categorical.append((evaluation_id, categories))+ else:+ no_category.append(evaluation_id)+ elif definition.passing_result is False:+ inverted.append(evaluation_id)++ placeholders: dict[str, ast.Expr] = {}+ graded_branches: list[str] = []+ passed_branches: list[str] = []++ # A kind without evaluations gets no branch, because HogQL cannot type an empty array constant.+ def branch(name: str, ids: Sequence[str], graded: str, passed: str) -> None:+ placeholders[f"{name}_ids"] = ast.Constant(value=list(ids))+ matches = f"{_EVALUATION_ID} IN {{{name}_ids}}"+ graded_branches.extend([matches, graded])+ passed_branches.extend([matches, passed])++ def rule_branch(name: str, ids: Sequence[str], event_predicate: str, predicates: RulePredicates) -> None:+ branch(name, ids, _graded_from(event_predicate, predicates.passed, predicates.failed), predicates.passed)++ if no_rule:+ branch("no_rule", no_rule, "false", "false")+ if inverted:+ definition = get_outcome_definition(OutputType.BOOLEAN, true_is_failure=True)+ outcome = definition.outcome_predicates+ rule_branch(+ "inverted",+ inverted,+ definition.event_predicate,+ RulePredicates(passed=outcome["pass"], failed=outcome["fail"]),+ )+ numeric_event = get_outcome_definition(OutputType.NUMERIC).event_predicate+ for operator, rules in numeric.items():+ name = f"numeric_{operator}"+ placeholders[f"{name}_thresholds"] = ast.Constant(value=[threshold for _, threshold in rules])+ # `transform` looks up a constant array through a hash table, where `indexOf` scans it on every run.+ threshold = f"transform({_EVALUATION_ID}, {{{name}_ids}}, {{{name}_thresholds}}, 0)"+ ids = [evaluation_id for evaluation_id, _ in rules]+ rule_branch(name, ids, numeric_event, numeric_rule_predicates(operator, threshold))+ categorical_event = get_outcome_definition(OutputType.CATEGORICAL).event_predicate+ if no_category:+ rule_branch("no_category", no_category, categorical_event, categorical_rule_predicates(None))+ if categorical:+ placeholders["categorical_passing"] = ast.Constant(value=[categories for _, categories in categorical])+ placeholders["categorical_positions"] = ast.Constant(value=list(range(1, len(categorical) + 1)))+ # `transform` cannot return an array, so it returns the position of the evaluation's categories instead.+ position = f"transform({_EVALUATION_ID}, {{categorical_ids}}, {{categorical_positions}}, 0)"+ passing = f"arrayElement({{categorical_passing}}, {position})"+ ids = [evaluation_id for evaluation_id, _ in categorical]+ rule_branch("categorical", ids, categorical_event, categorical_rule_predicates(passing))++ graded = _multi_if(graded_branches, _graded(_DEFAULT_DEFINITION))+ passes = _multi_if(passed_branches, _DEFAULT_DEFINITION.outcome_predicates["pass"])+ return PassRules(+ graded=parse_expr(graded, placeholders=placeholders), passes=parse_expr(passes, placeholders=placeholders)+ )+++def _own_rules(+ team_id: int, evaluation_ids: Sequence[str] | None+) -> Iterator[tuple[str, EvaluationReportOutcomeDefinition | None]]:+ """Each evaluation whose runs the default rule does not grade, with its rule, or None for a config that no longer+ loads. Soft-deleted evaluations stay in, because their past events still need their own rule. Reads the writer,+ because a receiver of evaluation_output_changed calls this right after the commit, before a replica can catch+ up."""+ evaluations = Evaluation.objects.using("default").filter(team_id=team_id)+ if evaluation_ids is not None:+ evaluations = evaluations.filter(id__in=evaluation_ids)+ evaluations_with_own_rule = evaluations.filter(+ Q(output_type__in=[OutputType.NUMERIC, OutputType.CATEGORICAL, OutputType.SENTIMENT])+ | Q(output_config__true_is_failure=True)+ ).values_list("id", "output_type", "output_config")+ for evaluation_id, output_type, output_config in evaluations_with_own_rule:+ try:+ definition = get_outcome_definition(+ output_type, true_is_failure=output_config.get("true_is_failure") is True, output_config=output_config+ )+ except ValueError:+ definition = None+ yield str(evaluation_id), definition++def _graded(definition: EvaluationReportOutcomeDefinition) -> str:predicates = definition.outcome_predicates- return f"{definition.event_predicate} AND (({predicates['pass']}) OR ({predicates['fail']}))"+ return _graded_from(definition.event_predicate, predicates["pass"], predicates["fail"])+++def _graded_from(event_predicate: str, passed: str, failed: str) -> str:+ return f"{event_predicate} AND (({passed}) OR ({failed}))"def _multi_if(branches: list[str], default: str) -> str:diff --git a/products/ai_observability/backend/logic/evaluations/period.py b/products/ai_observability/backend/logic/evaluations/period.pynew file mode 100644index 000000000000..831c880e02c2--- /dev/null+++ b/products/ai_observability/backend/logic/evaluations/period.py@@ -0,0 +1,112 @@+from datetime import UTC, date, datetime, time, timedelta, tzinfo+from typing import Literal++from posthog.dataclasses import frozen++Interval = Literal["hour", "day"]++# A period this short or shorter is bucketed by hour, so a day or two still draws a line rather than one or two dots.+MAX_HOURLY_SPAN = timedelta(days=2)+DEFAULT_DAYS = 14+MAX_DAYS = 90+++def _resolve_bound(value: date | datetime, tz: tzinfo, *, is_end: bool) -> datetime:+ """A date is a whole day in the project's timezone, so as an end it means the following midnight."""+ if isinstance(value, datetime):+ return value.astimezone(tz) if value.tzinfo else value.replace(tzinfo=tz)+ return datetime.combine(value + timedelta(days=1) if is_end else value, time.min, tz)+++class InvalidPeriodError(ValueError):+ def __init__(self, message: str, *, field: str | None = None) -> None:+ super().__init__(message)+ # The request parameter at fault, or None when the period as a whole is out of range.+ self.field = field+++@frozen+class PeriodBounds:+ start: datetime+ end: datetime # Exclusive.++ @classmethod+ def from_request(+ cls, date_from: date | datetime | None, date_to: date | datetime | None, tz: tzinfo+ ) -> "PeriodBounds":+ """A date is a whole day in the project's timezone, a datetime is an exact instant. Without bounds, the period+ is the last 14 days up to and including today."""+ try:+ date_to = date_to or datetime.now(tz).date()+ if date_from is None:+ date_from = (+ date_to - timedelta(days=DEFAULT_DAYS)+ if isinstance(date_to, datetime)+ else date_to - timedelta(days=DEFAULT_DAYS - 1)+ )+ end = _resolve_bound(date_to, tz, is_end=True)+ start = _resolve_bound(date_from, tz, is_end=False)+ # Both bounds share the project's tzinfo, which compares wall time, so this compares instants instead.+ length = end.astimezone(UTC) - start.astimezone(UTC)+ # The overview also reads the period of equal length before this one, so its start must exist too.+ start.astimezone(UTC) - length+ except OverflowError as error:+ raise InvalidPeriodError("The period is outside the supported date range.") from error+ if length <= timedelta(0):+ raise InvalidPeriodError("Must be before date_to.", field="date_from")+ # Whole days count by the calendar, because a day across a daylight saving change is 23 or 25 hours long.+ # An instant has no such change, so a period with one counts real time.+ if not isinstance(date_from, datetime) and not isinstance(date_to, datetime):+ too_long = (date_to - date_from).days + 1 > MAX_DAYS+ else:+ too_long = length > timedelta(days=MAX_DAYS)+ if too_long:+ raise InvalidPeriodError(f"The period can cover at most {MAX_DAYS} days.", field="date_from")+ return cls(start=start, end=end)+++@frozen+class OverviewPeriod:+ start: datetime+ end: datetime+ previous_start: datetime+ # Before `start` when the current period is still running, so both periods cover the same length of time.+ previous_end: datetime+ interval: Interval+ # The start of each bucket in the project's timezone. The first one can start before `start`.+ buckets: tuple[datetime, ...]+++def overview_period(start: datetime, end: datetime, now: datetime, tz: tzinfo) -> OverviewPeriod:+ # Two datetimes that share a tzinfo subtract and compare by wall time, so a daylight saving change would alter+ # the period's length. UTC keeps every length and comparison in real time.+ start_utc, end_utc = start.astimezone(UTC), end.astimezone(UTC)+ duration = end_utc - start_utc+ previous_start = start_utc - duration+ interval: Interval = "hour" if duration <= MAX_HOURLY_SPAN else "day"+ return OverviewPeriod(+ start=start,+ end=end,+ previous_start=previous_start.astimezone(tz),+ previous_end=max(min(end_utc, now) - duration, previous_start).astimezone(tz),+ interval=interval,+ buckets=_buckets(start.astimezone(tz), end_utc, interval, tz),+ )+++def _buckets(start: datetime, end: datetime, interval: Interval, tz: tzinfo) -> tuple[datetime, ...]:+ """`end` is in UTC, so each comparison with a local bucket is by instant."""+ buckets: list[datetime] = []+ if interval == "day":+ # Midnight by local date, because a day is 23 or 25 hours long across a daylight saving change.+ day = start.date()+ while (bucket := datetime.combine(day, time.min, tz)) < end:+ buckets.append(bucket)+ day += timedelta(days=1)+ return tuple(buckets)+ # Stepping in UTC keeps each step one real hour across a daylight saving change.+ bucket = start.replace(minute=0, second=0, microsecond=0)+ while bucket < end:+ buckets.append(bucket)+ bucket = (bucket.astimezone(UTC) + timedelta(hours=1)).astimezone(tz)+ return tuple(buckets)diff --git a/products/ai_observability/backend/logic/evaluations/runs.py b/products/ai_observability/backend/logic/evaluations/runs.pynew file mode 100644index 000000000000..d3d8d061957d--- /dev/null+++ b/products/ai_observability/backend/logic/evaluations/runs.py@@ -0,0 +1,89 @@+from typing import Literal++from posthog.hogql import ast+from posthog.hogql.parser import parse_expr+from posthog.hogql.visitor import clone_expr++from posthog.models import Team, User++from ...facade import contracts+from .overview_queries import (+ EVALUATION_RUNS_WHERE,+ GRADED_WITHOUT_RULE,+ evaluation_runs_placeholders,+ run_evaluation_query,+)+from .pass_filters import PassRules+from .period import PeriodBounds++RunOutcome = Literal["passed", "failed"]++REASONING_MAX_CHARS = 1000++_RUNS_QUERY = f"""+SELECT+ timestamp,+ if({{graded}}, if({{passes}}, 'passed', 'failed'), NULL),+ properties.$ai_evaluation_reasoning,+ properties.$ai_target_event_id,+ properties.$ai_trace_id+FROM events+WHERE {EVALUATION_RUNS_WHERE} AND {{matches}}+ORDER BY timestamp DESC+LIMIT {{limit}}+"""+++def _text(value: object) -> str | None:+ return value if isinstance(value, str) and value else None+++def _truncate(value: object) -> str | None:+ text = _text(value)+ if text is None or len(text) <= REASONING_MAX_CHARS:+ return text+ return f"{text[: REASONING_MAX_CHARS - 1]}…"+++def _matches(pass_rules: PassRules, outcome: RunOutcome | None) -> ast.Expr:+ # Copies, because the select reads the same rule expressions.+ graded, passes = clone_expr(pass_rules.graded), clone_expr(pass_rules.passes)+ if outcome == "passed":+ return ast.And(exprs=[graded, passes])+ if outcome == "failed":+ return ast.And(exprs=[graded, ast.Not(expr=passes)])+ return ast.Or(exprs=[graded, parse_expr(GRADED_WITHOUT_RULE)])+++def load_runs(+ team: Team,+ user: User | None,+ evaluation_id: str,+ period: PeriodBounds,+ pass_rules: PassRules,+ outcome: RunOutcome | None,+ limit: int,+) -> list[contracts.EvaluationRun]:+ rows = run_evaluation_query(+ "ai_evaluations_runs",+ _RUNS_QUERY,+ team,+ user,+ {+ **evaluation_runs_placeholders(evaluation_id, period),+ "graded": pass_rules.graded,+ "passes": pass_rules.passes,+ "matches": _matches(pass_rules, outcome),+ "limit": ast.Constant(value=limit),+ },+ )+ return [+ contracts.EvaluationRun(+ timestamp=row[0],+ outcome=row[1],+ reasoning=_truncate(row[2]),+ target_event_id=_text(row[3]),+ trace_id=_text(row[4]),+ )+ for row in rows+ ]diff --git a/products/ai_observability/backend/logic/evaluations/score_distribution.py b/products/ai_observability/backend/logic/evaluations/score_distribution.pynew file mode 100644index 000000000000..66878b120fd4--- /dev/null+++ b/products/ai_observability/backend/logic/evaluations/score_distribution.py@@ -0,0 +1,84 @@+from posthog.hogql import ast++from posthog.dataclasses import frozen+from posthog.models import Team, User++from products.ai_observability.backend.models.evaluation_outcomes import NUMERIC_SCORED_PREDICATE, SCORE_EXPRESSION++from ...facade import contracts+from .overview_queries import EVALUATION_RUNS_WHERE, evaluation_runs_placeholders, run_evaluation_query+from .period import PeriodBounds++# An integer scale of 2 to this many steps gets one bucket per integer, any other scale gets equal buckets.+MAX_INTEGER_BUCKETS = 20+EQUAL_BUCKETS = 10++# Scores outside the configured scale land in the end buckets. The emitter skips such runs, so this only catches+# runs scored before the scale changed. The epsilon keeps a score on a bucket edge in the bucket it starts, where+# float division can land just under it: (0.3 - 0) / 0.1 is 2.9999999999999996.+_HISTOGRAM_QUERY = f"""+SELECT least(greatest(toInt(floor(({SCORE_EXPRESSION} - {{low}}) / {{width}} + 1e-9)), 0), {{last}}) AS bucket, count()+FROM events+WHERE {EVALUATION_RUNS_WHERE} AND {NUMERIC_SCORED_PREDICATE}+GROUP BY bucket+LIMIT {MAX_INTEGER_BUCKETS + 1}+"""+++@frozen+class ScoreBuckets:+ low: float+ width: float+ labels: tuple[str, ...]+++def score_buckets(scale: contracts.ScoreScale, step: float | None) -> ScoreBuckets | None:+ span = scale.max - scale.min+ if span <= 0:+ return None+ # A 0 to 1 scale is a pair of integers too, but its scores are fractions unless a whole step says otherwise.+ integer_steps = step.is_integer() if step is not None else span >= 2+ if scale.min.is_integer() and scale.max.is_integer() and integer_steps and span <= MAX_INTEGER_BUCKETS:+ # Centering each bucket on its integer keeps a score of 3 in the "3" bucket however it rounds.+ return ScoreBuckets(+ low=scale.min - 0.5,+ width=1,+ labels=tuple(str(int(scale.min) + offset) for offset in range(int(span) + 1)),+ )+ width = span / EQUAL_BUCKETS+ return ScoreBuckets(+ low=scale.min,+ width=width,+ labels=tuple(+ f"{scale.min + index * width:g}-{scale.min + (index + 1) * width:g}" for index in range(EQUAL_BUCKETS)+ ),+ )+++def load_score_distribution(+ team: Team,+ user: User | None,+ evaluation_id: str,+ period: PeriodBounds,+ scale: contracts.ScoreScale,+ buckets: ScoreBuckets,+) -> contracts.ScoreHistogram:+ rows = run_evaluation_query(+ "ai_evaluations_score_distribution",+ _HISTOGRAM_QUERY,+ team,+ user,+ {+ **evaluation_runs_placeholders(evaluation_id, period),+ "low": ast.Constant(value=buckets.low),+ "width": ast.Constant(value=buckets.width),+ "last": ast.Constant(value=len(buckets.labels) - 1),+ },+ )+ counts = {int(bucket): count for bucket, count in rows}+ return contracts.ScoreHistogram(+ scale=scale,+ buckets=[+ contracts.ScoreBucket(label=label, count=counts.get(index, 0)) for index, label in enumerate(buckets.labels)+ ],+ )diff --git a/products/ai_observability/backend/models/evaluation_outcomes.py b/products/ai_observability/backend/models/evaluation_outcomes.pyindex 3067f197ea9e..0c11a6c97c5a 100644--- a/products/ai_observability/backend/models/evaluation_outcomes.py+++ b/products/ai_observability/backend/models/evaluation_outcomes.py@@ -7,9 +7,12 @@ that reads only those is correct without passing `true_is_failure` or `output_cofrom collections.abc import Callable, Mappingfrom dataclasses import dataclass+from typing import Literalfrom posthog.hogql.escape_sql import escape_hogql_string+from posthog.dataclasses import frozen+from products.ai_observability.backend.models.evaluation_configs import CategoricalOutputConfig, NumericOutputConfigSENTIMENT_LABELS = ("positive", "neutral", "negative")@@ -69,9 +72,9 @@ class EvaluationReportOutcomeDefinition:# String comparison is deliberate: an unregistered JSON bool property extracts as the# string 'true', which the bool literal cannot compare against, while a registered# Boolean property coerces correctly against the string form.-_NOT_SKIPPED_PREDICATE = "(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')"+NOT_SKIPPED_PREDICATE = "(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')"-_APPLICABLE_PREDICATE = (+APPLICABLE_PREDICATE = ("(isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false')")@@ -83,13 +86,13 @@ def _boolean_definition(true_is_failure: bool) -> EvaluationReportOutcomeDefinitreturn EvaluationReportOutcomeDefinition(outcomes=("pass", "fail", "na"),outcome_predicates={- "pass": f"properties.$ai_evaluation_result = {pass_literal} AND {_APPLICABLE_PREDICATE}",- "fail": f"properties.$ai_evaluation_result = {fail_literal} AND {_APPLICABLE_PREDICATE}",+ "pass": f"properties.$ai_evaluation_result = {pass_literal} AND {APPLICABLE_PREDICATE}",+ "fail": f"properties.$ai_evaluation_result = {fail_literal} AND {APPLICABLE_PREDICATE}","na": "properties.$ai_evaluation_applicable = 'false'",},event_predicate=("(properties.$ai_evaluation_result_type = 'boolean' OR isNull(properties.$ai_evaluation_result_type)) "- f"AND {_NOT_SKIPPED_PREDICATE}"+ f"AND {NOT_SKIPPED_PREDICATE}"),result_expression="properties.$ai_evaluation_result",applicable_expression="properties.$ai_evaluation_applicable",@@ -102,7 +105,7 @@ def _sentiment_definition(true_is_failure: bool) -> EvaluationReportOutcomeDefinreturn EvaluationReportOutcomeDefinition(outcomes=SENTIMENT_LABELS,outcome_predicates={label: f"properties.$ai_sentiment_label = '{label}'" for label in SENTIMENT_LABELS},- event_predicate=f"properties.$ai_evaluation_result_type = 'sentiment' AND {_NOT_SKIPPED_PREDICATE}",+ event_predicate=f"properties.$ai_evaluation_result_type = 'sentiment' AND {NOT_SKIPPED_PREDICATE}",result_expression="properties.$ai_sentiment_label",applicable_expression="NULL",score_expression="properties.$ai_sentiment_score",@@ -118,21 +121,59 @@ _DEFINITION_BUILDERS: Mapping[str, Callable[[bool], EvaluationReportOutcomeDefinSUPPORTED_EVAL_REPORT_OUTPUT_TYPES = (*_DEFINITION_BUILDERS, "numeric", "categorical")+CATEGORIES_EXPRESSION = "JSONExtract(ifNull(properties.$ai_evaluation_categorical_result, '[]'), 'Array(String)')"+SCORE_EXPRESSION = "toFloat(properties.$ai_evaluation_numeric_result)"+_NUMERIC_EVENT_PREDICATE = f"properties.$ai_evaluation_result_type = 'numeric' AND {NOT_SKIPPED_PREDICATE}"+_CATEGORICAL_EVENT_PREDICATE = f"properties.$ai_evaluation_result_type = 'categorical' AND {NOT_SKIPPED_PREDICATE}"+# A categorical run is graded only when the judge said it applies, where a numeric run is graded unless it said not.+_CATEGORICAL_APPLICABLE_PREDICATE = "properties.$ai_evaluation_applicable = 'true'"+# A run that produced a score or chose its categories, with or without a passing rule. Same as+# EVALUATION_NUMERIC_GRADED_HOGQL and EVALUATION_CATEGORICAL_GRADED_HOGQL in the frontend's constants.ts.+NUMERIC_SCORED_PREDICATE = (+ f"{_NUMERIC_EVENT_PREDICATE} AND properties.$ai_evaluation_numeric_result IS NOT NULL AND {APPLICABLE_PREDICATE}"+)+CATEGORIZED_PREDICATE = f"{_CATEGORICAL_EVENT_PREDICATE} AND {_CATEGORICAL_APPLICABLE_PREDICATE}"+++@frozen+class RulePredicates:+ passed: str+ failed: str+++def categorical_rule_predicates(passing_categories: str | None) -> RulePredicates:+ """The pass and fail predicates of a categorical passing rule. `passing_categories` is a HogQL array, or None+ for a rule that passes only a run that chose no category."""+ if passing_categories is None:+ passing = f"empty({CATEGORIES_EXPRESSION})"+ else:+ passing = f"(notEmpty({CATEGORIES_EXPRESSION}) AND hasAll({passing_categories}, {CATEGORIES_EXPRESSION}))"+ return RulePredicates(+ passed=f"{passing} AND {_CATEGORICAL_APPLICABLE_PREDICATE}",+ failed=f"NOT {passing} AND {_CATEGORICAL_APPLICABLE_PREDICATE}",+ )+++def numeric_rule_predicates(operator: Literal["gte", "lte"], threshold: str) -> RulePredicates:+ """The pass and fail predicates of a numeric passing rule. `threshold` is a HogQL expression."""+ passed_op, failed_op = (">=", "<") if operator == "gte" else ("<=", ">")+ return RulePredicates(+ passed=f"{SCORE_EXPRESSION} {passed_op} {threshold} AND {APPLICABLE_PREDICATE}",+ failed=f"{SCORE_EXPRESSION} {failed_op} {threshold} AND {APPLICABLE_PREDICATE}",+ )++def _categorical_definition(output_config: dict | None) -> EvaluationReportOutcomeDefinition:config = CategoricalOutputConfig.model_validate(output_config) if output_config else None- categories_expression = "JSONExtract(ifNull(properties.$ai_evaluation_categorical_result, '[]'), 'Array(String)')"- applicable_predicate = "properties.$ai_evaluation_applicable = 'true'"passed_predicate = "false"failed_predicate = "false"if config is not None and config.passing_rule is not None:- if config.passing_rule.categories:- passing = ", ".join(escape_hogql_string(category) for category in config.passing_rule.categories)- passing_condition = f"(notEmpty({categories_expression}) AND hasAll([{passing}], {categories_expression}))"- else:- passing_condition = f"empty({categories_expression})"- passed_predicate = f"{passing_condition} AND {applicable_predicate}"- failed_predicate = f"NOT {passing_condition} AND {applicable_predicate}"+ categories = config.passing_rule.categories+ predicates = categorical_rule_predicates(+ f"[{', '.join(escape_hogql_string(category) for category in categories)}]" if categories else None+ )+ passed_predicate, failed_predicate = predicates.passed, predicates.failedreturn EvaluationReportOutcomeDefinition(outcomes=("pass", "fail", "na"),@@ -141,8 +182,8 @@ def _categorical_definition(output_config: dict | None) -> EvaluationReportOutco"fail": failed_predicate,"na": "properties.$ai_evaluation_applicable = 'false'",},- event_predicate=f"properties.$ai_evaluation_result_type = 'categorical' AND {_NOT_SKIPPED_PREDICATE}",- result_expression=categories_expression,+ event_predicate=_CATEGORICAL_EVENT_PREDICATE,+ result_expression=CATEGORIES_EXPRESSION,applicable_expression="properties.$ai_evaluation_applicable",score_expression="NULL",passing_result=None,@@ -152,21 +193,17 @@ def _categorical_definition(output_config: dict | None) -> EvaluationReportOutcodef _numeric_definition(output_config: dict | None) -> EvaluationReportOutcomeDefinition:config = NumericOutputConfig.model_validate(output_config or {})- score = "toFloat(properties.$ai_evaluation_numeric_result)"- applicable = "(isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false')"passed = failed = "false"if config.passing_rule is not None:- passed_op, failed_op = (">=", "<") if config.passing_rule.operator == "gte" else ("<=", ">")- threshold = repr(config.passing_rule.threshold)- passed = f"{score} {passed_op} {threshold} AND {applicable}"- failed = f"{score} {failed_op} {threshold} AND {applicable}"+ predicates = numeric_rule_predicates(config.passing_rule.operator, repr(config.passing_rule.threshold))+ passed, failed = predicates.passed, predicates.failedreturn EvaluationReportOutcomeDefinition(outcomes=("pass", "fail", "na"),outcome_predicates={"pass": passed, "fail": failed, "na": "properties.$ai_evaluation_applicable = 'false'"},- event_predicate=f"properties.$ai_evaluation_result_type = 'numeric' AND {_NOT_SKIPPED_PREDICATE}",- result_expression=score,+ event_predicate=_NUMERIC_EVENT_PREDICATE,+ result_expression=SCORE_EXPRESSION,applicable_expression="properties.$ai_evaluation_applicable",- score_expression=score,+ score_expression=SCORE_EXPRESSION,passing_result=None,numeric_config=config,)diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_overview.py b/products/ai_observability/backend/tests/logic/evaluations/test_overview.pynew file mode 100644index 000000000000..49ba16b0fffb--- /dev/null+++ b/products/ai_observability/backend/tests/logic/evaluations/test_overview.py@@ -0,0 +1,242 @@+from datetime import UTC, datetime+from typing import Any++from parameterized import parameterized++from products.ai_observability.backend.facade import contracts+from products.ai_observability.backend.logic.evaluations.overview import (+ CategoryOption,+ EvaluationMeta,+ SummaryKind,+ evaluation_result,+ overview_totals,+ pass_rate_overview,+)+from products.ai_observability.backend.logic.evaluations.overview_queries import (+ CategoryBucketCount,+ EvaluationBucketCounts,+)++BUCKETS = [datetime(2026, 3, day, tzinfo=UTC) for day in (1, 2, 3, 4)]+++def _meta(kind: SummaryKind = "pass_rate", **fields: Any) -> EvaluationMeta:+ defaults: dict[str, Any] = {+ "id": "eval-1",+ "name": "Example",+ "directory_name": None,+ "evaluation_type": "llm_judge",+ "output_type": {"pass_rate": "boolean", "score": "numeric", "categories": "categorical"}.get(kind, kind),+ "enabled": True,+ "errored": False,+ "status_reason": None,+ "created_by": None,+ "kind": kind,+ "scale": None,+ "score_step": None,+ "categories": (),+ "multiple": False,+ }+ return EvaluationMeta(**{**defaults, **fields})+++def _bucket(bucket: datetime | None, **counts: int | float) -> EvaluationBucketCounts:+ fields: dict[str, Any] = {+ "passed": 0,+ "failed": 0,+ "not_applicable": 0,+ "skipped": 0,+ "score_count": 0,+ "score_sum": 0.0,+ "categorized": 0,+ "positive": 0,+ "neutral": 0,+ "negative": 0,+ **counts,+ }+ return EvaluationBucketCounts(evaluation_id="eval-1", bucket=bucket, last_run_at=None, **fields)+++def _result(+ meta: EvaluationMeta, counts: list[EvaluationBucketCounts], categories: list[CategoryBucketCount] | None = None+) -> contracts.EvaluationResult:+ return evaluation_result(meta, BUCKETS, counts, categories or [])+++ [+ ("healthy", {}, [_bucket(BUCKETS[0], passed=90, failed=10), _bucket(None, passed=90, failed=10)], []),+ ("paused never asks", {"enabled": False}, [_bucket(None, passed=90, failed=10)], []),+ ("silent", {}, [], ["silent"]),+ # The error status disables the evaluation, so it has no results, but it is neither silent nor ignored.+ ("errored", {"enabled": False, "errored": True}, [_bucket(None, passed=80, failed=20)], ["errored"]),+ (+ "pass rate fell 2pp",+ {},+ [_bucket(BUCKETS[0], passed=88, failed=12), _bucket(None, passed=90, failed=10)],+ ["worse"],+ ),+ (+ "pass rate fell under 2pp",+ {},+ [_bucket(BUCKETS[0], passed=89, failed=11), _bucket(None, passed=90, failed=10)],+ [],+ ),+ (+ "thin previous period is never worse",+ {},+ [_bucket(BUCKETS[0], passed=450, failed=50), _bucket(None, passed=2)],+ [],+ ),+ ]+)+def test_attention_reasons(+ _name: str, meta: dict[str, Any], counts: list[EvaluationBucketCounts], expected: list[str]+) -> None:+ assert _result(_meta(**meta), counts).attention_reasons == expected+++ [+ ("compared", [_bucket(BUCKETS[0], passed=80, failed=20), _bucket(None, passed=85, failed=15)], -5.0, False),+ (+ "low evidence is not compared",+ [_bucket(BUCKETS[0], passed=7, failed=5), _bucket(None, passed=9, failed=1)],+ None,+ True,+ ),+ ("nothing before", [_bucket(BUCKETS[0], passed=80, failed=20)], None, False),+ (+ "thin previous period is not compared",+ [_bucket(BUCKETS[0], passed=450, failed=50), _bucket(None, passed=2)],+ None,+ False,+ ),+ # 50 - 49.75 rounds half up like Math.round, where Python's round() would give 0.2.+ (+ "rounds half up",+ [_bucket(BUCKETS[0], passed=40, failed=40), _bucket(None, passed=199, failed=201)],+ 0.3,+ False,+ ),+ ]+)+def test_pass_rate_change_and_low_evidence(+ _name: str, counts: list[EvaluationBucketCounts], change: float | None, low_evidence: bool+) -> None:+ result = _result(_meta(), counts)+ assert (result.change.value if result.change else None) == change+ assert result.low_evidence is low_evidence+++def test_bucket_status_against_previous_period() -> None:+ result = _result(+ _meta(),+ [+ _bucket(BUCKETS[0], passed=90, failed=10),+ _bucket(BUCKETS[1], passed=86, failed=14),+ # Further below the previous rate, but too thin to judge.+ _bucket(BUCKETS[3], passed=20, failed=5),+ _bucket(None, passed=90, failed=10),+ ],+ )+ assert result.trend.values == [90.0, 86.0, None, 80.0]+ assert result.bucket_status == ["normal", "dip", "none", "normal"]+++def test_bucket_status_falls_back_to_own_median_without_previous_period() -> None:+ result = _result(+ _meta(),+ [+ _bucket(BUCKETS[0], passed=90, failed=10),+ _bucket(BUCKETS[1], passed=90, failed=10),+ _bucket(BUCKETS[2], passed=80, failed=20),+ ],+ )+ assert result.bucket_status == ["normal", "normal", "dip", "none"]+++def test_score_summary() -> None:+ meta = _meta("score", scale=contracts.ScoreScale(min=0, max=1))+ result = _result(+ meta,+ [+ _bucket(BUCKETS[0], score_count=40, score_sum=32.0),+ _bucket(BUCKETS[1], score_count=40, score_sum=32.0),+ _bucket(None, score_count=40, score_sum=26.0),+ ],+ )+ assert isinstance(result.summary, contracts.ScoreSummary)+ assert result.summary.score == contracts.ScoreDistribution(mean=0.8, scale=meta.scale)+ assert result.change == contracts.ResultChange(value=0.15, unit="points", subject=None)+ # A score moving 0.05 of its scale counts as unusual either way, with no worse direction.+ assert result.bucket_status == ["dip", "dip", "none", "none"]+ assert result.attention_reasons == []+++def test_categories_summary_tracks_the_label_that_moved_most() -> None:+ meta = _meta(+ "categories",+ multiple=True,+ categories=(CategoryOption(key="billing", label="Billing"), CategoryOption(key="bug", label="Bug")),+ )+ categories = [+ CategoryBucketCount(evaluation_id="eval-1", bucket=BUCKETS[0], key="billing", runs=30),+ CategoryBucketCount(evaluation_id="eval-1", bucket=BUCKETS[0], key="bug", runs=12),+ CategoryBucketCount(evaluation_id="eval-1", bucket=BUCKETS[0], key="", runs=8),+ CategoryBucketCount(evaluation_id="eval-1", bucket=None, key="billing", runs=30),+ CategoryBucketCount(evaluation_id="eval-1", bucket=None, key="bug", runs=2),+ CategoryBucketCount(evaluation_id="eval-1", bucket=None, key="refund", runs=4),+ ]+ result = _result(meta, [_bucket(BUCKETS[0], categorized=50), _bucket(None, categorized=40)], categories)+ assert result.summary == contracts.CategoriesSummary(+ kind="categories",+ multiple=True,+ categories=[+ contracts.CategoryCount(key="billing", label="Billing", count=30, previous_share=75.0),+ contracts.CategoryCount(key="bug", label="Bug", count=12, previous_share=5.0),+ contracts.CategoryCount(key="refund", label="refund", count=0, previous_share=10.0),+ ],+ no_categories_count=8,+ )+ assert result.change == contracts.ResultChange(value=19.0, unit="pp", subject="Bug")+ assert result.tracked_category == "bug"+ assert result.trend.values == [24.0, None, None, None]+++def test_rising_negative_sentiment_is_worse() -> None:+ result = _result(+ _meta("sentiment"),+ [_bucket(BUCKETS[0], positive=20, neutral=20, negative=10), _bucket(None, positive=25, neutral=20, negative=5)],+ )+ assert result.change == contracts.ResultChange(value=10.0, unit="pp", subject="negative")+ assert result.attention_reasons == ["worse"]+++def test_overview_totals_compare_only_evaluations_with_results_in_both_periods() -> None:+ steady = _result(_meta(), [_bucket(BUCKETS[0], passed=60, failed=20), _bucket(None, passed=40, failed=10)])+ new = _result(_meta(), [_bucket(BUCKETS[1], passed=30, failed=0)])+ sentiment = _result(+ _meta("sentiment"), [_bucket(BUCKETS[1], negative=5, neutral=15), _bucket(None, negative=3, neutral=27)]+ )+ paused = _result(_meta(enabled=False), [_bucket(BUCKETS[0], passed=1000), _bucket(None, passed=1000)])++ totals = overview_totals([steady, new, sentiment, paused], len(BUCKETS))++ assert totals.graded == 130+ assert totals.graded_comparison == contracts.PeriodComparison(current=100, previous=80)+ assert totals.graded_trend == [80, 50, 0, 0]+ assert totals.sentiment == contracts.OverviewSentiment(+ evaluation_id="eval-1", negative_share=25.0, previous_share=10.0+ )+ assert overview_totals([new], len(BUCKETS)).graded_comparison is None+++def test_pass_rate_median_change_uses_the_same_evaluations_in_both_periods() -> None:+ steady = _result(_meta(), [_bucket(BUCKETS[0], passed=80, failed=20), _bucket(None, passed=90, failed=10)])+ new = _result(_meta(), [_bucket(BUCKETS[0], passed=40, failed=60)])++ overview = pass_rate_overview([steady, new], len(BUCKETS))++ assert overview.median == 60.0+ assert overview.median_comparison == contracts.PeriodComparison(current=80.0, previous=90.0)diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.py b/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.pynew file mode 100644index 000000000000..12a829bd3865--- /dev/null+++ b/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.py@@ -0,0 +1,372 @@+from datetime import UTC, date, datetime, timedelta+from zoneinfo import ZoneInfo++import pytest+import time_machine+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events+from unittest.mock import patch++from parameterized import parameterized++from posthog.hogql.context import HogQLContext+from posthog.hogql.database.database import Database+from posthog.hogql.printer.utils import prepare_and_print_ast++from products.ai_observability.backend.facade.api import (+ evaluations_period,+ get_evaluation_runs,+ get_evaluations_overview,+)+from products.ai_observability.backend.logic.evaluations.overview_queries import (+ CategoryBucketCount,+ load_bucket_counts,+ load_category_counts,+ run_evaluation_query,+)+from products.ai_observability.backend.logic.evaluations.pass_filters import build_pass_rules+from products.ai_observability.backend.logic.evaluations.period import (+ InvalidPeriodError,+ OverviewPeriod,+ PeriodBounds,+ overview_period,+)+from products.ai_observability.backend.models.evaluations import Evaluation++PERIOD = OverviewPeriod(+ start=datetime(2026, 3, 8, tzinfo=UTC),+ end=datetime(2026, 3, 15, tzinfo=UTC),+ previous_start=datetime(2026, 3, 1, tzinfo=UTC),+ previous_end=datetime(2026, 3, 8, tzinfo=UTC),+ interval="day",+ buckets=tuple(datetime(2026, 3, day, tzinfo=UTC) for day in range(8, 15)),+)+++NEW_YORK = ZoneInfo("America/New_York")+++def _midnight(day: int) -> datetime:+ return datetime(2026, 3, day, tzinfo=UTC)+++ [+ # Clocks spring forward on March 8, so 48 real hours span 49 hours on the wall.+ ("spring_forward", datetime(2026, 3, 9, 12, tzinfo=UTC), 48),+ # Clocks fall back on November 1, so 24 real hours span 23 hours on the wall.+ ("fall_back", datetime(2026, 11, 1, 18, tzinfo=UTC), 24),+ ]+)+def test_rolling_hours_keep_their_real_length_across_daylight_saving(_name: str, now: datetime, hours: int) -> None:+ length = timedelta(hours=hours)+ period = overview_period((now - length).astimezone(NEW_YORK), now.astimezone(NEW_YORK), now, NEW_YORK)++ assert period.interval == "hour"+ assert len(period.buckets) == hours+ assert (period.previous_start, period.previous_end) == (now - 2 * length, now - length)+++ [+ ("end before start", date(2026, 3, 10), date(2026, 3, 9), "date_from"),+ ("91 days", date(2026, 8, 4), date(2026, 11, 2), "date_from"),+ ("last day of year 9999", None, date(9999, 12, 31), "out of range"),+ ("first day of year 1", None, date(1, 1, 1), "out of range"),+ # Clocks fall back on November 1, so these 90 days are 2161 hours long.+ ("90 days across daylight saving", date(2026, 8, 5), date(2026, 11, 2), None),+ (+ "90 days of instants",+ datetime(2026, 8, 5, tzinfo=UTC),+ datetime(2026, 11, 3, tzinfo=UTC),+ None,+ ),+ (+ "an instant past 90 days",+ datetime(2026, 8, 5, tzinfo=UTC),+ datetime(2026, 11, 3, 0, 0, 1, tzinfo=UTC),+ "date_from",+ ),+ ]+)+def test_period_from_request(+ _name: str, date_from: date | datetime | None, date_to: date | datetime, error: str | None+) -> None:+ if error is None:+ PeriodBounds.from_request(date_from, date_to, NEW_YORK)+ return+ with pytest.raises(InvalidPeriodError) as raised:+ PeriodBounds.from_request(date_from, date_to, NEW_YORK)+ assert (raised.value.field or "out of range") == error+++class TestEvaluationsOverviewQueries(ClickhouseTestMixin, BaseTest):+ def _judge(self) -> Evaluation:+ return Evaluation.objects.create(+ team=self.team,+ name="Judge",+ evaluation_type="hog",+ evaluation_config={"source": "return true"},+ output_type="boolean",+ output_config={},+ )++ def _run(self, evaluation: Evaluation, timestamp: str, **properties: object) -> None:+ _create_event(+ team=self.team,+ event="$ai_evaluation",+ distinct_id="user-1",+ timestamp=timestamp,+ properties={"$ai_evaluation_id": str(evaluation.id), **properties},+ )++ @parameterized.expand(+ [+ ("one query per kind", 50_000, 25_000, 2),+ ("a batch per evaluation", 50_000, 1, 3),+ # A full page splits its batch, down to one evaluation, which pages through its own rows.+ ("full pages", 2, 50_000, 8),+ ]+ )+ def test_groups_runs_by_evaluation_day_and_period(+ self, _name: str, page_size: int, batch_rows: int, queries: int+ ) -> None:+ judge = self._judge()+ topic = Evaluation.objects.create(+ team=self.team,+ name="Topic",+ evaluation_type="hog",+ evaluation_config={"source": "return []"},+ output_type="categorical",+ output_config={+ "options": [{"key": "billing", "label": "Billing"}, {"key": "bug", "label": "Bug"}],+ "selection_mode": "multiple",+ },+ )+ unrequested = Evaluation.objects.create(+ team=self.team, name="Not asked for", evaluation_type="hog", output_type="boolean", output_config={}+ )+ categorical = {"$ai_evaluation_result_type": "categorical", "$ai_evaluation_applicable": True}+ self._run(judge, "2026-03-09T10:00:00Z", **{"$ai_evaluation_result": True})+ self._run(judge, "2026-03-09T11:00:00Z", **{"$ai_evaluation_result": False})+ self._run(judge, "2026-03-09T12:00:00Z", **{"$ai_evaluation_applicable": False})+ skipped = {"$ai_evaluation_result": False, "$ai_evaluation_skipped": True}+ self._run(judge, "2026-03-10T10:00:00Z", **skipped, **{"$ai_evaluation_skip_reason": "hog_input_error"})+ self._run(judge, "2026-03-10T11:00:00Z", **skipped, **{"$ai_evaluation_skip_reason": "trace_not_found"})+ self._run(judge, "2026-03-02T10:00:00Z", **{"$ai_evaluation_result": True})+ self._run(judge, "2026-03-07T23:00:00Z", **{"$ai_evaluation_result": True})+ self._run(judge, "2026-02-28T23:00:00Z", **{"$ai_evaluation_result": True})+ self._run(+ topic, "2026-03-09T10:00:00Z", **categorical, **{"$ai_evaluation_categorical_result": ["billing", "bug"]}+ )+ self._run(topic, "2026-03-09T11:00:00Z", **categorical, **{"$ai_evaluation_categorical_result": []})+ self._run(topic, "2026-03-03T10:00:00Z", **categorical, **{"$ai_evaluation_categorical_result": ["bug"]})+ self._run(unrequested, "2026-03-09T10:00:00Z", **{"$ai_evaluation_result": True})+ flush_persons_and_events()++ ids = [str(judge.id), str(topic.id)]+ module = "products.ai_observability.backend.logic.evaluations.overview_queries"+ with (+ patch(f"{module}.PAGE_SIZE", page_size),+ patch(f"{module}.BATCH_ROWS", batch_rows),+ patch(f"{module}.run_evaluation_query", wraps=run_evaluation_query) as run,+ ):+ bucket_counts = load_bucket_counts(self.team, None, PERIOD, ids, build_pass_rules(self.team.pk, ids))+ category_counts = load_category_counts(self.team, None, PERIOD, {str(topic.id): 2})+ counts = {+ (row.evaluation_id, row.bucket): (row.passed, row.failed, row.not_applicable, row.skipped, row.categorized)+ for row in bucket_counts+ }+ categories = sorted(category_counts, key=lambda row: (row.bucket or datetime.min.replace(tzinfo=UTC), row.key))++ assert counts == {+ (str(judge.id), _midnight(9)): (1, 1, 1, 0, 0),+ (str(judge.id), _midnight(10)): (0, 0, 0, 2, 0),+ (str(judge.id), None): (2, 0, 0, 0, 0),+ (str(topic.id), _midnight(9)): (0, 0, 0, 0, 2),+ (str(topic.id), None): (0, 0, 0, 0, 1),+ }+ assert categories == [+ CategoryBucketCount(evaluation_id=str(topic.id), bucket=None, key="bug", runs=1),+ CategoryBucketCount(evaluation_id=str(topic.id), bucket=_midnight(9), key="", runs=1),+ CategoryBucketCount(evaluation_id=str(topic.id), bucket=_midnight(9), key="billing", runs=1),+ CategoryBucketCount(evaluation_id=str(topic.id), bucket=_midnight(9), key="bug", runs=1),+ ]+ assert run.call_count == queries++ @time_machine.travel(datetime(2026, 3, 10, 3, 0, tzinfo=UTC), tick=False)+ def test_periods_follow_the_project_timezone_and_a_running_day(self) -> None:+ self.team.timezone = "Asia/Tokyo"+ self.team.save()+ judge = self._judge()+ # Now is 12:00 on March 10 in Tokyo, so the previous period runs from March 7 to 12:00 on March 8.+ for timestamp in (+ "2026-03-06T14:00:00Z", # 23:00 March 6, before the previous period+ "2026-03-06T16:00:00Z", # 01:00 March 7+ "2026-03-08T02:00:00Z", # 11:00 March 8+ "2026-03-08T05:00:00Z", # 14:00 March 8, later in the day than now+ "2026-03-08T16:00:00Z", # 01:00 March 9+ "2026-03-09T16:00:00Z", # 01:00 March 10+ ):+ self._run(judge, timestamp, **{"$ai_evaluation_result": True})+ flush_persons_and_events()++ overview = get_evaluations_overview(+ self.team,+ None,+ [str(judge.id)],+ evaluations_period(self.team, date(2026, 3, 9), date(2026, 3, 10)),+ )++ [row] = overview.evaluations+ # Two whole days are short enough for hourly buckets, which start on Tokyo's hours.+ assert overview.interval == "hour"+ assert overview.buckets[0] == datetime(2026, 3, 9, tzinfo=ZoneInfo("Asia/Tokyo"))+ assert [index for index, volume in enumerate(row.trend.volume) if volume] == [1, 25]+ assert row.previous_graded_count == 2++ @time_machine.travel(datetime(2026, 3, 12, 12, 0, tzinfo=UTC), tick=False)+ def test_days_follow_the_project_timezone_when_the_team_turns_off_converting_to_it(self) -> None:+ self.team.timezone = "America/New_York"+ self.team.modifiers = {"convertToProjectTimezone": False}+ self.team.save()+ judge = self._judge()+ # 22:00 on March 9 in New York, already March 10 in UTC.+ self._run(judge, "2026-03-10T02:00:00Z", **{"$ai_evaluation_result": True})+ flush_persons_and_events()++ overview = get_evaluations_overview(+ self.team,+ None,+ [str(judge.id)],+ evaluations_period(self.team, date(2026, 3, 8), date(2026, 3, 11)),+ )++ [row] = overview.evaluations+ assert overview.interval == "day"+ assert row.trend.volume == [0, 1, 0, 0]++ @parameterized.expand(+ [+ # 01:30 on November 1 in New York, before and after the clocks fall back.+ ("new_york_fall_back", "America/New_York", datetime(2026, 11, 1, 18, tzinfo=UTC), 24, ["05:30", "06:30"]),+ # Lord Howe moves its clocks by 30 minutes on October 4, so from then on the buckets start at half past+ # the local hour, and 01:10 and 01:40 on October 5 fall in different buckets.+ (+ "lord_howe_spring_forward",+ "Australia/Lord_Howe",+ datetime(2026, 10, 4, 20, tzinfo=UTC),+ 48,+ ["2026-10-03T10:00", "2026-10-04T14:10", "2026-10-04T14:40"],+ ),+ ("kolkata_half_hour_offset", "Asia/Kolkata", datetime(2026, 3, 10, 12, tzinfo=UTC), 24, ["05:15", "06:45"]),+ # 09:59 and 10:01 in Kathmandu, either side of an hour that starts at a quarter past in UTC.+ (+ "kathmandu_quarter_hour_offset",+ "Asia/Kathmandu",+ datetime(2026, 3, 10, 12, tzinfo=UTC),+ 24,+ ["04:14", "04:16"],+ ),+ ]+ )+ def test_every_hourly_run_lands_in_the_trend(+ self, _name: str, timezone: str, now: datetime, hours: int, times: list[str]+ ) -> None:+ self.team.timezone = timezone+ self.team.save()+ judge = self._judge()+ for at in times:+ timestamp = at if "T" in at else f"{now.date().isoformat()}T{at}"+ self._run(judge, f"{timestamp}:00Z", **{"$ai_evaluation_result": True})+ flush_persons_and_events()++ with time_machine.travel(now, tick=False):+ overview = get_evaluations_overview(+ self.team, None, [str(judge.id)], PeriodBounds(start=now - timedelta(hours=hours), end=now)+ )++ [row] = overview.evaluations+ assert overview.interval == "hour"+ assert [volume for volume in row.trend.volume if volume] == [1] * len(times)+ assert row.graded_count == len(times)++ @time_machine.travel(datetime(2026, 11, 1, 18, 0, tzinfo=UTC), tick=False)+ def test_a_window_inside_a_repeated_daylight_saving_hour_reads_its_runs(self) -> None:+ self.team.timezone = "America/New_York"+ self.team.save()+ judge = self._judge()+ # 01:00 on November 1 in New York after the clocks fall back, between 01:40 before and 01:20 after it.+ self._run(judge, "2026-11-01T06:00:00Z", **{"$ai_evaluation_result": False})+ flush_persons_and_events()+ period = PeriodBounds(+ start=datetime(2026, 11, 1, 5, 40, tzinfo=UTC), end=datetime(2026, 11, 1, 6, 20, tzinfo=UTC)+ )++ overview = get_evaluations_overview(self.team, None, [str(judge.id)], period)+ runs = get_evaluation_runs(self.team, None, str(judge.id), period, "failed", 5)++ assert overview.evaluations[0].graded_count == 1+ assert len(runs.results) == 1++ def test_query_size_does_not_grow_with_the_number_of_evaluations(self) -> None:+ def printed_size(count: int) -> int:+ Evaluation.objects.filter(team=self.team).delete()+ evaluations = Evaluation.objects.bulk_create(+ Evaluation(+ team=self.team,+ name=f"Score {index}",+ evaluation_type="hog",+ output_type="numeric",+ output_config={"passing_rule": {"operator": "gte", "threshold": index}},+ )+ for index in range(count)+ )+ ids = [str(evaluation.id) for evaluation in evaluations]+ with patch(+ "products.ai_observability.backend.logic.evaluations.overview_queries.execute_hogql_query"+ ) as execute:+ execute.return_value.results = []+ load_bucket_counts(self.team, None, PERIOD, ids, build_pass_rules(self.team.pk, ids))+ context = HogQLContext(+ team_id=self.team.pk,+ team=self.team,+ enable_select_queries=True,+ database=Database.create_for_posthog_tables(self.team),+ )+ sql, _ = prepare_and_print_ast(execute.call_args.kwargs["query"], context, "clickhouse")+ return len(sql)++ assert printed_size(300) < printed_size(10) * 1.1++ @time_machine.travel(datetime(2026, 3, 10, 12, 30, tzinfo=UTC), tick=False)+ def test_a_day_or_shorter_buckets_by_hour_against_the_hours_before(self) -> None:+ judge = self._judge()+ for timestamp in (+ "2026-03-08T12:00:00Z", # before the previous 24 hours+ "2026-03-08T13:00:00Z", # previous 24 hours+ "2026-03-09T12:00:00Z", # previous 24 hours, in the hour the current period starts+ "2026-03-09T12:45:00Z",+ "2026-03-09T13:10:00Z",+ "2026-03-09T13:50:00Z",+ "2026-03-10T12:10:00Z",+ ):+ self._run(judge, timestamp, **{"$ai_evaluation_result": True})+ flush_persons_and_events()+ now = datetime(2026, 3, 10, 12, 30, tzinfo=UTC)++ overview = get_evaluations_overview(+ self.team, None, [str(judge.id)], PeriodBounds(start=now - timedelta(hours=24), end=now)+ )++ [row] = overview.evaluations+ assert overview.interval == "hour"+ assert overview.buckets[0] == datetime(2026, 3, 9, 12, tzinfo=UTC)+ assert len(overview.buckets) == 25+ assert (overview.previous_start, overview.previous_end) == (+ now - timedelta(hours=48),+ now - timedelta(hours=24),+ )+ assert row.trend.volume[:2] == [1, 2]+ assert row.trend.volume[-1] == 1+ assert sum(row.trend.volume) == 4+ assert row.previous_graded_count == 2diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py b/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.pyindex 70726ceb13dc..f520f35d15cd 100644--- a/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py+++ b/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py@@ -4,10 +4,12 @@ from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flusfrom parameterized import parameterized+from posthog.hogql.parser import parse_exprfrom posthog.hogql.query import execute_hogql_queryfrom products.ai_observability.backend.facade.api import get_evaluation_pass_filtersfrom products.ai_observability.backend.facade.evaluation_signals import evaluation_output_changed+from products.ai_observability.backend.logic.evaluations.pass_filters import PassRules, build_pass_rulesfrom products.ai_observability.backend.models.evaluations import EvaluationCATEGORY_OPTIONS = [{"key": "ok", "label": "Ok"}, {"key": "bad", "label": "Bad"}]@@ -20,18 +22,33 @@ def _evaluation(team, output_type: str = "boolean", output_config: dict | None =)+def _per_evaluation_rules(team_id: int, _evaluation_ids: list[str]) -> PassRules:+ filters = get_evaluation_pass_filters(team_id)+ return PassRules(graded=parse_expr(filters.graded), passes=parse_expr(filters.passed))++class TestEvaluationPassFilters(ClickhouseTestMixin, BaseTest):- def test_grades_each_kind_of_run_by_its_evaluation_rule(self) -> None:+ @parameterized.expand([("per_evaluation", _per_evaluation_rules), ("by_kind_of_rule", build_pass_rules)])+ def test_grades_each_kind_of_run_by_its_evaluation_rule(+ self, _name: str, build: Callable[[int, list[str]], PassRules]+ ) -> None:plain = _evaluation(self.team)detector = _evaluation(self.team, output_config={"true_is_failure": True})deleted_detector = _evaluation(self.team, output_config={"true_is_failure": True}, deleted=True)numeric = _evaluation(self.team, "numeric", {"passing_rule": {"operator": "gte", "threshold": 4}})+ numeric_lower = _evaluation(self.team, "numeric", {"passing_rule": {"operator": "gte", "threshold": 2}})+ numeric_at_most = _evaluation(self.team, "numeric", {"passing_rule": {"operator": "lte", "threshold": 4}})numeric_without_rule = _evaluation(self.team, "numeric", {})categorical = _evaluation(self.team,"categorical",{"options": CATEGORY_OPTIONS, "selection_mode": "single", "passing_rule": {"categories": ["ok"]}},)+ bad_passes = _evaluation(+ self.team,+ "categorical",+ {"options": CATEGORY_OPTIONS, "selection_mode": "multiple", "passing_rule": {"categories": ["bad"]}},+ )empty_passes = _evaluation(self.team,"categorical",@@ -49,6 +66,19 @@ class TestEvaluationPassFilters(ClickhouseTestMixin, BaseTest):"deleted detector false": (deleted_detector, {"$ai_evaluation_result": False}, (True, True)),"numeric at threshold": (numeric, {"$ai_evaluation_numeric_result": 4}, (True, True)),"numeric below threshold": (numeric, {"$ai_evaluation_numeric_result": 3.5}, (True, False)),+ "lower threshold below the other": (numeric_lower, {"$ai_evaluation_numeric_result": 3.5}, (True, True)),+ "at most, above": (numeric_at_most, {"$ai_evaluation_numeric_result": 4.5}, (True, False)),+ "at most, at threshold": (numeric_at_most, {"$ai_evaluation_numeric_result": 4}, (True, True)),+ "numeric n/a": (+ numeric,+ {"$ai_evaluation_numeric_result": 5, "$ai_evaluation_applicable": False},+ (False, False),+ ),+ "numeric skipped": (+ numeric,+ {"$ai_evaluation_numeric_result": 5, "$ai_evaluation_skipped": True},+ (False, False),+ ),"numeric without rule": (numeric_without_rule, {"$ai_evaluation_numeric_result": 5}, (False, False)),"categorical passing key": (categorical,@@ -60,6 +90,16 @@ class TestEvaluationPassFilters(ClickhouseTestMixin, BaseTest):{"$ai_evaluation_categorical_result": ["bad"], "$ai_evaluation_applicable": True},(True, False),),+ "other passing key": (+ bad_passes,+ {"$ai_evaluation_categorical_result": ["bad"], "$ai_evaluation_applicable": True},+ (True, True),+ ),+ "other passing key, both chosen": (+ bad_passes,+ {"$ai_evaluation_categorical_result": ["ok", "bad"], "$ai_evaluation_applicable": True},+ (True, False),+ ),"empty passes, empty": (empty_passes,{"$ai_evaluation_categorical_result": [], "$ai_evaluation_applicable": True},@@ -91,19 +131,20 @@ class TestEvaluationPassFilters(ClickhouseTestMixin, BaseTest):)flush_persons_and_events()- filters = get_evaluation_pass_filters(self.team.id)+ rules = build(self.team.id, list({str(evaluation.id) for evaluation, _, _ in cases.values()}))response = execute_hogql_query(- f"""+ """SELECTJSONExtractString(properties, 'case'),- countIf({filters.graded}) > 0,- countIf({filters.passed}) > 0,- countIf({filters.graded} AND NOT {filters.passed}) > 0+ countIf({graded}) > 0,+ countIf({graded} AND {passes}) > 0,+ countIf({graded} AND NOT {passes}) > 0FROM eventsWHERE event = '$ai_evaluation'GROUP BY 1""",team=self.team,+ placeholders={"graded": rules.graded, "passes": rules.passes},)graded_passed_failed = {diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_runs_and_scores.py b/products/ai_observability/backend/tests/logic/evaluations/test_runs_and_scores.pynew file mode 100644index 000000000000..7e69f8971d0b--- /dev/null+++ b/products/ai_observability/backend/tests/logic/evaluations/test_runs_and_scores.py@@ -0,0 +1,114 @@+from datetime import UTC, datetime++import pytest+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events++from parameterized import parameterized++from products.ai_observability.backend.facade import contracts+from products.ai_observability.backend.facade.api import (+ NoPassingRuleError,+ NoScoreScaleError,+ get_evaluation_runs,+ get_evaluation_score_distribution,+)+from products.ai_observability.backend.logic.evaluations.period import PeriodBounds+from products.ai_observability.backend.logic.evaluations.score_distribution import score_buckets+from products.ai_observability.backend.models.evaluations import Evaluation++PERIOD = PeriodBounds(start=datetime(2026, 3, 8, tzinfo=UTC), end=datetime(2026, 3, 11, tzinfo=UTC))+++ [+ ("small integer scale", 1, 5, None, ("1", "2", "3", "4", "5")),+ ("zero to one", 0, 1, None, tuple(f"{index / 10:g}-{(index + 1) / 10:g}" for index in range(10))),+ ("whole step on a short scale", 0, 1, 1, ("0", "1")),+ ("fractional step", 1, 5, 0.5, tuple(f"{1 + index * 0.4:g}-{1 + (index + 1) * 0.4:g}" for index in range(10))),+ ("wide integer scale", 0, 100, None, tuple(f"{index * 10}-{(index + 1) * 10}" for index in range(10))),+ ]+)+def test_score_buckets(_name: str, low: float, high: float, step: float | None, labels: tuple[str, ...]) -> None:+ buckets = score_buckets(contracts.ScoreScale(min=low, max=high), step)+ assert buckets is not None and buckets.labels == labels+++class TestEvaluationRunsAndScores(ClickhouseTestMixin, BaseTest):+ def _evaluation(self, output_type: str, output_config: dict) -> Evaluation:+ return Evaluation.objects.create(+ team=self.team,+ name=output_type,+ evaluation_type="hog",+ evaluation_config={"source": "return true"},+ output_type=output_type,+ output_config=output_config,+ )++ def _run(self, evaluation: Evaluation, timestamp: str, **properties: object) -> None:+ _create_event(+ team=self.team,+ event="$ai_evaluation",+ distinct_id="user-1",+ timestamp=timestamp,+ properties={"$ai_evaluation_id": str(evaluation.id), **properties},+ )++ def test_runs_and_score_distribution(self) -> None:+ judge = self._evaluation("boolean", {})+ relevance = self._evaluation(+ "numeric", {"min": 1, "max": 5, "passing_rule": {"operator": "gte", "threshold": 4}}+ )+ fraction = self._evaluation("numeric", {"min": 0, "max": 1})+ for hour in range(7):+ self._run(+ judge,+ f"2026-03-09T1{hour}:00:00Z",+ **{+ "$ai_evaluation_result": False,+ "$ai_evaluation_reasoning": "x" * 2000 if hour == 6 else f"reason {hour}",+ "$ai_target_event_id": f"generation-{hour}",+ "$ai_trace_id": f"trace-{hour}",+ },+ )+ self._run(judge, "2026-03-09T20:00:00Z", **{"$ai_evaluation_result": True, "$ai_trace_id": "trace-passed"})+ self._run(judge, "2026-03-09T21:00:00Z", **{"$ai_evaluation_result": False, "$ai_evaluation_skipped": True})+ self._run(judge, "2026-03-01T10:00:00Z", **{"$ai_evaluation_result": False})+ numeric = {"$ai_evaluation_result_type": "numeric"}+ for score in (1, 4, 4, 5, 4.4):+ self._run(relevance, "2026-03-09T10:00:00Z", **numeric, **{"$ai_evaluation_numeric_result": score})+ # Float division puts 0.3, 0.6 and 0.7 just under their bucket's lower edge.+ for score in (0.0, 0.3, 0.6, 0.7, 1.0):+ self._run(fraction, "2026-03-09T10:00:00Z", **numeric, **{"$ai_evaluation_numeric_result": score})+ flush_persons_and_events()++ failures = get_evaluation_runs(self.team, None, str(judge.id), PERIOD, "failed", 5).results+ assert [run.trace_id for run in failures] == ["trace-6", "trace-5", "trace-4", "trace-3", "trace-2"]+ assert {run.outcome for run in failures} == {"failed"}+ assert failures[0].reasoning == "x" * 999 + "…"+ assert failures[1].target_event_id == "generation-5"+ passed = get_evaluation_runs(self.team, None, str(judge.id), PERIOD, "passed", 5).results+ assert [(run.trace_id, run.outcome) for run in passed] == [("trace-passed", "passed")]+ # Without an outcome, the skipped run at 21:00 is left out.+ graded = get_evaluation_runs(self.team, None, str(judge.id), PERIOD, None, 2).results+ assert [run.outcome for run in graded] == ["passed", "failed"]+ relevance_failures = get_evaluation_runs(self.team, None, str(relevance.id), PERIOD, "failed", 5).results+ assert [run.timestamp.hour for run in relevance_failures] == [10]+ # An evaluation without a passing rule still lists its scored runs, with no outcome.+ scored = get_evaluation_runs(self.team, None, str(fraction.id), PERIOD, None, 50).results+ assert len(scored) == 5 and {run.outcome for run in scored} == {None}+ with pytest.raises(NoPassingRuleError):+ get_evaluation_runs(self.team, None, str(fraction.id), PERIOD, "failed", 5)++ assert get_evaluation_score_distribution(+ self.team, None, str(relevance.id), PERIOD+ ) == contracts.ScoreHistogram(+ scale=contracts.ScoreScale(min=1, max=5),+ buckets=[+ contracts.ScoreBucket(label=str(score), count=count)+ for score, count in zip(range(1, 6), [1, 0, 0, 3, 1])+ ],+ )+ histogram = get_evaluation_score_distribution(self.team, None, str(fraction.id), PERIOD)+ assert [bucket.count for bucket in histogram.buckets] == [1, 0, 0, 1, 0, 0, 1, 1, 0, 1]+ with pytest.raises(NoScoreScaleError):+ get_evaluation_score_distribution(self.team, None, str(judge.id), PERIOD)diff --git a/products/ai_observability/frontend/evaluations/constants.ts b/products/ai_observability/frontend/evaluations/constants.tsindex 2544eaeef2f9..227e31315595 100644--- a/products/ai_observability/frontend/evaluations/constants.ts+++ b/products/ai_observability/frontend/evaluations/constants.ts@@ -16,7 +16,7 @@ const EVALUATION_RESULT_FALSE_HOGQL = "properties.$ai_evaluation_result = 'false// A skipped run was never graded, but an evaluation that disallows N/A still emits result=false// alongside skipped=true, so any count that doesn't exclude skips reads them as failures. Mirrors-// _NOT_SKIPPED_PREDICATE in products/ai_observability/backend/models/evaluation_outcomes.py, which+// NOT_SKIPPED_PREDICATE in products/ai_observability/backend/models/evaluation_outcomes.py, which// the report metrics already apply — without this the two surfaces disagree about the same runs.export const EVALUATION_NOT_SKIPPED_HOGQL ="(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')"diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.tsindex bd2e7ca5d821..5a6edcfd7226 100644--- a/products/ai_observability/frontend/generated/api.schemas.ts+++ b/products/ai_observability/frontend/generated/api.schemas.ts@@ -2613,6 +2613,234 @@ export interface PatchedEvaluationApi {readonly user_access_level?: string | null}+/**+ * Null when the evaluation has no passing rule, or the run was not graded by it.+ */+export type EvaluationRunApiOutcome = (typeof EvaluationRunApiOutcome)[keyof typeof EvaluationRunApiOutcome] | null++export const EvaluationRunApiOutcome = {+ Passed: 'passed',+ Failed: 'failed',+} as const++export interface EvaluationRunApi {+ timestamp: string+ /** Null when the evaluation has no passing rule, or the run was not graded by it. */+ outcome: EvaluationRunApiOutcome+ /** The evaluator's reasoning, cut to about 1,000 characters. */+ reasoning: string | null+ /** The generation the run graded. Null for trace and session runs. */+ targetEventId: string | null+ traceId: string | null+}++export interface EvaluationRunsApi {+ /** Newest first. */+ results: EvaluationRunApi[]+}++export interface ScoreScaleApi {+ min: number+ max: number+}++export interface ScoreBucketApi {+ label: string+ count: number+}++export interface ScoreHistogramApi {+ scale: ScoreScaleApi+ /** One bucket per integer on a small integer scale, else ten equal buckets across the scale. */+ buckets: ScoreBucketApi[]+}++export type EvaluationsOverviewIntervalEnumApi =+ (typeof EvaluationsOverviewIntervalEnumApi)[keyof typeof EvaluationsOverviewIntervalEnumApi]++export const EvaluationsOverviewIntervalEnumApi = {+ Hour: 'hour',+ Day: 'day',+} as const++export interface PassFailCountsApi {+ passed: number+ failed: number+}++export interface ScoreDistributionApi {+ mean: number+ /** The configured score bounds, or null when the evaluation sets none. */+ scale: ScoreScaleApi | null+}++export interface PassRateSummaryApi {+ kind: 'pass_rate'+ passed: number+ failed: number+ /** Counts in the previous period, or null when it graded too few to compare. */+ previous: PassFailCountsApi | null+ /** The score of numeric evaluations with a passing rule. */+ score: ScoreDistributionApi | null+}++export interface ScoreSummaryApi {+ kind: 'score'+ /** Null when the current period scored nothing. */+ score: ScoreDistributionApi | null+ previousMean: number | null+}++export interface CategoryCountApi {+ key: string+ label: string+ count: number+ /** Percentage of the previous period's graded runs with this category, or null when it graded too few to compare. */+ previousShare: number | null+}++export interface CategoriesSummaryApi {+ kind: 'categories'+ multiple: boolean+ /** Sorted by count, most frequent first. */+ categories: CategoryCountApi[]+ /** Multiple-selection runs that returned no category. */+ noCategoriesCount: number+}++export interface SentimentSummaryApi {+ kind: 'sentiment'+ positive: number+ neutral: number+ negative: number+ previousNegativeShare: number | null+}++export interface EvaluationResultTrendApi {+ /** Headline value per bucket, aligned with the overview's `buckets`. Null where nothing was graded. */+ values: (number | null)[]+ /** Graded runs per bucket. */+ volume: number[]+}++export type ResultChangeUnitEnumApi = (typeof ResultChangeUnitEnumApi)[keyof typeof ResultChangeUnitEnumApi]++export const ResultChangeUnitEnumApi = {+ Pp: 'pp',+ Points: 'points',+} as const++export interface ResultChangeApi {+ /** Signed difference against the previous period, rounded for display. */+ value: number+ unit: ResultChangeUnitEnumApi+ /** What changed when it is not the headline, such as a category label. */+ subject: string | null+}++export type AttentionReasonsEnumApi = (typeof AttentionReasonsEnumApi)[keyof typeof AttentionReasonsEnumApi]++export const AttentionReasonsEnumApi = {+ Errored: 'errored',+ Silent: 'silent',+ Worse: 'worse',+} as const++export type BucketStatusEnumApi = (typeof BucketStatusEnumApi)[keyof typeof BucketStatusEnumApi]++export const BucketStatusEnumApi = {+ None: 'none',+ Dip: 'dip',+ Normal: 'normal',+} as const++export interface EvaluationResultApi {+ id: string+ name: string+ directoryName: string | null+ evaluationType: string+ outputType: string+ enabled: boolean+ /** Why the evaluation stopped with an error, as an evaluation status reason, or null when it has none. */+ statusReason: string | null+ createdBy: string | null+ gradedCount: number+ /** Graded runs in the previous period, or null when none. */+ previousGradedCount: number | null+ notApplicableCount: number+ /** Runs skipped before scoring. */+ skippedCount: number+ /** The latest graded run in the current or previous period. */+ lastRunAt: string | null+ summary: PassRateSummaryApi | ScoreSummaryApi | CategoriesSummaryApi | SentimentSummaryApi+ trend: EvaluationResultTrendApi+ /** Change against the previous period, or null when not comparable. */+ change: ResultChangeApi | null+ lowEvidence: boolean+ attentionReasons: AttentionReasonsEnumApi[]+ /** Each bucket against the previous period. A bucket with too few graded runs to judge is never a dip. */+ bucketStatus: BucketStatusEnumApi[]+ /** Key of the category a categorical evaluation's trend follows. */+ trackedCategory: string | null+}++/**+ * The same figure for both periods, over only the evaluations with results in both.+ */+export interface PeriodComparisonApi {+ current: number+ previous: number+}++export interface OverviewSentimentApi {+ evaluationId: string+ negativeShare: number+ previousShare: number | null+}++export interface OverviewTotalsApi {+ graded: number+ /** Graded runs in each period, over evaluations with results in both. Null when none has. */+ gradedComparison: PeriodComparisonApi | null+ gradedTrend: number[]+ sentiment: OverviewSentimentApi | null+}++export interface PassRateOverviewEntryApi {+ evaluationId: string+ rate: number+ lowEvidence: boolean+}++export interface PassRateOverviewApi {+ /** Enabled pass rate evaluations, lowest rate first. */+ entries: PassRateOverviewEntryApi[]+ /** Median of the per-evaluation pass rates without low evidence. */+ median: number | null+ /** Median pass rate in each period, over evaluations with results in both. Null when none has. */+ medianComparison: PeriodComparisonApi | null+ /** Median pass rate per bucket, aligned with `buckets`. */+ medianTrend: (number | null)[]+}++export interface EvaluationsOverviewApi {+ /** Start of the period, in the project's timezone. */+ start: string+ /** End of the period, exclusive. */+ end: string+ previousStart: string+ /** End of the previous period. While the current period runs, the previous one stops at the same point, so both cover the same length of time. */+ previousEnd: string+ /** `hour` for a period of two days or less, else `day`. */+ interval: EvaluationsOverviewIntervalEnumApi+ /** Start of each bucket, in the project's timezone. Every series is aligned with it. The first bucket can start before `start`. */+ buckets: string[]+ evaluations: EvaluationResultApi[]+ /** Totals over the enabled evaluations. */+ totals: OverviewTotalsApi+ passRateOverview: PassRateOverviewApi+}+export type TestHogRequestApiOutputConfigOptionsItem = {/*** Stable category key.@@ -5427,6 +5655,77 @@ export type EvaluationsBackfillsListParams = {offset?: number}+export type EvaluationsRunsRetrieveParams = {+ /**+ * Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. Defaults to 13 days before `date_to`.+ */+ date_from?: string+ /**+ * End of the period: a date includes that whole day, a datetime is an exclusive instant. Defaults to today.+ */+ date_to?: string+ /**+ * How many runs to return, newest first.+ * @minimum 1+ * @maximum 50+ */+ limit?: number+ /**+ * Only runs with this outcome under the evaluation's passing rule. Without it, every graded run, including scored, categorized and sentiment runs. An evaluation without a passing rule rejects this filter with a 400.+ *+ * * `passed` - Passed+ * * `failed` - Failed+ * @minLength 1+ */+ outcome?: EvaluationsRunsRetrieveOutcome+ /**+ * Skip the result cached for the last five minutes and compute it again.+ */+ refresh?: boolean+}++export type EvaluationsRunsRetrieveOutcome =+ (typeof EvaluationsRunsRetrieveOutcome)[keyof typeof EvaluationsRunsRetrieveOutcome]++export const EvaluationsRunsRetrieveOutcome = {+ Passed: 'passed',+ Failed: 'failed',+} as const++export type EvaluationsScoreDistributionRetrieveParams = {+ /**+ * Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. Defaults to 13 days before `date_to`.+ */+ date_from?: string+ /**+ * End of the period: a date includes that whole day, a datetime is an exclusive instant. Defaults to today.+ */+ date_to?: string+ /**+ * Skip the result cached for the last five minutes and compute it again.+ */+ refresh?: boolean+}++export type EvaluationsOverviewRetrieveParams = {+ /**+ * Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. Defaults to 13 days before `date_to`.+ */+ date_from?: string+ /**+ * End of the period: a date includes that whole day, a datetime is an exclusive instant. Defaults to today.+ */+ date_to?: string+ /**+ * Only these evaluations. Defaults to every evaluation the user can see.+ */+ evaluation_ids?: string[]+ /**+ * Skip the result cached for the last five minutes and compute it again.+ */+ refresh?: boolean+}+export type LlmAnalyticsClusteringJobsListParams = {/*** Number of results to return per page.diff --git a/products/ai_observability/frontend/generated/api.ts b/products/ai_observability/frontend/generated/api.tsindex bf3d0381929c..1575dcf51300 100644--- a/products/ai_observability/frontend/generated/api.ts+++ b/products/ai_observability/frontend/generated/api.ts@@ -49,8 +49,13 @@ import type {EvaluationReportUpdateApi,EvaluationRunRequestApi,EvaluationRunResponseApi,+ EvaluationRunsApi,EvaluationsBackfillsListParams,EvaluationsListParams,+ EvaluationsOverviewApi,+ EvaluationsOverviewRetrieveParams,+ EvaluationsRunsRetrieveParams,+ EvaluationsScoreDistributionRetrieveParams,ExperimentReceiptApi,ExperimentSubmissionApi,InstrumentationCheckActionApi,@@ -131,6 +136,7 @@ import type {ScoreDefinitionNewVersionApi,ScoreDefinitionVersionApi,ScoreDefinitionVersionPageApi,+ ScoreHistogramApi,SummarizeRequestApi,SummarizeResponseApi,TaggerApi,@@ -1464,6 +1470,106 @@ export const evaluationsDestroy = async (projectId: string, id: string, options?})}+export const getEvaluationsRunsRetrieveUrl = (+ projectId: string,+ id: string,+ params?: EvaluationsRunsRetrieveParams+) => {+ const normalizedParams = new URLSearchParams()++ Object.entries(params || {}).forEach(([key, value]) => {+ if (value !== undefined) {+ normalizedParams.append(key, value === null ? 'null' : String(value))+ }+ })++ const stringifiedParams = normalizedParams.toString()++ return stringifiedParams.length > 0+ ? `/api/projects/${projectId}/evaluations/${id}/runs/?${stringifiedParams}`+ : `/api/projects/${projectId}/evaluations/${id}/runs/`+}++/**+ * One evaluation's latest graded runs in a period, newest first, optionally only the passed or failed ones.+ */+export const evaluationsRunsRetrieve = async (+ projectId: string,+ id: string,+ params?: EvaluationsRunsRetrieveParams,+ options?: RequestInit+): Promise<EvaluationRunsApi> => {+ return apiMutator<EvaluationRunsApi>(getEvaluationsRunsRetrieveUrl(projectId, id, params), {+ ...options,+ method: 'GET',+ })+}++export const getEvaluationsScoreDistributionRetrieveUrl = (+ projectId: string,+ id: string,+ params?: EvaluationsScoreDistributionRetrieveParams+) => {+ const normalizedParams = new URLSearchParams()++ Object.entries(params || {}).forEach(([key, value]) => {+ if (value !== undefined) {+ normalizedParams.append(key, value === null ? 'null' : String(value))+ }+ })++ const stringifiedParams = normalizedParams.toString()++ return stringifiedParams.length > 0+ ? `/api/projects/${projectId}/evaluations/${id}/score_distribution/?${stringifiedParams}`+ : `/api/projects/${projectId}/evaluations/${id}/score_distribution/`+}++/**+ * How one numeric evaluation's scores in a period spread across its configured scale.+ */+export const evaluationsScoreDistributionRetrieve = async (+ projectId: string,+ id: string,+ params?: EvaluationsScoreDistributionRetrieveParams,+ options?: RequestInit+): Promise<ScoreHistogramApi> => {+ return apiMutator<ScoreHistogramApi>(getEvaluationsScoreDistributionRetrieveUrl(projectId, id, params), {+ ...options,+ method: 'GET',+ })+}++export const getEvaluationsOverviewRetrieveUrl = (projectId: string, params?: EvaluationsOverviewRetrieveParams) => {+ const normalizedParams = new URLSearchParams()++ Object.entries(params || {}).forEach(([key, value]) => {+ if (value !== undefined) {+ normalizedParams.append(key, value === null ? 'null' : String(value))+ }+ })++ const stringifiedParams = normalizedParams.toString()++ return stringifiedParams.length > 0+ ? `/api/projects/${projectId}/evaluations/overview/?${stringifiedParams}`+ : `/api/projects/${projectId}/evaluations/overview/`+}++/**+ * Each evaluation's results over a period, compared with the period of equal length before it.+ */+export const evaluationsOverviewRetrieve = async (+ projectId: string,+ params?: EvaluationsOverviewRetrieveParams,+ options?: RequestInit+): Promise<EvaluationsOverviewApi> => {+ return apiMutator<EvaluationsOverviewApi>(getEvaluationsOverviewRetrieveUrl(projectId, params), {+ ...options,+ method: 'GET',+ })+}+export const getEvaluationsTestHogCreateUrl = (projectId: string) => {return `/api/projects/${projectId}/evaluations/test_hog/`}diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 14fb2b14d8eb..2f246e41adf8 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -13328,6 +13328,15 @@ export namespace Schemas {limit: number;}+ export type AttentionReasonsEnum = typeof AttentionReasonsEnum[keyof typeof AttentionReasonsEnum];+++ export const AttentionReasonsEnum = {+ Errored: 'errored',+ Silent: 'silent',+ Worse: 'worse',+ } as const;+export interface AttributeBreakdownRow {count: number;error_count: number;@@ -19188,6 +19197,15 @@ export namespace Schemas {limit: number;}+ export type BucketStatusEnum = typeof BucketStatusEnum[keyof typeof BucketStatusEnum];+++ export const BucketStatusEnum = {+ None: 'none',+ Dip: 'dip',+ Normal: 'normal',+ } as const;+/*** * `distinct_id` - User ID (default)* * `device_id` - Device ID@@ -22254,6 +22272,23 @@ export namespace Schemas {passing_rule?: CategoricalScorePassingRule | null;}+ export interface CategoryCount {+ key: string;+ label: string;+ count: number;+ /** Percentage of the previous period's graded runs with this category, or null when it graded too few to compare. */+ previousShare: number | null;+ }++ export interface CategoriesSummary {+ kind: 'categories';+ multiple: boolean;+ /** Sorted by count, most frequent first. */+ categories: CategoryCount[];+ /** Multiple-selection runs that returned no category. */+ noCategoriesCount: number;+ }+export interface CdcEnableResponse {/** Whether CDC was enabled on the source. */success: boolean;@@ -43995,6 +44030,122 @@ export namespace Schemas {readonly created_at: string;}+ export interface PassFailCounts {+ passed: number;+ failed: number;+ }++ export interface ScoreScale {+ min: number;+ max: number;+ }++ export interface ScoreDistribution {+ mean: number;+ /** The configured score bounds, or null when the evaluation sets none. */+ scale: ScoreScale | null;+ }++ export interface PassRateSummary {+ kind: 'pass_rate';+ passed: number;+ failed: number;+ /** Counts in the previous period, or null when it graded too few to compare. */+ previous: PassFailCounts | null;+ /** The score of numeric evaluations with a passing rule. */+ score: ScoreDistribution | null;+ }++ export interface ScoreSummary {+ kind: 'score';+ /** Null when the current period scored nothing. */+ score: ScoreDistribution | null;+ previousMean: number | null;+ }++ export interface SentimentSummary {+ kind: 'sentiment';+ positive: number;+ neutral: number;+ negative: number;+ previousNegativeShare: number | null;+ }++ export interface EvaluationResultTrend {+ /** Headline value per bucket, aligned with the overview's `buckets`. Null where nothing was graded. */+ values: (number | null)[];+ /** Graded runs per bucket. */+ volume: number[];+ }++ export type ResultChangeUnitEnum = typeof ResultChangeUnitEnum[keyof typeof ResultChangeUnitEnum];+++ export const ResultChangeUnitEnum = {+ Pp: 'pp',+ Points: 'points',+ } as const;++ export interface ResultChange {+ /** Signed difference against the previous period, rounded for display. */+ value: number;+ unit: ResultChangeUnitEnum;+ /** What changed when it is not the headline, such as a category label. */+ subject: string | null;+ }++ export interface EvaluationResult {+ id: string;+ name: string;+ directoryName: string | null;+ evaluationType: string;+ outputType: string;+ enabled: boolean;+ /** Why the evaluation stopped with an error, as an evaluation status reason, or null when it has none. */+ statusReason: string | null;+ createdBy: string | null;+ gradedCount: number;+ /** Graded runs in the previous period, or null when none. */+ previousGradedCount: number | null;+ notApplicableCount: number;+ /** Runs skipped before scoring. */+ skippedCount: number;+ /** The latest graded run in the current or previous period. */+ lastRunAt: string | null;+ summary: PassRateSummary | ScoreSummary | CategoriesSummary | SentimentSummary;+ trend: EvaluationResultTrend;+ /** Change against the previous period, or null when not comparable. */+ change: ResultChange | null;+ lowEvidence: boolean;+ attentionReasons: AttentionReasonsEnum[];+ /** Each bucket against the previous period. A bucket with too few graded runs to judge is never a dip. */+ bucketStatus: BucketStatusEnum[];+ /** Key of the category a categorical evaluation's trend follows. */+ trackedCategory: string | null;+ }++ /**+ * Null when the evaluation has no passing rule, or the run was not graded by it.+ */+ export type EvaluationRunOutcome = typeof EvaluationRunOutcome[keyof typeof EvaluationRunOutcome] | null;+++ export const EvaluationRunOutcome = {+ Passed: 'passed',+ Failed: 'failed',+ } as const;++ export interface EvaluationRun {+ timestamp: string;+ /** Null when the evaluation has no passing rule, or the run was not graded by it. */+ outcome: EvaluationRunOutcome;+ /** The evaluator's reasoning, cut to about 1,000 characters. */+ reasoning: string | null;+ /** The generation the run graded. Null for trace and session runs. */+ targetEventId: string | null;+ traceId: string | null;+ }+export interface EvaluationRunEvaluation {/** UUID of the evaluation being run. */id: string;@@ -44029,6 +44180,76 @@ export namespace Schemas {target_event_id: string;}+ export interface EvaluationRuns {+ /** Newest first. */+ results: EvaluationRun[];+ }++ export type EvaluationsOverviewIntervalEnum = typeof EvaluationsOverviewIntervalEnum[keyof typeof EvaluationsOverviewIntervalEnum];+++ export const EvaluationsOverviewIntervalEnum = {+ Hour: 'hour',+ Day: 'day',+ } as const;++ /**+ * The same figure for both periods, over only the evaluations with results in both.+ */+ export interface PeriodComparison {+ current: number;+ previous: number;+ }++ export interface OverviewSentiment {+ evaluationId: string;+ negativeShare: number;+ previousShare: number | null;+ }++ export interface OverviewTotals {+ graded: number;+ /** Graded runs in each period, over evaluations with results in both. Null when none has. */+ gradedComparison: PeriodComparison | null;+ gradedTrend: number[];+ sentiment: OverviewSentiment | null;+ }++ export interface PassRateOverviewEntry {+ evaluationId: string;+ rate: number;+ lowEvidence: boolean;+ }++ export interface PassRateOverview {+ /** Enabled pass rate evaluations, lowest rate first. */+ entries: PassRateOverviewEntry[];+ /** Median of the per-evaluation pass rates without low evidence. */+ median: number | null;+ /** Median pass rate in each period, over evaluations with results in both. Null when none has. */+ medianComparison: PeriodComparison | null;+ /** Median pass rate per bucket, aligned with `buckets`. */+ medianTrend: (number | null)[];+ }++ export interface EvaluationsOverview {+ /** Start of the period, in the project's timezone. */+ start: string;+ /** End of the period, exclusive. */+ end: string;+ previousStart: string;+ /** End of the previous period. While the current period runs, the previous one stops at the same point, so both cover the same length of time. */+ previousEnd: string;+ /** `hour` for a period of two days or less, else `day`. */+ interval: EvaluationsOverviewIntervalEnum;+ /** Start of each bucket, in the project's timezone. Every series is aligned with it. The first bucket can start before `start`. */+ buckets: string[];+ evaluations: EvaluationResult[];+ /** Totals over the enabled evaluations. */+ totals: OverviewTotals;+ passRateOverview: PassRateOverview;+ }+export interface EventDefinitionBasic {id: string;name: string;@@ -103505,6 +103726,11 @@ export namespace Schemas {effective_destination_ids?: string[];}+ export interface ScoreBucket {+ label: string;+ count: number;+ }+export interface ScoreDefinitionCreate {/*** Tag names. Replaces existing tags; send [] to remove all tags. Names are trimmed and lowercased.@@ -103597,6 +103823,12 @@ export namespace Schemas {results: ScoreDefinitionVersion[];}+ export interface ScoreHistogram {+ scale: ScoreScale;+ /** One bucket per integer on a small integer scale, else ten equal buckets across the scale. */+ buckets: ScoreBucket[];+ }+export interface ScoringActivateRequest {/** Saved scoring version to activate for subsequent evaluations. */config_id: string;@@ -129486,6 +129718,77 @@ export namespace Schemas {offset?: number;};+ export type EvaluationsRunsRetrieveParams = {+ /**+ * Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. Defaults to 13 days before `date_to`.+ */+ date_from?: string;+ /**+ * End of the period: a date includes that whole day, a datetime is an exclusive instant. Defaults to today.+ */+ date_to?: string;+ /**+ * How many runs to return, newest first.+ * @minimum 1+ * @maximum 50+ */+ limit?: number;+ /**+ * Only runs with this outcome under the evaluation's passing rule. Without it, every graded run, including scored, categorized and sentiment runs. An evaluation without a passing rule rejects this filter with a 400.+ *+ * * `passed` - Passed+ * * `failed` - Failed+ * @minLength 1+ */+ outcome?: EvaluationsRunsRetrieveOutcome;+ /**+ * Skip the result cached for the last five minutes and compute it again.+ */+ refresh?: boolean;+ };++ export type EvaluationsRunsRetrieveOutcome = typeof EvaluationsRunsRetrieveOutcome[keyof typeof EvaluationsRunsRetrieveOutcome];+++ export const EvaluationsRunsRetrieveOutcome = {+ Passed: 'passed',+ Failed: 'failed',+ } as const;++ export type EvaluationsScoreDistributionRetrieveParams = {+ /**+ * Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. Defaults to 13 days before `date_to`.+ */+ date_from?: string;+ /**+ * End of the period: a date includes that whole day, a datetime is an exclusive instant. Defaults to today.+ */+ date_to?: string;+ /**+ * Skip the result cached for the last five minutes and compute it again.+ */+ refresh?: boolean;+ };++ export type EvaluationsOverviewRetrieveParams = {+ /**+ * Start of the period: a date for the whole day in the project's timezone, or an ISO 8601 datetime. Defaults to 13 days before `date_to`.+ */+ date_from?: string;+ /**+ * End of the period: a date includes that whole day, a datetime is an exclusive instant. Defaults to today.+ */+ date_to?: string;+ /**+ * Only these evaluations. Defaults to every evaluation the user can see.+ */+ evaluation_ids?: string[];+ /**+ * Skip the result cached for the last five minutes and compute it again.+ */+ refresh?: boolean;+ };+export type EventDefinitionsListParams = {/*** `event_custom` keeps only names without a `$` prefix and `event_posthog` only names with one. Default `event`.