:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): show results over time on the evaluation page (#123427)
Bernat Torresbfd2b7ae0037
diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.pyindex e6a11e200495..df4377acccf5 100644--- a/products/ai_observability/backend/api/evaluations.py+++ b/products/ai_observability/backend/api/evaluations.py@@ -1629,7 +1629,7 @@ class EvaluationViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, Forbiquery_serializer=EvaluationsOverviewQuerySerializer,responses={200: OpenApiResponse(response=contracts.EvaluationsOverview),- 400: OpenApiResponse(description="The period is malformed, empty, longer than 90 days or out of range."),+ 400: OpenApiResponse(description="The period is malformed, empty, longer than 91 days or out of range."),429: OpenApiResponse(description="Too many queries are running for the project or organization."),},)diff --git a/products/ai_observability/backend/facade/api.py b/products/ai_observability/backend/facade/api.pyindex e33c2afef281..a81869fbb9db 100644--- a/products/ai_observability/backend/facade/api.py+++ b/products/ai_observability/backend/facade/api.py@@ -225,7 +225,7 @@ def _to_evaluation_result(result: EvaluationResult) -> contracts.TraceEvaluationdef evaluations_period(team: Team, date_from: date | datetime | None, date_to: date | datetime | None) -> PeriodBounds:- """Raises `InvalidPeriodError` for a period that is empty, longer than 90 days or out of range."""+ """Raises `InvalidPeriodError` for a period that is empty, longer than 91 days or out of range."""return PeriodBounds.from_request(date_from, date_to, team.timezone_info)diff --git a/products/ai_observability/backend/logic/evaluations/period.py b/products/ai_observability/backend/logic/evaluations/period.pyindex 831c880e02c2..7d04637f6b87 100644--- a/products/ai_observability/backend/logic/evaluations/period.py+++ b/products/ai_observability/backend/logic/evaluations/period.py@@ -8,7 +8,8 @@ Interval = Literal["hour", "day"]# A period this short or shorter is bucketed by hour, so a day or two still draws a line rather than one or two dots.MAX_HOURLY_SPAN = timedelta(days=2)DEFAULT_DAYS = 14-MAX_DAYS = 90+# The longest preset, "Last 90 days", covers today and the 90 days before it.+MAX_DAYS = 91def _resolve_bound(value: date | datetime, tz: tzinfo, *, is_end: bool) -> datetime:diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.py b/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.pyindex 12a829bd3865..7554de7e2d27 100644--- a/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.py+++ b/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.py@@ -69,20 +69,20 @@ def test_rolling_hours_keep_their_real_length_across_daylight_saving(_name: str,@parameterized.expand([("end before start", date(2026, 3, 10), date(2026, 3, 9), "date_from"),- ("91 days", date(2026, 8, 4), date(2026, 11, 2), "date_from"),+ ("92 days", date(2026, 8, 3), date(2026, 11, 2), "date_from"),("last day of year 9999", None, date(9999, 12, 31), "out of range"),("first day of year 1", None, date(1, 1, 1), "out of range"),- # Clocks fall back on November 1, so these 90 days are 2161 hours long.- ("90 days across daylight saving", date(2026, 8, 5), date(2026, 11, 2), None),+ # Clocks fall back on November 1, so these 91 days are 2185 hours long.+ ("91 days across daylight saving", date(2026, 8, 4), date(2026, 11, 2), None),(- "90 days of instants",- datetime(2026, 8, 5, tzinfo=UTC),+ "91 days of instants",+ datetime(2026, 8, 4, tzinfo=UTC),datetime(2026, 11, 3, tzinfo=UTC),None,),(- "an instant past 90 days",- datetime(2026, 8, 5, tzinfo=UTC),+ "an instant past 91 days",+ datetime(2026, 8, 4, tzinfo=UTC),datetime(2026, 11, 3, 0, 0, 1, tzinfo=UTC),"date_from",),diff --git a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsxindex dad46be4d9c5..1d248a5f9ab7 100644--- a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx+++ b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx@@ -74,6 +74,7 @@ import {hasUnsetConditionRollout,llmEvaluationLogic,} from './llmEvaluationLogic'+import { EvaluationDetailResults } from './overview/EvaluationDetailResults'import { statusReasonLabel, statusReasonRecoveryLabel } from './statusDisplay'import { EvaluationSettleStrategy, EvaluationTarget, EvaluationType } from './types'@@ -106,6 +107,7 @@ export function AIObservabilityEvaluation(): JSX.Element {const { searchParams } = useValues(router)const { featureFlags } = useValues(featureFlagLogic)const settlingStrategyEnabled = !!featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EVAL_SETTLING_STRATEGY]+ const resultsCardsEnabled = !!featureFlags[FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW]const {setEvaluationName,setEvaluationDescription,@@ -466,6 +468,11 @@ export function AIObservabilityEvaluation(): JSX.Element {)}</LemonBanner>)}+ {resultsCardsEnabled && !runsBackfillId && (+ <div className="mb-4">+ <EvaluationDetailResults evaluationId={evaluation.id} />+ </div>+ )}<EvaluationRunsTable /></div>),diff --git a/products/ai_observability/frontend/evaluations/constants.ts b/products/ai_observability/frontend/evaluations/constants.tsindex 227e31315595..61f9370ef5c0 100644--- a/products/ai_observability/frontend/evaluations/constants.ts+++ b/products/ai_observability/frontend/evaluations/constants.ts@@ -61,6 +61,14 @@ export const EVALUATION_NUMERIC_GRADED_HOGQL = `properties.$ai_evaluation_resultexport const EVALUATION_BOOLEAN_GRADED_HOGQL = `(isNull(properties.$ai_evaluation_result_type) OR properties.$ai_evaluation_result_type = 'boolean') AND properties.$ai_evaluation_result IS NOT NULL AND ${EVALUATION_NOT_SKIPPED_HOGQL}`export const EVALUATION_NUMERIC_MEAN_HOGQL = `avgIf(toFloat(properties.$ai_evaluation_numeric_result), ${EVALUATION_NUMERIC_GRADED_HOGQL})`+export function evaluationGradedHogQL(outputType: string): string {+ return outputType === 'categorical'+ ? EVALUATION_CATEGORICAL_GRADED_HOGQL+ : outputType === 'numeric'+ ? EVALUATION_NUMERIC_GRADED_HOGQL+ : EVALUATION_BOOLEAN_GRADED_HOGQL+}+export function numericEvaluationPassedHogQL(evaluation: Pick<EvaluationConfig, 'output_config'>): string {const rule = evaluation.output_config.passing_ruleif (!rule || !('threshold' in rule) || !Number.isFinite(rule.threshold)) {diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.tsindex a85df921f56c..700ba3057380 100644--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts@@ -10,7 +10,7 @@ import { teamLogic } from 'scenes/teamLogic'import { urls } from 'scenes/urls'import { SIDE_PANEL_CONTEXT_KEY, SidePanelSceneContext } from '~/layout/navigation-3000/sidepanel/types'-import { DateRange, InsightVizNode, NodeKind } from '~/queries/schema/schema-general'+import { DateRange, InsightVizNode, NodeKind, TrendsQuery } from '~/queries/schema/schema-general'import { MaxContextInput, createMaxContextHelpers } from '~/scenes/max/maxTypes'import { ActivityScope, Breadcrumb, ChartDisplayType, HogQLMathType } from '~/types'@@ -359,6 +359,7 @@ export interface llmEvaluationLogicValues {selectedPickerProviderKeyId: string | nullsidePanelContext: SidePanelSceneContext | nulltrendInsightUrl: string | null+ trendsQuery: TrendsQuery | nullusesDecisionModel: boolean}@@ -566,7 +567,8 @@ export interface llmEvaluationLogicMeta {evaluation: EvaluationConfig | null,originalEvaluation: EvaluationConfig | null) => boolean- trendInsightUrl: (originalEvaluation: EvaluationConfig | null) => string | null+ trendsQuery: (originalEvaluation: EvaluationConfig | null) => TrendsQuery | null+ trendInsightUrl: (trendsQuery: TrendsQuery | null) => string | nullisNewEvaluation: (evaluationId: string) => booleanevaluationBackTarget: (isNewEvaluation: boolean, searchParams: Record<string, any>) => EvaluationBackTargetmodelSelectionRequired: (@@ -1544,75 +1546,80 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([originalEvaluation: EvaluationConfig | null): boolean => evaluationSupportsReports(isNewEvaluation ? evaluation : originalEvaluation),],- trendInsightUrl: [+ // The evaluation's headline result as a trends query: the pass rate, or the mean score without a passing rule.+ trendsQuery: [(s) => [s.originalEvaluation],- (evaluation: EvaluationConfig | null): string | null =>+ (evaluation: EvaluationConfig | null): TrendsQuery | null =>evaluation &&(evaluationSupportsRunOutcomes(evaluation) || evaluation.output_type === 'numeric') &&evaluation.id- ? urls.insightNew({- query: {- kind: NodeKind.InsightVizNode,- source: {- kind: NodeKind.TrendsQuery,- series: [+ ? ({+ kind: NodeKind.TrendsQuery,+ series: [+ {+ kind: NodeKind.EventsNode,+ event: '$ai_evaluation',+ custom_name: `${evaluation.name} - ${evaluation.output_type === 'numeric' && !evaluation.output_config.passing_rule ? 'Mean score' : 'Pass rate'}`,+ math: HogQLMathType.HogQL,+ math_hogql:+ evaluation.output_type === 'categorical'+ ? evaluationPassRateHogQL(+ categoricalEvaluationPassedHogQL(evaluation),+ EVALUATION_CATEGORICAL_GRADED_HOGQL+ )+ : evaluation.output_type === 'numeric'+ ? evaluation.output_config.passing_rule+ ? evaluationPassRateHogQL(+ numericEvaluationPassedHogQL(evaluation),+ EVALUATION_NUMERIC_GRADED_HOGQL+ )+ : EVALUATION_NUMERIC_MEAN_HOGQL+ : evaluationPassRateHogQL(evaluationPassedHogQL(evaluation)),+ properties: [{- kind: NodeKind.EventsNode,- event: '$ai_evaluation',- custom_name: `${evaluation.name} - ${evaluation.output_type === 'numeric' && !evaluation.output_config.passing_rule ? 'Mean score' : 'Pass rate'}`,- math: HogQLMathType.HogQL,- math_hogql:- evaluation.output_type === 'categorical'- ? evaluationPassRateHogQL(- categoricalEvaluationPassedHogQL(evaluation),- EVALUATION_CATEGORICAL_GRADED_HOGQL- )- : evaluation.output_type === 'numeric'- ? evaluation.output_config.passing_rule- ? evaluationPassRateHogQL(- numericEvaluationPassedHogQL(evaluation),- EVALUATION_NUMERIC_GRADED_HOGQL- )- : EVALUATION_NUMERIC_MEAN_HOGQL- : evaluationPassRateHogQL(evaluationPassedHogQL(evaluation)),- properties: [- {- key: '$ai_evaluation_id',- value: evaluation.id,- operator: 'exact',- type: 'event',- },- ],+ key: '$ai_evaluation_id',+ value: evaluation.id,+ operator: 'exact',+ type: 'event',},- ...(evaluation.output_type === 'boolean' && evaluation.output_config.allows_na- ? [- {- kind: NodeKind.EventsNode as const,- event: '$ai_evaluation',- custom_name: `${evaluation.name} — N/A rate`,- math: HogQLMathType.HogQL as const,- math_hogql: `if(count() > 0, countIf(properties.$ai_evaluation_result IS NULL) / count() * 100, 0)`,- properties: [- {- key: '$ai_evaluation_id',- value: evaluation.id,- operator: 'exact' as const,- type: 'event' as const,- },- ],- },- ]- : []),],- trendsFilter: {- display: ChartDisplayType.ActionsLineGraph,- },- dateRange: {- date_from: '-7d',- },- interval: 'day',},- } as InsightVizNode,+ ...(evaluation.output_type === 'boolean' && evaluation.output_config.allows_na+ ? [+ {+ kind: NodeKind.EventsNode as const,+ event: '$ai_evaluation',+ custom_name: `${evaluation.name} — N/A rate`,+ math: HogQLMathType.HogQL as const,+ math_hogql: `if(count() > 0, countIf(properties.$ai_evaluation_result IS NULL) / count() * 100, 0)`,+ properties: [+ {+ key: '$ai_evaluation_id',+ value: evaluation.id,+ operator: 'exact' as const,+ type: 'event' as const,+ },+ ],+ },+ ]+ : []),+ ],+ trendsFilter: {+ display: ChartDisplayType.ActionsLineGraph,+ },+ dateRange: {+ date_from: '-7d',+ },+ interval: 'day',+ } as TrendsQuery)+ : null,+ ],+ trendInsightUrl: [+ (s) => [s.trendsQuery],+ (trendsQuery: TrendsQuery | null): string | null =>+ trendsQuery+ ? urls.insightNew({+ query: { kind: NodeKind.InsightVizNode, source: trendsQuery } as InsightVizNode,}): null,],diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationBreakdownPicker.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationBreakdownPicker.tsxnew file mode 100644index 000000000000..3cdd167e820e--- /dev/null+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationBreakdownPicker.tsx@@ -0,0 +1,37 @@+import { useActions, useValues } from 'kea'++import { LemonSelect } from '@posthog/lemon-ui'++import { llmEvaluationLogic } from '../llmEvaluationLogic'+import { evaluationDetailResultsLogic } from './evaluationDetailResultsLogic'++/** Properties every result event carries. The prompt ones are copied from the generation the evaluation scored. */+const BREAKDOWN_OPTIONS = [+ { value: '$ai_prompt_version', label: 'Prompt version' },+ { value: '$ai_prompt_name', label: 'Prompt name' },+ { value: '$ai_evaluation_model', label: 'Judge model', judgeOnly: true },+ { value: '$ai_evaluation_trigger', label: 'Live or backfill' },+]++export function EvaluationBreakdownPicker({ evaluationId }: { evaluationId: string }): JSX.Element {+ const logic = evaluationDetailResultsLogic({ evaluationId })+ const { breakdownProperty, breakdownSeriesLoading } = useValues(logic)+ const { setBreakdownProperty } = useActions(logic)+ const { originalEvaluation } = useValues(llmEvaluationLogic({ evaluationId }))++ return (+ <LemonSelect+ size="small"+ value={breakdownProperty}+ onChange={setBreakdownProperty}+ loading={breakdownSeriesLoading}+ options={[+ { value: null, label: 'No breakdown' },+ ...BREAKDOWN_OPTIONS.filter(+ (option) => !option.judgeOnly || originalEvaluation?.evaluation_type === 'llm_judge'+ ).map(({ value, label }) => ({ value, label: `By ${label.toLowerCase()}` })),+ ]}+ data-attr="llma-evaluation-results-breakdown"+ />+ )+}diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationDetailResults.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationDetailResults.tsxnew file mode 100644index 000000000000..63e7cffb178e--- /dev/null+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationDetailResults.tsx@@ -0,0 +1,107 @@+import { useActions, useValues } from 'kea'++import { IconClockRewind } from '@posthog/icons'+import { LemonBanner, LemonButton, LemonSkeleton } from '@posthog/lemon-ui'++import { teamLogic } from 'scenes/teamLogic'++import { EvaluationBreakdownPicker } from './EvaluationBreakdownPicker'+import { evaluationDetailResultsLogic } from './evaluationDetailResultsLogic'+import { RATE_LIMITED_MESSAGE, hasIsolatedPoints } from './evaluationOverviewDisplay'+import { EvaluationResultCards } from './EvaluationResultCards'++export function EvaluationDetailResults({ evaluationId }: { evaluationId: string }): JSX.Element | null {+ const logic = evaluationDetailResultsLogic({ evaluationId })+ const {+ results,+ resultsLoading,+ details,+ detailsLoading,+ request,+ failure,+ breakdownProperty,+ breakdownSeries,+ breakdownSeriesLoading,+ breakdownFailure,+ trendsQuery,+ } = useValues(logic)+ const { loadResults, loadDetails, loadBreakdown, selectBucket, runOnPastData } = useActions(logic)+ const { timezone } = useValues(teamLogic)++ if (request.notice) {+ return <LemonBanner type="info">{request.notice}</LemonBanner>+ }+ if (failure) {+ return (+ <LemonBanner+ type={failure === 'rate_limited' ? 'warning' : 'error'}+ action={{+ children: 'Try again',+ onClick: () => loadResults(),+ 'data-attr': 'llma-evaluation-results-retry',+ }}+ >+ {failure === 'rate_limited' ? RATE_LIMITED_MESSAGE : "Couldn't load the results over time."}+ </LemonBanner>+ )+ }+ if (!results) {+ return resultsLoading ? <LemonSkeleton className="h-56 w-full" /> : null+ }+ const { row, buckets, interval } = results+ // Nothing graded leaves the runs table below to say so, so the cards would only repeat it.+ if (!row || row.gradedCount === 0) {+ return null+ }++ return (+ <EvaluationResultCards+ row={row}+ buckets={buckets}+ interval={interval}+ timezone={timezone}+ // A retry shows the skeleton, so the error's Try again can't be clicked again while it runs.+ details={detailsLoading ? null : details}+ onRetryDetails={() => loadDetails()}+ onSelectBucket={selectBucket}+ showFailingExamples={false}+ chartAction={trendsQuery ? <EvaluationBreakdownPicker evaluationId={evaluationId} /> : null}+ breakdown={breakdownProperty ? breakdownSeries : null}+ chartFooter={+ breakdownFailure ? (+ <div className="flex flex-wrap items-center gap-x-2 text-xs text-secondary">+ <span>+ {breakdownFailure === 'rate_limited'+ ? RATE_LIMITED_MESSAGE+ : "Couldn't split the results by this property."}+ </span>+ <LemonButton+ size="xsmall"+ type="tertiary"+ onClick={() => loadBreakdown()}+ loading={breakdownSeriesLoading}+ data-attr="llma-evaluation-results-breakdown-retry"+ >+ Try again+ </LemonButton>+ </div>+ ) : (+ hasIsolatedPoints(row) && (+ <div className="flex flex-wrap items-center gap-x-2 text-xs text-secondary">+ <span>{`This evaluation has ${interval}s with no results, so some points have no line.`}</span>+ <LemonButton+ size="xsmall"+ type="tertiary"+ icon={<IconClockRewind />}+ onClick={runOnPastData}+ data-attr="llma-evaluation-results-backfill-hint"+ >+ Run on past data+ </LemonButton>+ </div>+ )+ )+ }+ />+ )+}diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationResultCards.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationResultCards.tsxnew file mode 100644index 000000000000..3cd57c65b1d1--- /dev/null+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationResultCards.tsx@@ -0,0 +1,121 @@+import { cn } from 'lib/utils/css-classes'++import type { EvaluationResultApi } from '../../generated/api.schemas'+import { EvaluationCategoryShares } from './EvaluationCategoryShares'+import { EvaluationFailingExamples } from './EvaluationFailingExamples'+import type { BucketInterval } from './evaluationOverviewDisplay'+import { EvaluationResultDetails } from './EvaluationResultDetails'+import type { DetailsLoad } from './evaluationResultDrawerLogic'+import { EvaluationResultOutcomes } from './EvaluationResultOutcomes'+import { EvaluationResultTrendChart, type TrendBreakdownSeries } from './EvaluationResultTrendChart'+import { EvaluationScoreHistogram } from './EvaluationScoreHistogram'++function Card({+ title,+ action,+ className,+ children,+}: {+ title: string+ action?: React.ReactNode+ className?: string+ children: React.ReactNode+}): JSX.Element {+ return (+ <section className={cn('border rounded bg-surface-primary p-3 flex flex-col gap-2 min-w-0', className)}>+ <div className="flex flex-wrap items-center justify-between gap-2">+ <h4 className="mb-0 text-sm font-semibold">{title}</h4>+ {action}+ </div>+ {children}+ </section>+ )+}++export function EvaluationResultCards({+ row,+ buckets,+ interval,+ timezone,+ details,+ onRetryDetails,+ onSelectBucket,+ chartFooter,+ chartAction,+ breakdown,+ showFailingExamples = true,+}: {+ row: EvaluationResultApi+ buckets: string[]+ interval: BucketInterval+ timezone: string+ details: DetailsLoad | null+ onRetryDetails: () => void+ onSelectBucket?: (bucket: string) => void+ chartFooter?: React.ReactNode+ chartAction?: React.ReactNode+ breakdown?: TrendBreakdownSeries[] | null+ /** The evaluation page lists every run below the cards, failing ones a filter away, so it leaves them out. */+ showFailingExamples?: boolean+}): JSX.Element {+ const { summary } = row+ const hasScore = summary.kind === 'score' || (summary.kind === 'pass_rate' && row.outputType === 'numeric')+ // Outcomes alone on its row takes the full width rather than leaving half of it empty.+ const outcomesAlone = !hasScore && !showFailingExamples++ return (+ <div className="@container">+ <div className="grid gap-3 grid-cols-1 @2xl:grid-cols-2">+ <Card title="Over time" action={chartAction} className="@2xl:col-span-2">+ <EvaluationResultTrendChart+ row={row}+ buckets={buckets}+ interval={interval}+ timezone={timezone}+ onSelectBucket={onSelectBucket}+ footer={chartFooter}+ breakdown={breakdown}+ />+ </Card>+ {summary.kind === 'pass_rate' && (+ <Card title="Outcomes" className={outcomesAlone ? '@2xl:col-span-2' : undefined}>+ <EvaluationResultOutcomes row={row} summary={summary} />+ </Card>+ )}+ {(summary.kind === 'categories' || summary.kind === 'sentiment') && (+ <Card title={summary.kind === 'sentiment' ? 'Sentiment split' : 'Labels'}>+ <EvaluationCategoryShares summary={summary} gradedCount={row.gradedCount} />+ </Card>+ )}+ {hasScore && (+ <Card title="Score distribution">+ <EvaluationResultDetails load={details} onRetry={onRetryDetails}>+ {(details) =>+ details.scoreDistribution ? (+ <EvaluationScoreHistogram histogram={details.scoreDistribution} />+ ) : (+ <div className="text-sm text-secondary">+ Set score bounds on the evaluation to see how scores spread.+ </div>+ )+ }+ </EvaluationResultDetails>+ </Card>+ )}+ {summary.kind === 'pass_rate' && showFailingExamples && (+ <Card title="Recent failing examples">+ <EvaluationResultDetails load={details} onRetry={onRetryDetails}>+ {(details) =>+ details.recentFailures && details.recentFailures.length > 0 ? (+ <EvaluationFailingExamples failures={details.recentFailures} />+ ) : (+ <div className="text-sm text-secondary">No failed runs in this range.</div>+ )+ }+ </EvaluationResultDetails>+ </Card>+ )}+ </div>+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationResultDetails.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationResultDetails.tsxindex 0b999f1b5525..df93b2521e34 100644--- a/products/ai_observability/frontend/evaluations/overview/EvaluationResultDetails.tsx+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationResultDetails.tsx@@ -1,21 +1,17 @@-import { useActions } from 'kea'-import { LemonButton, LemonSkeleton } from '@posthog/lemon-ui'import { RATE_LIMITED_MESSAGE } from './evaluationOverviewDisplay'-import { type DetailsLoad, type EvaluationDetails, evaluationResultDrawerLogic } from './evaluationResultDrawerLogic'+import type { DetailsLoad, EvaluationDetails } from './evaluationResultDrawerLogic'export function EvaluationResultDetails({- evaluationId,load,+ onRetry,children,}: {- evaluationId: stringload: DetailsLoad | null+ onRetry: () => voidchildren: (details: EvaluationDetails) => JSX.Element}): JSX.Element {- const { loadDetails } = useActions(evaluationResultDrawerLogic)-if (load?.status === 'ready') {return children(load.details)}@@ -28,7 +24,7 @@ export function EvaluationResultDetails({<LemonButtonsize="small"type="secondary"- onClick={() => loadDetails(evaluationId, 'open')}+ onClick={onRetry}data-attr="evaluation-overview-drawer-retry">Try againdiff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationResultExpanded.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationResultExpanded.tsxindex d7570f48caf2..baa1bfbb899f 100644--- a/products/ai_observability/frontend/evaluations/overview/EvaluationResultExpanded.tsx+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationResultExpanded.tsx@@ -1,46 +1,22 @@-import { useValues } from 'kea'+import { useActions, useValues } from 'kea'import { LemonBanner, Link } from '@posthog/lemon-ui'-import { cn } from 'lib/utils/css-classes'-import type { EvaluationResultApi } from '../../generated/api.schemas'-import { EvaluationCategoryShares } from './EvaluationCategoryShares'+import { EvaluationBackfillHint } from './EvaluationBackfillHint'import { EvaluationDrawerEmptyState } from './EvaluationDrawerEmptyState'-import { EvaluationFailingExamples } from './EvaluationFailingExamples'-import { describeAttention } from './evaluationOverviewDisplay'+import { describeAttention, hasIsolatedPoints } from './evaluationOverviewDisplay'import { EvaluationPausedNotice } from './EvaluationPausedNotice'-import { EvaluationResultDetails } from './EvaluationResultDetails'+import { EvaluationResultCards } from './EvaluationResultCards'import { evaluationResultDrawerLogic } from './evaluationResultDrawerLogic'-import { EvaluationResultOutcomes } from './EvaluationResultOutcomes'-import { EvaluationResultTrendChart } from './EvaluationResultTrendChart'-import { EvaluationScoreHistogram } from './EvaluationScoreHistogram'import { evaluationsOverviewLogic } from './evaluationsOverviewLogic'-function Card({- title,- className,- children,-}: {- title: string- className?: string- children: React.ReactNode-}): JSX.Element {- return (- <section className={cn('border rounded bg-surface-primary p-3 flex flex-col gap-2 min-w-0', className)}>- <h4 className="mb-0 text-sm font-semibold">{title}</h4>- {children}- </section>- )-}-export function EvaluationResultExpanded({ row }: { row: EvaluationResultApi }): JSX.Element {- const { configById, timezone, evaluationUrl } = useValues(evaluationsOverviewLogic)+ const { configById, timezone, evaluationUrl, buckets, interval } = useValues(evaluationsOverviewLogic)const { openDetails } = useValues(evaluationResultDrawerLogic)- const { summary } = row+ const { loadDetails } = useActions(evaluationResultDrawerLogic)const errored = row.attentionReasons.includes('errored')const enabled = configById.get(row.id)?.enabled ?? row.enabled- const hasScore = summary.kind === 'score' || (summary.kind === 'pass_rate' && row.outputType === 'numeric')const notice = errored ? (<LemonBanner type="error">@@ -58,49 +34,15 @@ export function EvaluationResultExpanded({ row }: { row: EvaluationResultApi }):return (<div className="@container flex flex-col gap-3">{notice}- <div className="grid gap-3 grid-cols-1 @2xl:grid-cols-2">- <Card title="Over time" className="@2xl:col-span-2">- <EvaluationResultTrendChart row={row} />- </Card>- {summary.kind === 'pass_rate' && (- <Card title="Outcomes">- <EvaluationResultOutcomes row={row} summary={summary} />- </Card>- )}- {(summary.kind === 'categories' || summary.kind === 'sentiment') && (- <Card title={summary.kind === 'sentiment' ? 'Sentiment split' : 'Labels'}>- <EvaluationCategoryShares summary={summary} gradedCount={row.gradedCount} />- </Card>- )}- {hasScore && (- <Card title="Score distribution">- <EvaluationResultDetails evaluationId={row.id} load={openDetails}>- {(details) =>- details.scoreDistribution ? (- <EvaluationScoreHistogram histogram={details.scoreDistribution} />- ) : (- <div className="text-sm text-secondary">- Set score bounds on the evaluation to see how scores spread.- </div>- )- }- </EvaluationResultDetails>- </Card>- )}- {summary.kind === 'pass_rate' && (- <Card title="Recent failing examples">- <EvaluationResultDetails evaluationId={row.id} load={openDetails}>- {(details) =>- details.recentFailures && details.recentFailures.length > 0 ? (- <EvaluationFailingExamples failures={details.recentFailures} />- ) : (- <div className="text-sm text-secondary">No failed runs in this range.</div>- )- }- </EvaluationResultDetails>- </Card>- )}- </div>+ <EvaluationResultCards+ row={row}+ buckets={buckets}+ interval={interval}+ timezone={timezone}+ details={openDetails}+ onRetryDetails={() => loadDetails(row.id, 'open')}+ chartFooter={hasIsolatedPoints(row) && <EvaluationBackfillHint row={row} source="drawer" />}+ /></div>)}diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationResultTrendChart.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationResultTrendChart.tsxindex 5283bb66743f..3f19d4588efe 100644--- a/products/ai_observability/frontend/evaluations/overview/EvaluationResultTrendChart.tsx+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationResultTrendChart.tsx@@ -1,4 +1,3 @@-import { useValues } from 'kea'import { useMemo } from 'react'import { TimeSeriesComboChart, TooltipSurface, TooltipSwatch } from '@posthog/quill-charts'@@ -9,16 +8,14 @@ import { getColorVar } from 'lib/colors'import { humanFriendlyNumber } from 'lib/utils/numbers'import type { EvaluationResultApi } from '../../generated/api.schemas'-import { EvaluationBackfillHint } from './EvaluationBackfillHint'import {+ type BucketInterval,formatBucket,- hasIsolatedPoints,hourTick,passFailCounts,toChartValues,trackedCategory,} from './evaluationOverviewDisplay'-import { evaluationsOverviewLogic } from './evaluationsOverviewLogic'const VOLUME_KEY = 'volume'const PASSED_KEY = 'passed'@@ -44,9 +41,38 @@ function withAlpha(color: string, alpha: number): string {: color}-export function EvaluationResultTrendChart({ row }: { row: EvaluationResultApi }): JSX.Element {+/** One value of a breakdown property, aligned with the buckets. NaN marks a bucket where it had nothing graded. */+export interface TrendBreakdownSeries {+ key: string+ label: string+ values: number[]+ graded: number[]+}++/** The overall line uses the theme's first color, so breakdown lines start at the second. */+function breakdownColor(colors: string[], index: number): string {+ return colors[(index + 1) % colors.length]+}++export function EvaluationResultTrendChart({+ row,+ buckets,+ interval,+ timezone,+ onSelectBucket,+ footer,+ breakdown,+}: {+ row: EvaluationResultApi+ buckets: string[]+ interval: BucketInterval+ timezone: string+ /** Called with the clicked bucket. Without it the chart is not clickable. */+ onSelectBucket?: (bucket: string) => void+ footer?: React.ReactNode+ breakdown?: TrendBreakdownSeries[] | null+}): JSX.Element {const theme = useChartTheme()- const { buckets, timezone, interval } = useValues(evaluationsOverviewLogic)const { summary, trend } = rowconst isScore = summary.kind === 'score'const label = trackedCategory(row)?.label ?? HEADLINE_LABELS[summary.kind]@@ -97,20 +123,33 @@ export function EvaluationResultTrendChart({ row }: { row: EvaluationResultApi }color: 'transparent',visibility: { tooltip: false, total: false, valueLabel: false },},- {- key: 'headline',- label,- // NaN leaves a gap, so a bucket with no graded results never reads as a measured 0%.- data: values,- type: 'line',- color: theme.colors[0],- points: { radius: 3 },- },+ ...(breakdown+ ? breakdown.map(+ (entry, index): Series => ({+ key: `breakdown-${entry.key}`,+ label: entry.label,+ data: entry.values,+ type: 'line',+ color: breakdownColor(theme.colors, index),+ points: { radius: 3 },+ })+ )+ : [+ {+ key: 'headline',+ label,+ // NaN leaves a gap, so a bucket with no graded results never reads as a measured 0%.+ data: values,+ type: 'line' as const,+ color: theme.colors[0],+ points: { radius: 3 },+ },+ ]),]- }, [trend, values, counts, label, theme])+ }, [trend, values, counts, label, theme, breakdown])const config = useMemo<TimeSeriesComboChartConfig>(() => {- const finite = values.filter(Number.isFinite)+ const finite = (breakdown ? breakdown.flatMap((entry) => entry.values) : values).filter(Number.isFinite)// Zoom in when every point sits high, and let the axis labels say so.const floor = finite.length > 0 ? Math.max(0, Math.floor((Math.min(...finite) - 10) / 10) * 10) : 0const scale = summary.kind === 'score' ? summary.score?.scale : null@@ -126,7 +165,7 @@ export function EvaluationResultTrendChart({ row }: { row: EvaluationResultApi }],valueDomain: isScore ? (scale ? { min: scale.min, max: scale.max } : undefined) : { min: floor, max: 100 },}- }, [isScore, summary, values, timezone, interval])+ }, [isScore, summary, values, timezone, interval, breakdown])const tooltip = (ctx: TooltipContext): JSX.Element => {const value = values[ctx.dataIndex]@@ -134,7 +173,24 @@ export function EvaluationResultTrendChart({ row }: { row: EvaluationResultApi }return (<TooltipSurface><div className="font-semibold">{formatBucket(String(ctx.label), interval)}</div>- {Number.isFinite(value) ? (+ {breakdown ? (+ breakdown.map((entry, index) =>+ Number.isFinite(entry.values[ctx.dataIndex]) ? (+ <div key={entry.key} className="flex items-center gap-2 mt-1">+ <TooltipSwatch color={breakdownColor(theme.colors, index)} />+ <span className="truncate max-w-60">{entry.label}</span>+ <strong className="ml-auto tabular-nums">+ {isScore+ ? entry.values[ctx.dataIndex].toFixed(2)+ : `${entry.values[ctx.dataIndex].toFixed(1)}%`}+ </strong>+ <span className="opacity-70 tabular-nums">+ {`${humanFriendlyNumber(entry.graded[ctx.dataIndex] ?? 0)} graded`}+ </span>+ </div>+ ) : null+ )+ ) : Number.isFinite(value) ? (<div className="flex items-center gap-2 mt-1">{line && <TooltipSwatch color={line.color} />}<span>{label}</span>@@ -181,14 +237,34 @@ export function EvaluationResultTrendChart({ row }: { row: EvaluationResultApi }theme={theme}config={config}tooltip={tooltip}+ onPointClick={+ onSelectBucket+ ? ({ dataIndex }) => buckets[dataIndex] && onSelectBucket(buckets[dataIndex])+ : undefined+ }/></div><div className="text-xs text-secondary">{counts? `Bars show how many results passed and failed each ${interval}.`: `Shaded bars show how many results were graded each ${interval}.`}+ {onSelectBucket ? ` Click a${interval === 'hour' ? 'n' : ''} ${interval} to see its runs.` : ''}</div>- {hasIsolatedPoints(row) && <EvaluationBackfillHint row={row} source="drawer" />}+ {breakdown && breakdown.length > 0 && (+ <div className="flex flex-wrap gap-x-3 gap-y-1 text-xs">+ {breakdown.map((entry, index) => (+ <span key={entry.key} className="flex items-center gap-1 min-w-0">+ <span+ className="inline-block size-2 shrink-0 rounded-full"+ // eslint-disable-next-line react/forbid-dom-props+ style={{ backgroundColor: breakdownColor(theme.colors, index) }}+ />+ <span className="truncate max-w-60">{entry.label}</span>+ </span>+ ))}+ </div>+ )}+ {footer}</div>)}diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationDetailResultsLogic.ts b/products/ai_observability/frontend/evaluations/overview/evaluationDetailResultsLogic.tsnew file mode 100644index 000000000000..e22b29e30159--- /dev/null+++ b/products/ai_observability/frontend/evaluations/overview/evaluationDetailResultsLogic.ts@@ -0,0 +1,574 @@+import { MakeLogicType, actions, afterMount, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea'+import { loaders } from 'kea-loaders'+import posthog from 'posthog-js'++import { dayjs } from 'lib/dayjs'+import { uuid } from 'lib/utils/dom'+import { teamLogic } from 'scenes/teamLogic'++import { cancelQueryOnServer } from '~/queries/nodes/DataNode/dataNodeLogic'+import { performQuery } from '~/queries/query'+import { DateRange, NodeKind, TrendsQuery, TrendsQueryResponse } from '~/queries/schema/schema-general'+import { escapeHogQLString, escapePropertyAsHogQLIdentifier } from '~/queries/utils'+import { HogQLMathType, type HogQLPropertyFilter, PropertyFilterType } from '~/types'++import { evaluationsOverviewRetrieve } from '../../generated/api'+import type { EvaluationResultApi, EvaluationsOverviewRetrieveParams } from '../../generated/api.schemas'+import type { EvaluationsOverviewIntervalEnumApi } from '../../generated/api.schemas'+import { evaluationGradedHogQL } from '../constants'+import { llmEvaluationLogic } from '../llmEvaluationLogic'+import type { EvaluationConfig } from '../types'+import { type BucketInterval, bucketDateRange } from './evaluationOverviewDisplay'+import { type DetailsLoad, fetchEvaluationDetails } from './evaluationResultDrawerLogic'+import type { TrendBreakdownSeries } from './EvaluationResultTrendChart'+import {+ OVERVIEW_MAX_DAYS,+ type OverviewFailure,+ type OverviewRequest,+ failureOf,+ markRefreshedAfterChange,+ needsRefreshAfterChange,+ overviewPeriod,+} from './evaluationsOverviewLogic'++export interface EvaluationDetailResultsLogicProps {+ evaluationId: string+}++/** How many values get their own line. The rest share one "Other" line. */+const BREAKDOWN_LIMIT = 5+/**+ * How many values the query returns. Trends ranks each series on its own and sums the rest into its "Other", which+ * is wrong for a rate, so the top values and "Other" come from these instead.+ */+// ponytail: values past this cap drop out of "Other"; query the counts with HogQL if properties get that wide.+const BREAKDOWN_QUERY_LIMIT = 100+const BREAKDOWN_NULL = '$$_posthog_breakdown_null_$$'+const BREAKDOWN_OTHER = '$$_posthog_breakdown_other_$$'+/** Trends cuts a breakdown value to this length. */+const BREAKDOWN_VALUE_MAX_LENGTH = 400+/** A rolling range of whole days, weeks, months, quarters or years. */+const ROLLING_DAYS = /^-\d+[dwmqy]$/++/** The key trends gives a bucket: the day, or the day and hour, in the project's time zone. */+function trendsBucketKey(bucket: string, interval: BucketInterval): string {+ return interval === 'hour' ? `${bucket.slice(0, 10)} ${bucket.slice(11, 19)}` : bucket.slice(0, 10)+}++interface BreakdownCounts {+ result?: number[]+ graded?: number[]+ days: string[]+}++/** Each breakdown value's result and graded count per bucket. */+function breakdownCounts(response: TrendsQueryResponse): Map<string, BreakdownCounts> {+ const byValue = new Map<string, BreakdownCounts>()+ for (const entry of response.results ?? []) {+ const raw = Array.isArray(entry.breakdown_value) ? entry.breakdown_value[0] : entry.breakdown_value+ const key = String(raw ?? BREAKDOWN_NULL)+ const current: BreakdownCounts = byValue.get(key) ?? { days: entry.days ?? [] }+ current[entry.action?.order === 1 ? 'graded' : 'result'] = entry.data ?? []+ byValue.set(key, current)+ }+ return byValue+}++/** The breakdown query, limited to the given values, so each series returns all of them. */+function restrictBreakdown(query: TrendsQuery, values: string[]): TrendsQuery {+ const property = String(query.breakdownFilter?.breakdowns?.[0]?.property ?? '')+ // The same value Trends breaks down by, so the keys it returned match.+ const value = `ifNull(nullIf(left(toString(properties.${escapePropertyAsHogQLIdentifier(property)}), ${BREAKDOWN_VALUE_MAX_LENGTH}), ''), ${escapeHogQLString(BREAKDOWN_NULL)})`+ const filter: HogQLPropertyFilter = {+ type: PropertyFilterType.HogQL,+ key: `${value} IN (${values.map(escapeHogQLString).join(', ')})`,+ }+ return {+ ...query,+ series: query.series.map((series) => ({ ...series, properties: [...(series.properties ?? []), filter] })),+ }+}++/**+ * The values with the most graded runs keep their own line, and the rest share an "Other" line. A result is a share or+ * a mean of the graded runs, so the "Other" result weighs each value's result by its graded count.+ */+export function topBreakdownSeries(series: TrendBreakdownSeries[], limit: number): TrendBreakdownSeries[] {+ const total = (entry: TrendBreakdownSeries): number => entry.graded.reduce((sum, count) => sum + count, 0)+ const ranked = series.filter((entry) => total(entry) > 0).sort((a, b) => total(b) - total(a))+ const rest = ranked.slice(limit)+ if (rest.length === 0) {+ return ranked+ }+ const graded = rest[0].graded.map((_, bucket) => rest.reduce((sum, entry) => sum + entry.graded[bucket], 0))+ const values = graded.map((count, bucket) =>+ count === 0+ ? NaN+ : rest.reduce(+ (sum, entry) => sum + (entry.graded[bucket] > 0 ? entry.values[bucket] * entry.graded[bucket] : 0),+ 0+ ) / count+ )+ return [...ranked.slice(0, limit), { key: BREAKDOWN_OTHER, label: 'Other', values, graded }]+}++export interface EvaluationDetailResults {+ row: EvaluationResultApi | null+ buckets: string[]+ interval: BucketInterval+ /** Whether this load skipped the backend's cache, so the loads that follow it skip it too. */+ skippedCache: boolean+}++// Generated by kea-typegen. Update if you're an agent, ignore if you're human.+export interface evaluationDetailResultsLogicValues {+ originalEvaluation: EvaluationConfig | null // llmEvaluationLogic+ runsDateRange: DateRange // llmEvaluationLogic+ trendsQuery: TrendsQuery | null // llmEvaluationLogic+ currentTeamId: number | null // teamLogic+ timezone: string // teamLogic+ weekStartDay: number // teamLogic+ breakdownFailure: OverviewFailure | null+ breakdownProperty: string | null+ breakdownQuery: TrendsQuery | null+ breakdownSeries: TrendBreakdownSeries[] | null+ breakdownSeriesLoading: boolean+ details: DetailsLoad | null+ detailsLoading: boolean+ failure: OverviewFailure | null+ periodNow: number+ request: OverviewRequest+ results: EvaluationDetailResults | null+ resultsLoading: boolean+}++// Generated by kea-typegen. Update if you're an agent, ignore if you're human.+export interface evaluationDetailResultsLogicActions {+ refreshEvaluationRuns: () => {+ value: true+ } // llmEvaluationLogic+ setActiveTab: (tab: string) => {+ tab: string+ } // llmEvaluationLogic+ setRunsDates: (+ dateFrom: string | null,+ dateTo: string | null+ ) => {+ dateFrom: string | null+ dateTo: string | null+ } // llmEvaluationLogic+ loadBreakdown: (refresh?: boolean) => boolean+ loadBreakdownFailure: (+ error: string,+ errorObject?: any+ ) => {+ error: string+ errorObject?: any+ }+ loadBreakdownSuccess: (+ breakdownSeries: TrendBreakdownSeries[] | null,+ payload?: boolean+ ) => {+ breakdownSeries: TrendBreakdownSeries[] | null+ payload?: boolean+ }+ loadDetails: (refresh?: boolean) => boolean+ loadDetailsFailure: (+ error: string,+ errorObject?: any+ ) => {+ error: string+ errorObject?: any+ }+ loadDetailsSuccess: (+ details: DetailsLoad | null,+ payload?: boolean+ ) => {+ details: DetailsLoad | null+ payload?: boolean+ }+ loadResults: (refresh?: boolean) => boolean+ loadResultsFailure: (+ error: string,+ errorObject?: any+ ) => {+ error: string+ errorObject?: any+ }+ loadResultsSuccess: (+ results: {+ buckets: string[]+ interval: EvaluationsOverviewIntervalEnumApi+ row: EvaluationResultApi | null+ skippedCache: boolean+ } | null,+ payload?: boolean+ ) => {+ results: {+ buckets: string[]+ interval: EvaluationsOverviewIntervalEnumApi+ row: EvaluationResultApi | null+ skippedCache: boolean+ } | null+ payload?: boolean+ }+ runOnPastData: () => {+ value: true+ }+ selectBucket: (bucket: string) => {+ bucket: string+ }+ setBreakdownFailure: (failure: OverviewFailure | null) => {+ failure: OverviewFailure | null+ }+ setBreakdownProperty: (property: string | null) => {+ property: string | null+ }+ setFailure: (failure: OverviewFailure | null) => {+ failure: OverviewFailure | null+ }+}++// Generated by kea-typegen. Update if you're an agent, ignore if you're human.+export interface evaluationDetailResultsLogicMeta {+ key: string+ __keaTypeGenInternalSelectorTypes: {+ breakdownQuery: (+ trendsQuery: TrendsQuery | null,+ originalEvaluation: EvaluationConfig | null,+ breakdownProperty: string | null,+ request: OverviewRequest,+ results: EvaluationDetailResults | null+ ) => TrendsQuery | null+ request: (+ runsDateRange: DateRange,+ timezone: string,+ weekStartDay: number,+ periodNow: number,+ arg: any+ ) => OverviewRequest+ }+}++export type evaluationDetailResultsLogicType = MakeLogicType<+ evaluationDetailResultsLogicValues,+ evaluationDetailResultsLogicActions,+ EvaluationDetailResultsLogicProps,+ evaluationDetailResultsLogicMeta+>++export const evaluationDetailResultsLogic = kea<evaluationDetailResultsLogicType>([+ path((key) => [+ 'products',+ 'ai_observability',+ 'frontend',+ 'evaluations',+ 'overview',+ 'evaluationDetailResultsLogic',+ key,+ ]),+ props({} as EvaluationDetailResultsLogicProps),+ key((props) => props.evaluationId),+ connect((props: EvaluationDetailResultsLogicProps) => ({+ values: [+ llmEvaluationLogic({ evaluationId: props.evaluationId }),+ ['runsDateRange', 'trendsQuery', 'originalEvaluation'],+ teamLogic,+ ['currentTeamId', 'timezone', 'weekStartDay'],+ ],+ actions: [+ llmEvaluationLogic({ evaluationId: props.evaluationId }),+ ['setRunsDates', 'setActiveTab', 'refreshEvaluationRuns'],+ ],+ })),+ actions({+ setFailure: (failure: OverviewFailure | null) => ({ failure }),+ setBreakdownFailure: (failure: OverviewFailure | null) => ({ failure }),+ selectBucket: (bucket: string) => ({ bucket }),+ runOnPastData: true,+ setBreakdownProperty: (property: string | null) => ({ property }),+ }),+ loaders(({ values, props, actions, cache }) => ({+ results: [+ null as EvaluationDetailResults | null,+ {+ loadResults: async (refresh: boolean = false, breakpoint) => {+ const { request } = values+ if (!values.currentTeamId || !request.ready) {+ return null+ }+ const requestKey = JSON.stringify(request)+ const skipCache = refresh || needsRefreshAfterChange(requestKey)+ try {+ const overview = await evaluationsOverviewRetrieve(String(values.currentTeamId), {+ ...request.params,+ ...(skipCache ? { refresh: true } : {}),+ })+ breakpoint()+ if (skipCache) {+ markRefreshedAfterChange(requestKey)+ }+ return {+ row: overview.evaluations.find((row) => row.id === props.evaluationId) ?? null,+ buckets: overview.buckets,+ interval: overview.interval,+ skippedCache: skipCache,+ }+ } catch (error) {+ breakpoint()+ actions.setFailure(failureOf(error))+ return null+ }+ },+ },+ ],+ breakdownSeries: [+ null as TrendBreakdownSeries[] | null,+ {+ loadBreakdown: async (refresh: boolean = false, breakpoint) => {+ const query = values.breakdownQuery+ const results = values.results+ if (!query || !results) {+ return null+ }+ const mode = refresh ? 'force_blocking' : undefined+ const run = (+ q: TrendsQuery+ ): Promise<Map<string, BreakdownCounts> | { failure: OverviewFailure }> => {+ const controller = new AbortController()+ const queryId = uuid()+ let finished = false+ // A newer load or an unmount stops this query, here and on the server.+ cache.disposables.add(+ () => () => {+ controller.abort()+ if (!finished) {+ void cancelQueryOnServer(values.currentTeamId, queryId)+ }+ },+ 'breakdownQuery',+ { pauseOnPageHidden: false }+ )+ return performQuery(q, { signal: controller.signal }, mode, queryId)+ .finally(() => {+ finished = true+ })+ .then(breakdownCounts, (error) => ({ failure: failureOf(error) }))+ }+ let byValue = await run(query)+ breakpoint()+ if (!('failure' in byValue)) {+ // Trends ranks each series on its own, so a value with many runs and a low result can miss+ // the result series. Then the query runs again on the values the graded series kept.+ const counts = byValue+ const graded = [...counts.entries()]+ .filter(([key, entry]) => key !== BREAKDOWN_OTHER && entry.graded)+ .map(([key]) => key)+ if (graded.some((key) => !counts.get(key)?.result)) {+ byValue = await run(restrictBreakdown(query, graded))+ breakpoint()+ }+ }+ if ('failure' in byValue) {+ actions.setBreakdownFailure(byValue.failure)+ return null+ }+ const series = [...byValue.entries()]+ .filter(([key]) => key !== BREAKDOWN_OTHER)+ .map(([key, { result = [], graded = [], days }]) => {+ // An hour repeated when clocks fall back has one key for two buckets, so match them in order.+ const index = new Map<string, number[]>()+ days.forEach((day, position) => index.set(day, [...(index.get(day) ?? []), position]))+ const seen = new Map<string, number>()+ const at = results.buckets.map((bucket) => {+ const day = trendsBucketKey(bucket, results.interval)+ const occurrence = seen.get(day) ?? 0+ seen.set(day, occurrence + 1)+ return index.get(day)?.[occurrence]+ })+ const gradedCounts = at.map((position) =>+ position === undefined ? 0 : (graded[position] ?? 0)+ )+ return {+ key,+ label: key === BREAKDOWN_NULL ? 'Not set' : key,+ // Trends fills a bucket without runs with 0, which is no result at all.+ values: at.map((position, bucketIndex) =>+ position === undefined || gradedCounts[bucketIndex] === 0+ ? NaN+ : (result[position] ?? NaN)+ ),+ graded: gradedCounts,+ }+ })+ return topBreakdownSeries(series, BREAKDOWN_LIMIT)+ },+ },+ ],+ details: [+ null as DetailsLoad | null,+ {+ loadDetails: async (refresh: boolean = false, breakpoint) => {+ const row = values.results?.row+ if (!values.currentTeamId || !row || row.gradedCount === 0) {+ return null+ }+ const { date_from, date_to } = values.request.params+ // The runs table lists failing runs, so this page leaves the failing examples out.+ const load = await fetchEvaluationDetails(+ String(values.currentTeamId),+ row,+ { date_from, date_to, ...(refresh ? { refresh: true } : {}) },+ { recentFailures: false }+ )+ breakpoint()+ return load+ },+ },+ ],+ })),+ reducers({+ // The instant a rolling range ends at, read when results load, so the request selector reads no clock.+ periodNow: [+ Date.now(),+ {+ loadResults: () => Date.now(),+ },+ ],+ // Lines of the old range would be drawn on the new range's buckets until the breakdown reloads.+ breakdownSeries: {+ loadResults: () => null,+ },+ breakdownProperty: [+ null as string | null,+ {+ setBreakdownProperty: (_, { property }) => property,+ },+ ],+ breakdownFailure: [+ null as OverviewFailure | null,+ {+ setBreakdownFailure: (_, { failure }) => failure,+ loadBreakdown: () => null,+ },+ ],+ failure: [+ null as OverviewFailure | null,+ {+ setFailure: (_, { failure }) => failure,+ loadResults: () => null,+ },+ ],+ }),+ selectors({+ // The headline result split by one property, per bucket, next to each value's graded count.+ breakdownQuery: [+ (s) => [s.trendsQuery, s.originalEvaluation, s.breakdownProperty, s.request, s.results],+ (+ trendsQuery: TrendsQuery | null,+ evaluation: EvaluationConfig | null,+ breakdownProperty: string | null,+ request: OverviewRequest,+ results: EvaluationDetailResults | null+ ): TrendsQuery | null => {+ if (!trendsQuery || !evaluation || !breakdownProperty || !request.ready || !results) {+ return null+ }+ const [headline] = trendsQuery.series+ const { date_from = null, date_to = null } = request.params+ return {+ ...trendsQuery,+ kind: NodeKind.TrendsQuery,+ series: [+ headline,+ {+ ...headline,+ custom_name: 'Graded',+ math: HogQLMathType.HogQL,+ math_hogql: `countIf(${evaluationGradedHogQL(evaluation.output_type)})`,+ },+ ],+ breakdownFilter: {+ breakdowns: [{ property: breakdownProperty, type: 'event' }],+ breakdown_limit: BREAKDOWN_QUERY_LIMIT,+ },+ // Trends reads a date-only end as that day's midnight when it buckets by hour.+ dateRange: {+ date_from,+ date_to:+ results.interval === 'hour' && date_to && /^\d{4}-\d{2}-\d{2}$/.test(date_to)+ ? `${date_to}T23:59:59`+ : date_to,+ // A rolling range of hours starts mid-hour, and the overview counts runs from that instant.+ ...(date_from && date_from.includes('T') ? { explicitDate: true } : {}),+ },+ interval: results.interval,+ }+ },+ ],+ request: [+ (s) => [s.runsDateRange, s.timezone, s.weekStartDay, s.periodNow, (_, props) => props.evaluationId],+ (+ runsDateRange: DateRange,+ timezone: string,+ weekStartDay: number,+ periodNow: number,+ evaluationId: string+ ): OverviewRequest => {+ const period = overviewPeriod(+ { dateFrom: runsDateRange.date_from ?? null, dateTo: runsDateRange.date_to ?? null },+ timezone,+ dayjs(periodNow),+ weekStartDay+ )+ if (!period || period.days > OVERVIEW_MAX_DAYS) {+ return {+ params: {+ date_from: period?.dateFrom,+ date_to: period?.dateTo,+ evaluation_ids: [evaluationId],+ },+ ready: false,+ notice: `Pick a range of ${OVERVIEW_MAX_DAYS} days or less to see results over time.`,+ }+ }+ // The runs table starts a rolling range at midnight N units ago, a day before the overview does.+ const dateFrom =+ ROLLING_DAYS.test(runsDateRange.date_from ?? '') && !runsDateRange.date_to+ ? dayjs.utc(period.dateFrom).subtract(1, 'day').format('YYYY-MM-DD')+ : period.dateFrom+ const params: EvaluationsOverviewRetrieveParams = {+ date_from: dateFrom,+ date_to: period.dateTo,+ evaluation_ids: [evaluationId],+ }+ return { params, ready: true, notice: null }+ },+ ],+ }),+ listeners(({ actions, values }) => ({+ setBreakdownProperty: ({ property }) => {+ posthog.capture('llma evaluation detail breakdown changed', { property })+ actions.loadBreakdown()+ },+ setRunsDates: () => actions.loadResults(),+ refreshEvaluationRuns: () => actions.loadResults(true),+ loadResultsSuccess: ({ results }) => {+ const refresh = results?.skippedCache ?? false+ actions.loadDetails(refresh)+ actions.loadBreakdown(refresh)+ },+ runOnPastData: () => {+ posthog.capture('llma evaluation detail backfill hint clicked')+ actions.setActiveTab('backfills')+ },+ selectBucket: ({ bucket }) => {+ posthog.capture('llma evaluation detail bucket selected', { interval: values.results?.interval })+ const { dateFrom, dateTo } = bucketDateRange(bucket, values.results?.interval ?? 'day')+ actions.setRunsDates(dateFrom, dateTo)+ },+ })),+ afterMount(({ actions }) => {+ actions.loadResults()+ }),+])diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationOverviewDisplay.ts b/products/ai_observability/frontend/evaluations/overview/evaluationOverviewDisplay.tsindex 589605e2c5a4..66110b9c961b 100644--- a/products/ai_observability/frontend/evaluations/overview/evaluationOverviewDisplay.ts+++ b/products/ai_observability/frontend/evaluations/overview/evaluationOverviewDisplay.ts@@ -259,3 +259,19 @@ export function hourTick(label: string): string | null {}return hour === 0 ? dayjs(label.slice(0, 10)).format('MMM D') : label.slice(11, 16)}++/**+ * The date range that covers one bucket. A day's bucket starts at local midnight, so its first ten characters are that+ * day in the project's time zone. An hour is exact instants, which the date filters read as given. Its end is the next+ * hour's start, because the filters exclude the end.+ */+export function bucketDateRange(bucket: string, interval: BucketInterval): { dateFrom: string; dateTo: string } {+ if (interval === 'hour') {+ const start = dayjs(bucket).utc()+ return {+ dateFrom: start.format('YYYY-MM-DDTHH:mm:ss[Z]'),+ dateTo: start.add(1, 'hour').format('YYYY-MM-DDTHH:mm:ss[Z]'),+ }+ }+ return { dateFrom: bucket.slice(0, 10), dateTo: bucket.slice(0, 10) }+}diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationResultDrawerLogic.ts b/products/ai_observability/frontend/evaluations/overview/evaluationResultDrawerLogic.tsindex 705c5fdf4c30..5440be98bcb2 100644--- a/products/ai_observability/frontend/evaluations/overview/evaluationResultDrawerLogic.ts+++ b/products/ai_observability/frontend/evaluations/overview/evaluationResultDrawerLogic.ts@@ -32,6 +32,34 @@ export type DetailsLoad =| { status: 'ready'; details: EvaluationDetails }| { status: 'error'; failure: OverviewFailure }+/** Reads the failed runs and the score spread a row's cards show. Each endpoint rejects an evaluation it does not+ * apply to, so the row decides which ones to read. */+export async function fetchEvaluationDetails(+ projectId: string,+ row: EvaluationResultApi,+ params: { date_from?: string; date_to?: string; refresh?: boolean },+ { recentFailures = true }: { recentFailures?: boolean } = {}+): Promise<DetailsLoad> {+ const { summary } = row+ const scale = 'score' in summary ? summary.score?.scale : null+ return await Promise.all([+ recentFailures && summary.kind === 'pass_rate'+ ? evaluationsRunsRetrieve(projectId, row.id, {+ ...params,+ outcome: 'failed',+ limit: RECENT_FAILURES_LIMIT,+ }).then((runs) => runs.results)+ : null,+ scale && scale.min < scale.max ? evaluationsScoreDistributionRetrieve(projectId, row.id, params) : null,+ ]).then(+ ([recentFailures, scoreDistribution]): DetailsLoad => ({+ status: 'ready',+ details: { recentFailures, scoreDistribution },+ }),+ (error: unknown): DetailsLoad => ({ status: 'error', failure: failureOf(error) })+ )+}+function detailsKey(evaluationId: string, request: OverviewRequest): string {return `${evaluationId}/${request.params.date_from}/${request.params.date_to}`}@@ -240,29 +268,14 @@ export const evaluationResultDrawerLogic = kea<evaluationResultDrawerLogicType>(const { date_from, date_to } = values.request.paramsconst refresh = cache.refreshKeys?.has(key) === trueconst generation = cache.generation ?? 0- const projectId = String(values.currentTeamId)- const params = { date_from, date_to, ...(refresh ? { refresh: true } : {}) }- // Each endpoint rejects an evaluation it does not apply to, so the row decides which ones to read.- const summary = values.rowsById.get(evaluationId)?.summary- const scale = summary && 'score' in summary ? summary.score?.scale : null- const load = await Promise.all([- summary?.kind === 'pass_rate'- ? evaluationsRunsRetrieve(projectId, evaluationId, {- ...params,- outcome: 'failed',- limit: RECENT_FAILURES_LIMIT,- }).then((runs) => runs.results)- : null,- scale && scale.min < scale.max- ? evaluationsScoreDistributionRetrieve(projectId, evaluationId, params)- : null,- ]).then(- ([recentFailures, scoreDistribution]): DetailsLoad => ({- status: 'ready',- details: { recentFailures, scoreDistribution },- }),- (error: unknown): DetailsLoad => ({ status: 'error', failure: failureOf(error) })- )+ const row = values.rowsById.get(evaluationId)+ const load: DetailsLoad = row+ ? await fetchEvaluationDetails(String(values.currentTeamId), row, {+ date_from,+ date_to,+ ...(refresh ? { refresh: true } : {}),+ })+ : { status: 'ready', details: { recentFailures: null, scoreDistribution: null } }if (cache.prefetchingKey === key) {cache.prefetchingKey = null}diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.tsindex f5a68d9a0017..b07af08319d6 100644--- a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts+++ b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts@@ -10,7 +10,8 @@ import { initKeaTests } from '~/test/init'import type { EvaluationApi, EvaluationResultApi } from '../../generated/api.schemas'import { llmEvaluationsLogic } from '../llmEvaluationsLogic'-import { hasIsolatedPoints, passFailCounts } from './evaluationOverviewDisplay'+import { topBreakdownSeries } from './evaluationDetailResultsLogic'+import { bucketDateRange, hasIsolatedPoints, passFailCounts } from './evaluationOverviewDisplay'import { evaluationResultDrawerLogic } from './evaluationResultDrawerLogic'import {evaluationsOverviewLogic,@@ -148,6 +149,25 @@ describe('evaluationsOverviewLogic', () => {expect(hasIsolatedPoints(sparse)).toBe(expected)})+ test.each([+ ['a day in a time zone ahead of UTC', '2026-10-20T00:00:00+02:00', 'day', '2026-10-20', '2026-10-20'],+ ['an hour', '2026-10-20T14:00:00+02:00', 'hour', '2026-10-20T12:00:00Z', '2026-10-20T13:00:00Z'],+ ] as const)('narrows the runs to %s', (_, bucket, interval, dateFrom, dateTo) => {+ expect(bucketDateRange(bucket, interval)).toEqual({ dateFrom, dateTo })+ })++ it('keeps the values with the most graded runs and weighs the rest into one Other result', () => {+ const series = [+ { key: 'a', label: 'a', values: [50, NaN], graded: [10, 0] },+ { key: 'b', label: 'b', values: [80, 100], graded: [1, 3] },+ { key: 'c', label: 'c', values: [20, NaN], graded: [3, 0] },+ ]+ expect(topBreakdownSeries(series, 1)).toEqual([+ series[0],+ { key: '$$_posthog_breakdown_other_$$', label: 'Other', values: [35, 100], graded: [4, 3] },+ ])+ })+it('splits pass rate volume into exact passed and failed counts, with empty buckets as zero', () => {const row = { ...ROWS[0], trend: { values: [(2 / 3) * 100, null, 100], volume: [3, 0, 7] } }expect(passFailCounts(row)).toEqual({ passed: [2, 0, 7], failed: [1, 0, 0] })diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.tsindex 8b48a2b4954e..dac160e52826 100644--- a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts+++ b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts@@ -49,8 +49,8 @@ const SCOPE_CHANGE_DEBOUNCE_MS = 300/** At or below this many evaluations, the page keeps a way to create another in view. */const FEW_EVALUATIONS_MAX = 3-/** The backend rejects a longer period. */-const OVERVIEW_MAX_DAYS = 90+/** The longest period people can pick. The backend accepts one day more, for the extra day the runs table counts. */+export const OVERVIEW_MAX_DAYS = 90/** How many evaluations the chart picks on its first render. People can pick more. */const DEFAULT_COMPARED = 3@@ -88,12 +88,17 @@ export function markEvaluationsOverviewStale(): void {evaluationsOverviewLogic.findMounted()?.actions.reloadOverview()}+/** Records that a read for this request skipped the cache, so later reads for it may use the cache again. */+export function markRefreshedAfterChange(requestKey: string): void {+ refreshedSinceChange.add(requestKey)+}+export function resetEvaluationsOverviewStale(): void {lastChangeAt = nullrefreshedSinceChange.clear()}-function needsRefreshAfterChange(requestKey: string): boolean {+export function needsRefreshAfterChange(requestKey: string): boolean {if (lastChangeAt === null || Date.now() - lastChangeAt > BACKEND_CACHE_MS) {return false}