Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): show results over time on the evaluation page (#123427)

Bernat Torresbfd2b7ae0037
diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.py
index e6a11e200495..df4377acccf5 100644
--- a/products/ai_observability/backend/api/evaluations.py
+++ b/products/ai_observability/backend/api/evaluations.py
@@ -1629,7 +1629,7 @@ class EvaluationViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, Forbi
query_serializer=EvaluationsOverviewQuerySerializer,
responses={
200: OpenApiResponse(response=contracts.EvaluationsOverview),
- 400: OpenApiResponse(description="The period is malformed, empty, longer than 90 days or out of range."),
+ 400: OpenApiResponse(description="The period is malformed, empty, longer than 91 days or out of range."),
429: OpenApiResponse(description="Too many queries are running for the project or organization."),
},
)
diff --git a/products/ai_observability/backend/facade/api.py b/products/ai_observability/backend/facade/api.py
index e33c2afef281..a81869fbb9db 100644
--- a/products/ai_observability/backend/facade/api.py
+++ b/products/ai_observability/backend/facade/api.py
@@ -225,7 +225,7 @@ def _to_evaluation_result(result: EvaluationResult) -> contracts.TraceEvaluation
def evaluations_period(team: Team, date_from: date | datetime | None, date_to: date | datetime | None) -> PeriodBounds:
- """Raises `InvalidPeriodError` for a period that is empty, longer than 90 days or out of range."""
+ """Raises `InvalidPeriodError` for a period that is empty, longer than 91 days or out of range."""
return PeriodBounds.from_request(date_from, date_to, team.timezone_info)
diff --git a/products/ai_observability/backend/logic/evaluations/period.py b/products/ai_observability/backend/logic/evaluations/period.py
index 831c880e02c2..7d04637f6b87 100644
--- a/products/ai_observability/backend/logic/evaluations/period.py
+++ b/products/ai_observability/backend/logic/evaluations/period.py
@@ -8,7 +8,8 @@ Interval = Literal["hour", "day"]
# A period this short or shorter is bucketed by hour, so a day or two still draws a line rather than one or two dots.
MAX_HOURLY_SPAN = timedelta(days=2)
DEFAULT_DAYS = 14
-MAX_DAYS = 90
+# The longest preset, "Last 90 days", covers today and the 90 days before it.
+MAX_DAYS = 91
def _resolve_bound(value: date | datetime, tz: tzinfo, *, is_end: bool) -> datetime:
diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.py b/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.py
index 12a829bd3865..7554de7e2d27 100644
--- a/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.py
+++ b/products/ai_observability/backend/tests/logic/evaluations/test_overview_queries.py
@@ -69,20 +69,20 @@ def test_rolling_hours_keep_their_real_length_across_daylight_saving(_name: str,
@parameterized.expand(
[
("end before start", date(2026, 3, 10), date(2026, 3, 9), "date_from"),
- ("91 days", date(2026, 8, 4), date(2026, 11, 2), "date_from"),
+ ("92 days", date(2026, 8, 3), date(2026, 11, 2), "date_from"),
("last day of year 9999", None, date(9999, 12, 31), "out of range"),
("first day of year 1", None, date(1, 1, 1), "out of range"),
- # Clocks fall back on November 1, so these 90 days are 2161 hours long.
- ("90 days across daylight saving", date(2026, 8, 5), date(2026, 11, 2), None),
+ # Clocks fall back on November 1, so these 91 days are 2185 hours long.
+ ("91 days across daylight saving", date(2026, 8, 4), date(2026, 11, 2), None),
(
- "90 days of instants",
- datetime(2026, 8, 5, tzinfo=UTC),
+ "91 days of instants",
+ datetime(2026, 8, 4, tzinfo=UTC),
datetime(2026, 11, 3, tzinfo=UTC),
None,
),
(
- "an instant past 90 days",
- datetime(2026, 8, 5, tzinfo=UTC),
+ "an instant past 91 days",
+ datetime(2026, 8, 4, tzinfo=UTC),
datetime(2026, 11, 3, 0, 0, 1, tzinfo=UTC),
"date_from",
),
diff --git a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
index dad46be4d9c5..1d248a5f9ab7 100644
--- a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
+++ b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
@@ -74,6 +74,7 @@ import {
hasUnsetConditionRollout,
llmEvaluationLogic,
} from './llmEvaluationLogic'
+import { EvaluationDetailResults } from './overview/EvaluationDetailResults'
import { statusReasonLabel, statusReasonRecoveryLabel } from './statusDisplay'
import { EvaluationSettleStrategy, EvaluationTarget, EvaluationType } from './types'
@@ -106,6 +107,7 @@ export function AIObservabilityEvaluation(): JSX.Element {
const { searchParams } = useValues(router)
const { featureFlags } = useValues(featureFlagLogic)
const settlingStrategyEnabled = !!featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EVAL_SETTLING_STRATEGY]
+ const resultsCardsEnabled = !!featureFlags[FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW]
const {
setEvaluationName,
setEvaluationDescription,
@@ -466,6 +468,11 @@ export function AIObservabilityEvaluation(): JSX.Element {
)}
</LemonBanner>
)}
+ {resultsCardsEnabled && !runsBackfillId && (
+ <div className="mb-4">
+ <EvaluationDetailResults evaluationId={evaluation.id} />
+ </div>
+ )}
<EvaluationRunsTable />
</div>
),
diff --git a/products/ai_observability/frontend/evaluations/constants.ts b/products/ai_observability/frontend/evaluations/constants.ts
index 227e31315595..61f9370ef5c0 100644
--- a/products/ai_observability/frontend/evaluations/constants.ts
+++ b/products/ai_observability/frontend/evaluations/constants.ts
@@ -61,6 +61,14 @@ export const EVALUATION_NUMERIC_GRADED_HOGQL = `properties.$ai_evaluation_result
export const EVALUATION_BOOLEAN_GRADED_HOGQL = `(isNull(properties.$ai_evaluation_result_type) OR properties.$ai_evaluation_result_type = 'boolean') AND properties.$ai_evaluation_result IS NOT NULL AND ${EVALUATION_NOT_SKIPPED_HOGQL}`
export const EVALUATION_NUMERIC_MEAN_HOGQL = `avgIf(toFloat(properties.$ai_evaluation_numeric_result), ${EVALUATION_NUMERIC_GRADED_HOGQL})`
+export function evaluationGradedHogQL(outputType: string): string {
+ return outputType === 'categorical'
+ ? EVALUATION_CATEGORICAL_GRADED_HOGQL
+ : outputType === 'numeric'
+ ? EVALUATION_NUMERIC_GRADED_HOGQL
+ : EVALUATION_BOOLEAN_GRADED_HOGQL
+}
+
export function numericEvaluationPassedHogQL(evaluation: Pick<EvaluationConfig, 'output_config'>): string {
const rule = evaluation.output_config.passing_rule
if (!rule || !('threshold' in rule) || !Number.isFinite(rule.threshold)) {
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
index a85df921f56c..700ba3057380 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
@@ -10,7 +10,7 @@ import { teamLogic } from 'scenes/teamLogic'
import { urls } from 'scenes/urls'
import { SIDE_PANEL_CONTEXT_KEY, SidePanelSceneContext } from '~/layout/navigation-3000/sidepanel/types'
-import { DateRange, InsightVizNode, NodeKind } from '~/queries/schema/schema-general'
+import { DateRange, InsightVizNode, NodeKind, TrendsQuery } from '~/queries/schema/schema-general'
import { MaxContextInput, createMaxContextHelpers } from '~/scenes/max/maxTypes'
import { ActivityScope, Breadcrumb, ChartDisplayType, HogQLMathType } from '~/types'
@@ -359,6 +359,7 @@ export interface llmEvaluationLogicValues {
selectedPickerProviderKeyId: string | null
sidePanelContext: SidePanelSceneContext | null
trendInsightUrl: string | null
+ trendsQuery: TrendsQuery | null
usesDecisionModel: boolean
}
@@ -566,7 +567,8 @@ export interface llmEvaluationLogicMeta {
evaluation: EvaluationConfig | null,
originalEvaluation: EvaluationConfig | null
) => boolean
- trendInsightUrl: (originalEvaluation: EvaluationConfig | null) => string | null
+ trendsQuery: (originalEvaluation: EvaluationConfig | null) => TrendsQuery | null
+ trendInsightUrl: (trendsQuery: TrendsQuery | null) => string | null
isNewEvaluation: (evaluationId: string) => boolean
evaluationBackTarget: (isNewEvaluation: boolean, searchParams: Record<string, any>) => EvaluationBackTarget
modelSelectionRequired: (
@@ -1544,75 +1546,80 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
originalEvaluation: EvaluationConfig | null
): boolean => evaluationSupportsReports(isNewEvaluation ? evaluation : originalEvaluation),
],
- trendInsightUrl: [
+ // The evaluation's headline result as a trends query: the pass rate, or the mean score without a passing rule.
+ trendsQuery: [
(s) => [s.originalEvaluation],
- (evaluation: EvaluationConfig | null): string | null =>
+ (evaluation: EvaluationConfig | null): TrendsQuery | null =>
evaluation &&
(evaluationSupportsRunOutcomes(evaluation) || evaluation.output_type === 'numeric') &&
evaluation.id
- ? urls.insightNew({
- query: {
- kind: NodeKind.InsightVizNode,
- source: {
- kind: NodeKind.TrendsQuery,
- series: [
+ ? ({
+ kind: NodeKind.TrendsQuery,
+ series: [
+ {
+ kind: NodeKind.EventsNode,
+ event: '$ai_evaluation',
+ custom_name: `${evaluation.name} - ${evaluation.output_type === 'numeric' && !evaluation.output_config.passing_rule ? 'Mean score' : 'Pass rate'}`,
+ math: HogQLMathType.HogQL,
+ math_hogql:
+ evaluation.output_type === 'categorical'
+ ? evaluationPassRateHogQL(
+ categoricalEvaluationPassedHogQL(evaluation),
+ EVALUATION_CATEGORICAL_GRADED_HOGQL
+ )
+ : evaluation.output_type === 'numeric'
+ ? evaluation.output_config.passing_rule
+ ? evaluationPassRateHogQL(
+ numericEvaluationPassedHogQL(evaluation),
+ EVALUATION_NUMERIC_GRADED_HOGQL
+ )
+ : EVALUATION_NUMERIC_MEAN_HOGQL
+ : evaluationPassRateHogQL(evaluationPassedHogQL(evaluation)),
+ properties: [
{
- kind: NodeKind.EventsNode,
- event: '$ai_evaluation',
- custom_name: `${evaluation.name} - ${evaluation.output_type === 'numeric' && !evaluation.output_config.passing_rule ? 'Mean score' : 'Pass rate'}`,
- math: HogQLMathType.HogQL,
- math_hogql:
- evaluation.output_type === 'categorical'
- ? evaluationPassRateHogQL(
- categoricalEvaluationPassedHogQL(evaluation),
- EVALUATION_CATEGORICAL_GRADED_HOGQL
- )
- : evaluation.output_type === 'numeric'
- ? evaluation.output_config.passing_rule
- ? evaluationPassRateHogQL(
- numericEvaluationPassedHogQL(evaluation),
- EVALUATION_NUMERIC_GRADED_HOGQL
- )
- : EVALUATION_NUMERIC_MEAN_HOGQL
- : evaluationPassRateHogQL(evaluationPassedHogQL(evaluation)),
- properties: [
- {
- key: '$ai_evaluation_id',
- value: evaluation.id,
- operator: 'exact',
- type: 'event',
- },
- ],
+ key: '$ai_evaluation_id',
+ value: evaluation.id,
+ operator: 'exact',
+ type: 'event',
},
- ...(evaluation.output_type === 'boolean' && evaluation.output_config.allows_na
- ? [
- {
- kind: NodeKind.EventsNode as const,
- event: '$ai_evaluation',
- custom_name: `${evaluation.name} — N/A rate`,
- math: HogQLMathType.HogQL as const,
- math_hogql: `if(count() > 0, countIf(properties.$ai_evaluation_result IS NULL) / count() * 100, 0)`,
- properties: [
- {
- key: '$ai_evaluation_id',
- value: evaluation.id,
- operator: 'exact' as const,
- type: 'event' as const,
- },
- ],
- },
- ]
- : []),
],
- trendsFilter: {
- display: ChartDisplayType.ActionsLineGraph,
- },
- dateRange: {
- date_from: '-7d',
- },
- interval: 'day',
},
- } as InsightVizNode,
+ ...(evaluation.output_type === 'boolean' && evaluation.output_config.allows_na
+ ? [
+ {
+ kind: NodeKind.EventsNode as const,
+ event: '$ai_evaluation',
+ custom_name: `${evaluation.name} — N/A rate`,
+ math: HogQLMathType.HogQL as const,
+ math_hogql: `if(count() > 0, countIf(properties.$ai_evaluation_result IS NULL) / count() * 100, 0)`,
+ properties: [
+ {
+ key: '$ai_evaluation_id',
+ value: evaluation.id,
+ operator: 'exact' as const,
+ type: 'event' as const,
+ },
+ ],
+ },
+ ]
+ : []),
+ ],
+ trendsFilter: {
+ display: ChartDisplayType.ActionsLineGraph,
+ },
+ dateRange: {
+ date_from: '-7d',
+ },
+ interval: 'day',
+ } as TrendsQuery)
+ : null,
+ ],
+ trendInsightUrl: [
+ (s) => [s.trendsQuery],
+ (trendsQuery: TrendsQuery | null): string | null =>
+ trendsQuery
+ ? urls.insightNew({
+ query: { kind: NodeKind.InsightVizNode, source: trendsQuery } as InsightVizNode,
})
: null,
],
diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationBreakdownPicker.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationBreakdownPicker.tsx
new file mode 100644
index 000000000000..3cdd167e820e
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationBreakdownPicker.tsx
@@ -0,0 +1,37 @@
+import { useActions, useValues } from 'kea'
+
+import { LemonSelect } from '@posthog/lemon-ui'
+
+import { llmEvaluationLogic } from '../llmEvaluationLogic'
+import { evaluationDetailResultsLogic } from './evaluationDetailResultsLogic'
+
+/** Properties every result event carries. The prompt ones are copied from the generation the evaluation scored. */
+const BREAKDOWN_OPTIONS = [
+ { value: '$ai_prompt_version', label: 'Prompt version' },
+ { value: '$ai_prompt_name', label: 'Prompt name' },
+ { value: '$ai_evaluation_model', label: 'Judge model', judgeOnly: true },
+ { value: '$ai_evaluation_trigger', label: 'Live or backfill' },
+]
+
+export function EvaluationBreakdownPicker({ evaluationId }: { evaluationId: string }): JSX.Element {
+ const logic = evaluationDetailResultsLogic({ evaluationId })
+ const { breakdownProperty, breakdownSeriesLoading } = useValues(logic)
+ const { setBreakdownProperty } = useActions(logic)
+ const { originalEvaluation } = useValues(llmEvaluationLogic({ evaluationId }))
+
+ return (
+ <LemonSelect
+ size="small"
+ value={breakdownProperty}
+ onChange={setBreakdownProperty}
+ loading={breakdownSeriesLoading}
+ options={[
+ { value: null, label: 'No breakdown' },
+ ...BREAKDOWN_OPTIONS.filter(
+ (option) => !option.judgeOnly || originalEvaluation?.evaluation_type === 'llm_judge'
+ ).map(({ value, label }) => ({ value, label: `By ${label.toLowerCase()}` })),
+ ]}
+ data-attr="llma-evaluation-results-breakdown"
+ />
+ )
+}
diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationDetailResults.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationDetailResults.tsx
new file mode 100644
index 000000000000..63e7cffb178e
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationDetailResults.tsx
@@ -0,0 +1,107 @@
+import { useActions, useValues } from 'kea'
+
+import { IconClockRewind } from '@posthog/icons'
+import { LemonBanner, LemonButton, LemonSkeleton } from '@posthog/lemon-ui'
+
+import { teamLogic } from 'scenes/teamLogic'
+
+import { EvaluationBreakdownPicker } from './EvaluationBreakdownPicker'
+import { evaluationDetailResultsLogic } from './evaluationDetailResultsLogic'
+import { RATE_LIMITED_MESSAGE, hasIsolatedPoints } from './evaluationOverviewDisplay'
+import { EvaluationResultCards } from './EvaluationResultCards'
+
+export function EvaluationDetailResults({ evaluationId }: { evaluationId: string }): JSX.Element | null {
+ const logic = evaluationDetailResultsLogic({ evaluationId })
+ const {
+ results,
+ resultsLoading,
+ details,
+ detailsLoading,
+ request,
+ failure,
+ breakdownProperty,
+ breakdownSeries,
+ breakdownSeriesLoading,
+ breakdownFailure,
+ trendsQuery,
+ } = useValues(logic)
+ const { loadResults, loadDetails, loadBreakdown, selectBucket, runOnPastData } = useActions(logic)
+ const { timezone } = useValues(teamLogic)
+
+ if (request.notice) {
+ return <LemonBanner type="info">{request.notice}</LemonBanner>
+ }
+ if (failure) {
+ return (
+ <LemonBanner
+ type={failure === 'rate_limited' ? 'warning' : 'error'}
+ action={{
+ children: 'Try again',
+ onClick: () => loadResults(),
+ 'data-attr': 'llma-evaluation-results-retry',
+ }}
+ >
+ {failure === 'rate_limited' ? RATE_LIMITED_MESSAGE : "Couldn't load the results over time."}
+ </LemonBanner>
+ )
+ }
+ if (!results) {
+ return resultsLoading ? <LemonSkeleton className="h-56 w-full" /> : null
+ }
+ const { row, buckets, interval } = results
+ // Nothing graded leaves the runs table below to say so, so the cards would only repeat it.
+ if (!row || row.gradedCount === 0) {
+ return null
+ }
+
+ return (
+ <EvaluationResultCards
+ row={row}
+ buckets={buckets}
+ interval={interval}
+ timezone={timezone}
+ // A retry shows the skeleton, so the error's Try again can't be clicked again while it runs.
+ details={detailsLoading ? null : details}
+ onRetryDetails={() => loadDetails()}
+ onSelectBucket={selectBucket}
+ showFailingExamples={false}
+ chartAction={trendsQuery ? <EvaluationBreakdownPicker evaluationId={evaluationId} /> : null}
+ breakdown={breakdownProperty ? breakdownSeries : null}
+ chartFooter={
+ breakdownFailure ? (
+ <div className="flex flex-wrap items-center gap-x-2 text-xs text-secondary">
+ <span>
+ {breakdownFailure === 'rate_limited'
+ ? RATE_LIMITED_MESSAGE
+ : "Couldn't split the results by this property."}
+ </span>
+ <LemonButton
+ size="xsmall"
+ type="tertiary"
+ onClick={() => loadBreakdown()}
+ loading={breakdownSeriesLoading}
+ data-attr="llma-evaluation-results-breakdown-retry"
+ >
+ Try again
+ </LemonButton>
+ </div>
+ ) : (
+ hasIsolatedPoints(row) && (
+ <div className="flex flex-wrap items-center gap-x-2 text-xs text-secondary">
+ <span>{`This evaluation has ${interval}s with no results, so some points have no line.`}</span>
+ <LemonButton
+ size="xsmall"
+ type="tertiary"
+ icon={<IconClockRewind />}
+ onClick={runOnPastData}
+ data-attr="llma-evaluation-results-backfill-hint"
+ >
+ Run on past data
+ </LemonButton>
+ </div>
+ )
+ )
+ }
+ />
+ )
+}
diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationResultCards.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationResultCards.tsx
new file mode 100644
index 000000000000..3cd57c65b1d1
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationResultCards.tsx
@@ -0,0 +1,121 @@
+import { cn } from 'lib/utils/css-classes'
+
+import type { EvaluationResultApi } from '../../generated/api.schemas'
+import { EvaluationCategoryShares } from './EvaluationCategoryShares'
+import { EvaluationFailingExamples } from './EvaluationFailingExamples'
+import type { BucketInterval } from './evaluationOverviewDisplay'
+import { EvaluationResultDetails } from './EvaluationResultDetails'
+import type { DetailsLoad } from './evaluationResultDrawerLogic'
+import { EvaluationResultOutcomes } from './EvaluationResultOutcomes'
+import { EvaluationResultTrendChart, type TrendBreakdownSeries } from './EvaluationResultTrendChart'
+import { EvaluationScoreHistogram } from './EvaluationScoreHistogram'
+
+function Card({
+ title,
+ action,
+ className,
+ children,
+}: {
+ title: string
+ action?: React.ReactNode
+ className?: string
+ children: React.ReactNode
+}): JSX.Element {
+ return (
+ <section className={cn('border rounded bg-surface-primary p-3 flex flex-col gap-2 min-w-0', className)}>
+ <div className="flex flex-wrap items-center justify-between gap-2">
+ <h4 className="mb-0 text-sm font-semibold">{title}</h4>
+ {action}
+ </div>
+ {children}
+ </section>
+ )
+}
+
+export function EvaluationResultCards({
+ row,
+ buckets,
+ interval,
+ timezone,
+ details,
+ onRetryDetails,
+ onSelectBucket,
+ chartFooter,
+ chartAction,
+ breakdown,
+ showFailingExamples = true,
+}: {
+ row: EvaluationResultApi
+ buckets: string[]
+ interval: BucketInterval
+ timezone: string
+ details: DetailsLoad | null
+ onRetryDetails: () => void
+ onSelectBucket?: (bucket: string) => void
+ chartFooter?: React.ReactNode
+ chartAction?: React.ReactNode
+ breakdown?: TrendBreakdownSeries[] | null
+ /** The evaluation page lists every run below the cards, failing ones a filter away, so it leaves them out. */
+ showFailingExamples?: boolean
+}): JSX.Element {
+ const { summary } = row
+ const hasScore = summary.kind === 'score' || (summary.kind === 'pass_rate' && row.outputType === 'numeric')
+ // Outcomes alone on its row takes the full width rather than leaving half of it empty.
+ const outcomesAlone = !hasScore && !showFailingExamples
+
+ return (
+ <div className="@container">
+ <div className="grid gap-3 grid-cols-1 @2xl:grid-cols-2">
+ <Card title="Over time" action={chartAction} className="@2xl:col-span-2">
+ <EvaluationResultTrendChart
+ row={row}
+ buckets={buckets}
+ interval={interval}
+ timezone={timezone}
+ onSelectBucket={onSelectBucket}
+ footer={chartFooter}
+ breakdown={breakdown}
+ />
+ </Card>
+ {summary.kind === 'pass_rate' && (
+ <Card title="Outcomes" className={outcomesAlone ? '@2xl:col-span-2' : undefined}>
+ <EvaluationResultOutcomes row={row} summary={summary} />
+ </Card>
+ )}
+ {(summary.kind === 'categories' || summary.kind === 'sentiment') && (
+ <Card title={summary.kind === 'sentiment' ? 'Sentiment split' : 'Labels'}>
+ <EvaluationCategoryShares summary={summary} gradedCount={row.gradedCount} />
+ </Card>
+ )}
+ {hasScore && (
+ <Card title="Score distribution">
+ <EvaluationResultDetails load={details} onRetry={onRetryDetails}>
+ {(details) =>
+ details.scoreDistribution ? (
+ <EvaluationScoreHistogram histogram={details.scoreDistribution} />
+ ) : (
+ <div className="text-sm text-secondary">
+ Set score bounds on the evaluation to see how scores spread.
+ </div>
+ )
+ }
+ </EvaluationResultDetails>
+ </Card>
+ )}
+ {summary.kind === 'pass_rate' && showFailingExamples && (
+ <Card title="Recent failing examples">
+ <EvaluationResultDetails load={details} onRetry={onRetryDetails}>
+ {(details) =>
+ details.recentFailures && details.recentFailures.length > 0 ? (
+ <EvaluationFailingExamples failures={details.recentFailures} />
+ ) : (
+ <div className="text-sm text-secondary">No failed runs in this range.</div>
+ )
+ }
+ </EvaluationResultDetails>
+ </Card>
+ )}
+ </div>
+ </div>
+ )
+}
diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationResultDetails.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationResultDetails.tsx
index 0b999f1b5525..df93b2521e34 100644
--- a/products/ai_observability/frontend/evaluations/overview/EvaluationResultDetails.tsx
+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationResultDetails.tsx
@@ -1,21 +1,17 @@
-import { useActions } from 'kea'
-
import { LemonButton, LemonSkeleton } from '@posthog/lemon-ui'
import { RATE_LIMITED_MESSAGE } from './evaluationOverviewDisplay'
-import { type DetailsLoad, type EvaluationDetails, evaluationResultDrawerLogic } from './evaluationResultDrawerLogic'
+import type { DetailsLoad, EvaluationDetails } from './evaluationResultDrawerLogic'
export function EvaluationResultDetails({
- evaluationId,
load,
+ onRetry,
children,
}: {
- evaluationId: string
load: DetailsLoad | null
+ onRetry: () => void
children: (details: EvaluationDetails) => JSX.Element
}): JSX.Element {
- const { loadDetails } = useActions(evaluationResultDrawerLogic)
-
if (load?.status === 'ready') {
return children(load.details)
}
@@ -28,7 +24,7 @@ export function EvaluationResultDetails({
<LemonButton
size="small"
type="secondary"
- onClick={() => loadDetails(evaluationId, 'open')}
+ onClick={onRetry}
data-attr="evaluation-overview-drawer-retry"
>
Try again
diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationResultExpanded.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationResultExpanded.tsx
index d7570f48caf2..baa1bfbb899f 100644
--- a/products/ai_observability/frontend/evaluations/overview/EvaluationResultExpanded.tsx
+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationResultExpanded.tsx
@@ -1,46 +1,22 @@
-import { useValues } from 'kea'
+import { useActions, useValues } from 'kea'
import { LemonBanner, Link } from '@posthog/lemon-ui'
-import { cn } from 'lib/utils/css-classes'
-
import type { EvaluationResultApi } from '../../generated/api.schemas'
-import { EvaluationCategoryShares } from './EvaluationCategoryShares'
+import { EvaluationBackfillHint } from './EvaluationBackfillHint'
import { EvaluationDrawerEmptyState } from './EvaluationDrawerEmptyState'
-import { EvaluationFailingExamples } from './EvaluationFailingExamples'
-import { describeAttention } from './evaluationOverviewDisplay'
+import { describeAttention, hasIsolatedPoints } from './evaluationOverviewDisplay'
import { EvaluationPausedNotice } from './EvaluationPausedNotice'
-import { EvaluationResultDetails } from './EvaluationResultDetails'
+import { EvaluationResultCards } from './EvaluationResultCards'
import { evaluationResultDrawerLogic } from './evaluationResultDrawerLogic'
-import { EvaluationResultOutcomes } from './EvaluationResultOutcomes'
-import { EvaluationResultTrendChart } from './EvaluationResultTrendChart'
-import { EvaluationScoreHistogram } from './EvaluationScoreHistogram'
import { evaluationsOverviewLogic } from './evaluationsOverviewLogic'
-function Card({
- title,
- className,
- children,
-}: {
- title: string
- className?: string
- children: React.ReactNode
-}): JSX.Element {
- return (
- <section className={cn('border rounded bg-surface-primary p-3 flex flex-col gap-2 min-w-0', className)}>
- <h4 className="mb-0 text-sm font-semibold">{title}</h4>
- {children}
- </section>
- )
-}
-
export function EvaluationResultExpanded({ row }: { row: EvaluationResultApi }): JSX.Element {
- const { configById, timezone, evaluationUrl } = useValues(evaluationsOverviewLogic)
+ const { configById, timezone, evaluationUrl, buckets, interval } = useValues(evaluationsOverviewLogic)
const { openDetails } = useValues(evaluationResultDrawerLogic)
- const { summary } = row
+ const { loadDetails } = useActions(evaluationResultDrawerLogic)
const errored = row.attentionReasons.includes('errored')
const enabled = configById.get(row.id)?.enabled ?? row.enabled
- const hasScore = summary.kind === 'score' || (summary.kind === 'pass_rate' && row.outputType === 'numeric')
const notice = errored ? (
<LemonBanner type="error">
@@ -58,49 +34,15 @@ export function EvaluationResultExpanded({ row }: { row: EvaluationResultApi }):
return (
<div className="@container flex flex-col gap-3">
{notice}
- <div className="grid gap-3 grid-cols-1 @2xl:grid-cols-2">
- <Card title="Over time" className="@2xl:col-span-2">
- <EvaluationResultTrendChart row={row} />
- </Card>
- {summary.kind === 'pass_rate' && (
- <Card title="Outcomes">
- <EvaluationResultOutcomes row={row} summary={summary} />
- </Card>
- )}
- {(summary.kind === 'categories' || summary.kind === 'sentiment') && (
- <Card title={summary.kind === 'sentiment' ? 'Sentiment split' : 'Labels'}>
- <EvaluationCategoryShares summary={summary} gradedCount={row.gradedCount} />
- </Card>
- )}
- {hasScore && (
- <Card title="Score distribution">
- <EvaluationResultDetails evaluationId={row.id} load={openDetails}>
- {(details) =>
- details.scoreDistribution ? (
- <EvaluationScoreHistogram histogram={details.scoreDistribution} />
- ) : (
- <div className="text-sm text-secondary">
- Set score bounds on the evaluation to see how scores spread.
- </div>
- )
- }
- </EvaluationResultDetails>
- </Card>
- )}
- {summary.kind === 'pass_rate' && (
- <Card title="Recent failing examples">
- <EvaluationResultDetails evaluationId={row.id} load={openDetails}>
- {(details) =>
- details.recentFailures && details.recentFailures.length > 0 ? (
- <EvaluationFailingExamples failures={details.recentFailures} />
- ) : (
- <div className="text-sm text-secondary">No failed runs in this range.</div>
- )
- }
- </EvaluationResultDetails>
- </Card>
- )}
- </div>
+ <EvaluationResultCards
+ row={row}
+ buckets={buckets}
+ interval={interval}
+ timezone={timezone}
+ details={openDetails}
+ onRetryDetails={() => loadDetails(row.id, 'open')}
+ chartFooter={hasIsolatedPoints(row) && <EvaluationBackfillHint row={row} source="drawer" />}
+ />
</div>
)
}
diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationResultTrendChart.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationResultTrendChart.tsx
index 5283bb66743f..3f19d4588efe 100644
--- a/products/ai_observability/frontend/evaluations/overview/EvaluationResultTrendChart.tsx
+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationResultTrendChart.tsx
@@ -1,4 +1,3 @@
-import { useValues } from 'kea'
import { useMemo } from 'react'
import { TimeSeriesComboChart, TooltipSurface, TooltipSwatch } from '@posthog/quill-charts'
@@ -9,16 +8,14 @@ import { getColorVar } from 'lib/colors'
import { humanFriendlyNumber } from 'lib/utils/numbers'
import type { EvaluationResultApi } from '../../generated/api.schemas'
-import { EvaluationBackfillHint } from './EvaluationBackfillHint'
import {
+ type BucketInterval,
formatBucket,
- hasIsolatedPoints,
hourTick,
passFailCounts,
toChartValues,
trackedCategory,
} from './evaluationOverviewDisplay'
-import { evaluationsOverviewLogic } from './evaluationsOverviewLogic'
const VOLUME_KEY = 'volume'
const PASSED_KEY = 'passed'
@@ -44,9 +41,38 @@ function withAlpha(color: string, alpha: number): string {
: color
}
-export function EvaluationResultTrendChart({ row }: { row: EvaluationResultApi }): JSX.Element {
+/** One value of a breakdown property, aligned with the buckets. NaN marks a bucket where it had nothing graded. */
+export interface TrendBreakdownSeries {
+ key: string
+ label: string
+ values: number[]
+ graded: number[]
+}
+
+/** The overall line uses the theme's first color, so breakdown lines start at the second. */
+function breakdownColor(colors: string[], index: number): string {
+ return colors[(index + 1) % colors.length]
+}
+
+export function EvaluationResultTrendChart({
+ row,
+ buckets,
+ interval,
+ timezone,
+ onSelectBucket,
+ footer,
+ breakdown,
+}: {
+ row: EvaluationResultApi
+ buckets: string[]
+ interval: BucketInterval
+ timezone: string
+ /** Called with the clicked bucket. Without it the chart is not clickable. */
+ onSelectBucket?: (bucket: string) => void
+ footer?: React.ReactNode
+ breakdown?: TrendBreakdownSeries[] | null
+}): JSX.Element {
const theme = useChartTheme()
- const { buckets, timezone, interval } = useValues(evaluationsOverviewLogic)
const { summary, trend } = row
const isScore = summary.kind === 'score'
const label = trackedCategory(row)?.label ?? HEADLINE_LABELS[summary.kind]
@@ -97,20 +123,33 @@ export function EvaluationResultTrendChart({ row }: { row: EvaluationResultApi }
color: 'transparent',
visibility: { tooltip: false, total: false, valueLabel: false },
},
- {
- key: 'headline',
- label,
- // NaN leaves a gap, so a bucket with no graded results never reads as a measured 0%.
- data: values,
- type: 'line',
- color: theme.colors[0],
- points: { radius: 3 },
- },
+ ...(breakdown
+ ? breakdown.map(
+ (entry, index): Series => ({
+ key: `breakdown-${entry.key}`,
+ label: entry.label,
+ data: entry.values,
+ type: 'line',
+ color: breakdownColor(theme.colors, index),
+ points: { radius: 3 },
+ })
+ )
+ : [
+ {
+ key: 'headline',
+ label,
+ // NaN leaves a gap, so a bucket with no graded results never reads as a measured 0%.
+ data: values,
+ type: 'line' as const,
+ color: theme.colors[0],
+ points: { radius: 3 },
+ },
+ ]),
]
- }, [trend, values, counts, label, theme])
+ }, [trend, values, counts, label, theme, breakdown])
const config = useMemo<TimeSeriesComboChartConfig>(() => {
- const finite = values.filter(Number.isFinite)
+ const finite = (breakdown ? breakdown.flatMap((entry) => entry.values) : values).filter(Number.isFinite)
// Zoom in when every point sits high, and let the axis labels say so.
const floor = finite.length > 0 ? Math.max(0, Math.floor((Math.min(...finite) - 10) / 10) * 10) : 0
const scale = summary.kind === 'score' ? summary.score?.scale : null
@@ -126,7 +165,7 @@ export function EvaluationResultTrendChart({ row }: { row: EvaluationResultApi }
],
valueDomain: isScore ? (scale ? { min: scale.min, max: scale.max } : undefined) : { min: floor, max: 100 },
}
- }, [isScore, summary, values, timezone, interval])
+ }, [isScore, summary, values, timezone, interval, breakdown])
const tooltip = (ctx: TooltipContext): JSX.Element => {
const value = values[ctx.dataIndex]
@@ -134,7 +173,24 @@ export function EvaluationResultTrendChart({ row }: { row: EvaluationResultApi }
return (
<TooltipSurface>
<div className="font-semibold">{formatBucket(String(ctx.label), interval)}</div>
- {Number.isFinite(value) ? (
+ {breakdown ? (
+ breakdown.map((entry, index) =>
+ Number.isFinite(entry.values[ctx.dataIndex]) ? (
+ <div key={entry.key} className="flex items-center gap-2 mt-1">
+ <TooltipSwatch color={breakdownColor(theme.colors, index)} />
+ <span className="truncate max-w-60">{entry.label}</span>
+ <strong className="ml-auto tabular-nums">
+ {isScore
+ ? entry.values[ctx.dataIndex].toFixed(2)
+ : `${entry.values[ctx.dataIndex].toFixed(1)}%`}
+ </strong>
+ <span className="opacity-70 tabular-nums">
+ {`${humanFriendlyNumber(entry.graded[ctx.dataIndex] ?? 0)} graded`}
+ </span>
+ </div>
+ ) : null
+ )
+ ) : Number.isFinite(value) ? (
<div className="flex items-center gap-2 mt-1">
{line && <TooltipSwatch color={line.color} />}
<span>{label}</span>
@@ -181,14 +237,34 @@ export function EvaluationResultTrendChart({ row }: { row: EvaluationResultApi }
theme={theme}
config={config}
tooltip={tooltip}
+ onPointClick={
+ onSelectBucket
+ ? ({ dataIndex }) => buckets[dataIndex] && onSelectBucket(buckets[dataIndex])
+ : undefined
+ }
/>
</div>
<div className="text-xs text-secondary">
{counts
? `Bars show how many results passed and failed each ${interval}.`
: `Shaded bars show how many results were graded each ${interval}.`}
+ {onSelectBucket ? ` Click a${interval === 'hour' ? 'n' : ''} ${interval} to see its runs.` : ''}
</div>
- {hasIsolatedPoints(row) && <EvaluationBackfillHint row={row} source="drawer" />}
+ {breakdown && breakdown.length > 0 && (
+ <div className="flex flex-wrap gap-x-3 gap-y-1 text-xs">
+ {breakdown.map((entry, index) => (
+ <span key={entry.key} className="flex items-center gap-1 min-w-0">
+ <span
+ className="inline-block size-2 shrink-0 rounded-full"
+ // eslint-disable-next-line react/forbid-dom-props
+ style={{ backgroundColor: breakdownColor(theme.colors, index) }}
+ />
+ <span className="truncate max-w-60">{entry.label}</span>
+ </span>
+ ))}
+ </div>
+ )}
+ {footer}
</div>
)
}
diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationDetailResultsLogic.ts b/products/ai_observability/frontend/evaluations/overview/evaluationDetailResultsLogic.ts
new file mode 100644
index 000000000000..e22b29e30159
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/overview/evaluationDetailResultsLogic.ts
@@ -0,0 +1,574 @@
+import { MakeLogicType, actions, afterMount, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea'
+import { loaders } from 'kea-loaders'
+import posthog from 'posthog-js'
+
+import { dayjs } from 'lib/dayjs'
+import { uuid } from 'lib/utils/dom'
+import { teamLogic } from 'scenes/teamLogic'
+
+import { cancelQueryOnServer } from '~/queries/nodes/DataNode/dataNodeLogic'
+import { performQuery } from '~/queries/query'
+import { DateRange, NodeKind, TrendsQuery, TrendsQueryResponse } from '~/queries/schema/schema-general'
+import { escapeHogQLString, escapePropertyAsHogQLIdentifier } from '~/queries/utils'
+import { HogQLMathType, type HogQLPropertyFilter, PropertyFilterType } from '~/types'
+
+import { evaluationsOverviewRetrieve } from '../../generated/api'
+import type { EvaluationResultApi, EvaluationsOverviewRetrieveParams } from '../../generated/api.schemas'
+import type { EvaluationsOverviewIntervalEnumApi } from '../../generated/api.schemas'
+import { evaluationGradedHogQL } from '../constants'
+import { llmEvaluationLogic } from '../llmEvaluationLogic'
+import type { EvaluationConfig } from '../types'
+import { type BucketInterval, bucketDateRange } from './evaluationOverviewDisplay'
+import { type DetailsLoad, fetchEvaluationDetails } from './evaluationResultDrawerLogic'
+import type { TrendBreakdownSeries } from './EvaluationResultTrendChart'
+import {
+ OVERVIEW_MAX_DAYS,
+ type OverviewFailure,
+ type OverviewRequest,
+ failureOf,
+ markRefreshedAfterChange,
+ needsRefreshAfterChange,
+ overviewPeriod,
+} from './evaluationsOverviewLogic'
+
+export interface EvaluationDetailResultsLogicProps {
+ evaluationId: string
+}
+
+/** How many values get their own line. The rest share one "Other" line. */
+const BREAKDOWN_LIMIT = 5
+/**
+ * How many values the query returns. Trends ranks each series on its own and sums the rest into its "Other", which
+ * is wrong for a rate, so the top values and "Other" come from these instead.
+ */
+// ponytail: values past this cap drop out of "Other"; query the counts with HogQL if properties get that wide.
+const BREAKDOWN_QUERY_LIMIT = 100
+const BREAKDOWN_NULL = '$$_posthog_breakdown_null_$$'
+const BREAKDOWN_OTHER = '$$_posthog_breakdown_other_$$'
+/** Trends cuts a breakdown value to this length. */
+const BREAKDOWN_VALUE_MAX_LENGTH = 400
+/** A rolling range of whole days, weeks, months, quarters or years. */
+const ROLLING_DAYS = /^-\d+[dwmqy]$/
+
+/** The key trends gives a bucket: the day, or the day and hour, in the project's time zone. */
+function trendsBucketKey(bucket: string, interval: BucketInterval): string {
+ return interval === 'hour' ? `${bucket.slice(0, 10)} ${bucket.slice(11, 19)}` : bucket.slice(0, 10)
+}
+
+interface BreakdownCounts {
+ result?: number[]
+ graded?: number[]
+ days: string[]
+}
+
+/** Each breakdown value's result and graded count per bucket. */
+function breakdownCounts(response: TrendsQueryResponse): Map<string, BreakdownCounts> {
+ const byValue = new Map<string, BreakdownCounts>()
+ for (const entry of response.results ?? []) {
+ const raw = Array.isArray(entry.breakdown_value) ? entry.breakdown_value[0] : entry.breakdown_value
+ const key = String(raw ?? BREAKDOWN_NULL)
+ const current: BreakdownCounts = byValue.get(key) ?? { days: entry.days ?? [] }
+ current[entry.action?.order === 1 ? 'graded' : 'result'] = entry.data ?? []
+ byValue.set(key, current)
+ }
+ return byValue
+}
+
+/** The breakdown query, limited to the given values, so each series returns all of them. */
+function restrictBreakdown(query: TrendsQuery, values: string[]): TrendsQuery {
+ const property = String(query.breakdownFilter?.breakdowns?.[0]?.property ?? '')
+ // The same value Trends breaks down by, so the keys it returned match.
+ const value = `ifNull(nullIf(left(toString(properties.${escapePropertyAsHogQLIdentifier(property)}), ${BREAKDOWN_VALUE_MAX_LENGTH}), ''), ${escapeHogQLString(BREAKDOWN_NULL)})`
+ const filter: HogQLPropertyFilter = {
+ type: PropertyFilterType.HogQL,
+ key: `${value} IN (${values.map(escapeHogQLString).join(', ')})`,
+ }
+ return {
+ ...query,
+ series: query.series.map((series) => ({ ...series, properties: [...(series.properties ?? []), filter] })),
+ }
+}
+
+/**
+ * The values with the most graded runs keep their own line, and the rest share an "Other" line. A result is a share or
+ * a mean of the graded runs, so the "Other" result weighs each value's result by its graded count.
+ */
+export function topBreakdownSeries(series: TrendBreakdownSeries[], limit: number): TrendBreakdownSeries[] {
+ const total = (entry: TrendBreakdownSeries): number => entry.graded.reduce((sum, count) => sum + count, 0)
+ const ranked = series.filter((entry) => total(entry) > 0).sort((a, b) => total(b) - total(a))
+ const rest = ranked.slice(limit)
+ if (rest.length === 0) {
+ return ranked
+ }
+ const graded = rest[0].graded.map((_, bucket) => rest.reduce((sum, entry) => sum + entry.graded[bucket], 0))
+ const values = graded.map((count, bucket) =>
+ count === 0
+ ? NaN
+ : rest.reduce(
+ (sum, entry) => sum + (entry.graded[bucket] > 0 ? entry.values[bucket] * entry.graded[bucket] : 0),
+ 0
+ ) / count
+ )
+ return [...ranked.slice(0, limit), { key: BREAKDOWN_OTHER, label: 'Other', values, graded }]
+}
+
+export interface EvaluationDetailResults {
+ row: EvaluationResultApi | null
+ buckets: string[]
+ interval: BucketInterval
+ /** Whether this load skipped the backend's cache, so the loads that follow it skip it too. */
+ skippedCache: boolean
+}
+
+// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
+export interface evaluationDetailResultsLogicValues {
+ originalEvaluation: EvaluationConfig | null // llmEvaluationLogic
+ runsDateRange: DateRange // llmEvaluationLogic
+ trendsQuery: TrendsQuery | null // llmEvaluationLogic
+ currentTeamId: number | null // teamLogic
+ timezone: string // teamLogic
+ weekStartDay: number // teamLogic
+ breakdownFailure: OverviewFailure | null
+ breakdownProperty: string | null
+ breakdownQuery: TrendsQuery | null
+ breakdownSeries: TrendBreakdownSeries[] | null
+ breakdownSeriesLoading: boolean
+ details: DetailsLoad | null
+ detailsLoading: boolean
+ failure: OverviewFailure | null
+ periodNow: number
+ request: OverviewRequest
+ results: EvaluationDetailResults | null
+ resultsLoading: boolean
+}
+
+// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
+export interface evaluationDetailResultsLogicActions {
+ refreshEvaluationRuns: () => {
+ value: true
+ } // llmEvaluationLogic
+ setActiveTab: (tab: string) => {
+ tab: string
+ } // llmEvaluationLogic
+ setRunsDates: (
+ dateFrom: string | null,
+ dateTo: string | null
+ ) => {
+ dateFrom: string | null
+ dateTo: string | null
+ } // llmEvaluationLogic
+ loadBreakdown: (refresh?: boolean) => boolean
+ loadBreakdownFailure: (
+ error: string,
+ errorObject?: any
+ ) => {
+ error: string
+ errorObject?: any
+ }
+ loadBreakdownSuccess: (
+ breakdownSeries: TrendBreakdownSeries[] | null,
+ payload?: boolean
+ ) => {
+ breakdownSeries: TrendBreakdownSeries[] | null
+ payload?: boolean
+ }
+ loadDetails: (refresh?: boolean) => boolean
+ loadDetailsFailure: (
+ error: string,
+ errorObject?: any
+ ) => {
+ error: string
+ errorObject?: any
+ }
+ loadDetailsSuccess: (
+ details: DetailsLoad | null,
+ payload?: boolean
+ ) => {
+ details: DetailsLoad | null
+ payload?: boolean
+ }
+ loadResults: (refresh?: boolean) => boolean
+ loadResultsFailure: (
+ error: string,
+ errorObject?: any
+ ) => {
+ error: string
+ errorObject?: any
+ }
+ loadResultsSuccess: (
+ results: {
+ buckets: string[]
+ interval: EvaluationsOverviewIntervalEnumApi
+ row: EvaluationResultApi | null
+ skippedCache: boolean
+ } | null,
+ payload?: boolean
+ ) => {
+ results: {
+ buckets: string[]
+ interval: EvaluationsOverviewIntervalEnumApi
+ row: EvaluationResultApi | null
+ skippedCache: boolean
+ } | null
+ payload?: boolean
+ }
+ runOnPastData: () => {
+ value: true
+ }
+ selectBucket: (bucket: string) => {
+ bucket: string
+ }
+ setBreakdownFailure: (failure: OverviewFailure | null) => {
+ failure: OverviewFailure | null
+ }
+ setBreakdownProperty: (property: string | null) => {
+ property: string | null
+ }
+ setFailure: (failure: OverviewFailure | null) => {
+ failure: OverviewFailure | null
+ }
+}
+
+// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
+export interface evaluationDetailResultsLogicMeta {
+ key: string
+ __keaTypeGenInternalSelectorTypes: {
+ breakdownQuery: (
+ trendsQuery: TrendsQuery | null,
+ originalEvaluation: EvaluationConfig | null,
+ breakdownProperty: string | null,
+ request: OverviewRequest,
+ results: EvaluationDetailResults | null
+ ) => TrendsQuery | null
+ request: (
+ runsDateRange: DateRange,
+ timezone: string,
+ weekStartDay: number,
+ periodNow: number,
+ arg: any
+ ) => OverviewRequest
+ }
+}
+
+export type evaluationDetailResultsLogicType = MakeLogicType<
+ evaluationDetailResultsLogicValues,
+ evaluationDetailResultsLogicActions,
+ EvaluationDetailResultsLogicProps,
+ evaluationDetailResultsLogicMeta
+>
+
+export const evaluationDetailResultsLogic = kea<evaluationDetailResultsLogicType>([
+ path((key) => [
+ 'products',
+ 'ai_observability',
+ 'frontend',
+ 'evaluations',
+ 'overview',
+ 'evaluationDetailResultsLogic',
+ key,
+ ]),
+ props({} as EvaluationDetailResultsLogicProps),
+ key((props) => props.evaluationId),
+ connect((props: EvaluationDetailResultsLogicProps) => ({
+ values: [
+ llmEvaluationLogic({ evaluationId: props.evaluationId }),
+ ['runsDateRange', 'trendsQuery', 'originalEvaluation'],
+ teamLogic,
+ ['currentTeamId', 'timezone', 'weekStartDay'],
+ ],
+ actions: [
+ llmEvaluationLogic({ evaluationId: props.evaluationId }),
+ ['setRunsDates', 'setActiveTab', 'refreshEvaluationRuns'],
+ ],
+ })),
+ actions({
+ setFailure: (failure: OverviewFailure | null) => ({ failure }),
+ setBreakdownFailure: (failure: OverviewFailure | null) => ({ failure }),
+ selectBucket: (bucket: string) => ({ bucket }),
+ runOnPastData: true,
+ setBreakdownProperty: (property: string | null) => ({ property }),
+ }),
+ loaders(({ values, props, actions, cache }) => ({
+ results: [
+ null as EvaluationDetailResults | null,
+ {
+ loadResults: async (refresh: boolean = false, breakpoint) => {
+ const { request } = values
+ if (!values.currentTeamId || !request.ready) {
+ return null
+ }
+ const requestKey = JSON.stringify(request)
+ const skipCache = refresh || needsRefreshAfterChange(requestKey)
+ try {
+ const overview = await evaluationsOverviewRetrieve(String(values.currentTeamId), {
+ ...request.params,
+ ...(skipCache ? { refresh: true } : {}),
+ })
+ breakpoint()
+ if (skipCache) {
+ markRefreshedAfterChange(requestKey)
+ }
+ return {
+ row: overview.evaluations.find((row) => row.id === props.evaluationId) ?? null,
+ buckets: overview.buckets,
+ interval: overview.interval,
+ skippedCache: skipCache,
+ }
+ } catch (error) {
+ breakpoint()
+ actions.setFailure(failureOf(error))
+ return null
+ }
+ },
+ },
+ ],
+ breakdownSeries: [
+ null as TrendBreakdownSeries[] | null,
+ {
+ loadBreakdown: async (refresh: boolean = false, breakpoint) => {
+ const query = values.breakdownQuery
+ const results = values.results
+ if (!query || !results) {
+ return null
+ }
+ const mode = refresh ? 'force_blocking' : undefined
+ const run = (
+ q: TrendsQuery
+ ): Promise<Map<string, BreakdownCounts> | { failure: OverviewFailure }> => {
+ const controller = new AbortController()
+ const queryId = uuid()
+ let finished = false
+ // A newer load or an unmount stops this query, here and on the server.
+ cache.disposables.add(
+ () => () => {
+ controller.abort()
+ if (!finished) {
+ void cancelQueryOnServer(values.currentTeamId, queryId)
+ }
+ },
+ 'breakdownQuery',
+ { pauseOnPageHidden: false }
+ )
+ return performQuery(q, { signal: controller.signal }, mode, queryId)
+ .finally(() => {
+ finished = true
+ })
+ .then(breakdownCounts, (error) => ({ failure: failureOf(error) }))
+ }
+ let byValue = await run(query)
+ breakpoint()
+ if (!('failure' in byValue)) {
+ // Trends ranks each series on its own, so a value with many runs and a low result can miss
+ // the result series. Then the query runs again on the values the graded series kept.
+ const counts = byValue
+ const graded = [...counts.entries()]
+ .filter(([key, entry]) => key !== BREAKDOWN_OTHER && entry.graded)
+ .map(([key]) => key)
+ if (graded.some((key) => !counts.get(key)?.result)) {
+ byValue = await run(restrictBreakdown(query, graded))
+ breakpoint()
+ }
+ }
+ if ('failure' in byValue) {
+ actions.setBreakdownFailure(byValue.failure)
+ return null
+ }
+ const series = [...byValue.entries()]
+ .filter(([key]) => key !== BREAKDOWN_OTHER)
+ .map(([key, { result = [], graded = [], days }]) => {
+ // An hour repeated when clocks fall back has one key for two buckets, so match them in order.
+ const index = new Map<string, number[]>()
+ days.forEach((day, position) => index.set(day, [...(index.get(day) ?? []), position]))
+ const seen = new Map<string, number>()
+ const at = results.buckets.map((bucket) => {
+ const day = trendsBucketKey(bucket, results.interval)
+ const occurrence = seen.get(day) ?? 0
+ seen.set(day, occurrence + 1)
+ return index.get(day)?.[occurrence]
+ })
+ const gradedCounts = at.map((position) =>
+ position === undefined ? 0 : (graded[position] ?? 0)
+ )
+ return {
+ key,
+ label: key === BREAKDOWN_NULL ? 'Not set' : key,
+ // Trends fills a bucket without runs with 0, which is no result at all.
+ values: at.map((position, bucketIndex) =>
+ position === undefined || gradedCounts[bucketIndex] === 0
+ ? NaN
+ : (result[position] ?? NaN)
+ ),
+ graded: gradedCounts,
+ }
+ })
+ return topBreakdownSeries(series, BREAKDOWN_LIMIT)
+ },
+ },
+ ],
+ details: [
+ null as DetailsLoad | null,
+ {
+ loadDetails: async (refresh: boolean = false, breakpoint) => {
+ const row = values.results?.row
+ if (!values.currentTeamId || !row || row.gradedCount === 0) {
+ return null
+ }
+ const { date_from, date_to } = values.request.params
+ // The runs table lists failing runs, so this page leaves the failing examples out.
+ const load = await fetchEvaluationDetails(
+ String(values.currentTeamId),
+ row,
+ { date_from, date_to, ...(refresh ? { refresh: true } : {}) },
+ { recentFailures: false }
+ )
+ breakpoint()
+ return load
+ },
+ },
+ ],
+ })),
+ reducers({
+ // The instant a rolling range ends at, read when results load, so the request selector reads no clock.
+ periodNow: [
+ Date.now(),
+ {
+ loadResults: () => Date.now(),
+ },
+ ],
+ // Lines of the old range would be drawn on the new range's buckets until the breakdown reloads.
+ breakdownSeries: {
+ loadResults: () => null,
+ },
+ breakdownProperty: [
+ null as string | null,
+ {
+ setBreakdownProperty: (_, { property }) => property,
+ },
+ ],
+ breakdownFailure: [
+ null as OverviewFailure | null,
+ {
+ setBreakdownFailure: (_, { failure }) => failure,
+ loadBreakdown: () => null,
+ },
+ ],
+ failure: [
+ null as OverviewFailure | null,
+ {
+ setFailure: (_, { failure }) => failure,
+ loadResults: () => null,
+ },
+ ],
+ }),
+ selectors({
+ // The headline result split by one property, per bucket, next to each value's graded count.
+ breakdownQuery: [
+ (s) => [s.trendsQuery, s.originalEvaluation, s.breakdownProperty, s.request, s.results],
+ (
+ trendsQuery: TrendsQuery | null,
+ evaluation: EvaluationConfig | null,
+ breakdownProperty: string | null,
+ request: OverviewRequest,
+ results: EvaluationDetailResults | null
+ ): TrendsQuery | null => {
+ if (!trendsQuery || !evaluation || !breakdownProperty || !request.ready || !results) {
+ return null
+ }
+ const [headline] = trendsQuery.series
+ const { date_from = null, date_to = null } = request.params
+ return {
+ ...trendsQuery,
+ kind: NodeKind.TrendsQuery,
+ series: [
+ headline,
+ {
+ ...headline,
+ custom_name: 'Graded',
+ math: HogQLMathType.HogQL,
+ math_hogql: `countIf(${evaluationGradedHogQL(evaluation.output_type)})`,
+ },
+ ],
+ breakdownFilter: {
+ breakdowns: [{ property: breakdownProperty, type: 'event' }],
+ breakdown_limit: BREAKDOWN_QUERY_LIMIT,
+ },
+ // Trends reads a date-only end as that day's midnight when it buckets by hour.
+ dateRange: {
+ date_from,
+ date_to:
+ results.interval === 'hour' && date_to && /^\d{4}-\d{2}-\d{2}$/.test(date_to)
+ ? `${date_to}T23:59:59`
+ : date_to,
+ // A rolling range of hours starts mid-hour, and the overview counts runs from that instant.
+ ...(date_from && date_from.includes('T') ? { explicitDate: true } : {}),
+ },
+ interval: results.interval,
+ }
+ },
+ ],
+ request: [
+ (s) => [s.runsDateRange, s.timezone, s.weekStartDay, s.periodNow, (_, props) => props.evaluationId],
+ (
+ runsDateRange: DateRange,
+ timezone: string,
+ weekStartDay: number,
+ periodNow: number,
+ evaluationId: string
+ ): OverviewRequest => {
+ const period = overviewPeriod(
+ { dateFrom: runsDateRange.date_from ?? null, dateTo: runsDateRange.date_to ?? null },
+ timezone,
+ dayjs(periodNow),
+ weekStartDay
+ )
+ if (!period || period.days > OVERVIEW_MAX_DAYS) {
+ return {
+ params: {
+ date_from: period?.dateFrom,
+ date_to: period?.dateTo,
+ evaluation_ids: [evaluationId],
+ },
+ ready: false,
+ notice: `Pick a range of ${OVERVIEW_MAX_DAYS} days or less to see results over time.`,
+ }
+ }
+ // The runs table starts a rolling range at midnight N units ago, a day before the overview does.
+ const dateFrom =
+ ROLLING_DAYS.test(runsDateRange.date_from ?? '') && !runsDateRange.date_to
+ ? dayjs.utc(period.dateFrom).subtract(1, 'day').format('YYYY-MM-DD')
+ : period.dateFrom
+ const params: EvaluationsOverviewRetrieveParams = {
+ date_from: dateFrom,
+ date_to: period.dateTo,
+ evaluation_ids: [evaluationId],
+ }
+ return { params, ready: true, notice: null }
+ },
+ ],
+ }),
+ listeners(({ actions, values }) => ({
+ setBreakdownProperty: ({ property }) => {
+ posthog.capture('llma evaluation detail breakdown changed', { property })
+ actions.loadBreakdown()
+ },
+ setRunsDates: () => actions.loadResults(),
+ refreshEvaluationRuns: () => actions.loadResults(true),
+ loadResultsSuccess: ({ results }) => {
+ const refresh = results?.skippedCache ?? false
+ actions.loadDetails(refresh)
+ actions.loadBreakdown(refresh)
+ },
+ runOnPastData: () => {
+ posthog.capture('llma evaluation detail backfill hint clicked')
+ actions.setActiveTab('backfills')
+ },
+ selectBucket: ({ bucket }) => {
+ posthog.capture('llma evaluation detail bucket selected', { interval: values.results?.interval })
+ const { dateFrom, dateTo } = bucketDateRange(bucket, values.results?.interval ?? 'day')
+ actions.setRunsDates(dateFrom, dateTo)
+ },
+ })),
+ afterMount(({ actions }) => {
+ actions.loadResults()
+ }),
+])
diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationOverviewDisplay.ts b/products/ai_observability/frontend/evaluations/overview/evaluationOverviewDisplay.ts
index 589605e2c5a4..66110b9c961b 100644
--- a/products/ai_observability/frontend/evaluations/overview/evaluationOverviewDisplay.ts
+++ b/products/ai_observability/frontend/evaluations/overview/evaluationOverviewDisplay.ts
@@ -259,3 +259,19 @@ export function hourTick(label: string): string | null {
}
return hour === 0 ? dayjs(label.slice(0, 10)).format('MMM D') : label.slice(11, 16)
}
+
+/**
+ * The date range that covers one bucket. A day's bucket starts at local midnight, so its first ten characters are that
+ * day in the project's time zone. An hour is exact instants, which the date filters read as given. Its end is the next
+ * hour's start, because the filters exclude the end.
+ */
+export function bucketDateRange(bucket: string, interval: BucketInterval): { dateFrom: string; dateTo: string } {
+ if (interval === 'hour') {
+ const start = dayjs(bucket).utc()
+ return {
+ dateFrom: start.format('YYYY-MM-DDTHH:mm:ss[Z]'),
+ dateTo: start.add(1, 'hour').format('YYYY-MM-DDTHH:mm:ss[Z]'),
+ }
+ }
+ return { dateFrom: bucket.slice(0, 10), dateTo: bucket.slice(0, 10) }
+}
diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationResultDrawerLogic.ts b/products/ai_observability/frontend/evaluations/overview/evaluationResultDrawerLogic.ts
index 705c5fdf4c30..5440be98bcb2 100644
--- a/products/ai_observability/frontend/evaluations/overview/evaluationResultDrawerLogic.ts
+++ b/products/ai_observability/frontend/evaluations/overview/evaluationResultDrawerLogic.ts
@@ -32,6 +32,34 @@ export type DetailsLoad =
| { status: 'ready'; details: EvaluationDetails }
| { status: 'error'; failure: OverviewFailure }
+/** Reads the failed runs and the score spread a row's cards show. Each endpoint rejects an evaluation it does not
+ * apply to, so the row decides which ones to read. */
+export async function fetchEvaluationDetails(
+ projectId: string,
+ row: EvaluationResultApi,
+ params: { date_from?: string; date_to?: string; refresh?: boolean },
+ { recentFailures = true }: { recentFailures?: boolean } = {}
+): Promise<DetailsLoad> {
+ const { summary } = row
+ const scale = 'score' in summary ? summary.score?.scale : null
+ return await Promise.all([
+ recentFailures && summary.kind === 'pass_rate'
+ ? evaluationsRunsRetrieve(projectId, row.id, {
+ ...params,
+ outcome: 'failed',
+ limit: RECENT_FAILURES_LIMIT,
+ }).then((runs) => runs.results)
+ : null,
+ scale && scale.min < scale.max ? evaluationsScoreDistributionRetrieve(projectId, row.id, params) : null,
+ ]).then(
+ ([recentFailures, scoreDistribution]): DetailsLoad => ({
+ status: 'ready',
+ details: { recentFailures, scoreDistribution },
+ }),
+ (error: unknown): DetailsLoad => ({ status: 'error', failure: failureOf(error) })
+ )
+}
+
function detailsKey(evaluationId: string, request: OverviewRequest): string {
return `${evaluationId}/${request.params.date_from}/${request.params.date_to}`
}
@@ -240,29 +268,14 @@ export const evaluationResultDrawerLogic = kea<evaluationResultDrawerLogicType>(
const { date_from, date_to } = values.request.params
const refresh = cache.refreshKeys?.has(key) === true
const generation = cache.generation ?? 0
- const projectId = String(values.currentTeamId)
- const params = { date_from, date_to, ...(refresh ? { refresh: true } : {}) }
- // Each endpoint rejects an evaluation it does not apply to, so the row decides which ones to read.
- const summary = values.rowsById.get(evaluationId)?.summary
- const scale = summary && 'score' in summary ? summary.score?.scale : null
- const load = await Promise.all([
- summary?.kind === 'pass_rate'
- ? evaluationsRunsRetrieve(projectId, evaluationId, {
- ...params,
- outcome: 'failed',
- limit: RECENT_FAILURES_LIMIT,
- }).then((runs) => runs.results)
- : null,
- scale && scale.min < scale.max
- ? evaluationsScoreDistributionRetrieve(projectId, evaluationId, params)
- : null,
- ]).then(
- ([recentFailures, scoreDistribution]): DetailsLoad => ({
- status: 'ready',
- details: { recentFailures, scoreDistribution },
- }),
- (error: unknown): DetailsLoad => ({ status: 'error', failure: failureOf(error) })
- )
+ const row = values.rowsById.get(evaluationId)
+ const load: DetailsLoad = row
+ ? await fetchEvaluationDetails(String(values.currentTeamId), row, {
+ date_from,
+ date_to,
+ ...(refresh ? { refresh: true } : {}),
+ })
+ : { status: 'ready', details: { recentFailures: null, scoreDistribution: null } }
if (cache.prefetchingKey === key) {
cache.prefetchingKey = null
}
diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts
index f5a68d9a0017..b07af08319d6 100644
--- a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts
+++ b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts
@@ -10,7 +10,8 @@ import { initKeaTests } from '~/test/init'
import type { EvaluationApi, EvaluationResultApi } from '../../generated/api.schemas'
import { llmEvaluationsLogic } from '../llmEvaluationsLogic'
-import { hasIsolatedPoints, passFailCounts } from './evaluationOverviewDisplay'
+import { topBreakdownSeries } from './evaluationDetailResultsLogic'
+import { bucketDateRange, hasIsolatedPoints, passFailCounts } from './evaluationOverviewDisplay'
import { evaluationResultDrawerLogic } from './evaluationResultDrawerLogic'
import {
evaluationsOverviewLogic,
@@ -148,6 +149,25 @@ describe('evaluationsOverviewLogic', () => {
expect(hasIsolatedPoints(sparse)).toBe(expected)
})
+ test.each([
+ ['a day in a time zone ahead of UTC', '2026-10-20T00:00:00+02:00', 'day', '2026-10-20', '2026-10-20'],
+ ['an hour', '2026-10-20T14:00:00+02:00', 'hour', '2026-10-20T12:00:00Z', '2026-10-20T13:00:00Z'],
+ ] as const)('narrows the runs to %s', (_, bucket, interval, dateFrom, dateTo) => {
+ expect(bucketDateRange(bucket, interval)).toEqual({ dateFrom, dateTo })
+ })
+
+ it('keeps the values with the most graded runs and weighs the rest into one Other result', () => {
+ const series = [
+ { key: 'a', label: 'a', values: [50, NaN], graded: [10, 0] },
+ { key: 'b', label: 'b', values: [80, 100], graded: [1, 3] },
+ { key: 'c', label: 'c', values: [20, NaN], graded: [3, 0] },
+ ]
+ expect(topBreakdownSeries(series, 1)).toEqual([
+ series[0],
+ { key: '$$_posthog_breakdown_other_$$', label: 'Other', values: [35, 100], graded: [4, 3] },
+ ])
+ })
+
it('splits pass rate volume into exact passed and failed counts, with empty buckets as zero', () => {
const row = { ...ROWS[0], trend: { values: [(2 / 3) * 100, null, 100], volume: [3, 0, 7] } }
expect(passFailCounts(row)).toEqual({ passed: [2, 0, 7], failed: [1, 0, 0] })
diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts
index 8b48a2b4954e..dac160e52826 100644
--- a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts
+++ b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts
@@ -49,8 +49,8 @@ const SCOPE_CHANGE_DEBOUNCE_MS = 300
/** At or below this many evaluations, the page keeps a way to create another in view. */
const FEW_EVALUATIONS_MAX = 3
-/** The backend rejects a longer period. */
-const OVERVIEW_MAX_DAYS = 90
+/** The longest period people can pick. The backend accepts one day more, for the extra day the runs table counts. */
+export const OVERVIEW_MAX_DAYS = 90
/** How many evaluations the chart picks on its first render. People can pick more. */
const DEFAULT_COMPARED = 3
@@ -88,12 +88,17 @@ export function markEvaluationsOverviewStale(): void {
evaluationsOverviewLogic.findMounted()?.actions.reloadOverview()
}
+/** Records that a read for this request skipped the cache, so later reads for it may use the cache again. */
+export function markRefreshedAfterChange(requestKey: string): void {
+ refreshedSinceChange.add(requestKey)
+}
+
export function resetEvaluationsOverviewStale(): void {
lastChangeAt = null
refreshedSinceChange.clear()
}
-function needsRefreshAfterChange(requestKey: string): boolean {
+export function needsRefreshAfterChange(requestKey: string): boolean {
if (lastChangeAt === null || Date.now() - lastChangeAt > BACKEND_CACHE_MS) {
return false
}