Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): count paused evaluations' results on the evals overview (#125580)

Bernat Torres4c0dd36fcf2b
diff --git a/products/ai_observability/backend/facade/contracts.py b/products/ai_observability/backend/facade/contracts.py
index 2d1d407b9bed..c1645696c4e2 100644
--- a/products/ai_observability/backend/facade/contracts.py
+++ b/products/ai_observability/backend/facade/contracts.py
@@ -415,7 +415,9 @@ class PassRateOverviewEntry(Contract):
class PassRateOverview(Contract):
- entries: list[PassRateOverviewEntry] = Field(description="Enabled pass rate evaluations, lowest rate first.")
+ entries: list[PassRateOverviewEntry] = Field(
+ description="Pass rate evaluations with results in the period, paused ones included, lowest rate first."
+ )
median: float | None = Field(description="Median of the per-evaluation pass rates without low evidence.")
median_comparison: PeriodComparison | None = Field(
description="Median pass rate in each period, over evaluations with results in both. Null when none has."
@@ -455,7 +457,9 @@ class EvaluationsOverview(OverviewPeriodFields):
"bucket can start before `start`."
)
evaluations: list[EvaluationResult]
- totals: OverviewTotals = Field(description="Totals over the enabled evaluations.")
+ totals: OverviewTotals = Field(
+ description="Totals over every evaluation's results in the period, paused ones included."
+ )
pass_rate_overview: PassRateOverview
diff --git a/products/ai_observability/backend/logic/evaluations/overview.py b/products/ai_observability/backend/logic/evaluations/overview.py
index 4e24ab1131c6..4819ae686205 100644
--- a/products/ai_observability/backend/logic/evaluations/overview.py
+++ b/products/ai_observability/backend/logic/evaluations/overview.py
@@ -432,7 +432,7 @@ def pass_rate_overview(rows: Sequence[contracts.EvaluationResult], bucket_count:
compared: list[tuple[float, float]] = []
for row in rows:
summary = row.summary
- if not isinstance(summary, contracts.PassRateSummary) or not row.enabled:
+ if not isinstance(summary, contracts.PassRateSummary):
continue
rate = pass_rate(summary.passed, summary.failed)
if rate is None:
@@ -469,12 +469,11 @@ def pass_rate_overview(rows: Sequence[contracts.EvaluationResult], bucket_count:
def overview_totals(rows: Sequence[contracts.EvaluationResult], bucket_count: int) -> contracts.OverviewTotals:
- """Totals for the metric cards at the top of the overview."""
- active = [row for row in rows if row.enabled]
+ """Totals for the metric cards at the top of the overview, paused evaluations' results included."""
# Only evaluations with results in both periods, so a new or newly silent evaluation does not read as a change.
- both = [row for row in active if row.graded_count > 0 and row.previous_graded_count is not None]
+ both = [row for row in rows if row.graded_count > 0 and row.previous_graded_count is not None]
sentiment = None
- for row in active:
+ for row in rows:
if isinstance(row.summary, contracts.SentimentSummary) and row.graded_count > 0:
summary = row.summary
sentiment = contracts.OverviewSentiment(
@@ -484,15 +483,13 @@ def overview_totals(rows: Sequence[contracts.EvaluationResult], bucket_count: in
)
break
return contracts.OverviewTotals(
- graded=sum(row.graded_count for row in active),
+ graded=sum(row.graded_count for row in rows),
graded_comparison=contracts.PeriodComparison(
current=sum(row.graded_count for row in both),
previous=sum(row.previous_graded_count or 0 for row in both),
)
if both
else None,
- graded_trend=[sum(row.trend.volume[index] for row in active) for index in range(bucket_count)]
- if active
- else [],
+ graded_trend=[sum(row.trend.volume[index] for row in rows) for index in range(bucket_count)] if rows else [],
sentiment=sentiment,
)
diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_overview.py b/products/ai_observability/backend/tests/logic/evaluations/test_overview.py
index 49ba16b0fffb..8de9b2881f31 100644
--- a/products/ai_observability/backend/tests/logic/evaluations/test_overview.py
+++ b/products/ai_observability/backend/tests/logic/evaluations/test_overview.py
@@ -223,9 +223,9 @@ def test_overview_totals_compare_only_evaluations_with_results_in_both_periods()
totals = overview_totals([steady, new, sentiment, paused], len(BUCKETS))
- assert totals.graded == 130
- assert totals.graded_comparison == contracts.PeriodComparison(current=100, previous=80)
- assert totals.graded_trend == [80, 50, 0, 0]
+ assert totals.graded == 1130
+ assert totals.graded_comparison == contracts.PeriodComparison(current=1100, previous=1080)
+ assert totals.graded_trend == [1080, 50, 0, 0]
assert totals.sentiment == contracts.OverviewSentiment(
evaluation_id="eval-1", negative_share=25.0, previous_share=10.0
)
@@ -233,7 +233,9 @@ def test_overview_totals_compare_only_evaluations_with_results_in_both_periods()
def test_pass_rate_median_change_uses_the_same_evaluations_in_both_periods() -> None:
- steady = _result(_meta(), [_bucket(BUCKETS[0], passed=80, failed=20), _bucket(None, passed=90, failed=10)])
+ steady = _result(
+ _meta(enabled=False), [_bucket(BUCKETS[0], passed=80, failed=20), _bucket(None, passed=90, failed=10)]
+ )
new = _result(_meta(), [_bucket(BUCKETS[0], passed=40, failed=60)])
overview = pass_rate_overview([steady, new], len(BUCKETS))
diff --git a/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.ts b/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.ts
index 1f80407c754a..3a5c9049d2f9 100644
--- a/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.ts
+++ b/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.ts
@@ -120,26 +120,26 @@ function buildEvaluationBreakdownExpression(evaluations: EvaluationConfig[]): st
return `multiIf(${conditionsAndLabels.join(', ')}, toString(properties.$ai_evaluation_id))`
}
-/** Pass rate over time with one line per evaluation, or null when none of them has a pass rate. */
+/** Pass rate over time with one line per evaluation, or null when none of them has a pass rate. Callers pick which evaluations to chart. */
export function buildEvaluationPassRateTrendsQuery(
evaluations: EvaluationConfig[],
dateFilter: { dateFrom: string | null; dateTo: string | null },
intervalOverride?: 'hour' | 'day'
): TrendsQuery | null {
- const enabledEvaluations = evaluations
- .filter((evaluation) => evaluation.enabled && !evaluation.deleted && hasPassRate(evaluation))
+ const charted = evaluations
+ .filter((evaluation) => !evaluation.deleted && hasPassRate(evaluation))
.slice(0, MAX_EVALUATION_CHART_SERIES)
- if (enabledEvaluations.length === 0) {
+ if (charted.length === 0) {
return null
}
const dateFrom = dateFilter.dateFrom || '-7d'
const dateTo = dateFilter.dateTo || null
const interval = intervalOverride ?? getIntervalFromDateRange(dateFrom)
- const numericEvaluations = enabledEvaluations.filter((evaluation) => evaluation.output_type === 'numeric')
+ const numericEvaluations = charted.filter((evaluation) => evaluation.output_type === 'numeric')
let passedExpression = evaluationPassedHogQLForMany(
- enabledEvaluations.filter(evaluationIsDetector).map((evaluation) => evaluation.id)
+ charted.filter(evaluationIsDetector).map((evaluation) => evaluation.id)
)
let gradedExpression = EVALUATION_BOOLEAN_GRADED_HOGQL
if (numericEvaluations.length > 0) {
@@ -152,8 +152,8 @@ export function buildEvaluationPassRateTrendsQuery(
gradedExpression = `if(properties.$ai_evaluation_id IN (${numericIds}), ${EVALUATION_NUMERIC_GRADED_HOGQL}, ${EVALUATION_BOOLEAN_GRADED_HOGQL})`
}
- if (enabledEvaluations.some((evaluation) => evaluation.output_type === 'categorical')) {
- passedExpression = `if(properties.$ai_evaluation_result_type = 'categorical', ${categoricalEvaluationsPassedHogQL(enabledEvaluations)}, ${passedExpression})`
+ if (charted.some((evaluation) => evaluation.output_type === 'categorical')) {
+ passedExpression = `if(properties.$ai_evaluation_result_type = 'categorical', ${categoricalEvaluationsPassedHogQL(charted)}, ${passedExpression})`
gradedExpression = `if(properties.$ai_evaluation_result_type = 'categorical', ${EVALUATION_CATEGORICAL_GRADED_HOGQL}, ${gradedExpression})`
}
@@ -168,7 +168,7 @@ export function buildEvaluationPassRateTrendsQuery(
properties: [
{
key: '$ai_evaluation_id',
- value: enabledEvaluations.map((evaluation) => evaluation.id),
+ value: charted.map((evaluation) => evaluation.id),
operator: PropertyOperator.Exact,
type: PropertyFilterType.Event,
},
@@ -176,7 +176,7 @@ export function buildEvaluationPassRateTrendsQuery(
},
],
breakdownFilter: {
- breakdown: buildEvaluationBreakdownExpression(enabledEvaluations),
+ breakdown: buildEvaluationBreakdownExpression(charted),
breakdown_type: 'hogql',
breakdown_limit: MAX_EVALUATION_CHART_SERIES,
breakdown_hide_other_aggregation: true,
@@ -479,7 +479,11 @@ export const evaluationMetricsLogic = kea<evaluationMetricsLogicType>([
(
evaluationsForMetrics: EvaluationConfig[],
dateFilter: { dateFrom: string | null; dateTo: string | null }
- ): TrendsQuery | null => buildEvaluationPassRateTrendsQuery(evaluationsForMetrics, dateFilter),
+ ): TrendsQuery | null =>
+ buildEvaluationPassRateTrendsQuery(
+ evaluationsForMetrics.filter((evaluation) => evaluation.enabled),
+ dateFilter
+ ),
],
}),
diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts
index b07af08319d6..d7e9ad829fa2 100644
--- a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts
+++ b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts
@@ -112,6 +112,7 @@ const ATTENTION_ROWS = [
attentionRow('worse-12', 'worse', PASS_RATE, -12),
attentionRow('worse-8', 'worse', PASS_RATE, -8),
attentionRow('errored', 'errored', PASS_RATE, null),
+ { ...attentionRow('paused', null, PASS_RATE, 1), enabled: false },
]
const HISTOGRAM = { scale: SCALE, buckets: [{ label: '7', count: 10 }] }
@@ -471,6 +472,28 @@ describe('evaluationsOverviewLogic', () => {
expect(drawerLogic.values.openDetails).toMatchObject({ status: 'ready' })
})
+ it('opens paused evaluations in Explore only when they have results in the period', async () => {
+ useMocks({
+ get: {
+ '/api/projects/:teamId/evaluations/': {
+ count: 2,
+ next: null,
+ results: [
+ { ...evaluation('eval-a', null), enabled: false },
+ { ...evaluation('eval-z', null), enabled: false },
+ ],
+ },
+ },
+ })
+ llmEvaluationsLogic.actions.loadEvaluations()
+ await expectLogic(llmEvaluationsLogic).toFinishAllListeners()
+ await expectLogic(logic).toFinishAllListeners()
+
+ const url = decodeURIComponent(logic.values.exploreUrl ?? '')
+ expect(url).toContain('eval-a')
+ expect(url).not.toContain('eval-z')
+ })
+
test.each([
{ dateFrom: '-14d', dateTo: null, expected: '"date_from":"2026-10-09","date_to":"2026-10-22"' },
// Hourly buckets read a date-only end as midnight, which would drop the whole day.
@@ -575,9 +598,13 @@ describe('evaluationsOverviewLogic', () => {
test.each([
{
selector: 'tableRows' as const,
- expected: ['errored', 'worse-sentiment', 'worse-12', 'worse-8', 'worse-5', 'silent', 'healthy'],
+ expected: ['errored', 'worse-sentiment', 'worse-12', 'worse-8', 'worse-5', 'silent', 'healthy', 'paused'],
},
{ selector: 'chartedIds' as const, expected: ['worse-sentiment', 'worse-12', 'worse-8'] },
+ {
+ selector: 'compareListRows' as const,
+ expected: ['worse-sentiment', 'worse-12', 'worse-8', 'worse-5', 'healthy', 'paused', 'errored', 'silent'],
+ },
])('puts the worst regression first in $selector', async ({ selector, expected }) => {
useMocks({
get: {
diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts
index dac160e52826..8c351d954194 100644
--- a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts
+++ b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts
@@ -521,7 +521,8 @@ export interface evaluationsOverviewLogicMeta {
evaluations: EvaluationConfig[],
scopedIds: string[] | null,
request: OverviewRequest,
- interval: BucketInterval
+ interval: BucketInterval,
+ rowsById: Map<string, EvaluationResultApi>
) => string | null
evaluationListFailed: (
evaluationsLoadFailed: boolean,
@@ -873,7 +874,7 @@ export const evaluationsOverviewLogic = kea<evaluationsOverviewLogicType>([
(rows: EvaluationResultApi[], nothingGraded: boolean): EvaluationResultApi[] => {
const listed = nothingGraded
? rows
- : rows.filter((row) => row.attentionReasons.length > 0 || (row.enabled && row.gradedCount > 0))
+ : rows.filter((row) => row.attentionReasons.length > 0 || row.gradedCount > 0)
// Rows with a line to draw come first, so evaluations that never ran do not push them down.
return [
...attentionFirst(listed.filter(isChartable)),
@@ -975,19 +976,23 @@ export const evaluationsOverviewLogic = kea<evaluationsOverviewLogicType>([
},
],
exploreUrl: [
- (s) => [s.evaluations, s.scopedIds, s.request, s.interval],
+ (s) => [s.evaluations, s.scopedIds, s.request, s.interval, s.rowsById],
(
evaluations: EvaluationConfig[],
scopedIds: string[] | null,
request: OverviewRequest,
- interval: BucketInterval
+ interval: BucketInterval,
+ rowsById: Map<string, EvaluationResultApi>
): string | null => {
if (!request.ready) {
return null
}
- const scoped = scopedIds
- ? evaluations.filter((evaluation) => scopedIds.includes(evaluation.id))
- : evaluations
+ // Paused evaluations with results in the period count, as they do on the overview chart.
+ const scoped = evaluations.filter(
+ (evaluation) =>
+ (!scopedIds || scopedIds.includes(evaluation.id)) &&
+ (evaluation.enabled || (rowsById.get(evaluation.id)?.gradedCount ?? 0) > 0)
+ )
const { date_from = null, date_to = null } = request.params
// Trends reads a date-only end as that day's midnight when it buckets by hour, so the day gets an end.
const dateTo =
diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.ts
index b76e649d8b1f..46b95cf35287 100644
--- a/products/ai_observability/frontend/generated/api.schemas.ts
+++ b/products/ai_observability/frontend/generated/api.schemas.ts
@@ -2999,7 +2999,7 @@ export interface PassRateOverviewEntryApi {
}
export interface PassRateOverviewApi {
- /** Enabled pass rate evaluations, lowest rate first. */
+ /** Pass rate evaluations with results in the period, paused ones included, lowest rate first. */
entries: PassRateOverviewEntryApi[]
/** Median of the per-evaluation pass rates without low evidence. */
median: number | null
@@ -3022,7 +3022,7 @@ export interface EvaluationsOverviewApi {
/** Start of each bucket, in the project's timezone. Every series is aligned with it. The first bucket can start before `start`. */
buckets: string[]
evaluations: EvaluationResultApi[]
- /** Totals over the enabled evaluations. */
+ /** Totals over every evaluation's results in the period, paused ones included. */
totals: OverviewTotalsApi
passRateOverview: PassRateOverviewApi
}
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 3648274c40cd..2763003a5a2f 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -46354,7 +46354,7 @@ export namespace Schemas {
}
export interface PassRateOverview {
- /** Enabled pass rate evaluations, lowest rate first. */
+ /** Pass rate evaluations with results in the period, paused ones included, lowest rate first. */
entries: PassRateOverviewEntry[];
/** Median of the per-evaluation pass rates without low evidence. */
median: number | null;
@@ -46377,7 +46377,7 @@ export namespace Schemas {
/** Start of each bucket, in the project's timezone. Every series is aligned with it. The first bucket can start before `start`. */
buckets: string[];
evaluations: EvaluationResult[];
- /** Totals over the enabled evaluations. */
+ /** Totals over every evaluation's results in the period, paused ones included. */
totals: OverviewTotals;
passRateOverview: PassRateOverview;
}