:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): count paused evaluations' results on the evals overview (#125580)
Bernat Torres4c0dd36fcf2b
diff --git a/products/ai_observability/backend/facade/contracts.py b/products/ai_observability/backend/facade/contracts.pyindex 2d1d407b9bed..c1645696c4e2 100644--- a/products/ai_observability/backend/facade/contracts.py+++ b/products/ai_observability/backend/facade/contracts.py@@ -415,7 +415,9 @@ class PassRateOverviewEntry(Contract):class PassRateOverview(Contract):- entries: list[PassRateOverviewEntry] = Field(description="Enabled pass rate evaluations, lowest rate first.")+ entries: list[PassRateOverviewEntry] = Field(+ description="Pass rate evaluations with results in the period, paused ones included, lowest rate first."+ )median: float | None = Field(description="Median of the per-evaluation pass rates without low evidence.")median_comparison: PeriodComparison | None = Field(description="Median pass rate in each period, over evaluations with results in both. Null when none has."@@ -455,7 +457,9 @@ class EvaluationsOverview(OverviewPeriodFields):"bucket can start before `start`.")evaluations: list[EvaluationResult]- totals: OverviewTotals = Field(description="Totals over the enabled evaluations.")+ totals: OverviewTotals = Field(+ description="Totals over every evaluation's results in the period, paused ones included."+ )pass_rate_overview: PassRateOverviewdiff --git a/products/ai_observability/backend/logic/evaluations/overview.py b/products/ai_observability/backend/logic/evaluations/overview.pyindex 4e24ab1131c6..4819ae686205 100644--- a/products/ai_observability/backend/logic/evaluations/overview.py+++ b/products/ai_observability/backend/logic/evaluations/overview.py@@ -432,7 +432,7 @@ def pass_rate_overview(rows: Sequence[contracts.EvaluationResult], bucket_count:compared: list[tuple[float, float]] = []for row in rows:summary = row.summary- if not isinstance(summary, contracts.PassRateSummary) or not row.enabled:+ if not isinstance(summary, contracts.PassRateSummary):continuerate = pass_rate(summary.passed, summary.failed)if rate is None:@@ -469,12 +469,11 @@ def pass_rate_overview(rows: Sequence[contracts.EvaluationResult], bucket_count:def overview_totals(rows: Sequence[contracts.EvaluationResult], bucket_count: int) -> contracts.OverviewTotals:- """Totals for the metric cards at the top of the overview."""- active = [row for row in rows if row.enabled]+ """Totals for the metric cards at the top of the overview, paused evaluations' results included."""# Only evaluations with results in both periods, so a new or newly silent evaluation does not read as a change.- both = [row for row in active if row.graded_count > 0 and row.previous_graded_count is not None]+ both = [row for row in rows if row.graded_count > 0 and row.previous_graded_count is not None]sentiment = None- for row in active:+ for row in rows:if isinstance(row.summary, contracts.SentimentSummary) and row.graded_count > 0:summary = row.summarysentiment = contracts.OverviewSentiment(@@ -484,15 +483,13 @@ def overview_totals(rows: Sequence[contracts.EvaluationResult], bucket_count: in)breakreturn contracts.OverviewTotals(- graded=sum(row.graded_count for row in active),+ graded=sum(row.graded_count for row in rows),graded_comparison=contracts.PeriodComparison(current=sum(row.graded_count for row in both),previous=sum(row.previous_graded_count or 0 for row in both),)if bothelse None,- graded_trend=[sum(row.trend.volume[index] for row in active) for index in range(bucket_count)]- if active- else [],+ graded_trend=[sum(row.trend.volume[index] for row in rows) for index in range(bucket_count)] if rows else [],sentiment=sentiment,)diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_overview.py b/products/ai_observability/backend/tests/logic/evaluations/test_overview.pyindex 49ba16b0fffb..8de9b2881f31 100644--- a/products/ai_observability/backend/tests/logic/evaluations/test_overview.py+++ b/products/ai_observability/backend/tests/logic/evaluations/test_overview.py@@ -223,9 +223,9 @@ def test_overview_totals_compare_only_evaluations_with_results_in_both_periods()totals = overview_totals([steady, new, sentiment, paused], len(BUCKETS))- assert totals.graded == 130- assert totals.graded_comparison == contracts.PeriodComparison(current=100, previous=80)- assert totals.graded_trend == [80, 50, 0, 0]+ assert totals.graded == 1130+ assert totals.graded_comparison == contracts.PeriodComparison(current=1100, previous=1080)+ assert totals.graded_trend == [1080, 50, 0, 0]assert totals.sentiment == contracts.OverviewSentiment(evaluation_id="eval-1", negative_share=25.0, previous_share=10.0)@@ -233,7 +233,9 @@ def test_overview_totals_compare_only_evaluations_with_results_in_both_periods()def test_pass_rate_median_change_uses_the_same_evaluations_in_both_periods() -> None:- steady = _result(_meta(), [_bucket(BUCKETS[0], passed=80, failed=20), _bucket(None, passed=90, failed=10)])+ steady = _result(+ _meta(enabled=False), [_bucket(BUCKETS[0], passed=80, failed=20), _bucket(None, passed=90, failed=10)]+ )new = _result(_meta(), [_bucket(BUCKETS[0], passed=40, failed=60)])overview = pass_rate_overview([steady, new], len(BUCKETS))diff --git a/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.ts b/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.tsindex 1f80407c754a..3a5c9049d2f9 100644--- a/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.ts+++ b/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.ts@@ -120,26 +120,26 @@ function buildEvaluationBreakdownExpression(evaluations: EvaluationConfig[]): streturn `multiIf(${conditionsAndLabels.join(', ')}, toString(properties.$ai_evaluation_id))`}-/** Pass rate over time with one line per evaluation, or null when none of them has a pass rate. */+/** Pass rate over time with one line per evaluation, or null when none of them has a pass rate. Callers pick which evaluations to chart. */export function buildEvaluationPassRateTrendsQuery(evaluations: EvaluationConfig[],dateFilter: { dateFrom: string | null; dateTo: string | null },intervalOverride?: 'hour' | 'day'): TrendsQuery | null {- const enabledEvaluations = evaluations- .filter((evaluation) => evaluation.enabled && !evaluation.deleted && hasPassRate(evaluation))+ const charted = evaluations+ .filter((evaluation) => !evaluation.deleted && hasPassRate(evaluation)).slice(0, MAX_EVALUATION_CHART_SERIES)- if (enabledEvaluations.length === 0) {+ if (charted.length === 0) {return null}const dateFrom = dateFilter.dateFrom || '-7d'const dateTo = dateFilter.dateTo || nullconst interval = intervalOverride ?? getIntervalFromDateRange(dateFrom)- const numericEvaluations = enabledEvaluations.filter((evaluation) => evaluation.output_type === 'numeric')+ const numericEvaluations = charted.filter((evaluation) => evaluation.output_type === 'numeric')let passedExpression = evaluationPassedHogQLForMany(- enabledEvaluations.filter(evaluationIsDetector).map((evaluation) => evaluation.id)+ charted.filter(evaluationIsDetector).map((evaluation) => evaluation.id))let gradedExpression = EVALUATION_BOOLEAN_GRADED_HOGQLif (numericEvaluations.length > 0) {@@ -152,8 +152,8 @@ export function buildEvaluationPassRateTrendsQuery(gradedExpression = `if(properties.$ai_evaluation_id IN (${numericIds}), ${EVALUATION_NUMERIC_GRADED_HOGQL}, ${EVALUATION_BOOLEAN_GRADED_HOGQL})`}- if (enabledEvaluations.some((evaluation) => evaluation.output_type === 'categorical')) {- passedExpression = `if(properties.$ai_evaluation_result_type = 'categorical', ${categoricalEvaluationsPassedHogQL(enabledEvaluations)}, ${passedExpression})`+ if (charted.some((evaluation) => evaluation.output_type === 'categorical')) {+ passedExpression = `if(properties.$ai_evaluation_result_type = 'categorical', ${categoricalEvaluationsPassedHogQL(charted)}, ${passedExpression})`gradedExpression = `if(properties.$ai_evaluation_result_type = 'categorical', ${EVALUATION_CATEGORICAL_GRADED_HOGQL}, ${gradedExpression})`}@@ -168,7 +168,7 @@ export function buildEvaluationPassRateTrendsQuery(properties: [{key: '$ai_evaluation_id',- value: enabledEvaluations.map((evaluation) => evaluation.id),+ value: charted.map((evaluation) => evaluation.id),operator: PropertyOperator.Exact,type: PropertyFilterType.Event,},@@ -176,7 +176,7 @@ export function buildEvaluationPassRateTrendsQuery(},],breakdownFilter: {- breakdown: buildEvaluationBreakdownExpression(enabledEvaluations),+ breakdown: buildEvaluationBreakdownExpression(charted),breakdown_type: 'hogql',breakdown_limit: MAX_EVALUATION_CHART_SERIES,breakdown_hide_other_aggregation: true,@@ -479,7 +479,11 @@ export const evaluationMetricsLogic = kea<evaluationMetricsLogicType>([(evaluationsForMetrics: EvaluationConfig[],dateFilter: { dateFrom: string | null; dateTo: string | null }- ): TrendsQuery | null => buildEvaluationPassRateTrendsQuery(evaluationsForMetrics, dateFilter),+ ): TrendsQuery | null =>+ buildEvaluationPassRateTrendsQuery(+ evaluationsForMetrics.filter((evaluation) => evaluation.enabled),+ dateFilter+ ),],}),diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.tsindex b07af08319d6..d7e9ad829fa2 100644--- a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts+++ b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.test.ts@@ -112,6 +112,7 @@ const ATTENTION_ROWS = [attentionRow('worse-12', 'worse', PASS_RATE, -12),attentionRow('worse-8', 'worse', PASS_RATE, -8),attentionRow('errored', 'errored', PASS_RATE, null),+ { ...attentionRow('paused', null, PASS_RATE, 1), enabled: false },]const HISTOGRAM = { scale: SCALE, buckets: [{ label: '7', count: 10 }] }@@ -471,6 +472,28 @@ describe('evaluationsOverviewLogic', () => {expect(drawerLogic.values.openDetails).toMatchObject({ status: 'ready' })})+ it('opens paused evaluations in Explore only when they have results in the period', async () => {+ useMocks({+ get: {+ '/api/projects/:teamId/evaluations/': {+ count: 2,+ next: null,+ results: [+ { ...evaluation('eval-a', null), enabled: false },+ { ...evaluation('eval-z', null), enabled: false },+ ],+ },+ },+ })+ llmEvaluationsLogic.actions.loadEvaluations()+ await expectLogic(llmEvaluationsLogic).toFinishAllListeners()+ await expectLogic(logic).toFinishAllListeners()++ const url = decodeURIComponent(logic.values.exploreUrl ?? '')+ expect(url).toContain('eval-a')+ expect(url).not.toContain('eval-z')+ })+test.each([{ dateFrom: '-14d', dateTo: null, expected: '"date_from":"2026-10-09","date_to":"2026-10-22"' },// Hourly buckets read a date-only end as midnight, which would drop the whole day.@@ -575,9 +598,13 @@ describe('evaluationsOverviewLogic', () => {test.each([{selector: 'tableRows' as const,- expected: ['errored', 'worse-sentiment', 'worse-12', 'worse-8', 'worse-5', 'silent', 'healthy'],+ expected: ['errored', 'worse-sentiment', 'worse-12', 'worse-8', 'worse-5', 'silent', 'healthy', 'paused'],},{ selector: 'chartedIds' as const, expected: ['worse-sentiment', 'worse-12', 'worse-8'] },+ {+ selector: 'compareListRows' as const,+ expected: ['worse-sentiment', 'worse-12', 'worse-8', 'worse-5', 'healthy', 'paused', 'errored', 'silent'],+ },])('puts the worst regression first in $selector', async ({ selector, expected }) => {useMocks({get: {diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.tsindex dac160e52826..8c351d954194 100644--- a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts+++ b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts@@ -521,7 +521,8 @@ export interface evaluationsOverviewLogicMeta {evaluations: EvaluationConfig[],scopedIds: string[] | null,request: OverviewRequest,- interval: BucketInterval+ interval: BucketInterval,+ rowsById: Map<string, EvaluationResultApi>) => string | nullevaluationListFailed: (evaluationsLoadFailed: boolean,@@ -873,7 +874,7 @@ export const evaluationsOverviewLogic = kea<evaluationsOverviewLogicType>([(rows: EvaluationResultApi[], nothingGraded: boolean): EvaluationResultApi[] => {const listed = nothingGraded? rows- : rows.filter((row) => row.attentionReasons.length > 0 || (row.enabled && row.gradedCount > 0))+ : rows.filter((row) => row.attentionReasons.length > 0 || row.gradedCount > 0)// Rows with a line to draw come first, so evaluations that never ran do not push them down.return [...attentionFirst(listed.filter(isChartable)),@@ -975,19 +976,23 @@ export const evaluationsOverviewLogic = kea<evaluationsOverviewLogicType>([},],exploreUrl: [- (s) => [s.evaluations, s.scopedIds, s.request, s.interval],+ (s) => [s.evaluations, s.scopedIds, s.request, s.interval, s.rowsById],(evaluations: EvaluationConfig[],scopedIds: string[] | null,request: OverviewRequest,- interval: BucketInterval+ interval: BucketInterval,+ rowsById: Map<string, EvaluationResultApi>): string | null => {if (!request.ready) {return null}- const scoped = scopedIds- ? evaluations.filter((evaluation) => scopedIds.includes(evaluation.id))- : evaluations+ // Paused evaluations with results in the period count, as they do on the overview chart.+ const scoped = evaluations.filter(+ (evaluation) =>+ (!scopedIds || scopedIds.includes(evaluation.id)) &&+ (evaluation.enabled || (rowsById.get(evaluation.id)?.gradedCount ?? 0) > 0)+ )const { date_from = null, date_to = null } = request.params// Trends reads a date-only end as that day's midnight when it buckets by hour, so the day gets an end.const dateTo =diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.tsindex b76e649d8b1f..46b95cf35287 100644--- a/products/ai_observability/frontend/generated/api.schemas.ts+++ b/products/ai_observability/frontend/generated/api.schemas.ts@@ -2999,7 +2999,7 @@ export interface PassRateOverviewEntryApi {}export interface PassRateOverviewApi {- /** Enabled pass rate evaluations, lowest rate first. */+ /** Pass rate evaluations with results in the period, paused ones included, lowest rate first. */entries: PassRateOverviewEntryApi[]/** Median of the per-evaluation pass rates without low evidence. */median: number | null@@ -3022,7 +3022,7 @@ export interface EvaluationsOverviewApi {/** Start of each bucket, in the project's timezone. Every series is aligned with it. The first bucket can start before `start`. */buckets: string[]evaluations: EvaluationResultApi[]- /** Totals over the enabled evaluations. */+ /** Totals over every evaluation's results in the period, paused ones included. */totals: OverviewTotalsApipassRateOverview: PassRateOverviewApi}diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 3648274c40cd..2763003a5a2f 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -46354,7 +46354,7 @@ export namespace Schemas {}export interface PassRateOverview {- /** Enabled pass rate evaluations, lowest rate first. */+ /** Pass rate evaluations with results in the period, paused ones included, lowest rate first. */entries: PassRateOverviewEntry[];/** Median of the per-evaluation pass rates without low evidence. */median: number | null;@@ -46377,7 +46377,7 @@ export namespace Schemas {/** Start of each bucket, in the project's timezone. Every series is aligned with it. The first bucket can start before `start`. */buckets: string[];evaluations: EvaluationResult[];- /** Totals over the enabled evaluations. */+ /** Totals over every evaluation's results in the period, paused ones included. */totals: OverviewTotals;passRateOverview: PassRateOverview;}