Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(autoresearch): show online accuracy over time on the accuracy tab (#119807)

posthog[bot]5a25de4269aa
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 35cf94879498..11af65a700c5 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -6659,17 +6659,21 @@ snapshots:
products-autoresearch-model-card--training--light:
hash: v1.k794b7964.b358f33a0d879e85f4c8bdb4ae65330f392cdaa8a869e8bce7e07cdcb22d13a9.EAfBa_-86ylT0UXKPW7EAqtGV9zNLRT4VdSYkWFWj_M
products-autoresearch-model-detail-scene--accuracy--narrow--dark:
- hash: v1.k794b7964.cc5c80f54b366af72ee206c132ff16855a53d9f0152b5fcd665badc347dfe76a.VrTGlwRaHpwfI1QBWePyr2ARuWCj1HoG2ztzwfXESAE
+ hash: v1.k794b7964.4cab77a4d9bfb7975449b4c47648e536f506002c38ed147a73524af54da5a6b5.orHlfN-LtUJeeKkONCkVAUx3PZ31dQw-lxvYmfjfFZw
products-autoresearch-model-detail-scene--accuracy--narrow--light:
- hash: v1.k794b7964.fd34150ddd7a416e7fe6d25aad92f32348db555b75b8b9074eb984313d326ae5.regqIjpzTv7BhVk_HRbkv2QTV49o_yP7SXcVGHbuOgM
+ hash: v1.k794b7964.d408d0bce9b8c87c96b7da7acaeabb25004967fbeec3c62584bca0c6c8ac1379.TNDdvZN3B87CYwfV0JrlH4mmOfD7zy_o78jJPJMOVVU
products-autoresearch-model-detail-scene--accuracy--wide--dark:
- hash: v1.k794b7964.8079c8a10c44ea9de9d26d8ce7c73c380b7523fecb0bc6c5217fcbe757d48ddc.s4wWq4ykRqBZKIokdQ7VJwm6NlFNienMd_5OeTxFEGE
+ hash: v1.k794b7964.5682a1f2d01f1443bf621305ee8ffdc331cea7a60bd316c1a6f7ea7bf2169dae.DTCmGuHIF_7jXkNbXDx61qO3R5StzjzR1Ss-eJXz9uM
products-autoresearch-model-detail-scene--accuracy--wide--light:
- hash: v1.k794b7964.93b1cbc7ef465154449463c0410d44fa6b468c1272015c66aeda987b45e55c82.3KzsV_fKafMRSkfLwl5Tmb0lJ0BtsaXYCePf36mOhDA
+ hash: v1.k794b7964.914e11340da36659c95e64018017601f426f3a76e6d6f3922d1f0526d263dbd4.u_QcTRlMQZGm_G9T2NxseEBkUI5cBldEUjUefQuyzxc
products-autoresearch-model-detail-scene--accuracy-before-precision-recall--wide--dark:
- hash: v1.k794b7964.392e2f039d6619c9344d299e9bdde59ba18e44f3c6a52f9485d16b1d12f71143.iws2WKI1cVn8mtF7gQ7yXjLl6Zer9ClpXnKDt4saXMw
+ hash: v1.k794b7964.288332b7abc333b0b2e5bf103793af5bd21af15b445d01051597452b186345d9.GG4kd4k9wLG0iij6X36W9pCzDSJi338S8DuUM5iT1Mc
products-autoresearch-model-detail-scene--accuracy-before-precision-recall--wide--light:
- hash: v1.k794b7964.f9e9a4a3106781c287215c4814de98cfcd66afa6c3bba6f3a9b5e028943a035d.GFjkvK159EBC_ohj72MNqX56nSfXlla66-89avAQ16I
+ hash: v1.k794b7964.331ed0c5d5a26f0caadf6483ed07a718afc022e84d63ce3a1a9d2825870b3d15.ZEzPvoPtVQJN-IDnPBS3ZgWRJ7MYBELQaXBvBPbDWww
+ products-autoresearch-model-detail-scene--accuracy-hit-rate-with-challengers--wide--dark:
+ hash: v1.k794b7964.1ac9a20b8616dad37424909ca867b45106763943c61739470b298c8651854a5b.Mk_e6ArRT0NCHD-DnzB69VprJZmBeiuBJqHghsG2AcY
+ products-autoresearch-model-detail-scene--accuracy-hit-rate-with-challengers--wide--light:
+ hash: v1.k794b7964.9ba1c3edd96fb8572bf98877a23b1ae71c1cabb39c4dd900935751ea9a62a243.vJA1Rx5am3ih3Wu-xMWyJZzd1UMZabJm810pFmVhEgg
products-autoresearch-model-detail-scene--agent-research-narrow--narrow--dark:
hash: v1.k794b7964.0889baa7d2b6bca50dfd14487b80f287e8bd27a098a0e3482cb9da086c840e3e.5vUCdyEAF1AXUusb51kIrj4Jbpwm8BYmrR6O3ATOz34
products-autoresearch-model-detail-scene--agent-research-narrow--narrow--light:
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index c4e6086ec177..bf95d04cf8d5 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -2652,6 +2652,12 @@ importers:
'@posthog/quill-charts':
specifier: workspace:*
version: link:../../packages/quill/packages/charts
+ '@testing-library/dom':
+ specifier: '*'
+ version: 10.4.0
+ '@testing-library/user-event':
+ specifier: '*'
+ version: 14.6.1(@testing-library/[email protected])
'@types/react':
specifier: 18.3.27
version: 18.3.27
diff --git a/products/autoresearch/frontend/AGENTS.md b/products/autoresearch/frontend/AGENTS.md
index 88d35f056132..f2fbee90fcb0 100644
--- a/products/autoresearch/frontend/AGENTS.md
+++ b/products/autoresearch/frontend/AGENTS.md
@@ -20,7 +20,7 @@ The product is mostly a backend, and the UI is deliberately thin — it reads st
| `pipelineLifecycle.ts` | The five lifecycle steps (question set to checked against reality) and which one is current. |
| `trainingScheduleLogic.ts` | Setup tab training schedule: the research interval and experiments per run, their save, and the one-line phase and next run status. |
| `agentSearch.ts` | The agent's search across training runs: chart points, best score so far, change per experiment within a run, log groups and status brief. |
-| `onlinePerformance.ts` | Accuracy tab helpers over the `online_performance` endpoint: the stat tiles, the realized AUC series, predicted against actual by segment. |
+| `onlinePerformance.ts` | Accuracy tab helpers over the `online_performance` endpoint: the stat tiles, the per-metric accuracy trend, predicted against actual. |
| `predictionCoverage.ts` | Predictions tab coverage from the runs list: the measured summary and the coverage and score age per day. |
| `predictionSegments.ts` | The likelihood segments from the cut points the `online_performance` endpoint returns: lift over the base rate, or a fixed pair until then. |
| `PipelineStatusTag.tsx` | Status tag + tooltip shared by the list and detail scenes. |
@@ -53,7 +53,7 @@ Follow `frontend/src/AGENTS.md` — it applies to product frontends too.
- TypeScript with explicit return types; Tailwind utilities rather than inline styles.
- Reuse Lemon / quill components instead of hand-rolling tables, badges, or tags.
- Any button that fires a request must guard against double-submission — `loading` / `disabledReason` on `LemonButton`, reset on both success and error paths. The lifecycle actions here (train, score, archive, pause, resume) are all network calls, and several are expensive: firing `train` twice starts two sandbox agent runs and spends the budget twice.
-- Usage events are part of the API: renaming one breaks every insight built on it. The list logic captures `autoresearch model list viewed`, `autoresearch model list load failed`, `autoresearch model deleted` / `paused` / `resumed`, `autoresearch model training started` (with `source: 'list'` from a card), `autoresearch model card clicked`, and `autoresearch model action failed` (with `action`). The create logic captures `autoresearch model template selected` (with `template_key`), `autoresearch model custom chosen`, `autoresearch model advanced opened`, `autoresearch model created` (with `action`: `start_training` or `save_draft`), and `autoresearch model training started` / `training start failed`. The detail logic captures `autoresearch model tab changed` (with `tab`) when a person picks a tab, `autoresearch model experiment log filter changed` (with `filter`), `autoresearch model accuracy cutoff changed` (with `cutoff`), `autoresearch model search point clicked` (with `run_id`, `iteration_number`, `status`), `autoresearch model experiment details opened` (with `run_id`, `iteration_number`) when a person opens an experiment's working notes, `autoresearch model suggestion sent` (with `priority`), `autoresearch agent research experiment opened` (with `run_id`, `iteration_number`, `status`, `source`: `chart` or `log`) when a person opens the experiment drawer, `autoresearch agent research steer opened` (with `queued_count`) `autoresearch agent research full notes opened` (with `run_number`) and `autoresearch training schedule changed` (with `research_interval_days` and `iterations_per_run`); buttons carry a `data-attr` for autocapture. Extend the same names when you add a scene.
+- Usage events are part of the API: renaming one breaks every insight built on it. The list logic captures `autoresearch model list viewed`, `autoresearch model list load failed`, `autoresearch model deleted` / `paused` / `resumed`, `autoresearch model training started` (with `source: 'list'` from a card), `autoresearch model card clicked`, and `autoresearch model action failed` (with `action`). The create logic captures `autoresearch model template selected` (with `template_key`), `autoresearch model custom chosen`, `autoresearch model advanced opened`, `autoresearch model created` (with `action`: `start_training` or `save_draft`), and `autoresearch model training started` / `training start failed`. The detail logic captures `autoresearch model tab changed` (with `tab`) when a person picks a tab, `autoresearch model experiment log filter changed` (with `filter`), `autoresearch model accuracy cutoff changed` (with `cutoff`, and `source`: `flagged` or `trend` for the card that changed it), `autoresearch model accuracy trend metric changed` (with `metric`), `autoresearch model accuracy trend challengers toggled` (with `shown`), `autoresearch model search point clicked` (with `run_id`, `iteration_number`, `status`), `autoresearch model experiment details opened` (with `run_id`, `iteration_number`) when a person opens an experiment's working notes, `autoresearch model suggestion sent` (with `priority`), `autoresearch agent research experiment opened` (with `run_id`, `iteration_number`, `status`, `source`: `chart` or `log`) when a person opens the experiment drawer, `autoresearch agent research steer opened` (with `queued_count`) `autoresearch agent research full notes opened` (with `run_number`) and `autoresearch training schedule changed` (with `research_interval_days` and `iterations_per_run`); buttons carry a `data-attr` for autocapture. Extend the same names when you add a scene.
## Copy
diff --git a/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx b/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx
index e035090c1261..e9b905e65305 100644
--- a/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx
+++ b/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx
@@ -1,4 +1,6 @@
import { Meta, StoryObj } from '@storybook/react'
+import { within } from '@testing-library/dom'
+import userEvent from '@testing-library/user-event'
import { FEATURE_FLAGS } from 'lib/constants'
import { App } from 'scenes/App'
@@ -142,44 +144,82 @@ function iteration(
}
}
+const PERFORMANCE_DATES = ['2026-02-10', '2026-02-11', '2026-02-12', '2026-02-13', '2026-02-14']
+
+/** Top 10% counts out of 47,000 people scored, 1,410 of whom did it. */
+function topTenCounts(tp: number): Record<string, number> {
+ return {
+ tp,
+ fp: 4700 - tp,
+ fn: 1410 - tp,
+ tn: 47000 - 4700 - (1410 - tp),
+ n_flagged: 4700,
+ precision: tp / 4700,
+ recall: tp / 1410,
+ }
+}
+
+function performanceRow(prediction_date: string, index: number): { prediction_date: string } & Record<string, unknown> {
+ return {
+ validation_run_id: `validate-${index}`,
+ prediction_date,
+ horizon_days: 7,
+ weekday: 1,
+ model_id: champion.id,
+ emitted_role: 'champion',
+ current_role: 'champion',
+ n_scored: 47000,
+ n_positive: 1410,
+ base_rate: 0.03,
+ mean_p_y: 0.033,
+ realized_auc: [0.78, 0.8, 0.79, 0.82, 0.81][index],
+ realized_auc_ci_low: [0.76, 0.78, 0.77, 0.8, 0.79][index],
+ realized_auc_ci_high: [0.8, 0.82, 0.81, 0.84, 0.83][index],
+ brier_score: 0.06,
+ calibration_error: 0.04,
+ lift_at_10: 3.4,
+ lift_at_20: 2.6,
+ average_precision: 0.21,
+ confusion: {
+ top_10: topTenCounts([440, 480, 470, 510, 455][index]),
+ top_20: { tp: 735, fp: 8665, fn: 675, tn: 36925, n_flagged: 9400, precision: 0.0782, recall: 0.5213 },
+ likely: { tp: 700, fp: 3700, fn: 710, tn: 41890, n_flagged: 4400, precision: 0.1591, recall: 0.4965 },
+ },
+ likely_threshold: 0.09,
+ calibration_bins: [
+ { n: 37600, mean_p_y: 0.01, positive_rate: 0.008 },
+ { n: 4700, mean_p_y: 0.08, positive_rate: 0.07 },
+ { n: 2350, mean_p_y: 0.3, positive_rate: 0.27 },
+ { n: 2350, mean_p_y: 0.68, positive_rate: 0.6 },
+ ],
+ warning: null,
+ validated_at: '2026-02-21T03:00:00Z',
+ }
+}
+
+// The challenger started scoring next to the champion on Feb 12, so its line covers the last three dates.
+const challengerPerformance = PERFORMANCE_DATES.slice(2).map((prediction_date, index) => ({
+ ...performanceRow(prediction_date, index + 2),
+ validation_run_id: `validate-challenger-${index}`,
+ model_id: challenger.id,
+ emitted_role: 'challenger',
+ current_role: 'challenger',
+ mean_p_y: 0.031,
+ realized_auc: [0.82, 0.84, 0.83][index],
+ realized_auc_ci_low: [0.8, 0.82, 0.81][index],
+ realized_auc_ci_high: [0.84, 0.86, 0.85][index],
+ lift_at_10: [3.6, 3.9, 3.8][index],
+ confusion: {
+ top_10: { tp: 520, fp: 4180, fn: 890, tn: 41410, n_flagged: 4700, precision: 0.1106, recall: 0.3688 },
+ top_20: { tp: 760, fp: 8640, fn: 650, tn: 36950, n_flagged: 9400, precision: 0.0809, recall: 0.539 },
+ likely: { tp: 720, fp: 3600, fn: 690, tn: 41990, n_flagged: 4320, precision: 0.1667, recall: 0.5106 },
+ },
+}))
+
const onlinePerformance = {
- rows: ['2026-02-10', '2026-02-11', '2026-02-12', '2026-02-13', '2026-02-14']
- .map((prediction_date, index) => ({
- validation_run_id: `validate-${index}`,
- prediction_date,
- horizon_days: 7,
- weekday: 1,
- model_id: champion.id,
- emitted_role: 'champion',
- current_role: 'champion',
- n_scored: 47000,
- n_positive: 1410,
- base_rate: 0.03,
- mean_p_y: 0.033,
- realized_auc: [0.78, 0.8, 0.79, 0.82, 0.81][index],
- realized_auc_ci_low: [0.76, 0.78, 0.77, 0.8, 0.79][index],
- realized_auc_ci_high: [0.8, 0.82, 0.81, 0.84, 0.83][index],
- brier_score: 0.06,
- calibration_error: 0.04,
- lift_at_10: 3.4,
- lift_at_20: 2.6,
- average_precision: 0.21,
- confusion: {
- top_10: { tp: 480, fp: 4220, fn: 930, tn: 41370, n_flagged: 4700, precision: 0.1021, recall: 0.3404 },
- top_20: { tp: 735, fp: 8665, fn: 675, tn: 36925, n_flagged: 9400, precision: 0.0782, recall: 0.5213 },
- likely: { tp: 700, fp: 3700, fn: 710, tn: 41890, n_flagged: 4400, precision: 0.1591, recall: 0.4965 },
- },
- likely_threshold: 0.09,
- calibration_bins: [
- { n: 37600, mean_p_y: 0.01, positive_rate: 0.008 },
- { n: 4700, mean_p_y: 0.08, positive_rate: 0.07 },
- { n: 2350, mean_p_y: 0.3, positive_rate: 0.27 },
- { n: 2350, mean_p_y: 0.68, positive_rate: 0.6 },
- ],
- warning: null,
- validated_at: '2026-02-21T03:00:00Z',
- }))
- .reverse(),
+ rows: [...PERFORMANCE_DATES.map(performanceRow), ...challengerPerformance].sort((a, b) =>
+ b.prediction_date.localeCompare(a.prediction_date)
+ ),
segment_thresholds: {
likely_threshold: 0.09,
possible_threshold: 0.03,
@@ -394,6 +434,26 @@ export const Accuracy: Story = {
},
}
+export const AccuracyHitRateWithChallengers: Story = {
+ parameters: {
+ pageUrl: `${urls.autoresearchPipeline(PIPELINE_ID)}?tab=accuracy`,
+ testOptions: {
+ viewportWidths: ['wide'],
+ waitForSelector: '[data-attr="autoresearch-model-accuracy-trend-cutoff-top_10"]',
+ },
+ },
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement)
+ await canvas.findByText('Accuracy over time', {}, { timeout: 15000 })
+ await userEvent.click(
+ canvasElement.querySelector(
+ '[data-attr="autoresearch-model-accuracy-trend-metric-hit_rate"]'
+ ) as HTMLElement
+ )
+ await userEvent.click(await canvas.findByText('Compare challengers'))
+ },
+}
+
export const AccuracyBeforePrecisionRecall: Story = {
decorators: [
mswDecorator({
diff --git a/products/autoresearch/frontend/autoresearchPipelineLogic.ts b/products/autoresearch/frontend/autoresearchPipelineLogic.ts
index 242c3cc9d409..685b0012a71c 100644
--- a/products/autoresearch/frontend/autoresearchPipelineLogic.ts
+++ b/products/autoresearch/frontend/autoresearchPipelineLogic.ts
@@ -64,14 +64,15 @@ import {
import type { ConfusionByCutoffApi } from './generated/api.schemas'
import {
type AccuracyCutoff,
+ type AccuracyTrend,
+ type AccuracyTrendMetric,
type PooledConfusion,
- RealizedAucPoint,
SegmentCalibration,
+ accuracyTrend,
calibrationBySegment,
firstCheckDate,
latestChampionRow,
pooledConfusion,
- realizedAucSeries,
validatedPredictionDates,
} from './onlinePerformance'
import { LifecycleStep, pipelineLifecycle } from './pipelineLifecycle'
@@ -90,6 +91,9 @@ import type { PredictionsPeopleView } from './predictionsPeopleQuery'
/** Where a person opened an experiment from. */
export type ExperimentOpenSource = 'chart' | 'log'
+/** The card whose cutoff picker changed the shared cutoff. */
+export type AccuracyCutoffSource = 'flagged' | 'trend'
+
export interface AutoresearchPipelineLogicProps {
id: string
}
@@ -288,6 +292,8 @@ export interface autoresearchPipelineLogicValues {
currentProjectId: number | null // projectLogic
currentTeamId: number | null // teamLogic
accuracyCutoff: AccuracyCutoff
+ accuracyTrend: AccuracyTrend
+ accuracyTrendMetric: AccuracyTrendMetric
activeScoreRun: AutoresearchRunApi | null
activeTab: AutoresearchPipelineTab
agentSearch: AgentSearch
@@ -308,7 +314,9 @@ export interface autoresearchPipelineLogicValues {
experimentLogFilter: ExperimentLogFilter
experimentLogGroups: ExperimentLogGroup[]
firstCheck: dayjs.Dayjs | null
+ hasChallengerPerformance: boolean
hasLiveTrainingRun: boolean
+ hoveredAccuracyDate: string | null
hoveredSearchPoint: SearchPoint | null
latestChampionPerformance: OnlinePerformanceRowApi | null
latestPropertyCleanupRun: AutoresearchRunApi | null
@@ -335,7 +343,6 @@ export interface autoresearchPipelineLogicValues {
probabilityDistributionLoading: boolean
probabilityHistogram: ProbabilityBucket[] | null
queuedSuggestionCount: number
- realizedAucPoints: RealizedAucPoint[]
reportByRun: Record<string, string | null>
reportByRunLoading: boolean
runRoles: Record<string, RunRole>
@@ -353,6 +360,7 @@ export interface autoresearchPipelineLogicValues {
segmentDefinitions: PredictionSegmentDefinition[] | null
segmentThresholds: PredictionSegmentThresholdsApi | null
selectedTab: AutoresearchPipelineTab | null
+ showChallengersInTrend: boolean
startTrainingResult: AutoresearchTrainingRunApi | null
startTrainingResultLoading: boolean
statusBrief: StatusBrief | null
@@ -680,8 +688,15 @@ export interface autoresearchPipelineLogicActions {
searchPointClicked: (point: SearchPoint) => {
point: SearchPoint
}
- setAccuracyCutoff: (cutoff: AccuracyCutoff) => {
+ setAccuracyCutoff: (
+ cutoff: AccuracyCutoff,
+ source: AccuracyCutoffSource
+ ) => {
cutoff: keyof ConfusionByCutoffApi
+ source: AccuracyCutoffSource
+ }
+ setAccuracyTrendMetric: (metric: AccuracyTrendMetric) => {
+ metric: AccuracyTrendMetric
}
setActiveScoreRun: (run: AutoresearchRunApi | null) => {
run: AutoresearchRunApi | null
@@ -692,6 +707,9 @@ export interface autoresearchPipelineLogicActions {
setExperimentLogFilter: (filter: ExperimentLogFilter) => {
filter: ExperimentLogFilter
}
+ setHoveredAccuracyDate: (date: string | null) => {
+ date: string | null
+ }
setHoveredSearchPoint: (point: SearchPoint | null) => {
point: SearchPoint | null
}
@@ -701,6 +719,9 @@ export interface autoresearchPipelineLogicActions {
setSegmentThresholds: (thresholds: PredictionSegmentThresholdsApi) => {
thresholds: PredictionSegmentThresholdsApi
}
+ setShowChallengersInTrend: (show: boolean) => {
+ show: boolean
+ }
setSuggestionDraft: (draft: string) => {
draft: string
}
@@ -819,7 +840,13 @@ export interface autoresearchPipelineLogicMeta {
) => CoverageSummary | null
coverageHistory: (runs: AutoresearchRunApi[]) => CoveragePoint[]
latestChampionPerformance: (onlinePerformance: OnlinePerformanceRowApi[]) => OnlinePerformanceRowApi | null
- realizedAucPoints: (onlinePerformance: OnlinePerformanceRowApi[]) => RealizedAucPoint[]
+ accuracyTrend: (
+ onlinePerformance: OnlinePerformanceRowApi[],
+ accuracyTrendMetric: AccuracyTrendMetric,
+ accuracyCutoff: keyof ConfusionByCutoffApi,
+ showChallengersInTrend: boolean
+ ) => AccuracyTrend
+ hasChallengerPerformance: (onlinePerformance: OnlinePerformanceRowApi[]) => boolean
segmentDefinitions: (
segmentThresholds: PredictionSegmentThresholdsApi | null
) => PredictionSegmentDefinition[] | null
@@ -893,7 +920,10 @@ export const autoresearchPipelineLogic = kea<autoresearchPipelineLogicType>([
actions({
setActiveTab: (tab: AutoresearchPipelineTab) => ({ tab }),
setPredictionsPeopleView: (view: PredictionsPeopleView) => ({ view }),
- setAccuracyCutoff: (cutoff: AccuracyCutoff) => ({ cutoff }),
+ setAccuracyCutoff: (cutoff: AccuracyCutoff, source: AccuracyCutoffSource) => ({ cutoff, source }),
+ setAccuracyTrendMetric: (metric: AccuracyTrendMetric) => ({ metric }),
+ setShowChallengersInTrend: (show: boolean) => ({ show }),
+ setHoveredAccuracyDate: (date: string | null) => ({ date }),
setTabFromUrl: (tab: AutoresearchPipelineTab | null) => ({ tab }),
loadDetail: true,
toggleRunArtifacts: (runId: string) => ({ runId }),
@@ -948,6 +978,24 @@ export const autoresearchPipelineLogic = kea<autoresearchPipelineLogicType>([
setAccuracyCutoff: (_, { cutoff }) => cutoff,
},
],
+ accuracyTrendMetric: [
+ 'ranking' as AccuracyTrendMetric,
+ {
+ setAccuracyTrendMetric: (_, { metric }) => metric,
+ },
+ ],
+ showChallengersInTrend: [
+ false,
+ {
+ setShowChallengersInTrend: (_, { show }) => show,
+ },
+ ],
+ hoveredAccuracyDate: [
+ null as string | null,
+ {
+ setHoveredAccuracyDate: (_, { date }) => date,
+ },
+ ],
activeScoreRun: [
null as AutoresearchRunApi | null,
{
@@ -1456,9 +1504,20 @@ export const autoresearchPipelineLogic = kea<autoresearchPipelineLogicType>([
(onlinePerformance: OnlinePerformanceRowApi[]): OnlinePerformanceRowApi | null =>
latestChampionRow(onlinePerformance),
],
- realizedAucPoints: [
+ accuracyTrend: [
+ (s) => [s.onlinePerformance, s.accuracyTrendMetric, s.accuracyCutoff, s.showChallengersInTrend],
+ (
+ onlinePerformance: OnlinePerformanceRowApi[],
+ accuracyTrendMetric: AccuracyTrendMetric,
+ accuracyCutoff: AccuracyCutoff,
+ showChallengersInTrend: boolean
+ ): AccuracyTrend =>
+ accuracyTrend(onlinePerformance, accuracyTrendMetric, accuracyCutoff, showChallengersInTrend),
+ ],
+ hasChallengerPerformance: [
(s) => [s.onlinePerformance],
- (onlinePerformance: OnlinePerformanceRowApi[]): RealizedAucPoint[] => realizedAucSeries(onlinePerformance),
+ (onlinePerformance: OnlinePerformanceRowApi[]): boolean =>
+ onlinePerformance.some((row) => row.emitted_role !== 'champion'),
],
segmentDefinitions: [
(s) => [s.segmentThresholds],
@@ -1859,8 +1918,20 @@ export const autoresearchPipelineLogic = kea<autoresearchPipelineLogicType>([
actions.loadPredictionSegments()
}
},
- setAccuracyCutoff: ({ cutoff }) => {
- posthog.capture('autoresearch model accuracy cutoff changed', { pipeline_id: props.id, cutoff })
+ setAccuracyCutoff: ({ cutoff, source }) => {
+ // pinned: analytics event name and properties. Renaming breaks insights built on them.
+ posthog.capture('autoresearch model accuracy cutoff changed', { pipeline_id: props.id, cutoff, source })
+ },
+ setAccuracyTrendMetric: ({ metric }) => {
+ // pinned: analytics event name and properties. Renaming breaks insights built on them.
+ posthog.capture('autoresearch model accuracy trend metric changed', { pipeline_id: props.id, metric })
+ },
+ setShowChallengersInTrend: ({ show }) => {
+ // pinned: analytics event name and properties. Renaming breaks insights built on them.
+ posthog.capture('autoresearch model accuracy trend challengers toggled', {
+ pipeline_id: props.id,
+ shown: show,
+ })
},
setWritePersonProperty: async ({ enabled, source }) => {
if (!values.currentTeamId) {
diff --git a/products/autoresearch/frontend/onlinePerformance.test.ts b/products/autoresearch/frontend/onlinePerformance.test.ts
index e53153c31eb9..56386f177026 100644
--- a/products/autoresearch/frontend/onlinePerformance.test.ts
+++ b/products/autoresearch/frontend/onlinePerformance.test.ts
@@ -2,9 +2,10 @@ import { AutoresearchRunApi, ConfusionByCutoffApi, OnlinePerformanceRowApi } fro
import {
accuracyStats,
accuracySummary,
+ accuracyTrend,
+ accuracyTrendSummary,
calibrationBySegment,
pooledConfusion,
- realizedAucSeries,
validatedPredictionDates,
} from './onlinePerformance'
@@ -20,8 +21,20 @@ function row(overrides: Partial<OnlinePerformanceRowApi>): OnlinePerformanceRowA
}
function counts(tp: number, fp: number, fn: number, tn: number): ConfusionByCutoffApi {
- const cutoff = { tp, fp, fn, tn, n_flagged: tp + fp, precision: null, recall: null }
- return { top_10: cutoff, top_20: cutoff, likely: { ...cutoff, tp: 0, fp: 0, n_flagged: 0, tn: tn + tp + fp } }
+ const cutoff = {
+ tp,
+ fp,
+ fn,
+ tn,
+ n_flagged: tp + fp,
+ precision: tp + fp > 0 ? tp / (tp + fp) : null,
+ recall: tp + fn > 0 ? tp / (tp + fn) : null,
+ }
+ return {
+ top_10: cutoff,
+ top_20: cutoff,
+ likely: { ...cutoff, tp: 0, fp: 0, n_flagged: 0, tn: tn + tp + fp, precision: null, recall: 0 },
+ }
}
describe('onlinePerformance', () => {
@@ -43,20 +56,87 @@ describe('onlinePerformance', () => {
])
})
- test('realizedAucSeries keeps the champion rows with an AUC, oldest date first', () => {
- const points = realizedAucSeries([
- row({ prediction_date: '2026-02-12', realized_auc: 0.82 }),
- row({ prediction_date: '2026-02-12', emitted_role: 'challenger', realized_auc: 0.7 }),
- row({ prediction_date: '2026-02-11', realized_auc: null }),
- row({ prediction_date: '2026-02-10', realized_auc: 0.79 }),
- ])
+ test('accuracyTrend keeps each role on its own line, oldest date first, with gaps for missing values', () => {
+ const rows = [
+ row({
+ prediction_date: '2026-02-12',
+ model_id: 'champ',
+ realized_auc: 0.82,
+ confusion: counts(3, 1, 2, 94),
+ }),
+ row({ prediction_date: '2026-02-12', model_id: 'chal', emitted_role: 'challenger', realized_auc: 0.7 }),
+ row({ prediction_date: '2026-02-11', model_id: 'champ', realized_auc: null, confusion: null }),
+ row({
+ prediction_date: '2026-02-10',
+ model_id: 'champ',
+ realized_auc: 0.79,
+ confusion: counts(1, 3, 2, 94),
+ }),
+ row({ prediction_date: '2026-02-10', model_id: 'chal', emitted_role: 'challenger', realized_auc: 0.74 }),
+ ].map((r) => ({ ...r, n_positive: 5, base_rate: 0.05 }))
+ const values = (trend: ReturnType<typeof accuracyTrend>): [string, [string, number][]][] =>
+ trend.series.map((s) => [s.label, s.points.map((p) => [p.date, p.value])])
- expect(points.map((p) => [p.date, p.auc])).toEqual([
- ['2026-02-10', 0.79],
- ['2026-02-12', 0.82],
+ expect(values(accuracyTrend(rows, 'ranking', 'top_10', false))).toEqual([
+ [
+ 'Champion',
+ [
+ ['2026-02-10', 0.79],
+ ['2026-02-12', 0.82],
+ ],
+ ],
+ ])
+ expect(values(accuracyTrend(rows, 'ranking', 'top_10', true))[1]).toEqual([
+ 'Challenger',
+ [
+ ['2026-02-10', 0.74],
+ ['2026-02-12', 0.7],
+ ],
+ ])
+ // Likely lift is the Likely hit rate over the base rate: 3/4 against 5%.
+ expect(values(accuracyTrend(rows, 'lift', 'likely', false))).toEqual([['Champion', []]])
+ const likely = rows.map((r) =>
+ r.confusion ? { ...r, confusion: { ...r.confusion, likely: r.confusion.top_10 } } : r
+ )
+ expect(values(accuracyTrend(likely, 'lift', 'likely', false))[0][1]).toEqual([
+ ['2026-02-10', expect.closeTo(0.25 / 0.05)],
+ ['2026-02-12', expect.closeTo(0.75 / 0.05)],
+ ])
+ expect(values(accuracyTrend(rows, 'calibration', 'top_10', false)).map(([label]) => label)).toEqual([
+ 'Champion score',
+ 'Actual rate',
])
})
+ test.each([
+ {
+ metric: 'ranking' as const,
+ expected:
+ 'Over 2 checked dates, realized AUC ranged from 0.790 to 0.820. It matched or beat its holdout AUC of 0.800 on 1 of 2 dates.',
+ },
+ {
+ metric: 'hit_rate' as const,
+ expected:
+ 'On a typical date, 30.0% of the top 10% the model flagged did “File shared”, against 4.0% of everyone.',
+ },
+ {
+ metric: 'calibration' as const,
+ expected: "Over 2 checked dates, the champion's average score stayed within 2.0 points of the actual rate.",
+ },
+ ])('accuracyTrendSummary describes the champion over the checked dates for $metric', ({ metric, expected }) => {
+ const rows = [
+ row({ prediction_date: '2026-02-12', realized_auc: 0.82, base_rate: 0.03, mean_p_y: 0.05 }),
+ row({ prediction_date: '2026-02-12', emitted_role: 'challenger', realized_auc: 0.5, mean_p_y: 0.5 }),
+ row({ prediction_date: '2026-02-10', realized_auc: 0.79, base_rate: 0.05, mean_p_y: 0.04 }),
+ ].map((r, index) => ({
+ ...r,
+ n_positive: 5,
+ confusion: index === 0 ? counts(4, 6, 1, 89) : counts(2, 8, 3, 87),
+ }))
+
+ expect(accuracyTrendSummary(rows, metric, 'top_10', 'File shared', 0.8)).toEqual(expected)
+ })
+
test('pooledConfusion sums the current champion dates and counts pre-change dates apart', () => {
const rows = [
row({ prediction_date: '2026-02-13', model_id: 'new', confusion: counts(3, 1, 2, 94) }),
diff --git a/products/autoresearch/frontend/onlinePerformance.ts b/products/autoresearch/frontend/onlinePerformance.ts
index 15b228515c69..4acaaee78d90 100644
--- a/products/autoresearch/frontend/onlinePerformance.ts
+++ b/products/autoresearch/frontend/onlinePerformance.ts
@@ -15,13 +15,6 @@ import {
predictionSegmentDefinitions,
} from './predictionSegments'
-export interface RealizedAucPoint {
- date: string
- auc: number
- low: number | null
- high: number | null
-}
-
export interface SegmentCalibration {
segment: PredictionSegmentDefinition
people: number
@@ -121,17 +114,164 @@ export function accuracyStats(row: OnlinePerformanceRowApi, target: string): Acc
]
}
-/** The champion's realized AUC per date, oldest first, for the trend chart. */
-export function realizedAucSeries(rows: OnlinePerformanceRowApi[]): RealizedAucPoint[] {
- return rows
- .filter((row) => row.emitted_role === 'champion' && row.realized_auc != null)
- .map((row) => ({
- date: row.prediction_date,
- auc: row.realized_auc as number,
- low: row.realized_auc_ci_low,
- high: row.realized_auc_ci_high,
- }))
- .sort((a, b) => a.date.localeCompare(b.date))
+export type AccuracyTrendMetric = 'ranking' | 'hit_rate' | 'coverage' | 'lift' | 'calibration'
+
+export const ACCURACY_TREND_METRICS: { key: AccuracyTrendMetric; label: string }[] = [
+ { key: 'ranking', label: 'Ranking (AUC)' },
+ { key: 'hit_rate', label: 'Hit rate' },
+ { key: 'coverage', label: 'Coverage' },
+ { key: 'lift', label: 'Lift' },
+ { key: 'calibration', label: 'Predicted and actual' },
+]
+
+export function trendMetricUsesCutoff(metric: AccuracyTrendMetric): boolean {
+ return metric === 'hit_rate' || metric === 'coverage' || metric === 'lift'
+}
+
+export interface AccuracyTrendPoint {
+ date: string
+ value: number
+ /** The 95% AUC interval. Null for every metric except ranking. */
+ low: number | null
+ high: number | null
+}
+
+export interface AccuracyTrendSeries {
+ key: string
+ label: string
+ /** `actual` is the share of people who did the target, drawn next to the average score. */
+ role: 'champion' | 'challenger' | 'actual'
+ /** Oldest date first. A date the model has no value for is missing, not zero. */
+ points: AccuracyTrendPoint[]
+}
+
+export interface AccuracyTrend {
+ /** Every date any series has a value for, oldest first. */
+ dates: string[]
+ /** The champion series comes first. */
+ series: AccuracyTrendSeries[]
+ /** The champion's people scored and people who did the target, per date. */
+ people: Record<string, { scored: number; positive: number }>
+}
+
+function liftAt(row: OnlinePerformanceRowApi, cutoff: AccuracyCutoff): number | null {
+ // Lift has no meaning when no one did the target, and the backend reports 0 then.
+ if (row.n_positive === 0) {
+ return null
+ }
+ switch (cutoff) {
+ case 'top_10':
+ return row.lift_at_10
+ case 'top_20':
+ return row.lift_at_20
+ case 'likely': {
+ const precision = row.confusion?.likely.precision
+ return precision != null && row.base_rate > 0 ? precision / row.base_rate : null
+ }
+ }
+}
+
+function trendPoint(
+ row: OnlinePerformanceRowApi,
+ metric: AccuracyTrendMetric,
+ cutoff: AccuracyCutoff
+): AccuracyTrendPoint | null {
+ const point = (
+ value: number | null | undefined,
+ low: number | null = null,
+ high: number | null = null
+ ): AccuracyTrendPoint | null => (value != null ? { date: row.prediction_date, value, low, high } : null)
+ switch (metric) {
+ case 'ranking':
+ return point(row.realized_auc, row.realized_auc_ci_low, row.realized_auc_ci_high)
+ case 'hit_rate':
+ return point(row.confusion?.[cutoff].precision)
+ case 'coverage':
+ return point(row.confusion?.[cutoff].recall)
+ case 'lift':
+ return point(liftAt(row, cutoff))
+ case 'calibration':
+ return point(row.mean_p_y)
+ }
+}
+
+/**
+ * One metric per prediction date, for the champion and, when asked, each model that emitted
+ * predictions in another role. A row keeps the role its model held on that date, so a promoted
+ * challenger's older dates stay on its challenger line.
+ */
+export function accuracyTrend(
+ rows: OnlinePerformanceRowApi[],
+ metric: AccuracyTrendMetric,
+ cutoff: AccuracyCutoff,
+ withChallengers: boolean
+): AccuracyTrend {
+ const oldestFirst = [...rows].sort((a, b) => a.prediction_date.localeCompare(b.prediction_date))
+ const championRows = oldestFirst.filter((row) => row.emitted_role === 'champion')
+ const pointsOf = (modelRows: OnlinePerformanceRowApi[]): AccuracyTrendPoint[] =>
+ modelRows.flatMap((row) => trendPoint(row, metric, cutoff) ?? [])
+
+ const series: AccuracyTrendSeries[] = [
+ {
+ key: 'champion',
+ label: metric === 'calibration' ? 'Champion score' : 'Champion',
+ role: 'champion',
+ points: pointsOf(championRows),
+ },
+ ]
+ if (withChallengers) {
+ const challengerRows = oldestFirst.filter((row) => row.emitted_role !== 'champion')
+ const challengers = [...new Set(challengerRows.map((row) => row.model_id))]
+ .map((modelId) => pointsOf(challengerRows.filter((row) => row.model_id === modelId)))
+ .filter((points) => points.length > 0)
+ challengers.forEach((points, index) =>
+ series.push({
+ key: `challenger-${index}`,
+ label: challengers.length > 1 ? `Challenger ${index + 1}` : 'Challenger',
+ role: 'challenger',
+ points,
+ })
+ )
+ }
+ if (metric === 'calibration') {
+ series.push({
+ key: 'actual',
+ label: 'Actual rate',
+ role: 'actual',
+ points: championRows.map((row) => ({
+ date: row.prediction_date,
+ value: row.base_rate,
+ low: null,
+ high: null,
+ })),
+ })
+ }
+
+ return {
+ dates: [...new Set(series.flatMap((s) => s.points.map((point) => point.date)))].sort(),
+ series,
+ people: Object.fromEntries(
+ championRows.map((row) => [row.prediction_date, { scored: row.n_scored, positive: row.n_positive }])
+ ),
+ }
+}
+
+/** A metric value in its own unit. `precise` adds a decimal place to AUC for the per-date details. */
+export function formatTrendValue(metric: AccuracyTrendMetric, value: number, precise = false): string {
+ switch (metric) {
+ case 'ranking':
+ return value.toFixed(precise ? 3 : 2)
+ case 'lift':
+ return `${value.toFixed(1)}×`
+ default:
+ return percent(value)
+ }
+}
+
+/** The gap between the average score and the actual rate, in percentage points with a sign. */
+export function percentagePoints(gap: number): string {
+ const points = (gap * 100).toFixed(1)
+ return `${gap > 0 ? '+' : ''}${points} pts`
}
/**
@@ -238,6 +378,71 @@ export function accuracySummary(confusion: PooledConfusion, cutoff: AccuracyCuto
return `${period}, ${percent(confusion.precision)} of ${cutoffGroup(cutoff)} did ${eventName(target)}, against ${percent(overall)} of everyone. They include ${percent(confusion.recall)} of everyone who did it.`
}
+function median(values: number[]): number {
+ const sorted = [...values].sort((a, b) => a - b)
+ const middle = Math.floor(sorted.length / 2)
+ return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2
+}
+
+/** One sentence on how the champion did on the picked metric over the checked dates. Null with no values. */
+export function accuracyTrendSummary(
+ rows: OnlinePerformanceRowApi[],
+ metric: AccuracyTrendMetric,
+ cutoff: AccuracyCutoff,
+ target: string,
+ holdoutAuc: number | null
+): string | null {
+ const checked = rows
+ .filter((row) => row.emitted_role === 'champion')
+ .flatMap((row) => {
+ const point = trendPoint(row, metric, cutoff)
+ return point ? [{ row, value: point.value }] : []
+ })
+ if (checked.length === 0) {
+ return null
+ }
+ const values = checked.map(({ value }) => value)
+ const typical = median(values)
+ const single = checked.length === 1
+ const lead = single ? `On ${dayjs(checked[0].row.prediction_date).format('MMM D')}` : 'On a typical date'
+ const event = eventName(target)
+
+ switch (metric) {
+ case 'ranking': {
+ if (single) {
+ return `${lead}, realized AUC was ${values[0].toFixed(3)}.`
+ }
+ const range = `Over ${checked.length} checked dates, realized AUC ranged from ${Math.min(...values).toFixed(3)} to ${Math.max(...values).toFixed(3)}.`
+ if (holdoutAuc == null) {
+ return range
+ }
+ const holdout = holdoutAuc.toFixed(3)
+ const beat = values.filter((value) => value >= holdoutAuc).length
+ if (beat === 0) {
+ return `${range} It stayed below its holdout AUC of ${holdout} on every date.`
+ }
+ const dates = beat === checked.length ? 'every date' : `${beat} of ${checked.length} dates`
+ return `${range} It matched or beat its holdout AUC of ${holdout} on ${dates}.`
+ }
+ case 'hit_rate': {
+ const everyone = median(checked.map(({ row }) => row.base_rate))
+ return `${lead}, ${percent(typical)} of ${cutoffGroup(cutoff)} did ${event}, against ${percent(everyone)} of everyone.`
+ }
+ case 'coverage':
+ return `${lead}, ${cutoffGroup(cutoff)} included ${percent(typical)} of everyone who did ${event}.`
+ case 'lift':
+ return `${lead}, ${cutoffGroup(cutoff)} did ${event} at ${typical.toFixed(1)}× the rate of everyone.`
+ case 'calibration': {
+ const gaps = checked.map(({ row, value }) => value - row.base_rate)
+ if (single) {
+ return `${lead}, the champion's average score was ${(Math.abs(gaps[0]) * 100).toFixed(1)} points ${gaps[0] >= 0 ? 'above' : 'below'} the actual rate.`
+ }
+ const widest = Math.max(...gaps.map(Math.abs))
+ return `Over ${checked.length} checked dates, the champion's average score stayed within ${(widest * 100).toFixed(1)} points of the actual rate.`
+ }
+ }
+}
+
/** When the first scored date matures: the earliest completed scoring run plus the horizon. */
export function firstCheckDate(
runs: Pick<AutoresearchRunApi, 'run_type' | 'status' | 'created_at'>[],
diff --git a/products/autoresearch/frontend/pipeline/AccuracyTrendCard.tsx b/products/autoresearch/frontend/pipeline/AccuracyTrendCard.tsx
new file mode 100644
index 000000000000..a4d1bc6cc5cd
--- /dev/null
+++ b/products/autoresearch/frontend/pipeline/AccuracyTrendCard.tsx
@@ -0,0 +1,122 @@
+import { useActions, useValues } from 'kea'
+
+import { LemonSegmentedButton, LemonSwitch } from '@posthog/lemon-ui'
+
+import { autoresearchPipelineLogic } from '../autoresearchPipelineLogic'
+import {
+ ACCURACY_CUTOFFS,
+ ACCURACY_TREND_METRICS,
+ AccuracyTrendMetric,
+ accuracyTrendSummary,
+ trendMetricUsesCutoff,
+} from '../onlinePerformance'
+import { eventName } from '../pipelineQuestion'
+import { AccuracyTrendChart } from './AccuracyTrendChart'
+import { AccuracyTrendDatePanel } from './AccuracyTrendDatePanel'
+import { SectionCard } from './SectionCard'
+
+function description(metric: AccuracyTrendMetric, target: string): string {
+ switch (metric) {
+ case 'ranking':
+ return `How well the model ranked people who did ${target} above people who did not, on each checked date. 0.5 is a coin flip and 1 is a perfect ranking. The dashed line is the holdout AUC the model got in training.`
+ case 'hit_rate':
+ return `The share of flagged people who did ${target}, on each checked date.`
+ case 'coverage':
+ return `The share of everyone who did ${target} that the model flagged, on each checked date.`
+ case 'lift':
+ return `How many times more often flagged people did ${target} than everyone scored, on each checked date.`
+ case 'calibration':
+ return `The average score the model gave against the share of people who really did ${target}. When the lines stay close, you can read the scores as probabilities.`
+ }
+}
+
+/** The picked online metric per prediction date, for the champion and optionally its challengers. */
+export function AccuracyTrendCard(): JSX.Element {
+ const {
+ pipeline,
+ champion,
+ onlinePerformance,
+ accuracyTrend,
+ accuracyTrendMetric: metric,
+ accuracyCutoff,
+ showChallengersInTrend,
+ hasChallengerPerformance,
+ hoveredAccuracyDate,
+ } = useValues(autoresearchPipelineLogic)
+ const { setAccuracyTrendMetric, setAccuracyCutoff, setShowChallengersInTrend, setHoveredAccuracyDate } =
+ useActions(autoresearchPipelineLogic)
+ const targetEvent = pipeline?.target_event ?? ''
+ const holdoutAuc = champion?.holdout_score ?? null
+ const { dates } = accuracyTrend
+ const panelDate =
+ hoveredAccuracyDate && dates.includes(hoveredAccuracyDate) ? hoveredAccuracyDate : dates[dates.length - 1]
+ const summary = accuracyTrendSummary(onlinePerformance, metric, accuracyCutoff, targetEvent, holdoutAuc)
+
+ return (
+ <SectionCard
+ title="Accuracy over time"
+ description={description(metric, eventName(targetEvent))}
+ actions={
+ <LemonSwitch
+ label="Compare challengers"
+ checked={showChallengersInTrend}
+ onChange={setShowChallengersInTrend}
+ disabledReason={
+ !hasChallengerPerformance ? 'No challenger has been checked against real outcomes yet.' : null
+ }
+ bordered
+ size="small"
+ data-attr="autoresearch-model-accuracy-trend-challengers"
+ />
+ }
+ >
+ <div className="flex flex-wrap items-center justify-between gap-2">
+ <LemonSegmentedButton
+ size="small"
+ value={metric}
+ onChange={setAccuracyTrendMetric}
+ options={ACCURACY_TREND_METRICS.map(({ key, label }) => ({
+ value: key,
+ label,
+ 'data-attr': `autoresearch-model-accuracy-trend-metric-${key}`,
+ }))}
+ />
+ {trendMetricUsesCutoff(metric) && (
+ <LemonSegmentedButton
+ size="small"
+ value={accuracyCutoff}
+ onChange={(cutoff) => setAccuracyCutoff(cutoff, 'trend')}
+ options={ACCURACY_CUTOFFS.map(({ key, label }) => ({
+ value: key,
+ label,
+ 'data-attr': `autoresearch-model-accuracy-trend-cutoff-${key}`,
+ }))}
+ />
+ )}
+ </div>
+ {accuracyTrend.series[0].points.length === 0 || !panelDate ? (
+ <p className="text-sm text-muted mb-0">
+ {metric === 'ranking'
+ ? `No checked date had both people who did ${eventName(targetEvent)} and people who did not, so there is no ranking score yet.`
+ : 'This metric starts from the next checked date. Dates checked before it existed have no value.'}
+ </p>
+ ) : (
+ <div className="@container">
+ <div className="flex flex-col @3xl:flex-row gap-4">
+ <div className="flex-1 min-w-0">
+ <AccuracyTrendChart
+ trend={accuracyTrend}
+ metric={metric}
+ holdoutAuc={holdoutAuc}
+ hoveredDate={hoveredAccuracyDate}
+ onHover={setHoveredAccuracyDate}
+ />
+ </div>
+ <AccuracyTrendDatePanel trend={accuracyTrend} metric={metric} date={panelDate} />
+ </div>
+ </div>
+ )}
+ {summary && <p className="text-sm mb-0">{summary}</p>}
+ </SectionCard>
+ )
+}
diff --git a/products/autoresearch/frontend/pipeline/AccuracyTrendChart.tsx b/products/autoresearch/frontend/pipeline/AccuracyTrendChart.tsx
new file mode 100644
index 000000000000..ebd38abfaf71
--- /dev/null
+++ b/products/autoresearch/frontend/pipeline/AccuracyTrendChart.tsx
@@ -0,0 +1,175 @@
+import { dayjs } from 'lib/dayjs'
+
+import { AccuracyTrend, AccuracyTrendMetric, AccuracyTrendPoint, formatTrendValue } from '../onlinePerformance'
+import { trendLineStyles } from './accuracyTrendStyle'
+
+const WIDTH = 600
+const HEIGHT = 160
+
+/** Runs of points on consecutive dates. A date a series has no value for breaks its line instead of joining across it. */
+function lineRuns(points: AccuracyTrendPoint[], dateIndex: Map<string, number>): { index: number; value: number }[][] {
+ const runs: { index: number; value: number }[][] = []
+ let previous = -2
+ for (const point of points) {
+ const index = dateIndex.get(point.date) as number
+ if (index !== previous + 1) {
+ runs.push([])
+ }
+ runs[runs.length - 1].push({ index, value: point.value })
+ previous = index
+ }
+ return runs
+}
+
+/** The y range: every value plus the reference line, padded, and kept inside 0 to 1 for metrics that are shares. */
+function yDomain(trend: AccuracyTrend, metric: AccuracyTrendMetric, reference: number | null): [number, number] {
+ const values = trend.series.flatMap((s) =>
+ s.points.flatMap((p) => (s.role === 'champion' ? [p.value, p.low ?? p.value, p.high ?? p.value] : [p.value]))
+ )
+ if (reference != null) {
+ values.push(reference)
+ }
+ const min = Math.min(...values)
+ const max = Math.max(...values)
+ const pad = (max - min) * 0.1 || Math.abs(max) * 0.1 || 0.01
+ return [Math.max(min - pad, 0), metric === 'lift' ? max + pad : Math.min(max + pad, 1)]
+}
+
+/** One metric per prediction date, a line per model, with the AUC interval and holdout AUC in ranking mode. No chart deps, like MetricSparkline. */
+export function AccuracyTrendChart({
+ trend,
+ metric,
+ holdoutAuc,
+ hoveredDate,
+ onHover,
+}: {
+ trend: AccuracyTrend
+ metric: AccuracyTrendMetric
+ holdoutAuc: number | null
+ hoveredDate: string | null
+ onHover: (date: string | null) => void
+}): JSX.Element {
+ const { dates, series } = trend
+ const styles = trendLineStyles(series)
+ const reference = metric === 'ranking' ? holdoutAuc : null
+ const [yMin, yMax] = yDomain(trend, metric, reference)
+ const dateIndex = new Map(dates.map((date, index) => [date, index]))
+ const x = (index: number): number => (dates.length === 1 ? WIDTH / 2 : (index * WIDTH) / (dates.length - 1))
+ const y = (value: number): number => HEIGHT - ((value - yMin) / (yMax - yMin)) * HEIGHT
+ const columnWidth = dates.length === 1 ? WIDTH : WIDTH / (dates.length - 1)
+ const hoveredIndex = hoveredDate != null ? dateIndex.get(hoveredDate) : undefined
+
+ const champion = series[0]
+ const banded = metric === 'ranking' ? champion.points.filter((p): boolean => p.low != null && p.high != null) : []
+ const band = [
+ ...banded.map((p) => `${x(dateIndex.get(p.date) as number)},${y(p.high as number)}`),
+ ...[...banded].reverse().map((p) => `${x(dateIndex.get(p.date) as number)},${y(p.low as number)}`),
+ ].join(' ')
+
+ return (
+ <div className="space-y-1 min-w-0">
+ {(banded.length > 0 || reference != null) && (
+ <div className="flex flex-wrap items-baseline gap-x-4 gap-y-1 text-xs text-muted">
+ {banded.length > 0 && (
+ <span>
+ <span className="inline-block w-3 h-2 align-middle mr-1 bg-[var(--data-color-1)] opacity-20" />
+ Champion 95% interval
+ </span>
+ )}
+ {reference != null && (
+ <span>
+ <span className="inline-block w-3 align-middle mr-1 border-t border-dashed border-current" />
+ Holdout AUC {reference.toFixed(2)}
+ </span>
+ )}
+ </div>
+ )}
+ <div className="grid grid-cols-[auto_minmax(0,1fr)] gap-x-2 gap-y-1">
+ <div className="flex flex-col justify-between text-xs text-muted tabular-nums h-40 text-right">
+ <span>{formatTrendValue(metric, yMax)}</span>
+ <span>{formatTrendValue(metric, yMin)}</span>
+ </div>
+ <svg
+ viewBox={`0 0 ${WIDTH} ${HEIGHT}`}
+ preserveAspectRatio="none"
+ className="w-full h-40 overflow-visible border-b border-l"
+ role="img"
+ aria-label="Accuracy over prediction dates"
+ onMouseLeave={() => onHover(null)}
+ >
+ {banded.length > 0 && <polygon points={band} className="fill-[var(--data-color-1)] opacity-20" />}
+ {reference != null && (
+ <line
+ x1={0}
+ x2={WIDTH}
+ y1={y(reference)}
+ y2={y(reference)}
+ className="stroke-current text-secondary"
+ strokeDasharray="6 4"
+ vectorEffect="non-scaling-stroke"
+ />
+ )}
+ {hoveredIndex != null && (
+ <line
+ x1={x(hoveredIndex)}
+ x2={x(hoveredIndex)}
+ y1={0}
+ y2={HEIGHT}
+ className="stroke-current text-muted"
+ vectorEffect="non-scaling-stroke"
+ />
+ )}
+ {[...series].reverse().map((s, reversedIndex) => {
+ const style = styles[series.length - 1 - reversedIndex]
+ return (
+ <g key={s.key} className={style.stroke}>
+ {lineRuns(s.points, dateIndex).map((run) => (
+ <polyline
+ key={run[0].index}
+ points={run.map((p) => `${x(p.index)},${y(p.value)}`).join(' ')}
+ fill="none"
+ strokeWidth={2}
+ strokeDasharray={style.dash}
+ vectorEffect="non-scaling-stroke"
+ />
+ ))}
+ {s.points.map((p) => {
+ const index = dateIndex.get(p.date) as number
+ return (
+ // A zero-length line with round caps stays a round dot when the SVG stretches.
+ <line
+ key={p.date}
+ x1={x(index)}
+ x2={x(index)}
+ y1={y(p.value)}
+ y2={y(p.value)}
+ strokeWidth={s.role === 'actual' ? 4 : 6}
+ strokeLinecap="round"
+ vectorEffect="non-scaling-stroke"
+ />
+ )
+ })}
+ </g>
+ )
+ })}
+ {dates.map((date, index) => (
+ <rect
+ key={date}
+ x={x(index) - columnWidth / 2}
+ y={0}
+ width={columnWidth}
+ height={HEIGHT}
+ fill="transparent"
+ onMouseEnter={() => onHover(date)}
+ />
+ ))}
+ </svg>
+ <div />
+ <div className="flex justify-between text-xs text-muted">
+ <span>{dayjs(dates[0]).format('MMM D')}</span>
+ {dates.length > 1 && <span>{dayjs(dates[dates.length - 1]).format('MMM D')}</span>}
+ </div>
+ </div>
+ </div>
+ )
+}
diff --git a/products/autoresearch/frontend/pipeline/AccuracyTrendDatePanel.tsx b/products/autoresearch/frontend/pipeline/AccuracyTrendDatePanel.tsx
new file mode 100644
index 000000000000..c3180ca88e9b
--- /dev/null
+++ b/products/autoresearch/frontend/pipeline/AccuracyTrendDatePanel.tsx
@@ -0,0 +1,61 @@
+import { dayjs } from 'lib/dayjs'
+
+import { AccuracyTrend, AccuracyTrendMetric, formatTrendValue, percentagePoints } from '../onlinePerformance'
+import { TrendLineStyle, trendLineStyles } from './accuracyTrendStyle'
+
+function LineSample({ style }: { style: TrendLineStyle }): JSX.Element {
+ return (
+ <svg viewBox="0 0 16 4" className={`w-4 h-1 shrink-0 ${style.stroke}`} aria-hidden>
+ <line x1={0} x2={16} y1={2} y2={2} strokeWidth={2} strokeDasharray={style.dash} />
+ </svg>
+ )
+}
+
+/** Each line's value on one date, with the AUC interval or the gap to the actual rate, and the people checked. */
+export function AccuracyTrendDatePanel({
+ trend,
+ metric,
+ date,
+}: {
+ trend: AccuracyTrend
+ metric: AccuracyTrendMetric
+ date: string
+}): JSX.Element {
+ const styles = trendLineStyles(trend.series)
+ const actual = trend.series.find((s) => s.role === 'actual')?.points.find((p) => p.date === date)
+ const people = trend.people[date]
+
+ return (
+ <div className="w-full @3xl:w-64 shrink-0 rounded border p-3 space-y-2 text-xs">
+ <div className="text-muted">Scored on {dayjs(date).format('MMM D, YYYY')}</div>
+ {trend.series.map((s, index) => {
+ const point = s.points.find((p) => p.date === date)
+ const detail =
+ point == null
+ ? null
+ : metric === 'ranking' && point.low != null && point.high != null
+ ? `95% interval ${point.low.toFixed(3)} to ${point.high.toFixed(3)}`
+ : s.role !== 'actual' && actual != null
+ ? `${percentagePoints(point.value - actual.value)} against the actual rate`
+ : null
+ return (
+ <div key={s.key} className="min-w-0">
+ <div className="flex items-center gap-2">
+ <LineSample style={styles[index]} />
+ <span className="flex-1 min-w-0 truncate">{s.label}</span>
+ <span className="font-semibold tabular-nums">
+ {point ? formatTrendValue(metric, point.value, true) : '—'}
+ </span>
+ </div>
+ {detail && <div className="text-secondary pl-6">{detail}</div>}
+ </div>
+ )
+ })}
+ {people && (
+ <div className="text-secondary border-t pt-2">
+ {people.scored.toLocaleString()} scored, {people.positive.toLocaleString()} did it
+ </div>
+ )}
+ </div>
+ )
+}
diff --git a/products/autoresearch/frontend/pipeline/ConfusionMatrixCard.tsx b/products/autoresearch/frontend/pipeline/ConfusionMatrixCard.tsx
index 04025521c7f0..d912182ee72e 100644
--- a/products/autoresearch/frontend/pipeline/ConfusionMatrixCard.tsx
+++ b/products/autoresearch/frontend/pipeline/ConfusionMatrixCard.tsx
@@ -58,9 +58,12 @@ export function ConfusionMatrixCard(): JSX.Element {
<LemonSegmentedButton
size="small"
value={accuracyCutoff}
- onChange={setAccuracyCutoff}
- options={ACCURACY_CUTOFFS.map(({ key, label }) => ({ value: key, label }))}
- data-attr="autoresearch-model-accuracy-cutoff"
+ onChange={(cutoff) => setAccuracyCutoff(cutoff, 'flagged')}
+ options={ACCURACY_CUTOFFS.map(({ key, label }) => ({
+ value: key,
+ label,
+ 'data-attr': `autoresearch-model-accuracy-cutoff-${key}`,
+ }))}
/>
}
>
diff --git a/products/autoresearch/frontend/pipeline/OnlinePerformanceTab.tsx b/products/autoresearch/frontend/pipeline/OnlinePerformanceTab.tsx
index a1594ec3e93c..1acc3624a2fc 100644
--- a/products/autoresearch/frontend/pipeline/OnlinePerformanceTab.tsx
+++ b/products/autoresearch/frontend/pipeline/OnlinePerformanceTab.tsx
@@ -9,11 +9,11 @@ import { pluralize } from 'lib/utils/strings'
import { autoresearchPipelineLogic } from '../autoresearchPipelineLogic'
import { accuracyStats, accuracySummary, percent } from '../onlinePerformance'
import { eventName } from '../pipelineQuestion'
+import { AccuracyTrendCard } from './AccuracyTrendCard'
import { ConfusionMatrixCard } from './ConfusionMatrixCard'
import { EmptyTab } from './EmptyTab'
import { LiveModelDrivers } from './LiveModelDrivers'
import { MetricCard } from './MetricCard'
-import { RealizedAucChart } from './RealizedAucChart'
import { ScoreNowButton } from './ScoreNowButton'
import { SectionCard } from './SectionCard'
import { SegmentCalibrationBars } from './SegmentCalibrationBars'
@@ -50,12 +50,10 @@ function NotCheckedYet(): JSX.Element {
export function OnlinePerformanceTab(): JSX.Element {
const {
pipeline,
- champion,
onlinePerformance,
onlinePerformanceLoading,
onlinePerformanceError,
latestChampionPerformance,
- realizedAucPoints,
segmentCalibration,
accuracyCutoff,
championConfusion,
@@ -115,20 +113,7 @@ export function OnlinePerformanceTab(): JSX.Element {
</SectionCard>
)}
- {realizedAucPoints.length > 0 && (
- <SectionCard
- title="Realized AUC over time"
- description={
- <>
- Realized AUC measures how well the model ranked people who did {target} above people who did
- not. 0.5 is a coin flip and 1 is a perfect ranking. The holdout line is the score the model
- got in training.
- </>
- }
- >
- <RealizedAucChart points={realizedAucPoints} holdoutAuc={champion?.holdout_score ?? null} />
- </SectionCard>
- )}
+ <AccuracyTrendCard />
<LiveModelDrivers />
diff --git a/products/autoresearch/frontend/pipeline/RealizedAucChart.tsx b/products/autoresearch/frontend/pipeline/RealizedAucChart.tsx
deleted file mode 100644
index c443567659f3..000000000000
--- a/products/autoresearch/frontend/pipeline/RealizedAucChart.tsx
+++ /dev/null
@@ -1,107 +0,0 @@
-import { dayjs } from 'lib/dayjs'
-
-import { RealizedAucPoint } from '../onlinePerformance'
-
-const WIDTH = 600
-const HEIGHT = 160
-const Y_MIN = 0.5
-const Y_MAX = 1
-
-/** Champion realized AUC per prediction date, with its 95% interval and the holdout AUC for reference. No chart deps, like MetricSparkline. */
-export function RealizedAucChart({
- points,
- holdoutAuc,
-}: {
- points: RealizedAucPoint[]
- holdoutAuc: number | null
-}): JSX.Element {
- const x = (index: number): number => (points.length === 1 ? WIDTH / 2 : (index * WIDTH) / (points.length - 1))
- const y = (value: number): number =>
- HEIGHT - ((Math.min(Math.max(value, Y_MIN), Y_MAX) - Y_MIN) / (Y_MAX - Y_MIN)) * HEIGHT
- const banded = points.map((point, index) => ({ ...point, index })).filter((p) => p.low != null && p.high != null)
- const band = [
- ...banded.map((p) => `${x(p.index)},${y(p.high as number)}`),
- ...[...banded].reverse().map((p) => `${x(p.index)},${y(p.low as number)}`),
- ].join(' ')
- const line = points.map((p, index) => `${x(index)},${y(p.auc)}`).join(' ')
- const latest = points[points.length - 1]
-
- return (
- <div className="space-y-1 max-w-4xl">
- <div className="flex flex-wrap items-baseline gap-x-4 gap-y-1 text-xs text-muted">
- <span>
- <span className="inline-block w-3 h-0.5 align-middle mr-1 bg-[var(--data-color-1)]" />
- Realized AUC, latest {latest.auc.toFixed(2)}
- </span>
- <span>
- <span className="inline-block w-3 h-2 align-middle mr-1 bg-[var(--data-color-1)] opacity-20" />
- 95% interval
- </span>
- {holdoutAuc != null && (
- <span>
- <span className="inline-block w-3 align-middle mr-1 border-t border-dashed border-current" />
- Holdout AUC {holdoutAuc.toFixed(2)}
- </span>
- )}
- </div>
- <div className="flex gap-2">
- <div className="flex flex-col justify-between text-xs text-muted tabular-nums h-40">
- <span>{Y_MAX.toFixed(1)}</span>
- <span>{Y_MIN.toFixed(1)}</span>
- </div>
- <svg
- viewBox={`0 0 ${WIDTH} ${HEIGHT}`}
- preserveAspectRatio="none"
- className="flex-1 min-w-0 h-40 overflow-visible border-b border-l"
- role="img"
- aria-label="Realized AUC over prediction dates"
- >
- {banded.length > 0 && <polygon points={band} className="fill-[var(--data-color-1)] opacity-20" />}
- {holdoutAuc != null && (
- <line
- x1={0}
- x2={WIDTH}
- y1={y(holdoutAuc)}
- y2={y(holdoutAuc)}
- className="stroke-current text-secondary"
- strokeDasharray="6 4"
- vectorEffect="non-scaling-stroke"
- />
- )}
- <polyline
- points={line}
- fill="none"
- className="stroke-[var(--data-color-1)]"
- strokeWidth={2}
- vectorEffect="non-scaling-stroke"
- />
- {points.map((p, index) => (
- // A zero-length line with round caps stays a round dot when the SVG stretches.
- <line
- key={p.date}
- x1={x(index)}
- x2={x(index)}
- y1={y(p.auc)}
- y2={y(p.auc)}
- className="stroke-[var(--data-color-1)]"
- strokeWidth={6}
- strokeLinecap="round"
- vectorEffect="non-scaling-stroke"
- >
- <title>
- {dayjs(p.date).format('MMM D, YYYY')}: AUC {p.auc.toFixed(3)}
- {p.low != null && p.high != null
- ? ` (95% interval ${p.low.toFixed(3)} to ${p.high.toFixed(3)})`
- : ''}
- </title>
- </line>
- ))}
- </svg>
- </div>
- <div className="flex justify-between text-xs text-muted pl-6">
- <span>{dayjs(points[0].date).format('MMM D')}</span>
- {points.length > 1 && <span>{dayjs(latest.date).format('MMM D')}</span>}
- </div>
- </div>
- )
-}
diff --git a/products/autoresearch/frontend/pipeline/accuracyTrendStyle.ts b/products/autoresearch/frontend/pipeline/accuracyTrendStyle.ts
new file mode 100644
index 000000000000..4070b8175b48
--- /dev/null
+++ b/products/autoresearch/frontend/pipeline/accuracyTrendStyle.ts
@@ -0,0 +1,36 @@
+import { AccuracyTrendSeries } from '../onlinePerformance'
+
+export interface TrendLineStyle {
+ /** Tailwind classes that set the stroke color. */
+ stroke: string
+ dash: string | undefined
+}
+
+// Static class names, so Tailwind keeps them.
+const CHALLENGER_STROKES = [
+ 'stroke-[var(--data-color-2)]',
+ 'stroke-[var(--data-color-3)]',
+ 'stroke-[var(--data-color-4)]',
+ 'stroke-[var(--data-color-5)]',
+]
+const CHALLENGER_DASHES = ['6 4', '2 3']
+
+/** One style per series, in series order. Lines differ by dash as well as color, so they stay apart without color. */
+export function trendLineStyles(series: AccuracyTrendSeries[]): TrendLineStyle[] {
+ let challengerIndex = 0
+ return series.map((s) => {
+ switch (s.role) {
+ case 'champion':
+ return { stroke: 'stroke-[var(--data-color-1)]', dash: undefined }
+ case 'actual':
+ return { stroke: 'stroke-current text-secondary', dash: '6 4' }
+ case 'challenger': {
+ const index = challengerIndex++
+ return {
+ stroke: CHALLENGER_STROKES[index % CHALLENGER_STROKES.length],
+ dash: CHALLENGER_DASHES[index % CHALLENGER_DASHES.length],
+ }
+ }
+ }
+ })
+}
diff --git a/products/autoresearch/package.json b/products/autoresearch/package.json
index 9c463861b2d0..d18053d12678 100644
--- a/products/autoresearch/package.json
+++ b/products/autoresearch/package.json
@@ -23,6 +23,8 @@
"@posthog/brand": "catalog:",
"@posthog/icons": "catalog:",
"@posthog/lemon-ui": "*",
+ "@testing-library/dom": "*",
+ "@testing-library/user-event": "*",
"@types/react": "*",
"react": "*"
}