:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(autoresearch): show online accuracy over time on the accuracy tab (#119807)
posthog[bot]5a25de4269aa
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex 35cf94879498..11af65a700c5 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -6659,17 +6659,21 @@ snapshots:products-autoresearch-model-card--training--light:hash: v1.k794b7964.b358f33a0d879e85f4c8bdb4ae65330f392cdaa8a869e8bce7e07cdcb22d13a9.EAfBa_-86ylT0UXKPW7EAqtGV9zNLRT4VdSYkWFWj_Mproducts-autoresearch-model-detail-scene--accuracy--narrow--dark:- hash: v1.k794b7964.cc5c80f54b366af72ee206c132ff16855a53d9f0152b5fcd665badc347dfe76a.VrTGlwRaHpwfI1QBWePyr2ARuWCj1HoG2ztzwfXESAE+ hash: v1.k794b7964.4cab77a4d9bfb7975449b4c47648e536f506002c38ed147a73524af54da5a6b5.orHlfN-LtUJeeKkONCkVAUx3PZ31dQw-lxvYmfjfFZwproducts-autoresearch-model-detail-scene--accuracy--narrow--light:- hash: v1.k794b7964.fd34150ddd7a416e7fe6d25aad92f32348db555b75b8b9074eb984313d326ae5.regqIjpzTv7BhVk_HRbkv2QTV49o_yP7SXcVGHbuOgM+ hash: v1.k794b7964.d408d0bce9b8c87c96b7da7acaeabb25004967fbeec3c62584bca0c6c8ac1379.TNDdvZN3B87CYwfV0JrlH4mmOfD7zy_o78jJPJMOVVUproducts-autoresearch-model-detail-scene--accuracy--wide--dark:- hash: v1.k794b7964.8079c8a10c44ea9de9d26d8ce7c73c380b7523fecb0bc6c5217fcbe757d48ddc.s4wWq4ykRqBZKIokdQ7VJwm6NlFNienMd_5OeTxFEGE+ hash: v1.k794b7964.5682a1f2d01f1443bf621305ee8ffdc331cea7a60bd316c1a6f7ea7bf2169dae.DTCmGuHIF_7jXkNbXDx61qO3R5StzjzR1Ss-eJXz9uMproducts-autoresearch-model-detail-scene--accuracy--wide--light:- hash: v1.k794b7964.93b1cbc7ef465154449463c0410d44fa6b468c1272015c66aeda987b45e55c82.3KzsV_fKafMRSkfLwl5Tmb0lJ0BtsaXYCePf36mOhDA+ hash: v1.k794b7964.914e11340da36659c95e64018017601f426f3a76e6d6f3922d1f0526d263dbd4.u_QcTRlMQZGm_G9T2NxseEBkUI5cBldEUjUefQuyzxcproducts-autoresearch-model-detail-scene--accuracy-before-precision-recall--wide--dark:- hash: v1.k794b7964.392e2f039d6619c9344d299e9bdde59ba18e44f3c6a52f9485d16b1d12f71143.iws2WKI1cVn8mtF7gQ7yXjLl6Zer9ClpXnKDt4saXMw+ hash: v1.k794b7964.288332b7abc333b0b2e5bf103793af5bd21af15b445d01051597452b186345d9.GG4kd4k9wLG0iij6X36W9pCzDSJi338S8DuUM5iT1Mcproducts-autoresearch-model-detail-scene--accuracy-before-precision-recall--wide--light:- hash: v1.k794b7964.f9e9a4a3106781c287215c4814de98cfcd66afa6c3bba6f3a9b5e028943a035d.GFjkvK159EBC_ohj72MNqX56nSfXlla66-89avAQ16I+ hash: v1.k794b7964.331ed0c5d5a26f0caadf6483ed07a718afc022e84d63ce3a1a9d2825870b3d15.ZEzPvoPtVQJN-IDnPBS3ZgWRJ7MYBELQaXBvBPbDWww+ products-autoresearch-model-detail-scene--accuracy-hit-rate-with-challengers--wide--dark:+ hash: v1.k794b7964.1ac9a20b8616dad37424909ca867b45106763943c61739470b298c8651854a5b.Mk_e6ArRT0NCHD-DnzB69VprJZmBeiuBJqHghsG2AcY+ products-autoresearch-model-detail-scene--accuracy-hit-rate-with-challengers--wide--light:+ hash: v1.k794b7964.9ba1c3edd96fb8572bf98877a23b1ae71c1cabb39c4dd900935751ea9a62a243.vJA1Rx5am3ih3Wu-xMWyJZzd1UMZabJm810pFmVhEggproducts-autoresearch-model-detail-scene--agent-research-narrow--narrow--dark:hash: v1.k794b7964.0889baa7d2b6bca50dfd14487b80f287e8bd27a098a0e3482cb9da086c840e3e.5vUCdyEAF1AXUusb51kIrj4Jbpwm8BYmrR6O3ATOz34products-autoresearch-model-detail-scene--agent-research-narrow--narrow--light:diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex c4e6086ec177..bf95d04cf8d5 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -2652,6 +2652,12 @@ importers:'@posthog/quill-charts':specifier: workspace:*version: link:../../packages/quill/packages/charts+ '@testing-library/dom':+ specifier: '*'+ version: 10.4.0+ '@testing-library/user-event':+ specifier: '*'+ version: 14.6.1(@testing-library/[email protected])'@types/react':specifier: 18.3.27version: 18.3.27diff --git a/products/autoresearch/frontend/AGENTS.md b/products/autoresearch/frontend/AGENTS.mdindex 88d35f056132..f2fbee90fcb0 100644--- a/products/autoresearch/frontend/AGENTS.md+++ b/products/autoresearch/frontend/AGENTS.md@@ -20,7 +20,7 @@ The product is mostly a backend, and the UI is deliberately thin — it reads st| `pipelineLifecycle.ts` | The five lifecycle steps (question set to checked against reality) and which one is current. || `trainingScheduleLogic.ts` | Setup tab training schedule: the research interval and experiments per run, their save, and the one-line phase and next run status. || `agentSearch.ts` | The agent's search across training runs: chart points, best score so far, change per experiment within a run, log groups and status brief. |-| `onlinePerformance.ts` | Accuracy tab helpers over the `online_performance` endpoint: the stat tiles, the realized AUC series, predicted against actual by segment. |+| `onlinePerformance.ts` | Accuracy tab helpers over the `online_performance` endpoint: the stat tiles, the per-metric accuracy trend, predicted against actual. || `predictionCoverage.ts` | Predictions tab coverage from the runs list: the measured summary and the coverage and score age per day. || `predictionSegments.ts` | The likelihood segments from the cut points the `online_performance` endpoint returns: lift over the base rate, or a fixed pair until then. || `PipelineStatusTag.tsx` | Status tag + tooltip shared by the list and detail scenes. |@@ -53,7 +53,7 @@ Follow `frontend/src/AGENTS.md` — it applies to product frontends too.- TypeScript with explicit return types; Tailwind utilities rather than inline styles.- Reuse Lemon / quill components instead of hand-rolling tables, badges, or tags.- Any button that fires a request must guard against double-submission — `loading` / `disabledReason` on `LemonButton`, reset on both success and error paths. The lifecycle actions here (train, score, archive, pause, resume) are all network calls, and several are expensive: firing `train` twice starts two sandbox agent runs and spends the budget twice.-- Usage events are part of the API: renaming one breaks every insight built on it. The list logic captures `autoresearch model list viewed`, `autoresearch model list load failed`, `autoresearch model deleted` / `paused` / `resumed`, `autoresearch model training started` (with `source: 'list'` from a card), `autoresearch model card clicked`, and `autoresearch model action failed` (with `action`). The create logic captures `autoresearch model template selected` (with `template_key`), `autoresearch model custom chosen`, `autoresearch model advanced opened`, `autoresearch model created` (with `action`: `start_training` or `save_draft`), and `autoresearch model training started` / `training start failed`. The detail logic captures `autoresearch model tab changed` (with `tab`) when a person picks a tab, `autoresearch model experiment log filter changed` (with `filter`), `autoresearch model accuracy cutoff changed` (with `cutoff`), `autoresearch model search point clicked` (with `run_id`, `iteration_number`, `status`), `autoresearch model experiment details opened` (with `run_id`, `iteration_number`) when a person opens an experiment's working notes, `autoresearch model suggestion sent` (with `priority`), `autoresearch agent research experiment opened` (with `run_id`, `iteration_number`, `status`, `source`: `chart` or `log`) when a person opens the experiment drawer, `autoresearch agent research steer opened` (with `queued_count`) `autoresearch agent research full notes opened` (with `run_number`) and `autoresearch training schedule changed` (with `research_interval_days` and `iterations_per_run`); buttons carry a `data-attr` for autocapture. Extend the same names when you add a scene.+- Usage events are part of the API: renaming one breaks every insight built on it. The list logic captures `autoresearch model list viewed`, `autoresearch model list load failed`, `autoresearch model deleted` / `paused` / `resumed`, `autoresearch model training started` (with `source: 'list'` from a card), `autoresearch model card clicked`, and `autoresearch model action failed` (with `action`). The create logic captures `autoresearch model template selected` (with `template_key`), `autoresearch model custom chosen`, `autoresearch model advanced opened`, `autoresearch model created` (with `action`: `start_training` or `save_draft`), and `autoresearch model training started` / `training start failed`. The detail logic captures `autoresearch model tab changed` (with `tab`) when a person picks a tab, `autoresearch model experiment log filter changed` (with `filter`), `autoresearch model accuracy cutoff changed` (with `cutoff`, and `source`: `flagged` or `trend` for the card that changed it), `autoresearch model accuracy trend metric changed` (with `metric`), `autoresearch model accuracy trend challengers toggled` (with `shown`), `autoresearch model search point clicked` (with `run_id`, `iteration_number`, `status`), `autoresearch model experiment details opened` (with `run_id`, `iteration_number`) when a person opens an experiment's working notes, `autoresearch model suggestion sent` (with `priority`), `autoresearch agent research experiment opened` (with `run_id`, `iteration_number`, `status`, `source`: `chart` or `log`) when a person opens the experiment drawer, `autoresearch agent research steer opened` (with `queued_count`) `autoresearch agent research full notes opened` (with `run_number`) and `autoresearch training schedule changed` (with `research_interval_days` and `iterations_per_run`); buttons carry a `data-attr` for autocapture. Extend the same names when you add a scene.## Copydiff --git a/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx b/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsxindex e035090c1261..e9b905e65305 100644--- a/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx+++ b/products/autoresearch/frontend/AutoresearchPipelineScene.stories.tsx@@ -1,4 +1,6 @@import { Meta, StoryObj } from '@storybook/react'+import { within } from '@testing-library/dom'+import userEvent from '@testing-library/user-event'import { FEATURE_FLAGS } from 'lib/constants'import { App } from 'scenes/App'@@ -142,44 +144,82 @@ function iteration(}}+const PERFORMANCE_DATES = ['2026-02-10', '2026-02-11', '2026-02-12', '2026-02-13', '2026-02-14']++/** Top 10% counts out of 47,000 people scored, 1,410 of whom did it. */+function topTenCounts(tp: number): Record<string, number> {+ return {+ tp,+ fp: 4700 - tp,+ fn: 1410 - tp,+ tn: 47000 - 4700 - (1410 - tp),+ n_flagged: 4700,+ precision: tp / 4700,+ recall: tp / 1410,+ }+}++function performanceRow(prediction_date: string, index: number): { prediction_date: string } & Record<string, unknown> {+ return {+ validation_run_id: `validate-${index}`,+ prediction_date,+ horizon_days: 7,+ weekday: 1,+ model_id: champion.id,+ emitted_role: 'champion',+ current_role: 'champion',+ n_scored: 47000,+ n_positive: 1410,+ base_rate: 0.03,+ mean_p_y: 0.033,+ realized_auc: [0.78, 0.8, 0.79, 0.82, 0.81][index],+ realized_auc_ci_low: [0.76, 0.78, 0.77, 0.8, 0.79][index],+ realized_auc_ci_high: [0.8, 0.82, 0.81, 0.84, 0.83][index],+ brier_score: 0.06,+ calibration_error: 0.04,+ lift_at_10: 3.4,+ lift_at_20: 2.6,+ average_precision: 0.21,+ confusion: {+ top_10: topTenCounts([440, 480, 470, 510, 455][index]),+ top_20: { tp: 735, fp: 8665, fn: 675, tn: 36925, n_flagged: 9400, precision: 0.0782, recall: 0.5213 },+ likely: { tp: 700, fp: 3700, fn: 710, tn: 41890, n_flagged: 4400, precision: 0.1591, recall: 0.4965 },+ },+ likely_threshold: 0.09,+ calibration_bins: [+ { n: 37600, mean_p_y: 0.01, positive_rate: 0.008 },+ { n: 4700, mean_p_y: 0.08, positive_rate: 0.07 },+ { n: 2350, mean_p_y: 0.3, positive_rate: 0.27 },+ { n: 2350, mean_p_y: 0.68, positive_rate: 0.6 },+ ],+ warning: null,+ validated_at: '2026-02-21T03:00:00Z',+ }+}++// The challenger started scoring next to the champion on Feb 12, so its line covers the last three dates.+const challengerPerformance = PERFORMANCE_DATES.slice(2).map((prediction_date, index) => ({+ ...performanceRow(prediction_date, index + 2),+ validation_run_id: `validate-challenger-${index}`,+ model_id: challenger.id,+ emitted_role: 'challenger',+ current_role: 'challenger',+ mean_p_y: 0.031,+ realized_auc: [0.82, 0.84, 0.83][index],+ realized_auc_ci_low: [0.8, 0.82, 0.81][index],+ realized_auc_ci_high: [0.84, 0.86, 0.85][index],+ lift_at_10: [3.6, 3.9, 3.8][index],+ confusion: {+ top_10: { tp: 520, fp: 4180, fn: 890, tn: 41410, n_flagged: 4700, precision: 0.1106, recall: 0.3688 },+ top_20: { tp: 760, fp: 8640, fn: 650, tn: 36950, n_flagged: 9400, precision: 0.0809, recall: 0.539 },+ likely: { tp: 720, fp: 3600, fn: 690, tn: 41990, n_flagged: 4320, precision: 0.1667, recall: 0.5106 },+ },+}))+const onlinePerformance = {- rows: ['2026-02-10', '2026-02-11', '2026-02-12', '2026-02-13', '2026-02-14']- .map((prediction_date, index) => ({- validation_run_id: `validate-${index}`,- prediction_date,- horizon_days: 7,- weekday: 1,- model_id: champion.id,- emitted_role: 'champion',- current_role: 'champion',- n_scored: 47000,- n_positive: 1410,- base_rate: 0.03,- mean_p_y: 0.033,- realized_auc: [0.78, 0.8, 0.79, 0.82, 0.81][index],- realized_auc_ci_low: [0.76, 0.78, 0.77, 0.8, 0.79][index],- realized_auc_ci_high: [0.8, 0.82, 0.81, 0.84, 0.83][index],- brier_score: 0.06,- calibration_error: 0.04,- lift_at_10: 3.4,- lift_at_20: 2.6,- average_precision: 0.21,- confusion: {- top_10: { tp: 480, fp: 4220, fn: 930, tn: 41370, n_flagged: 4700, precision: 0.1021, recall: 0.3404 },- top_20: { tp: 735, fp: 8665, fn: 675, tn: 36925, n_flagged: 9400, precision: 0.0782, recall: 0.5213 },- likely: { tp: 700, fp: 3700, fn: 710, tn: 41890, n_flagged: 4400, precision: 0.1591, recall: 0.4965 },- },- likely_threshold: 0.09,- calibration_bins: [- { n: 37600, mean_p_y: 0.01, positive_rate: 0.008 },- { n: 4700, mean_p_y: 0.08, positive_rate: 0.07 },- { n: 2350, mean_p_y: 0.3, positive_rate: 0.27 },- { n: 2350, mean_p_y: 0.68, positive_rate: 0.6 },- ],- warning: null,- validated_at: '2026-02-21T03:00:00Z',- }))- .reverse(),+ rows: [...PERFORMANCE_DATES.map(performanceRow), ...challengerPerformance].sort((a, b) =>+ b.prediction_date.localeCompare(a.prediction_date)+ ),segment_thresholds: {likely_threshold: 0.09,possible_threshold: 0.03,@@ -394,6 +434,26 @@ export const Accuracy: Story = {},}+export const AccuracyHitRateWithChallengers: Story = {+ parameters: {+ pageUrl: `${urls.autoresearchPipeline(PIPELINE_ID)}?tab=accuracy`,+ testOptions: {+ viewportWidths: ['wide'],+ waitForSelector: '[data-attr="autoresearch-model-accuracy-trend-cutoff-top_10"]',+ },+ },+ play: async ({ canvasElement }) => {+ const canvas = within(canvasElement)+ await canvas.findByText('Accuracy over time', {}, { timeout: 15000 })+ await userEvent.click(+ canvasElement.querySelector(+ '[data-attr="autoresearch-model-accuracy-trend-metric-hit_rate"]'+ ) as HTMLElement+ )+ await userEvent.click(await canvas.findByText('Compare challengers'))+ },+}+export const AccuracyBeforePrecisionRecall: Story = {decorators: [mswDecorator({diff --git a/products/autoresearch/frontend/autoresearchPipelineLogic.ts b/products/autoresearch/frontend/autoresearchPipelineLogic.tsindex 242c3cc9d409..685b0012a71c 100644--- a/products/autoresearch/frontend/autoresearchPipelineLogic.ts+++ b/products/autoresearch/frontend/autoresearchPipelineLogic.ts@@ -64,14 +64,15 @@ import {import type { ConfusionByCutoffApi } from './generated/api.schemas'import {type AccuracyCutoff,+ type AccuracyTrend,+ type AccuracyTrendMetric,type PooledConfusion,- RealizedAucPoint,SegmentCalibration,+ accuracyTrend,calibrationBySegment,firstCheckDate,latestChampionRow,pooledConfusion,- realizedAucSeries,validatedPredictionDates,} from './onlinePerformance'import { LifecycleStep, pipelineLifecycle } from './pipelineLifecycle'@@ -90,6 +91,9 @@ import type { PredictionsPeopleView } from './predictionsPeopleQuery'/** Where a person opened an experiment from. */export type ExperimentOpenSource = 'chart' | 'log'+/** The card whose cutoff picker changed the shared cutoff. */+export type AccuracyCutoffSource = 'flagged' | 'trend'+export interface AutoresearchPipelineLogicProps {id: string}@@ -288,6 +292,8 @@ export interface autoresearchPipelineLogicValues {currentProjectId: number | null // projectLogiccurrentTeamId: number | null // teamLogicaccuracyCutoff: AccuracyCutoff+ accuracyTrend: AccuracyTrend+ accuracyTrendMetric: AccuracyTrendMetricactiveScoreRun: AutoresearchRunApi | nullactiveTab: AutoresearchPipelineTabagentSearch: AgentSearch@@ -308,7 +314,9 @@ export interface autoresearchPipelineLogicValues {experimentLogFilter: ExperimentLogFilterexperimentLogGroups: ExperimentLogGroup[]firstCheck: dayjs.Dayjs | null+ hasChallengerPerformance: booleanhasLiveTrainingRun: boolean+ hoveredAccuracyDate: string | nullhoveredSearchPoint: SearchPoint | nulllatestChampionPerformance: OnlinePerformanceRowApi | nulllatestPropertyCleanupRun: AutoresearchRunApi | null@@ -335,7 +343,6 @@ export interface autoresearchPipelineLogicValues {probabilityDistributionLoading: booleanprobabilityHistogram: ProbabilityBucket[] | nullqueuedSuggestionCount: number- realizedAucPoints: RealizedAucPoint[]reportByRun: Record<string, string | null>reportByRunLoading: booleanrunRoles: Record<string, RunRole>@@ -353,6 +360,7 @@ export interface autoresearchPipelineLogicValues {segmentDefinitions: PredictionSegmentDefinition[] | nullsegmentThresholds: PredictionSegmentThresholdsApi | nullselectedTab: AutoresearchPipelineTab | null+ showChallengersInTrend: booleanstartTrainingResult: AutoresearchTrainingRunApi | nullstartTrainingResultLoading: booleanstatusBrief: StatusBrief | null@@ -680,8 +688,15 @@ export interface autoresearchPipelineLogicActions {searchPointClicked: (point: SearchPoint) => {point: SearchPoint}- setAccuracyCutoff: (cutoff: AccuracyCutoff) => {+ setAccuracyCutoff: (+ cutoff: AccuracyCutoff,+ source: AccuracyCutoffSource+ ) => {cutoff: keyof ConfusionByCutoffApi+ source: AccuracyCutoffSource+ }+ setAccuracyTrendMetric: (metric: AccuracyTrendMetric) => {+ metric: AccuracyTrendMetric}setActiveScoreRun: (run: AutoresearchRunApi | null) => {run: AutoresearchRunApi | null@@ -692,6 +707,9 @@ export interface autoresearchPipelineLogicActions {setExperimentLogFilter: (filter: ExperimentLogFilter) => {filter: ExperimentLogFilter}+ setHoveredAccuracyDate: (date: string | null) => {+ date: string | null+ }setHoveredSearchPoint: (point: SearchPoint | null) => {point: SearchPoint | null}@@ -701,6 +719,9 @@ export interface autoresearchPipelineLogicActions {setSegmentThresholds: (thresholds: PredictionSegmentThresholdsApi) => {thresholds: PredictionSegmentThresholdsApi}+ setShowChallengersInTrend: (show: boolean) => {+ show: boolean+ }setSuggestionDraft: (draft: string) => {draft: string}@@ -819,7 +840,13 @@ export interface autoresearchPipelineLogicMeta {) => CoverageSummary | nullcoverageHistory: (runs: AutoresearchRunApi[]) => CoveragePoint[]latestChampionPerformance: (onlinePerformance: OnlinePerformanceRowApi[]) => OnlinePerformanceRowApi | null- realizedAucPoints: (onlinePerformance: OnlinePerformanceRowApi[]) => RealizedAucPoint[]+ accuracyTrend: (+ onlinePerformance: OnlinePerformanceRowApi[],+ accuracyTrendMetric: AccuracyTrendMetric,+ accuracyCutoff: keyof ConfusionByCutoffApi,+ showChallengersInTrend: boolean+ ) => AccuracyTrend+ hasChallengerPerformance: (onlinePerformance: OnlinePerformanceRowApi[]) => booleansegmentDefinitions: (segmentThresholds: PredictionSegmentThresholdsApi | null) => PredictionSegmentDefinition[] | null@@ -893,7 +920,10 @@ export const autoresearchPipelineLogic = kea<autoresearchPipelineLogicType>([actions({setActiveTab: (tab: AutoresearchPipelineTab) => ({ tab }),setPredictionsPeopleView: (view: PredictionsPeopleView) => ({ view }),- setAccuracyCutoff: (cutoff: AccuracyCutoff) => ({ cutoff }),+ setAccuracyCutoff: (cutoff: AccuracyCutoff, source: AccuracyCutoffSource) => ({ cutoff, source }),+ setAccuracyTrendMetric: (metric: AccuracyTrendMetric) => ({ metric }),+ setShowChallengersInTrend: (show: boolean) => ({ show }),+ setHoveredAccuracyDate: (date: string | null) => ({ date }),setTabFromUrl: (tab: AutoresearchPipelineTab | null) => ({ tab }),loadDetail: true,toggleRunArtifacts: (runId: string) => ({ runId }),@@ -948,6 +978,24 @@ export const autoresearchPipelineLogic = kea<autoresearchPipelineLogicType>([setAccuracyCutoff: (_, { cutoff }) => cutoff,},],+ accuracyTrendMetric: [+ 'ranking' as AccuracyTrendMetric,+ {+ setAccuracyTrendMetric: (_, { metric }) => metric,+ },+ ],+ showChallengersInTrend: [+ false,+ {+ setShowChallengersInTrend: (_, { show }) => show,+ },+ ],+ hoveredAccuracyDate: [+ null as string | null,+ {+ setHoveredAccuracyDate: (_, { date }) => date,+ },+ ],activeScoreRun: [null as AutoresearchRunApi | null,{@@ -1456,9 +1504,20 @@ export const autoresearchPipelineLogic = kea<autoresearchPipelineLogicType>([(onlinePerformance: OnlinePerformanceRowApi[]): OnlinePerformanceRowApi | null =>latestChampionRow(onlinePerformance),],- realizedAucPoints: [+ accuracyTrend: [+ (s) => [s.onlinePerformance, s.accuracyTrendMetric, s.accuracyCutoff, s.showChallengersInTrend],+ (+ onlinePerformance: OnlinePerformanceRowApi[],+ accuracyTrendMetric: AccuracyTrendMetric,+ accuracyCutoff: AccuracyCutoff,+ showChallengersInTrend: boolean+ ): AccuracyTrend =>+ accuracyTrend(onlinePerformance, accuracyTrendMetric, accuracyCutoff, showChallengersInTrend),+ ],+ hasChallengerPerformance: [(s) => [s.onlinePerformance],- (onlinePerformance: OnlinePerformanceRowApi[]): RealizedAucPoint[] => realizedAucSeries(onlinePerformance),+ (onlinePerformance: OnlinePerformanceRowApi[]): boolean =>+ onlinePerformance.some((row) => row.emitted_role !== 'champion'),],segmentDefinitions: [(s) => [s.segmentThresholds],@@ -1859,8 +1918,20 @@ export const autoresearchPipelineLogic = kea<autoresearchPipelineLogicType>([actions.loadPredictionSegments()}},- setAccuracyCutoff: ({ cutoff }) => {- posthog.capture('autoresearch model accuracy cutoff changed', { pipeline_id: props.id, cutoff })+ setAccuracyCutoff: ({ cutoff, source }) => {+ // pinned: analytics event name and properties. Renaming breaks insights built on them.+ posthog.capture('autoresearch model accuracy cutoff changed', { pipeline_id: props.id, cutoff, source })+ },+ setAccuracyTrendMetric: ({ metric }) => {+ // pinned: analytics event name and properties. Renaming breaks insights built on them.+ posthog.capture('autoresearch model accuracy trend metric changed', { pipeline_id: props.id, metric })+ },+ setShowChallengersInTrend: ({ show }) => {+ // pinned: analytics event name and properties. Renaming breaks insights built on them.+ posthog.capture('autoresearch model accuracy trend challengers toggled', {+ pipeline_id: props.id,+ shown: show,+ })},setWritePersonProperty: async ({ enabled, source }) => {if (!values.currentTeamId) {diff --git a/products/autoresearch/frontend/onlinePerformance.test.ts b/products/autoresearch/frontend/onlinePerformance.test.tsindex e53153c31eb9..56386f177026 100644--- a/products/autoresearch/frontend/onlinePerformance.test.ts+++ b/products/autoresearch/frontend/onlinePerformance.test.ts@@ -2,9 +2,10 @@ import { AutoresearchRunApi, ConfusionByCutoffApi, OnlinePerformanceRowApi } froimport {accuracyStats,accuracySummary,+ accuracyTrend,+ accuracyTrendSummary,calibrationBySegment,pooledConfusion,- realizedAucSeries,validatedPredictionDates,} from './onlinePerformance'@@ -20,8 +21,20 @@ function row(overrides: Partial<OnlinePerformanceRowApi>): OnlinePerformanceRowA}function counts(tp: number, fp: number, fn: number, tn: number): ConfusionByCutoffApi {- const cutoff = { tp, fp, fn, tn, n_flagged: tp + fp, precision: null, recall: null }- return { top_10: cutoff, top_20: cutoff, likely: { ...cutoff, tp: 0, fp: 0, n_flagged: 0, tn: tn + tp + fp } }+ const cutoff = {+ tp,+ fp,+ fn,+ tn,+ n_flagged: tp + fp,+ precision: tp + fp > 0 ? tp / (tp + fp) : null,+ recall: tp + fn > 0 ? tp / (tp + fn) : null,+ }+ return {+ top_10: cutoff,+ top_20: cutoff,+ likely: { ...cutoff, tp: 0, fp: 0, n_flagged: 0, tn: tn + tp + fp, precision: null, recall: 0 },+ }}describe('onlinePerformance', () => {@@ -43,20 +56,87 @@ describe('onlinePerformance', () => {])})- test('realizedAucSeries keeps the champion rows with an AUC, oldest date first', () => {- const points = realizedAucSeries([- row({ prediction_date: '2026-02-12', realized_auc: 0.82 }),- row({ prediction_date: '2026-02-12', emitted_role: 'challenger', realized_auc: 0.7 }),- row({ prediction_date: '2026-02-11', realized_auc: null }),- row({ prediction_date: '2026-02-10', realized_auc: 0.79 }),- ])+ test('accuracyTrend keeps each role on its own line, oldest date first, with gaps for missing values', () => {+ const rows = [+ row({+ prediction_date: '2026-02-12',+ model_id: 'champ',+ realized_auc: 0.82,+ confusion: counts(3, 1, 2, 94),+ }),+ row({ prediction_date: '2026-02-12', model_id: 'chal', emitted_role: 'challenger', realized_auc: 0.7 }),+ row({ prediction_date: '2026-02-11', model_id: 'champ', realized_auc: null, confusion: null }),+ row({+ prediction_date: '2026-02-10',+ model_id: 'champ',+ realized_auc: 0.79,+ confusion: counts(1, 3, 2, 94),+ }),+ row({ prediction_date: '2026-02-10', model_id: 'chal', emitted_role: 'challenger', realized_auc: 0.74 }),+ ].map((r) => ({ ...r, n_positive: 5, base_rate: 0.05 }))+ const values = (trend: ReturnType<typeof accuracyTrend>): [string, [string, number][]][] =>+ trend.series.map((s) => [s.label, s.points.map((p) => [p.date, p.value])])- expect(points.map((p) => [p.date, p.auc])).toEqual([- ['2026-02-10', 0.79],- ['2026-02-12', 0.82],+ expect(values(accuracyTrend(rows, 'ranking', 'top_10', false))).toEqual([+ [+ 'Champion',+ [+ ['2026-02-10', 0.79],+ ['2026-02-12', 0.82],+ ],+ ],+ ])+ expect(values(accuracyTrend(rows, 'ranking', 'top_10', true))[1]).toEqual([+ 'Challenger',+ [+ ['2026-02-10', 0.74],+ ['2026-02-12', 0.7],+ ],+ ])+ // Likely lift is the Likely hit rate over the base rate: 3/4 against 5%.+ expect(values(accuracyTrend(rows, 'lift', 'likely', false))).toEqual([['Champion', []]])+ const likely = rows.map((r) =>+ r.confusion ? { ...r, confusion: { ...r.confusion, likely: r.confusion.top_10 } } : r+ )+ expect(values(accuracyTrend(likely, 'lift', 'likely', false))[0][1]).toEqual([+ ['2026-02-10', expect.closeTo(0.25 / 0.05)],+ ['2026-02-12', expect.closeTo(0.75 / 0.05)],+ ])+ expect(values(accuracyTrend(rows, 'calibration', 'top_10', false)).map(([label]) => label)).toEqual([+ 'Champion score',+ 'Actual rate',])})+ test.each([+ {+ metric: 'ranking' as const,+ expected:+ 'Over 2 checked dates, realized AUC ranged from 0.790 to 0.820. It matched or beat its holdout AUC of 0.800 on 1 of 2 dates.',+ },+ {+ metric: 'hit_rate' as const,+ expected:+ 'On a typical date, 30.0% of the top 10% the model flagged did “File shared”, against 4.0% of everyone.',+ },+ {+ metric: 'calibration' as const,+ expected: "Over 2 checked dates, the champion's average score stayed within 2.0 points of the actual rate.",+ },+ ])('accuracyTrendSummary describes the champion over the checked dates for $metric', ({ metric, expected }) => {+ const rows = [+ row({ prediction_date: '2026-02-12', realized_auc: 0.82, base_rate: 0.03, mean_p_y: 0.05 }),+ row({ prediction_date: '2026-02-12', emitted_role: 'challenger', realized_auc: 0.5, mean_p_y: 0.5 }),+ row({ prediction_date: '2026-02-10', realized_auc: 0.79, base_rate: 0.05, mean_p_y: 0.04 }),+ ].map((r, index) => ({+ ...r,+ n_positive: 5,+ confusion: index === 0 ? counts(4, 6, 1, 89) : counts(2, 8, 3, 87),+ }))++ expect(accuracyTrendSummary(rows, metric, 'top_10', 'File shared', 0.8)).toEqual(expected)+ })+test('pooledConfusion sums the current champion dates and counts pre-change dates apart', () => {const rows = [row({ prediction_date: '2026-02-13', model_id: 'new', confusion: counts(3, 1, 2, 94) }),diff --git a/products/autoresearch/frontend/onlinePerformance.ts b/products/autoresearch/frontend/onlinePerformance.tsindex 15b228515c69..4acaaee78d90 100644--- a/products/autoresearch/frontend/onlinePerformance.ts+++ b/products/autoresearch/frontend/onlinePerformance.ts@@ -15,13 +15,6 @@ import {predictionSegmentDefinitions,} from './predictionSegments'-export interface RealizedAucPoint {- date: string- auc: number- low: number | null- high: number | null-}-export interface SegmentCalibration {segment: PredictionSegmentDefinitionpeople: number@@ -121,17 +114,164 @@ export function accuracyStats(row: OnlinePerformanceRowApi, target: string): Acc]}-/** The champion's realized AUC per date, oldest first, for the trend chart. */-export function realizedAucSeries(rows: OnlinePerformanceRowApi[]): RealizedAucPoint[] {- return rows- .filter((row) => row.emitted_role === 'champion' && row.realized_auc != null)- .map((row) => ({- date: row.prediction_date,- auc: row.realized_auc as number,- low: row.realized_auc_ci_low,- high: row.realized_auc_ci_high,- }))- .sort((a, b) => a.date.localeCompare(b.date))+export type AccuracyTrendMetric = 'ranking' | 'hit_rate' | 'coverage' | 'lift' | 'calibration'++export const ACCURACY_TREND_METRICS: { key: AccuracyTrendMetric; label: string }[] = [+ { key: 'ranking', label: 'Ranking (AUC)' },+ { key: 'hit_rate', label: 'Hit rate' },+ { key: 'coverage', label: 'Coverage' },+ { key: 'lift', label: 'Lift' },+ { key: 'calibration', label: 'Predicted and actual' },+]++export function trendMetricUsesCutoff(metric: AccuracyTrendMetric): boolean {+ return metric === 'hit_rate' || metric === 'coverage' || metric === 'lift'+}++export interface AccuracyTrendPoint {+ date: string+ value: number+ /** The 95% AUC interval. Null for every metric except ranking. */+ low: number | null+ high: number | null+}++export interface AccuracyTrendSeries {+ key: string+ label: string+ /** `actual` is the share of people who did the target, drawn next to the average score. */+ role: 'champion' | 'challenger' | 'actual'+ /** Oldest date first. A date the model has no value for is missing, not zero. */+ points: AccuracyTrendPoint[]+}++export interface AccuracyTrend {+ /** Every date any series has a value for, oldest first. */+ dates: string[]+ /** The champion series comes first. */+ series: AccuracyTrendSeries[]+ /** The champion's people scored and people who did the target, per date. */+ people: Record<string, { scored: number; positive: number }>+}++function liftAt(row: OnlinePerformanceRowApi, cutoff: AccuracyCutoff): number | null {+ // Lift has no meaning when no one did the target, and the backend reports 0 then.+ if (row.n_positive === 0) {+ return null+ }+ switch (cutoff) {+ case 'top_10':+ return row.lift_at_10+ case 'top_20':+ return row.lift_at_20+ case 'likely': {+ const precision = row.confusion?.likely.precision+ return precision != null && row.base_rate > 0 ? precision / row.base_rate : null+ }+ }+}++function trendPoint(+ row: OnlinePerformanceRowApi,+ metric: AccuracyTrendMetric,+ cutoff: AccuracyCutoff+): AccuracyTrendPoint | null {+ const point = (+ value: number | null | undefined,+ low: number | null = null,+ high: number | null = null+ ): AccuracyTrendPoint | null => (value != null ? { date: row.prediction_date, value, low, high } : null)+ switch (metric) {+ case 'ranking':+ return point(row.realized_auc, row.realized_auc_ci_low, row.realized_auc_ci_high)+ case 'hit_rate':+ return point(row.confusion?.[cutoff].precision)+ case 'coverage':+ return point(row.confusion?.[cutoff].recall)+ case 'lift':+ return point(liftAt(row, cutoff))+ case 'calibration':+ return point(row.mean_p_y)+ }+}++/**+ * One metric per prediction date, for the champion and, when asked, each model that emitted+ * predictions in another role. A row keeps the role its model held on that date, so a promoted+ * challenger's older dates stay on its challenger line.+ */+export function accuracyTrend(+ rows: OnlinePerformanceRowApi[],+ metric: AccuracyTrendMetric,+ cutoff: AccuracyCutoff,+ withChallengers: boolean+): AccuracyTrend {+ const oldestFirst = [...rows].sort((a, b) => a.prediction_date.localeCompare(b.prediction_date))+ const championRows = oldestFirst.filter((row) => row.emitted_role === 'champion')+ const pointsOf = (modelRows: OnlinePerformanceRowApi[]): AccuracyTrendPoint[] =>+ modelRows.flatMap((row) => trendPoint(row, metric, cutoff) ?? [])++ const series: AccuracyTrendSeries[] = [+ {+ key: 'champion',+ label: metric === 'calibration' ? 'Champion score' : 'Champion',+ role: 'champion',+ points: pointsOf(championRows),+ },+ ]+ if (withChallengers) {+ const challengerRows = oldestFirst.filter((row) => row.emitted_role !== 'champion')+ const challengers = [...new Set(challengerRows.map((row) => row.model_id))]+ .map((modelId) => pointsOf(challengerRows.filter((row) => row.model_id === modelId)))+ .filter((points) => points.length > 0)+ challengers.forEach((points, index) =>+ series.push({+ key: `challenger-${index}`,+ label: challengers.length > 1 ? `Challenger ${index + 1}` : 'Challenger',+ role: 'challenger',+ points,+ })+ )+ }+ if (metric === 'calibration') {+ series.push({+ key: 'actual',+ label: 'Actual rate',+ role: 'actual',+ points: championRows.map((row) => ({+ date: row.prediction_date,+ value: row.base_rate,+ low: null,+ high: null,+ })),+ })+ }++ return {+ dates: [...new Set(series.flatMap((s) => s.points.map((point) => point.date)))].sort(),+ series,+ people: Object.fromEntries(+ championRows.map((row) => [row.prediction_date, { scored: row.n_scored, positive: row.n_positive }])+ ),+ }+}++/** A metric value in its own unit. `precise` adds a decimal place to AUC for the per-date details. */+export function formatTrendValue(metric: AccuracyTrendMetric, value: number, precise = false): string {+ switch (metric) {+ case 'ranking':+ return value.toFixed(precise ? 3 : 2)+ case 'lift':+ return `${value.toFixed(1)}×`+ default:+ return percent(value)+ }+}++/** The gap between the average score and the actual rate, in percentage points with a sign. */+export function percentagePoints(gap: number): string {+ const points = (gap * 100).toFixed(1)+ return `${gap > 0 ? '+' : ''}${points} pts`}/**@@ -238,6 +378,71 @@ export function accuracySummary(confusion: PooledConfusion, cutoff: AccuracyCutoreturn `${period}, ${percent(confusion.precision)} of ${cutoffGroup(cutoff)} did ${eventName(target)}, against ${percent(overall)} of everyone. They include ${percent(confusion.recall)} of everyone who did it.`}+function median(values: number[]): number {+ const sorted = [...values].sort((a, b) => a - b)+ const middle = Math.floor(sorted.length / 2)+ return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2+}++/** One sentence on how the champion did on the picked metric over the checked dates. Null with no values. */+export function accuracyTrendSummary(+ rows: OnlinePerformanceRowApi[],+ metric: AccuracyTrendMetric,+ cutoff: AccuracyCutoff,+ target: string,+ holdoutAuc: number | null+): string | null {+ const checked = rows+ .filter((row) => row.emitted_role === 'champion')+ .flatMap((row) => {+ const point = trendPoint(row, metric, cutoff)+ return point ? [{ row, value: point.value }] : []+ })+ if (checked.length === 0) {+ return null+ }+ const values = checked.map(({ value }) => value)+ const typical = median(values)+ const single = checked.length === 1+ const lead = single ? `On ${dayjs(checked[0].row.prediction_date).format('MMM D')}` : 'On a typical date'+ const event = eventName(target)++ switch (metric) {+ case 'ranking': {+ if (single) {+ return `${lead}, realized AUC was ${values[0].toFixed(3)}.`+ }+ const range = `Over ${checked.length} checked dates, realized AUC ranged from ${Math.min(...values).toFixed(3)} to ${Math.max(...values).toFixed(3)}.`+ if (holdoutAuc == null) {+ return range+ }+ const holdout = holdoutAuc.toFixed(3)+ const beat = values.filter((value) => value >= holdoutAuc).length+ if (beat === 0) {+ return `${range} It stayed below its holdout AUC of ${holdout} on every date.`+ }+ const dates = beat === checked.length ? 'every date' : `${beat} of ${checked.length} dates`+ return `${range} It matched or beat its holdout AUC of ${holdout} on ${dates}.`+ }+ case 'hit_rate': {+ const everyone = median(checked.map(({ row }) => row.base_rate))+ return `${lead}, ${percent(typical)} of ${cutoffGroup(cutoff)} did ${event}, against ${percent(everyone)} of everyone.`+ }+ case 'coverage':+ return `${lead}, ${cutoffGroup(cutoff)} included ${percent(typical)} of everyone who did ${event}.`+ case 'lift':+ return `${lead}, ${cutoffGroup(cutoff)} did ${event} at ${typical.toFixed(1)}× the rate of everyone.`+ case 'calibration': {+ const gaps = checked.map(({ row, value }) => value - row.base_rate)+ if (single) {+ return `${lead}, the champion's average score was ${(Math.abs(gaps[0]) * 100).toFixed(1)} points ${gaps[0] >= 0 ? 'above' : 'below'} the actual rate.`+ }+ const widest = Math.max(...gaps.map(Math.abs))+ return `Over ${checked.length} checked dates, the champion's average score stayed within ${(widest * 100).toFixed(1)} points of the actual rate.`+ }+ }+}+/** When the first scored date matures: the earliest completed scoring run plus the horizon. */export function firstCheckDate(runs: Pick<AutoresearchRunApi, 'run_type' | 'status' | 'created_at'>[],diff --git a/products/autoresearch/frontend/pipeline/AccuracyTrendCard.tsx b/products/autoresearch/frontend/pipeline/AccuracyTrendCard.tsxnew file mode 100644index 000000000000..a4d1bc6cc5cd--- /dev/null+++ b/products/autoresearch/frontend/pipeline/AccuracyTrendCard.tsx@@ -0,0 +1,122 @@+import { useActions, useValues } from 'kea'++import { LemonSegmentedButton, LemonSwitch } from '@posthog/lemon-ui'++import { autoresearchPipelineLogic } from '../autoresearchPipelineLogic'+import {+ ACCURACY_CUTOFFS,+ ACCURACY_TREND_METRICS,+ AccuracyTrendMetric,+ accuracyTrendSummary,+ trendMetricUsesCutoff,+} from '../onlinePerformance'+import { eventName } from '../pipelineQuestion'+import { AccuracyTrendChart } from './AccuracyTrendChart'+import { AccuracyTrendDatePanel } from './AccuracyTrendDatePanel'+import { SectionCard } from './SectionCard'++function description(metric: AccuracyTrendMetric, target: string): string {+ switch (metric) {+ case 'ranking':+ return `How well the model ranked people who did ${target} above people who did not, on each checked date. 0.5 is a coin flip and 1 is a perfect ranking. The dashed line is the holdout AUC the model got in training.`+ case 'hit_rate':+ return `The share of flagged people who did ${target}, on each checked date.`+ case 'coverage':+ return `The share of everyone who did ${target} that the model flagged, on each checked date.`+ case 'lift':+ return `How many times more often flagged people did ${target} than everyone scored, on each checked date.`+ case 'calibration':+ return `The average score the model gave against the share of people who really did ${target}. When the lines stay close, you can read the scores as probabilities.`+ }+}++/** The picked online metric per prediction date, for the champion and optionally its challengers. */+export function AccuracyTrendCard(): JSX.Element {+ const {+ pipeline,+ champion,+ onlinePerformance,+ accuracyTrend,+ accuracyTrendMetric: metric,+ accuracyCutoff,+ showChallengersInTrend,+ hasChallengerPerformance,+ hoveredAccuracyDate,+ } = useValues(autoresearchPipelineLogic)+ const { setAccuracyTrendMetric, setAccuracyCutoff, setShowChallengersInTrend, setHoveredAccuracyDate } =+ useActions(autoresearchPipelineLogic)+ const targetEvent = pipeline?.target_event ?? ''+ const holdoutAuc = champion?.holdout_score ?? null+ const { dates } = accuracyTrend+ const panelDate =+ hoveredAccuracyDate && dates.includes(hoveredAccuracyDate) ? hoveredAccuracyDate : dates[dates.length - 1]+ const summary = accuracyTrendSummary(onlinePerformance, metric, accuracyCutoff, targetEvent, holdoutAuc)++ return (+ <SectionCard+ title="Accuracy over time"+ description={description(metric, eventName(targetEvent))}+ actions={+ <LemonSwitch+ label="Compare challengers"+ checked={showChallengersInTrend}+ onChange={setShowChallengersInTrend}+ disabledReason={+ !hasChallengerPerformance ? 'No challenger has been checked against real outcomes yet.' : null+ }+ bordered+ size="small"+ data-attr="autoresearch-model-accuracy-trend-challengers"+ />+ }+ >+ <div className="flex flex-wrap items-center justify-between gap-2">+ <LemonSegmentedButton+ size="small"+ value={metric}+ onChange={setAccuracyTrendMetric}+ options={ACCURACY_TREND_METRICS.map(({ key, label }) => ({+ value: key,+ label,+ 'data-attr': `autoresearch-model-accuracy-trend-metric-${key}`,+ }))}+ />+ {trendMetricUsesCutoff(metric) && (+ <LemonSegmentedButton+ size="small"+ value={accuracyCutoff}+ onChange={(cutoff) => setAccuracyCutoff(cutoff, 'trend')}+ options={ACCURACY_CUTOFFS.map(({ key, label }) => ({+ value: key,+ label,+ 'data-attr': `autoresearch-model-accuracy-trend-cutoff-${key}`,+ }))}+ />+ )}+ </div>+ {accuracyTrend.series[0].points.length === 0 || !panelDate ? (+ <p className="text-sm text-muted mb-0">+ {metric === 'ranking'+ ? `No checked date had both people who did ${eventName(targetEvent)} and people who did not, so there is no ranking score yet.`+ : 'This metric starts from the next checked date. Dates checked before it existed have no value.'}+ </p>+ ) : (+ <div className="@container">+ <div className="flex flex-col @3xl:flex-row gap-4">+ <div className="flex-1 min-w-0">+ <AccuracyTrendChart+ trend={accuracyTrend}+ metric={metric}+ holdoutAuc={holdoutAuc}+ hoveredDate={hoveredAccuracyDate}+ onHover={setHoveredAccuracyDate}+ />+ </div>+ <AccuracyTrendDatePanel trend={accuracyTrend} metric={metric} date={panelDate} />+ </div>+ </div>+ )}+ {summary && <p className="text-sm mb-0">{summary}</p>}+ </SectionCard>+ )+}diff --git a/products/autoresearch/frontend/pipeline/AccuracyTrendChart.tsx b/products/autoresearch/frontend/pipeline/AccuracyTrendChart.tsxnew file mode 100644index 000000000000..ebd38abfaf71--- /dev/null+++ b/products/autoresearch/frontend/pipeline/AccuracyTrendChart.tsx@@ -0,0 +1,175 @@+import { dayjs } from 'lib/dayjs'++import { AccuracyTrend, AccuracyTrendMetric, AccuracyTrendPoint, formatTrendValue } from '../onlinePerformance'+import { trendLineStyles } from './accuracyTrendStyle'++const WIDTH = 600+const HEIGHT = 160++/** Runs of points on consecutive dates. A date a series has no value for breaks its line instead of joining across it. */+function lineRuns(points: AccuracyTrendPoint[], dateIndex: Map<string, number>): { index: number; value: number }[][] {+ const runs: { index: number; value: number }[][] = []+ let previous = -2+ for (const point of points) {+ const index = dateIndex.get(point.date) as number+ if (index !== previous + 1) {+ runs.push([])+ }+ runs[runs.length - 1].push({ index, value: point.value })+ previous = index+ }+ return runs+}++/** The y range: every value plus the reference line, padded, and kept inside 0 to 1 for metrics that are shares. */+function yDomain(trend: AccuracyTrend, metric: AccuracyTrendMetric, reference: number | null): [number, number] {+ const values = trend.series.flatMap((s) =>+ s.points.flatMap((p) => (s.role === 'champion' ? [p.value, p.low ?? p.value, p.high ?? p.value] : [p.value]))+ )+ if (reference != null) {+ values.push(reference)+ }+ const min = Math.min(...values)+ const max = Math.max(...values)+ const pad = (max - min) * 0.1 || Math.abs(max) * 0.1 || 0.01+ return [Math.max(min - pad, 0), metric === 'lift' ? max + pad : Math.min(max + pad, 1)]+}++/** One metric per prediction date, a line per model, with the AUC interval and holdout AUC in ranking mode. No chart deps, like MetricSparkline. */+export function AccuracyTrendChart({+ trend,+ metric,+ holdoutAuc,+ hoveredDate,+ onHover,+}: {+ trend: AccuracyTrend+ metric: AccuracyTrendMetric+ holdoutAuc: number | null+ hoveredDate: string | null+ onHover: (date: string | null) => void+}): JSX.Element {+ const { dates, series } = trend+ const styles = trendLineStyles(series)+ const reference = metric === 'ranking' ? holdoutAuc : null+ const [yMin, yMax] = yDomain(trend, metric, reference)+ const dateIndex = new Map(dates.map((date, index) => [date, index]))+ const x = (index: number): number => (dates.length === 1 ? WIDTH / 2 : (index * WIDTH) / (dates.length - 1))+ const y = (value: number): number => HEIGHT - ((value - yMin) / (yMax - yMin)) * HEIGHT+ const columnWidth = dates.length === 1 ? WIDTH : WIDTH / (dates.length - 1)+ const hoveredIndex = hoveredDate != null ? dateIndex.get(hoveredDate) : undefined++ const champion = series[0]+ const banded = metric === 'ranking' ? champion.points.filter((p): boolean => p.low != null && p.high != null) : []+ const band = [+ ...banded.map((p) => `${x(dateIndex.get(p.date) as number)},${y(p.high as number)}`),+ ...[...banded].reverse().map((p) => `${x(dateIndex.get(p.date) as number)},${y(p.low as number)}`),+ ].join(' ')++ return (+ <div className="space-y-1 min-w-0">+ {(banded.length > 0 || reference != null) && (+ <div className="flex flex-wrap items-baseline gap-x-4 gap-y-1 text-xs text-muted">+ {banded.length > 0 && (+ <span>+ <span className="inline-block w-3 h-2 align-middle mr-1 bg-[var(--data-color-1)] opacity-20" />+ Champion 95% interval+ </span>+ )}+ {reference != null && (+ <span>+ <span className="inline-block w-3 align-middle mr-1 border-t border-dashed border-current" />+ Holdout AUC {reference.toFixed(2)}+ </span>+ )}+ </div>+ )}+ <div className="grid grid-cols-[auto_minmax(0,1fr)] gap-x-2 gap-y-1">+ <div className="flex flex-col justify-between text-xs text-muted tabular-nums h-40 text-right">+ <span>{formatTrendValue(metric, yMax)}</span>+ <span>{formatTrendValue(metric, yMin)}</span>+ </div>+ <svg+ viewBox={`0 0 ${WIDTH} ${HEIGHT}`}+ preserveAspectRatio="none"+ className="w-full h-40 overflow-visible border-b border-l"+ role="img"+ aria-label="Accuracy over prediction dates"+ onMouseLeave={() => onHover(null)}+ >+ {banded.length > 0 && <polygon points={band} className="fill-[var(--data-color-1)] opacity-20" />}+ {reference != null && (+ <line+ x1={0}+ x2={WIDTH}+ y1={y(reference)}+ y2={y(reference)}+ className="stroke-current text-secondary"+ strokeDasharray="6 4"+ vectorEffect="non-scaling-stroke"+ />+ )}+ {hoveredIndex != null && (+ <line+ x1={x(hoveredIndex)}+ x2={x(hoveredIndex)}+ y1={0}+ y2={HEIGHT}+ className="stroke-current text-muted"+ vectorEffect="non-scaling-stroke"+ />+ )}+ {[...series].reverse().map((s, reversedIndex) => {+ const style = styles[series.length - 1 - reversedIndex]+ return (+ <g key={s.key} className={style.stroke}>+ {lineRuns(s.points, dateIndex).map((run) => (+ <polyline+ key={run[0].index}+ points={run.map((p) => `${x(p.index)},${y(p.value)}`).join(' ')}+ fill="none"+ strokeWidth={2}+ strokeDasharray={style.dash}+ vectorEffect="non-scaling-stroke"+ />+ ))}+ {s.points.map((p) => {+ const index = dateIndex.get(p.date) as number+ return (+ // A zero-length line with round caps stays a round dot when the SVG stretches.+ <line+ key={p.date}+ x1={x(index)}+ x2={x(index)}+ y1={y(p.value)}+ y2={y(p.value)}+ strokeWidth={s.role === 'actual' ? 4 : 6}+ strokeLinecap="round"+ vectorEffect="non-scaling-stroke"+ />+ )+ })}+ </g>+ )+ })}+ {dates.map((date, index) => (+ <rect+ key={date}+ x={x(index) - columnWidth / 2}+ y={0}+ width={columnWidth}+ height={HEIGHT}+ fill="transparent"+ onMouseEnter={() => onHover(date)}+ />+ ))}+ </svg>+ <div />+ <div className="flex justify-between text-xs text-muted">+ <span>{dayjs(dates[0]).format('MMM D')}</span>+ {dates.length > 1 && <span>{dayjs(dates[dates.length - 1]).format('MMM D')}</span>}+ </div>+ </div>+ </div>+ )+}diff --git a/products/autoresearch/frontend/pipeline/AccuracyTrendDatePanel.tsx b/products/autoresearch/frontend/pipeline/AccuracyTrendDatePanel.tsxnew file mode 100644index 000000000000..c3180ca88e9b--- /dev/null+++ b/products/autoresearch/frontend/pipeline/AccuracyTrendDatePanel.tsx@@ -0,0 +1,61 @@+import { dayjs } from 'lib/dayjs'++import { AccuracyTrend, AccuracyTrendMetric, formatTrendValue, percentagePoints } from '../onlinePerformance'+import { TrendLineStyle, trendLineStyles } from './accuracyTrendStyle'++function LineSample({ style }: { style: TrendLineStyle }): JSX.Element {+ return (+ <svg viewBox="0 0 16 4" className={`w-4 h-1 shrink-0 ${style.stroke}`} aria-hidden>+ <line x1={0} x2={16} y1={2} y2={2} strokeWidth={2} strokeDasharray={style.dash} />+ </svg>+ )+}++/** Each line's value on one date, with the AUC interval or the gap to the actual rate, and the people checked. */+export function AccuracyTrendDatePanel({+ trend,+ metric,+ date,+}: {+ trend: AccuracyTrend+ metric: AccuracyTrendMetric+ date: string+}): JSX.Element {+ const styles = trendLineStyles(trend.series)+ const actual = trend.series.find((s) => s.role === 'actual')?.points.find((p) => p.date === date)+ const people = trend.people[date]++ return (+ <div className="w-full @3xl:w-64 shrink-0 rounded border p-3 space-y-2 text-xs">+ <div className="text-muted">Scored on {dayjs(date).format('MMM D, YYYY')}</div>+ {trend.series.map((s, index) => {+ const point = s.points.find((p) => p.date === date)+ const detail =+ point == null+ ? null+ : metric === 'ranking' && point.low != null && point.high != null+ ? `95% interval ${point.low.toFixed(3)} to ${point.high.toFixed(3)}`+ : s.role !== 'actual' && actual != null+ ? `${percentagePoints(point.value - actual.value)} against the actual rate`+ : null+ return (+ <div key={s.key} className="min-w-0">+ <div className="flex items-center gap-2">+ <LineSample style={styles[index]} />+ <span className="flex-1 min-w-0 truncate">{s.label}</span>+ <span className="font-semibold tabular-nums">+ {point ? formatTrendValue(metric, point.value, true) : '—'}+ </span>+ </div>+ {detail && <div className="text-secondary pl-6">{detail}</div>}+ </div>+ )+ })}+ {people && (+ <div className="text-secondary border-t pt-2">+ {people.scored.toLocaleString()} scored, {people.positive.toLocaleString()} did it+ </div>+ )}+ </div>+ )+}diff --git a/products/autoresearch/frontend/pipeline/ConfusionMatrixCard.tsx b/products/autoresearch/frontend/pipeline/ConfusionMatrixCard.tsxindex 04025521c7f0..d912182ee72e 100644--- a/products/autoresearch/frontend/pipeline/ConfusionMatrixCard.tsx+++ b/products/autoresearch/frontend/pipeline/ConfusionMatrixCard.tsx@@ -58,9 +58,12 @@ export function ConfusionMatrixCard(): JSX.Element {<LemonSegmentedButtonsize="small"value={accuracyCutoff}- onChange={setAccuracyCutoff}- options={ACCURACY_CUTOFFS.map(({ key, label }) => ({ value: key, label }))}- data-attr="autoresearch-model-accuracy-cutoff"+ onChange={(cutoff) => setAccuracyCutoff(cutoff, 'flagged')}+ options={ACCURACY_CUTOFFS.map(({ key, label }) => ({+ value: key,+ label,+ 'data-attr': `autoresearch-model-accuracy-cutoff-${key}`,+ }))}/>}>diff --git a/products/autoresearch/frontend/pipeline/OnlinePerformanceTab.tsx b/products/autoresearch/frontend/pipeline/OnlinePerformanceTab.tsxindex a1594ec3e93c..1acc3624a2fc 100644--- a/products/autoresearch/frontend/pipeline/OnlinePerformanceTab.tsx+++ b/products/autoresearch/frontend/pipeline/OnlinePerformanceTab.tsx@@ -9,11 +9,11 @@ import { pluralize } from 'lib/utils/strings'import { autoresearchPipelineLogic } from '../autoresearchPipelineLogic'import { accuracyStats, accuracySummary, percent } from '../onlinePerformance'import { eventName } from '../pipelineQuestion'+import { AccuracyTrendCard } from './AccuracyTrendCard'import { ConfusionMatrixCard } from './ConfusionMatrixCard'import { EmptyTab } from './EmptyTab'import { LiveModelDrivers } from './LiveModelDrivers'import { MetricCard } from './MetricCard'-import { RealizedAucChart } from './RealizedAucChart'import { ScoreNowButton } from './ScoreNowButton'import { SectionCard } from './SectionCard'import { SegmentCalibrationBars } from './SegmentCalibrationBars'@@ -50,12 +50,10 @@ function NotCheckedYet(): JSX.Element {export function OnlinePerformanceTab(): JSX.Element {const {pipeline,- champion,onlinePerformance,onlinePerformanceLoading,onlinePerformanceError,latestChampionPerformance,- realizedAucPoints,segmentCalibration,accuracyCutoff,championConfusion,@@ -115,20 +113,7 @@ export function OnlinePerformanceTab(): JSX.Element {</SectionCard>)}- {realizedAucPoints.length > 0 && (- <SectionCard- title="Realized AUC over time"- description={- <>- Realized AUC measures how well the model ranked people who did {target} above people who did- not. 0.5 is a coin flip and 1 is a perfect ranking. The holdout line is the score the model- got in training.- </>- }- >- <RealizedAucChart points={realizedAucPoints} holdoutAuc={champion?.holdout_score ?? null} />- </SectionCard>- )}+ <AccuracyTrendCard /><LiveModelDrivers />diff --git a/products/autoresearch/frontend/pipeline/RealizedAucChart.tsx b/products/autoresearch/frontend/pipeline/RealizedAucChart.tsxdeleted file mode 100644index c443567659f3..000000000000--- a/products/autoresearch/frontend/pipeline/RealizedAucChart.tsx+++ /dev/null@@ -1,107 +0,0 @@-import { dayjs } from 'lib/dayjs'--import { RealizedAucPoint } from '../onlinePerformance'--const WIDTH = 600-const HEIGHT = 160-const Y_MIN = 0.5-const Y_MAX = 1--/** Champion realized AUC per prediction date, with its 95% interval and the holdout AUC for reference. No chart deps, like MetricSparkline. */-export function RealizedAucChart({- points,- holdoutAuc,-}: {- points: RealizedAucPoint[]- holdoutAuc: number | null-}): JSX.Element {- const x = (index: number): number => (points.length === 1 ? WIDTH / 2 : (index * WIDTH) / (points.length - 1))- const y = (value: number): number =>- HEIGHT - ((Math.min(Math.max(value, Y_MIN), Y_MAX) - Y_MIN) / (Y_MAX - Y_MIN)) * HEIGHT- const banded = points.map((point, index) => ({ ...point, index })).filter((p) => p.low != null && p.high != null)- const band = [- ...banded.map((p) => `${x(p.index)},${y(p.high as number)}`),- ...[...banded].reverse().map((p) => `${x(p.index)},${y(p.low as number)}`),- ].join(' ')- const line = points.map((p, index) => `${x(index)},${y(p.auc)}`).join(' ')- const latest = points[points.length - 1]-- return (- <div className="space-y-1 max-w-4xl">- <div className="flex flex-wrap items-baseline gap-x-4 gap-y-1 text-xs text-muted">- <span>- <span className="inline-block w-3 h-0.5 align-middle mr-1 bg-[var(--data-color-1)]" />- Realized AUC, latest {latest.auc.toFixed(2)}- </span>- <span>- <span className="inline-block w-3 h-2 align-middle mr-1 bg-[var(--data-color-1)] opacity-20" />- 95% interval- </span>- {holdoutAuc != null && (- <span>- <span className="inline-block w-3 align-middle mr-1 border-t border-dashed border-current" />- Holdout AUC {holdoutAuc.toFixed(2)}- </span>- )}- </div>- <div className="flex gap-2">- <div className="flex flex-col justify-between text-xs text-muted tabular-nums h-40">- <span>{Y_MAX.toFixed(1)}</span>- <span>{Y_MIN.toFixed(1)}</span>- </div>- <svg- viewBox={`0 0 ${WIDTH} ${HEIGHT}`}- preserveAspectRatio="none"- className="flex-1 min-w-0 h-40 overflow-visible border-b border-l"- role="img"- aria-label="Realized AUC over prediction dates"- >- {banded.length > 0 && <polygon points={band} className="fill-[var(--data-color-1)] opacity-20" />}- {holdoutAuc != null && (- <line- x1={0}- x2={WIDTH}- y1={y(holdoutAuc)}- y2={y(holdoutAuc)}- className="stroke-current text-secondary"- strokeDasharray="6 4"- vectorEffect="non-scaling-stroke"- />- )}- <polyline- points={line}- fill="none"- className="stroke-[var(--data-color-1)]"- strokeWidth={2}- vectorEffect="non-scaling-stroke"- />- {points.map((p, index) => (- // A zero-length line with round caps stays a round dot when the SVG stretches.- <line- key={p.date}- x1={x(index)}- x2={x(index)}- y1={y(p.auc)}- y2={y(p.auc)}- className="stroke-[var(--data-color-1)]"- strokeWidth={6}- strokeLinecap="round"- vectorEffect="non-scaling-stroke"- >- <title>- {dayjs(p.date).format('MMM D, YYYY')}: AUC {p.auc.toFixed(3)}- {p.low != null && p.high != null- ? ` (95% interval ${p.low.toFixed(3)} to ${p.high.toFixed(3)})`- : ''}- </title>- </line>- ))}- </svg>- </div>- <div className="flex justify-between text-xs text-muted pl-6">- <span>{dayjs(points[0].date).format('MMM D')}</span>- {points.length > 1 && <span>{dayjs(latest.date).format('MMM D')}</span>}- </div>- </div>- )-}diff --git a/products/autoresearch/frontend/pipeline/accuracyTrendStyle.ts b/products/autoresearch/frontend/pipeline/accuracyTrendStyle.tsnew file mode 100644index 000000000000..4070b8175b48--- /dev/null+++ b/products/autoresearch/frontend/pipeline/accuracyTrendStyle.ts@@ -0,0 +1,36 @@+import { AccuracyTrendSeries } from '../onlinePerformance'++export interface TrendLineStyle {+ /** Tailwind classes that set the stroke color. */+ stroke: string+ dash: string | undefined+}++// Static class names, so Tailwind keeps them.+const CHALLENGER_STROKES = [+ 'stroke-[var(--data-color-2)]',+ 'stroke-[var(--data-color-3)]',+ 'stroke-[var(--data-color-4)]',+ 'stroke-[var(--data-color-5)]',+]+const CHALLENGER_DASHES = ['6 4', '2 3']++/** One style per series, in series order. Lines differ by dash as well as color, so they stay apart without color. */+export function trendLineStyles(series: AccuracyTrendSeries[]): TrendLineStyle[] {+ let challengerIndex = 0+ return series.map((s) => {+ switch (s.role) {+ case 'champion':+ return { stroke: 'stroke-[var(--data-color-1)]', dash: undefined }+ case 'actual':+ return { stroke: 'stroke-current text-secondary', dash: '6 4' }+ case 'challenger': {+ const index = challengerIndex+++ return {+ stroke: CHALLENGER_STROKES[index % CHALLENGER_STROKES.length],+ dash: CHALLENGER_DASHES[index % CHALLENGER_DASHES.length],+ }+ }+ }+ })+}diff --git a/products/autoresearch/package.json b/products/autoresearch/package.jsonindex 9c463861b2d0..d18053d12678 100644--- a/products/autoresearch/package.json+++ b/products/autoresearch/package.json@@ -23,6 +23,8 @@"@posthog/brand": "catalog:","@posthog/icons": "catalog:","@posthog/lemon-ui": "*",+ "@testing-library/dom": "*",+ "@testing-library/user-event": "*","@types/react": "*","react": "*"}