:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): tighten the eval create flow and eval page (#129476)
Bernat Torres9fd797ede37b
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex dd98616477c8..bada9c3f6361 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -9227,17 +9227,17 @@ snapshots:scenes-app-ai-observability-new-evaluation--from-scratch--light:hash: v1.k794b7964.816d9e830ebfeb8ac1303b3b59efe94150a1d2371cb78e3338153f176f521444.XhZKS15jlRkqnRcA2vpvX4Fs7ZEuvE23jm48lIAMqGEscenes-app-ai-observability-new-evaluation--from-template--dark:- hash: v1.k794b7964.522cb4a66474488f0a1a5c64f26c09f1beda25de9d3025a7d9d708a5b638fb5e.FTdDwtnvQSREwVqU1obRvn4JMO2BZIA6_hQq53n1gIY+ hash: v1.k794b7964.07380f3dfb3e23227666db25699716cb64a1ad87908c2f44ee8526d727195662.oibYuMDeIcQcBB8-xT_4kUNOi9WaFpEX734xD-sLBQYscenes-app-ai-observability-new-evaluation--from-template--light:- hash: v1.k794b7964.2293fbca6fae992a9b19200cc73bf6ac0a9c06056d24cc4b3ccbcb6a592a6cd5.FGIm1AeCqfdM9Py8enk2QX65UJr1tn8ZlUAlAIG2bUA+ hash: v1.k794b7964.9c1d8e6148d26b50a85c618819388cc6108f52d5c6d71e40d661f6d403994e63.shdmDJxZKXSPIOJBz_AHBjJ-mDZgf3E0SJjRK7MP0VEscenes-app-ai-observability-new-evaluation--nothing-matches--dark:- hash: v1.k794b7964.9042eb2d03e3fad492dfdfc755ffe45d651856cf53a23c1b126820af2af4a157.dsssE0qpI7l2eda6yDFu7etQyalm08flGpQ5pF8Xbs4+ hash: v1.k794b7964.8de1c6adc167189439f173be655c9ba621d431c569d490186e2edb21b5adab9a.y68wObiGXDm_gk9dpuMWLRhMgcLq9W8w2O3BR23qYzgscenes-app-ai-observability-new-evaluation--nothing-matches--light:- hash: v1.k794b7964.e57e16dfee3e4b83740a8d14a8ce425fa22a2bed2410c9c82f8308ab83bf5a81.bMBzNe0faqqHTUOVJ3CMGDuHtbEEw7nRqfcNXQVqgIs+ hash: v1.k794b7964.19005b94e6654d7728de11e62a833dd04ca2232cf230a0ae15115a5ea803b631.DazOYwUIoHXTkv5XSJlVGspprGh0ALMDe_SxkhllWt8scenes-app-ai-observability-new-evaluation--reports-open--dark:- hash: v1.k794b7964.f38dbca5c0f4a4666976c917d4f027924f1dc0d1550775c38654360889cbb23c.byoQrLQhAAMUFDzn9EJZVZPcMCP5MiyIfHEgHD9K-pE+ hash: v1.k794b7964.7dd06c6e687d65fb2d75ed92dbc344ba3f9277ba17a0107d7f6eb410d31bd721.V9xJdZjH3bGMs9ZxKTtpZ5YJ58Mm1zzcz_UGRpjJZ3Mscenes-app-ai-observability-new-evaluation--reports-open--light:- hash: v1.k794b7964.421b25300229d08d96da3d1f65a030df1365b26e795695d85478539543e58682._Yjl0PGso1sxk98zetDxwSc6PZoc_uRl637y0px88GA+ hash: v1.k794b7964.7679f6ae83d3c6db477d8be83a813390bdeffda70944a9b2b3fd93d5c1220bed.mmWadDp5K9oYJGHsYiDnkfu5wDAXFYOqsPTqrgwYT7Uscenes-app-ai-observability-new-evaluation--sentiment-evaluation--dark:hash: v1.k794b7964.ac47d8aaeb570a78393cc59504adfa570a40a1a544ace57840c0dae235680603.4O7QEnavU1PIlLZw-fCCE7TO-6r3nrqU2WG0_02WyKcscenes-app-ai-observability-new-evaluation--sentiment-evaluation--light:diff --git a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsxindex fdcaa5dbccc5..d61c691aa29d 100644--- a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx+++ b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx@@ -1,8 +1,7 @@import { useActions, useValues } from 'kea'import { combineUrl, router } from 'kea-router'-import { IconArrowLeft, IconPlay, IconTrends, IconWarning } from '@posthog/icons'-import { LemonBanner, LemonButton, LemonSkeleton, LemonTabs, LemonTag, Link } from '@posthog/lemon-ui'+import { LemonBanner, LemonButton, LemonSkeleton, LemonTabs, Link } from '@posthog/lemon-ui'import { AccessControlAction } from 'lib/components/AccessControlAction'import { NotFound } from 'lib/components/NotFound'@@ -11,8 +10,6 @@ import { lemonToast } from 'lib/lemon-ui/LemonToast'import { pluralize } from 'lib/utils/strings'import { SceneExport } from 'scenes/sceneTypes'-import { SceneBreadcrumbBackButton } from '~/layout/scenes/components/SceneBreadcrumbs'-import { SceneStickyBar } from '~/layout/scenes/components/SceneStickyBar'import { urls } from '~/scenes/urls'import { AccessControlLevel, AccessControlResourceType } from '~/types'@@ -26,6 +23,7 @@ import { EvaluationLatestReport } from './components/EvaluationLatestReport'import { EvaluationReportsCallout } from './components/EvaluationReportsCallout'import { EvaluationReportsTab } from './components/EvaluationReportsTab'import { EvaluationRunsTable } from './components/EvaluationRunsTable'+import { EvaluationSceneHeader } from './components/EvaluationSceneHeader'import {formatNumericEvaluationScore,booleanOutputConfigError,@@ -52,7 +50,6 @@ export function AIObservabilityEvaluation(): JSX.Element {evaluation,originalEvaluation,isReportableEvaluation,- trendInsightUrl,evaluationBackTarget,evaluationLoading,evaluationFormSubmitting,@@ -182,70 +179,14 @@ export function AIObservabilityEvaluation(): JSX.Element {return (<div className="@container/evaluation min-w-0 space-y-6 pb-8">- <SceneBreadcrumbBackButton />- {/* Header */}- <SceneStickyBar- hasSceneTitleSection={false}- className="flex flex-col justify-between items-start gap-4 space-y-0 @min-[64rem]/evaluation:flex-row"- >- <div className="space-y-2 min-w-0">- <h1 className="text-2xl font-semibold break-words">{evaluation.name}</h1>- <div className="flex flex-wrap items-center gap-2">- {evaluation.status === 'error' ? (- <LemonTag type="danger" icon={<IconWarning />}>- Error- </LemonTag>- ) : (- <LemonTag type={evaluation.enabled ? 'success' : 'default'}>- {evaluation.enabled ? 'Enabled' : 'Disabled'}- </LemonTag>- )}- {unsaved && <LemonTag type="warning">Unsaved changes</LemonTag>}- </div>- </div>- <div className="flex flex-wrap gap-2 max-w-full">- {trendInsightUrl ? (- <LemonButton- type="secondary"- icon={<IconTrends />}- to={trendInsightUrl}- targetBlank- data-attr="llma-evaluation-trend-insight"- >- Trend insight- </LemonButton>- ) : null}- {openInPlaygroundUrl ? (- <LemonButton- type="secondary"- icon={<IconPlay />}- to={openInPlaygroundUrl}- data-attr="llma-playground-open-from-evaluation"- >- Open in Playground- </LemonButton>- ) : null}- <LemonButton type="secondary" icon={<IconArrowLeft />} onClick={handleCancel}>- {unsaved ? 'Cancel' : 'Back'}- </LemonButton>- {showSaveAction && (- <AccessControlAction- resourceType={AccessControlResourceType.Evaluation}- minAccessLevel={AccessControlLevel.Editor}- userAccessLevel={evaluation.user_access_level ?? undefined}- >- <LemonButton- type="primary"- onClick={handleSave}- disabledReason={saveButtonDisabledReason}- loading={evaluationFormSubmitting}- >- Save changes- </LemonButton>- </AccessControlAction>- )}- </div>- </SceneStickyBar>+ <EvaluationSceneHeader+ playgroundUrl={openInPlaygroundUrl}+ unsaved={unsaved}+ showSaveAction={showSaveAction}+ saveDisabledReason={saveButtonDisabledReason}+ onSave={handleSave}+ onCancel={handleCancel}+ />{evaluation.status === 'error' && (<LemonBanner type="error">diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationAgentActions.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationAgentActions.tsxindex 87c9b2d28c0d..ebe042e7b763 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationAgentActions.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationAgentActions.tsx@@ -65,6 +65,7 @@ export function EvaluationAgentActions(): JSX.Element | null {actions={[{ key: 'prompt', label: 'prompt', buildPrompt: () => prompt }]}labelMode="destination"appearance="lemon"+ excludeAgentKeys={['posthog-code']}onRun={({ agentKey }) => reportAgentOpened(SURFACE_KEY, 'card', agentKey)}data-attr="mcp-hint-card-agent-button"/>diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationFirstResultStatus.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationFirstResultStatus.tsxindex f7ccf09a96cf..2f941ea4f5d1 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationFirstResultStatus.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationFirstResultStatus.tsx@@ -3,8 +3,11 @@ import { BindLogic, useActions, useMountedLogic, useValues } from 'kea'import { LemonBanner } from '@posthog/lemon-ui'import { dayjs } from 'lib/dayjs'+import { getAccessControlDisabledReason } from 'lib/utils/accessControlUtils'import { humanFriendlyDuration } from 'lib/utils/durations'+import { AccessControlLevel, AccessControlResourceType } from '~/types'+import { settleStrategy } from '../constants'import { VOLUME_LOOKBACK_DAYS, evaluationTarget, runRate } from '../evaluationVolume'import { evaluationVolumeLogic } from '../evaluationVolumeLogic'@@ -47,19 +50,27 @@ export function EvaluationFirstResultStatus(): JSX.Element | null {}function PausedBanner({ evaluation }: { evaluation: EvaluationConfig }): JSX.Element {- const { setActiveTab } = useActions(llmEvaluationLogic)+ const { enabledSaving, enableSavedReason } = useValues(llmEvaluationLogic)+ const { saveEnabled } = useActions(llmEvaluationLogic)+ const accessReason = getAccessControlDisabledReason(+ AccessControlResourceType.Evaluation,+ AccessControlLevel.Editor,+ evaluation.user_access_level ?? undefined+ )return (<LemonBannertype="info"className="mb-4"action={{- children: 'Open configuration',- onClick: () => setActiveTab('configuration'),- 'data-attr': 'llma-evaluation-first-result-triggers',+ children: 'Turn on',+ onClick: () => saveEnabled(true),+ loading: enabledSaving,+ disabledReason: accessReason ?? enableSavedReason ?? undefined,+ 'data-attr': 'llma-evaluation-first-result-turn-on',}}>- <span className="font-semibold">No results yet, because this evaluation is paused.</span> Turn it on in its- configuration to start checking new {UNIT_NOUNS[evaluationTarget(evaluation)]}.+ <span className="font-semibold">No results yet, because this evaluation is paused.</span> Turn it on to+ start checking new {UNIT_NOUNS[evaluationTarget(evaluation)]}.</LemonBanner>)}diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationRunsTable.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationRunsTable.tsxindex 3323a73d1785..1e573b680616 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationRunsTable.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationRunsTable.tsx@@ -1,6 +1,6 @@import { useActions, useValues } from 'kea'-import { IconRefresh } from '@posthog/icons'+import { IconRefresh, IconTrends } from '@posthog/icons'import { LemonBanner, LemonButton, LemonSegmentedButton, LemonTable, LemonTag } from '@posthog/lemon-ui'import { DateFilter } from 'lib/components/DateFilter/DateFilter'@@ -52,6 +52,25 @@ function SentimentEvaluationRunsFilters(): JSX.Element {)}+function TrendInsightButton(): JSX.Element | null {+ const { trendInsightUrl } = useValues(llmEvaluationLogic)+ if (!trendInsightUrl) {+ return null+ }+ return (+ <LemonButton+ type="secondary"+ icon={<IconTrends />}+ to={trendInsightUrl}+ targetBlank+ size="small"+ data-attr="llma-evaluation-trend-insight"+ >+ Trend insight+ </LemonButton>+ )+}+export function EvaluationRunsTable(): JSX.Element {const {filteredEvaluationRuns,@@ -207,6 +226,7 @@ export function EvaluationRunsTable(): JSX.Element {)}</>)}+ <TrendInsightButton /><LemonButtontype="secondary"icon={<IconRefresh />}diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationSceneHeader.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationSceneHeader.tsxnew file mode 100644index 000000000000..f753676b20e1--- /dev/null+++ b/products/ai_observability/frontend/evaluations/components/EvaluationSceneHeader.tsx@@ -0,0 +1,146 @@+import { useActions, useValues } from 'kea'++import { IconPause, IconPlay, IconWarning } from '@posthog/icons'+import { LemonButton, LemonTag } from '@posthog/lemon-ui'++import { AccessControlAction } from 'lib/components/AccessControlAction'+import { FEATURE_FLAGS } from 'lib/constants'+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'++import { SceneTitleSection } from '~/layout/scenes/components/SceneTitleSection'+import { AccessControlLevel, AccessControlResourceType } from '~/types'++import { llmEvaluationLogic } from '../llmEvaluationLogic'+import { EvaluationSceneMenuBar } from './EvaluationSceneMenuBar'++function StatusTags({+ errored,+ running,+ unsaved,+}: {+ errored: boolean+ running: boolean+ unsaved: boolean+}): JSX.Element {+ return (+ <>+ {errored ? (+ <LemonTag type="danger" icon={<IconWarning />}>+ Error+ </LemonTag>+ ) : (+ <LemonTag type={running ? 'success' : 'default'}>{running ? 'Running' : 'Paused'}</LemonTag>+ )}+ {unsaved && <LemonTag type="warning">Unsaved changes</LemonTag>}+ </>+ )+}++function RunStateButton({+ running,+ accessLevel,+}: {+ running: boolean+ accessLevel: AccessControlLevel | undefined+}): JSX.Element {+ const { enabledSaving, enableSavedReason } = useValues(llmEvaluationLogic)+ const { saveEnabled } = useActions(llmEvaluationLogic)+ return (+ <AccessControlAction+ resourceType={AccessControlResourceType.Evaluation}+ minAccessLevel={AccessControlLevel.Editor}+ userAccessLevel={accessLevel}+ >+ <LemonButton+ type="secondary"+ size="small"+ icon={running ? <IconPause /> : <IconPlay />}+ onClick={() => saveEnabled(!running)}+ loading={enabledSaving}+ disabledReason={running ? undefined : (enableSavedReason ?? undefined)}+ data-attr="llma-evaluation-header-enabled"+ >+ {running ? 'Pause' : 'Turn on'}+ </LemonButton>+ </AccessControlAction>+ )+}++/** The saved evaluation's title row, its run state, and the actions that change it. */+export function EvaluationSceneHeader({+ playgroundUrl,+ unsaved,+ showSaveAction,+ saveDisabledReason,+ onSave,+ onCancel,+}: {+ playgroundUrl: string | null+ unsaved: boolean+ showSaveAction: boolean+ saveDisabledReason: string | undefined+ onSave: () => void+ onCancel: () => void+}): JSX.Element | null {+ const { evaluation, originalEvaluation, evaluationFormSubmitting } = useValues(llmEvaluationLogic)+ const { featureFlags } = useValues(featureFlagLogic)+ const sceneMenuBarEnabled = !!featureFlags[FEATURE_FLAGS.SCENE_MENU_BAR]++ if (!evaluation) {+ return null+ }+ // Pause and Turn on save at once, so they follow the saved state, not an unsaved edit from the configuration tab.+ const running = originalEvaluation?.enabled ?? evaluation.enabled+ const accessLevel = evaluation.user_access_level ?? undefined++ return (+ <>+ {sceneMenuBarEnabled && <EvaluationSceneMenuBar playgroundUrl={playgroundUrl} />}+ <SceneTitleSection+ name={evaluation.name}+ description={evaluation.description || null}+ resourceType={{ type: 'llm_evaluations' }}+ hideProductSetupButton+ nameSuffix={<StatusTags errored={evaluation.status === 'error'} running={running} unsaved={unsaved} />}+ actions={+ <>+ {!sceneMenuBarEnabled && playgroundUrl && (+ <LemonButton+ type="secondary"+ size="small"+ icon={<IconPlay />}+ to={playgroundUrl}+ data-attr="llma-playground-open-from-evaluation"+ >+ Open in Playground+ </LemonButton>+ )}+ <RunStateButton running={running} accessLevel={accessLevel} />+ {unsaved && (+ <LemonButton type="secondary" size="small" onClick={onCancel}>+ Cancel+ </LemonButton>+ )}+ {showSaveAction && (+ <AccessControlAction+ resourceType={AccessControlResourceType.Evaluation}+ minAccessLevel={AccessControlLevel.Editor}+ userAccessLevel={accessLevel}+ >+ <LemonButton+ type="primary"+ size="small"+ onClick={onSave}+ disabledReason={saveDisabledReason}+ loading={evaluationFormSubmitting}+ >+ Save changes+ </LemonButton>+ </AccessControlAction>+ )}+ </>+ }+ />+ </>+ )+}diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationSceneMenuBar.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationSceneMenuBar.tsxnew file mode 100644index 000000000000..c3bc81372868--- /dev/null+++ b/products/ai_observability/frontend/evaluations/components/EvaluationSceneMenuBar.tsx@@ -0,0 +1,92 @@+import { useActions, useValues } from 'kea'+import { router } from 'kea-router'++import { IconGear, IconPause, IconPlay, IconTrends } from '@posthog/icons'++import { AccessControlAction } from 'lib/components/AccessControlAction'+import { sceneFileLogic } from 'lib/components/Scenes/sceneFileLogic'+import { SceneMenuBarFileItems } from 'lib/components/Scenes/SceneMenuBarFileItems'++import {+ SceneMenuBar,+ SceneMenuBarItem,+ SceneMenuBarMenu,+ SceneMenuBarSeparator,+} from '~/layout/scenes/components/SceneMenuBar'+import { AccessControlLevel, AccessControlResourceType } from '~/types'++import { llmEvaluationLogic } from '../llmEvaluationLogic'++/** The evaluation page's menu bar. The caller renders it only behind the scene menu bar flag. */+export function EvaluationSceneMenuBar({ playgroundUrl }: { playgroundUrl: string | null }): JSX.Element | null {+ const { originalEvaluation, enabledSaving, enableSavedReason, trendInsightUrl } = useValues(llmEvaluationLogic)+ const { saveEnabled, setActiveTab } = useActions(llmEvaluationLogic)+ const { hasFileItems } = useValues(sceneFileLogic)++ if (!originalEvaluation) {+ return null+ }+ const running = originalEvaluation.enabled++ return (+ <SceneMenuBar>+ <SceneMenuBarMenu label="File" dataAttr="llma-evaluation-menubar-file" disabled={!hasFileItems}>+ {hasFileItems && <SceneMenuBarFileItems dataAttrKey="llm_evaluation" />}+ </SceneMenuBarMenu>+ <SceneMenuBarMenu label="Edit" dataAttr="llma-evaluation-menubar-edit">+ <SceneMenuBarItem+ onClick={() => setActiveTab('configuration')}+ data-attr="llma-evaluation-menubar-configure"+ >+ <IconGear />+ Edit configuration+ </SceneMenuBarItem>+ <SceneMenuBarSeparator />+ <AccessControlAction+ resourceType={AccessControlResourceType.Evaluation}+ minAccessLevel={AccessControlLevel.Editor}+ userAccessLevel={originalEvaluation.user_access_level ?? undefined}+ >+ {({ disabledReason }) => {+ const reason = disabledReason ?? (running ? null : enableSavedReason)+ return (+ <SceneMenuBarItem+ onClick={() => saveEnabled(!running)}+ disabled={!!reason || enabledSaving}+ tooltip={reason ?? undefined}+ data-attr="llma-evaluation-menubar-enabled"+ >+ {running ? <IconPause /> : <IconPlay />}+ {running ? 'Pause' : 'Turn on'}+ </SceneMenuBarItem>+ )+ }}+ </AccessControlAction>+ </SceneMenuBarMenu>+ <SceneMenuBarMenu+ label="View"+ dataAttr="llma-evaluation-menubar-view"+ disabled={!trendInsightUrl && !playgroundUrl}+ >+ {trendInsightUrl && (+ <SceneMenuBarItem+ onClick={() => window.open(trendInsightUrl, '_blank', 'noopener,noreferrer')}+ data-attr="llma-evaluation-menubar-trend-insight"+ >+ <IconTrends />+ Trend insight+ </SceneMenuBarItem>+ )}+ {playgroundUrl && (+ <SceneMenuBarItem+ onClick={() => router.actions.push(playgroundUrl)}+ data-attr="llma-evaluation-menubar-playground"+ >+ <IconPlay />+ Open in Playground+ </SceneMenuBarItem>+ )}+ </SceneMenuBarMenu>+ </SceneMenuBar>+ )+}diff --git a/products/ai_observability/frontend/evaluations/components/templateSummary.ts b/products/ai_observability/frontend/evaluations/components/templateSummary.tsindex bf5dd26e996f..8f502f88c140 100644--- a/products/ai_observability/frontend/evaluations/components/templateSummary.ts+++ b/products/ai_observability/frontend/evaluations/components/templateSummary.ts@@ -24,7 +24,7 @@ export function templateSummary(template: EvaluationTemplate): string {}/** One line that says what a run returns, e.g. "Score from 1 to 5". */-export function templateOutputText(template: EvaluationTemplate): string {+export function templateOutputText(template: Pick<EvaluationTemplate, 'output_type' | 'output_config'>): string {const { min, max, options = [], selection_mode } = template.output_configswitch (template.output_type) {case 'boolean':diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.tsxindex fc15b46bc4f4..133eeea9e2ea 100644--- a/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.tsx+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.tsx@@ -3,8 +3,6 @@ import { router } from 'kea-router'import posthog from 'posthog-js'import { useMemo, useRef } from 'react'-import { LemonTag } from '@posthog/lemon-ui'-import { lemonToast } from 'lib/lemon-ui/LemonToast'import { SceneBreadcrumbBackButton } from '~/layout/scenes/components/SceneBreadcrumbs'@@ -107,10 +105,9 @@ export function EvaluationCreatePage({ templateKey }: { templateKey?: string }):hasSceneTitleSection={false}className="flex flex-wrap items-center justify-between gap-2">- <div className="flex flex-wrap items-center gap-2">- <h1 className="mb-0 text-2xl font-semibold">New evaluation</h1>- {template && <LemonTag type="muted">{`From the ${template.name} template`}</LemonTag>}- </div>+ <h1 className="mb-0 text-2xl font-semibold">+ {template ? `New ${template.name.toLowerCase()} evaluation` : 'New evaluation'}+ </h1></SceneStickyBar><div className="w-full max-w-4xl mx-auto"><EvaluationFormSteps checklist={checklist} onCreate={handleCreate} onCancel={handleCancel} />diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCreateReview.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCreateReview.tsxindex 05528c3f43bc..0b779ab0bc83 100644--- a/products/ai_observability/frontend/evaluations/create/EvaluationCreateReview.tsx+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCreateReview.tsx@@ -1,23 +1,17 @@-import { useActions, useMountedLogic, useValues } from 'kea'+import { useActions, useValues } from 'kea'-import { IconCheckCircle, IconCircleDashed, IconClock, IconNotification, IconSparkles } from '@posthog/icons'-import { LemonSwitch, Link } from '@posthog/lemon-ui'+import { IconCheckCircle, IconCircleDashed } from '@posthog/icons'+import { LemonCheckbox, Link } from '@posthog/lemon-ui'import { cn } from 'lib/utils/css-classes'-import { capitalizeFirstLetter } from 'lib/utils/strings'+import { pluralize } from 'lib/utils/strings'-import { PROVIDER_SETTINGS_URL } from '../../ModelPicker'-import { buildDeliveryTargets, evaluationReportLogic } from '../evaluationReportLogic'-import { runRate } from '../evaluationVolume'+import { VOLUME_LOOKBACK_DAYS } from '../evaluationVolume'import { evaluationVolumeLogic } from '../evaluationVolumeLogic'import { llmEvaluationLogic } from '../llmEvaluationLogic'import type { ChecklistItem, ChecklistKey } from './creationChecklist'-import { reportText } from './evaluationSummary'-/**- * What the evaluation needs before it can be created, as a checklist, next to what it will do once it runs:- * how often, at what cost per run, where reports go, and whether it starts running.- */+/** What the evaluation needs before it can be created, next to what happens once it is created. */export function EvaluationCreateReview({checklist,onFix,@@ -26,60 +20,29 @@ export function EvaluationCreateReview({/** Goes to the step that sets an item. */onFix: (key: ChecklistKey) => void}): JSX.Element | null {- const { props } = useMountedLogic(llmEvaluationLogic)- const { evaluation, canEnable, canEnableReason, isReportableEvaluation, selectedModel, selectedModelName } =- useValues(llmEvaluationLogic)- const { setEvaluationEnabled } = useActions(llmEvaluationLogic)+ const { evaluation, canEnableReason, checkPastTraffic } = useValues(llmEvaluationLogic)+ const { setCheckPastTraffic } = useActions(llmEvaluationLogic)const { volume } = useValues(evaluationVolumeLogic)- const { configDraft } = useValues(evaluationReportLogic({ evaluationId: props.evaluationId }))if (!evaluation) {return null}- const destinations = buildDeliveryTargets(configDraft).length- const reportsOn = configDraft.enabled && isReportableEvaluation- const facts: { key: string; icon: JSX.Element; title: string; detail: string }[] = [- ...(volume- ? [- {- key: 'volume',- icon: <IconClock />,- title: capitalizeFirstLetter(runRate(volume.runs)),- detail: "From last week's matching traffic",- },- ]- : []),- {- key: 'judge',- icon: <IconSparkles />,- title:- evaluation.evaluation_type === 'llm_judge'- ? `Judged by ${selectedModelName ?? (selectedModel || 'the model you pick')}`- : 'No AI key needed',- detail:- evaluation.evaluation_type === 'llm_judge'- ? 'Uses your own AI provider key'- : evaluation.evaluation_type === 'hog'- ? 'Your code decides each result'- : 'A built-in classifier decides each result',- },- {- key: 'reports',- icon: <IconNotification />,- title: reportsOn- ? `${capitalizeFirstLetter(reportText({ ...configDraft, enabled: true }))} reports`- : 'No reports',- detail: reportsOn- ? destinations > 0- ? `Sent to ${destinations === 1 ? 'one destination' : `${destinations} destinations`}`- : 'In the Reports tab'- : 'Turn them on in the Reports step',- },- ]+ // Unknown volume (still counting, or a filter on content) still offers it; only a known zero hides it.+ const hasPastTraffic = volume?.runs !== 0+ const units = volume ? pluralize(volume.matchingUnits, volume.target) : null+ // Below 100% sampling the exact count depends on which units the sample picks, so it names the pool instead.+ const pastTrafficCount = !volume+ ? ''+ : volume.runs === volume.matchingUnits+ ? ` It will evaluate ${units}.`+ : ` It will evaluate a sample of the ${units} that matched.`+ const pastTrafficDetail = `See results right away instead of waiting for new traffic.${pastTrafficCount}${+ evaluation.evaluation_type === 'llm_judge' ? ' Each one will be a call to your judge model.' : ''+ }`const done = checklist.filter((item) => item.done).lengthreturn (- <div className="grid grid-cols-1 gap-4 @min-[40rem]/evaluation:grid-cols-2">+ <div className="flex flex-col gap-3 w-full max-w-xl mx-auto"><div className="border rounded p-4 flex flex-col gap-3"><div className="flex items-center justify-between gap-2"><h4 className="mb-0 text-sm font-semibold">Checklist</h4>@@ -95,55 +58,31 @@ export function EvaluationCreateReview({)}<span className="flex flex-col min-w-0"><span className={cn(item.done ? 'text-secondary' : 'font-semibold')}>{item.label}</span>- {!item.done &&- (item.fixInSettings ? (- <Link- to={PROVIDER_SETTINGS_URL}- target="_blank"- data-attr="llma-evaluation-create-add-key"- >- {item.todo}- </Link>- ) : (- <Link onClick={() => onFix(item.key)}>{item.todo}</Link>- ))}+ {item.done && item.summary && (+ <span className="text-xs text-secondary truncate">{item.summary}</span>+ )}+ {!item.done && <Link onClick={() => onFix(item.key)}>{item.todo}</Link>}</span></li>))}</ul></div>- <div className="border rounded p-4 flex flex-col gap-3" data-attr="llma-evaluation-create-glance">- <h4 className="mb-0 text-sm font-semibold">At a glance</h4>- <ul className="m-0 p-0 list-none flex flex-col gap-3">- {facts.map((fact) => (- <li key={fact.key} className="flex items-start gap-3">- <span className="flex size-8 shrink-0 items-center justify-center rounded bg-fill-highlight-100 text-lg text-secondary">- {fact.icon}- </span>- <span className="flex flex-col min-w-0">- <span className="text-sm font-semibold">{fact.title}</span>- <span className="text-xs text-secondary">{fact.detail}</span>- </span>- </li>- ))}- </ul>- <div className="mt-auto flex flex-col gap-1 pt-2">- <LemonSwitch- checked={evaluation.enabled}- onChange={setEvaluationEnabled}- disabledReason={!canEnable && !evaluation.enabled ? (canEnableReason ?? undefined) : undefined}- label="Start running when created"- bordered- fullWidth- data-attr="llma-evaluation-create-enabled"+ {evaluation.enabled && hasPastTraffic && (+ <div className="border rounded px-4 py-3 flex flex-col gap-1">+ <LemonCheckbox+ checked={checkPastTraffic}+ onChange={setCheckPastTraffic}+ label={`Also evaluate the last ${VOLUME_LOOKBACK_DAYS} days`}+ data-attr="llma-evaluation-create-past-traffic"/>- <span className="text-xs text-secondary">- {evaluation.enabled- ? 'It checks new matching traffic as soon as you create it.'- : 'It stays paused. Turn it on from its page when you are ready.'}- </span>+ <span className="text-xs text-secondary pl-6">{pastTrafficDetail}</span></div>- </div>+ )}+ {!evaluation.enabled && (+ <p className="mb-0 text-sm text-secondary">+ {`It is created paused. ${canEnableReason ?? ''} Then turn it on from its page.`}+ </p>+ )}</div>)}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsxindex 7bc119608d52..ffb10944a7e0 100644--- a/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx+++ b/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx@@ -8,7 +8,7 @@ import * as officeWorkerPng from '@posthog/brand/hoggies/png/office-worker'import * as reporterPng from '@posthog/brand/hoggies/png/reporter'import * as stampApprovedPng from '@posthog/brand/hoggies/png/stamp-approved'import { IconCheck } from '@posthog/icons'-import { LemonButton, LemonSwitch } from '@posthog/lemon-ui'+import { LemonButton } from '@posthog/lemon-ui'import { pngHoggie } from 'lib/brand/hoggies'import { cn } from 'lib/utils/css-classes'@@ -96,33 +96,15 @@ const STEPS: Step[] = [},]-function StepContent({ step, isNew }: { step: Step['key']; isNew: boolean }): JSX.Element | null {- const { evaluation, canEnable, canEnableReason } = useValues(llmEvaluationLogic)- const { setEvaluationEnabled } = useActions(llmEvaluationLogic)+function StepContent({ step }: { step: Step['key'] }): JSX.Element | null {+ const { evaluation } = useValues(llmEvaluationLogic)if (!evaluation) {return null}switch (step) {case 'name':- return (- <>- <EvaluationNameFields />- {/* A new evaluation sets this on its create step. */}- {!isNew && (- <LemonSwitch- checked={evaluation.enabled}- onChange={setEvaluationEnabled}- disabledReason={- !canEnable && !evaluation.enabled ? (canEnableReason ?? undefined) : undefined- }- label={evaluation.enabled ? 'Running on new traffic' : 'Paused'}- bordered- data-attr="llma-evaluation-configuration-enabled"- />- )}- </>- )+ return <EvaluationNameFields />case 'check':return (<>@@ -269,7 +251,7 @@ export function EvaluationFormSteps({</div>{!isCreateStep && (<div className="flex flex-col gap-4 min-w-0">- <StepContent step={step.key} isNew={!!onCreate} />+ <StepContent step={step.key} /></div>)}{isCreateStep && (diff --git a/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts b/products/ai_observability/frontend/evaluations/create/creationChecklist.test.tsindex ec058e4bf4be..c939d1fb193b 100644--- a/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts+++ b/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts@@ -43,6 +43,21 @@ describe('creationChecklist', () => {expect(firstTodo(overrides)).toBe(expected)})+ it('sums up each done item, with only the first line of a long prompt', () => {+ const evaluation = {+ ...judge('Is it helpful?\nAnswer true or false.'),+ target: 'generation',+ model_configuration: { provider: 'openai', model: 'gpt-5-mini', provider_key_id: 'key-1' },+ } as EvaluationConfig+ expect(creationChecklist(input({ evaluation })).map((item) => item.summary)).toEqual([+ 'Helpfulness',+ 'Is it helpful?',+ 'Pass or fail',+ 'gpt-5-mini',+ '100% of all generations',+ ])+ })+it('asks for no prompt, result or model on a sentiment evaluation', () => {const sentiment = { ...judge(''), evaluation_type: 'sentiment', output_type: 'sentiment' } as EvaluationConfigexpect(diff --git a/products/ai_observability/frontend/evaluations/create/creationChecklist.ts b/products/ai_observability/frontend/evaluations/create/creationChecklist.tsindex 89644519ba80..4d59a0424a56 100644--- a/products/ai_observability/frontend/evaluations/create/creationChecklist.ts+++ b/products/ai_observability/frontend/evaluations/create/creationChecklist.ts@@ -1,5 +1,9 @@+import { capitalizeFirstLetter, pluralize } from 'lib/utils/strings'++import { templateOutputText } from '../components/templateSummary'import { categoricalOutputConfigError } from '../constants'import type { EvaluationConfig, EvaluationOutputConfig } from '../types'+import { triggerText } from './evaluationSummary'export type ChecklistKey = 'name' | 'criteria' | 'result' | 'model' | 'sampling'@@ -9,8 +13,8 @@ export interface ChecklistItem {done: boolean/** What to do next when the item is not done. Also the reason the create button is disabled. */todo: string- /** Set when the fix lives on another page, such as adding a provider key in settings. */- fixInSettings?: boolean+ /** What was chosen, shown once the item is done. */+ summary?: string}export interface ChecklistInput {@@ -33,14 +37,28 @@ const CRITERIA: Record<'llm_judge' | 'hog', { label: string; todo: string }> = {export function creationChecklist(input: ChecklistInput): ChecklistItem[] {const { evaluation } = inputconst items: ChecklistItem[] = [- { key: 'name', label: 'Name', done: evaluation.name.trim().length > 0, todo: 'Give the evaluation a name' },+ {+ key: 'name',+ label: 'Name',+ done: evaluation.name.trim().length > 0,+ todo: 'Give the evaluation a name',+ summary: evaluation.name.trim(),+ },]if (evaluation.evaluation_type === 'llm_judge' || evaluation.evaluation_type === 'hog') {const content =evaluation.evaluation_type === 'llm_judge'? evaluation.evaluation_config.prompt: evaluation.evaluation_config.source- items.push({ key: 'criteria', done: content.trim().length > 0, ...CRITERIA[evaluation.evaluation_type] })+ items.push({+ key: 'criteria',+ done: content.trim().length > 0,+ summary:+ evaluation.evaluation_type === 'llm_judge'+ ? content.trim().split('\n')[0]+ : pluralize(content.trim().split('\n').length, 'line') + ' of code',+ ...CRITERIA[evaluation.evaluation_type],+ })}if (evaluation.evaluation_type !== 'sentiment') {items.push({@@ -48,6 +66,7 @@ export function creationChecklist(input: ChecklistInput): ChecklistItem[] {label: 'Result',done: input.resultError === null,todo: input.resultError ?? '',+ summary: templateOutputText(evaluation),})}if (input.modelSelectionRequired) {@@ -58,7 +77,7 @@ export function creationChecklist(input: ChecklistInput): ChecklistItem[] {todo: input.hasAvailableModels? 'Pick the model that judges each run': 'Add an API key for an AI provider, then pick a model',- fixInSettings: !input.hasAvailableModels,+ summary: evaluation.model_configuration?.model,})}items.push({@@ -66,6 +85,7 @@ export function creationChecklist(input: ChecklistInput): ChecklistItem[] {label: 'Sampling',done: !input.hasUnsetSampling,todo: 'Set a sampling percentage between 0.1% and 100%',+ summary: capitalizeFirstLetter(triggerText(evaluation)),})return items}diff --git a/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.test.ts b/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.test.tsindex f4bed492358c..a3a3ee384cea 100644--- a/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.test.ts+++ b/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.test.ts@@ -1,5 +1,6 @@import { MOCK_DEFAULT_TEAM } from 'lib/api.mock'+import { router } from 'kea-router'import { expectLogic } from 'kea-test-utils'import posthog from 'posthog-js'@@ -21,6 +22,7 @@ import {import type { EvaluationBackfillApi, EvaluationBackfillEstimateApi } from '../generated/api.schemas'import { evaluationBackfillsLogic } from './evaluationBackfillsLogic'import { llmEvaluationLogic } from './llmEvaluationLogic'+import { requestBackfillStart } from './pendingBackfillStarts'import type { EvaluationConfig } from './types'jest.mock('../generated/api', () => ({@@ -461,6 +463,35 @@ describe('evaluationBackfillsLogic', () => {expect(logic.cache.disposables.registry.has('backfillPoll')).toBe(false)})+ it('starts the default range once counted when a create asked for it, and only once', async () => {+ requestBackfillStart(EVALUATION_ID)+ mountLogic()+ await expectLogic(logic).toDispatchActions(['requestEstimateSuccess', 'createBackfill', 'createBackfillDone'])+ expect(createMock).toHaveBeenCalledTimes(1)++ logic.actions.requestEstimateSuccess(estimate())+ await expectLogic(logic).toFinishAllListeners()+ expect(createMock).toHaveBeenCalledTimes(1)+ })++ it('starts nothing from a link that only claims a start was requested', async () => {+ router.actions.push(router.values.location.pathname, { backfill_start: 'requested' })+ mountLogic()+ await expectLogic(logic).toDispatchActions(['requestEstimateSuccess'])+ await expectLogic(logic).toFinishAllListeners()+ expect(createMock).not.toHaveBeenCalled()+ })++ it('still starts a requested backfill after leaving the tab before the count landed', async () => {+ requestBackfillStart(EVALUATION_ID)+ mountLogic()+ logic.unmount()++ mountLogic()+ await expectLogic(logic).toDispatchActions(['createBackfill', 'createBackfillDone'])+ expect(createMock).toHaveBeenCalledTimes(1)+ })+it('keeps polling a finished run until its coverage lands', async () => {listMock.mockResolvedValueOnce({ count: 1, results: [backfill({ status: 'running' })] })mountLogic()diff --git a/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.ts b/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.tsindex e0ec98516221..955b7be8e340 100644--- a/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.ts+++ b/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.ts@@ -25,6 +25,7 @@ import type {import { evaluationErrorMessage } from './apiErrors'import { backfillRangeDateFormat } from './backfillConditions'import { hasUnsetConditionRollout, llmEvaluationLogic, normalizeConditionRollouts } from './llmEvaluationLogic'+import { takeBackfillStart } from './pendingBackfillStarts'import type { EvaluationConditionSet } from './types'import type { EvaluationConfig } from './types'@@ -556,6 +557,12 @@ export const evaluationBackfillsLogic = kea<evaluationBackfillsLogicType>([},loadBackfillsSuccess: schedulePoll,loadBackfillsFailure: schedulePoll,+ // A count cut short by leaving the tab never lands here, so the request waits for the next visit.+ requestEstimateSuccess: () => {+ if (takeBackfillStart(props.evaluationId) && !values.startDisabledReason) {+ actions.createBackfill()+ }+ },seedConditions: () => actions.requestEstimate(),setWindowRange: () => actions.requestEstimate(),setConditions: () => actions.requestEstimate(),@@ -624,7 +631,7 @@ export const evaluationBackfillsLogic = kea<evaluationBackfillsLogicType>([// The units this run covers now have a result, so the count it started from is stale.cache.lastEstimate = nullcache.recountWhenIdle = true- lemonToast.success('Backfill started')+ lemonToast.success('Backfill started', { autoClose: 15000 })actions.loadBackfills()} catch (error) {lemonToast.error(backfillErrorMessage(error, 'Couldn’t start the backfill. Try again.'))diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.tsindex 2a6f607f5635..95df79953e4b 100644--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts@@ -17,6 +17,7 @@ import { EVALUATION_RUNS_QUERY_LIMIT, numericScorePasses } from './constants'import { evaluationReportLogic } from './evaluationReportLogic'import { DEFAULT_HOG_SOURCE, llmEvaluationLogic } from './llmEvaluationLogic'import { llmEvaluationsLogic } from './llmEvaluationsLogic'+import { takeBackfillStart } from './pendingBackfillStarts'import { EvaluationConfig, EvaluationReport, EvaluationRun } from './types'const mockProviderKeys: LLMProviderKey[] = [@@ -285,16 +286,6 @@ describe('llmEvaluationLogic', () => {expect(logic.values.evaluation?.enabled).toBe(expected)})- it('keeps a new evaluation paused once someone turned it off by hand', async () => {- await expectLogic(keysLogic).toFinishAllListeners()- logic.actions.setEvaluationType('hog')- logic.actions.setEvaluationEnabled(false)- // Through a judge and back, so canEnable changes and the default would turn it on again.- logic.actions.setEvaluationType('llm_judge')- logic.actions.setEvaluationType('hog')- expect(logic.values.evaluation?.enabled).toBe(false)- })-it.each([[DEFAULT_HOG_SOURCE, 'return 0;'],['return 42;', 'return 42;'],@@ -1696,16 +1687,27 @@ return result`,})})- it('setEvaluationType switches back to llm_judge config shape', async () => {+ it('switching method and back keeps the prompt and code already typed', async () => {await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])+ logic.actions.setEvaluationPrompt('Is the answer polite?')logic.actions.setEvaluationType('hog')+ logic.actions.setHogSource('return true;')logic.actions.setEvaluationType('llm_judge')await expectLogic(logic).toMatchValues({evaluation: expect.objectContaining({evaluation_type: 'llm_judge',- evaluation_config: { prompt: '' },+ evaluation_config: { prompt: 'Is the answer polite?' },+ }),+ })++ logic.actions.setEvaluationType('hog')++ await expectLogic(logic).toMatchValues({+ evaluation: expect.objectContaining({+ evaluation_type: 'hog',+ evaluation_config: { source: 'return true;' },}),})})@@ -2292,11 +2294,58 @@ return result`,})})+ describe('saveEvaluation past traffic', () => {+ it.each([+ { enabled: true, expectedTab: 'backfills', requestsStart: true, offersPastData: false },+ { enabled: false, expectedTab: null, requestsStart: false, offersPastData: true },+ ])(+ 'a new evaluation created enabled=$enabled asks the Backfills tab to start: $requestsStart',+ async ({ enabled, expectedTab, requestsStart, offersPastData }) => {+ const pushSpy = jest.spyOn(router.actions, 'push').mockImplementation(() => {})+ const infoSpy = jest.spyOn(lemonToast, 'info')+ let backfillRequests = 0+ useMocks({+ post: {+ '/api/projects/:teamId/evaluations/': () => ({ ...mockSentimentEvaluation, enabled }),+ '/api/projects/:teamId/evaluations/:id/backfills/': () => {+ backfillRequests += 1+ return [201, {}]+ },+ },+ })+ logic = llmEvaluationLogic({ evaluationId: 'new', evaluationType: 'sentiment' })+ logic.mount()++ try {+ await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])+ logic.actions.setEvaluationName('Sentiment evaluation')+ logic.actions.setCheckPastTraffic(true)+ logic.actions.saveEvaluation()+ await expectLogic(logic).toDispatchActions(['saveEvaluationSuccess'])+ await expectLogic(logic).toFinishAllListeners()++ expect(backfillRequests).toBe(0)+ const params = new URL(String(pushSpy.mock.calls.at(-1)?.[0]), 'http://x').searchParams+ expect(params.get('evaluation_tab')).toBe(expectedTab)+ // The request stays in memory, never in a URL someone could share.+ expect(params.get('backfill_start')).toBeNull()+ expect(takeBackfillStart(mockSentimentEvaluation.id)).toBe(requestsStart)+ expect(+ infoSpy.mock.calls.some(+ ([message]) => message === 'This evaluation grades new data from now on.'+ )+ ).toBe(offersPastData)+ } finally {+ pushSpy.mockRestore()+ infoSpy.mockRestore()+ }+ }+ )+ })+describe('saveEvaluation updates', () => {it.each([{ change: 'name', expected: { name: 'Renamed decision evaluation' }, rejected: false },- { change: 'disable', expected: { enabled: false }, rejected: false },- { change: 'enable', expected: { enabled: true }, rejected: true },{change: 'output_config',expected: { output_config: { allows_na: true } },@@ -2325,7 +2374,7 @@ return result`,async ({ change, expected, rejected }) => {const evaluation: EvaluationConfig = {...mockEvaluation,- enabled: change !== 'enable',+ enabled: true,evaluation_type: 'llm_judge',evaluation_config: { prompt: 'Is this response helpful?', judge_method: 'decision' },output_type: 'boolean',@@ -2364,10 +2413,6 @@ return result`,case 'name':logic.actions.setEvaluationName(expected.name)break- case 'disable':- case 'enable':- logic.actions.setEvaluationEnabled(expected.enabled)- breakcase 'output_config':logic.actions.setAllowsNA(true)break@@ -2391,6 +2436,58 @@ return result`,)})+ describe('saveEnabled', () => {+ it('lets a saved code evaluation turn on while its unsaved draft is a judge without a key', async () => {+ const evaluation: EvaluationConfig = {+ ...mockEvaluation,+ enabled: false,+ evaluation_type: 'hog',+ evaluation_config: { source: 'return true;' },+ model_configuration: null,+ } as EvaluationConfig+ useMocks({ get: { '/api/projects/:teamId/evaluations/:id/': evaluation } })+ logic = llmEvaluationLogic({ evaluationId: 'eval-123' })+ logic.mount()+ await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])+ await expectLogic(keysLogic).toFinishAllListeners()++ logic.actions.setEvaluationType('llm_judge')++ expect(logic.values.canEnable).toBe(false)+ expect(logic.values.enableSavedReason).toBeNull()+ })++ it('saves only the run state and keeps unsaved configuration edits', async () => {+ const evaluation: EvaluationConfig = { ...mockEvaluation, enabled: false }+ let savedPayload: unknown+ useMocks({+ get: { '/api/projects/:teamId/evaluations/:id/': evaluation },+ patch: {+ '/api/projects/:teamId/evaluations/:id/': async ({ request }) => {+ savedPayload = await request.json()+ return { ...evaluation, enabled: true }+ },+ },+ })+ logic = llmEvaluationLogic({ evaluationId: 'eval-123' })+ logic.mount()+ await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])++ logic.actions.setEvaluationName('Renamed but not saved')+ logic.actions.saveEnabled(true)++ await expectLogic(logic)+ .toDispatchActions(['saveEnabledSuccess'])+ .toMatchValues({+ enabledSaving: false,+ evaluation: expect.objectContaining({ enabled: true, name: 'Renamed but not saved' }),+ originalEvaluation: expect.objectContaining({ enabled: true, name: evaluation.name }),+ hasUnsavedChanges: true,+ })+ expect(savedPayload).toEqual({ enabled: true })+ })+ })+describe('saveEvaluation report persistence', () => {it('does not create an evaluation when the report threshold is invalid', async () => {let evaluationCreateCount = 0diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.tsindex e04ad0e18f37..e456266aafab 100644--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts@@ -60,9 +60,11 @@ import {} from './evaluationCapabilities'import { EvaluationBackTarget, getEvaluationBackTarget } from './evaluationNavigation'import { evaluationReportLogic, persistReportDraft } from './evaluationReportLogic'+import { patchEvaluation } from './evaluationsApi'import { getHogEvalExample } from './hogEvalExamples'import { llmEvaluationsLogic } from './llmEvaluationsLogic'import { markEvaluationsOverviewStale } from './overview/evaluationsOverviewLogic'+import { requestBackfillStart } from './pendingBackfillStarts'import { EvaluationTemplateKey, defaultEvaluationTemplates } from './templates'import type {EvaluationConditionSet,@@ -212,6 +214,34 @@ function toSentimentEvaluation(evaluation: EvaluationConfig): SentimentEvaluatio}}+/** Offers past data for a new evaluation, or hands a requested past-traffic check to the Backfills tab. Returns the URL params to open the evaluation with. */+function coverPastTraffic(+ evaluation: EvaluationConfig | undefined,+ requested: boolean,+ isNew: boolean+): Record<string, string> {+ if (!isNew || !evaluation?.id) {+ return {}+ }+ // The backfill API refuses a paused evaluation, so past traffic is checked only when it runs.+ if (!evaluation.enabled || !requested) {+ // Saving navigates away, so the offer to cover past data travels with the user.+ lemonToast.info('This evaluation grades new data from now on.', {+ button: {+ label: 'Evaluate past data',+ action: () =>+ router.actions.push(+ combineUrl(urls.aiObservabilityEvaluation(evaluation.id), { evaluation_tab: 'backfills' }).url+ ),+ },+ })+ return {}+ }+ // The Backfills tab counts the range and starts the run, with its own errors and polling.+ requestBackfillStart(evaluation.id)+ return { evaluation_tab: 'backfills' }+}+function filterEvaluationRuns(runs: EvaluationRun[],filter: EvaluationRunsFilter,@@ -260,6 +290,11 @@ function filterEvaluationRuns(type TestableHogEvaluation = HogEvaluation+export interface MethodConfigDrafts {+ prompt?: string+ hogSource?: string+}+function isTestableHogEvaluation(evaluation: EvaluationConfig | null): evaluation is TestableHogEvaluation {return evaluation?.evaluation_type === 'hog'}@@ -325,7 +360,9 @@ export interface llmEvaluationLogicValues {breadcrumbs: Breadcrumb[]canEnable: booleancanEnableReason: string | null- enabledChosen: boolean+ checkPastTraffic: boolean+ enableSavedReason: string | null+ enabledSaving: booleanevaluation: EvaluationConfig | nullevaluationBackTarget: EvaluationBackTargetevaluationFormSubmitting: boolean@@ -350,6 +387,7 @@ export interface llmEvaluationLogicValues {judgeMethod: JudgeMethodjudgeModelGroups: ProviderModelGroup[]maxContext: MaxContextInput[]+ methodConfigDrafts: MethodConfigDraftsmodelSelectionRequired: booleannumericBoundsRequired: booleanoriginalEvaluation: EvaluationConfig | null@@ -472,6 +510,15 @@ export interface llmEvaluationLogicActions {resetEvaluation: () => {value: true}+ saveEnabled: (enabled: boolean) => {+ enabled: boolean+ }+ saveEnabledFailure: () => {+ value: true+ }+ saveEnabledSuccess: (evaluation: EvaluationConfig) => {+ evaluation: EvaluationConfig+ }saveEvaluation: () => {value: true}@@ -494,6 +541,9 @@ export interface llmEvaluationLogicActions {setAllowsNA: (allowsNA: boolean) => {allowsNA: boolean}+ setCheckPastTraffic: (checkPastTraffic: boolean) => {+ checkPastTraffic: boolean+ }setDefaultEnabled: (enabled: boolean) => {enabled: boolean}@@ -503,9 +553,6 @@ export interface llmEvaluationLogicActions {setEvaluationDirectory: (directoryId: string | null) => {directoryId: string | null}- setEvaluationEnabled: (enabled: boolean) => {- enabled: boolean- }setEvaluationName: (name: string) => {name: string}@@ -527,6 +574,8 @@ export interface llmEvaluationLogicActions {}setEvaluationType: (evaluationType: EvaluationType) => {evaluationType: EvaluationType+ methodConfigDrafts: MethodConfigDrafts+ previousEvaluation: EvaluationConfig | null}setFormStep: (step: string) => {step: string@@ -633,6 +682,10 @@ export interface llmEvaluationLogicMeta {evaluation: EvaluationConfig | null,activeProviderKey: LLMProviderKey | null | undefined) => boolean+ enableSavedReason: (+ originalEvaluation: EvaluationConfig | null,+ activeProviderKey: LLMProviderKey | null | undefined+ ) => string | nullcanEnableReason: (canEnable: boolean) => string | nullevaluationProviderKeyIssue: (evaluation: EvaluationConfig | null,@@ -696,7 +749,6 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([setEvaluationDescription: (description: string) => ({ description }),setEvaluationDirectory: (directoryId: string | null) => ({ directoryId }),setEvaluationPrompt: (prompt: string) => ({ prompt }),- setEvaluationEnabled: (enabled: boolean) => ({ enabled }),setDefaultEnabled: (enabled: boolean) => ({ enabled }),setOutputType: (outputType: 'boolean' | 'numeric' | 'categorical') => ({outputType,@@ -711,7 +763,11 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([setJudgeMethod: (judgeMethod: JudgeMethod) => ({ judgeMethod }),setIncludeMedia: (includeMedia: boolean) => ({ includeMedia }),setReasoningLevel: (reasoningLevel: ReasoningLevel | null) => ({ reasoningLevel }),- setEvaluationType: (evaluationType: EvaluationType) => ({ evaluationType }),+ setEvaluationType: (evaluationType: EvaluationType) => ({+ evaluationType,+ previousEvaluation: values.evaluation,+ methodConfigDrafts: values.methodConfigDrafts,+ }),setEvaluationTarget: (target: EvaluationTarget) => ({ target }),setSettleStrategy: (strategy: EvaluationSettleStrategy) => ({ strategy }),// Duration fields only — switching strategy must go through setSettleStrategy so the@@ -728,6 +784,10 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([// Evaluation management actionssaveEvaluation: true,saveEvaluationSuccess: (evaluation: EvaluationConfig) => ({ evaluation }),+ saveEnabled: (enabled: boolean) => ({ enabled }),+ setCheckPastTraffic: (checkPastTraffic: boolean) => ({ checkPastTraffic }),+ saveEnabledSuccess: (evaluation: EvaluationConfig) => ({ evaluation }),+ saveEnabledFailure: true,saveEvaluationFailure: (error: string) => ({ error }),loadEvaluation: true,loadEvaluationSuccess: (evaluation: EvaluationConfig | null) => ({@@ -921,11 +981,32 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([: state,},],+ methodConfigDrafts: [+ {} as MethodConfigDrafts,+ {+ loadEvaluationSuccess: () => ({}),+ setEvaluationType: (state, { previousEvaluation }) =>+ previousEvaluation?.evaluation_type === 'llm_judge'+ ? { ...state, prompt: previousEvaluation.evaluation_config.prompt }+ : previousEvaluation?.evaluation_type === 'hog'+ ? {+ ...state,+ // An untouched default is regenerated on return so it matches the output type chosen meanwhile.+ hogSource:+ previousEvaluation.evaluation_config.source ===+ toHogEvaluation(previousEvaluation).evaluation_config.source+ ? undefined+ : previousEvaluation.evaluation_config.source,+ }+ : state,+ },+ ],originalEvaluation: [null as EvaluationConfig | null,{loadEvaluationSuccess: (_, { evaluation }) => evaluation,saveEvaluationSuccess: (_, { evaluation }) => evaluation,+ saveEnabledSuccess: (_, { evaluation }) => evaluation,},],evaluation: [@@ -939,7 +1020,6 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([state && isLLMJudgeEvaluation(state)? { ...state, evaluation_config: { ...state.evaluation_config, prompt } }: state,- setEvaluationEnabled: (state, { enabled }) => (state ? { ...state, enabled } : null),setDefaultEnabled: (state, { enabled }) => (state ? { ...state, enabled } : null),setOutputType: (state, { outputType, outputConfig }) => {if (@@ -1053,12 +1133,15 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([state && isLLMJudgeEvaluation(state)? { ...state, evaluation_config: { ...state.evaluation_config, include_media: includeMedia } }: state,- setEvaluationType: (state, { evaluationType }) => {+ setEvaluationType: (state, { evaluationType, methodConfigDrafts }) => {if (!state) {return null}if (evaluationType === 'hog') {- return toHogEvaluation(state)+ const hogEvaluation = toHogEvaluation(state)+ return methodConfigDrafts.hogSource === undefined+ ? hogEvaluation+ : { ...hogEvaluation, evaluation_config: { source: methodConfigDrafts.hogSource } }}if (evaluationType === 'sentiment') {if (@@ -1069,7 +1152,10 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([}return toSentimentEvaluation(state)}- return toLLMJudgeEvaluation(state)+ const judgeEvaluation = toLLMJudgeEvaluation(state)+ return methodConfigDrafts.prompt === undefined+ ? judgeEvaluation+ : { ...judgeEvaluation, evaluation_config: { prompt: methodConfigDrafts.prompt } }},setEvaluationTarget: (state, { target }) => {if (!state) {@@ -1107,6 +1193,17 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([: state,loadEvaluationSuccess: (_, { evaluation }) => evaluation,saveEvaluationSuccess: (_, { evaluation }) => evaluation,+ // Takes only the run state, so configuration edits that are not saved yet stay unsaved.+ saveEnabledSuccess: (state, { evaluation }) =>+ state+ ? {+ ...state,+ enabled: evaluation.enabled,+ status: evaluation.status,+ status_reason: evaluation.status_reason,+ status_reason_detail: evaluation.status_reason_detail,+ }+ : state,},],hogTestResults: {@@ -1179,13 +1276,19 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([saveEvaluationFailure: () => false,},],- // Whether someone switched "enabled" by hand. Until then a new evaluation follows whether it can run.- enabledChosen: [+ checkPastTraffic: [false,{- setEvaluationEnabled: () => true,+ setCheckPastTraffic: (_, { checkPastTraffic }) => checkPastTraffic,loadEvaluationSuccess: () => false,- resetEvaluation: () => false,+ },+ ],+ enabledSaving: [+ false,+ {+ saveEnabled: () => true,+ saveEnabledSuccess: () => false,+ saveEnabledFailure: () => false,},],hasUnsavedChanges: [@@ -1195,7 +1298,6 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([setEvaluationDescription: () => true,setEvaluationDirectory: () => true,setEvaluationPrompt: () => true,- setEvaluationEnabled: () => true,setOutputType: () => true,patchOutputConfig: () => true,setAllowsNA: () => true,@@ -1461,6 +1563,26 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([}},+ saveEnabled: async ({ enabled }) => {+ const teamId = teamLogic.values.currentTeamId+ if (!teamId) {+ actions.saveEnabledFailure()+ return+ }+ try {+ actions.saveEnabledSuccess(await patchEvaluation(teamId.toString(), props.evaluationId, { enabled }))+ llmEvaluationsLogic.findMounted()?.actions.loadEvaluations()+ markEvaluationsOverviewStale()+ } catch (error) {+ lemonToast.error(+ evaluationErrorMessage(+ error,+ enabled ? 'Couldn’t turn on this evaluation.' : 'Couldn’t pause this evaluation.'+ )+ )+ actions.saveEnabledFailure()+ }+ },saveEvaluation: async () => {try {const teamId = teamLogic.values.currentTeamId@@ -1511,6 +1633,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([props.evaluationId,evaluationData as Parameters<typeof evaluationsPartialUpdate>[2])) as unknown as EvaluationConfig+ const openParams = coverPastTraffic(response, values.checkPastTraffic, isNew)actions.saveEvaluationSuccess(response)// The list and the self-driving table read from llmEvaluationsLogic, which only// fetches on mount. Refresh it here so they show the eval we just saved.@@ -1518,20 +1641,6 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([markEvaluationsOverviewStale()if (isNew) {globalSetupLogic.findMounted()?.actions.markTaskAsCompleted(SetupTaskId.SetUpLlmEvaluation)- // Saving navigates away, so the offer to cover past data travels with the user.- if (response?.id) {- lemonToast.info('This evaluation grades new data from now on.', {- button: {- label: 'Evaluate past data',- action: () =>- router.actions.push(- combineUrl(urls.aiObservabilityEvaluation(response.id), {- evaluation_tab: 'backfills',- }).url- ),- },- })- }}// Piggyback the scheduled-report draft onto the main save so the single@@ -1566,7 +1675,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([const opensEvaluation = isNew && !!response?.idrouter.actions.push(opensEvaluation- ? urls.aiObservabilityEvaluation(response.id)+ ? combineUrl(urls.aiObservabilityEvaluation(response.id), openParams).url: getEvaluationBackTarget(false, router.values.searchParams).path)} catch (error) {@@ -1882,6 +1991,18 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([},],+ // Pause and Turn on change only the saved evaluation, so they check it rather than an unsaved draft.+ enableSavedReason: [+ (s) => [s.originalEvaluation, s.activeProviderKey],+ (+ originalEvaluation: EvaluationConfig | null,+ activeProviderKey: LLMProviderKey | null | undefined+ ): string | null =>+ !originalEvaluation || evaluationCanResolveModel(originalEvaluation, activeProviderKey)+ ? null+ : 'Add a provider API key to enable this evaluation.',+ ],+canEnableReason: [(s) => [s.canEnable],(canEnable: boolean): string | null => {@@ -2070,12 +2191,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([subscriptions(({ actions, values, props }) => ({// A new evaluation starts running when it can: code and sentiment need no key, a judge needs a working one.canEnable: (canEnable: boolean) => {- if (- props.evaluationId === 'new' &&- !values.enabledChosen &&- values.evaluation &&- values.evaluation.enabled !== canEnable- ) {+ if (props.evaluationId === 'new' && values.evaluation && values.evaluation.enabled !== canEnable) {actions.setDefaultEnabled(canEnable)}},diff --git a/products/ai_observability/frontend/evaluations/pendingBackfillStarts.ts b/products/ai_observability/frontend/evaluations/pendingBackfillStarts.tsnew file mode 100644index 000000000000..bcaf7cf81bc9--- /dev/null+++ b/products/ai_observability/frontend/evaluations/pendingBackfillStarts.ts@@ -0,0 +1,14 @@+/**+ * New evaluations whose creator asked to also evaluate past traffic. It lives in memory, not in the URL,+ * so a link cannot start a backfill, and a request survives leaving the Backfills tab before its count lands.+ */+const requested = new Set<string>()++export function requestBackfillStart(evaluationId: string): void {+ requested.add(evaluationId)+}++/** Returns whether a start was requested, and forgets it, so each request starts at most one backfill. */+export function takeBackfillStart(evaluationId: string): boolean {+ return requested.delete(evaluationId)+}