Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): tighten the eval create flow and eval page (#129476)

Bernat Torres9fd797ede37b
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index dd98616477c8..bada9c3f6361 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -9227,17 +9227,17 @@ snapshots:
scenes-app-ai-observability-new-evaluation--from-scratch--light:
hash: v1.k794b7964.816d9e830ebfeb8ac1303b3b59efe94150a1d2371cb78e3338153f176f521444.XhZKS15jlRkqnRcA2vpvX4Fs7ZEuvE23jm48lIAMqGE
scenes-app-ai-observability-new-evaluation--from-template--dark:
- hash: v1.k794b7964.522cb4a66474488f0a1a5c64f26c09f1beda25de9d3025a7d9d708a5b638fb5e.FTdDwtnvQSREwVqU1obRvn4JMO2BZIA6_hQq53n1gIY
+ hash: v1.k794b7964.07380f3dfb3e23227666db25699716cb64a1ad87908c2f44ee8526d727195662.oibYuMDeIcQcBB8-xT_4kUNOi9WaFpEX734xD-sLBQY
scenes-app-ai-observability-new-evaluation--from-template--light:
- hash: v1.k794b7964.2293fbca6fae992a9b19200cc73bf6ac0a9c06056d24cc4b3ccbcb6a592a6cd5.FGIm1AeCqfdM9Py8enk2QX65UJr1tn8ZlUAlAIG2bUA
+ hash: v1.k794b7964.9c1d8e6148d26b50a85c618819388cc6108f52d5c6d71e40d661f6d403994e63.shdmDJxZKXSPIOJBz_AHBjJ-mDZgf3E0SJjRK7MP0VE
scenes-app-ai-observability-new-evaluation--nothing-matches--dark:
- hash: v1.k794b7964.9042eb2d03e3fad492dfdfc755ffe45d651856cf53a23c1b126820af2af4a157.dsssE0qpI7l2eda6yDFu7etQyalm08flGpQ5pF8Xbs4
+ hash: v1.k794b7964.8de1c6adc167189439f173be655c9ba621d431c569d490186e2edb21b5adab9a.y68wObiGXDm_gk9dpuMWLRhMgcLq9W8w2O3BR23qYzg
scenes-app-ai-observability-new-evaluation--nothing-matches--light:
- hash: v1.k794b7964.e57e16dfee3e4b83740a8d14a8ce425fa22a2bed2410c9c82f8308ab83bf5a81.bMBzNe0faqqHTUOVJ3CMGDuHtbEEw7nRqfcNXQVqgIs
+ hash: v1.k794b7964.19005b94e6654d7728de11e62a833dd04ca2232cf230a0ae15115a5ea803b631.DazOYwUIoHXTkv5XSJlVGspprGh0ALMDe_SxkhllWt8
scenes-app-ai-observability-new-evaluation--reports-open--dark:
- hash: v1.k794b7964.f38dbca5c0f4a4666976c917d4f027924f1dc0d1550775c38654360889cbb23c.byoQrLQhAAMUFDzn9EJZVZPcMCP5MiyIfHEgHD9K-pE
+ hash: v1.k794b7964.7dd06c6e687d65fb2d75ed92dbc344ba3f9277ba17a0107d7f6eb410d31bd721.V9xJdZjH3bGMs9ZxKTtpZ5YJ58Mm1zzcz_UGRpjJZ3M
scenes-app-ai-observability-new-evaluation--reports-open--light:
- hash: v1.k794b7964.421b25300229d08d96da3d1f65a030df1365b26e795695d85478539543e58682._Yjl0PGso1sxk98zetDxwSc6PZoc_uRl637y0px88GA
+ hash: v1.k794b7964.7679f6ae83d3c6db477d8be83a813390bdeffda70944a9b2b3fd93d5c1220bed.mmWadDp5K9oYJGHsYiDnkfu5wDAXFYOqsPTqrgwYT7U
scenes-app-ai-observability-new-evaluation--sentiment-evaluation--dark:
hash: v1.k794b7964.ac47d8aaeb570a78393cc59504adfa570a40a1a544ace57840c0dae235680603.4O7QEnavU1PIlLZw-fCCE7TO-6r3nrqU2WG0_02WyKc
scenes-app-ai-observability-new-evaluation--sentiment-evaluation--light:
diff --git a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
index fdcaa5dbccc5..d61c691aa29d 100644
--- a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
+++ b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
@@ -1,8 +1,7 @@
import { useActions, useValues } from 'kea'
import { combineUrl, router } from 'kea-router'
-import { IconArrowLeft, IconPlay, IconTrends, IconWarning } from '@posthog/icons'
-import { LemonBanner, LemonButton, LemonSkeleton, LemonTabs, LemonTag, Link } from '@posthog/lemon-ui'
+import { LemonBanner, LemonButton, LemonSkeleton, LemonTabs, Link } from '@posthog/lemon-ui'
import { AccessControlAction } from 'lib/components/AccessControlAction'
import { NotFound } from 'lib/components/NotFound'
@@ -11,8 +10,6 @@ import { lemonToast } from 'lib/lemon-ui/LemonToast'
import { pluralize } from 'lib/utils/strings'
import { SceneExport } from 'scenes/sceneTypes'
-import { SceneBreadcrumbBackButton } from '~/layout/scenes/components/SceneBreadcrumbs'
-import { SceneStickyBar } from '~/layout/scenes/components/SceneStickyBar'
import { urls } from '~/scenes/urls'
import { AccessControlLevel, AccessControlResourceType } from '~/types'
@@ -26,6 +23,7 @@ import { EvaluationLatestReport } from './components/EvaluationLatestReport'
import { EvaluationReportsCallout } from './components/EvaluationReportsCallout'
import { EvaluationReportsTab } from './components/EvaluationReportsTab'
import { EvaluationRunsTable } from './components/EvaluationRunsTable'
+import { EvaluationSceneHeader } from './components/EvaluationSceneHeader'
import {
formatNumericEvaluationScore,
booleanOutputConfigError,
@@ -52,7 +50,6 @@ export function AIObservabilityEvaluation(): JSX.Element {
evaluation,
originalEvaluation,
isReportableEvaluation,
- trendInsightUrl,
evaluationBackTarget,
evaluationLoading,
evaluationFormSubmitting,
@@ -182,70 +179,14 @@ export function AIObservabilityEvaluation(): JSX.Element {
return (
<div className="@container/evaluation min-w-0 space-y-6 pb-8">
- <SceneBreadcrumbBackButton />
- {/* Header */}
- <SceneStickyBar
- hasSceneTitleSection={false}
- className="flex flex-col justify-between items-start gap-4 space-y-0 @min-[64rem]/evaluation:flex-row"
- >
- <div className="space-y-2 min-w-0">
- <h1 className="text-2xl font-semibold break-words">{evaluation.name}</h1>
- <div className="flex flex-wrap items-center gap-2">
- {evaluation.status === 'error' ? (
- <LemonTag type="danger" icon={<IconWarning />}>
- Error
- </LemonTag>
- ) : (
- <LemonTag type={evaluation.enabled ? 'success' : 'default'}>
- {evaluation.enabled ? 'Enabled' : 'Disabled'}
- </LemonTag>
- )}
- {unsaved && <LemonTag type="warning">Unsaved changes</LemonTag>}
- </div>
- </div>
- <div className="flex flex-wrap gap-2 max-w-full">
- {trendInsightUrl ? (
- <LemonButton
- type="secondary"
- icon={<IconTrends />}
- to={trendInsightUrl}
- targetBlank
- data-attr="llma-evaluation-trend-insight"
- >
- Trend insight
- </LemonButton>
- ) : null}
- {openInPlaygroundUrl ? (
- <LemonButton
- type="secondary"
- icon={<IconPlay />}
- to={openInPlaygroundUrl}
- data-attr="llma-playground-open-from-evaluation"
- >
- Open in Playground
- </LemonButton>
- ) : null}
- <LemonButton type="secondary" icon={<IconArrowLeft />} onClick={handleCancel}>
- {unsaved ? 'Cancel' : 'Back'}
- </LemonButton>
- {showSaveAction && (
- <AccessControlAction
- resourceType={AccessControlResourceType.Evaluation}
- minAccessLevel={AccessControlLevel.Editor}
- userAccessLevel={evaluation.user_access_level ?? undefined}
- >
- <LemonButton
- type="primary"
- onClick={handleSave}
- disabledReason={saveButtonDisabledReason}
- loading={evaluationFormSubmitting}
- >
- Save changes
- </LemonButton>
- </AccessControlAction>
- )}
- </div>
- </SceneStickyBar>
+ <EvaluationSceneHeader
+ playgroundUrl={openInPlaygroundUrl}
+ unsaved={unsaved}
+ showSaveAction={showSaveAction}
+ saveDisabledReason={saveButtonDisabledReason}
+ onSave={handleSave}
+ onCancel={handleCancel}
+ />
{evaluation.status === 'error' && (
<LemonBanner type="error">
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationAgentActions.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationAgentActions.tsx
index 87c9b2d28c0d..ebe042e7b763 100644
--- a/products/ai_observability/frontend/evaluations/components/EvaluationAgentActions.tsx
+++ b/products/ai_observability/frontend/evaluations/components/EvaluationAgentActions.tsx
@@ -65,6 +65,7 @@ export function EvaluationAgentActions(): JSX.Element | null {
actions={[{ key: 'prompt', label: 'prompt', buildPrompt: () => prompt }]}
labelMode="destination"
appearance="lemon"
+ excludeAgentKeys={['posthog-code']}
onRun={({ agentKey }) => reportAgentOpened(SURFACE_KEY, 'card', agentKey)}
data-attr="mcp-hint-card-agent-button"
/>
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationFirstResultStatus.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationFirstResultStatus.tsx
index f7ccf09a96cf..2f941ea4f5d1 100644
--- a/products/ai_observability/frontend/evaluations/components/EvaluationFirstResultStatus.tsx
+++ b/products/ai_observability/frontend/evaluations/components/EvaluationFirstResultStatus.tsx
@@ -3,8 +3,11 @@ import { BindLogic, useActions, useMountedLogic, useValues } from 'kea'
import { LemonBanner } from '@posthog/lemon-ui'
import { dayjs } from 'lib/dayjs'
+import { getAccessControlDisabledReason } from 'lib/utils/accessControlUtils'
import { humanFriendlyDuration } from 'lib/utils/durations'
+import { AccessControlLevel, AccessControlResourceType } from '~/types'
+
import { settleStrategy } from '../constants'
import { VOLUME_LOOKBACK_DAYS, evaluationTarget, runRate } from '../evaluationVolume'
import { evaluationVolumeLogic } from '../evaluationVolumeLogic'
@@ -47,19 +50,27 @@ export function EvaluationFirstResultStatus(): JSX.Element | null {
}
function PausedBanner({ evaluation }: { evaluation: EvaluationConfig }): JSX.Element {
- const { setActiveTab } = useActions(llmEvaluationLogic)
+ const { enabledSaving, enableSavedReason } = useValues(llmEvaluationLogic)
+ const { saveEnabled } = useActions(llmEvaluationLogic)
+ const accessReason = getAccessControlDisabledReason(
+ AccessControlResourceType.Evaluation,
+ AccessControlLevel.Editor,
+ evaluation.user_access_level ?? undefined
+ )
return (
<LemonBanner
type="info"
className="mb-4"
action={{
- children: 'Open configuration',
- onClick: () => setActiveTab('configuration'),
- 'data-attr': 'llma-evaluation-first-result-triggers',
+ children: 'Turn on',
+ onClick: () => saveEnabled(true),
+ loading: enabledSaving,
+ disabledReason: accessReason ?? enableSavedReason ?? undefined,
+ 'data-attr': 'llma-evaluation-first-result-turn-on',
}}
>
- <span className="font-semibold">No results yet, because this evaluation is paused.</span> Turn it on in its
- configuration to start checking new {UNIT_NOUNS[evaluationTarget(evaluation)]}.
+ <span className="font-semibold">No results yet, because this evaluation is paused.</span> Turn it on to
+ start checking new {UNIT_NOUNS[evaluationTarget(evaluation)]}.
</LemonBanner>
)
}
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationRunsTable.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationRunsTable.tsx
index 3323a73d1785..1e573b680616 100644
--- a/products/ai_observability/frontend/evaluations/components/EvaluationRunsTable.tsx
+++ b/products/ai_observability/frontend/evaluations/components/EvaluationRunsTable.tsx
@@ -1,6 +1,6 @@
import { useActions, useValues } from 'kea'
-import { IconRefresh } from '@posthog/icons'
+import { IconRefresh, IconTrends } from '@posthog/icons'
import { LemonBanner, LemonButton, LemonSegmentedButton, LemonTable, LemonTag } from '@posthog/lemon-ui'
import { DateFilter } from 'lib/components/DateFilter/DateFilter'
@@ -52,6 +52,25 @@ function SentimentEvaluationRunsFilters(): JSX.Element {
)
}
+function TrendInsightButton(): JSX.Element | null {
+ const { trendInsightUrl } = useValues(llmEvaluationLogic)
+ if (!trendInsightUrl) {
+ return null
+ }
+ return (
+ <LemonButton
+ type="secondary"
+ icon={<IconTrends />}
+ to={trendInsightUrl}
+ targetBlank
+ size="small"
+ data-attr="llma-evaluation-trend-insight"
+ >
+ Trend insight
+ </LemonButton>
+ )
+}
+
export function EvaluationRunsTable(): JSX.Element {
const {
filteredEvaluationRuns,
@@ -207,6 +226,7 @@ export function EvaluationRunsTable(): JSX.Element {
)}
</>
)}
+ <TrendInsightButton />
<LemonButton
type="secondary"
icon={<IconRefresh />}
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationSceneHeader.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationSceneHeader.tsx
new file mode 100644
index 000000000000..f753676b20e1
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/components/EvaluationSceneHeader.tsx
@@ -0,0 +1,146 @@
+import { useActions, useValues } from 'kea'
+
+import { IconPause, IconPlay, IconWarning } from '@posthog/icons'
+import { LemonButton, LemonTag } from '@posthog/lemon-ui'
+
+import { AccessControlAction } from 'lib/components/AccessControlAction'
+import { FEATURE_FLAGS } from 'lib/constants'
+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
+
+import { SceneTitleSection } from '~/layout/scenes/components/SceneTitleSection'
+import { AccessControlLevel, AccessControlResourceType } from '~/types'
+
+import { llmEvaluationLogic } from '../llmEvaluationLogic'
+import { EvaluationSceneMenuBar } from './EvaluationSceneMenuBar'
+
+function StatusTags({
+ errored,
+ running,
+ unsaved,
+}: {
+ errored: boolean
+ running: boolean
+ unsaved: boolean
+}): JSX.Element {
+ return (
+ <>
+ {errored ? (
+ <LemonTag type="danger" icon={<IconWarning />}>
+ Error
+ </LemonTag>
+ ) : (
+ <LemonTag type={running ? 'success' : 'default'}>{running ? 'Running' : 'Paused'}</LemonTag>
+ )}
+ {unsaved && <LemonTag type="warning">Unsaved changes</LemonTag>}
+ </>
+ )
+}
+
+function RunStateButton({
+ running,
+ accessLevel,
+}: {
+ running: boolean
+ accessLevel: AccessControlLevel | undefined
+}): JSX.Element {
+ const { enabledSaving, enableSavedReason } = useValues(llmEvaluationLogic)
+ const { saveEnabled } = useActions(llmEvaluationLogic)
+ return (
+ <AccessControlAction
+ resourceType={AccessControlResourceType.Evaluation}
+ minAccessLevel={AccessControlLevel.Editor}
+ userAccessLevel={accessLevel}
+ >
+ <LemonButton
+ type="secondary"
+ size="small"
+ icon={running ? <IconPause /> : <IconPlay />}
+ onClick={() => saveEnabled(!running)}
+ loading={enabledSaving}
+ disabledReason={running ? undefined : (enableSavedReason ?? undefined)}
+ data-attr="llma-evaluation-header-enabled"
+ >
+ {running ? 'Pause' : 'Turn on'}
+ </LemonButton>
+ </AccessControlAction>
+ )
+}
+
+/** The saved evaluation's title row, its run state, and the actions that change it. */
+export function EvaluationSceneHeader({
+ playgroundUrl,
+ unsaved,
+ showSaveAction,
+ saveDisabledReason,
+ onSave,
+ onCancel,
+}: {
+ playgroundUrl: string | null
+ unsaved: boolean
+ showSaveAction: boolean
+ saveDisabledReason: string | undefined
+ onSave: () => void
+ onCancel: () => void
+}): JSX.Element | null {
+ const { evaluation, originalEvaluation, evaluationFormSubmitting } = useValues(llmEvaluationLogic)
+ const { featureFlags } = useValues(featureFlagLogic)
+ const sceneMenuBarEnabled = !!featureFlags[FEATURE_FLAGS.SCENE_MENU_BAR]
+
+ if (!evaluation) {
+ return null
+ }
+ // Pause and Turn on save at once, so they follow the saved state, not an unsaved edit from the configuration tab.
+ const running = originalEvaluation?.enabled ?? evaluation.enabled
+ const accessLevel = evaluation.user_access_level ?? undefined
+
+ return (
+ <>
+ {sceneMenuBarEnabled && <EvaluationSceneMenuBar playgroundUrl={playgroundUrl} />}
+ <SceneTitleSection
+ name={evaluation.name}
+ description={evaluation.description || null}
+ resourceType={{ type: 'llm_evaluations' }}
+ hideProductSetupButton
+ nameSuffix={<StatusTags errored={evaluation.status === 'error'} running={running} unsaved={unsaved} />}
+ actions={
+ <>
+ {!sceneMenuBarEnabled && playgroundUrl && (
+ <LemonButton
+ type="secondary"
+ size="small"
+ icon={<IconPlay />}
+ to={playgroundUrl}
+ data-attr="llma-playground-open-from-evaluation"
+ >
+ Open in Playground
+ </LemonButton>
+ )}
+ <RunStateButton running={running} accessLevel={accessLevel} />
+ {unsaved && (
+ <LemonButton type="secondary" size="small" onClick={onCancel}>
+ Cancel
+ </LemonButton>
+ )}
+ {showSaveAction && (
+ <AccessControlAction
+ resourceType={AccessControlResourceType.Evaluation}
+ minAccessLevel={AccessControlLevel.Editor}
+ userAccessLevel={accessLevel}
+ >
+ <LemonButton
+ type="primary"
+ size="small"
+ onClick={onSave}
+ disabledReason={saveDisabledReason}
+ loading={evaluationFormSubmitting}
+ >
+ Save changes
+ </LemonButton>
+ </AccessControlAction>
+ )}
+ </>
+ }
+ />
+ </>
+ )
+}
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationSceneMenuBar.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationSceneMenuBar.tsx
new file mode 100644
index 000000000000..c3bc81372868
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/components/EvaluationSceneMenuBar.tsx
@@ -0,0 +1,92 @@
+import { useActions, useValues } from 'kea'
+import { router } from 'kea-router'
+
+import { IconGear, IconPause, IconPlay, IconTrends } from '@posthog/icons'
+
+import { AccessControlAction } from 'lib/components/AccessControlAction'
+import { sceneFileLogic } from 'lib/components/Scenes/sceneFileLogic'
+import { SceneMenuBarFileItems } from 'lib/components/Scenes/SceneMenuBarFileItems'
+
+import {
+ SceneMenuBar,
+ SceneMenuBarItem,
+ SceneMenuBarMenu,
+ SceneMenuBarSeparator,
+} from '~/layout/scenes/components/SceneMenuBar'
+import { AccessControlLevel, AccessControlResourceType } from '~/types'
+
+import { llmEvaluationLogic } from '../llmEvaluationLogic'
+
+/** The evaluation page's menu bar. The caller renders it only behind the scene menu bar flag. */
+export function EvaluationSceneMenuBar({ playgroundUrl }: { playgroundUrl: string | null }): JSX.Element | null {
+ const { originalEvaluation, enabledSaving, enableSavedReason, trendInsightUrl } = useValues(llmEvaluationLogic)
+ const { saveEnabled, setActiveTab } = useActions(llmEvaluationLogic)
+ const { hasFileItems } = useValues(sceneFileLogic)
+
+ if (!originalEvaluation) {
+ return null
+ }
+ const running = originalEvaluation.enabled
+
+ return (
+ <SceneMenuBar>
+ <SceneMenuBarMenu label="File" dataAttr="llma-evaluation-menubar-file" disabled={!hasFileItems}>
+ {hasFileItems && <SceneMenuBarFileItems dataAttrKey="llm_evaluation" />}
+ </SceneMenuBarMenu>
+ <SceneMenuBarMenu label="Edit" dataAttr="llma-evaluation-menubar-edit">
+ <SceneMenuBarItem
+ onClick={() => setActiveTab('configuration')}
+ data-attr="llma-evaluation-menubar-configure"
+ >
+ <IconGear />
+ Edit configuration
+ </SceneMenuBarItem>
+ <SceneMenuBarSeparator />
+ <AccessControlAction
+ resourceType={AccessControlResourceType.Evaluation}
+ minAccessLevel={AccessControlLevel.Editor}
+ userAccessLevel={originalEvaluation.user_access_level ?? undefined}
+ >
+ {({ disabledReason }) => {
+ const reason = disabledReason ?? (running ? null : enableSavedReason)
+ return (
+ <SceneMenuBarItem
+ onClick={() => saveEnabled(!running)}
+ disabled={!!reason || enabledSaving}
+ tooltip={reason ?? undefined}
+ data-attr="llma-evaluation-menubar-enabled"
+ >
+ {running ? <IconPause /> : <IconPlay />}
+ {running ? 'Pause' : 'Turn on'}
+ </SceneMenuBarItem>
+ )
+ }}
+ </AccessControlAction>
+ </SceneMenuBarMenu>
+ <SceneMenuBarMenu
+ label="View"
+ dataAttr="llma-evaluation-menubar-view"
+ disabled={!trendInsightUrl && !playgroundUrl}
+ >
+ {trendInsightUrl && (
+ <SceneMenuBarItem
+ onClick={() => window.open(trendInsightUrl, '_blank', 'noopener,noreferrer')}
+ data-attr="llma-evaluation-menubar-trend-insight"
+ >
+ <IconTrends />
+ Trend insight
+ </SceneMenuBarItem>
+ )}
+ {playgroundUrl && (
+ <SceneMenuBarItem
+ onClick={() => router.actions.push(playgroundUrl)}
+ data-attr="llma-evaluation-menubar-playground"
+ >
+ <IconPlay />
+ Open in Playground
+ </SceneMenuBarItem>
+ )}
+ </SceneMenuBarMenu>
+ </SceneMenuBar>
+ )
+}
diff --git a/products/ai_observability/frontend/evaluations/components/templateSummary.ts b/products/ai_observability/frontend/evaluations/components/templateSummary.ts
index bf5dd26e996f..8f502f88c140 100644
--- a/products/ai_observability/frontend/evaluations/components/templateSummary.ts
+++ b/products/ai_observability/frontend/evaluations/components/templateSummary.ts
@@ -24,7 +24,7 @@ export function templateSummary(template: EvaluationTemplate): string {
}
/** One line that says what a run returns, e.g. "Score from 1 to 5". */
-export function templateOutputText(template: EvaluationTemplate): string {
+export function templateOutputText(template: Pick<EvaluationTemplate, 'output_type' | 'output_config'>): string {
const { min, max, options = [], selection_mode } = template.output_config
switch (template.output_type) {
case 'boolean':
diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.tsx
index fc15b46bc4f4..133eeea9e2ea 100644
--- a/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.tsx
+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.tsx
@@ -3,8 +3,6 @@ import { router } from 'kea-router'
import posthog from 'posthog-js'
import { useMemo, useRef } from 'react'
-import { LemonTag } from '@posthog/lemon-ui'
-
import { lemonToast } from 'lib/lemon-ui/LemonToast'
import { SceneBreadcrumbBackButton } from '~/layout/scenes/components/SceneBreadcrumbs'
@@ -107,10 +105,9 @@ export function EvaluationCreatePage({ templateKey }: { templateKey?: string }):
hasSceneTitleSection={false}
className="flex flex-wrap items-center justify-between gap-2"
>
- <div className="flex flex-wrap items-center gap-2">
- <h1 className="mb-0 text-2xl font-semibold">New evaluation</h1>
- {template && <LemonTag type="muted">{`From the ${template.name} template`}</LemonTag>}
- </div>
+ <h1 className="mb-0 text-2xl font-semibold">
+ {template ? `New ${template.name.toLowerCase()} evaluation` : 'New evaluation'}
+ </h1>
</SceneStickyBar>
<div className="w-full max-w-4xl mx-auto">
<EvaluationFormSteps checklist={checklist} onCreate={handleCreate} onCancel={handleCancel} />
diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCreateReview.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCreateReview.tsx
index 05528c3f43bc..0b779ab0bc83 100644
--- a/products/ai_observability/frontend/evaluations/create/EvaluationCreateReview.tsx
+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCreateReview.tsx
@@ -1,23 +1,17 @@
-import { useActions, useMountedLogic, useValues } from 'kea'
+import { useActions, useValues } from 'kea'
-import { IconCheckCircle, IconCircleDashed, IconClock, IconNotification, IconSparkles } from '@posthog/icons'
-import { LemonSwitch, Link } from '@posthog/lemon-ui'
+import { IconCheckCircle, IconCircleDashed } from '@posthog/icons'
+import { LemonCheckbox, Link } from '@posthog/lemon-ui'
import { cn } from 'lib/utils/css-classes'
-import { capitalizeFirstLetter } from 'lib/utils/strings'
+import { pluralize } from 'lib/utils/strings'
-import { PROVIDER_SETTINGS_URL } from '../../ModelPicker'
-import { buildDeliveryTargets, evaluationReportLogic } from '../evaluationReportLogic'
-import { runRate } from '../evaluationVolume'
+import { VOLUME_LOOKBACK_DAYS } from '../evaluationVolume'
import { evaluationVolumeLogic } from '../evaluationVolumeLogic'
import { llmEvaluationLogic } from '../llmEvaluationLogic'
import type { ChecklistItem, ChecklistKey } from './creationChecklist'
-import { reportText } from './evaluationSummary'
-/**
- * What the evaluation needs before it can be created, as a checklist, next to what it will do once it runs:
- * how often, at what cost per run, where reports go, and whether it starts running.
- */
+/** What the evaluation needs before it can be created, next to what happens once it is created. */
export function EvaluationCreateReview({
checklist,
onFix,
@@ -26,60 +20,29 @@ export function EvaluationCreateReview({
/** Goes to the step that sets an item. */
onFix: (key: ChecklistKey) => void
}): JSX.Element | null {
- const { props } = useMountedLogic(llmEvaluationLogic)
- const { evaluation, canEnable, canEnableReason, isReportableEvaluation, selectedModel, selectedModelName } =
- useValues(llmEvaluationLogic)
- const { setEvaluationEnabled } = useActions(llmEvaluationLogic)
+ const { evaluation, canEnableReason, checkPastTraffic } = useValues(llmEvaluationLogic)
+ const { setCheckPastTraffic } = useActions(llmEvaluationLogic)
const { volume } = useValues(evaluationVolumeLogic)
- const { configDraft } = useValues(evaluationReportLogic({ evaluationId: props.evaluationId }))
if (!evaluation) {
return null
}
- const destinations = buildDeliveryTargets(configDraft).length
- const reportsOn = configDraft.enabled && isReportableEvaluation
- const facts: { key: string; icon: JSX.Element; title: string; detail: string }[] = [
- ...(volume
- ? [
- {
- key: 'volume',
- icon: <IconClock />,
- title: capitalizeFirstLetter(runRate(volume.runs)),
- detail: "From last week's matching traffic",
- },
- ]
- : []),
- {
- key: 'judge',
- icon: <IconSparkles />,
- title:
- evaluation.evaluation_type === 'llm_judge'
- ? `Judged by ${selectedModelName ?? (selectedModel || 'the model you pick')}`
- : 'No AI key needed',
- detail:
- evaluation.evaluation_type === 'llm_judge'
- ? 'Uses your own AI provider key'
- : evaluation.evaluation_type === 'hog'
- ? 'Your code decides each result'
- : 'A built-in classifier decides each result',
- },
- {
- key: 'reports',
- icon: <IconNotification />,
- title: reportsOn
- ? `${capitalizeFirstLetter(reportText({ ...configDraft, enabled: true }))} reports`
- : 'No reports',
- detail: reportsOn
- ? destinations > 0
- ? `Sent to ${destinations === 1 ? 'one destination' : `${destinations} destinations`}`
- : 'In the Reports tab'
- : 'Turn them on in the Reports step',
- },
- ]
+ // Unknown volume (still counting, or a filter on content) still offers it; only a known zero hides it.
+ const hasPastTraffic = volume?.runs !== 0
+ const units = volume ? pluralize(volume.matchingUnits, volume.target) : null
+ // Below 100% sampling the exact count depends on which units the sample picks, so it names the pool instead.
+ const pastTrafficCount = !volume
+ ? ''
+ : volume.runs === volume.matchingUnits
+ ? ` It will evaluate ${units}.`
+ : ` It will evaluate a sample of the ${units} that matched.`
+ const pastTrafficDetail = `See results right away instead of waiting for new traffic.${pastTrafficCount}${
+ evaluation.evaluation_type === 'llm_judge' ? ' Each one will be a call to your judge model.' : ''
+ }`
const done = checklist.filter((item) => item.done).length
return (
- <div className="grid grid-cols-1 gap-4 @min-[40rem]/evaluation:grid-cols-2">
+ <div className="flex flex-col gap-3 w-full max-w-xl mx-auto">
<div className="border rounded p-4 flex flex-col gap-3">
<div className="flex items-center justify-between gap-2">
<h4 className="mb-0 text-sm font-semibold">Checklist</h4>
@@ -95,55 +58,31 @@ export function EvaluationCreateReview({
)}
<span className="flex flex-col min-w-0">
<span className={cn(item.done ? 'text-secondary' : 'font-semibold')}>{item.label}</span>
- {!item.done &&
- (item.fixInSettings ? (
- <Link
- to={PROVIDER_SETTINGS_URL}
- target="_blank"
- data-attr="llma-evaluation-create-add-key"
- >
- {item.todo}
- </Link>
- ) : (
- <Link onClick={() => onFix(item.key)}>{item.todo}</Link>
- ))}
+ {item.done && item.summary && (
+ <span className="text-xs text-secondary truncate">{item.summary}</span>
+ )}
+ {!item.done && <Link onClick={() => onFix(item.key)}>{item.todo}</Link>}
</span>
</li>
))}
</ul>
</div>
- <div className="border rounded p-4 flex flex-col gap-3" data-attr="llma-evaluation-create-glance">
- <h4 className="mb-0 text-sm font-semibold">At a glance</h4>
- <ul className="m-0 p-0 list-none flex flex-col gap-3">
- {facts.map((fact) => (
- <li key={fact.key} className="flex items-start gap-3">
- <span className="flex size-8 shrink-0 items-center justify-center rounded bg-fill-highlight-100 text-lg text-secondary">
- {fact.icon}
- </span>
- <span className="flex flex-col min-w-0">
- <span className="text-sm font-semibold">{fact.title}</span>
- <span className="text-xs text-secondary">{fact.detail}</span>
- </span>
- </li>
- ))}
- </ul>
- <div className="mt-auto flex flex-col gap-1 pt-2">
- <LemonSwitch
- checked={evaluation.enabled}
- onChange={setEvaluationEnabled}
- disabledReason={!canEnable && !evaluation.enabled ? (canEnableReason ?? undefined) : undefined}
- label="Start running when created"
- bordered
- fullWidth
- data-attr="llma-evaluation-create-enabled"
+ {evaluation.enabled && hasPastTraffic && (
+ <div className="border rounded px-4 py-3 flex flex-col gap-1">
+ <LemonCheckbox
+ checked={checkPastTraffic}
+ onChange={setCheckPastTraffic}
+ label={`Also evaluate the last ${VOLUME_LOOKBACK_DAYS} days`}
+ data-attr="llma-evaluation-create-past-traffic"
/>
- <span className="text-xs text-secondary">
- {evaluation.enabled
- ? 'It checks new matching traffic as soon as you create it.'
- : 'It stays paused. Turn it on from its page when you are ready.'}
- </span>
+ <span className="text-xs text-secondary pl-6">{pastTrafficDetail}</span>
</div>
- </div>
+ )}
+ {!evaluation.enabled && (
+ <p className="mb-0 text-sm text-secondary">
+ {`It is created paused. ${canEnableReason ?? ''} Then turn it on from its page.`}
+ </p>
+ )}
</div>
)
}
diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx
index 7bc119608d52..ffb10944a7e0 100644
--- a/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx
+++ b/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx
@@ -8,7 +8,7 @@ import * as officeWorkerPng from '@posthog/brand/hoggies/png/office-worker'
import * as reporterPng from '@posthog/brand/hoggies/png/reporter'
import * as stampApprovedPng from '@posthog/brand/hoggies/png/stamp-approved'
import { IconCheck } from '@posthog/icons'
-import { LemonButton, LemonSwitch } from '@posthog/lemon-ui'
+import { LemonButton } from '@posthog/lemon-ui'
import { pngHoggie } from 'lib/brand/hoggies'
import { cn } from 'lib/utils/css-classes'
@@ -96,33 +96,15 @@ const STEPS: Step[] = [
},
]
-function StepContent({ step, isNew }: { step: Step['key']; isNew: boolean }): JSX.Element | null {
- const { evaluation, canEnable, canEnableReason } = useValues(llmEvaluationLogic)
- const { setEvaluationEnabled } = useActions(llmEvaluationLogic)
+function StepContent({ step }: { step: Step['key'] }): JSX.Element | null {
+ const { evaluation } = useValues(llmEvaluationLogic)
if (!evaluation) {
return null
}
switch (step) {
case 'name':
- return (
- <>
- <EvaluationNameFields />
- {/* A new evaluation sets this on its create step. */}
- {!isNew && (
- <LemonSwitch
- checked={evaluation.enabled}
- onChange={setEvaluationEnabled}
- disabledReason={
- !canEnable && !evaluation.enabled ? (canEnableReason ?? undefined) : undefined
- }
- label={evaluation.enabled ? 'Running on new traffic' : 'Paused'}
- bordered
- data-attr="llma-evaluation-configuration-enabled"
- />
- )}
- </>
- )
+ return <EvaluationNameFields />
case 'check':
return (
<>
@@ -269,7 +251,7 @@ export function EvaluationFormSteps({
</div>
{!isCreateStep && (
<div className="flex flex-col gap-4 min-w-0">
- <StepContent step={step.key} isNew={!!onCreate} />
+ <StepContent step={step.key} />
</div>
)}
{isCreateStep && (
diff --git a/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts b/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts
index ec058e4bf4be..c939d1fb193b 100644
--- a/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts
+++ b/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts
@@ -43,6 +43,21 @@ describe('creationChecklist', () => {
expect(firstTodo(overrides)).toBe(expected)
})
+ it('sums up each done item, with only the first line of a long prompt', () => {
+ const evaluation = {
+ ...judge('Is it helpful?\nAnswer true or false.'),
+ target: 'generation',
+ model_configuration: { provider: 'openai', model: 'gpt-5-mini', provider_key_id: 'key-1' },
+ } as EvaluationConfig
+ expect(creationChecklist(input({ evaluation })).map((item) => item.summary)).toEqual([
+ 'Helpfulness',
+ 'Is it helpful?',
+ 'Pass or fail',
+ 'gpt-5-mini',
+ '100% of all generations',
+ ])
+ })
+
it('asks for no prompt, result or model on a sentiment evaluation', () => {
const sentiment = { ...judge(''), evaluation_type: 'sentiment', output_type: 'sentiment' } as EvaluationConfig
expect(
diff --git a/products/ai_observability/frontend/evaluations/create/creationChecklist.ts b/products/ai_observability/frontend/evaluations/create/creationChecklist.ts
index 89644519ba80..4d59a0424a56 100644
--- a/products/ai_observability/frontend/evaluations/create/creationChecklist.ts
+++ b/products/ai_observability/frontend/evaluations/create/creationChecklist.ts
@@ -1,5 +1,9 @@
+import { capitalizeFirstLetter, pluralize } from 'lib/utils/strings'
+
+import { templateOutputText } from '../components/templateSummary'
import { categoricalOutputConfigError } from '../constants'
import type { EvaluationConfig, EvaluationOutputConfig } from '../types'
+import { triggerText } from './evaluationSummary'
export type ChecklistKey = 'name' | 'criteria' | 'result' | 'model' | 'sampling'
@@ -9,8 +13,8 @@ export interface ChecklistItem {
done: boolean
/** What to do next when the item is not done. Also the reason the create button is disabled. */
todo: string
- /** Set when the fix lives on another page, such as adding a provider key in settings. */
- fixInSettings?: boolean
+ /** What was chosen, shown once the item is done. */
+ summary?: string
}
export interface ChecklistInput {
@@ -33,14 +37,28 @@ const CRITERIA: Record<'llm_judge' | 'hog', { label: string; todo: string }> = {
export function creationChecklist(input: ChecklistInput): ChecklistItem[] {
const { evaluation } = input
const items: ChecklistItem[] = [
- { key: 'name', label: 'Name', done: evaluation.name.trim().length > 0, todo: 'Give the evaluation a name' },
+ {
+ key: 'name',
+ label: 'Name',
+ done: evaluation.name.trim().length > 0,
+ todo: 'Give the evaluation a name',
+ summary: evaluation.name.trim(),
+ },
]
if (evaluation.evaluation_type === 'llm_judge' || evaluation.evaluation_type === 'hog') {
const content =
evaluation.evaluation_type === 'llm_judge'
? evaluation.evaluation_config.prompt
: evaluation.evaluation_config.source
- items.push({ key: 'criteria', done: content.trim().length > 0, ...CRITERIA[evaluation.evaluation_type] })
+ items.push({
+ key: 'criteria',
+ done: content.trim().length > 0,
+ summary:
+ evaluation.evaluation_type === 'llm_judge'
+ ? content.trim().split('\n')[0]
+ : pluralize(content.trim().split('\n').length, 'line') + ' of code',
+ ...CRITERIA[evaluation.evaluation_type],
+ })
}
if (evaluation.evaluation_type !== 'sentiment') {
items.push({
@@ -48,6 +66,7 @@ export function creationChecklist(input: ChecklistInput): ChecklistItem[] {
label: 'Result',
done: input.resultError === null,
todo: input.resultError ?? '',
+ summary: templateOutputText(evaluation),
})
}
if (input.modelSelectionRequired) {
@@ -58,7 +77,7 @@ export function creationChecklist(input: ChecklistInput): ChecklistItem[] {
todo: input.hasAvailableModels
? 'Pick the model that judges each run'
: 'Add an API key for an AI provider, then pick a model',
- fixInSettings: !input.hasAvailableModels,
+ summary: evaluation.model_configuration?.model,
})
}
items.push({
@@ -66,6 +85,7 @@ export function creationChecklist(input: ChecklistInput): ChecklistItem[] {
label: 'Sampling',
done: !input.hasUnsetSampling,
todo: 'Set a sampling percentage between 0.1% and 100%',
+ summary: capitalizeFirstLetter(triggerText(evaluation)),
})
return items
}
diff --git a/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.test.ts b/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.test.ts
index f4bed492358c..a3a3ee384cea 100644
--- a/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.test.ts
+++ b/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.test.ts
@@ -1,5 +1,6 @@
import { MOCK_DEFAULT_TEAM } from 'lib/api.mock'
+import { router } from 'kea-router'
import { expectLogic } from 'kea-test-utils'
import posthog from 'posthog-js'
@@ -21,6 +22,7 @@ import {
import type { EvaluationBackfillApi, EvaluationBackfillEstimateApi } from '../generated/api.schemas'
import { evaluationBackfillsLogic } from './evaluationBackfillsLogic'
import { llmEvaluationLogic } from './llmEvaluationLogic'
+import { requestBackfillStart } from './pendingBackfillStarts'
import type { EvaluationConfig } from './types'
jest.mock('../generated/api', () => ({
@@ -461,6 +463,35 @@ describe('evaluationBackfillsLogic', () => {
expect(logic.cache.disposables.registry.has('backfillPoll')).toBe(false)
})
+ it('starts the default range once counted when a create asked for it, and only once', async () => {
+ requestBackfillStart(EVALUATION_ID)
+ mountLogic()
+ await expectLogic(logic).toDispatchActions(['requestEstimateSuccess', 'createBackfill', 'createBackfillDone'])
+ expect(createMock).toHaveBeenCalledTimes(1)
+
+ logic.actions.requestEstimateSuccess(estimate())
+ await expectLogic(logic).toFinishAllListeners()
+ expect(createMock).toHaveBeenCalledTimes(1)
+ })
+
+ it('starts nothing from a link that only claims a start was requested', async () => {
+ router.actions.push(router.values.location.pathname, { backfill_start: 'requested' })
+ mountLogic()
+ await expectLogic(logic).toDispatchActions(['requestEstimateSuccess'])
+ await expectLogic(logic).toFinishAllListeners()
+ expect(createMock).not.toHaveBeenCalled()
+ })
+
+ it('still starts a requested backfill after leaving the tab before the count landed', async () => {
+ requestBackfillStart(EVALUATION_ID)
+ mountLogic()
+ logic.unmount()
+
+ mountLogic()
+ await expectLogic(logic).toDispatchActions(['createBackfill', 'createBackfillDone'])
+ expect(createMock).toHaveBeenCalledTimes(1)
+ })
+
it('keeps polling a finished run until its coverage lands', async () => {
listMock.mockResolvedValueOnce({ count: 1, results: [backfill({ status: 'running' })] })
mountLogic()
diff --git a/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.ts b/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.ts
index e0ec98516221..955b7be8e340 100644
--- a/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.ts
+++ b/products/ai_observability/frontend/evaluations/evaluationBackfillsLogic.ts
@@ -25,6 +25,7 @@ import type {
import { evaluationErrorMessage } from './apiErrors'
import { backfillRangeDateFormat } from './backfillConditions'
import { hasUnsetConditionRollout, llmEvaluationLogic, normalizeConditionRollouts } from './llmEvaluationLogic'
+import { takeBackfillStart } from './pendingBackfillStarts'
import type { EvaluationConditionSet } from './types'
import type { EvaluationConfig } from './types'
@@ -556,6 +557,12 @@ export const evaluationBackfillsLogic = kea<evaluationBackfillsLogicType>([
},
loadBackfillsSuccess: schedulePoll,
loadBackfillsFailure: schedulePoll,
+ // A count cut short by leaving the tab never lands here, so the request waits for the next visit.
+ requestEstimateSuccess: () => {
+ if (takeBackfillStart(props.evaluationId) && !values.startDisabledReason) {
+ actions.createBackfill()
+ }
+ },
seedConditions: () => actions.requestEstimate(),
setWindowRange: () => actions.requestEstimate(),
setConditions: () => actions.requestEstimate(),
@@ -624,7 +631,7 @@ export const evaluationBackfillsLogic = kea<evaluationBackfillsLogicType>([
// The units this run covers now have a result, so the count it started from is stale.
cache.lastEstimate = null
cache.recountWhenIdle = true
- lemonToast.success('Backfill started')
+ lemonToast.success('Backfill started', { autoClose: 15000 })
actions.loadBackfills()
} catch (error) {
lemonToast.error(backfillErrorMessage(error, 'Couldn’t start the backfill. Try again.'))
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
index 2a6f607f5635..95df79953e4b 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
@@ -17,6 +17,7 @@ import { EVALUATION_RUNS_QUERY_LIMIT, numericScorePasses } from './constants'
import { evaluationReportLogic } from './evaluationReportLogic'
import { DEFAULT_HOG_SOURCE, llmEvaluationLogic } from './llmEvaluationLogic'
import { llmEvaluationsLogic } from './llmEvaluationsLogic'
+import { takeBackfillStart } from './pendingBackfillStarts'
import { EvaluationConfig, EvaluationReport, EvaluationRun } from './types'
const mockProviderKeys: LLMProviderKey[] = [
@@ -285,16 +286,6 @@ describe('llmEvaluationLogic', () => {
expect(logic.values.evaluation?.enabled).toBe(expected)
})
- it('keeps a new evaluation paused once someone turned it off by hand', async () => {
- await expectLogic(keysLogic).toFinishAllListeners()
- logic.actions.setEvaluationType('hog')
- logic.actions.setEvaluationEnabled(false)
- // Through a judge and back, so canEnable changes and the default would turn it on again.
- logic.actions.setEvaluationType('llm_judge')
- logic.actions.setEvaluationType('hog')
- expect(logic.values.evaluation?.enabled).toBe(false)
- })
-
it.each([
[DEFAULT_HOG_SOURCE, 'return 0;'],
['return 42;', 'return 42;'],
@@ -1696,16 +1687,27 @@ return result`,
})
})
- it('setEvaluationType switches back to llm_judge config shape', async () => {
+ it('switching method and back keeps the prompt and code already typed', async () => {
await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])
+ logic.actions.setEvaluationPrompt('Is the answer polite?')
logic.actions.setEvaluationType('hog')
+ logic.actions.setHogSource('return true;')
logic.actions.setEvaluationType('llm_judge')
await expectLogic(logic).toMatchValues({
evaluation: expect.objectContaining({
evaluation_type: 'llm_judge',
- evaluation_config: { prompt: '' },
+ evaluation_config: { prompt: 'Is the answer polite?' },
+ }),
+ })
+
+ logic.actions.setEvaluationType('hog')
+
+ await expectLogic(logic).toMatchValues({
+ evaluation: expect.objectContaining({
+ evaluation_type: 'hog',
+ evaluation_config: { source: 'return true;' },
}),
})
})
@@ -2292,11 +2294,58 @@ return result`,
})
})
+ describe('saveEvaluation past traffic', () => {
+ it.each([
+ { enabled: true, expectedTab: 'backfills', requestsStart: true, offersPastData: false },
+ { enabled: false, expectedTab: null, requestsStart: false, offersPastData: true },
+ ])(
+ 'a new evaluation created enabled=$enabled asks the Backfills tab to start: $requestsStart',
+ async ({ enabled, expectedTab, requestsStart, offersPastData }) => {
+ const pushSpy = jest.spyOn(router.actions, 'push').mockImplementation(() => {})
+ const infoSpy = jest.spyOn(lemonToast, 'info')
+ let backfillRequests = 0
+ useMocks({
+ post: {
+ '/api/projects/:teamId/evaluations/': () => ({ ...mockSentimentEvaluation, enabled }),
+ '/api/projects/:teamId/evaluations/:id/backfills/': () => {
+ backfillRequests += 1
+ return [201, {}]
+ },
+ },
+ })
+ logic = llmEvaluationLogic({ evaluationId: 'new', evaluationType: 'sentiment' })
+ logic.mount()
+
+ try {
+ await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])
+ logic.actions.setEvaluationName('Sentiment evaluation')
+ logic.actions.setCheckPastTraffic(true)
+ logic.actions.saveEvaluation()
+ await expectLogic(logic).toDispatchActions(['saveEvaluationSuccess'])
+ await expectLogic(logic).toFinishAllListeners()
+
+ expect(backfillRequests).toBe(0)
+ const params = new URL(String(pushSpy.mock.calls.at(-1)?.[0]), 'http://x').searchParams
+ expect(params.get('evaluation_tab')).toBe(expectedTab)
+ // The request stays in memory, never in a URL someone could share.
+ expect(params.get('backfill_start')).toBeNull()
+ expect(takeBackfillStart(mockSentimentEvaluation.id)).toBe(requestsStart)
+ expect(
+ infoSpy.mock.calls.some(
+ ([message]) => message === 'This evaluation grades new data from now on.'
+ )
+ ).toBe(offersPastData)
+ } finally {
+ pushSpy.mockRestore()
+ infoSpy.mockRestore()
+ }
+ }
+ )
+ })
+
describe('saveEvaluation updates', () => {
it.each([
{ change: 'name', expected: { name: 'Renamed decision evaluation' }, rejected: false },
- { change: 'disable', expected: { enabled: false }, rejected: false },
- { change: 'enable', expected: { enabled: true }, rejected: true },
{
change: 'output_config',
expected: { output_config: { allows_na: true } },
@@ -2325,7 +2374,7 @@ return result`,
async ({ change, expected, rejected }) => {
const evaluation: EvaluationConfig = {
...mockEvaluation,
- enabled: change !== 'enable',
+ enabled: true,
evaluation_type: 'llm_judge',
evaluation_config: { prompt: 'Is this response helpful?', judge_method: 'decision' },
output_type: 'boolean',
@@ -2364,10 +2413,6 @@ return result`,
case 'name':
logic.actions.setEvaluationName(expected.name)
break
- case 'disable':
- case 'enable':
- logic.actions.setEvaluationEnabled(expected.enabled)
- break
case 'output_config':
logic.actions.setAllowsNA(true)
break
@@ -2391,6 +2436,58 @@ return result`,
)
})
+ describe('saveEnabled', () => {
+ it('lets a saved code evaluation turn on while its unsaved draft is a judge without a key', async () => {
+ const evaluation: EvaluationConfig = {
+ ...mockEvaluation,
+ enabled: false,
+ evaluation_type: 'hog',
+ evaluation_config: { source: 'return true;' },
+ model_configuration: null,
+ } as EvaluationConfig
+ useMocks({ get: { '/api/projects/:teamId/evaluations/:id/': evaluation } })
+ logic = llmEvaluationLogic({ evaluationId: 'eval-123' })
+ logic.mount()
+ await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])
+ await expectLogic(keysLogic).toFinishAllListeners()
+
+ logic.actions.setEvaluationType('llm_judge')
+
+ expect(logic.values.canEnable).toBe(false)
+ expect(logic.values.enableSavedReason).toBeNull()
+ })
+
+ it('saves only the run state and keeps unsaved configuration edits', async () => {
+ const evaluation: EvaluationConfig = { ...mockEvaluation, enabled: false }
+ let savedPayload: unknown
+ useMocks({
+ get: { '/api/projects/:teamId/evaluations/:id/': evaluation },
+ patch: {
+ '/api/projects/:teamId/evaluations/:id/': async ({ request }) => {
+ savedPayload = await request.json()
+ return { ...evaluation, enabled: true }
+ },
+ },
+ })
+ logic = llmEvaluationLogic({ evaluationId: 'eval-123' })
+ logic.mount()
+ await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])
+
+ logic.actions.setEvaluationName('Renamed but not saved')
+ logic.actions.saveEnabled(true)
+
+ await expectLogic(logic)
+ .toDispatchActions(['saveEnabledSuccess'])
+ .toMatchValues({
+ enabledSaving: false,
+ evaluation: expect.objectContaining({ enabled: true, name: 'Renamed but not saved' }),
+ originalEvaluation: expect.objectContaining({ enabled: true, name: evaluation.name }),
+ hasUnsavedChanges: true,
+ })
+ expect(savedPayload).toEqual({ enabled: true })
+ })
+ })
+
describe('saveEvaluation report persistence', () => {
it('does not create an evaluation when the report threshold is invalid', async () => {
let evaluationCreateCount = 0
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
index e04ad0e18f37..e456266aafab 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
@@ -60,9 +60,11 @@ import {
} from './evaluationCapabilities'
import { EvaluationBackTarget, getEvaluationBackTarget } from './evaluationNavigation'
import { evaluationReportLogic, persistReportDraft } from './evaluationReportLogic'
+import { patchEvaluation } from './evaluationsApi'
import { getHogEvalExample } from './hogEvalExamples'
import { llmEvaluationsLogic } from './llmEvaluationsLogic'
import { markEvaluationsOverviewStale } from './overview/evaluationsOverviewLogic'
+import { requestBackfillStart } from './pendingBackfillStarts'
import { EvaluationTemplateKey, defaultEvaluationTemplates } from './templates'
import type {
EvaluationConditionSet,
@@ -212,6 +214,34 @@ function toSentimentEvaluation(evaluation: EvaluationConfig): SentimentEvaluatio
}
}
+/** Offers past data for a new evaluation, or hands a requested past-traffic check to the Backfills tab. Returns the URL params to open the evaluation with. */
+function coverPastTraffic(
+ evaluation: EvaluationConfig | undefined,
+ requested: boolean,
+ isNew: boolean
+): Record<string, string> {
+ if (!isNew || !evaluation?.id) {
+ return {}
+ }
+ // The backfill API refuses a paused evaluation, so past traffic is checked only when it runs.
+ if (!evaluation.enabled || !requested) {
+ // Saving navigates away, so the offer to cover past data travels with the user.
+ lemonToast.info('This evaluation grades new data from now on.', {
+ button: {
+ label: 'Evaluate past data',
+ action: () =>
+ router.actions.push(
+ combineUrl(urls.aiObservabilityEvaluation(evaluation.id), { evaluation_tab: 'backfills' }).url
+ ),
+ },
+ })
+ return {}
+ }
+ // The Backfills tab counts the range and starts the run, with its own errors and polling.
+ requestBackfillStart(evaluation.id)
+ return { evaluation_tab: 'backfills' }
+}
+
function filterEvaluationRuns(
runs: EvaluationRun[],
filter: EvaluationRunsFilter,
@@ -260,6 +290,11 @@ function filterEvaluationRuns(
type TestableHogEvaluation = HogEvaluation
+export interface MethodConfigDrafts {
+ prompt?: string
+ hogSource?: string
+}
+
function isTestableHogEvaluation(evaluation: EvaluationConfig | null): evaluation is TestableHogEvaluation {
return evaluation?.evaluation_type === 'hog'
}
@@ -325,7 +360,9 @@ export interface llmEvaluationLogicValues {
breadcrumbs: Breadcrumb[]
canEnable: boolean
canEnableReason: string | null
- enabledChosen: boolean
+ checkPastTraffic: boolean
+ enableSavedReason: string | null
+ enabledSaving: boolean
evaluation: EvaluationConfig | null
evaluationBackTarget: EvaluationBackTarget
evaluationFormSubmitting: boolean
@@ -350,6 +387,7 @@ export interface llmEvaluationLogicValues {
judgeMethod: JudgeMethod
judgeModelGroups: ProviderModelGroup[]
maxContext: MaxContextInput[]
+ methodConfigDrafts: MethodConfigDrafts
modelSelectionRequired: boolean
numericBoundsRequired: boolean
originalEvaluation: EvaluationConfig | null
@@ -472,6 +510,15 @@ export interface llmEvaluationLogicActions {
resetEvaluation: () => {
value: true
}
+ saveEnabled: (enabled: boolean) => {
+ enabled: boolean
+ }
+ saveEnabledFailure: () => {
+ value: true
+ }
+ saveEnabledSuccess: (evaluation: EvaluationConfig) => {
+ evaluation: EvaluationConfig
+ }
saveEvaluation: () => {
value: true
}
@@ -494,6 +541,9 @@ export interface llmEvaluationLogicActions {
setAllowsNA: (allowsNA: boolean) => {
allowsNA: boolean
}
+ setCheckPastTraffic: (checkPastTraffic: boolean) => {
+ checkPastTraffic: boolean
+ }
setDefaultEnabled: (enabled: boolean) => {
enabled: boolean
}
@@ -503,9 +553,6 @@ export interface llmEvaluationLogicActions {
setEvaluationDirectory: (directoryId: string | null) => {
directoryId: string | null
}
- setEvaluationEnabled: (enabled: boolean) => {
- enabled: boolean
- }
setEvaluationName: (name: string) => {
name: string
}
@@ -527,6 +574,8 @@ export interface llmEvaluationLogicActions {
}
setEvaluationType: (evaluationType: EvaluationType) => {
evaluationType: EvaluationType
+ methodConfigDrafts: MethodConfigDrafts
+ previousEvaluation: EvaluationConfig | null
}
setFormStep: (step: string) => {
step: string
@@ -633,6 +682,10 @@ export interface llmEvaluationLogicMeta {
evaluation: EvaluationConfig | null,
activeProviderKey: LLMProviderKey | null | undefined
) => boolean
+ enableSavedReason: (
+ originalEvaluation: EvaluationConfig | null,
+ activeProviderKey: LLMProviderKey | null | undefined
+ ) => string | null
canEnableReason: (canEnable: boolean) => string | null
evaluationProviderKeyIssue: (
evaluation: EvaluationConfig | null,
@@ -696,7 +749,6 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
setEvaluationDescription: (description: string) => ({ description }),
setEvaluationDirectory: (directoryId: string | null) => ({ directoryId }),
setEvaluationPrompt: (prompt: string) => ({ prompt }),
- setEvaluationEnabled: (enabled: boolean) => ({ enabled }),
setDefaultEnabled: (enabled: boolean) => ({ enabled }),
setOutputType: (outputType: 'boolean' | 'numeric' | 'categorical') => ({
outputType,
@@ -711,7 +763,11 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
setJudgeMethod: (judgeMethod: JudgeMethod) => ({ judgeMethod }),
setIncludeMedia: (includeMedia: boolean) => ({ includeMedia }),
setReasoningLevel: (reasoningLevel: ReasoningLevel | null) => ({ reasoningLevel }),
- setEvaluationType: (evaluationType: EvaluationType) => ({ evaluationType }),
+ setEvaluationType: (evaluationType: EvaluationType) => ({
+ evaluationType,
+ previousEvaluation: values.evaluation,
+ methodConfigDrafts: values.methodConfigDrafts,
+ }),
setEvaluationTarget: (target: EvaluationTarget) => ({ target }),
setSettleStrategy: (strategy: EvaluationSettleStrategy) => ({ strategy }),
// Duration fields only — switching strategy must go through setSettleStrategy so the
@@ -728,6 +784,10 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
// Evaluation management actions
saveEvaluation: true,
saveEvaluationSuccess: (evaluation: EvaluationConfig) => ({ evaluation }),
+ saveEnabled: (enabled: boolean) => ({ enabled }),
+ setCheckPastTraffic: (checkPastTraffic: boolean) => ({ checkPastTraffic }),
+ saveEnabledSuccess: (evaluation: EvaluationConfig) => ({ evaluation }),
+ saveEnabledFailure: true,
saveEvaluationFailure: (error: string) => ({ error }),
loadEvaluation: true,
loadEvaluationSuccess: (evaluation: EvaluationConfig | null) => ({
@@ -921,11 +981,32 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
: state,
},
],
+ methodConfigDrafts: [
+ {} as MethodConfigDrafts,
+ {
+ loadEvaluationSuccess: () => ({}),
+ setEvaluationType: (state, { previousEvaluation }) =>
+ previousEvaluation?.evaluation_type === 'llm_judge'
+ ? { ...state, prompt: previousEvaluation.evaluation_config.prompt }
+ : previousEvaluation?.evaluation_type === 'hog'
+ ? {
+ ...state,
+ // An untouched default is regenerated on return so it matches the output type chosen meanwhile.
+ hogSource:
+ previousEvaluation.evaluation_config.source ===
+ toHogEvaluation(previousEvaluation).evaluation_config.source
+ ? undefined
+ : previousEvaluation.evaluation_config.source,
+ }
+ : state,
+ },
+ ],
originalEvaluation: [
null as EvaluationConfig | null,
{
loadEvaluationSuccess: (_, { evaluation }) => evaluation,
saveEvaluationSuccess: (_, { evaluation }) => evaluation,
+ saveEnabledSuccess: (_, { evaluation }) => evaluation,
},
],
evaluation: [
@@ -939,7 +1020,6 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
state && isLLMJudgeEvaluation(state)
? { ...state, evaluation_config: { ...state.evaluation_config, prompt } }
: state,
- setEvaluationEnabled: (state, { enabled }) => (state ? { ...state, enabled } : null),
setDefaultEnabled: (state, { enabled }) => (state ? { ...state, enabled } : null),
setOutputType: (state, { outputType, outputConfig }) => {
if (
@@ -1053,12 +1133,15 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
state && isLLMJudgeEvaluation(state)
? { ...state, evaluation_config: { ...state.evaluation_config, include_media: includeMedia } }
: state,
- setEvaluationType: (state, { evaluationType }) => {
+ setEvaluationType: (state, { evaluationType, methodConfigDrafts }) => {
if (!state) {
return null
}
if (evaluationType === 'hog') {
- return toHogEvaluation(state)
+ const hogEvaluation = toHogEvaluation(state)
+ return methodConfigDrafts.hogSource === undefined
+ ? hogEvaluation
+ : { ...hogEvaluation, evaluation_config: { source: methodConfigDrafts.hogSource } }
}
if (evaluationType === 'sentiment') {
if (
@@ -1069,7 +1152,10 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
}
return toSentimentEvaluation(state)
}
- return toLLMJudgeEvaluation(state)
+ const judgeEvaluation = toLLMJudgeEvaluation(state)
+ return methodConfigDrafts.prompt === undefined
+ ? judgeEvaluation
+ : { ...judgeEvaluation, evaluation_config: { prompt: methodConfigDrafts.prompt } }
},
setEvaluationTarget: (state, { target }) => {
if (!state) {
@@ -1107,6 +1193,17 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
: state,
loadEvaluationSuccess: (_, { evaluation }) => evaluation,
saveEvaluationSuccess: (_, { evaluation }) => evaluation,
+ // Takes only the run state, so configuration edits that are not saved yet stay unsaved.
+ saveEnabledSuccess: (state, { evaluation }) =>
+ state
+ ? {
+ ...state,
+ enabled: evaluation.enabled,
+ status: evaluation.status,
+ status_reason: evaluation.status_reason,
+ status_reason_detail: evaluation.status_reason_detail,
+ }
+ : state,
},
],
hogTestResults: {
@@ -1179,13 +1276,19 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
saveEvaluationFailure: () => false,
},
],
- // Whether someone switched "enabled" by hand. Until then a new evaluation follows whether it can run.
- enabledChosen: [
+ checkPastTraffic: [
false,
{
- setEvaluationEnabled: () => true,
+ setCheckPastTraffic: (_, { checkPastTraffic }) => checkPastTraffic,
loadEvaluationSuccess: () => false,
- resetEvaluation: () => false,
+ },
+ ],
+ enabledSaving: [
+ false,
+ {
+ saveEnabled: () => true,
+ saveEnabledSuccess: () => false,
+ saveEnabledFailure: () => false,
},
],
hasUnsavedChanges: [
@@ -1195,7 +1298,6 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
setEvaluationDescription: () => true,
setEvaluationDirectory: () => true,
setEvaluationPrompt: () => true,
- setEvaluationEnabled: () => true,
setOutputType: () => true,
patchOutputConfig: () => true,
setAllowsNA: () => true,
@@ -1461,6 +1563,26 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
}
},
+ saveEnabled: async ({ enabled }) => {
+ const teamId = teamLogic.values.currentTeamId
+ if (!teamId) {
+ actions.saveEnabledFailure()
+ return
+ }
+ try {
+ actions.saveEnabledSuccess(await patchEvaluation(teamId.toString(), props.evaluationId, { enabled }))
+ llmEvaluationsLogic.findMounted()?.actions.loadEvaluations()
+ markEvaluationsOverviewStale()
+ } catch (error) {
+ lemonToast.error(
+ evaluationErrorMessage(
+ error,
+ enabled ? 'Couldn’t turn on this evaluation.' : 'Couldn’t pause this evaluation.'
+ )
+ )
+ actions.saveEnabledFailure()
+ }
+ },
saveEvaluation: async () => {
try {
const teamId = teamLogic.values.currentTeamId
@@ -1511,6 +1633,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
props.evaluationId,
evaluationData as Parameters<typeof evaluationsPartialUpdate>[2]
)) as unknown as EvaluationConfig
+ const openParams = coverPastTraffic(response, values.checkPastTraffic, isNew)
actions.saveEvaluationSuccess(response)
// The list and the self-driving table read from llmEvaluationsLogic, which only
// fetches on mount. Refresh it here so they show the eval we just saved.
@@ -1518,20 +1641,6 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
markEvaluationsOverviewStale()
if (isNew) {
globalSetupLogic.findMounted()?.actions.markTaskAsCompleted(SetupTaskId.SetUpLlmEvaluation)
- // Saving navigates away, so the offer to cover past data travels with the user.
- if (response?.id) {
- lemonToast.info('This evaluation grades new data from now on.', {
- button: {
- label: 'Evaluate past data',
- action: () =>
- router.actions.push(
- combineUrl(urls.aiObservabilityEvaluation(response.id), {
- evaluation_tab: 'backfills',
- }).url
- ),
- },
- })
- }
}
// Piggyback the scheduled-report draft onto the main save so the single
@@ -1566,7 +1675,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
const opensEvaluation = isNew && !!response?.id
router.actions.push(
opensEvaluation
- ? urls.aiObservabilityEvaluation(response.id)
+ ? combineUrl(urls.aiObservabilityEvaluation(response.id), openParams).url
: getEvaluationBackTarget(false, router.values.searchParams).path
)
} catch (error) {
@@ -1882,6 +1991,18 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
},
],
+ // Pause and Turn on change only the saved evaluation, so they check it rather than an unsaved draft.
+ enableSavedReason: [
+ (s) => [s.originalEvaluation, s.activeProviderKey],
+ (
+ originalEvaluation: EvaluationConfig | null,
+ activeProviderKey: LLMProviderKey | null | undefined
+ ): string | null =>
+ !originalEvaluation || evaluationCanResolveModel(originalEvaluation, activeProviderKey)
+ ? null
+ : 'Add a provider API key to enable this evaluation.',
+ ],
+
canEnableReason: [
(s) => [s.canEnable],
(canEnable: boolean): string | null => {
@@ -2070,12 +2191,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
subscriptions(({ actions, values, props }) => ({
// A new evaluation starts running when it can: code and sentiment need no key, a judge needs a working one.
canEnable: (canEnable: boolean) => {
- if (
- props.evaluationId === 'new' &&
- !values.enabledChosen &&
- values.evaluation &&
- values.evaluation.enabled !== canEnable
- ) {
+ if (props.evaluationId === 'new' && values.evaluation && values.evaluation.enabled !== canEnable) {
actions.setDefaultEnabled(canEnable)
}
},
diff --git a/products/ai_observability/frontend/evaluations/pendingBackfillStarts.ts b/products/ai_observability/frontend/evaluations/pendingBackfillStarts.ts
new file mode 100644
index 000000000000..bcaf7cf81bc9
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/pendingBackfillStarts.ts
@@ -0,0 +1,14 @@
+/**
+ * New evaluations whose creator asked to also evaluate past traffic. It lives in memory, not in the URL,
+ * so a link cannot start a backfill, and a request survives leaving the Backfills tab before its count lands.
+ */
+const requested = new Set<string>()
+
+export function requestBackfillStart(evaluationId: string): void {
+ requested.add(evaluationId)
+}
+
+/** Returns whether a start was requested, and forgets it, so each request starts at most one backfill. */
+export function takeBackfillStart(evaluationId: string): boolean {
+ return requested.delete(evaluationId)
+}