Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): guide teams without an AI provider key to evals that run (#127069)

Bernat Torresd6607537467b
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 22905ab897e8..1ed6ac083780 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -8911,9 +8911,9 @@ snapshots:
scenes-app-ai-observability-byok-model-picker-notice--models-failed-to-load--light:
hash: v1.k794b7964.76cfc4cc5e788cae6f7eab832fd2d85089afc7b2bdfcbba8813b33b1e212b90a.rqyS3wFRk8IcQnR9MkuOBrJviIiAqlOz342XtDrw1DI
scenes-app-ai-observability-byok-model-picker-notice--no-provider-keys--dark:
- hash: v1.k794b7964.fd75ac130df7efede8c38a25a7d0c2c7675bca398496e38bd347dd8568f8522a.ppty7IQt0-lGuwpFlzBe_RHxc2pYgeRqIpcjSzwLGwA
+ hash: v1.k794b7964.26f6a546df80753a2fb3f8832d3fd4738f84268449603cdf497ae41b414ed5b6.lD6qcAM4H9ocYAkOSWeyBNRVdg_36snWFJxBRguxQR0
scenes-app-ai-observability-byok-model-picker-notice--no-provider-keys--light:
- hash: v1.k794b7964.00f1c6684767d0c95bf81ce8144f18acecf494b8590c2cd0efc6cc0e61c30afe.c7O5Tq5l2b_JFWj-fNfa7erXDEH7ZYORyDXCJHXz6AI
+ hash: v1.k794b7964.aeb93c3858b6a0dc6a42f9529d4aa03a4ee96e40867e88bcc13f5cdf8900980d.9BoUebeBw6_p5_djebdTYSXy2rDYTE3zQ4Me0BluMX0
scenes-app-ai-observability-byok-model-picker-notice--no-usable-provider-keys--dark:
hash: v1.k794b7964.a260abcedfa1372c2c4caf36dbf8e51580ae6d6303b7d18617a6799fbdcb9841.pfjNZI_gu55WYAo4V3uxORMyk_GHRA7uQMkkBaMaYX8
scenes-app-ai-observability-byok-model-picker-notice--no-usable-provider-keys--light:
@@ -9007,13 +9007,13 @@ snapshots:
scenes-app-ai-observability-evaluation-explanation--system-one-zero-probability--light:
hash: v1.k794b7964.f56325c23a185f032b8e7c331463345ea780e4e1b9a85f44a2f560869f06afa4.KcU1H2opmRuSoQcBHacqKya9WVvI5GATsFn_lgLD8rM
scenes-app-ai-observability-evaluations-onboarding--narrow--dark:
- hash: v1.k794b7964.2fe8551c13c48d3966c607923d7457eb1edba026b14430e8c40a1052a5223554.h1DTp9Ar3Xk9DmfrMviNCD90-dEDhm_xLCi-Fl8C0Ps
+ hash: v1.k794b7964.effa3fb1ba5852100b35391d3ab39e8346a57dadd0590b06cb9d49f64c01a737.djrzTGO7oZZpIpNVyXF7UP_CZMcUUGELjb-xsc04N88
scenes-app-ai-observability-evaluations-onboarding--narrow--light:
- hash: v1.k794b7964.725c9cd138e9974a36814447547b53b57f74adf288df0362133840fb7b04a416.ZgYC21heeJMxtaLWGjfeDXIrZAhqovb5dhfEtPO-TCc
+ hash: v1.k794b7964.9ff3b23f67aef9bca395669b8c6b8fd866d5540a43d9d1b646336be4ca1304f2.L6AzmeevlP-C222Eg3uV8umWZVUdUTB4g1MFk-5JMYM
scenes-app-ai-observability-evaluations-onboarding--no-online-evaluations--dark:
- hash: v1.k794b7964.3caf7af67991fbfda18f590ef7a86787d67dd96e02dd2b3806a9d4fc321837ee.hz1-YxbzgDhzEHRO8TndIgNIVd_qAKCY35yNf2k4UbI
+ hash: v1.k794b7964.29f809615534c4d871203afc5ef9c55b017f4db6cf24462005b99069a7a72f47.NdFTYxlZdPrwJtYsyGOErDGNc7lbady-ag9hYKxEdvY
scenes-app-ai-observability-evaluations-onboarding--no-online-evaluations--light:
- hash: v1.k794b7964.1dbea4bbbd7b0731b841cfdf0dc28826e9bba9caf9735cc7257e0fc4e0a67026.ezM9M-EOjWsTZXJk602I44gvDXvOzobiHJq09E0OEac
+ hash: v1.k794b7964.469add53e73b02ed08f3f15d5f1d4fb8608174c7b50e9aeefab0abe607b6e071.IYZzSNa-qIVMxHbkk-kGbn2tlkVE8d4EwYAT2yQZY6A
scenes-app-ai-observability-evaluations-overview--directory-open--dark:
hash: v1.k794b7964.9e8467472787599992a607e83fc725f1808762fc1c33cd0ec4f097e457e1411c.VdW-6vKl9WLQcs3_PcFk76quSN32MHTKKi65Mr2AtIw
scenes-app-ai-observability-evaluations-overview--directory-open--light:
@@ -13470,6 +13470,10 @@ snapshots:
hash: v1.k794b7964.ea887c384da7d18b38230ba0b420b1f87d0a2995899d5844290ddc16eb476dc3.8F9-jOoonwrPVwHD8_XR0eud3gqkF0OueJM1tArPbn0
scenes-app-llm-observability-evaluation-templates--narrow--light:
hash: v1.k794b7964.acb588946dd41382c78934be3f8912589d1a15df06ef9a976f617fad21f1ddd2.0LsEsqRoncEFZhEVh5X8gbA_QoGZelV2t1ABDsxGeao
+ scenes-app-llm-observability-evaluation-templates--without-api-key--dark:
+ hash: v1.k794b7964.50d81e23177f49a10b50aa963a9a617dc26c8b2cce062a8e313430cf4868446e.MlMWfByO92mw-UakenXEkHJEuedG86aYeWonqvOAILA
+ scenes-app-llm-observability-evaluation-templates--without-api-key--light:
+ hash: v1.k794b7964.539c391440bb76c34d73f3b2dde3540960a8c521e97968ba1fbfe5b88e2a339d.GXDp0COFtN21PhNIdkfuMAKL2O1mGPpyPcWqTdJCZTI
scenes-app-llm-observability-evaluationreportconfig--lemon-ui--dark:
hash: v1.k794b7964.195aa4a5a39e8f947e17fe67f5c23303b7dd0579c278907761be24a08780be92.IkGQtAI95T60kJKVZcmmFBIWNCd4meuJv-bcNDoohwU
scenes-app-llm-observability-evaluationreportconfig--lemon-ui--light:
diff --git a/products/ai_observability/frontend/ByokModelPickerNotice.tsx b/products/ai_observability/frontend/ByokModelPickerNotice.tsx
index a90791031d6f..0d1962ec82a3 100644
--- a/products/ai_observability/frontend/ByokModelPickerNotice.tsx
+++ b/products/ai_observability/frontend/ByokModelPickerNotice.tsx
@@ -4,6 +4,8 @@ import { LemonBanner, Link } from '@posthog/lemon-ui'
import { PROVIDER_SETTINGS_URL } from './ModelPicker'
import { modelPickerLogic } from './modelPickerLogic'
+import { llmProviderKeysLogic } from './settings/llmProviderKeysLogic'
+import { LLMProviderKeysSettings } from './settings/LLMProviderKeysSettings'
import { providerLabel } from './settings/providerKeyStateUtils'
/** Explains an empty or short model picker on the surfaces that only run on the team's own provider keys. */
@@ -11,6 +13,17 @@ export function ByokModelPickerNotice({ forEvaluation = false }: { forEvaluation
const { byokModelNotice: generativeModelNotice, evaluationModelNotice } = useValues(modelPickerLogic)
const byokModelNotice = forEvaluation ? evaluationModelNotice : generativeModelNotice
const { loadByokModels } = useActions(modelPickerLogic)
+ const { newKeyModalOpen, providerKeys } = useValues(llmProviderKeysLogic)
+
+ // Saving a key reloads the keys, which clears the notice for a moment. Keep the view mounted
+ // while its add-key modal is open, so a failed save keeps what the person typed.
+ if (byokModelNotice?.kind === 'no-keys' || (newKeyModalOpen && providerKeys.length === 0)) {
+ return (
+ <div className="mt-2">
+ <LLMProviderKeysSettings />
+ </div>
+ )
+ }
switch (byokModelNotice?.kind) {
case 'models-failed': {
@@ -48,12 +61,6 @@ export function ByokModelPickerNotice({ forEvaluation = false }: { forEvaluation
</p>
)
}
- case 'no-keys':
- return (
- <p className="text-xs text-secondary mt-1">
- No models available. <Link to={PROVIDER_SETTINGS_URL}>Add your own API keys</Link> to choose one.
- </p>
- )
case 'no-usable-keys':
return (
<p className="text-xs text-secondary mt-1">
diff --git a/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx b/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx
index 3a3019a17c30..30324f60e98b 100644
--- a/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx
+++ b/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx
@@ -1,4 +1,4 @@
-import { useActions } from 'kea'
+import { useActions, useValues } from 'kea'
import { router } from 'kea-router'
import posthog from 'posthog-js'
@@ -12,15 +12,12 @@ import { urls } from 'scenes/urls'
import { ProductKey } from '~/queries/schema/schema-general'
import { AccessControlLevel, AccessControlResourceType } from '~/types'
+import { llmProviderKeysLogic } from '../settings/llmProviderKeysLogic'
import { ScratchTemplateCard } from './components/ScratchTemplateCard'
import { TemplateCard } from './components/TemplateCard'
import { getEvaluationTemplateSelectionUrl } from './evaluationNavigation'
-import { RECOMMENDED_TEMPLATE_KEYS } from './EvaluationTemplates'
-import { type EvaluationTemplate, defaultEvaluationTemplates } from './templates'
-
-const QUICK_START_TEMPLATES = RECOMMENDED_TEMPLATE_KEYS.slice(0, 3).flatMap((key) =>
- defaultEvaluationTemplates.filter((template) => template.key === key)
-)
+import { recommendedTemplates } from './EvaluationTemplates'
+import type { EvaluationTemplate } from './templates'
export function EvaluationQuickStart({
mode,
@@ -30,6 +27,8 @@ export function EvaluationQuickStart({
preview: boolean
}): JSX.Element | null {
const { reportSetupInteraction } = useActions(productSetupStatusLogic({ productKey: ProductKey.LLM_EVALUATIONS }))
+ const { requiresProviderKey } = useValues(llmProviderKeysLogic)
+ const quickStartTemplates = recommendedTemplates(requiresProviderKey).slice(0, 3)
const selectTemplate = (template: EvaluationTemplate, placement: 'card' | 'example'): void => {
// pinned: same analytics event as the template picker, so template funnels count both entry points
@@ -61,7 +60,7 @@ export function EvaluationQuickStart({
return (
<div className="flex flex-col gap-2">
<div className="grid gap-2">
- {QUICK_START_TEMPLATES.map((template) => (
+ {quickStartTemplates.map((template) => (
<TemplateCard key={template.key} template={template} onSelect={selectTemplate} />
))}
<ScratchTemplateCard onSelect={startFromScratch} />
diff --git a/products/ai_observability/frontend/evaluations/EvaluationTemplates.stories.tsx b/products/ai_observability/frontend/evaluations/EvaluationTemplates.stories.tsx
index 5f83af4eab1c..ab828a0f3f64 100644
--- a/products/ai_observability/frontend/evaluations/EvaluationTemplates.stories.tsx
+++ b/products/ai_observability/frontend/evaluations/EvaluationTemplates.stories.tsx
@@ -1,5 +1,7 @@
import type { Meta, StoryObj } from '@storybook/react'
+import { mswDecorator } from '~/mocks/browser'
+
import { EvaluationTemplatesScene } from './EvaluationTemplates'
const meta: Meta<typeof EvaluationTemplatesScene> = {
@@ -23,3 +25,13 @@ export const Narrow: Story = {
</div>
),
}
+
+export const WithoutApiKey: Story = {
+ decorators: [
+ mswDecorator({
+ get: {
+ '/api/environments/:team_id/llm_analytics/evaluation_config/': { active_provider_key: null },
+ },
+ }),
+ ],
+}
diff --git a/products/ai_observability/frontend/evaluations/EvaluationTemplates.tsx b/products/ai_observability/frontend/evaluations/EvaluationTemplates.tsx
index 50a63f288ff9..dd632d2ddcd6 100644
--- a/products/ai_observability/frontend/evaluations/EvaluationTemplates.tsx
+++ b/products/ai_observability/frontend/evaluations/EvaluationTemplates.tsx
@@ -10,6 +10,7 @@ import { LemonButton, LemonTabs, Link } from '@posthog/lemon-ui'
import { pngHoggie } from 'lib/brand/hoggies'
import { SceneExport } from 'scenes/sceneTypes'
+import { llmProviderKeysLogic } from '../settings/llmProviderKeysLogic'
import { EvaluationAgentActions } from './components/EvaluationAgentActions'
import { TemplateCard } from './components/TemplateCard'
import { getEvaluationBackTarget, getEvaluationTemplateSelectionUrl } from './evaluationNavigation'
@@ -22,7 +23,7 @@ const POPULAR_TEMPLATE_KEYS = ['relevance', 'cost_latency', 'helpfulness', 'hall
// Sentiment leads because more teams run sentiment checks than any other kind.
// The score and category templates complete the first row so people discover those output types.
// The second row holds the other templates that people most often keep running.
-export const RECOMMENDED_TEMPLATE_KEYS = [
+const RECOMMENDED_TEMPLATE_KEYS = [
'sentiment',
'response_quality',
'task_outcome',
@@ -30,6 +31,15 @@ export const RECOMMENDED_TEMPLATE_KEYS = [
'hallucination',
'helpfulness',
]
+// An LLM judge needs the team's own provider key, so a team without one gets a first row it can run as is.
+const NO_KEY_RECOMMENDED_TEMPLATE_KEYS = [
+ 'sentiment',
+ 'cost_latency',
+ 'repeated_tool_calls',
+ 'response_quality',
+ 'task_outcome',
+ 'hallucination',
+]
type TemplateTab = 'recommended' | 'all' | EvaluationTemplateCategory
@@ -44,14 +54,18 @@ const CATEGORY_TABS: { key: EvaluationTemplateCategory; label: string }[] = [
const byKeys = (keys: string[]): EvaluationTemplate[] =>
keys.flatMap((key) => defaultEvaluationTemplates.filter((template) => template.key === key))
+export function recommendedTemplates(requiresProviderKey: boolean): EvaluationTemplate[] {
+ return byKeys(requiresProviderKey ? NO_KEY_RECOMMENDED_TEMPLATE_KEYS : RECOMMENDED_TEMPLATE_KEYS)
+}
+
const ALL_TEMPLATES_POPULAR_FIRST = [
...byKeys(POPULAR_TEMPLATE_KEYS),
...defaultEvaluationTemplates.filter((template) => !POPULAR_TEMPLATE_KEYS.includes(template.key)),
]
-function templatesForTab(tab: TemplateTab): EvaluationTemplate[] {
+function templatesForTab(tab: TemplateTab, requiresProviderKey: boolean): EvaluationTemplate[] {
if (tab === 'recommended') {
- return byKeys(RECOMMENDED_TEMPLATE_KEYS)
+ return recommendedTemplates(requiresProviderKey)
}
if (tab === 'all') {
return ALL_TEMPLATES_POPULAR_FIRST
@@ -71,8 +85,9 @@ interface TemplatePickerProps {
function TemplatePicker({ title, showBackButton = false, learnMoreUrl }: TemplatePickerProps): JSX.Element {
const { searchParams } = useValues(router)
+ const { requiresProviderKey } = useValues(llmProviderKeysLogic)
const [tab, setTab] = useState<TemplateTab>('recommended')
- const templates = templatesForTab(tab)
+ const templates = templatesForTab(tab, requiresProviderKey)
const selectTab = (key: TemplateTab): void => {
setTab(key)