:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): guide teams without an AI provider key to evals that run (#127069)
Bernat Torresd6607537467b
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex 22905ab897e8..1ed6ac083780 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -8911,9 +8911,9 @@ snapshots:scenes-app-ai-observability-byok-model-picker-notice--models-failed-to-load--light:hash: v1.k794b7964.76cfc4cc5e788cae6f7eab832fd2d85089afc7b2bdfcbba8813b33b1e212b90a.rqyS3wFRk8IcQnR9MkuOBrJviIiAqlOz342XtDrw1DIscenes-app-ai-observability-byok-model-picker-notice--no-provider-keys--dark:- hash: v1.k794b7964.fd75ac130df7efede8c38a25a7d0c2c7675bca398496e38bd347dd8568f8522a.ppty7IQt0-lGuwpFlzBe_RHxc2pYgeRqIpcjSzwLGwA+ hash: v1.k794b7964.26f6a546df80753a2fb3f8832d3fd4738f84268449603cdf497ae41b414ed5b6.lD6qcAM4H9ocYAkOSWeyBNRVdg_36snWFJxBRguxQR0scenes-app-ai-observability-byok-model-picker-notice--no-provider-keys--light:- hash: v1.k794b7964.00f1c6684767d0c95bf81ce8144f18acecf494b8590c2cd0efc6cc0e61c30afe.c7O5Tq5l2b_JFWj-fNfa7erXDEH7ZYORyDXCJHXz6AI+ hash: v1.k794b7964.aeb93c3858b6a0dc6a42f9529d4aa03a4ee96e40867e88bcc13f5cdf8900980d.9BoUebeBw6_p5_djebdTYSXy2rDYTE3zQ4Me0BluMX0scenes-app-ai-observability-byok-model-picker-notice--no-usable-provider-keys--dark:hash: v1.k794b7964.a260abcedfa1372c2c4caf36dbf8e51580ae6d6303b7d18617a6799fbdcb9841.pfjNZI_gu55WYAo4V3uxORMyk_GHRA7uQMkkBaMaYX8scenes-app-ai-observability-byok-model-picker-notice--no-usable-provider-keys--light:@@ -9007,13 +9007,13 @@ snapshots:scenes-app-ai-observability-evaluation-explanation--system-one-zero-probability--light:hash: v1.k794b7964.f56325c23a185f032b8e7c331463345ea780e4e1b9a85f44a2f560869f06afa4.KcU1H2opmRuSoQcBHacqKya9WVvI5GATsFn_lgLD8rMscenes-app-ai-observability-evaluations-onboarding--narrow--dark:- hash: v1.k794b7964.2fe8551c13c48d3966c607923d7457eb1edba026b14430e8c40a1052a5223554.h1DTp9Ar3Xk9DmfrMviNCD90-dEDhm_xLCi-Fl8C0Ps+ hash: v1.k794b7964.effa3fb1ba5852100b35391d3ab39e8346a57dadd0590b06cb9d49f64c01a737.djrzTGO7oZZpIpNVyXF7UP_CZMcUUGELjb-xsc04N88scenes-app-ai-observability-evaluations-onboarding--narrow--light:- hash: v1.k794b7964.725c9cd138e9974a36814447547b53b57f74adf288df0362133840fb7b04a416.ZgYC21heeJMxtaLWGjfeDXIrZAhqovb5dhfEtPO-TCc+ hash: v1.k794b7964.9ff3b23f67aef9bca395669b8c6b8fd866d5540a43d9d1b646336be4ca1304f2.L6AzmeevlP-C222Eg3uV8umWZVUdUTB4g1MFk-5JMYMscenes-app-ai-observability-evaluations-onboarding--no-online-evaluations--dark:- hash: v1.k794b7964.3caf7af67991fbfda18f590ef7a86787d67dd96e02dd2b3806a9d4fc321837ee.hz1-YxbzgDhzEHRO8TndIgNIVd_qAKCY35yNf2k4UbI+ hash: v1.k794b7964.29f809615534c4d871203afc5ef9c55b017f4db6cf24462005b99069a7a72f47.NdFTYxlZdPrwJtYsyGOErDGNc7lbady-ag9hYKxEdvYscenes-app-ai-observability-evaluations-onboarding--no-online-evaluations--light:- hash: v1.k794b7964.1dbea4bbbd7b0731b841cfdf0dc28826e9bba9caf9735cc7257e0fc4e0a67026.ezM9M-EOjWsTZXJk602I44gvDXvOzobiHJq09E0OEac+ hash: v1.k794b7964.469add53e73b02ed08f3f15d5f1d4fb8608174c7b50e9aeefab0abe607b6e071.IYZzSNa-qIVMxHbkk-kGbn2tlkVE8d4EwYAT2yQZY6Ascenes-app-ai-observability-evaluations-overview--directory-open--dark:hash: v1.k794b7964.9e8467472787599992a607e83fc725f1808762fc1c33cd0ec4f097e457e1411c.VdW-6vKl9WLQcs3_PcFk76quSN32MHTKKi65Mr2AtIwscenes-app-ai-observability-evaluations-overview--directory-open--light:@@ -13470,6 +13470,10 @@ snapshots:hash: v1.k794b7964.ea887c384da7d18b38230ba0b420b1f87d0a2995899d5844290ddc16eb476dc3.8F9-jOoonwrPVwHD8_XR0eud3gqkF0OueJM1tArPbn0scenes-app-llm-observability-evaluation-templates--narrow--light:hash: v1.k794b7964.acb588946dd41382c78934be3f8912589d1a15df06ef9a976f617fad21f1ddd2.0LsEsqRoncEFZhEVh5X8gbA_QoGZelV2t1ABDsxGeao+ scenes-app-llm-observability-evaluation-templates--without-api-key--dark:+ hash: v1.k794b7964.50d81e23177f49a10b50aa963a9a617dc26c8b2cce062a8e313430cf4868446e.MlMWfByO92mw-UakenXEkHJEuedG86aYeWonqvOAILA+ scenes-app-llm-observability-evaluation-templates--without-api-key--light:+ hash: v1.k794b7964.539c391440bb76c34d73f3b2dde3540960a8c521e97968ba1fbfe5b88e2a339d.GXDp0COFtN21PhNIdkfuMAKL2O1mGPpyPcWqTdJCZTIscenes-app-llm-observability-evaluationreportconfig--lemon-ui--dark:hash: v1.k794b7964.195aa4a5a39e8f947e17fe67f5c23303b7dd0579c278907761be24a08780be92.IkGQtAI95T60kJKVZcmmFBIWNCd4meuJv-bcNDoohwUscenes-app-llm-observability-evaluationreportconfig--lemon-ui--light:diff --git a/products/ai_observability/frontend/ByokModelPickerNotice.tsx b/products/ai_observability/frontend/ByokModelPickerNotice.tsxindex a90791031d6f..0d1962ec82a3 100644--- a/products/ai_observability/frontend/ByokModelPickerNotice.tsx+++ b/products/ai_observability/frontend/ByokModelPickerNotice.tsx@@ -4,6 +4,8 @@ import { LemonBanner, Link } from '@posthog/lemon-ui'import { PROVIDER_SETTINGS_URL } from './ModelPicker'import { modelPickerLogic } from './modelPickerLogic'+import { llmProviderKeysLogic } from './settings/llmProviderKeysLogic'+import { LLMProviderKeysSettings } from './settings/LLMProviderKeysSettings'import { providerLabel } from './settings/providerKeyStateUtils'/** Explains an empty or short model picker on the surfaces that only run on the team's own provider keys. */@@ -11,6 +13,17 @@ export function ByokModelPickerNotice({ forEvaluation = false }: { forEvaluationconst { byokModelNotice: generativeModelNotice, evaluationModelNotice } = useValues(modelPickerLogic)const byokModelNotice = forEvaluation ? evaluationModelNotice : generativeModelNoticeconst { loadByokModels } = useActions(modelPickerLogic)+ const { newKeyModalOpen, providerKeys } = useValues(llmProviderKeysLogic)++ // Saving a key reloads the keys, which clears the notice for a moment. Keep the view mounted+ // while its add-key modal is open, so a failed save keeps what the person typed.+ if (byokModelNotice?.kind === 'no-keys' || (newKeyModalOpen && providerKeys.length === 0)) {+ return (+ <div className="mt-2">+ <LLMProviderKeysSettings />+ </div>+ )+ }switch (byokModelNotice?.kind) {case 'models-failed': {@@ -48,12 +61,6 @@ export function ByokModelPickerNotice({ forEvaluation = false }: { forEvaluation</p>)}- case 'no-keys':- return (- <p className="text-xs text-secondary mt-1">- No models available. <Link to={PROVIDER_SETTINGS_URL}>Add your own API keys</Link> to choose one.- </p>- )case 'no-usable-keys':return (<p className="text-xs text-secondary mt-1">diff --git a/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx b/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsxindex 3a3019a17c30..30324f60e98b 100644--- a/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx+++ b/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx@@ -1,4 +1,4 @@-import { useActions } from 'kea'+import { useActions, useValues } from 'kea'import { router } from 'kea-router'import posthog from 'posthog-js'@@ -12,15 +12,12 @@ import { urls } from 'scenes/urls'import { ProductKey } from '~/queries/schema/schema-general'import { AccessControlLevel, AccessControlResourceType } from '~/types'+import { llmProviderKeysLogic } from '../settings/llmProviderKeysLogic'import { ScratchTemplateCard } from './components/ScratchTemplateCard'import { TemplateCard } from './components/TemplateCard'import { getEvaluationTemplateSelectionUrl } from './evaluationNavigation'-import { RECOMMENDED_TEMPLATE_KEYS } from './EvaluationTemplates'-import { type EvaluationTemplate, defaultEvaluationTemplates } from './templates'--const QUICK_START_TEMPLATES = RECOMMENDED_TEMPLATE_KEYS.slice(0, 3).flatMap((key) =>- defaultEvaluationTemplates.filter((template) => template.key === key)-)+import { recommendedTemplates } from './EvaluationTemplates'+import type { EvaluationTemplate } from './templates'export function EvaluationQuickStart({mode,@@ -30,6 +27,8 @@ export function EvaluationQuickStart({preview: boolean}): JSX.Element | null {const { reportSetupInteraction } = useActions(productSetupStatusLogic({ productKey: ProductKey.LLM_EVALUATIONS }))+ const { requiresProviderKey } = useValues(llmProviderKeysLogic)+ const quickStartTemplates = recommendedTemplates(requiresProviderKey).slice(0, 3)const selectTemplate = (template: EvaluationTemplate, placement: 'card' | 'example'): void => {// pinned: same analytics event as the template picker, so template funnels count both entry points@@ -61,7 +60,7 @@ export function EvaluationQuickStart({return (<div className="flex flex-col gap-2"><div className="grid gap-2">- {QUICK_START_TEMPLATES.map((template) => (+ {quickStartTemplates.map((template) => (<TemplateCard key={template.key} template={template} onSelect={selectTemplate} />))}<ScratchTemplateCard onSelect={startFromScratch} />diff --git a/products/ai_observability/frontend/evaluations/EvaluationTemplates.stories.tsx b/products/ai_observability/frontend/evaluations/EvaluationTemplates.stories.tsxindex 5f83af4eab1c..ab828a0f3f64 100644--- a/products/ai_observability/frontend/evaluations/EvaluationTemplates.stories.tsx+++ b/products/ai_observability/frontend/evaluations/EvaluationTemplates.stories.tsx@@ -1,5 +1,7 @@import type { Meta, StoryObj } from '@storybook/react'+import { mswDecorator } from '~/mocks/browser'+import { EvaluationTemplatesScene } from './EvaluationTemplates'const meta: Meta<typeof EvaluationTemplatesScene> = {@@ -23,3 +25,13 @@ export const Narrow: Story = {</div>),}++export const WithoutApiKey: Story = {+ decorators: [+ mswDecorator({+ get: {+ '/api/environments/:team_id/llm_analytics/evaluation_config/': { active_provider_key: null },+ },+ }),+ ],+}diff --git a/products/ai_observability/frontend/evaluations/EvaluationTemplates.tsx b/products/ai_observability/frontend/evaluations/EvaluationTemplates.tsxindex 50a63f288ff9..dd632d2ddcd6 100644--- a/products/ai_observability/frontend/evaluations/EvaluationTemplates.tsx+++ b/products/ai_observability/frontend/evaluations/EvaluationTemplates.tsx@@ -10,6 +10,7 @@ import { LemonButton, LemonTabs, Link } from '@posthog/lemon-ui'import { pngHoggie } from 'lib/brand/hoggies'import { SceneExport } from 'scenes/sceneTypes'+import { llmProviderKeysLogic } from '../settings/llmProviderKeysLogic'import { EvaluationAgentActions } from './components/EvaluationAgentActions'import { TemplateCard } from './components/TemplateCard'import { getEvaluationBackTarget, getEvaluationTemplateSelectionUrl } from './evaluationNavigation'@@ -22,7 +23,7 @@ const POPULAR_TEMPLATE_KEYS = ['relevance', 'cost_latency', 'helpfulness', 'hall// Sentiment leads because more teams run sentiment checks than any other kind.// The score and category templates complete the first row so people discover those output types.// The second row holds the other templates that people most often keep running.-export const RECOMMENDED_TEMPLATE_KEYS = [+const RECOMMENDED_TEMPLATE_KEYS = ['sentiment','response_quality','task_outcome',@@ -30,6 +31,15 @@ export const RECOMMENDED_TEMPLATE_KEYS = ['hallucination','helpfulness',]+// An LLM judge needs the team's own provider key, so a team without one gets a first row it can run as is.+const NO_KEY_RECOMMENDED_TEMPLATE_KEYS = [+ 'sentiment',+ 'cost_latency',+ 'repeated_tool_calls',+ 'response_quality',+ 'task_outcome',+ 'hallucination',+]type TemplateTab = 'recommended' | 'all' | EvaluationTemplateCategory@@ -44,14 +54,18 @@ const CATEGORY_TABS: { key: EvaluationTemplateCategory; label: string }[] = [const byKeys = (keys: string[]): EvaluationTemplate[] =>keys.flatMap((key) => defaultEvaluationTemplates.filter((template) => template.key === key))+export function recommendedTemplates(requiresProviderKey: boolean): EvaluationTemplate[] {+ return byKeys(requiresProviderKey ? NO_KEY_RECOMMENDED_TEMPLATE_KEYS : RECOMMENDED_TEMPLATE_KEYS)+}+const ALL_TEMPLATES_POPULAR_FIRST = [...byKeys(POPULAR_TEMPLATE_KEYS),...defaultEvaluationTemplates.filter((template) => !POPULAR_TEMPLATE_KEYS.includes(template.key)),]-function templatesForTab(tab: TemplateTab): EvaluationTemplate[] {+function templatesForTab(tab: TemplateTab, requiresProviderKey: boolean): EvaluationTemplate[] {if (tab === 'recommended') {- return byKeys(RECOMMENDED_TEMPLATE_KEYS)+ return recommendedTemplates(requiresProviderKey)}if (tab === 'all') {return ALL_TEMPLATES_POPULAR_FIRST@@ -71,8 +85,9 @@ interface TemplatePickerProps {function TemplatePicker({ title, showBackButton = false, learnMoreUrl }: TemplatePickerProps): JSX.Element {const { searchParams } = useValues(router)+ const { requiresProviderKey } = useValues(llmProviderKeysLogic)const [tab, setTab] = useState<TemplateTab>('recommended')- const templates = templatesForTab(tab)+ const templates = templatesForTab(tab, requiresProviderKey)const selectTab = (key: TemplateTab): void => {setTab(key)