:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): make start from scratch a quick start card in the evals empty state (#123223)
Bernat Torres2061090f48a4
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex d5928dfcfda1..95d204339605 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -2369,9 +2369,9 @@ snapshots:components-product-empty-state--error-tracking-waiting-for-data-new-wizard--light:hash: v1.k794b7964.1c2478e9c9e74eb47d896585f4f78e377c27a838e4f60cb35f7e685709d4fff9.xW8y7FqybbzrWn2szBUXyfUTJMNMrCVmSVBDVa6Fpbgcomponents-product-empty-state--evaluations-needs-setup--dark:- hash: v1.k794b7964.a4e0a12b91a43ce92f9aada132ba4f9a96ad95b5ea9e6e7b76933a3d029957fb.SlyVebf5LwsdXeSis0flvdlhCf3em1xogUXAFvIDJqg+ hash: v1.k794b7964.1cd862f86d149aedf2780a397e7ec0f37f0bb2e7f59a3e3f7d0c9245400a1007.WcFvJCJjYAdAMGJmTjD2a0gRE3mvrMMq7lSXR9c0xdYcomponents-product-empty-state--evaluations-needs-setup--light:- hash: v1.k794b7964.efbbe0197700f5ce0654df7fa093ecdf36ff8b42619695deaa64f14d91dfa3bb.uQvA6WPeH3g2Y06kgYuF6ageuPVzA46wZsOOIzSzrCI+ hash: v1.k794b7964.1ccb9bba2ed6a41c77ec890370be38020510f80cac86fe58ee3b66066965789b.9l1dhLw_-LV8QUelevSLCcoH_CfnTINd_9tFCa2irCkcomponents-product-empty-state--experiments-needs-setup--dark:hash: v1.k794b7964.2cdc47f158cecd85687618856c238c7ac65456dfa887db9e5e66baa9476cd145.mBKJtWHPn-vr4FCycrTMD2WE4K4a6cOT8SGg4p6MzbEcomponents-product-empty-state--experiments-needs-setup--light:@@ -8953,13 +8953,13 @@ snapshots:scenes-app-ai-observability-evaluation-explanation--system-one-zero-probability--light:hash: v1.k794b7964.f56325c23a185f032b8e7c331463345ea780e4e1b9a85f44a2f560869f06afa4.KcU1H2opmRuSoQcBHacqKya9WVvI5GATsFn_lgLD8rMscenes-app-ai-observability-evaluations-onboarding--narrow--dark:- hash: v1.k794b7964.0ed6014f2a3fd0292708929997d9545f397b66c41641b668a95cfb685841988c.-Ik8mLqrjRuPSfcYXl2fo2qdH5gqVoGTMWFmXIWOpyE+ hash: v1.k794b7964.2fe8551c13c48d3966c607923d7457eb1edba026b14430e8c40a1052a5223554.h1DTp9Ar3Xk9DmfrMviNCD90-dEDhm_xLCi-Fl8C0Psscenes-app-ai-observability-evaluations-onboarding--narrow--light:- hash: v1.k794b7964.5a62442e74a47e86a74aeec554f673c52a9970f5da6174f5457d90f6b02e315e.oUAuAgENqNeiKNV8VgSujnuwWyAd34NGdnBM0ZNXH8U+ hash: v1.k794b7964.725c9cd138e9974a36814447547b53b57f74adf288df0362133840fb7b04a416.ZgYC21heeJMxtaLWGjfeDXIrZAhqovb5dhfEtPO-TCcscenes-app-ai-observability-evaluations-onboarding--no-online-evaluations--dark:- hash: v1.k794b7964.56307b43a9acf90ace571f095a991f5b8575398a28e75d0f3daafc0093461cdd.I4i1suNWgDO5QPJuG6JDF0hACCAlsXD0lmwk8OHkYzw+ hash: v1.k794b7964.3caf7af67991fbfda18f590ef7a86787d67dd96e02dd2b3806a9d4fc321837ee.hz1-YxbzgDhzEHRO8TndIgNIVd_qAKCY35yNf2k4UbIscenes-app-ai-observability-evaluations-onboarding--no-online-evaluations--light:- hash: v1.k794b7964.8aafd41513ae8814fe0bfa1245ed00c5cd3d7892c22d0a59fdc7a4be6d8dd8e6.kzvWxk7VWyCvrX-3kSaFjBXjuWU9KJ_8fPOod34uGVw+ hash: v1.k794b7964.1dbea4bbbd7b0731b841cfdf0dc28826e9bba9caf9735cc7257e0fc4e0a67026.ezM9M-EOjWsTZXJk602I44gvDXvOzobiHJq09E0OEacscenes-app-ai-observability-evaluations-overview--directory-open--dark:hash: v1.k794b7964.cb88d8783a0c7ab63525f13c5aafbdddcb35a5448c12a774c8465122228257d5.vVBMLgF2jQd6tJ_DlQjocyXVoe4XtpNjDKGijdEz8oUscenes-app-ai-observability-evaluations-overview--directory-open--light:diff --git a/products/ai_observability/frontend/emptyState/evaluationsEmptyState.tsx b/products/ai_observability/frontend/emptyState/evaluationsEmptyState.tsxindex 1e80c9f3e63a..a8854eb95a0b 100644--- a/products/ai_observability/frontend/emptyState/evaluationsEmptyState.tsx+++ b/products/ai_observability/frontend/emptyState/evaluationsEmptyState.tsx@@ -3,10 +3,8 @@ import { IconListCheck } from '@posthog/icons'import { pngHoggie } from 'lib/brand/hoggies'import type { SceneProductEmptyState } from 'lib/components/ProductEmptyState/types'-import { urls } from 'scenes/urls'import { ProductKey } from '~/queries/schema/schema-general'-import { AccessControlLevel, AccessControlResourceType } from '~/types'import { EvaluationQuickStart } from '../evaluations/EvaluationQuickStart'import { EvaluationsTabs } from '../evaluations/EvaluationsTabs'@@ -33,15 +31,8 @@ export const evaluationsEmptyState: SceneProductEmptyState = {lead: 'Score your generations, traces, or sessions in production. Use an LLM as a judge, Hog code, or sentiment analysis for a wide range of checks. Start with a template, from scratch, or with PostHog AI, then view pass rates over time and jump straight into the runs that failed.',},},- primaryAction: {- label: 'Create your first evaluation',- to: urls.aiObservabilityEvaluationTemplates(),- accessControl: {- resourceType: AccessControlResourceType.Evaluation,- minAccessLevel: AccessControlLevel.Editor,- },- dataAttr: 'create-evaluation-button',- },+ // Owns the primary-action slot so the shared fallback button doesn't come back; the quick start below is the only CTA.+ PrimaryAction: () => null,SetupActions: EvaluationQuickStart,docsUrl: 'https://posthog.com/docs/ai-evals/evaluations',skippable: false,diff --git a/products/ai_observability/frontend/evaluations/EvaluationQuickStart.test.tsx b/products/ai_observability/frontend/evaluations/EvaluationQuickStart.test.tsxindex 615b3c7c097d..6b5b7e90f1d0 100644--- a/products/ai_observability/frontend/evaluations/EvaluationQuickStart.test.tsx+++ b/products/ai_observability/frontend/evaluations/EvaluationQuickStart.test.tsx@@ -2,6 +2,7 @@ import '@testing-library/jest-dom'import { cleanup, render } from '@testing-library/react'import { router } from 'kea-router'+import posthog from 'posthog-js'import { initKeaTests } from '~/test/init'import { AccessControlLevel, AccessControlResourceType, AppContext } from '~/types'@@ -17,6 +18,7 @@ describe('EvaluationQuickStart', () => {afterEach(() => {cleanup()+ jest.restoreAllMocks()window.POSTHOG_APP_CONTEXT = originalAppContext})@@ -31,13 +33,31 @@ describe('EvaluationQuickStart', () => {return render(<EvaluationQuickStart mode="needs-setup" preview={false} />).container}- it('opens a new evaluation from the clicked template', () => {+ it('offers three recommended templates and a scratch card', () => {const container = renderWithAccess(AccessControlLevel.Editor)- container.querySelector<HTMLButtonElement>('[data-attr="evaluation-template-sentiment"]')!.click()+ const cards = Array.from(container.querySelectorAll('[data-attr^="evaluation-template-"]')).filter(+ (element) => !element.getAttribute('data-attr')!.startsWith('evaluation-template-example-')+ )+ expect(cards).toHaveLength(4)+ expect(cards[3]).toHaveAttribute('data-attr', 'evaluation-template-scratch')+ })++ it.each([+ ['sentiment', 'sentiment', 'sentiment'],+ ['scratch', undefined, 'blank'],+ ])('opens a new evaluation from the %s card', (cardKey, expectedTemplate, trackedKey) => {+ const capture = jest.spyOn(posthog, 'capture').mockImplementation(() => undefined)+ const container = renderWithAccess(AccessControlLevel.Editor)++ container.querySelector<HTMLButtonElement>(`[data-attr="evaluation-template-${cardKey}"]`)!.click()expect(router.values.location.pathname).toContain('/evaluations/new')- expect(router.values.searchParams.template).toEqual('sentiment')+ expect(router.values.searchParams.template).toEqual(expectedTemplate)+ expect(capture).toHaveBeenCalledWith(+ 'llm evaluation template selected',+ expect.objectContaining({ template_key: trackedKey, source: 'empty_state_quick_start' })+ )})it('shows no templates to a user who cannot create evaluations', () => {diff --git a/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx b/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsxindex eab30345da28..3a3019a17c30 100644--- a/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx+++ b/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx@@ -12,12 +12,13 @@ import { urls } from 'scenes/urls'import { ProductKey } from '~/queries/schema/schema-general'import { AccessControlLevel, AccessControlResourceType } from '~/types'+import { ScratchTemplateCard } from './components/ScratchTemplateCard'import { TemplateCard } from './components/TemplateCard'import { getEvaluationTemplateSelectionUrl } from './evaluationNavigation'import { RECOMMENDED_TEMPLATE_KEYS } from './EvaluationTemplates'import { type EvaluationTemplate, defaultEvaluationTemplates } from './templates'-const QUICK_START_TEMPLATES = RECOMMENDED_TEMPLATE_KEYS.slice(0, 4).flatMap((key) =>+const QUICK_START_TEMPLATES = RECOMMENDED_TEMPLATE_KEYS.slice(0, 3).flatMap((key) =>defaultEvaluationTemplates.filter((template) => template.key === key))@@ -42,18 +43,28 @@ export function EvaluationQuickStart({router.actions.push(getEvaluationTemplateSelectionUrl(router.values.searchParams, template.key))}- // Viewers see the disabled primary action instead, because every card opens the creation form.+ const startFromScratch = (): void => {+ posthog.capture('llm evaluation template selected', {+ template_key: 'blank',+ placement: 'card',+ source: 'empty_state_quick_start',+ })+ reportSetupInteraction('template selected', mode, null, preview)+ router.actions.push(getEvaluationTemplateSelectionUrl(router.values.searchParams))+ }++ // Every card opens the creation form, so a viewer sees no quick start at all.if (!userHasAccess(AccessControlResourceType.Evaluation, AccessControlLevel.Editor)) {return null}return (<div className="flex flex-col gap-2">- <p className="m-0 text-sm text-secondary">Start from a template or create your own.</p>- <div className="flex flex-col gap-2">+ <div className="grid gap-2">{QUICK_START_TEMPLATES.map((template) => (<TemplateCard key={template.key} template={template} onSelect={selectTemplate} />))}+ <ScratchTemplateCard onSelect={startFromScratch} /></div><Linkto={urls.aiObservabilityEvaluationTemplates()}diff --git a/products/ai_observability/frontend/evaluations/EvaluationsScene.stories.tsx b/products/ai_observability/frontend/evaluations/EvaluationsScene.stories.tsxindex b3bfecef5aad..e4937c01c405 100644--- a/products/ai_observability/frontend/evaluations/EvaluationsScene.stories.tsx+++ b/products/ai_observability/frontend/evaluations/EvaluationsScene.stories.tsx@@ -17,7 +17,7 @@ const meta: Meta<typeof EvaluationsScene> = {pageUrl: urls.aiObservabilityEvaluations(),mockDate: '2026-10-01',featureFlags: [FEATURE_FLAGS.LLM_ANALYTICS_DATASETS, FEATURE_FLAGS.AI_OBSERVABILITY_OFFLINE_EVALUATIONS],- testOptions: { waitForSelector: '[data-attr="create-evaluation-button"]' },+ testOptions: { waitForSelector: '[data-attr="evaluation-template-scratch"]' },},decorators: [(Story, { parameters }) => (diff --git a/products/ai_observability/frontend/evaluations/components/ScratchTemplateCard.tsx b/products/ai_observability/frontend/evaluations/components/ScratchTemplateCard.tsxnew file mode 100644index 000000000000..baacc02af2ed--- /dev/null+++ b/products/ai_observability/frontend/evaluations/components/ScratchTemplateCard.tsx@@ -0,0 +1,25 @@+import { IconPlus } from '@posthog/icons'++export function ScratchTemplateCard({ onSelect }: { onSelect: () => void }): JSX.Element {+ return (+ <div className="relative flex h-full gap-3 p-3 rounded-lg border border-primary bg-surface-primary hover:border-accent transition-colors motion-reduce:transition-none">+ <div className="flex items-center justify-center size-10 shrink-0 rounded-lg bg-fill-secondary text-secondary">+ <IconPlus className="size-5 shrink-0" />+ </div>+ <div className="flex flex-col flex-1 min-w-0 gap-0.5">+ <h3 className="m-0 text-sm font-semibold truncate">+ {/* The pseudo-element stretches this button over the whole card, so the card is one click target. */}+ <button+ type="button"+ className="text-left cursor-pointer after:absolute after:inset-0 after:rounded-lg focus-visible:outline-none focus-visible:after:ring-2 focus-visible:after:ring-inset focus-visible:after:ring-accent"+ data-attr="evaluation-template-scratch"+ onClick={onSelect}+ >+ Start from scratch+ </button>+ </h3>+ <p className="m-0 text-xs text-secondary">Write your own prompt or code</p>+ </div>+ </div>+ )+}