Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): make start from scratch a quick start card in the evals empty state (#123223)

Bernat Torres2061090f48a4
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index d5928dfcfda1..95d204339605 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -2369,9 +2369,9 @@ snapshots:
components-product-empty-state--error-tracking-waiting-for-data-new-wizard--light:
hash: v1.k794b7964.1c2478e9c9e74eb47d896585f4f78e377c27a838e4f60cb35f7e685709d4fff9.xW8y7FqybbzrWn2szBUXyfUTJMNMrCVmSVBDVa6Fpbg
components-product-empty-state--evaluations-needs-setup--dark:
- hash: v1.k794b7964.a4e0a12b91a43ce92f9aada132ba4f9a96ad95b5ea9e6e7b76933a3d029957fb.SlyVebf5LwsdXeSis0flvdlhCf3em1xogUXAFvIDJqg
+ hash: v1.k794b7964.1cd862f86d149aedf2780a397e7ec0f37f0bb2e7f59a3e3f7d0c9245400a1007.WcFvJCJjYAdAMGJmTjD2a0gRE3mvrMMq7lSXR9c0xdY
components-product-empty-state--evaluations-needs-setup--light:
- hash: v1.k794b7964.efbbe0197700f5ce0654df7fa093ecdf36ff8b42619695deaa64f14d91dfa3bb.uQvA6WPeH3g2Y06kgYuF6ageuPVzA46wZsOOIzSzrCI
+ hash: v1.k794b7964.1ccb9bba2ed6a41c77ec890370be38020510f80cac86fe58ee3b66066965789b.9l1dhLw_-LV8QUelevSLCcoH_CfnTINd_9tFCa2irCk
components-product-empty-state--experiments-needs-setup--dark:
hash: v1.k794b7964.2cdc47f158cecd85687618856c238c7ac65456dfa887db9e5e66baa9476cd145.mBKJtWHPn-vr4FCycrTMD2WE4K4a6cOT8SGg4p6MzbE
components-product-empty-state--experiments-needs-setup--light:
@@ -8953,13 +8953,13 @@ snapshots:
scenes-app-ai-observability-evaluation-explanation--system-one-zero-probability--light:
hash: v1.k794b7964.f56325c23a185f032b8e7c331463345ea780e4e1b9a85f44a2f560869f06afa4.KcU1H2opmRuSoQcBHacqKya9WVvI5GATsFn_lgLD8rM
scenes-app-ai-observability-evaluations-onboarding--narrow--dark:
- hash: v1.k794b7964.0ed6014f2a3fd0292708929997d9545f397b66c41641b668a95cfb685841988c.-Ik8mLqrjRuPSfcYXl2fo2qdH5gqVoGTMWFmXIWOpyE
+ hash: v1.k794b7964.2fe8551c13c48d3966c607923d7457eb1edba026b14430e8c40a1052a5223554.h1DTp9Ar3Xk9DmfrMviNCD90-dEDhm_xLCi-Fl8C0Ps
scenes-app-ai-observability-evaluations-onboarding--narrow--light:
- hash: v1.k794b7964.5a62442e74a47e86a74aeec554f673c52a9970f5da6174f5457d90f6b02e315e.oUAuAgENqNeiKNV8VgSujnuwWyAd34NGdnBM0ZNXH8U
+ hash: v1.k794b7964.725c9cd138e9974a36814447547b53b57f74adf288df0362133840fb7b04a416.ZgYC21heeJMxtaLWGjfeDXIrZAhqovb5dhfEtPO-TCc
scenes-app-ai-observability-evaluations-onboarding--no-online-evaluations--dark:
- hash: v1.k794b7964.56307b43a9acf90ace571f095a991f5b8575398a28e75d0f3daafc0093461cdd.I4i1suNWgDO5QPJuG6JDF0hACCAlsXD0lmwk8OHkYzw
+ hash: v1.k794b7964.3caf7af67991fbfda18f590ef7a86787d67dd96e02dd2b3806a9d4fc321837ee.hz1-YxbzgDhzEHRO8TndIgNIVd_qAKCY35yNf2k4UbI
scenes-app-ai-observability-evaluations-onboarding--no-online-evaluations--light:
- hash: v1.k794b7964.8aafd41513ae8814fe0bfa1245ed00c5cd3d7892c22d0a59fdc7a4be6d8dd8e6.kzvWxk7VWyCvrX-3kSaFjBXjuWU9KJ_8fPOod34uGVw
+ hash: v1.k794b7964.1dbea4bbbd7b0731b841cfdf0dc28826e9bba9caf9735cc7257e0fc4e0a67026.ezM9M-EOjWsTZXJk602I44gvDXvOzobiHJq09E0OEac
scenes-app-ai-observability-evaluations-overview--directory-open--dark:
hash: v1.k794b7964.cb88d8783a0c7ab63525f13c5aafbdddcb35a5448c12a774c8465122228257d5.vVBMLgF2jQd6tJ_DlQjocyXVoe4XtpNjDKGijdEz8oU
scenes-app-ai-observability-evaluations-overview--directory-open--light:
diff --git a/products/ai_observability/frontend/emptyState/evaluationsEmptyState.tsx b/products/ai_observability/frontend/emptyState/evaluationsEmptyState.tsx
index 1e80c9f3e63a..a8854eb95a0b 100644
--- a/products/ai_observability/frontend/emptyState/evaluationsEmptyState.tsx
+++ b/products/ai_observability/frontend/emptyState/evaluationsEmptyState.tsx
@@ -3,10 +3,8 @@ import { IconListCheck } from '@posthog/icons'
import { pngHoggie } from 'lib/brand/hoggies'
import type { SceneProductEmptyState } from 'lib/components/ProductEmptyState/types'
-import { urls } from 'scenes/urls'
import { ProductKey } from '~/queries/schema/schema-general'
-import { AccessControlLevel, AccessControlResourceType } from '~/types'
import { EvaluationQuickStart } from '../evaluations/EvaluationQuickStart'
import { EvaluationsTabs } from '../evaluations/EvaluationsTabs'
@@ -33,15 +31,8 @@ export const evaluationsEmptyState: SceneProductEmptyState = {
lead: 'Score your generations, traces, or sessions in production. Use an LLM as a judge, Hog code, or sentiment analysis for a wide range of checks. Start with a template, from scratch, or with PostHog AI, then view pass rates over time and jump straight into the runs that failed.',
},
},
- primaryAction: {
- label: 'Create your first evaluation',
- to: urls.aiObservabilityEvaluationTemplates(),
- accessControl: {
- resourceType: AccessControlResourceType.Evaluation,
- minAccessLevel: AccessControlLevel.Editor,
- },
- dataAttr: 'create-evaluation-button',
- },
+ // Owns the primary-action slot so the shared fallback button doesn't come back; the quick start below is the only CTA.
+ PrimaryAction: () => null,
SetupActions: EvaluationQuickStart,
docsUrl: 'https://posthog.com/docs/ai-evals/evaluations',
skippable: false,
diff --git a/products/ai_observability/frontend/evaluations/EvaluationQuickStart.test.tsx b/products/ai_observability/frontend/evaluations/EvaluationQuickStart.test.tsx
index 615b3c7c097d..6b5b7e90f1d0 100644
--- a/products/ai_observability/frontend/evaluations/EvaluationQuickStart.test.tsx
+++ b/products/ai_observability/frontend/evaluations/EvaluationQuickStart.test.tsx
@@ -2,6 +2,7 @@ import '@testing-library/jest-dom'
import { cleanup, render } from '@testing-library/react'
import { router } from 'kea-router'
+import posthog from 'posthog-js'
import { initKeaTests } from '~/test/init'
import { AccessControlLevel, AccessControlResourceType, AppContext } from '~/types'
@@ -17,6 +18,7 @@ describe('EvaluationQuickStart', () => {
afterEach(() => {
cleanup()
+ jest.restoreAllMocks()
window.POSTHOG_APP_CONTEXT = originalAppContext
})
@@ -31,13 +33,31 @@ describe('EvaluationQuickStart', () => {
return render(<EvaluationQuickStart mode="needs-setup" preview={false} />).container
}
- it('opens a new evaluation from the clicked template', () => {
+ it('offers three recommended templates and a scratch card', () => {
const container = renderWithAccess(AccessControlLevel.Editor)
- container.querySelector<HTMLButtonElement>('[data-attr="evaluation-template-sentiment"]')!.click()
+ const cards = Array.from(container.querySelectorAll('[data-attr^="evaluation-template-"]')).filter(
+ (element) => !element.getAttribute('data-attr')!.startsWith('evaluation-template-example-')
+ )
+ expect(cards).toHaveLength(4)
+ expect(cards[3]).toHaveAttribute('data-attr', 'evaluation-template-scratch')
+ })
+
+ it.each([
+ ['sentiment', 'sentiment', 'sentiment'],
+ ['scratch', undefined, 'blank'],
+ ])('opens a new evaluation from the %s card', (cardKey, expectedTemplate, trackedKey) => {
+ const capture = jest.spyOn(posthog, 'capture').mockImplementation(() => undefined)
+ const container = renderWithAccess(AccessControlLevel.Editor)
+
+ container.querySelector<HTMLButtonElement>(`[data-attr="evaluation-template-${cardKey}"]`)!.click()
expect(router.values.location.pathname).toContain('/evaluations/new')
- expect(router.values.searchParams.template).toEqual('sentiment')
+ expect(router.values.searchParams.template).toEqual(expectedTemplate)
+ expect(capture).toHaveBeenCalledWith(
+ 'llm evaluation template selected',
+ expect.objectContaining({ template_key: trackedKey, source: 'empty_state_quick_start' })
+ )
})
it('shows no templates to a user who cannot create evaluations', () => {
diff --git a/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx b/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx
index eab30345da28..3a3019a17c30 100644
--- a/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx
+++ b/products/ai_observability/frontend/evaluations/EvaluationQuickStart.tsx
@@ -12,12 +12,13 @@ import { urls } from 'scenes/urls'
import { ProductKey } from '~/queries/schema/schema-general'
import { AccessControlLevel, AccessControlResourceType } from '~/types'
+import { ScratchTemplateCard } from './components/ScratchTemplateCard'
import { TemplateCard } from './components/TemplateCard'
import { getEvaluationTemplateSelectionUrl } from './evaluationNavigation'
import { RECOMMENDED_TEMPLATE_KEYS } from './EvaluationTemplates'
import { type EvaluationTemplate, defaultEvaluationTemplates } from './templates'
-const QUICK_START_TEMPLATES = RECOMMENDED_TEMPLATE_KEYS.slice(0, 4).flatMap((key) =>
+const QUICK_START_TEMPLATES = RECOMMENDED_TEMPLATE_KEYS.slice(0, 3).flatMap((key) =>
defaultEvaluationTemplates.filter((template) => template.key === key)
)
@@ -42,18 +43,28 @@ export function EvaluationQuickStart({
router.actions.push(getEvaluationTemplateSelectionUrl(router.values.searchParams, template.key))
}
- // Viewers see the disabled primary action instead, because every card opens the creation form.
+ const startFromScratch = (): void => {
+ posthog.capture('llm evaluation template selected', {
+ template_key: 'blank',
+ placement: 'card',
+ source: 'empty_state_quick_start',
+ })
+ reportSetupInteraction('template selected', mode, null, preview)
+ router.actions.push(getEvaluationTemplateSelectionUrl(router.values.searchParams))
+ }
+
+ // Every card opens the creation form, so a viewer sees no quick start at all.
if (!userHasAccess(AccessControlResourceType.Evaluation, AccessControlLevel.Editor)) {
return null
}
return (
<div className="flex flex-col gap-2">
- <p className="m-0 text-sm text-secondary">Start from a template or create your own.</p>
- <div className="flex flex-col gap-2">
+ <div className="grid gap-2">
{QUICK_START_TEMPLATES.map((template) => (
<TemplateCard key={template.key} template={template} onSelect={selectTemplate} />
))}
+ <ScratchTemplateCard onSelect={startFromScratch} />
</div>
<Link
to={urls.aiObservabilityEvaluationTemplates()}
diff --git a/products/ai_observability/frontend/evaluations/EvaluationsScene.stories.tsx b/products/ai_observability/frontend/evaluations/EvaluationsScene.stories.tsx
index b3bfecef5aad..e4937c01c405 100644
--- a/products/ai_observability/frontend/evaluations/EvaluationsScene.stories.tsx
+++ b/products/ai_observability/frontend/evaluations/EvaluationsScene.stories.tsx
@@ -17,7 +17,7 @@ const meta: Meta<typeof EvaluationsScene> = {
pageUrl: urls.aiObservabilityEvaluations(),
mockDate: '2026-10-01',
featureFlags: [FEATURE_FLAGS.LLM_ANALYTICS_DATASETS, FEATURE_FLAGS.AI_OBSERVABILITY_OFFLINE_EVALUATIONS],
- testOptions: { waitForSelector: '[data-attr="create-evaluation-button"]' },
+ testOptions: { waitForSelector: '[data-attr="evaluation-template-scratch"]' },
},
decorators: [
(Story, { parameters }) => (
diff --git a/products/ai_observability/frontend/evaluations/components/ScratchTemplateCard.tsx b/products/ai_observability/frontend/evaluations/components/ScratchTemplateCard.tsx
new file mode 100644
index 000000000000..baacc02af2ed
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/components/ScratchTemplateCard.tsx
@@ -0,0 +1,25 @@
+import { IconPlus } from '@posthog/icons'
+
+export function ScratchTemplateCard({ onSelect }: { onSelect: () => void }): JSX.Element {
+ return (
+ <div className="relative flex h-full gap-3 p-3 rounded-lg border border-primary bg-surface-primary hover:border-accent transition-colors motion-reduce:transition-none">
+ <div className="flex items-center justify-center size-10 shrink-0 rounded-lg bg-fill-secondary text-secondary">
+ <IconPlus className="size-5 shrink-0" />
+ </div>
+ <div className="flex flex-col flex-1 min-w-0 gap-0.5">
+ <h3 className="m-0 text-sm font-semibold truncate">
+ {/* The pseudo-element stretches this button over the whole card, so the card is one click target. */}
+ <button
+ type="button"
+ className="text-left cursor-pointer after:absolute after:inset-0 after:rounded-lg focus-visible:outline-none focus-visible:after:ring-2 focus-visible:after:ring-inset focus-visible:after:ring-accent"
+ data-attr="evaluation-template-scratch"
+ onClick={onSelect}
+ >
+ Start from scratch
+ </button>
+ </h3>
+ <p className="m-0 text-xs text-secondary">Write your own prompt or code</p>
+ </div>
+ </div>
+ )
+}