:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): ask what an eval runs on before the check, with examples (#129982)
Bernat Torres3bb55d567acf
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex e95056db1b3c..94e7a09dfe9a 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -9219,25 +9219,25 @@ snapshots:scenes-app-ai-observability-instrumentation-checklist-card--warnings-with-a-dismissal--light:hash: v1.k794b7964.b0e06944ee7afe4af8cb39363280135111e71ba215aab56a3e410c79de62238b.WtW8suUOWanaTrvjxhV2Z_ANVDRmEW3bqHpTrTe5GoIscenes-app-ai-observability-new-evaluation--from-scratch--dark:- hash: v1.k794b7964.1623c532467fc2fb9021319640ab56552a83391fee86222a42627bb17667601b.nSDxQwNFpkX-6WGaA9yhItS9n2HoyhdGiqyEDOv1eX8+ hash: v1.k794b7964.fc91e3ff0dd9de6b0bea2d07d4395bc19e55b31b7831b3c7174f1e8584b41f64.kr-lIMIpsqbG6Ed42CkU6Kz-87cjHHOkLiH9jbTHRQwscenes-app-ai-observability-new-evaluation--from-scratch--light:- hash: v1.k794b7964.a3d8880f3f1cbf434a0c7a18914eb050b5d033574c89979d90945d04f1537ba5.9Th6uVg6fJ22ZWsbpTTGixJtC7pikKrioeL0YJ8vdY8+ hash: v1.k794b7964.55ee1fb64bab604d15996ff98496bf23f3614c4e06a064b5f3cca0301a4617b4.sEHRuW4DpUhc6lR2Ao7Ugmm7NCcUa5dN4HaEfMhGVCUscenes-app-ai-observability-new-evaluation--from-template--dark:- hash: v1.k794b7964.f9d9a09d9303c25bafcaaa20f59fd9628594e76e815539f2d7220fe5f96dcbc5.6hzX_X1xGupZ5Eatw5zyummoMJiz8rRAEHLPuK4cAok+ hash: v1.k794b7964.6b7adcdc3816c5b0527680571747bcd99c503f2ca943d05ffdc23449b93159d2.G9H6b_z2VPF2-yyntQSi_nd6T1CJa2P8wbkqk49S9XAscenes-app-ai-observability-new-evaluation--from-template--light:- hash: v1.k794b7964.54150a3b308abd9258f3eacbbb1194fce5cb6f429dac2ab8ca052e6180264fa8._bonOninP0AOBH3rqZ1MwCAGZxtS6SO7SGOa9gL_3DU+ hash: v1.k794b7964.d211644f72bfd103c304609e42d8bf6705beb79e13907a6e9fafaa7aac159216.ZMoZLzkVwLX0ibM4gN9kpWTh9WoUR6cHSzqfM87FTdsscenes-app-ai-observability-new-evaluation--nothing-matches--dark:- hash: v1.k794b7964.396e08ce75fb2c312044909b41b1765deaca062000bbb525db0c321d9ace4516.QwHRq-3wwGDBXMGuGY33f6qntgpARIZDNbPC24WcMqY+ hash: v1.k794b7964.0f75171a4469acec19eec631e7d1298e64d16e07507d602f199d6c733bb03ec5.7_ml8JjB28h13MFpF95zACT3_t5r3XrbpXDdaKFAjDUscenes-app-ai-observability-new-evaluation--nothing-matches--light:- hash: v1.k794b7964.27310b4f90ce455265af48454939d09269e1cea8ea398a4e74ba4db951428767.bM2eMOm2joan81_iWeCnSovmnW-TCmWOYh7J35QfFuE+ hash: v1.k794b7964.94b8c046ebc609ccc3eea39ea6a451f5e58ad5207def3a12431ec3277ccc8fe8.PcWnKTB1kz43hiIYi-FQx6EnI0wlf2xVGAlvqEqUGSUscenes-app-ai-observability-new-evaluation--reports-open--dark:- hash: v1.k794b7964.5ad9bdae0566f15dcdc5238c62fffb399b9cdeaa1d5a02f1929b2cab5ca4dc52.XAeaho41k-IUXOFQWLIjcbw9CP5uZsS-DUuPk8xIGaU+ hash: v1.k794b7964.a8c97a7c23359ece86c736fd2b5353d47a3c4b09acc36d0c48975ff2bab45802.2WkW7lPzPeGCpWmLTLs0yIqxPdV7bbXQL5r6bEbie8Uscenes-app-ai-observability-new-evaluation--reports-open--light:- hash: v1.k794b7964.2f347185de1abeeea3ac1cf1695f2d4ba31538107122355e8b85d0ef61ce927b.3eEJ2iM0k2Vzpd_A4wIFOV3VrZ5SsH3cX3WV97t3IWs+ hash: v1.k794b7964.34abbbe9167fb6e3e6b5a662d83130129b0e6627d4fe1d26f54e932d2caa0be5.mEd3vcxOZfsKGhLaDOAyUcp50eB9_OtiAOnVTyeKSywscenes-app-ai-observability-new-evaluation--sentiment-evaluation--dark:- hash: v1.k794b7964.8c336b6ebde49b2757c37c13a6aa941963b43e634e95cb3e7a88c3cd12557442.CbHuQYy2RT5HfqU_87nmhziWlrb84bZryG-MXR9sNes+ hash: v1.k794b7964.1ab345c62ae3ddb2665c37dc8510cd553dd0b161af5ea22a3152cd2e763eade7.CPiDb0h0SrNyFuxB-NxYQEnYQ1W4Gy_xHtL4yn9qxqwscenes-app-ai-observability-new-evaluation--sentiment-evaluation--light:- hash: v1.k794b7964.7111ca85011a30f773806bdcc42d96233c092103e3a16e9786da74a098cc354a.x0BG3GEY8hKgskXiUXY2VpX3JpOfdYKMy5xkJ4t2TtA+ hash: v1.k794b7964.40e4877066bd2b81a8bf276ae541c2e6ee0d3d1852874844352a88035afbfc8d.F4fC4xAFV9xXc1TsUmk8i7nDJKWekRtlJlYgb68_eFMscenes-app-ai-observability-offline-experiments-experiment--all-scorers--dark:hash: v1.k794b7964.5e999325599c0186b678a4aaba8423a09a578de4789220ddb18fbf7c64245764.9GJhQsDjcB9bKZSm5iX9O2RQApiIhl3KFVBhf1nv3_Iscenes-app-ai-observability-offline-experiments-experiment--all-scorers--light:diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationTriggers.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationTriggers.tsxindex 590486ee9e7a..4f5684d0e160 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationTriggers.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationTriggers.tsx@@ -1,11 +1,11 @@import { useActions, useValues } from 'kea'-import { IconCopy, IconPlus, IconTrash } from '@posthog/icons'+import { IconEllipsis, IconPlus } from '@posthog/icons'import { LemonButton, LemonInput } from '@posthog/lemon-ui'import { PropertyFilters } from 'lib/components/PropertyFilters/PropertyFilters'import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types'-import { LemonSlider } from 'lib/lemon-ui/LemonSlider'+import { LemonMenu } from 'lib/lemon-ui/LemonMenu'import { pluralize } from 'lib/utils/strings'import { llmEvaluationLogic } from '../llmEvaluationLogic'@@ -70,113 +70,82 @@ export function EvaluationTriggers({ conditions, onChange, unit }: EvaluationTriconst unitPlural = pluralize(2, unit ?? evaluation.target, undefined, false)return (- <div className="space-y-6">+ <div className="flex flex-col gap-1">{current.map((condition, index) => {const percentageValue = condition.rollout_percentage || 0+ const hasFilters = (condition.properties ?? []).length > 0return (- <div key={condition.id} className="bg-bg-light border rounded p-4 space-y-4">- {/* Header */}- <div className="flex flex-wrap justify-between items-center gap-2">- <div className="flex flex-wrap items-center gap-2">- <h4 className="font-semibold">Condition set {index + 1}</h4>- {current.length > 1 && (- <div className="text-sm text-muted">{index === 0 ? 'IF' : 'OR IF'}</div>- )}- </div>- <div className="flex gap-1">- <LemonButton- icon={<IconCopy />}+ <div key={condition.id} className="flex flex-col gap-1">+ <div className="flex items-start gap-2" data-attr="llma-evaluation-condition-set">+ <div className="flex flex-wrap items-center gap-2 text-sm min-w-0 flex-1">+ {/* Fixed width, so the inputs of every set line up under each other. */}+ <span className="w-12 shrink-0 text-right text-secondary">+ {index === 0 ? 'Run on' : 'or'}+ </span>+ <LemonInput+ type="number"size="small"- type="secondary"- onClick={() => duplicateConditionSet(index)}- tooltip="Duplicate condition set"+ className="w-20"+ value={percentageValue}+ onChange={(value) =>+ updateConditionSet(index, { rollout_percentage: Number(value) || 0 })+ }+ min={0.1}+ max={100}+ step={0.1}+ suffix={<span>%</span>}+ aria-label="Sampling percentage"+ status={percentageValue === 0 ? 'danger' : undefined}+ />+ <span>{hasFilters ? `of ${unitPlural} where` : `of all ${unitPlural}`}</span>+ <PropertyFilters+ propertyFilters={condition.properties}+ onChange={(properties) => updateConditionSet(index, { properties })}+ pageKey={`evaluation-condition-${condition.id}`}+ taxonomicGroupTypes={[+ TaxonomicFilterGroupType.EventProperties,+ TaxonomicFilterGroupType.EventMetadata,+ TaxonomicFilterGroupType.PersonProperties,+ ]}+ buttonSize="small"+ hasRowOperator={false}+ sendAllKeyUpdates+ allowRelativeDateOptions={false}/>- {current.length > 1 && (+ <LemonMenu+ items={[+ { label: 'Duplicate', onClick: () => duplicateConditionSet(index) },+ {+ label: 'Remove',+ status: 'danger',+ onClick: () => removeConditionSet(index),+ disabledReason:+ current.length === 1+ ? 'It needs at least one condition set'+ : undefined,+ },+ ]}+ ><LemonButton- icon={<IconTrash />}+ icon={<IconEllipsis />}size="small"- type="secondary"- status="danger"- onClick={() => removeConditionSet(index)}- tooltip="Remove condition set"+ aria-label={`Condition set ${index + 1} actions`}/>- )}+ </LemonMenu></div></div>-- {/* Percentage Control */}- <div className="space-y-2">- <label className="block text-sm font-medium">- Sampling percentage <span className="text-danger">*</span>- </label>- <div className="flex items-center gap-4 max-w-md">- <div className="flex-1 min-w-0">- <LemonSlider- value={percentageValue}- onChange={(value) => updateConditionSet(index, { rollout_percentage: value })}- min={0.1}- max={100}- step={0.1}- />- </div>- <div className="w-24 shrink-0">- <LemonInput- type="number"- value={percentageValue}- onChange={(value) =>- updateConditionSet(index, { rollout_percentage: Number(value) || 0 })- }- min={0.1}- max={100}- step={0.1}- suffix={<span>%</span>}- placeholder="Set percentage"- status={percentageValue === 0 ? 'danger' : undefined}- />- </div>- </div>- {percentageValue === 0 ? (- <div className="text-xs text-danger">- Please set a sampling percentage between 0.1% and 100%- </div>- ) : (- <div className="text-xs text-muted">- This evaluation will run on {percentageValue.toFixed(2)}% of matching {unitPlural}- </div>- )}- </div>-- {/* Property Filters */}- <div className="space-y-2">- <label className="block text-sm font-medium">Filter conditions</label>- <div className="text-sm text-muted mb-2">- Filter by generation event properties or person properties to target specific users or- generations. Leave empty to match all generations.- </div>- <PropertyFilters- propertyFilters={condition.properties}- onChange={(properties) => updateConditionSet(index, { properties })}- pageKey={`evaluation-condition-${condition.id}`}- taxonomicGroupTypes={[- TaxonomicFilterGroupType.EventProperties,- TaxonomicFilterGroupType.EventMetadata,- TaxonomicFilterGroupType.PersonProperties,- ]}- addText="Add filter condition"- hasRowOperator={false}- sendAllKeyUpdates- allowRelativeDateOptions={false}- />- </div>+ {percentageValue === 0 && (+ <span className="text-xs text-danger pl-14">+ Set a sampling percentage between 0.1% and 100%.+ </span>+ )}</div>)})}-- {/* Add Condition Set Button */}- <div className="flex justify-center">- <LemonButton type="secondary" icon={<IconPlus />} onClick={addConditionSet}>- Add Condition Set+ <div className="pl-12 pt-1">+ <LemonButton type="tertiary" size="small" icon={<IconPlus />} onClick={addConditionSet}>+ Add condition set</LemonButton></div></div>diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationTriggersPanel.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationTriggersPanel.tsxindex ea1d65d902c3..da8f215d84d9 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationTriggersPanel.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationTriggersPanel.tsx@@ -1,51 +1,75 @@-import { useValues } from 'kea'+import { useActions, useValues } from 'kea'++import { IconChevronDown } from '@posthog/icons'+import { LemonButton } from '@posthog/lemon-ui'++import { LemonMenu } from 'lib/lemon-ui/LemonMenu'+import { teamLogic } from 'scenes/teamLogic'import { llmEvaluationLogic } from '../llmEvaluationLogic'+import { QuickFilterKey, applyQuickFilter, internalUserFilters, quickFilterApplied } from './evaluationQuickFilters'import { EvaluationTriggers } from './EvaluationTriggers'-/** The condition sets an evaluation runs on, with how they combine and a few examples. */+const QUICK_FILTERS: { key: QuickFilterKey; label: string }[] = [+ { key: 'internal', label: 'Exclude internal users' },+ { key: 'errors', label: 'Only errors' },+ { key: 'expensive', label: 'Only calls over $0.01' },+ { key: 'sample', label: 'Sample 10%' },+]+export function EvaluationTriggersPanel(): JSX.Element | null {const { evaluation } = useValues(llmEvaluationLogic)+ const { setTriggerConditions } = useActions(llmEvaluationLogic)+ const { currentTeam } = useValues(teamLogic)if (!evaluation) {return null}- const hasFilters = evaluation.conditions.some((condition) => (condition.properties ?? []).length > 0)+ const testAccountFilters = currentTeam?.test_account_filters ?? []+ const usableInternalFilters = internalUserFilters(testAccountFilters).length+ const disabledReason = (key: QuickFilterKey): string | undefined => {+ // A generation judge skips errored generations without grading them. Trace and session judges grade the+ // whole transcript, errors included, so the filter stays useful there.+ if (key === 'errors' && evaluation.evaluation_type === 'llm_judge' && evaluation.target === 'generation') {+ return 'The LLM judge skips generations with errors, so this would leave it nothing to grade.'+ }+ if (quickFilterApplied(key, evaluation.conditions, testAccountFilters)) {+ return 'Already added'+ }+ if (key === 'internal' && usableInternalFilters === 0) {+ return 'Add internal and test user filters in project settings first.'+ }+ return undefined+ }+ const tooltip = (key: QuickFilterKey): string | undefined =>+ key === 'internal' && usableInternalFilters < testAccountFilters.length+ ? "Cohort and group filters from project settings can't be used here, so they're left out."+ : undefined+return (- <div>- <p className="text-muted text-sm mb-4">- The evaluation runs on generations that match any one of these condition sets. Within a set, all filters- must match, and sampling decides how many of the matching generations get evaluated.- {evaluation.target === 'trace' && (- <>- {' '}- Conditions match individual generations. The whole trace is evaluated once any of its- generations matches, and sampling applies per trace.- </>- )}- </p>+ <div className="flex flex-col gap-2">+ <div className="flex items-center justify-between gap-2">+ <h4 className="mb-0 text-sm font-semibold">Which traffic</h4>+ <LemonMenu+ items={QUICK_FILTERS.map(({ key, label }) => ({+ label,+ onClick: () =>+ setTriggerConditions(applyQuickFilter(key, evaluation.conditions, testAccountFilters)),+ disabledReason: disabledReason(key),+ tooltip: tooltip(key),+ 'data-attr': `llma-evaluation-quick-filter-${key}`,+ }))}+ >+ <LemonButton type="tertiary" size="xsmall" sideIcon={<IconChevronDown />}>+ <span className="text-xs font-normal text-secondary">Quick filters</span>+ </LemonButton>+ </LemonMenu>+ </div><EvaluationTriggers />- {/* Examples help before the first filter. Once someone filters, they only take room. */}- {!hasFilters && (- <div className="bg-bg-light border rounded p-3 text-sm mt-6">- <h4 className="font-semibold mb-2">Examples:</h4>- <ul className="space-y-1 text-muted list-disc list-inside">- <li>- <strong>10% of all generations:</strong> Set 10% sampling with no filter conditions- </li>- <li>- <strong>5% of one model's generations:</strong> Set 5% sampling with $ai_model containing- your model's name- </li>- <li>- <strong>Exclude internal users:</strong> Set 100% sampling with person property is_internal- ≠ true- </li>- <li>- <strong>High-cost generations:</strong> Set 100% sampling with $ai_total_cost_usd > 0.01- </li>- </ul>- </div>+ {evaluation.target !== 'generation' && (+ <p className="mb-0 text-xs text-secondary">+ A {evaluation.target} matches when one of its generations does.+ </p>)}</div>)diff --git a/products/ai_observability/frontend/evaluations/components/evaluationQuickFilters.test.ts b/products/ai_observability/frontend/evaluations/components/evaluationQuickFilters.test.tsnew file mode 100644index 000000000000..60b800f2a7d5--- /dev/null+++ b/products/ai_observability/frontend/evaluations/components/evaluationQuickFilters.test.ts@@ -0,0 +1,65 @@+import { AnyPropertyFilter, PropertyFilterType, PropertyOperator } from '~/types'++import type { EvaluationConditionSet } from '../types'+import { QuickFilterKey, applyQuickFilter, quickFilterApplied } from './evaluationQuickFilters'++const conditionSets = (): EvaluationConditionSet[] => [+ { id: 'a', rollout_percentage: 100, properties: [] },+ {+ id: 'b',+ rollout_percentage: 50,+ properties: [+ { type: PropertyFilterType.Event, key: '$ai_model', operator: PropertyOperator.Exact, value: ['x'] },+ ],+ },+]++const internalFilter: AnyPropertyFilter = {+ type: PropertyFilterType.Person,+ key: 'email',+ operator: PropertyOperator.NotIContains,+ value: '@example.com',+}+const cohortFilter = { type: PropertyFilterType.Cohort, key: 'id', value: 1 } as AnyPropertyFilter++describe('evaluationQuickFilters', () => {+ test.each<[QuickFilterKey, AnyPropertyFilter[], AnyPropertyFilter | null, number[]]>([+ ['internal', [internalFilter, cohortFilter], internalFilter, [100, 50]],+ [+ 'errors',+ [],+ { type: PropertyFilterType.Event, key: '$ai_is_error', operator: PropertyOperator.Exact, value: ['true'] },+ [100, 50],+ ],+ [+ 'expensive',+ [],+ {+ type: PropertyFilterType.Event,+ key: '$ai_total_cost_usd',+ operator: PropertyOperator.GreaterThan,+ value: 0.01,+ },+ [100, 50],+ ],+ ['sample', [], null, [10, 10]],+ ])(+ '%s reaches every condition set once, and a second click adds nothing',+ (key, testAccountFilters, added, rollouts) => {+ const once = applyQuickFilter(key, conditionSets(), testAccountFilters)+ const twice = applyQuickFilter(key, once, testAccountFilters)++ expect(quickFilterApplied(key, once, testAccountFilters)).toBe(true)+ expect(twice).toEqual(once)+ expect(once.map((set) => set.rollout_percentage)).toEqual(rollouts)+ expect(once.map((set) => set.properties?.length)).toEqual(added ? [1, 2] : [0, 1])+ once.forEach((set) => expect(set.properties).toEqual(expect.arrayContaining(added ? [added] : [])))+ expect(once.flatMap((set) => set.properties ?? [])).not.toContainEqual(cohortFilter)+ }+ )++ it('treats internal users as not set up when the team only has filters a trigger cannot hold', () => {+ expect(applyQuickFilter('internal', conditionSets(), [cohortFilter])).toEqual(conditionSets())+ expect(quickFilterApplied('internal', conditionSets(), [cohortFilter])).toBe(false)+ })+})diff --git a/products/ai_observability/frontend/evaluations/components/evaluationQuickFilters.ts b/products/ai_observability/frontend/evaluations/components/evaluationQuickFilters.tsnew file mode 100644index 000000000000..a99277ad69a5--- /dev/null+++ b/products/ai_observability/frontend/evaluations/components/evaluationQuickFilters.ts@@ -0,0 +1,77 @@+import { objectsEqual } from 'lib/utils/objects'++import { AnyPropertyFilter, PropertyFilterType, PropertyOperator } from '~/types'++import type { EvaluationConditionSet } from '../types'++export type QuickFilterKey = 'internal' | 'errors' | 'expensive' | 'sample'++const QUICK_SAMPLE_PERCENTAGE = 10++const ERRORS_FILTER: AnyPropertyFilter = {+ type: PropertyFilterType.Event,+ key: '$ai_is_error',+ operator: PropertyOperator.Exact,+ value: ['true'],+}++const EXPENSIVE_FILTER: AnyPropertyFilter = {+ type: PropertyFilterType.Event,+ key: '$ai_total_cost_usd',+ operator: PropertyOperator.GreaterThan,+ value: 0.01,+}++/** The team's internal and test user filters that a condition set can hold. Triggers take event and person properties only. */+export function internalUserFilters(testAccountFilters: AnyPropertyFilter[]): AnyPropertyFilter[] {+ return testAccountFilters.filter(+ (filter) => filter.type === PropertyFilterType.Event || filter.type === PropertyFilterType.Person+ )+}++function filtersFor(key: QuickFilterKey, testAccountFilters: AnyPropertyFilter[]): AnyPropertyFilter[] {+ switch (key) {+ case 'internal':+ return internalUserFilters(testAccountFilters)+ case 'errors':+ return [ERRORS_FILTER]+ case 'expensive':+ return [EXPENSIVE_FILTER]+ case 'sample':+ return []+ }+}++const hasFilter = (condition: EvaluationConditionSet, filter: AnyPropertyFilter): boolean =>+ (condition.properties ?? []).some((existing) => objectsEqual(existing, filter))++/** Whether every condition set already has what the quick filter adds. */+export function quickFilterApplied(+ key: QuickFilterKey,+ conditions: EvaluationConditionSet[],+ testAccountFilters: AnyPropertyFilter[]+): boolean {+ if (key === 'sample') {+ return conditions.every((condition) => condition.rollout_percentage === QUICK_SAMPLE_PERCENTAGE)+ }+ const filters = filtersFor(key, testAccountFilters)+ return (+ filters.length > 0 && conditions.every((condition) => filters.every((filter) => hasFilter(condition, filter)))+ )+}++/** Adds the quick filter to every condition set, so it holds whichever set matches. */+export function applyQuickFilter(+ key: QuickFilterKey,+ conditions: EvaluationConditionSet[],+ testAccountFilters: AnyPropertyFilter[]+): EvaluationConditionSet[] {+ if (key === 'sample') {+ return conditions.map((condition) => ({ ...condition, rollout_percentage: QUICK_SAMPLE_PERCENTAGE }))+ }+ const filters = filtersFor(key, testAccountFilters)+ return conditions.map((condition) => ({+ ...condition,+ properties: [...(condition.properties ?? []), ...filters.filter((filter) => !hasFilter(condition, filter))],+ }))+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsxindex 744b05bb4eac..31f23b10064e 100644--- a/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx@@ -94,7 +94,7 @@ export const NothingMatches: Story = {},decorators: [scene(0)],play: async ({ canvasElement }) => {- await openPart(canvasElement, 'triggers', 'Triggers')+ await openPart(canvasElement, 'triggers', 'Quick filters')await waitFor(() => {if (!canvasElement.querySelector('[data-attr="llma-evaluation-create-volume-none"]')) {throw new Error('Waiting for the volume estimate')diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsxindex 5dc9e556f8fa..99ed57f0716c 100644--- a/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx+++ b/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx@@ -65,6 +65,15 @@ const STEPS: Step[] = [parts: [],checks: ['result'],},+ {+ key: 'traffic',+ title: 'Runs on',+ heading: 'What it runs on',+ description: ['Choose what one run covers and which traffic it checks.'],+ hoggie: pngHoggie(magnifyingGlassPng),+ parts: ['target', 'triggers'],+ checks: ['sampling'],+ },{key: 'check',title: 'Check',@@ -74,15 +83,6 @@ const STEPS: Step[] = [parts: ['model'],checks: ['criteria', 'model'],},- {- key: 'traffic',- title: 'Traffic',- heading: 'What it runs on',- description: ['Pick the live traffic it checks, and how much of it.'],- hoggie: pngHoggie(magnifyingGlassPng),- parts: ['target', 'triggers'],- checks: ['sampling'],- },{key: 'reports',title: 'Reports',@@ -122,23 +122,20 @@ function StepContent({ step }: { step: Step['key'] }): JSX.Element | null {)case 'output':return <EvaluationOutputFields />- case 'check':+ case 'traffic':return (<>- {evaluation.evaluation_type === 'llm_judge' && <EvaluationModelPanel />}- {evaluation.evaluation_type === 'llm_judge' && <EvaluationPromptEditor />}- {evaluation.evaluation_type === 'hog' && <EvaluationCodeEditor />}+ <EvaluationTargetPanel />+ <EvaluationTriggersPanel />+ <EvaluationVolumeEstimate /></>)- case 'traffic':+ case 'check':return (<>- <EvaluationTargetPanel />- <div className="flex flex-col gap-2">- <h4 className="mb-0 text-base font-semibold">Triggers</h4>- <EvaluationTriggersPanel />- </div>- <EvaluationVolumeEstimate />+ {evaluation.evaluation_type === 'llm_judge' && <EvaluationModelPanel />}+ {evaluation.evaluation_type === 'llm_judge' && <EvaluationPromptEditor />}+ {evaluation.evaluation_type === 'hog' && <EvaluationCodeEditor />}</>)case 'reports':diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationLevelDiagram.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationLevelDiagram.tsxnew file mode 100644index 000000000000..f2d6f83a493e--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationLevelDiagram.tsx@@ -0,0 +1,115 @@+import { Link } from '@posthog/lemon-ui'++import { cn } from 'lib/utils/css-classes'++import type { TraceNodeApi } from '../../generated/api.schemas'+import { NodeKindGlyph } from '../../redesign/trace/components/NodeKindGlyph'+import { NodeStatsLine } from '../../redesign/trace/components/NodeStatsLine'+import type { EvaluationTarget } from '../types'++const stats = (costUsd: number | null, latencyMs: number): TraceNodeApi['stats'] => ({+ costUsd,+ latencyMs,+ inputTokens: null,+ outputTokens: null,+ cacheReadTokens: null,+ cacheWriteTokens: null,+})++const node = (+ id: string,+ kind: TraceNodeApi['kind'],+ name: string,+ nodeStats: TraceNodeApi['stats'],+ children: TraceNodeApi[] = [],+ model: string | null = null+): TraceNodeApi => ({ id, kind, name, model, stats: nodeStats, hasError: false, children })++const SAMPLE_SESSION: TraceNodeApi[] = [+ node('trace-1', 'trace', 'support_chat', stats(0.0041, 2300), [+ node('span-1', 'span', 'search_docs', stats(null, 400)),+ node('gen-1', 'generation', 'answer_question', stats(0.0032, 1600), [], 'gpt-4.1-mini'),+ ]),+ node('trace-2', 'trace', 'support_chat', stats(0.0028, 1400), [+ node('gen-3', 'generation', 'answer_question', stats(0.0028, 1400), [], 'gpt-4.1-mini'),+ ]),+]++const HIGHLIGHT = 'rounded bg-accent-highlight-light ring-1 ring-accent'++function RunLabel(): JSX.Element {+ return <span className="ml-auto shrink-0 pl-2 text-xs font-semibold text-accent">1 run</span>+}++function Row({ item, depth, oneRun }: { item: TraceNodeApi; depth: number; oneRun: boolean }): JSX.Element {+ return (+ <div+ // Same indent as the trace view's tree rows.+ className={cn(+ 'flex min-w-0 items-center gap-2 py-0.5 pr-1.5',+ depth === 0 ? 'pl-1.5' : 'pl-4.5',+ oneRun && HIGHLIGHT+ )}+ >+ <NodeKindGlyph kind={item.kind} />+ <span className="truncate text-xs font-medium">{item.name}</span>+ <NodeStatsLine stats={item.stats} model={item.model} compact className="min-w-0" />+ {oneRun && <RunLabel />}+ </div>+ )+}++export function EvaluationLevelDiagram({ level }: { level: EvaluationTarget }): JSX.Element {+ return (+ <div className="flex flex-col gap-2">+ <div+ aria-hidden+ className={cn(+ 'flex flex-col gap-0.5 rounded border border-primary bg-surface-primary p-1',+ level === 'session' && HIGHLIGHT+ )}+ >+ <div className="flex items-center px-1.5 py-0.5 text-xs text-secondary">+ Example session+ {level === 'session' && <RunLabel />}+ </div>+ {SAMPLE_SESSION.map((trace, traceIndex) => {+ const traceIsRun = level === 'trace' && traceIndex === 0+ return (+ <div key={trace.id} className={cn(traceIsRun && HIGHLIGHT)}>+ <div className="flex items-center">+ <div className="min-w-0 flex-1">+ <Row item={trace} depth={0} oneRun={false} />+ </div>+ {traceIsRun && (+ <span className="pr-1.5">+ <RunLabel />+ </span>+ )}+ </div>+ {trace.children.map((child) => (+ <Row+ key={child.id}+ item={child}+ depth={1}+ oneRun={level === 'generation' && child.id === 'gen-1'}+ />+ ))}+ </div>+ )+ })}+ </div>+ <span className="text-xs text-secondary">+ Learn about{' '}+ <Link to="https://posthog.com/docs/ai-observability/traces" target="_blank">+ traces+ </Link>{' '}+ and{' '}+ <Link to="https://posthog.com/docs/ai-observability/sessions" target="_blank">+ sessions+ </Link>+ .+ </span>+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationTargetPanel.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationTargetPanel.tsxindex e1c0d7037b98..50c1412b35f6 100644--- a/products/ai_observability/frontend/evaluations/create/EvaluationTargetPanel.tsx+++ b/products/ai_observability/frontend/evaluations/create/EvaluationTargetPanel.tsx@@ -1,22 +1,20 @@import { useActions, useValues } from 'kea'-import { LemonSegmentedButton } from '@posthog/lemon-ui'+import { IconChat, IconMessage, IconStack } from '@posthog/icons'+import { LemonCollapse } from '@posthog/lemon-ui'import { FEATURE_FLAGS } from 'lib/constants'-import { LemonField } from 'lib/lemon-ui/LemonField'import { featureFlagLogic } from 'lib/logic/featureFlagLogic'import { EvaluationSettleConfig } from '../components/EvaluationSettleConfig'import { evaluationOffersSessionTarget } from '../evaluationCapabilities'import { llmEvaluationLogic } from '../llmEvaluationLogic'import type { EvaluationTarget } from '../types'+import { EvaluationChoiceCards } from './EvaluationChoiceCards'+import { EvaluationLevelDiagram } from './EvaluationLevelDiagram'+import { settleSummary } from './evaluationSummary'-const TARGET_HINTS: Record<EvaluationTarget, string> = {- generation: 'Runs on each matching generation as soon as it arrives.',- trace: 'Runs once per trace, after the trace has had time to finish. Only generations sent with a trace ID count.',- session:- 'Runs once per session, after the session has had time to finish. Only generations sent with a session ID and a trace ID count.',-}+const SENTIMENT_ONLY_GENERATION = 'Sentiment reads one message at a time, so it runs on each generation.'/** What one run covers, and how long a trace or session gets to finish first. */export function EvaluationTargetPanel(): JSX.Element | null {@@ -33,25 +31,73 @@ export function EvaluationTargetPanel(): JSX.Element | null {evaluation,!!featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EVAL_SETTLING_STRATEGY])+ const summary = settleSummary(evaluation)return (- <div className="flex flex-col gap-4">- {!isSentiment && (- <LemonField.Pure label="Evaluate" help={TARGET_HINTS[target]}>- <LemonSegmentedButton+ <div className="flex flex-col gap-3">+ <div className="grid grid-cols-1 items-start gap-4 @min-[40rem]/evaluation:grid-cols-2">+ <div className="flex flex-col gap-3 min-w-0">+ <EvaluationChoiceCards<EvaluationTarget>+ label="What one run covers"value={target}onChange={setEvaluationTarget}+ dataAttr="llma-evaluation-create-target"+ stackedoptions={[- { value: 'generation', label: 'Each generation' },- { value: 'trace', label: 'Whole trace' },- ...(offersSession ? [{ value: 'session' as const, label: 'Whole session' }] : []),+ {+ value: 'generation',+ label: 'Each generation',+ description: 'One LLM call. It runs as soon as the call arrives.',+ icon: <IconMessage />,+ },+ {+ value: 'trace',+ label: 'Whole trace',+ description:+ 'One user request with all its LLM calls. Needs a trace ID ($ai_trace_id).',+ icon: <IconStack />,+ disabledReason: isSentiment ? SENTIMENT_ONLY_GENERATION : undefined,+ },+ ...(offersSession+ ? [+ {+ value: 'session' as const,+ label: 'Whole session',+ description:+ 'One conversation with all its traces. Needs a session ID ($ai_session_id) and a trace ID ($ai_trace_id).',+ icon: <IconChat />,+ disabledReason: isSentiment ? SENTIMENT_ONLY_GENERATION : undefined,+ },+ ]+ : []),]}- fullWidth- data-attr="llma-evaluation-create-target"/>- </LemonField.Pure>- )}- <EvaluationSettleConfig />+ {summary && (+ <LemonCollapse+ embedded+ size="small"+ panels={[+ {+ key: 'advanced',+ header: (+ <span className="flex min-w-0 items-center gap-2 text-left">+ <span className="shrink-0 text-sm font-medium">Advanced</span>+ <span className="min-w-0 truncate text-xs text-secondary">{summary}</span>+ </span>+ ),+ content: (+ <div className="flex flex-col gap-4 pt-2">+ <EvaluationSettleConfig />+ </div>+ ),+ dataAttr: 'llma-evaluation-create-target-advanced',+ },+ ]}+ />+ )}+ </div>+ <EvaluationLevelDiagram level={target} />+ </div></div>)}diff --git a/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts b/products/ai_observability/frontend/evaluations/create/creationChecklist.test.tsindex b94569197936..e243497e702a 100644--- a/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts+++ b/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts@@ -52,9 +52,9 @@ describe('creationChecklist', () => {expect(creationChecklist(input({ evaluation })).map((item) => item.summary)).toEqual(['Helpfulness','Pass or fail',+ '100% of all generations','Is it helpful?','gpt-5-mini',- '100% of all generations',])})diff --git a/products/ai_observability/frontend/evaluations/create/creationChecklist.ts b/products/ai_observability/frontend/evaluations/create/creationChecklist.tsindex 7d7acead9bab..daf4cc985636 100644--- a/products/ai_observability/frontend/evaluations/create/creationChecklist.ts+++ b/products/ai_observability/frontend/evaluations/create/creationChecklist.ts@@ -54,6 +54,13 @@ export function creationChecklist(input: ChecklistInput): ChecklistItem[] {summary: templateOutputText(evaluation),})}+ items.push({+ key: 'sampling',+ label: 'Sampling',+ done: !input.hasUnsetSampling,+ todo: 'Set a sampling percentage between 0.1% and 100%',+ summary: capitalizeFirstLetter(triggerText(evaluation)),+ })if (evaluation.evaluation_type === 'llm_judge' || evaluation.evaluation_type === 'hog') {const content =evaluation.evaluation_type === 'llm_judge'@@ -80,13 +87,6 @@ export function creationChecklist(input: ChecklistInput): ChecklistItem[] {summary: evaluation.model_configuration?.model,})}- items.push({- key: 'sampling',- label: 'Sampling',- done: !input.hasUnsetSampling,- todo: 'Set a sampling percentage between 0.1% and 100%',- summary: capitalizeFirstLetter(triggerText(evaluation)),- })return items}diff --git a/products/ai_observability/frontend/evaluations/create/evaluationSummary.test.ts b/products/ai_observability/frontend/evaluations/create/evaluationSummary.test.tsindex e11a8133d4a3..e2d34ae37dee 100644--- a/products/ai_observability/frontend/evaluations/create/evaluationSummary.test.ts+++ b/products/ai_observability/frontend/evaluations/create/evaluationSummary.test.ts@@ -1,5 +1,5 @@import type { EvaluationConditionSet, EvaluationConfig } from '../types'-import { reportText, triggerText } from './evaluationSummary'+import { reportText, settleSummary, triggerText } from './evaluationSummary'const evaluation = (overrides: Partial<EvaluationConfig>): EvaluationConfig =>({@@ -52,4 +52,28 @@ describe('evaluation sentence parts', () => {])('reports: $name', ({ input, expected }) => {expect(reportText({ scheduleCadence: 'weekly', triggerThreshold: 100, ...input })).toEqual(expected)})++ test.each([+ { name: 'a generation runs at once', input: evaluation({ target: 'generation' }), expected: null },+ {+ name: 'a trace waits a fixed delay by default',+ input: evaluation({ target: 'trace', target_config: {} }),+ expected: 'It runs 30 minutes after the first matching generation.',+ },+ {+ name: 'a session waits for quiet by default',+ input: evaluation({ target: 'session', target_config: {} }),+ expected: 'It runs once the session has been quiet for 1 hour, or 1 day after it starts.',+ },+ {+ name: 'a stored quiet period wins over the default',+ input: evaluation({+ target: 'trace',+ target_config: { strategy: 'inactivity', quiet_period_seconds: 120, max_age_seconds: 3600 },+ } as Partial<EvaluationConfig>),+ expected: 'It runs once the trace has been quiet for 2 minutes, or 1 hour after it starts.',+ },+ ])('timing: $name', ({ input, expected }) => {+ expect(settleSummary(input)?.replace(/\u00a0/g, ' ') ?? null).toEqual(expected)+ })})diff --git a/products/ai_observability/frontend/evaluations/create/evaluationSummary.ts b/products/ai_observability/frontend/evaluations/create/evaluationSummary.tsindex 41d153ee1249..cc8678396d0a 100644--- a/products/ai_observability/frontend/evaluations/create/evaluationSummary.ts+++ b/products/ai_observability/frontend/evaluations/create/evaluationSummary.ts@@ -1,6 +1,15 @@import { pluralize } from 'lib/utils/strings'+import { settleStrategy } from '../constants'import { evaluationTarget } from '../evaluationVolume'+import {+ DEFAULT_SESSION_MAX_AGE_SECONDS,+ DEFAULT_SESSION_QUIET_PERIOD_SECONDS,+ DEFAULT_SESSION_WINDOW_SECONDS,+ DEFAULT_TRACE_MAX_AGE_SECONDS,+ DEFAULT_TRACE_QUIET_PERIOD_SECONDS,+ DEFAULT_TRACE_WINDOW_SECONDS,+} from '../llmEvaluationLogic'import type { EvaluationConfig, EvaluationTarget } from '../types'const UNIT_NOUNS: Record<EvaluationTarget, string> = {@@ -66,3 +75,37 @@ export function reportsUnavailableReason(evaluation: EvaluationConfig, isReporta? 'Reports need a pass rate. In the Returns step, pick which categories count as a pass.': 'Reports are not available for this kind of evaluation.'}++const DURATION_UNITS: [string, number][] = [+ ['day', 86400],+ ['hour', 3600],+ ['minute', 60],+ ['second', 1],+]++/** A duration in its largest whole unit, such as "30 minutes". The duration picker only stores whole units. */+export function durationInWords(seconds: number): string {+ const [unit, size] = DURATION_UNITS.find(([, unitSize]) => seconds % unitSize === 0) ?? ['second', 1]+ return pluralize(seconds / size, unit)+}++/** When a trace or session run starts, in one sentence, so the timing settings can stay folded away. */+export function settleSummary(evaluation: EvaluationConfig): string | null {+ if (evaluation.target !== 'trace' && evaluation.target !== 'session') {+ return null+ }+ const isSession = evaluation.target === 'session'+ const config = evaluation.target_config+ const duration = durationInWords+ if (settleStrategy(config.strategy, evaluation.target) === 'fixed_window') {+ const wait =+ config.window_seconds ?? (isSession ? DEFAULT_SESSION_WINDOW_SECONDS : DEFAULT_TRACE_WINDOW_SECONDS)+ return `It runs ${duration(wait)} after the first matching generation.`+ }+ const quiet =+ config.quiet_period_seconds ??+ (isSession ? DEFAULT_SESSION_QUIET_PERIOD_SECONDS : DEFAULT_TRACE_QUIET_PERIOD_SECONDS)+ const maxAge =+ config.max_age_seconds ?? (isSession ? DEFAULT_SESSION_MAX_AGE_SECONDS : DEFAULT_TRACE_MAX_AGE_SECONDS)+ return `It runs once the ${evaluation.target} has been quiet for ${duration(quiet)}, or ${duration(maxAge)} after it starts.`+}