Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): ask what an eval runs on before the check, with examples (#129982)

Bernat Torres3bb55d567acf
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index e95056db1b3c..94e7a09dfe9a 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -9219,25 +9219,25 @@ snapshots:
scenes-app-ai-observability-instrumentation-checklist-card--warnings-with-a-dismissal--light:
hash: v1.k794b7964.b0e06944ee7afe4af8cb39363280135111e71ba215aab56a3e410c79de62238b.WtW8suUOWanaTrvjxhV2Z_ANVDRmEW3bqHpTrTe5GoI
scenes-app-ai-observability-new-evaluation--from-scratch--dark:
- hash: v1.k794b7964.1623c532467fc2fb9021319640ab56552a83391fee86222a42627bb17667601b.nSDxQwNFpkX-6WGaA9yhItS9n2HoyhdGiqyEDOv1eX8
+ hash: v1.k794b7964.fc91e3ff0dd9de6b0bea2d07d4395bc19e55b31b7831b3c7174f1e8584b41f64.kr-lIMIpsqbG6Ed42CkU6Kz-87cjHHOkLiH9jbTHRQw
scenes-app-ai-observability-new-evaluation--from-scratch--light:
- hash: v1.k794b7964.a3d8880f3f1cbf434a0c7a18914eb050b5d033574c89979d90945d04f1537ba5.9Th6uVg6fJ22ZWsbpTTGixJtC7pikKrioeL0YJ8vdY8
+ hash: v1.k794b7964.55ee1fb64bab604d15996ff98496bf23f3614c4e06a064b5f3cca0301a4617b4.sEHRuW4DpUhc6lR2Ao7Ugmm7NCcUa5dN4HaEfMhGVCU
scenes-app-ai-observability-new-evaluation--from-template--dark:
- hash: v1.k794b7964.f9d9a09d9303c25bafcaaa20f59fd9628594e76e815539f2d7220fe5f96dcbc5.6hzX_X1xGupZ5Eatw5zyummoMJiz8rRAEHLPuK4cAok
+ hash: v1.k794b7964.6b7adcdc3816c5b0527680571747bcd99c503f2ca943d05ffdc23449b93159d2.G9H6b_z2VPF2-yyntQSi_nd6T1CJa2P8wbkqk49S9XA
scenes-app-ai-observability-new-evaluation--from-template--light:
- hash: v1.k794b7964.54150a3b308abd9258f3eacbbb1194fce5cb6f429dac2ab8ca052e6180264fa8._bonOninP0AOBH3rqZ1MwCAGZxtS6SO7SGOa9gL_3DU
+ hash: v1.k794b7964.d211644f72bfd103c304609e42d8bf6705beb79e13907a6e9fafaa7aac159216.ZMoZLzkVwLX0ibM4gN9kpWTh9WoUR6cHSzqfM87FTds
scenes-app-ai-observability-new-evaluation--nothing-matches--dark:
- hash: v1.k794b7964.396e08ce75fb2c312044909b41b1765deaca062000bbb525db0c321d9ace4516.QwHRq-3wwGDBXMGuGY33f6qntgpARIZDNbPC24WcMqY
+ hash: v1.k794b7964.0f75171a4469acec19eec631e7d1298e64d16e07507d602f199d6c733bb03ec5.7_ml8JjB28h13MFpF95zACT3_t5r3XrbpXDdaKFAjDU
scenes-app-ai-observability-new-evaluation--nothing-matches--light:
- hash: v1.k794b7964.27310b4f90ce455265af48454939d09269e1cea8ea398a4e74ba4db951428767.bM2eMOm2joan81_iWeCnSovmnW-TCmWOYh7J35QfFuE
+ hash: v1.k794b7964.94b8c046ebc609ccc3eea39ea6a451f5e58ad5207def3a12431ec3277ccc8fe8.PcWnKTB1kz43hiIYi-FQx6EnI0wlf2xVGAlvqEqUGSU
scenes-app-ai-observability-new-evaluation--reports-open--dark:
- hash: v1.k794b7964.5ad9bdae0566f15dcdc5238c62fffb399b9cdeaa1d5a02f1929b2cab5ca4dc52.XAeaho41k-IUXOFQWLIjcbw9CP5uZsS-DUuPk8xIGaU
+ hash: v1.k794b7964.a8c97a7c23359ece86c736fd2b5353d47a3c4b09acc36d0c48975ff2bab45802.2WkW7lPzPeGCpWmLTLs0yIqxPdV7bbXQL5r6bEbie8U
scenes-app-ai-observability-new-evaluation--reports-open--light:
- hash: v1.k794b7964.2f347185de1abeeea3ac1cf1695f2d4ba31538107122355e8b85d0ef61ce927b.3eEJ2iM0k2Vzpd_A4wIFOV3VrZ5SsH3cX3WV97t3IWs
+ hash: v1.k794b7964.34abbbe9167fb6e3e6b5a662d83130129b0e6627d4fe1d26f54e932d2caa0be5.mEd3vcxOZfsKGhLaDOAyUcp50eB9_OtiAOnVTyeKSyw
scenes-app-ai-observability-new-evaluation--sentiment-evaluation--dark:
- hash: v1.k794b7964.8c336b6ebde49b2757c37c13a6aa941963b43e634e95cb3e7a88c3cd12557442.CbHuQYy2RT5HfqU_87nmhziWlrb84bZryG-MXR9sNes
+ hash: v1.k794b7964.1ab345c62ae3ddb2665c37dc8510cd553dd0b161af5ea22a3152cd2e763eade7.CPiDb0h0SrNyFuxB-NxYQEnYQ1W4Gy_xHtL4yn9qxqw
scenes-app-ai-observability-new-evaluation--sentiment-evaluation--light:
- hash: v1.k794b7964.7111ca85011a30f773806bdcc42d96233c092103e3a16e9786da74a098cc354a.x0BG3GEY8hKgskXiUXY2VpX3JpOfdYKMy5xkJ4t2TtA
+ hash: v1.k794b7964.40e4877066bd2b81a8bf276ae541c2e6ee0d3d1852874844352a88035afbfc8d.F4fC4xAFV9xXc1TsUmk8i7nDJKWekRtlJlYgb68_eFM
scenes-app-ai-observability-offline-experiments-experiment--all-scorers--dark:
hash: v1.k794b7964.5e999325599c0186b678a4aaba8423a09a578de4789220ddb18fbf7c64245764.9GJhQsDjcB9bKZSm5iX9O2RQApiIhl3KFVBhf1nv3_I
scenes-app-ai-observability-offline-experiments-experiment--all-scorers--light:
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationTriggers.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationTriggers.tsx
index 590486ee9e7a..4f5684d0e160 100644
--- a/products/ai_observability/frontend/evaluations/components/EvaluationTriggers.tsx
+++ b/products/ai_observability/frontend/evaluations/components/EvaluationTriggers.tsx
@@ -1,11 +1,11 @@
import { useActions, useValues } from 'kea'
-import { IconCopy, IconPlus, IconTrash } from '@posthog/icons'
+import { IconEllipsis, IconPlus } from '@posthog/icons'
import { LemonButton, LemonInput } from '@posthog/lemon-ui'
import { PropertyFilters } from 'lib/components/PropertyFilters/PropertyFilters'
import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types'
-import { LemonSlider } from 'lib/lemon-ui/LemonSlider'
+import { LemonMenu } from 'lib/lemon-ui/LemonMenu'
import { pluralize } from 'lib/utils/strings'
import { llmEvaluationLogic } from '../llmEvaluationLogic'
@@ -70,113 +70,82 @@ export function EvaluationTriggers({ conditions, onChange, unit }: EvaluationTri
const unitPlural = pluralize(2, unit ?? evaluation.target, undefined, false)
return (
- <div className="space-y-6">
+ <div className="flex flex-col gap-1">
{current.map((condition, index) => {
const percentageValue = condition.rollout_percentage || 0
+ const hasFilters = (condition.properties ?? []).length > 0
return (
- <div key={condition.id} className="bg-bg-light border rounded p-4 space-y-4">
- {/* Header */}
- <div className="flex flex-wrap justify-between items-center gap-2">
- <div className="flex flex-wrap items-center gap-2">
- <h4 className="font-semibold">Condition set {index + 1}</h4>
- {current.length > 1 && (
- <div className="text-sm text-muted">{index === 0 ? 'IF' : 'OR IF'}</div>
- )}
- </div>
- <div className="flex gap-1">
- <LemonButton
- icon={<IconCopy />}
+ <div key={condition.id} className="flex flex-col gap-1">
+ <div className="flex items-start gap-2" data-attr="llma-evaluation-condition-set">
+ <div className="flex flex-wrap items-center gap-2 text-sm min-w-0 flex-1">
+ {/* Fixed width, so the inputs of every set line up under each other. */}
+ <span className="w-12 shrink-0 text-right text-secondary">
+ {index === 0 ? 'Run on' : 'or'}
+ </span>
+ <LemonInput
+ type="number"
size="small"
- type="secondary"
- onClick={() => duplicateConditionSet(index)}
- tooltip="Duplicate condition set"
+ className="w-20"
+ value={percentageValue}
+ onChange={(value) =>
+ updateConditionSet(index, { rollout_percentage: Number(value) || 0 })
+ }
+ min={0.1}
+ max={100}
+ step={0.1}
+ suffix={<span>%</span>}
+ aria-label="Sampling percentage"
+ status={percentageValue === 0 ? 'danger' : undefined}
+ />
+ <span>{hasFilters ? `of ${unitPlural} where` : `of all ${unitPlural}`}</span>
+ <PropertyFilters
+ propertyFilters={condition.properties}
+ onChange={(properties) => updateConditionSet(index, { properties })}
+ pageKey={`evaluation-condition-${condition.id}`}
+ taxonomicGroupTypes={[
+ TaxonomicFilterGroupType.EventProperties,
+ TaxonomicFilterGroupType.EventMetadata,
+ TaxonomicFilterGroupType.PersonProperties,
+ ]}
+ buttonSize="small"
+ hasRowOperator={false}
+ sendAllKeyUpdates
+ allowRelativeDateOptions={false}
/>
- {current.length > 1 && (
+ <LemonMenu
+ items={[
+ { label: 'Duplicate', onClick: () => duplicateConditionSet(index) },
+ {
+ label: 'Remove',
+ status: 'danger',
+ onClick: () => removeConditionSet(index),
+ disabledReason:
+ current.length === 1
+ ? 'It needs at least one condition set'
+ : undefined,
+ },
+ ]}
+ >
<LemonButton
- icon={<IconTrash />}
+ icon={<IconEllipsis />}
size="small"
- type="secondary"
- status="danger"
- onClick={() => removeConditionSet(index)}
- tooltip="Remove condition set"
+ aria-label={`Condition set ${index + 1} actions`}
/>
- )}
+ </LemonMenu>
</div>
</div>
-
- {/* Percentage Control */}
- <div className="space-y-2">
- <label className="block text-sm font-medium">
- Sampling percentage <span className="text-danger">*</span>
- </label>
- <div className="flex items-center gap-4 max-w-md">
- <div className="flex-1 min-w-0">
- <LemonSlider
- value={percentageValue}
- onChange={(value) => updateConditionSet(index, { rollout_percentage: value })}
- min={0.1}
- max={100}
- step={0.1}
- />
- </div>
- <div className="w-24 shrink-0">
- <LemonInput
- type="number"
- value={percentageValue}
- onChange={(value) =>
- updateConditionSet(index, { rollout_percentage: Number(value) || 0 })
- }
- min={0.1}
- max={100}
- step={0.1}
- suffix={<span>%</span>}
- placeholder="Set percentage"
- status={percentageValue === 0 ? 'danger' : undefined}
- />
- </div>
- </div>
- {percentageValue === 0 ? (
- <div className="text-xs text-danger">
- Please set a sampling percentage between 0.1% and 100%
- </div>
- ) : (
- <div className="text-xs text-muted">
- This evaluation will run on {percentageValue.toFixed(2)}% of matching {unitPlural}
- </div>
- )}
- </div>
-
- {/* Property Filters */}
- <div className="space-y-2">
- <label className="block text-sm font-medium">Filter conditions</label>
- <div className="text-sm text-muted mb-2">
- Filter by generation event properties or person properties to target specific users or
- generations. Leave empty to match all generations.
- </div>
- <PropertyFilters
- propertyFilters={condition.properties}
- onChange={(properties) => updateConditionSet(index, { properties })}
- pageKey={`evaluation-condition-${condition.id}`}
- taxonomicGroupTypes={[
- TaxonomicFilterGroupType.EventProperties,
- TaxonomicFilterGroupType.EventMetadata,
- TaxonomicFilterGroupType.PersonProperties,
- ]}
- addText="Add filter condition"
- hasRowOperator={false}
- sendAllKeyUpdates
- allowRelativeDateOptions={false}
- />
- </div>
+ {percentageValue === 0 && (
+ <span className="text-xs text-danger pl-14">
+ Set a sampling percentage between 0.1% and 100%.
+ </span>
+ )}
</div>
)
})}
-
- {/* Add Condition Set Button */}
- <div className="flex justify-center">
- <LemonButton type="secondary" icon={<IconPlus />} onClick={addConditionSet}>
- Add Condition Set
+ <div className="pl-12 pt-1">
+ <LemonButton type="tertiary" size="small" icon={<IconPlus />} onClick={addConditionSet}>
+ Add condition set
</LemonButton>
</div>
</div>
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationTriggersPanel.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationTriggersPanel.tsx
index ea1d65d902c3..da8f215d84d9 100644
--- a/products/ai_observability/frontend/evaluations/components/EvaluationTriggersPanel.tsx
+++ b/products/ai_observability/frontend/evaluations/components/EvaluationTriggersPanel.tsx
@@ -1,51 +1,75 @@
-import { useValues } from 'kea'
+import { useActions, useValues } from 'kea'
+
+import { IconChevronDown } from '@posthog/icons'
+import { LemonButton } from '@posthog/lemon-ui'
+
+import { LemonMenu } from 'lib/lemon-ui/LemonMenu'
+import { teamLogic } from 'scenes/teamLogic'
import { llmEvaluationLogic } from '../llmEvaluationLogic'
+import { QuickFilterKey, applyQuickFilter, internalUserFilters, quickFilterApplied } from './evaluationQuickFilters'
import { EvaluationTriggers } from './EvaluationTriggers'
-/** The condition sets an evaluation runs on, with how they combine and a few examples. */
+const QUICK_FILTERS: { key: QuickFilterKey; label: string }[] = [
+ { key: 'internal', label: 'Exclude internal users' },
+ { key: 'errors', label: 'Only errors' },
+ { key: 'expensive', label: 'Only calls over $0.01' },
+ { key: 'sample', label: 'Sample 10%' },
+]
+
export function EvaluationTriggersPanel(): JSX.Element | null {
const { evaluation } = useValues(llmEvaluationLogic)
+ const { setTriggerConditions } = useActions(llmEvaluationLogic)
+ const { currentTeam } = useValues(teamLogic)
if (!evaluation) {
return null
}
- const hasFilters = evaluation.conditions.some((condition) => (condition.properties ?? []).length > 0)
+ const testAccountFilters = currentTeam?.test_account_filters ?? []
+ const usableInternalFilters = internalUserFilters(testAccountFilters).length
+ const disabledReason = (key: QuickFilterKey): string | undefined => {
+ // A generation judge skips errored generations without grading them. Trace and session judges grade the
+ // whole transcript, errors included, so the filter stays useful there.
+ if (key === 'errors' && evaluation.evaluation_type === 'llm_judge' && evaluation.target === 'generation') {
+ return 'The LLM judge skips generations with errors, so this would leave it nothing to grade.'
+ }
+ if (quickFilterApplied(key, evaluation.conditions, testAccountFilters)) {
+ return 'Already added'
+ }
+ if (key === 'internal' && usableInternalFilters === 0) {
+ return 'Add internal and test user filters in project settings first.'
+ }
+ return undefined
+ }
+ const tooltip = (key: QuickFilterKey): string | undefined =>
+ key === 'internal' && usableInternalFilters < testAccountFilters.length
+ ? "Cohort and group filters from project settings can't be used here, so they're left out."
+ : undefined
+
return (
- <div>
- <p className="text-muted text-sm mb-4">
- The evaluation runs on generations that match any one of these condition sets. Within a set, all filters
- must match, and sampling decides how many of the matching generations get evaluated.
- {evaluation.target === 'trace' && (
- <>
- {' '}
- Conditions match individual generations. The whole trace is evaluated once any of its
- generations matches, and sampling applies per trace.
- </>
- )}
- </p>
+ <div className="flex flex-col gap-2">
+ <div className="flex items-center justify-between gap-2">
+ <h4 className="mb-0 text-sm font-semibold">Which traffic</h4>
+ <LemonMenu
+ items={QUICK_FILTERS.map(({ key, label }) => ({
+ label,
+ onClick: () =>
+ setTriggerConditions(applyQuickFilter(key, evaluation.conditions, testAccountFilters)),
+ disabledReason: disabledReason(key),
+ tooltip: tooltip(key),
+ 'data-attr': `llma-evaluation-quick-filter-${key}`,
+ }))}
+ >
+ <LemonButton type="tertiary" size="xsmall" sideIcon={<IconChevronDown />}>
+ <span className="text-xs font-normal text-secondary">Quick filters</span>
+ </LemonButton>
+ </LemonMenu>
+ </div>
<EvaluationTriggers />
- {/* Examples help before the first filter. Once someone filters, they only take room. */}
- {!hasFilters && (
- <div className="bg-bg-light border rounded p-3 text-sm mt-6">
- <h4 className="font-semibold mb-2">Examples:</h4>
- <ul className="space-y-1 text-muted list-disc list-inside">
- <li>
- <strong>10% of all generations:</strong> Set 10% sampling with no filter conditions
- </li>
- <li>
- <strong>5% of one model's generations:</strong> Set 5% sampling with $ai_model containing
- your model's name
- </li>
- <li>
- <strong>Exclude internal users:</strong> Set 100% sampling with person property is_internal
- ≠ true
- </li>
- <li>
- <strong>High-cost generations:</strong> Set 100% sampling with $ai_total_cost_usd &gt; 0.01
- </li>
- </ul>
- </div>
+ {evaluation.target !== 'generation' && (
+ <p className="mb-0 text-xs text-secondary">
+ A {evaluation.target} matches when one of its generations does.
+ </p>
)}
</div>
)
diff --git a/products/ai_observability/frontend/evaluations/components/evaluationQuickFilters.test.ts b/products/ai_observability/frontend/evaluations/components/evaluationQuickFilters.test.ts
new file mode 100644
index 000000000000..60b800f2a7d5
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/components/evaluationQuickFilters.test.ts
@@ -0,0 +1,65 @@
+import { AnyPropertyFilter, PropertyFilterType, PropertyOperator } from '~/types'
+
+import type { EvaluationConditionSet } from '../types'
+import { QuickFilterKey, applyQuickFilter, quickFilterApplied } from './evaluationQuickFilters'
+
+const conditionSets = (): EvaluationConditionSet[] => [
+ { id: 'a', rollout_percentage: 100, properties: [] },
+ {
+ id: 'b',
+ rollout_percentage: 50,
+ properties: [
+ { type: PropertyFilterType.Event, key: '$ai_model', operator: PropertyOperator.Exact, value: ['x'] },
+ ],
+ },
+]
+
+const internalFilter: AnyPropertyFilter = {
+ type: PropertyFilterType.Person,
+ key: 'email',
+ operator: PropertyOperator.NotIContains,
+ value: '@example.com',
+}
+const cohortFilter = { type: PropertyFilterType.Cohort, key: 'id', value: 1 } as AnyPropertyFilter
+
+describe('evaluationQuickFilters', () => {
+ test.each<[QuickFilterKey, AnyPropertyFilter[], AnyPropertyFilter | null, number[]]>([
+ ['internal', [internalFilter, cohortFilter], internalFilter, [100, 50]],
+ [
+ 'errors',
+ [],
+ { type: PropertyFilterType.Event, key: '$ai_is_error', operator: PropertyOperator.Exact, value: ['true'] },
+ [100, 50],
+ ],
+ [
+ 'expensive',
+ [],
+ {
+ type: PropertyFilterType.Event,
+ key: '$ai_total_cost_usd',
+ operator: PropertyOperator.GreaterThan,
+ value: 0.01,
+ },
+ [100, 50],
+ ],
+ ['sample', [], null, [10, 10]],
+ ])(
+ '%s reaches every condition set once, and a second click adds nothing',
+ (key, testAccountFilters, added, rollouts) => {
+ const once = applyQuickFilter(key, conditionSets(), testAccountFilters)
+ const twice = applyQuickFilter(key, once, testAccountFilters)
+
+ expect(quickFilterApplied(key, once, testAccountFilters)).toBe(true)
+ expect(twice).toEqual(once)
+ expect(once.map((set) => set.rollout_percentage)).toEqual(rollouts)
+ expect(once.map((set) => set.properties?.length)).toEqual(added ? [1, 2] : [0, 1])
+ once.forEach((set) => expect(set.properties).toEqual(expect.arrayContaining(added ? [added] : [])))
+ expect(once.flatMap((set) => set.properties ?? [])).not.toContainEqual(cohortFilter)
+ }
+ )
+
+ it('treats internal users as not set up when the team only has filters a trigger cannot hold', () => {
+ expect(applyQuickFilter('internal', conditionSets(), [cohortFilter])).toEqual(conditionSets())
+ expect(quickFilterApplied('internal', conditionSets(), [cohortFilter])).toBe(false)
+ })
+})
diff --git a/products/ai_observability/frontend/evaluations/components/evaluationQuickFilters.ts b/products/ai_observability/frontend/evaluations/components/evaluationQuickFilters.ts
new file mode 100644
index 000000000000..a99277ad69a5
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/components/evaluationQuickFilters.ts
@@ -0,0 +1,77 @@
+import { objectsEqual } from 'lib/utils/objects'
+
+import { AnyPropertyFilter, PropertyFilterType, PropertyOperator } from '~/types'
+
+import type { EvaluationConditionSet } from '../types'
+
+export type QuickFilterKey = 'internal' | 'errors' | 'expensive' | 'sample'
+
+const QUICK_SAMPLE_PERCENTAGE = 10
+
+const ERRORS_FILTER: AnyPropertyFilter = {
+ type: PropertyFilterType.Event,
+ key: '$ai_is_error',
+ operator: PropertyOperator.Exact,
+ value: ['true'],
+}
+
+const EXPENSIVE_FILTER: AnyPropertyFilter = {
+ type: PropertyFilterType.Event,
+ key: '$ai_total_cost_usd',
+ operator: PropertyOperator.GreaterThan,
+ value: 0.01,
+}
+
+/** The team's internal and test user filters that a condition set can hold. Triggers take event and person properties only. */
+export function internalUserFilters(testAccountFilters: AnyPropertyFilter[]): AnyPropertyFilter[] {
+ return testAccountFilters.filter(
+ (filter) => filter.type === PropertyFilterType.Event || filter.type === PropertyFilterType.Person
+ )
+}
+
+function filtersFor(key: QuickFilterKey, testAccountFilters: AnyPropertyFilter[]): AnyPropertyFilter[] {
+ switch (key) {
+ case 'internal':
+ return internalUserFilters(testAccountFilters)
+ case 'errors':
+ return [ERRORS_FILTER]
+ case 'expensive':
+ return [EXPENSIVE_FILTER]
+ case 'sample':
+ return []
+ }
+}
+
+const hasFilter = (condition: EvaluationConditionSet, filter: AnyPropertyFilter): boolean =>
+ (condition.properties ?? []).some((existing) => objectsEqual(existing, filter))
+
+/** Whether every condition set already has what the quick filter adds. */
+export function quickFilterApplied(
+ key: QuickFilterKey,
+ conditions: EvaluationConditionSet[],
+ testAccountFilters: AnyPropertyFilter[]
+): boolean {
+ if (key === 'sample') {
+ return conditions.every((condition) => condition.rollout_percentage === QUICK_SAMPLE_PERCENTAGE)
+ }
+ const filters = filtersFor(key, testAccountFilters)
+ return (
+ filters.length > 0 && conditions.every((condition) => filters.every((filter) => hasFilter(condition, filter)))
+ )
+}
+
+/** Adds the quick filter to every condition set, so it holds whichever set matches. */
+export function applyQuickFilter(
+ key: QuickFilterKey,
+ conditions: EvaluationConditionSet[],
+ testAccountFilters: AnyPropertyFilter[]
+): EvaluationConditionSet[] {
+ if (key === 'sample') {
+ return conditions.map((condition) => ({ ...condition, rollout_percentage: QUICK_SAMPLE_PERCENTAGE }))
+ }
+ const filters = filtersFor(key, testAccountFilters)
+ return conditions.map((condition) => ({
+ ...condition,
+ properties: [...(condition.properties ?? []), ...filters.filter((filter) => !hasFilter(condition, filter))],
+ }))
+}
diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx
index 744b05bb4eac..31f23b10064e 100644
--- a/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx
+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx
@@ -94,7 +94,7 @@ export const NothingMatches: Story = {
},
decorators: [scene(0)],
play: async ({ canvasElement }) => {
- await openPart(canvasElement, 'triggers', 'Triggers')
+ await openPart(canvasElement, 'triggers', 'Quick filters')
await waitFor(() => {
if (!canvasElement.querySelector('[data-attr="llma-evaluation-create-volume-none"]')) {
throw new Error('Waiting for the volume estimate')
diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx
index 5dc9e556f8fa..99ed57f0716c 100644
--- a/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx
+++ b/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx
@@ -65,6 +65,15 @@ const STEPS: Step[] = [
parts: [],
checks: ['result'],
},
+ {
+ key: 'traffic',
+ title: 'Runs on',
+ heading: 'What it runs on',
+ description: ['Choose what one run covers and which traffic it checks.'],
+ hoggie: pngHoggie(magnifyingGlassPng),
+ parts: ['target', 'triggers'],
+ checks: ['sampling'],
+ },
{
key: 'check',
title: 'Check',
@@ -74,15 +83,6 @@ const STEPS: Step[] = [
parts: ['model'],
checks: ['criteria', 'model'],
},
- {
- key: 'traffic',
- title: 'Traffic',
- heading: 'What it runs on',
- description: ['Pick the live traffic it checks, and how much of it.'],
- hoggie: pngHoggie(magnifyingGlassPng),
- parts: ['target', 'triggers'],
- checks: ['sampling'],
- },
{
key: 'reports',
title: 'Reports',
@@ -122,23 +122,20 @@ function StepContent({ step }: { step: Step['key'] }): JSX.Element | null {
)
case 'output':
return <EvaluationOutputFields />
- case 'check':
+ case 'traffic':
return (
<>
- {evaluation.evaluation_type === 'llm_judge' && <EvaluationModelPanel />}
- {evaluation.evaluation_type === 'llm_judge' && <EvaluationPromptEditor />}
- {evaluation.evaluation_type === 'hog' && <EvaluationCodeEditor />}
+ <EvaluationTargetPanel />
+ <EvaluationTriggersPanel />
+ <EvaluationVolumeEstimate />
</>
)
- case 'traffic':
+ case 'check':
return (
<>
- <EvaluationTargetPanel />
- <div className="flex flex-col gap-2">
- <h4 className="mb-0 text-base font-semibold">Triggers</h4>
- <EvaluationTriggersPanel />
- </div>
- <EvaluationVolumeEstimate />
+ {evaluation.evaluation_type === 'llm_judge' && <EvaluationModelPanel />}
+ {evaluation.evaluation_type === 'llm_judge' && <EvaluationPromptEditor />}
+ {evaluation.evaluation_type === 'hog' && <EvaluationCodeEditor />}
</>
)
case 'reports':
diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationLevelDiagram.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationLevelDiagram.tsx
new file mode 100644
index 000000000000..f2d6f83a493e
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/create/EvaluationLevelDiagram.tsx
@@ -0,0 +1,115 @@
+import { Link } from '@posthog/lemon-ui'
+
+import { cn } from 'lib/utils/css-classes'
+
+import type { TraceNodeApi } from '../../generated/api.schemas'
+import { NodeKindGlyph } from '../../redesign/trace/components/NodeKindGlyph'
+import { NodeStatsLine } from '../../redesign/trace/components/NodeStatsLine'
+import type { EvaluationTarget } from '../types'
+
+const stats = (costUsd: number | null, latencyMs: number): TraceNodeApi['stats'] => ({
+ costUsd,
+ latencyMs,
+ inputTokens: null,
+ outputTokens: null,
+ cacheReadTokens: null,
+ cacheWriteTokens: null,
+})
+
+const node = (
+ id: string,
+ kind: TraceNodeApi['kind'],
+ name: string,
+ nodeStats: TraceNodeApi['stats'],
+ children: TraceNodeApi[] = [],
+ model: string | null = null
+): TraceNodeApi => ({ id, kind, name, model, stats: nodeStats, hasError: false, children })
+
+const SAMPLE_SESSION: TraceNodeApi[] = [
+ node('trace-1', 'trace', 'support_chat', stats(0.0041, 2300), [
+ node('span-1', 'span', 'search_docs', stats(null, 400)),
+ node('gen-1', 'generation', 'answer_question', stats(0.0032, 1600), [], 'gpt-4.1-mini'),
+ ]),
+ node('trace-2', 'trace', 'support_chat', stats(0.0028, 1400), [
+ node('gen-3', 'generation', 'answer_question', stats(0.0028, 1400), [], 'gpt-4.1-mini'),
+ ]),
+]
+
+const HIGHLIGHT = 'rounded bg-accent-highlight-light ring-1 ring-accent'
+
+function RunLabel(): JSX.Element {
+ return <span className="ml-auto shrink-0 pl-2 text-xs font-semibold text-accent">1 run</span>
+}
+
+function Row({ item, depth, oneRun }: { item: TraceNodeApi; depth: number; oneRun: boolean }): JSX.Element {
+ return (
+ <div
+ // Same indent as the trace view's tree rows.
+ className={cn(
+ 'flex min-w-0 items-center gap-2 py-0.5 pr-1.5',
+ depth === 0 ? 'pl-1.5' : 'pl-4.5',
+ oneRun && HIGHLIGHT
+ )}
+ >
+ <NodeKindGlyph kind={item.kind} />
+ <span className="truncate text-xs font-medium">{item.name}</span>
+ <NodeStatsLine stats={item.stats} model={item.model} compact className="min-w-0" />
+ {oneRun && <RunLabel />}
+ </div>
+ )
+}
+
+export function EvaluationLevelDiagram({ level }: { level: EvaluationTarget }): JSX.Element {
+ return (
+ <div className="flex flex-col gap-2">
+ <div
+ aria-hidden
+ className={cn(
+ 'flex flex-col gap-0.5 rounded border border-primary bg-surface-primary p-1',
+ level === 'session' && HIGHLIGHT
+ )}
+ >
+ <div className="flex items-center px-1.5 py-0.5 text-xs text-secondary">
+ Example session
+ {level === 'session' && <RunLabel />}
+ </div>
+ {SAMPLE_SESSION.map((trace, traceIndex) => {
+ const traceIsRun = level === 'trace' && traceIndex === 0
+ return (
+ <div key={trace.id} className={cn(traceIsRun && HIGHLIGHT)}>
+ <div className="flex items-center">
+ <div className="min-w-0 flex-1">
+ <Row item={trace} depth={0} oneRun={false} />
+ </div>
+ {traceIsRun && (
+ <span className="pr-1.5">
+ <RunLabel />
+ </span>
+ )}
+ </div>
+ {trace.children.map((child) => (
+ <Row
+ key={child.id}
+ item={child}
+ depth={1}
+ oneRun={level === 'generation' && child.id === 'gen-1'}
+ />
+ ))}
+ </div>
+ )
+ })}
+ </div>
+ <span className="text-xs text-secondary">
+ Learn about{' '}
+ <Link to="https://posthog.com/docs/ai-observability/traces" target="_blank">
+ traces
+ </Link>{' '}
+ and{' '}
+ <Link to="https://posthog.com/docs/ai-observability/sessions" target="_blank">
+ sessions
+ </Link>
+ .
+ </span>
+ </div>
+ )
+}
diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationTargetPanel.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationTargetPanel.tsx
index e1c0d7037b98..50c1412b35f6 100644
--- a/products/ai_observability/frontend/evaluations/create/EvaluationTargetPanel.tsx
+++ b/products/ai_observability/frontend/evaluations/create/EvaluationTargetPanel.tsx
@@ -1,22 +1,20 @@
import { useActions, useValues } from 'kea'
-import { LemonSegmentedButton } from '@posthog/lemon-ui'
+import { IconChat, IconMessage, IconStack } from '@posthog/icons'
+import { LemonCollapse } from '@posthog/lemon-ui'
import { FEATURE_FLAGS } from 'lib/constants'
-import { LemonField } from 'lib/lemon-ui/LemonField'
import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { EvaluationSettleConfig } from '../components/EvaluationSettleConfig'
import { evaluationOffersSessionTarget } from '../evaluationCapabilities'
import { llmEvaluationLogic } from '../llmEvaluationLogic'
import type { EvaluationTarget } from '../types'
+import { EvaluationChoiceCards } from './EvaluationChoiceCards'
+import { EvaluationLevelDiagram } from './EvaluationLevelDiagram'
+import { settleSummary } from './evaluationSummary'
-const TARGET_HINTS: Record<EvaluationTarget, string> = {
- generation: 'Runs on each matching generation as soon as it arrives.',
- trace: 'Runs once per trace, after the trace has had time to finish. Only generations sent with a trace ID count.',
- session:
- 'Runs once per session, after the session has had time to finish. Only generations sent with a session ID and a trace ID count.',
-}
+const SENTIMENT_ONLY_GENERATION = 'Sentiment reads one message at a time, so it runs on each generation.'
/** What one run covers, and how long a trace or session gets to finish first. */
export function EvaluationTargetPanel(): JSX.Element | null {
@@ -33,25 +31,73 @@ export function EvaluationTargetPanel(): JSX.Element | null {
evaluation,
!!featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EVAL_SETTLING_STRATEGY]
)
+ const summary = settleSummary(evaluation)
return (
- <div className="flex flex-col gap-4">
- {!isSentiment && (
- <LemonField.Pure label="Evaluate" help={TARGET_HINTS[target]}>
- <LemonSegmentedButton
+ <div className="flex flex-col gap-3">
+ <div className="grid grid-cols-1 items-start gap-4 @min-[40rem]/evaluation:grid-cols-2">
+ <div className="flex flex-col gap-3 min-w-0">
+ <EvaluationChoiceCards<EvaluationTarget>
+ label="What one run covers"
value={target}
onChange={setEvaluationTarget}
+ dataAttr="llma-evaluation-create-target"
+ stacked
options={[
- { value: 'generation', label: 'Each generation' },
- { value: 'trace', label: 'Whole trace' },
- ...(offersSession ? [{ value: 'session' as const, label: 'Whole session' }] : []),
+ {
+ value: 'generation',
+ label: 'Each generation',
+ description: 'One LLM call. It runs as soon as the call arrives.',
+ icon: <IconMessage />,
+ },
+ {
+ value: 'trace',
+ label: 'Whole trace',
+ description:
+ 'One user request with all its LLM calls. Needs a trace ID ($ai_trace_id).',
+ icon: <IconStack />,
+ disabledReason: isSentiment ? SENTIMENT_ONLY_GENERATION : undefined,
+ },
+ ...(offersSession
+ ? [
+ {
+ value: 'session' as const,
+ label: 'Whole session',
+ description:
+ 'One conversation with all its traces. Needs a session ID ($ai_session_id) and a trace ID ($ai_trace_id).',
+ icon: <IconChat />,
+ disabledReason: isSentiment ? SENTIMENT_ONLY_GENERATION : undefined,
+ },
+ ]
+ : []),
]}
- fullWidth
- data-attr="llma-evaluation-create-target"
/>
- </LemonField.Pure>
- )}
- <EvaluationSettleConfig />
+ {summary && (
+ <LemonCollapse
+ embedded
+ size="small"
+ panels={[
+ {
+ key: 'advanced',
+ header: (
+ <span className="flex min-w-0 items-center gap-2 text-left">
+ <span className="shrink-0 text-sm font-medium">Advanced</span>
+ <span className="min-w-0 truncate text-xs text-secondary">{summary}</span>
+ </span>
+ ),
+ content: (
+ <div className="flex flex-col gap-4 pt-2">
+ <EvaluationSettleConfig />
+ </div>
+ ),
+ dataAttr: 'llma-evaluation-create-target-advanced',
+ },
+ ]}
+ />
+ )}
+ </div>
+ <EvaluationLevelDiagram level={target} />
+ </div>
</div>
)
}
diff --git a/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts b/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts
index b94569197936..e243497e702a 100644
--- a/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts
+++ b/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts
@@ -52,9 +52,9 @@ describe('creationChecklist', () => {
expect(creationChecklist(input({ evaluation })).map((item) => item.summary)).toEqual([
'Helpfulness',
'Pass or fail',
+ '100% of all generations',
'Is it helpful?',
'gpt-5-mini',
- '100% of all generations',
])
})
diff --git a/products/ai_observability/frontend/evaluations/create/creationChecklist.ts b/products/ai_observability/frontend/evaluations/create/creationChecklist.ts
index 7d7acead9bab..daf4cc985636 100644
--- a/products/ai_observability/frontend/evaluations/create/creationChecklist.ts
+++ b/products/ai_observability/frontend/evaluations/create/creationChecklist.ts
@@ -54,6 +54,13 @@ export function creationChecklist(input: ChecklistInput): ChecklistItem[] {
summary: templateOutputText(evaluation),
})
}
+ items.push({
+ key: 'sampling',
+ label: 'Sampling',
+ done: !input.hasUnsetSampling,
+ todo: 'Set a sampling percentage between 0.1% and 100%',
+ summary: capitalizeFirstLetter(triggerText(evaluation)),
+ })
if (evaluation.evaluation_type === 'llm_judge' || evaluation.evaluation_type === 'hog') {
const content =
evaluation.evaluation_type === 'llm_judge'
@@ -80,13 +87,6 @@ export function creationChecklist(input: ChecklistInput): ChecklistItem[] {
summary: evaluation.model_configuration?.model,
})
}
- items.push({
- key: 'sampling',
- label: 'Sampling',
- done: !input.hasUnsetSampling,
- todo: 'Set a sampling percentage between 0.1% and 100%',
- summary: capitalizeFirstLetter(triggerText(evaluation)),
- })
return items
}
diff --git a/products/ai_observability/frontend/evaluations/create/evaluationSummary.test.ts b/products/ai_observability/frontend/evaluations/create/evaluationSummary.test.ts
index e11a8133d4a3..e2d34ae37dee 100644
--- a/products/ai_observability/frontend/evaluations/create/evaluationSummary.test.ts
+++ b/products/ai_observability/frontend/evaluations/create/evaluationSummary.test.ts
@@ -1,5 +1,5 @@
import type { EvaluationConditionSet, EvaluationConfig } from '../types'
-import { reportText, triggerText } from './evaluationSummary'
+import { reportText, settleSummary, triggerText } from './evaluationSummary'
const evaluation = (overrides: Partial<EvaluationConfig>): EvaluationConfig =>
({
@@ -52,4 +52,28 @@ describe('evaluation sentence parts', () => {
])('reports: $name', ({ input, expected }) => {
expect(reportText({ scheduleCadence: 'weekly', triggerThreshold: 100, ...input })).toEqual(expected)
})
+
+ test.each([
+ { name: 'a generation runs at once', input: evaluation({ target: 'generation' }), expected: null },
+ {
+ name: 'a trace waits a fixed delay by default',
+ input: evaluation({ target: 'trace', target_config: {} }),
+ expected: 'It runs 30 minutes after the first matching generation.',
+ },
+ {
+ name: 'a session waits for quiet by default',
+ input: evaluation({ target: 'session', target_config: {} }),
+ expected: 'It runs once the session has been quiet for 1 hour, or 1 day after it starts.',
+ },
+ {
+ name: 'a stored quiet period wins over the default',
+ input: evaluation({
+ target: 'trace',
+ target_config: { strategy: 'inactivity', quiet_period_seconds: 120, max_age_seconds: 3600 },
+ } as Partial<EvaluationConfig>),
+ expected: 'It runs once the trace has been quiet for 2 minutes, or 1 hour after it starts.',
+ },
+ ])('timing: $name', ({ input, expected }) => {
+ expect(settleSummary(input)?.replace(/\u00a0/g, ' ') ?? null).toEqual(expected)
+ })
})
diff --git a/products/ai_observability/frontend/evaluations/create/evaluationSummary.ts b/products/ai_observability/frontend/evaluations/create/evaluationSummary.ts
index 41d153ee1249..cc8678396d0a 100644
--- a/products/ai_observability/frontend/evaluations/create/evaluationSummary.ts
+++ b/products/ai_observability/frontend/evaluations/create/evaluationSummary.ts
@@ -1,6 +1,15 @@
import { pluralize } from 'lib/utils/strings'
+import { settleStrategy } from '../constants'
import { evaluationTarget } from '../evaluationVolume'
+import {
+ DEFAULT_SESSION_MAX_AGE_SECONDS,
+ DEFAULT_SESSION_QUIET_PERIOD_SECONDS,
+ DEFAULT_SESSION_WINDOW_SECONDS,
+ DEFAULT_TRACE_MAX_AGE_SECONDS,
+ DEFAULT_TRACE_QUIET_PERIOD_SECONDS,
+ DEFAULT_TRACE_WINDOW_SECONDS,
+} from '../llmEvaluationLogic'
import type { EvaluationConfig, EvaluationTarget } from '../types'
const UNIT_NOUNS: Record<EvaluationTarget, string> = {
@@ -66,3 +75,37 @@ export function reportsUnavailableReason(evaluation: EvaluationConfig, isReporta
? 'Reports need a pass rate. In the Returns step, pick which categories count as a pass.'
: 'Reports are not available for this kind of evaluation.'
}
+
+const DURATION_UNITS: [string, number][] = [
+ ['day', 86400],
+ ['hour', 3600],
+ ['minute', 60],
+ ['second', 1],
+]
+
+/** A duration in its largest whole unit, such as "30 minutes". The duration picker only stores whole units. */
+export function durationInWords(seconds: number): string {
+ const [unit, size] = DURATION_UNITS.find(([, unitSize]) => seconds % unitSize === 0) ?? ['second', 1]
+ return pluralize(seconds / size, unit)
+}
+
+/** When a trace or session run starts, in one sentence, so the timing settings can stay folded away. */
+export function settleSummary(evaluation: EvaluationConfig): string | null {
+ if (evaluation.target !== 'trace' && evaluation.target !== 'session') {
+ return null
+ }
+ const isSession = evaluation.target === 'session'
+ const config = evaluation.target_config
+ const duration = durationInWords
+ if (settleStrategy(config.strategy, evaluation.target) === 'fixed_window') {
+ const wait =
+ config.window_seconds ?? (isSession ? DEFAULT_SESSION_WINDOW_SECONDS : DEFAULT_TRACE_WINDOW_SECONDS)
+ return `It runs ${duration(wait)} after the first matching generation.`
+ }
+ const quiet =
+ config.quiet_period_seconds ??
+ (isSession ? DEFAULT_SESSION_QUIET_PERIOD_SECONDS : DEFAULT_TRACE_QUIET_PERIOD_SECONDS)
+ const maxAge =
+ config.max_age_seconds ?? (isSession ? DEFAULT_SESSION_MAX_AGE_SECONDS : DEFAULT_TRACE_MAX_AGE_SECONDS)
+ return `It runs once the ${evaluation.target} has been quiet for ${duration(quiet)}, or ${duration(maxAge)} after it starts.`
+}