Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(aio): remove the evals overview feature flag (#126192)

Bernat Torresc0e85fb76fa7
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index bdbf0f740e57..29889385547d 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -13430,22 +13430,6 @@ snapshots:
hash: v1.k794b7964.5e28b3fd6a08be3be25b6a06934b466602ec39fb25b271798ff5df41688522b4.youqoqtK4pdutgYf3Q1PJR2_G-XKgNvVLsQVvrY1abk
scenes-app-legal-documents--new-dpat-swift--light:
hash: v1.k794b7964.e87918b59eea97e7aaf2b8f103d045e0f1ba279539677f983d37191c65ca26fe.ZgqoHeykE3Lt4Oqlg7CLX0cA7ejpZCN9DXWt5AZqQgI
- scenes-app-llm-observability-categoricalevaluationconfig--default--dark:
- hash: v1.k794b7964.ada2ad9acdfbff7b24546c99eb5a9d230f6382ea79c85efea0702d97502ec71e.J-Q7sqVTRhiJi3_pXpETIh1UPiUNEadlSk-newKsp_U
- scenes-app-llm-observability-categoricalevaluationconfig--default--light:
- hash: v1.k794b7964.02e9ebb08a3452ccd3d9d4bd2f88d16af51166514c8d99206a3332b4ac2178dc.CpX4ruvV0x6n6jgLdcPDKEPopK0MFnzhOzGgx1oVPq8
- scenes-app-llm-observability-categoricalevaluationconfig--duplicate-keys--dark:
- hash: v1.k794b7964.2f6add612f640b9a83c0b7b670fed2241e5838919bd562081771f3beabab88cd.rM-CuNIz3Hg8PenSy4j5HW1bU3DbitfhT8kyf1Nf2aY
- scenes-app-llm-observability-categoricalevaluationconfig--duplicate-keys--light:
- hash: v1.k794b7964.1ea3209a9d2358a1a69bede92656d70058ce51d3ee012132aae4d7486a663f63.Qlu7Dg-bgksM_HJHARKdhJ0gwIW8O4xJyHFR3Kwy-jc
- scenes-app-llm-observability-categoricalevaluationconfig--invalid-passing-rule--dark:
- hash: v1.k794b7964.da59d7c00ac5b84cf617c3ec127a633e735d22702a74374271d9869e25166d1a.KKEzg2vLfjMGpCmnLry9Yy26AdLK-7VUu2XzAQdgeVY
- scenes-app-llm-observability-categoricalevaluationconfig--invalid-passing-rule--light:
- hash: v1.k794b7964.8c39683ef7ed907d88d445412ef9c5893b78a2ea3677e7ffea4a76169b8fd72e.A5q_lgCmp8JX6i4xUFabSGrAPOQzFPZMJJPWwjIcrG8
- scenes-app-llm-observability-categoricalevaluationconfig--narrow--dark:
- hash: v1.k794b7964.7d559096946c7fb95a3c0b63549bedb9b208c1725d50c6f4429444abb5e18325.kUhWPi1f7gBuRO9-ksG1UG2p4-PqmJRWldGLqAzrFoI
- scenes-app-llm-observability-categoricalevaluationconfig--narrow--light:
- hash: v1.k794b7964.c96e17b2cd655cfc8cb4085ca2efb87c19ea743c6200e65c368aa6538af92cab.Fc2w6LHTyrCIM3oJi1NA8eTxZm4y0vcVfE2nfIkD-1w
scenes-app-llm-observability-evaluation-templates--default--dark:
hash: v1.k794b7964.07a0809fec2e7a04e7ed1e7ca06aa3570d62106e9754c1938662a127839a7d43.09gLEWuTLF5IIQazReIApdLh_V8h-FDxjGyNj2hkMus
scenes-app-llm-observability-evaluation-templates--default--light:
@@ -13454,34 +13438,6 @@ snapshots:
hash: v1.k794b7964.ea887c384da7d18b38230ba0b420b1f87d0a2995899d5844290ddc16eb476dc3.8F9-jOoonwrPVwHD8_XR0eud3gqkF0OueJM1tArPbn0
scenes-app-llm-observability-evaluation-templates--narrow--light:
hash: v1.k794b7964.acb588946dd41382c78934be3f8912589d1a15df06ef9a976f617fad21f1ddd2.0LsEsqRoncEFZhEVh5X8gbA_QoGZelV2t1ABDsxGeao
- scenes-app-llm-observability-evaluationmodelpicker--decision--dark:
- hash: v1.k794b7964.8cf03ee1f25ccdee62cc5104d48589d196f85dd3dbad84d5a838cc4c8bfe9607.G7gP4btJiT-u-RfsschCP6cV1btlQnw85BFAhZ1D86M
- scenes-app-llm-observability-evaluationmodelpicker--decision--light:
- hash: v1.k794b7964.6b288367a286d1858e3032bab65a034553c5f5ae79dacd0274cb92d1f8bab818.CM-LkhqTNhtSyn0O3Y6c6Yd_gLfSXe31n3Dxaj5uEZI
- scenes-app-llm-observability-evaluationmodelpicker--decision-narrow--dark:
- hash: v1.k794b7964.e75b3c58f2b1922e7ce4b3f8512fa7dc81021d0d988622599b3f52bb40f32eeb.PLEVbwWEAXbpJdC_PSjXyTKP-lLKLDdAkRcrP9jOOXU
- scenes-app-llm-observability-evaluationmodelpicker--decision-narrow--light:
- hash: v1.k794b7964.8ae8ae696fb4cd22a44ccd18c901236eea8a0e6ac6273b686dc946821f3f7962.Dw7EaOMcP1cHXUw3o5SoYiAZ66QGAYk7WzhqDGcCrLY
- scenes-app-llm-observability-evaluationmodelpicker--llm--dark:
- hash: v1.k794b7964.a42ccc41fa5864dee67f21add5ecc3aa6838805fb494f6686b826688cc325e9e.tHDxeEbDTVatdZYPj2xVDQtJpukqzNyUHU9iScfQ2AY
- scenes-app-llm-observability-evaluationmodelpicker--llm--light:
- hash: v1.k794b7964.24f4da8cefdaef0c33f3a1783746888c340cf60cb1d47edc654b1325f5ff644b.gDAO1ukg0FfytAXK4R2OSPF5mkNti7adPrdbA1bwGGI
- scenes-app-llm-observability-evaluationmodelpicker--native-open-ai--dark:
- hash: v1.k794b7964.8cf03ee1f25ccdee62cc5104d48589d196f85dd3dbad84d5a838cc4c8bfe9607.kKKEqvhd_Wb8zZVyl0fmkbE6RBz9odp8Iyhq8HPAw3I
- scenes-app-llm-observability-evaluationmodelpicker--native-open-ai--light:
- hash: v1.k794b7964.6b288367a286d1858e3032bab65a034553c5f5ae79dacd0274cb92d1f8bab818.nTYWu7M1TfGyOlu52uGxd98cNvdj-nyoyzcWUovs6Ys
- scenes-app-llm-observability-evaluationmodelpicker--native-open-ai-narrow--dark:
- hash: v1.k794b7964.e75b3c58f2b1922e7ce4b3f8512fa7dc81021d0d988622599b3f52bb40f32eeb.qdYv6zfUkJUHXikWSwr_1jyDM3PvnFOPTj8qFIdrqr8
- scenes-app-llm-observability-evaluationmodelpicker--native-open-ai-narrow--light:
- hash: v1.k794b7964.8ae8ae696fb4cd22a44ccd18c901236eea8a0e6ac6273b686dc946821f3f7962._qdgUVltz2sd1ZHnkaFq8J5CXje346MVRlc4WOsz-pg
- scenes-app-llm-observability-evaluationmodelpicker--no-decision-models--dark:
- hash: v1.k794b7964.ed596389ff3b9a8a505c3b1ad5cbfabadc1cb83507c341f99dd54b004f8d257e.Jm_sWi9-7s4eTkxjyQUcGApiT5QcFzke8u5k2hK6gaE
- scenes-app-llm-observability-evaluationmodelpicker--no-decision-models--light:
- hash: v1.k794b7964.b10b433ed0507465953ef6174a8872546e4e695407914798a8e11a6e7f8aa38d.jQ8Xqfw0JjPdrTXWnuYodzsfn46fs4aKri7d1Pzl7QU
- scenes-app-llm-observability-evaluationmodelpicker--reasoning-level-narrow--dark:
- hash: v1.k794b7964.c135d25cde9ab12d1bbf35f7cc6cc87ab8c838586bdbd7316b62a59d54d85b9e.QVUXtySXrLSrHHvMht-mZvR9AIKvry19dFXdhLIHs30
- scenes-app-llm-observability-evaluationmodelpicker--reasoning-level-narrow--light:
- hash: v1.k794b7964.d0372da3513c43cccbb1d2c7e9b6212567f86ca3e20f33cfaf0fba84c2eb8072.1Lrr0sxzbqQN9iCbO_JrkZtixXdNV2MOgqu6TI5qjDw
scenes-app-llm-observability-evaluationreportconfig--lemon-ui--dark:
hash: v1.k794b7964.195aa4a5a39e8f947e17fe67f5c23303b7dd0579c278907761be24a08780be92.IkGQtAI95T60kJKVZcmmFBIWNCd4meuJv-bcNDoohwU
scenes-app-llm-observability-evaluationreportconfig--lemon-ui--light:
diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx
index e5f8b02c96e6..c1694727969a 100644
--- a/frontend/src/lib/constants.tsx
+++ b/frontend/src/lib/constants.tsx
@@ -252,7 +252,6 @@ export const FEATURE_FLAGS = {
// Temporary feature flags, still WIP, should be removed eventually
ADVANCE_MARKETING_ANALYTICS_SETTINGS: 'advance-marketing-analytics-settings', // owner: @jabahamondes #team-web-analytics
- AI_OBSERVABILITY_EVALUATIONS_OVERVIEW: 'ai-observability-evaluations-overview', // owner: #team-ai-observability
AI_OBSERVABILITY_OFFLINE_EVALUATIONS: 'ai-observability-offline-evaluations', // owner: #team-ai-observability
AI_OBSERVABILITY_TRACE_REDESIGN: 'ai-observability-trace-redesign', // owner: #team-ai-observability, gates the redesigned trace view
/** Alert edit modal: check history chart + chart/table toggle (table remains when off). */
diff --git a/products/ai_observability/frontend/AIObservabilityReloadAction.tsx b/products/ai_observability/frontend/AIObservabilityReloadAction.tsx
index 21eaf2673656..d8b1e5a7cebd 100644
--- a/products/ai_observability/frontend/AIObservabilityReloadAction.tsx
+++ b/products/ai_observability/frontend/AIObservabilityReloadAction.tsx
@@ -14,8 +14,6 @@ import { dataNodeCollectionLogic } from '~/queries/nodes/DataNode/dataNodeCollec
import { DashboardPlacement } from '~/types'
import { AI_OBSERVABILITY_DATA_COLLECTION_NODE_ID, aiObservabilitySharedLogic } from './aiObservabilitySharedLogic'
-import { EVALUATION_METRICS_COLLECTION_ID } from './evaluations/components/EvaluationMetrics'
-import { evaluationMetricsLogic } from './evaluations/evaluationMetricsLogic'
import { llmEvaluationsLogic } from './evaluations/llmEvaluationsLogic'
import { evaluationsOverviewLogic } from './evaluations/overview/evaluationsOverviewLogic'
import { aiObservabilityDashboardLogic } from './tabs/aiObservabilityDashboardLogic'
@@ -53,9 +51,6 @@ export function AIObservabilityReloadAction(): JSX.Element {
const { triggerDashboardRefresh } = useActions(boundLogic)
const { reloadAll } = useActions(dataNodeCollectionLogic({ key: AI_OBSERVABILITY_DATA_COLLECTION_NODE_ID }))
- const { reloadAll: reloadEvaluationMetrics } = useActions(
- dataNodeCollectionLogic({ key: EVALUATION_METRICS_COLLECTION_ID })
- )
const { loadEvaluations } = useActions(llmEvaluationsLogic)
const { sessionsLoading } = useValues(aiObservabilitySessionsViewLogic)
const { loadSessions } = useActions(aiObservabilitySessionsViewLogic)
@@ -67,12 +62,9 @@ export function AIObservabilityReloadAction(): JSX.Element {
if (activeTab === 'dashboard') {
triggerDashboardRefresh()
} else if (activeTab === 'evaluations') {
- // Refresh evaluations list and metrics. Only the view on screen has its logic mounted, and mounting the
- // other one from here would start its queries for nothing.
+ // The overview logic is mounted only while its tab is on screen, so mounting it from here would query for nothing.
loadEvaluations()
- evaluationMetricsLogic.findMounted()?.actions.refreshMetrics()
evaluationsOverviewLogic.findMounted()?.actions.reloadOverview()
- reloadEvaluationMetrics()
} else if (activeTab === 'sessions') {
// The sessions list is a custom loader, not part of the data-node collection reloadAll() drives.
// Force a recompute so the button surfaces newly-ingested sessions instead of a cached result.
diff --git a/products/ai_observability/frontend/e2e/evaluation-polarity.spec.ts b/products/ai_observability/frontend/e2e/evaluation-polarity.spec.ts
index f3c6998b0b70..fa30daa31de9 100644
--- a/products/ai_observability/frontend/e2e/evaluation-polarity.spec.ts
+++ b/products/ai_observability/frontend/e2e/evaluation-polarity.spec.ts
@@ -12,13 +12,24 @@ if (not result) {
}
return result`
-const SCREENSHOT_DIR = path.join(__dirname, '../../../../.playwright-screenshots')
-
interface BooleanRunSpec {
result: boolean | null
skipped?: boolean
}
+// Identical runs on both evaluations, so any difference in how they render comes from the declared polarity.
+const BOOLEAN_RUNS: BooleanRunSpec[] = [
+ { result: true },
+ { result: true },
+ { result: true },
+ { result: true },
+ { result: false },
+ { result: null },
+ { result: false, skipped: true },
+]
+
+const SCREENSHOT_DIR = path.join(__dirname, '../../../../.playwright-screenshots')
+
function buildBooleanRunEvents(
evaluationId: string,
evaluationName: string,
@@ -88,108 +99,44 @@ test.describe('AI observability boolean evaluation polarity', () => {
const workspace = await playwrightSetup.createWorkspace({ skip_onboarding: true, no_demo_data: true })
await playwrightSetup.login(page, workspace)
- await test.step('the switch renders under Allow N/A responses and round-trips through save + reload', async () => {
- await page.goto('/ai-evals/evaluations/new')
- await expect(page.getByRole('heading', { name: 'New evaluation' })).toBeVisible()
-
- await page.getByPlaceholder('e.g., Helpfulness Check').fill('User struggled')
- await page.getByRole('button', { name: 'LLM as a judge' }).click()
- await page.getByRole('menuitem', { name: 'Hog code' }).click()
-
- const trueIsFailureSwitch = page.getByTestId('llma-evaluation-true-is-failure-switch')
- const allowsNaLabel = page.getByText('Allow N/A responses', { exact: true })
- const trueIsFailureLabel = page.getByText('A true result flags a problem', { exact: true })
-
- // Settle scrolling before taking any measurement — every bounding box read below has
- // to share one coordinate space with the eventual screenshot clip, and a later click
- // auto-scrolling the page would otherwise invalidate boxes read before it.
- await trueIsFailureSwitch.scrollIntoViewIfNeeded()
-
- await expect(allowsNaLabel).toBeVisible()
- await expect(trueIsFailureLabel).toBeVisible()
- const naBox = await allowsNaLabel.boundingBox()
- const flagBox = await trueIsFailureLabel.boundingBox()
- if (!naBox || !flagBox) {
- throw new Error('Expected both switch labels to have a bounding box')
- }
- expect(flagBox.y).toBeGreaterThan(naBox.y)
-
- await expect(trueIsFailureSwitch).toHaveAttribute('aria-checked', 'false')
- await expect(page.getByText('A true result counts as a pass', { exact: true })).toBeVisible()
-
- await trueIsFailureSwitch.click()
- await expect(trueIsFailureSwitch).toHaveAttribute('aria-checked', 'true')
- const failDescription = page.getByText('A true result counts as a fail', { exact: true })
- await expect(failDescription).toBeVisible()
-
- const naDescription = page.getByText('Evaluation must return true or false', { exact: true })
- const failBox = await failDescription.boundingBox()
- const switchBox = await trueIsFailureSwitch.boundingBox()
- const naDescriptionBox = await naDescription.boundingBox()
- if (failBox && switchBox && naDescriptionBox) {
- const pad = 24
- const left = Math.min(naBox.x, switchBox.x) - pad
- const top = naBox.y - 10
- const right =
- Math.max(
- naBox.x + naBox.width,
- naDescriptionBox.x + naDescriptionBox.width,
- failBox.x + failBox.width
- ) + pad
- const bottom = failBox.y + failBox.height + pad
- await page.screenshot({
- path: path.join(SCREENSHOT_DIR, '01-switch.png'),
- clip: { x: left, y: top, width: right - left, height: bottom - top },
- animations: 'disabled',
- })
- }
+ await test.step('the polarity choice round-trips through save + reload', async () => {
+ // Created through the API, not seeded: a save needs at least one condition set, which a seed lacks.
+ const createResponse = await page.request.post(`/api/projects/${workspace.team_id}/evaluations/`, {
+ headers: { Authorization: `Bearer ${workspace.personal_api_key}` },
+ data: {
+ name: 'User struggled',
+ evaluation_type: 'hog',
+ evaluation_config: { source: HOG_SOURCE },
+ output_type: 'boolean',
+ output_config: {},
+ conditions: [{ id: 'all', rollout_percentage: 100, properties: [] }],
+ },
+ })
+ expect(createResponse.ok()).toBe(true)
+ const evaluationId = ((await createResponse.json()) as { id: string }).id
- const [createResponse] = await Promise.all([
- page.waitForResponse(
- (resp) => resp.url().includes('/evaluations/') && resp.request().method() === 'POST'
- ),
- page.getByTestId('llma-evaluation-save-bottom').click(),
- ])
- await expect(page).toHaveURL(/\/ai-evals\/evaluations$/, { timeout: 30_000 })
- const created = await createResponse.json()
- const evalAId = created.id as string
-
- const gotoEvalA = async (): Promise<void> => {
- await page.goto(`/ai-evals/evaluations/${evalAId}`)
+ const passOption = page.getByTestId('llma-evaluation-true-is-pass')
+ const failOption = page.getByTestId('llma-evaluation-true-is-failure')
+ const gotoConfiguration = async (): Promise<void> => {
+ await page.goto(`/ai-evals/evaluations/${evaluationId}`)
await page.getByTestId('llma-evaluation-configuration-tab').click()
+ await page.getByTestId('llma-evaluation-create-step-check').click()
}
- await gotoEvalA()
- await expect(page.getByTestId('llma-evaluation-true-is-failure-switch')).toHaveAttribute(
- 'aria-checked',
- 'true'
- )
- await page.reload()
- await page.getByTestId('llma-evaluation-configuration-tab').click()
- await expect(page.getByTestId('llma-evaluation-true-is-failure-switch')).toHaveAttribute(
- 'aria-checked',
- 'true'
- )
-
- await page.getByTestId('llma-evaluation-true-is-failure-switch').click()
- await expect(page.getByTestId('llma-evaluation-true-is-failure-switch')).toHaveAttribute(
- 'aria-checked',
- 'false'
- )
- await page.getByTestId('llma-evaluation-save-bottom').click()
- await expect(page).toHaveURL(/\/ai-evals\/evaluations$/, { timeout: 30_000 })
-
- await gotoEvalA()
- await expect(page.getByTestId('llma-evaluation-true-is-failure-switch')).toHaveAttribute(
- 'aria-checked',
- 'false'
- )
- await page.reload()
- await page.getByTestId('llma-evaluation-configuration-tab').click()
- await expect(page.getByTestId('llma-evaluation-true-is-failure-switch')).toHaveAttribute(
- 'aria-checked',
- 'false'
- )
+ for (const option of [failOption, passOption]) {
+ await gotoConfiguration()
+ await option.click()
+ await Promise.all([
+ page.waitForResponse(
+ (resp) => resp.url().includes('/evaluations/') && resp.request().method() === 'PATCH'
+ ),
+ page.getByTestId('llma-evaluation-save-bottom').click(),
+ ])
+
+ // A full page load, so the choice comes back from the API rather than form state.
+ await gotoConfiguration()
+ await expect(option).toHaveClass(/LemonButton--primary/)
+ }
})
await test.step('a legacy evaluation whose config never had the key behaves as pass-on-true', async () => {
@@ -204,15 +151,12 @@ test.describe('AI observability boolean evaluation polarity', () => {
await page.goto(`/ai-evals/evaluations/${legacy.evaluation_id}`)
await page.getByTestId('llma-evaluation-configuration-tab').click()
- await expect(page.getByTestId('llma-evaluation-true-is-failure-switch')).toHaveAttribute(
- 'aria-checked',
- 'false'
- )
- await expect(page.getByText('A true result counts as a pass', { exact: true })).toBeVisible()
+ await page.getByTestId('llma-evaluation-create-step-check').click()
+ await expect(page.getByTestId('llma-evaluation-true-is-pass')).toHaveClass(/LemonButton--primary/)
})
})
- test('seeded results propagate correctly across the list, the runs table, and edge cases', async ({
+ test('seeded results propagate correctly across the runs table and edge cases', async ({
page,
playwrightSetup,
}) => {
@@ -221,8 +165,6 @@ test.describe('AI observability boolean evaluation polarity', () => {
const workspace = await playwrightSetup.createWorkspace({ skip_onboarding: true, no_demo_data: true })
await playwrightSetup.login(page, workspace)
- // Enabled so a reviewer glancing at the list screenshot doesn't have to wonder
- // whether the rates are stale because the evaluations are off.
const detector = await playwrightSetup.seedEvaluation({
team_id: workspace.team_id,
name: 'User struggled',
@@ -245,85 +187,10 @@ test.describe('AI observability boolean evaluation polarity', () => {
})
const evalBId = quality.evaluation_id
- await test.step('identical results invert the pass rate between a detector and a quality check', async () => {
- const now = Date.now()
- const detectorEvents = buildBooleanRunEvents(
- evalAId,
- 'User struggled',
- [
- { result: true },
- { result: true },
- { result: true },
- { result: true },
- { result: false },
- { result: null },
- { result: false, skipped: true },
- ],
- now
- )
- const qualityEvents = buildBooleanRunEvents(
- evalBId,
- 'Response is accurate',
- [
- { result: true },
- { result: true },
- { result: true },
- { result: true },
- { result: false },
- { result: null },
- { result: false, skipped: true },
- ],
- now
- )
- await playwrightSetup.seedEvents(workspace.team_id, [...detectorEvents, ...qualityEvents])
-
- await page.goto('/ai-evals/evaluations')
- const detectorRow = page.getByRole('row', { name: /User struggled/ })
- const qualityRow = page.getByRole('row', { name: /Response is accurate/ })
- await expect(detectorRow).toBeVisible()
- await expect(qualityRow).toBeVisible()
-
- // Scroll before measuring — the table can render below the fold on a fresh
- // page load, and a clip built from boxes measured pre-scroll lands off-image.
- await detectorRow.scrollIntoViewIfNeeded()
-
- // Byte-identical seed data on both evaluations (same 4 true / 1 false / 1 null /
- // 1 skipped pattern) — the run count has to match too, or a reviewer glancing at the
- // screenshot could mistake the inverted rates for a difference in the underlying data
- // rather than in the declared polarity.
- const detectorRuns = detectorRow.getByText('7 runs', { exact: true })
- const qualityRuns = qualityRow.getByText('7 runs', { exact: true })
- await expect(detectorRuns).toBeVisible()
- await expect(qualityRuns).toBeVisible()
-
- const detectorRate = detectorRow.getByText('20%', { exact: true })
- const qualityRate = qualityRow.getByText('80%', { exact: true })
- await expect(detectorRate).toBeVisible()
- await expect(detectorRate).toHaveClass(/text-danger/)
- await expect(qualityRate).toBeVisible()
- await expect(qualityRate).toHaveClass(/text-success/)
-
- const detectorBox = await detectorRow.boundingBox()
- const qualityBox = await qualityRow.boundingBox()
- const detectorRateBox = await detectorRate.boundingBox()
- const qualityRateBox = await qualityRate.boundingBox()
- if (detectorBox && qualityBox && detectorRateBox && qualityRateBox) {
- const pad = 12
- const top = Math.max(0, Math.min(detectorBox.y, qualityBox.y) - pad)
- const bottom = Math.max(detectorBox.y + detectorBox.height, qualityBox.y + qualityBox.height) + pad
- const left = Math.max(0, Math.min(detectorBox.x, qualityBox.x) - pad)
- // End at the Runs column's right edge, not the row's full width — the Actions
- // column past it (folder/edit/delete buttons) adds nothing to the story and,
- // depending on row width, could clip a button mid-icon with a white gutter after it.
- const right =
- Math.max(detectorRateBox.x + detectorRateBox.width, qualityRateBox.x + qualityRateBox.width) + pad
- await page.screenshot({
- path: path.join(SCREENSHOT_DIR, '02-list-pass-rate.png'),
- clip: { x: left, y: top, width: right - left, height: bottom - top },
- animations: 'disabled',
- })
- }
- })
+ await playwrightSetup.seedEvents(workspace.team_id, [
+ ...buildBooleanRunEvents(evalAId, 'User struggled', BOOLEAN_RUNS, Date.now()),
+ ...buildBooleanRunEvents(evalBId, 'Response is accurate', BOOLEAN_RUNS, Date.now()),
+ ])
await test.step('a true run renders as the undesirable outcome, and the Passing/Failing filter inverts', async () => {
await page.goto(`/ai-evals/evaluations/${evalAId}`)
@@ -408,8 +275,8 @@ test.describe('AI observability boolean evaluation polarity', () => {
await page.goto(`/ai-evals/evaluations/${evalCId}`)
await page.getByTestId('llma-evaluation-configuration-tab').click()
- await expect(page.getByText('Allow N/A responses', { exact: true })).toHaveCount(0)
- await expect(page.getByText('A true result flags a problem', { exact: true })).toHaveCount(0)
+ await page.getByTestId('llma-evaluation-create-step-check').click()
+ await expect(page.getByTestId('llma-evaluation-true-is-failure')).toHaveCount(0)
const sentimentEvents = buildSentimentRunEvents(
evalCId,
diff --git a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
index c0ef2b00898a..aeaa825ea34b 100644
--- a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
+++ b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx
@@ -1,29 +1,13 @@
import { useActions, useValues } from 'kea'
import { combineUrl, router } from 'kea-router'
-import { useRef } from 'react'
-import { IconArrowLeft, IconInfo, IconPlay, IconTrends, IconWarning } from '@posthog/icons'
-import {
- LemonBanner,
- LemonButton,
- LemonInput,
- LemonSelect,
- LemonSkeleton,
- LemonSwitch,
- LemonTabs,
- LemonTag,
- LemonTextArea,
- Link,
- Tooltip,
-} from '@posthog/lemon-ui'
+import { IconArrowLeft, IconPlay, IconTrends, IconWarning } from '@posthog/icons'
+import { LemonBanner, LemonButton, LemonSkeleton, LemonTabs, LemonTag, Link } from '@posthog/lemon-ui'
import { AccessControlAction } from 'lib/components/AccessControlAction'
import { NotFound } from 'lib/components/NotFound'
import { TZLabel } from 'lib/components/TZLabel'
-import { FEATURE_FLAGS } from 'lib/constants'
-import { LemonField } from 'lib/lemon-ui/LemonField'
import { lemonToast } from 'lib/lemon-ui/LemonToast'
-import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { pluralize } from 'lib/utils/strings'
import { SceneExport } from 'scenes/sceneTypes'
@@ -35,20 +19,12 @@ import { AccessControlLevel, AccessControlResourceType } from '~/types'
import { useAttachedContext } from 'products/posthog_ai/frontend/api/logics'
import { providerKeyStateIssueDescription, providerLabel } from '../settings/providerKeyStateUtils'
-import { CategoricalEvaluationConfig } from './components/CategoricalEvaluationConfig'
import { EvaluationBackfillsTab } from './components/EvaluationBackfillsTab'
-import { EvaluationCodeEditor } from './components/EvaluationCodeEditor'
import { EvaluationFirstResultStatus } from './components/EvaluationFirstResultStatus'
import { EvaluationLatestReport } from './components/EvaluationLatestReport'
-import { EvaluationModelPicker } from './components/EvaluationModelPicker'
-import { EvaluationPromptEditor } from './components/EvaluationPromptEditor'
-import { EvaluationReportConfig } from './components/EvaluationReportConfig'
import { EvaluationReportsCallout } from './components/EvaluationReportsCallout'
import { EvaluationReportsTab } from './components/EvaluationReportsTab'
import { EvaluationRunsTable } from './components/EvaluationRunsTable'
-import { EvaluationSettleConfig } from './components/EvaluationSettleConfig'
-import { EvaluationTriggersPanel } from './components/EvaluationTriggersPanel'
-import { NumericEvaluationConfig } from './components/NumericEvaluationConfig'
import {
formatNumericEvaluationScore,
booleanOutputConfigError,
@@ -58,19 +34,15 @@ import {
import { EvaluationConfigurationForm } from './create/EvaluationConfigurationForm'
import { EvaluationCreatePage } from './create/EvaluationCreatePage'
import {
- evaluationOffersSessionTarget,
evaluationSupportsReportHistory,
evaluationSupportsReports,
evaluationSupportsRunOutcomes,
- evaluationTypeHasEditableCriteria,
evaluationTypeUsesModelConfiguration,
- isBooleanEvaluationOutput,
} from './evaluationCapabilities'
import { evaluationReportLogic } from './evaluationReportLogic'
import { LLMEvaluationLogicProps, hasUnsetConditionRollout, llmEvaluationLogic } from './llmEvaluationLogic'
import { EvaluationDetailResults } from './overview/EvaluationDetailResults'
import { statusReasonLabel, statusReasonRecoveryLabel } from './statusDisplay'
-import { EvaluationTarget, EvaluationType } from './types'
const RUNS_BACKFILL_TIME_FORMAT = { formatDate: 'MMM D, YYYY', formatTime: 'HH:mm' }
@@ -92,38 +64,18 @@ export function AIObservabilityEvaluation(): JSX.Element {
runsDateRange,
evaluationProviderKeyIssue,
activeTab,
- canEnable,
- canEnableReason,
modelSelectionRequired,
numericBoundsRequired,
usesDecisionModel,
} = useValues(llmEvaluationLogic)
const { searchParams } = useValues(router)
- const { featureFlags } = useValues(featureFlagLogic)
- const settlingStrategyEnabled = !!featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EVAL_SETTLING_STRATEGY]
- const resultsCardsEnabled = !!featureFlags[FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW]
const { isConfigDirty: reportConfigDirty } = useValues(
evaluationReportLogic({ evaluationId: isNewEvaluation || !evaluation?.id ? 'new' : evaluation.id })
)
// The step layout edits reports with the rest of the configuration, so a report-only edit is unsaved too.
- const unsaved = hasUnsavedChanges || (resultsCardsEnabled && !isNewEvaluation && reportConfigDirty)
- const {
- setEvaluationName,
- setEvaluationDescription,
- setEvaluationEnabled,
- setAllowsNA,
- setTrueIsFailure,
- saveEvaluation,
- resetEvaluation,
- setEvaluationType,
- setOutputType,
- patchOutputConfig,
- setEvaluationTarget,
- setActiveTab,
- setFormStep,
- } = useActions(llmEvaluationLogic)
+ const unsaved = hasUnsavedChanges || (!isNewEvaluation && reportConfigDirty)
+ const { saveEvaluation, resetEvaluation, setActiveTab, setFormStep } = useActions(llmEvaluationLogic)
const { push } = useActions(router)
- const triggersRef = useRef<HTMLDivElement>(null)
const settingsUrl = combineUrl(urls.aiObservabilityEvaluations(), { ...searchParams, tab: 'settings' }).url
useAttachedContext(
@@ -137,7 +89,7 @@ export function AIObservabilityEvaluation(): JSX.Element {
if (!evaluation) {
return <NotFound object="evaluation" />
}
- if (isNewEvaluation && featureFlags[FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW]) {
+ if (isNewEvaluation) {
return (
<EvaluationCreatePage
templateKey={typeof searchParams.template === 'string' ? searchParams.template : undefined}
@@ -151,11 +103,7 @@ export function AIObservabilityEvaluation(): JSX.Element {
const isHog = evaluation.evaluation_type === 'hog'
const isSentiment = evaluation.evaluation_type === 'sentiment'
- const isSessionTarget = evaluation.target === 'session'
- const offersSessionTarget = evaluationOffersSessionTarget(evaluation, settlingStrategyEnabled)
const supportsRunOutcomes = evaluationSupportsRunOutcomes(originalEvaluation)
- const isBooleanOutput = isBooleanEvaluationOutput(evaluation.output_type)
- const hasEditableCriteria = evaluationTypeHasEditableCriteria(evaluation.evaluation_type)
const configValid = isHog
? evaluation.evaluation_config.source.trim().length > 0
@@ -191,9 +139,6 @@ export function AIObservabilityEvaluation(): JSX.Element {
const focusTriggers = (): void => {
setActiveTab('configuration')
setFormStep('traffic')
- requestAnimationFrame(() => {
- triggersRef.current?.scrollIntoView({ behavior: 'smooth', block: 'center' })
- })
}
const handleSave = (): void => {
@@ -215,7 +160,7 @@ export function AIObservabilityEvaluation(): JSX.Element {
return
}
- const reportLogic = evaluationReportLogic({ evaluationId: isNewEvaluation ? 'new' : evaluation.id })
+ const reportLogic = evaluationReportLogic({ evaluationId: evaluation.id })
if (evaluationSupportsReports(evaluation) && reportLogic.isMounted() && reportLogic.values.configError) {
lemonToast.error(reportLogic.values.configError)
return
@@ -231,24 +176,6 @@ export function AIObservabilityEvaluation(): JSX.Element {
push(evaluationBackTarget.path)
}
- const hogEvaluationMethodOptions: { value: EvaluationType; label: string }[] = [
- {
- value: 'hog',
- label: 'Hog code',
- },
- ]
- const evaluationMethodOptions: { value: EvaluationType; label: string }[] = [
- {
- value: 'llm_judge',
- label: 'LLM as a judge',
- },
- ...hogEvaluationMethodOptions,
- {
- value: 'sentiment',
- label: 'Sentiment analysis',
- },
- ]
-
return (
<div className="@container/evaluation min-w-0 space-y-6 pb-8">
<SceneBreadcrumbBackButton />
@@ -258,26 +185,18 @@ export function AIObservabilityEvaluation(): JSX.Element {
className="flex flex-col justify-between items-start gap-4 space-y-0 @min-[64rem]/evaluation:flex-row"
>
<div className="space-y-2 min-w-0">
- <h1 className="text-2xl font-semibold break-words">
- {isNewEvaluation ? 'New evaluation' : evaluation.name}
- </h1>
+ <h1 className="text-2xl font-semibold break-words">{evaluation.name}</h1>
<div className="flex flex-wrap items-center gap-2">
- {isNewEvaluation ? (
- <LemonTag type="primary">New</LemonTag>
+ {evaluation.status === 'error' ? (
+ <LemonTag type="danger" icon={<IconWarning />}>
+ Error
+ </LemonTag>
) : (
- <>
- {evaluation.status === 'error' ? (
- <LemonTag type="danger" icon={<IconWarning />}>
- Error
- </LemonTag>
- ) : (
- <LemonTag type={evaluation.enabled ? 'success' : 'default'}>
- {evaluation.enabled ? 'Enabled' : 'Disabled'}
- </LemonTag>
- )}
- {unsaved && <LemonTag type="warning">Unsaved changes</LemonTag>}
- </>
+ <LemonTag type={evaluation.enabled ? 'success' : 'default'}>
+ {evaluation.enabled ? 'Enabled' : 'Disabled'}
+ </LemonTag>
)}
+ {unsaved && <LemonTag type="warning">Unsaved changes</LemonTag>}
</div>
</div>
<div className="flex flex-wrap gap-2 max-w-full">
@@ -317,7 +236,7 @@ export function AIObservabilityEvaluation(): JSX.Element {
disabledReason={saveButtonDisabledReason}
loading={evaluationFormSubmitting}
>
- {isNewEvaluation ? 'Create evaluation' : 'Save changes'}
+ Save changes
</LemonButton>
</AccessControlAction>
)}
@@ -364,7 +283,7 @@ export function AIObservabilityEvaluation(): JSX.Element {
onChange={(key) => setActiveTab(key)}
data-attr="llma-evaluation-tabs"
tabs={[
- !isNewEvaluation && {
+ {
key: 'runs',
label: 'Runs',
'data-attr': 'llma-evaluation-runs-tab',
@@ -464,18 +383,14 @@ export function AIObservabilityEvaluation(): JSX.Element {
)}
</LemonBanner>
)}
- {resultsCardsEnabled && (
- <>
- <EvaluationFirstResultStatus />
- {evaluationSupportsReportHistory(originalEvaluation) && (
- <EvaluationLatestReport
- evaluationId={evaluation.id}
- onOpenReports={() => setActiveTab('reports')}
- />
- )}
- </>
+ <EvaluationFirstResultStatus />
+ {evaluationSupportsReportHistory(originalEvaluation) && (
+ <EvaluationLatestReport
+ evaluationId={evaluation.id}
+ onOpenReports={() => setActiveTab('reports')}
+ />
)}
- {resultsCardsEnabled && !runsBackfillId && (
+ {!runsBackfillId && (
<div className="mb-4">
<EvaluationDetailResults evaluationId={evaluation.id} />
</div>
@@ -484,25 +399,22 @@ export function AIObservabilityEvaluation(): JSX.Element {
</div>
),
},
- !isNewEvaluation &&
- evaluationSupportsReportHistory(originalEvaluation) && {
- key: 'reports',
- label: 'Reports',
- 'data-attr': 'llma-evaluation-reports-tab',
- content: (
- <EvaluationReportsTab
- evaluationId={evaluation.id}
- generationDisabledReason={
- isReportableEvaluation
- ? undefined
- : 'Add a passing rule to generate new reports.'
- }
- userAccessLevel={evaluation.user_access_level ?? undefined}
- onConfigureClick={() => setActiveTab('configuration')}
- />
- ),
- },
- !isNewEvaluation && {
+ evaluationSupportsReportHistory(originalEvaluation) && {
+ key: 'reports',
+ label: 'Reports',
+ 'data-attr': 'llma-evaluation-reports-tab',
+ content: (
+ <EvaluationReportsTab
+ evaluationId={evaluation.id}
+ generationDisabledReason={
+ isReportableEvaluation ? undefined : 'Add a passing rule to generate new reports.'
+ }
+ userAccessLevel={evaluation.user_access_level ?? undefined}
+ onConfigureClick={() => setActiveTab('configuration')}
+ />
+ ),
+ },
+ {
key: 'backfills',
label: 'Backfills',
'data-attr': 'llma-evaluation-backfills-tab',
@@ -519,287 +431,8 @@ export function AIObservabilityEvaluation(): JSX.Element {
label: 'Configuration',
'data-attr': 'llma-evaluation-configuration-tab',
content: (
- // The step layout is centered, like the new evaluation page. The old form stays on the left.
- <div
- className={resultsCardsEnabled && !isNewEvaluation ? 'max-w-4xl mx-auto' : 'max-w-4xl'}
- >
- {resultsCardsEnabled && !isNewEvaluation ? (
- <EvaluationConfigurationForm />
- ) : (
- <div className="space-y-6">
- {/* Basic Information */}
- <div className="bg-bg-light border rounded p-4 @min-[40rem]/evaluation:p-6">
- <h3 className="text-lg font-semibold mb-4">Basic information</h3>
-
- <div className="space-y-4">
- <LemonField.Pure label="Name">
- <LemonInput
- value={evaluation.name}
- onChange={setEvaluationName}
- placeholder="e.g., Helpfulness Check"
- maxLength={100}
- />
- </LemonField.Pure>
-
- {evaluationMethodOptions.length > 1 && (
- <LemonField.Pure label="Method">
- <LemonSelect
- value={evaluation.evaluation_type}
- onChange={(value) =>
- setEvaluationType(value as EvaluationType)
- }
- options={evaluationMethodOptions.map((option) => ({
- ...option,
- disabledReason:
- !isNewEvaluation &&
- ['numeric', 'categorical'].includes(
- evaluation.output_type
- ) &&
- option.value === 'sentiment'
- ? 'Create a new evaluation to change its output type.'
- : undefined,
- }))}
- fullWidth
- />
- </LemonField.Pure>
- )}
- <p className="text-muted text-sm -mt-2">
- {isSentiment ? (
- 'Classify the sentiment of only the last user message on each matching generation event with a sentiment classifier, not LLM calls. The classifier is trained on English, so labels are unreliable for other languages. For a multilingual agent, use an LLM judge instead.'
- ) : isHog ? (
- <>
- Run deterministic{' '}
- <Link to="https://posthog.com/docs/hog" target="_blank">
- Hog code
- </Link>{' '}
- against{' '}
- {isSessionTarget
- ? 'the whole session'
- : evaluation.target === 'trace'
- ? 'the whole trace'
- : 'each generation'}
- . No LLM cost.
- </>
- ) : (
- `Use an LLM to evaluate ${
- isSessionTarget
- ? 'the whole session'
- : evaluation.target === 'trace'
- ? 'the whole trace'
- : 'each generation'
- } against a natural-language prompt.`
- )}
- </p>
-
- {!isSentiment && (
- <>
- <LemonField.Pure label="Evaluate">
- <LemonSelect<EvaluationTarget>
- value={evaluation.target ?? 'generation'}
- onChange={setEvaluationTarget}
- options={[
- {
- value: 'generation',
- label: 'Each generation',
- },
- {
- value: 'trace',
- label: 'Whole trace',
- },
- ...(offersSessionTarget
- ? [
- {
- value: 'session' as const,
- label: 'Whole session',
- },
- ]
- : []),
- ]}
- fullWidth
- />
- </LemonField.Pure>
- <p className="text-muted text-sm -mt-2">
- {isSessionTarget
- ? 'Runs once per session on every trace it contains, after the session settles. Only fires for generations that have an $ai_session_id property.'
- : evaluation.target === 'trace'
- ? 'Runs once per trace on all of its events together, after it settles. Only fires for generations that have an $ai_trace_id property.'
- : 'Runs on each matching generation event individually, right after it is ingested.'}
- </p>
- <EvaluationSettleConfig />
- </>
- )}
-
- {!isSentiment && (
- <LemonField.Pure label="Output type">
- <LemonSelect
- value={evaluation.output_type}
- options={[
- { value: 'boolean', label: 'Boolean' },
- { value: 'categorical', label: 'Categorical' },
- { value: 'numeric', label: 'Numeric score' },
- ]}
- onChange={(value) =>
- setOutputType(
- value as 'boolean' | 'numeric' | 'categorical'
- )
- }
- disabledReason={
- !isNewEvaluation
- ? 'Create a new evaluation to change its output type.'
- : undefined
- }
- data-attr="llma-evaluation-output-type"
- />
- </LemonField.Pure>
- )}
- {evaluation.output_type === 'categorical' && (
- <CategoricalEvaluationConfig
- config={evaluation.output_config}
- onChange={patchOutputConfig}
- />
- )}
- {evaluation.output_type === 'numeric' && (
- <NumericEvaluationConfig
- config={evaluation.output_config}
- onChange={patchOutputConfig}
- requiresBounds={numericBoundsRequired}
- />
- )}
- <LemonField.Pure label="Description (optional)">
- <LemonTextArea
- value={evaluation.description || ''}
- onChange={setEvaluationDescription}
- placeholder="Describe what this evaluation checks for..."
- rows={2}
- maxLength={500}
- />
- </LemonField.Pure>
-
- <div className="flex flex-wrap items-center gap-2">
- <Tooltip
- title={canEnableReason}
- visible={canEnableReason ? undefined : false}
- >
- <span>
- <LemonSwitch
- checked={evaluation.enabled}
- onChange={setEvaluationEnabled}
- label="Enable evaluation"
- disabled={!canEnable && !evaluation.enabled}
- />
- </span>
- </Tooltip>
- <span className="text-muted text-sm">
- {!canEnable && !evaluation.enabled
- ? canEnableReason
- : evaluation.enabled
- ? 'This evaluation will run automatically based on triggers'
- : 'This evaluation is paused and will not run'}
- </span>
- </div>
-
- {isBooleanOutput && (
- <LemonField.Pure
- label={
- <div className="flex items-center gap-1">
- <span>Allow N/A responses</span>
- <Tooltip
- title={
- isHog
- ? 'When enabled, returning null from your Hog code means "not applicable" instead of being treated as an error.'
- : 'Sometimes forcing a True or False is not enough and you want the LLM to decide if the evaluation is applicable or not. Enable this when the evaluation criteria may not apply to all generations.'
- }
- >
- <IconInfo className="text-muted text-base" />
- </Tooltip>
- </div>
- }
- >
- <div className="flex flex-wrap items-center gap-2">
- <LemonSwitch
- checked={evaluation.output_config.allows_na ?? false}
- onChange={setAllowsNA}
- />
- <span className="text-muted text-sm">
- {evaluation.output_config.allows_na
- ? isHog
- ? 'Returning null means "Not Applicable"'
- : 'Evaluation can return "Not Applicable" when criteria doesn\'t apply'
- : isHog
- ? 'Evaluation must return true or false'
- : 'Evaluation returns true or false'}
- </span>
- </div>
- </LemonField.Pure>
- )}
-
- {isBooleanOutput && (
- <LemonField.Pure
- label={
- <div className="flex items-center gap-1">
- <span>A true result flags a problem</span>
- <Tooltip title="Turn this on when the evaluation looks for something you don't want, like a struggling user or a hallucination. A true result then counts as a fail in reports and pass rates. The evaluation results themselves don't change.">
- <IconInfo className="text-muted text-base" />
- </Tooltip>
- </div>
- }
- >
- <div className="flex flex-wrap items-center gap-2">
- <LemonSwitch
- checked={
- evaluation.output_config.true_is_failure ?? false
- }
- onChange={setTrueIsFailure}
- data-attr="llma-evaluation-true-is-failure-switch"
- />
- <span className="text-muted text-sm">
- {evaluation.output_config.true_is_failure
- ? 'A true result counts as a fail'
- : 'A true result counts as a pass'}
- </span>
- </div>
- </LemonField.Pure>
- )}
- </div>
- </div>
-
- {/* Prompt / Code Configuration */}
- {hasEditableCriteria && (
- <div className="bg-bg-light border rounded p-4 @min-[40rem]/evaluation:p-6">
- <h3 className="text-lg font-semibold mb-4">
- {isHog ? 'Evaluation code' : 'Evaluation prompt'}
- </h3>
- {isHog ? <EvaluationCodeEditor /> : <EvaluationPromptEditor />}
- </div>
- )}
-
- {/* Judge Model Configuration (LLM judge only) */}
- {evaluationTypeUsesModelConfiguration(evaluation.evaluation_type) && (
- <EvaluationModelPicker />
- )}
-
- {/* Trigger Configuration */}
- <div
- ref={triggersRef}
- className="bg-bg-light border rounded p-4 @min-[40rem]/evaluation:p-6"
- >
- <h3 className="text-lg font-semibold mb-4">Triggers</h3>
- <EvaluationTriggersPanel />
- </div>
-
- {/* Scheduled Reports (inline config for new evaluations) */}
- {isNewEvaluation && isReportableEvaluation && (
- <EvaluationReportConfig evaluationId="new" />
- )}
- </div>
- )}
-
- {/* Scheduled Reports (for existing evaluations, outside the form). The new form sets them in its sentence. */}
- {!resultsCardsEnabled && !isNewEvaluation && isReportableEvaluation && (
- <div className="mt-6">
- <EvaluationReportConfig evaluationId={evaluation.id} />
- </div>
- )}
+ <div className="max-w-4xl mx-auto">
+ <EvaluationConfigurationForm />
<div className="flex justify-end mt-6">
<AccessControlAction
resourceType={AccessControlResourceType.Evaluation}
@@ -813,7 +446,7 @@ export function AIObservabilityEvaluation(): JSX.Element {
loading={evaluationFormSubmitting}
data-attr="llma-evaluation-save-bottom"
>
- {isNewEvaluation ? 'Create evaluation' : 'Save changes'}
+ Save changes
</LemonButton>
</AccessControlAction>
</div>
diff --git a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluationsScene.tsx b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluationsScene.tsx
index f2b64c487c03..e9621c3b8012 100644
--- a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluationsScene.tsx
+++ b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluationsScene.tsx
@@ -1,589 +1,47 @@
import { BindLogic, useActions, useMountedLogic, useValues } from 'kea'
import { combineUrl, router } from 'kea-router'
-import { useEffect, useRef, useState } from 'react'
-import {
- IconArrowLeft,
- IconEye,
- IconFolder,
- IconHide,
- IconPencil,
- IconPlus,
- IconSearch,
- IconTrash,
- IconWarning,
-} from '@posthog/icons'
-import {
- LemonBanner,
- LemonButton,
- LemonDialog,
- LemonInput,
- LemonMenu,
- LemonModal,
- LemonSwitch,
- LemonTable,
- LemonTag,
- Link,
- Tooltip,
-} from '@posthog/lemon-ui'
+import { IconArrowLeft, IconPlus } from '@posthog/icons'
+import { LemonBanner, LemonButton, LemonInput, LemonModal, Link } from '@posthog/lemon-ui'
import { Button } from '@posthog/quill'
import { AccessControlAction } from 'lib/components/AccessControlAction'
-import { DateFilter } from 'lib/components/DateFilter/DateFilter'
import { useFeatureFlag } from 'lib/hooks/useFeatureFlag'
-import { LemonTableColumns } from 'lib/lemon-ui/LemonTable'
import { LinkPrimitive } from 'lib/lemon-ui/Link'
-import { ProfilePicture } from 'lib/lemon-ui/ProfilePicture'
-import { getAccessControlDisabledReason } from 'lib/utils/accessControlUtils'
-import { fullName } from 'lib/utils/strings'
import { urls } from 'scenes/urls'
import { SceneContent } from '~/layout/scenes/components/SceneContent'
import { SceneTitleSection } from '~/layout/scenes/components/SceneTitleSection'
import { AccessControlLevel, AccessControlResourceType } from '~/types'
-import type { EvaluationDirectoryApi } from '../generated/api.schemas'
-import { LLMProviderKey } from '../settings/llmProviderKeysLogic'
-import {
- getUnhealthyProviderKey,
- providerKeyStateIssueDescription,
- providerKeyStateLabel,
- providerLabel,
-} from '../settings/providerKeyStateUtils'
-import {
- EvaluationMetrics,
- PASS_RATE_SUCCESS_THRESHOLD,
- PASS_RATE_WARNING_THRESHOLD,
-} from './components/EvaluationMetrics'
-import { formatNumericEvaluationScore } from './constants'
-import { evaluationTypeUsesProviderKey } from './evaluationCapabilities'
-import { EvaluationStats, evaluationMetricsLogic } from './evaluationMetricsLogic'
+import { providerKeyStateLabel, providerLabel } from '../settings/providerKeyStateUtils'
import { EvaluationsTabs } from './EvaluationsTabs'
import { EvaluationTemplatesEmptyState } from './EvaluationTemplates'
import { llmEvaluationsLogic } from './llmEvaluationsLogic'
import { EvaluationResultsTable } from './overview/EvaluationResultsTable'
import { EvaluationsOverview } from './overview/EvaluationsOverview'
-import { statusReasonLabel } from './statusDisplay'
-import { EvaluationConfig } from './types'
-
-function getProviderKeyIssue(evaluation: EvaluationConfig, providerKeys: LLMProviderKey[]): LLMProviderKey | null {
- if (!evaluationTypeUsesProviderKey(evaluation.evaluation_type)) {
- return null
- }
-
- return getUnhealthyProviderKey(providerKeys, evaluation.model_configuration?.provider_key_id)
-}
-
-function getEvaluationMethodLabel(evaluation: EvaluationConfig): string {
- if (evaluation.evaluation_type === 'hog') {
- return 'Hog'
- }
- if (evaluation.evaluation_type === 'sentiment') {
- return 'Sentiment'
- }
- return 'LLM judge'
-}
-
-function getEvaluationMethodTagType(evaluation: EvaluationConfig): 'option' | 'highlight' | 'caution' {
- if (evaluation.evaluation_type === 'hog') {
- return 'option'
- }
- if (evaluation.evaluation_type === 'sentiment') {
- return 'highlight'
- }
- return 'caution'
-}
-
-function getEvaluationConfigPreview(evaluation: EvaluationConfig): string {
- if (evaluation.evaluation_type === 'hog') {
- return evaluation.evaluation_config.source
- }
- if (evaluation.evaluation_type === 'sentiment') {
- return 'User messages'
- }
- return evaluation.evaluation_config.prompt
-}
-
-function EvaluationDescription({ description }: { description: string }): JSX.Element {
- const [expanded, setExpanded] = useState(false)
- const [isClamped, setIsClamped] = useState(false)
- const textRef = useRef<HTMLDivElement | null>(null)
-
- // line-clamp-1 constrains clientHeight to a single line; if scrollHeight exceeds it the text is
- // truncated and worth a toggle. Only measure while collapsed — expanded, scrollHeight === clientHeight.
- useEffect(() => {
- if (textRef.current && !expanded) {
- setIsClamped(textRef.current.scrollHeight > textRef.current.clientHeight)
- }
- }, [description, expanded])
-
- return (
- <div className="flex items-start gap-1">
- <div ref={textRef} className={`text-muted text-sm ${expanded ? '' : 'line-clamp-1'}`}>
- {description}
- </div>
- {(isClamped || expanded) && (
- <LemonButton
- size="xsmall"
- onClick={() => setExpanded(!expanded)}
- data-attr="toggle-evaluation-description"
- >
- {expanded ? 'Show less' : 'Show more'}
- </LemonButton>
- )}
- </div>
- )
-}
-
-/** The table without the overview. It alone reads the metrics, so the overview never runs their queries. */
-function EvaluationsWithMetricsTable({
- columns,
- displayedEvaluations,
- loading,
-}: {
- columns: LemonTableColumns<EvaluationConfig>
- displayedEvaluations: EvaluationConfig[]
- loading: boolean
-}): JSX.Element {
- const { evaluationsWithMetrics } = useValues(evaluationMetricsLogic)
- const displayedIds = new Set(displayedEvaluations.map((evaluation) => evaluation.id))
-
- return (
- <LemonTable
- columns={columns}
- dataSource={evaluationsWithMetrics.filter((evaluation: EvaluationConfig) =>
- displayedIds.has(evaluation.id)
- )}
- loading={loading}
- rowKey="id"
- pagination={{
- pageSize: 50,
- }}
- nouns={['evaluation', 'evaluations']}
- />
- )
-}
function AIObservabilityEvaluationsContent(): JSX.Element {
const evaluationsLogic = llmEvaluationsLogic()
const {
evaluations,
- displayedEvaluations,
evaluationsLoading,
evaluationsLoadFailed,
- evaluationsFilter,
- showDisabledEvaluations,
- dateFilter,
- providerKeys,
unhealthyProviderKeysUsedByEvaluations,
- canEnableEvaluation,
evaluationDirectories,
evaluationDirectoriesLoading,
selectedDirectory,
- selectedDirectoryId,
directoryEditor,
submitDirectoryLoading,
- deletingDirectoryId,
- movingEvaluationId,
} = useValues(evaluationsLogic)
- const {
- setEvaluationsFilter,
- setShowDisabledEvaluations,
- toggleEvaluationEnabled,
- setDates,
- openCreateDirectory,
- openRenameDirectory,
- closeDirectoryEditor,
- setDirectoryEditorName,
- submitDirectory,
- deleteDirectory,
- moveEvaluation,
- selectDirectory,
- deleteEvaluation,
- } = useActions(evaluationsLogic)
- const overviewEnabled = useFeatureFlag('AI_OBSERVABILITY_EVALUATIONS_OVERVIEW')
- const { push } = useActions(router)
+ const { closeDirectoryEditor, setDirectoryEditorName, submitDirectory, selectDirectory } =
+ useActions(evaluationsLogic)
const { searchParams } = useValues(router)
- const evaluationUrl = (id: string): string => combineUrl(urls.aiObservabilityEvaluation(id), searchParams).url
const settingsUrl = urls.settings('project-ai-observability', 'ai-observability-byok')
- const moveEvaluationDisabledReason = (evaluation: EvaluationConfig): string | null =>
- getAccessControlDisabledReason(
- AccessControlResourceType.Evaluation,
- AccessControlLevel.Editor,
- evaluation.user_access_level ?? undefined
- )
-
- const directoryById = new Map(evaluationDirectories.map((directory) => [directory.id, directory]))
-
- const columns: LemonTableColumns<EvaluationConfig> = [
- {
- title: 'Name',
- key: 'name',
- render: (_, evaluation) => (
- <div className="flex flex-col">
- <Link to={evaluationUrl(evaluation.id)} className="font-semibold text-primary">
- {evaluation.name}
- </Link>
- {evaluation.description && <EvaluationDescription description={evaluation.description} />}
- </div>
- ),
- sorter: (a, b) => a.name.localeCompare(b.name),
- },
- {
- title: 'Directory',
- key: 'directory',
- render: (_, evaluation) =>
- evaluation.directory_id ? directoryById.get(evaluation.directory_id)?.name : 'Top level',
- sorter: (a, b) => {
- const directoryName = (evaluation: EvaluationConfig): string =>
- evaluation.directory_id ? directoryById.get(evaluation.directory_id)?.name || '' : ''
- return directoryName(a).localeCompare(directoryName(b))
- },
- },
- {
- title: 'Status',
- key: 'status',
- render: (_, evaluation) => {
- // When the system has marked an eval as errored, the toggle is misleading — flipping it
- // would just fail. Show an error pill instead so the row is visibly different and users
- // click through to the detail page to see what's wrong and how to fix it.
- if (evaluation.status === 'error') {
- return (
- <Tooltip title={`${statusReasonLabel(evaluation.status_reason)}. Open to fix.`}>
- <LemonTag
- type="danger"
- icon={<IconWarning />}
- forceClickable
- onClick={() => push(evaluationUrl(evaluation.id))}
- data-attr="evaluation-status-error"
- >
- Error
- </LemonTag>
- </Tooltip>
- )
- }
- const providerKeyIssue = evaluation.enabled ? getProviderKeyIssue(evaluation, providerKeys) : null
- if (providerKeyIssue) {
- return (
- <Tooltip
- title={`Paused because API key ${providerKeyIssue.name} ${providerKeyStateIssueDescription(
- providerKeyIssue.state
- )}. Open settings to fix.`}
- >
- <LemonTag
- type="warning"
- icon={<IconWarning />}
- forceClickable
- onClick={() => push(settingsUrl)}
- data-attr="evaluation-status-key-issue"
- >
- Key issue
- </LemonTag>
- </Tooltip>
- )
- }
- const canEnable = canEnableEvaluation(evaluation)
- const isBlocked = !canEnable && !evaluation.enabled
- const blockedReason = 'Add a provider API key to enable this evaluation.'
- return (
- <div className="flex flex-wrap items-center gap-2">
- <AccessControlAction
- resourceType={AccessControlResourceType.Evaluation}
- minAccessLevel={AccessControlLevel.Editor}
- userAccessLevel={evaluation.user_access_level ?? undefined}
- >
- <Tooltip title={isBlocked ? blockedReason : undefined}>
- <span>
- <LemonSwitch
- checked={evaluation.enabled}
- onChange={() => toggleEvaluationEnabled(evaluation.id)}
- size="small"
- disabled={isBlocked}
- data-attr="toggle-evaluation-enabled"
- />
- </span>
- </Tooltip>
- </AccessControlAction>
- <span className={evaluation.enabled ? 'text-success' : 'text-muted'}>
- {evaluation.enabled ? 'Enabled' : 'Disabled'}
- </span>
- </div>
- )
- },
- // Sort: errors first (most attention-demanding), then enabled, then paused.
- sorter: (a, b) => {
- const rank = (e: EvaluationConfig): number => (e.status === 'error' ? 0 : e.enabled ? 1 : 2)
- return rank(a) - rank(b)
- },
- },
- {
- title: 'Method',
- key: 'method',
- render: (_, evaluation) => (
- <LemonTag type={getEvaluationMethodTagType(evaluation)}>
- {getEvaluationMethodLabel(evaluation)}
- </LemonTag>
- ),
- },
- {
- title: 'Config',
- key: 'config',
- render: (_, evaluation) => {
- const preview = getEvaluationConfigPreview(evaluation)
- return (
- <div className="max-w-md">
- <div className="text-sm font-mono bg-bg-light border rounded px-2 py-1 truncate">
- {preview || '(empty)'}
- </div>
- </div>
- )
- },
- },
- {
- title: 'Triggers',
- key: 'conditions',
- render: (_, evaluation) => (
- <div className="flex flex-wrap gap-1">
- {evaluation.conditions.map((condition) => {
- const propertyCount = condition.properties?.length ?? 0
- return (
- <LemonTag key={condition.id} type="option">
- {parseFloat((condition.rollout_percentage ?? 0).toFixed(2))}%
- {propertyCount > 0 &&
- ` when ${propertyCount} condition${propertyCount !== 1 ? 's' : ''}`}
- </LemonTag>
- )
- })}
- {evaluation.conditions.length === 0 && <span className="text-muted text-sm">No triggers</span>}
- </div>
- ),
- },
- {
- title: 'Runs',
- key: 'recent_stats',
- render: (_, evaluation: EvaluationConfig & { stats?: EvaluationStats }) => {
- const stats = evaluation.stats
- if (!stats || stats.runs_count === 0) {
- return <span className="text-muted text-sm">No runs</span>
- }
-
- if (evaluation.output_type === 'numeric') {
- return (
- <div className="text-sm">
- <div>{`${stats.runs_count} runs`}</div>
- <div>{`Mean score: ${stats.score_mean == null ? '–' : formatNumericEvaluationScore(stats.score_mean)}`}</div>
- {evaluation.output_config.passing_rule && (
- <div>{`Pass rate: ${stats.pass_rate == null ? '–' : `${stats.pass_rate.toFixed(1)}%`}`}</div>
- )}
- </div>
- )
- }
- // Sentiment evals classify rather than pass/fail, so a pass rate is meaningless
- if (evaluation.evaluation_type === 'sentiment') {
- return (
- <div className="text-sm">
- {stats.runs_count} run{stats.runs_count !== 1 ? 's' : ''}
- </div>
- )
- }
-
- const passRateColor =
- stats.pass_rate == null
- ? 'text-muted'
- : stats.pass_rate >= PASS_RATE_SUCCESS_THRESHOLD
- ? 'text-success'
- : stats.pass_rate >= PASS_RATE_WARNING_THRESHOLD
- ? 'text-warning'
- : 'text-danger'
-
- return (
- <div className="flex flex-col items-center">
- <div className="text-sm">
- {stats.runs_count} run{stats.runs_count !== 1 ? 's' : ''}
- </div>
- <div className={`font-semibold ${passRateColor}`}>
- {stats.pass_rate == null ? '–' : `${parseFloat(stats.pass_rate.toFixed(2))}%`}
- </div>
- </div>
- )
- },
- },
- {
- title: 'Created by',
- key: 'created_by',
- render: (_, evaluation) =>
- evaluation.created_by ? (
- <ProfilePicture user={evaluation.created_by} size="md" showName />
- ) : (
- <span className="text-muted text-sm">–</span>
- ),
- sorter: (a, b) => {
- // Match the displayed identity: full name, falling back to email when no name is set.
- const sortKey = (e: EvaluationConfig): string =>
- e.created_by ? fullName(e.created_by) || e.created_by.email : ''
- return sortKey(a).localeCompare(sortKey(b))
- },
- },
- {
- title: 'Actions',
- key: 'actions',
- render: (_, evaluation) => (
- <div className="flex gap-1">
- <LemonMenu
- items={[
- {
- icon: <IconFolder />,
- label: 'Top level',
- onClick: () => moveEvaluation(evaluation.id, null),
- disabledReason:
- !evaluation.directory_id || movingEvaluationId === evaluation.id
- ? 'Evaluation is already here'
- : undefined,
- },
- ...evaluationDirectories.map((directory) => ({
- icon: <IconFolder />,
- label: directory.name,
- onClick: () => moveEvaluation(evaluation.id, directory.id),
- disabledReason:
- evaluation.directory_id === directory.id || movingEvaluationId === evaluation.id
- ? 'Evaluation is already here'
- : undefined,
- })),
- ]}
- >
- <LemonButton
- size="small"
- type="secondary"
- icon={<IconFolder />}
- tooltip="Move evaluation"
- loading={movingEvaluationId === evaluation.id}
- disabledReason={moveEvaluationDisabledReason(evaluation)}
- />
- </LemonMenu>
- <AccessControlAction
- resourceType={AccessControlResourceType.Evaluation}
- minAccessLevel={AccessControlLevel.Editor}
- userAccessLevel={evaluation.user_access_level ?? undefined}
- >
- <LemonButton
- size="small"
- type="secondary"
- icon={<IconPencil />}
- onClick={() => push(evaluationUrl(evaluation.id))}
- />
- </AccessControlAction>
- <AccessControlAction
- resourceType={AccessControlResourceType.Evaluation}
- minAccessLevel={AccessControlLevel.Editor}
- userAccessLevel={evaluation.user_access_level ?? undefined}
- >
- <LemonButton
- size="small"
- type="secondary"
- status="danger"
- icon={<IconTrash />}
- onClick={() => {
- LemonDialog.open({
- title: `Delete ${evaluation.name}?`,
- description: 'Are you sure you want to delete this evaluation?',
- primaryButton: {
- children: 'Delete',
- type: 'primary',
- status: 'danger',
- 'data-attr': 'confirm-delete-evaluation',
- onClick: () => deleteEvaluation(evaluation.id),
- },
- secondaryButton: {
- children: 'Cancel',
- type: 'secondary',
- },
- })
- }}
- />
- </AccessControlAction>
- </div>
- ),
- },
- ]
-
- const directoryColumns: LemonTableColumns<EvaluationDirectoryApi> = [
- {
- title: 'Name',
- key: 'name',
- render: (_, directory) => (
- <LemonButton
- type="tertiary"
- icon={<IconFolder />}
- onClick={() => selectDirectory(directory.id)}
- data-attr="evaluation-directory-link"
- >
- {directory.name}
- </LemonButton>
- ),
- sorter: (a, b) => a.name.localeCompare(b.name),
- },
- {
- title: 'Evaluations',
- key: 'evaluation_count',
- render: (_, directory) => directory.evaluation_count,
- sorter: (a, b) => a.evaluation_count - b.evaluation_count,
- },
- {
- title: 'Actions',
- key: 'actions',
- align: 'right',
- width: 0,
- render: (_, directory) => (
- <div className="flex justify-end gap-1">
- <AccessControlAction
- resourceType={AccessControlResourceType.Evaluation}
- minAccessLevel={AccessControlLevel.Editor}
- >
- <LemonButton
- size="small"
- type="secondary"
- icon={<IconPencil />}
- tooltip="Rename directory"
- onClick={() => openRenameDirectory(directory)}
- />
- </AccessControlAction>
- <AccessControlAction
- resourceType={AccessControlResourceType.Evaluation}
- minAccessLevel={AccessControlLevel.Editor}
- >
- <LemonButton
- size="small"
- type="secondary"
- status="danger"
- icon={<IconTrash />}
- tooltip="Delete directory"
- loading={deletingDirectoryId === directory.id}
- onClick={() => {
- LemonDialog.open({
- title: `Delete ${directory.name}?`,
- description: `${directory.evaluation_count} evaluation${
- directory.evaluation_count === 1 ? '' : 's'
- } will move to the top level.`,
- primaryButton: {
- children: 'Delete directory',
- status: 'danger',
- onClick: () => deleteDirectory(directory.id),
- },
- secondaryButton: {
- children: 'Cancel',
- },
- })
- }}
- />
- </AccessControlAction>
- </div>
- ),
- },
- ]
// A failed list is not an empty project. The overview still loads, and in a directory it shows a retry.
const showEmptyState =
- !(overviewEnabled && evaluationsLoadFailed) &&
+ !evaluationsLoadFailed &&
!evaluationsLoading &&
!evaluationDirectoriesLoading &&
evaluations.length === 0 &&
@@ -633,21 +91,6 @@ function AIObservabilityEvaluationsContent(): JSX.Element {
</p>
</div>
<div className="flex flex-wrap items-center gap-2">
- {!selectedDirectoryId && !overviewEnabled && (
- <AccessControlAction
- resourceType={AccessControlResourceType.Evaluation}
- minAccessLevel={AccessControlLevel.Editor}
- >
- <LemonButton
- type="secondary"
- icon={<IconFolder />}
- onClick={() => openCreateDirectory()}
- data-attr="create-evaluation-directory-button"
- >
- New directory
- </LemonButton>
- </AccessControlAction>
- )}
<AccessControlAction
resourceType={AccessControlResourceType.Evaluation}
minAccessLevel={AccessControlLevel.Editor}
@@ -669,59 +112,8 @@ function AIObservabilityEvaluationsContent(): JSX.Element {
<EvaluationTemplatesEmptyState />
) : (
<>
- {overviewEnabled ? (
- <EvaluationsOverview />
- ) : (
- <>
- <DateFilter dateFrom={dateFilter.dateFrom} dateTo={dateFilter.dateTo} onChange={setDates} />
-
- <EvaluationMetrics />
- </>
- )}
-
- {!overviewEnabled && (
- <div className="flex flex-wrap items-center gap-2">
- <LemonInput
- type="search"
- placeholder="Search online evals..."
- value={evaluationsFilter}
- data-attr="evaluations-search-input"
- onChange={setEvaluationsFilter}
- prefix={<IconSearch />}
- className="max-w-sm"
- />
- <LemonButton
- type="secondary"
- active={!showDisabledEvaluations}
- icon={showDisabledEvaluations ? <IconEye /> : <IconHide />}
- onClick={() => setShowDisabledEvaluations(!showDisabledEvaluations)}
- data-attr="toggle-show-disabled-evaluations"
- tooltip={showDisabledEvaluations ? 'Hide disabled evals' : 'Show disabled evals'}
- >
- {showDisabledEvaluations ? 'Hide disabled' : 'Show disabled'}
- </LemonButton>
- </div>
- )}
-
- {!overviewEnabled && !evaluationsFilter && !selectedDirectoryId && (
- <LemonTable
- columns={directoryColumns}
- dataSource={evaluationDirectories}
- loading={evaluationDirectoriesLoading}
- rowKey="id"
- nouns={['directory', 'directories']}
- />
- )}
-
- {overviewEnabled ? (
- <EvaluationResultsTable />
- ) : (
- <EvaluationsWithMetricsTable
- columns={columns}
- displayedEvaluations={displayedEvaluations}
- loading={evaluationsLoading || evaluationDirectoriesLoading}
- />
- )}
+ <EvaluationsOverview />
+ <EvaluationResultsTable />
</>
)}
@@ -760,65 +152,45 @@ function AIObservabilityEvaluationsContent(): JSX.Element {
)
}
-/**
- * Binding mounts the metrics for the scene's lifetime. Attaching them to llmEvaluationsLogic instead would hold both
- * mounted, so the list would never reload on the next visit. The overview never shows them, so it skips their queries.
- */
-function LegacyMetricsBinding({ bound, children }: { bound: boolean; children: React.ReactNode }): JSX.Element {
- return bound ? (
- <BindLogic logic={evaluationMetricsLogic} props={{}}>
- {children}
- </BindLogic>
- ) : (
- <>{children}</>
- )
-}
-
export function AIObservabilityEvaluationsScene(): JSX.Element {
useMountedLogic(llmEvaluationsLogic())
const todayRailNav = useFeatureFlag('TODAY_RAIL_NAV')
- const overviewEnabled = useFeatureFlag('AI_OBSERVABILITY_EVALUATIONS_OVERVIEW')
return (
<BindLogic logic={llmEvaluationsLogic} props={{}}>
- <LegacyMetricsBinding bound={!overviewEnabled}>
- <SceneContent>
- <SceneTitleSection
- name="Evaluations"
- description="Configure and monitor automated LLM output evaluations."
- resourceType={{
- type: 'llm_evaluations',
- }}
- actions={
- todayRailNav ? (
- <Button
- variant="outline"
- render={
- <LinkPrimitive
- to="https://posthog.com/docs/ai-evals/evaluations"
- target="_blank"
- />
- }
- >
- Documentation
- </Button>
- ) : (
- <LemonButton
- to="https://posthog.com/docs/ai-evals/evaluations"
- type="secondary"
- targetBlank
- size="small"
- >
- Documentation
- </LemonButton>
- )
- }
- />
- <EvaluationsTabs activeTab="online-evals">
- <AIObservabilityEvaluationsContent />
- </EvaluationsTabs>
- </SceneContent>
- </LegacyMetricsBinding>
+ <SceneContent>
+ <SceneTitleSection
+ name="Evaluations"
+ description="Configure and monitor automated LLM output evaluations."
+ resourceType={{
+ type: 'llm_evaluations',
+ }}
+ actions={
+ todayRailNav ? (
+ <Button
+ variant="outline"
+ render={
+ <LinkPrimitive to="https://posthog.com/docs/ai-evals/evaluations" target="_blank" />
+ }
+ >
+ Documentation
+ </Button>
+ ) : (
+ <LemonButton
+ to="https://posthog.com/docs/ai-evals/evaluations"
+ type="secondary"
+ targetBlank
+ size="small"
+ >
+ Documentation
+ </LemonButton>
+ )
+ }
+ />
+ <EvaluationsTabs activeTab="online-evals">
+ <AIObservabilityEvaluationsContent />
+ </EvaluationsTabs>
+ </SceneContent>
</BindLogic>
)
}
diff --git a/products/ai_observability/frontend/evaluations/components/CategoricalEvaluationConfig.stories.tsx b/products/ai_observability/frontend/evaluations/components/CategoricalEvaluationConfig.stories.tsx
deleted file mode 100644
index 3863149c97a6..000000000000
--- a/products/ai_observability/frontend/evaluations/components/CategoricalEvaluationConfig.stories.tsx
+++ /dev/null
@@ -1,60 +0,0 @@
-import type { Meta, StoryObj } from '@storybook/react'
-import { useState } from 'react'
-
-import type { EvaluationOutputConfig } from '../types'
-import { CategoricalEvaluationConfig } from './CategoricalEvaluationConfig'
-
-const meta: Meta<typeof CategoricalEvaluationConfig> = {
- title: 'Scenes-App/LLM observability/CategoricalEvaluationConfig',
- component: CategoricalEvaluationConfig,
- parameters: { layout: 'padded' },
-}
-export default meta
-
-type Story = StoryObj<typeof CategoricalEvaluationConfig>
-
-function Template({
- narrow = false,
- initialConfig,
-}: {
- narrow?: boolean
- initialConfig?: EvaluationOutputConfig
-}): JSX.Element {
- const [config, setConfig] = useState<EvaluationOutputConfig>({
- options: [
- { key: 'resolved', label: 'Resolved' },
- { key: 'needs_follow_up', label: 'Needs follow-up' },
- ],
- selection_mode: 'multiple',
- allows_na: true,
- passing_rule: { categories: ['resolved'] },
- ...initialConfig,
- })
- return (
- <div className={narrow ? 'max-w-lg' : 'max-w-4xl'}>
- <CategoricalEvaluationConfig
- config={config}
- onChange={(patch) => setConfig((current) => ({ ...current, ...patch }))}
- />
- </div>
- )
-}
-
-export const Default: Story = { render: () => <Template /> }
-export const Narrow: Story = { render: () => <Template narrow /> }
-export const InvalidPassingRule: Story = {
- render: () => <Template narrow initialConfig={{ selection_mode: 'single', passing_rule: { categories: [] } }} />,
-}
-export const DuplicateKeys: Story = {
- render: () => (
- <Template
- narrow
- initialConfig={{
- options: [
- { key: 'resolved', label: 'Resolved' },
- { key: 'resolved', label: 'Needs follow-up' },
- ],
- }}
- />
- ),
-}
diff --git a/products/ai_observability/frontend/evaluations/components/CategoricalEvaluationConfig.tsx b/products/ai_observability/frontend/evaluations/components/CategoricalEvaluationConfig.tsx
deleted file mode 100644
index 3e7b33de72ab..000000000000
--- a/products/ai_observability/frontend/evaluations/components/CategoricalEvaluationConfig.tsx
+++ /dev/null
@@ -1,164 +0,0 @@
-import { IconPlus, IconTrash } from '@posthog/icons'
-import { LemonButton, LemonCheckbox, LemonInput, LemonSegmentedButton, LemonSwitch } from '@posthog/lemon-ui'
-
-import { LemonField } from 'lib/lemon-ui/LemonField'
-
-import { MAX_CATEGORICAL_OPTIONS, categoricalOptionsError, categoricalPassingRuleError } from '../constants'
-import type { EvaluationOutputConfig } from '../types'
-
-export function CategoricalEvaluationConfig({
- config,
- onChange,
-}: {
- config: EvaluationOutputConfig
- onChange: (patch: EvaluationOutputConfig) => void
-}): JSX.Element {
- const options = config.options ?? []
- const rule = config.passing_rule && 'categories' in config.passing_rule ? config.passing_rule : null
- const updateOption = (index: number, patch: Partial<{ key: string; label: string }>): void =>
- onChange({ options: options.map((item, i) => (i === index ? { ...item, ...patch } : item)) })
-
- return (
- <div className="flex flex-col gap-4">
- <LemonField.Pure label="Categories per result">
- <LemonSegmentedButton
- value={config.selection_mode ?? 'single'}
- onChange={(selection_mode) => onChange({ selection_mode })}
- options={[
- { value: 'single', label: 'One category' },
- { value: 'multiple', label: 'Several categories' },
- ]}
- fullWidth
- data-attr="llma-evaluation-categorical-selection"
- />
- </LemonField.Pure>
- <LemonField.Pure
- label="Categories"
- error={categoricalOptionsError(config)}
- help="Results store the key. Keep a key unchanged to keep its past results."
- >
- <div className="flex flex-col gap-2">
- {options.length === 0 && (
- <span className="text-sm text-secondary">Add a category to define the possible results.</span>
- )}
- {options.map((option, index) => (
- <div key={index} className="flex items-center gap-2">
- <LemonInput
- className="flex-1 min-w-0"
- aria-label={`Category ${index + 1} label`}
- data-attr="llma-evaluation-categorical-label"
- placeholder="Label, for example Resolved"
- value={option.label}
- maxLength={256}
- onChange={(label) => updateOption(index, { label })}
- />
- <LemonInput
- className="w-36 shrink-0 font-mono"
- aria-label={`Category ${index + 1} key`}
- data-attr="llma-evaluation-categorical-key"
- placeholder="key"
- value={option.key}
- maxLength={128}
- onChange={(key) =>
- onChange({
- options: options.map((item, i) => (i === index ? { ...item, key } : item)),
- ...(rule
- ? {
- passing_rule: {
- categories: rule.categories.map((value) =>
- value === option.key ? key : value
- ),
- },
- }
- : {}),
- })
- }
- />
- {rule && (
- <LemonCheckbox
- label="Passes"
- data-attr="llma-evaluation-categorical-passing-category"
- disabledReason={option.key ? undefined : 'Add a key first'}
- checked={!!option.key && rule.categories.includes(option.key)}
- onChange={(checked) =>
- onChange({
- passing_rule: {
- categories: checked
- ? [...rule.categories, option.key]
- : rule.categories.filter((key) => key !== option.key),
- },
- })
- }
- />
- )}
- <LemonButton
- icon={<IconTrash />}
- size="small"
- tooltip="Remove category"
- aria-label={`Remove category ${option.label || index + 1}`}
- data-attr="llma-evaluation-categorical-remove"
- onClick={() =>
- onChange({
- options: options.filter((_, i) => i !== index),
- ...(rule
- ? {
- passing_rule: {
- categories: rule.categories.filter((key) => key !== option.key),
- },
- }
- : {}),
- })
- }
- />
- </div>
- ))}
- <div>
- <LemonButton
- type="secondary"
- size="small"
- icon={<IconPlus />}
- data-attr="llma-evaluation-categorical-add"
- disabledReason={
- options.length >= MAX_CATEGORICAL_OPTIONS
- ? `You can add up to ${MAX_CATEGORICAL_OPTIONS} categories.`
- : undefined
- }
- onClick={() => onChange({ options: [...options, { key: '', label: '' }] })}
- >
- Add category
- </LemonButton>
- </div>
- </div>
- </LemonField.Pure>
- <LemonSwitch
- label="Allow N/A responses"
- data-attr="llma-evaluation-categorical-allows-na"
- checked={config.allows_na ?? false}
- onChange={(allows_na) => onChange({ allows_na })}
- bordered
- fullWidth
- />
- <LemonField.Pure
- error={categoricalPassingRuleError(config)}
- help={
- rule
- ? `A result passes when every category it returns is marked as passing.${
- config.selection_mode === 'multiple'
- ? ' An empty selection passes only when no category is marked.'
- : ''
- } Changing this also changes how past results count.`
- : 'Without a passing rule, results show their categories but no pass rate or reports.'
- }
- >
- <LemonSwitch
- label="Set a passing rule"
- data-attr="llma-evaluation-categorical-passing-rule"
- checked={!!rule}
- onChange={(enabled) => onChange({ passing_rule: enabled ? { categories: [] } : null })}
- bordered
- fullWidth
- />
- </LemonField.Pure>
- </div>
- )
-}
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationMetrics.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationMetrics.tsx
deleted file mode 100644
index 5107f0feafa4..000000000000
--- a/products/ai_observability/frontend/evaluations/components/EvaluationMetrics.tsx
+++ /dev/null
@@ -1,129 +0,0 @@
-import { useValues } from 'kea'
-
-import { LemonSkeleton } from '@posthog/lemon-ui'
-
-import { Query } from '~/queries/Query/Query'
-import { InsightVizNode, NodeKind } from '~/queries/schema/schema-general'
-
-// eslint-disable-next-line import/no-cycle
-import { evaluationMetricsLogic } from '../evaluationMetricsLogic'
-import { llmEvaluationsLogic } from '../llmEvaluationsLogic'
-
-export const EVALUATION_METRICS_COLLECTION_ID = 'evaluation-metrics'
-
-export const PASS_RATE_SUCCESS_THRESHOLD = 70
-export const PASS_RATE_WARNING_THRESHOLD = 50
-
-function getPassRateColor(passRate: number): string {
- if (passRate >= PASS_RATE_SUCCESS_THRESHOLD) {
- return 'text-success'
- }
- if (passRate >= PASS_RATE_WARNING_THRESHOLD) {
- return 'text-warning'
- }
- return 'text-danger'
-}
-
-function SummaryCard({
- title,
- value,
- subtitle,
- colorClass,
-}: {
- title: string
- value: string | number
- subtitle?: string
- colorClass?: string
-}): JSX.Element {
- return (
- <div className="bg-bg-light border rounded p-4 flex flex-col">
- <div className="text-muted text-xs font-medium uppercase mb-2">{title}</div>
- <div className={`text-3xl font-semibold ${colorClass || ''}`}>{value}</div>
- {subtitle && <div className="text-muted text-sm mt-1">{subtitle}</div>}
- </div>
- )
-}
-
-export function EvaluationMetrics(): JSX.Element {
- const { summaryMetrics, statsLoading, chartQuery } = useValues(evaluationMetricsLogic)
-
- const { evaluations, evaluationsLoading } = useValues(llmEvaluationsLogic)
-
- const enabledEvaluationsCount = evaluations.filter((e) => e.enabled && !e.deleted).length
-
- return (
- <div className="@container/evaluation-metrics mb-6">
- <div className="flex flex-col gap-4 @min-[56rem]/evaluation-metrics:flex-row">
- {evaluationsLoading ? (
- <LemonSkeleton className="flex-1 h-96" />
- ) : chartQuery ? (
- <div className="flex-1 min-w-0 bg-bg-light rounded p-4 flex flex-col InsightCard h-96">
- <h3 className="text-lg font-semibold mb-2">Pass rate over time</h3>
- <p className="text-muted text-sm mb-4">Showing pass rate trends for enabled evaluations</p>
- <div className="flex-1 flex flex-col min-h-0">
- <Query
- query={{ kind: NodeKind.InsightVizNode, source: chartQuery } as InsightVizNode}
- readOnly
- embedded
- inSharedMode
- context={{
- insightProps: {
- dashboardItemId: 'new-evaluation-metrics-chart',
- dataNodeCollectionId: EVALUATION_METRICS_COLLECTION_ID,
- },
- }}
- />
- </div>
- </div>
- ) : (
- <div className="flex-1 min-w-0 min-h-48 bg-bg-light border rounded p-8 flex items-center justify-center">
- <div className="text-muted text-center">
- No enabled evaluations with pass rates. Enable a boolean evaluation or set a passing rule on
- a numeric or categorical evaluation.
- </div>
- </div>
- )}
-
- <div className="flex-1 min-w-0 grid grid-cols-2 gap-4">
- {statsLoading ? (
- Array.from({ length: 4 }, (_, index) => <LemonSkeleton key={index} className="h-full w-full" />)
- ) : (
- <>
- <SummaryCard
- title="Enabled evaluations"
- value={enabledEvaluationsCount}
- subtitle={`${evaluations.length} total`}
- />
- <SummaryCard
- title="Runs"
- value={summaryMetrics.total_runs}
- subtitle={summaryMetrics.total_runs === 0 ? 'No activity' : undefined}
- />
- <SummaryCard
- title="Pass rate"
- value={
- summaryMetrics.overall_pass_rate == null
- ? 'N/A'
- : `${summaryMetrics.overall_pass_rate}%`
- }
- colorClass={
- summaryMetrics.overall_pass_rate != null
- ? getPassRateColor(summaryMetrics.overall_pass_rate)
- : ''
- }
- />
- <SummaryCard
- title="Failing evaluations"
- value={summaryMetrics.failing_evaluations_count}
- subtitle="< 70% pass rate"
- colorClass={
- summaryMetrics.failing_evaluations_count > 0 ? 'text-danger' : 'text-success'
- }
- />
- </>
- )}
- </div>
- </div>
- </div>
- )
-}
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx
deleted file mode 100644
index f75f3d11e37b..000000000000
--- a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx
+++ /dev/null
@@ -1,130 +0,0 @@
-import type { Meta, StoryObj } from '@storybook/react'
-import { BindLogic } from 'kea'
-
-import { FEATURE_FLAGS } from 'lib/constants'
-
-import { useStorybookMocks } from '~/mocks/browser'
-
-import { llmEvaluationLogic } from '../llmEvaluationLogic'
-import type { JudgeMethod, ReasoningLevel } from '../types'
-import { EvaluationModelPicker } from './EvaluationModelPicker'
-
-const meta: Meta<{
- method?: JudgeMethod
- narrow?: boolean
- empty?: boolean
- nativeOpenAI?: boolean
- reasoningLevel?: ReasoningLevel
-}> = {
- title: 'Scenes-App/LLM observability/EvaluationModelPicker',
- component: EvaluationModelPicker,
- parameters: {
- layout: 'padded',
- },
- render: ({ method = 'llm', narrow = false, empty = false, nativeOpenAI = false, reasoningLevel }) => {
- useStorybookMocks({
- get: {
- '/api/environments/:team_id/llm_analytics/provider_keys/': {
- results: [
- nativeOpenAI
- ? { id: 'example-openai-key', name: 'OpenAI', provider: 'openai', state: 'ok' }
- : { id: 'example-openrouter-key', name: 'OpenRouter', provider: 'openrouter', state: 'ok' },
- ],
- },
- '/api/environments/:team_id/llm_analytics/evaluation_config/': { active_provider_key: null },
- '/api/llm_proxy/models/': ({ request }) => [
- 200,
- new URL(request.url).searchParams.has('provider_key_id')
- ? nativeOpenAI
- ? [
- {
- id: 'gpt-5-mini',
- name: 'gpt-5-mini',
- provider: 'OpenAI',
- supports_chat: true,
- supports_decisions: false,
- supports_reasoning: true,
- },
- {
- id: 'gpt-6-luna',
- name: 'gpt-6-luna',
- provider: 'OpenAI',
- supports_chat: true,
- supports_decisions: true,
- },
- ]
- : [
- {
- id: 'example/chat-model',
- name: 'example/chat-model',
- provider: 'OpenRouter',
- supports_chat: true,
- supports_decisions: false,
- },
- ...(!empty
- ? [
- {
- id: 'example/decision-model',
- name: 'example/decision-model',
- provider: 'OpenRouter',
- supports_chat: false,
- supports_decisions: true,
- },
- {
- id: 'example/dual-model',
- name: 'example/dual-model',
- provider: 'OpenRouter',
- supports_chat: true,
- supports_decisions: true,
- },
- ]
- : []),
- ]
- : [],
- ],
- '/api/projects/:team_id/evaluations/example-evaluation/': {
- id: 'example-evaluation',
- name: 'Response quality',
- enabled: false,
- evaluation_type: 'llm_judge',
- evaluation_config: {
- prompt: 'Does the response answer the question?',
- judge_method: method,
- reasoning_level: reasoningLevel,
- },
- output_type: 'boolean',
- output_config: {},
- conditions: [],
- target: 'generation',
- target_config: {},
- model_configuration: reasoningLevel
- ? { provider: 'openai', model: 'gpt-5-mini', provider_key_id: 'example-openai-key' }
- : null,
- },
- },
- })
- return (
- <BindLogic logic={llmEvaluationLogic} props={{ evaluationId: 'example-evaluation' }}>
- <div className={narrow ? 'w-[32rem]' : 'max-w-4xl'}>
- <EvaluationModelPicker />
- </div>
- </BindLogic>
- )
- },
-}
-export default meta
-type Story = StoryObj<typeof meta>
-
-export const LLM: Story = { args: { method: 'llm' } }
-export const Decision: Story = { args: { method: 'decision' } }
-export const DecisionNarrow: Story = { args: { method: 'decision', narrow: true } }
-export const NoDecisionModels: Story = { args: { method: 'decision', empty: true } }
-export const NativeOpenAI: Story = { args: { method: 'decision', nativeOpenAI: true } }
-export const NativeOpenAINarrow: Story = { args: { method: 'decision', nativeOpenAI: true, narrow: true } }
-export const WithAttachments: Story = {
- args: { method: 'llm' },
- parameters: { featureFlags: [FEATURE_FLAGS.LLM_ANALYTICS_JUDGE_IMAGES], testOptions: { snapshotBrowsers: [] } },
-}
-export const ReasoningLevelNarrow: Story = {
- args: { method: 'llm', nativeOpenAI: true, narrow: true, reasoningLevel: 'medium' },
-}
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx
deleted file mode 100644
index 55b6aa67405e..000000000000
--- a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx
+++ /dev/null
@@ -1,101 +0,0 @@
-import { useActions, useValues } from 'kea'
-
-import { LemonSegmentedButton, LemonSwitch } from '@posthog/lemon-ui'
-
-import { FEATURE_FLAGS } from 'lib/constants'
-import { LemonField } from 'lib/lemon-ui/LemonField'
-import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
-
-import { modelPickerLogic } from '../../modelPickerLogic'
-import { booleanOutputConfigError, decisionModelHint } from '../constants'
-import { llmEvaluationLogic } from '../llmEvaluationLogic'
-import type { JudgeMethod } from '../types'
-import { DecisionThresholdInput } from './DecisionThresholdInput'
-import { EvaluationJudgeModelPicker } from './EvaluationJudgeModelPicker'
-
-export function EvaluationModelPicker(): JSX.Element {
- const { byokModelsLoading, providerKeysLoading } = useValues(modelPickerLogic)
- const { evaluation, usesDecisionModel, judgeMethod, includeMedia } = useValues(llmEvaluationLogic)
- const { setJudgeMethod, setIncludeMedia } = useActions(llmEvaluationLogic)
- const { featureFlags } = useValues(featureFlagLogic)
-
- const loading = byokModelsLoading || providerKeysLoading
-
- return (
- <div className="bg-bg-light border rounded p-4 @min-[40rem]/evaluation:p-6">
- <div className="@container/judge-model space-y-4">
- <div>
- <h3 className="text-lg font-semibold mb-2">Judge model</h3>
- <p className="text-muted text-sm mb-0">
- Choose how to judge responses, then select a model from your configured AI providers.
- </p>
- </div>
- <LemonField.Pure label="Judge method">
- <LemonSegmentedButton<JudgeMethod>
- value={judgeMethod}
- onChange={setJudgeMethod}
- className="max-w-full"
- disabledReason={loading ? 'Loading model capabilities.' : undefined}
- options={[
- {
- value: 'llm',
- label: (
- <span className="whitespace-normal @min-[24rem]/judge-model:whitespace-nowrap">
- LLM with explanation
- </span>
- ),
- },
- {
- value: 'decision',
- label: (
- <span className="whitespace-normal @min-[24rem]/judge-model:whitespace-nowrap">
- Decision model
- </span>
- ),
- },
- ]}
- />
- </LemonField.Pure>
- <LemonField.Pure label="Model">
- <div>
- <EvaluationJudgeModelPicker />
- {evaluation && usesDecisionModel && (
- <p className="text-sm text-muted mt-2">{decisionModelHint(evaluation.output_type)}</p>
- )}
- </div>
- </LemonField.Pure>
- {featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_JUDGE_IMAGES] &&
- evaluation?.evaluation_type === 'llm_judge' &&
- !usesDecisionModel && (
- <LemonField.Pure label="Attachments">
- <LemonSwitch
- checked={includeMedia}
- onChange={setIncludeMedia}
- bordered
- label="Send to the judge"
- data-attr="llma-evaluation-include-attachments"
- />
- <p className="text-muted text-sm">
- The judge sees the images in what it evaluates. Other file types are not supported yet.
- </p>
- </LemonField.Pure>
- )}
- {evaluation?.evaluation_type === 'llm_judge' &&
- evaluation.output_type === 'boolean' &&
- usesDecisionModel && (
- <LemonField.Pure
- label="Decision threshold"
- htmlFor="llma-evaluation-decision-threshold"
- error={booleanOutputConfigError(evaluation.output_config)}
- >
- <DecisionThresholdInput id="llma-evaluation-decision-threshold" />
- <p className="text-muted text-sm">
- A probability at or above this threshold produces a true result. Applies to new runs;
- past results stay unchanged.
- </p>
- </LemonField.Pure>
- )}
- </div>
- </div>
- )
-}
diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx
index 238a9b4798ed..c3aebb6213d1 100644
--- a/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx
+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx
@@ -1,7 +1,6 @@
import { Meta, StoryObj } from '@storybook/react'
import { waitFor, within } from '@testing-library/dom'
-import { FEATURE_FLAGS } from 'lib/constants'
import { App } from 'scenes/App'
import { urls } from 'scenes/urls'
@@ -38,7 +37,6 @@ const meta: Meta = {
viewMode: 'story',
mockDate: '2026-10-20',
pageUrl: urls.aiObservabilityEvaluation('new'),
- featureFlags: [FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW],
testOptions: {
waitForSelector: '[data-attr="llma-evaluation-create-step-body-name"]',
},
diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx
index 1b8fafb3df55..3dee7c4c5f35 100644
--- a/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx
+++ b/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx
@@ -1,4 +1,10 @@
-import { useValues } from 'kea'
+import { useActions, useValues } from 'kea'
+
+import { LemonSwitch } from '@posthog/lemon-ui'
+
+import { FEATURE_FLAGS } from 'lib/constants'
+import { LemonField } from 'lib/lemon-ui/LemonField'
+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { EvaluationJudgeModelPicker } from '../components/EvaluationJudgeModelPicker'
import { decisionModelHint } from '../constants'
@@ -6,21 +12,39 @@ import { llmEvaluationLogic } from '../llmEvaluationLogic'
/** The model a judge or decision model runs on, from the team's AI providers, with what that model does. */
export function EvaluationModelPanel(): JSX.Element | null {
- const { evaluation, usesDecisionModel } = useValues(llmEvaluationLogic)
+ const { evaluation, usesDecisionModel, includeMedia } = useValues(llmEvaluationLogic)
+ const { setIncludeMedia } = useActions(llmEvaluationLogic)
+ const { featureFlags } = useValues(featureFlagLogic)
if (!evaluation) {
return null
}
return (
- <div className="flex flex-wrap items-start gap-x-6 gap-y-2">
- <div className="w-80 max-w-full">
- <EvaluationJudgeModelPicker />
+ <div className="flex flex-col gap-4">
+ <div className="flex flex-wrap items-start gap-x-6 gap-y-2">
+ <div className="w-80 max-w-full">
+ <EvaluationJudgeModelPicker />
+ </div>
+ <p className="mb-0 flex-1 basis-64 text-sm text-secondary">
+ {usesDecisionModel
+ ? decisionModelHint(evaluation.output_type)
+ : 'Runs on your own AI provider key. Each run is one call to this model.'}
+ </p>
</div>
- <p className="mb-0 flex-1 basis-64 text-sm text-secondary">
- {usesDecisionModel
- ? decisionModelHint(evaluation.output_type)
- : 'Runs on your own AI provider key. Each run is one call to this model.'}
- </p>
+ {featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_JUDGE_IMAGES] && !usesDecisionModel && (
+ <LemonField.Pure label="Attachments">
+ <LemonSwitch
+ checked={includeMedia}
+ onChange={setIncludeMedia}
+ bordered
+ label="Send to the judge"
+ data-attr="llma-evaluation-include-attachments"
+ />
+ <p className="text-muted text-sm">
+ The judge sees the images in what it evaluates. Other file types are not supported yet.
+ </p>
+ </LemonField.Pure>
+ )}
</div>
)
}
diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationOutputFields.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationOutputFields.tsx
index a476342a91f2..5f83f44d8683 100644
--- a/products/ai_observability/frontend/evaluations/create/EvaluationOutputFields.tsx
+++ b/products/ai_observability/frontend/evaluations/create/EvaluationOutputFields.tsx
@@ -83,10 +83,9 @@ export function EvaluationOutputFields(): JSX.Element | null {
value={config.true_is_failure ? 'fail' : 'pass'}
onChange={(value) => setTrueIsFailure(value === 'fail')}
options={[
- { value: 'pass', label: 'Pass' },
- { value: 'fail', label: 'Fail' },
+ { value: 'pass', label: 'Pass', 'data-attr': 'llma-evaluation-true-is-pass' },
+ { value: 'fail', label: 'Fail', 'data-attr': 'llma-evaluation-true-is-failure' },
]}
- data-attr="llma-evaluation-true-is-failure-switch"
/>
</>
)}
diff --git a/products/ai_observability/frontend/evaluations/evaluationCapabilities.ts b/products/ai_observability/frontend/evaluations/evaluationCapabilities.ts
index 33c90a8111bb..9935ef4faada 100644
--- a/products/ai_observability/frontend/evaluations/evaluationCapabilities.ts
+++ b/products/ai_observability/frontend/evaluations/evaluationCapabilities.ts
@@ -93,7 +93,3 @@ export function evaluationCanResolveModel(
export function evaluationTypeDefaultsToBooleanOutput(evaluationType: EvaluationType | null | undefined): boolean {
return evaluationType === 'llm_judge' || evaluationType === 'hog'
}
-
-export function evaluationTypeHasEditableCriteria(evaluationType: EvaluationType | null | undefined): boolean {
- return evaluationTypeDefaultsToBooleanOutput(evaluationType)
-}
diff --git a/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.test.ts b/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.test.ts
deleted file mode 100644
index 8e6919168a8e..000000000000
--- a/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.test.ts
+++ /dev/null
@@ -1,340 +0,0 @@
-import { waitFor } from '@testing-library/react'
-import { combineUrl, router } from 'kea-router'
-import { expectLogic } from 'kea-test-utils'
-
-import { urls } from 'scenes/urls'
-
-import { useMocks } from '~/mocks/jest'
-import { initKeaTests } from '~/test/init'
-
-import {
- EVALUATION_BOOLEAN_GRADED_HOGQL,
- EVALUATION_NOT_SKIPPED_HOGQL,
- EVALUATION_NUMERIC_GRADED_HOGQL,
- EVALUATION_RESULT_TRUE_HOGQL,
-} from './constants'
-import { evaluationMetricsLogic, EvaluationStatsRow } from './evaluationMetricsLogic'
-import { llmEvaluationsLogic } from './llmEvaluationsLogic'
-import { LLMJudgeEvaluation } from './types'
-
-const queryMock = jest.fn().mockResolvedValue({ results: [] })
-
-jest.mock('lib/api', () => {
- const actual = jest.requireActual('lib/api')
-
- return {
- __esModule: true,
- ...actual,
- default: {
- ...actual.default,
- query: (...args: unknown[]) => queryMock(...args),
- },
- }
-})
-
-const evaluation = (id: string, directoryId: string | null, name = `Evaluation ${id}`): LLMJudgeEvaluation => ({
- id,
- name,
- description: '',
- directory_id: directoryId,
- enabled: true,
- status: 'active',
- status_reason: null,
- status_reason_detail: null,
- evaluation_type: 'llm_judge',
- evaluation_config: { prompt: 'Prompt' },
- output_type: 'boolean',
- output_config: {},
- conditions: [{ id: `condition-${id}`, rollout_percentage: 100, properties: [] }],
- target: 'generation',
- target_config: {},
- model_configuration: null,
- total_runs: 0,
- created_at: '2024-01-01T00:00:00Z',
- updated_at: '2024-01-01T00:00:00Z',
-})
-
-const stats = (evaluationId: string, runsCount: number, trueCount = runsCount): EvaluationStatsRow => ({
- evaluation_id: evaluationId,
- runs_count: runsCount,
- applicable_count: runsCount,
- true_count: trueCount,
- applicability_rate: 100,
-})
-
-describe('evaluationMetricsLogic', () => {
- let evaluationsLogic: ReturnType<typeof llmEvaluationsLogic.build>
- let metricsLogic: ReturnType<typeof evaluationMetricsLogic.build>
-
- beforeEach(async () => {
- useMocks({
- get: {
- '/api/environments/:teamId/llm_analytics/provider_keys/': { results: [] },
- '/api/environments/:teamId/llm_analytics/evaluation_config/': {
- active_provider_key: null,
- created_at: '2024-01-01T00:00:00Z',
- updated_at: '2024-01-01T00:00:00Z',
- },
- '/api/projects/:teamId/evaluations/': { results: [] },
- '/api/projects/:teamId/evaluation_directories/': [],
- },
- })
- initKeaTests()
- evaluationsLogic = llmEvaluationsLogic()
- evaluationsLogic.mount()
- metricsLogic = evaluationMetricsLogic()
- metricsLogic.mount()
-
- await expectLogic(metricsLogic, () => metricsLogic.actions.loadStats()).toFinishAllListeners()
- })
-
- afterEach(() => {
- metricsLogic.unmount()
- evaluationsLogic.unmount()
- })
-
- it('includes categorical pass rates while excluding N/A from their denominator', async () => {
- evaluationsLogic.actions.loadEvaluationsSuccess([
- {
- ...evaluation('categories', null),
- output_type: 'categorical',
- output_config: {
- options: [{ key: 'resolved', label: 'Resolved' }],
- passing_rule: { categories: ['resolved'] },
- },
- },
- ])
- queryMock.mockResolvedValueOnce({ results: [['categories', 4, 0, 0, 0, null, 0, 3, 2]] })
- await expectLogic(metricsLogic, () => metricsLogic.actions.loadStats()).toFinishAllListeners()
- expect(metricsLogic.values.evaluationsWithMetrics[0].stats).toMatchObject({
- applicable_count: 3,
- pass_count: 2,
- pass_rate: (2 / 3) * 100,
- })
- expect(metricsLogic.values.summaryMetrics.overall_pass_rate).toBe(66.7)
- })
-
- it('loads stats once after evaluations finish loading on mount', async () => {
- metricsLogic.unmount()
- queryMock.mockClear()
- evaluationsLogic.actions.loadEvaluations()
- metricsLogic.mount()
- await waitFor(() => expect(queryMock).toHaveBeenCalledTimes(1))
- await expectLogic(metricsLogic).toFinishAllListeners()
- expect(queryMock).toHaveBeenCalledTimes(1)
- })
-
- it('keeps the latest stats when an older request finishes last', async () => {
- queryMock.mockClear()
- let resolveOld!: (response: { results: unknown[][] }) => void
- queryMock.mockImplementationOnce(
- () =>
- new Promise((resolve) => {
- resolveOld = resolve
- })
- )
- metricsLogic.actions.loadStats()
- await waitFor(() => expect(queryMock).toHaveBeenCalledTimes(1))
-
- queryMock.mockResolvedValueOnce({ results: [['new', 4, 0, 0, 4, 50, 2]] })
- evaluationsLogic.actions.loadEvaluationsSuccess([
- {
- ...evaluation('new', null),
- output_type: 'numeric',
- output_config: { passing_rule: { operator: 'gte', threshold: 50 } },
- },
- ])
- await waitFor(() => expect(metricsLogic.values.stats[0]?.evaluation_id).toBe('new'))
- expect(queryMock.mock.calls[1][0].query).toContain(
- ">= transform(properties.$ai_evaluation_id, ['new'], [50], toFloat(NULL))"
- )
-
- resolveOld({ results: [['old', 4, 0, 0, 4, 50, 0]] })
- await expectLogic(metricsLogic).toFinishAllListeners()
- expect(metricsLogic.values.stats[0].evaluation_id).toBe('new')
- })
-
- it.each(['gte', 'lte'] as const)('includes numeric %s outcomes in the overview pass rate', async (operator) => {
- evaluationsLogic.actions.loadEvaluationsSuccess([
- evaluation('boolean', null),
- {
- ...evaluation('numeric', null),
- output_type: 'numeric',
- output_config: { passing_rule: { operator, threshold: 0 } },
- },
- { ...evaluation('ungraded', null), output_type: 'numeric', output_config: {} },
- ])
- await expectLogic(metricsLogic).toFinishAllListeners()
- const query = queryMock.mock.calls.at(-1)?.[0].query
- expect(query).toContain(
- `${operator === 'gte' ? '>=' : '<='} transform(properties.$ai_evaluation_id, ['numeric'], [0], toFloat(NULL))`
- )
- expect(query).not.toContain("'ungraded'")
- metricsLogic.actions.loadStatsSuccess([
- stats('boolean', 10, 10),
- {
- ...stats('numeric', 100, 0),
- applicable_count: 0,
- score_count: 80,
- score_mean: 6.5,
- numeric_pass_count: 40,
- },
- { ...stats('ungraded', 20, 0), score_count: 20, numeric_pass_count: 0 },
- ])
- expect(metricsLogic.values.summaryMetrics).toMatchObject({
- total_runs: 130,
- overall_pass_rate: 55.6,
- failing_evaluations_count: 1,
- })
- expect(metricsLogic.values.evaluationsWithMetrics[1].stats).toMatchObject({
- score_mean: 6.5,
- pass_rate: 50,
- applicable_count: 80,
- })
- expect(metricsLogic.values.chartQuery?.series[0].properties).toEqual([
- expect.objectContaining({ value: ['boolean', 'numeric'] }),
- ])
- const math = metricsLogic.values.chartQuery?.series[0].math_hogql ?? ''
- expect(math).toContain(
- `toFloat(properties.$ai_evaluation_numeric_result) ${operator === 'gte' ? '>=' : '<='} 0`
- )
- expect(math).toContain("properties.$ai_evaluation_id = 'numeric'")
- expect(math).toContain('properties.$ai_evaluation_applicable')
- expect(math).toContain(EVALUATION_NOT_SKIPPED_HOGQL)
- })
-
- it('keeps thresholds aligned with their evaluation IDs and comparison directions', async () => {
- evaluationsLogic.actions.loadEvaluationsSuccess([
- ...(
- [
- { id: "quality'check", operator: 'gte', threshold: 7 },
- { id: 'cost', operator: 'lte', threshold: 0.05 },
- { id: 'accuracy', operator: 'gte', threshold: 0.9 },
- { id: 'invalid', operator: 'gte', threshold: NaN },
- { id: 'infinite', operator: 'lte', threshold: Infinity },
- ] as const
- ).map(({ id, operator, threshold }) => ({
- ...evaluation(id, null),
- output_type: 'numeric' as const,
- output_config: { passing_rule: { operator, threshold } },
- })),
- evaluation('boolean', null),
- ])
- await expectLogic(metricsLogic).toFinishAllListeners()
-
- const query = queryMock.mock.calls.at(-1)?.[0].query
- expect(query).toContain(
- ">= transform(properties.$ai_evaluation_id, ['quality\\'check', 'accuracy'], [7, 0.9], toFloat(NULL))"
- )
- expect(query).toContain("<= transform(properties.$ai_evaluation_id, ['cost'], [0.05], toFloat(NULL))")
- expect(query).not.toMatch(/'invalid'|'infinite'|NaN|Infinity/)
- expect(query).toContain(`AND ${EVALUATION_NUMERIC_GRADED_HOGQL}) as numeric_pass_count`)
- })
-
- it.each([false, true])('has no pass rate when numeric runs have no graded outcomes (rule: %s)', (hasRule) => {
- evaluationsLogic.actions.loadEvaluationsSuccess([
- {
- ...evaluation('numeric', null),
- output_type: 'numeric',
- output_config: hasRule ? { passing_rule: { operator: 'gte', threshold: 100 } } : {},
- },
- ])
- metricsLogic.actions.loadStatsSuccess([
- { ...stats('numeric', 4, 0), score_count: hasRule ? 0 : 4, numeric_pass_count: 0 },
- ])
- expect(metricsLogic.values.summaryMetrics).toEqual({
- total_runs: 4,
- overall_pass_rate: null,
- failing_evaluations_count: 0,
- })
- expect(metricsLogic.values.chartQuery !== null).toBe(hasRule)
- expect(metricsLogic.values.evaluationsWithMetrics[0].stats?.pass_rate).toBeNull()
- })
-
- it('uses one filtered breakdown query and scopes metrics to the selected directory', () => {
- const firstRootEvaluation = evaluation('root-one', null, "Root's evaluation")
- const secondRootEvaluation = evaluation('root-two', null, "Root's evaluation")
- const directoryEvaluation = evaluation('inside', 'directory-a')
-
- evaluationsLogic.actions.loadEvaluationsSuccess([
- firstRootEvaluation,
- secondRootEvaluation,
- directoryEvaluation,
- ])
- metricsLogic.actions.loadStatsSuccess([
- stats(firstRootEvaluation.id, 4),
- stats(secondRootEvaluation.id, 3),
- stats(directoryEvaluation.id, 2),
- ])
-
- expect(metricsLogic.values.chartQuery?.series).toEqual([
- expect.objectContaining({
- properties: [expect.objectContaining({ value: [firstRootEvaluation.id, secondRootEvaluation.id] })],
- }),
- ])
- expect(metricsLogic.values.chartQuery?.breakdownFilter).toEqual(
- expect.objectContaining({
- breakdown_type: 'hogql',
- breakdown_hide_other_aggregation: true,
- })
- )
- expect(metricsLogic.values.chartQuery?.breakdownFilter?.breakdown).toContain("'Root\\'s evaluation (1)'")
- expect(metricsLogic.values.chartQuery?.breakdownFilter?.breakdown).toContain("'Root\\'s evaluation (2)'")
- expect(metricsLogic.values.summaryMetrics.total_runs).toBe(7)
-
- router.actions.push(
- combineUrl(urls.aiObservabilityEvaluations(), {
- directory: 'directory-a',
- }).url
- )
-
- expect(metricsLogic.values.chartQuery?.series).toEqual([
- expect.objectContaining({
- properties: [expect.objectContaining({ value: [directoryEvaluation.id] })],
- }),
- ])
- expect(metricsLogic.values.chartQuery?.breakdownFilter?.breakdown).toContain(directoryEvaluation.name)
- expect(metricsLogic.values.chartQuery?.breakdownFilter?.breakdown).not.toContain(firstRootEvaluation.id)
- expect(metricsLogic.values.summaryMetrics.total_runs).toBe(2)
- })
-
- it('reads a detector pass rate from its false results', () => {
- const detector = { ...evaluation('detector', null), output_config: { true_is_failure: true } }
- const quality = evaluation('quality', null)
-
- evaluationsLogic.actions.loadEvaluationsSuccess([detector, quality])
- metricsLogic.actions.loadStatsSuccess([stats('detector', 100, 80), stats('quality', 100, 80)])
-
- expect(metricsLogic.values.evaluationsWithMetrics).toEqual(
- expect.arrayContaining([
- expect.objectContaining({ id: 'detector', stats: expect.objectContaining({ pass_rate: 20 }) }),
- expect.objectContaining({ id: 'quality', stats: expect.objectContaining({ pass_rate: 80 }) }),
- ])
- )
- })
-
- it('scopes the chart pass expression to a detector among the enabled evaluations', () => {
- const detector = { ...evaluation('detector', null), output_config: { true_is_failure: true } }
- const quality = evaluation('quality', null)
-
- evaluationsLogic.actions.loadEvaluationsSuccess([detector, quality])
- metricsLogic.actions.loadStatsSuccess([stats('detector', 10, 2), stats('quality', 10, 8)])
-
- const mathHogql = metricsLogic.values.chartQuery?.series?.[0].math_hogql ?? ''
- expect(mathHogql).toContain("properties.$ai_evaluation_id IN ('detector')")
- expect(mathHogql).toContain("properties.$ai_evaluation_result = 'false'")
- // Both sides of the ratio drop skipped runs — otherwise the false a skip stores reads as
- // a detector pass, and the chart disagrees with the list, which already excludes them.
- expect(mathHogql).toContain(`AND ${EVALUATION_NOT_SKIPPED_HOGQL})) /`)
- expect(mathHogql).toContain(`IS NOT NULL AND ${EVALUATION_NOT_SKIPPED_HOGQL}`)
- })
-
- it('excludes skipped runs from both counts, so a detector cannot count them as a pass', async () => {
- await expectLogic(metricsLogic, () => metricsLogic.actions.loadStats()).toFinishAllListeners()
-
- const query = queryMock.mock.calls.at(-1)?.[0]
- expect(query.query).toContain(`IS NOT NULL AND ${EVALUATION_NOT_SKIPPED_HOGQL}`)
- expect(query.query).toContain(`${EVALUATION_RESULT_TRUE_HOGQL} AND ${EVALUATION_BOOLEAN_GRADED_HOGQL}`)
- expect(query.query).toContain(`countIf((false) AND ${EVALUATION_NUMERIC_GRADED_HOGQL})`)
- })
-})
diff --git a/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.ts b/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.ts
deleted file mode 100644
index 3a5c9049d2f9..000000000000
--- a/products/ai_observability/frontend/evaluations/evaluationMetricsLogic.ts
+++ /dev/null
@@ -1,507 +0,0 @@
-import { MakeLogicType, actions, afterMount, connect, kea, listeners, path, props, selectors } from 'kea'
-import { loaders } from 'kea-loaders'
-
-import api from 'lib/api'
-import { dayjs } from 'lib/dayjs'
-
-import { HogQLQuery, NodeKind, TrendsQuery } from '~/queries/schema/schema-general'
-import { escapeHogQLString } from '~/queries/utils'
-import { ChartDisplayType, HogQLMathType, PropertyFilterType, PropertyOperator } from '~/types'
-
-// eslint-disable-next-line import/no-cycle
-import { PASS_RATE_SUCCESS_THRESHOLD } from './components/EvaluationMetrics'
-import {
- EVALUATION_BOOLEAN_GRADED_HOGQL,
- EVALUATION_NUMERIC_GRADED_HOGQL,
- EVALUATION_NUMERIC_MEAN_HOGQL,
- numericEvaluationPassedHogQL,
- categoricalEvaluationsPassedHogQL,
- EVALUATION_CATEGORICAL_GRADED_HOGQL,
- EVALUATION_RESULT_TRUE_HOGQL,
- evaluationIsDetector,
- evaluationPassedHogQLForMany,
- evaluationPassRateHogQL,
-} from './constants'
-import { llmEvaluationsLogic, waitForEvaluationsSettled } from './llmEvaluationsLogic'
-import { EvaluationConfig } from './types'
-
-const MIN_RUNS_FOR_FAILING_STATUS = 3
-const MAX_EVALUATION_CHART_SERIES = 10
-
-/** One row from the stats query. Polarity-free by design — the query never claims a pass. */
-export interface EvaluationStatsRow {
- evaluation_id: string
- runs_count: number
- applicable_count: number
- true_count: number
- applicability_rate: number
- score_count?: number
- score_mean?: number | null
- categorical_count?: number
- categorical_pass_count?: number
- numeric_pass_count?: number
-}
-
-/** A stats row with the evaluation's own polarity applied. */
-export interface EvaluationStats extends EvaluationStatsRow {
- pass_count: number
- pass_rate: number | null
-}
-
-export interface SummaryMetrics {
- total_runs: number
- overall_pass_rate: number | null
- failing_evaluations_count: number
-}
-
-type RawStatsRow = [
- evaluation_id: string,
- runs_count: number,
- applicable_count: number,
- true_count: number,
- score_count?: number,
- score_mean?: number | null,
- numeric_pass_count?: number,
- categorical_count?: number,
- categorical_pass_count?: number,
-]
-
-function hasPassRate(evaluation: EvaluationConfig): boolean {
- return (
- evaluation.output_type === 'boolean' ||
- (['numeric', 'categorical'].includes(evaluation.output_type) && !!evaluation.output_config.passing_rule)
- )
-}
-
-export type EvaluationMetricsLogicProps = Record<string, never>
-
-function getIntervalFromDateRange(dateFrom: string | null): 'hour' | 'day' {
- if (!dateFrom) {
- return 'day'
- }
-
- // Handle "today" formats
- if (dateFrom === 'dStart' || dateFrom === '-0d' || dateFrom === '-0dStart') {
- return 'hour'
- }
-
- // Handle relative date strings like "-24h", "-1d", "-7d"
- const match = dateFrom.match(/^-(\d+)([hdwmy])/i)
- if (match) {
- const value = parseInt(match[1])
- const unit = match[2].toLowerCase()
- const hoursMap: Record<string, number> = { h: 1, d: 24, w: 168, m: 720, y: 8760 }
- const hours = value * (hoursMap[unit] || 24)
- return hours <= 24 ? 'hour' : 'day'
- }
-
- // Handle absolute dates
- const duration = dayjs.duration(dayjs().diff(dayjs(dateFrom)))
- return duration.asDays() <= 1 ? 'hour' : 'day'
-}
-
-function buildEvaluationBreakdownExpression(evaluations: EvaluationConfig[]): string {
- const evaluationNameCounts = new Map<string, number>()
- const evaluationNameIndexes = new Map<string, number>()
-
- for (const evaluation of evaluations) {
- evaluationNameCounts.set(evaluation.name, (evaluationNameCounts.get(evaluation.name) ?? 0) + 1)
- }
-
- const conditionsAndLabels = evaluations.flatMap((evaluation) => {
- const nameIndex = (evaluationNameIndexes.get(evaluation.name) ?? 0) + 1
- evaluationNameIndexes.set(evaluation.name, nameIndex)
- const label =
- evaluationNameCounts.get(evaluation.name) === 1 ? evaluation.name : `${evaluation.name} (${nameIndex})`
-
- return [`properties.$ai_evaluation_id = ${escapeHogQLString(evaluation.id)}`, escapeHogQLString(label)]
- })
-
- return `multiIf(${conditionsAndLabels.join(', ')}, toString(properties.$ai_evaluation_id))`
-}
-
-/** Pass rate over time with one line per evaluation, or null when none of them has a pass rate. Callers pick which evaluations to chart. */
-export function buildEvaluationPassRateTrendsQuery(
- evaluations: EvaluationConfig[],
- dateFilter: { dateFrom: string | null; dateTo: string | null },
- intervalOverride?: 'hour' | 'day'
-): TrendsQuery | null {
- const charted = evaluations
- .filter((evaluation) => !evaluation.deleted && hasPassRate(evaluation))
- .slice(0, MAX_EVALUATION_CHART_SERIES)
-
- if (charted.length === 0) {
- return null
- }
-
- const dateFrom = dateFilter.dateFrom || '-7d'
- const dateTo = dateFilter.dateTo || null
- const interval = intervalOverride ?? getIntervalFromDateRange(dateFrom)
- const numericEvaluations = charted.filter((evaluation) => evaluation.output_type === 'numeric')
- let passedExpression = evaluationPassedHogQLForMany(
- charted.filter(evaluationIsDetector).map((evaluation) => evaluation.id)
- )
- let gradedExpression = EVALUATION_BOOLEAN_GRADED_HOGQL
- if (numericEvaluations.length > 0) {
- const numericCases = numericEvaluations.flatMap((evaluation) => [
- `properties.$ai_evaluation_id = ${escapeHogQLString(evaluation.id)}`,
- numericEvaluationPassedHogQL(evaluation),
- ])
- const numericIds = numericEvaluations.map((evaluation) => escapeHogQLString(evaluation.id)).join(', ')
- passedExpression = `multiIf(${numericCases.join(', ')}, ${passedExpression})`
- gradedExpression = `if(properties.$ai_evaluation_id IN (${numericIds}), ${EVALUATION_NUMERIC_GRADED_HOGQL}, ${EVALUATION_BOOLEAN_GRADED_HOGQL})`
- }
-
- if (charted.some((evaluation) => evaluation.output_type === 'categorical')) {
- passedExpression = `if(properties.$ai_evaluation_result_type = 'categorical', ${categoricalEvaluationsPassedHogQL(charted)}, ${passedExpression})`
- gradedExpression = `if(properties.$ai_evaluation_result_type = 'categorical', ${EVALUATION_CATEGORICAL_GRADED_HOGQL}, ${gradedExpression})`
- }
-
- return {
- kind: NodeKind.TrendsQuery,
- series: [
- {
- kind: NodeKind.EventsNode,
- event: '$ai_evaluation',
- math: HogQLMathType.HogQL,
- math_hogql: evaluationPassRateHogQL(passedExpression, gradedExpression),
- properties: [
- {
- key: '$ai_evaluation_id',
- value: charted.map((evaluation) => evaluation.id),
- operator: PropertyOperator.Exact,
- type: PropertyFilterType.Event,
- },
- ],
- },
- ],
- breakdownFilter: {
- breakdown: buildEvaluationBreakdownExpression(charted),
- breakdown_type: 'hogql',
- breakdown_limit: MAX_EVALUATION_CHART_SERIES,
- breakdown_hide_other_aggregation: true,
- },
- trendsFilter: {
- display: ChartDisplayType.ActionsLineGraph,
- },
- dateRange: {
- date_from: dateFrom,
- date_to: dateTo,
- },
- interval,
- }
-}
-
-// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
-export interface evaluationMetricsLogicValues {
- dateFilter: {
- dateFrom: string | null
- dateTo: string | null
- } // llmEvaluationsLogic
- evaluations: EvaluationConfig[] // llmEvaluationsLogic
- selectedDirectoryId: string | null // llmEvaluationsLogic
- chartQuery: TrendsQuery | null
- evaluationsForMetrics: EvaluationConfig[]
- evaluationsWithMetrics: Array<
- EvaluationConfig & {
- stats?: EvaluationStats
- }
- >
- stats: EvaluationStatsRow[]
- statsLoading: boolean
- summaryMetrics: SummaryMetrics
-}
-
-// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
-export interface evaluationMetricsLogicActions {
- setDates: (
- dateFrom: string | null,
- dateTo: string | null
- ) => {
- dateFrom: string | null
- dateTo: string | null
- } // llmEvaluationsLogic
- loadStats: (_?: void) => void
- loadStatsFailure: (
- error: string,
- errorObject?: any
- ) => {
- error: string
- errorObject?: any
- }
- loadStatsSuccess: (
- stats: EvaluationStatsRow[],
- payload?: void
- ) => {
- stats: EvaluationStatsRow[]
- payload?: void
- }
- refreshMetrics: () => {
- value: true
- }
-}
-
-// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
-export interface evaluationMetricsLogicMeta {
- __keaTypeGenInternalSelectorTypes: {
- evaluationsForMetrics: (
- evaluations: EvaluationConfig[],
- selectedDirectoryId: string | null
- ) => EvaluationConfig[]
- evaluationsWithMetrics: (
- evaluations: EvaluationConfig[],
- stats: EvaluationStatsRow[]
- ) => Array<
- EvaluationConfig & {
- stats?: EvaluationStats
- }
- >
- summaryMetrics: (
- evaluationsWithMetrics: (EvaluationConfig & {
- stats?: EvaluationStats
- })[],
- evaluationsForMetrics: EvaluationConfig[]
- ) => SummaryMetrics
- chartQuery: (
- evaluationsForMetrics: EvaluationConfig[],
- dateFilter: {
- dateFrom: string | null
- dateTo: string | null
- }
- ) => TrendsQuery | null
- }
-}
-
-export type evaluationMetricsLogicType = MakeLogicType<
- evaluationMetricsLogicValues,
- evaluationMetricsLogicActions,
- EvaluationMetricsLogicProps,
- evaluationMetricsLogicMeta
->
-
-export const evaluationMetricsLogic = kea<evaluationMetricsLogicType>([
- path(['products', 'ai_observability', 'frontend', 'evaluations', 'evaluationMetricsLogic']),
- props({} as EvaluationMetricsLogicProps),
-
- connect(() => ({
- values: [llmEvaluationsLogic, ['evaluations', 'dateFilter', 'selectedDirectoryId']],
- actions: [llmEvaluationsLogic, ['setDates']],
- })),
-
- actions({
- refreshMetrics: true,
- }),
-
- loaders(({ values }) => ({
- stats: [
- [] as EvaluationStatsRow[],
- {
- loadStats: async (_?: void, breakpoint?: () => void): Promise<EvaluationStatsRow[]> => {
- await waitForEvaluationsSettled()
- breakpoint?.()
- const dateFrom = values.dateFilter.dateFrom || '-1d'
- const dateTo = values.dateFilter.dateTo || null
-
- // Threshold lookups avoid evaluating a separate condition for every evaluation on each run.
- const numericPass =
- (['gte', 'lte'] as const)
- .flatMap((operator) => {
- const rules = values.evaluations.flatMap((evaluation) => {
- const rule = evaluation.output_config.passing_rule
- return evaluation.output_type === 'numeric' &&
- rule &&
- 'threshold' in rule &&
- rule.operator === operator &&
- Number.isFinite(rule.threshold)
- ? [{ id: evaluation.id, threshold: rule.threshold }]
- : []
- })
- if (rules.length === 0) {
- return []
- }
- const ids = rules.map(({ id }) => escapeHogQLString(id)).join(', ')
- const thresholds = rules.map(({ threshold }) => threshold).join(', ')
- return [
- `toFloat(properties.$ai_evaluation_numeric_result) ${operator === 'gte' ? '>=' : '<='} transform(properties.$ai_evaluation_id, [${ids}], [${thresholds}], toFloat(NULL))`,
- ]
- })
- .join(' OR ') || 'false'
- const query: HogQLQuery = {
- kind: NodeKind.HogQLQuery,
- query: `
- SELECT
- properties.$ai_evaluation_id as evaluation_id,
- count() as runs_count,
- countIf(${EVALUATION_BOOLEAN_GRADED_HOGQL}) as applicable_count,
- countIf(${EVALUATION_RESULT_TRUE_HOGQL} AND ${EVALUATION_BOOLEAN_GRADED_HOGQL}) as true_count,
- countIf(${EVALUATION_NUMERIC_GRADED_HOGQL}) as score_count,
- ${EVALUATION_NUMERIC_MEAN_HOGQL} as score_mean,
- countIf((${numericPass}) AND ${EVALUATION_NUMERIC_GRADED_HOGQL}) as numeric_pass_count,
- countIf(${EVALUATION_CATEGORICAL_GRADED_HOGQL}) as categorical_count,
- countIf((${categoricalEvaluationsPassedHogQL(values.evaluations)}) AND ${EVALUATION_CATEGORICAL_GRADED_HOGQL}) as categorical_pass_count
- FROM events
- WHERE event = '$ai_evaluation' AND {filters}
- GROUP BY evaluation_id
- `,
- filters: {
- dateRange: {
- date_from: dateFrom,
- date_to: dateTo,
- },
- },
- }
-
- try {
- const response = await api.query(query)
- breakpoint?.()
-
- return (response.results || []).map((row: RawStatsRow) => {
- const runs_count = row[1]
- const applicable_count = row[2]
- const true_count = row[3]
- const applicability_rate = runs_count > 0 ? (applicable_count / runs_count) * 100 : 0
-
- return {
- evaluation_id: row[0],
- score_count: row[4] ?? 0,
- score_mean: row[4] && row[5] != null && Number.isFinite(row[5]) ? row[5] : null,
- numeric_pass_count: row[6] ?? 0,
- categorical_count: row[7] ?? 0,
- categorical_pass_count: row[8] ?? 0,
- runs_count,
- applicable_count,
- true_count,
- applicability_rate: Math.round(applicability_rate * 10) / 10,
- }
- })
- } catch (error) {
- breakpoint?.()
- console.error('Failed to load stats:', error)
- return []
- }
- },
- },
- ],
- })),
-
- selectors({
- evaluationsForMetrics: [
- (s) => [s.evaluations, s.selectedDirectoryId],
- (evaluations: EvaluationConfig[], selectedDirectoryId: string | null): EvaluationConfig[] =>
- evaluations.filter((evaluation) => (evaluation.directory_id ?? null) === selectedDirectoryId),
- ],
- evaluationsWithMetrics: [
- (s) => [s.evaluations, s.stats],
- (
- evaluations: EvaluationConfig[],
- stats: EvaluationStatsRow[]
- ): Array<EvaluationConfig & { stats?: EvaluationStats }> => {
- const statsMap = new Map(stats.map((stat) => [stat.evaluation_id, stat]))
-
- return evaluations.map((evaluation) => {
- const stat = statsMap.get(evaluation.id)
- if (!stat) {
- return { ...evaluation }
- }
- if (evaluation.output_type === 'numeric' || evaluation.output_type === 'categorical') {
- const applicable_count =
- (evaluation.output_type === 'categorical' ? stat.categorical_count : stat.score_count) ?? 0
- const pass_count =
- (evaluation.output_type === 'categorical'
- ? stat.categorical_pass_count
- : stat.numeric_pass_count) ?? 0
- return {
- ...evaluation,
- stats: {
- ...stat,
- applicable_count,
- pass_count,
- applicability_rate: stat.runs_count ? (applicable_count / stat.runs_count) * 100 : 0,
- pass_rate:
- applicable_count && evaluation.output_config.passing_rule
- ? (pass_count / applicable_count) * 100
- : null,
- },
- }
- }
- // Pass rate excludes N/A results (uses applicable_count as denominator)
- const pass_count = evaluationIsDetector(evaluation)
- ? stat.applicable_count - stat.true_count
- : stat.true_count
- const pass_rate =
- stat.applicable_count > 0 ? Math.round((pass_count / stat.applicable_count) * 1000) / 10 : null
- return { ...evaluation, stats: { ...stat, pass_count, pass_rate } }
- })
- },
- ],
-
- summaryMetrics: [
- (s) => [s.evaluationsWithMetrics, s.evaluationsForMetrics],
- (
- evaluationsWithMetrics: Array<EvaluationConfig & { stats?: EvaluationStats }>,
- evaluationsForMetrics: EvaluationConfig[]
- ): SummaryMetrics => {
- const ids = new Set(evaluationsForMetrics.map((evaluation) => evaluation.id))
- const statsForMetrics = evaluationsWithMetrics
- .filter((evaluation) => ids.has(evaluation.id) && hasPassRate(evaluation))
- .map((evaluation) => evaluation.stats)
- .filter((stat): stat is EvaluationStats => stat != null)
-
- const total_runs = evaluationsWithMetrics
- .filter((evaluation) => ids.has(evaluation.id))
- .reduce((sum, evaluation) => sum + (evaluation.stats?.runs_count ?? 0), 0)
- const total_applicable = statsForMetrics.reduce((sum, stat) => sum + stat.applicable_count, 0)
- const total_passes = statsForMetrics.reduce((sum, stat) => sum + stat.pass_count, 0)
- // Overall pass rate excludes N/A results
- // Keep an unmeasured pass rate distinct from measured zero passes.
- const overall_pass_rate =
- total_applicable > 0 ? Math.round((total_passes / total_applicable) * 1000) / 10 : null
-
- const failing_count = statsForMetrics.filter((stat) => {
- // Use applicable_count for minimum runs check
- return (
- stat.applicable_count >= MIN_RUNS_FOR_FAILING_STATUS &&
- stat.pass_rate != null &&
- stat.pass_rate < PASS_RATE_SUCCESS_THRESHOLD
- )
- }).length
-
- return {
- total_runs,
- overall_pass_rate,
- failing_evaluations_count: failing_count,
- }
- },
- ],
-
- chartQuery: [
- (s) => [s.evaluationsForMetrics, s.dateFilter],
- (
- evaluationsForMetrics: EvaluationConfig[],
- dateFilter: { dateFrom: string | null; dateTo: string | null }
- ): TrendsQuery | null =>
- buildEvaluationPassRateTrendsQuery(
- evaluationsForMetrics.filter((evaluation) => evaluation.enabled),
- dateFilter
- ),
- ],
- }),
-
- listeners(({ actions }) => ({
- refreshMetrics: () => {
- actions.loadStats()
- },
- [llmEvaluationsLogic.actionTypes.loadEvaluationsSuccess]: () => {
- actions.loadStats()
- },
- setDates: () => {
- actions.loadStats()
- },
- })),
-
- afterMount(({ actions }) => {
- if (llmEvaluationsLogic.values.evaluationsSettled) {
- actions.loadStats()
- }
- }),
-])
diff --git a/products/ai_observability/frontend/evaluations/evaluationPassRateTrendsQuery.test.ts b/products/ai_observability/frontend/evaluations/evaluationPassRateTrendsQuery.test.ts
new file mode 100644
index 000000000000..fd8871166c10
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/evaluationPassRateTrendsQuery.test.ts
@@ -0,0 +1,86 @@
+import { EVALUATION_NOT_SKIPPED_HOGQL } from './constants'
+import { buildEvaluationPassRateTrendsQuery } from './evaluationPassRateTrendsQuery'
+import { LLMJudgeEvaluation } from './types'
+
+const evaluation = (id: string, name = `Evaluation ${id}`): LLMJudgeEvaluation => ({
+ id,
+ name,
+ description: '',
+ directory_id: null,
+ enabled: true,
+ status: 'active',
+ status_reason: null,
+ status_reason_detail: null,
+ evaluation_type: 'llm_judge',
+ evaluation_config: { prompt: 'Prompt' },
+ output_type: 'boolean',
+ output_config: {},
+ conditions: [{ id: `condition-${id}`, rollout_percentage: 100, properties: [] }],
+ target: 'generation',
+ target_config: {},
+ model_configuration: null,
+ total_runs: 0,
+ created_at: '2024-01-01T00:00:00Z',
+ updated_at: '2024-01-01T00:00:00Z',
+})
+
+const PERIOD = { dateFrom: '-7d', dateTo: null }
+
+describe('buildEvaluationPassRateTrendsQuery', () => {
+ it.each(['gte', 'lte'] as const)(
+ 'charts numeric %s pass rules and skips numeric evaluations without one',
+ (operator) => {
+ const query = buildEvaluationPassRateTrendsQuery(
+ [
+ evaluation('boolean'),
+ {
+ ...evaluation('numeric'),
+ output_type: 'numeric',
+ output_config: { passing_rule: { operator, threshold: 0 } },
+ },
+ { ...evaluation('ungraded'), output_type: 'numeric', output_config: {} },
+ ],
+ PERIOD,
+ 'day'
+ )
+
+ expect(query?.series[0].properties).toEqual([expect.objectContaining({ value: ['boolean', 'numeric'] })])
+ const math = query?.series[0].math_hogql ?? ''
+ expect(math).toContain(
+ `toFloat(properties.$ai_evaluation_numeric_result) ${operator === 'gte' ? '>=' : '<='} 0`
+ )
+ expect(math).toContain("properties.$ai_evaluation_id = 'numeric'")
+ expect(math).toContain(EVALUATION_NOT_SKIPPED_HOGQL)
+ }
+ )
+
+ it('returns null when no evaluation has a pass rate', () => {
+ const ungraded = { ...evaluation('numeric'), output_type: 'numeric' as const, output_config: {} }
+ expect(buildEvaluationPassRateTrendsQuery([ungraded], PERIOD, 'day')).toBeNull()
+ })
+
+ it('labels evaluations that share a name apart in one breakdown', () => {
+ const query = buildEvaluationPassRateTrendsQuery(
+ [evaluation('root-one', "Root's evaluation"), evaluation('root-two', "Root's evaluation")],
+ PERIOD,
+ 'day'
+ )
+
+ expect(query?.breakdownFilter).toEqual(
+ expect.objectContaining({ breakdown_type: 'hogql', breakdown_hide_other_aggregation: true })
+ )
+ expect(query?.breakdownFilter?.breakdown).toContain("'Root\\'s evaluation (1)'")
+ expect(query?.breakdownFilter?.breakdown).toContain("'Root\\'s evaluation (2)'")
+ })
+
+ it('reads a detector pass from its false results and drops skipped runs from both counts', () => {
+ const detector = { ...evaluation('detector'), output_config: { true_is_failure: true } }
+ const math = buildEvaluationPassRateTrendsQuery([detector, evaluation('quality')], PERIOD, 'day')?.series[0]
+ .math_hogql
+
+ expect(math).toContain("properties.$ai_evaluation_id IN ('detector')")
+ expect(math).toContain("properties.$ai_evaluation_result = 'false'")
+ expect(math).toContain(`AND ${EVALUATION_NOT_SKIPPED_HOGQL})) /`)
+ expect(math).toContain(`IS NOT NULL AND ${EVALUATION_NOT_SKIPPED_HOGQL}`)
+ })
+})
diff --git a/products/ai_observability/frontend/evaluations/evaluationPassRateTrendsQuery.ts b/products/ai_observability/frontend/evaluations/evaluationPassRateTrendsQuery.ts
new file mode 100644
index 000000000000..7caccf663cee
--- /dev/null
+++ b/products/ai_observability/frontend/evaluations/evaluationPassRateTrendsQuery.ts
@@ -0,0 +1,115 @@
+import { NodeKind, TrendsQuery } from '~/queries/schema/schema-general'
+import { escapeHogQLString } from '~/queries/utils'
+import { ChartDisplayType, HogQLMathType, PropertyFilterType, PropertyOperator } from '~/types'
+
+import {
+ EVALUATION_BOOLEAN_GRADED_HOGQL,
+ EVALUATION_CATEGORICAL_GRADED_HOGQL,
+ EVALUATION_NUMERIC_GRADED_HOGQL,
+ categoricalEvaluationsPassedHogQL,
+ evaluationIsDetector,
+ evaluationPassedHogQLForMany,
+ evaluationPassRateHogQL,
+ numericEvaluationPassedHogQL,
+} from './constants'
+import { EvaluationConfig } from './types'
+
+const MAX_EVALUATION_CHART_SERIES = 10
+
+function hasPassRate(evaluation: EvaluationConfig): boolean {
+ return (
+ evaluation.output_type === 'boolean' ||
+ (['numeric', 'categorical'].includes(evaluation.output_type) && !!evaluation.output_config.passing_rule)
+ )
+}
+
+function buildEvaluationBreakdownExpression(evaluations: EvaluationConfig[]): string {
+ const evaluationNameCounts = new Map<string, number>()
+ const evaluationNameIndexes = new Map<string, number>()
+
+ for (const evaluation of evaluations) {
+ evaluationNameCounts.set(evaluation.name, (evaluationNameCounts.get(evaluation.name) ?? 0) + 1)
+ }
+
+ const conditionsAndLabels = evaluations.flatMap((evaluation) => {
+ const nameIndex = (evaluationNameIndexes.get(evaluation.name) ?? 0) + 1
+ evaluationNameIndexes.set(evaluation.name, nameIndex)
+ const label =
+ evaluationNameCounts.get(evaluation.name) === 1 ? evaluation.name : `${evaluation.name} (${nameIndex})`
+
+ return [`properties.$ai_evaluation_id = ${escapeHogQLString(evaluation.id)}`, escapeHogQLString(label)]
+ })
+
+ return `multiIf(${conditionsAndLabels.join(', ')}, toString(properties.$ai_evaluation_id))`
+}
+
+/** Pass rate over time with one line per evaluation, or null when none of them has a pass rate. Callers pick which evaluations to chart. */
+export function buildEvaluationPassRateTrendsQuery(
+ evaluations: EvaluationConfig[],
+ dateFilter: { dateFrom: string | null; dateTo: string | null },
+ interval: 'hour' | 'day'
+): TrendsQuery | null {
+ const charted = evaluations
+ .filter((evaluation) => !evaluation.deleted && hasPassRate(evaluation))
+ .slice(0, MAX_EVALUATION_CHART_SERIES)
+
+ if (charted.length === 0) {
+ return null
+ }
+
+ const dateFrom = dateFilter.dateFrom || '-7d'
+ const dateTo = dateFilter.dateTo || null
+ const numericEvaluations = charted.filter((evaluation) => evaluation.output_type === 'numeric')
+ let passedExpression = evaluationPassedHogQLForMany(
+ charted.filter(evaluationIsDetector).map((evaluation) => evaluation.id)
+ )
+ let gradedExpression = EVALUATION_BOOLEAN_GRADED_HOGQL
+ if (numericEvaluations.length > 0) {
+ const numericCases = numericEvaluations.flatMap((evaluation) => [
+ `properties.$ai_evaluation_id = ${escapeHogQLString(evaluation.id)}`,
+ numericEvaluationPassedHogQL(evaluation),
+ ])
+ const numericIds = numericEvaluations.map((evaluation) => escapeHogQLString(evaluation.id)).join(', ')
+ passedExpression = `multiIf(${numericCases.join(', ')}, ${passedExpression})`
+ gradedExpression = `if(properties.$ai_evaluation_id IN (${numericIds}), ${EVALUATION_NUMERIC_GRADED_HOGQL}, ${EVALUATION_BOOLEAN_GRADED_HOGQL})`
+ }
+
+ if (charted.some((evaluation) => evaluation.output_type === 'categorical')) {
+ passedExpression = `if(properties.$ai_evaluation_result_type = 'categorical', ${categoricalEvaluationsPassedHogQL(charted)}, ${passedExpression})`
+ gradedExpression = `if(properties.$ai_evaluation_result_type = 'categorical', ${EVALUATION_CATEGORICAL_GRADED_HOGQL}, ${gradedExpression})`
+ }
+
+ return {
+ kind: NodeKind.TrendsQuery,
+ series: [
+ {
+ kind: NodeKind.EventsNode,
+ event: '$ai_evaluation',
+ math: HogQLMathType.HogQL,
+ math_hogql: evaluationPassRateHogQL(passedExpression, gradedExpression),
+ properties: [
+ {
+ key: '$ai_evaluation_id',
+ value: charted.map((evaluation) => evaluation.id),
+ operator: PropertyOperator.Exact,
+ type: PropertyFilterType.Event,
+ },
+ ],
+ },
+ ],
+ breakdownFilter: {
+ breakdown: buildEvaluationBreakdownExpression(charted),
+ breakdown_type: 'hogql',
+ breakdown_limit: MAX_EVALUATION_CHART_SERIES,
+ breakdown_hide_other_aggregation: true,
+ },
+ trendsFilter: {
+ display: ChartDisplayType.ActionsLineGraph,
+ },
+ dateRange: {
+ date_from: dateFrom,
+ date_to: dateTo,
+ },
+ interval,
+ }
+}
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
index 120e4aabcdb9..6d79fd4ae147 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
@@ -5,9 +5,7 @@ import { subscriptions } from 'kea-subscriptions'
import posthog from 'posthog-js'
import { SetupTaskId, globalSetupLogic } from 'lib/components/ProductSetup'
-import { FEATURE_FLAGS } from 'lib/constants'
import { lemonToast } from 'lib/lemon-ui/LemonToast'
-import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { objectsEqual } from 'lib/utils/objects'
import { teamLogic } from 'scenes/teamLogic'
import { urls } from 'scenes/urls'
@@ -1564,12 +1562,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
}
// The new creation page lands on the evaluation itself, which says when its first result arrives.
- const opensEvaluation =
- isNew &&
- !!response?.id &&
- !!featureFlagLogic.findMounted()?.values.featureFlags[
- FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW
- ]
+ const opensEvaluation = isNew && !!response?.id
router.actions.push(
opensEvaluation
? urls.aiObservabilityEvaluation(response.id)
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationsLogic.test.ts b/products/ai_observability/frontend/evaluations/llmEvaluationsLogic.test.ts
index effcca602db9..d9a24c0c4fe7 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationsLogic.test.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationsLogic.test.ts
@@ -1,8 +1,6 @@
import { combineUrl, router } from 'kea-router'
import { expectLogic } from 'kea-test-utils'
-import { FEATURE_FLAGS } from 'lib/constants'
-import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { teamLogic } from 'scenes/teamLogic'
import { urls } from 'scenes/urls'
@@ -309,47 +307,7 @@ describe('llmEvaluationsLogic', () => {
})
})
- describe('filteredEvaluations', () => {
- const enabledEval = evaluationWithKey('eval-enabled', null)
- const disabledEval = { ...evaluationWithKey('eval-disabled', null), enabled: false }
-
- it('includes disabled evaluations by default and excludes them when hidden', async () => {
- logic.actions.loadEvaluationsSuccess([enabledEval, disabledEval])
-
- await expectLogic(logic).toMatchValues({
- filteredEvaluations: [enabledEval, disabledEval],
- })
-
- logic.actions.setShowDisabledEvaluations(false)
-
- await expectLogic(logic).toMatchValues({
- filteredEvaluations: [enabledEval],
- })
- })
-
- it('scopes the list to a directory but searches across all directories', async () => {
- const rootEvaluation = evaluationWithKey('root', null)
- const directoryEvaluation = evaluationWithKey('inside', null, 'directory-a')
-
- router.actions.push(
- combineUrl(urls.aiObservabilityEvaluations(), {
- directory: 'directory-a',
- }).url
- )
- logic.actions.loadEvaluationsSuccess([rootEvaluation, directoryEvaluation])
-
- await expectLogic(logic).toMatchValues({
- selectedDirectoryId: 'directory-a',
- displayedEvaluations: [directoryEvaluation],
- })
-
- logic.actions.setEvaluationsFilter('root')
-
- await expectLogic(logic).toMatchValues({
- displayedEvaluations: [rootEvaluation],
- })
- })
-
+ describe('selected directory', () => {
it('does not reload evaluations when the selected directory changes', async () => {
const evaluation = evaluationWithKey('local-state', null)
await expectLogic(logic).toFinishAllListeners()
@@ -381,13 +339,9 @@ describe('llmEvaluationsLogic', () => {
describe('default date range', () => {
it.each([
- { flagOn: true, dateFrom: undefined, expected: '-7d' },
- { flagOn: false, dateFrom: undefined, expected: '-24h' },
- { flagOn: true, dateFrom: '-30d', expected: '-30d' },
- ])('flag on $flagOn with date_from $dateFrom starts at $expected', async ({ flagOn, dateFrom, expected }) => {
- const flag = FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW
- featureFlagLogic.actions.setFeatureFlags(flagOn ? [flag] : [], { [flag]: flagOn })
-
+ { dateFrom: undefined, expected: '-7d' },
+ { dateFrom: '-30d', expected: '-30d' },
+ ])('date_from $dateFrom starts at $expected', async ({ dateFrom, expected }) => {
router.actions.push(combineUrl(urls.aiObservabilityEvaluations(), { date_from: dateFrom }).url)
await expectLogic(logic).toFinishAllListeners()
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationsLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationsLogic.ts
index 61d3c269e605..2b8f72282a83 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationsLogic.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationsLogic.ts
@@ -14,9 +14,7 @@ import {
import { loaders } from 'kea-loaders'
import { router, urlToAction } from 'kea-router'
-import { FEATURE_FLAGS } from 'lib/constants'
import { lemonToast } from 'lib/lemon-ui/LemonToast'
-import { type FeatureFlagsSet, featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { trackedActionToUrl } from 'lib/logic/scenes/trackedActionToUrl'
import { teamLogic } from 'scenes/teamLogic'
import { urls } from 'scenes/urls'
@@ -36,9 +34,8 @@ import { evaluationCanResolveModel } from './evaluationCapabilities'
import { listAllEvaluations, patchEvaluation } from './evaluationsApi'
import { EvaluationConfig } from './types'
-const INITIAL_DATE_FROM = '-24h' as string | null
// The overview compares each day with the previous period, which a 24 hour range is too short to show.
-const OVERVIEW_INITIAL_DATE_FROM = '-7d'
+const INITIAL_DATE_FROM = '-7d' as string | null
const INITIAL_DATE_TO = null as string | null
export type LLMEvaluationsLogicProps = Record<string, never>
@@ -51,7 +48,6 @@ export interface EvaluationDirectoryEditor {
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface llmEvaluationsLogicValues {
- featureFlags: FeatureFlagsSet // featureFlagLogic
activeProviderKey: LLMProviderKey | null | undefined // llmProviderKeysLogic
providerKeys: LLMProviderKey[] // llmProviderKeysLogic
searchParams: Record<string, any> // router
@@ -60,23 +56,18 @@ export interface llmEvaluationsLogicValues {
dateFrom: string | null
dateTo: string | null
}
- defaultDateFrom: string | null
deletingDirectoryId: string | null
detectorEvaluationIds: string[]
directoryEditor: EvaluationDirectoryEditor | null
- displayedEvaluations: EvaluationConfig[]
evaluationDirectories: EvaluationDirectoryApi[]
evaluationDirectoriesLoading: boolean
evaluations: EvaluationConfig[]
- evaluationsFilter: string
evaluationsLoadFailed: boolean
evaluationsLoading: boolean
evaluationsSettled: boolean
- filteredEvaluations: EvaluationConfig[]
movingEvaluationId: string | null
selectedDirectory: EvaluationDirectoryApi | null
selectedDirectoryId: string | null
- showDisabledEvaluations: boolean
submitDirectoryLoading: boolean
unhealthyProviderKeysUsedByEvaluations: LLMProviderKey[]
}
@@ -173,12 +164,6 @@ export interface llmEvaluationsLogicActions {
setDirectoryEditorName: (name: string) => {
name: string
}
- setEvaluationsFilter: (filter: string) => {
- filter: string
- }
- setShowDisabledEvaluations: (show: boolean) => {
- show: boolean
- }
submitDirectory: () => {
value: true
}
@@ -207,22 +192,11 @@ export interface llmEvaluationsLogicActions {
export interface llmEvaluationsLogicMeta {
__keaTypeGenInternalSelectorTypes: {
detectorEvaluationIds: (evaluations: EvaluationConfig[]) => string[]
- filteredEvaluations: (
- evaluations: EvaluationConfig[],
- evaluationsFilter: string,
- showDisabledEvaluations: boolean
- ) => EvaluationConfig[]
- defaultDateFrom: (featureFlags: FeatureFlagsSet) => string | null
selectedDirectoryId: (searchParams: Record<string, any>) => string | null
selectedDirectory: (
evaluationDirectories: EvaluationDirectoryApi[],
selectedDirectoryId: string | null
) => EvaluationDirectoryApi | null
- displayedEvaluations: (
- filteredEvaluations: EvaluationConfig[],
- evaluationsFilter: string,
- selectedDirectoryId: string | null
- ) => EvaluationConfig[]
canEnableEvaluation: (
activeProviderKey: LLMProviderKey | null | undefined
) => (evaluation: EvaluationConfig) => boolean
@@ -244,14 +218,7 @@ export const llmEvaluationsLogic = kea<llmEvaluationsLogicType>([
path(['products', 'ai_observability', 'evaluations', 'llmEvaluationsLogic']),
props({} as LLMEvaluationsLogicProps),
connect(() => ({
- values: [
- llmProviderKeysLogic,
- ['providerKeys', 'activeProviderKey'],
- router,
- ['searchParams'],
- featureFlagLogic,
- ['featureFlags'],
- ],
+ values: [llmProviderKeysLogic, ['providerKeys', 'activeProviderKey'], router, ['searchParams']],
actions: [llmProviderKeysLogic, ['loadProviderKeys']],
})),
@@ -280,8 +247,6 @@ export const llmEvaluationsLogic = kea<llmEvaluationsLogicType>([
toggleEvaluationEnabled: (id: string) => ({ id }),
toggleEvaluationEnabledSuccess: (id: string) => ({ id }),
toggleEvaluationEnabledFailure: (id: string, error: string) => ({ id, error }),
- setEvaluationsFilter: (filter: string) => ({ filter }),
- setShowDisabledEvaluations: (show: boolean) => ({ show }),
}),
reducers({
@@ -355,18 +320,6 @@ export const llmEvaluationsLogic = kea<llmEvaluationsLogicType>([
loadEvaluationsSuccess: () => false,
},
],
- evaluationsFilter: [
- '',
- {
- setEvaluationsFilter: (_, { filter }) => filter,
- },
- ],
- showDisabledEvaluations: [
- true,
- {
- setShowDisabledEvaluations: (_, { show }) => show,
- },
- ],
directoryEditor: [
null as EvaluationDirectoryEditor | null,
{
@@ -579,33 +532,6 @@ export const llmEvaluationsLogic = kea<llmEvaluationsLogicType>([
(evaluations: EvaluationConfig[]): string[] =>
evaluations.filter((evaluation) => evaluationIsDetector(evaluation)).map((evaluation) => evaluation.id),
],
- filteredEvaluations: [
- (s) => [s.evaluations, s.evaluationsFilter, s.showDisabledEvaluations],
- (evaluations: EvaluationConfig[], filter: string, showDisabledEvaluations: boolean) => {
- const visible = showDisabledEvaluations
- ? evaluations
- : evaluations.filter((e: EvaluationConfig) => e.enabled)
- if (!filter) {
- return visible
- }
- return visible.filter(
- (e: EvaluationConfig) =>
- e.name.toLowerCase().includes(filter.toLowerCase()) ||
- e.description?.toLowerCase().includes(filter.toLowerCase()) ||
- ('prompt' in e.evaluation_config &&
- e.evaluation_config.prompt.toLowerCase().includes(filter.toLowerCase())) ||
- ('source' in e.evaluation_config &&
- e.evaluation_config.source.toLowerCase().includes(filter.toLowerCase()))
- )
- },
- ],
- defaultDateFrom: [
- (s) => [s.featureFlags],
- (featureFlags: FeatureFlagsSet): string | null =>
- featureFlags[FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW]
- ? OVERVIEW_INITIAL_DATE_FROM
- : INITIAL_DATE_FROM,
- ],
selectedDirectoryId: [
(s) => [s.searchParams],
(searchParams: Record<string, unknown>): string | null =>
@@ -619,19 +545,6 @@ export const llmEvaluationsLogic = kea<llmEvaluationsLogicType>([
): EvaluationDirectoryApi | null =>
directories.find((directory) => directory.id === selectedDirectoryId) ?? null,
],
- displayedEvaluations: [
- (s) => [s.filteredEvaluations, s.evaluationsFilter, s.selectedDirectoryId],
- (
- filteredEvaluations: EvaluationConfig[],
- evaluationsFilter: string,
- selectedDirectoryId: string | null
- ): EvaluationConfig[] =>
- evaluationsFilter
- ? filteredEvaluations
- : filteredEvaluations.filter(
- (evaluation) => (evaluation.directory_id ?? null) === selectedDirectoryId
- ),
- ],
canEnableEvaluation: [
(s) => [s.activeProviderKey],
(activeProviderKey: LLMProviderKey | null | undefined) => {
@@ -673,7 +586,7 @@ export const llmEvaluationsLogic = kea<llmEvaluationsLogicType>([
return
}
- const dateFrom = (searchParams.date_from as string | null) || values.defaultDateFrom
+ const dateFrom = (searchParams.date_from as string | null) || INITIAL_DATE_FROM
const dateTo = (searchParams.date_to as string | null) || INITIAL_DATE_TO
if (dateFrom !== values.dateFilter.dateFrom || dateTo !== values.dateFilter.dateTo) {
@@ -682,12 +595,12 @@ export const llmEvaluationsLogic = kea<llmEvaluationsLogicType>([
},
})),
- trackedActionToUrl(({ values }) => ({
+ trackedActionToUrl(() => ({
setDates: ({ dateFrom, dateTo }) => [
urls.aiObservabilityEvaluations(),
{
...router.values.searchParams,
- date_from: dateFrom === values.defaultDateFrom ? undefined : dateFrom || undefined,
+ date_from: dateFrom === INITIAL_DATE_FROM ? undefined : dateFrom || undefined,
date_to: dateTo || undefined,
},
],
diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationsOverview.stories.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationsOverview.stories.tsx
index 8fd3c48f2fde..7cdde6044006 100644
--- a/products/ai_observability/frontend/evaluations/overview/EvaluationsOverview.stories.tsx
+++ b/products/ai_observability/frontend/evaluations/overview/EvaluationsOverview.stories.tsx
@@ -2,7 +2,6 @@ import { Meta, StoryObj } from '@storybook/react'
import { within } from '@testing-library/dom'
import userEvent from '@testing-library/user-event'
-import { FEATURE_FLAGS } from 'lib/constants'
import { App } from 'scenes/App'
import { urls } from 'scenes/urls'
@@ -377,7 +376,6 @@ const meta: Meta = {
viewMode: 'story',
mockDate: '2026-10-20',
pageUrl: urls.aiObservabilityEvaluations(),
- featureFlags: [FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW],
testOptions: { waitForLoadersToDisappear: true },
},
}
diff --git a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts
index 8c351d954194..cfb1f489ab20 100644
--- a/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts
+++ b/products/ai_observability/frontend/evaluations/overview/evaluationsOverviewLogic.ts
@@ -22,7 +22,7 @@ import type {
EvaluationsOverviewApi,
EvaluationsOverviewRetrieveParams,
} from '../../generated/api.schemas'
-import { buildEvaluationPassRateTrendsQuery } from '../evaluationMetricsLogic'
+import { buildEvaluationPassRateTrendsQuery } from '../evaluationPassRateTrendsQuery'
import { llmEvaluationsLogic } from '../llmEvaluationsLogic'
import type { EvaluationDirectoryEditor } from '../llmEvaluationsLogic'
import type { EvaluationConfig } from '../types'
diff --git a/products/ai_observability/frontend/utils.ts b/products/ai_observability/frontend/utils.ts
index 18df0ef1f384..c2459fd02757 100644
--- a/products/ai_observability/frontend/utils.ts
+++ b/products/ai_observability/frontend/utils.ts
@@ -1413,8 +1413,7 @@ export interface EvaluationRunsStats {
// Counts every matching run server-side. queryEvaluationRuns caps its fetch at
// EVALUATION_RUNS_QUERY_LIMIT, so summary stats can't be derived from that list without
-// undercounting. Counting semantics mirror the evaluations list view (evaluationMetricsLogic)
-// so both surfaces report the same totals.
+// undercounting.
export async function queryEvaluationRunsStats(params: {
evaluation?: EvaluationConfig | null
evaluationId?: string