:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): rebuild the new evaluation page as numbered steps (#123594)
Bernat Torres658b98280cbf
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex bac16d6ad09d..a440f67ee371 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -9126,6 +9126,26 @@ snapshots:hash: v1.k794b7964.6f9f209d5daaf8573020fb50615421d426055c8090d0ee73a2629487d30cbf29.stsWopH6BI50ASXjhDxCWn-Xa46oqy_cH4v0xAO4VIQscenes-app-ai-observability-instrumentation-checklist-card--warnings-with-a-dismissal--light:hash: v1.k794b7964.b0e06944ee7afe4af8cb39363280135111e71ba215aab56a3e410c79de62238b.WtW8suUOWanaTrvjxhV2Z_ANVDRmEW3bqHpTrTe5GoI+ scenes-app-ai-observability-new-evaluation--from-scratch--dark:+ hash: v1.k794b7964.7ccaf69cf0e22860ac3c6b4f7f7f26701eafd010d5123e37c56fc848e6fe74a7.OpK0ahu9jaxpIlHEQ8mw1Sh7mdzqOONp4aw1LVbwzuA+ scenes-app-ai-observability-new-evaluation--from-scratch--light:+ hash: v1.k794b7964.816d9e830ebfeb8ac1303b3b59efe94150a1d2371cb78e3338153f176f521444.XhZKS15jlRkqnRcA2vpvX4Fs7ZEuvE23jm48lIAMqGE+ scenes-app-ai-observability-new-evaluation--from-template--dark:+ hash: v1.k794b7964.522cb4a66474488f0a1a5c64f26c09f1beda25de9d3025a7d9d708a5b638fb5e.FTdDwtnvQSREwVqU1obRvn4JMO2BZIA6_hQq53n1gIY+ scenes-app-ai-observability-new-evaluation--from-template--light:+ hash: v1.k794b7964.2293fbca6fae992a9b19200cc73bf6ac0a9c06056d24cc4b3ccbcb6a592a6cd5.FGIm1AeCqfdM9Py8enk2QX65UJr1tn8ZlUAlAIG2bUA+ scenes-app-ai-observability-new-evaluation--nothing-matches--dark:+ hash: v1.k794b7964.9042eb2d03e3fad492dfdfc755ffe45d651856cf53a23c1b126820af2af4a157.dsssE0qpI7l2eda6yDFu7etQyalm08flGpQ5pF8Xbs4+ scenes-app-ai-observability-new-evaluation--nothing-matches--light:+ hash: v1.k794b7964.e57e16dfee3e4b83740a8d14a8ce425fa22a2bed2410c9c82f8308ab83bf5a81.bMBzNe0faqqHTUOVJ3CMGDuHtbEEw7nRqfcNXQVqgIs+ scenes-app-ai-observability-new-evaluation--reports-open--dark:+ hash: v1.k794b7964.f38dbca5c0f4a4666976c917d4f027924f1dc0d1550775c38654360889cbb23c.byoQrLQhAAMUFDzn9EJZVZPcMCP5MiyIfHEgHD9K-pE+ scenes-app-ai-observability-new-evaluation--reports-open--light:+ hash: v1.k794b7964.421b25300229d08d96da3d1f65a030df1365b26e795695d85478539543e58682._Yjl0PGso1sxk98zetDxwSc6PZoc_uRl637y0px88GA+ scenes-app-ai-observability-new-evaluation--sentiment-evaluation--dark:+ hash: v1.k794b7964.ac47d8aaeb570a78393cc59504adfa570a40a1a544ace57840c0dae235680603.4O7QEnavU1PIlLZw-fCCE7TO-6r3nrqU2WG0_02WyKc+ scenes-app-ai-observability-new-evaluation--sentiment-evaluation--light:+ hash: v1.k794b7964.69868b16ba3c403a48e038ee39dd0a2befa608feb9a3fc21375118ff54bb3e4e.TFLBq7ma7X2uapVb-eu18RUZakc5-WVt9OQ5QFiXwz8scenes-app-ai-observability-offline-experiments-experiment--all-scorers--dark:hash: v1.k794b7964.5e999325599c0186b678a4aaba8423a09a578de4789220ddb18fbf7c64245764.9GJhQsDjcB9bKZSm5iX9O2RQApiIhl3KFVBhf1nv3_Iscenes-app-ai-observability-offline-experiments-experiment--all-scorers--light:@@ -13215,21 +13235,21 @@ snapshots:scenes-app-legal-documents--new-dpat-swift--light:hash: v1.k794b7964.e87918b59eea97e7aaf2b8f103d045e0f1ba279539677f983d37191c65ca26fe.ZgqoHeykE3Lt4Oqlg7CLX0cA7ejpZCN9DXWt5AZqQgIscenes-app-llm-observability-categoricalevaluationconfig--default--dark:- hash: v1.k794b7964.5d13f8e4da4472d9afaa7bdaa4684b69e83d718b13ce996057ac181df792afec.QAvBIAMjHaItLYd-Zgj2X088lA-gxJadN8ijiqhGkIo+ hash: v1.k794b7964.ada2ad9acdfbff7b24546c99eb5a9d230f6382ea79c85efea0702d97502ec71e.J-Q7sqVTRhiJi3_pXpETIh1UPiUNEadlSk-newKsp_Uscenes-app-llm-observability-categoricalevaluationconfig--default--light:- hash: v1.k794b7964.2f27936e1adef36981db2b3099b05057bc603b40489f7b4dcf6d1696af85ddbe.aFk-vFyGv0uHtYAAEWuW0qN1ktNAOvdOy69XkwjtSxU+ hash: v1.k794b7964.02e9ebb08a3452ccd3d9d4bd2f88d16af51166514c8d99206a3332b4ac2178dc.CpX4ruvV0x6n6jgLdcPDKEPopK0MFnzhOzGgx1oVPq8scenes-app-llm-observability-categoricalevaluationconfig--duplicate-keys--dark:- hash: v1.k794b7964.94005aed5ae5353e8d8c2cb9631b683e95408030720ddfcb3e5fb9820e15ee90.TgwkYuTA4wd1Ci2isCxadZ_szsY_kI7my_KQnE0wq-A+ hash: v1.k794b7964.2f6add612f640b9a83c0b7b670fed2241e5838919bd562081771f3beabab88cd.rM-CuNIz3Hg8PenSy4j5HW1bU3DbitfhT8kyf1Nf2aYscenes-app-llm-observability-categoricalevaluationconfig--duplicate-keys--light:- hash: v1.k794b7964.ed68af50f92b6e4e398fc257a17ea72d979de6f9de9f6da0d9d2acb9fdf9609c.Lo2e2uFUHe7A_4hglEhLCaqZPRY0638jI0Fcp_BFVj8+ hash: v1.k794b7964.1ea3209a9d2358a1a69bede92656d70058ce51d3ee012132aae4d7486a663f63.Qlu7Dg-bgksM_HJHARKdhJ0gwIW8O4xJyHFR3Kwy-jcscenes-app-llm-observability-categoricalevaluationconfig--invalid-passing-rule--dark:- hash: v1.k794b7964.a85d83120f135c73b50fb434836988ec2a417bdacda581cb6ad7f741296e3114.pRo6Tp791ruMRMXEFypwDj7u2bbEQ7c-b1CGSjQtdXQ+ hash: v1.k794b7964.da59d7c00ac5b84cf617c3ec127a633e735d22702a74374271d9869e25166d1a.KKEzg2vLfjMGpCmnLry9Yy26AdLK-7VUu2XzAQdgeVYscenes-app-llm-observability-categoricalevaluationconfig--invalid-passing-rule--light:- hash: v1.k794b7964.d27e7eb29b2bbf297aac6ed17e32ab9bb4179243955bc524aaa22b8856260a1a.1WZBFuxI-8PXuuTG2u7H1YlOmzpw5nX3IK10XIhWJ4Y+ hash: v1.k794b7964.8c39683ef7ed907d88d445412ef9c5893b78a2ea3677e7ffea4a76169b8fd72e.A5q_lgCmp8JX6i4xUFabSGrAPOQzFPZMJJPWwjIcrG8scenes-app-llm-observability-categoricalevaluationconfig--narrow--dark:- hash: v1.k794b7964.436dcd7ee00a47ccca4d46685c0066615f888983b15844ac3945b8e51a79e508.zpSy36nqAxQSuvSHzElPGHXJZgnJKRjQmxBwDamre9w+ hash: v1.k794b7964.7d559096946c7fb95a3c0b63549bedb9b208c1725d50c6f4429444abb5e18325.kUhWPi1f7gBuRO9-ksG1UG2p4-PqmJRWldGLqAzrFoIscenes-app-llm-observability-categoricalevaluationconfig--narrow--light:- hash: v1.k794b7964.8489284cc30b609b61e118a4c29d24a2ded7d4bd674c30cc5cba80d1cd97f76b.fbzkOm6zlJje7ju3NPx5a-8JiaQdshwaLbVhRvplR6E+ hash: v1.k794b7964.c96e17b2cd655cfc8cb4085ca2efb87c19ea743c6200e65c368aa6538af92cab.Fc2w6LHTyrCIM3oJi1NA8eTxZm4y0vcVfE2nfIkD-1wscenes-app-llm-observability-evaluation-templates--default--dark:hash: v1.k794b7964.07a0809fec2e7a04e7ed1e7ca06aa3570d62106e9754c1938662a127839a7d43.09gLEWuTLF5IIQazReIApdLh_V8h-FDxjGyNj2hkMusscenes-app-llm-observability-evaluation-templates--default--light:@@ -13239,29 +13259,29 @@ snapshots:scenes-app-llm-observability-evaluation-templates--narrow--light:hash: v1.k794b7964.acb588946dd41382c78934be3f8912589d1a15df06ef9a976f617fad21f1ddd2.0LsEsqRoncEFZhEVh5X8gbA_QoGZelV2t1ABDsxGeaoscenes-app-llm-observability-evaluationmodelpicker--decision--dark:- hash: v1.k794b7964.0325f8084a7ecb6745b8164ee4c198f67f133e02c8c6e7431b611a560d757bbf.txtMNRNYHw5Xmf1gnMpp7hAyShiaVIDV2m1SYroxnME+ hash: v1.k794b7964.8cf03ee1f25ccdee62cc5104d48589d196f85dd3dbad84d5a838cc4c8bfe9607.G7gP4btJiT-u-RfsschCP6cV1btlQnw85BFAhZ1D86Mscenes-app-llm-observability-evaluationmodelpicker--decision--light:- hash: v1.k794b7964.a3ad1f30e680fd2cdf3add2e75822037390ff220d3c7aef549f1f32cebdcc7c6.-bNzbWVEQXCqou9cnKIz70PTxSFkha0pDknYqaC5tnM+ hash: v1.k794b7964.6b288367a286d1858e3032bab65a034553c5f5ae79dacd0274cb92d1f8bab818.CM-LkhqTNhtSyn0O3Y6c6Yd_gLfSXe31n3Dxaj5uEZIscenes-app-llm-observability-evaluationmodelpicker--decision-narrow--dark:- hash: v1.k794b7964.58dc8d73bf85d6c8bcd7a10b08c1e38efefd840fbe912cbbdff34bc4596041d8.5_RSGKERITVltMJ5ap2pmDAF2IC77kXkHPRfyXJi2oY+ hash: v1.k794b7964.8cf03ee1f25ccdee62cc5104d48589d196f85dd3dbad84d5a838cc4c8bfe9607.W_9QnaVZtdekweLtoPuF32I_r0G0fCdjRWUVyBg3kEIscenes-app-llm-observability-evaluationmodelpicker--decision-narrow--light:- hash: v1.k794b7964.28a30a7cb478ba7b585a0736e53e74735c643e3b61b0b84059494f6040493c29.haO943nTQB3hFa_f-TgDhACaCqM3_bOKVEdmp0SXTzg+ hash: v1.k794b7964.6b288367a286d1858e3032bab65a034553c5f5ae79dacd0274cb92d1f8bab818.HuCQ8PyBI05CTQEjr38moKf85RyyPvF9SCxuwytomXQscenes-app-llm-observability-evaluationmodelpicker--llm--dark:- hash: v1.k794b7964.43b905fdb235f4c15e7b906f14467c4ee9e8df010a3f26e476a2b3e6f59a4323.MZUQFFzWautdc3zzDJVWS6YbBa4qxX69-2HNwn-4wXQ+ hash: v1.k794b7964.a42ccc41fa5864dee67f21add5ecc3aa6838805fb494f6686b826688cc325e9e.tHDxeEbDTVatdZYPj2xVDQtJpukqzNyUHU9iScfQ2AYscenes-app-llm-observability-evaluationmodelpicker--llm--light:- hash: v1.k794b7964.5fbd678972f802f1baf3bb1e1dc4869a6d2b78c8279bc5f03e3b55e149d24c8f.FAWGdhvRzIqww3vfNkkv-FISpmlDCBK9QHYPiQ6sS6c+ hash: v1.k794b7964.24f4da8cefdaef0c33f3a1783746888c340cf60cb1d47edc654b1325f5ff644b.gDAO1ukg0FfytAXK4R2OSPF5mkNti7adPrdbA1bwGGIscenes-app-llm-observability-evaluationmodelpicker--native-open-ai--dark:- hash: v1.k794b7964.0325f8084a7ecb6745b8164ee4c198f67f133e02c8c6e7431b611a560d757bbf.hqOKsQgqwHGk8LsGxX1mnQMpHyhzl00XVFCNrp8uPe0+ hash: v1.k794b7964.8cf03ee1f25ccdee62cc5104d48589d196f85dd3dbad84d5a838cc4c8bfe9607.kKKEqvhd_Wb8zZVyl0fmkbE6RBz9odp8Iyhq8HPAw3Iscenes-app-llm-observability-evaluationmodelpicker--native-open-ai--light:- hash: v1.k794b7964.a3ad1f30e680fd2cdf3add2e75822037390ff220d3c7aef549f1f32cebdcc7c6.wlwriSaiiF198e3yVP5IaIdmT9kaWmMlGSOehEhg5rU+ hash: v1.k794b7964.6b288367a286d1858e3032bab65a034553c5f5ae79dacd0274cb92d1f8bab818.nTYWu7M1TfGyOlu52uGxd98cNvdj-nyoyzcWUovs6Ysscenes-app-llm-observability-evaluationmodelpicker--native-open-ai-narrow--dark:- hash: v1.k794b7964.58dc8d73bf85d6c8bcd7a10b08c1e38efefd840fbe912cbbdff34bc4596041d8.u5g-WppnRiIjPg1k0jD2eqEX-5Qt1nlyABrap6Toqas+ hash: v1.k794b7964.8cf03ee1f25ccdee62cc5104d48589d196f85dd3dbad84d5a838cc4c8bfe9607.gZEs23POnv-eBuhQT-V2gxTuoBiA7R32dhshLPHHyVsscenes-app-llm-observability-evaluationmodelpicker--native-open-ai-narrow--light:- hash: v1.k794b7964.28a30a7cb478ba7b585a0736e53e74735c643e3b61b0b84059494f6040493c29.DaQknPZWc0jSBbq13iEjT3m5vvDAlm5QHELIewdA7PY+ hash: v1.k794b7964.6b288367a286d1858e3032bab65a034553c5f5ae79dacd0274cb92d1f8bab818.DO5O6V3lvkGYD6JJ5RfEBG2IbsSvb0aNeCME9dvKEWkscenes-app-llm-observability-evaluationmodelpicker--no-decision-models--dark:- hash: v1.k794b7964.681693474e46e0164b0a832b7d33c991f1b6c1c944cf5f23c64e3ee90a223096.MrFeO9h-IUD2Yd0l-zbg2KzxcUkskx57GnPNKwKhcv0+ hash: v1.k794b7964.ed596389ff3b9a8a505c3b1ad5cbfabadc1cb83507c341f99dd54b004f8d257e.Jm_sWi9-7s4eTkxjyQUcGApiT5QcFzke8u5k2hK6gaEscenes-app-llm-observability-evaluationmodelpicker--no-decision-models--light:- hash: v1.k794b7964.39f6167f6c51638beef826219c123100b5ae9207c817bfaa6811f5428beb4da5._LdpuKYQp9L79AcfpQpKH-XHosP2Tymw21tcAFJDWCk+ hash: v1.k794b7964.b10b433ed0507465953ef6174a8872546e4e695407914798a8e11a6e7f8aa38d.jQ8Xqfw0JjPdrTXWnuYodzsfn46fs4aKri7d1Pzl7QUscenes-app-llm-observability-evaluationreportconfig--lemon-ui--dark:hash: v1.k794b7964.195aa4a5a39e8f947e17fe67f5c23303b7dd0579c278907761be24a08780be92.IkGQtAI95T60kJKVZcmmFBIWNCd4meuJv-bcNDoohwUscenes-app-llm-observability-evaluationreportconfig--lemon-ui--light:diff --git a/posthog/temporal/ai_observability/message_utils.py b/posthog/temporal/ai_observability/message_utils.pyindex 177216c8c255..9f8c7b799117 100644--- a/posthog/temporal/ai_observability/message_utils.py+++ b/posthog/temporal/ai_observability/message_utils.py@@ -51,6 +51,30 @@ def extract_text_from_messages(messages: Union[str, list, dict, None]) -> str:return "\n".join(rendered for rendered in rendered_messages if rendered is not None)+def last_message_text(messages: Union[str, list, dict, None], role: str) -> str:+ """Text of the last message in a role, or all the text when no message has that role.++ A preview built from the start of the input shows the system prompt, which is the same on every sample.+ """+ if isinstance(messages, list):+ for msg in reversed(messages):+ if isinstance(msg, dict) and msg.get("role") == role and not _only_tool_results(msg):+ rendered = extract_text_from_messages([msg])+ if rendered:+ return rendered.removeprefix(f"{role}: ")+ return extract_text_from_messages(messages)+++def _only_tool_results(msg: dict) -> bool:+ # Anthropic sends tool output back as a user message made only of tool_result blocks.+ content = msg.get("content")+ return (+ isinstance(content, list)+ and len(content) > 0+ and all(isinstance(part, dict) and part.get("type") == "tool_result" for part in content)+ )++def _render_message(msg: dict) -> str | None:"""Render a single message dict into a `role: body` line.diff --git a/posthog/temporal/ai_observability/test_message_utils.py b/posthog/temporal/ai_observability/test_message_utils.pyindex 99bce708483e..548c36ccf6e1 100644--- a/posthog/temporal/ai_observability/test_message_utils.py+++ b/posthog/temporal/ai_observability/test_message_utils.py@@ -4,7 +4,11 @@ import jsonimport pytest-from posthog.temporal.ai_observability.message_utils import extract_text_from_messages, format_tool_definitions+from posthog.temporal.ai_observability.message_utils import (+ extract_text_from_messages,+ format_tool_definitions,+ last_message_text,+)class TestExtractTextFromMessages:@@ -764,3 +768,34 @@ class TestFormatToolDefinitions:# rather than silently dropped — the judge can still see something.result = format_tool_definitions(tools)assert result == json.dumps(tools, default=str)+++ "messages,role,expected",+ [+ (+ [+ {"role": "system", "content": "You are a support agent."},+ {"role": "user", "content": "First question"},+ {"role": "assistant", "content": "First answer"},+ {"role": "user", "content": [{"type": "text", "text": "Second question"}]},+ ],+ "user",+ "Second question",+ ),+ (+ [+ {"role": "user", "content": "What is the weather in Paris?"},+ {"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "weather", "input": {}}]},+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "18C sunny"}]},+ ],+ "user",+ "What is the weather in Paris?",+ ),+ ([{"role": "assistant", "content": "The answer"}], "assistant", "The answer"),+ ([{"role": "system", "content": "Only a system prompt"}], "user", "system: Only a system prompt"),+ ("A plain string", "user", "A plain string"),+ ],+)+def test_last_message_text(messages, role, expected):+ assert last_message_text(messages, role) == expecteddiff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.pyindex ca1b961c85b1..940f9589e4fe 100644--- a/products/ai_observability/backend/api/evaluations.py+++ b/products/ai_observability/backend/api/evaluations.py@@ -37,7 +37,7 @@ from posthog.hogql_queries.ai.ai_table_resolver import AIEventsUnavailableError,from posthog.hogql_queries.ai.utils import HEAVY_COLUMN_NAMES, merge_heavy_propertiesfrom posthog.models.team import Teamfrom posthog.permissions import AccessControlPermission, is_service_auth-from posthog.temporal.ai_observability.message_utils import extract_text_from_messages+from posthog.temporal.ai_observability.message_utils import last_message_textfrom posthog.temporal.ai_observability.model_resolution import active_key_fallbackfrom posthog.temporal.ai_observability.run_evaluation import extract_event_io, run_hog_evalfrom posthog.temporal.ai_observability.run_session_evaluation import run_hog_eval_over_recent_sessions@@ -1874,8 +1874,8 @@ class EvaluationViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, Forbi)io = extract_event_io(event_type, properties)- input_preview = extract_text_from_messages(io.input_raw)[:200]- output_preview = extract_text_from_messages(io.output_raw)[:200]+ input_preview = last_message_text(io.input_raw, "user")[:200]+ output_preview = last_message_text(io.output_raw, "assistant")[:200]results.append({diff --git a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsxindex 2fb64bc8b812..c0ef2b00898a 100644--- a/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx+++ b/products/ai_observability/frontend/evaluations/AIObservabilityEvaluation.tsx@@ -18,7 +18,6 @@ import {} from '@posthog/lemon-ui'import { AccessControlAction } from 'lib/components/AccessControlAction'-import { DurationPicker } from 'lib/components/DurationPicker/DurationPicker'import { NotFound } from 'lib/components/NotFound'import { TZLabel } from 'lib/components/TZLabel'import { FEATURE_FLAGS } from 'lib/constants'@@ -39,13 +38,16 @@ import { providerKeyStateIssueDescription, providerLabel } from '../settings/proimport { CategoricalEvaluationConfig } from './components/CategoricalEvaluationConfig'import { EvaluationBackfillsTab } from './components/EvaluationBackfillsTab'import { EvaluationCodeEditor } from './components/EvaluationCodeEditor'+import { EvaluationFirstResultStatus } from './components/EvaluationFirstResultStatus'+import { EvaluationLatestReport } from './components/EvaluationLatestReport'import { EvaluationModelPicker } from './components/EvaluationModelPicker'import { EvaluationPromptEditor } from './components/EvaluationPromptEditor'import { EvaluationReportConfig } from './components/EvaluationReportConfig'import { EvaluationReportsCallout } from './components/EvaluationReportsCallout'import { EvaluationReportsTab } from './components/EvaluationReportsTab'import { EvaluationRunsTable } from './components/EvaluationRunsTable'-import { EvaluationTriggers } from './components/EvaluationTriggers'+import { EvaluationSettleConfig } from './components/EvaluationSettleConfig'+import { EvaluationTriggersPanel } from './components/EvaluationTriggersPanel'import { NumericEvaluationConfig } from './components/NumericEvaluationConfig'import {formatNumericEvaluationScore,@@ -53,6 +55,8 @@ import {numericOutputConfigError,categoricalOutputConfigError,} from './constants'+import { EvaluationConfigurationForm } from './create/EvaluationConfigurationForm'+import { EvaluationCreatePage } from './create/EvaluationCreatePage'import {evaluationOffersSessionTarget,evaluationSupportsReportHistory,@@ -63,20 +67,10 @@ import {isBooleanEvaluationOutput,} from './evaluationCapabilities'import { evaluationReportLogic } from './evaluationReportLogic'-import {- DEFAULT_SESSION_MAX_AGE_SECONDS,- DEFAULT_SESSION_QUIET_PERIOD_SECONDS,- DEFAULT_SESSION_WINDOW_SECONDS,- DEFAULT_TRACE_MAX_AGE_SECONDS,- DEFAULT_TRACE_QUIET_PERIOD_SECONDS,- DEFAULT_TRACE_WINDOW_SECONDS,- LLMEvaluationLogicProps,- hasUnsetConditionRollout,- llmEvaluationLogic,-} from './llmEvaluationLogic'+import { LLMEvaluationLogicProps, hasUnsetConditionRollout, llmEvaluationLogic } from './llmEvaluationLogic'import { EvaluationDetailResults } from './overview/EvaluationDetailResults'import { statusReasonLabel, statusReasonRecoveryLabel } from './statusDisplay'-import { EvaluationSettleStrategy, EvaluationTarget, EvaluationType } from './types'+import { EvaluationTarget, EvaluationType } from './types'const RUNS_BACKFILL_TIME_FORMAT = { formatDate: 'MMM D, YYYY', formatTime: 'HH:mm' }@@ -108,6 +102,11 @@ export function AIObservabilityEvaluation(): JSX.Element {const { featureFlags } = useValues(featureFlagLogic)const settlingStrategyEnabled = !!featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EVAL_SETTLING_STRATEGY]const resultsCardsEnabled = !!featureFlags[FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW]+ const { isConfigDirty: reportConfigDirty } = useValues(+ evaluationReportLogic({ evaluationId: isNewEvaluation || !evaluation?.id ? 'new' : evaluation.id })+ )+ // The step layout edits reports with the rest of the configuration, so a report-only edit is unsaved too.+ const unsaved = hasUnsavedChanges || (resultsCardsEnabled && !isNewEvaluation && reportConfigDirty)const {setEvaluationName,setEvaluationDescription,@@ -120,9 +119,8 @@ export function AIObservabilityEvaluation(): JSX.Element {setOutputType,patchOutputConfig,setEvaluationTarget,- setSettleStrategy,- patchTargetConfig,setActiveTab,+ setFormStep,} = useActions(llmEvaluationLogic)const { push } = useActions(router)const triggersRef = useRef<HTMLDivElement>(null)@@ -139,6 +137,13 @@ export function AIObservabilityEvaluation(): JSX.Element {if (!evaluation) {return <NotFound object="evaluation" />}+ if (isNewEvaluation && featureFlags[FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW]) {+ return (+ <EvaluationCreatePage+ templateKey={typeof searchParams.template === 'string' ? searchParams.template : undefined}+ />+ )+ }const openInPlaygroundUrl =evaluationTypeUsesModelConfiguration(evaluation.evaluation_type) && evaluation.id && !usesDecisionModel? combineUrl(urls.aiObservabilityPlayground(), { source_evaluation_id: evaluation.id }).url@@ -146,14 +151,8 @@ export function AIObservabilityEvaluation(): JSX.Element {const isHog = evaluation.evaluation_type === 'hog'const isSentiment = evaluation.evaluation_type === 'sentiment'- const isAggregateTarget = evaluation.target === 'trace' || evaluation.target === 'session'const isSessionTarget = evaluation.target === 'session'const offersSessionTarget = evaluationOffersSessionTarget(evaluation, settlingStrategyEnabled)- // Read the stored strategy whether or not the flag is on. The flag gates the *picker*, not what- // the row contains: an API- or MCP-created session eval is inactivity, and forcing fixed_window- // here rendered a "wait 30 minutes" field holding a default the config never had.- const effectiveStrategy: EvaluationSettleStrategy =- evaluation.target_config.strategy ?? (isSessionTarget ? 'inactivity' : 'fixed_window')const supportsRunOutcomes = evaluationSupportsRunOutcomes(originalEvaluation)const isBooleanOutput = isBooleanEvaluationOutput(evaluation.output_type)const hasEditableCriteria = evaluationTypeHasEditableCriteria(evaluation.evaluation_type)@@ -191,6 +190,7 @@ export function AIObservabilityEvaluation(): JSX.Element {const focusTriggers = (): void => {setActiveTab('configuration')+ setFormStep('traffic')requestAnimationFrame(() => {triggersRef.current?.scrollIntoView({ behavior: 'smooth', block: 'center' })})@@ -225,7 +225,7 @@ export function AIObservabilityEvaluation(): JSX.Element {}const handleCancel = (): void => {- if (hasUnsavedChanges) {+ if (unsaved) {resetEvaluation()}push(evaluationBackTarget.path)@@ -275,7 +275,7 @@ export function AIObservabilityEvaluation(): JSX.Element {{evaluation.enabled ? 'Enabled' : 'Disabled'}</LemonTag>)}- {hasUnsavedChanges && <LemonTag type="warning">Unsaved changes</LemonTag>}+ {unsaved && <LemonTag type="warning">Unsaved changes</LemonTag>}</>)}</div>@@ -303,7 +303,7 @@ export function AIObservabilityEvaluation(): JSX.Element {</LemonButton>) : null}<LemonButton type="secondary" icon={<IconArrowLeft />} onClick={handleCancel}>- {hasUnsavedChanges ? 'Cancel' : 'Back'}+ {unsaved ? 'Cancel' : 'Back'}</LemonButton>{showSaveAction && (<AccessControlAction@@ -464,6 +464,17 @@ export function AIObservabilityEvaluation(): JSX.Element {)}</LemonBanner>)}+ {resultsCardsEnabled && (+ <>+ <EvaluationFirstResultStatus />+ {evaluationSupportsReportHistory(originalEvaluation) && (+ <EvaluationLatestReport+ evaluationId={evaluation.id}+ onOpenReports={() => setActiveTab('reports')}+ />+ )}+ </>+ )}{resultsCardsEnabled && !runsBackfillId && (<div className="mb-4"><EvaluationDetailResults evaluationId={evaluation.id} />@@ -508,410 +519,283 @@ export function AIObservabilityEvaluation(): JSX.Element {label: 'Configuration','data-attr': 'llma-evaluation-configuration-tab',content: (- <div className="max-w-4xl">- <div className="space-y-6">- {/* Basic Information */}- <div className="bg-bg-light border rounded p-4 @min-[40rem]/evaluation:p-6">- <h3 className="text-lg font-semibold mb-4">Basic information</h3>-- <div className="space-y-4">- <LemonField.Pure label="Name">- <LemonInput- value={evaluation.name}- onChange={setEvaluationName}- placeholder="e.g., Helpfulness Check"- maxLength={100}- />- </LemonField.Pure>-- {evaluationMethodOptions.length > 1 && (- <LemonField.Pure label="Method">- <LemonSelect- value={evaluation.evaluation_type}- onChange={(value) => setEvaluationType(value as EvaluationType)}- options={evaluationMethodOptions.map((option) => ({- ...option,- disabledReason:- !isNewEvaluation &&- ['numeric', 'categorical'].includes(- evaluation.output_type- ) &&- option.value === 'sentiment'- ? 'Create a new evaluation to change its output type.'- : undefined,- }))}- fullWidth+ // The step layout is centered, like the new evaluation page. The old form stays on the left.+ <div+ className={resultsCardsEnabled && !isNewEvaluation ? 'max-w-4xl mx-auto' : 'max-w-4xl'}+ >+ {resultsCardsEnabled && !isNewEvaluation ? (+ <EvaluationConfigurationForm />+ ) : (+ <div className="space-y-6">+ {/* Basic Information */}+ <div className="bg-bg-light border rounded p-4 @min-[40rem]/evaluation:p-6">+ <h3 className="text-lg font-semibold mb-4">Basic information</h3>++ <div className="space-y-4">+ <LemonField.Pure label="Name">+ <LemonInput+ value={evaluation.name}+ onChange={setEvaluationName}+ placeholder="e.g., Helpfulness Check"+ maxLength={100}/></LemonField.Pure>- )}- <p className="text-muted text-sm -mt-2">- {isSentiment ? (- 'Classify the sentiment of only the last user message on each matching generation event with a sentiment classifier, not LLM calls. The classifier is trained on English, so labels are unreliable for other languages. For a multilingual agent, use an LLM judge instead.'- ) : isHog ? (++ {evaluationMethodOptions.length > 1 && (+ <LemonField.Pure label="Method">+ <LemonSelect+ value={evaluation.evaluation_type}+ onChange={(value) =>+ setEvaluationType(value as EvaluationType)+ }+ options={evaluationMethodOptions.map((option) => ({+ ...option,+ disabledReason:+ !isNewEvaluation &&+ ['numeric', 'categorical'].includes(+ evaluation.output_type+ ) &&+ option.value === 'sentiment'+ ? 'Create a new evaluation to change its output type.'+ : undefined,+ }))}+ fullWidth+ />+ </LemonField.Pure>+ )}+ <p className="text-muted text-sm -mt-2">+ {isSentiment ? (+ 'Classify the sentiment of only the last user message on each matching generation event with a sentiment classifier, not LLM calls. The classifier is trained on English, so labels are unreliable for other languages. For a multilingual agent, use an LLM judge instead.'+ ) : isHog ? (+ <>+ Run deterministic{' '}+ <Link to="https://posthog.com/docs/hog" target="_blank">+ Hog code+ </Link>{' '}+ against{' '}+ {isSessionTarget+ ? 'the whole session'+ : evaluation.target === 'trace'+ ? 'the whole trace'+ : 'each generation'}+ . No LLM cost.+ </>+ ) : (+ `Use an LLM to evaluate ${+ isSessionTarget+ ? 'the whole session'+ : evaluation.target === 'trace'+ ? 'the whole trace'+ : 'each generation'+ } against a natural-language prompt.`+ )}+ </p>++ {!isSentiment && (<>- Run deterministic{' '}- <Link to="https://posthog.com/docs/hog" target="_blank">- Hog code- </Link>{' '}- against{' '}- {isSessionTarget- ? 'the whole session'- : evaluation.target === 'trace'- ? 'the whole trace'- : 'each generation'}- . No LLM cost.+ <LemonField.Pure label="Evaluate">+ <LemonSelect<EvaluationTarget>+ value={evaluation.target ?? 'generation'}+ onChange={setEvaluationTarget}+ options={[+ {+ value: 'generation',+ label: 'Each generation',+ },+ {+ value: 'trace',+ label: 'Whole trace',+ },+ ...(offersSessionTarget+ ? [+ {+ value: 'session' as const,+ label: 'Whole session',+ },+ ]+ : []),+ ]}+ fullWidth+ />+ </LemonField.Pure>+ <p className="text-muted text-sm -mt-2">+ {isSessionTarget+ ? 'Runs once per session on every trace it contains, after the session settles. Only fires for generations that have an $ai_session_id property.'+ : evaluation.target === 'trace'+ ? 'Runs once per trace on all of its events together, after it settles. Only fires for generations that have an $ai_trace_id property.'+ : 'Runs on each matching generation event individually, right after it is ingested.'}+ </p>+ <EvaluationSettleConfig /></>- ) : (- `Use an LLM to evaluate ${- isSessionTarget- ? 'the whole session'- : evaluation.target === 'trace'- ? 'the whole trace'- : 'each generation'- } against a natural-language prompt.`)}- </p>-- {!isSentiment && (- <>- <LemonField.Pure label="Evaluate">- <LemonSelect<EvaluationTarget>- value={evaluation.target ?? 'generation'}- onChange={setEvaluationTarget}++ {!isSentiment && (+ <LemonField.Pure label="Output type">+ <LemonSelect+ value={evaluation.output_type}options={[- {- value: 'generation',- label: 'Each generation',- },- {- value: 'trace',- label: 'Whole trace',- },- ...(offersSessionTarget- ? [- {- value: 'session' as const,- label: 'Whole session',- },- ]- : []),+ { value: 'boolean', label: 'Boolean' },+ { value: 'categorical', label: 'Categorical' },+ { value: 'numeric', label: 'Numeric score' },]}- fullWidth+ onChange={(value) =>+ setOutputType(+ value as 'boolean' | 'numeric' | 'categorical'+ )+ }+ disabledReason={+ !isNewEvaluation+ ? 'Create a new evaluation to change its output type.'+ : undefined+ }+ data-attr="llma-evaluation-output-type"/></LemonField.Pure>- <p className="text-muted text-sm -mt-2">- {isSessionTarget- ? 'Runs once per session on every trace it contains, after the session settles. Only fires for generations that have an $ai_session_id property.'- : evaluation.target === 'trace'- ? 'Runs once per trace on all of its events together, after it settles. Only fires for generations that have an $ai_trace_id property.'- : 'Runs on each matching generation event individually, right after it is ingested.'}- </p>- {isAggregateTarget && (- <>- {settlingStrategyEnabled && (- <LemonField.Pure label="Evaluate when">- <LemonSelect<EvaluationSettleStrategy>- value={effectiveStrategy}- onChange={setSettleStrategy}- options={[- {- value: 'fixed_window',- label: 'A fixed delay has passed',- },- {- value: 'inactivity',- label: isSessionTarget- ? 'The session goes quiet'- : 'The trace goes quiet',- },- ]}- fullWidth- />- </LemonField.Pure>- )}- {effectiveStrategy === 'fixed_window' ? (- <LemonField.Pure label="Wait before evaluating">- <div className="space-y-1">- <DurationPicker- value={- evaluation.target_config- .window_seconds ??- (isSessionTarget- ? DEFAULT_SESSION_WINDOW_SECONDS- : DEFAULT_TRACE_WINDOW_SECONDS)- }- onChange={(value) => {- if (- evaluation.target_config- .strategy === 'inactivity'- ) {- setSettleStrategy('fixed_window')- }- patchTargetConfig({- window_seconds: value,- })- }}- />- <p className="text-muted text-xs">- {isSessionTarget- ? "How long to wait after the first matching generation before pulling the whole session (10s to 7 days). Captured when the run is scheduled, so changing it won't affect runs already in flight."- : "How long to wait after the first matching generation before pulling the whole trace (10s–2h). Captured when the run is scheduled — changing it won't affect trace runs already in flight."}- </p>- </div>- </LemonField.Pure>- ) : (- <>- <LemonField.Pure label="Quiet period">- <div className="space-y-1">- <DurationPicker- value={- evaluation.target_config- .quiet_period_seconds ??- (isSessionTarget- ? DEFAULT_SESSION_QUIET_PERIOD_SECONDS- : DEFAULT_TRACE_QUIET_PERIOD_SECONDS)- }- onChange={(value) =>- patchTargetConfig({- quiet_period_seconds: value,- })- }- />- <p className="text-muted text-xs">- {isSessionTarget- ? 'Evaluate once the session has had no new activity for this long (10s to 24 hours). Sessions often pause for a while mid-conversation, so a longer quiet period means fewer verdicts on a session that was not finished.'- : 'Evaluate once the trace has had no new activity for this long (10s–30m).'}- </p>- </div>- </LemonField.Pure>- <LemonField.Pure label="Evaluate by">- <div className="space-y-1">- <DurationPicker- value={- evaluation.target_config- .max_age_seconds ??- (isSessionTarget- ? DEFAULT_SESSION_MAX_AGE_SECONDS- : DEFAULT_TRACE_MAX_AGE_SECONDS)- }- onChange={(value) =>- patchTargetConfig({- max_age_seconds: value,- })- }- />- <p className="text-muted text-xs">- {isSessionTarget- ? 'Always evaluate once the session is this old, even if it is still active (1m to 7 days).'- : "Always evaluate once the trace is this old, even if it's still active (1m–2h)."}- </p>- </div>- </LemonField.Pure>- </>- )}- </>- )}- </>- )}-- {!isSentiment && (- <LemonField.Pure label="Output type">- <LemonSelect- value={evaluation.output_type}- options={[- { value: 'boolean', label: 'Boolean' },- { value: 'categorical', label: 'Categorical' },- { value: 'numeric', label: 'Numeric score' },- ]}- onChange={(value) =>- setOutputType(- value as 'boolean' | 'numeric' | 'categorical'- )- }- disabledReason={- !isNewEvaluation- ? 'Create a new evaluation to change its output type.'- : undefined- }- data-attr="llma-evaluation-output-type"+ )}+ {evaluation.output_type === 'categorical' && (+ <CategoricalEvaluationConfig+ config={evaluation.output_config}+ onChange={patchOutputConfig}+ />+ )}+ {evaluation.output_type === 'numeric' && (+ <NumericEvaluationConfig+ config={evaluation.output_config}+ onChange={patchOutputConfig}+ requiresBounds={numericBoundsRequired}+ />+ )}+ <LemonField.Pure label="Description (optional)">+ <LemonTextArea+ value={evaluation.description || ''}+ onChange={setEvaluationDescription}+ placeholder="Describe what this evaluation checks for..."+ rows={2}+ maxLength={500}/></LemonField.Pure>- )}- {evaluation.output_type === 'categorical' && (- <CategoricalEvaluationConfig- config={evaluation.output_config}- onChange={patchOutputConfig}- />- )}- {evaluation.output_type === 'numeric' && (- <NumericEvaluationConfig- config={evaluation.output_config}- onChange={patchOutputConfig}- requiresBounds={numericBoundsRequired}- />- )}- <LemonField.Pure label="Description (optional)">- <LemonTextArea- value={evaluation.description || ''}- onChange={setEvaluationDescription}- placeholder="Describe what this evaluation checks for..."- rows={2}- maxLength={500}- />- </LemonField.Pure>-- <div className="flex flex-wrap items-center gap-2">- <Tooltip- title={canEnableReason}- visible={canEnableReason ? undefined : false}- >- <span>- <LemonSwitch- checked={evaluation.enabled}- onChange={setEvaluationEnabled}- label="Enable evaluation"- disabled={!canEnable && !evaluation.enabled}- />++ <div className="flex flex-wrap items-center gap-2">+ <Tooltip+ title={canEnableReason}+ visible={canEnableReason ? undefined : false}+ >+ <span>+ <LemonSwitch+ checked={evaluation.enabled}+ onChange={setEvaluationEnabled}+ label="Enable evaluation"+ disabled={!canEnable && !evaluation.enabled}+ />+ </span>+ </Tooltip>+ <span className="text-muted text-sm">+ {!canEnable && !evaluation.enabled+ ? canEnableReason+ : evaluation.enabled+ ? 'This evaluation will run automatically based on triggers'+ : 'This evaluation is paused and will not run'}</span>- </Tooltip>- <span className="text-muted text-sm">- {!canEnable && !evaluation.enabled- ? canEnableReason- : evaluation.enabled- ? 'This evaluation will run automatically based on triggers'- : 'This evaluation is paused and will not run'}- </span>- </div>+ </div>- {isBooleanOutput && (- <LemonField.Pure- label={- <div className="flex items-center gap-1">- <span>Allow N/A responses</span>- <Tooltip- title={- isHog- ? 'When enabled, returning null from your Hog code means "not applicable" instead of being treated as an error.'- : 'Sometimes forcing a True or False is not enough and you want the LLM to decide if the evaluation is applicable or not. Enable this when the evaluation criteria may not apply to all generations.'- }- >- <IconInfo className="text-muted text-base" />- </Tooltip>+ {isBooleanOutput && (+ <LemonField.Pure+ label={+ <div className="flex items-center gap-1">+ <span>Allow N/A responses</span>+ <Tooltip+ title={+ isHog+ ? 'When enabled, returning null from your Hog code means "not applicable" instead of being treated as an error.'+ : 'Sometimes forcing a True or False is not enough and you want the LLM to decide if the evaluation is applicable or not. Enable this when the evaluation criteria may not apply to all generations.'+ }+ >+ <IconInfo className="text-muted text-base" />+ </Tooltip>+ </div>+ }+ >+ <div className="flex flex-wrap items-center gap-2">+ <LemonSwitch+ checked={evaluation.output_config.allows_na ?? false}+ onChange={setAllowsNA}+ />+ <span className="text-muted text-sm">+ {evaluation.output_config.allows_na+ ? isHog+ ? 'Returning null means "Not Applicable"'+ : 'Evaluation can return "Not Applicable" when criteria doesn\'t apply'+ : isHog+ ? 'Evaluation must return true or false'+ : 'Evaluation returns true or false'}+ </span></div>- }- >- <div className="flex flex-wrap items-center gap-2">- <LemonSwitch- checked={evaluation.output_config.allows_na ?? false}- onChange={setAllowsNA}- />- <span className="text-muted text-sm">- {evaluation.output_config.allows_na- ? isHog- ? 'Returning null means "Not Applicable"'- : 'Evaluation can return "Not Applicable" when criteria doesn\'t apply'- : isHog- ? 'Evaluation must return true or false'- : 'Evaluation returns true or false'}- </span>- </div>- </LemonField.Pure>- )}-- {isBooleanOutput && (- <LemonField.Pure- label={- <div className="flex items-center gap-1">- <span>A true result flags a problem</span>- <Tooltip title="Turn this on when the evaluation looks for something you don't want, like a struggling user or a hallucination. A true result then counts as a fail in reports and pass rates. The evaluation results themselves don't change.">- <IconInfo className="text-muted text-base" />- </Tooltip>+ </LemonField.Pure>+ )}++ {isBooleanOutput && (+ <LemonField.Pure+ label={+ <div className="flex items-center gap-1">+ <span>A true result flags a problem</span>+ <Tooltip title="Turn this on when the evaluation looks for something you don't want, like a struggling user or a hallucination. A true result then counts as a fail in reports and pass rates. The evaluation results themselves don't change.">+ <IconInfo className="text-muted text-base" />+ </Tooltip>+ </div>+ }+ >+ <div className="flex flex-wrap items-center gap-2">+ <LemonSwitch+ checked={+ evaluation.output_config.true_is_failure ?? false+ }+ onChange={setTrueIsFailure}+ data-attr="llma-evaluation-true-is-failure-switch"+ />+ <span className="text-muted text-sm">+ {evaluation.output_config.true_is_failure+ ? 'A true result counts as a fail'+ : 'A true result counts as a pass'}+ </span></div>- }- >- <div className="flex flex-wrap items-center gap-2">- <LemonSwitch- checked={evaluation.output_config.true_is_failure ?? false}- onChange={setTrueIsFailure}- data-attr="llma-evaluation-true-is-failure-switch"- />- <span className="text-muted text-sm">- {evaluation.output_config.true_is_failure- ? 'A true result counts as a fail'- : 'A true result counts as a pass'}- </span>- </div>- </LemonField.Pure>- )}+ </LemonField.Pure>+ )}+ </div></div>- </div>- {/* Prompt / Code Configuration */}- {hasEditableCriteria && (- <div className="bg-bg-light border rounded p-4 @min-[40rem]/evaluation:p-6">- <h3 className="text-lg font-semibold mb-4">- {isHog ? 'Evaluation code' : 'Evaluation prompt'}- </h3>- {isHog ? <EvaluationCodeEditor /> : <EvaluationPromptEditor />}- </div>- )}+ {/* Prompt / Code Configuration */}+ {hasEditableCriteria && (+ <div className="bg-bg-light border rounded p-4 @min-[40rem]/evaluation:p-6">+ <h3 className="text-lg font-semibold mb-4">+ {isHog ? 'Evaluation code' : 'Evaluation prompt'}+ </h3>+ {isHog ? <EvaluationCodeEditor /> : <EvaluationPromptEditor />}+ </div>+ )}- {/* Judge Model Configuration (LLM judge only) */}- {evaluationTypeUsesModelConfiguration(evaluation.evaluation_type) && (- <EvaluationModelPicker />- )}+ {/* Judge Model Configuration (LLM judge only) */}+ {evaluationTypeUsesModelConfiguration(evaluation.evaluation_type) && (+ <EvaluationModelPicker />+ )}- {/* Trigger Configuration */}- <div- ref={triggersRef}- className="bg-bg-light border rounded p-4 @min-[40rem]/evaluation:p-6"- >- <h3 className="text-lg font-semibold mb-4">Triggers</h3>- <p className="text-muted text-sm mb-4">- The evaluation runs on generations that match any one of these condition- sets. Within a set, all filters must match, and sampling decides how many of- the matching generations get evaluated.- {evaluation.target === 'trace' && (- <>- {' '}- Conditions match individual generations. The whole trace is- evaluated once any of its generations matches, and sampling applies- per trace.- </>- )}- </p>- <EvaluationTriggers />- <div className="bg-bg-light border rounded p-3 text-sm mt-6">- <h4 className="font-semibold mb-2">Examples:</h4>- <ul className="space-y-1 text-muted list-disc list-inside">- <li>- <strong>10% of all generations:</strong> Set 10% sampling with no- filter conditions- </li>- <li>- <strong>5% of GPT-4 generations:</strong> Set 5% sampling with- $ai_model = "gpt-4o"- </li>- <li>- <strong>Exclude internal users:</strong> Set 100% sampling with- person property is_internal ≠ true- </li>- <li>- <strong>High-cost generations:</strong> Set 100% sampling with- $ai_total_cost_usd > 0.01- </li>- </ul>+ {/* Trigger Configuration */}+ <div+ ref={triggersRef}+ className="bg-bg-light border rounded p-4 @min-[40rem]/evaluation:p-6"+ >+ <h3 className="text-lg font-semibold mb-4">Triggers</h3>+ <EvaluationTriggersPanel /></div>- </div>- {/* Scheduled Reports (inline config for new evaluations) */}- {isNewEvaluation && isReportableEvaluation && (- <EvaluationReportConfig evaluationId="new" />- )}- </div>+ {/* Scheduled Reports (inline config for new evaluations) */}+ {isNewEvaluation && isReportableEvaluation && (+ <EvaluationReportConfig evaluationId="new" />+ )}+ </div>+ )}- {/* Scheduled Reports (for existing evaluations, outside the form) */}- {!isNewEvaluation && isReportableEvaluation && (+ {/* Scheduled Reports (for existing evaluations, outside the form). The new form sets them in its sentence. */}+ {!resultsCardsEnabled && !isNewEvaluation && isReportableEvaluation && (<div className="mt-6"><EvaluationReportConfig evaluationId={evaluation.id} /></div>diff --git a/products/ai_observability/frontend/evaluations/components/CategoricalEvaluationConfig.tsx b/products/ai_observability/frontend/evaluations/components/CategoricalEvaluationConfig.tsxindex 13679ed42102..3e7b33de72ab 100644--- a/products/ai_observability/frontend/evaluations/components/CategoricalEvaluationConfig.tsx+++ b/products/ai_observability/frontend/evaluations/components/CategoricalEvaluationConfig.tsx@@ -1,7 +1,5 @@-import { useId } from 'react'-import { IconPlus, IconTrash } from '@posthog/icons'-import { LemonButton, LemonCheckbox, LemonInput, LemonSelect, LemonSwitch, LemonTable } from '@posthog/lemon-ui'+import { LemonButton, LemonCheckbox, LemonInput, LemonSegmentedButton, LemonSwitch } from '@posthog/lemon-ui'import { LemonField } from 'lib/lemon-ui/LemonField'@@ -15,186 +13,152 @@ export function CategoricalEvaluationConfig({config: EvaluationOutputConfigonChange: (patch: EvaluationOutputConfig) => void}): JSX.Element {- const id = useId()const options = config.options ?? []const rule = config.passing_rule && 'categories' in config.passing_rule ? config.passing_rule : null+ const updateOption = (index: number, patch: Partial<{ key: string; label: string }>): void =>+ onChange({ options: options.map((item, i) => (i === index ? { ...item, ...patch } : item)) })+return (- <div className="space-y-4">- <LemonField.Pure- label="Categories per result"- htmlFor={`${id}-selection`}- help="Choose whether each result can contain one category or several."- >- <LemonSelect- id={`${id}-selection`}- data-attr="llma-evaluation-categorical-selection"+ <div className="flex flex-col gap-4">+ <LemonField.Pure label="Categories per result">+ <LemonSegmentedButtonvalue={config.selection_mode ?? 'single'}+ onChange={(selection_mode) => onChange({ selection_mode })}options={[{ value: 'single', label: 'One category' },- { value: 'multiple', label: 'Multiple categories' },+ { value: 'multiple', label: 'Several categories' },]}- onChange={(selection_mode) => onChange({ selection_mode })}+ fullWidth+ data-attr="llma-evaluation-categorical-selection"/></LemonField.Pure>- <LemonField.Pure label="Categories" error={categoricalOptionsError(config)}>- <LemonTable- dataSource={options}- size="small"- embedded- stealth- inset- tableLayout="fixed"- uppercaseHeader={false}- emptyState="Add a category to define the possible results."- columns={[- {- title: 'Label',- key: 'label',- render: (_, option, index) => (- <LemonInput- aria-label={`Category ${index + 1} label`}- data-attr="llma-evaluation-categorical-label"- placeholder="Resolved"- fullWidth- value={option.label}- maxLength={256}- onChange={(label) =>- onChange({- options: options.map((item, i) =>- i === index ? { ...item, label } : item- ),- })- }- />- ),- },- {- title: 'Key',- key: 'key',- render: (_, option, index) => (- <LemonInput- aria-label={`Category ${index + 1} key`}- data-attr="llma-evaluation-categorical-key"- placeholder="resolved"- fullWidth- value={option.key}- maxLength={128}- onChange={(key) =>- onChange({- options: options.map((item, i) => (i === index ? { ...item, key } : item)),- ...(rule- ? {- passing_rule: {- categories: rule.categories.map((value) =>- value === option.key ? key : value- ),- },- }- : {}),- })- }- />- ),- },- {- key: 'actions',- width: 48,- render: (_, option, index) => (- <LemonButton- icon={<IconTrash />}- size="small"- tooltip="Remove category"- aria-label={`Remove category ${option.label || index + 1}`}- data-attr="llma-evaluation-categorical-remove"- onClick={() =>+ <LemonField.Pure+ label="Categories"+ error={categoricalOptionsError(config)}+ help="Results store the key. Keep a key unchanged to keep its past results."+ >+ <div className="flex flex-col gap-2">+ {options.length === 0 && (+ <span className="text-sm text-secondary">Add a category to define the possible results.</span>+ )}+ {options.map((option, index) => (+ <div key={index} className="flex items-center gap-2">+ <LemonInput+ className="flex-1 min-w-0"+ aria-label={`Category ${index + 1} label`}+ data-attr="llma-evaluation-categorical-label"+ placeholder="Label, for example Resolved"+ value={option.label}+ maxLength={256}+ onChange={(label) => updateOption(index, { label })}+ />+ <LemonInput+ className="w-36 shrink-0 font-mono"+ aria-label={`Category ${index + 1} key`}+ data-attr="llma-evaluation-categorical-key"+ placeholder="key"+ value={option.key}+ maxLength={128}+ onChange={(key) =>+ onChange({+ options: options.map((item, i) => (i === index ? { ...item, key } : item)),+ ...(rule+ ? {+ passing_rule: {+ categories: rule.categories.map((value) =>+ value === option.key ? key : value+ ),+ },+ }+ : {}),+ })+ }+ />+ {rule && (+ <LemonCheckbox+ label="Passes"+ data-attr="llma-evaluation-categorical-passing-category"+ disabledReason={option.key ? undefined : 'Add a key first'}+ checked={!!option.key && rule.categories.includes(option.key)}+ onChange={(checked) =>onChange({- options: options.filter((_, i) => i !== index),- ...(rule- ? {- passing_rule: {- categories: rule.categories.filter(- (key) => key !== option.key- ),- },- }- : {}),+ passing_rule: {+ categories: checked+ ? [...rule.categories, option.key]+ : rule.categories.filter((key) => key !== option.key),+ },})}/>- ),- },- ]}- />- <LemonButton- type="secondary"- size="small"- icon={<IconPlus />}- className="self-start"- data-attr="llma-evaluation-categorical-add"- disabledReason={- options.length >= MAX_CATEGORICAL_OPTIONS- ? `You can add up to ${MAX_CATEGORICAL_OPTIONS} categories.`- : undefined- }- onClick={() => onChange({ options: [...options, { key: '', label: '' }] })}- >- Add category- </LemonButton>+ )}+ <LemonButton+ icon={<IconTrash />}+ size="small"+ tooltip="Remove category"+ aria-label={`Remove category ${option.label || index + 1}`}+ data-attr="llma-evaluation-categorical-remove"+ onClick={() =>+ onChange({+ options: options.filter((_, i) => i !== index),+ ...(rule+ ? {+ passing_rule: {+ categories: rule.categories.filter((key) => key !== option.key),+ },+ }+ : {}),+ })+ }+ />+ </div>+ ))}+ <div>+ <LemonButton+ type="secondary"+ size="small"+ icon={<IconPlus />}+ data-attr="llma-evaluation-categorical-add"+ disabledReason={+ options.length >= MAX_CATEGORICAL_OPTIONS+ ? `You can add up to ${MAX_CATEGORICAL_OPTIONS} categories.`+ : undefined+ }+ onClick={() => onChange({ options: [...options, { key: '', label: '' }] })}+ >+ Add category+ </LemonButton>+ </div>+ </div></LemonField.Pure>- <p className="text-muted text-sm">- Results store category keys. Keep keys unchanged to preserve historical results.- </p><LemonSwitchlabel="Allow N/A responses"data-attr="llma-evaluation-categorical-allows-na"checked={config.allows_na ?? false}onChange={(allows_na) => onChange({ allows_na })}+ bordered+ fullWidth/>- <LemonField.Pure error={categoricalPassingRuleError(config)}>+ <LemonField.Pure+ error={categoricalPassingRuleError(config)}+ help={+ rule+ ? `A result passes when every category it returns is marked as passing.${+ config.selection_mode === 'multiple'+ ? ' An empty selection passes only when no category is marked.'+ : ''+ } Changing this also changes how past results count.`+ : 'Without a passing rule, results show their categories but no pass rate or reports.'+ }+ ><LemonSwitchlabel="Set a passing rule"data-attr="llma-evaluation-categorical-passing-rule"checked={!!rule}onChange={(enabled) => onChange({ passing_rule: enabled ? { categories: [] } : null })}+ bordered+ fullWidth/>- {rule && (- <LemonField.Pure label="Passing categories" className="mt-2">- {options- .filter((option) => option.key)- .map((option, index) => (- <LemonCheckbox- key={index}- label={option.label || option.key}- data-attr="llma-evaluation-categorical-passing-category"- checked={rule.categories.includes(option.key)}- onChange={(checked) =>- onChange({- passing_rule: {- categories: checked- ? [...rule.categories, option.key]- : rule.categories.filter((key) => key !== option.key),- },- })- }- />- ))}- </LemonField.Pure>- )}</LemonField.Pure>- {rule ? (- <p className="text-muted text-sm">- Every returned category must be marked as passing.- {config.selection_mode === 'multiple'- ? ' An empty selection passes only when no categories are marked as passing.'- : ''}{' '}- Changing this rule also updates how historical results count as passes.- </p>- ) : (- <p className="text-muted text-sm">- Results show their categories without a pass or fail status. Add a passing rule to enable pass rates- and reports.- </p>- )}</div>)}diff --git a/products/ai_observability/frontend/evaluations/components/DecisionThresholdInput.tsx b/products/ai_observability/frontend/evaluations/components/DecisionThresholdInput.tsxnew file mode 100644index 000000000000..d99c6a549730--- /dev/null+++ b/products/ai_observability/frontend/evaluations/components/DecisionThresholdInput.tsx@@ -0,0 +1,32 @@+import { useActions, useValues } from 'kea'++import { LemonInput } from '@posthog/lemon-ui'++import { llmEvaluationLogic } from '../llmEvaluationLogic'++/** The probability, as a percentage, at which a decision model answers true. */+export function DecisionThresholdInput({ id }: { id?: string }): JSX.Element | null {+ const { evaluation } = useValues(llmEvaluationLogic)+ const { patchOutputConfig } = useActions(llmEvaluationLogic)++ if (!evaluation) {+ return null+ }+ return (+ <LemonInput+ id={id}+ type="number"+ className="w-24"+ value={Number(((evaluation.output_config.decision_threshold ?? 0.5) * 100).toPrecision(15))}+ onChange={(percentage) =>+ patchOutputConfig({ decision_threshold: percentage == null ? NaN : percentage / 100 })+ }+ min={0}+ max={100}+ step="any"+ suffix={<span>%</span>}+ aria-label="Decision threshold"+ data-attr="llma-evaluation-decision-threshold"+ />+ )+}diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationCodeEditor.test.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationCodeEditor.test.tsxindex c4e1b606c33c..1345e3c5a6d9 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationCodeEditor.test.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationCodeEditor.test.tsx@@ -68,6 +68,8 @@ function renderEditor(): void {</BindLogic></Provider>)+ // The examples and tips sit in a fold that starts closed.+ fireEvent.click(screen.getByText('Examples and reference'))}describe('EvaluationCodeEditor', () => {diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationCodeEditor.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationCodeEditor.tsxindex 166e1b28b60e..be9eda21a8a6 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationCodeEditor.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationCodeEditor.tsx@@ -1,13 +1,11 @@import { useActions, useValues } from 'kea'import { Fragment } from 'react'-import { IconCheck, IconExternal, IconMinus, IconSparkles, IconWarning, IconX } from '@posthog/icons'-import { LemonButton, LemonTable, LemonTag, Link, Tooltip } from '@posthog/lemon-ui'+import { IconCheck, IconExternal, IconMinus, IconWarning, IconX } from '@posthog/icons'+import { LemonButton, LemonCollapse, LemonTable, LemonTag, Link, Tooltip } from '@posthog/lemon-ui'import { CodeEditorResizeable } from 'lib/monaco/CodeEditorResizable'-import { todayShellLogic } from '~/layout/today/todayShellLogic'-import { useOpenAi } from '~/scenes/max/useOpenAi'import { urls } from '~/scenes/urls'import { EvaluationResultTag } from '../../components/EvaluationResultTag'@@ -196,6 +194,17 @@ const HOG_EVAL_GLOBALS_BY_TARGET = {session: HOG_EVAL_COMMON_GLOBALS_BY_TARGET.session,}+/** One line of a sample's text, with the full text on hover. */+function SamplePreview({ text }: { text: string }): JSX.Element {+ return text ? (+ <Tooltip title={<span className="whitespace-pre-wrap">{text}</span>}>+ <div className="truncate text-sm max-w-80">{text}</div>+ </Tooltip>+ ) : (+ <span className="text-sm text-muted">Empty</span>+ )+}+export function HogTestResultsPanel(): JSX.Element | null {const { hogTestResults, hogTestResultsLoading, hogTestMessage, evaluation } = useValues(llmEvaluationLogic)const { clearHogTestResults } = useActions(llmEvaluationLogic)@@ -319,14 +328,27 @@ export function HogTestResultsPanel(): JSX.Element | null {},},{- title: 'Reasoning',- key: 'reasoning',- render: (_, row) => (- <div className="truncate text-sm">- {row.reasoning || row.error || <span className="text-muted italic">none</span>}- </div>- ),+ title: 'Input',+ key: 'input',+ render: (_, row) => <SamplePreview text={row.input_preview} />,+ },+ {+ title: 'Output',+ key: 'output',+ render: (_, row) => <SamplePreview text={row.output_preview} />,},+ // Most code returns no reasoning, so the column shows only when a row has something in it.+ ...(hogTestResults?.some((row) => row.reasoning || row.error)+ ? [+ {+ title: 'Reasoning',+ key: 'reasoning',+ render: (_: unknown, row: TestHogResultItemApi) => (+ <SamplePreview text={row.reasoning || row.error || ''} />+ ),+ },+ ]+ : []),{title: '',key: 'link',@@ -372,8 +394,6 @@ export function HogTestResultsPanel(): JSX.Element | null {export function EvaluationCodeEditor(): JSX.Element {const { evaluation, hogTestResultsLoading } = useValues(llmEvaluationLogic)const { setHogSource, setAllowsNA, testHogOnSample } = useActions(llmEvaluationLogic)- const { openAi } = useOpenAi()- const { todayRailEnabled } = useValues(todayShellLogic)if (!evaluation || evaluation.evaluation_type !== 'hog') {return <div>Loading...</div>@@ -432,27 +452,6 @@ export function EvaluationCodeEditor(): JSX.Element {Test on sample</LemonButton></Tooltip>- {!todayRailEnabled && (- <LemonButton- type="secondary"- size="xsmall"- icon={<IconSparkles />}- onClick={() =>- openAi('Help me write Hog evaluation code for this evaluation', {- evaluation: {- id: evaluation.id,- name: evaluation.name,- description: evaluation.description,- evaluation_type: evaluation.evaluation_type,- hog_source: evaluation.evaluation_config.source,- },- })- }- data-attr="llma-evaluation-generate-with-ai"- >- Generate with AI- </LemonButton>- )}</div><div className="flex items-center gap-2"><span>Expected output:</span>@@ -475,108 +474,129 @@ export function EvaluationCodeEditor(): JSX.Element {<HogTestResultsPanel />- <div className="bg-bg-light border rounded p-3">- <div className="flex items-center justify-between mb-2">- <div>- <h4 className="text-sm font-semibold m-0">Examples</h4>- <p className="text-xs text-muted m-0">Starter examples to play with</p>- </div>- <Link to="https://posthog.com/docs/hog" target="_blank" className="text-sm">- Hog language reference <IconExternal className="inline text-xs" />- </Link>- </div>- <div className="flex flex-wrap gap-1.5 mb-3">- {(evaluation.output_type === 'categorical'- ? (evaluation.output_config.options ?? [])- .slice(0, 2)- .map(({ key, label }) => ({ label, source: `return ['${key}'];` }))- : evaluation.output_type === 'numeric'- ? [- { label: 'Latency', source: 'return target.total_latency_seconds;' },- { label: 'Cost', source: 'return target.total_cost_usd;' },- ]- : HOG_EVAL_EXAMPLES- ).map((example) => (- <LemonButton- key={example.label}- type="secondary"- size="xsmall"- onClick={() => {- if (evaluation.output_type === 'numeric') {- setAllowsNA(true)- }- setHogSource(example.source)- }}- >- {example.label}- </LemonButton>- ))}- </div>- {evaluation.output_type === 'numeric' && (- <p className="text-xs text-muted mb-3">- These examples enable N/A for missing cost or latency measurements.- </p>- )}- <h4 className="text-sm font-semibold mb-2">Available globals</h4>- <dl className="grid grid-cols-[max-content_minmax(0,1fr)] items-start gap-x-3 gap-y-2 text-sm text-muted">- {HOG_EVAL_COMMON_GLOBAL_DEFINITIONS.map((globalDefinition) => (- <Fragment key={globalDefinition.name}>- <dt>- <code className={GLOBAL_NAME_CODE_CLASS}>{globalDefinition.name}</code>- </dt>- <dd className="m-0">- <p className="m-0">- {resolveHogGlobalDescription(globalDefinition.description, evaluation.target)}- </p>- <dl className="grid grid-cols-[max-content_minmax(0,1fr)] items-baseline gap-x-2 gap-y-1 mt-1.5">- {globalDefinition.fields.map((field) => (- <Fragment key={field.name}>+ {/* Folded by default, so the editor and its test action stay close to the rest of the form. */}+ <LemonCollapse+ size="small"+ panels={[+ {+ key: 'reference',+ header: 'Examples and reference',+ dataAttr: 'llma-evaluation-hog-reference',+ content: (+ <div>+ <div className="flex items-center justify-between mb-2">+ <div>+ <h4 className="text-sm font-semibold m-0">Examples</h4>+ <p className="text-xs text-muted m-0">Starter examples to play with</p>+ </div>+ <Link to="https://posthog.com/docs/hog" target="_blank" className="text-sm">+ Hog language reference <IconExternal className="inline text-xs" />+ </Link>+ </div>+ <div className="flex flex-wrap gap-1.5 mb-3">+ {(evaluation.output_type === 'categorical'+ ? (evaluation.output_config.options ?? [])+ .slice(0, 2)+ .map(({ key, label }) => ({ label, source: `return ['${key}'];` }))+ : evaluation.output_type === 'numeric'+ ? [+ { label: 'Latency', source: 'return target.total_latency_seconds;' },+ { label: 'Cost', source: 'return target.total_cost_usd;' },+ ]+ : HOG_EVAL_EXAMPLES+ ).map((example) => (+ <LemonButton+ key={example.label}+ type="secondary"+ size="xsmall"+ onClick={() => {+ if (evaluation.output_type === 'numeric') {+ setAllowsNA(true)+ }+ setHogSource(example.source)+ }}+ >+ {example.label}+ </LemonButton>+ ))}+ </div>+ {evaluation.output_type === 'numeric' && (+ <p className="text-xs text-muted mb-3">+ These examples enable N/A for missing cost or latency measurements.+ </p>+ )}+ <h4 className="text-sm font-semibold mb-2">Available globals</h4>+ <dl className="grid grid-cols-[max-content_minmax(0,1fr)] items-start gap-x-3 gap-y-2 text-sm text-muted">+ {HOG_EVAL_COMMON_GLOBAL_DEFINITIONS.map((globalDefinition) => (+ <Fragment key={globalDefinition.name}><dt>- <code className={PROPERTY_NAME_CODE_CLASS}>{field.name}</code>+ <code className={GLOBAL_NAME_CODE_CLASS}>{globalDefinition.name}</code></dt><dd className="m-0">- {resolveHogGlobalDescription(field.description, evaluation.target)}+ <p className="m-0">+ {resolveHogGlobalDescription(+ globalDefinition.description,+ evaluation.target+ )}+ </p>+ <dl className="grid grid-cols-[max-content_minmax(0,1fr)] items-baseline gap-x-2 gap-y-1 mt-1.5">+ {globalDefinition.fields.map((field) => (+ <Fragment key={field.name}>+ <dt>+ <code className={PROPERTY_NAME_CODE_CLASS}>+ {field.name}+ </code>+ </dt>+ <dd className="m-0">+ {resolveHogGlobalDescription(+ field.description,+ evaluation.target+ )}+ </dd>+ </Fragment>+ ))}+ </dl></dd></Fragment>))}</dl>- </dd>- </Fragment>- ))}- </dl>- <h4 className="text-sm font-semibold mt-3 mb-2">Tips</h4>- <ul className="text-sm text-muted space-y-1 list-disc list-inside">- <li>- {evaluation.output_type === 'categorical' ? (- <span>- Return configured category keys in a list. An empty list is valid for multiple- selection; null means N/A when allowed.- </span>- ) : evaluation.output_type === 'numeric' ? (- <span>Return a number within the configured bounds.</span>- ) : (- <>- Return <code>true</code> ({trueIsFailure ? 'fail' : 'pass'}) or <code>false</code> (- {trueIsFailure ? 'pass' : 'fail'})- {evaluation.output_config.allows_na ? (- <>- {' '}- or <code>null</code> (N/A)- </>- ) : null}- </>- )}- </li>- {evaluation.output_config.allows_na && (- <li>- Return <code>null</code> when the evaluation criteria doesn't apply- </li>- )}- <li>- Use <code>print()</code> to add reasoning (visible in evaluation runs)- </li>- </ul>- </div>+ <h4 className="text-sm font-semibold mt-3 mb-2">Tips</h4>+ <ul className="text-sm text-muted space-y-1 list-disc list-inside">+ <li>+ {evaluation.output_type === 'categorical' ? (+ <span>+ Return configured category keys in a list. An empty list is valid for+ multiple selection; null means N/A when allowed.+ </span>+ ) : evaluation.output_type === 'numeric' ? (+ <span>Return a number within the configured bounds.</span>+ ) : (+ <>+ Return <code>true</code> ({trueIsFailure ? 'fail' : 'pass'}) or{' '}+ <code>false</code> ({trueIsFailure ? 'pass' : 'fail'})+ {evaluation.output_config.allows_na ? (+ <>+ {' '}+ or <code>null</code> (N/A)+ </>+ ) : null}+ </>+ )}+ </li>+ {evaluation.output_config.allows_na && (+ <li>+ Return <code>null</code> when the evaluation criteria doesn't apply+ </li>+ )}+ <li>+ Use <code>print()</code> to add reasoning (visible in evaluation runs)+ </li>+ </ul>+ </div>+ ),+ },+ ]}+ /></div>)}diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationFirstResultStatus.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationFirstResultStatus.tsxnew file mode 100644index 000000000000..f7ccf09a96cf--- /dev/null+++ b/products/ai_observability/frontend/evaluations/components/EvaluationFirstResultStatus.tsx@@ -0,0 +1,103 @@+import { BindLogic, useActions, useMountedLogic, useValues } from 'kea'++import { LemonBanner } from '@posthog/lemon-ui'++import { dayjs } from 'lib/dayjs'+import { humanFriendlyDuration } from 'lib/utils/durations'++import { settleStrategy } from '../constants'+import { VOLUME_LOOKBACK_DAYS, evaluationTarget, runRate } from '../evaluationVolume'+import { evaluationVolumeLogic } from '../evaluationVolumeLogic'+import { DEFAULT_SESSION_WINDOW_SECONDS, DEFAULT_TRACE_WINDOW_SECONDS, llmEvaluationLogic } from '../llmEvaluationLogic'+import type { EvaluationConfig } from '../types'++const UNIT_NOUNS = { generation: 'generations', trace: 'traces', session: 'sessions' } as const++/**+ * Says why a new evaluation has no results yet: it is paused, nothing matches its triggers, or the first run is+ * on its way. Shows only while the default range holds no runs at all and the evaluation is younger than that range,+ * because only then does it mean the evaluation has never run.+ */+export function EvaluationFirstResultStatus(): JSX.Element | null {+ const { originalEvaluation, runsStats, runsStatsLoading, runsBackfillId, runsDateRange } =+ useValues(llmEvaluationLogic)+ const { props } = useMountedLogic(llmEvaluationLogic)++ const neverRan =+ !!originalEvaluation &&+ !runsStatsLoading &&+ runsStats?.total === 0 &&+ !runsBackfillId &&+ runsDateRange.date_from === `-${VOLUME_LOOKBACK_DAYS}d` &&+ !runsDateRange.date_to &&+ dayjs(originalEvaluation.created_at).isAfter(dayjs().subtract(VOLUME_LOOKBACK_DAYS, 'day'))+ if (!neverRan || !originalEvaluation) {+ return null+ }+ if (!originalEvaluation.enabled) {+ return <PausedBanner evaluation={originalEvaluation} />+ }+ return (+ <BindLogic logic={evaluationVolumeLogic} props={props}>+ <div>+ <FirstResultBanner evaluation={originalEvaluation} />+ </div>+ </BindLogic>+ )+}++function PausedBanner({ evaluation }: { evaluation: EvaluationConfig }): JSX.Element {+ const { setActiveTab } = useActions(llmEvaluationLogic)+ return (+ <LemonBanner+ type="info"+ className="mb-4"+ action={{+ children: 'Open configuration',+ onClick: () => setActiveTab('configuration'),+ 'data-attr': 'llma-evaluation-first-result-triggers',+ }}+ >+ <span className="font-semibold">No results yet, because this evaluation is paused.</span> Turn it on in its+ configuration to start checking new {UNIT_NOUNS[evaluationTarget(evaluation)]}.+ </LemonBanner>+ )+}++function FirstResultBanner({ evaluation }: { evaluation: EvaluationConfig }): JSX.Element {+ const { volume } = useValues(evaluationVolumeLogic)+ const { setActiveTab } = useActions(llmEvaluationLogic)+ const target = evaluationTarget(evaluation)+ const units = UNIT_NOUNS[target]++ // Matches, not runs: low sampling rounds a week of runs down to zero while traffic still matches.+ if (volume && volume.matchingUnits === 0) {+ return (+ <LemonBanner+ type="warning"+ className="mb-4"+ action={{+ children: 'Check the triggers',+ onClick: () => setActiveTab('configuration'),+ 'data-attr': 'llma-evaluation-first-result-triggers',+ }}+ >+ <span className="font-semibold">No results yet.</span> Nothing matched its triggers in the last{' '}+ {VOLUME_LOOKBACK_DAYS} days, so it runs only once matching {units} arrive.+ </LemonBanner>+ )+ }+ const waitSeconds =+ target !== 'generation' && settleStrategy(evaluation.target_config.strategy, target) === 'fixed_window'+ ? (evaluation.target_config.window_seconds ??+ (target === 'session' ? DEFAULT_SESSION_WINDOW_SECONDS : DEFAULT_TRACE_WINDOW_SECONDS))+ : null+ return (+ <LemonBanner type="info" className="mb-4">+ <span className="font-semibold">Waiting for the first result.</span>+ {volume && ` At the traffic of the last ${VOLUME_LOOKBACK_DAYS} days, it runs ${runRate(volume.runs)}.`}+ {waitSeconds !== null &&+ ` Each ${target} gets ${humanFriendlyDuration(waitSeconds)} after its first generation before it runs.`}+ </LemonBanner>+ )+}diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationJudgeModelPicker.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationJudgeModelPicker.tsxnew file mode 100644index 000000000000..0d8cbc35c1c8--- /dev/null+++ b/products/ai_observability/frontend/evaluations/components/EvaluationJudgeModelPicker.tsx@@ -0,0 +1,37 @@+import { useActions, useValues } from 'kea'++import { ByokModelPickerNotice } from '../../ByokModelPickerNotice'+import { getModelPickerFooterLink, ModelPicker } from '../../ModelPicker'+import { modelPickerLogic } from '../../modelPickerLogic'+import { llmEvaluationLogic } from '../llmEvaluationLogic'++/** The judge model picker with its notices, for the models the current judge method can use. */+export function EvaluationJudgeModelPicker(): JSX.Element {+ const { byokModelsLoading, providerKeysLoading } = useValues(modelPickerLogic)+ const { selectedModel, selectedModelName, selectedPickerProviderKeyId, usesDecisionModel, judgeModelGroups } =+ useValues(llmEvaluationLogic)+ const { selectModelFromPicker } = useActions(llmEvaluationLogic)++ const loading = byokModelsLoading || providerKeysLoading++ return (+ <div>+ <ModelPicker+ model={selectedModel}+ selectedProviderKeyId={selectedPickerProviderKeyId}+ onSelect={selectModelFromPicker}+ groups={judgeModelGroups}+ loading={loading}+ footerLink={getModelPickerFooterLink(judgeModelGroups.some((group) => !group.disabledReason))}+ selectedModelName={selectedModelName}+ data-attr="evaluation-model-selector"+ />+ <ByokModelPickerNotice forEvaluation />+ {usesDecisionModel && !loading && !judgeModelGroups.some((group) => group.models.length > 0) && (+ <p className="text-sm text-muted mt-2">+ No decision models are available from your configured AI providers.+ </p>+ )}+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationLatestReport.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationLatestReport.tsxnew file mode 100644index 000000000000..e0f2734729a7--- /dev/null+++ b/products/ai_observability/frontend/evaluations/components/EvaluationLatestReport.tsx@@ -0,0 +1,54 @@+import { useValues } from 'kea'++import { LemonButton } from '@posthog/lemon-ui'++import { dayjs } from 'lib/dayjs'++import { evaluationReportLogic } from '../evaluationReportLogic'+import { ReportSectionContent } from './EvaluationReportViewer'++/**+ * The newest report's headline and summary, on the tab people actually open. Few people visit the Reports tab,+ * so the report comes to them here, with a link to the full version.+ */+export function EvaluationLatestReport({+ evaluationId,+ onOpenReports,+}: {+ evaluationId: string+ onOpenReports: () => void+}): JSX.Element | null {+ const { reportRuns } = useValues(evaluationReportLogic({ evaluationId }))++ const latest = [...reportRuns].sort((a, b) => b.created_at.localeCompare(a.created_at))[0]+ if (!latest) {+ return null+ }+ const content = latest.content+ const [summary] = content.sections ?? []++ return (+ <section className="mb-4 rounded border bg-surface-primary p-4" data-attr="llma-evaluation-latest-report">+ <div className="flex flex-wrap items-start justify-between gap-2">+ <div className="flex flex-col gap-1 min-w-0">+ <span className="text-xs text-secondary">{`Latest report, ${dayjs(latest.created_at).fromNow()}`}</span>+ <h3 className="mb-0 text-base font-semibold">{content.title || 'Evaluation report'}</h3>+ </div>+ <LemonButton+ type="secondary"+ size="small"+ onClick={onOpenReports}+ data-attr="llma-evaluation-latest-report-open"+ >+ Read the full report+ </LemonButton>+ </div>+ {summary?.content && (+ // Clipped to a few lines, since the full report is one click away.+ <div className="mt-2 max-h-28 overflow-hidden">+ <ReportSectionContent section={summary} citations={content.citations ?? []} />+ </div>+ )}+ </section>+ )+}diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsxindex 07b4b2de9cd1..79801c6a2d87 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx@@ -1,45 +1,32 @@import { useActions, useValues } from 'kea'-import { LemonInput, LemonSegmentedButton } from '@posthog/lemon-ui'+import { LemonSegmentedButton } from '@posthog/lemon-ui'import { LemonField } from 'lib/lemon-ui/LemonField'-import { ByokModelPickerNotice } from '../../ByokModelPickerNotice'-import { getModelPickerFooterLink, ModelPicker } from '../../ModelPicker'import { modelPickerLogic } from '../../modelPickerLogic'-import { booleanOutputConfigError } from '../constants'+import { booleanOutputConfigError, decisionModelHint } from '../constants'import { llmEvaluationLogic } from '../llmEvaluationLogic'import type { JudgeMethod } from '../types'+import { DecisionThresholdInput } from './DecisionThresholdInput'+import { EvaluationJudgeModelPicker } from './EvaluationJudgeModelPicker'export function EvaluationModelPicker(): JSX.Element {- const { byokModels, byokModelsLoading, providerKeysLoading } = useValues(modelPickerLogic)- const {- selectedModel,- selectedPickerProviderKeyId,- modelSelectionRequired,- evaluation,- usesDecisionModel,- judgeMethod,- judgeModelGroups: groups,- } = useValues(llmEvaluationLogic)- const { selectModelFromPicker, setJudgeMethod, patchOutputConfig } = useActions(llmEvaluationLogic)+ const { byokModelsLoading, providerKeysLoading } = useValues(modelPickerLogic)+ const { evaluation, usesDecisionModel, judgeMethod } = useValues(llmEvaluationLogic)+ const { setJudgeMethod } = useActions(llmEvaluationLogic)- // Evals always run on the team's own provider key, so only BYOK models are offered.- const selectedModelName = byokModels.find(- (m) => m.id === selectedModel && m.providerKeyId === selectedPickerProviderKeyId- )?.nameconst loading = byokModelsLoading || providerKeysLoading- const footerLink = getModelPickerFooterLink(groups.some((group) => !group.disabledReason))-return (<div className="bg-bg-light border rounded p-4 @min-[40rem]/evaluation:p-6">- <h3 className="text-lg font-semibold mb-2">Judge model</h3>- <p className="text-muted text-sm mb-4">- Choose how to judge responses, then select a model from your configured AI providers.- </p>-<div className="@container/judge-model space-y-4">+ <div>+ <h3 className="text-lg font-semibold mb-2">Judge model</h3>+ <p className="text-muted text-sm mb-0">+ Choose how to judge responses, then select a model from your configured AI providers.+ </p>+ </div><LemonField.Pure label="Judge method"><LemonSegmentedButton<JudgeMethod>value={judgeMethod}@@ -68,33 +55,9 @@ export function EvaluationModelPicker(): JSX.Element {</LemonField.Pure><LemonField.Pure label="Model"><div>- <ModelPicker- model={selectedModel}- selectedProviderKeyId={selectedPickerProviderKeyId}- onSelect={selectModelFromPicker}- groups={groups}- loading={loading}- footerLink={footerLink}- selectedModelName={selectedModelName}- data-attr="evaluation-model-selector"- />- <ByokModelPickerNotice forEvaluation />- {usesDecisionModel && !loading && !groups.some((group) => group.models.length > 0) && (- <p className="text-sm text-muted mt-2">- No decision models are available from your configured AI providers.- </p>- )}+ <EvaluationJudgeModelPicker />{evaluation && usesDecisionModel && (- <p className="text-sm text-muted mt-2">- {evaluation.output_type === 'categorical'- ? 'This decision model selects categories without written reasoning. For multiple selections, each category is included when its probability is 50% or higher.'- : evaluation.output_type === 'numeric'- ? 'This decision model estimates a score between your minimum and maximum without written reasoning. Define what low and high scores mean in your evaluation prompt. Scores can be fractional.'- : 'This decision model returns a probability without written reasoning. The decision threshold determines whether the result is true.'}- </p>- )}- {modelSelectionRequired && !selectedModel && (- <p className="text-sm text-danger mt-1">Select a judge model.</p>+ <p className="text-sm text-muted mt-2">{decisionModelHint(evaluation.output_type)}</p>)}</div></LemonField.Pure>@@ -106,23 +69,7 @@ export function EvaluationModelPicker(): JSX.Element {htmlFor="llma-evaluation-decision-threshold"error={booleanOutputConfigError(evaluation.output_config)}>- <LemonInput- id="llma-evaluation-decision-threshold"- type="number"- value={Number(- ((evaluation.output_config.decision_threshold ?? 0.5) * 100).toPrecision(15)- )}- onChange={(percentage) =>- patchOutputConfig({- decision_threshold: percentage == null ? NaN : percentage / 100,- })- }- min={0}- max={100}- step="any"- suffix={<span>%</span>}- data-attr="llma-evaluation-decision-threshold"- />+ <DecisionThresholdInput id="llma-evaluation-decision-threshold" /><p className="text-muted text-sm">A probability at or above this threshold produces a true result. Applies to new runs;past results stay unchanged.diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationPromptEditor.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationPromptEditor.tsxindex 3142dc3c974d..6fd99e127049 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationPromptEditor.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationPromptEditor.tsx@@ -1,6 +1,6 @@import { useActions, useValues } from 'kea'-import { LemonTag, LemonTextArea } from '@posthog/lemon-ui'+import { LemonTextArea } from '@posthog/lemon-ui'import { LemonField } from 'lib/lemon-ui/LemonField'@@ -26,7 +26,7 @@ export function EvaluationPromptEditor(): JSX.Element {onChange={setEvaluationPrompt}placeholder={evaluation.output_type === 'categorical'- ? 'Describe when each category applies. Return the matching category keys.'+ ? 'Describe when each category applies.': evaluation.output_type === 'numeric'? 'Describe how to score the response. Return a numeric score within the configured bounds.': allowsNA@@ -41,54 +41,8 @@ Example: Is this response helpful and accurate? Return true if yes, false if no.className="font-mono text-sm"maxLength={2000}/>- <div className="flex flex-wrap gap-2 justify-between items-center text-sm text-muted">- <div>{prompt.length}/2000 characters</div>- <div className="flex items-center gap-2">- <span>Expected output:</span>- <LemonTag type="completion">- {evaluation.output_type === 'categorical'- ? allowsNA- ? 'Category keys or N/A'- : 'Category keys'- : evaluation.output_type === 'numeric'- ? allowsNA- ? 'Number or N/A'- : 'Number'- : allowsNA- ? 'Boolean (true/false/NA)'- : 'Boolean (true/false)'}- </LemonTag>- </div>- </div></div></LemonField.Pure>-- {prompt.length > 0 && (- <div className="bg-bg-light border rounded p-3">- <h4 className="text-sm font-semibold mb-2">Prompt guidelines:</h4>- <ul className="text-sm text-muted space-y-1 list-disc list-inside">- <li>Be specific about what you want to evaluate</li>- <li>- {evaluation.output_type === 'categorical' ? (- <span>Explain what each category means and when to select it</span>- ) : evaluation.output_type === 'numeric' ? (- <span>Describe the scoring scale and what each score means</span>- ) : allowsNA ? (- <>- Clearly instruct to return <code>true</code>, <code>false</code>, or{' '}- <code>N/A</code> when criteria doesn't apply- </>- ) : (- <>- Clearly instruct to return <code>true</code> or <code>false</code>- </>- )}- </li>- <li>Consider the context: input prompt, model output</li>- <li>Keep it concise but comprehensive</li>- </ul>- </div>- )}</div>)}diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationReportConfig.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationReportConfig.tsxindex bdc7141a0846..f17b54f5a244 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationReportConfig.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationReportConfig.tsx@@ -1,8 +1,10 @@import { useActions, useValues } from 'kea'+import { IconLetter, IconPlus, IconTrash } from '@posthog/icons'import {LemonButton,LemonInput,+ LemonMenu,LemonSegmentedButton,LemonSkeleton,LemonSwitch,@@ -13,6 +15,7 @@ import {import { IntegrationChoice } from 'lib/components/CyclotronJob/integrations/IntegrationChoice'import { integrationsLogic } from 'lib/integrations/integrationsLogic'import { SlackChannelPicker, SlackNotConfiguredBanner } from 'lib/integrations/SlackIntegrationHelpers'+import { IconSlack } from 'lib/lemon-ui/icons'import {COOLDOWN_HOURS_MAX,@@ -22,9 +25,9 @@ import {TRIGGER_THRESHOLD_MIN,WEEKDAY_OPTIONS,} from '../evaluationReportLogic'-import type { ReportScheduleCadence, ReportScheduleWeekday } from '../evaluationReportLogic'+import type { ReportDestination, ReportScheduleCadence, ReportScheduleWeekday } from '../evaluationReportLogic'-const GUIDANCE_PLACEHOLDER =+export const GUIDANCE_PLACEHOLDER ="Optional guidance for the report agent. e.g. 'Focus on cost regressions across models', 'Compare latency between gpt-4o-mini and claude-sonnet', 'Keep it to 2 sections max'"const FREQUENCY_OPTIONS = [@@ -142,14 +145,49 @@ function CooldownConfig({)}-/** Shared delivery targets configuration */-function DeliveryTargetsConfig({+/** One delivery destination: its name, a remove button, and its settings. */+function DestinationRow({+ icon,+ title,+ onRemove,+ children,+}: {+ icon: JSX.Element+ title: string+ onRemove: () => void+ children: React.ReactNode+}): JSX.Element {+ return (+ <div className="border rounded p-3 space-y-2">+ <div className="flex items-center justify-between gap-2">+ <span className="flex items-center gap-2 font-semibold text-sm">+ {icon}+ {title}+ </span>+ <LemonButton+ size="xsmall"+ icon={<IconTrash />}+ tooltip={`Remove ${title.toLowerCase()}`}+ aria-label={`Remove ${title.toLowerCase()}`}+ onClick={onRemove}+ />+ </div>+ {children}+ </div>+ )+}++/** Where reports go besides the Reports tab. Each destination is added from a menu, so unused ones take no room. */+export function DeliveryTargetsConfig({emailValue,onEmailChange,slackIntegrationId,onSlackIntegrationChange,slackChannelValue,onSlackChannelChange,+ shown,+ onAdd,+ onRemove,}: {emailValue: stringonEmailChange: (value: string) => void@@ -157,57 +195,86 @@ function DeliveryTargetsConfig({onSlackIntegrationChange: (value: number | null) => voidslackChannelValue: stringonSlackChannelChange: (value: string) => void+ shown: ReportDestination[]+ onAdd: (destination: ReportDestination) => void+ onRemove: (destination: ReportDestination) => void}): JSX.Element {const { slackIntegrations, integrations } = useValues(integrationsLogic)const selectedIntegration =slackIntegrationId !== null ? integrations?.find((i) => i.id === slackIntegrationId) : undefined+ const showEmail = shown.includes('email')+ const showSlack = shown.includes('slack')+ const menuItems = [+ ...(showEmail ? [] : [{ label: 'Email', icon: <IconLetter />, onClick: () => onAdd('email') }]),+ ...(showSlack ? [] : [{ label: 'Slack', icon: <IconSlack />, onClick: () => onAdd('slack') }]),+ ]return (- <>+ <div className="space-y-2"><div>- <label className="font-semibold text-sm">Email recipients</label>- <LemonInput- value={emailValue}- onChange={onEmailChange}- placeholder="[email protected], [email protected]"- fullWidth- />- <p className="text-xs text-muted mt-1">Comma-separated email addresses</p>+ <label className="font-semibold text-sm">Destinations</label>+ <p className="text-xs text-muted mb-0">Reports always show in the Reports tab. Also send them to:</p></div>- <div>- <label className="font-semibold text-sm">Slack channel</label>- {!slackIntegrations?.length ? (- <SlackNotConfiguredBanner />- ) : (- <div className="space-y-2">- <IntegrationChoice- integration="slack"- value={slackIntegrationId ?? undefined}- onChange={(newValue) => {- if (newValue !== slackIntegrationId) {- onSlackChannelChange('')- }- onSlackIntegrationChange(newValue)- }}+ <div className="grid grid-cols-1 items-start gap-2 @min-[48rem]/evaluation:grid-cols-2">+ {showEmail && (+ <DestinationRow icon={<IconLetter />} title="Email" onRemove={() => onRemove('email')}>+ <LemonInput+ value={emailValue}+ onChange={onEmailChange}+ placeholder="[email protected], [email protected]"+ fullWidth/>- {selectedIntegration && (- <SlackChannelPicker- value={slackChannelValue}- onChange={(val) => onSlackChannelChange(val || '')}- integration={selectedIntegration}- />+ <p className="text-xs text-muted mb-0">Comma-separated email addresses</p>+ </DestinationRow>+ )}+ {showSlack && (+ <DestinationRow icon={<IconSlack />} title="Slack" onRemove={() => onRemove('slack')}>+ {!slackIntegrations?.length ? (+ <SlackNotConfiguredBanner />+ ) : (+ <div className="space-y-2">+ <IntegrationChoice+ integration="slack"+ value={slackIntegrationId ?? undefined}+ onChange={(newValue) => {+ if (newValue !== slackIntegrationId) {+ onSlackChannelChange('')+ }+ onSlackIntegrationChange(newValue)+ }}+ />+ {selectedIntegration && (+ <SlackChannelPicker+ value={slackChannelValue}+ onChange={(val) => onSlackChannelChange(val || '')}+ integration={selectedIntegration}+ />+ )}+ </div>)}- </div>+ </DestinationRow>)}</div>- </>+ {menuItems.length > 0 && (+ <LemonMenu items={menuItems}>+ <LemonButton+ type="secondary"+ size="small"+ icon={<IconPlus />}+ data-attr="llma-evaluation-report-add-destination"+ >+ Add destination+ </LemonButton>+ </LemonMenu>+ )}+ </div>)}/** Shared form body used by both the create-evaluation and edit-existing-evaluation paths.* All state is backed by `evaluationReportLogic.configDraft` keyed by evaluationId. */-function ReportFormFields({ evaluationId }: { evaluationId: string }): JSX.Element {- const { configDraft, configErrors } = useValues(evaluationReportLogic({ evaluationId }))+export function ReportFormFields({ evaluationId }: { evaluationId: string }): JSX.Element {+ const { configDraft, configErrors, shownDestinations } = useValues(evaluationReportLogic({ evaluationId }))const {setDraftFrequency,setDraftScheduleCadence,@@ -218,10 +285,23 @@ function ReportFormFields({ evaluationId }: { evaluationId: string }): JSX.ElemesetDraftReportPromptGuidance,setDraftTriggerThreshold,setDraftCooldownHours,+ addDraftDestination,+ removeDraftDestination,} = useActions(evaluationReportLogic({ evaluationId }))return (<div className="space-y-4 mt-4">+ <DeliveryTargetsConfig+ emailValue={configDraft.emailValue}+ onEmailChange={setDraftEmailValue}+ slackIntegrationId={configDraft.slackIntegrationId}+ onSlackIntegrationChange={setDraftSlackIntegrationId}+ slackChannelValue={configDraft.slackChannelValue}+ onSlackChannelChange={setDraftSlackChannelValue}+ shown={shownDestinations}+ onAdd={addDraftDestination}+ onRemove={removeDraftDestination}+ /><div><label className="font-semibold text-sm">Frequency</label><LemonSegmentedButton@@ -232,7 +312,7 @@ function ReportFormFields({ evaluationId }: { evaluationId: string }): JSX.Eleme/></div>{configDraft.frequency === 'every_n' && (- <>+ <div className="grid grid-cols-1 gap-4 @min-[48rem]/evaluation:grid-cols-2"><ThresholdConfigvalue={configDraft.triggerThreshold}error={configErrors.triggerThreshold}@@ -243,7 +323,7 @@ function ReportFormFields({ evaluationId }: { evaluationId: string }): JSX.Elemeerror={configErrors.cooldownHours}onChange={setDraftCooldownHours}/>- </>+ </div>)}{configDraft.frequency === 'scheduled' && (<ScheduleConfig@@ -253,14 +333,6 @@ function ReportFormFields({ evaluationId }: { evaluationId: string }): JSX.ElemeonWeekdayToggle={toggleDraftScheduleWeekday}/>)}- <DeliveryTargetsConfig- emailValue={configDraft.emailValue}- onEmailChange={setDraftEmailValue}- slackIntegrationId={configDraft.slackIntegrationId}- onSlackIntegrationChange={setDraftSlackIntegrationId}- slackChannelValue={configDraft.slackChannelValue}- onSlackChannelChange={setDraftSlackChannelValue}- /><div><label className="font-semibold text-sm">Report agent guidance (optional)</label><LemonTextAreadiff --git a/products/ai_observability/frontend/evaluations/components/EvaluationReportViewer.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationReportViewer.tsxindex c3ae7774657d..b2e9fffa80a8 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationReportViewer.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationReportViewer.tsx@@ -133,7 +133,7 @@ function stripRedundantLeadingHeading(content: string, sectionTitle: string): streturn lines.slice(startIdx).join('\n')}-function ReportSectionContent({+export function ReportSectionContent({section,citations,}: {@@ -144,8 +144,9 @@ function ReportSectionContent({stripRedundantLeadingHeading(section.content ?? '', section.title ?? ''),citations)+ // A model writes reports from trace content, so an injected image would call an outside server for each viewer.return (- <LemonMarkdown lowKeyHeadings className="text-sm">+ <LemonMarkdown lowKeyHeadings disableImages className="text-sm">{markdown}</LemonMarkdown>)diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationSettleConfig.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationSettleConfig.tsxnew file mode 100644index 000000000000..17a66be14941--- /dev/null+++ b/products/ai_observability/frontend/evaluations/components/EvaluationSettleConfig.tsx@@ -0,0 +1,129 @@+import { useActions, useValues } from 'kea'++import { LemonSelect } from '@posthog/lemon-ui'++import { DurationPicker } from 'lib/components/DurationPicker/DurationPicker'+import { FEATURE_FLAGS } from 'lib/constants'+import { LemonField } from 'lib/lemon-ui/LemonField'+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'++import { settleStrategy } from '../constants'+import {+ DEFAULT_SESSION_MAX_AGE_SECONDS,+ DEFAULT_SESSION_QUIET_PERIOD_SECONDS,+ DEFAULT_SESSION_WINDOW_SECONDS,+ DEFAULT_TRACE_MAX_AGE_SECONDS,+ DEFAULT_TRACE_QUIET_PERIOD_SECONDS,+ DEFAULT_TRACE_WINDOW_SECONDS,+ llmEvaluationLogic,+} from '../llmEvaluationLogic'+import { EvaluationSettleStrategy } from '../types'++/** When a trace or session evaluation runs: after a fixed delay, or once the trace or session goes quiet. */+export function EvaluationSettleConfig(): JSX.Element | null {+ const { evaluation } = useValues(llmEvaluationLogic)+ const { setSettleStrategy, patchTargetConfig } = useActions(llmEvaluationLogic)+ const { featureFlags } = useValues(featureFlagLogic)+ const settlingStrategyEnabled = !!featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EVAL_SETTLING_STRATEGY]++ if (!evaluation || (evaluation.target !== 'trace' && evaluation.target !== 'session')) {+ return null+ }+ const isSessionTarget = evaluation.target === 'session'+ // Read the stored strategy whether or not the flag is on. The flag gates the picker, not what the row contains.+ const effectiveStrategy = settleStrategy(evaluation.target_config.strategy, evaluation.target)++ return (+ <>+ {settlingStrategyEnabled && (+ <LemonField.Pure label="Evaluate when">+ <LemonSelect<EvaluationSettleStrategy>+ value={effectiveStrategy}+ onChange={setSettleStrategy}+ options={[+ {+ value: 'fixed_window',+ label: 'A fixed delay has passed',+ },+ {+ value: 'inactivity',+ label: isSessionTarget ? 'The session goes quiet' : 'The trace goes quiet',+ },+ ]}+ fullWidth+ />+ </LemonField.Pure>+ )}+ {effectiveStrategy === 'fixed_window' ? (+ <LemonField.Pure label="Wait before evaluating">+ <div className="flex flex-wrap items-center gap-x-3 gap-y-1">+ <DurationPicker+ value={+ evaluation.target_config.window_seconds ??+ (isSessionTarget ? DEFAULT_SESSION_WINDOW_SECONDS : DEFAULT_TRACE_WINDOW_SECONDS)+ }+ onChange={(value) => {+ if (evaluation.target_config.strategy === 'inactivity') {+ setSettleStrategy('fixed_window')+ }+ patchTargetConfig({+ window_seconds: value,+ })+ }}+ />+ <p className="text-muted text-xs mb-0 flex-1 basis-60">+ {isSessionTarget+ ? 'After the first matching generation, from 10 seconds to 7 days. Runs already scheduled keep the wait they started with.'+ : 'After the first matching generation, from 10 seconds to 2 hours. Runs already scheduled keep the wait they started with.'}+ </p>+ </div>+ </LemonField.Pure>+ ) : (+ <>+ <LemonField.Pure label="Quiet period">+ <div className="flex flex-wrap items-center gap-x-3 gap-y-1">+ <DurationPicker+ value={+ evaluation.target_config.quiet_period_seconds ??+ (isSessionTarget+ ? DEFAULT_SESSION_QUIET_PERIOD_SECONDS+ : DEFAULT_TRACE_QUIET_PERIOD_SECONDS)+ }+ onChange={(value) =>+ patchTargetConfig({+ quiet_period_seconds: value,+ })+ }+ />+ <p className="text-muted text-xs mb-0 flex-1 basis-60">+ {isSessionTarget+ ? 'Evaluate once the session has had no new activity for this long (10s to 24 hours). Sessions often pause for a while mid-conversation, so a longer quiet period means fewer verdicts on a session that was not finished.'+ : 'Evaluate once the trace has had no new activity for this long (10 seconds to 30 minutes).'}+ </p>+ </div>+ </LemonField.Pure>+ <LemonField.Pure label="Evaluate by">+ <div className="flex flex-wrap items-center gap-x-3 gap-y-1">+ <DurationPicker+ value={+ evaluation.target_config.max_age_seconds ??+ (isSessionTarget ? DEFAULT_SESSION_MAX_AGE_SECONDS : DEFAULT_TRACE_MAX_AGE_SECONDS)+ }+ onChange={(value) =>+ patchTargetConfig({+ max_age_seconds: value,+ })+ }+ />+ <p className="text-muted text-xs mb-0 flex-1 basis-60">+ {isSessionTarget+ ? 'Always evaluate once the session is this old, even if it is still active (1m to 7 days).'+ : "Always evaluate once the trace is this old, even if it's still active (1 minute to 2 hours)."}+ </p>+ </div>+ </LemonField.Pure>+ </>+ )}+ </>+ )+}diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationTriggersPanel.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationTriggersPanel.tsxnew file mode 100644index 000000000000..ea1d65d902c3--- /dev/null+++ b/products/ai_observability/frontend/evaluations/components/EvaluationTriggersPanel.tsx@@ -0,0 +1,52 @@+import { useValues } from 'kea'++import { llmEvaluationLogic } from '../llmEvaluationLogic'+import { EvaluationTriggers } from './EvaluationTriggers'++/** The condition sets an evaluation runs on, with how they combine and a few examples. */+export function EvaluationTriggersPanel(): JSX.Element | null {+ const { evaluation } = useValues(llmEvaluationLogic)++ if (!evaluation) {+ return null+ }+ const hasFilters = evaluation.conditions.some((condition) => (condition.properties ?? []).length > 0)+ return (+ <div>+ <p className="text-muted text-sm mb-4">+ The evaluation runs on generations that match any one of these condition sets. Within a set, all filters+ must match, and sampling decides how many of the matching generations get evaluated.+ {evaluation.target === 'trace' && (+ <>+ {' '}+ Conditions match individual generations. The whole trace is evaluated once any of its+ generations matches, and sampling applies per trace.+ </>+ )}+ </p>+ <EvaluationTriggers />+ {/* Examples help before the first filter. Once someone filters, they only take room. */}+ {!hasFilters && (+ <div className="bg-bg-light border rounded p-3 text-sm mt-6">+ <h4 className="font-semibold mb-2">Examples:</h4>+ <ul className="space-y-1 text-muted list-disc list-inside">+ <li>+ <strong>10% of all generations:</strong> Set 10% sampling with no filter conditions+ </li>+ <li>+ <strong>5% of one model's generations:</strong> Set 5% sampling with $ai_model containing+ your model's name+ </li>+ <li>+ <strong>Exclude internal users:</strong> Set 100% sampling with person property is_internal+ ≠ true+ </li>+ <li>+ <strong>High-cost generations:</strong> Set 100% sampling with $ai_total_cost_usd > 0.01+ </li>+ </ul>+ </div>+ )}+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/constants.ts b/products/ai_observability/frontend/evaluations/constants.tsindex 61f9370ef5c0..e271b0f620bb 100644--- a/products/ai_observability/frontend/evaluations/constants.ts+++ b/products/ai_observability/frontend/evaluations/constants.ts@@ -1,6 +1,20 @@import { escapeHogQLString } from '~/queries/utils'-import type { EvaluationConfig, EvaluationOutputConfig, EvaluationRunsFilter } from './types'+import type {+ EvaluationConfig,+ EvaluationOutputConfig,+ EvaluationOutputType,+ EvaluationRunsFilter,+ EvaluationSettleStrategy,+} from './types'++/** The strategy a trace or session evaluation settles with. Rows without one use the backend's default for their target. */+export function settleStrategy(+ strategy: EvaluationSettleStrategy | undefined,+ target: 'trace' | 'session'+): EvaluationSettleStrategy {+ return strategy ?? (target === 'session' ? 'inactivity' : 'fixed_window')+}// Most-recent runs fetched for an evaluation's runs table and its result badgesexport const EVALUATION_RUNS_QUERY_LIMIT = 250@@ -248,3 +262,12 @@ export function categoricalEvaluationsPassedHogQL(const passingCategories = `arrayElement([${categories}], transform(properties.$ai_evaluation_id, [${ids}], [${indexes}], 0))`return `properties.$ai_evaluation_id IN (${ids}) AND if(empty(${EVALUATION_CATEGORIES_HOGQL}), empty(${passingCategories}), hasAll(${passingCategories}, ${EVALUATION_CATEGORIES_HOGQL}))`}++/** What a decision model returns for each result type, since it writes no reasoning. */+export function decisionModelHint(outputType: EvaluationOutputType): string {+ return outputType === 'categorical'+ ? 'This decision model selects categories without written reasoning. For multiple selections, each category is included when its probability is 50% or higher.'+ : outputType === 'numeric'+ ? 'This decision model estimates a score between your minimum and maximum without written reasoning. Define what low and high scores mean in your evaluation prompt. Scores can be fractional.'+ : 'This decision model returns a probability without written reasoning. The decision threshold determines whether the result is true.'+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCategoryEditor.test.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCategoryEditor.test.tsxnew file mode 100644index 000000000000..a4255db2e4cd--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCategoryEditor.test.tsx@@ -0,0 +1,71 @@+import '@testing-library/jest-dom'++import { cleanup, fireEvent, render, screen } from '@testing-library/react'++import type { EvaluationOutputConfig } from '../types'+import { EvaluationCategoryEditor } from './EvaluationCategoryEditor'++describe('EvaluationCategoryEditor', () => {+ afterEach(cleanup)++ it.each([+ { name: 'a new category takes its key from the label', storedKeys: [], key: 'fixed' },+ {+ name: 'a saved category keeps its stored key, which past results use',+ storedKeys: ['resolved'],+ key: 'resolved',+ },+ ])('$name', ({ storedKeys, key }) => {+ const onChange = jest.fn()+ const config: EvaluationOutputConfig = {+ selection_mode: 'single',+ options: [{ key: 'resolved', label: 'Resolved' }],+ passing_rule: { categories: ['resolved'] },+ }+ render(+ <EvaluationCategoryEditor config={config} onChange={onChange} showKeys={false} storedKeys={storedKeys} />+ )++ fireEvent.change(screen.getByLabelText('Category 1'), { target: { value: 'Fixed' } })++ expect(onChange).toHaveBeenCalledWith({+ options: [{ key, label: 'Fixed' }],+ passing_rule: { categories: [key] },+ })+ })+ it('a new row whose label passes through a saved one takes its own key and leaves the pass mark', () => {+ const onChange = jest.fn()+ const config: EvaluationOutputConfig = {+ selection_mode: 'single',+ options: [+ { key: 'helpful', label: 'Helpful' },+ { key: 'helpful', label: 'Helpful' },+ ],+ passing_rule: { categories: ['helpful'] },+ }+ render(+ <EvaluationCategoryEditor config={config} onChange={onChange} showKeys={false} storedKeys={['helpful']} />+ )++ fireEvent.change(screen.getByLabelText('Category 2'), { target: { value: 'Helpful but long' } })++ expect(onChange).toHaveBeenCalledWith({+ options: [config.options![0], { key: 'helpful-but-long', label: 'Helpful but long' }],+ passing_rule: { categories: ['helpful'] },+ })+ })++ it('turning grading off drops the passing rule, so runs only get a category', () => {+ const onChange = jest.fn()+ const config: EvaluationOutputConfig = {+ selection_mode: 'multiple',+ options: [{ key: 'helpful', label: 'Helpful' }],+ passing_rule: { categories: [] },+ }+ render(<EvaluationCategoryEditor config={config} onChange={onChange} showKeys={false} />)++ fireEvent.click(screen.getByLabelText('Grade runs as pass or fail'))++ expect(onChange).toHaveBeenCalledWith({ passing_rule: null })+ })+})diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCategoryEditor.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCategoryEditor.tsxnew file mode 100644index 000000000000..35653d7a4a3f--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCategoryEditor.tsx@@ -0,0 +1,160 @@+import { IconPlus, IconTrash } from '@posthog/icons'+import { LemonButton, LemonInput, LemonSwitch, Tooltip } from '@posthog/lemon-ui'++import { cn } from 'lib/utils/css-classes'++import { MAX_CATEGORICAL_OPTIONS } from '../constants'+import type { EvaluationOutputConfig } from '../types'+import { categoryKey } from './categoryKey'++/**+ * The categories of an evaluation. A new category's key follows its label. With grading on, the categories marked as a+ * pass are the passing rule; with it off, runs get a category and no pass or fail.+ */+export function EvaluationCategoryEditor({+ config,+ onChange,+ showKeys,+ storedKeys = [],+}: {+ config: EvaluationOutputConfig+ onChange: (patch: EvaluationOutputConfig) => void+ /** Code returns the key, so a code evaluation shows it next to each name. */+ showKeys: boolean+ /** Keys the saved evaluation already stores. Past results and code use them, so renaming keeps them. */+ storedKeys?: string[]+}): JSX.Element {+ const options = config.options ?? []+ const passing = config.passing_rule && 'categories' in config.passing_rule ? config.passing_rule.categories : []+ const named = options.filter((option) => option.key)+ const grades = !!config.passing_rule+ const withPassing = (keys: string[]): EvaluationOutputConfig['passing_rule'] =>+ grades ? { categories: keys } : null+ // Only the first row with a saved key keeps it. A new row whose label passes through a saved one keeps deriving.+ const keepsStoredKey = (key: string, index: number): boolean =>+ storedKeys.includes(key) && options.findIndex((item) => item.key === key) === index++ return (+ <div className="flex flex-col gap-2">+ {options.map((option, index) => (+ <div key={index} className="flex items-center gap-2">+ <Tooltip title={option.key ? `Stored as ${option.key}` : undefined} placement="top-start">+ <LemonInput+ className="flex-1 min-w-0"+ aria-label={`Category ${index + 1}`}+ placeholder={index === 0 ? 'For example, Resolved' : 'Category'}+ value={option.label}+ maxLength={256}+ onChange={(label) => {+ const key = keepsStoredKey(option.key, index) ? option.key : categoryKey(label)+ // While another row still holds the old key, its pass mark stays with that row.+ const shared = options.some((item, i) => i !== index && item.key === option.key)+ onChange({+ options: options.map((item, i) => (i === index ? { label, key } : item)),+ passing_rule: withPassing(+ shared ? passing : passing.map((value) => (value === option.key ? key : value))+ ),+ })+ }}+ suffix={+ showKeys && option.key ? (+ <code className="text-xs text-secondary">{`returns '${option.key}'`}</code>+ ) : undefined+ }+ data-attr="llma-evaluation-create-category-label"+ />+ </Tooltip>+ <LemonButton+ icon={<IconTrash />}+ size="small"+ tooltip="Remove category"+ aria-label={`Remove category ${option.label || index + 1}`}+ onClick={() =>+ onChange({+ options: options.filter((_, i) => i !== index),+ // Another row with the same key keeps its pass mark.+ passing_rule: withPassing(+ options.some((item, i) => i !== index && item.key === option.key)+ ? passing+ : passing.filter((value) => value !== option.key)+ ),+ })+ }+ />+ </div>+ ))}+ <div>+ <LemonButton+ type="secondary"+ size="small"+ icon={<IconPlus />}+ disabledReason={+ options.length >= MAX_CATEGORICAL_OPTIONS+ ? `You can add up to ${MAX_CATEGORICAL_OPTIONS} categories.`+ : undefined+ }+ onClick={() => onChange({ options: [...options, { key: '', label: '' }] })}+ data-attr="llma-evaluation-create-category-add"+ >+ Add category+ </LemonButton>+ </div>+ {named.length > 0 && (+ <div className="flex flex-col gap-2 pt-2">+ <LemonSwitch+ label="Grade runs as pass or fail"+ checked={grades}+ onChange={(on) => onChange({ passing_rule: on ? { categories: passing } : null })}+ data-attr="llma-evaluation-create-category-grading"+ />+ {grades ? (+ <div className="flex flex-col gap-1">+ <span className="text-sm font-semibold">Which ones count as a pass?</span>+ <div className="flex flex-wrap gap-1">+ {named.map((option, index) => {+ const passes = passing.includes(option.key)+ return (+ <button+ key={`${index}-${option.key}`}+ type="button"+ aria-pressed={passes}+ className={cn(+ 'rounded border px-2 py-1 text-sm cursor-pointer transition-colors',+ passes+ ? 'bg-success-highlight border-success text-success font-semibold'+ : 'bg-surface-primary hover:bg-fill-highlight-100'+ )}+ onClick={() =>+ onChange({+ passing_rule: withPassing(+ passes+ ? passing.filter((value) => value !== option.key)+ : [...passing, option.key]+ ),+ })+ }+ data-attr="llma-evaluation-create-category-pass"+ >+ {option.label}+ </button>+ )+ })}+ </div>+ <span className="text-xs text-secondary">+ {passing.length > 0+ ? 'Runs with any other category fail. Pass rates and reports use this.'+ : config.selection_mode === 'multiple'+ ? 'With none picked, only runs with no category pass.'+ : 'Pick at least one category that counts as a pass.'}+ </span>+ </div>+ ) : (+ <span className="text-xs text-secondary">+ Runs only get a category, with no pass rate or reports.+ </span>+ )}+ </div>+ )}+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationConfigurationForm.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationConfigurationForm.tsxnew file mode 100644index 000000000000..2cc761d96424--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationConfigurationForm.tsx@@ -0,0 +1,16 @@+import { BindLogic, useMountedLogic } from 'kea'++import { evaluationVolumeLogic } from '../evaluationVolumeLogic'+import { llmEvaluationLogic } from '../llmEvaluationLogic'+import { EvaluationFormSteps } from './EvaluationFormSteps'++/** A saved evaluation's settings, in the same steps as the new evaluation page. The page header saves them. */+export function EvaluationConfigurationForm(): JSX.Element {+ const { props } = useMountedLogic(llmEvaluationLogic)++ return (+ <BindLogic logic={evaluationVolumeLogic} props={props}>+ <EvaluationFormSteps checklist={[]} />+ </BindLogic>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCreateButton.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCreateButton.tsxnew file mode 100644index 000000000000..2b44ee2fd920--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCreateButton.tsx@@ -0,0 +1,47 @@+import { useValues } from 'kea'++import { LemonButton } from '@posthog/lemon-ui'++import { AccessControlAction } from 'lib/components/AccessControlAction'++import { AccessControlLevel, AccessControlResourceType } from '~/types'++import { llmEvaluationLogic } from '../llmEvaluationLogic'++/** The create action, named for whether the evaluation starts running, and held back for people without edit access. */+export function EvaluationCreateButton({+ onCreate,+ disabledReason,+ fullWidth = false,+ 'data-attr': dataAttr,+}: {+ onCreate: () => void+ disabledReason?: string+ fullWidth?: boolean+ 'data-attr': string+}): JSX.Element | null {+ const { evaluation, evaluationFormSubmitting } = useValues(llmEvaluationLogic)++ if (!evaluation) {+ return null+ }+ return (+ <AccessControlAction+ resourceType={AccessControlResourceType.Evaluation}+ minAccessLevel={AccessControlLevel.Editor}+ userAccessLevel={evaluation.user_access_level ?? undefined}+ >+ <LemonButton+ type="primary"+ center={fullWidth}+ fullWidth={fullWidth}+ onClick={onCreate}+ loading={evaluationFormSubmitting}+ disabledReason={disabledReason}+ data-attr={dataAttr}+ >+ {evaluation.enabled ? 'Create and start' : 'Create paused'}+ </LemonButton>+ </AccessControlAction>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsxnew file mode 100644index 000000000000..238a9b4798ed--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.stories.tsx@@ -0,0 +1,106 @@+import { Meta, StoryObj } from '@storybook/react'+import { waitFor, within } from '@testing-library/dom'++import { FEATURE_FLAGS } from 'lib/constants'+import { App } from 'scenes/App'+import { urls } from 'scenes/urls'++import { mswDecorator } from '~/mocks/browser'+import { toPaginatedResponse } from '~/mocks/handlers'++/** One aggregated count per series, the shape a bold-number trends query returns. */+function counts(...values: number[]): { results: { aggregated_value: number }[] } {+ return { results: values.map((value) => ({ aggregated_value: value })) }+}++function scene(matchingGenerations: number): ReturnType<typeof mswDecorator> {+ return mswDecorator({+ get: {+ '/api/projects/:team_id/evaluations/': toPaginatedResponse([]),+ '/api/projects/:team_id/evaluation_directories/': [],+ '/api/environments/:team_id/llm_analytics/provider_keys/': toPaginatedResponse([]),+ '/api/projects/:team_id/llm_analytics/provider_keys/': toPaginatedResponse([]),+ '/api/environments/:team_id/llm_analytics/evaluation_config/': { active_provider_key: null },+ },+ // The query client puts the query kind in the path.+ post: {+ '/api/environments/:team_id/query/:kind/': counts(matchingGenerations, Math.round(matchingGenerations / 3)),+ '/api/environments/:team_id/query/': counts(matchingGenerations, Math.round(matchingGenerations / 3)),+ },+ })+}++const meta: Meta = {+ component: App,+ title: 'Scenes-App/AI observability/New evaluation',+ parameters: {+ layout: 'fullscreen',+ viewMode: 'story',+ mockDate: '2026-10-20',+ pageUrl: urls.aiObservabilityEvaluation('new'),+ featureFlags: [FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW],+ testOptions: {+ waitForSelector: '[data-attr="llma-evaluation-create-step-body-name"]',+ },+ },+}+export default meta+type Story = StoryObj<{}>++export const FromScratch: Story = { decorators: [scene(1240)] }++export const FromTemplate: Story = {+ parameters: { pageUrl: `${urls.aiObservabilityEvaluation('new')}?template=helpfulness` },+ decorators: [scene(1240)],+}++/** Clicks one part of the sentence, the way someone does, and waits for the step it opens. */+async function openPart(canvasElement: HTMLElement, part: string, panelText: string): Promise<void> {+ const button = await waitFor(() => {+ const element = canvasElement.querySelector<HTMLElement>(`[data-attr="llma-evaluation-create-part-${part}"]`)+ if (!element) {+ throw new Error(`Waiting for the ${part} part`)+ }+ return element+ })+ button.click()+ await within(canvasElement).findByText(panelText)+}++export const ReportsOpen: Story = {+ parameters: {+ pageUrl: `${urls.aiObservabilityEvaluation('new')}?template=helpfulness`,+ testOptions: { waitForSelector: '[data-attr="llma-evaluation-create-step-body-reports"]' },+ },+ decorators: [scene(1240)],+ play: async ({ canvasElement }) => {+ await openPart(canvasElement, 'reports', 'Send a report')+ },+}++export const SentimentEvaluation: Story = {+ parameters: { testOptions: { waitForSelector: '[data-attr="llma-evaluation-create-step-body-check"]' } },+ decorators: [scene(1240)],+ play: async ({ canvasElement }) => {+ await openPart(canvasElement, 'method', 'Sentiment')+ within(canvasElement).getByText('Sentiment').click()+ await within(canvasElement).findByText('a sentiment classifier')+ },+}++/** The "What it runs on" step, with the warning shown when no generation matched. */+export const NothingMatches: Story = {+ parameters: {+ pageUrl: `${urls.aiObservabilityEvaluation('new')}?template=helpfulness`,+ testOptions: { waitForSelector: '[data-attr="llma-evaluation-create-volume-none"]' },+ },+ decorators: [scene(0)],+ play: async ({ canvasElement }) => {+ await openPart(canvasElement, 'triggers', 'Triggers')+ await waitFor(() => {+ if (!canvasElement.querySelector('[data-attr="llma-evaluation-create-volume-none"]')) {+ throw new Error('Waiting for the volume estimate')+ }+ })+ },+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.tsxnew file mode 100644index 000000000000..fc15b46bc4f4--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCreatePage.tsx@@ -0,0 +1,121 @@+import { BindLogic, useActions, useMountedLogic, useValues } from 'kea'+import { router } from 'kea-router'+import posthog from 'posthog-js'+import { useMemo, useRef } from 'react'++import { LemonTag } from '@posthog/lemon-ui'++import { lemonToast } from 'lib/lemon-ui/LemonToast'++import { SceneBreadcrumbBackButton } from '~/layout/scenes/components/SceneBreadcrumbs'+import { SceneStickyBar } from '~/layout/scenes/components/SceneStickyBar'++import { booleanOutputConfigError, numericOutputConfigError } from '../constants'+import { evaluationSupportsReports } from '../evaluationCapabilities'+import { buildDeliveryTargets, evaluationReportLogic } from '../evaluationReportLogic'+import { evaluationVolumeLogic } from '../evaluationVolumeLogic'+import { hasUnsetConditionRollout, llmEvaluationLogic } from '../llmEvaluationLogic'+import { defaultEvaluationTemplates } from '../templates'+import { categoryResultError, creationChecklist } from './creationChecklist'+import { EvaluationFormSteps } from './EvaluationFormSteps'++/** The new evaluation form: centered steps under a sentence that sums up the setup. */+export function EvaluationCreatePage({ templateKey }: { templateKey?: string }): JSX.Element | null {+ const {+ evaluation,+ evaluationBackTarget,+ modelSelectionRequired,+ numericBoundsRequired,+ usesDecisionModel,+ selectedModel,+ judgeModelGroups,+ isReportableEvaluation,+ } = useValues(llmEvaluationLogic)+ const { saveEvaluation, resetEvaluation } = useActions(llmEvaluationLogic)+ const { props: logicProps } = useMountedLogic(llmEvaluationLogic)+ const { volume } = useValues(evaluationVolumeLogic(logicProps))+ const template = defaultEvaluationTemplates.find((entry) => entry.key === templateKey)+ const headerRef = useRef<HTMLDivElement>(null)++ const checklist = useMemo(() => {+ if (!evaluation) {+ return []+ }+ const resultError =+ evaluation.output_type === 'boolean'+ ? booleanOutputConfigError(evaluation.output_config)+ : evaluation.output_type === 'numeric'+ ? numericOutputConfigError(evaluation.output_config, numericBoundsRequired)+ : evaluation.output_type === 'categorical'+ ? categoryResultError(evaluation.output_config)+ : null+ return creationChecklist({+ evaluation,+ resultError: resultError ?? null,+ modelSelectionRequired,+ hasSelectedModel: !!selectedModel,+ hasAvailableModels: judgeModelGroups.some((group) => group.models.length > 0),+ hasUnsetSampling:+ evaluation.conditions.length === 0 ||+ hasUnsetConditionRollout(evaluation.conditions) ||+ evaluation.conditions.some((condition) => (condition.rollout_percentage ?? 0) > 100),+ })+ }, [evaluation, numericBoundsRequired, modelSelectionRequired, selectedModel, judgeModelGroups])++ if (!evaluation) {+ return null+ }++ const handleCreate = (): void => {+ const reportLogic = evaluationReportLogic({ evaluationId: 'new' })+ if (evaluationSupportsReports(evaluation) && reportLogic.isMounted() && reportLogic.values.configError) {+ lemonToast.error(reportLogic.values.configError)+ return+ }+ // pinned: analytics event name and properties, because the creation funnel reads them+ posthog.capture('llma evaluation create submitted', {+ evaluation_type: evaluation.evaluation_type,+ output_type: evaluation.output_type,+ target: evaluation.target,+ template_key: templateKey ?? 'blank',+ condition_sets: evaluation.conditions.length,+ has_filters: evaluation.conditions.some((condition) => (condition.properties ?? []).length > 0),+ start_running: evaluation.enabled,+ uses_decision_model: usesDecisionModel,+ reports_enabled: reportLogic.isMounted()+ ? reportLogic.values.configDraft.enabled && isReportableEvaluation+ : null,+ report_frequency: reportLogic.isMounted() ? reportLogic.values.configDraft.frequency : null,+ report_destinations: reportLogic.isMounted()+ ? buildDeliveryTargets(reportLogic.values.configDraft).length+ : null,+ estimated_weekly_runs: volume?.runs ?? null,+ })+ saveEvaluation()+ }+ const handleCancel = (): void => {+ resetEvaluation()+ router.actions.push(evaluationBackTarget.path)+ }++ return (+ <BindLogic logic={evaluationVolumeLogic} props={logicProps}>+ <div className="@container/evaluation min-w-0 flex flex-col gap-4 pb-8">+ <SceneBreadcrumbBackButton />+ <SceneStickyBar+ ref={headerRef}+ hasSceneTitleSection={false}+ className="flex flex-wrap items-center justify-between gap-2"+ >+ <div className="flex flex-wrap items-center gap-2">+ <h1 className="mb-0 text-2xl font-semibold">New evaluation</h1>+ {template && <LemonTag type="muted">{`From the ${template.name} template`}</LemonTag>}+ </div>+ </SceneStickyBar>+ <div className="w-full max-w-4xl mx-auto">+ <EvaluationFormSteps checklist={checklist} onCreate={handleCreate} onCancel={handleCancel} />+ </div>+ </div>+ </BindLogic>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationCreateReview.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationCreateReview.tsxnew file mode 100644index 000000000000..05528c3f43bc--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationCreateReview.tsx@@ -0,0 +1,149 @@+import { useActions, useMountedLogic, useValues } from 'kea'++import { IconCheckCircle, IconCircleDashed, IconClock, IconNotification, IconSparkles } from '@posthog/icons'+import { LemonSwitch, Link } from '@posthog/lemon-ui'++import { cn } from 'lib/utils/css-classes'+import { capitalizeFirstLetter } from 'lib/utils/strings'++import { PROVIDER_SETTINGS_URL } from '../../ModelPicker'+import { buildDeliveryTargets, evaluationReportLogic } from '../evaluationReportLogic'+import { runRate } from '../evaluationVolume'+import { evaluationVolumeLogic } from '../evaluationVolumeLogic'+import { llmEvaluationLogic } from '../llmEvaluationLogic'+import type { ChecklistItem, ChecklistKey } from './creationChecklist'+import { reportText } from './evaluationSummary'++/**+ * What the evaluation needs before it can be created, as a checklist, next to what it will do once it runs:+ * how often, at what cost per run, where reports go, and whether it starts running.+ */+export function EvaluationCreateReview({+ checklist,+ onFix,+}: {+ checklist: ChecklistItem[]+ /** Goes to the step that sets an item. */+ onFix: (key: ChecklistKey) => void+}): JSX.Element | null {+ const { props } = useMountedLogic(llmEvaluationLogic)+ const { evaluation, canEnable, canEnableReason, isReportableEvaluation, selectedModel, selectedModelName } =+ useValues(llmEvaluationLogic)+ const { setEvaluationEnabled } = useActions(llmEvaluationLogic)+ const { volume } = useValues(evaluationVolumeLogic)+ const { configDraft } = useValues(evaluationReportLogic({ evaluationId: props.evaluationId }))++ if (!evaluation) {+ return null+ }+ const destinations = buildDeliveryTargets(configDraft).length+ const reportsOn = configDraft.enabled && isReportableEvaluation+ const facts: { key: string; icon: JSX.Element; title: string; detail: string }[] = [+ ...(volume+ ? [+ {+ key: 'volume',+ icon: <IconClock />,+ title: capitalizeFirstLetter(runRate(volume.runs)),+ detail: "From last week's matching traffic",+ },+ ]+ : []),+ {+ key: 'judge',+ icon: <IconSparkles />,+ title:+ evaluation.evaluation_type === 'llm_judge'+ ? `Judged by ${selectedModelName ?? (selectedModel || 'the model you pick')}`+ : 'No AI key needed',+ detail:+ evaluation.evaluation_type === 'llm_judge'+ ? 'Uses your own AI provider key'+ : evaluation.evaluation_type === 'hog'+ ? 'Your code decides each result'+ : 'A built-in classifier decides each result',+ },+ {+ key: 'reports',+ icon: <IconNotification />,+ title: reportsOn+ ? `${capitalizeFirstLetter(reportText({ ...configDraft, enabled: true }))} reports`+ : 'No reports',+ detail: reportsOn+ ? destinations > 0+ ? `Sent to ${destinations === 1 ? 'one destination' : `${destinations} destinations`}`+ : 'In the Reports tab'+ : 'Turn them on in the Reports step',+ },+ ]+ const done = checklist.filter((item) => item.done).length++ return (+ <div className="grid grid-cols-1 gap-4 @min-[40rem]/evaluation:grid-cols-2">+ <div className="border rounded p-4 flex flex-col gap-3">+ <div className="flex items-center justify-between gap-2">+ <h4 className="mb-0 text-sm font-semibold">Checklist</h4>+ <span className="text-xs text-secondary">{`${done} of ${checklist.length} done`}</span>+ </div>+ <ul className="flex flex-col gap-2 m-0 p-0 list-none" data-attr="llma-evaluation-create-checklist">+ {checklist.map((item) => (+ <li key={item.key} className="flex items-start gap-2 text-sm">+ {item.done ? (+ <IconCheckCircle className="text-success text-base shrink-0 mt-0.5" />+ ) : (+ <IconCircleDashed className="text-secondary text-base shrink-0 mt-0.5" />+ )}+ <span className="flex flex-col min-w-0">+ <span className={cn(item.done ? 'text-secondary' : 'font-semibold')}>{item.label}</span>+ {!item.done &&+ (item.fixInSettings ? (+ <Link+ to={PROVIDER_SETTINGS_URL}+ target="_blank"+ data-attr="llma-evaluation-create-add-key"+ >+ {item.todo}+ </Link>+ ) : (+ <Link onClick={() => onFix(item.key)}>{item.todo}</Link>+ ))}+ </span>+ </li>+ ))}+ </ul>+ </div>+ <div className="border rounded p-4 flex flex-col gap-3" data-attr="llma-evaluation-create-glance">+ <h4 className="mb-0 text-sm font-semibold">At a glance</h4>+ <ul className="m-0 p-0 list-none flex flex-col gap-3">+ {facts.map((fact) => (+ <li key={fact.key} className="flex items-start gap-3">+ <span className="flex size-8 shrink-0 items-center justify-center rounded bg-fill-highlight-100 text-lg text-secondary">+ {fact.icon}+ </span>+ <span className="flex flex-col min-w-0">+ <span className="text-sm font-semibold">{fact.title}</span>+ <span className="text-xs text-secondary">{fact.detail}</span>+ </span>+ </li>+ ))}+ </ul>+ <div className="mt-auto flex flex-col gap-1 pt-2">+ <LemonSwitch+ checked={evaluation.enabled}+ onChange={setEvaluationEnabled}+ disabledReason={!canEnable && !evaluation.enabled ? (canEnableReason ?? undefined) : undefined}+ label="Start running when created"+ bordered+ fullWidth+ data-attr="llma-evaluation-create-enabled"+ />+ <span className="text-xs text-secondary">+ {evaluation.enabled+ ? 'It checks new matching traffic as soon as you create it.'+ : 'It stays paused. Turn it on from its page when you are ready.'}+ </span>+ </div>+ </div>+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsxnew file mode 100644index 000000000000..7bc119608d52--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationFormSteps.tsx@@ -0,0 +1,322 @@+import { useActions, useMountedLogic, useValues } from 'kea'+import posthog from 'posthog-js'+import { Fragment } from 'react'++import * as judgePng from '@posthog/brand/hoggies/png/judge'+import * as magnifyingGlassPng from '@posthog/brand/hoggies/png/magnifying-glass'+import * as officeWorkerPng from '@posthog/brand/hoggies/png/office-worker'+import * as reporterPng from '@posthog/brand/hoggies/png/reporter'+import * as stampApprovedPng from '@posthog/brand/hoggies/png/stamp-approved'+import { IconCheck } from '@posthog/icons'+import { LemonButton, LemonSwitch } from '@posthog/lemon-ui'++import { pngHoggie } from 'lib/brand/hoggies'+import { cn } from 'lib/utils/css-classes'++import { EvaluationCodeEditor } from '../components/EvaluationCodeEditor'+import { EvaluationPromptEditor } from '../components/EvaluationPromptEditor'+import { EvaluationTriggersPanel } from '../components/EvaluationTriggersPanel'+import { llmEvaluationLogic } from '../llmEvaluationLogic'+import { llmEvaluationsLogic } from '../llmEvaluationsLogic'+import type { ChecklistItem, ChecklistKey } from './creationChecklist'+import { EvaluationCreateButton } from './EvaluationCreateButton'+import { EvaluationCreateReview } from './EvaluationCreateReview'+import { EvaluationMethodPanel } from './EvaluationMethodPanel'+import { EvaluationModelPanel } from './EvaluationModelPanel'+import { EvaluationNameFields } from './EvaluationNameFields'+import { EvaluationOutputFields } from './EvaluationOutputFields'+import { EvaluationReportsPanel } from './EvaluationReportsPanel'+import { EvaluationReportsSwitch } from './EvaluationReportsSwitch'+import { EvaluationSentence, type SentencePart } from './EvaluationSentence'+import { EvaluationTargetPanel } from './EvaluationTargetPanel'+import { EvaluationVolumeEstimate } from './EvaluationVolumeEstimate'++interface Step {+ // pinned: analytics property value, because the creation funnel reads it+ key: 'name' | 'check' | 'traffic' | 'reports' | 'create'+ title: string+ heading: string+ /** One paragraph per entry. */+ description: string[]+ hoggie: ReturnType<typeof pngHoggie>+ /** The parts of the sentence this step edits. */+ parts: SentencePart[]+ /** The checklist items this step completes. */+ checks: ChecklistKey[]+}++const STEPS: Step[] = [+ {+ key: 'name',+ title: 'Name',+ heading: 'Name your evaluation',+ description: ['People see this name on results and reports.'],+ hoggie: pngHoggie(officeWorkerPng),+ parts: [],+ checks: ['name'],+ },+ {+ key: 'check',+ title: 'Check',+ heading: 'What it checks',+ description: ['Choose how each run gets checked, then write the instructions and pick what it returns.'],+ hoggie: pngHoggie(judgePng),+ parts: ['method', 'model'],+ checks: ['criteria', 'result', 'model'],+ },+ {+ key: 'traffic',+ title: 'Traffic',+ heading: 'What it runs on',+ description: ['Pick the live traffic it checks, and how much of it.'],+ hoggie: pngHoggie(magnifyingGlassPng),+ parts: ['target', 'triggers'],+ checks: ['sampling'],+ },+ {+ key: 'reports',+ title: 'Reports',+ heading: 'Reports',+ description: [+ 'An agent reads the results and writes up what changed, with examples.',+ 'Choose how often a report arrives and where it goes.',+ ],+ hoggie: pngHoggie(reporterPng),+ parts: ['reports'],+ checks: [],+ },+ {+ key: 'create',+ title: 'Create',+ heading: 'Ready to create',+ description: ['Everything it needs is set.'],+ hoggie: pngHoggie(stampApprovedPng),+ parts: [],+ checks: [],+ },+]++function StepContent({ step, isNew }: { step: Step['key']; isNew: boolean }): JSX.Element | null {+ const { evaluation, canEnable, canEnableReason } = useValues(llmEvaluationLogic)+ const { setEvaluationEnabled } = useActions(llmEvaluationLogic)++ if (!evaluation) {+ return null+ }+ switch (step) {+ case 'name':+ return (+ <>+ <EvaluationNameFields />+ {/* A new evaluation sets this on its create step. */}+ {!isNew && (+ <LemonSwitch+ checked={evaluation.enabled}+ onChange={setEvaluationEnabled}+ disabledReason={+ !canEnable && !evaluation.enabled ? (canEnableReason ?? undefined) : undefined+ }+ label={evaluation.enabled ? 'Running on new traffic' : 'Paused'}+ bordered+ data-attr="llma-evaluation-configuration-enabled"+ />+ )}+ </>+ )+ case 'check':+ return (+ <>+ <EvaluationMethodPanel />+ {evaluation.evaluation_type === 'llm_judge' && <EvaluationModelPanel />}+ {evaluation.evaluation_type === 'llm_judge' && <EvaluationPromptEditor />}+ {evaluation.evaluation_type === 'hog' && <EvaluationCodeEditor />}+ <EvaluationOutputFields />+ </>+ )+ case 'traffic':+ return (+ <>+ <EvaluationTargetPanel />+ <div className="flex flex-col gap-2">+ <h4 className="mb-0 text-base font-semibold">Triggers</h4>+ <EvaluationTriggersPanel />+ </div>+ <EvaluationVolumeEstimate />+ </>+ )+ case 'reports':+ return <EvaluationReportsPanel />+ case 'create':+ return null+ }+}++/**+ * An evaluation's settings as steps, with the setup sentence above them as the overview of every choice so far.+ * Clicking a part of the sentence or a step goes straight there, so steps can be visited in any order. A new+ * evaluation ends on a create step. A saved one has no such step, because the page header saves it.+ */+export function EvaluationFormSteps({+ checklist,+ onCreate,+ onCancel,+}: {+ checklist: ChecklistItem[]+ /** Set for a new evaluation. Without it the steps edit a saved one. */+ onCreate?: () => void+ onCancel?: () => void+}): JSX.Element | null {+ const { evaluation, formStep } = useValues(llmEvaluationLogic)+ const { setFormStep } = useActions(llmEvaluationLogic)+ // Mounted here, not only in the name step, so moving between steps does not refetch the folders.+ useMountedLogic(llmEvaluationsLogic())++ if (!evaluation) {+ return null+ }+ const steps = onCreate ? STEPS : STEPS.filter((candidate) => candidate.key !== 'create')+ const index = Math.max(+ 0,+ steps.findIndex((candidate) => candidate.key === formStep)+ )+ const step = steps[index]+ const isLast = index === steps.length - 1+ const isCreateStep = step.key === 'create'+ const todo = checklist.filter((item) => !item.done)+ const stepDone = (candidate: Step): boolean =>+ candidate.checks.every((key) => checklist.find((item) => item.key === key)?.done ?? true)+ const goTo = (next: number, via: 'next' | 'back' | 'sentence' | 'stepper'): void => {+ if (next === index) {+ return+ }+ // pinned: analytics event name and properties, because the creation funnel reads them+ posthog.capture('llma evaluation create step viewed', { step: steps[next].key, via, is_new: !!onCreate })+ setFormStep(steps[next].key)+ }+ const Hoggie = step.hoggie++ return (+ <div className="flex flex-col gap-6">+ <EvaluationSentence+ activeParts={step.parts}+ onPartClick={(part) =>+ goTo(+ STEPS.findIndex((candidate) => candidate.parts.includes(part)),+ 'sentence'+ )+ }+ />+ <nav className="flex flex-wrap items-center justify-center gap-2" aria-label="Steps">+ {steps.map((candidate, candidateIndex) => {+ const current = candidateIndex === index+ // A step counts as done once you have moved past it and nothing in it is missing.+ const done = !current && candidateIndex < index && stepDone(candidate)+ return (+ <Fragment key={candidate.key}>+ {candidateIndex > 0 && (+ <span+ className={cn('h-px w-8 shrink-0', done ? 'bg-success' : 'bg-border')}+ aria-hidden+ />+ )}+ <button+ type="button"+ onClick={() => goTo(candidateIndex, 'stepper')}+ aria-current={current ? 'step' : undefined}+ className="flex items-center gap-2 rounded px-1 py-0.5 cursor-pointer hover:bg-fill-highlight-100"+ data-attr={`llma-evaluation-create-step-${candidate.key}`}+ >+ <span+ className={cn(+ 'flex size-6 items-center justify-center rounded-full text-xs font-semibold',+ current+ ? 'bg-accent text-primary-inverse'+ : done+ ? 'text-success border border-success'+ : 'border text-secondary'+ )}+ >+ {done ? <IconCheck /> : candidateIndex + 1}+ </span>+ <span className={cn('text-sm', current ? 'font-semibold' : 'text-secondary')}>+ {candidate.title}+ </span>+ </button>+ </Fragment>+ )+ })}+ </nav>+ <section+ className="bg-bg-light border rounded p-4 @min-[40rem]/evaluation:p-6 flex flex-col gap-6 min-w-0"+ data-attr={`llma-evaluation-create-step-body-${step.key}`}+ >+ <div className="flex items-center gap-4">+ <Hoggie className="w-16 shrink-0" />+ <div className="flex flex-col gap-1 min-w-0 flex-1">+ <h2 className="mb-0 text-lg font-semibold">+ {isCreateStep && todo.length > 0 ? 'Almost there' : step.heading}+ </h2>+ {(isCreateStep && todo.length > 0+ ? ['Finish the items below, then create it.']+ : step.description+ ).map((line) => (+ <p key={line} className="mb-0 text-sm text-secondary">+ {line}+ </p>+ ))}+ </div>+ {step.key === 'reports' && <EvaluationReportsSwitch />}+ </div>+ {!isCreateStep && (+ <div className="flex flex-col gap-4 min-w-0">+ <StepContent step={step.key} isNew={!!onCreate} />+ </div>+ )}+ {isCreateStep && (+ <EvaluationCreateReview+ checklist={checklist}+ onFix={(key) =>+ goTo(+ steps.findIndex((candidate) => candidate.checks.includes(key)),+ 'stepper'+ )+ }+ />+ )}+ <div className="flex items-center justify-between gap-2">+ {index > 0 ? (+ <LemonButton type="tertiary" onClick={() => goTo(index - 1, 'back')}>+ Back+ </LemonButton>+ ) : (+ <span />+ )}+ <div className="flex items-center gap-2">+ {onCancel && (+ <LemonButton type="tertiary" onClick={onCancel}>+ Cancel+ </LemonButton>+ )}+ {isCreateStep && onCreate ? (+ <EvaluationCreateButton+ onCreate={onCreate}+ disabledReason={todo[0]?.todo}+ data-attr="llma-evaluation-create-submit"+ />+ ) : (+ !isLast && (+ <LemonButton+ type="primary"+ onClick={() => goTo(index + 1, 'next')}+ data-attr="llma-evaluation-create-next-step"+ >+ {`Next: ${steps[index + 1].title}`}+ </LemonButton>+ )+ )}+ </div>+ </div>+ </section>+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationMethodPanel.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationMethodPanel.tsxnew file mode 100644index 000000000000..9138515109ff--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationMethodPanel.tsx@@ -0,0 +1,83 @@+import { useActions, useValues } from 'kea'++import { LemonSegmentedButton, Link } from '@posthog/lemon-ui'++import { modelPickerLogic } from '../../modelPickerLogic'+import { llmEvaluationLogic } from '../llmEvaluationLogic'++type Method = 'llm_judge' | 'decision' | 'hog' | 'sentiment'++const METHOD_HINTS: Record<Method, React.ReactNode> = {+ llm_judge: 'An LLM grades each run against your prompt and explains why, using your own AI provider key.',+ decision:+ 'A decision model answers your prompt with a probability and no written reasoning. It is faster and cheaper than an LLM judge.',+ hog: (+ <>+ Deterministic{' '}+ <Link to="https://posthog.com/docs/hog" target="_blank">+ Hog code+ </Link>{' '}+ returns the result. It makes no LLM calls, and you can test it on recent generations.+ </>+ ),+ sentiment:+ "Classifies the user's last message as positive, neutral, or negative. Works for English only, so use an LLM judge for other languages.",+}++/** Who checks each run: an LLM judge with its model, a decision model, code, or the sentiment classifier. */+export function EvaluationMethodPanel(): JSX.Element | null {+ const { evaluation, usesDecisionModel, isNewEvaluation } = useValues(llmEvaluationLogic)+ const { setEvaluationType, setJudgeMethod } = useActions(llmEvaluationLogic)+ const { byokModelsLoading, providerKeysLoading } = useValues(modelPickerLogic)++ if (!evaluation) {+ return null+ }+ const type = evaluation.evaluation_type+ const method: Method = type === 'llm_judge' && usesDecisionModel ? 'decision' : type+ const isJudge = type === 'llm_judge'+ // Without capabilities a saved model looks incompatible with the other judge method, and the switch would clear it.+ const judgeDisabledReason =+ isJudge && (byokModelsLoading || providerKeysLoading) ? 'Loading model capabilities.' : undefined+ const setMethod = (next: Method): void => {+ // Re-selecting a method would reset its prompt or code.+ if (next === method) {+ return+ }+ // Switching the type resets the prompt, so a move between the two judge methods keeps the type.+ if (next === 'llm_judge' || next === 'decision') {+ if (!isJudge) {+ setEvaluationType('llm_judge')+ }+ setJudgeMethod(next === 'decision' ? 'decision' : 'llm')+ } else {+ setEvaluationType(next)+ }+ }++ return (+ <div className="flex flex-col gap-2">+ <div>+ <LemonSegmentedButton+ value={method}+ onChange={setMethod}+ options={[+ { value: 'llm_judge', label: 'LLM judge', disabledReason: judgeDisabledReason },+ { value: 'decision', label: 'Decision model', disabledReason: judgeDisabledReason },+ { value: 'hog', label: 'Code' },+ {+ value: 'sentiment',+ label: 'Sentiment',+ disabledReason:+ !isNewEvaluation && ['numeric', 'categorical'].includes(evaluation.output_type)+ ? 'Create a new evaluation to change what it returns.'+ : undefined,+ },+ ]}+ data-attr="llma-evaluation-create-method"+ />+ </div>+ <p className="mb-0 text-sm text-secondary">{METHOD_HINTS[method]}</p>+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsxnew file mode 100644index 000000000000..1b8fafb3df55--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx@@ -0,0 +1,26 @@+import { useValues } from 'kea'++import { EvaluationJudgeModelPicker } from '../components/EvaluationJudgeModelPicker'+import { decisionModelHint } from '../constants'+import { llmEvaluationLogic } from '../llmEvaluationLogic'++/** The model a judge or decision model runs on, from the team's AI providers, with what that model does. */+export function EvaluationModelPanel(): JSX.Element | null {+ const { evaluation, usesDecisionModel } = useValues(llmEvaluationLogic)++ if (!evaluation) {+ return null+ }+ return (+ <div className="flex flex-wrap items-start gap-x-6 gap-y-2">+ <div className="w-80 max-w-full">+ <EvaluationJudgeModelPicker />+ </div>+ <p className="mb-0 flex-1 basis-64 text-sm text-secondary">+ {usesDecisionModel+ ? decisionModelHint(evaluation.output_type)+ : 'Runs on your own AI provider key. Each run is one call to this model.'}+ </p>+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationNameFields.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationNameFields.tsxnew file mode 100644index 000000000000..9bd755775dd9--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationNameFields.tsx@@ -0,0 +1,83 @@+import { useActions, useValues } from 'kea'+import { useState } from 'react'++import { IconPlus } from '@posthog/icons'+import { LemonButton, LemonInput, LemonSelect } from '@posthog/lemon-ui'++import { LemonField } from 'lib/lemon-ui/LemonField'++import { llmEvaluationLogic } from '../llmEvaluationLogic'+import { llmEvaluationsLogic } from '../llmEvaluationsLogic'++/** The name people see on results and reports, the folder it lives in, and an optional note. */+export function EvaluationNameFields(): JSX.Element | null {+ const [descriptionOpen, setDescriptionOpen] = useState(false)+ const { evaluation } = useValues(llmEvaluationLogic)+ const { setEvaluationName, setEvaluationDescription, setEvaluationDirectory } = useActions(llmEvaluationLogic)+ const { evaluationDirectories, evaluationDirectoriesLoading } = useValues(llmEvaluationsLogic())++ if (!evaluation) {+ return null+ }++ return (+ // Both fields share the same label style, so the name input and the folder picker line up.+ <div className="grid grid-cols-1 gap-x-4 gap-y-2 items-start @min-[40rem]/evaluation:grid-cols-[minmax(0,1fr)_14rem]">+ <div className="flex flex-col gap-2 min-w-0">+ <LemonField.Pure label="Name" htmlFor="llma-evaluation-create-name">+ <LemonInput+ id="llma-evaluation-create-name"+ value={evaluation.name}+ onChange={setEvaluationName}+ placeholder="For example, Answer is grounded in the docs"+ maxLength={100}+ autoFocus={!evaluation.name}+ data-attr="llma-evaluation-create-name"+ />+ </LemonField.Pure>+ {/* The field waits behind this button, even when a template filled it in, so it takes no room. */}+ {descriptionOpen ? (+ <LemonField.Pure label="Description (optional)" htmlFor="llma-evaluation-create-description">+ <LemonInput+ id="llma-evaluation-create-description"+ value={evaluation.description || ''}+ onChange={setEvaluationDescription}+ placeholder="What it checks and when to use it, for your teammates"+ maxLength={500}+ autoFocus+ data-attr="llma-evaluation-create-description"+ />+ </LemonField.Pure>+ ) : (+ <div>+ <LemonButton+ type="tertiary"+ size="xsmall"+ icon={<IconPlus />}+ onClick={() => setDescriptionOpen(true)}+ data-attr="llma-evaluation-create-add-description"+ >+ {evaluation.description ? 'Edit description' : 'Add description'}+ </LemonButton>+ </div>+ )}+ </div>+ <LemonField.Pure label="Folder">+ <LemonSelect<string | null>+ value={evaluation.directory_id ?? null}+ onChange={setEvaluationDirectory}+ loading={evaluationDirectoriesLoading}+ options={[+ { value: null, label: 'Top level' },+ ...evaluationDirectories.map((directory) => ({+ value: directory.id,+ label: directory.name,+ })),+ ]}+ fullWidth+ data-attr="llma-evaluation-create-folder"+ />+ </LemonField.Pure>+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationOutputFields.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationOutputFields.tsxnew file mode 100644index 000000000000..a476342a91f2--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationOutputFields.tsx@@ -0,0 +1,139 @@+import { useActions, useValues } from 'kea'++import { LemonCheckbox, LemonInput, LemonSegmentedButton, LemonSelect } from '@posthog/lemon-ui'++import { DecisionThresholdInput } from '../components/DecisionThresholdInput'+import { booleanOutputConfigError, decisionModelHint } from '../constants'+import { llmEvaluationLogic } from '../llmEvaluationLogic'+import { EvaluationCategoryEditor } from './EvaluationCategoryEditor'+import { EvaluationScoreOutput } from './EvaluationScoreOutput'++type ReturnShape = 'boolean' | 'numeric' | 'categorical' | 'categorical_multiple'++/** What each run returns, read as one sentence, and which answers count as a pass. */+export function EvaluationOutputFields(): JSX.Element | null {+ const { evaluation, originalEvaluation, numericBoundsRequired, usesDecisionModel, isNewEvaluation } =+ useValues(llmEvaluationLogic)+ const { setOutputType, patchOutputConfig, setAllowsNA, setTrueIsFailure } = useActions(llmEvaluationLogic)++ // Sentiment always returns positive, neutral, or negative, which the method hint already says.+ if (!evaluation || evaluation.evaluation_type === 'sentiment') {+ return null+ }+ const config = evaluation.output_config+ const outputType = evaluation.output_type+ const shape: ReturnShape =+ outputType === 'categorical' && config.selection_mode === 'multiple'+ ? 'categorical_multiple'+ : (outputType as ReturnShape)+ const setShape = (next: ReturnShape): void => {+ const nextType = next === 'categorical_multiple' ? 'categorical' : next+ if (nextType !== outputType) {+ setOutputType(nextType)+ }+ if (nextType === 'categorical') {+ patchOutputConfig({ selection_mode: next === 'categorical_multiple' ? 'multiple' : 'single' })+ }+ }+ const usesThreshold = evaluation.evaluation_type === 'llm_judge' && usesDecisionModel && outputType === 'boolean'+ const bound = (field: 'min' | 'max'): JSX.Element => (+ <LemonInput+ type="number"+ size="small"+ className="w-20"+ value={config[field] ?? undefined}+ onChange={(value) => patchOutputConfig({ [field]: Number.isFinite(value) ? value : null })}+ placeholder={numericBoundsRequired ? field : 'any'}+ aria-label={field === 'min' ? 'Lowest score' : 'Highest score'}+ data-attr={`llma-evaluation-numeric-${field}`}+ />+ )++ return (+ <div className="flex flex-col gap-3">+ <div className="flex flex-wrap items-center gap-2 text-sm">+ <span className="font-semibold">Return</span>+ <LemonSelect<ReturnShape>+ size="small"+ value={shape}+ onChange={setShape}+ options={(+ [+ { value: 'boolean', label: 'true or false' },+ { value: 'numeric', label: 'a score' },+ { value: 'categorical', label: 'one category' },+ { value: 'categorical_multiple', label: 'several categories' },+ ] as const+ ).map((option) => ({+ ...option,+ // Past results keep their type, so a saved evaluation can only switch between category modes.+ disabledReason:+ !isNewEvaluation &&+ (option.value === 'categorical_multiple' ? 'categorical' : option.value) !== outputType+ ? 'Create a new evaluation to change what it returns.'+ : undefined,+ }))}+ data-attr="llma-evaluation-output-type"+ />+ {outputType === 'boolean' && (+ <>+ <span>where true counts as a</span>+ <LemonSegmentedButton+ size="small"+ value={config.true_is_failure ? 'fail' : 'pass'}+ onChange={(value) => setTrueIsFailure(value === 'fail')}+ options={[+ { value: 'pass', label: 'Pass' },+ { value: 'fail', label: 'Fail' },+ ]}+ data-attr="llma-evaluation-true-is-failure-switch"+ />+ </>+ )}+ {outputType === 'numeric' && (+ <>+ <span>from</span>+ {bound('min')}+ <span>to</span>+ {bound('max')}+ </>+ )}+ </div>+ {usesThreshold && (+ <div className="flex flex-col gap-1">+ <div className="flex flex-wrap items-center gap-2 text-sm">+ <span>Answer true when the probability is at least</span>+ <DecisionThresholdInput />+ </div>+ {booleanOutputConfigError(config) && (+ <span className="text-xs text-danger">{booleanOutputConfigError(config)}</span>+ )}+ </div>+ )}+ {outputType === 'numeric' && <EvaluationScoreOutput config={config} onChange={patchOutputConfig} />}+ {outputType === 'categorical' && (+ <EvaluationCategoryEditor+ config={config}+ onChange={patchOutputConfig}+ showKeys={evaluation.evaluation_type === 'hog'}+ storedKeys={+ isNewEvaluation+ ? []+ : (originalEvaluation?.output_config.options ?? []).map((option) => option.key)+ }+ />+ )}+ {usesDecisionModel && evaluation.evaluation_type === 'llm_judge' && (+ <p className="mb-0 text-xs text-secondary">{decisionModelHint(outputType)}</p>+ )}+ <LemonCheckbox+ label={`${+ evaluation.evaluation_type === 'hog' ? 'The code' : 'The judge'+ } can answer not applicable, when the check does not apply to a run`}+ checked={config.allows_na ?? false}+ onChange={setAllowsNA}+ data-attr="llma-evaluation-create-allows-na"+ />+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationReportsPanel.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationReportsPanel.tsxnew file mode 100644index 000000000000..19692353e6a7--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationReportsPanel.tsx@@ -0,0 +1,163 @@+import { useActions, useMountedLogic, useValues } from 'kea'+import { useState } from 'react'++import { IconPlus } from '@posthog/icons'+import { LemonButton, LemonInput, LemonSelect, LemonSkeleton, LemonTextArea } from '@posthog/lemon-ui'++import { DeliveryTargetsConfig, GUIDANCE_PLACEHOLDER } from '../components/EvaluationReportConfig'+import { WEEKDAY_OPTIONS, evaluationReportLogic } from '../evaluationReportLogic'+import { llmEvaluationLogic } from '../llmEvaluationLogic'+import { reportsUnavailableReason } from './evaluationSummary'++type Cadence = 'every_n' | 'daily' | 'weekly'++/** Scheduled reports in a few lines: when they arrive, where they go, and optional guidance for the agent. The step heading holds the on and off switch. */+export function EvaluationReportsPanel(): JSX.Element | null {+ const { props } = useMountedLogic(llmEvaluationLogic)+ const [guidanceOpen, setGuidanceOpen] = useState(false)+ const { evaluation, isReportableEvaluation } = useValues(llmEvaluationLogic)+ const { configDraft, configErrors, shownDestinations, reportsLoading, activeReport } = useValues(+ evaluationReportLogic({ evaluationId: props.evaluationId })+ )+ const {+ setDraftFrequency,+ setDraftScheduleCadence,+ toggleDraftScheduleWeekday,+ setDraftTriggerThreshold,+ setDraftCooldownHours,+ setDraftEmailValue,+ setDraftSlackIntegrationId,+ setDraftSlackChannelValue,+ setDraftReportPromptGuidance,+ addDraftDestination,+ removeDraftDestination,+ } = useActions(evaluationReportLogic({ evaluationId: props.evaluationId }))++ if (!evaluation) {+ return null+ }+ // The saved settings replace the draft once they load, so edits made before then would be lost.+ if (props.evaluationId !== 'new' && reportsLoading && !activeReport) {+ return (+ <div className="flex flex-col gap-3">+ <LemonSkeleton className="h-8 w-full" />+ <LemonSkeleton className="h-8 w-2/3" />+ </div>+ )+ }+ const unavailableReason = reportsUnavailableReason(evaluation, isReportableEvaluation)+ const enabled = configDraft.enabled && !unavailableReason+ const cadence: Cadence = configDraft.frequency === 'scheduled' ? configDraft.scheduleCadence : 'every_n'+ const setCadence = (next: Cadence): void => {+ if (next === 'every_n') {+ setDraftFrequency('every_n')+ } else {+ setDraftFrequency('scheduled')+ setDraftScheduleCadence(next)+ }+ }+ const error = configErrors.triggerThreshold ?? configErrors.cooldownHours++ return (+ <div className="flex flex-col gap-4">+ {unavailableReason && <p className="mb-0 text-sm text-secondary">{unavailableReason}</p>}+ {enabled && (+ <>+ <div className="flex flex-col gap-1">+ <div className="flex flex-wrap items-center gap-2 text-sm">+ <span>Send a report</span>+ <LemonSelect<Cadence>+ size="small"+ value={cadence}+ onChange={setCadence}+ options={[+ { value: 'every_n', label: 'every' },+ { value: 'daily', label: 'daily' },+ { value: 'weekly', label: 'weekly' },+ ]}+ data-attr="llma-evaluation-create-report-cadence"+ />+ {cadence === 'every_n' && (+ <>+ <LemonInput+ type="number"+ size="small"+ className="w-24"+ value={configDraft.triggerThreshold}+ onChange={(value) => setDraftTriggerThreshold(Number(value))}+ status={configErrors.triggerThreshold ? 'danger' : undefined}+ aria-label="Results per report"+ />+ <span>results, at most once every</span>+ <LemonInput+ type="number"+ size="small"+ className="w-16"+ value={configDraft.cooldownHours}+ onChange={(value) => setDraftCooldownHours(Number(value))}+ status={configErrors.cooldownHours ? 'danger' : undefined}+ aria-label="Minimum hours between reports"+ />+ <span>hours</span>+ </>+ )}+ {cadence === 'weekly' && (+ <>+ <span>on</span>+ {WEEKDAY_OPTIONS.map((option) => (+ <LemonButton+ key={option.value}+ size="xsmall"+ type={+ configDraft.scheduleWeekdays.includes(option.value)+ ? 'primary'+ : 'secondary'+ }+ aria-pressed={configDraft.scheduleWeekdays.includes(option.value)}+ onClick={() => toggleDraftScheduleWeekday(option.value)}+ >+ {option.label}+ </LemonButton>+ ))}+ </>+ )}+ </div>+ {error && <span className="text-xs text-danger">{error}</span>}+ </div>+ <DeliveryTargetsConfig+ emailValue={configDraft.emailValue}+ onEmailChange={setDraftEmailValue}+ slackIntegrationId={configDraft.slackIntegrationId}+ onSlackIntegrationChange={setDraftSlackIntegrationId}+ slackChannelValue={configDraft.slackChannelValue}+ onSlackChannelChange={setDraftSlackChannelValue}+ shown={shownDestinations}+ onAdd={addDraftDestination}+ onRemove={removeDraftDestination}+ />+ {guidanceOpen || configDraft.reportPromptGuidance ? (+ <LemonTextArea+ value={configDraft.reportPromptGuidance}+ onChange={setDraftReportPromptGuidance}+ placeholder={GUIDANCE_PLACEHOLDER}+ rows={2}+ autoFocus={guidanceOpen}+ />+ ) : (+ <div>+ <LemonButton+ type="tertiary"+ size="xsmall"+ icon={<IconPlus />}+ onClick={() => setGuidanceOpen(true)}+ data-attr="llma-evaluation-create-report-guidance"+ >+ Add guidance for the agent+ </LemonButton>+ </div>+ )}+ </>+ )}+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationReportsSwitch.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationReportsSwitch.tsxnew file mode 100644index 000000000000..99cf9f75c025--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationReportsSwitch.tsx@@ -0,0 +1,39 @@+import { useActions, useMountedLogic, useValues } from 'kea'++import { LemonSwitch } from '@posthog/lemon-ui'++import { evaluationReportLogic } from '../evaluationReportLogic'+import { llmEvaluationLogic } from '../llmEvaluationLogic'+import { reportsUnavailableReason } from './evaluationSummary'++/** Turns scheduled reports on or off, and says why it cannot while the evaluation has no pass rate. */+export function EvaluationReportsSwitch(): JSX.Element | null {+ const { props } = useMountedLogic(llmEvaluationLogic)+ const { evaluation, isReportableEvaluation } = useValues(llmEvaluationLogic)+ const { configDraft, reportsLoading, activeReport } = useValues(+ evaluationReportLogic({ evaluationId: props.evaluationId })+ )+ const { setDraftEnabled } = useActions(evaluationReportLogic({ evaluationId: props.evaluationId }))++ if (!evaluation) {+ return null+ }+ const unavailableReason = reportsUnavailableReason(evaluation, isReportableEvaluation)+ const enabled = configDraft.enabled && !unavailableReason+ return (+ <LemonSwitch+ checked={enabled}+ onChange={setDraftEnabled}+ // Loading the saved settings replaces the draft, so a toggle before then would be lost.+ disabledReason={+ unavailableReason ??+ (props.evaluationId !== 'new' && reportsLoading && !activeReport+ ? 'Loading the saved report settings'+ : undefined)+ }+ bordered+ label={enabled ? 'Enabled' : 'Disabled'}+ data-attr="llma-evaluation-create-reports-switch"+ />+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationScoreOutput.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationScoreOutput.tsxnew file mode 100644index 000000000000..89f84fd92db6--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationScoreOutput.tsx@@ -0,0 +1,58 @@+import { useState } from 'react'++import { LemonInput, LemonSelect } from '@posthog/lemon-ui'++import type { EvaluationOutputConfig } from '../types'++/** The score a run needs to pass. Without a passing score, scores carry no pass or fail. */+export function EvaluationScoreOutput({+ config,+ onChange,+}: {+ config: EvaluationOutputConfig+ onChange: (patch: EvaluationOutputConfig) => void+}): JSX.Element {+ const rule = config.passing_rule && 'threshold' in config.passing_rule ? config.passing_rule : null+ // The direction is kept here until a passing score exists, because a rule needs both.+ const [pendingOperator, setPendingOperator] = useState<'gte' | 'lte'>('gte')+ const operator = rule?.operator ?? pendingOperator+ return (+ <div className="flex flex-col gap-1">+ <div className="flex flex-wrap items-center gap-2 text-sm">+ <span>A run passes when the score is</span>+ <LemonSelect+ size="small"+ value={operator}+ options={[+ { value: 'gte', label: 'at least' },+ { value: 'lte', label: 'at most' },+ ]}+ onChange={(next) => {+ setPendingOperator(next)+ if (rule) {+ onChange({ passing_rule: { ...rule, operator: next } })+ }+ }}+ />+ <LemonInput+ type="number"+ size="small"+ className="w-20"+ value={rule && Number.isFinite(rule.threshold) ? rule.threshold : undefined}+ onChange={(threshold) =>+ onChange({+ passing_rule:+ threshold == null || !Number.isFinite(threshold) ? null : { operator, threshold },+ })+ }+ placeholder="score"+ aria-label="Passing score"+ data-attr="llma-evaluation-numeric-threshold"+ />+ </div>+ <span className="text-xs text-secondary">+ Leave the passing score empty to keep scores without a pass or fail. Pass rates and reports need one.+ </span>+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationSentence.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationSentence.tsxnew file mode 100644index 000000000000..c1b38d07c35a--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationSentence.tsx@@ -0,0 +1,80 @@+import { useMountedLogic, useValues } from 'kea'+import posthog from 'posthog-js'++import { LemonButton } from '@posthog/lemon-ui'++import { evaluationReportLogic } from '../evaluationReportLogic'+import { llmEvaluationLogic } from '../llmEvaluationLogic'+import { methodText, reportText, targetText, triggerText } from './evaluationSummary'++export type SentencePart = 'method' | 'model' | 'target' | 'triggers' | 'reports'++/**+ * The whole setup as one sentence: who checks, with which model, what one run covers, which traffic, and how often+ * a report arrives. It is the overview above the steps: it marks the parts the current step edits, and a click on a+ * part goes to the step that edits it.+ */+export function EvaluationSentence({+ activeParts,+ onPartClick,+}: {+ activeParts: SentencePart[]+ onPartClick: (part: SentencePart) => void+}): JSX.Element | null {+ const { props } = useMountedLogic(llmEvaluationLogic)+ const { evaluation, usesDecisionModel, selectedModelName, selectedModel, isReportableEvaluation, isNewEvaluation } =+ useValues(llmEvaluationLogic)+ const { configDraft } = useValues(evaluationReportLogic({ evaluationId: props.evaluationId }))++ if (!evaluation) {+ return null+ }+ const needsModel = evaluation.evaluation_type === 'llm_judge' && !selectedModel+ const part = (key: SentencePart, text: React.ReactNode, missing = false): JSX.Element => (+ <LemonButton+ // The parts of the current step stand out, so the sentence shows where you are.+ type={activeParts.includes(key) ? 'primary' : 'secondary'}+ size="small"+ status={missing ? 'danger' : 'default'}+ active={activeParts.includes(key)}+ onClick={() => {+ if (!activeParts.includes(key)) {+ // pinned: analytics event name and properties, because the creation funnel reads them+ posthog.capture('llma evaluation create part opened', { part: key, is_new: isNewEvaluation })+ }+ onPartClick(key)+ }}+ className="inline-flex align-middle"+ data-attr={`llma-evaluation-create-part-${key}`}+ >+ {text}+ </LemonButton>+ )+ return (+ // One line, wrapping only when the page is narrow.+ <section+ className="flex flex-wrap items-center justify-center gap-x-2 gap-y-1 text-base"+ data-attr="llma-evaluation-create-sentence"+ >+ <span>Run</span>+ {part('method', methodText(evaluation, usesDecisionModel))}+ {evaluation.evaluation_type === 'llm_judge' && (+ <>+ <span>with</span>+ {part('model', needsModel ? 'pick a model' : (selectedModelName ?? selectedModel), needsModel)}+ </>+ )}+ <span>on</span>+ {/* Sentiment always reads single generations, so there is nothing to choose. */}+ {evaluation.evaluation_type === 'sentiment' ? (+ <span className="font-semibold">{targetText(evaluation)}</span>+ ) : (+ part('target', targetText(evaluation))+ )}+ <span>for</span>+ {part('triggers', triggerText(evaluation))}+ <span>and alert me</span>+ {part('reports', reportText({ ...configDraft, enabled: configDraft.enabled && isReportableEvaluation }))}+ </section>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationTargetPanel.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationTargetPanel.tsxnew file mode 100644index 000000000000..e1c0d7037b98--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationTargetPanel.tsx@@ -0,0 +1,57 @@+import { useActions, useValues } from 'kea'++import { LemonSegmentedButton } from '@posthog/lemon-ui'++import { FEATURE_FLAGS } from 'lib/constants'+import { LemonField } from 'lib/lemon-ui/LemonField'+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'++import { EvaluationSettleConfig } from '../components/EvaluationSettleConfig'+import { evaluationOffersSessionTarget } from '../evaluationCapabilities'+import { llmEvaluationLogic } from '../llmEvaluationLogic'+import type { EvaluationTarget } from '../types'++const TARGET_HINTS: Record<EvaluationTarget, string> = {+ generation: 'Runs on each matching generation as soon as it arrives.',+ trace: 'Runs once per trace, after the trace has had time to finish. Only generations sent with a trace ID count.',+ session:+ 'Runs once per session, after the session has had time to finish. Only generations sent with a session ID and a trace ID count.',+}++/** What one run covers, and how long a trace or session gets to finish first. */+export function EvaluationTargetPanel(): JSX.Element | null {+ const { evaluation } = useValues(llmEvaluationLogic)+ const { setEvaluationTarget } = useActions(llmEvaluationLogic)+ const { featureFlags } = useValues(featureFlagLogic)++ if (!evaluation) {+ return null+ }+ const isSentiment = evaluation.evaluation_type === 'sentiment'+ const target: EvaluationTarget = isSentiment ? 'generation' : evaluation.target+ const offersSession = evaluationOffersSessionTarget(+ evaluation,+ !!featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EVAL_SETTLING_STRATEGY]+ )++ return (+ <div className="flex flex-col gap-4">+ {!isSentiment && (+ <LemonField.Pure label="Evaluate" help={TARGET_HINTS[target]}>+ <LemonSegmentedButton+ value={target}+ onChange={setEvaluationTarget}+ options={[+ { value: 'generation', label: 'Each generation' },+ { value: 'trace', label: 'Whole trace' },+ ...(offersSession ? [{ value: 'session' as const, label: 'Whole session' }] : []),+ ]}+ fullWidth+ data-attr="llma-evaluation-create-target"+ />+ </LemonField.Pure>+ )}+ <EvaluationSettleConfig />+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationVolumeEstimate.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationVolumeEstimate.tsxnew file mode 100644index 000000000000..6bcbd1aeaa26--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/EvaluationVolumeEstimate.tsx@@ -0,0 +1,78 @@+import { useActions, useValues } from 'kea'++import { LemonBanner, Spinner } from '@posthog/lemon-ui'++import { humanFriendlyNumber } from 'lib/utils/numbers'++import { VOLUME_LOOKBACK_DAYS, filtersOnContent, runRate } from '../evaluationVolume'+import { evaluationVolumeLogic } from '../evaluationVolumeLogic'+import { llmEvaluationLogic } from '../llmEvaluationLogic'++const UNIT_NOUNS = { trace: 'traces', session: 'sessions' } as const++/** How often the evaluation would have run last week, so a filter that matches nothing shows before saving. */+export function EvaluationVolumeEstimate(): JSX.Element | null {+ const { evaluation } = useValues(llmEvaluationLogic)+ const { setEvaluationTarget } = useActions(llmEvaluationLogic)+ const { volume, countsLoading, countsError } = useValues(evaluationVolumeLogic)++ if (countsError) {+ return <p className="mb-0 text-sm text-secondary">Couldn't estimate how often it would run.</p>+ }+ if (evaluation && filtersOnContent(evaluation.conditions)) {+ return (+ <p className="mb-0 text-sm text-secondary">+ Couldn't estimate how often it would run, because a filter checks the input or output.+ </p>+ )+ }+ if (!volume) {+ return countsLoading ? (+ <p className="mb-0 flex items-center gap-2 text-sm text-secondary">+ <Spinner />+ Counting matching generations+ </p>+ ) : null+ }+ if (volume.matchingGenerations === 0) {+ // The wrapper carries the data-attr, since LemonBanner does not pass one through.+ return (+ <div data-attr="llma-evaluation-create-volume-none">+ <LemonBanner type="warning">+ {`No generations matched in the last ${VOLUME_LOOKBACK_DAYS} days, so it would not have run. Check the filters, or keep them if the traffic has not started yet.`}+ </LemonBanner>+ </div>+ )+ }+ if (volume.target !== 'generation' && volume.matchingUnits === 0) {+ const ids =+ volume.target === 'trace'+ ? 'a trace ID ($ai_trace_id)'+ : 'both a session ID ($ai_session_id) and a trace ID ($ai_trace_id)'+ return (+ <LemonBanner+ type="warning"+ action={{+ children: 'Evaluate each generation',+ onClick: () => setEvaluationTarget('generation'),+ 'data-attr': 'llma-evaluation-create-use-generation',+ }}+ >+ {`Generations matched, but none were sent with ${ids}, so this ${volume.target} evaluation would not run.`}+ </LemonBanner>+ )+ }+ const matched =+ volume.target === 'generation'+ ? `${humanFriendlyNumber(volume.matchingGenerations)} generations matched`+ : `${humanFriendlyNumber(volume.matchingGenerations)} generations in ${humanFriendlyNumber(volume.matchingUnits)} ${UNIT_NOUNS[volume.target]} matched`+ return (+ <div className="flex items-start gap-2 text-sm text-secondary" data-attr="llma-evaluation-create-volume">+ {countsLoading && <Spinner className="mt-0.5" />}+ <p className="mb-0">+ {`In the last ${VOLUME_LOOKBACK_DAYS} days, ${matched}. With this sampling it would have run about ${humanFriendlyNumber(volume.runs)} times, ${runRate(volume.runs)}.`}+ {evaluation?.evaluation_type === 'llm_judge' && ' Each run is one call to your judge model.'}+ </p>+ </div>+ )+}diff --git a/products/ai_observability/frontend/evaluations/create/categoryKey.ts b/products/ai_observability/frontend/evaluations/create/categoryKey.tsnew file mode 100644index 000000000000..9cd3688fb4ba--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/categoryKey.ts@@ -0,0 +1,16 @@+import { hashCodeForString, slugify } from 'lib/utils/strings'++/** The key a new category gets from its label, in the format results store: lowercase words joined by hyphens. */+export function categoryKey(label: string): string {+ // Keys are capped at 128 characters, while labels can be longer.+ const slug = slugify(label)+ .replace(/_/g, '-')+ .replace(/-{2,}/g, '-')+ .slice(0, 128)+ .replace(/^-+|-+$/g, '')+ // Slugs keep only Latin letters and digits, so a label in another script gets a stable key from its text.+ if (!slug && /[\p{L}\p{N}]/u.test(label)) {+ return `category-${hashCodeForString(label.trim()).toString(36)}`+ }+ return slug+}diff --git a/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts b/products/ai_observability/frontend/evaluations/create/creationChecklist.test.tsnew file mode 100644index 000000000000..ec058e4bf4be--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/creationChecklist.test.ts@@ -0,0 +1,118 @@+import { runRate, summarizeVolume } from '../evaluationVolume'+import type { EvaluationConfig } from '../types'+import { categoryKey } from './categoryKey'+import { type ChecklistInput, categoryResultError, creationChecklist } from './creationChecklist'++const judge = (prompt: string, name = 'Helpfulness'): EvaluationConfig =>+ ({+ name,+ evaluation_type: 'llm_judge',+ evaluation_config: { prompt },+ output_type: 'boolean',+ output_config: {},+ conditions: [{ id: 'c1', rollout_percentage: 100, properties: [] }],+ }) as unknown as EvaluationConfig++const input = (overrides: Partial<ChecklistInput>): ChecklistInput => ({+ evaluation: judge('Is it helpful?'),+ resultError: null,+ modelSelectionRequired: true,+ hasSelectedModel: true,+ hasAvailableModels: true,+ hasUnsetSampling: false,+ ...overrides,+})++const firstTodo = (overrides: Partial<ChecklistInput>): string | undefined =>+ creationChecklist(input(overrides)).find((item) => !item.done)?.todo++describe('creationChecklist', () => {+ test.each([+ ['a complete judge', {}, undefined],+ ['a missing name', { evaluation: judge('Is it helpful?', ' ') }, 'Give the evaluation a name'],+ ['an empty prompt', { evaluation: judge(' ') }, 'Write the prompt the judge follows'],+ ['an invalid result setting', { resultError: 'Add at least one label' }, 'Add at least one label'],+ ['no model while models exist', { hasSelectedModel: false }, 'Pick the model that judges each run'],+ [+ 'no model and no provider key',+ { hasSelectedModel: false, hasAvailableModels: false },+ 'Add an API key for an AI provider, then pick a model',+ ],+ ['unset sampling', { hasUnsetSampling: true }, 'Set a sampling percentage between 0.1% and 100%'],+ ] as [string, Partial<ChecklistInput>, string | undefined][])('blocks create on %s', (_, overrides, expected) => {+ expect(firstTodo(overrides)).toBe(expected)+ })++ it('asks for no prompt, result or model on a sentiment evaluation', () => {+ const sentiment = { ...judge(''), evaluation_type: 'sentiment', output_type: 'sentiment' } as EvaluationConfig+ expect(+ creationChecklist(input({ evaluation: sentiment, modelSelectionRequired: false })).map((item) => item.key)+ ).toEqual(['name', 'sampling'])+ })++ test.each([+ [+ 'samples each condition set by its own percentage',+ [+ { generations: 1000, units: 1000, rolloutPercentage: 10 },+ { generations: 50, units: 50, rolloutPercentage: 100 },+ ],+ 'generation',+ { matchingGenerations: 1050, matchingUnits: 1050, runs: 150 },+ ],+ [+ 'counts traces, not generations, for a trace evaluation',+ [{ generations: 900, units: 120, rolloutPercentage: 50 }],+ 'trace',+ { matchingGenerations: 900, matchingUnits: 120, runs: 60 },+ ],+ ] as const)('estimates runs: %s', (_, conditions, target, expected) => {+ expect(summarizeVolume([...conditions], target)).toEqual({ target, ...expected })+ })++ test.each([+ [98, 'about 14 times a day'],+ [7, 'about once a day'],+ [3, 'about 3 times a week'],+ [1, 'once a week'],+ [0, 'less than once a week'],+ ])('describes %d runs a week as "%s"', (runs, expected) => {+ expect(runRate(runs)).toEqual(expected)+ })++ test.each([+ ['Resolved', 'resolved'],+ ['Handed off to a human', 'handed-off-to-a-human'],+ [' Partly resolved (needs follow-up)! ', 'partly-resolved-needs-follow-up'],+ ['snake_case label', 'snake-case-label'],+ ['已解决', 'category-ef9uq'],+ ])('turns the category label %j into the stored key %j', (label, key) => {+ expect(categoryKey(label)).toBe(key)+ expect(key).toMatch(/^[a-z0-9]+(?:[_-][a-z0-9]+)*$/)+ })++ test.each([+ ['no categories', [], 'Add at least one category'],+ [+ 'a nameless category',+ [{ key: '', label: '' }],+ 'Give every category a name with at least one letter or number',+ ],+ [+ 'a name with only symbols',+ [{ key: '', label: '!!' }],+ 'Give every category a name with at least one letter or number',+ ],+ [+ 'two names with the same key',+ [+ { key: 'resolved', label: 'Resolved' },+ { key: 'resolved', label: 'Resolved!' },+ ],+ 'Give each category a different name',+ ],+ ['valid categories', [{ key: 'resolved', label: 'Resolved' }], null],+ ])('explains %s without mentioning keys', (_, options, expected) => {+ expect(categoryResultError({ options })).toBe(expected)+ })+})diff --git a/products/ai_observability/frontend/evaluations/create/creationChecklist.ts b/products/ai_observability/frontend/evaluations/create/creationChecklist.tsnew file mode 100644index 000000000000..89644519ba80--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/creationChecklist.ts@@ -0,0 +1,86 @@+import { categoricalOutputConfigError } from '../constants'+import type { EvaluationConfig, EvaluationOutputConfig } from '../types'++export type ChecklistKey = 'name' | 'criteria' | 'result' | 'model' | 'sampling'++export interface ChecklistItem {+ key: ChecklistKey+ label: string+ done: boolean+ /** What to do next when the item is not done. Also the reason the create button is disabled. */+ todo: string+ /** Set when the fix lives on another page, such as adding a provider key in settings. */+ fixInSettings?: boolean+}++export interface ChecklistInput {+ evaluation: EvaluationConfig+ /** The output settings' validation error, or null when they are valid. */+ resultError: string | null+ modelSelectionRequired: boolean+ hasSelectedModel: boolean+ /** False when no configured provider offers a model to pick. */+ hasAvailableModels: boolean+ hasUnsetSampling: boolean+}++const CRITERIA: Record<'llm_judge' | 'hog', { label: string; todo: string }> = {+ llm_judge: { label: 'Evaluation prompt', todo: 'Write the prompt the judge follows' },+ hog: { label: 'Code', todo: 'Add the Hog code that returns the result' },+}++/** What a new evaluation still needs before it can be created, in the order the page asks for it. */+export function creationChecklist(input: ChecklistInput): ChecklistItem[] {+ const { evaluation } = input+ const items: ChecklistItem[] = [+ { key: 'name', label: 'Name', done: evaluation.name.trim().length > 0, todo: 'Give the evaluation a name' },+ ]+ if (evaluation.evaluation_type === 'llm_judge' || evaluation.evaluation_type === 'hog') {+ const content =+ evaluation.evaluation_type === 'llm_judge'+ ? evaluation.evaluation_config.prompt+ : evaluation.evaluation_config.source+ items.push({ key: 'criteria', done: content.trim().length > 0, ...CRITERIA[evaluation.evaluation_type] })+ }+ if (evaluation.evaluation_type !== 'sentiment') {+ items.push({+ key: 'result',+ label: 'Result',+ done: input.resultError === null,+ todo: input.resultError ?? '',+ })+ }+ if (input.modelSelectionRequired) {+ items.push({+ key: 'model',+ label: 'Judge model',+ done: input.hasSelectedModel,+ todo: input.hasAvailableModels+ ? 'Pick the model that judges each run'+ : 'Add an API key for an AI provider, then pick a model',+ fixInSettings: !input.hasAvailableModels,+ })+ }+ items.push({+ key: 'sampling',+ label: 'Sampling',+ done: !input.hasUnsetSampling,+ todo: 'Set a sampling percentage between 0.1% and 100%',+ })+ return items+}++/** The categories' validation error in the page's terms. The page derives keys from names, so it never mentions keys. */+export function categoryResultError(config: EvaluationOutputConfig): string | null {+ const options = config.options ?? []+ if (options.length === 0) {+ return 'Add at least one category'+ }+ if (options.some((option) => !option.label.trim() || !option.key)) {+ return 'Give every category a name with at least one letter or number'+ }+ if (new Set(options.map((option) => option.key)).size !== options.length) {+ return 'Give each category a different name'+ }+ return categoricalOutputConfigError(config)+}diff --git a/products/ai_observability/frontend/evaluations/create/evaluationSummary.test.ts b/products/ai_observability/frontend/evaluations/create/evaluationSummary.test.tsnew file mode 100644index 000000000000..e11a8133d4a3--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/evaluationSummary.test.ts@@ -0,0 +1,55 @@+import type { EvaluationConditionSet, EvaluationConfig } from '../types'+import { reportText, triggerText } from './evaluationSummary'++const evaluation = (overrides: Partial<EvaluationConfig>): EvaluationConfig =>+ ({+ name: 'Helpfulness',+ evaluation_type: 'llm_judge',+ evaluation_config: { prompt: 'Is it helpful?' },+ output_type: 'boolean',+ output_config: {},+ target: 'trace',+ conditions: [{ id: 'c1', rollout_percentage: 100, properties: [] }],+ ...overrides,+ }) as unknown as EvaluationConfig++const filtered = (rollout: number, filters: number): EvaluationConditionSet => ({+ id: `c-${rollout}`,+ rollout_percentage: rollout,+ properties: Array.from({ length: filters }, (_, i) => ({ key: `p${i}`, type: 'event' })) as any,+})++describe('evaluation sentence parts', () => {+ test.each([+ { name: 'all traffic', input: evaluation({}), expected: '100% of all traces' },+ {+ name: 'sampled, filtered traffic',+ input: evaluation({ conditions: [filtered(10, 2)] }),+ expected: '10% of matching traces (2 filters)',+ },+ {+ name: 'sentiment counts generations, whatever target is stored',+ input: evaluation({ evaluation_type: 'sentiment' } as Partial<EvaluationConfig>),+ expected: '100% of all generations',+ },+ {+ name: 'several condition sets sample on their own, so no single percentage',+ input: evaluation({ conditions: [filtered(100, 1), filtered(20, 0)] }),+ expected: '2 condition sets',+ },+ ])('triggers: $name', ({ input, expected }) => {+ expect(triggerText(input).replace(/\u00a0/g, ' ')).toEqual(expected)+ })++ test.each([+ { name: 'off', input: { enabled: false, frequency: 'scheduled' }, expected: 'never' },+ { name: 'scheduled', input: { enabled: true, frequency: 'scheduled' }, expected: 'weekly' },+ {+ name: 'by result count',+ input: { enabled: true, frequency: 'every_n' },+ expected: 'every 100 results',+ },+ ])('reports: $name', ({ input, expected }) => {+ expect(reportText({ scheduleCadence: 'weekly', triggerThreshold: 100, ...input })).toEqual(expected)+ })+})diff --git a/products/ai_observability/frontend/evaluations/create/evaluationSummary.ts b/products/ai_observability/frontend/evaluations/create/evaluationSummary.tsnew file mode 100644index 000000000000..29b41d771b39--- /dev/null+++ b/products/ai_observability/frontend/evaluations/create/evaluationSummary.ts@@ -0,0 +1,69 @@+import { pluralize } from 'lib/utils/strings'++import { evaluationTarget } from '../evaluationVolume'+import type { EvaluationConfig, EvaluationTarget } from '../types'++const UNIT_NOUNS: Record<EvaluationTarget, string> = {+ generation: 'generations',+ trace: 'traces',+ session: 'sessions',+}++/** What one run covers, such as "each trace". */+export function targetText(evaluation: EvaluationConfig): string {+ return `each ${evaluationTarget(evaluation)}`+}++/** How much traffic the triggers take. Each condition set samples on its own, so several sets get no one percentage. */+export function triggerText(evaluation: EvaluationConfig): string {+ if (evaluation.conditions.length > 1) {+ return `${evaluation.conditions.length} condition sets`+ }+ const [condition] = evaluation.conditions+ const filterCount = (condition?.properties ?? []).length+ return `${condition?.rollout_percentage ?? 0}% of ${filterCount ? 'matching' : 'all'} ${+ UNIT_NOUNS[evaluationTarget(evaluation)]+ }${filterCount ? ` (${pluralize(filterCount, 'filter')})` : ''}`+}++/** How often a report arrives, such as "weekly" or "every 100 results", or "never" when reports are off. */+export function reportText(report: {+ enabled: boolean+ frequency: string+ scheduleCadence: string+ triggerThreshold: number+}): string {+ if (!report.enabled) {+ return 'never'+ }+ if (report.frequency === 'scheduled') {+ return report.scheduleCadence+ }+ return `every ${report.triggerThreshold} results`+}++/** Who does the checking, such as "an LLM judge (gpt-4o-mini)". */+export function methodText(evaluation: EvaluationConfig, usesDecisionModel: boolean, modelName?: string): string {+ const model = modelName ? ` (${modelName})` : ''+ switch (evaluation.evaluation_type) {+ case 'llm_judge':+ return `${usesDecisionModel ? 'a decision model' : 'an LLM judge'}${model}`+ case 'hog':+ return 'your code'+ case 'sentiment':+ return 'a sentiment classifier'+ }+}++/** Why reports cannot be turned on yet, or null when they can. Reports read pass rates. */+export function reportsUnavailableReason(evaluation: EvaluationConfig, isReportable: boolean): string | null {+ if (isReportable) {+ return null+ }+ const instructions = evaluation.evaluation_type === 'hog' ? 'code' : 'prompt'+ return evaluation.output_type === 'numeric'+ ? `Reports need a pass rate. Below the ${instructions}, set the score a run needs to pass.`+ : evaluation.output_type === 'categorical'+ ? `Reports need a pass rate. Below the ${instructions}, pick which categories count as a pass.`+ : 'Reports are not available for this kind of evaluation.'+}diff --git a/products/ai_observability/frontend/evaluations/evaluationReportLogic.ts b/products/ai_observability/frontend/evaluations/evaluationReportLogic.tsindex fd83fb402da2..b5bbc0221ef3 100644--- a/products/ai_observability/frontend/evaluations/evaluationReportLogic.ts+++ b/products/ai_observability/frontend/evaluations/evaluationReportLogic.ts@@ -44,6 +44,17 @@ export interface ReportConfigDraft {cooldownHours: number}+/** Where a report goes besides the Reports tab. */+export type ReportDestination = 'email' | 'slack'++/** The destinations a saved report already sends to, so their settings show when the form opens. */+function destinationsFromDraft(draft: ReportConfigDraft): ReportDestination[] {+ return [+ ...(draft.emailValue ? (['email'] as const) : []),+ ...(draft.slackIntegrationId !== null ? (['slack'] as const) : []),+ ]+}+export interface ReportConfigErrors {triggerThreshold?: stringcooldownHours?: string@@ -83,6 +94,10 @@ const DEFAULT_CONFIG_DRAFT: ReportConfigDraft = {cooldownHours: COOLDOWN_HOURS_DEFAULT,}+// A new evaluation reports daily. Saved evaluations without a report keep the older default as their baseline,+// so opening one does not show a change nobody made.+const NEW_EVALUATION_CONFIG_DRAFT: ReportConfigDraft = { ...DEFAULT_CONFIG_DRAFT, frequency: 'scheduled' }+export function getReportConfigErrors(draft: ReportConfigDraft): ReportConfigErrors {if (draft.frequency !== 'every_n') {return {}@@ -290,10 +305,14 @@ export interface evaluationReportLogicValues {reports: EvaluationReport[]reportsLoading: booleanselectedReportRun: EvaluationReportRun | null+ shownDestinations: ReportDestination[]}// Generated by kea-typegen. Update if you're an agent, ignore if you're human.export interface evaluationReportLogicActions {+ addDraftDestination: (destination: ReportDestination) => {+ destination: ReportDestination+ }createReport: (params: {cooldown_minutes?: number | nulldelivery_targets: EvaluationReportDeliveryTarget[]@@ -390,6 +409,9 @@ export interface evaluationReportLogicActions {reports: EvaluationReport[]payload?: any}+ removeDraftDestination: (destination: ReportDestination) => {+ destination: ReportDestination+ }resetDraft: () => {value: true}@@ -498,6 +520,9 @@ export const evaluationReportLogic = kea<evaluationReportLogicType>([setDraftCooldownHours: (cooldownHours: number) => ({ cooldownHours }),seedDraftFromReport: (report: EvaluationReport) => ({ report }),resetDraft: true,+ addDraftDestination: (destination: ReportDestination) => ({ destination }),+ /** Removes a destination and clears its settings, so the report stops sending there. */+ removeDraftDestination: (destination: ReportDestination) => ({ destination }),/** Save the current draft against the active report — creates if none exists, updates otherwise. */saveDraft: true,@@ -505,11 +530,23 @@ export const evaluationReportLogic = kea<evaluationReportLogicType>([selectReportRun: (reportRun: EvaluationReportRun | null) => ({ reportRun }),}),- reducers({+ reducers(({ props }) => ({configDraft: [- DEFAULT_CONFIG_DRAFT as ReportConfigDraft,+ (props.evaluationId === 'new' ? NEW_EVALUATION_CONFIG_DRAFT : DEFAULT_CONFIG_DRAFT) as ReportConfigDraft,{- setDraftEnabled: (state, { enabled }) => ({ ...state, enabled }),+ setDraftEnabled: (state, { enabled }) => {+ if (enabled) {+ return { ...state, enabled }+ }+ // Turning reports off hides these fields, so a bad value in them must not block saving.+ const errors = getReportConfigErrors(state)+ return {+ ...state,+ enabled,+ triggerThreshold: errors.triggerThreshold ? TRIGGER_THRESHOLD_DEFAULT : state.triggerThreshold,+ cooldownHours: errors.cooldownHours ? COOLDOWN_HOURS_DEFAULT : state.cooldownHours,+ }+ },setDraftFrequency: (state, { frequency }) => {return { ...state, frequency }},@@ -555,7 +592,11 @@ export const evaluationReportLogic = kea<evaluationReportLogicType>([cooldownHours,}),seedDraftFromReport: (_, { report }) => draftFromReport(report),- resetDraft: () => DEFAULT_CONFIG_DRAFT,+ resetDraft: () => (props.evaluationId === 'new' ? NEW_EVALUATION_CONFIG_DRAFT : DEFAULT_CONFIG_DRAFT),+ removeDraftDestination: (state, { destination }) =>+ destination === 'email'+ ? { ...state, emailValue: '' }+ : { ...state, slackIntegrationId: null, slackChannelValue: '' },},],selectedReportRun: [@@ -564,7 +605,18 @@ export const evaluationReportLogic = kea<evaluationReportLogicType>([selectReportRun: (_, { reportRun }) => reportRun,},],- }),+ // Tracked apart from the values, so clearing an input keeps its destination open while someone types.+ shownDestinations: [+ [] as ReportDestination[],+ {+ addDraftDestination: (state, { destination }) =>+ state.includes(destination) ? state : [...state, destination],+ removeDraftDestination: (state, { destination }) => state.filter((item) => item !== destination),+ seedDraftFromReport: (_, { report }) => destinationsFromDraft(draftFromReport(report)),+ resetDraft: () => [],+ },+ ],+ })),loaders(({ props, values }) => ({reports: [diff --git a/products/ai_observability/frontend/evaluations/evaluationVolume.ts b/products/ai_observability/frontend/evaluations/evaluationVolume.tsnew file mode 100644index 000000000000..8cf2251753b3--- /dev/null+++ b/products/ai_observability/frontend/evaluations/evaluationVolume.ts@@ -0,0 +1,115 @@+import { humanFriendlyNumber } from 'lib/utils/numbers'++import { NodeKind, TrendsQuery } from '~/queries/schema/schema-general'+import { BaseMathType, ChartDisplayType, HogQLMathType } from '~/types'++import type { EvaluationConditionSet, EvaluationConfig, EvaluationTarget } from './types'++/** How far back the estimate looks. A week covers weekday and weekend traffic. */+export const VOLUME_LOOKBACK_DAYS = 7++/** What one run evaluates. Sentiment always reads single generations, whatever target the row stores. */+export function evaluationTarget(evaluation: EvaluationConfig): EvaluationTarget {+ return evaluation.evaluation_type === 'sentiment' ? 'generation' : evaluation.target+}++// Ingestion keeps these only on AI events storage, so a filter on them matches nothing in the events the estimate counts.+const CONTENT_PROPERTIES = new Set([+ '$ai_input',+ '$ai_output',+ '$ai_output_choices',+ '$ai_input_state',+ '$ai_output_state',+ '$ai_tools',+])++export function filtersOnContent(conditions: EvaluationConditionSet[]): boolean {+ return conditions.some((condition) =>+ (condition.properties ?? []).some(+ (property) => 'key' in property && CONTENT_PROPERTIES.has(String(property.key))+ )+ )+}++const HAS_TRACE_ID = "notEmpty(coalesce(properties.$ai_trace_id, ''))"++/** The scheduler skips a generation without a trace ID, and a session evaluation also needs a session ID. */+const UNIT_COUNT_HOGQL: Record<Exclude<EvaluationTarget, 'generation'>, string> = {+ trace: `countDistinctIf(properties.$ai_trace_id, ${HAS_TRACE_ID})`,+ session: `countDistinctIf(properties.$ai_session_id, ${HAS_TRACE_ID} AND notEmpty(coalesce(properties.$ai_session_id, '')))`,+}++/**+ * Counts what one condition set matched over the lookback: every matching generation, and for a trace or session+ * target the distinct traces or sessions among them. Evaluations only run on `$ai_generation` events.+ */+export function conditionVolumeQuery(condition: EvaluationConditionSet, target: EvaluationTarget): TrendsQuery {+ const base = {+ kind: NodeKind.EventsNode as const,+ event: '$ai_generation',+ properties: condition.properties ?? [],+ }+ return {+ kind: NodeKind.TrendsQuery,+ series: [+ { ...base, math: BaseMathType.TotalCount },+ ...(target === 'generation'+ ? []+ : [+ {+ ...base,+ math: HogQLMathType.HogQL,+ math_hogql: UNIT_COUNT_HOGQL[target],+ },+ ]),+ ],+ dateRange: { date_from: `-${VOLUME_LOOKBACK_DAYS}d` },+ interval: 'day',+ trendsFilter: { display: ChartDisplayType.BoldNumber },+ }+}++export interface ConditionCounts {+ generations: number+ /** Units of the target: generations, or distinct traces or sessions. */+ units: number+}++export interface ConditionVolume extends ConditionCounts {+ rolloutPercentage: number+}++export interface VolumeEstimate {+ target: EvaluationTarget+ /** Generations that matched any condition set. Overlapping sets count a generation once per set. */+ matchingGenerations: number+ matchingUnits: number+ /** Runs the evaluation would have made over the lookback, after sampling. */+ runs: number+}++export function summarizeVolume(conditions: ConditionVolume[], target: EvaluationTarget): VolumeEstimate {+ return {+ target,+ matchingGenerations: conditions.reduce((sum, condition) => sum + condition.generations, 0),+ matchingUnits: conditions.reduce((sum, condition) => sum + condition.units, 0),+ runs: Math.round(+ conditions.reduce((sum, condition) => sum + (condition.units * condition.rolloutPercentage) / 100, 0)+ ),+ }+}++/** How often the evaluation runs, from its runs over the lookback, such as "about 14 times a day". */+export function runRate(runs: number): string {+ const perDay = Math.round(runs / VOLUME_LOOKBACK_DAYS)+ if (perDay >= 2) {+ return `about ${humanFriendlyNumber(perDay)} times a day`+ }+ if (perDay === 1) {+ return 'about once a day'+ }+ if (runs === 0) {+ return 'less than once a week'+ }+ return runs === 1 ? 'once a week' : `about ${runs} times a week`+}diff --git a/products/ai_observability/frontend/evaluations/evaluationVolumeLogic.ts b/products/ai_observability/frontend/evaluations/evaluationVolumeLogic.tsnew file mode 100644index 000000000000..2fff71dda030--- /dev/null+++ b/products/ai_observability/frontend/evaluations/evaluationVolumeLogic.ts@@ -0,0 +1,182 @@+import { MakeLogicType, connect, kea, key, path, props, reducers, selectors } from 'kea'+import { loaders } from 'kea-loaders'+import { subscriptions } from 'kea-subscriptions'++import { uuid } from 'lib/utils/dom'+import { teamLogic } from 'scenes/teamLogic'++import { cancelQueryOnServer } from '~/queries/nodes/DataNode/dataNodeLogic'+import { performQuery } from '~/queries/query'++import {+ type ConditionCounts,+ type VolumeEstimate,+ conditionVolumeQuery,+ evaluationTarget,+ filtersOnContent,+ summarizeVolume,+} from './evaluationVolume'+import { type LLMEvaluationLogicProps, llmEvaluationLogic, llmEvaluationLogicKey } from './llmEvaluationLogic'+import type { EvaluationConditionSet, EvaluationConfig, EvaluationTarget } from './types'++interface VolumeInput {+ conditions: EvaluationConditionSet[]+ target: EvaluationTarget+}++/** Waits for typing in a filter to settle before querying. */+const VOLUME_DEBOUNCE_MS = 600++// Generated by kea-typegen. Update if you're an agent, ignore if you're human.+export interface evaluationVolumeLogicValues {+ evaluation: EvaluationConfig | null // llmEvaluationLogic+ currentTeamId: number | null // teamLogic+ counts: ConditionCounts[] | null+ countsError: boolean+ countsKey: string+ countsLoading: boolean+ volume: VolumeEstimate | null+ volumeInput: VolumeInput | null+}++// Generated by kea-typegen. Update if you're an agent, ignore if you're human.+export interface evaluationVolumeLogicActions {+ loadCounts: (_: void) => void+ loadCountsFailure: (+ error: string,+ errorObject?: any+ ) => {+ error: string+ errorObject?: any+ }+ loadCountsSuccess: (+ counts: ConditionCounts[] | null,+ payload?: void+ ) => {+ counts: ConditionCounts[] | null+ payload?: void+ }+}++// Generated by kea-typegen. Update if you're an agent, ignore if you're human.+export interface evaluationVolumeLogicMeta {+ key: string+ __keaTypeGenInternalSelectorTypes: {+ volumeInput: (evaluation: EvaluationConfig | null) => VolumeInput | null+ countsKey: (volumeInput: VolumeInput | null) => string+ volume: (counts: ConditionCounts[] | null, volumeInput: VolumeInput | null) => VolumeEstimate | null+ }+}++export type evaluationVolumeLogicType = MakeLogicType<+ evaluationVolumeLogicValues,+ evaluationVolumeLogicActions,+ LLMEvaluationLogicProps,+ evaluationVolumeLogicMeta+>++/** Estimates how often an unsaved evaluation would run, from the generations its triggers matched last week. */+export const evaluationVolumeLogic = kea<evaluationVolumeLogicType>([+ path((key) => ['products', 'ai_observability', 'frontend', 'evaluations', 'evaluationVolumeLogic', key]),+ // Same props and key as the evaluation it estimates, so a template draft reads its own evaluation.+ props({} as LLMEvaluationLogicProps),+ key(llmEvaluationLogicKey),+ connect((props: LLMEvaluationLogicProps) => ({+ values: [llmEvaluationLogic(props), ['evaluation'], teamLogic, ['currentTeamId']],+ })),+ loaders(({ values, cache }) => ({+ counts: [+ null as ConditionCounts[] | null,+ {+ loadCounts: async (_: void, breakpoint) => {+ await breakpoint(VOLUME_DEBOUNCE_MS)+ const input = values.volumeInput+ if (!input) {+ return null+ }+ const controller = new AbortController()+ const running = new Set<string>()+ cache.disposables.add(+ () => () => {+ controller.abort()+ running.forEach((queryId) => void cancelQueryOnServer(values.currentTeamId, queryId))+ },+ 'volumeQueries',+ { pauseOnPageHidden: false }+ )+ let counts: ConditionCounts[]+ try {+ counts = await Promise.all(+ input.conditions.map(async (condition): Promise<ConditionCounts> => {+ const queryId = uuid()+ running.add(queryId)+ const response = await performQuery(+ conditionVolumeQuery(condition, input.target),+ { signal: controller.signal },+ undefined,+ queryId+ ).finally(() => running.delete(queryId))+ const [generations = 0, units = generations] = (response.results ?? []).map(+ (series: { aggregated_value?: number }) => series.aggregated_value ?? 0+ )+ return { generations, units }+ })+ )+ } catch (error) {+ // A superseded load fails on the abort, so it ends here instead of showing an error.+ breakpoint()+ throw error+ }+ breakpoint()+ return counts+ },+ },+ ],+ })),+ selectors({+ volumeInput: [+ (s) => [s.evaluation],+ (evaluation: EvaluationConfig | null): VolumeInput | null =>+ evaluation && !filtersOnContent(evaluation.conditions)+ ? { conditions: evaluation.conditions, target: evaluationTarget(evaluation) }+ : null,+ ],+ // Sampling changes only scale the counts, so the key leaves it out and moving the slider queries nothing.+ countsKey: [+ (s) => [s.volumeInput],+ (input: VolumeInput | null): string =>+ JSON.stringify(+ input && {+ target: input.target,+ properties: input.conditions.map((condition) => condition.properties ?? []),+ }+ ),+ ],+ volume: [+ (s) => [s.counts, s.volumeInput],+ (counts: ConditionCounts[] | null, input: VolumeInput | null): VolumeEstimate | null =>+ counts && input && counts.length === input.conditions.length+ ? summarizeVolume(+ counts.map((count, index) => ({+ ...count,+ rolloutPercentage: input.conditions[index].rollout_percentage ?? 0,+ })),+ input.target+ )+ : null,+ ],+ }),+ reducers({+ countsError: [+ false,+ {+ loadCounts: () => false,+ loadCountsSuccess: () => false,+ loadCountsFailure: () => true,+ },+ ],+ }),+ subscriptions(({ actions }) => ({+ countsKey: () => actions.loadCounts(),+ })),+])diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.tsindex 5aaac3253909..ecbc2abbb9c7 100644--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts@@ -275,6 +275,26 @@ describe('llmEvaluationLogic', () => {logic.mount()})+ it.each([+ ['sentiment' as const, true],+ ['hog' as const, true],+ ['llm_judge' as const, false],+ ])('starts a new %s evaluation running only when it can run without a provider key', async (type, expected) => {+ await expectLogic(keysLogic).toFinishAllListeners()+ logic.actions.setEvaluationType(type)+ expect(logic.values.evaluation?.enabled).toBe(expected)+ })++ it('keeps a new evaluation paused once someone turned it off by hand', async () => {+ await expectLogic(keysLogic).toFinishAllListeners()+ logic.actions.setEvaluationType('hog')+ logic.actions.setEvaluationEnabled(false)+ // Through a judge and back, so canEnable changes and the default would turn it on again.+ logic.actions.setEvaluationType('llm_judge')+ logic.actions.setEvaluationType('hog')+ expect(logic.values.evaluation?.enabled).toBe(false)+ })+it.each([[DEFAULT_HOG_SOURCE, 'return 0;'],['return 42;', 'return 42;'],@@ -2317,6 +2337,7 @@ return result`,model: 'gpt-5-mini',provider_key_id: 'key-1',})+ reportLogic.actions.setDraftFrequency('every_n')reportLogic.actions.setDraftTriggerThreshold(67)expect(reportLogic.values.configErrors.triggerThreshold).toBe(@@ -2455,6 +2476,43 @@ return result`,})})+ describe('report destinations', () => {+ it.each([+ { name: 'a new evaluation reports daily', evaluationId: 'new', frequency: 'scheduled' },+ {+ name: 'a saved evaluation without a report keeps the older baseline, so it shows no unsaved change',+ evaluationId: 'saved-eval',+ frequency: 'every_n',+ },+ ])('$name', ({ evaluationId, frequency }) => {+ const reportLogic = evaluationReportLogic({ evaluationId })+ reportLogic.mount()+ try {+ expect(reportLogic.values.configDraft).toMatchObject({ frequency, scheduleCadence: 'daily' })+ } finally {+ reportLogic.unmount()+ }+ })++ it('keeps an emptied destination open until it is removed, then clears it', () => {+ const reportLogic = evaluationReportLogic({ evaluationId: 'new' })+ reportLogic.mount()+ try {+ reportLogic.actions.addDraftDestination('email')+ reportLogic.actions.setDraftEmailValue('[email protected]')+ reportLogic.actions.setDraftEmailValue('')+ expect(reportLogic.values.shownDestinations).toEqual(['email'])++ reportLogic.actions.setDraftEmailValue('[email protected]')+ reportLogic.actions.removeDraftDestination('email')+ expect(reportLogic.values.shownDestinations).toEqual([])+ expect(reportLogic.values.configDraft.emailValue).toEqual('')+ } finally {+ reportLogic.unmount()+ }+ })+ })+describe('saveEvaluation failure handling', () => {beforeEach(() => {useMocks({diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.tsindex 700ba3057380..a7794abd62c9 100644--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts@@ -1,10 +1,13 @@import { MakeLogicType, actions, afterMount, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea'import { loaders } from 'kea-loaders'import { actionToUrl, combineUrl, router, urlToAction } from 'kea-router'+import { subscriptions } from 'kea-subscriptions'import posthog from 'posthog-js'import { SetupTaskId, globalSetupLogic } from 'lib/components/ProductSetup'+import { FEATURE_FLAGS } from 'lib/constants'import { lemonToast } from 'lib/lemon-ui/LemonToast'+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'import { objectsEqual } from 'lib/utils/objects'import { teamLogic } from 'scenes/teamLogic'import { urls } from 'scenes/urls'@@ -297,6 +300,12 @@ function buildHogTestRequest(evaluation: TestableHogEvaluation, forComparison =return request}+export function llmEvaluationLogicKey(props: LLMEvaluationLogicProps): string {+ return `${props.evaluationId || 'new'}${props.templateKey ? `-${props.templateKey}` : ''}${+ props.evaluationType ? `-${props.evaluationType}` : ''+ }`+}+export interface LLMEvaluationLogicProps {evaluationId: stringtemplateKey?: EvaluationTemplateKey@@ -315,6 +324,7 @@ export interface llmEvaluationLogicValues {breadcrumbs: Breadcrumb[]canEnable: booleancanEnableReason: string | null+ enabledChosen: booleanevaluation: EvaluationConfig | nullevaluationBackTarget: EvaluationBackTargetevaluationFormSubmitting: boolean@@ -326,6 +336,7 @@ export interface llmEvaluationLogicValues {evaluationRunsHasMore: booleanevaluationRunsLoading: booleanfilteredEvaluationRuns: EvaluationRun[]+ formStep: stringformValid: booleanhasUnsavedChanges: booleanhogTestMessage: string | null@@ -356,6 +367,7 @@ export interface llmEvaluationLogicValues {total: number} | nullselectedModel: string+ selectedModelName: string | undefinedselectedPickerProviderKeyId: string | nullsidePanelContext: SidePanelSceneContext | nulltrendInsightUrl: string | null@@ -477,9 +489,15 @@ export interface llmEvaluationLogicActions {setAllowsNA: (allowsNA: boolean) => {allowsNA: boolean}+ setDefaultEnabled: (enabled: boolean) => {+ enabled: boolean+ }setEvaluationDescription: (description: string) => {description: string}+ setEvaluationDirectory: (directoryId: string | null) => {+ directoryId: string | null+ }setEvaluationEnabled: (enabled: boolean) => {enabled: boolean}@@ -505,6 +523,9 @@ export interface llmEvaluationLogicActions {setEvaluationType: (evaluationType: EvaluationType) => {evaluationType: EvaluationType}+ setFormStep: (step: string) => {+ step: string+ }setHogSource: (source: string) => {source: string}@@ -578,6 +599,11 @@ export interface llmEvaluationLogicMeta {) => booleanjudgeMethod: (evaluation: EvaluationConfig | null, byokModels: ModelOption[]) => JudgeMethodusesDecisionModel: (judgeMethod: JudgeMethod) => boolean+ selectedModelName: (+ byokModels: ModelOption[],+ selectedModel: string,+ selectedPickerProviderKeyId: string | null+ ) => string | undefinedjudgeModelGroups: (evaluationProviderModelGroups: ProviderModelGroup[],judgeMethod: JudgeMethod@@ -637,12 +663,7 @@ export type llmEvaluationLogicType = MakeLogicType<export const llmEvaluationLogic = kea<llmEvaluationLogicType>([path(['products', 'ai_observability', 'evaluations', 'llmEvaluationLogic']),props({} as LLMEvaluationLogicProps),- key(- (props) =>- `${props.evaluationId || 'new'}${props.templateKey ? `-${props.templateKey}` : ''}${- props.evaluationType ? `-${props.evaluationType}` : ''- }`- ),+ key(llmEvaluationLogicKey),connect(() => ({values: [@@ -658,8 +679,10 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([// Evaluation configuration actionssetEvaluationName: (name: string) => ({ name }),setEvaluationDescription: (description: string) => ({ description }),+ setEvaluationDirectory: (directoryId: string | null) => ({ directoryId }),setEvaluationPrompt: (prompt: string) => ({ prompt }),setEvaluationEnabled: (enabled: boolean) => ({ enabled }),+ setDefaultEnabled: (enabled: boolean) => ({ enabled }),setOutputType: (outputType: 'boolean' | 'numeric' | 'categorical') => ({outputType,previousEvaluation: values.evaluation,@@ -681,6 +704,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([// Tab navigationsetActiveTab: (tab: string) => ({ tab }),+ setFormStep: (step: string) => ({ step }),setRunsBackfillId: (backfillId: string | null) => ({ backfillId }),setRunsDates: (dateFrom: string | null, dateTo: string | null) => ({ dateFrom, dateTo }),@@ -764,6 +788,13 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([},]}+ // pinned: analytics event name and properties, because usage reviews read them+ posthog.capture('llma evaluation code tested', {+ is_new: props.evaluationId === 'new',+ target: evaluation.target,+ samples: results.filter((result) => result.sample_id !== 'error').length,+ errors: results.filter((result) => result.error).length,+ })breakpoint?.()const currentEvaluation = values.evaluation@@ -885,11 +916,14 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([{setEvaluationName: (state, { name }) => (state ? { ...state, name } : null),setEvaluationDescription: (state, { description }) => (state ? { ...state, description } : null),+ setEvaluationDirectory: (state, { directoryId }) =>+ state ? { ...state, directory_id: directoryId } : null,setEvaluationPrompt: (state, { prompt }) =>state && isLLMJudgeEvaluation(state)? { ...state, evaluation_config: { ...state.evaluation_config, prompt } }: state,setEvaluationEnabled: (state, { enabled }) => (state ? { ...state, enabled } : null),+ setDefaultEnabled: (state, { enabled }) => (state ? { ...state, enabled } : null),setOutputType: (state, { outputType, outputConfig }) => {if (!state ||@@ -1116,11 +1150,21 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([saveEvaluationFailure: () => false,},],+ // Whether someone switched "enabled" by hand. Until then a new evaluation follows whether it can run.+ enabledChosen: [+ false,+ {+ setEvaluationEnabled: () => true,+ loadEvaluationSuccess: () => false,+ resetEvaluation: () => false,+ },+ ],hasUnsavedChanges: [false,{setEvaluationName: () => true,setEvaluationDescription: () => true,+ setEvaluationDirectory: () => true,setEvaluationPrompt: () => true,setEvaluationEnabled: () => true,setOutputType: () => true,@@ -1173,6 +1217,13 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([setRunsBackfillId: () => null,},],+ // The open step of the step layout, here so a failed save can open the step that needs fixing.+ formStep: [+ 'name' as string,+ {+ setFormStep: (_, { step }) => step,+ },+ ],activeTab: ['configuration' as string,{@@ -1198,7 +1249,11 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([)}},- loadEvaluationSuccess: () => {+ loadEvaluationSuccess: ({ evaluation }) => {+ // A draft can open already able to run, so the canEnable subscription never sees a change to apply.+ if (props.evaluationId === 'new' && evaluation && evaluation.enabled !== values.canEnable) {+ actions.setDefaultEnabled(values.canEnable)+ }actions.loadRunsStats()// The filter query reads the evaluation's pass rules, and a sentiment evaluation opens on a non-'all' filter.if (values.evaluationRunsFilter !== 'all') {@@ -1213,7 +1268,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([values.evaluation?.enabled &&!evaluationCanResolveModel(values.evaluation, values.activeProviderKey)) {- actions.setEvaluationEnabled(false)+ actions.setDefaultEnabled(false)}},@@ -1476,7 +1531,18 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([}}- router.actions.push(getEvaluationBackTarget(false, router.values.searchParams).path)+ // The new creation page lands on the evaluation itself, which says when its first result arrives.+ const opensEvaluation =+ isNew &&+ !!response?.id &&+ !!featureFlagLogic.findMounted()?.values.featureFlags[+ FEATURE_FLAGS.AI_OBSERVABILITY_EVALUATIONS_OVERVIEW+ ]+ router.actions.push(+ opensEvaluation+ ? urls.aiObservabilityEvaluation(response.id)+ : getEvaluationBackTarget(false, router.values.searchParams).path+ )} catch (error) {const message = evaluationErrorMessage(error, 'Failed to save evaluation')lemonToast.error(message)@@ -1534,6 +1600,10 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([if (!values.isReportableEvaluation && values.activeTab === 'reports') {actions.setActiveTab('configuration')}+ // Code and sentiment drop the model, so the picker must not keep showing it for a later switch back.+ if (!values.evaluation?.model_configuration && values.selectedModel) {+ actions.setModelConfiguration(null)+ }},})),@@ -1675,6 +1745,12 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([},],usesDecisionModel: [(s) => [s.judgeMethod], (judgeMethod: JudgeMethod): boolean => judgeMethod === 'decision'],+ // Evals always run on the team's own provider key, so only BYOK models can be the selected one.+ selectedModelName: [+ (s) => [s.byokModels, s.selectedModel, s.selectedPickerProviderKeyId],+ (models: ModelOption[], selectedModel: string, providerKeyId: string | null): string | undefined =>+ models.find((m) => m.id === selectedModel && m.providerKeyId === providerKeyId)?.name,+ ],judgeModelGroups: [(s) => [s.evaluationProviderModelGroups, s.judgeMethod],(groups: ProviderModelGroup[], judgeMethod: JudgeMethod): ProviderModelGroup[] =>@@ -1930,6 +2006,19 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([},})),+ subscriptions(({ actions, values, props }) => ({+ // A new evaluation starts running when it can: code and sentiment need no key, a judge needs a working one.+ canEnable: (canEnable: boolean) => {+ if (+ props.evaluationId === 'new' &&+ !values.enabledChosen &&+ values.evaluation &&+ values.evaluation.enabled !== canEnable+ ) {+ actions.setDefaultEnabled(canEnable)+ }+ },+ })),afterMount(({ actions, props }) => {actions.loadProviderKeys()actions.loadEvaluation()