:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(experiments): grade prompt experiment eval pass rates with current evaluation rules (#121139)
Bernat Torres937ddd5a40cf
diff --git a/frontend/src/scenes/experiments/utils.test.ts b/frontend/src/scenes/experiments/utils.test.tsindex 7dedd9d1ba8e..e3d1a0150d53 100644--- a/frontend/src/scenes/experiments/utils.test.ts+++ b/frontend/src/scenes/experiments/utils.test.ts@@ -3,6 +3,7 @@ import { entityFilterLogic } from 'scenes/insights/filters/ActionFilter/entityFiimport experimentJson from '~/mocks/fixtures/api/experiments/_experiment_launched.json'import {CachedNewExperimentQueryResponse,+ EventsNode,ExperimentEventExposureConfig,ExperimentExposureConfig,ExperimentMetric,@@ -1080,6 +1081,34 @@ describe('getEventCountQuery', () => {expect(query?.tags).toEqual({ productKey: ProductKey.PRODUCT_ANALYTICS })})+ it('drops evaluation pass markers, which only the experiment results query can resolve', () => {+ const promptFilter = {+ key: '$ai_prompt_name',+ type: PropertyFilterType.Event,+ operator: PropertyOperator.Exact,+ value: 'p',+ } as const+ const passed: EventsNode = {+ kind: NodeKind.EventsNode,+ event: '$ai_evaluation',+ properties: [promptFilter, { key: 'aiEvaluationPassed()', type: PropertyFilterType.HogQL }],+ }+ const ratio: ExperimentMetric = {+ kind: NodeKind.ExperimentMetric,+ metric_type: ExperimentMetricType.RATIO,+ numerator: passed,+ denominator: { kind: NodeKind.EventsNode, event: '$ai_evaluation' },+ }+ const funnel: ExperimentMetric = {+ kind: NodeKind.ExperimentMetric,+ metric_type: ExperimentMetricType.FUNNEL,+ series: [passed],+ }++ expect(getEventCountQuery(ratio, true)?.series[0].properties).toEqual([promptFilter])+ expect(getEventCountQuery(funnel, true)?.series[0].properties).toEqual([promptFilter])+ })+it('returns null when series is empty', () => {const metric: ExperimentMetric = {uuid: 'test-metric',diff --git a/frontend/src/scenes/experiments/utils.ts b/frontend/src/scenes/experiments/utils.tsindex 59850ab036bd..3fe16439ad39 100644--- a/frontend/src/scenes/experiments/utils.ts+++ b/frontend/src/scenes/experiments/utils.ts@@ -29,6 +29,7 @@ import {} from '~/queries/schema/schema-general'import { isFunnelsQuery, isNodeWithSource, isTrendsQuery, isValidQueryForExperiment } from '~/queries/utils'import {+ AnyPropertyFilter,BreakdownAttributionType,ChartDisplayType,Experiment,@@ -38,6 +39,7 @@ import {FeatureFlagType,FunnelConversionWindowTimeUnit,MultivariateFlagVariant,+ PropertyFilterType,type InsightModel,UniversalFiltersGroupValue,} from '~/types'@@ -148,13 +150,21 @@ export function isEvenlyDistributed(variants: MultivariateFlagVariant[]): booleareturn variants.every((variant, index) => variant.rollout_percentage === evenPercentages[index])}+// The experiments backend replaces these HogQL filter keys with the team's current evaluation pass rules. Other query+// kinds cannot resolve them, so queries outside the experiment results drop them.+const EVALUATION_PASS_MARKERS = new Set(['aiEvaluationGraded()', 'aiEvaluationPassed()'])++function withoutEvaluationPassMarkers(properties: AnyPropertyFilter[]): AnyPropertyFilter[] {+ return properties.filter((p) => !(p.type === PropertyFilterType.HogQL && EVALUATION_PASS_MARKERS.has(p.key)))+}+function seriesToFilter(series: AnyEntityNode | ExperimentMetricSource): UniversalFiltersGroupValue | null {if (series.kind === NodeKind.EventsNode) {return {id: series.event ?? null,name: series.event as string,type: 'events',- properties: series.properties ?? [],+ properties: withoutEvaluationPassMarkers(series.properties ?? []),}}@@ -653,7 +663,9 @@ const getEventCountSeries = (metric: ExperimentMetric): AnyEntityNode[] => {event: lastStep.event,math: ExperimentMetricMathType.TotalCount,...(lastStep.properties &&- lastStep.properties.length > 0 && { properties: lastStep.properties }),+ lastStep.properties.length > 0 && {+ properties: withoutEvaluationPassMarkers(lastStep.properties),+ }),},]} else if (lastStep.kind === NodeKind.ActionsNode) {@@ -693,7 +705,10 @@ const getEventCountSeries = (metric: ExperimentMetric): AnyEntityNode[] => {name: eventsNode.event || undefined,event: eventsNode.event || undefined,math: ExperimentMetricMathType.TotalCount,- ...(eventsNode.properties && eventsNode.properties.length > 0 && { properties: eventsNode.properties }),+ ...(eventsNode.properties &&+ eventsNode.properties.length > 0 && {+ properties: withoutEvaluationPassMarkers(eventsNode.properties),+ }),},]).with({ kind: NodeKind.ActionsNode }, (actionsNode) => [diff --git a/products/ai_observability/backend/facade/api.py b/products/ai_observability/backend/facade/api.pyindex 578b0a0aa9c7..de542494bf27 100644--- a/products/ai_observability/backend/facade/api.py+++ b/products/ai_observability/backend/facade/api.py@@ -1,6 +1,8 @@from collections.abc import Sequencefrom datetime import date, datetime+from posthog.schema import HogQLPropertyFilter+from posthog.models import Team, Userfrom ..logic.evaluations.evaluation_metas import load_evaluation_meta, load_evaluation_metas@@ -178,6 +180,28 @@ def get_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFiltersreturn build_evaluation_pass_filters(team_id)+def resolve_evaluation_pass_markers[P](properties: list[P], team_id: int) -> list[P]:+ """Replace each evaluation pass marker with the team's current condition."""+ markers = {contracts.EVALUATION_GRADED_MARKER, contracts.EVALUATION_PASSED_MARKER}+ if not any(isinstance(prop, HogQLPropertyFilter) and prop.key in markers for prop in properties):+ return properties+ filters = build_evaluation_pass_filters(team_id)+ conditions = {+ contracts.EVALUATION_GRADED_MARKER: filters.graded,+ contracts.EVALUATION_PASSED_MARKER: filters.passed,+ }+ return [+ prop.model_copy(update={"key": conditions[prop.key]})+ if isinstance(prop, HogQLPropertyFilter) and prop.key in conditions+ else prop+ for prop in properties+ ]+++def mentions_evaluation_pass_markers(text: str) -> bool:+ return contracts.EVALUATION_GRADED_MARKER in text or contracts.EVALUATION_PASSED_MARKER in text++def get_trace_evaluation_results(team: Team, user: User | None, trace_id: str) -> contracts.TraceEvaluationResults:rows = load_evaluation_results(team, user, trace_id)configs = load_output_configs(team, user, {row.evaluation_id for row in rows if row.evaluation_id})diff --git a/products/ai_observability/backend/facade/contracts.py b/products/ai_observability/backend/facade/contracts.pyindex 3103e4335a35..58dae6336039 100644--- a/products/ai_observability/backend/facade/contracts.py+++ b/products/ai_observability/backend/facade/contracts.py@@ -111,6 +111,12 @@ class EvaluationPassFilters(Contract):passed: str # The run passed. Implies `graded`.+# HogQL filter keys that stand in for `EvaluationPassFilters` in a stored filter, so that the stored filter holds no+# evaluation rules and always grades with the current ones. `resolve_evaluation_pass_markers` replaces them at query time.+EVALUATION_GRADED_MARKER = "aiEvaluationGraded()"+EVALUATION_PASSED_MARKER = "aiEvaluationPassed()"++EvaluationOutcome = Literal["pass", "fail", "inconclusive", "unrated"]diff --git a/products/experiments/backend/hogql_queries/base_query_utils.py b/products/experiments/backend/hogql_queries/base_query_utils.pyindex e7e25cdd5422..e68f7fa7deef 100644--- a/products/experiments/backend/hogql_queries/base_query_utils.py+++ b/products/experiments/backend/hogql_queries/base_query_utils.py@@ -29,6 +29,7 @@ from posthog.clickhouse.query_tagging import tag_contains_user_hogqlfrom posthog.models.team.team import Teamfrom products.actions.backend.models.action import Action+from products.ai_observability.backend.facade.api import resolve_evaluation_pass_markersfrom products.experiments.backend.hogql_queries.hogql_aggregation_utils import (contains_aggregation,extract_aggregation_and_inner_expr,@@ -186,7 +187,8 @@ def event_or_action_to_filter()if entity_node.properties:- event_properties = ast.And(exprs=[property_to_expr(property, team) for property in entity_node.properties])+ properties = resolve_evaluation_pass_markers(entity_node.properties, team.pk)+ event_properties = ast.And(exprs=[property_to_expr(property, team) for property in properties])event_filter = ast.And(exprs=[event_filter, event_properties])return event_filterdiff --git a/products/experiments/backend/hogql_queries/experiment_query_runner.py b/products/experiments/backend/hogql_queries/experiment_query_runner.pyindex c9355d2b2474..8e75e95e2287 100644--- a/products/experiments/backend/hogql_queries/experiment_query_runner.py+++ b/products/experiments/backend/hogql_queries/experiment_query_runner.py@@ -1,3 +1,4 @@+import jsonimport uuidfrom datetime import UTC, datetime, timedeltafrom functools import cached_property@@ -36,6 +37,7 @@ from posthog.hogql_queries.query_runner import QueryRunnerfrom posthog.hogql_queries.utils.query_date_range import QueryDateRangefrom posthog.models.team.extensions import get_or_create_team_extension+from products.ai_observability.backend.facade.api import mentions_evaluation_pass_markersfrom products.analytics_platform.backend.lazy_computation.lazy_computation_executor import LazyComputationResultfrom products.experiments.backend.hogql_queries import MULTIPLE_VARIANT_KEY, get_baseline_variant_keyfrom products.experiments.backend.hogql_queries.base_query_utils import experiment_window, experiment_window_end@@ -438,15 +440,17 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):):modifiers.sessionIdPushdown = True- experiment_query_debug = get_experiment_query_debug(- experiment_query_ast,- self.team,- user=self.user,- bypass_warehouse_access_control=self.bypass_warehouse_access_control,- modifiers=modifiers,- )- self.hogql = experiment_query_debug[0]- self.clickhouse_sql = experiment_query_debug[1]+ # The debug SQL would expose the evaluation rules that the markers keep out of the browser.+ if not self._uses_evaluation_pass_markers():+ experiment_query_debug = get_experiment_query_debug(+ experiment_query_ast,+ self.team,+ user=self.user,+ bypass_warehouse_access_control=self.bypass_warehouse_access_control,+ modifiers=modifiers,+ )+ self.hogql = experiment_query_debug[0]+ self.clickhouse_sql = experiment_query_debug[1]settings = HogQLGlobalSettings(max_execution_time=self.max_execution_time,@@ -619,8 +623,18 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):return variants + variants_missingdef to_actors_query(self) -> ast.SelectQuery:+ # The actors response returns its generated SQL, which would expose the evaluation rules.+ if self._uses_evaluation_pass_markers():+ raise ValidationError("Persons are not available for evaluation pass rate metrics.")return build_experiment_actors_query(self)+ def _uses_evaluation_pass_markers(self) -> bool:+ return mentions_evaluation_pass_markers(+ self.metric.model_dump_json()+ + json.dumps(self.experiment.exposure_criteria or {})+ + (self.actors_query.model_dump_json() if self.actors_query else "")+ )+def to_query(self) -> ast.SelectQuery:raise ValidationError(f"Cannot convert source query of type {self.query.metric.kind} to query")diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_ratio_metric.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_ratio_metric.pyindex 76b71867f6f9..024ced5b53ee 100644--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_ratio_metric.py+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_ratio_metric.py@@ -24,8 +24,10 @@ from posthog.schema import (from posthog.test.test_journeys import journeys_forfrom products.actions.backend.models.action import Action+from products.ai_observability.backend.models.evaluations import Evaluationfrom products.experiments.backend.hogql_queries.experiment_query_runner import ExperimentQueryRunnerfrom products.experiments.backend.hogql_queries.test.experiment_query_runner.base import ExperimentQueryRunnerBaseTest+from products.experiments.backend.llm_metric_templates import build_eval_pass_rate_metric@override_settings(IN_UNIT_TESTING=True)@@ -182,6 +184,72 @@ class TestExperimentRatioMetric(ExperimentQueryRunnerBaseTest):self.assertIsNotNone(control_variant.numerator_denominator_sum_product)self.assertIsNotNone(test_variant.numerator_denominator_sum_product)+ @parameterized.expand(+ [+ ("direct", False),+ ("precomputed", True),+ ]+ )+ @time_machine.travel("2020-01-01T12:00:00Z", tick=False)+ def test_eval_pass_rate_grades_with_current_evaluation_rules(self, name: str, use_precomputation: bool) -> None:+ self._setup_precomputation_test(use_precomputation)+ feature_flag = self.create_feature_flag()+ experiment = self.create_experiment(feature_flag=feature_flag)+ experiment.stats_config = {"method": "frequentist"}+ metric = build_eval_pass_rate_metric("my-prompt")+ experiment.metrics = [metric.model_dump(mode="json")]+ self._save_experiment_with_precomputation(experiment, use_precomputation)+ detector = Evaluation.objects.create(+ team=self.team,+ name="detector",+ evaluation_type="hog",+ evaluation_config={"source": "return true"},+ output_type="boolean",+ output_config={"true_is_failure": True},+ )++ # A detector run passes when it detects nothing, so control passes 3 of 4 runs and test passes 1 of 4.+ for variant, results in (("control", [False, False, False, True]), ("test", [False, True, True, True])):+ for i, result in enumerate(results):+ distinct_id = f"user_{variant}_{i}"+ _create_person(distinct_ids=[distinct_id], team_id=self.team.pk)+ _create_event(+ team=self.team,+ event="$feature_flag_called",+ distinct_id=distinct_id,+ timestamp="2020-01-02T12:00:00Z",+ properties={"$feature_flag_response": variant, "$feature_flag": feature_flag.key},+ )+ _create_event(+ team=self.team,+ event="$ai_evaluation",+ distinct_id=distinct_id,+ timestamp="2020-01-02T12:01:00Z",+ properties={+ "$ai_prompt_name": "my-prompt",+ "$ai_evaluation_id": str(detector.id),+ "$ai_evaluation_result": result,+ },+ )+ flush_persons_and_events()++ def passed_and_graded() -> list[tuple[float, float | None]]:+ query = ExperimentQuery(experiment_id=experiment.id, kind="ExperimentQuery", metric=metric)+ runner = ExperimentQueryRunner(query=query, team=self.team)+ result = cast(ExperimentQueryResponse, runner.calculate())+ assert runner._metric_events_precomputed is use_precomputation+ assert result.baseline is not None and result.variant_results is not None+ assert result.hogql is None and result.clickhouse_sql is None+ return [(variant.sum, variant.denominator_sum) for variant in [result.baseline, *result.variant_results]]++ self.assertEqual(passed_and_graded(), [(3, 4), (1, 4)])++ # The experiment stores no rules, so the next query grades with the edited ones. On the precomputed path this+ # also proves that the edit changed the job hash, because reused jobs would still hold the old pass counts.+ detector.output_config = {}+ detector.save()+ self.assertEqual(passed_and_graded(), [(1, 4), (3, 4)])+@parameterized.expand([("direct", False),diff --git a/products/experiments/backend/hogql_queries/test/test_experiment_actors_query.py b/products/experiments/backend/hogql_queries/test/test_experiment_actors_query.pyindex 9b7da013acbf..2305095a22d0 100644--- a/products/experiments/backend/hogql_queries/test/test_experiment_actors_query.py+++ b/products/experiments/backend/hogql_queries/test/test_experiment_actors_query.py@@ -21,6 +21,7 @@ from unittest.mock import patchfrom django.test import override_settingsfrom parameterized import parameterized+from rest_framework.exceptions import ValidationErrorfrom posthog.schema import (ActorsQuery,@@ -29,12 +30,14 @@ from posthog.schema import (ExperimentEventExposureConfig,ExperimentFunnelMetric,ExperimentQuery,+ HogQLPropertyFilter,)from posthog.hogql.context import HogQLContextfrom posthog.hogql_queries.actors_query_runner import ActorsQueryRunner+from products.ai_observability.backend.facade.contracts import EVALUATION_PASSED_MARKERfrom products.cohorts.backend.models.cohort import Cohortfrom products.cohorts.backend.models.util import print_cohort_hogql_queryfrom products.experiments.backend.hogql_queries.exposure_query_logic import EXPERIMENT_EXPOSURE_EVENT_CUTOFF@@ -195,6 +198,36 @@ class TestExperimentActorsQuery(ExperimentQueryRunnerBaseTest, ClickhouseTestMixfor distinct_id in distinct_ids:assert distinct_id.startswith(f"user_{variant}_")+ @parameterized.expand([("metric",), ("saved_exposure_criteria",), ("request_exposure_override",)])+ def test_experiment_funnel_actors_refuses_evaluation_pass_markers(self, placement: str) -> None:+ _feature_flag, experiment, experiment_query = self._create_experiment_with_funnel()+ marker_exposure = ExperimentEventExposureConfig(+ event="$ai_evaluation", properties=[HogQLPropertyFilter(key=EVALUATION_PASSED_MARKER)]+ )+ if placement == "metric":+ experiment_query.metric = ExperimentFunnelMetric(+ series=[+ EventsNode(event="$ai_evaluation", properties=[HogQLPropertyFilter(key=EVALUATION_PASSED_MARKER)])+ ]+ )+ elif placement == "saved_exposure_criteria":+ experiment.exposure_criteria = {"exposure_config": marker_exposure.model_dump(mode="json")}+ experiment.save()++ actors_query = ActorsQuery(+ source=ExperimentActorsQuery(+ kind="ExperimentActorsQuery",+ source=experiment_query,+ funnelStep=1,+ funnelStepBreakdown="control",+ exposureConfig=marker_exposure if placement == "request_exposure_override" else None,+ ),+ select=["id", "person"],+ )++ with self.assertRaisesMessage(ValidationError, "Persons are not available for evaluation pass rate metrics."):+ ActorsQueryRunner(query=actors_query, team=self.team).calculate()+@time_machine.travel(EXPERIMENT_EXPOSURE_EVENT_CUTOFF + timedelta(days=4), tick=False)def test_experiment_funnel_actors_resolves_explicit_default_exposure_event(self) -> None:feature_flag, experiment, experiment_query = self._create_experiment_with_funnel()diff --git a/products/experiments/backend/llm_metric_templates.py b/products/experiments/backend/llm_metric_templates.pyindex 77341297f564..0f688fc20bd1 100644--- a/products/experiments/backend/llm_metric_templates.py+++ b/products/experiments/backend/llm_metric_templates.py@@ -11,6 +11,8 @@ from posthog.schema import (PropertyOperator,)+from products.ai_observability.backend.facade.contracts import EVALUATION_GRADED_MARKER, EVALUATION_PASSED_MARKER+LLMMetric = ExperimentMeanMetric | ExperimentRatioMetric@@ -49,34 +51,15 @@ def build_latency_metric(prompt_name: str) -> ExperimentMeanMetric:def build_eval_pass_rate_metric(prompt_name: str) -> ExperimentRatioMetric:- # Use explicit JSON extraction rather than `properties.X = true`. The latter goes through- # HogQL property type inference, which casts the property to Float64 if any event in the- # table has stored this key as a number — silently dropping JSON-boolean events. The- # JSONExtract* functions bypass the inference and read the JSON value directly.- # The applicable guard excludes N/A evaluations (applicable=false) but keeps events- # where the property isn't set at all (JSONExtractRaw returns '' for missing keys).- applicable_filter = HogQLPropertyFilter(key="JSONExtractRaw(properties, '$ai_evaluation_applicable') != 'false'")- boolean_filter = HogQLPropertyFilter(- key="JSONExtractString(properties, '$ai_evaluation_result_type') IN ('', 'boolean')"- )return ExperimentRatioMetric(name="Eval pass rate",numerator=EventsNode(event="$ai_evaluation",- properties=[- _prompt_filter(prompt_name),- HogQLPropertyFilter(key="JSONExtractBool(properties, '$ai_evaluation_result')"),- applicable_filter,- boolean_filter,- ],+ properties=[_prompt_filter(prompt_name), HogQLPropertyFilter(key=EVALUATION_PASSED_MARKER)],),denominator=EventsNode(event="$ai_evaluation",- properties=[- _prompt_filter(prompt_name),- applicable_filter,- boolean_filter,- ],+ properties=[_prompt_filter(prompt_name), HogQLPropertyFilter(key=EVALUATION_GRADED_MARKER)],),)diff --git a/products/experiments/backend/test/test_create_from_prompt.py b/products/experiments/backend/test/test_create_from_prompt.pyindex a9cb83c4f1c6..6a769cd4870e 100644--- a/products/experiments/backend/test/test_create_from_prompt.py+++ b/products/experiments/backend/test/test_create_from_prompt.py@@ -7,6 +7,7 @@ from rest_framework import statusfrom posthog.constants import AvailableFeaturefrom products.access_control.backend.models.access_control import AccessControl+from products.ai_observability.backend.facade.contracts import EVALUATION_GRADED_MARKER, EVALUATION_PASSED_MARKERfrom products.ai_observability.backend.models.llm_prompt import LLMPromptfrom products.experiments.backend.llm_metric_templates import TEMPLATE_NAMESfrom products.experiments.backend.models.experiment import Experiment@@ -120,11 +121,8 @@ class TestExperimentsCreateFromPrompt(APILicensedTest):prompt_filter = next(p for p in properties if p.get("key") == "$ai_prompt_name")self.assertEqual(prompt_filter["value"], self.prompt_name)if template_name == "eval_pass_rate":- for source in (metric["numerator"], metric["denominator"]):- self.assertIn(- "JSONExtractString(properties, '$ai_evaluation_result_type') IN ('', 'boolean')",- [prop["key"] for prop in source["properties"]],- )+ self.assertIn(EVALUATION_PASSED_MARKER, [prop["key"] for prop in metric["numerator"]["properties"]])+ self.assertIn(EVALUATION_GRADED_MARKER, [prop["key"] for prop in metric["denominator"]["properties"]])def test_multiple_templates_creates_one_metric_each(self) -> None:# Pick all available templates so we exercise both mean and ratio shapes.