Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): grade prompt experiment eval pass rates with current evaluation rules (#121139)

Bernat Torres937ddd5a40cf
diff --git a/frontend/src/scenes/experiments/utils.test.ts b/frontend/src/scenes/experiments/utils.test.ts
index 7dedd9d1ba8e..e3d1a0150d53 100644
--- a/frontend/src/scenes/experiments/utils.test.ts
+++ b/frontend/src/scenes/experiments/utils.test.ts
@@ -3,6 +3,7 @@ import { entityFilterLogic } from 'scenes/insights/filters/ActionFilter/entityFi
import experimentJson from '~/mocks/fixtures/api/experiments/_experiment_launched.json'
import {
CachedNewExperimentQueryResponse,
+ EventsNode,
ExperimentEventExposureConfig,
ExperimentExposureConfig,
ExperimentMetric,
@@ -1080,6 +1081,34 @@ describe('getEventCountQuery', () => {
expect(query?.tags).toEqual({ productKey: ProductKey.PRODUCT_ANALYTICS })
})
+ it('drops evaluation pass markers, which only the experiment results query can resolve', () => {
+ const promptFilter = {
+ key: '$ai_prompt_name',
+ type: PropertyFilterType.Event,
+ operator: PropertyOperator.Exact,
+ value: 'p',
+ } as const
+ const passed: EventsNode = {
+ kind: NodeKind.EventsNode,
+ event: '$ai_evaluation',
+ properties: [promptFilter, { key: 'aiEvaluationPassed()', type: PropertyFilterType.HogQL }],
+ }
+ const ratio: ExperimentMetric = {
+ kind: NodeKind.ExperimentMetric,
+ metric_type: ExperimentMetricType.RATIO,
+ numerator: passed,
+ denominator: { kind: NodeKind.EventsNode, event: '$ai_evaluation' },
+ }
+ const funnel: ExperimentMetric = {
+ kind: NodeKind.ExperimentMetric,
+ metric_type: ExperimentMetricType.FUNNEL,
+ series: [passed],
+ }
+
+ expect(getEventCountQuery(ratio, true)?.series[0].properties).toEqual([promptFilter])
+ expect(getEventCountQuery(funnel, true)?.series[0].properties).toEqual([promptFilter])
+ })
+
it('returns null when series is empty', () => {
const metric: ExperimentMetric = {
uuid: 'test-metric',
diff --git a/frontend/src/scenes/experiments/utils.ts b/frontend/src/scenes/experiments/utils.ts
index 59850ab036bd..3fe16439ad39 100644
--- a/frontend/src/scenes/experiments/utils.ts
+++ b/frontend/src/scenes/experiments/utils.ts
@@ -29,6 +29,7 @@ import {
} from '~/queries/schema/schema-general'
import { isFunnelsQuery, isNodeWithSource, isTrendsQuery, isValidQueryForExperiment } from '~/queries/utils'
import {
+ AnyPropertyFilter,
BreakdownAttributionType,
ChartDisplayType,
Experiment,
@@ -38,6 +39,7 @@ import {
FeatureFlagType,
FunnelConversionWindowTimeUnit,
MultivariateFlagVariant,
+ PropertyFilterType,
type InsightModel,
UniversalFiltersGroupValue,
} from '~/types'
@@ -148,13 +150,21 @@ export function isEvenlyDistributed(variants: MultivariateFlagVariant[]): boolea
return variants.every((variant, index) => variant.rollout_percentage === evenPercentages[index])
}
+// The experiments backend replaces these HogQL filter keys with the team's current evaluation pass rules. Other query
+// kinds cannot resolve them, so queries outside the experiment results drop them.
+const EVALUATION_PASS_MARKERS = new Set(['aiEvaluationGraded()', 'aiEvaluationPassed()'])
+
+function withoutEvaluationPassMarkers(properties: AnyPropertyFilter[]): AnyPropertyFilter[] {
+ return properties.filter((p) => !(p.type === PropertyFilterType.HogQL && EVALUATION_PASS_MARKERS.has(p.key)))
+}
+
function seriesToFilter(series: AnyEntityNode | ExperimentMetricSource): UniversalFiltersGroupValue | null {
if (series.kind === NodeKind.EventsNode) {
return {
id: series.event ?? null,
name: series.event as string,
type: 'events',
- properties: series.properties ?? [],
+ properties: withoutEvaluationPassMarkers(series.properties ?? []),
}
}
@@ -653,7 +663,9 @@ const getEventCountSeries = (metric: ExperimentMetric): AnyEntityNode[] => {
event: lastStep.event,
math: ExperimentMetricMathType.TotalCount,
...(lastStep.properties &&
- lastStep.properties.length > 0 && { properties: lastStep.properties }),
+ lastStep.properties.length > 0 && {
+ properties: withoutEvaluationPassMarkers(lastStep.properties),
+ }),
},
]
} else if (lastStep.kind === NodeKind.ActionsNode) {
@@ -693,7 +705,10 @@ const getEventCountSeries = (metric: ExperimentMetric): AnyEntityNode[] => {
name: eventsNode.event || undefined,
event: eventsNode.event || undefined,
math: ExperimentMetricMathType.TotalCount,
- ...(eventsNode.properties && eventsNode.properties.length > 0 && { properties: eventsNode.properties }),
+ ...(eventsNode.properties &&
+ eventsNode.properties.length > 0 && {
+ properties: withoutEvaluationPassMarkers(eventsNode.properties),
+ }),
},
])
.with({ kind: NodeKind.ActionsNode }, (actionsNode) => [
diff --git a/products/ai_observability/backend/facade/api.py b/products/ai_observability/backend/facade/api.py
index 578b0a0aa9c7..de542494bf27 100644
--- a/products/ai_observability/backend/facade/api.py
+++ b/products/ai_observability/backend/facade/api.py
@@ -1,6 +1,8 @@
from collections.abc import Sequence
from datetime import date, datetime
+from posthog.schema import HogQLPropertyFilter
+
from posthog.models import Team, User
from ..logic.evaluations.evaluation_metas import load_evaluation_meta, load_evaluation_metas
@@ -178,6 +180,28 @@ def get_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilters
return build_evaluation_pass_filters(team_id)
+def resolve_evaluation_pass_markers[P](properties: list[P], team_id: int) -> list[P]:
+ """Replace each evaluation pass marker with the team's current condition."""
+ markers = {contracts.EVALUATION_GRADED_MARKER, contracts.EVALUATION_PASSED_MARKER}
+ if not any(isinstance(prop, HogQLPropertyFilter) and prop.key in markers for prop in properties):
+ return properties
+ filters = build_evaluation_pass_filters(team_id)
+ conditions = {
+ contracts.EVALUATION_GRADED_MARKER: filters.graded,
+ contracts.EVALUATION_PASSED_MARKER: filters.passed,
+ }
+ return [
+ prop.model_copy(update={"key": conditions[prop.key]})
+ if isinstance(prop, HogQLPropertyFilter) and prop.key in conditions
+ else prop
+ for prop in properties
+ ]
+
+
+def mentions_evaluation_pass_markers(text: str) -> bool:
+ return contracts.EVALUATION_GRADED_MARKER in text or contracts.EVALUATION_PASSED_MARKER in text
+
+
def get_trace_evaluation_results(team: Team, user: User | None, trace_id: str) -> contracts.TraceEvaluationResults:
rows = load_evaluation_results(team, user, trace_id)
configs = load_output_configs(team, user, {row.evaluation_id for row in rows if row.evaluation_id})
diff --git a/products/ai_observability/backend/facade/contracts.py b/products/ai_observability/backend/facade/contracts.py
index 3103e4335a35..58dae6336039 100644
--- a/products/ai_observability/backend/facade/contracts.py
+++ b/products/ai_observability/backend/facade/contracts.py
@@ -111,6 +111,12 @@ class EvaluationPassFilters(Contract):
passed: str # The run passed. Implies `graded`.
+# HogQL filter keys that stand in for `EvaluationPassFilters` in a stored filter, so that the stored filter holds no
+# evaluation rules and always grades with the current ones. `resolve_evaluation_pass_markers` replaces them at query time.
+EVALUATION_GRADED_MARKER = "aiEvaluationGraded()"
+EVALUATION_PASSED_MARKER = "aiEvaluationPassed()"
+
+
EvaluationOutcome = Literal["pass", "fail", "inconclusive", "unrated"]
diff --git a/products/experiments/backend/hogql_queries/base_query_utils.py b/products/experiments/backend/hogql_queries/base_query_utils.py
index e7e25cdd5422..e68f7fa7deef 100644
--- a/products/experiments/backend/hogql_queries/base_query_utils.py
+++ b/products/experiments/backend/hogql_queries/base_query_utils.py
@@ -29,6 +29,7 @@ from posthog.clickhouse.query_tagging import tag_contains_user_hogql
from posthog.models.team.team import Team
from products.actions.backend.models.action import Action
+from products.ai_observability.backend.facade.api import resolve_evaluation_pass_markers
from products.experiments.backend.hogql_queries.hogql_aggregation_utils import (
contains_aggregation,
extract_aggregation_and_inner_expr,
@@ -186,7 +187,8 @@ def event_or_action_to_filter(
)
if entity_node.properties:
- event_properties = ast.And(exprs=[property_to_expr(property, team) for property in entity_node.properties])
+ properties = resolve_evaluation_pass_markers(entity_node.properties, team.pk)
+ event_properties = ast.And(exprs=[property_to_expr(property, team) for property in properties])
event_filter = ast.And(exprs=[event_filter, event_properties])
return event_filter
diff --git a/products/experiments/backend/hogql_queries/experiment_query_runner.py b/products/experiments/backend/hogql_queries/experiment_query_runner.py
index c9355d2b2474..8e75e95e2287 100644
--- a/products/experiments/backend/hogql_queries/experiment_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_query_runner.py
@@ -1,3 +1,4 @@
+import json
import uuid
from datetime import UTC, datetime, timedelta
from functools import cached_property
@@ -36,6 +37,7 @@ from posthog.hogql_queries.query_runner import QueryRunner
from posthog.hogql_queries.utils.query_date_range import QueryDateRange
from posthog.models.team.extensions import get_or_create_team_extension
+from products.ai_observability.backend.facade.api import mentions_evaluation_pass_markers
from products.analytics_platform.backend.lazy_computation.lazy_computation_executor import LazyComputationResult
from products.experiments.backend.hogql_queries import MULTIPLE_VARIANT_KEY, get_baseline_variant_key
from products.experiments.backend.hogql_queries.base_query_utils import experiment_window, experiment_window_end
@@ -438,15 +440,17 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
):
modifiers.sessionIdPushdown = True
- experiment_query_debug = get_experiment_query_debug(
- experiment_query_ast,
- self.team,
- user=self.user,
- bypass_warehouse_access_control=self.bypass_warehouse_access_control,
- modifiers=modifiers,
- )
- self.hogql = experiment_query_debug[0]
- self.clickhouse_sql = experiment_query_debug[1]
+ # The debug SQL would expose the evaluation rules that the markers keep out of the browser.
+ if not self._uses_evaluation_pass_markers():
+ experiment_query_debug = get_experiment_query_debug(
+ experiment_query_ast,
+ self.team,
+ user=self.user,
+ bypass_warehouse_access_control=self.bypass_warehouse_access_control,
+ modifiers=modifiers,
+ )
+ self.hogql = experiment_query_debug[0]
+ self.clickhouse_sql = experiment_query_debug[1]
settings = HogQLGlobalSettings(
max_execution_time=self.max_execution_time,
@@ -619,8 +623,18 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
return variants + variants_missing
def to_actors_query(self) -> ast.SelectQuery:
+ # The actors response returns its generated SQL, which would expose the evaluation rules.
+ if self._uses_evaluation_pass_markers():
+ raise ValidationError("Persons are not available for evaluation pass rate metrics.")
return build_experiment_actors_query(self)
+ def _uses_evaluation_pass_markers(self) -> bool:
+ return mentions_evaluation_pass_markers(
+ self.metric.model_dump_json()
+ + json.dumps(self.experiment.exposure_criteria or {})
+ + (self.actors_query.model_dump_json() if self.actors_query else "")
+ )
+
def to_query(self) -> ast.SelectQuery:
raise ValidationError(f"Cannot convert source query of type {self.query.metric.kind} to query")
diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_ratio_metric.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_ratio_metric.py
index 76b71867f6f9..024ced5b53ee 100644
--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_ratio_metric.py
+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_ratio_metric.py
@@ -24,8 +24,10 @@ from posthog.schema import (
from posthog.test.test_journeys import journeys_for
from products.actions.backend.models.action import Action
+from products.ai_observability.backend.models.evaluations import Evaluation
from products.experiments.backend.hogql_queries.experiment_query_runner import ExperimentQueryRunner
from products.experiments.backend.hogql_queries.test.experiment_query_runner.base import ExperimentQueryRunnerBaseTest
+from products.experiments.backend.llm_metric_templates import build_eval_pass_rate_metric
@override_settings(IN_UNIT_TESTING=True)
@@ -182,6 +184,72 @@ class TestExperimentRatioMetric(ExperimentQueryRunnerBaseTest):
self.assertIsNotNone(control_variant.numerator_denominator_sum_product)
self.assertIsNotNone(test_variant.numerator_denominator_sum_product)
+ @parameterized.expand(
+ [
+ ("direct", False),
+ ("precomputed", True),
+ ]
+ )
+ @time_machine.travel("2020-01-01T12:00:00Z", tick=False)
+ def test_eval_pass_rate_grades_with_current_evaluation_rules(self, name: str, use_precomputation: bool) -> None:
+ self._setup_precomputation_test(use_precomputation)
+ feature_flag = self.create_feature_flag()
+ experiment = self.create_experiment(feature_flag=feature_flag)
+ experiment.stats_config = {"method": "frequentist"}
+ metric = build_eval_pass_rate_metric("my-prompt")
+ experiment.metrics = [metric.model_dump(mode="json")]
+ self._save_experiment_with_precomputation(experiment, use_precomputation)
+ detector = Evaluation.objects.create(
+ team=self.team,
+ name="detector",
+ evaluation_type="hog",
+ evaluation_config={"source": "return true"},
+ output_type="boolean",
+ output_config={"true_is_failure": True},
+ )
+
+ # A detector run passes when it detects nothing, so control passes 3 of 4 runs and test passes 1 of 4.
+ for variant, results in (("control", [False, False, False, True]), ("test", [False, True, True, True])):
+ for i, result in enumerate(results):
+ distinct_id = f"user_{variant}_{i}"
+ _create_person(distinct_ids=[distinct_id], team_id=self.team.pk)
+ _create_event(
+ team=self.team,
+ event="$feature_flag_called",
+ distinct_id=distinct_id,
+ timestamp="2020-01-02T12:00:00Z",
+ properties={"$feature_flag_response": variant, "$feature_flag": feature_flag.key},
+ )
+ _create_event(
+ team=self.team,
+ event="$ai_evaluation",
+ distinct_id=distinct_id,
+ timestamp="2020-01-02T12:01:00Z",
+ properties={
+ "$ai_prompt_name": "my-prompt",
+ "$ai_evaluation_id": str(detector.id),
+ "$ai_evaluation_result": result,
+ },
+ )
+ flush_persons_and_events()
+
+ def passed_and_graded() -> list[tuple[float, float | None]]:
+ query = ExperimentQuery(experiment_id=experiment.id, kind="ExperimentQuery", metric=metric)
+ runner = ExperimentQueryRunner(query=query, team=self.team)
+ result = cast(ExperimentQueryResponse, runner.calculate())
+ assert runner._metric_events_precomputed is use_precomputation
+ assert result.baseline is not None and result.variant_results is not None
+ assert result.hogql is None and result.clickhouse_sql is None
+ return [(variant.sum, variant.denominator_sum) for variant in [result.baseline, *result.variant_results]]
+
+ self.assertEqual(passed_and_graded(), [(3, 4), (1, 4)])
+
+ # The experiment stores no rules, so the next query grades with the edited ones. On the precomputed path this
+ # also proves that the edit changed the job hash, because reused jobs would still hold the old pass counts.
+ detector.output_config = {}
+ detector.save()
+ self.assertEqual(passed_and_graded(), [(1, 4), (3, 4)])
+
@parameterized.expand(
[
("direct", False),
diff --git a/products/experiments/backend/hogql_queries/test/test_experiment_actors_query.py b/products/experiments/backend/hogql_queries/test/test_experiment_actors_query.py
index 9b7da013acbf..2305095a22d0 100644
--- a/products/experiments/backend/hogql_queries/test/test_experiment_actors_query.py
+++ b/products/experiments/backend/hogql_queries/test/test_experiment_actors_query.py
@@ -21,6 +21,7 @@ from unittest.mock import patch
from django.test import override_settings
from parameterized import parameterized
+from rest_framework.exceptions import ValidationError
from posthog.schema import (
ActorsQuery,
@@ -29,12 +30,14 @@ from posthog.schema import (
ExperimentEventExposureConfig,
ExperimentFunnelMetric,
ExperimentQuery,
+ HogQLPropertyFilter,
)
from posthog.hogql.context import HogQLContext
from posthog.hogql_queries.actors_query_runner import ActorsQueryRunner
+from products.ai_observability.backend.facade.contracts import EVALUATION_PASSED_MARKER
from products.cohorts.backend.models.cohort import Cohort
from products.cohorts.backend.models.util import print_cohort_hogql_query
from products.experiments.backend.hogql_queries.exposure_query_logic import EXPERIMENT_EXPOSURE_EVENT_CUTOFF
@@ -195,6 +198,36 @@ class TestExperimentActorsQuery(ExperimentQueryRunnerBaseTest, ClickhouseTestMix
for distinct_id in distinct_ids:
assert distinct_id.startswith(f"user_{variant}_")
+ @parameterized.expand([("metric",), ("saved_exposure_criteria",), ("request_exposure_override",)])
+ def test_experiment_funnel_actors_refuses_evaluation_pass_markers(self, placement: str) -> None:
+ _feature_flag, experiment, experiment_query = self._create_experiment_with_funnel()
+ marker_exposure = ExperimentEventExposureConfig(
+ event="$ai_evaluation", properties=[HogQLPropertyFilter(key=EVALUATION_PASSED_MARKER)]
+ )
+ if placement == "metric":
+ experiment_query.metric = ExperimentFunnelMetric(
+ series=[
+ EventsNode(event="$ai_evaluation", properties=[HogQLPropertyFilter(key=EVALUATION_PASSED_MARKER)])
+ ]
+ )
+ elif placement == "saved_exposure_criteria":
+ experiment.exposure_criteria = {"exposure_config": marker_exposure.model_dump(mode="json")}
+ experiment.save()
+
+ actors_query = ActorsQuery(
+ source=ExperimentActorsQuery(
+ kind="ExperimentActorsQuery",
+ source=experiment_query,
+ funnelStep=1,
+ funnelStepBreakdown="control",
+ exposureConfig=marker_exposure if placement == "request_exposure_override" else None,
+ ),
+ select=["id", "person"],
+ )
+
+ with self.assertRaisesMessage(ValidationError, "Persons are not available for evaluation pass rate metrics."):
+ ActorsQueryRunner(query=actors_query, team=self.team).calculate()
+
@time_machine.travel(EXPERIMENT_EXPOSURE_EVENT_CUTOFF + timedelta(days=4), tick=False)
def test_experiment_funnel_actors_resolves_explicit_default_exposure_event(self) -> None:
feature_flag, experiment, experiment_query = self._create_experiment_with_funnel()
diff --git a/products/experiments/backend/llm_metric_templates.py b/products/experiments/backend/llm_metric_templates.py
index 77341297f564..0f688fc20bd1 100644
--- a/products/experiments/backend/llm_metric_templates.py
+++ b/products/experiments/backend/llm_metric_templates.py
@@ -11,6 +11,8 @@ from posthog.schema import (
PropertyOperator,
)
+from products.ai_observability.backend.facade.contracts import EVALUATION_GRADED_MARKER, EVALUATION_PASSED_MARKER
+
LLMMetric = ExperimentMeanMetric | ExperimentRatioMetric
@@ -49,34 +51,15 @@ def build_latency_metric(prompt_name: str) -> ExperimentMeanMetric:
def build_eval_pass_rate_metric(prompt_name: str) -> ExperimentRatioMetric:
- # Use explicit JSON extraction rather than `properties.X = true`. The latter goes through
- # HogQL property type inference, which casts the property to Float64 if any event in the
- # table has stored this key as a number — silently dropping JSON-boolean events. The
- # JSONExtract* functions bypass the inference and read the JSON value directly.
- # The applicable guard excludes N/A evaluations (applicable=false) but keeps events
- # where the property isn't set at all (JSONExtractRaw returns '' for missing keys).
- applicable_filter = HogQLPropertyFilter(key="JSONExtractRaw(properties, '$ai_evaluation_applicable') != 'false'")
- boolean_filter = HogQLPropertyFilter(
- key="JSONExtractString(properties, '$ai_evaluation_result_type') IN ('', 'boolean')"
- )
return ExperimentRatioMetric(
name="Eval pass rate",
numerator=EventsNode(
event="$ai_evaluation",
- properties=[
- _prompt_filter(prompt_name),
- HogQLPropertyFilter(key="JSONExtractBool(properties, '$ai_evaluation_result')"),
- applicable_filter,
- boolean_filter,
- ],
+ properties=[_prompt_filter(prompt_name), HogQLPropertyFilter(key=EVALUATION_PASSED_MARKER)],
),
denominator=EventsNode(
event="$ai_evaluation",
- properties=[
- _prompt_filter(prompt_name),
- applicable_filter,
- boolean_filter,
- ],
+ properties=[_prompt_filter(prompt_name), HogQLPropertyFilter(key=EVALUATION_GRADED_MARKER)],
),
)
diff --git a/products/experiments/backend/test/test_create_from_prompt.py b/products/experiments/backend/test/test_create_from_prompt.py
index a9cb83c4f1c6..6a769cd4870e 100644
--- a/products/experiments/backend/test/test_create_from_prompt.py
+++ b/products/experiments/backend/test/test_create_from_prompt.py
@@ -7,6 +7,7 @@ from rest_framework import status
from posthog.constants import AvailableFeature
from products.access_control.backend.models.access_control import AccessControl
+from products.ai_observability.backend.facade.contracts import EVALUATION_GRADED_MARKER, EVALUATION_PASSED_MARKER
from products.ai_observability.backend.models.llm_prompt import LLMPrompt
from products.experiments.backend.llm_metric_templates import TEMPLATE_NAMES
from products.experiments.backend.models.experiment import Experiment
@@ -120,11 +121,8 @@ class TestExperimentsCreateFromPrompt(APILicensedTest):
prompt_filter = next(p for p in properties if p.get("key") == "$ai_prompt_name")
self.assertEqual(prompt_filter["value"], self.prompt_name)
if template_name == "eval_pass_rate":
- for source in (metric["numerator"], metric["denominator"]):
- self.assertIn(
- "JSONExtractString(properties, '$ai_evaluation_result_type') IN ('', 'boolean')",
- [prop["key"] for prop in source["properties"]],
- )
+ self.assertIn(EVALUATION_PASSED_MARKER, [prop["key"] for prop in metric["numerator"]["properties"]])
+ self.assertIn(EVALUATION_GRADED_MARKER, [prop["key"] for prop in metric["denominator"]["properties"]])
def test_multiple_templates_creates_one_metric_each(self) -> None:
# Pick all available templates so we exercise both mean and ratio shapes.