:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): expose evaluation pass filters through the facade (#119920)
Bernat Torresb75c59897612
diff --git a/.github/new-events-schema-targets.txt b/.github/new-events-schema-targets.txtindex 621549bb7e53..55745da1d56b 100644--- a/.github/new-events-schema-targets.txt+++ b/.github/new-events-schema-targets.txt@@ -23,6 +23,7 @@ posthog/hogql_queriesposthog/session_recordings/queriesposthog/clickhouse/testee/clickhouse+products/ai_observability/backend/tests/logic/evaluationsproducts/ai_observability/backend/tests/logic/tracesproducts/cohortsproducts/customer_analytics/backend/hogql_queriesdiff --git a/posthog/temporal/ai_observability/eval_reports/activities.py b/posthog/temporal/ai_observability/eval_reports/activities.pyindex 0a4fa1de2817..f5e89d25fff2 100644--- a/posthog/temporal/ai_observability/eval_reports/activities.py+++ b/posthog/temporal/ai_observability/eval_reports/activities.py@@ -31,7 +31,6 @@ from posthog.temporal.ai_observability.eval_reports.constants import (COUNT_TRIGGER_QUERY_TOTAL_BUDGET_SECONDS,COUNT_TRIGGER_QUERY_WIDTH,)-from posthog.temporal.ai_observability.eval_reports.output_types import get_outcome_definitionfrom posthog.temporal.ai_observability.eval_reports.targets import (GENERATION_TARGET,resolve_evaluation_target,@@ -57,6 +56,7 @@ from posthog.temporal.ai_observability.eval_reports.types import (from posthog.temporal.common.heartbeat import Heartbeaterfrom products.ai_observability.backend.models.evaluation_configs import evaluation_supports_reports+from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definitionif TYPE_CHECKING:from posthog.models import Teamdiff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/graph.py b/posthog/temporal/ai_observability/eval_reports/report_agent/graph.pyindex 8007bc55be0d..0d5e6d03d11f 100644--- a/posthog/temporal/ai_observability/eval_reports/report_agent/graph.py+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/graph.py@@ -11,7 +11,6 @@ from langgraph.prebuilt import create_react_agentfrom openai import APIStatusErrorfrom posthog.llm.gateway_client import team_distinct_id-from posthog.temporal.ai_observability.eval_reports.output_types import get_outcome_definitionfrom posthog.temporal.ai_observability.eval_reports.report_agent.prompts import build_eval_report_system_promptfrom posthog.temporal.ai_observability.eval_reports.report_agent.schema import (MAX_REPORT_SECTIONS,@@ -44,6 +43,8 @@ from posthog.temporal.ai_observability.eval_reports.targets import (from posthog.temporal.ai_observability.eval_reports.types import RunEvalReportAgentInputfrom posthog.temporal.ai_observability.llm_endpoint import build_flex_first_chat_client, build_langchain_callbacks+from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definition+logger = structlog.get_logger(__name__)diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py b/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.pyindex 8803d3c43dcb..98c911e1778b 100644--- a/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py@@ -2,7 +2,6 @@import json-from posthog.temporal.ai_observability.eval_reports.output_types import get_outcome_definitionfrom posthog.temporal.ai_observability.eval_reports.report_agent.schema import MAX_REPORT_SECTIONSfrom posthog.temporal.ai_observability.eval_reports.targets import (GENERATION_TARGET,@@ -11,6 +10,8 @@ from posthog.temporal.ai_observability.eval_reports.targets import (get_target_descriptor,)+from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definition+EVAL_REPORT_SYSTEM_PROMPT = """You are an evaluation report agent for PostHog's AI observability platform. Your job is to analyze results from an LLM evaluation and produce a concise, grounded, example-backed report.## What you're analyzingdiff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/schema.py b/posthog/temporal/ai_observability/eval_reports/report_agent/schema.pyindex 66aebb56f9e5..0e89126953f9 100644--- a/posthog/temporal/ai_observability/eval_reports/report_agent/schema.py+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/schema.py@@ -17,9 +17,10 @@ from dataclasses import dataclass, fieldfrom enum import StrEnumfrom typing import Literal, overload-from posthog.temporal.ai_observability.eval_reports.output_types import get_outcome_definitionfrom posthog.temporal.ai_observability.eval_reports.targets import GENERATION_TARGET, resolve_evaluation_target+from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definition+# Hard cap on the number of agent-chosen sections. Prevents section sprawl and# keeps reports scannable. The agent is also instructed to lean lean — quality# over quantity, merge related findings rather than fragmenting.diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_schema.py b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_schema.pyindex bb3b4a282c51..9eefd7805cf8 100644--- a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_schema.py+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_schema.py@@ -4,7 +4,6 @@ from django.test import SimpleTestCasefrom parameterized import parameterized-from posthog.temporal.ai_observability.eval_reports.output_types import SUPPORTED_EVAL_REPORT_OUTPUT_TYPESfrom posthog.temporal.ai_observability.eval_reports.report_agent.schema import (MAX_REPORT_SECTIONS,MIN_REPORT_SECTIONS,@@ -16,6 +15,7 @@ from posthog.temporal.ai_observability.eval_reports.report_agent.schema import ()from products.ai_observability.backend.models.evaluation_configs import REPORTABLE_OUTPUT_TYPES+from products.ai_observability.backend.models.evaluation_outcomes import SUPPORTED_EVAL_REPORT_OUTPUT_TYPESclass TestOutputTypeRegistry(SimpleTestCase):diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools.py b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools.pyindex 56a71cfaafe4..186bd9dd0b7a 100644--- a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools.py+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools.py@@ -248,7 +248,7 @@ class TestSummaryMetrics(SimpleTestCase):current_query = mock_execute_hogql.call_args_list[0].args[1]self.assertIn("properties.$ai_sentiment_label = 'positive'", current_query)self.assertIn("properties.$ai_evaluation_result_type = 'sentiment'", current_query)- self.assertNotIn("properties.$ai_evaluation_result = true", current_query)+ self.assertNotIn("properties.$ai_evaluation_result = 'true'", current_query)@patch("posthog.temporal.ai_observability.eval_reports.report_agent.tools._execute_hogql")def test_detector_polarity_counts_a_false_result_as_the_pass(self, mock_execute_hogql):@@ -273,7 +273,7 @@ class TestSummaryMetrics(SimpleTestCase):self.assertEqual(result["current_period"]["pass_rate"], 18.37)pass_column = re.search(r"countIf\((.*?)\) as pass_count", mock_execute_hogql.call_args_list[0].args[1], re.S)assert pass_column is not None- self.assertIn("properties.$ai_evaluation_result = false", pass_column.group(1))+ self.assertIn("properties.$ai_evaluation_result = 'false'", pass_column.group(1))@patch("posthog.temporal.ai_observability.eval_reports.report_agent.tools._execute_hogql")def test_state_without_polarity_counts_a_true_result_as_the_pass(self, mock_execute_hogql):@@ -296,7 +296,7 @@ class TestSummaryMetrics(SimpleTestCase):self.assertEqual(result["current_period"]["result_counts"], {"pass": 80, "fail": 18, "na": 2})pass_column = re.search(r"countIf\((.*?)\) as pass_count", mock_execute_hogql.call_args_list[0].args[1], re.S)assert pass_column is not None- self.assertIn("properties.$ai_evaluation_result = true", pass_column.group(1))+ self.assertIn("properties.$ai_evaluation_result = 'true'", pass_column.group(1))class TestTargetAwareEvalResults(SimpleTestCase):diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py b/posthog/temporal/ai_observability/eval_reports/report_agent/tools.pyindex c8896585440b..5695735159d1 100644--- a/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py@@ -26,10 +26,6 @@ from posthog.hogql import astfrom posthog.dataclasses import frozenfrom posthog.errors import CH_TRANSIENT_ERRORS-from posthog.temporal.ai_observability.eval_reports.output_types import (- EvaluationReportOutcomeDefinition,- get_outcome_definition,-)from posthog.temporal.ai_observability.eval_reports.report_agent.schema import (MAX_REPORT_SECTIONS,Citation,@@ -58,6 +54,11 @@ from posthog.temporal.ai_observability.trace_summarization.constants import ()from posthog.temporal.ai_observability.trace_summarization.fetch_and_format import _fetch_and_format_trace+from products.ai_observability.backend.models.evaluation_outcomes import (+ EvaluationReportOutcomeDefinition,+ get_outcome_definition,+)+if TYPE_CHECKING:from posthog.models import Team@@ -403,7 +404,6 @@ def _fetch_period_summary(AND timestamp < {{ts_end}}""",placeholders={- **definition.query_placeholders,"evaluation_id": ast.Constant(value=evaluation_id),"ts_start": ast.Constant(value=ts_start),"ts_end": ast.Constant(value=ts_end),@@ -508,7 +508,6 @@ def get_result_distribution_over_time(ORDER BY bucket""",placeholders={- **definition.query_placeholders,"evaluation_id": ast.Constant(value=evaluation_id),"ts_start": ast.Constant(value=ts_start),"ts_end": ast.Constant(value=ts_end),@@ -564,7 +563,6 @@ def list_all_eval_results(evaluation_target = resolve_evaluation_target(state.get("evaluation_target"))shared_placeholders = {- **definition.query_placeholders,"evaluation_id": ast.Constant(value=evaluation_id),"ts_start": ast.Constant(value=ts_start),"ts_end": ast.Constant(value=ts_end),@@ -711,7 +709,6 @@ def sample_eval_results(LIMIT {{limit}}""",placeholders={- **definition.query_placeholders,"evaluation_id": ast.Constant(value=evaluation_id),"ts_start": ast.Constant(value=ts_start),"ts_end": ast.Constant(value=ts_end),@@ -1547,7 +1544,6 @@ def get_top_outcome_reasons(LIMIT {{limit}}""",placeholders={- **definition.query_placeholders,"evaluation_id": ast.Constant(value=evaluation_id),"ts_start": ast.Constant(value=ts_start),"ts_end": ast.Constant(value=ts_end),diff --git a/posthog/temporal/ai_observability/eval_reports/test/test_report_prompt_polarity.py b/posthog/temporal/ai_observability/eval_reports/test/test_report_prompt_polarity.pynew file mode 100644index 000000000000..9992fd1af65a--- /dev/null+++ b/posthog/temporal/ai_observability/eval_reports/test/test_report_prompt_polarity.py@@ -0,0 +1,26 @@+from django.test import SimpleTestCase+++class TestReportPromptPolarity(SimpleTestCase):+ def _prompt(self, *, true_is_failure: bool) -> str:+ from posthog.temporal.ai_observability.eval_reports.report_agent.prompts import build_eval_report_system_prompt++ return build_eval_report_system_prompt(+ evaluation_name="Struggle detector",+ evaluation_description="",+ evaluation_prompt="",+ evaluation_type="llm_judge",+ output_type="boolean",+ period_start="2026-04-08T14:00:00+00:00",+ period_end="2026-04-08T15:00:00+00:00",+ true_is_failure=true_is_failure,+ )++ def test_detector_prompt_says_a_true_result_is_reported_as_a_fail(self):+ prompt = self._prompt(true_is_failure=True)+ self.assertIn("reported as a fail", prompt)++ def test_default_prompt_keeps_the_pass_on_true_story(self):+ prompt = self._prompt(true_is_failure=False)+ self.assertIn("satisfied the configured criteria", prompt)+ self.assertNotIn("reported as a fail", prompt)diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.pyindex b37a5f47ed90..78fc82a8b3ed 100644--- a/products/ai_observability/backend/api/evaluations.py+++ b/products/ai_observability/backend/api/evaluations.py@@ -31,7 +31,6 @@ from posthog.hogql_queries.ai.ai_table_resolver import AIEventsUnavailableError,from posthog.hogql_queries.ai.utils import HEAVY_COLUMN_NAMES, merge_heavy_propertiesfrom posthog.models.team import Teamfrom posthog.permissions import AccessControlPermission-from posthog.temporal.ai_observability.eval_reports.output_types import get_outcome_definitionfrom posthog.temporal.ai_observability.message_utils import extract_text_from_messagesfrom posthog.temporal.ai_observability.model_resolution import active_key_fallbackfrom posthog.temporal.ai_observability.run_evaluation import extract_event_io, run_hog_eval@@ -42,6 +41,7 @@ from products.access_control.backend.presentation.access_control import (AccessControlViewSetMixin,UserAccessControlSerializerMixin,)+from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definitionfrom ..evaluation_conditions import build_condition_filterfrom ..hog import compile_ai_observability_hogdiff --git a/products/ai_observability/backend/facade/api.py b/products/ai_observability/backend/facade/api.pyindex f5a7a75bb679..6b3033f9a66e 100644--- a/products/ai_observability/backend/facade/api.py+++ b/products/ai_observability/backend/facade/api.py@@ -2,6 +2,7 @@ from datetime import datetimefrom posthog.models import Team, User+from ..logic.evaluations.pass_filters import build_evaluation_pass_filtersfrom ..logic.traces.timeline import TimelineRowfrom ..logic.traces.trace import Tracefrom ..logic.traces.trace_queries import fallback_window, load_trace@@ -94,3 +95,10 @@ def _to_trace(trace: Trace) -> contracts.Trace:total_ms=trace.timeline.total_ms,thread_node_ids=trace.thread_node_ids,)+++def get_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilters:+ """Grade `$ai_evaluation` events with the same rules the evaluation UI applies: boolean polarity, the numeric+ or categorical passing rule, and no skipped, N/A or sentiment runs. The conditions change when an evaluation's+ output config changes, so a caller that stores them rebuilds them on `evaluation_output_changed`."""+ return build_evaluation_pass_filters(team_id)diff --git a/products/ai_observability/backend/facade/contracts.py b/products/ai_observability/backend/facade/contracts.pyindex 7806ec1677bb..859a9e87b032 100644--- a/products/ai_observability/backend/facade/contracts.py+++ b/products/ai_observability/backend/facade/contracts.py@@ -66,3 +66,10 @@ class Trace(Contract):timeline: list[TraceTimelineRow]total_ms: floatthread_node_ids: list[str]+++class EvaluationPassFilters(Contract):+ """HogQL conditions over `$ai_evaluation` events, built from each evaluation's current output config."""++ graded: str # The run counts toward a pass rate.+ passed: str # The run passed. Implies `graded`.diff --git a/products/ai_observability/backend/facade/evaluation_signals.py b/products/ai_observability/backend/facade/evaluation_signals.pynew file mode 100644index 000000000000..df8e37b4ecff--- /dev/null+++ b/products/ai_observability/backend/facade/evaluation_signals.py@@ -0,0 +1,12 @@+from collections.abc import Callable++from django.dispatch import Signal++evaluation_output_changed = Signal()+++def connect_evaluation_output_changed(receiver: Callable[..., object], *, dispatch_uid: str) -> None:+ """``receiver`` gets ``team_id=`` after an evaluation save that writes its output type or output config commits.+ Every full save writes them, so a receiver must be idempotent. A receiver that raises is logged and never fails+ the save."""+ evaluation_output_changed.connect(receiver, dispatch_uid=dispatch_uid)diff --git a/products/ai_observability/backend/logic/evaluations/__init__.py b/products/ai_observability/backend/logic/evaluations/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/ai_observability/backend/logic/evaluations/pass_filters.py b/products/ai_observability/backend/logic/evaluations/pass_filters.pynew file mode 100644index 000000000000..ae44fce458f9--- /dev/null+++ b/products/ai_observability/backend/logic/evaluations/pass_filters.py@@ -0,0 +1,64 @@+from django.db.models import Q++from posthog.hogql.escape_sql import escape_hogql_string++from products.ai_observability.backend.models.evaluation_configs import OutputType+from products.ai_observability.backend.models.evaluation_outcomes import (+ EvaluationReportOutcomeDefinition,+ get_outcome_definition,+)+from products.ai_observability.backend.models.evaluations import Evaluation++from ...facade import contracts++_DEFAULT_DEFINITION = get_outcome_definition(OutputType.BOOLEAN)+++def build_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilters:+ graded_branches: list[str] = []+ passed_branches: list[str] = []+ # Every other evaluation falls through to the default rule. Soft-deleted evaluations stay in, because their past+ # events still need their own rule. Reads the writer, because a receiver of evaluation_output_changed calls this+ # right after the commit, before a replica can catch up.+ evaluations_with_own_rule = (+ Evaluation.objects.using("default")+ .filter(team_id=team_id)+ .filter(+ Q(output_type__in=[OutputType.NUMERIC, OutputType.CATEGORICAL, OutputType.SENTIMENT])+ | Q(output_config__true_is_failure=True)+ )+ .values_list("id", "output_type", "output_config")+ )+ for evaluation_id, output_type, output_config in evaluations_with_own_rule:+ try:+ definition = get_outcome_definition(+ output_type, true_is_failure=output_config.get("true_is_failure") is True, output_config=output_config+ )+ except ValueError:+ definition = None+ matches_evaluation = f"properties.$ai_evaluation_id = {escape_hogql_string(str(evaluation_id))}"+ # An evaluation without a pass rule grades none of its runs, including boolean runs from before it+ # switched to sentiment, which the default branch would otherwise grade.+ if definition is None or not definition.has_pass_rule:+ graded_branches += [matches_evaluation, "false"]+ passed_branches += [matches_evaluation, "false"]+ continue+ graded_branches += [matches_evaluation, _graded(definition)]+ passed_branches += [matches_evaluation, definition.outcome_predicates["pass"]]++ graded = _multi_if(graded_branches, _graded(_DEFAULT_DEFINITION))+ return contracts.EvaluationPassFilters(+ graded=graded,+ passed=f"({graded} AND {_multi_if(passed_branches, _DEFAULT_DEFINITION.outcome_predicates['pass'])})",+ )+++def _graded(definition: EvaluationReportOutcomeDefinition) -> str:+ predicates = definition.outcome_predicates+ return f"{definition.event_predicate} AND (({predicates['pass']}) OR ({predicates['fail']}))"+++def _multi_if(branches: list[str], default: str) -> str:+ if not branches:+ return f"({default})"+ return f"multiIf({', '.join(branches)}, {default})"diff --git a/posthog/temporal/ai_observability/eval_reports/output_types.py b/products/ai_observability/backend/models/evaluation_outcomes.pysimilarity index 87%rename from posthog/temporal/ai_observability/eval_reports/output_types.pyrename to products/ai_observability/backend/models/evaluation_outcomes.pyindex 6b9cf8f4f307..3067f197ea9e 100644--- a/posthog/temporal/ai_observability/eval_reports/output_types.py+++ b/products/ai_observability/backend/models/evaluation_outcomes.py@@ -1,4 +1,4 @@-"""Trusted query definitions for evaluation report outcomes.+"""Trusted query definitions for evaluation outcomes, shared by reports and the pass filters other products use.Only `outcome_predicates` and `label_for` depend on the evaluation's passing rule. `event_predicate`,`outcomes`, `result_expression`, `applicable_expression` and `score_expression` do not, so a caller@@ -8,7 +8,7 @@ that reads only those is correct without passing `true_is_failure` or `output_cofrom collections.abc import Callable, Mappingfrom dataclasses import dataclass-from posthog.hogql import ast+from posthog.hogql.escape_sql import escape_hogql_stringfrom products.ai_observability.backend.models.evaluation_configs import CategoricalOutputConfig, NumericOutputConfig@@ -29,12 +29,12 @@ class EvaluationReportOutcomeDefinition:categorical_config: CategoricalOutputConfig | None = None@property- def query_placeholders(self) -> dict[str, ast.Constant]:- if self.categorical_config is not None and self.categorical_config.passing_rule is not None:- return {"passing_categories": ast.Constant(value=self.categorical_config.passing_rule.categories)}- if self.numeric_config is not None and self.numeric_config.passing_rule is not None:- return {"numeric_threshold": ast.Constant(value=self.numeric_config.passing_rule.threshold)}- return {}+ def has_pass_rule(self) -> bool:+ if self.numeric_config is not None:+ return self.numeric_config.passing_rule is not None+ if self.categorical_config is not None:+ return self.categorical_config.passing_rule is not None+ return self.passing_result is not Nonedef label_for(self, result: object, applicable: object = None) -> str | None:"""Map one raw result value to its outcome label, or None when it is not one we report."""@@ -72,20 +72,20 @@ class EvaluationReportOutcomeDefinition:_NOT_SKIPPED_PREDICATE = "(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')"_APPLICABLE_PREDICATE = (- "(isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != false)"+ "(isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false')")def _boolean_definition(true_is_failure: bool) -> EvaluationReportOutcomeDefinition:passing_result = not true_is_failure- pass_literal = "true" if passing_result else "false"- fail_literal = "false" if passing_result else "true"+ pass_literal = "'true'" if passing_result else "'false'"+ fail_literal = "'false'" if passing_result else "'true'"return EvaluationReportOutcomeDefinition(outcomes=("pass", "fail", "na"),outcome_predicates={"pass": f"properties.$ai_evaluation_result = {pass_literal} AND {_APPLICABLE_PREDICATE}","fail": f"properties.$ai_evaluation_result = {fail_literal} AND {_APPLICABLE_PREDICATE}",- "na": "properties.$ai_evaluation_applicable = false",+ "na": "properties.$ai_evaluation_applicable = 'false'",},event_predicate=("(properties.$ai_evaluation_result_type = 'boolean' OR isNull(properties.$ai_evaluation_result_type)) "@@ -127,9 +127,8 @@ def _categorical_definition(output_config: dict | None) -> EvaluationReportOutcoif config is not None and config.passing_rule is not None:if config.passing_rule.categories:- passing_condition = (- f"(notEmpty({categories_expression}) AND hasAll({{passing_categories}}, {categories_expression}))"- )+ passing = ", ".join(escape_hogql_string(category) for category in config.passing_rule.categories)+ passing_condition = f"(notEmpty({categories_expression}) AND hasAll([{passing}], {categories_expression}))"else:passing_condition = f"empty({categories_expression})"passed_predicate = f"{passing_condition} AND {applicable_predicate}"@@ -158,8 +157,9 @@ def _numeric_definition(output_config: dict | None) -> EvaluationReportOutcomeDepassed = failed = "false"if config.passing_rule is not None:passed_op, failed_op = (">=", "<") if config.passing_rule.operator == "gte" else ("<=", ">")- passed = f"{score} {passed_op} {{numeric_threshold}} AND {applicable}"- failed = f"{score} {failed_op} {{numeric_threshold}} AND {applicable}"+ threshold = repr(config.passing_rule.threshold)+ passed = f"{score} {passed_op} {threshold} AND {applicable}"+ failed = f"{score} {failed_op} {threshold} AND {applicable}"return EvaluationReportOutcomeDefinition(outcomes=("pass", "fail", "na"),outcome_predicates={"pass": passed, "fail": failed, "na": "properties.$ai_evaluation_applicable = 'false'"},diff --git a/products/ai_observability/backend/models/evaluations.py b/products/ai_observability/backend/models/evaluations.pyindex 6370f0ced475..0cd4e409c8c0 100644--- a/products/ai_observability/backend/models/evaluations.py+++ b/products/ai_observability/backend/models/evaluations.py@@ -12,6 +12,7 @@ from rest_framework.exceptions import ValidationErrorfrom posthog.models.activity_logging.model_activity import ModelActivityMixinfrom posthog.models.utils import UUIDTModel+from ..facade.evaluation_signals import evaluation_output_changedfrom .evaluation_configs import (EVALUATION_CONFIG_MODELS,EvaluationType,@@ -146,6 +147,8 @@ class Evaluation(ModelActivityMixin, UUIDTModel):created_by = models.ForeignKey("posthog.User", on_delete=models.SET_NULL, null=True, blank=True, related_name="+")deleted = models.BooleanField(default=False)+ _writes_output_on_save = False+def __init__(self, *args, **kwargs) -> None:super().__init__(*args, **kwargs)# Snapshot of the fields as last loaded / saved — used by _coerce_status_and_enabled to know which@@ -302,6 +305,11 @@ class Evaluation(ModelActivityMixin, UUIDTModel):raise ValidationError({"conditions": compile_error})self.conditions = compiled_conditions+ # Any save that writes an output field notifies, changed or not. Comparing with the stored row instead would+ # miss a concurrent save that writes a stale output after another save changed it.+ self._writes_output_on_save = update_fields is None or bool(+ {"output_type", "output_config"} & set(update_fields)+ )result = super().save(*args, **kwargs)# Refresh the baseline so the next save cycle compares against the post-save state.self._initial_enabled = self.enabled@@ -328,4 +336,18 @@ def evaluation_saved(sender, instance, created, **kwargs):# on_commit fires synchronously, preserving prior behavior.team_id = instance.team_idevaluation_id = str(instance.id)+ # Registered before the worker reload, because an on_commit callback that raises stops the ones after it.+ if instance._writes_output_on_save:+ transaction.on_commit(lambda: _notify_evaluation_output_changed(team_id))transaction.on_commit(lambda: reload_evaluations_on_workers(team_id=team_id, evaluation_ids=[evaluation_id]))+++def _notify_evaluation_output_changed(team_id: int) -> None:+ for receiver_fn, response in evaluation_output_changed.send_robust(sender=Evaluation, team_id=team_id):+ if isinstance(response, Exception):+ logger.error(+ "evaluation_output_changed_receiver_failed",+ receiver=getattr(receiver_fn, "__qualname__", repr(receiver_fn)),+ team_id=team_id,+ exc_info=response,+ )diff --git a/posthog/temporal/ai_observability/eval_reports/test/test_output_types.py b/products/ai_observability/backend/models/test/test_evaluation_outcomes.pysimilarity index 71%rename from posthog/temporal/ai_observability/eval_reports/test/test_output_types.pyrename to products/ai_observability/backend/models/test/test_evaluation_outcomes.pyindex c566a54ae56c..60314b56ef16 100644--- a/posthog/temporal/ai_observability/eval_reports/test/test_output_types.py+++ b/products/ai_observability/backend/models/test/test_evaluation_outcomes.py@@ -2,7 +2,7 @@ from django.test import SimpleTestCasefrom parameterized import parameterized-from posthog.temporal.ai_observability.eval_reports.output_types import (+from products.ai_observability.backend.models.evaluation_outcomes import (SUPPORTED_EVAL_REPORT_OUTPUT_TYPES,get_outcome_definition,)@@ -36,8 +36,6 @@ class TestOutcomeDefinitions(SimpleTestCase):self.assertEqual(definition.label_for(None, applicable=False), "na")self.assertIsNone(definition.label_for(None))self.assertIsNone(definition.label_for("resolved"))- if passing_categories is not None:- self.assertEqual(definition.query_placeholders["passing_categories"].value, passing_categories)@parameterized.expand([@@ -57,8 +55,6 @@ class TestOutcomeDefinitions(SimpleTestCase):self.assertEqual(definition.label_for(None, applicable=False), "na")self.assertIsNone(definition.label_for(True))self.assertIsNone(definition.label_for(float("nan")))- self.assertIn("{numeric_threshold}", definition.outcome_predicates["pass"])- self.assertEqual(definition.query_placeholders["numeric_threshold"].value, 7)def test_numeric_without_rule_has_no_pass_fail(self):definition = get_outcome_definition("numeric")@@ -66,15 +62,15 @@ class TestOutcomeDefinitions(SimpleTestCase):def test_absent_polarity_keeps_true_as_the_pass(self):definition = get_outcome_definition("boolean")- self.assertIn("properties.$ai_evaluation_result = true", definition.outcome_predicates["pass"])- self.assertIn("properties.$ai_evaluation_result = false", definition.outcome_predicates["fail"])+ self.assertIn("properties.$ai_evaluation_result = 'true'", definition.outcome_predicates["pass"])+ self.assertIn("properties.$ai_evaluation_result = 'false'", definition.outcome_predicates["fail"])self.assertEqual(definition.label_for(True), "pass")self.assertEqual(definition.label_for(False), "fail")def test_detector_polarity_makes_true_the_fail(self):definition = get_outcome_definition("boolean", true_is_failure=True)- self.assertIn("properties.$ai_evaluation_result = false", definition.outcome_predicates["pass"])- self.assertIn("properties.$ai_evaluation_result = true", definition.outcome_predicates["fail"])+ self.assertIn("properties.$ai_evaluation_result = 'false'", definition.outcome_predicates["pass"])+ self.assertIn("properties.$ai_evaluation_result = 'true'", definition.outcome_predicates["fail"])self.assertEqual(definition.label_for(True), "fail")self.assertEqual(definition.label_for(False), "pass")@@ -110,28 +106,3 @@ class TestOutcomeDefinitions(SimpleTestCase):def test_unsupported_type_raises(self):with self.assertRaises(ValueError):get_outcome_definition("unsupported")---class TestReportPromptPolarity(SimpleTestCase):- def _prompt(self, *, true_is_failure: bool) -> str:- from posthog.temporal.ai_observability.eval_reports.report_agent.prompts import build_eval_report_system_prompt-- return build_eval_report_system_prompt(- evaluation_name="Struggle detector",- evaluation_description="",- evaluation_prompt="",- evaluation_type="llm_judge",- output_type="boolean",- period_start="2026-04-08T14:00:00+00:00",- period_end="2026-04-08T15:00:00+00:00",- true_is_failure=true_is_failure,- )-- def test_detector_prompt_says_a_true_result_is_reported_as_a_fail(self):- prompt = self._prompt(true_is_failure=True)- self.assertIn("reported as a fail", prompt)-- def test_default_prompt_keeps_the_pass_on_true_story(self):- prompt = self._prompt(true_is_failure=False)- self.assertIn("satisfied the configured criteria", prompt)- self.assertNotIn("reported as a fail", prompt)diff --git a/products/ai_observability/backend/tests/logic/evaluations/__init__.py b/products/ai_observability/backend/tests/logic/evaluations/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py b/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.pynew file mode 100644index 000000000000..70726ceb13dc--- /dev/null+++ b/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py@@ -0,0 +1,156 @@+from collections.abc import Callable++from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events++from parameterized import parameterized++from posthog.hogql.query import execute_hogql_query++from products.ai_observability.backend.facade.api import get_evaluation_pass_filters+from products.ai_observability.backend.facade.evaluation_signals import evaluation_output_changed+from products.ai_observability.backend.models.evaluations import Evaluation++CATEGORY_OPTIONS = [{"key": "ok", "label": "Ok"}, {"key": "bad", "label": "Bad"}]+++def _evaluation(team, output_type: str = "boolean", output_config: dict | None = None, **fields) -> Evaluation:+ fields = {"evaluation_type": "hog", "evaluation_config": {"source": "return true"}, **fields}+ return Evaluation.objects.create(+ team=team, name="e", output_type=output_type, output_config=output_config or {}, **fields+ )+++class TestEvaluationPassFilters(ClickhouseTestMixin, BaseTest):+ def test_grades_each_kind_of_run_by_its_evaluation_rule(self) -> None:+ plain = _evaluation(self.team)+ detector = _evaluation(self.team, output_config={"true_is_failure": True})+ deleted_detector = _evaluation(self.team, output_config={"true_is_failure": True}, deleted=True)+ numeric = _evaluation(self.team, "numeric", {"passing_rule": {"operator": "gte", "threshold": 4}})+ numeric_without_rule = _evaluation(self.team, "numeric", {})+ categorical = _evaluation(+ self.team,+ "categorical",+ {"options": CATEGORY_OPTIONS, "selection_mode": "single", "passing_rule": {"categories": ["ok"]}},+ )+ empty_passes = _evaluation(+ self.team,+ "categorical",+ {"options": CATEGORY_OPTIONS, "selection_mode": "multiple", "passing_rule": {"categories": []}},+ )+ sentiment = _evaluation(self.team, "sentiment", {}, evaluation_type="sentiment", evaluation_config={})++ cases: dict[str, tuple[Evaluation, dict, tuple[bool, bool]]] = {+ "plain true": (plain, {"$ai_evaluation_result": True}, (True, True)),+ "plain false": (plain, {"$ai_evaluation_result": False}, (True, False)),+ "plain skipped": (plain, {"$ai_evaluation_result": False, "$ai_evaluation_skipped": True}, (False, False)),+ "plain n/a": (plain, {"$ai_evaluation_applicable": False}, (False, False)),+ "detector true": (detector, {"$ai_evaluation_result": True}, (True, False)),+ "detector false": (detector, {"$ai_evaluation_result": False}, (True, True)),+ "deleted detector false": (deleted_detector, {"$ai_evaluation_result": False}, (True, True)),+ "numeric at threshold": (numeric, {"$ai_evaluation_numeric_result": 4}, (True, True)),+ "numeric below threshold": (numeric, {"$ai_evaluation_numeric_result": 3.5}, (True, False)),+ "numeric without rule": (numeric_without_rule, {"$ai_evaluation_numeric_result": 5}, (False, False)),+ "categorical passing key": (+ categorical,+ {"$ai_evaluation_categorical_result": ["ok"], "$ai_evaluation_applicable": True},+ (True, True),+ ),+ "categorical failing key": (+ categorical,+ {"$ai_evaluation_categorical_result": ["bad"], "$ai_evaluation_applicable": True},+ (True, False),+ ),+ "empty passes, empty": (+ empty_passes,+ {"$ai_evaluation_categorical_result": [], "$ai_evaluation_applicable": True},+ (True, True),+ ),+ "empty passes, key": (+ empty_passes,+ {"$ai_evaluation_categorical_result": ["bad"], "$ai_evaluation_applicable": True},+ (True, False),+ ),+ "sentiment": (sentiment, {"$ai_sentiment_label": "positive"}, (False, False)),+ "boolean run before the switch to sentiment": (+ sentiment,+ {"$ai_evaluation_result_type": "boolean", "$ai_evaluation_result": True},+ (False, False),+ ),+ }+ for case, (evaluation, properties, _) in cases.items():+ _create_event(+ team=self.team,+ event="$ai_evaluation",+ distinct_id="user",+ properties={+ "case": case,+ "$ai_evaluation_id": str(evaluation.id),+ "$ai_evaluation_result_type": evaluation.output_type,+ **properties,+ },+ )+ flush_persons_and_events()++ filters = get_evaluation_pass_filters(self.team.id)+ response = execute_hogql_query(+ f"""+ SELECT+ JSONExtractString(properties, 'case'),+ countIf({filters.graded}) > 0,+ countIf({filters.passed}) > 0,+ countIf({filters.graded} AND NOT {filters.passed}) > 0+ FROM events+ WHERE event = '$ai_evaluation'+ GROUP BY 1+ """,+ team=self.team,+ )++ graded_passed_failed = {+ case: (bool(graded), bool(passed), bool(failed)) for case, graded, passed, failed in response.results+ }+ assert graded_passed_failed == {+ case: (graded, passed, graded and not passed) for case, (_, _, (graded, passed)) in cases.items()+ }+++def _save(evaluation: Evaluation, **changes: object) -> None:+ for field, value in changes.items():+ setattr(evaluation, field, value)+ evaluation.save()+++def _save_name_only(evaluation: Evaluation) -> None:+ evaluation.output_config = {"true_is_failure": True}+ evaluation.name = "renamed"+ evaluation.save(update_fields=["name"])+++class TestEvaluationOutputChangedSignal(BaseTest):+ @parameterized.expand(+ [+ ("created", None, True),+ ("output config changed", lambda e: _save(e, output_config={"true_is_failure": True}), True),+ ("full save without an output change", lambda e: _save(e, name="renamed"), True),+ ("status update", lambda e: e.set_status("paused", None), False),+ ("output edited but only the name saved", _save_name_only, False),+ ]+ )+ def test_notifies_on_saves_that_write_the_output(+ self, _name: str, change: Callable[[Evaluation], None] | None, should_notify: bool+ ) -> None:+ evaluation = None if change is None else _evaluation(self.team)+ notified: list[int] = []+ evaluation_output_changed.connect(+ lambda **kwargs: notified.append(kwargs["team_id"]), weak=False, dispatch_uid="test-output-changed"+ )+ self.addCleanup(evaluation_output_changed.disconnect, dispatch_uid="test-output-changed")++ with self.captureOnCommitCallbacks(execute=True):+ if change is None:+ _evaluation(self.team)+ else:+ assert evaluation is not None+ change(evaluation)++ assert notified == ([self.team.id] if should_notify else [])diff --git a/products/ai_observability/frontend/evaluations/constants.ts b/products/ai_observability/frontend/evaluations/constants.tsindex d82b11511241..789b1f064c0e 100644--- a/products/ai_observability/frontend/evaluations/constants.ts+++ b/products/ai_observability/frontend/evaluations/constants.ts@@ -10,13 +10,13 @@ export const EVALUATION_RUNS_QUERY_LIMIT = 250// JSON bool as the string 'true' (where a numeric literal throws), while Boolean-registered teams// still compare correctly against the string form. Reading the property directly rather than via// JSONExtractString keeps the lookup on the properties_group_ai column and its bloom filter index.-// Same reasoning as the predicates in posthog/temporal/ai_observability/eval_reports/output_types.py.+// Same reasoning as the predicates in products/ai_observability/backend/models/evaluation_outcomes.py.export const EVALUATION_RESULT_TRUE_HOGQL = "properties.$ai_evaluation_result = 'true'"const EVALUATION_RESULT_FALSE_HOGQL = "properties.$ai_evaluation_result = 'false'"// A skipped run was never graded, but an evaluation that disallows N/A still emits result=false// alongside skipped=true, so any count that doesn't exclude skips reads them as failures. Mirrors-// _NOT_SKIPPED_PREDICATE in posthog/temporal/ai_observability/eval_reports/output_types.py, which+// _NOT_SKIPPED_PREDICATE in products/ai_observability/backend/models/evaluation_outcomes.py, which// the report metrics already apply — without this the two surfaces disagree about the same runs.export const EVALUATION_NOT_SKIPPED_HOGQL ="(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')"