Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): expose evaluation pass filters through the facade (#119920)

Bernat Torresb75c59897612
diff --git a/.github/new-events-schema-targets.txt b/.github/new-events-schema-targets.txt
index 621549bb7e53..55745da1d56b 100644
--- a/.github/new-events-schema-targets.txt
+++ b/.github/new-events-schema-targets.txt
@@ -23,6 +23,7 @@ posthog/hogql_queries
posthog/session_recordings/queries
posthog/clickhouse/test
ee/clickhouse
+products/ai_observability/backend/tests/logic/evaluations
products/ai_observability/backend/tests/logic/traces
products/cohorts
products/customer_analytics/backend/hogql_queries
diff --git a/posthog/temporal/ai_observability/eval_reports/activities.py b/posthog/temporal/ai_observability/eval_reports/activities.py
index 0a4fa1de2817..f5e89d25fff2 100644
--- a/posthog/temporal/ai_observability/eval_reports/activities.py
+++ b/posthog/temporal/ai_observability/eval_reports/activities.py
@@ -31,7 +31,6 @@ from posthog.temporal.ai_observability.eval_reports.constants import (
COUNT_TRIGGER_QUERY_TOTAL_BUDGET_SECONDS,
COUNT_TRIGGER_QUERY_WIDTH,
)
-from posthog.temporal.ai_observability.eval_reports.output_types import get_outcome_definition
from posthog.temporal.ai_observability.eval_reports.targets import (
GENERATION_TARGET,
resolve_evaluation_target,
@@ -57,6 +56,7 @@ from posthog.temporal.ai_observability.eval_reports.types import (
from posthog.temporal.common.heartbeat import Heartbeater
from products.ai_observability.backend.models.evaluation_configs import evaluation_supports_reports
+from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definition
if TYPE_CHECKING:
from posthog.models import Team
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/graph.py b/posthog/temporal/ai_observability/eval_reports/report_agent/graph.py
index 8007bc55be0d..0d5e6d03d11f 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/graph.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/graph.py
@@ -11,7 +11,6 @@ from langgraph.prebuilt import create_react_agent
from openai import APIStatusError
from posthog.llm.gateway_client import team_distinct_id
-from posthog.temporal.ai_observability.eval_reports.output_types import get_outcome_definition
from posthog.temporal.ai_observability.eval_reports.report_agent.prompts import build_eval_report_system_prompt
from posthog.temporal.ai_observability.eval_reports.report_agent.schema import (
MAX_REPORT_SECTIONS,
@@ -44,6 +43,8 @@ from posthog.temporal.ai_observability.eval_reports.targets import (
from posthog.temporal.ai_observability.eval_reports.types import RunEvalReportAgentInput
from posthog.temporal.ai_observability.llm_endpoint import build_flex_first_chat_client, build_langchain_callbacks
+from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definition
+
logger = structlog.get_logger(__name__)
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py b/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
index 8803d3c43dcb..98c911e1778b 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
@@ -2,7 +2,6 @@
import json
-from posthog.temporal.ai_observability.eval_reports.output_types import get_outcome_definition
from posthog.temporal.ai_observability.eval_reports.report_agent.schema import MAX_REPORT_SECTIONS
from posthog.temporal.ai_observability.eval_reports.targets import (
GENERATION_TARGET,
@@ -11,6 +10,8 @@ from posthog.temporal.ai_observability.eval_reports.targets import (
get_target_descriptor,
)
+from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definition
+
EVAL_REPORT_SYSTEM_PROMPT = """You are an evaluation report agent for PostHog's AI observability platform. Your job is to analyze results from an LLM evaluation and produce a concise, grounded, example-backed report.
## What you're analyzing
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/schema.py b/posthog/temporal/ai_observability/eval_reports/report_agent/schema.py
index 66aebb56f9e5..0e89126953f9 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/schema.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/schema.py
@@ -17,9 +17,10 @@ from dataclasses import dataclass, field
from enum import StrEnum
from typing import Literal, overload
-from posthog.temporal.ai_observability.eval_reports.output_types import get_outcome_definition
from posthog.temporal.ai_observability.eval_reports.targets import GENERATION_TARGET, resolve_evaluation_target
+from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definition
+
# Hard cap on the number of agent-chosen sections. Prevents section sprawl and
# keeps reports scannable. The agent is also instructed to lean lean — quality
# over quantity, merge related findings rather than fragmenting.
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_schema.py b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_schema.py
index bb3b4a282c51..9eefd7805cf8 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_schema.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_schema.py
@@ -4,7 +4,6 @@ from django.test import SimpleTestCase
from parameterized import parameterized
-from posthog.temporal.ai_observability.eval_reports.output_types import SUPPORTED_EVAL_REPORT_OUTPUT_TYPES
from posthog.temporal.ai_observability.eval_reports.report_agent.schema import (
MAX_REPORT_SECTIONS,
MIN_REPORT_SECTIONS,
@@ -16,6 +15,7 @@ from posthog.temporal.ai_observability.eval_reports.report_agent.schema import (
)
from products.ai_observability.backend.models.evaluation_configs import REPORTABLE_OUTPUT_TYPES
+from products.ai_observability.backend.models.evaluation_outcomes import SUPPORTED_EVAL_REPORT_OUTPUT_TYPES
class TestOutputTypeRegistry(SimpleTestCase):
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools.py b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools.py
index 56a71cfaafe4..186bd9dd0b7a 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools.py
@@ -248,7 +248,7 @@ class TestSummaryMetrics(SimpleTestCase):
current_query = mock_execute_hogql.call_args_list[0].args[1]
self.assertIn("properties.$ai_sentiment_label = 'positive'", current_query)
self.assertIn("properties.$ai_evaluation_result_type = 'sentiment'", current_query)
- self.assertNotIn("properties.$ai_evaluation_result = true", current_query)
+ self.assertNotIn("properties.$ai_evaluation_result = 'true'", current_query)
@patch("posthog.temporal.ai_observability.eval_reports.report_agent.tools._execute_hogql")
def test_detector_polarity_counts_a_false_result_as_the_pass(self, mock_execute_hogql):
@@ -273,7 +273,7 @@ class TestSummaryMetrics(SimpleTestCase):
self.assertEqual(result["current_period"]["pass_rate"], 18.37)
pass_column = re.search(r"countIf\((.*?)\) as pass_count", mock_execute_hogql.call_args_list[0].args[1], re.S)
assert pass_column is not None
- self.assertIn("properties.$ai_evaluation_result = false", pass_column.group(1))
+ self.assertIn("properties.$ai_evaluation_result = 'false'", pass_column.group(1))
@patch("posthog.temporal.ai_observability.eval_reports.report_agent.tools._execute_hogql")
def test_state_without_polarity_counts_a_true_result_as_the_pass(self, mock_execute_hogql):
@@ -296,7 +296,7 @@ class TestSummaryMetrics(SimpleTestCase):
self.assertEqual(result["current_period"]["result_counts"], {"pass": 80, "fail": 18, "na": 2})
pass_column = re.search(r"countIf\((.*?)\) as pass_count", mock_execute_hogql.call_args_list[0].args[1], re.S)
assert pass_column is not None
- self.assertIn("properties.$ai_evaluation_result = true", pass_column.group(1))
+ self.assertIn("properties.$ai_evaluation_result = 'true'", pass_column.group(1))
class TestTargetAwareEvalResults(SimpleTestCase):
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py b/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py
index c8896585440b..5695735159d1 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py
@@ -26,10 +26,6 @@ from posthog.hogql import ast
from posthog.dataclasses import frozen
from posthog.errors import CH_TRANSIENT_ERRORS
-from posthog.temporal.ai_observability.eval_reports.output_types import (
- EvaluationReportOutcomeDefinition,
- get_outcome_definition,
-)
from posthog.temporal.ai_observability.eval_reports.report_agent.schema import (
MAX_REPORT_SECTIONS,
Citation,
@@ -58,6 +54,11 @@ from posthog.temporal.ai_observability.trace_summarization.constants import (
)
from posthog.temporal.ai_observability.trace_summarization.fetch_and_format import _fetch_and_format_trace
+from products.ai_observability.backend.models.evaluation_outcomes import (
+ EvaluationReportOutcomeDefinition,
+ get_outcome_definition,
+)
+
if TYPE_CHECKING:
from posthog.models import Team
@@ -403,7 +404,6 @@ def _fetch_period_summary(
AND timestamp < {{ts_end}}
""",
placeholders={
- **definition.query_placeholders,
"evaluation_id": ast.Constant(value=evaluation_id),
"ts_start": ast.Constant(value=ts_start),
"ts_end": ast.Constant(value=ts_end),
@@ -508,7 +508,6 @@ def get_result_distribution_over_time(
ORDER BY bucket
""",
placeholders={
- **definition.query_placeholders,
"evaluation_id": ast.Constant(value=evaluation_id),
"ts_start": ast.Constant(value=ts_start),
"ts_end": ast.Constant(value=ts_end),
@@ -564,7 +563,6 @@ def list_all_eval_results(
evaluation_target = resolve_evaluation_target(state.get("evaluation_target"))
shared_placeholders = {
- **definition.query_placeholders,
"evaluation_id": ast.Constant(value=evaluation_id),
"ts_start": ast.Constant(value=ts_start),
"ts_end": ast.Constant(value=ts_end),
@@ -711,7 +709,6 @@ def sample_eval_results(
LIMIT {{limit}}
""",
placeholders={
- **definition.query_placeholders,
"evaluation_id": ast.Constant(value=evaluation_id),
"ts_start": ast.Constant(value=ts_start),
"ts_end": ast.Constant(value=ts_end),
@@ -1547,7 +1544,6 @@ def get_top_outcome_reasons(
LIMIT {{limit}}
""",
placeholders={
- **definition.query_placeholders,
"evaluation_id": ast.Constant(value=evaluation_id),
"ts_start": ast.Constant(value=ts_start),
"ts_end": ast.Constant(value=ts_end),
diff --git a/posthog/temporal/ai_observability/eval_reports/test/test_report_prompt_polarity.py b/posthog/temporal/ai_observability/eval_reports/test/test_report_prompt_polarity.py
new file mode 100644
index 000000000000..9992fd1af65a
--- /dev/null
+++ b/posthog/temporal/ai_observability/eval_reports/test/test_report_prompt_polarity.py
@@ -0,0 +1,26 @@
+from django.test import SimpleTestCase
+
+
+class TestReportPromptPolarity(SimpleTestCase):
+ def _prompt(self, *, true_is_failure: bool) -> str:
+ from posthog.temporal.ai_observability.eval_reports.report_agent.prompts import build_eval_report_system_prompt
+
+ return build_eval_report_system_prompt(
+ evaluation_name="Struggle detector",
+ evaluation_description="",
+ evaluation_prompt="",
+ evaluation_type="llm_judge",
+ output_type="boolean",
+ period_start="2026-04-08T14:00:00+00:00",
+ period_end="2026-04-08T15:00:00+00:00",
+ true_is_failure=true_is_failure,
+ )
+
+ def test_detector_prompt_says_a_true_result_is_reported_as_a_fail(self):
+ prompt = self._prompt(true_is_failure=True)
+ self.assertIn("reported as a fail", prompt)
+
+ def test_default_prompt_keeps_the_pass_on_true_story(self):
+ prompt = self._prompt(true_is_failure=False)
+ self.assertIn("satisfied the configured criteria", prompt)
+ self.assertNotIn("reported as a fail", prompt)
diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.py
index b37a5f47ed90..78fc82a8b3ed 100644
--- a/products/ai_observability/backend/api/evaluations.py
+++ b/products/ai_observability/backend/api/evaluations.py
@@ -31,7 +31,6 @@ from posthog.hogql_queries.ai.ai_table_resolver import AIEventsUnavailableError,
from posthog.hogql_queries.ai.utils import HEAVY_COLUMN_NAMES, merge_heavy_properties
from posthog.models.team import Team
from posthog.permissions import AccessControlPermission
-from posthog.temporal.ai_observability.eval_reports.output_types import get_outcome_definition
from posthog.temporal.ai_observability.message_utils import extract_text_from_messages
from posthog.temporal.ai_observability.model_resolution import active_key_fallback
from posthog.temporal.ai_observability.run_evaluation import extract_event_io, run_hog_eval
@@ -42,6 +41,7 @@ from products.access_control.backend.presentation.access_control import (
AccessControlViewSetMixin,
UserAccessControlSerializerMixin,
)
+from products.ai_observability.backend.models.evaluation_outcomes import get_outcome_definition
from ..evaluation_conditions import build_condition_filter
from ..hog import compile_ai_observability_hog
diff --git a/products/ai_observability/backend/facade/api.py b/products/ai_observability/backend/facade/api.py
index f5a7a75bb679..6b3033f9a66e 100644
--- a/products/ai_observability/backend/facade/api.py
+++ b/products/ai_observability/backend/facade/api.py
@@ -2,6 +2,7 @@ from datetime import datetime
from posthog.models import Team, User
+from ..logic.evaluations.pass_filters import build_evaluation_pass_filters
from ..logic.traces.timeline import TimelineRow
from ..logic.traces.trace import Trace
from ..logic.traces.trace_queries import fallback_window, load_trace
@@ -94,3 +95,10 @@ def _to_trace(trace: Trace) -> contracts.Trace:
total_ms=trace.timeline.total_ms,
thread_node_ids=trace.thread_node_ids,
)
+
+
+def get_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilters:
+ """Grade `$ai_evaluation` events with the same rules the evaluation UI applies: boolean polarity, the numeric
+ or categorical passing rule, and no skipped, N/A or sentiment runs. The conditions change when an evaluation's
+ output config changes, so a caller that stores them rebuilds them on `evaluation_output_changed`."""
+ return build_evaluation_pass_filters(team_id)
diff --git a/products/ai_observability/backend/facade/contracts.py b/products/ai_observability/backend/facade/contracts.py
index 7806ec1677bb..859a9e87b032 100644
--- a/products/ai_observability/backend/facade/contracts.py
+++ b/products/ai_observability/backend/facade/contracts.py
@@ -66,3 +66,10 @@ class Trace(Contract):
timeline: list[TraceTimelineRow]
total_ms: float
thread_node_ids: list[str]
+
+
+class EvaluationPassFilters(Contract):
+ """HogQL conditions over `$ai_evaluation` events, built from each evaluation's current output config."""
+
+ graded: str # The run counts toward a pass rate.
+ passed: str # The run passed. Implies `graded`.
diff --git a/products/ai_observability/backend/facade/evaluation_signals.py b/products/ai_observability/backend/facade/evaluation_signals.py
new file mode 100644
index 000000000000..df8e37b4ecff
--- /dev/null
+++ b/products/ai_observability/backend/facade/evaluation_signals.py
@@ -0,0 +1,12 @@
+from collections.abc import Callable
+
+from django.dispatch import Signal
+
+evaluation_output_changed = Signal()
+
+
+def connect_evaluation_output_changed(receiver: Callable[..., object], *, dispatch_uid: str) -> None:
+ """``receiver`` gets ``team_id=`` after an evaluation save that writes its output type or output config commits.
+ Every full save writes them, so a receiver must be idempotent. A receiver that raises is logged and never fails
+ the save."""
+ evaluation_output_changed.connect(receiver, dispatch_uid=dispatch_uid)
diff --git a/products/ai_observability/backend/logic/evaluations/__init__.py b/products/ai_observability/backend/logic/evaluations/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/ai_observability/backend/logic/evaluations/pass_filters.py b/products/ai_observability/backend/logic/evaluations/pass_filters.py
new file mode 100644
index 000000000000..ae44fce458f9
--- /dev/null
+++ b/products/ai_observability/backend/logic/evaluations/pass_filters.py
@@ -0,0 +1,64 @@
+from django.db.models import Q
+
+from posthog.hogql.escape_sql import escape_hogql_string
+
+from products.ai_observability.backend.models.evaluation_configs import OutputType
+from products.ai_observability.backend.models.evaluation_outcomes import (
+ EvaluationReportOutcomeDefinition,
+ get_outcome_definition,
+)
+from products.ai_observability.backend.models.evaluations import Evaluation
+
+from ...facade import contracts
+
+_DEFAULT_DEFINITION = get_outcome_definition(OutputType.BOOLEAN)
+
+
+def build_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilters:
+ graded_branches: list[str] = []
+ passed_branches: list[str] = []
+ # Every other evaluation falls through to the default rule. Soft-deleted evaluations stay in, because their past
+ # events still need their own rule. Reads the writer, because a receiver of evaluation_output_changed calls this
+ # right after the commit, before a replica can catch up.
+ evaluations_with_own_rule = (
+ Evaluation.objects.using("default")
+ .filter(team_id=team_id)
+ .filter(
+ Q(output_type__in=[OutputType.NUMERIC, OutputType.CATEGORICAL, OutputType.SENTIMENT])
+ | Q(output_config__true_is_failure=True)
+ )
+ .values_list("id", "output_type", "output_config")
+ )
+ for evaluation_id, output_type, output_config in evaluations_with_own_rule:
+ try:
+ definition = get_outcome_definition(
+ output_type, true_is_failure=output_config.get("true_is_failure") is True, output_config=output_config
+ )
+ except ValueError:
+ definition = None
+ matches_evaluation = f"properties.$ai_evaluation_id = {escape_hogql_string(str(evaluation_id))}"
+ # An evaluation without a pass rule grades none of its runs, including boolean runs from before it
+ # switched to sentiment, which the default branch would otherwise grade.
+ if definition is None or not definition.has_pass_rule:
+ graded_branches += [matches_evaluation, "false"]
+ passed_branches += [matches_evaluation, "false"]
+ continue
+ graded_branches += [matches_evaluation, _graded(definition)]
+ passed_branches += [matches_evaluation, definition.outcome_predicates["pass"]]
+
+ graded = _multi_if(graded_branches, _graded(_DEFAULT_DEFINITION))
+ return contracts.EvaluationPassFilters(
+ graded=graded,
+ passed=f"({graded} AND {_multi_if(passed_branches, _DEFAULT_DEFINITION.outcome_predicates['pass'])})",
+ )
+
+
+def _graded(definition: EvaluationReportOutcomeDefinition) -> str:
+ predicates = definition.outcome_predicates
+ return f"{definition.event_predicate} AND (({predicates['pass']}) OR ({predicates['fail']}))"
+
+
+def _multi_if(branches: list[str], default: str) -> str:
+ if not branches:
+ return f"({default})"
+ return f"multiIf({', '.join(branches)}, {default})"
diff --git a/posthog/temporal/ai_observability/eval_reports/output_types.py b/products/ai_observability/backend/models/evaluation_outcomes.py
similarity index 87%
rename from posthog/temporal/ai_observability/eval_reports/output_types.py
rename to products/ai_observability/backend/models/evaluation_outcomes.py
index 6b9cf8f4f307..3067f197ea9e 100644
--- a/posthog/temporal/ai_observability/eval_reports/output_types.py
+++ b/products/ai_observability/backend/models/evaluation_outcomes.py
@@ -1,4 +1,4 @@
-"""Trusted query definitions for evaluation report outcomes.
+"""Trusted query definitions for evaluation outcomes, shared by reports and the pass filters other products use.
Only `outcome_predicates` and `label_for` depend on the evaluation's passing rule. `event_predicate`,
`outcomes`, `result_expression`, `applicable_expression` and `score_expression` do not, so a caller
@@ -8,7 +8,7 @@ that reads only those is correct without passing `true_is_failure` or `output_co
from collections.abc import Callable, Mapping
from dataclasses import dataclass
-from posthog.hogql import ast
+from posthog.hogql.escape_sql import escape_hogql_string
from products.ai_observability.backend.models.evaluation_configs import CategoricalOutputConfig, NumericOutputConfig
@@ -29,12 +29,12 @@ class EvaluationReportOutcomeDefinition:
categorical_config: CategoricalOutputConfig | None = None
@property
- def query_placeholders(self) -> dict[str, ast.Constant]:
- if self.categorical_config is not None and self.categorical_config.passing_rule is not None:
- return {"passing_categories": ast.Constant(value=self.categorical_config.passing_rule.categories)}
- if self.numeric_config is not None and self.numeric_config.passing_rule is not None:
- return {"numeric_threshold": ast.Constant(value=self.numeric_config.passing_rule.threshold)}
- return {}
+ def has_pass_rule(self) -> bool:
+ if self.numeric_config is not None:
+ return self.numeric_config.passing_rule is not None
+ if self.categorical_config is not None:
+ return self.categorical_config.passing_rule is not None
+ return self.passing_result is not None
def label_for(self, result: object, applicable: object = None) -> str | None:
"""Map one raw result value to its outcome label, or None when it is not one we report."""
@@ -72,20 +72,20 @@ class EvaluationReportOutcomeDefinition:
_NOT_SKIPPED_PREDICATE = "(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')"
_APPLICABLE_PREDICATE = (
- "(isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != false)"
+ "(isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false')"
)
def _boolean_definition(true_is_failure: bool) -> EvaluationReportOutcomeDefinition:
passing_result = not true_is_failure
- pass_literal = "true" if passing_result else "false"
- fail_literal = "false" if passing_result else "true"
+ pass_literal = "'true'" if passing_result else "'false'"
+ fail_literal = "'false'" if passing_result else "'true'"
return EvaluationReportOutcomeDefinition(
outcomes=("pass", "fail", "na"),
outcome_predicates={
"pass": f"properties.$ai_evaluation_result = {pass_literal} AND {_APPLICABLE_PREDICATE}",
"fail": f"properties.$ai_evaluation_result = {fail_literal} AND {_APPLICABLE_PREDICATE}",
- "na": "properties.$ai_evaluation_applicable = false",
+ "na": "properties.$ai_evaluation_applicable = 'false'",
},
event_predicate=(
"(properties.$ai_evaluation_result_type = 'boolean' OR isNull(properties.$ai_evaluation_result_type)) "
@@ -127,9 +127,8 @@ def _categorical_definition(output_config: dict | None) -> EvaluationReportOutco
if config is not None and config.passing_rule is not None:
if config.passing_rule.categories:
- passing_condition = (
- f"(notEmpty({categories_expression}) AND hasAll({{passing_categories}}, {categories_expression}))"
- )
+ passing = ", ".join(escape_hogql_string(category) for category in config.passing_rule.categories)
+ passing_condition = f"(notEmpty({categories_expression}) AND hasAll([{passing}], {categories_expression}))"
else:
passing_condition = f"empty({categories_expression})"
passed_predicate = f"{passing_condition} AND {applicable_predicate}"
@@ -158,8 +157,9 @@ def _numeric_definition(output_config: dict | None) -> EvaluationReportOutcomeDe
passed = failed = "false"
if config.passing_rule is not None:
passed_op, failed_op = (">=", "<") if config.passing_rule.operator == "gte" else ("<=", ">")
- passed = f"{score} {passed_op} {{numeric_threshold}} AND {applicable}"
- failed = f"{score} {failed_op} {{numeric_threshold}} AND {applicable}"
+ threshold = repr(config.passing_rule.threshold)
+ passed = f"{score} {passed_op} {threshold} AND {applicable}"
+ failed = f"{score} {failed_op} {threshold} AND {applicable}"
return EvaluationReportOutcomeDefinition(
outcomes=("pass", "fail", "na"),
outcome_predicates={"pass": passed, "fail": failed, "na": "properties.$ai_evaluation_applicable = 'false'"},
diff --git a/products/ai_observability/backend/models/evaluations.py b/products/ai_observability/backend/models/evaluations.py
index 6370f0ced475..0cd4e409c8c0 100644
--- a/products/ai_observability/backend/models/evaluations.py
+++ b/products/ai_observability/backend/models/evaluations.py
@@ -12,6 +12,7 @@ from rest_framework.exceptions import ValidationError
from posthog.models.activity_logging.model_activity import ModelActivityMixin
from posthog.models.utils import UUIDTModel
+from ..facade.evaluation_signals import evaluation_output_changed
from .evaluation_configs import (
EVALUATION_CONFIG_MODELS,
EvaluationType,
@@ -146,6 +147,8 @@ class Evaluation(ModelActivityMixin, UUIDTModel):
created_by = models.ForeignKey("posthog.User", on_delete=models.SET_NULL, null=True, blank=True, related_name="+")
deleted = models.BooleanField(default=False)
+ _writes_output_on_save = False
+
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
# Snapshot of the fields as last loaded / saved — used by _coerce_status_and_enabled to know which
@@ -302,6 +305,11 @@ class Evaluation(ModelActivityMixin, UUIDTModel):
raise ValidationError({"conditions": compile_error})
self.conditions = compiled_conditions
+ # Any save that writes an output field notifies, changed or not. Comparing with the stored row instead would
+ # miss a concurrent save that writes a stale output after another save changed it.
+ self._writes_output_on_save = update_fields is None or bool(
+ {"output_type", "output_config"} & set(update_fields)
+ )
result = super().save(*args, **kwargs)
# Refresh the baseline so the next save cycle compares against the post-save state.
self._initial_enabled = self.enabled
@@ -328,4 +336,18 @@ def evaluation_saved(sender, instance, created, **kwargs):
# on_commit fires synchronously, preserving prior behavior.
team_id = instance.team_id
evaluation_id = str(instance.id)
+ # Registered before the worker reload, because an on_commit callback that raises stops the ones after it.
+ if instance._writes_output_on_save:
+ transaction.on_commit(lambda: _notify_evaluation_output_changed(team_id))
transaction.on_commit(lambda: reload_evaluations_on_workers(team_id=team_id, evaluation_ids=[evaluation_id]))
+
+
+def _notify_evaluation_output_changed(team_id: int) -> None:
+ for receiver_fn, response in evaluation_output_changed.send_robust(sender=Evaluation, team_id=team_id):
+ if isinstance(response, Exception):
+ logger.error(
+ "evaluation_output_changed_receiver_failed",
+ receiver=getattr(receiver_fn, "__qualname__", repr(receiver_fn)),
+ team_id=team_id,
+ exc_info=response,
+ )
diff --git a/posthog/temporal/ai_observability/eval_reports/test/test_output_types.py b/products/ai_observability/backend/models/test/test_evaluation_outcomes.py
similarity index 71%
rename from posthog/temporal/ai_observability/eval_reports/test/test_output_types.py
rename to products/ai_observability/backend/models/test/test_evaluation_outcomes.py
index c566a54ae56c..60314b56ef16 100644
--- a/posthog/temporal/ai_observability/eval_reports/test/test_output_types.py
+++ b/products/ai_observability/backend/models/test/test_evaluation_outcomes.py
@@ -2,7 +2,7 @@ from django.test import SimpleTestCase
from parameterized import parameterized
-from posthog.temporal.ai_observability.eval_reports.output_types import (
+from products.ai_observability.backend.models.evaluation_outcomes import (
SUPPORTED_EVAL_REPORT_OUTPUT_TYPES,
get_outcome_definition,
)
@@ -36,8 +36,6 @@ class TestOutcomeDefinitions(SimpleTestCase):
self.assertEqual(definition.label_for(None, applicable=False), "na")
self.assertIsNone(definition.label_for(None))
self.assertIsNone(definition.label_for("resolved"))
- if passing_categories is not None:
- self.assertEqual(definition.query_placeholders["passing_categories"].value, passing_categories)
@parameterized.expand(
[
@@ -57,8 +55,6 @@ class TestOutcomeDefinitions(SimpleTestCase):
self.assertEqual(definition.label_for(None, applicable=False), "na")
self.assertIsNone(definition.label_for(True))
self.assertIsNone(definition.label_for(float("nan")))
- self.assertIn("{numeric_threshold}", definition.outcome_predicates["pass"])
- self.assertEqual(definition.query_placeholders["numeric_threshold"].value, 7)
def test_numeric_without_rule_has_no_pass_fail(self):
definition = get_outcome_definition("numeric")
@@ -66,15 +62,15 @@ class TestOutcomeDefinitions(SimpleTestCase):
def test_absent_polarity_keeps_true_as_the_pass(self):
definition = get_outcome_definition("boolean")
- self.assertIn("properties.$ai_evaluation_result = true", definition.outcome_predicates["pass"])
- self.assertIn("properties.$ai_evaluation_result = false", definition.outcome_predicates["fail"])
+ self.assertIn("properties.$ai_evaluation_result = 'true'", definition.outcome_predicates["pass"])
+ self.assertIn("properties.$ai_evaluation_result = 'false'", definition.outcome_predicates["fail"])
self.assertEqual(definition.label_for(True), "pass")
self.assertEqual(definition.label_for(False), "fail")
def test_detector_polarity_makes_true_the_fail(self):
definition = get_outcome_definition("boolean", true_is_failure=True)
- self.assertIn("properties.$ai_evaluation_result = false", definition.outcome_predicates["pass"])
- self.assertIn("properties.$ai_evaluation_result = true", definition.outcome_predicates["fail"])
+ self.assertIn("properties.$ai_evaluation_result = 'false'", definition.outcome_predicates["pass"])
+ self.assertIn("properties.$ai_evaluation_result = 'true'", definition.outcome_predicates["fail"])
self.assertEqual(definition.label_for(True), "fail")
self.assertEqual(definition.label_for(False), "pass")
@@ -110,28 +106,3 @@ class TestOutcomeDefinitions(SimpleTestCase):
def test_unsupported_type_raises(self):
with self.assertRaises(ValueError):
get_outcome_definition("unsupported")
-
-
-class TestReportPromptPolarity(SimpleTestCase):
- def _prompt(self, *, true_is_failure: bool) -> str:
- from posthog.temporal.ai_observability.eval_reports.report_agent.prompts import build_eval_report_system_prompt
-
- return build_eval_report_system_prompt(
- evaluation_name="Struggle detector",
- evaluation_description="",
- evaluation_prompt="",
- evaluation_type="llm_judge",
- output_type="boolean",
- period_start="2026-04-08T14:00:00+00:00",
- period_end="2026-04-08T15:00:00+00:00",
- true_is_failure=true_is_failure,
- )
-
- def test_detector_prompt_says_a_true_result_is_reported_as_a_fail(self):
- prompt = self._prompt(true_is_failure=True)
- self.assertIn("reported as a fail", prompt)
-
- def test_default_prompt_keeps_the_pass_on_true_story(self):
- prompt = self._prompt(true_is_failure=False)
- self.assertIn("satisfied the configured criteria", prompt)
- self.assertNotIn("reported as a fail", prompt)
diff --git a/products/ai_observability/backend/tests/logic/evaluations/__init__.py b/products/ai_observability/backend/tests/logic/evaluations/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py b/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py
new file mode 100644
index 000000000000..70726ceb13dc
--- /dev/null
+++ b/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py
@@ -0,0 +1,156 @@
+from collections.abc import Callable
+
+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
+
+from parameterized import parameterized
+
+from posthog.hogql.query import execute_hogql_query
+
+from products.ai_observability.backend.facade.api import get_evaluation_pass_filters
+from products.ai_observability.backend.facade.evaluation_signals import evaluation_output_changed
+from products.ai_observability.backend.models.evaluations import Evaluation
+
+CATEGORY_OPTIONS = [{"key": "ok", "label": "Ok"}, {"key": "bad", "label": "Bad"}]
+
+
+def _evaluation(team, output_type: str = "boolean", output_config: dict | None = None, **fields) -> Evaluation:
+ fields = {"evaluation_type": "hog", "evaluation_config": {"source": "return true"}, **fields}
+ return Evaluation.objects.create(
+ team=team, name="e", output_type=output_type, output_config=output_config or {}, **fields
+ )
+
+
+class TestEvaluationPassFilters(ClickhouseTestMixin, BaseTest):
+ def test_grades_each_kind_of_run_by_its_evaluation_rule(self) -> None:
+ plain = _evaluation(self.team)
+ detector = _evaluation(self.team, output_config={"true_is_failure": True})
+ deleted_detector = _evaluation(self.team, output_config={"true_is_failure": True}, deleted=True)
+ numeric = _evaluation(self.team, "numeric", {"passing_rule": {"operator": "gte", "threshold": 4}})
+ numeric_without_rule = _evaluation(self.team, "numeric", {})
+ categorical = _evaluation(
+ self.team,
+ "categorical",
+ {"options": CATEGORY_OPTIONS, "selection_mode": "single", "passing_rule": {"categories": ["ok"]}},
+ )
+ empty_passes = _evaluation(
+ self.team,
+ "categorical",
+ {"options": CATEGORY_OPTIONS, "selection_mode": "multiple", "passing_rule": {"categories": []}},
+ )
+ sentiment = _evaluation(self.team, "sentiment", {}, evaluation_type="sentiment", evaluation_config={})
+
+ cases: dict[str, tuple[Evaluation, dict, tuple[bool, bool]]] = {
+ "plain true": (plain, {"$ai_evaluation_result": True}, (True, True)),
+ "plain false": (plain, {"$ai_evaluation_result": False}, (True, False)),
+ "plain skipped": (plain, {"$ai_evaluation_result": False, "$ai_evaluation_skipped": True}, (False, False)),
+ "plain n/a": (plain, {"$ai_evaluation_applicable": False}, (False, False)),
+ "detector true": (detector, {"$ai_evaluation_result": True}, (True, False)),
+ "detector false": (detector, {"$ai_evaluation_result": False}, (True, True)),
+ "deleted detector false": (deleted_detector, {"$ai_evaluation_result": False}, (True, True)),
+ "numeric at threshold": (numeric, {"$ai_evaluation_numeric_result": 4}, (True, True)),
+ "numeric below threshold": (numeric, {"$ai_evaluation_numeric_result": 3.5}, (True, False)),
+ "numeric without rule": (numeric_without_rule, {"$ai_evaluation_numeric_result": 5}, (False, False)),
+ "categorical passing key": (
+ categorical,
+ {"$ai_evaluation_categorical_result": ["ok"], "$ai_evaluation_applicable": True},
+ (True, True),
+ ),
+ "categorical failing key": (
+ categorical,
+ {"$ai_evaluation_categorical_result": ["bad"], "$ai_evaluation_applicable": True},
+ (True, False),
+ ),
+ "empty passes, empty": (
+ empty_passes,
+ {"$ai_evaluation_categorical_result": [], "$ai_evaluation_applicable": True},
+ (True, True),
+ ),
+ "empty passes, key": (
+ empty_passes,
+ {"$ai_evaluation_categorical_result": ["bad"], "$ai_evaluation_applicable": True},
+ (True, False),
+ ),
+ "sentiment": (sentiment, {"$ai_sentiment_label": "positive"}, (False, False)),
+ "boolean run before the switch to sentiment": (
+ sentiment,
+ {"$ai_evaluation_result_type": "boolean", "$ai_evaluation_result": True},
+ (False, False),
+ ),
+ }
+ for case, (evaluation, properties, _) in cases.items():
+ _create_event(
+ team=self.team,
+ event="$ai_evaluation",
+ distinct_id="user",
+ properties={
+ "case": case,
+ "$ai_evaluation_id": str(evaluation.id),
+ "$ai_evaluation_result_type": evaluation.output_type,
+ **properties,
+ },
+ )
+ flush_persons_and_events()
+
+ filters = get_evaluation_pass_filters(self.team.id)
+ response = execute_hogql_query(
+ f"""
+ SELECT
+ JSONExtractString(properties, 'case'),
+ countIf({filters.graded}) > 0,
+ countIf({filters.passed}) > 0,
+ countIf({filters.graded} AND NOT {filters.passed}) > 0
+ FROM events
+ WHERE event = '$ai_evaluation'
+ GROUP BY 1
+ """,
+ team=self.team,
+ )
+
+ graded_passed_failed = {
+ case: (bool(graded), bool(passed), bool(failed)) for case, graded, passed, failed in response.results
+ }
+ assert graded_passed_failed == {
+ case: (graded, passed, graded and not passed) for case, (_, _, (graded, passed)) in cases.items()
+ }
+
+
+def _save(evaluation: Evaluation, **changes: object) -> None:
+ for field, value in changes.items():
+ setattr(evaluation, field, value)
+ evaluation.save()
+
+
+def _save_name_only(evaluation: Evaluation) -> None:
+ evaluation.output_config = {"true_is_failure": True}
+ evaluation.name = "renamed"
+ evaluation.save(update_fields=["name"])
+
+
+class TestEvaluationOutputChangedSignal(BaseTest):
+ @parameterized.expand(
+ [
+ ("created", None, True),
+ ("output config changed", lambda e: _save(e, output_config={"true_is_failure": True}), True),
+ ("full save without an output change", lambda e: _save(e, name="renamed"), True),
+ ("status update", lambda e: e.set_status("paused", None), False),
+ ("output edited but only the name saved", _save_name_only, False),
+ ]
+ )
+ def test_notifies_on_saves_that_write_the_output(
+ self, _name: str, change: Callable[[Evaluation], None] | None, should_notify: bool
+ ) -> None:
+ evaluation = None if change is None else _evaluation(self.team)
+ notified: list[int] = []
+ evaluation_output_changed.connect(
+ lambda **kwargs: notified.append(kwargs["team_id"]), weak=False, dispatch_uid="test-output-changed"
+ )
+ self.addCleanup(evaluation_output_changed.disconnect, dispatch_uid="test-output-changed")
+
+ with self.captureOnCommitCallbacks(execute=True):
+ if change is None:
+ _evaluation(self.team)
+ else:
+ assert evaluation is not None
+ change(evaluation)
+
+ assert notified == ([self.team.id] if should_notify else [])
diff --git a/products/ai_observability/frontend/evaluations/constants.ts b/products/ai_observability/frontend/evaluations/constants.ts
index d82b11511241..789b1f064c0e 100644
--- a/products/ai_observability/frontend/evaluations/constants.ts
+++ b/products/ai_observability/frontend/evaluations/constants.ts
@@ -10,13 +10,13 @@ export const EVALUATION_RUNS_QUERY_LIMIT = 250
// JSON bool as the string 'true' (where a numeric literal throws), while Boolean-registered teams
// still compare correctly against the string form. Reading the property directly rather than via
// JSONExtractString keeps the lookup on the properties_group_ai column and its bloom filter index.
-// Same reasoning as the predicates in posthog/temporal/ai_observability/eval_reports/output_types.py.
+// Same reasoning as the predicates in products/ai_observability/backend/models/evaluation_outcomes.py.
export const EVALUATION_RESULT_TRUE_HOGQL = "properties.$ai_evaluation_result = 'true'"
const EVALUATION_RESULT_FALSE_HOGQL = "properties.$ai_evaluation_result = 'false'"
// A skipped run was never graded, but an evaluation that disallows N/A still emits result=false
// alongside skipped=true, so any count that doesn't exclude skips reads them as failures. Mirrors
-// _NOT_SKIPPED_PREDICATE in posthog/temporal/ai_observability/eval_reports/output_types.py, which
+// _NOT_SKIPPED_PREDICATE in products/ai_observability/backend/models/evaluation_outcomes.py, which
// the report metrics already apply — without this the two surfaces disagree about the same runs.
export const EVALUATION_NOT_SKIPPED_HOGQL =
"(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')"