:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(aio): remove unused evaluation output signal and pass filters facade (#122700)
Bernat Torres8c20c4e803af
diff --git a/products/ai_observability/backend/facade/api.py b/products/ai_observability/backend/facade/api.pyindex de542494bf27..e33c2afef281 100644--- a/products/ai_observability/backend/facade/api.py+++ b/products/ai_observability/backend/facade/api.py@@ -173,13 +173,6 @@ def _to_node_detail(node_id: str, event: NodeEvent) -> contracts.TraceNodeDetail)-def get_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilters:- """Grade `$ai_evaluation` events with the same rules the evaluation UI applies: boolean polarity, the numeric- or categorical passing rule, and no skipped, N/A or sentiment runs. The conditions change when an evaluation's- output config changes, so a caller that stores them rebuilds them on `evaluation_output_changed`."""- return build_evaluation_pass_filters(team_id)--def resolve_evaluation_pass_markers[P](properties: list[P], team_id: int) -> list[P]:"""Replace each evaluation pass marker with the team's current condition."""markers = {contracts.EVALUATION_GRADED_MARKER, contracts.EVALUATION_PASSED_MARKER}diff --git a/products/ai_observability/backend/facade/evaluation_signals.py b/products/ai_observability/backend/facade/evaluation_signals.pydeleted file mode 100644index df8e37b4ecff..000000000000--- a/products/ai_observability/backend/facade/evaluation_signals.py+++ /dev/null@@ -1,12 +0,0 @@-from collections.abc import Callable--from django.dispatch import Signal--evaluation_output_changed = Signal()---def connect_evaluation_output_changed(receiver: Callable[..., object], *, dispatch_uid: str) -> None:- """``receiver`` gets ``team_id=`` after an evaluation save that writes its output type or output config commits.- Every full save writes them, so a receiver must be idempotent. A receiver that raises is logged and never fails- the save."""- evaluation_output_changed.connect(receiver, dispatch_uid=dispatch_uid)diff --git a/products/ai_observability/backend/logic/evaluations/pass_filters.py b/products/ai_observability/backend/logic/evaluations/pass_filters.pyindex a23dbcc2ca59..1be87154fbd9 100644--- a/products/ai_observability/backend/logic/evaluations/pass_filters.py+++ b/products/ai_observability/backend/logic/evaluations/pass_filters.py@@ -35,13 +35,10 @@ class PassRules:passes: ast.Expr-def build_evaluation_pass_filters(- team_id: int, evaluation_ids: Sequence[str] | None = None-) -> contracts.EvaluationPassFilters:- """`evaluation_ids` limits the rules to the runs of those evaluations, for a read that filters on them."""+def build_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilters:graded_branches: list[str] = []passed_branches: list[str] = []- for evaluation_id, definition in _own_rules(team_id, evaluation_ids):+ for evaluation_id, definition in _own_rules(team_id, None):matches_evaluation = f"{_EVALUATION_ID} = {escape_hogql_string(evaluation_id)}"# An evaluation without a pass rule grades none of its runs, including boolean runs from before it# switched to sentiment, which the default branch would otherwise grade.@@ -138,10 +135,8 @@ def _own_rules(team_id: int, evaluation_ids: Sequence[str] | None) -> Iterator[tuple[str, EvaluationReportOutcomeDefinition | None]]:"""Each evaluation whose runs the default rule does not grade, with its rule, or None for a config that no longer- loads. Soft-deleted evaluations stay in, because their past events still need their own rule. Reads the writer,- because a receiver of evaluation_output_changed calls this right after the commit, before a replica can catch- up."""- evaluations = Evaluation.objects.using("default").filter(team_id=team_id)+ loads. Soft-deleted evaluations stay in, because their past events still need their own rule."""+ evaluations = Evaluation.objects.filter(team_id=team_id)if evaluation_ids is not None:evaluations = evaluations.filter(id__in=evaluation_ids)evaluations_with_own_rule = evaluations.filter(diff --git a/products/ai_observability/backend/models/evaluations.py b/products/ai_observability/backend/models/evaluations.pyindex 0cd4e409c8c0..6370f0ced475 100644--- a/products/ai_observability/backend/models/evaluations.py+++ b/products/ai_observability/backend/models/evaluations.py@@ -12,7 +12,6 @@ from rest_framework.exceptions import ValidationErrorfrom posthog.models.activity_logging.model_activity import ModelActivityMixinfrom posthog.models.utils import UUIDTModel-from ..facade.evaluation_signals import evaluation_output_changedfrom .evaluation_configs import (EVALUATION_CONFIG_MODELS,EvaluationType,@@ -147,8 +146,6 @@ class Evaluation(ModelActivityMixin, UUIDTModel):created_by = models.ForeignKey("posthog.User", on_delete=models.SET_NULL, null=True, blank=True, related_name="+")deleted = models.BooleanField(default=False)- _writes_output_on_save = False-def __init__(self, *args, **kwargs) -> None:super().__init__(*args, **kwargs)# Snapshot of the fields as last loaded / saved — used by _coerce_status_and_enabled to know which@@ -305,11 +302,6 @@ class Evaluation(ModelActivityMixin, UUIDTModel):raise ValidationError({"conditions": compile_error})self.conditions = compiled_conditions- # Any save that writes an output field notifies, changed or not. Comparing with the stored row instead would- # miss a concurrent save that writes a stale output after another save changed it.- self._writes_output_on_save = update_fields is None or bool(- {"output_type", "output_config"} & set(update_fields)- )result = super().save(*args, **kwargs)# Refresh the baseline so the next save cycle compares against the post-save state.self._initial_enabled = self.enabled@@ -336,18 +328,4 @@ def evaluation_saved(sender, instance, created, **kwargs):# on_commit fires synchronously, preserving prior behavior.team_id = instance.team_idevaluation_id = str(instance.id)- # Registered before the worker reload, because an on_commit callback that raises stops the ones after it.- if instance._writes_output_on_save:- transaction.on_commit(lambda: _notify_evaluation_output_changed(team_id))transaction.on_commit(lambda: reload_evaluations_on_workers(team_id=team_id, evaluation_ids=[evaluation_id]))---def _notify_evaluation_output_changed(team_id: int) -> None:- for receiver_fn, response in evaluation_output_changed.send_robust(sender=Evaluation, team_id=team_id):- if isinstance(response, Exception):- logger.error(- "evaluation_output_changed_receiver_failed",- receiver=getattr(receiver_fn, "__qualname__", repr(receiver_fn)),- team_id=team_id,- exc_info=response,- )diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py b/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.pyindex f520f35d15cd..fe2cff12bef4 100644--- a/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py+++ b/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py@@ -7,9 +7,11 @@ from parameterized import parameterizedfrom posthog.hogql.parser import parse_exprfrom posthog.hogql.query import execute_hogql_query-from products.ai_observability.backend.facade.api import get_evaluation_pass_filters-from products.ai_observability.backend.facade.evaluation_signals import evaluation_output_changed-from products.ai_observability.backend.logic.evaluations.pass_filters import PassRules, build_pass_rules+from products.ai_observability.backend.logic.evaluations.pass_filters import (+ PassRules,+ build_evaluation_pass_filters,+ build_pass_rules,+)from products.ai_observability.backend.models.evaluations import EvaluationCATEGORY_OPTIONS = [{"key": "ok", "label": "Ok"}, {"key": "bad", "label": "Bad"}]@@ -23,7 +25,7 @@ def _evaluation(team, output_type: str = "boolean", output_config: dict | None =def _per_evaluation_rules(team_id: int, _evaluation_ids: list[str]) -> PassRules:- filters = get_evaluation_pass_filters(team_id)+ filters = build_evaluation_pass_filters(team_id)return PassRules(graded=parse_expr(filters.graded), passes=parse_expr(filters.passed))@@ -153,45 +155,3 @@ class TestEvaluationPassFilters(ClickhouseTestMixin, BaseTest):assert graded_passed_failed == {case: (graded, passed, graded and not passed) for case, (_, _, (graded, passed)) in cases.items()}---def _save(evaluation: Evaluation, **changes: object) -> None:- for field, value in changes.items():- setattr(evaluation, field, value)- evaluation.save()---def _save_name_only(evaluation: Evaluation) -> None:- evaluation.output_config = {"true_is_failure": True}- evaluation.name = "renamed"- evaluation.save(update_fields=["name"])---class TestEvaluationOutputChangedSignal(BaseTest):- @parameterized.expand(- [- ("created", None, True),- ("output config changed", lambda e: _save(e, output_config={"true_is_failure": True}), True),- ("full save without an output change", lambda e: _save(e, name="renamed"), True),- ("status update", lambda e: e.set_status("paused", None), False),- ("output edited but only the name saved", _save_name_only, False),- ]- )- def test_notifies_on_saves_that_write_the_output(- self, _name: str, change: Callable[[Evaluation], None] | None, should_notify: bool- ) -> None:- evaluation = None if change is None else _evaluation(self.team)- notified: list[int] = []- evaluation_output_changed.connect(- lambda **kwargs: notified.append(kwargs["team_id"]), weak=False, dispatch_uid="test-output-changed"- )- self.addCleanup(evaluation_output_changed.disconnect, dispatch_uid="test-output-changed")-- with self.captureOnCommitCallbacks(execute=True):- if change is None:- _evaluation(self.team)- else:- assert evaluation is not None- change(evaluation)-- assert notified == ([self.team.id] if should_notify else [])