Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(aio): remove unused evaluation output signal and pass filters facade (#122700)

Bernat Torres8c20c4e803af
diff --git a/products/ai_observability/backend/facade/api.py b/products/ai_observability/backend/facade/api.py
index de542494bf27..e33c2afef281 100644
--- a/products/ai_observability/backend/facade/api.py
+++ b/products/ai_observability/backend/facade/api.py
@@ -173,13 +173,6 @@ def _to_node_detail(node_id: str, event: NodeEvent) -> contracts.TraceNodeDetail
)
-def get_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilters:
- """Grade `$ai_evaluation` events with the same rules the evaluation UI applies: boolean polarity, the numeric
- or categorical passing rule, and no skipped, N/A or sentiment runs. The conditions change when an evaluation's
- output config changes, so a caller that stores them rebuilds them on `evaluation_output_changed`."""
- return build_evaluation_pass_filters(team_id)
-
-
def resolve_evaluation_pass_markers[P](properties: list[P], team_id: int) -> list[P]:
"""Replace each evaluation pass marker with the team's current condition."""
markers = {contracts.EVALUATION_GRADED_MARKER, contracts.EVALUATION_PASSED_MARKER}
diff --git a/products/ai_observability/backend/facade/evaluation_signals.py b/products/ai_observability/backend/facade/evaluation_signals.py
deleted file mode 100644
index df8e37b4ecff..000000000000
--- a/products/ai_observability/backend/facade/evaluation_signals.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from collections.abc import Callable
-
-from django.dispatch import Signal
-
-evaluation_output_changed = Signal()
-
-
-def connect_evaluation_output_changed(receiver: Callable[..., object], *, dispatch_uid: str) -> None:
- """``receiver`` gets ``team_id=`` after an evaluation save that writes its output type or output config commits.
- Every full save writes them, so a receiver must be idempotent. A receiver that raises is logged and never fails
- the save."""
- evaluation_output_changed.connect(receiver, dispatch_uid=dispatch_uid)
diff --git a/products/ai_observability/backend/logic/evaluations/pass_filters.py b/products/ai_observability/backend/logic/evaluations/pass_filters.py
index a23dbcc2ca59..1be87154fbd9 100644
--- a/products/ai_observability/backend/logic/evaluations/pass_filters.py
+++ b/products/ai_observability/backend/logic/evaluations/pass_filters.py
@@ -35,13 +35,10 @@ class PassRules:
passes: ast.Expr
-def build_evaluation_pass_filters(
- team_id: int, evaluation_ids: Sequence[str] | None = None
-) -> contracts.EvaluationPassFilters:
- """`evaluation_ids` limits the rules to the runs of those evaluations, for a read that filters on them."""
+def build_evaluation_pass_filters(team_id: int) -> contracts.EvaluationPassFilters:
graded_branches: list[str] = []
passed_branches: list[str] = []
- for evaluation_id, definition in _own_rules(team_id, evaluation_ids):
+ for evaluation_id, definition in _own_rules(team_id, None):
matches_evaluation = f"{_EVALUATION_ID} = {escape_hogql_string(evaluation_id)}"
# An evaluation without a pass rule grades none of its runs, including boolean runs from before it
# switched to sentiment, which the default branch would otherwise grade.
@@ -138,10 +135,8 @@ def _own_rules(
team_id: int, evaluation_ids: Sequence[str] | None
) -> Iterator[tuple[str, EvaluationReportOutcomeDefinition | None]]:
"""Each evaluation whose runs the default rule does not grade, with its rule, or None for a config that no longer
- loads. Soft-deleted evaluations stay in, because their past events still need their own rule. Reads the writer,
- because a receiver of evaluation_output_changed calls this right after the commit, before a replica can catch
- up."""
- evaluations = Evaluation.objects.using("default").filter(team_id=team_id)
+ loads. Soft-deleted evaluations stay in, because their past events still need their own rule."""
+ evaluations = Evaluation.objects.filter(team_id=team_id)
if evaluation_ids is not None:
evaluations = evaluations.filter(id__in=evaluation_ids)
evaluations_with_own_rule = evaluations.filter(
diff --git a/products/ai_observability/backend/models/evaluations.py b/products/ai_observability/backend/models/evaluations.py
index 0cd4e409c8c0..6370f0ced475 100644
--- a/products/ai_observability/backend/models/evaluations.py
+++ b/products/ai_observability/backend/models/evaluations.py
@@ -12,7 +12,6 @@ from rest_framework.exceptions import ValidationError
from posthog.models.activity_logging.model_activity import ModelActivityMixin
from posthog.models.utils import UUIDTModel
-from ..facade.evaluation_signals import evaluation_output_changed
from .evaluation_configs import (
EVALUATION_CONFIG_MODELS,
EvaluationType,
@@ -147,8 +146,6 @@ class Evaluation(ModelActivityMixin, UUIDTModel):
created_by = models.ForeignKey("posthog.User", on_delete=models.SET_NULL, null=True, blank=True, related_name="+")
deleted = models.BooleanField(default=False)
- _writes_output_on_save = False
-
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
# Snapshot of the fields as last loaded / saved — used by _coerce_status_and_enabled to know which
@@ -305,11 +302,6 @@ class Evaluation(ModelActivityMixin, UUIDTModel):
raise ValidationError({"conditions": compile_error})
self.conditions = compiled_conditions
- # Any save that writes an output field notifies, changed or not. Comparing with the stored row instead would
- # miss a concurrent save that writes a stale output after another save changed it.
- self._writes_output_on_save = update_fields is None or bool(
- {"output_type", "output_config"} & set(update_fields)
- )
result = super().save(*args, **kwargs)
# Refresh the baseline so the next save cycle compares against the post-save state.
self._initial_enabled = self.enabled
@@ -336,18 +328,4 @@ def evaluation_saved(sender, instance, created, **kwargs):
# on_commit fires synchronously, preserving prior behavior.
team_id = instance.team_id
evaluation_id = str(instance.id)
- # Registered before the worker reload, because an on_commit callback that raises stops the ones after it.
- if instance._writes_output_on_save:
- transaction.on_commit(lambda: _notify_evaluation_output_changed(team_id))
transaction.on_commit(lambda: reload_evaluations_on_workers(team_id=team_id, evaluation_ids=[evaluation_id]))
-
-
-def _notify_evaluation_output_changed(team_id: int) -> None:
- for receiver_fn, response in evaluation_output_changed.send_robust(sender=Evaluation, team_id=team_id):
- if isinstance(response, Exception):
- logger.error(
- "evaluation_output_changed_receiver_failed",
- receiver=getattr(receiver_fn, "__qualname__", repr(receiver_fn)),
- team_id=team_id,
- exc_info=response,
- )
diff --git a/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py b/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py
index f520f35d15cd..fe2cff12bef4 100644
--- a/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py
+++ b/products/ai_observability/backend/tests/logic/evaluations/test_pass_filters.py
@@ -7,9 +7,11 @@ from parameterized import parameterized
from posthog.hogql.parser import parse_expr
from posthog.hogql.query import execute_hogql_query
-from products.ai_observability.backend.facade.api import get_evaluation_pass_filters
-from products.ai_observability.backend.facade.evaluation_signals import evaluation_output_changed
-from products.ai_observability.backend.logic.evaluations.pass_filters import PassRules, build_pass_rules
+from products.ai_observability.backend.logic.evaluations.pass_filters import (
+ PassRules,
+ build_evaluation_pass_filters,
+ build_pass_rules,
+)
from products.ai_observability.backend.models.evaluations import Evaluation
CATEGORY_OPTIONS = [{"key": "ok", "label": "Ok"}, {"key": "bad", "label": "Bad"}]
@@ -23,7 +25,7 @@ def _evaluation(team, output_type: str = "boolean", output_config: dict | None =
def _per_evaluation_rules(team_id: int, _evaluation_ids: list[str]) -> PassRules:
- filters = get_evaluation_pass_filters(team_id)
+ filters = build_evaluation_pass_filters(team_id)
return PassRules(graded=parse_expr(filters.graded), passes=parse_expr(filters.passed))
@@ -153,45 +155,3 @@ class TestEvaluationPassFilters(ClickhouseTestMixin, BaseTest):
assert graded_passed_failed == {
case: (graded, passed, graded and not passed) for case, (_, _, (graded, passed)) in cases.items()
}
-
-
-def _save(evaluation: Evaluation, **changes: object) -> None:
- for field, value in changes.items():
- setattr(evaluation, field, value)
- evaluation.save()
-
-
-def _save_name_only(evaluation: Evaluation) -> None:
- evaluation.output_config = {"true_is_failure": True}
- evaluation.name = "renamed"
- evaluation.save(update_fields=["name"])
-
-
-class TestEvaluationOutputChangedSignal(BaseTest):
- @parameterized.expand(
- [
- ("created", None, True),
- ("output config changed", lambda e: _save(e, output_config={"true_is_failure": True}), True),
- ("full save without an output change", lambda e: _save(e, name="renamed"), True),
- ("status update", lambda e: e.set_status("paused", None), False),
- ("output edited but only the name saved", _save_name_only, False),
- ]
- )
- def test_notifies_on_saves_that_write_the_output(
- self, _name: str, change: Callable[[Evaluation], None] | None, should_notify: bool
- ) -> None:
- evaluation = None if change is None else _evaluation(self.team)
- notified: list[int] = []
- evaluation_output_changed.connect(
- lambda **kwargs: notified.append(kwargs["team_id"]), weak=False, dispatch_uid="test-output-changed"
- )
- self.addCleanup(evaluation_output_changed.disconnect, dispatch_uid="test-output-changed")
-
- with self.captureOnCommitCallbacks(execute=True):
- if change is None:
- _evaluation(self.team)
- else:
- assert evaluation is not None
- change(evaluation)
-
- assert notified == ([self.team.id] if should_notify else [])