Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): skip hog recompile on status-only evaluation saves (#124417)

posthog[bot]7821e35c2fb7
diff --git a/products/ai_observability/backend/models/evaluations.py b/products/ai_observability/backend/models/evaluations.py
index 6370f0ced475..a6a1c6a76886 100644
--- a/products/ai_observability/backend/models/evaluations.py
+++ b/products/ai_observability/backend/models/evaluations.py
@@ -1,3 +1,5 @@
+from typing import Any
+
from django.db import models, transaction
from django.db.models.signals import post_save
from django.dispatch import receiver
@@ -238,6 +240,18 @@ class Evaluation(ModelActivityMixin, UUIDTModel):
return True
return [c.get("properties", []) for c in self.conditions] != [c.get("properties", []) for c in stored]
+ def _stored_hog_bytecode_if_source_unchanged(self) -> list[Any] | None:
+ if self._state.adding:
+ return None
+ stored = (
+ Evaluation.objects.filter(pk=self.pk, team_id=self.team_id)
+ .values_list("evaluation_config", flat=True)
+ .first()
+ )
+ if not stored or stored.get("source") != self.evaluation_config["source"]:
+ return None
+ return stored.get("bytecode") or None
+
def save(self, *args, **kwargs):
from products.cdp.backend.filters import compile_filters_bytecode
@@ -272,13 +286,26 @@ class Evaluation(ModelActivityMixin, UUIDTModel):
except ValueError as e:
raise ValidationError({"target_config": str(e)})
+ update_fields = kwargs.get("update_fields")
+
# Compile Hog source to bytecode
- if self.evaluation_type == EvaluationType.HOG and self.evaluation_config.get("source"):
+ writes_evaluation_config = update_fields is None or "evaluation_config" in update_fields
+ if (
+ writes_evaluation_config
+ and self.evaluation_type == EvaluationType.HOG
+ and self.evaluation_config.get("source")
+ ):
try:
bytecode = compile_ai_observability_hog(self.evaluation_config["source"], "destination")
- self.evaluation_config["bytecode"] = bytecode
except Exception as e:
- raise ValidationError({"evaluation_config": f"Failed to compile Hog code: {e}"})
+ # A newer compiler can reject source that an older one accepted. Keep the stored bytecode so the
+ # owner can still pause or delete the evaluation, but never let broken code start to run again.
+ activating = self.status == EvaluationStatus.ACTIVE and self._initial_status != self.status
+ stored_bytecode = None if activating else self._stored_hog_bytecode_if_source_unchanged()
+ if stored_bytecode is None:
+ raise ValidationError({"evaluation_config": f"Failed to compile Hog code: {e}"})
+ bytecode = stored_bytecode
+ self.evaluation_config["bytecode"] = bytecode
# Compile bytecode for each condition
compiled_conditions = []
@@ -296,7 +323,6 @@ class Evaluation(ModelActivityMixin, UUIDTModel):
# The scheduler never matches a condition that has no bytecode. Reject the save when the caller
# changes the filters. Other saves skip this check, so pause, delete and status changes work.
- update_fields = kwargs.get("update_fields")
writes_conditions = update_fields is None or "conditions" in update_fields
if compile_error and writes_conditions and self._condition_filters_changed():
raise ValidationError({"conditions": compile_error})
diff --git a/products/ai_observability/backend/models/test/test_evaluations.py b/products/ai_observability/backend/models/test/test_evaluations.py
index dc40d313660d..a6c997e17f14 100644
--- a/products/ai_observability/backend/models/test/test_evaluations.py
+++ b/products/ai_observability/backend/models/test/test_evaluations.py
@@ -6,6 +6,8 @@ from rest_framework.exceptions import ValidationError
from products.ai_observability.backend.models.evaluations import Evaluation, EvaluationStatus, EvaluationStatusReason
+REJECTED_HOG_SOURCE = "let output := output\nreturn true"
+
class TestEvaluationModel(BaseTest):
def test_compiles_bytecode_for_conditions_with_properties(self):
@@ -41,18 +43,31 @@ class TestEvaluationModel(BaseTest):
@parameterized.expand(
[
- ("set_status", None, "status", EvaluationStatus.ERROR),
- ("pause", {"enabled": False}, "enabled", False),
- ("delete", {"deleted": True}, "deleted", True),
+ (f"{broken}_{action}", broken, updates, field, expected)
+ for broken in ("condition", "hog_source")
+ for action, updates, field, expected in [
+ ("set_status", None, "status", EvaluationStatus.ERROR),
+ ("pause", {"enabled": False}, "enabled", False),
+ ("delete", {"deleted": True}, "deleted", True),
+ ]
]
)
- def test_save_without_filter_change_succeeds_when_condition_no_longer_compiles(
- self, _name, updates, field, expected
+ def test_save_without_code_change_succeeds_when_stored_code_no_longer_compiles(
+ self, _name, broken, updates, field, expected
):
evaluation = self._create_hog_evaluation()
-
- with patch("products.cdp.backend.filters.compile_filters_bytecode") as mock_compile:
- mock_compile.return_value = {"bytecode": None, "bytecode_error": "Invalid property filter"}
+ stored_bytecode = evaluation.evaluation_config["bytecode"]
+ if broken == "hog_source":
+ self._store_rejected_hog_source(evaluation)
+
+ condition_result = (
+ {"bytecode": None, "bytecode_error": "Invalid property filter"}
+ if broken == "condition"
+ else {"bytecode": ["_H", 1], "bytecode_error": None}
+ )
+ with patch(
+ "products.cdp.backend.filters.compile_filters_bytecode", return_value=condition_result
+ ) as mock_compile:
if updates is None:
evaluation.set_status(EvaluationStatus.ERROR, EvaluationStatusReason.HOG_ERROR)
else:
@@ -63,6 +78,7 @@ class TestEvaluationModel(BaseTest):
evaluation.refresh_from_db()
self.assertEqual(getattr(evaluation, field), expected)
+ self.assertEqual(evaluation.evaluation_config["bytecode"], stored_bytecode)
def test_full_save_rejects_filter_edited_in_place_after_status_only_save(self):
evaluation = self._create_hog_evaluation()
@@ -75,6 +91,36 @@ class TestEvaluationModel(BaseTest):
evaluation.refresh_from_db()
self.assertEqual(evaluation.conditions[0]["properties"], [])
+ def test_full_save_keeps_stored_bytecode_when_resubmitted_hog_source_no_longer_compiles(self):
+ evaluation = self._create_hog_evaluation()
+ stored_bytecode = evaluation.evaluation_config["bytecode"]
+ self._store_rejected_hog_source(evaluation)
+
+ evaluation.evaluation_config = {"source": REJECTED_HOG_SOURCE}
+ evaluation.save()
+
+ evaluation.refresh_from_db()
+ self.assertEqual(evaluation.evaluation_config["bytecode"], stored_bytecode)
+
+ @parameterized.expand(
+ [
+ ("edit_source_in_place", lambda e: e.evaluation_config.update(source="let x := x\nreturn false")),
+ ("reenable", lambda e: setattr(e, "enabled", True)),
+ ]
+ )
+ def test_full_save_rejects_stored_hog_source_that_no_longer_compiles(self, _name, change):
+ evaluation = self._create_hog_evaluation()
+ self._store_rejected_hog_source(evaluation)
+ evaluation.set_status(EvaluationStatus.ERROR, EvaluationStatusReason.HOG_ERROR)
+
+ change(evaluation)
+ with self.assertRaises(ValidationError):
+ evaluation.save()
+
+ evaluation.refresh_from_db()
+ self.assertEqual(evaluation.status, EvaluationStatus.ERROR)
+ self.assertEqual(evaluation.evaluation_config["source"], REJECTED_HOG_SOURCE)
+
def _create_hog_evaluation(self) -> Evaluation:
return Evaluation.objects.create(
team=self.team,
@@ -86,6 +132,12 @@ class TestEvaluationModel(BaseTest):
conditions=[{"id": "cond-1", "rollout_percentage": 100, "properties": []}],
)
+ def _store_rejected_hog_source(self, evaluation: Evaluation) -> None:
+ Evaluation.objects.filter(id=evaluation.id).update(
+ evaluation_config={**evaluation.evaluation_config, "source": REJECTED_HOG_SOURCE}
+ )
+ evaluation.refresh_from_db()
+
def test_handles_empty_properties_list(self):
"""
Conditions with empty properties should still be saved correctly
@@ -244,13 +296,19 @@ class TestEvaluationModel(BaseTest):
self.assertFalse(result["verdict"])
self.assertIsNone(result["error"])
- def test_hog_evaluation_invalid_source_raises_validation_error(self):
+ @parameterized.expand(
+ [
+ ("syntax_error", "this is not valid hog {{{{"),
+ ("local_read_in_own_declaration", REJECTED_HOG_SOURCE),
+ ]
+ )
+ def test_hog_evaluation_invalid_source_raises_validation_error(self, _name, source):
with self.assertRaises(ValidationError):
Evaluation.objects.create(
team=self.team,
name="Bad Hog Eval",
evaluation_type="hog",
- evaluation_config={"source": "this is not valid hog {{{{"},
+ evaluation_config={"source": source},
output_type="boolean",
output_config={},
enabled=True,