:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): skip hog recompile on status-only evaluation saves (#124417)
posthog[bot]7821e35c2fb7
diff --git a/products/ai_observability/backend/models/evaluations.py b/products/ai_observability/backend/models/evaluations.pyindex 6370f0ced475..a6a1c6a76886 100644--- a/products/ai_observability/backend/models/evaluations.py+++ b/products/ai_observability/backend/models/evaluations.py@@ -1,3 +1,5 @@+from typing import Any+from django.db import models, transactionfrom django.db.models.signals import post_savefrom django.dispatch import receiver@@ -238,6 +240,18 @@ class Evaluation(ModelActivityMixin, UUIDTModel):return Truereturn [c.get("properties", []) for c in self.conditions] != [c.get("properties", []) for c in stored]+ def _stored_hog_bytecode_if_source_unchanged(self) -> list[Any] | None:+ if self._state.adding:+ return None+ stored = (+ Evaluation.objects.filter(pk=self.pk, team_id=self.team_id)+ .values_list("evaluation_config", flat=True)+ .first()+ )+ if not stored or stored.get("source") != self.evaluation_config["source"]:+ return None+ return stored.get("bytecode") or None+def save(self, *args, **kwargs):from products.cdp.backend.filters import compile_filters_bytecode@@ -272,13 +286,26 @@ class Evaluation(ModelActivityMixin, UUIDTModel):except ValueError as e:raise ValidationError({"target_config": str(e)})+ update_fields = kwargs.get("update_fields")+# Compile Hog source to bytecode- if self.evaluation_type == EvaluationType.HOG and self.evaluation_config.get("source"):+ writes_evaluation_config = update_fields is None or "evaluation_config" in update_fields+ if (+ writes_evaluation_config+ and self.evaluation_type == EvaluationType.HOG+ and self.evaluation_config.get("source")+ ):try:bytecode = compile_ai_observability_hog(self.evaluation_config["source"], "destination")- self.evaluation_config["bytecode"] = bytecodeexcept Exception as e:- raise ValidationError({"evaluation_config": f"Failed to compile Hog code: {e}"})+ # A newer compiler can reject source that an older one accepted. Keep the stored bytecode so the+ # owner can still pause or delete the evaluation, but never let broken code start to run again.+ activating = self.status == EvaluationStatus.ACTIVE and self._initial_status != self.status+ stored_bytecode = None if activating else self._stored_hog_bytecode_if_source_unchanged()+ if stored_bytecode is None:+ raise ValidationError({"evaluation_config": f"Failed to compile Hog code: {e}"})+ bytecode = stored_bytecode+ self.evaluation_config["bytecode"] = bytecode# Compile bytecode for each conditioncompiled_conditions = []@@ -296,7 +323,6 @@ class Evaluation(ModelActivityMixin, UUIDTModel):# The scheduler never matches a condition that has no bytecode. Reject the save when the caller# changes the filters. Other saves skip this check, so pause, delete and status changes work.- update_fields = kwargs.get("update_fields")writes_conditions = update_fields is None or "conditions" in update_fieldsif compile_error and writes_conditions and self._condition_filters_changed():raise ValidationError({"conditions": compile_error})diff --git a/products/ai_observability/backend/models/test/test_evaluations.py b/products/ai_observability/backend/models/test/test_evaluations.pyindex dc40d313660d..a6c997e17f14 100644--- a/products/ai_observability/backend/models/test/test_evaluations.py+++ b/products/ai_observability/backend/models/test/test_evaluations.py@@ -6,6 +6,8 @@ from rest_framework.exceptions import ValidationErrorfrom products.ai_observability.backend.models.evaluations import Evaluation, EvaluationStatus, EvaluationStatusReason+REJECTED_HOG_SOURCE = "let output := output\nreturn true"+class TestEvaluationModel(BaseTest):def test_compiles_bytecode_for_conditions_with_properties(self):@@ -41,18 +43,31 @@ class TestEvaluationModel(BaseTest):@parameterized.expand([- ("set_status", None, "status", EvaluationStatus.ERROR),- ("pause", {"enabled": False}, "enabled", False),- ("delete", {"deleted": True}, "deleted", True),+ (f"{broken}_{action}", broken, updates, field, expected)+ for broken in ("condition", "hog_source")+ for action, updates, field, expected in [+ ("set_status", None, "status", EvaluationStatus.ERROR),+ ("pause", {"enabled": False}, "enabled", False),+ ("delete", {"deleted": True}, "deleted", True),+ ]])- def test_save_without_filter_change_succeeds_when_condition_no_longer_compiles(- self, _name, updates, field, expected+ def test_save_without_code_change_succeeds_when_stored_code_no_longer_compiles(+ self, _name, broken, updates, field, expected):evaluation = self._create_hog_evaluation()-- with patch("products.cdp.backend.filters.compile_filters_bytecode") as mock_compile:- mock_compile.return_value = {"bytecode": None, "bytecode_error": "Invalid property filter"}+ stored_bytecode = evaluation.evaluation_config["bytecode"]+ if broken == "hog_source":+ self._store_rejected_hog_source(evaluation)++ condition_result = (+ {"bytecode": None, "bytecode_error": "Invalid property filter"}+ if broken == "condition"+ else {"bytecode": ["_H", 1], "bytecode_error": None}+ )+ with patch(+ "products.cdp.backend.filters.compile_filters_bytecode", return_value=condition_result+ ) as mock_compile:if updates is None:evaluation.set_status(EvaluationStatus.ERROR, EvaluationStatusReason.HOG_ERROR)else:@@ -63,6 +78,7 @@ class TestEvaluationModel(BaseTest):evaluation.refresh_from_db()self.assertEqual(getattr(evaluation, field), expected)+ self.assertEqual(evaluation.evaluation_config["bytecode"], stored_bytecode)def test_full_save_rejects_filter_edited_in_place_after_status_only_save(self):evaluation = self._create_hog_evaluation()@@ -75,6 +91,36 @@ class TestEvaluationModel(BaseTest):evaluation.refresh_from_db()self.assertEqual(evaluation.conditions[0]["properties"], [])+ def test_full_save_keeps_stored_bytecode_when_resubmitted_hog_source_no_longer_compiles(self):+ evaluation = self._create_hog_evaluation()+ stored_bytecode = evaluation.evaluation_config["bytecode"]+ self._store_rejected_hog_source(evaluation)++ evaluation.evaluation_config = {"source": REJECTED_HOG_SOURCE}+ evaluation.save()++ evaluation.refresh_from_db()+ self.assertEqual(evaluation.evaluation_config["bytecode"], stored_bytecode)++ @parameterized.expand(+ [+ ("edit_source_in_place", lambda e: e.evaluation_config.update(source="let x := x\nreturn false")),+ ("reenable", lambda e: setattr(e, "enabled", True)),+ ]+ )+ def test_full_save_rejects_stored_hog_source_that_no_longer_compiles(self, _name, change):+ evaluation = self._create_hog_evaluation()+ self._store_rejected_hog_source(evaluation)+ evaluation.set_status(EvaluationStatus.ERROR, EvaluationStatusReason.HOG_ERROR)++ change(evaluation)+ with self.assertRaises(ValidationError):+ evaluation.save()++ evaluation.refresh_from_db()+ self.assertEqual(evaluation.status, EvaluationStatus.ERROR)+ self.assertEqual(evaluation.evaluation_config["source"], REJECTED_HOG_SOURCE)+def _create_hog_evaluation(self) -> Evaluation:return Evaluation.objects.create(team=self.team,@@ -86,6 +132,12 @@ class TestEvaluationModel(BaseTest):conditions=[{"id": "cond-1", "rollout_percentage": 100, "properties": []}],)+ def _store_rejected_hog_source(self, evaluation: Evaluation) -> None:+ Evaluation.objects.filter(id=evaluation.id).update(+ evaluation_config={**evaluation.evaluation_config, "source": REJECTED_HOG_SOURCE}+ )+ evaluation.refresh_from_db()+def test_handles_empty_properties_list(self):"""Conditions with empty properties should still be saved correctly@@ -244,13 +296,19 @@ class TestEvaluationModel(BaseTest):self.assertFalse(result["verdict"])self.assertIsNone(result["error"])- def test_hog_evaluation_invalid_source_raises_validation_error(self):+ @parameterized.expand(+ [+ ("syntax_error", "this is not valid hog {{{{"),+ ("local_read_in_own_declaration", REJECTED_HOG_SOURCE),+ ]+ )+ def test_hog_evaluation_invalid_source_raises_validation_error(self, _name, source):with self.assertRaises(ValidationError):Evaluation.objects.create(team=self.team,name="Bad Hog Eval",evaluation_type="hog",- evaluation_config={"source": "this is not valid hog {{{{"},+ evaluation_config={"source": source},output_type="boolean",output_config={},enabled=True,