Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

16.7 KB
Raw Blame
from collections.abc import Collection
from typing import Any, Self

from django.db import models, transaction
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.functional import Promise

import structlog

# DRF's ValidationError (not Django's) so save() raise sites surface as 400s via the DRF exception
# handler — Django's variant falls through to the unhandled-exception path and returns a 500.
from rest_framework.exceptions import ValidationError

from posthog.models.activity_logging.model_activity import ModelActivityMixin
from posthog.models.utils import UUIDTModel

from .evaluation_configs import (
    EVALUATION_CONFIG_MODELS,
    EvaluationType,
    OutputType,
    evaluation_configs_allow_empty,
    evaluation_uses_model_configuration,
    validate_evaluation_configs,
    validate_target_config,
)

logger = structlog.get_logger(__name__)


class EvaluationStatus(models.TextChoices):
    ACTIVE = "active", "Active"
    PAUSED = "paused", "Paused"
    ERROR = "error", "Error"


class EvaluationStatusReason(models.TextChoices):
    PROVIDER_KEY_REQUIRED = "provider_key_required", "No provider API key configured"
    PROVIDER_KEY_DELETED = "provider_key_deleted", "Provider API key was deleted"
    NO_DEFAULT_MODEL = "no_default_model", "No default model available for the selected provider"
    PROVIDER_KEY_INVALID = "provider_key_invalid", "Provider API key is invalid"
    PROVIDER_KEY_PERMISSION_DENIED = "provider_key_permission_denied", "Provider API key lacks model access"
    PROVIDER_KEY_QUOTA_EXCEEDED = "provider_key_quota_exceeded", "Provider API key quota exceeded"
    PROVIDER_KEY_RATE_LIMITED = "provider_key_rate_limited", "Provider API key is rate limited"
    MODEL_NOT_FOUND = "model_not_found", "Model not found"
    MODEL_NOT_SUPPORTED = "model_not_supported", "Model does not support chat completions"
    HOG_ERROR = "hog_error", "Hog evaluation code failed"


def evaluation_status_reason_choices() -> list[tuple[str, str | Promise]]:
    # Callable so growing the enum doesn't generate a no-op migration.
    return list(EvaluationStatusReason.choices)


class EvaluationQuerySet(models.QuerySet):
    def using_provider_keys(self) -> "EvaluationQuerySet":
        return self.filter(evaluation_type=EvaluationType.LLM_JUDGE)


class EvaluationTarget(models.TextChoices):
    GENERATION = "generation", "Generation"
    TRACE = "trace", "Trace"
    SESSION = "session", "Session"


class Evaluation(ModelActivityMixin, UUIDTModel):
    class Meta:
        db_table = "llm_analytics_evaluation"
        ordering = ["-created_at", "id"]
        indexes = [
            models.Index(fields=["team", "-created_at", "id"]),
            models.Index(fields=["team", "enabled"]),
            models.Index(fields=["model_configuration"], name="llm_analyti_model_c_idx"),
            models.Index(
                fields=["team", "directory", "-created_at", "id"],
                name="llma_eval_team_dir_created_idx",
            ),
            # Deleting a directory makes Django run the SET_NULL update with a filter on directory_id only,
            # and the index above cannot serve that filter because it leads with team_id.
            models.Index(
                fields=["directory"],
                name="llma_eval_directory_idx",
                condition=models.Q(directory__isnull=False),
            ),
        ]
        constraints = [
            models.CheckConstraint(
                name="model_config_only_on_llm_judge",
                condition=models.Q(model_configuration__isnull=True)
                | models.Q(evaluation_type=EvaluationType.LLM_JUDGE),
            ),
        ]

    objects = EvaluationQuerySet.as_manager()

    # Core fields
    team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+")
    name = models.CharField(max_length=400)
    description = models.TextField(blank=True, default="")
    directory = models.ForeignKey(
        "ai_observability.EvaluationDirectory",
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
        related_name="evaluations",
        db_index=False,
        db_constraint=False,
    )

    # Lifecycle state. `status` is authoritative; `enabled` is a boolean projection kept in sync by save() for
    # backwards compatibility with existing API / DB callers. When status is ERROR, status_reason must be set.
    enabled = models.BooleanField(default=False)
    status = models.CharField(max_length=20, choices=EvaluationStatus, default=EvaluationStatus.PAUSED)
    status_reason = models.CharField(max_length=50, choices=evaluation_status_reason_choices, null=True, blank=True)
    status_reason_detail = models.TextField(null=True, blank=True)

    evaluation_type = models.CharField(max_length=50, choices=EvaluationType)
    evaluation_config = models.JSONField(default=dict)
    output_type = models.CharField(max_length=50, choices=OutputType)
    output_config = models.JSONField(default=dict)

    conditions = models.JSONField(default=list)

    # What unit the evaluation runs on: a single $ai_generation event, or the whole trace
    # (debounced and pulled from ClickHouse after an aggregation window).
    target = models.CharField(
        max_length=20,
        choices=EvaluationTarget,
        default=EvaluationTarget.GENERATION,
        db_default=EvaluationTarget.GENERATION,
    )
    # Target-specific settings, keyed off `target` (parallel to evaluation_config/output_config).
    # Trace targets carry {window_seconds}; generation targets carry nothing.
    target_config = models.JSONField(default=dict, db_default=models.Value("{}"))

    # Model configuration for the LLM judge
    model_configuration = models.ForeignKey(
        "ai_observability.LLMModelConfiguration",
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
        related_name="evaluations",
        db_index=False,
    )

    # Timestamps
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    created_by = models.ForeignKey("posthog.User", on_delete=models.SET_NULL, null=True, blank=True, related_name="+")
    deleted = models.BooleanField(default=False)

    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
        # Snapshot of the fields as last loaded / saved — used by _coerce_status_and_enabled to know which
        # field the caller actually moved when enabled and status disagree.
        self._initial_enabled = self.enabled
        self._initial_status = self.status

    @classmethod
    def from_db(cls, db: str | None, field_names: Collection[str], values: Collection[Any], **kwargs: Any) -> Self:
        instance = super().from_db(db, field_names, values, **kwargs)
        instance._initial_enabled = instance.enabled
        instance._initial_status = instance.status
        return instance

    def refresh_from_db(self, *args, **kwargs) -> None:
        # Refreshing replaces the in-memory state with DB state — the change-tracking baseline must
        # follow, otherwise a subsequent caller-side edit would be compared against stale initials and
        # the coerce step could misinterpret which field moved.
        super().refresh_from_db(*args, **kwargs)
        self._initial_enabled = self.enabled
        self._initial_status = self.status

    def __str__(self):
        return self.name

    def _coerce_status_and_enabled(self) -> None:
        """Reconcile status with enabled at save time.

        New records: derive status from enabled unless the caller explicitly set status (e.g. ERROR + reason).
        Existing records: whichever field actually changed since load drives the other. This lets existing
        callers keep PATCHing `enabled` without knowing about status, while system transitions that write
        `status` directly take effect even when `enabled` is stale in memory.

        Post-reconciliation invariants are always enforced:
            - ACTIVE  → enabled=True,  status_reason/detail=None
            - PAUSED  → enabled=False, status_reason/detail=None
            - ERROR   → enabled=False, status_reason required (raises otherwise)
        """
        # UUIDTModel assigns pk in __init__, so we can't use `self.pk is None` to detect new rows.
        is_new = self._state.adding
        if is_new:
            if self.status == EvaluationStatus.PAUSED and self.enabled:
                self.status = EvaluationStatus.ACTIVE
            elif self.status == EvaluationStatus.ACTIVE and not self.enabled:
                self.status = EvaluationStatus.PAUSED
        else:
            enabled_changed = self.enabled != self._initial_enabled
            status_changed = self.status != self._initial_status
            if status_changed and not enabled_changed:
                self.enabled = self.status == EvaluationStatus.ACTIVE
            elif enabled_changed and not status_changed:
                self.status = EvaluationStatus.ACTIVE if self.enabled else EvaluationStatus.PAUSED

        if self.status == EvaluationStatus.ACTIVE:
            self.enabled = True
            self.status_reason = None
            self.status_reason_detail = None
        elif self.status == EvaluationStatus.PAUSED:
            self.enabled = False
            self.status_reason = None
            self.status_reason_detail = None
        elif self.status == EvaluationStatus.ERROR:
            self.enabled = False
            if not self.status_reason:
                raise ValidationError({"status_reason": "status_reason is required when status is ERROR"})

    def set_status(
        self,
        status: "EvaluationStatus | str",
        reason: "EvaluationStatusReason | str | None" = None,
        reason_detail: str | None = None,
    ) -> None:
        """Transition helper. Prefer this (or .save()) over .update() so invariants stay enforced.

        Callers using QuerySet.update() bypass save() — they must write the status fields together.
        """
        self.status = EvaluationStatus(status)
        self.status_reason = EvaluationStatusReason(reason) if reason else None
        self.status_reason_detail = reason_detail
        self.save(update_fields=["status", "status_reason", "status_reason_detail", "enabled", "updated_at"])

    def _condition_filters_changed(self) -> bool:
        # Compare against the stored row, not an in-memory snapshot, so in-place edits count as changes.
        if self._state.adding:
            return True
        stored = (
            Evaluation.objects.filter(pk=self.pk, team_id=self.team_id).values_list("conditions", flat=True).first()
        )
        if stored is None:
            return True
        return [c.get("properties", []) for c in self.conditions] != [c.get("properties", []) for c in stored]

    def _stored_hog_bytecode_if_source_unchanged(self) -> list[Any] | None:
        if self._state.adding:
            return None
        stored = (
            Evaluation.objects.filter(pk=self.pk, team_id=self.team_id)
            .values_list("evaluation_config", flat=True)
            .first()
        )
        if not stored or stored.get("source") != self.evaluation_config["source"]:
            return None
        return stored.get("bytecode") or None

    def save(self, *args, **kwargs):
        from products.cdp.backend.filters import compile_filters_bytecode

        from ..hog import compile_ai_observability_hog  # noqa: PLC0415 - keeps Hog compiler off model import path

        # Coerce status and enabled into a consistent pair. status is authoritative, but we accept writes to
        # either field — typically `enabled` from user PATCHes, `status` from system transitions.
        self._coerce_status_and_enabled()

        if not evaluation_uses_model_configuration(self.evaluation_type) and self.model_configuration_id:
            raise ValidationError({"model_configuration": "This evaluation type does not use model configuration."})

        if (self.evaluation_type, self.output_type) not in EVALUATION_CONFIG_MODELS:
            raise ValidationError(f"Unsupported combination: {self.evaluation_type} + {self.output_type}")

        # Validate configs when callers provide them, or when the selected config models can supply every default.
        if (
            evaluation_configs_allow_empty(self.evaluation_type, self.output_type)
            or self.evaluation_config
            or self.output_config
        ):
            try:
                self.evaluation_config, self.output_config = validate_evaluation_configs(
                    self.evaluation_type, self.output_type, self.evaluation_config, self.output_config
                )
            except ValueError as e:
                raise ValidationError(str(e))

        # Validate target config (defaults the trace window when absent, strips it for generation).
        try:
            self.target_config = validate_target_config(self.target, self.target_config)
        except ValueError as e:
            raise ValidationError({"target_config": str(e)})

        update_fields = kwargs.get("update_fields")

        # Compile Hog source to bytecode
        writes_evaluation_config = update_fields is None or "evaluation_config" in update_fields
        if (
            writes_evaluation_config
            and self.evaluation_type == EvaluationType.HOG
            and self.evaluation_config.get("source")
        ):
            try:
                bytecode = compile_ai_observability_hog(self.evaluation_config["source"], "destination")
            except Exception as e:
                # A newer compiler can reject source that an older one accepted. Keep the stored bytecode so the
                # owner can still pause or delete the evaluation, but never let broken code start to run again.
                activating = self.status == EvaluationStatus.ACTIVE and self._initial_status != self.status
                stored_bytecode = None if activating else self._stored_hog_bytecode_if_source_unchanged()
                if stored_bytecode is None:
                    raise ValidationError({"evaluation_config": f"Failed to compile Hog code: {e}"})
                bytecode = stored_bytecode
            self.evaluation_config["bytecode"] = bytecode

        # Compile bytecode for each condition
        compiled_conditions = []
        compile_error: str | None = None
        for index, condition in enumerate(self.conditions):
            compiled_condition = {**condition}
            filters = {"properties": condition.get("properties", [])}
            compiled = compile_filters_bytecode(filters, self.team)
            bytecode_error = compiled.get("bytecode_error")
            if bytecode_error and compile_error is None:
                compile_error = f"Condition set {index + 1} has a filter that evaluations cannot run. {bytecode_error}"
            compiled_condition["bytecode"] = compiled.get("bytecode")
            compiled_condition["bytecode_error"] = bytecode_error
            compiled_conditions.append(compiled_condition)

        # The scheduler never matches a condition that has no bytecode. Reject the save when the caller
        # changes the filters. Other saves skip this check, so pause, delete and status changes work.
        writes_conditions = update_fields is None or "conditions" in update_fields
        if compile_error and writes_conditions and self._condition_filters_changed():
            raise ValidationError({"conditions": compile_error})

        self.conditions = compiled_conditions
        result = super().save(*args, **kwargs)
        # Refresh the baseline so the next save cycle compares against the post-save state.
        self._initial_enabled = self.enabled
        self._initial_status = self.status
        return result


@receiver(post_save, sender=Evaluation)
def evaluation_saved(sender, instance, created, **kwargs):
    from posthog.plugins.plugin_server_api import reload_evaluations_on_workers

    from .evaluation_reports import EvaluationReport

    if instance.deleted:
        EvaluationReport.objects.filter(evaluation_id=instance.id, deleted=False).update(deleted=True, enabled=False)
    elif instance.status == EvaluationStatus.PAUSED:
        # A user pause should persist on the report too. Error states are only filtered from delivery
        # temporarily so the report can resume when the evaluation recovers.
        EvaluationReport.objects.filter(evaluation_id=instance.id, deleted=False, enabled=True).update(enabled=False)

    # Defer publishing to workers until the surrounding transaction commits — otherwise
    # workers can fire before the row is visible, especially now that perform_create wraps
    # the save + EvaluationReport create in transaction.atomic(). In auto-commit mode
    # on_commit fires synchronously, preserving prior behavior.
    team_id = instance.team_id
    evaluation_id = str(instance.id)
    transaction.on_commit(lambda: reload_evaluations_on_workers(team_id=team_id, evaluation_ids=[evaluation_id]))