Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.4 KB
Raw Blame
import copy
import hashlib
from typing import TYPE_CHECKING, Any

from posthog.models.activity_logging.activity_log import (
    ActivityLog,
    Change,
    ChangeAction,
    Detail,
    LogActivityEntry,
    bulk_log_activity,
    changes_between,
    log_activity,
)
from posthog.models.activity_logging.utils import activity_storage
from posthog.models.signals import model_activity_signal, mutable_receiver

from products.ai_observability.backend.models.evaluation_directories import EvaluationDirectory
from products.ai_observability.backend.models.evaluations import Evaluation
from products.ai_observability.backend.models.llm_prompt import LLMPromptLabel

if TYPE_CHECKING:
    from posthog.models import Team, User

ACTIVITY_LOG_ITEM_ID_MAX_LENGTH: int = ActivityLog._meta.get_field("item_id").max_length or 72


def prompt_activity_item_id(prompt_name: str) -> str:
    """Deterministic activity-log key for a prompt, at most 72 chars (item_id is varchar(72)).

    Short names are used as-is. Longer names become a readable prefix plus a sha256 digest
    of the full name (128 bits kept), so a user cannot craft a second prompt name whose key
    collides with another prompt's. '#' cannot appear in a prompt name, so hashed keys can't
    collide with literal ones. The frontend does not mirror this: the History tab reads the
    key from the API (activity_item_id on the prompt serializer).
    """
    if len(prompt_name) <= ACTIVITY_LOG_ITEM_ID_MAX_LENGTH:
        return prompt_name
    digest = hashlib.sha256(prompt_name.encode("utf-8")).hexdigest()[:32]
    prefix = prompt_name[: ACTIVITY_LOG_ITEM_ID_MAX_LENGTH - 33]
    return f"{prefix}#{digest}"


def log_llm_prompt_activity(
    *,
    team: "Team",
    user: "User | None",
    prompt_name: str,
    activity: str,
    changes: list[Change] | None = None,
) -> None:
    """Audit-log a prompt lifecycle event (create/publish/archive/duplicate).

    Uses the same item_id as label activity so the prompt History tab gets both
    streams in one query. Called explicitly from the prompt service functions —
    LLMPrompt has no ModelActivityMixin because each row is a version, and a
    per-row "created" entry per publish would say nothing about the lifecycle.
    Impersonation comes from the request-scoped activity storage, same as the
    mixin path.
    """
    log_activity(
        organization_id=team.organization_id,
        team_id=team.id,
        user=user,
        was_impersonated=activity_storage.get_was_impersonated(),
        item_id=prompt_activity_item_id(prompt_name),
        scope="LLMPrompt",
        activity=activity,
        detail=Detail(name=prompt_name, changes=changes),
    )


# Lives here, not in api/evaluations.py, so it can wire at AppConfig.ready() without dragging the
# evaluations viewset (which pulls scipy / google.genai / the ai_observability Temporal worker) onto
# the django.setup() path. Evaluations are mutated outside web requests, so the audit log must
# connect in every process.

_COMPILED_KEYS = ("bytecode", "bytecode_error")


def _strip_compiled_from_conditions(conditions: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
    if not conditions:
        return []
    return [{k: v for k, v in cond.items() if k not in _COMPILED_KEYS} for cond in conditions]


def _strip_compiled_from_eval_config(config: dict[str, Any] | None) -> dict[str, Any]:
    if not config:
        return {}
    return {k: v for k, v in config.items() if k not in _COMPILED_KEYS}


def _evaluation_directory_change(before_update: Evaluation | None, after_update: Evaluation | None) -> Change | None:
    if before_update is None or after_update is None:
        return None

    before_directory_id = str(before_update.directory_id) if before_update.directory_id else None
    after_directory_id = str(after_update.directory_id) if after_update.directory_id else None
    if before_directory_id == after_directory_id:
        return None

    action: ChangeAction = "changed"
    if before_directory_id is None:
        action = "created"
    elif after_directory_id is None:
        action = "deleted"

    return Change(
        type="Evaluation",
        action=action,
        field="directory",
        before=before_directory_id,
        after=after_directory_id,
    )


@mutable_receiver(model_activity_signal, sender=Evaluation)
def handle_evaluation_change(
    sender, scope, before_update, after_update, activity, user, was_impersonated=False, **kwargs
):
    # `save()` re-derives `conditions[*].bytecode` and `evaluation_config.bytecode` on every write.
    # Strip them on shallow copies so the diff reflects user intent, not the compiler — mutating
    # `after_update` directly would also mutate the live instance DRF serialises for the response.
    before_log = copy.copy(before_update) if before_update is not None else None
    after_log = copy.copy(after_update) if after_update is not None else None
    for snapshot in (before_log, after_log):
        if snapshot is not None:
            snapshot.conditions = _strip_compiled_from_conditions(snapshot.conditions)
            snapshot.evaluation_config = _strip_compiled_from_eval_config(snapshot.evaluation_config)

    if before_update and after_update:
        before_deleted = getattr(before_update, "deleted", None)
        after_deleted = getattr(after_update, "deleted", None)
        if before_deleted is not None and after_deleted is not None and before_deleted != after_deleted:
            activity = "restored" if after_deleted is False else "deleted"

    changes = changes_between(scope, before_log, after_log)
    directory_change = _evaluation_directory_change(before_update, after_update)
    if directory_change is not None:
        changes.append(directory_change)

    log_activity(
        organization_id=after_update.team.organization_id,
        team_id=after_update.team_id,
        user=user,
        was_impersonated=was_impersonated,
        item_id=after_update.id,
        scope=scope,
        activity=activity,
        detail=Detail(
            changes=changes,
            name=after_update.name,
        ),
    )


def log_evaluations_moved_to_top_level(evaluations: list[Evaluation]) -> None:
    user = activity_storage.get_user()
    was_impersonated = activity_storage.get_was_impersonated()
    bulk_log_activity(
        [
            LogActivityEntry(
                organization_id=None,
                team_id=evaluation.team_id,
                user=user,
                was_impersonated=was_impersonated,
                item_id=str(evaluation.id),
                scope="Evaluation",
                activity="updated",
                detail=Detail(
                    name=evaluation.name,
                    changes=[
                        Change(
                            type="Evaluation",
                            action="deleted",
                            field="directory",
                            before=str(evaluation.directory_id),
                            after=None,
                        )
                    ],
                ),
            )
            for evaluation in evaluations
        ]
    )


@mutable_receiver(model_activity_signal, sender=EvaluationDirectory)
def handle_evaluation_directory_change(
    sender, scope, before_update, after_update, activity, user, was_impersonated=False, **kwargs
):
    directory = after_update or before_update
    if directory is None:
        return

    log_activity(
        organization_id=directory.team.organization_id,
        team_id=directory.team_id,
        user=user,
        was_impersonated=was_impersonated,
        item_id=directory.id,
        scope=scope,
        activity=activity,
        detail=Detail(
            name=directory.name,
            changes=changes_between(scope, before_update, after_update),
        ),
    )


@mutable_receiver(model_activity_signal, sender=LLMPromptLabel)
def handle_llm_prompt_label_change(
    sender, scope, before_update, after_update, activity, user, was_impersonated=False, **kwargs
):
    # A label move changes which prompt version production traffic receives, with no code
    # deploy — the from/to version numbers are the audit trail, so build the change by hand
    # instead of diffing the raw FK (which would only show opaque version-row UUIDs).
    instance = after_update if after_update is not None else before_update
    if instance is None:
        return

    before_version = before_update.prompt.version if before_update is not None else None
    after_version = after_update.prompt.version if after_update is not None else None
    action: ChangeAction = "changed"
    if activity == "created":
        action = "created"
    elif activity == "deleted":
        action = "deleted"

    log_activity(
        organization_id=instance.team.organization_id,
        team_id=instance.team_id,
        user=user,
        was_impersonated=was_impersonated,
        # The prompt name, not the label row id: the History tab lists all label activity
        # for one prompt, and label rows are recreated on delete + re-add.
        item_id=prompt_activity_item_id(instance.prompt_name),
        scope="LLMPromptLabel",
        activity=activity,
        detail=Detail(
            name=f"{instance.prompt_name}: {instance.name}",
            changes=[
                Change(
                    type="LLMPromptLabel",
                    action=action,
                    field="version",
                    before=before_version,
                    after=after_version,
                )
            ],
        ),
    )