Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

24.1 KB
Raw Blame
from collections.abc import Iterable
from datetime import timedelta
from enum import StrEnum
from typing import Any, Literal

from django.conf import settings
from django.db.models import Max
from django.utils import timezone

from pydantic import ValidationError

from posthog.schema import ArtifactContentType, InsightVizNode

from posthog.api.search import (
    FEATURE_FLAG_SEARCH_CONFIG,
    LIMIT as SEARCH_LIMIT,
    EntityConfig,
    FacadeSearch,
    search_entities as search_entities_fts,
    search_live_actions,
)
from posthog.models import Team, User
from posthog.sync import database_sync_to_async

from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.alerts.backend.models.alert import AlertConfiguration
from products.cohorts.backend.models.cohort import Cohort
from products.customer_analytics.backend.facade import api as customer_analytics_facade
from products.customer_analytics.backend.facade.api import build_account_deeplink
from products.customer_analytics.backend.facade.contracts import AccountRef
from products.dashboards.backend.models.dashboard import Dashboard
from products.experiments.backend.models.experiment import Experiment
from products.feature_flags.backend.flag_status import (
    FeatureFlagStatus,
    FeatureFlagStatusChecker,
    filter_flags_by_active_param,
)
from products.feature_flags.backend.models.feature_flag import FeatureFlag
from products.notebooks.backend.models import Notebook
from products.product_analytics.backend.facade.models import Insight
from products.surveys.backend.models import Survey
from products.workflows.backend.facade.api import search_workflows

from ee.hogai.artifacts.handlers.base import get_handler_for_content_type
from ee.hogai.context.context import AssistantContextManager


class EntityKind(StrEnum):
    INSIGHTS = "insights"
    DASHBOARDS = "dashboards"
    COHORTS = "cohorts"
    ACTIONS = "actions"
    EXPERIMENTS = "experiments"
    FEATURE_FLAGS = "feature_flags"
    NOTEBOOKS = "notebooks"
    SURVEYS = "surveys"
    ALERTS = "alerts"
    ACCOUNTS = "accounts"
    WORKFLOWS = "workflows"
    ALL = "all"


SEARCH_KIND_TO_DATABASE_ENTITY_TYPE: dict[EntityKind, str] = {
    EntityKind.INSIGHTS: "insight",
    EntityKind.DASHBOARDS: "dashboard",
    EntityKind.EXPERIMENTS: "experiment",
    EntityKind.FEATURE_FLAGS: "feature_flag",
    EntityKind.NOTEBOOKS: "notebook",
    EntityKind.ACTIONS: "action",
    EntityKind.COHORTS: "cohort",
    EntityKind.SURVEYS: "survey",
    EntityKind.ALERTS: "alert_configuration",
    EntityKind.ACCOUNTS: "account",
    EntityKind.WORKFLOWS: "hog_flow",
}

ENTITY_MAP: dict[str, EntityConfig] = {
    "insight": {
        "klass": Insight,
        "search_fields": {"name": "A", "description": "C", "query_metadata": "B"},
        "extra_fields": ["name", "description", "query_metadata", "query"],
        "filters": {"deleted": False, "saved": True},
    },
    "dashboard": {
        "klass": Dashboard,
        "search_fields": {"name": "A", "description": "C"},
        "extra_fields": ["name", "description"],
        "filters": {"deleted": False},
    },
    "experiment": {
        "klass": Experiment,
        "search_fields": {"name": "A", "description": "C"},
        "extra_fields": ["name", "description"],
        "filters": {"deleted": False},
    },
    "feature_flag": FEATURE_FLAG_SEARCH_CONFIG,
    "cohort": {
        "klass": Cohort,
        "search_fields": {"name": "A", "description": "C", "filters": "B"},
        "extra_fields": ["name", "description", "filters"],
        "filters": {"deleted": False},
    },
    "survey": {
        "klass": Survey,
        "search_fields": {"name": "A", "description": "C"},
        "extra_fields": ["name", "description"],
        "filters": {"archived": False},
    },
    "alert_configuration": {
        "klass": AlertConfiguration,
        "search_fields": {"name": "A"},
        "extra_fields": ["name", "enabled", "state", "calculation_interval"],
        "filters": {},
    },
    "notebook": {
        "klass": Notebook,
        "search_fields": {"title": "A", "text_content": "C"},
        "extra_fields": ["title", "text_content"],
        "filters": {"deleted": False},
    },
}
"""
Map of entity names to their class, search_fields and extra_fields.

The value in search_fields corresponds to the PostgreSQL weighting i.e. A, B, C or D.
"""


# Actions are searched through the actions facade rather than unioned by model class, so they are not in ENTITY_MAP.
ACTION_ENTITY = "action"
FACADE_SEARCHES: dict[str, FacadeSearch] = {ACTION_ENTITY: search_live_actions}
_UNION_ENTITY_NAMES = [*ENTITY_MAP]
# Actions keep their place between feature flags and cohorts in the counts.
COUNTED_ENTITIES = [
    *_UNION_ENTITY_NAMES[: _UNION_ENTITY_NAMES.index("cohort")],
    ACTION_ENTITY,
    *_UNION_ENTITY_NAMES[_UNION_ENTITY_NAMES.index("cohort") :],
]

EXCLUDE_FROM_DISPLAY: dict[str, set[str]] = {
    "insight": {"query", "query_metadata"},
    "notebook": {"title"},
}
"""Fields to exclude from display output per entity type (they're still used for search ranking)."""

MAX_FIELD_LENGTH = 1000
"""Maximum length for field values in search result display to avoid bloating context."""

INSIGHTS_CUTOFF_DAYS = 180
"""Only include insights viewed within this many days."""


class EntitySearchContext:
    """Context manager for searching/listing and formatting Django models."""

    def __init__(self, team: Team, user: User, context_manager: AssistantContextManager):
        self._team = team
        self._user = user
        self._context_manager = context_manager

    @property
    def user_access_control(self) -> UserAccessControl:
        # The `search_entities` function uses this field for access control.
        return UserAccessControl(user=self._user, team=self._team, organization_id=self._team.organization.id)

    async def search_entities(
        self,
        entity_types: set[str] | Literal["all"],
        query: str,
    ) -> tuple[list[dict], dict[str, int | None]]:
        """
        Search for entities by query and entity types.

        Args:
            entity_types: Set of entity type strings to search or "all" to search all entities
            query: Search query string

        Returns:
            Tuple of (results list, counts dict)
        """
        if entity_types == "all":
            entity_types = {*ENTITY_MAP, ACTION_ENTITY, "hog_flow"}

        results: list[dict] = []
        counts: dict[str, int | None] = {}
        workflow_requested = "hog_flow" in entity_types
        workflow_results: list[dict[str, Any]] = []

        if "account" in entity_types:
            # Account uses a fail-closed manager and is not in ENTITY_MAP, so it can't go through the shared FTS path
            entity_types = entity_types - {"account"}
            account_results, account_count = await self._search_accounts(query)
            results.extend(account_results)
            counts["account"] = account_count

        if workflow_requested:
            entity_types = entity_types - {"hog_flow"}
            workflow_results, workflow_count = await database_sync_to_async(search_workflows, thread_sensitive=False)(
                project_id=self._team.project_id,
                query=query,
                access_control=self.user_access_control,
                limit=SEARCH_LIMIT,
            )
            counts["hog_flow"] = workflow_count

        fts_results: list[dict[str, Any]] = []
        if entity_types:
            fts_results, fts_counts, _ = await database_sync_to_async(search_entities_fts, thread_sensitive=False)(
                entity_types,
                query,
                self._team.project_id,
                self,  # type: ignore
                ENTITY_MAP,
                facade_searches=FACADE_SEARCHES,
                counted_entities=COUNTED_ENTITIES,
            )
            assert fts_counts is not None
            counts.update(fts_counts)

        if workflow_requested:
            database_results = [*workflow_results, *fts_results]
            if query:
                database_results.sort(key=lambda result: result.get("rank", 0), reverse=True)
            else:
                database_results.sort(
                    key=lambda result: (
                        result["type"],
                        result.get("extra_fields", {}).get("name") or result.get("extra_fields", {}).get("title") or "",
                    )
                )
            results.extend(database_results[:SEARCH_LIMIT])
        else:
            results.extend(fts_results)

        return results, counts

    async def list_entities(
        self,
        entity_type: str,
        limit: int = 100,
        offset: int = 0,
    ) -> tuple[list[dict[str, Any]], int]:
        """
        List entities with pagination support, sorted by updated_at DESC.

        Args:
            entity_type: Type of entity to list (insight, dashboard, artifact, etc.)
            limit: Number of entities to return
            offset: Number of entities to skip

        Returns:
            Tuple of (entities list, total count)
        """
        all_entities: list[dict[str, Any]] = []
        total_count: int = 0

        if entity_type == "artifact":
            (
                artifacts,
                total_count,
            ) = await self._context_manager.artifacts.aget_conversation_artifacts(limit, offset)

            # Convert artifacts to the same format as database entities
            for artifact in artifacts:
                try:
                    content = artifact.content
                    handler = get_handler_for_content_type(ArtifactContentType(content.content_type))
                    extra_fields = handler.get_metadata(content) if handler else {}
                    all_entities.append(
                        {
                            "type": "artifact",
                            "result_id": artifact.artifact_id,
                            "extra_fields": extra_fields,
                        }
                    )
                except ValidationError:
                    # Skip artifacts that can't be parsed
                    continue
        elif entity_type == "insight":
            # Use specialized queryset filtered by recent view time
            return await self._list_insights(limit, offset)
        elif entity_type == "account":
            # Account uses a fail-closed manager, so it can't go through the shared FTS path
            return await self._list_accounts(limit, offset)
        elif entity_type == "hog_flow":
            return await database_sync_to_async(search_workflows, thread_sensitive=False)(
                project_id=self._team.project_id,
                query=None,
                access_control=self.user_access_control,
                limit=limit,
                offset=offset,
            )
        elif entity_type == "feature_flag":
            # Specialized queryset so we can surface each flag's status
            return await self.list_feature_flags(limit, offset)
        else:
            # Fetch database entities
            db_results, _, maybe_count = await database_sync_to_async(search_entities_fts, thread_sensitive=False)(
                entities={entity_type},
                query=None,  # No search query, just listing
                project_id=self._team.project_id,
                view=self,  # type: ignore
                entity_map=ENTITY_MAP,
                limit=limit,
                offset=offset,
                facade_searches=FACADE_SEARCHES,
                counted_entities=COUNTED_ENTITIES,
            )
            all_entities.extend(db_results)
            assert maybe_count is not None
            total_count = maybe_count

        return all_entities, total_count

    def format_entities(self, entities: list[dict]) -> str:
        """
        Format a list of entities as a CSV-like pipe-separated matrix.

        Args:
            entities: List of entity result dicts

        Returns:
            Pipe-separated matrix string with header row
        """
        if not entities:
            return ""

        # Determine if we have multiple entity types
        entity_types = {e["type"] for e in entities}
        multiple_types = len(entity_types) > 1

        # Collect all extra field keys across all entities (excluding standard columns)
        standard_columns = {"name", "id", "url", "type"}
        extra_columns: set[str] = set()
        for entity in entities:
            entity_type = entity["type"]
            exclude_fields = EXCLUDE_FROM_DISPLAY.get(entity_type, set())
            extra_fields = entity.get("extra_fields", {})
            for key in extra_fields:
                if key not in standard_columns and key not in exclude_fields:
                    extra_columns.add(key)

        # Build column order: ID, Name, extra fields alphabetically, URL last
        extra_columns_sorted = sorted(extra_columns)

        # Build rows
        rows: list[str] = []

        if multiple_types:
            # Header with Entity type column - each row will have its own type_id
            header_cols = [
                "entity_type",
                "entity_id",
                "name",
                *extra_columns_sorted,
                "url",
            ]
            rows.append("|".join(header_cols))
        else:
            # Single type: add type header line and use type-specific id column
            single_type = next(iter(entity_types))
            rows.append(f"entity_type: {single_type}")
            # Header with type-specific id column (e.g., insight_id, dashboard_id)
            header_cols = [f"{single_type}_id", "name", *extra_columns_sorted, "url"]
            rows.append("|".join(header_cols))

        # Data rows
        for entity in entities:
            row_values = self._get_entity_row_values(entity, extra_columns_sorted, multiple_types)
            rows.append("|".join(row_values))

        return "\n".join(rows)

    async def _list_insights(self, limit: int = 100, offset: int = 0) -> tuple[list[dict[str, Any]], int]:
        """
        List insights filtered by recent view time (same queryset as InsightSearchNode).
        Only includes insights viewed within INSIGHTS_CUTOFF_DAYS.

        Returns:
            Tuple of (entities list in format_entities format, total count)
        """
        return await database_sync_to_async(self._list_insights_sync, thread_sensitive=False)(limit, offset)

    def _list_insights_sync(self, limit: int = 100, offset: int = 0) -> tuple[list[dict[str, Any]], int]:
        """Sync implementation of _list_insights for use with database_sync_to_async."""
        cutoff_date = timezone.now() - timedelta(days=INSIGHTS_CUTOFF_DAYS)
        queryset = (
            Insight.objects.filter(team=self._team, deleted=False)
            .annotate(latest_view_time=Max("insightviewed__last_viewed_at"))
            .filter(latest_view_time__gte=cutoff_date)
            .order_by("-latest_view_time")
        )
        # Apply access control filtering
        queryset = self.user_access_control.filter_queryset_by_access_level(queryset)

        total_count = queryset.count()
        insights = list(
            queryset[offset : offset + limit].values("id", "name", "description", "query", "derived_name", "short_id")
        )

        all_entities: list[dict[str, Any]] = []
        for insight in insights:
            all_entities.append(
                {
                    "type": "insight",
                    "result_id": insight["short_id"],
                    "extra_fields": {
                        "name": insight["name"] or insight["derived_name"] or "Unnamed",
                        "description": insight["description"],
                        "insight_type": self._extract_insight_type(insight["query"]),
                    },
                }
            )

        return all_entities, total_count

    @staticmethod
    def _account_entities(accounts: Iterable[AccountRef]) -> list[dict[str, Any]]:
        return [
            {
                "type": "account",
                "result_id": account.id,
                "extra_fields": {"name": account.name, "external_id": account.external_id},
            }
            for account in accounts
        ]

    async def list_feature_flags(
        self, limit: int = 100, offset: int = 0, active_filter: str | None = None
    ) -> tuple[list[dict[str, Any]], int]:
        """
        List feature flags, newest first, surfacing each flag's status so stale flags can be
        identified without reading them one by one. `active_filter` ("STALE"/"true"/"false")
        reuses the same backend filter as the feature_flags API.
        """
        return await database_sync_to_async(self._list_feature_flags_sync, thread_sensitive=False)(
            limit, offset, active_filter
        )

    def _list_feature_flags_sync(
        self, limit: int = 100, offset: int = 0, active_filter: str | None = None
    ) -> tuple[list[dict[str, Any]], int]:
        # Stricter than filter_queryset_by_access_level's fail-closed baseline: a caller without
        # feature flag access gets nothing, not even flags they created (also reachable via list_data).
        if not self.user_access_control.check_access_level_for_resource("feature_flag", "viewer"):
            return [], 0

        queryset = self.user_access_control.filter_queryset_by_access_level(
            FeatureFlag.objects.filter(team=self._team, deleted=False)
        )
        if active_filter is not None:
            queryset = filter_flags_by_active_param(queryset, active_filter)
        queryset = queryset.order_by("-updated_at")

        total_count = queryset.count()
        flags = list(queryset[offset : offset + limit])

        all_entities: list[dict[str, Any]] = []
        for flag in flags:
            status, _ = FeatureFlagStatusChecker(feature_flag=flag).get_status()
            # Map to the tool's stale/enabled/disabled vocabulary. The checker reports ACTIVE for
            # disabled flags (it skips staleness for them) and "active" isn't a value the tool
            # exposes, so derive the label from `active` + STALE instead of using status.value.
            if not flag.active:
                display_status = "disabled"
            elif status == FeatureFlagStatus.STALE:
                display_status = "stale"
            else:
                display_status = "enabled"
            all_entities.append(
                {
                    "type": "feature_flag",
                    "result_id": str(flag.id),
                    "extra_fields": {
                        "key": flag.key,
                        "name": flag.name,
                        "status": display_status,
                        "active": flag.active,
                        "rollout": self._flag_rollout_summary(flag),
                    },
                }
            )
        return all_entities, total_count

    def _flag_rollout_summary(self, flag: FeatureFlag) -> str | None:
        """Highest release-condition rollout percentage, as a display hint (not staleness logic)."""
        groups = (flag.filters or {}).get("groups", [])
        percentages = [g.get("rollout_percentage") for g in groups if g.get("rollout_percentage") is not None]
        return f"{max(percentages)}%" if percentages else None

    async def _search_accounts(self, query: str) -> tuple[list[dict[str, Any]], int]:
        """Search accounts by name, external id, known email, or email domain."""
        return await database_sync_to_async(self._search_accounts_sync, thread_sensitive=False)(query)

    def _search_accounts_sync(self, query: str) -> tuple[list[dict[str, Any]], int]:
        accounts, total_count = customer_analytics_facade.search_accounts(
            self._team.id, query, self.user_access_control, SEARCH_LIMIT
        )
        return self._account_entities(accounts), total_count

    async def _list_accounts(self, limit: int = 100, offset: int = 0) -> tuple[list[dict[str, Any]], int]:
        """List customer accounts, newest first."""
        return await database_sync_to_async(self._list_accounts_sync, thread_sensitive=False)(limit, offset)

    def _list_accounts_sync(self, limit: int = 100, offset: int = 0) -> tuple[list[dict[str, Any]], int]:
        accounts, total_count = customer_analytics_facade.list_accounts(
            self._team.id, offset, limit, self.user_access_control
        )
        return self._account_entities(accounts), total_count

    def _get_entity_row_values(self, result: dict, extra_columns: list[str], include_type: bool) -> list[str]:
        """
        Get the row values for an entity.

        Args:
            result: Entity result dict with 'type', 'result_id', and 'extra_fields'
            extra_columns: List of extra column names to include
            include_type: Whether to include the entity type as first column

        Returns:
            List of escaped string values for the row
        """
        entity_type = result["type"]
        result_id = result["result_id"]
        extra_fields = result.get("extra_fields", {})

        # Get name (from extra_fields or generate default)
        name = extra_fields.get("name") or extra_fields.get("title") or f"{entity_type.upper()} {result_id}"

        # Get URL if available
        try:
            url = self._build_url(entity_type, result_id, self._team.id)
        except ValueError:
            url = ""

        # Build row values: Entity type (if multiple), ID, Name, extra fields, URL last
        values: list[str] = []
        if include_type:
            values.append(self._escape_value(entity_type.replace("_", " ").title()))
        values.append(self._escape_value(result_id))
        values.append(self._escape_value(name))

        # Add extra field values
        exclude_fields = EXCLUDE_FROM_DISPLAY.get(entity_type, set())
        for col in extra_columns:
            if col in exclude_fields:
                values.append("-")
            else:
                val = extra_fields.get(col)
                if isinstance(val, str) and len(val) > MAX_FIELD_LENGTH:
                    val = val[:MAX_FIELD_LENGTH] + "..."
                values.append(self._escape_value(val))

        # URL goes last
        values.append(self._escape_value(url))

        return values

    def _escape_value(self, value: Any) -> str:
        """Escape a value for CSV output, quoting if it contains pipe characters."""
        if value is None or value == "":
            return "-"
        str_val = str(value)
        return f'"{str_val}"' if "|" in str_val else str_val

    def _extract_insight_type(self, query: Any) -> str | None:
        """Extract human-readable insight type from an InsightVizNode query."""
        try:
            node = InsightVizNode.model_validate(query)
            return node.source.kind.lower().replace("query", "")
        except ValidationError:
            return None

    def _build_url(self, entity_type: str, result_id: str, team_id: int) -> str:
        """Build a URL for an entity based on its type and ID."""
        base_url = f"{settings.SITE_URL}/project/{team_id}"
        match entity_type:
            case "insight":
                return f"{base_url}/insights/{result_id}"
            case "dashboard":
                return f"{base_url}/dashboard/{result_id}"
            case "experiment":
                return f"{base_url}/experiments/{result_id}"
            case "feature_flag":
                return f"{base_url}/feature-flags/{result_id}"
            case "notebook":
                return f"{base_url}/notebooks/{result_id}"
            case "action":
                return f"{base_url}/data-management/actions/{result_id}"
            case "cohort":
                return f"{base_url}/cohorts/{result_id}"
            case "survey":
                return f"{base_url}/surveys/{result_id}"
            case "error_tracking_issue":
                return f"{base_url}/error-tracking/{result_id}"
            case "alert_configuration":
                return f"{base_url}/alerts?alert_id={result_id}"
            case "hog_flow":
                return f"{base_url}/workflows/{result_id}/workflow"
            case "account":
                # Deep-link to the specific account (filtered + expanded) rather than the bare list.
                return f"{base_url}{build_account_deeplink(account_id=result_id)}"
            case _:
                raise ValueError(f"Unknown entity type: {entity_type}")