Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

28.2 KB
Raw Blame
"""Materialization lifecycle for endpoint versions.

``EndpointMaterializationService`` owns enabling/disabling materialization
(creating and reverting the backing ``DataWarehouseSavedQuery``), the
materialization preview, and the status payload. The AST-level analysis and
query transforms live in ``products.endpoints.backend.materialization_transforms``.
"""

import dataclasses
from datetime import timedelta
from typing import cast

from django.db import DatabaseError, transaction

import structlog
from rest_framework.exceptions import APIException, ValidationError
from rest_framework.request import Request

from posthog.hogql import ast
from posthog.hogql.context import HogQLContext
from posthog.hogql.errors import ExposedHogQLError
from posthog.hogql.parser import parse_select
from posthog.hogql.printer import to_printed_hogql
from posthog.hogql.printer.utils import print_prepared_ast

from posthog.clickhouse.query_tagging import Product
from posthog.exceptions_capture import capture_exception
from posthog.helpers.impersonation import is_impersonated
from posthog.models import Team, User
from posthog.models.activity_logging.activity_log import Detail, log_activity

from products.data_modeling.backend.facade.api import (
    HasDependentsError,
    SchedulingUnavailableError,
    UnsatisfiableFrequencyError,
    UnsupportedFrequencyTargetError,
    is_materialization_fresh,
    latest_saved_query_materialization_job,
    link_endpoint_nodes,
    saved_query_materialized_at,
    sync_saved_query_to_dag,
)
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
from products.endpoints.backend.logic.activity import EndpointContext
from products.endpoints.backend.logic.materialization_dependencies import EndpointMaterializationDependencyService
from products.endpoints.backend.logic.strategies import apply_where_filter, strategy_for
from products.endpoints.backend.materialization_transforms import (
    MaterializationNotSupportedError,
    _extract_aggregate_name,
    analyze_variables_for_materialization,
    build_endpoint_hogql,
    convert_insight_query_to_hogql,
    get_reaggregation,
    transform_query_for_materialization,
)
from products.endpoints.backend.metrics import ENDPOINT_MATERIALIZATION_EVENT_TOTAL
from products.endpoints.backend.models import Endpoint, EndpointVersion
from products.endpoints.backend.rate_limit import clear_endpoint_materialization_cache

logger = structlog.get_logger(__name__)


class OrphanedEndpointSavedQueryError(Exception):
    pass


class EndpointMaterializationTurnedOffError(MaterializationNotSupportedError):
    pass


def log_materialization_activity(
    team: Team,
    endpoint: Endpoint,
    version: EndpointVersion,
    activity: str,
    *,
    saved_query_id: str,
    saved_query_name: str,
    user: User | None = None,
    was_impersonated: bool = False,
    reason: str | None = None,
) -> None:
    # Two rows: the model's history reads the saved query, the endpoint's history reads the
    # endpoint. A read-time join can't replace this, since disabling unlinks the saved query.
    for item_id, scope, name in (
        (saved_query_id, "DataWarehouseSavedQuery", saved_query_name),
        (str(endpoint.id), "EndpointVersion", endpoint.name),
    ):
        log_activity(
            organization_id=team.organization_id,
            team_id=team.pk,
            user=user,
            was_impersonated=was_impersonated,
            item_id=item_id,
            scope=scope,
            activity=activity,
            detail=Detail(name=name, context=EndpointContext(version=version.version, reason=reason)),
        )


def prepare_executable_query(saved_query: DataWarehouseSavedQuery) -> None:
    """Rebuild the saved query's executable HogQL from its endpoint version.

    Called by the data-modeling Temporal workflow before each materialization run,
    so query-printer changes and bucket overrides are always reflected.
    """
    version = EndpointVersion.objects.filter(saved_query=saved_query).first()
    if version is None:
        raise OrphanedEndpointSavedQueryError(
            f"Saved query {saved_query.id} ({saved_query.name}) has no linked EndpointVersion"
        )

    try:
        saved_query.query = build_endpoint_hogql(
            version.query,
            saved_query.team,
            bucket_overrides=version.bucket_overrides,
            # Temporal rebuilds are userless and operate on an already-owned endpoint saved query.
            bypass_warehouse_access_control=True,
        )
    except MaterializationNotSupportedError as e:
        # Readers of the backing table would break, so keep failing until they are updated.
        try:
            EndpointMaterializationDependencyService(saved_query.team_id).check_deletable(version)
        except HasDependentsError:
            raise e
        # The version can never materialize again, so turn it off once instead of failing every run.
        saved_query_id, saved_query_name = str(saved_query.id), saved_query.name
        with transaction.atomic():
            version.disable_materialization()
        clear_endpoint_materialization_cache(saved_query.team_id, version.endpoint.name, versions=[version.version])
        reason = str(e).rstrip(".")
        log_materialization_activity(
            saved_query.team,
            version.endpoint,
            version,
            "materialization_disabled",
            saved_query_id=saved_query_id,
            saved_query_name=saved_query_name,
            reason=reason,
        )
        raise EndpointMaterializationTurnedOffError(
            f"Materialization was turned off for this endpoint version: {reason}. "
            "The endpoint now runs its query directly."
        ) from e
    saved_query.save(update_fields=["query", "updated_at"])


ELIGIBILITY_CHECK_FAILED_REASON = (
    "Couldn't check whether this query can be materialized. Try again, and if it keeps happening contact support."
)


def build_materialization_info(version: EndpointVersion, endpoint_name: str | None = None) -> dict:
    """Build the materialization status dict for a version."""
    if version.saved_query:
        latest_job = latest_saved_query_materialization_job(version.saved_query)
        materialization_status = latest_job.status if latest_job else version.saved_query.status
        materialization_error = latest_job.error if latest_job else version.saved_query.latest_error
        if latest_job and latest_job.status == DataWarehouseSavedQuery.Status.COMPLETED:
            materialized_at = max(
                timestamp
                for timestamp in (latest_job.last_run_at, version.saved_query.last_run_at)
                if timestamp is not None
            )
        else:
            # A failed or running attempt may follow a successful build that remains serveable.
            materialized_at = saved_query_materialized_at(version.saved_query)
        result = {
            "enabled": bool(version.saved_query.is_materialized),
            "ready": bool(
                version.saved_query.table_id
                and is_materialization_fresh(materialized_at, version.data_freshness_seconds)
            ),
            "status": materialization_status or "Unknown",
            "can_materialize": True,
            "last_materialized_at": materialized_at.isoformat() if materialized_at else None,
            "error": (materialization_error or "")
            if materialization_status != DataWarehouseSavedQuery.Status.COMPLETED
            else "",
            "saved_query_id": str(version.saved_query.id),
        }
    else:
        try:
            can_mat, reason = version.can_materialize()
        except Exception as e:
            capture_exception(e, {"endpoint_version_id": str(version.id), "team_id": version.endpoint.team_id})
            can_mat, reason = False, ELIGIBILITY_CHECK_FAILED_REASON
        result = {
            "enabled": False,
            "ready": False,
            "can_materialize": can_mat,
            "reason": reason if not can_mat else None,
        }

    if endpoint_name is not None:
        result["name"] = endpoint_name
    return result


@dataclasses.dataclass(frozen=True)
class MaterializationPreview:
    """Payload of the materialization-preview endpoint."""

    can_materialize: bool
    reason: str | None = None
    transformed_query: str | None = None
    execution_query: str | None = None
    display_execution_query: str | None = None
    range_pairs: list[dict] = dataclasses.field(default_factory=list)
    aggregates: list[dict] = dataclasses.field(default_factory=list)

    @classmethod
    def cant_materialize(cls, reason: str) -> "MaterializationPreview":
        return cls(can_materialize=False, reason=reason)


class EndpointMaterializationService:
    """Enable, disable, and preview materialization for endpoint versions."""

    def __init__(self, team: Team, request: Request):
        self.team = team
        self.request = request
        self.user = cast(User, request.user)

    def enable_materialization(
        self,
        endpoint: Endpoint,
        version: EndpointVersion,
        data_freshness_seconds: int,
        bucket_overrides: dict[str, str] | None = None,
    ) -> None:
        """Enable materialization for an endpoint version.

        Each version gets its own saved_query named {endpoint_name}_v{version}.
        """
        try:
            self._enable_materialization_inner(endpoint, version, data_freshness_seconds, bucket_overrides)
            # The throttle may hold a not-ready snapshot cached before the enable. Drop it so the
            # first completed run is picked up on the next request, not when the entry expires.
            transaction.on_commit(
                lambda: clear_endpoint_materialization_cache(self.team.pk, endpoint.name, versions=[version.version]),
                robust=True,
            )
            ENDPOINT_MATERIALIZATION_EVENT_TOTAL.labels(action="enable", status="success").inc()
            if version.saved_query:
                self._log_materialization_activity(
                    endpoint,
                    version,
                    "materialization_enabled",
                    saved_query_id=str(version.saved_query.id),
                    saved_query_name=version.saved_query.name,
                )
        except ValidationError:
            ENDPOINT_MATERIALIZATION_EVENT_TOTAL.labels(action="enable", status="validation_error").inc()
            raise
        except (ExposedHogQLError, MaterializationNotSupportedError) as e:
            # A bad user query, not a system fault — surface as a 400. Pre-flight validation
            # (can_materialize) normally catches these, so reaching here is a backstop.
            ENDPOINT_MATERIALIZATION_EVENT_TOTAL.labels(action="enable", status="validation_error").inc()
            raise ValidationError(f"Cannot materialize endpoint. Reason: {e}")
        except Exception as e:
            ENDPOINT_MATERIALIZATION_EVENT_TOTAL.labels(action="enable", status="error").inc()
            # Genuine system fault (user-query limitations are handled above). Log + capture so
            # status="error" — which the alert fires on — is never a blind spot.
            logger.exception(
                "Failed to enable endpoint materialization",
                endpoint_name=endpoint.name,
                version=version.version,
                team_id=self.team.pk,
            )
            capture_exception(
                e,
                {
                    "product": Product.ENDPOINTS,
                    "team_id": self.team.pk,
                    "endpoint_name": endpoint.name,
                },
            )
            # Not a request-validation problem — surface as a server error, not a 400.
            raise APIException("Failed to enable materialization.")

    def _enable_materialization_inner(
        self,
        endpoint: Endpoint,
        version: EndpointVersion,
        data_freshness_seconds: int,
        bucket_overrides: dict[str, str] | None = None,
    ) -> None:
        can_mat, reason = version.can_materialize()
        if not can_mat:
            raise ValidationError(f"Cannot materialize endpoint. Reason: {reason}")

        # Atomic so a rejected freshness target unwinds the whole enable (saved query, version link,
        # node) instead of leaving a dangling materialization. schedule_materialization's side effect
        # is deferred to on_commit (tiered) or terminal (v1), so nothing rolls back after it fires.
        with transaction.atomic():
            saved_query = self._get_or_build_saved_query(version)
            # No live saved query row means materialization is being stood up fresh (first
            # enable, re-enable after disable, or a version bump). Only that case gets an
            # immediate first run; a retained enable (e.g. a metadata-only endpoint update
            # re-reconciling materialization) must not restart the workflow.
            newly_materialized = saved_query._state.adding
            self._configure_saved_query(saved_query, version, data_freshness_seconds, bucket_overrides)
            version.enable_materialization(saved_query, bucket_overrides)

            # The DAG node must exist before scheduling: the v2 detection and the freshness-target
            # write-through both resolve this saved query through its Node row.
            try:
                sync_saved_query_to_dag(saved_query)
                link_endpoint_nodes(
                    team_id=saved_query.team_id,
                    saved_query_id=saved_query.id,
                    endpoint_name=endpoint.name,
                    version=version.version,
                )
            except DatabaseError:
                # An infrastructure fault, not a lineage problem the author can fix: let it surface
                # as a server error so the status="error" metric sees it.
                raise
            except Exception as e:
                logger.exception(
                    "Failed to sync endpoint node to DAG",
                    endpoint_name=endpoint.name,
                    saved_query_id=saved_query.id,
                )
                capture_exception(
                    e,
                    {
                        "product": Product.ENDPOINTS,
                        "team_id": self.team.pk,
                        "endpoint_name": endpoint.name,
                        "saved_query_id": saved_query.id,
                    },
                )
                # The node is missing because the query's dependencies did not resolve, which
                # the author fixes by editing the query. Reporting that as a server error would
                # tell them to retry, and would page on-call through the status="error" metric.
                raise ValidationError(f"Cannot materialize endpoint. Reason: {e}")

            # NOTE: on v1, schedule_materialization only triggers an immediate run when it CREATES
            # the Temporal schedule; re-enabling an existing materialization just (re)syncs it.
            # trigger_immediate_run mirrors that on v2: first run only for a newly created saved
            # query (deferred to on_commit, so it sees the version link above).
            try:
                saved_query.schedule_materialization(
                    trigger_immediate_run=newly_materialized, triggered_by_id=self.user.pk
                )
            except (UnsatisfiableFrequencyError, UnsupportedFrequencyTargetError) as e:
                # The chosen data freshness can't be honored (e.g. finer than an upstream import
                # delivers) — a request problem, not a server one.
                raise ValidationError(str(e))
            except SchedulingUnavailableError as e:
                # Raising unwinds the atomic block, so the endpoint keeps the state it had before.
                raise APIException(f"Failed to schedule materialization for endpoint {endpoint.name}.") from e

            # schedule_materialization applies a disable-on-failure contract rather than raising,
            # so a failure to schedule shows up only as is_materialized flipping back. Checking it
            # is what stops the enable reporting success on an endpoint nothing will ever refresh.
            saved_query.refresh_from_db(fields=["is_materialized"])
            if not saved_query.is_materialized:
                raise APIException(f"Failed to schedule materialization for endpoint {endpoint.name}.")

    def _get_or_build_saved_query(self, version: EndpointVersion) -> DataWarehouseSavedQuery:
        """Find this version's saved query, or build a new (unsaved) one.

        SECURITY: only adopt a saved query that this endpoint owns (ENDPOINT origin and not
        linked to a different version). Without this, a user-created saved query whose name
        happens to collide with {endpoint}_v{n} would be silently taken over — its query
        overwritten and served as the endpoint's data.
        """
        name = version.materialized_view_name
        existing = DataWarehouseSavedQuery.objects.filter(name=name, team=self.team, deleted=False).first()
        if existing is None:
            return DataWarehouseSavedQuery(
                name=name,
                team=self.team,
                origin=DataWarehouseSavedQuery.Origin.ENDPOINT,
            )

        is_foreign = existing.origin != DataWarehouseSavedQuery.Origin.ENDPOINT or (
            EndpointVersion.objects.filter(saved_query=existing).exclude(pk=version.pk).exists()
        )
        if is_foreign:
            raise ValidationError(
                f"A saved query named '{name}' already exists and is not managed by this endpoint. "
                "Rename or delete it before enabling materialization."
            )
        return existing

    def _configure_saved_query(
        self,
        saved_query: DataWarehouseSavedQuery,
        version: EndpointVersion,
        data_freshness_seconds: int,
        bucket_overrides: dict[str, str] | None,
    ) -> None:
        """Point the saved query at the version's materializable HogQL and sync cadence."""
        saved_query.query = build_endpoint_hogql(
            version.query,
            self.team,
            bucket_overrides=bucket_overrides,
            user=self.user,
        )
        saved_query.external_tables = saved_query.s3_tables
        saved_query.is_materialized = True
        saved_query.sync_frequency_interval = timedelta(seconds=data_freshness_seconds)
        saved_query.save()

    def disable_materialization(
        self, endpoint: Endpoint, version: EndpointVersion, *, cache_ttl_seconds: int | None = None
    ) -> None:
        """Disable materialization for an endpoint version."""
        if version.saved_query:
            saved_query_id = str(version.saved_query.id)
            saved_query_name = version.saved_query.name
            try:
                version.disable_materialization(cache_ttl_seconds=cache_ttl_seconds)
            except Exception:
                ENDPOINT_MATERIALIZATION_EVENT_TOTAL.labels(action="disable", status="error").inc()
                raise
            ENDPOINT_MATERIALIZATION_EVENT_TOTAL.labels(action="disable", status="success").inc()
            self._log_materialization_activity(
                endpoint,
                version,
                "materialization_disabled",
                saved_query_id=saved_query_id,
                saved_query_name=saved_query_name,
            )
        # Clears this version's throttle-readiness key plus the "current" key (the disabled
        # version may be the current one) — the next request lazily re-checks the DB.
        clear_endpoint_materialization_cache(self.team.pk, endpoint.name, versions=[version.version])

    def _log_materialization_activity(
        self,
        endpoint: Endpoint,
        version: EndpointVersion,
        activity: str,
        *,
        saved_query_id: str,
        saved_query_name: str,
    ) -> None:
        log_materialization_activity(
            self.team,
            endpoint,
            version,
            activity,
            saved_query_id=saved_query_id,
            saved_query_name=saved_query_name,
            user=self.user,
            was_impersonated=is_impersonated(self.request),
        )

    def preview(
        self,
        endpoint: Endpoint,
        version: EndpointVersion,
        bucket_overrides: dict[str, str] | None = None,
    ) -> MaterializationPreview:
        """Preview the materialization transform without enabling it."""
        try:
            return self._build_preview(endpoint, version, bucket_overrides)
        except (ExposedHogQLError, MaterializationNotSupportedError) as e:
            # A bad user query, not a system fault, so report it as unmaterializable.
            return MaterializationPreview.cant_materialize(str(e))

    def _build_preview(
        self,
        endpoint: Endpoint,
        version: EndpointVersion,
        bucket_overrides: dict[str, str] | None,
    ) -> MaterializationPreview:
        can_mat, reason = version.can_materialize()
        if not can_mat:
            return MaterializationPreview.cant_materialize(reason)

        hogql_query = convert_insight_query_to_hogql(version.query, self.team, user=self.user)

        range_pairs: list[dict] = []
        aggregates: list[dict] = []
        transformed_query_str: str | None = None
        variable_infos: list = []

        if version.query.get("variables"):
            can_materialize_vars, var_reason, variable_infos = analyze_variables_for_materialization(
                version.query, bucket_overrides=bucket_overrides
            )

            if not can_materialize_vars:
                return MaterializationPreview.cant_materialize(var_reason)

            if variable_infos:
                transformed = transform_query_for_materialization(
                    hogql_query,
                    variable_infos,
                    self.team,
                    bucket_overrides=bucket_overrides,
                    user=self.user,
                )
                transformed_query_str = transformed.get("query")

                # Extract range pairs grouped by column
                seen_columns: dict[str, dict] = {}
                for v in variable_infos:
                    if v.bucket_fn is not None:
                        col_key = ".".join(v.column_chain) if v.column_chain else v.column_expression
                        if col_key not in seen_columns:
                            seen_columns[col_key] = {
                                "column": col_key,
                                "variables": [],
                                "bucket_fn": v.bucket_fn,
                            }
                        seen_columns[col_key]["variables"].append(v.code_name)
                range_pairs = list(seen_columns.values())

                query_str = hogql_query.get("query", "")
                if query_str:
                    try:
                        parsed = parse_select(query_str)
                    except Exception as e:
                        # Preview degrades (no aggregate re-aggregation info), the request succeeds.
                        logger.warning(
                            "materialization_preview: converted HogQL failed to parse; "
                            "skipping aggregate re-aggregation preview",
                            endpoint_name=endpoint.name,
                            team_id=self.team.pk,
                        )
                        capture_exception(
                            e,
                            {
                                "product": Product.ENDPOINTS,
                                "team_id": self.team.pk,
                                "endpoint_name": endpoint.name,
                                "materialization_preview": True,
                            },
                        )
                    else:
                        if isinstance(parsed, ast.SelectQuery) and parsed.select:
                            for expr in parsed.select:
                                agg_name = _extract_aggregate_name(expr)
                                if agg_name:
                                    reagg_info = get_reaggregation(agg_name)
                                    reagg = reagg_info.reaggregate_fn if reagg_info else None
                                    if isinstance(expr, ast.Alias):
                                        label = expr.alias
                                    else:
                                        label = expr.to_hogql()
                                    aggregates.append(
                                        {
                                            "expression": label,
                                            "reaggregate_fn": reagg,
                                        }
                                    )
        else:
            # No variables — just show the converted query as-is
            transformed_query_str = hogql_query.get("query")

        # Build the execution query preview — what runs at request time against the materialized table
        execution_query_str: str | None = None
        display_execution_query_str: str | None = None
        try:
            strategy = strategy_for(endpoint, version, self.team)

            def _build_exec_preview(table_name: str) -> ast.SelectQuery:
                q, _ = strategy.build_materialized_select(
                    table_name=table_name,
                    variable_infos=variable_infos or None,
                )
                for v in variable_infos:
                    apply_where_filter(
                        q,
                        v.code_name,
                        f"{{variables.{v.code_name}}}",
                        op=v.operator,
                        value_wrapper_fns=v.value_wrapper_fns,
                        bucket_fn=v.bucket_fn,
                    )
                return q

            # Each call builds a fresh SelectQuery, so WHERE mutations don't leak between calls.
            # Type resolution (to_printed_hogql) needs the materialized table to exist in the
            # database, which only holds once materialization has completed. Previewing a
            # not-yet-materialized version means that table is absent, so we'd otherwise hit
            # "Unknown table". Fall back to printing without type resolution in that case — the
            # frontend uses execution_query only as a presence flag and renders the display variant.
            if version.is_materialized:
                execution_query_str = to_printed_hogql(
                    _build_exec_preview(version.materialized_view_name),
                    team=self.team,
                    # Display-only text for the endpoint's own materialized view, and no rows are read.
                    # Without the bypass, the userless database denies the view and the print fails.
                    bypass_warehouse_access_control=True,
                )
            else:
                execution_query_str = print_prepared_ast(
                    node=_build_exec_preview(version.materialized_view_name),
                    context=HogQLContext(team_id=self.team.pk, enable_select_queries=True),
                    dialect="hogql",
                    pretty=True,
                )

            # Display variant uses the friendly endpoint name — printed without type resolution
            # since the friendly name isn't a real table in the database
            display_execution_query_str = print_prepared_ast(
                node=_build_exec_preview(endpoint.name),
                context=HogQLContext(team_id=self.team.pk, enable_select_queries=True),
                dialect="hogql",
                pretty=True,
            )
        except Exception as e:
            logger.warning(
                "Failed to build materialization execution query preview",
                endpoint_name=endpoint.name,
                team_id=self.team.pk,
                exc_info=True,
            )
            capture_exception(
                e,
                {
                    "product": Product.ENDPOINTS,
                    "team_id": self.team.pk,
                    "endpoint_name": endpoint.name,
                    "materialization_preview": True,
                },
            )

        return MaterializationPreview(
            can_materialize=True,
            transformed_query=transformed_query_str,
            execution_query=execution_query_str,
            display_execution_query=display_execution_query_str,
            range_pairs=range_pairs,
            aggregates=aggregates,
        )