Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

19.0 KB
Raw Blame
"""Public Python interface for creating and retrieving one-off exports."""

import io
from copy import deepcopy
from datetime import datetime, timedelta
from typing import Any

from django.conf import settings
from django.db.models import Q
from django.http.response import HttpResponseBase
from django.utils.timezone import now

import structlog
from asgiref.sync import async_to_sync
from PIL import Image, UnidentifiedImageError
from temporalio.common import WorkflowIDReusePolicy

from posthog.hogql.constants import LimitContext

from posthog.hogql_queries.query_runner import ExecutionMode
from posthog.models import Team, User
from posthog.storage import object_storage
from posthog.temporal.common.client import async_connect
from posthog.temporal.exports.workflows import ExportAssetWorkflow, ExportAssetWorkflowInputs

from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.dashboards.backend.models.dashboard import Dashboard
from products.exports.backend.models.exported_asset import (
    DATASET_EXPORT_KIND as DATASET_EXPORT_KIND,
    ExportedAsset,
    get_content_response,
    save_content as _save_content,
    save_content_from_file as _save_content_from_file,
)
from products.exports.backend.tasks.failure_handler import (
    USER_QUERY_ERROR_NAMES,
    InvalidExportContext as InvalidExportContext,
    RetryableExportError as RetryableExportError,
)
from products.product_analytics.backend.facade.models import Insight

logger = structlog.get_logger(__name__)

JSONL_EXPORT_FORMAT = ExportedAsset.ExportFormat.JSONL

# Caps the whole workflow including retries; callers block on this, so it must stay
# well under the web tier's request timeout.
RENDER_TIMEOUT = timedelta(seconds=90)
_MAX_IMAGE_PIXELS = 25_000_000
_PNG_MODES = frozenset({"1", "L", "LA", "I", "I;16", "P", "RGB", "RGBA"})
EXPORT_WORKFLOW_TIMEOUT = timedelta(minutes=35)


def create_export_asset_async(
    *,
    team: Team,
    created_by: User,
    export_format: str,
    export_context: dict[str, object],
    expires_after: datetime | None = None,
) -> ExportedAsset:
    if export_format not in ExportedAsset.get_supported_format_values():
        raise ValueError(f"Unsupported export format: {export_format}")

    asset = ExportedAsset.objects.create(
        team=team,
        created_by=created_by,
        export_format=export_format,
        export_context=export_context,
        # None keeps the model's format-default TTL (see ExportedAsset.save).
        expires_after=ExportedAsset.round_up_expiry(expires_after) if expires_after is not None else None,
    )

    async def _start() -> None:
        client = await async_connect()
        await client.start_workflow(
            ExportAssetWorkflow.run,
            ExportAssetWorkflowInputs(
                exported_asset_id=asset.id,
                team_id=team.id,
                distinct_id=str(created_by.distinct_id),
            ),
            id=f"export-asset-{asset.id}",
            task_queue=settings.ANALYTICS_PLATFORM_TASK_QUEUE,
            id_reuse_policy=WorkflowIDReusePolicy.TERMINATE_IF_RUNNING,
            execution_timeout=EXPORT_WORKFLOW_TIMEOUT,
        )

    try:
        async_to_sync(_start)()
    except Exception as error:
        logger.info("export_workflow_failed_gracefully", asset_id=asset.id, error=str(error))
        asset.refresh_from_db()
        if not asset.exception:
            asset.exception = "The export could not be started. Try again."
            asset.exception_type = type(error).__name__
            asset.save(update_fields=["exception", "exception_type"])
    asset.refresh_from_db()
    return asset


def get_export_asset(*, team_id: int, asset_id: int) -> ExportedAsset | None:
    return ExportedAsset.objects.filter(team_id=team_id, id=asset_id).first()


def is_export_failure_final(*, exception_type: str | None, failure_type: str | None) -> bool:
    """Whether a recorded export failure is final, rather than one the export workflow retries."""
    # Every failure the workflow records carries a failure type, so one without it comes from a workflow that never
    # started, and nothing will retry it.
    if failure_type is None:
        return True
    # Mirrors the non-retryable error types of EXPORT_RETRY_POLICY.
    return exception_type in USER_QUERY_ERROR_NAMES


def get_export_asset_content_response(*, asset: ExportedAsset, download: bool) -> HttpResponseBase:
    return get_content_response(asset, download=download)


def read_export_asset_content(*, team_id: int, asset_id: int) -> bytes | None:
    """The stored bytes of an export, or None when the export has no content or does not exist in the team."""
    asset = get_export_asset(team_id=team_id, asset_id=asset_id)
    if asset is None:
        return None
    if asset.content is not None:
        return bytes(asset.content)
    if asset.content_location:
        return object_storage.read_bytes(asset.content_location)
    return None


def save_export_asset_content_from_file(
    *,
    asset: ExportedAsset,
    file_path: str,
    max_database_bytes: int | None = None,
) -> None:
    _save_content_from_file(asset, file_path, max_database_bytes=max_database_bytes)


# The limit contexts an export writer can pin, keyed by the string it stores in export_context.
# The API rejects this key from clients, so only PostHog's own writers reach this map.
_PINNABLE_EXPORT_LIMIT_CONTEXTS = {"posthog_ai": LimitContext.POSTHOG_AI}


def export_limit_context(export_context: dict | None) -> LimitContext:
    requested = (export_context or {}).get("limit_context")
    if isinstance(requested, str):
        return _PINNABLE_EXPORT_LIMIT_CONTEXTS.get(requested, LimitContext.QUERY)
    return LimitContext.QUERY


# The execution modes an insight export can warm its cache with. Async modes return no result on a
# cache miss, so the render page would recompute inside its load timeout.
_EXPORT_EXECUTION_MODES = frozenset(
    {ExecutionMode.CALCULATE_BLOCKING_ALWAYS, ExecutionMode.RECENT_CACHE_CALCULATE_BLOCKING_IF_STALE}
)


def export_execution_mode(export_context: dict | None) -> ExecutionMode:
    requested = (export_context or {}).get("execution_mode")
    if requested in _EXPORT_EXECUTION_MODES:
        return ExecutionMode(requested)
    return ExecutionMode.CALCULATE_BLOCKING_ALWAYS


def _validate_adhoc_export_context(export_context: dict) -> None:
    """The ad-hoc render pipeline (viewport sizing, the exporter page's Query dispatch) draws a
    chart for an InsightVizNode-wrapped source, or for a DataVisualizationNode over HogQL. Anything
    else renders a JSON dump instead of a chart, so reject it here with a real error instead."""
    source = export_context.get("source")
    if isinstance(source, dict):
        kind = source.get("kind")
        if kind == "InsightVizNode":
            return
        inner = source.get("source")
        if (
            kind in ("DataVisualizationNode", "BIVisualizationNode")
            and isinstance(inner, dict)
            and inner.get("kind") == "HogQLQuery"
        ):
            return
    raise ValueError("export_context.source must be an InsightVizNode- or DataVisualizationNode-wrapped query")


def get_delivery_image_url(
    *, team_id: int, asset_id: int, expiry_delta: timedelta, created_by_id: int | None = None
) -> str | None:
    """Mint a delivery-purposed url for one of the team's own rendered images.

    The token authenticates anonymously and bypasses the org's publicly-shared-resources
    setting, so it is minted on demand rather than stored anywhere a lower-privileged
    reader could reach. Callers are responsible for only passing an ``asset_id`` they
    established server-side; the format and team filters bound the damage if one leaks —
    an image url can never be turned into a CSV or XLSX download. The manager also
    excludes assets past their TTL.

    Pass ``created_by_id`` when the ``asset_id`` comes from a store a caller cannot fully
    trust (e.g. an id read back from a shared cache): it pins the asset to the user that
    rendered it, so a substituted id belonging to another same-team user mints nothing.
    """
    asset_filter = {"team_id": team_id, "id": asset_id, "export_format": ExportedAsset.ExportFormat.PNG}
    if created_by_id is not None:
        asset_filter["created_by_id"] = created_by_id
    asset = ExportedAsset.objects.filter(**asset_filter).first()
    if asset is None:
        return None
    return asset.get_subscription_delivery_content_url(expiry_delta=expiry_delta)


def create_png_delivery_url(
    *, team_id: int, created_by_id: int, content: bytes, filename: str, url_ttl: timedelta, asset_ttl: timedelta
) -> str | None:
    """Store image bytes as a PNG export and mint a delivery url for it. None when the bytes are not an image.

    The url opens without a login, so the caller must check that the user may see the image.
    """
    png = _as_png(content)
    if png is None:
        return None
    asset = ExportedAsset.objects.create(
        team_id=team_id,
        created_by_id=created_by_id,
        export_format=ExportedAsset.ExportFormat.PNG,
        export_context={"filename": filename},
        expires_after=now() + asset_ttl,
    )
    _save_content(asset, png)
    return asset.get_subscription_delivery_content_url(expiry_delta=url_ttl)


def _as_png(content: bytes) -> bytes | None:
    try:
        with Image.open(io.BytesIO(content)) as image:
            # A small compressed file can decode to a very large bitmap.
            if image.width * image.height > _MAX_IMAGE_PIXELS:
                return None
            if image.format == "PNG":
                image.verify()
                return content
            image.load()
            png_ready: Image.Image = image
            if image.mode not in _PNG_MODES:
                png_ready = image.convert("RGBA" if "A" in image.getbands() else "RGB")
            output = io.BytesIO()
            png_ready.save(output, format="PNG")
            return output.getvalue()
    except (UnidentifiedImageError, Image.DecompressionBombError, OSError, ValueError):
        return None


def _create_png_export_asset(
    *,
    team: Team,
    created_by: User,
    export_context: dict | None = None,
    insight_id: int | None = None,
    insight_short_id: str | None = None,
    dashboard_id: int | None = None,
    is_system: bool = False,
    expires_after: datetime | None = None,
    execution_mode: ExecutionMode = ExecutionMode.CALCULATE_BLOCKING_ALWAYS,
    reference_lines: list[dict[str, Any]] | None = None,
) -> ExportedAsset:
    if execution_mode not in _EXPORT_EXECUTION_MODES:
        raise ValueError(f"Unsupported execution mode for an export: {execution_mode}")
    if created_by is None:
        # Access control below resolves against created_by; a principal-less render would
        # silently skip it, so service callers must attribute the render to a real user.
        raise ValueError("created_by is required")
    if sum(value is not None for value in (export_context, insight_id, insight_short_id, dashboard_id)) != 1:
        raise ValueError("Provide exactly one of export_context, insight_id, insight_short_id or dashboard_id")
    if export_context is not None:
        _validate_adhoc_export_context(export_context)
        # An ad-hoc render runs whatever query the caller supplies, so it needs the same gate as
        # running that query directly; the object-level check below covers only saved insights.
        if not UserAccessControl(user=created_by, team=team).check_access_level_for_resource("query", "viewer"):
            raise ValueError("You need query access to render this export")
    if insight_id is not None or insight_short_id is not None:
        insight_filter = {"id": insight_id} if insight_id is not None else {"short_id": insight_short_id}
        insight = Insight.objects.filter(team_id=team.id, deleted=False, **insight_filter).first()
        # Object-level access matters here: created_by may not be allowed to view the insight.
        if insight is None or not UserAccessControl(user=created_by, team=team).check_access_level_for_object(
            insight, "viewer"
        ):
            raise ValueError("Insight not found")
        insight_id = insight.id
    if dashboard_id is not None:
        dashboard = Dashboard.objects.filter(team_id=team.id, id=dashboard_id, deleted=False).first()
        if dashboard is None or not UserAccessControl(user=created_by, team=team).check_access_level_for_object(
            dashboard, "viewer"
        ):
            raise ValueError("Dashboard not found")
    if execution_mode != ExecutionMode.CALCULATE_BLOCKING_ALWAYS:
        export_context = {**(export_context or {}), "execution_mode": execution_mode.value}

    if reference_lines:
        export_context = {**(export_context or {}), "reference_lines": reference_lines}

    asset = ExportedAsset.objects.create(
        team=team,
        created_by=created_by,
        export_format=ExportedAsset.ExportFormat.PNG,
        export_context=export_context,
        insight_id=insight_id,
        dashboard_id=dashboard_id,
        is_system=is_system,
        # None keeps the model's format-default TTL (see ExportedAsset.save).
        expires_after=expires_after,
    )
    return asset


def create_png_export(
    *,
    team: Team,
    created_by: User,
    insight_id: int | None = None,
    query: dict | None = None,
    is_system: bool = False,
    expires_after: datetime | None = None,
    execution_mode: ExecutionMode = ExecutionMode.CALCULATE_BLOCKING_ALWAYS,
) -> int:
    """Check access and create the PNG asset that ``ExportAssetWorkflow`` renders, without rendering it.

    Returns the asset id. For a caller that is itself a Temporal workflow: it runs the export as a
    child workflow instead of blocking a worker on ``render_png_export``. The access rules and the
    ``is_system`` and ``expires_after`` arguments are the same as for ``render_png_export``.

    Pass exactly one of a saved ``insight_id`` or an ad hoc ``query``. A query must be an
    InsightVizNode or a DataVisualizationNode over HogQL. A request that breaks a rule raises ``ValueError``.

    ``execution_mode`` sets how a saved insight warms its cache before the render. The default,
    ``CALCULATE_BLOCKING_ALWAYS``, recomputes the insight on every render.
    ``RECENT_CACHE_CALCULATE_BLOCKING_IF_STALE`` uses a recent cached result, and computes and waits
    only when the cache is stale. Use it where a slightly older chart is acceptable and query load
    matters, such as link previews. Other modes raise ``ValueError``. An ad hoc query always renders
    with the recent cache.
    """
    return _create_png_export_asset(
        team=team,
        created_by=created_by,
        insight_id=insight_id,
        export_context={"source": query} if query is not None else None,
        is_system=is_system,
        expires_after=expires_after,
        execution_mode=execution_mode,
    ).id


def query_with_reference_lines(query: dict[str, Any], reference_lines: list[dict[str, Any]]) -> dict[str, Any]:
    query = deepcopy(query)
    target = query
    if query.get("kind") == "InsightVizNode":
        target = query.get("source") or {}
    settings_key = {
        "TrendsQuery": "trendsFilter",
        "FunnelsQuery": "funnelsFilter",
        "RetentionQuery": "retentionFilter",
        "DataVisualizationNode": "chartSettings",
        "BIVisualizationNode": "chartSettings",
        "MetricsQuery": "display",
    }.get(target.get("kind", ""))
    if settings_key is None:
        return query
    settings = target.get(settings_key) or {}
    target[settings_key] = settings
    existing = settings.get("goalLines") or []
    settings["goalLines"] = existing + [line for line in reference_lines if line not in existing]
    return query


def png_export_has_content(*, team_id: int, asset_id: int) -> bool:
    return (
        ExportedAsset.objects.filter(team_id=team_id, id=asset_id, export_format=ExportedAsset.ExportFormat.PNG)
        .filter(Q(exception__isnull=True) | Q(exception=""))
        .filter(Q(content__isnull=False) | (Q(content_location__isnull=False) & ~Q(content_location="")))
        .exists()
    )


def render_png_export(
    *,
    team: Team,
    created_by: User,
    export_context: dict | None = None,
    insight_id: int | None = None,
    insight_short_id: str | None = None,
    dashboard_id: int | None = None,
    is_system: bool = False,
    expires_after: datetime | None = None,
    reference_lines: list[dict[str, Any]] | None = None,
) -> tuple[ExportedAsset, bytes | None]:
    """Render a PNG export synchronously and return the asset together with its content bytes.

    Blocks until the export workflow finishes (typically a few seconds). On failure the
    returned bytes are None and ``asset.exception`` carries the error.

    ``is_system`` marks the asset as created by an internal process, which excludes it from
    the user's export listings and the per-team export quota. ``expires_after`` overrides
    the format's default TTL; pass it when the render backs a short-lived delivery URL so
    the stored bytes do not outlive their only consumer by months.
    """
    asset = _create_png_export_asset(
        team=team,
        created_by=created_by,
        export_context=export_context,
        insight_id=insight_id,
        insight_short_id=insight_short_id,
        dashboard_id=dashboard_id,
        is_system=is_system,
        expires_after=expires_after,
        reference_lines=reference_lines,
    )

    async def _run() -> None:
        client = await async_connect()
        await client.execute_workflow(
            ExportAssetWorkflow.run,
            ExportAssetWorkflowInputs(
                exported_asset_id=asset.id,
                team_id=team.id,
                distinct_id=str(created_by.distinct_id),
            ),
            id=f"export-asset-{asset.id}",
            task_queue=settings.ANALYTICS_PLATFORM_TASK_QUEUE,
            id_reuse_policy=WorkflowIDReusePolicy.TERMINATE_IF_RUNNING,
            execution_timeout=RENDER_TIMEOUT,
        )

    try:
        async_to_sync(_run)()
    except Exception as e:
        # export_asset_direct records activity failures on the asset, but a dispatch
        # failure (Temporal unreachable, workflow never started) would leave
        # asset.exception empty — record it so the documented failure contract
        # ("bytes are None and asset.exception carries the error") always holds.
        logger.info("render_png_export_failed", asset_id=asset.id, error=str(e))
        asset.refresh_from_db()
        if not asset.exception:
            asset.exception = str(e) or "Export dispatch failed"
            asset.save(update_fields=["exception"])

    asset.refresh_from_db()
    if asset.exception:
        return asset, None
    content = asset.content
    if not content and asset.content_location:
        content = object_storage.read_bytes(asset.content_location)
    if not content:
        # A workflow that reports success but stores nothing would otherwise return no bytes
        # and no exception, leaving callers unable to tell it apart from a successful render.
        asset.exception = "Export produced no content"
        asset.save(update_fields=["exception"])
        return asset, None
    return asset, bytes(content)