:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
"""Public Python interface for creating and retrieving one-off exports."""
import io
from copy import deepcopy
from datetime import datetime, timedelta
from typing import Any
from django.conf import settings
from django.db.models import Q
from django.http.response import HttpResponseBase
from django.utils.timezone import now
import structlog
from asgiref.sync import async_to_sync
from PIL import Image, UnidentifiedImageError
from temporalio.common import WorkflowIDReusePolicy
from posthog.hogql.constants import LimitContext
from posthog.hogql_queries.query_runner import ExecutionMode
from posthog.models import Team, User
from posthog.storage import object_storage
from posthog.temporal.common.client import async_connect
from posthog.temporal.exports.workflows import ExportAssetWorkflow, ExportAssetWorkflowInputs
from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.dashboards.backend.models.dashboard import Dashboard
from products.exports.backend.models.exported_asset import (
DATASET_EXPORT_KIND as DATASET_EXPORT_KIND,
ExportedAsset,
get_content_response,
save_content as _save_content,
save_content_from_file as _save_content_from_file,
)
from products.exports.backend.tasks.failure_handler import (
USER_QUERY_ERROR_NAMES,
InvalidExportContext as InvalidExportContext,
RetryableExportError as RetryableExportError,
)
from products.product_analytics.backend.facade.models import Insight
logger = structlog.get_logger(__name__)
JSONL_EXPORT_FORMAT = ExportedAsset.ExportFormat.JSONL
# Caps the whole workflow including retries; callers block on this, so it must stay
# well under the web tier's request timeout.
RENDER_TIMEOUT = timedelta(seconds=90)
_MAX_IMAGE_PIXELS = 25_000_000
_PNG_MODES = frozenset({"1", "L", "LA", "I", "I;16", "P", "RGB", "RGBA"})
EXPORT_WORKFLOW_TIMEOUT = timedelta(minutes=35)
def create_export_asset_async(
*,
team: Team,
created_by: User,
export_format: str,
export_context: dict[str, object],
expires_after: datetime | None = None,
) -> ExportedAsset:
if export_format not in ExportedAsset.get_supported_format_values():
raise ValueError(f"Unsupported export format: {export_format}")
asset = ExportedAsset.objects.create(
team=team,
created_by=created_by,
export_format=export_format,
export_context=export_context,
# None keeps the model's format-default TTL (see ExportedAsset.save).
expires_after=ExportedAsset.round_up_expiry(expires_after) if expires_after is not None else None,
)
async def _start() -> None:
client = await async_connect()
await client.start_workflow(
ExportAssetWorkflow.run,
ExportAssetWorkflowInputs(
exported_asset_id=asset.id,
team_id=team.id,
distinct_id=str(created_by.distinct_id),
),
id=f"export-asset-{asset.id}",
task_queue=settings.ANALYTICS_PLATFORM_TASK_QUEUE,
id_reuse_policy=WorkflowIDReusePolicy.TERMINATE_IF_RUNNING,
execution_timeout=EXPORT_WORKFLOW_TIMEOUT,
)
try:
async_to_sync(_start)()
except Exception as error:
logger.info("export_workflow_failed_gracefully", asset_id=asset.id, error=str(error))
asset.refresh_from_db()
if not asset.exception:
asset.exception = "The export could not be started. Try again."
asset.exception_type = type(error).__name__
asset.save(update_fields=["exception", "exception_type"])
asset.refresh_from_db()
return asset
def get_export_asset(*, team_id: int, asset_id: int) -> ExportedAsset | None:
return ExportedAsset.objects.filter(team_id=team_id, id=asset_id).first()
def is_export_failure_final(*, exception_type: str | None, failure_type: str | None) -> bool:
"""Whether a recorded export failure is final, rather than one the export workflow retries."""
# Every failure the workflow records carries a failure type, so one without it comes from a workflow that never
# started, and nothing will retry it.
if failure_type is None:
return True
# Mirrors the non-retryable error types of EXPORT_RETRY_POLICY.
return exception_type in USER_QUERY_ERROR_NAMES
def get_export_asset_content_response(*, asset: ExportedAsset, download: bool) -> HttpResponseBase:
return get_content_response(asset, download=download)
def read_export_asset_content(*, team_id: int, asset_id: int) -> bytes | None:
"""The stored bytes of an export, or None when the export has no content or does not exist in the team."""
asset = get_export_asset(team_id=team_id, asset_id=asset_id)
if asset is None:
return None
if asset.content is not None:
return bytes(asset.content)
if asset.content_location:
return object_storage.read_bytes(asset.content_location)
return None
def save_export_asset_content_from_file(
*,
asset: ExportedAsset,
file_path: str,
max_database_bytes: int | None = None,
) -> None:
_save_content_from_file(asset, file_path, max_database_bytes=max_database_bytes)
# The limit contexts an export writer can pin, keyed by the string it stores in export_context.
# The API rejects this key from clients, so only PostHog's own writers reach this map.
_PINNABLE_EXPORT_LIMIT_CONTEXTS = {"posthog_ai": LimitContext.POSTHOG_AI}
def export_limit_context(export_context: dict | None) -> LimitContext:
requested = (export_context or {}).get("limit_context")
if isinstance(requested, str):
return _PINNABLE_EXPORT_LIMIT_CONTEXTS.get(requested, LimitContext.QUERY)
return LimitContext.QUERY
# The execution modes an insight export can warm its cache with. Async modes return no result on a
# cache miss, so the render page would recompute inside its load timeout.
_EXPORT_EXECUTION_MODES = frozenset(
{ExecutionMode.CALCULATE_BLOCKING_ALWAYS, ExecutionMode.RECENT_CACHE_CALCULATE_BLOCKING_IF_STALE}
)
def export_execution_mode(export_context: dict | None) -> ExecutionMode:
requested = (export_context or {}).get("execution_mode")
if requested in _EXPORT_EXECUTION_MODES:
return ExecutionMode(requested)
return ExecutionMode.CALCULATE_BLOCKING_ALWAYS
def _validate_adhoc_export_context(export_context: dict) -> None:
"""The ad-hoc render pipeline (viewport sizing, the exporter page's Query dispatch) draws a
chart for an InsightVizNode-wrapped source, or for a DataVisualizationNode over HogQL. Anything
else renders a JSON dump instead of a chart, so reject it here with a real error instead."""
source = export_context.get("source")
if isinstance(source, dict):
kind = source.get("kind")
if kind == "InsightVizNode":
return
inner = source.get("source")
if (
kind in ("DataVisualizationNode", "BIVisualizationNode")
and isinstance(inner, dict)
and inner.get("kind") == "HogQLQuery"
):
return
raise ValueError("export_context.source must be an InsightVizNode- or DataVisualizationNode-wrapped query")
def get_delivery_image_url(
*, team_id: int, asset_id: int, expiry_delta: timedelta, created_by_id: int | None = None
) -> str | None:
"""Mint a delivery-purposed url for one of the team's own rendered images.
The token authenticates anonymously and bypasses the org's publicly-shared-resources
setting, so it is minted on demand rather than stored anywhere a lower-privileged
reader could reach. Callers are responsible for only passing an ``asset_id`` they
established server-side; the format and team filters bound the damage if one leaks —
an image url can never be turned into a CSV or XLSX download. The manager also
excludes assets past their TTL.
Pass ``created_by_id`` when the ``asset_id`` comes from a store a caller cannot fully
trust (e.g. an id read back from a shared cache): it pins the asset to the user that
rendered it, so a substituted id belonging to another same-team user mints nothing.
"""
asset_filter = {"team_id": team_id, "id": asset_id, "export_format": ExportedAsset.ExportFormat.PNG}
if created_by_id is not None:
asset_filter["created_by_id"] = created_by_id
asset = ExportedAsset.objects.filter(**asset_filter).first()
if asset is None:
return None
return asset.get_subscription_delivery_content_url(expiry_delta=expiry_delta)
def create_png_delivery_url(
*, team_id: int, created_by_id: int, content: bytes, filename: str, url_ttl: timedelta, asset_ttl: timedelta
) -> str | None:
"""Store image bytes as a PNG export and mint a delivery url for it. None when the bytes are not an image.
The url opens without a login, so the caller must check that the user may see the image.
"""
png = _as_png(content)
if png is None:
return None
asset = ExportedAsset.objects.create(
team_id=team_id,
created_by_id=created_by_id,
export_format=ExportedAsset.ExportFormat.PNG,
export_context={"filename": filename},
expires_after=now() + asset_ttl,
)
_save_content(asset, png)
return asset.get_subscription_delivery_content_url(expiry_delta=url_ttl)
def _as_png(content: bytes) -> bytes | None:
try:
with Image.open(io.BytesIO(content)) as image:
# A small compressed file can decode to a very large bitmap.
if image.width * image.height > _MAX_IMAGE_PIXELS:
return None
if image.format == "PNG":
image.verify()
return content
image.load()
png_ready: Image.Image = image
if image.mode not in _PNG_MODES:
png_ready = image.convert("RGBA" if "A" in image.getbands() else "RGB")
output = io.BytesIO()
png_ready.save(output, format="PNG")
return output.getvalue()
except (UnidentifiedImageError, Image.DecompressionBombError, OSError, ValueError):
return None
def _create_png_export_asset(
*,
team: Team,
created_by: User,
export_context: dict | None = None,
insight_id: int | None = None,
insight_short_id: str | None = None,
dashboard_id: int | None = None,
is_system: bool = False,
expires_after: datetime | None = None,
execution_mode: ExecutionMode = ExecutionMode.CALCULATE_BLOCKING_ALWAYS,
reference_lines: list[dict[str, Any]] | None = None,
) -> ExportedAsset:
if execution_mode not in _EXPORT_EXECUTION_MODES:
raise ValueError(f"Unsupported execution mode for an export: {execution_mode}")
if created_by is None:
# Access control below resolves against created_by; a principal-less render would
# silently skip it, so service callers must attribute the render to a real user.
raise ValueError("created_by is required")
if sum(value is not None for value in (export_context, insight_id, insight_short_id, dashboard_id)) != 1:
raise ValueError("Provide exactly one of export_context, insight_id, insight_short_id or dashboard_id")
if export_context is not None:
_validate_adhoc_export_context(export_context)
# An ad-hoc render runs whatever query the caller supplies, so it needs the same gate as
# running that query directly; the object-level check below covers only saved insights.
if not UserAccessControl(user=created_by, team=team).check_access_level_for_resource("query", "viewer"):
raise ValueError("You need query access to render this export")
if insight_id is not None or insight_short_id is not None:
insight_filter = {"id": insight_id} if insight_id is not None else {"short_id": insight_short_id}
insight = Insight.objects.filter(team_id=team.id, deleted=False, **insight_filter).first()
# Object-level access matters here: created_by may not be allowed to view the insight.
if insight is None or not UserAccessControl(user=created_by, team=team).check_access_level_for_object(
insight, "viewer"
):
raise ValueError("Insight not found")
insight_id = insight.id
if dashboard_id is not None:
dashboard = Dashboard.objects.filter(team_id=team.id, id=dashboard_id, deleted=False).first()
if dashboard is None or not UserAccessControl(user=created_by, team=team).check_access_level_for_object(
dashboard, "viewer"
):
raise ValueError("Dashboard not found")
if execution_mode != ExecutionMode.CALCULATE_BLOCKING_ALWAYS:
export_context = {**(export_context or {}), "execution_mode": execution_mode.value}
if reference_lines:
export_context = {**(export_context or {}), "reference_lines": reference_lines}
asset = ExportedAsset.objects.create(
team=team,
created_by=created_by,
export_format=ExportedAsset.ExportFormat.PNG,
export_context=export_context,
insight_id=insight_id,
dashboard_id=dashboard_id,
is_system=is_system,
# None keeps the model's format-default TTL (see ExportedAsset.save).
expires_after=expires_after,
)
return asset
def create_png_export(
*,
team: Team,
created_by: User,
insight_id: int | None = None,
query: dict | None = None,
is_system: bool = False,
expires_after: datetime | None = None,
execution_mode: ExecutionMode = ExecutionMode.CALCULATE_BLOCKING_ALWAYS,
) -> int:
"""Check access and create the PNG asset that ``ExportAssetWorkflow`` renders, without rendering it.
Returns the asset id. For a caller that is itself a Temporal workflow: it runs the export as a
child workflow instead of blocking a worker on ``render_png_export``. The access rules and the
``is_system`` and ``expires_after`` arguments are the same as for ``render_png_export``.
Pass exactly one of a saved ``insight_id`` or an ad hoc ``query``. A query must be an
InsightVizNode or a DataVisualizationNode over HogQL. A request that breaks a rule raises ``ValueError``.
``execution_mode`` sets how a saved insight warms its cache before the render. The default,
``CALCULATE_BLOCKING_ALWAYS``, recomputes the insight on every render.
``RECENT_CACHE_CALCULATE_BLOCKING_IF_STALE`` uses a recent cached result, and computes and waits
only when the cache is stale. Use it where a slightly older chart is acceptable and query load
matters, such as link previews. Other modes raise ``ValueError``. An ad hoc query always renders
with the recent cache.
"""
return _create_png_export_asset(
team=team,
created_by=created_by,
insight_id=insight_id,
export_context={"source": query} if query is not None else None,
is_system=is_system,
expires_after=expires_after,
execution_mode=execution_mode,
).id
def query_with_reference_lines(query: dict[str, Any], reference_lines: list[dict[str, Any]]) -> dict[str, Any]:
query = deepcopy(query)
target = query
if query.get("kind") == "InsightVizNode":
target = query.get("source") or {}
settings_key = {
"TrendsQuery": "trendsFilter",
"FunnelsQuery": "funnelsFilter",
"RetentionQuery": "retentionFilter",
"DataVisualizationNode": "chartSettings",
"BIVisualizationNode": "chartSettings",
"MetricsQuery": "display",
}.get(target.get("kind", ""))
if settings_key is None:
return query
settings = target.get(settings_key) or {}
target[settings_key] = settings
existing = settings.get("goalLines") or []
settings["goalLines"] = existing + [line for line in reference_lines if line not in existing]
return query
def png_export_has_content(*, team_id: int, asset_id: int) -> bool:
return (
ExportedAsset.objects.filter(team_id=team_id, id=asset_id, export_format=ExportedAsset.ExportFormat.PNG)
.filter(Q(exception__isnull=True) | Q(exception=""))
.filter(Q(content__isnull=False) | (Q(content_location__isnull=False) & ~Q(content_location="")))
.exists()
)
def render_png_export(
*,
team: Team,
created_by: User,
export_context: dict | None = None,
insight_id: int | None = None,
insight_short_id: str | None = None,
dashboard_id: int | None = None,
is_system: bool = False,
expires_after: datetime | None = None,
reference_lines: list[dict[str, Any]] | None = None,
) -> tuple[ExportedAsset, bytes | None]:
"""Render a PNG export synchronously and return the asset together with its content bytes.
Blocks until the export workflow finishes (typically a few seconds). On failure the
returned bytes are None and ``asset.exception`` carries the error.
``is_system`` marks the asset as created by an internal process, which excludes it from
the user's export listings and the per-team export quota. ``expires_after`` overrides
the format's default TTL; pass it when the render backs a short-lived delivery URL so
the stored bytes do not outlive their only consumer by months.
"""
asset = _create_png_export_asset(
team=team,
created_by=created_by,
export_context=export_context,
insight_id=insight_id,
insight_short_id=insight_short_id,
dashboard_id=dashboard_id,
is_system=is_system,
expires_after=expires_after,
reference_lines=reference_lines,
)
async def _run() -> None:
client = await async_connect()
await client.execute_workflow(
ExportAssetWorkflow.run,
ExportAssetWorkflowInputs(
exported_asset_id=asset.id,
team_id=team.id,
distinct_id=str(created_by.distinct_id),
),
id=f"export-asset-{asset.id}",
task_queue=settings.ANALYTICS_PLATFORM_TASK_QUEUE,
id_reuse_policy=WorkflowIDReusePolicy.TERMINATE_IF_RUNNING,
execution_timeout=RENDER_TIMEOUT,
)
try:
async_to_sync(_run)()
except Exception as e:
# export_asset_direct records activity failures on the asset, but a dispatch
# failure (Temporal unreachable, workflow never started) would leave
# asset.exception empty — record it so the documented failure contract
# ("bytes are None and asset.exception carries the error") always holds.
logger.info("render_png_export_failed", asset_id=asset.id, error=str(e))
asset.refresh_from_db()
if not asset.exception:
asset.exception = str(e) or "Export dispatch failed"
asset.save(update_fields=["exception"])
asset.refresh_from_db()
if asset.exception:
return asset, None
content = asset.content
if not content and asset.content_location:
content = object_storage.read_bytes(asset.content_location)
if not content:
# A workflow that reports success but stores nothing would otherwise return no bytes
# and no exception, leaving callers unable to tell it apart from a successful render.
asset.exception = "Export produced no content"
asset.save(update_fields=["exception"])
return asset, None
return asset, bytes(content)