:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(exports): key export handlers by format and kind (#124372)
Carlos Marchalcea88a1b0902
diff --git a/posthog/tasks/exporter.py b/posthog/tasks/exporter.pyindex 9e14090167cd..892c9e0d07f6 100644--- a/posthog/tasks/exporter.py+++ b/posthog/tasks/exporter.py@@ -122,15 +122,17 @@ def export_asset_direct(try:assert_export_authorization(exported_asset)- if exported_asset.export_format in (ExportedAsset.ExportFormat.CSV, ExportedAsset.ExportFormat.XLSX):+ export_handler = get_export_format_handler(+ export_format=exported_asset.export_format, kind=exported_asset.export_kind+ )+ if export_handler is not None:+ export_handler(exported_asset)+ elif exported_asset.export_format in (ExportedAsset.ExportFormat.CSV, ExportedAsset.ExportFormat.XLSX):csv_exporter.export_tabular(exported_asset, limit=limit, source=export_source)elif exported_asset.export_format == ExportedAsset.ExportFormat.JSONL:- if not exported_asset.is_dataset_export:- raise InvalidExportContext("JSONL exports require a dataset export context.", reportable=True)- export_handler = get_export_format_handler(exported_asset.export_format)- if export_handler is None:- raise InvalidExportContext("No handler is registered for this export format.", reportable=True)- export_handler(exported_asset)+ raise InvalidExportContext(+ "This export can't be generated. Contact support if this keeps happening.", reportable=True+ )else:image_exporter.export_image(exported_asset, max_height_pixels=max_height_pixels, source=export_source)diff --git a/posthog/tasks/test/test_exporter.py b/posthog/tasks/test/test_exporter.pyindex 8f91fd0065d9..0507056ef057 100644--- a/posthog/tasks/test/test_exporter.py+++ b/posthog/tasks/test/test_exporter.py@@ -16,6 +16,7 @@ from posthog.models.utils import generate_random_token_personal, hash_key_valuefrom posthog.tasks import exporterfrom products.dashboards.backend.models.dashboard import Dashboard+from products.exports.backend.facade import exporters as export_handlersfrom products.exports.backend.models.exported_asset import DATASET_EXPORT_KIND, ExportedAssetfrom products.exports.backend.tasks.failure_handler import (FAILURE_TYPE_SYSTEM,@@ -122,6 +123,31 @@ class TestExporterTask(APIBaseTest):assert asset.exception_type == "QueryError"assert asset.failure_type == FAILURE_TYPE_USER+ @patch("products.exports.backend.tasks.image_exporter.export_image")+ def test_routes_a_registered_format_and_kind_to_its_handler(+ self, mock_export_image: MagicMock, mock_uuid: MagicMock+ ) -> None:+ handler = MagicMock()+ with patch.dict(export_handlers._export_format_handlers):+ export_handlers.register_export_format_handler(+ export_format="application/json",+ kind="test_kind",+ handler=handler,+ content_path=lambda *, team_id, asset_id, export_context: None,+ )+ asset = ExportedAsset.objects.create(+ team=self.team,+ export_format=ExportedAsset.ExportFormat.JSON,+ export_context={"kind": "test_kind"},+ )++ exporter.export_asset(asset.id)++ handler.assert_called_once_with(asset)+ mock_export_image.assert_not_called()+ asset.refresh_from_db()+ assert asset.exception is None+class TestExportAssetFailureRecording(APIBaseTest):@parameterized.expand(@@ -129,7 +155,7 @@ class TestExportAssetFailureRecording(APIBaseTest):("missing_dataset_kind",{"dataset_id": "caller-metadata"},- "JSONL exports require a dataset export context.",+ "This export can't be generated. Contact support if this keeps happening.",),("invalid_dataset_context",diff --git a/products/ai_observability/backend/api/dataset_exports.py b/products/ai_observability/backend/api/dataset_exports.pyindex d282d8579fd4..289b4ef6cad6 100644--- a/products/ai_observability/backend/api/dataset_exports.py+++ b/products/ai_observability/backend/api/dataset_exports.py@@ -1,6 +1,8 @@+from collections.abc import Mappingfrom datetime import timedeltafrom typing import Literal+from django.urls import reversefrom django.utils.timezone import nowfrom rest_framework import serializers@@ -146,3 +148,17 @@ def get_dataset_export(if str(context.get("dataset_id")) != str(dataset.id):return Nonereturn asset+++def dataset_export_content_path(*, team_id: int, asset_id: int, export_context: Mapping[str, object]) -> str | None:+ dataset_id = export_context.get("dataset_id")+ if not isinstance(dataset_id, str):+ return None+ return reverse(+ "project_datasets-export-content",+ kwargs={+ "parent_lookup_team_id": team_id,+ "pk": dataset_id,+ "export_id": asset_id,+ },+ )diff --git a/products/ai_observability/backend/apps.py b/products/ai_observability/backend/apps.pyindex 29318c16e5e4..9830141f47ac 100644--- a/products/ai_observability/backend/apps.py+++ b/products/ai_observability/backend/apps.py@@ -1,3 +1,4 @@+from collections.abc import Mappingfrom typing import TYPE_CHECKINGfrom django.apps import AppConfig@@ -14,6 +15,14 @@ def _export_dataset_jsonl(asset: "ExportedAsset") -> None:export_dataset_jsonl(asset)+def _dataset_export_content_path(*, team_id: int, asset_id: int, export_context: Mapping[str, object]) -> str | None:+ from products.ai_observability.backend.api.dataset_exports import ( # noqa: PLC0415 -- keeps the dataset API off the Django startup path+ dataset_export_content_path,+ )++ return dataset_export_content_path(team_id=team_id, asset_id=asset_id, export_context=export_context)++class AIObservabilityConfig(AppConfig):default_auto_field = "django.db.models.BigAutoField"name = "products.ai_observability.backend"@@ -25,7 +34,13 @@ class AIObservabilityConfig(AppConfig):# the ai_observability Temporal worker) so wiring it here stays off the django.setup() path.from products.ai_observability.backend import activity_logging # noqa: F401, PLC0415from products.exports.backend.facade.exporters import ( # noqa: PLC0415 -- registration belongs at app population+ DATASET_EXPORT_KIND,register_export_format_handler,)- register_export_format_handler("application/x-ndjson", _export_dataset_jsonl)+ register_export_format_handler(+ export_format="application/x-ndjson",+ kind=DATASET_EXPORT_KIND,+ handler=_export_dataset_jsonl,+ content_path=_dataset_export_content_path,+ )diff --git a/products/exports/backend/api/exports.py b/products/exports/backend/api/exports.pyindex a21c8b9bffb0..c7d1ccf1bf14 100644--- a/products/exports/backend/api/exports.py+++ b/products/exports/backend/api/exports.py@@ -7,7 +7,6 @@ from typing import Any, Literalfrom django.conf import settingsfrom django.http import HttpResponse, HttpResponseRedirectfrom django.shortcuts import get_object_or_404-from django.urls import reversefrom django.utils.timezone import nowimport structlog@@ -43,6 +42,7 @@ from products.access_control.backend.presentation.access_control import (AccessControlViewSetMixin,UserAccessControlSerializerMixin,)+from products.exports.backend.facade.exporters import get_export_content_path, is_product_exportfrom products.exports.backend.models.exported_asset import (ExportedAsset,get_content_response,@@ -184,7 +184,7 @@ class ExportedAssetSerializer(UserAccessControlSerializerMixin, serializers.Moderaise ValidationError({"insight": ["This insight does not belong to your team."]})export_context = data.get("export_context") or {}- for internal_key in ("limit_context", "execution_mode"):+ for internal_key in ("limit_context", "execution_mode", "kind"):if internal_key in export_context:raise ValidationError({"export_context": [f"{internal_key} is not supported for exports."]})if export_context.get("path") and (@@ -625,7 +625,12 @@ class ExportedAssetViewSet(instance = get_object_or_404(queryset, pk=self.kwargs["pk"])export_context = instance.export_context or {}- if instance.is_dataset_export and self.action != "content":+ # The owning product authorizes these on its own endpoints, so this API only lists them and+ # redirects their content.+ if (+ is_product_export(export_format=instance.export_format, kind=instance.export_kind)+ and self.action != "content"+ ):raise NotFound()if not instance.is_session_recording_export and instance.created_by_id != self.request.user.id:@@ -677,20 +682,18 @@ class ExportedAssetViewSet(@action(methods=["GET"], detail=True, required_scopes=["export:read"])def content(self, request: Request, *args: Any, **kwargs: Any) -> HttpResponse:instance = self.get_object()- if instance.is_dataset_export:- dataset_id = (instance.export_context or {}).get("dataset_id")- if not isinstance(dataset_id, str):- raise NotFound()- return HttpResponseRedirect(- reverse(- "project_datasets-export-content",- kwargs={- "parent_lookup_team_id": instance.team_id,- "pk": dataset_id,- "export_id": instance.id,- },- )+ if is_product_export(export_format=instance.export_format, kind=instance.export_kind):+ owner_path = get_export_content_path(+ export_format=instance.export_format,+ kind=instance.export_kind,+ team_id=instance.team_id,+ asset_id=instance.id,+ export_context=instance.export_context or {},)+ if owner_path is None:+ raise NotFound()+ query = request.META.get("QUERY_STRING")+ return HttpResponseRedirect(f"{owner_path}?{query}" if query else owner_path)return get_content_response(instance,download=request.query_params.get("download") == "true",diff --git a/products/exports/backend/api/test/test_exports.py b/products/exports/backend/api/test/test_exports.pyindex 5534dc6509ce..ec46a24f91aa 100644--- a/products/exports/backend/api/test/test_exports.py+++ b/products/exports/backend/api/test/test_exports.py@@ -44,6 +44,7 @@ from products.access_control.backend.models.access_control import AccessControlfrom products.dashboards.backend.models.dashboard import Dashboardfrom products.dashboards.backend.models.dashboard_tile import DashboardTilefrom products.exports.backend.api.exports import BLOCKING_EXPORTS_PER_TEAM, _blocking_exports_limiter+from products.exports.backend.facade import exporters as export_handlersfrom products.exports.backend.facade.api import EXPORT_WORKFLOW_TIMEOUTfrom products.exports.backend.models.exported_asset import DATASET_EXPORT_KIND, ExportedAssetfrom products.exports.backend.source_authentication import required_scopes_for_export_target@@ -502,7 +503,7 @@ class TestExports(APIBaseTest):},)- @parameterized.expand([("limit_context", "posthog_ai"), ("execution_mode", "blocking")])+ @parameterized.expand([("limit_context", "posthog_ai"), ("execution_mode", "blocking"), ("kind", "dataset")])def test_errors_if_the_request_picks_an_internal_export_setting(self, key: str, value: str) -> None:response = self.client.post(f"/api/projects/{self.team.id}/exports",@@ -823,6 +824,36 @@ class TestExports(APIBaseTest):fetch_redirect_response=False,)+ @parameterized.expand(+ [+ ("an owner path", "/api/owner/content", 302, "/api/owner/content?download=true"),+ ("no owner path", None, 404, None),+ ]+ )+ def test_registered_exports_are_hidden_and_their_content_goes_to_the_owner(+ self, _name: str, owner_path: str | None, content_status: int, location: str | None+ ) -> None:+ with patch.dict(export_handlers._export_format_handlers):+ export_handlers.register_export_format_handler(+ export_format="application/json",+ kind="test_kind",+ handler=lambda asset: None,+ content_path=lambda *, team_id, asset_id, export_context: owner_path,+ )+ asset = ExportedAsset.objects.create(+ team=self.team,+ export_format=ExportedAsset.ExportFormat.JSON,+ export_context={"kind": "test_kind"},+ created_by=self.user,+ content=b"{}",+ )++ retrieve = self.client.get(f"/api/projects/{self.team.id}/exports/{asset.id}")+ content = self.client.get(f"/api/projects/{self.team.id}/exports/{asset.id}/content?download=true")++ self.assertEqual(retrieve.status_code, status.HTTP_404_NOT_FOUND)+ self.assertEqual((content.status_code, content.get("Location")), (content_status, location))+def test_dataset_id_metadata_does_not_hide_an_ordinary_export(self) -> None:ordinary_export = ExportedAsset.objects.create(team=self.team,diff --git a/products/exports/backend/facade/api.py b/products/exports/backend/facade/api.pyindex 940fc8119922..6dc739181c6f 100644--- a/products/exports/backend/facade/api.py+++ b/products/exports/backend/facade/api.py@@ -56,6 +56,7 @@ def create_export_asset_async(created_by: User,export_format: str,export_context: dict[str, object],+ expires_after: datetime | None = None,) -> ExportedAsset:if export_format not in ExportedAsset.get_supported_format_values():raise ValueError(f"Unsupported export format: {export_format}")@@ -65,6 +66,8 @@ def create_export_asset_async(created_by=created_by,export_format=export_format,export_context=export_context,+ # None keeps the model's format-default TTL (see ExportedAsset.save).+ expires_after=ExportedAsset.round_up_expiry(expires_after) if expires_after is not None else None,)async def _start() -> None:diff --git a/products/exports/backend/facade/exporters.py b/products/exports/backend/facade/exporters.pyindex 203e2488c517..15563bb041fa 100644--- a/products/exports/backend/facade/exporters.py+++ b/products/exports/backend/facade/exporters.py@@ -1,21 +1,64 @@-from collections.abc import Callable-from typing import TYPE_CHECKING+from collections.abc import Callable, Mapping+from typing import TYPE_CHECKING, Protocol++from posthog.dataclasses import frozenif TYPE_CHECKING:from products.exports.backend.models.exported_asset import ExportedAsset+DATASET_EXPORT_KIND = "dataset"ExportFormatHandler = Callable[["ExportedAsset"], None]-_export_format_handlers: dict[str, ExportFormatHandler] = {}++class ExportContentPath(Protocol):+ def __call__(self, *, team_id: int, asset_id: int, export_context: Mapping[str, object]) -> str | None: ...+++@frozen+class _Registration:+ handler: ExportFormatHandler+ content_path: ExportContentPath+++_export_format_handlers: dict[tuple[str, str], _Registration] = {}+++def register_export_format_handler(+ *, export_format: str, kind: str, handler: ExportFormatHandler, content_path: ExportContentPath+) -> None:+ """Route exports of this format and `export_context.kind` to a product's handler.++ The product owns the export end to end: the generic exports API returns 404 when asked for it and+ redirects its content requests, query string included, to `content_path`, an endpoint the product+ authorizes. It still lists the export for its creator.+ """+ registration = _Registration(handler=handler, content_path=content_path)+ existing = _export_format_handlers.get((export_format, kind))+ if existing is not None and existing != registration:+ raise ValueError(f"An export handler is already registered for {export_format} exports of kind {kind}.")+ _export_format_handlers[(export_format, kind)] = registration+++def get_export_format_handler(*, export_format: str, kind: str | None) -> ExportFormatHandler | None:+ registration = _registration(export_format, kind)+ return registration.handler if registration is not None else None+++def is_product_export(*, export_format: str, kind: str | None) -> bool:+ return _registration(export_format, kind) is not None-def register_export_format_handler(export_format: str, handler: ExportFormatHandler) -> None:- existing_handler = _export_format_handlers.get(export_format)- if existing_handler is not None and existing_handler is not handler:- raise ValueError(f"An export handler is already registered for {export_format}.")- _export_format_handlers[export_format] = handler+def get_export_content_path(+ *, export_format: str, kind: str | None, team_id: int, asset_id: int, export_context: Mapping[str, object]+) -> str | None:+ registration = _registration(export_format, kind)+ if registration is None:+ return None+ return registration.content_path(team_id=team_id, asset_id=asset_id, export_context=export_context)-def get_export_format_handler(export_format: str) -> ExportFormatHandler | None:- return _export_format_handlers.get(export_format)+def _registration(export_format: str, kind: str | None) -> _Registration | None:+ if kind is None:+ return None+ return _export_format_handlers.get((export_format, kind))diff --git a/products/exports/backend/models/exported_asset.py b/products/exports/backend/models/exported_asset.pyindex dac072a85549..1658d639c34e 100644--- a/products/exports/backend/models/exported_asset.py+++ b/products/exports/backend/models/exported_asset.py@@ -1,6 +1,6 @@import reimport secrets-from datetime import datetime, timedelta+from datetime import UTC, datetime, timedeltafrom pathlib import Pathfrom typing import Optionalfrom urllib.parse import urlparse@@ -24,13 +24,14 @@ from posthog.storage import object_storagefrom posthog.storage.object_storage import ObjectStorageErrorfrom posthog.utils import absolute_uri+from products.exports.backend.facade.exporters import DATASET_EXPORT_KIND as DATASET_EXPORT_KIND+logger = structlog.get_logger(__name__)PUBLIC_ACCESS_TOKEN_EXP_DAYS = 365MAX_AGE_CONTENT = 86400 # 1 dayEXPORTED_ASSET_PURPOSE_RENDER = "render"EXPORTED_ASSET_PURPOSE_SUBSCRIPTION_DELIVERY = "subscription_delivery"-DATASET_EXPORT_KIND = "dataset"_EXPIRY_DELETE_BATCH = 1000@@ -175,10 +176,14 @@ class ExportedAsset(models.Model):@classmethoddef compute_expires_after(cls, export_format: str) -> datetime:+ return cls.round_up_expiry(now() + cls.get_expiry_delta(export_format))++ @staticmethod+ def round_up_expiry(expiry: datetime) -> datetime:# Rounded up, because S3 rounds a lifecycle rule up to the next UTC midnight; rounding down# retires the row while the object it points at is still there.- expiry_datetime = now() + cls.get_expiry_delta(export_format) + timedelta(days=1)- return expiry_datetime.replace(hour=0, minute=0, second=0, microsecond=0)+ next_day = expiry.astimezone(UTC) + timedelta(days=1)+ return next_day.replace(hour=0, minute=0, second=0, microsecond=0)@propertydef has_content(self):@@ -224,12 +229,14 @@ class ExportedAsset(models.Model):return DATASET_EXPORT_KINDreturn "unknown"+ @property+ def export_kind(self) -> str | None:+ kind = (self.export_context or {}).get("kind")+ return kind if isinstance(kind, str) and kind else None+@propertydef is_dataset_export(self) -> bool:- return (- self.export_format == self.ExportFormat.JSONL- and (self.export_context or {}).get("kind") == DATASET_EXPORT_KIND- )+ return self.export_format == self.ExportFormat.JSONL and self.export_kind == DATASET_EXPORT_KIND@propertydef is_rasterized_export(self) -> bool:@@ -248,6 +255,7 @@ class ExportedAsset(models.Model):"export_format": self.export_format,"dashboard_id": self.dashboard_id,"insight_id": self.insight_id,+ "export_kind": self.export_kind,# Scanner-driven renders (replay_vision) report through the same pipeline; without this# flag their volume is indistinguishable from user exports in failure-rate comparisons."is_system": bool(self.is_system),diff --git a/products/exports/backend/stuck_exports.py b/products/exports/backend/stuck_exports.pyindex e4bae4b814d8..70315ddb68b9 100644--- a/products/exports/backend/stuck_exports.py+++ b/products/exports/backend/stuck_exports.py@@ -10,6 +10,7 @@ from posthog.temporal.session_replay.rasterize_recording.types import RASTERIZE_from products.exports.backend.analytics import capture_export_eventfrom products.exports.backend.facade.api import EXPORT_WORKFLOW_TIMEOUT+from products.exports.backend.facade.exporters import is_product_exportfrom products.exports.backend.models.exported_asset import ExportedAssetfrom products.exports.backend.tasks.failure_handler import FAILURE_TYPE_TIMEOUT_GENERATION@@ -29,12 +30,12 @@ def stuck_export_threshold(instance: ExportedAsset) -> timedelta:"""How long an export may sit without content before nothing can still be working on it.Matched to whichever pipeline renders the format: video renders get the rasterize workflow's- envelope, dataset exports the export workflow's, and everything else the HogQL query timeout it- inherits from the Celery exporter.+ envelope, product-owned exports get the export workflow's, and everything else gets the HogQL+ query timeout it inherits from the Celery exporter."""if instance.is_rasterized_export:return RASTERIZE_WORKFLOW_TIMEOUT- if instance.is_dataset_export:+ if is_product_export(export_format=instance.export_format, kind=instance.export_kind):return EXPORT_WORKFLOW_TIMEOUTreturn timedelta(seconds=HOGQL_INCREASED_MAX_EXECUTION_TIME)diff --git a/products/exports/backend/tests/test_exporters.py b/products/exports/backend/tests/test_exporters.pynew file mode 100644index 000000000000..bbd09c4ad0ce--- /dev/null+++ b/products/exports/backend/tests/test_exporters.py@@ -0,0 +1,49 @@+from collections.abc import Mapping++from unittest.mock import MagicMock, patch++from django.test import SimpleTestCase++from parameterized import parameterized++from products.exports.backend.facade import exporters+++def _content_path(*, team_id: int, asset_id: int, export_context: Mapping[str, object]) -> str:+ return f"/api/projects/{team_id}/owner/{asset_id}/content"+++class TestExportFormatHandlers(SimpleTestCase):+ def setUp(self) -> None:+ registry_patch = patch.dict(exporters._export_format_handlers, clear=True)+ registry_patch.start()+ self.addCleanup(registry_patch.stop)++ def _register(self, handler: MagicMock) -> None:+ exporters.register_export_format_handler(+ export_format="application/json", kind="test_kind", handler=handler, content_path=_content_path+ )++ @parameterized.expand(+ [+ ("the registered pair", "application/json", "test_kind", True),+ ("the same kind in another format", "application/x-ndjson", "test_kind", False),+ ("another kind", "application/json", "other_kind", False),+ ("no kind", "application/json", None, False),+ ]+ )+ def test_finds_a_handler_by_format_and_kind(+ self, _name: str, export_format: str, kind: str | None, found: bool+ ) -> None:+ handler = MagicMock()+ self._register(handler)++ assert (exporters.get_export_format_handler(export_format=export_format, kind=kind) is handler) == found++ def test_registering_the_same_handler_again_is_allowed_but_a_different_one_is_refused(self) -> None:+ handler = MagicMock()+ self._register(handler)+ self._register(handler)++ with self.assertRaises(ValueError):+ self._register(MagicMock())diff --git a/products/exports/backend/tests/test_facade.py b/products/exports/backend/tests/test_facade.pyindex 4a6cb7b8b8bf..a165917abb3f 100644--- a/products/exports/backend/tests/test_facade.py+++ b/products/exports/backend/tests/test_facade.py@@ -1,8 +1,9 @@from copy import deepcopy-from datetime import timedelta+from datetime import UTC, datetime, timedelta+import time_machinefrom posthog.test.base import BaseTest-from unittest.mock import MagicMock, patch+from unittest.mock import AsyncMock, MagicMock, patchfrom django.test import SimpleTestCase@@ -14,6 +15,7 @@ from posthog.models.user import Userfrom products.dashboards.backend.models.dashboard import Dashboardfrom products.exports.backend.facade.api import (_validate_adhoc_export_context,+ create_export_asset_async,get_delivery_image_url,query_with_reference_lines,render_png_export,@@ -169,6 +171,23 @@ class TestRenderPngExportInsightLookup(BaseTest):render_png_export(team=self.team, created_by=self.user, **kwargs)+class TestCreateExportAssetAsync(BaseTest):+ @time_machine.travel(datetime(2026, 3, 2, 12, 0, tzinfo=UTC), tick=False)+ @patch("products.exports.backend.facade.api.async_connect", new_callable=AsyncMock)+ def test_create_export_asset_async_rounds_an_explicit_expiry_up_to_utc_midnight(+ self, async_connect: AsyncMock+ ) -> None:+ asset = create_export_asset_async(+ team=self.team,+ created_by=self.user,+ export_format=ExportedAsset.ExportFormat.JSON,+ export_context={"kind": "test_kind"},+ expires_after=datetime(2026, 3, 9, 12, 0, tzinfo=UTC),+ )++ assert asset.expires_after == datetime(2026, 3, 10, 0, 0, tzinfo=UTC)++class TestGetDeliveryImageUrl(BaseTest):def test_created_by_filter_pins_the_asset_to_its_renderer(self):other = User.objects.create(email="[email protected]")