:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(model): Add model-specific export types (#115541)
Tomás Farías Santanafabe7fdc3e01
diff --git a/products/batch_exports/backend/service.py b/products/batch_exports/backend/service.pyindex 730d641fd983..b38a2a1b5124 100644--- a/products/batch_exports/backend/service.py+++ b/products/batch_exports/backend/service.py@@ -3,6 +3,7 @@ import typesimport typingimport datetime as dtfrom dataclasses import Field, asdict, dataclass, field, fields+from typing import Literalfrom uuid import UUIDfrom django.conf import settings@@ -37,6 +38,7 @@ from posthog.temporal.common.schedule import ()from products.batch_exports.backend.facade.contracts import AWSCredentials+from products.batch_exports.backend.facade.enums import IncrementalModefrom products.batch_exports.backend.models.batch_export import (BatchExport,BatchExportBackfill,@@ -152,6 +154,164 @@ class BatchExportModel:hogql_modifiers: dict[str, typing.Any] | None = None+class _BatchExportModelProtocol(typing.Protocol):+ """Read-only protocol covering all batch export models."""++ @property+ def name(self) -> str: ...++ @property+ def incremental_mode(self) -> IncrementalMode: ...++ @property+ def primary_key(self) -> tuple[str, ...]: ...++ @property+ def version_key(self) -> tuple[str, ...]: ...+++@frozen+class BatchExportEventsModel:+ # NOTE: In this and classes that follow, 'name' should really be a class+ # variable. However, fields() and asdict() exclude attributes with ClassVar+ # annotations. In the prescence of a type union, this would make it+ # possible for Temporal to accidentaly deserialize into the wrong class, if+ # we are not strict with type hints. So, to have the additional check 'name'+ # provides, I would use a field(init=False) for name to approximate a class+ # variable that is included in serialization/deserialization. However, this+ # is not supported in Temporal==1.34.0. A fix has landed on main but has not+ # released yet.+ # TODO: On temporal==1.35.0 (or later) add init=False.+ name: Literal["events"] = field(default="events")+ # NOTE: primary_key and version_key are hardcoded for pre-defined models.+ # This maintains backwards compatibility, and rejects any user input for+ # models maintained by PostHog.+ primary_key: tuple[Literal["uuid"]] = field(default=("uuid",))+ version_key: tuple[Literal["timestamp"]] = field(default=("timestamp",))+ incremental_mode: IncrementalMode = IncrementalMode.APPEND+ filters: list[dict[str, str | bool | float | list[str | bool | float] | None]] = field(default_factory=list)+++@frozen+class BatchExportEventsCustomModel:+ name: Literal["events"] = field(default="events")+ schema: BatchExportSchema+ # NOTE: With a custom schema, we can't know if any of the key fields are+ # present or not, so we enforce an empty key just to keep up with the+ # protocol.+ primary_key: tuple[str, ...] = field(default=())+ version_key: tuple[str, ...] = field(default=())+ incremental_mode: IncrementalMode = field(default=IncrementalMode.APPEND)+ filters: list[dict[str, str | bool | float | list[str | bool | float] | None]] = field(default_factory=list)++ def __post_init__(self) -> None:+ if self.incremental_mode != IncrementalMode.APPEND:+ # NOTE: Ideally, we would express this restriction using different+ # types: an append model and a merge model as required. But Temporal+ # encoding/decoding cannot reconstruct the types correctly, as+ # Literal types just pass their SERIALIZED value as-is. So we+ # have to validate. At least the validation happens at construction+ # (or right after) so if you have a model instance, it's valid!+ raise ValueError("Model is APPEND-only")+++def _validate_merge_only_model(model: _BatchExportModelProtocol) -> None:+ if model.incremental_mode != IncrementalMode.MERGE:+ raise ValueError("Model is MERGE-only")++ _validate_keys(model)+++def _validate_keys(model: _BatchExportModelProtocol) -> None:+ if not model.primary_key:+ raise ValueError("MERGE requires a primary key")+++@frozen+class BatchExportPersonsModel:+ name: Literal["persons"] = field(default="persons")+ primary_key: tuple[Literal["team_id"], Literal["distinct_id"]] = field(default=("team_id", "distinct_id"))+ version_key: tuple[Literal["person_version"], Literal["person_distinct_id_version"]] = field(+ default=(+ "person_version",+ "person_distinct_id_version",+ ),+ )+ # NOTE: I am not sure if appending makes sense for persons or sessions, so+ # this is hardcoded to merge. Happy to be corrected!+ incremental_mode: IncrementalMode = field(default=IncrementalMode.MERGE)++ def __post_init__(self) -> None:+ _validate_merge_only_model(self)+++@frozen+class BatchExportSessionsModel:+ name: Literal["sessions"] = field(default="sessions")+ primary_key: tuple[Literal["team_id"], Literal["session_id"]] = field(default=("team_id", "session_id"))+ version_key: tuple[Literal["end_timestamp"]] = field(default=("end_timestamp",))+ incremental_mode: IncrementalMode = field(default=IncrementalMode.MERGE)++ def __post_init__(self):+ _validate_merge_only_model(self)+++@frozen+class BatchExportHogQLModel:+ name: Literal["hogql"] = field(default="hogql")+ query: str+ user_id: int+ modifiers: dict[str, typing.Any]+ primary_key: tuple[str, ...] = ()+ version_key: tuple[str, ...] = ()+ incremental_mode: IncrementalMode = IncrementalMode.APPEND++ def __post_init__(self) -> None:+ if self.incremental_mode == IncrementalMode.MERGE:+ _validate_keys(self)+++AnyBatchExportModel = (+ BatchExportEventsCustomModel+ | BatchExportEventsModel+ | BatchExportPersonsModel+ | BatchExportSessionsModel+ | BatchExportHogQLModel+)++BatchExportModelInput = AnyBatchExportModel | BatchExportModel+++def migrate_batch_export_model(old: BatchExportModelInput | None, /) -> AnyBatchExportModel:+ if old is None:+ return BatchExportEventsModel()+ if isinstance(old, AnyBatchExportModel):+ return old++ match old.name:+ case "events":+ if old.schema is not None:+ return BatchExportEventsCustomModel(schema=old.schema, filters=old.filters or [])+ return BatchExportEventsModel(filters=old.filters or [])+ case "persons":+ return BatchExportPersonsModel()+ case "sessions":+ return BatchExportSessionsModel()+ case "hogql":+ # These two should be set in the model, but we must be strict.+ if old.user_id is None:+ raise ValueError("HogQL model requires user_id")+ if old.hogql_query is None:+ raise ValueError("HogQL model requires a query")+ return BatchExportHogQLModel(+ query=old.hogql_query,+ user_id=old.user_id,+ modifiers=old.hogql_modifiers if old.hogql_modifiers is not None else {},+ )+ case s:+ raise ValueError(f"Invalid model name: '{s}'")++@dataclassclass BackfillDetails:backfill_id: str | None@@ -202,7 +362,7 @@ class BaseBatchExportInputs:batch_export_model: BatchExportModel | None = Noneintegration_id: int | None = None- def __post_init__(self):+ def __post_init__(self) -> None:"""Coerce string values to their declared types.EncryptedJSONField may not preserve Python types, so fields can@@ -411,7 +571,7 @@ class RedshiftBatchExportInputs(BaseBatchExportInputs):mode: typing.Literal["COPY", "INSERT"] = "INSERT"copy_inputs: RedshiftCopyInputs | None = None- def __post_init__(self):+ def __post_init__(self) -> None:super().__post_init__()if (self.mode == "COPY"diff --git a/products/batch_exports/backend/tests/test_service.py b/products/batch_exports/backend/tests/test_service.pyindex 956174e8a9cc..a90e73113c87 100644--- a/products/batch_exports/backend/tests/test_service.py+++ b/products/batch_exports/backend/tests/test_service.py@@ -5,6 +5,7 @@ import pytestfrom temporalio.converter import JSONPlainPayloadConverter+from products.batch_exports.backend.facade.enums import IncrementalModefrom products.batch_exports.backend.models.batch_export import (BatchExport,BatchExportBackfill,@@ -12,9 +13,15 @@ from products.batch_exports.backend.models.batch_export import (BatchExportRun,)from products.batch_exports.backend.service import (+ AnyBatchExportModel,AWSCredentials,AzureBlobBatchExportInputs,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportHogQLModel,BatchExportModel,+ BatchExportPersonsModel,+ BatchExportSessionsModel,BigQueryBatchExportInputs,DatabricksBatchExportInputs,PostgresBatchExportInputs,@@ -24,8 +31,102 @@ from products.batch_exports.backend.service import (S3CompatibleBatchExportInputs,aget_or_create_batch_export_backfill,align_timestamp_to_interval,+ migrate_batch_export_model,)++class TestBatchExportModelMigration:+ @pytest.mark.parametrize(+ "legacy,expected",+ [+ (None, BatchExportEventsModel()),+ (BatchExportModel(name="events", schema=None), BatchExportEventsModel()),+ (+ BatchExportModel(name="events", schema=None, filters=[{"key": "amount", "value": [1.5, True]}]),+ BatchExportEventsModel(filters=[{"key": "amount", "value": [1.5, True]}]),+ ),+ (+ BatchExportModel(name="events", schema={"fields": [], "values": {"value": 1.5}}),+ BatchExportEventsCustomModel(schema={"fields": [], "values": {"value": 1.5}}),+ ),+ (BatchExportModel(name="persons", schema=None), BatchExportPersonsModel()),+ (BatchExportModel(name="sessions", schema=None), BatchExportSessionsModel()),+ (+ BatchExportModel(name="hogql", schema=None, hogql_query="SELECT event FROM events", user_id=42),+ BatchExportHogQLModel(query="SELECT event FROM events", user_id=42, modifiers={}),+ ),+ (+ BatchExportModel(+ name="hogql",+ schema=None,+ hogql_query="SELECT event FROM events",+ user_id=42,+ hogql_modifiers={"convertToProjectTimezone": False},+ ),+ BatchExportHogQLModel(+ query="SELECT event FROM events",+ user_id=42,+ modifiers={"convertToProjectTimezone": False},+ ),+ ),+ ],+ )+ def test_migration_preserves_legacy_payload(+ self, legacy: BatchExportModel | None, expected: AnyBatchExportModel+ ) -> None:+ converter = JSONPlainPayloadConverter()+ if legacy is not None:+ payload = converter.to_payload(legacy)+ assert payload is not None+ legacy = converter.from_payload(payload, BatchExportModel)+ migrated = migrate_batch_export_model(legacy)+ assert migrated == expected+ assert migrate_batch_export_model(migrated) is migrated++ @pytest.mark.parametrize(+ "legacy,message",+ [+ (BatchExportModel(name="unknown", schema=None), "Invalid model name"),+ (BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1"), "requires user_id"),+ (BatchExportModel(name="hogql", schema=None, user_id=42), "requires a query"),+ ],+ )+ def test_invalid_legacy_model_is_rejected(self, legacy: BatchExportModel, message: str) -> None:+ with pytest.raises(ValueError, match=message):+ migrate_batch_export_model(legacy)++ @pytest.mark.parametrize("version_key", [(), ("updated_at",)])+ def test_hogql_merge_requires_primary_key(self, version_key: tuple[str, ...]) -> None:+ with pytest.raises(ValueError, match="MERGE requires a primary key"):+ BatchExportHogQLModel(+ query="SELECT 1",+ user_id=42,+ modifiers={},+ incremental_mode=IncrementalMode.MERGE,+ primary_key=(),+ version_key=version_key,+ )++ @pytest.mark.parametrize("version_key", [(), ("updated_at",)])+ def test_migration_preserves_hogql_merge_settings(self, version_key: tuple[str, ...]) -> None:+ model = BatchExportHogQLModel(+ query="SELECT id, updated_at FROM events",+ user_id=42,+ modifiers={},+ incremental_mode=IncrementalMode.MERGE,+ primary_key=("id",),+ version_key=version_key,+ )+ assert model.incremental_mode == IncrementalMode.MERGE+ assert model.primary_key == ("id",)+ assert model.version_key == version_key+ assert migrate_batch_export_model(model) is model+ converter = JSONPlainPayloadConverter()+ payload = converter.to_payload(model)+ assert payload is not None+ assert converter.from_payload(payload, BatchExportHogQLModel) == model++DESTINATION_INPUTS = {"Databricks": DatabricksBatchExportInputs(batch_export_id="test",