:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
refactor(batch-exports): read export constants and config checks through the facade (#122666)
Ross0b8cd37acce5
diff --git a/products/batch_exports/backend/facade/api.py b/products/batch_exports/backend/facade/api.pyindex 1d3c4ef824ef..6e50a2e6a18a 100644--- a/products/batch_exports/backend/facade/api.py+++ b/products/batch_exports/backend/facade/api.py@@ -11,8 +11,9 @@ is visible in one place.and ``temporalio`` on the ``django.setup()`` path. Core config-time consumers import thisfacade, so it reaches the service only through ``_service()``, which imports it at calltime. For the same reason the Temporal client is built in ``_get_temporal_client()``, and-the HogQL, file download, run and backfill modules load through ``_import_hogql_source()``,-``_import_file_downloads()``, ``_import_runs()`` and ``_import_backfills()``.+the HogQL, destination config, file download, run and backfill modules load through+``_import_hogql_source()``, ``_import_destination_config()``, ``_import_file_downloads()``,+``_import_runs()`` and ``_import_backfills()``.Temporal workflow and activity registration crosses as objects, not data, so it lives in``facade/temporal.py``. The destination clients the warehouse writers reuse live in@@ -35,9 +36,11 @@ import structlogfrom posthog.models import Team, Userfrom posthog.models.integration import Integration-from products.batch_exports.backend import destination_tests+from products.batch_exports.backend import (+ destination_tests,+ filters as batch_export_filters,+)from products.batch_exports.backend.billing import exclude_non_billable_runs-from products.batch_exports.backend.filters import validate_batch_export_filtersfrom products.batch_exports.backend.models.batch_export import (BATCH_EXPORT_INTERVALS,BatchExport,@@ -59,6 +62,7 @@ __all__ = ["cancel_batch_export_backfill","cancel_batch_export_run","cancel_file_download",+ "coerce_destination_config","count_batch_exports_for_teams","count_hogql_rows","create_batch_export",@@ -89,6 +93,8 @@ __all__ = ["retry_batch_export_run","run_destination_test_step","start_file_download",+ "validate_batch_export_filters",+ "validate_destination_config","validate_hogql_query",]@@ -135,6 +141,15 @@ def _import_hogql_source() -> ModuleType:return hogql_source+def _import_destination_config() -> ModuleType:+ """Import the destination config checks at call time. See the module docstring for why."""+ from products.batch_exports.backend.logic import ( # noqa: PLC0415 — keeps the service layer and temporalio off the django.setup() path+ destination_config,+ )++ return destination_config++def _import_file_downloads() -> ModuleType:"""Import the file download module at call time. See the module docstring for why."""from products.batch_exports.backend.logic import file_downloads@@ -422,6 +437,15 @@ def create_batch_export(return _to_detail(batch_export)+def validate_batch_export_filters(filters: object) -> None:+ """Check the event filters a batch export stores.++ Raises ``contracts.InvalidBatchExportFilters`` unless ``filters`` is None or a list of+ supported filters.+ """+ batch_export_filters.validate_batch_export_filters(filters)++def create_workflows_backfill_export(team_id: int,*,@@ -478,6 +502,25 @@ def list_supported_intervals() -> tuple[str, ...]:return tuple(interval for interval, _ in BATCH_EXPORT_INTERVALS)+def validate_destination_config(destination_type: str, config: Mapping[str, Any], *, partial: bool) -> dict[str, Any]:+ """Check a destination config against the workflow inputs of its destination type.++ With ``partial``, a required field may be missing, as in a patch. Returns a copy of the+ config with values converted to the declared types. Raises+ ``contracts.InvalidDestinationConfigError`` when a field is unknown, missing or of the+ wrong type.+ """+ return _import_destination_config().validate_destination_config(destination_type, config, partial=partial)+++def coerce_destination_config(destination_type: str, config: dict[str, Any]) -> dict[str, Any]:+ """Return a stored destination config with its values restored to their declared types.++ The stored config reads back with stringified scalars, so ``"False"`` becomes ``False``.+ """+ return _service().coerce_config_to_declared_types(destination_type, config)++def describe_destination_test(destination_type: str) -> contracts.DestinationTest:"""Return the steps of a destination's connection test, with no step run."""destination_test = destination_tests.get_destination_test(destination=destination_type)diff --git a/products/batch_exports/backend/facade/contracts.py b/products/batch_exports/backend/facade/contracts.pyindex 3a3a5f44c352..3b82bd06bd73 100644--- a/products/batch_exports/backend/facade/contracts.py+++ b/products/batch_exports/backend/facade/contracts.py@@ -33,6 +33,13 @@ class InvalidBatchExportFilters(ValueError):"""Raised when event filters are not a list of filters a batch export can apply."""+class InvalidDestinationConfigError(ValueError):+ """Raised when a destination config does not match the workflow inputs of its destination type.++ The message says which field is wrong.+ """++class UnsupportedDestinationTestError(ValueError):"""Raised when a destination type has no connection test."""diff --git a/products/batch_exports/backend/facade/destinations/azure_blob.py b/products/batch_exports/backend/facade/destinations/azure_blob.pyindex cb91c1b4b353..7f101a01d466 100644--- a/products/batch_exports/backend/facade/destinations/azure_blob.py+++ b/products/batch_exports/backend/facade/destinations/azure_blob.py@@ -8,13 +8,13 @@ Importing this module loads the destination's vendor SDK, so keep it off the``django.setup()`` path. See ``posthog/test/repo_invariants/test_startup_import_budget.py``."""+from products.batch_exports.backend.facade.enums import AZURE_BLOB_SUPPORTED_COMPRESSIONSfrom products.batch_exports.backend.temporal.destinations.azure_blob_batch_export import (AzureBlobIntegrationNotFoundError,MalformedConnectionStringError,_get_azure_blob_integration as get_azure_blob_integration,_is_authorization_failure_response_error as is_authorization_failure_response_error,)-from products.batch_exports.backend.temporal.destinations.constants import AZURE_BLOB_SUPPORTED_COMPRESSIONS__all__ = ["AZURE_BLOB_SUPPORTED_COMPRESSIONS",diff --git a/products/batch_exports/backend/facade/destinations/s3.py b/products/batch_exports/backend/facade/destinations/s3.pyindex e7d2896b8799..df9d726f855c 100644--- a/products/batch_exports/backend/facade/destinations/s3.py+++ b/products/batch_exports/backend/facade/destinations/s3.py@@ -8,7 +8,7 @@ Importing this module loads the destination's vendor SDK, so keep it off the``django.setup()`` path. See ``posthog/test/repo_invariants/test_startup_import_budget.py``."""-from products.batch_exports.backend.temporal.destinations.constants import S3_SUPPORTED_COMPRESSIONS+from products.batch_exports.backend.facade.enums import S3_SUPPORTED_COMPRESSIONSfrom products.batch_exports.backend.temporal.destinations.s3_batch_export import (ConcurrentS3Consumer,ExternalRoleArnNotConfiguredError,diff --git a/products/batch_exports/backend/facade/enums.py b/products/batch_exports/backend/facade/enums.pyindex 276cf17388f4..9847bb2eb06b 100644--- a/products/batch_exports/backend/facade/enums.py+++ b/products/batch_exports/backend/facade/enums.py@@ -1,11 +1,16 @@-"""Enumerations the batch_exports models use.+"""Enumerations and fixed value sets the batch_exports models and API use.A consumer that writes one of these values reads the enum from here and does not need themodel class. The models keep each enum as a class attribute (``BatchExportRun.Status``) soexisting call sites are unchanged. In an annotation the class attribute is not a valid type, soannotate with the enum from this module directly.++The value sets at the end of the module are the single source for the models, the Temporal+destinations and the API serializers, so they must stay free of framework imports."""+import pytz+from posthog.enums import LabeledStrEnum@@ -78,3 +83,64 @@ class IncrementalMode(LabeledStrEnum):APPEND = "APPEND"MERGE = "MERGE"# TODO: Add 'FULL_REFRESH': data is re-created on every run.+++# this is what is used by the Team model+# (we could use common_timezones instead; this has 433 timezones vs 596 for all_timezones)+TIMEZONES = [(tz, tz) for tz in pytz.all_timezones]++S3_FAMILY_TYPES: frozenset[str] = frozenset(+ {BatchExportDestinationType.AWS_S3, BatchExportDestinationType.S3_COMPATIBLE}+)++# Destinations that write files to object storage, as opposed to the data warehouse destinations+# (BigQuery, Databricks, Postgres, Redshift, Snowflake) that write rows into a table. File format,+# compression and object key layout are questions only this half has to answer.+#+# Also includes "FileDownload", which writes to a PostHog bucket rather than a customer one.+# TODO: it probably makes sense to introduce a 'DestinationKind' abstraction in the future,+# to properly categorize object storage and data warehouse destinations.+OBJECT_STORAGE_DESTINATIONS: frozenset[str] = frozenset(+ {+ BatchExportDestinationType.AWS_S3,+ BatchExportDestinationType.S3_COMPATIBLE,+ BatchExportDestinationType.AZURE_BLOB,+ BatchExportDestinationType.FILE_DOWNLOAD,+ }+)++S3_SUPPORTED_COMPRESSIONS: dict[str, list[str]] = {+ "Parquet": ["zstd", "lz4", "snappy", "gzip", "brotli"],+ "JSONLines": ["gzip", "brotli"],+}++AZURE_BLOB_SUPPORTED_COMPRESSIONS: dict[str, list[str]] = {+ "Parquet": ["zstd", "lz4", "snappy", "gzip", "brotli"],+ "JSONLines": ["gzip", "brotli"],+}++# Projected columns a HogQL query can select.+# NOTE: this is just for the 'old' version of our HogQL support, where a user can define a custom+# SELECT FROM clause, useful for defining a custom schema. It's not related to the new HogQL model,+# in which a user can use any arbitrary SELECT query.+EXPORTABLE_EVENTS_MODEL_FIELDS = frozenset(+ {+ "team_id",+ "timestamp",+ "event",+ "distinct_id",+ "uuid",+ "created_at",+ "elements_chain",+ "person_id",+ "properties",+ "person_properties",+ }+)++# Single source of truth for the serialized filter `type` values we accept. Enforced at write+# time by `validate_batch_export_filters` and at query time by `compose_filters_clause`.+SUPPORTED_FILTER_TYPES = frozenset({"event", "person", "hogql"})++# Quoted, sorted rendering of the supported filter types for user-facing messages.+SUPPORTED_FILTER_TYPES_DISPLAY = ", ".join(repr(t) for t in sorted(SUPPORTED_FILTER_TYPES))diff --git a/products/batch_exports/backend/filters.py b/products/batch_exports/backend/filters.pyindex 1fee7ec895af..1eeccdb831e9 100644--- a/products/batch_exports/backend/filters.py+++ b/products/batch_exports/backend/filters.py@@ -1,19 +1,13 @@"""The event filters a batch export stores, and the checks they must pass on write.-This module imports no framework and no Temporal code, so the DRF serializer and the facade-share one set of checks. ``service.py`` imports ``temporalio``, so the checks cannot live-there: the facade is on the ``django.setup()`` path.+This module imports no framework and no Temporal code, because the facade imports it and the+facade is on the ``django.setup()`` path. ``service.py`` imports ``temporalio``, so the checks+cannot live there."""from products.batch_exports.backend.facade.contracts import InvalidBatchExportFilters--# Single source of truth for the serialized filter `type` values we accept. Enforced at write-# time by `validate_batch_export_filters` and at query time by `compose_filters_clause`.-SUPPORTED_FILTER_TYPES = {"event", "person", "hogql"}--# Quoted, sorted rendering of the supported filter types for user-facing messages.-SUPPORTED_FILTER_TYPES_DISPLAY = ", ".join(repr(t) for t in sorted(SUPPORTED_FILTER_TYPES))+from products.batch_exports.backend.facade.enums import SUPPORTED_FILTER_TYPES, SUPPORTED_FILTER_TYPES_DISPLAYdef validate_batch_export_filters(filters: object) -> None:diff --git a/products/batch_exports/backend/logic/destination_config.py b/products/batch_exports/backend/logic/destination_config.pynew file mode 100644index 000000000000..e42ce78e8d0f--- /dev/null+++ b/products/batch_exports/backend/logic/destination_config.py@@ -0,0 +1,105 @@+"""The config a batch export destination stores: check it against the workflow inputs.++The facade reaches this module through a call-time import, because the workflow input+dataclasses live in the service layer, which may not load on the ``django.setup()`` path.+"""++import typing+import builtins+import dataclasses+from collections.abc import Callable, Mapping++from posthog.utils import str_to_bool++from products.batch_exports.backend.facade import contracts+from products.batch_exports.backend.service import DESTINATION_WORKFLOWS, BaseBatchExportInputs+++def _try_convert_to_type(value: typing.Any, target_type: type) -> tuple[typing.Any, bool]:+ """Attempt to convert value to target type based on well-known casting functions.++ This doesn't raise any exceptions but rather returns a tuple with a bool indicating+ if the value in the first position was successfully casted to `target_type` or not.+ If casting fails, the value in the first position is returned unchanged, otherwise a+ new value of type `target_type` is returned.+ """+ current_type = type(value)++ match (current_type, target_type):+ case (builtins.str, builtins.bool):+ cast_func: Callable[[typing.Any], typing.Any] = str_to_bool+ case (builtins.str, builtins.int):+ cast_func = int+ case _:+ return (value, False)++ try:+ new_value = cast_func(value)+ except Exception:+ return (value, False)++ return (new_value, True)+++def validate_destination_config(+ destination_type: str, config: Mapping[str, typing.Any], *, partial: bool+) -> dict[str, typing.Any]:+ """Check a destination config against the workflow inputs of its destination type.++ Ensure that the config passes the following checks:+ * Does NOT contain fields that do not exist in workflow inputs.+ * Contains all required fields as defined by workflow inputs, unless ``partial``.+ * Provided values match types required by workflow inputs.++ Returns a copy of the config with values converted to the declared types.++ Raises:+ A `contracts.InvalidDestinationConfigError` if any of these checks fail.+ """+ validated = dict(config)++ _, workflow_inputs = DESTINATION_WORKFLOWS[destination_type]+ base_field_names = {field.name for field in dataclasses.fields(BaseBatchExportInputs)}+ workflow_fields = dataclasses.fields(workflow_inputs)+ destination_fields = {field for field in workflow_fields if field.name not in base_field_names}++ extra_fields = validated.keys() - {field.name for field in destination_fields} - base_field_names+ if extra_fields:+ str_fields = ", ".join(f"'{extra_field}'" for extra_field in sorted(extra_fields))+ raise contracts.InvalidDestinationConfigError(f"Configuration has unknown field/s: {str_fields}")++ # Destination config fields without a dataclass default must be provided.+ for destination_field in destination_fields:+ is_required = (+ destination_field.default == dataclasses.MISSING+ and destination_field.default_factory == dataclasses.MISSING+ )+ if destination_field.name not in validated:+ if is_required and not partial:+ # When patching we expect a partial configuration. So, we don't+ # error on missing required fields.+ raise contracts.InvalidDestinationConfigError(+ f"Configuration missing required field: '{destination_field.name}'"+ )+ else:+ continue++ config_value = validated[destination_field.name]+ field_type = destination_field.type++ if not isinstance(field_type, type):+ # `dataclasses.Field.type` could be something we can't work with.+ # TODO: Validate these ones too?+ continue++ if not isinstance(config_value, field_type):+ config_value, success = _try_convert_to_type(config_value, field_type)++ if not success:+ raise contracts.InvalidDestinationConfigError(+ f"Configuration has invalid type: got '{type(config_value).__name__}', expected '{field_type.__name__}'"+ )++ validated[destination_field.name] = config_value++ return validateddiff --git a/products/batch_exports/backend/models/batch_export.py b/products/batch_exports/backend/models/batch_export.pyindex 7076d53ee353..32a93faeabfa 100644--- a/products/batch_exports/backend/models/batch_export.py+++ b/products/batch_exports/backend/models/batch_export.py@@ -9,8 +9,6 @@ from django.contrib.postgres.fields import ArrayFieldfrom django.db import modelsfrom django.db.models import Q-import pytz-# This import prevents a circular import during Django app loading.# The import chain: apps.py -> tasks -> async_migrations/definition.py -> models.utils# triggers models/__init__.py which imports batch_exports.models. Without this import,@@ -22,21 +20,7 @@ from posthog.models.scoping.root_mixin import TeamScopedRootMixinfrom posthog.models.utils import UUIDTModelfrom products.batch_exports.backend.facade import enums--# this is what is used by the Team model-# (we could use common_timezones instead; this has 433 timezones vs 596 for all_timezones)-TIMEZONES = [(tz, tz) for tz in pytz.all_timezones]--S3_FAMILY_TYPES: frozenset[str] = frozenset({"AwsS3", "S3Compatible"})--# Destinations that write files to object storage, as opposed to the data warehouse destinations-# (BigQuery, Databricks, Postgres, Redshift, Snowflake) that write rows into a table. File format,-# compression and object key layout are questions only this half has to answer.-#-# Also includes "FileDownload", which writes to a PostHog bucket rather than a customer one.-# TODO: it probably makes sense to introduce a 'DestinationKind' abstraction in the future,-# to properly categorize object storage and data warehouse destinations.-OBJECT_STORAGE_DESTINATIONS: frozenset[str] = frozenset({"AwsS3", "S3Compatible", "AzureBlob", "FileDownload"})+from products.batch_exports.backend.facade.enums import TIMEZONESclass DayOfWeek(IntEnum):diff --git a/products/batch_exports/backend/presentation/views/batch_export/destinations.py b/products/batch_exports/backend/presentation/views/batch_export/destinations.pyindex 24e9ef1b2279..7c9539927a8f 100644--- a/products/batch_exports/backend/presentation/views/batch_export/destinations.py+++ b/products/batch_exports/backend/presentation/views/batch_export/destinations.py@@ -1,8 +1,6 @@"""Batch export destination serializers: the typed config and request shape of each destination type."""import typing-import builtins-import dataclassesimport collections.abcfrom django.db import models@@ -12,18 +10,16 @@ from rest_framework import serializersfrom posthog.api.scoped_related_fields import TeamScopedPrimaryKeyRelatedFieldfrom posthog.models.integration import Integration-from posthog.utils import str_to_bool-from products.batch_exports.backend.models.batch_export import OBJECT_STORAGE_DESTINATIONS, BatchExportDestination-from products.batch_exports.backend.service import (- DESTINATION_WORKFLOWS,- BaseBatchExportInputs,- coerce_config_to_declared_types,-)-from products.batch_exports.backend.temporal.destinations.constants import (+from products.batch_exports.backend.facade import api as batch_exports_api+from products.batch_exports.backend.facade.contracts import InvalidDestinationConfigError+from products.batch_exports.backend.facade.enums import (AZURE_BLOB_SUPPORTED_COMPRESSIONS,+ OBJECT_STORAGE_DESTINATIONS,S3_SUPPORTED_COMPRESSIONS,+ BatchExportDestinationType,)+from products.batch_exports.backend.models.batch_export import BatchExportDestinationclass DatabricksDestinationConfigSerializer(serializers.Serializer):@@ -522,8 +518,8 @@ class BatchExportDestinationRequestField(serializers.JSONField):# the linked integration's kind. Adding a future S3-family destination (e.g. a# first-class GCS-via-S3 type) is a one-line addition here.S3_DESTINATION_TO_INTEGRATION_KIND: dict[str, Integration.IntegrationKind] = {- BatchExportDestination.Destination.AWS_S3: Integration.IntegrationKind.AWS_S3,- BatchExportDestination.Destination.S3_COMPATIBLE: Integration.IntegrationKind.S3_COMPATIBLE,+ BatchExportDestinationType.AWS_S3: Integration.IntegrationKind.AWS_S3,+ BatchExportDestinationType.S3_COMPATIBLE: Integration.IntegrationKind.S3_COMPATIBLE,}@@ -535,10 +531,10 @@ def _writes_compressed_parquet(config: dict[str, typing.Any]) -> bool:def _uses_legacy_parquet_extension(destination_type: str, stored_config: dict[str, typing.Any]) -> bool:"""Whether an export's stored config already writes the codec into its Parquet file names.- Reads through `coerce_config_to_declared_types`, because `EncryptedJSONField` stringifies+ Reads through `coerce_destination_config`, because `EncryptedJSONField` stringifiesscalars on write and the string "False" is truthy."""- coerced = coerce_config_to_declared_types(destination_type, stored_config)+ coerced = batch_exports_api.coerce_destination_config(destination_type, stored_config)stored = coerced.get("legacy_parquet_extension")if stored is None:# export was created before the `legacy_parquet_extension` field was added@@ -636,64 +632,17 @@ class BatchExportDestinationSerializer(serializers.ModelSerializer):export_destination = BatchExportDestination.objects.create(**validated_data)return export_destination- def validate(self, attrs: collections.abc.Mapping[str, typing.Any]) -> collections.abc.Mapping[str, typing.Any]:- """Validate the destination configuration based on workflow inputs.-- Ensure that the submitted destination configuration passes the following checks:- * Does NOT contain fields that do not exist in workflow inputs.- * Contains all required fields as defined by workflow inputs.- * Provided values match types required by workflow inputs.-- Raises:- A `serializers.ValidationError` if any of these checks fail.- """- export_type, config = attrs["type"], attrs["config"]+ def validate(self, attrs: dict[str, typing.Any]) -> dict[str, typing.Any]:+ """Check the config against the destination's workflow inputs, through the facade."""request = self.context.get("request")is_patch = request is not None and request.method == "PATCH"- _, workflow_inputs = DESTINATION_WORKFLOWS[export_type]- base_field_names = {field.name for field in dataclasses.fields(BaseBatchExportInputs)}- workflow_fields = dataclasses.fields(workflow_inputs)- destination_fields = {field for field in workflow_fields if field.name not in base_field_names}-- extra_fields = config.keys() - {field.name for field in destination_fields} - base_field_names- if extra_fields:- str_fields = ", ".join(f"'{extra_field}'" for extra_field in sorted(extra_fields))- raise serializers.ValidationError(f"Configuration has unknown field/s: {str_fields}")-- # Destination config fields without a dataclass default must be provided.- for destination_field in destination_fields:- is_required = (- destination_field.default == dataclasses.MISSING- and destination_field.default_factory == dataclasses.MISSING+ try:+ attrs["config"] = batch_exports_api.validate_destination_config(+ attrs["type"], attrs["config"], partial=is_patch)- if destination_field.name not in config:- if is_required and not is_patch:- # When patching we expect a partial configuration. So, we don't- # error on missing required fields.- raise serializers.ValidationError(- f"Configuration missing required field: '{destination_field.name}'"- )- else:- continue-- config_value = config[destination_field.name]- field_type = destination_field.type-- if not isinstance(field_type, type):- # `dataclasses.Field.type` could be something we can't work with.- # TODO: Validate these ones too?- continue-- if not isinstance(config_value, field_type):- config_value, success = try_convert_to_type(config_value, field_type)-- if not success:- raise serializers.ValidationError(- f"Configuration has invalid type: got '{type(config_value).__name__}', expected '{field_type.__name__}'"- )-- config[destination_field.name] = config_value+ except InvalidDestinationConfigError as e:+ raise serializers.ValidationError(str(e))return attrs@@ -714,35 +663,6 @@ class BatchExportDestinationSerializer(serializers.ModelSerializer):return targetconfig = remove_secret_fields_recursive(data["config"])- data["config"] = coerce_config_to_declared_types(instance.type, config)+ data["config"] = batch_exports_api.coerce_destination_config(instance.type, config)return data---Success = bool---def try_convert_to_type(value: typing.Any, target_type: type) -> tuple[typing.Any, Success]:- """Attempt to convert value to target type based on well-known casting functions.-- This doesn't raise any exceptions but rather returns a tuple with a bool indicating- if the value in the first position was successfully casted to `target_type` or not.- If casting fails, the value in the first position is returned unchanged, otherwise a- new value of type `target_type` is returned.- """- current_type = type(value)-- match (current_type, target_type):- case (builtins.str, builtins.bool):- cast_func: typing.Callable[[typing.Any], typing.Any] = str_to_bool- case (builtins.str, builtins.int):- cast_func = int- case _:- return (value, False)-- try:- new_value = cast_func(value)- except Exception:- return (value, False)-- return (new_value, True)diff --git a/products/batch_exports/backend/presentation/views/batch_export/exports.py b/products/batch_exports/backend/presentation/views/batch_export/exports.pyindex 41a9a2f392e6..2ce3ce178fa6 100644--- a/products/batch_exports/backend/presentation/views/batch_export/exports.py+++ b/products/batch_exports/backend/presentation/views/batch_export/exports.py@@ -44,21 +44,25 @@ from posthog.temporal.common.client import sync_connectfrom products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_teamfrom products.batch_exports.backend.facade import api as batch_exports_apifrom products.batch_exports.backend.facade.contracts import InvalidBatchExportFilters, UnsupportedDestinationTestError-from products.batch_exports.backend.filters import SUPPORTED_FILTER_TYPES_DISPLAY, validate_batch_export_filters+from products.batch_exports.backend.facade.enums import (+ AZURE_BLOB_SUPPORTED_COMPRESSIONS,+ EXPORTABLE_EVENTS_MODEL_FIELDS,+ S3_FAMILY_TYPES,+ S3_SUPPORTED_COMPRESSIONS,+ SUPPORTED_FILTER_TYPES_DISPLAY,+ TIMEZONES,+ BatchExportDestinationType,+ BatchExportInterval,+ BatchExportModel,+ IncrementalMode,+)from products.batch_exports.backend.hogql_source import (UnsupportedHogQLQueryError,load_hogql_modifiers,serialize_batch_export_query,validate_hogql_query_for_batch_export,)-from products.batch_exports.backend.models.batch_export import (- BATCH_EXPORT_INTERVALS,- S3_FAMILY_TYPES,- TIMEZONES,- BatchExport,- BatchExportDestination,- BatchExportSource,-)+from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportDestination, BatchExportSourcefrom products.batch_exports.backend.presentation.views.utils import (HOGQL_MODIFIERS_HELP_TEXT,HogQLModifiersField,@@ -77,11 +81,6 @@ from products.batch_exports.backend.service import (sync_batch_export,unpause_batch_export,)-from products.batch_exports.backend.temporal.destinations.constants import (- AZURE_BLOB_SUPPORTED_COMPRESSIONS,- S3_SUPPORTED_COMPRESSIONS,-)-from products.batch_exports.backend.temporal.sql.events import EXPORTABLE_EVENTS_MODEL_FIELDSfrom . import (destinations as destination_views,@@ -123,8 +122,8 @@ class BatchExportRequestSerializer(serializers.Serializer):name = serializers.CharField(help_text="Human-readable name for the batch export.")model = serializers.ChoiceField(- choices=BatchExport.Model.choices,- default=serializers.CreateOnlyDefault(BatchExport.Model.EVENTS), # type: ignore[arg-type]+ choices=BatchExportModel.choices,+ default=serializers.CreateOnlyDefault(BatchExportModel.EVENTS), # type: ignore[arg-type]help_text=("Which data model to export: events, persons, sessions, or hogql. ""The hogql model exports the results of hogql_query."@@ -134,7 +133,7 @@ class BatchExportRequestSerializer(serializers.Serializer):help_text="Destination configuration. Required integration_id is enforced per destination type.",)interval = serializers.ChoiceField(- choices=BATCH_EXPORT_INTERVALS,+ choices=BatchExportInterval.choices,help_text="How often the batch export should run.",)paused = serializers.BooleanField(required=False, help_text="Whether the batch export is paused.")@@ -163,8 +162,8 @@ class BatchExportRequestSerializer(serializers.Serializer):help_text="The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.",)incremental_mode = serializers.ChoiceField(- choices=BatchExport.IncrementalMode.choices,- default=serializers.CreateOnlyDefault(BatchExport.IncrementalMode.APPEND), # type: ignore[arg-type]+ choices=IncrementalMode.choices,+ default=serializers.CreateOnlyDefault(IncrementalMode.APPEND), # type: ignore[arg-type]help_text="How this batch export handles incremental updates.",)filters = serializers.JSONField(@@ -270,8 +269,8 @@ class BatchExportSerializer(serializers.ModelSerializer):"""Serializer for a BatchExport model."""model = serializers.ChoiceField(- choices=BatchExport.Model.choices,- default=serializers.CreateOnlyDefault(BatchExport.Model.EVENTS), # type: ignore[arg-type]+ choices=BatchExportModel.choices,+ default=serializers.CreateOnlyDefault(BatchExportModel.EVENTS), # type: ignore[arg-type]help_text=("Which data model to export: events, persons, sessions, or hogql. ""The hogql model exports the results of hogql_query."@@ -286,7 +285,7 @@ class BatchExportSerializer(serializers.ModelSerializer):help_text="The 10 most recent runs of this batch export, ordered newest first.",)interval = serializers.ChoiceField(- choices=BATCH_EXPORT_INTERVALS,+ choices=BatchExportInterval.choices,help_text="How often the batch export should run.",)hogql_query = serializers.CharField(@@ -314,8 +313,8 @@ class BatchExportSerializer(serializers.ModelSerializer):help_text="The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.",)incremental_mode = serializers.ChoiceField(- choices=BatchExport.IncrementalMode.choices,- default=serializers.CreateOnlyDefault(BatchExport.IncrementalMode.APPEND), # type: ignore[arg-type]+ choices=IncrementalMode.choices,+ default=serializers.CreateOnlyDefault(IncrementalMode.APPEND), # type: ignore[arg-type]help_text="How this batch export handles incremental updates.",)timezone = serializers.ChoiceField(@@ -378,19 +377,19 @@ class BatchExportSerializer(serializers.ModelSerializer):model = self._get_model(attrs)destination = attrs.get("destination")- if destination and destination.get("type") == BatchExportDestination.Destination.HTTP:- if model != BatchExport.Model.EVENTS:+ if destination and destination.get("type") == BatchExportDestinationType.HTTP:+ if model != BatchExportModel.EVENTS:raise serializers.ValidationError("HTTP batch exports only support the events model")- self._validate_attribute_only_supported_for_model(attrs, name="filters", model=BatchExport.Model.EVENTS)- self._validate_attribute_only_supported_for_model(attrs, name="hogql_modifiers", model=BatchExport.Model.HOGQL)+ self._validate_attribute_only_supported_for_model(attrs, name="filters", model=BatchExportModel.EVENTS)+ self._validate_attribute_only_supported_for_model(attrs, name="hogql_modifiers", model=BatchExportModel.HOGQL)# Pre-defined models have these values hard-coded, only HogQL allows user input- self._validate_attribute_only_supported_for_model(attrs, name="primary_key", model=BatchExport.Model.HOGQL)- self._validate_attribute_only_supported_for_model(attrs, name="version_key", model=BatchExport.Model.HOGQL)+ self._validate_attribute_only_supported_for_model(attrs, name="primary_key", model=BatchExportModel.HOGQL)+ self._validate_attribute_only_supported_for_model(attrs, name="version_key", model=BatchExportModel.HOGQL)# TODO: Once these teams are migrated, remove this.- if model == BatchExport.Model.EVENTS and (hogql_query := attrs.get("hogql_query")) is not None:+ if model == BatchExportModel.EVENTS and (hogql_query := attrs.get("hogql_query")) is not None:is_grandfathered_team_id = BatchExport.objects.filter(schema__isnull=False, team=self.context["get_team"]()).exists()@@ -398,12 +397,12 @@ class BatchExportSerializer(serializers.ModelSerializer):attrs["hogql_query"] = self._validate_events_hogql_query(hogql_query)else:self._validate_attribute_only_supported_for_model(- attrs, name="hogql_query", model=BatchExport.Model.HOGQL+ attrs, name="hogql_query", model=BatchExportModel.HOGQL)else:- self._validate_attribute_only_supported_for_model(attrs, name="hogql_query", model=BatchExport.Model.HOGQL)+ self._validate_attribute_only_supported_for_model(attrs, name="hogql_query", model=BatchExportModel.HOGQL)- if model == BatchExport.Model.HOGQL:+ if model == BatchExportModel.HOGQL:self._validate_hogql_query(attrs)return attrs@@ -523,7 +522,7 @@ class BatchExportSerializer(serializers.ModelSerializer):saved_primary_key = source.primary_key if source is not None else Noneprimary_key = set(attrs.get("primary_key", saved_primary_key) or ())- if not primary_key and incremental_mode == BatchExport.IncrementalMode.MERGE:+ if not primary_key and incremental_mode == IncrementalMode.MERGE:raise serializers.ValidationError({"primary_key": "'primary_key' is required when 'incremental_mode' is 'MERGE'"})@@ -566,7 +565,7 @@ class BatchExportSerializer(serializers.ModelSerializer):def validate_model(self, model: str) -> str:"""Validate the model is new or not changing when provided."""- if model == BatchExport.Model.HOGQL:+ if model == BatchExportModel.HOGQL:team = self.context["get_team"]()check_hogql_batch_exports_enabled(team=team)@@ -660,7 +659,7 @@ class BatchExportSerializer(serializers.ModelSerializer):def validate_filters(self, filters):try:- validate_batch_export_filters(filters)+ batch_exports_api.validate_batch_export_filters(filters)except InvalidBatchExportFilters as e:raise serializers.ValidationError(str(e))return filters@@ -703,12 +702,12 @@ class BatchExportSerializer(serializers.ModelSerializer):merged_config = recursive_dict_merge(existing_config, config)# SSRF protection for HTTP batch exports- if destination_type == BatchExportDestination.Destination.HTTP:+ if destination_type == BatchExportDestinationType.HTTP:url = merged_config.get("url")if url and url not in ("https://us.i.posthog.com/batch/", "https://eu.i.posthog.com/batch/"):raise serializers.ValidationError(f"Invalid destination URL: {url}")- if destination_type == BatchExportDestination.Destination.SNOWFLAKE:+ if destination_type == BatchExportDestinationType.SNOWFLAKE:integration: Integration | None = destination_attrs.get("integration")if integration is None and "integration" not in destination_attrs and instance is not None:# A PATCH may send config alone, which keeps the export's existing integration.@@ -773,7 +772,7 @@ class BatchExportSerializer(serializers.ModelSerializer):f"Compression {compression} is not supported for file format {file_format}. Supported compressions are {S3_SUPPORTED_COMPRESSIONS[file_format]}")- if destination_type == BatchExportDestination.Destination.DATABRICKS:+ if destination_type == BatchExportDestinationType.DATABRICKS:# validate the Integration is valid (this is mandatory for Databricks batch exports)integration = destination_attrs.get("integration")if integration is None:@@ -786,7 +785,7 @@ class BatchExportSerializer(serializers.ModelSerializer):except DatabricksIntegrationError as e:raise serializers.ValidationError(str(e))- if destination_type == BatchExportDestination.Destination.POSTGRES:+ if destination_type == BatchExportDestinationType.POSTGRES:integration = destination_attrs.get("integration")# New Postgres exports must use an Integration for credentials. Exports created before# integrations existed keep their inline credentials, so only require it on create@@ -796,14 +795,14 @@ class BatchExportSerializer(serializers.ModelSerializer):if integration is not None and integration.kind != Integration.IntegrationKind.POSTGRESQL:raise serializers.ValidationError("Integration is not a PostgreSQL integration.")- if destination_type == BatchExportDestination.Destination.BIGQUERY:+ if destination_type == BatchExportDestinationType.BIGQUERY:integration = destination_attrs.get("integration")if integration is None:raise serializers.ValidationError("Integration is required for BigQuery batch exports")if integration.kind != Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT:raise serializers.ValidationError("Integration is not a Google Cloud service account integration.")- if destination_type == BatchExportDestination.Destination.AZURE_BLOB:+ if destination_type == BatchExportDestinationType.AZURE_BLOB:# validate the Integration is valid (this is mandatory for Azure Blob batch exports)integration = destination_attrs.get("integration")if integration is None:@@ -828,7 +827,7 @@ class BatchExportSerializer(serializers.ModelSerializer):f"Compression {compression} is not supported for file format {file_format}. Supported compressions are {AZURE_BLOB_SUPPORTED_COMPRESSIONS[file_format]}")- if destination_type == BatchExportDestination.Destination.REDSHIFT:+ if destination_type == BatchExportDestinationType.REDSHIFT:integration = destination_attrs.get("integration")# Sticky integration: an export that uses one cannot drop back to inline credentials.@@ -925,7 +924,7 @@ class BatchExportSerializer(serializers.ModelSerializer):"Authorization for 'COPY' must be an IAM role ARN, AWS credentials, or an integration ID.")- if destination_type == BatchExportDestination.Destination.WORKFLOWS:+ if destination_type == BatchExportDestinationType.WORKFLOWS:team_id = self.context["team_id"]team = Team.objects.get(id=team_id)@@ -944,8 +943,8 @@ class BatchExportSerializer(serializers.ModelSerializer):raise PermissionDenied("Backfilling Workflows is not enabled for this team.")if destination_type in (- BatchExportDestination.Destination.POSTGRES,- BatchExportDestination.Destination.REDSHIFT,+ BatchExportDestinationType.POSTGRES,+ BatchExportDestinationType.REDSHIFT,):# PostgreSQL-server integrations (Postgres, plain Redshift) keep the host in the# linked Integration; AWS Redshift integrations and inline configs keep it in@@ -977,7 +976,7 @@ class BatchExportSerializer(serializers.ModelSerializer):version_key = validated_data.pop("version_key", None)source = None- if model == BatchExport.Model.HOGQL:+ if model == BatchExportModel.HOGQL:source = BatchExportSource(team_id=team_id,hogql_query=hogql_query,@@ -1130,7 +1129,7 @@ class BatchExportSerializer(serializers.ModelSerializer):integration = destination_data.get("integration", batch_export.destination.integration)batch_export.destination.integration = integration- if batch_export.model == BatchExport.Model.HOGQL:+ if batch_export.model == BatchExportModel.HOGQL:hogql_attributes = (hogql_query, hogql_modifiers, primary_key, version_key)if any(attr is not missing for attr in hogql_attributes):source = batch_export.source or BatchExportSource(team_id=batch_export.team_id)diff --git a/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py b/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.pyindex 5a553acf27ac..f36635d3a6d4 100644--- a/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py@@ -22,6 +22,7 @@ from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.heartbeat import Heartbeaterfrom posthog.temporal.common.logger import get_write_only_logger+from products.batch_exports.backend.facade.enums import AZURE_BLOB_SUPPORTED_COMPRESSIONS as SUPPORTED_COMPRESSIONSfrom products.batch_exports.backend.service import (AzureBlobBatchExportInputs,BatchExportField,@@ -36,10 +37,7 @@ from products.batch_exports.backend.temporal.batch_exports import (is_over_billing_limit_error,start_batch_export_run,)-from products.batch_exports.backend.temporal.destinations.constants import (- AZURE_BLOB_SUPPORTED_COMPRESSIONS as SUPPORTED_COMPRESSIONS,- FILE_FORMAT_EXTENSIONS,-)+from products.batch_exports.backend.temporal.destinations.constants import FILE_FORMAT_EXTENSIONSfrom products.batch_exports.backend.temporal.destinations.utils import EXTERNAL_LOGGER, get_manifest_key, get_object_keyfrom products.batch_exports.backend.temporal.pipeline.consumer import Consumer, run_consumer_from_stagefrom products.batch_exports.backend.temporal.pipeline.entrypoint import execute_batch_export_using_internal_stagediff --git a/products/batch_exports/backend/temporal/destinations/constants.py b/products/batch_exports/backend/temporal/destinations/constants.pyindex 92930847904a..90495eb62f4c 100644--- a/products/batch_exports/backend/temporal/destinations/constants.py+++ b/products/batch_exports/backend/temporal/destinations/constants.py@@ -1,20 +1,10 @@"""Destination capability tables that must stay import-light.The destination modules import their vendor SDKs at module scope, so anything that only-needs these tables — like the batch exports API serializers validating file format and-compression combinations — should import them from here instead of the destination module.+needs these tables should import them from here instead of the destination module. The+compression tables live in ``facade/enums.py``, because the API serializers read them too."""-S3_SUPPORTED_COMPRESSIONS: dict[str, list[str]] = {- "Parquet": ["zstd", "lz4", "snappy", "gzip", "brotli"],- "JSONLines": ["gzip", "brotli"],-}--AZURE_BLOB_SUPPORTED_COMPRESSIONS: dict[str, list[str]] = {- "Parquet": ["zstd", "lz4", "snappy", "gzip", "brotli"],- "JSONLines": ["gzip", "brotli"],-}-FILE_FORMAT_EXTENSIONS: dict[str, str] = {"Parquet": "parquet","JSONLines": "jsonl",diff --git a/products/batch_exports/backend/temporal/destinations/s3_batch_export.py b/products/batch_exports/backend/temporal/destinations/s3_batch_export.pyindex 8651c5fe0467..8091c11b6a55 100644--- a/products/batch_exports/backend/temporal/destinations/s3_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/s3_batch_export.py@@ -41,6 +41,7 @@ from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.heartbeat import Heartbeaterfrom posthog.temporal.common.logger import get_logger, get_write_only_logger+from products.batch_exports.backend.facade.enums import S3_SUPPORTED_COMPRESSIONS as SUPPORTED_COMPRESSIONSfrom products.batch_exports.backend.service import (AWSCredentials,BatchExportField,@@ -56,10 +57,7 @@ from products.batch_exports.backend.temporal.batch_exports import (is_over_billing_limit_error,start_batch_export_run,)-from products.batch_exports.backend.temporal.destinations.constants import (- FILE_FORMAT_EXTENSIONS,- S3_SUPPORTED_COMPRESSIONS as SUPPORTED_COMPRESSIONS,-)+from products.batch_exports.backend.temporal.destinations.constants import FILE_FORMAT_EXTENSIONSfrom products.batch_exports.backend.temporal.destinations.utils import (get_absolute_key_prefix,get_manifest_key,diff --git a/products/batch_exports/backend/temporal/filters.py b/products/batch_exports/backend/temporal/filters.pyindex 6fd166cc9a82..5af8efdaf899 100644--- a/products/batch_exports/backend/temporal/filters.py+++ b/products/batch_exports/backend/temporal/filters.py@@ -20,7 +20,7 @@ from posthog.hogql.visitor import TraversingVisitorfrom posthog.models import Team-from products.batch_exports.backend.filters import SUPPORTED_FILTER_TYPES+from products.batch_exports.backend.facade.enums import SUPPORTED_FILTER_TYPESclass UpdatePropertiesToPersonProperties(TraversingVisitor):diff --git a/products/batch_exports/backend/temporal/sql/events.py b/products/batch_exports/backend/temporal/sql/events.pyindex c21ef9a07710..a440f686b372 100644--- a/products/batch_exports/backend/temporal/sql/events.py+++ b/products/batch_exports/backend/temporal/sql/events.py@@ -4,25 +4,6 @@ from string import Templatefrom posthog.hogql.printer.events_json_document import event_document_sql, person_document_sql-# Projected columns a HogQL query can select.-# NOTE: this is just for the 'old' version of our HogQL support, where a user can define a custom-# SELECT FROM clause, useful for defining a custom schema. It's not related to the new HogQL model,-# in which a user can use any arbitrary SELECT query.-EXPORTABLE_EVENTS_MODEL_FIELDS = frozenset(- {- "team_id",- "timestamp",- "event",- "distinct_id",- "uuid",- "created_at",- "elements_chain",- "person_id",- "properties",- "person_properties",- }-)-SELECT_FROM_EVENTS_VIEW = Template("""SELECTdiff --git a/products/batch_exports/backend/tests/api/test_create_s3.py b/products/batch_exports/backend/tests/api/test_create_s3.pyindex 17879a9028f4..ac2683df5ccb 100644--- a/products/batch_exports/backend/tests/api/test_create_s3.py+++ b/products/batch_exports/backend/tests/api/test_create_s3.py@@ -5,7 +5,7 @@ from django.test.client import Client as HttpClientfrom rest_framework import statusfrom temporalio.client import ScheduleActionStartWorkflow-from products.batch_exports.backend.models.batch_export import S3_FAMILY_TYPES+from products.batch_exports.backend.facade.enums import S3_FAMILY_TYPESfrom products.batch_exports.backend.tests.api.conftest import describe_schedulefrom products.batch_exports.backend.tests.api.operations import create_batch_exportdiff --git a/pyproject.toml b/pyproject.tomlindex 6fcc5d848c24..f86b5bde7088 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -749,14 +749,9 @@ ignore_imports = [# -> serialize and delete each entry. backend:contract-check stays off until this list is# empty. Only delete entries here: a new import from these views goes through the facade."products.batch_exports.backend.presentation.views.batch_export.destinations -> products.batch_exports.backend.models.batch_export",- "products.batch_exports.backend.presentation.views.batch_export.destinations -> products.batch_exports.backend.service",- "products.batch_exports.backend.presentation.views.batch_export.destinations -> products.batch_exports.backend.temporal.destinations.constants",- "products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.filters","products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.hogql_source","products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.models.batch_export","products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.service",- "products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.temporal.destinations.constants",- "products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.temporal.sql.events",]# routes.py is the one module core reads to assemble a product's API router, so it is the