Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

refactor(batch-exports): read export constants and config checks through the facade (#122666)

Ross0b8cd37acce5
diff --git a/products/batch_exports/backend/facade/api.py b/products/batch_exports/backend/facade/api.py
index 1d3c4ef824ef..6e50a2e6a18a 100644
--- a/products/batch_exports/backend/facade/api.py
+++ b/products/batch_exports/backend/facade/api.py
@@ -11,8 +11,9 @@ is visible in one place.
and ``temporalio`` on the ``django.setup()`` path. Core config-time consumers import this
facade, so it reaches the service only through ``_service()``, which imports it at call
time. For the same reason the Temporal client is built in ``_get_temporal_client()``, and
-the HogQL, file download, run and backfill modules load through ``_import_hogql_source()``,
-``_import_file_downloads()``, ``_import_runs()`` and ``_import_backfills()``.
+the HogQL, destination config, file download, run and backfill modules load through
+``_import_hogql_source()``, ``_import_destination_config()``, ``_import_file_downloads()``,
+``_import_runs()`` and ``_import_backfills()``.
Temporal workflow and activity registration crosses as objects, not data, so it lives in
``facade/temporal.py``. The destination clients the warehouse writers reuse live in
@@ -35,9 +36,11 @@ import structlog
from posthog.models import Team, User
from posthog.models.integration import Integration
-from products.batch_exports.backend import destination_tests
+from products.batch_exports.backend import (
+ destination_tests,
+ filters as batch_export_filters,
+)
from products.batch_exports.backend.billing import exclude_non_billable_runs
-from products.batch_exports.backend.filters import validate_batch_export_filters
from products.batch_exports.backend.models.batch_export import (
BATCH_EXPORT_INTERVALS,
BatchExport,
@@ -59,6 +62,7 @@ __all__ = [
"cancel_batch_export_backfill",
"cancel_batch_export_run",
"cancel_file_download",
+ "coerce_destination_config",
"count_batch_exports_for_teams",
"count_hogql_rows",
"create_batch_export",
@@ -89,6 +93,8 @@ __all__ = [
"retry_batch_export_run",
"run_destination_test_step",
"start_file_download",
+ "validate_batch_export_filters",
+ "validate_destination_config",
"validate_hogql_query",
]
@@ -135,6 +141,15 @@ def _import_hogql_source() -> ModuleType:
return hogql_source
+def _import_destination_config() -> ModuleType:
+ """Import the destination config checks at call time. See the module docstring for why."""
+ from products.batch_exports.backend.logic import ( # noqa: PLC0415 — keeps the service layer and temporalio off the django.setup() path
+ destination_config,
+ )
+
+ return destination_config
+
+
def _import_file_downloads() -> ModuleType:
"""Import the file download module at call time. See the module docstring for why."""
from products.batch_exports.backend.logic import file_downloads
@@ -422,6 +437,15 @@ def create_batch_export(
return _to_detail(batch_export)
+def validate_batch_export_filters(filters: object) -> None:
+ """Check the event filters a batch export stores.
+
+ Raises ``contracts.InvalidBatchExportFilters`` unless ``filters`` is None or a list of
+ supported filters.
+ """
+ batch_export_filters.validate_batch_export_filters(filters)
+
+
def create_workflows_backfill_export(
team_id: int,
*,
@@ -478,6 +502,25 @@ def list_supported_intervals() -> tuple[str, ...]:
return tuple(interval for interval, _ in BATCH_EXPORT_INTERVALS)
+def validate_destination_config(destination_type: str, config: Mapping[str, Any], *, partial: bool) -> dict[str, Any]:
+ """Check a destination config against the workflow inputs of its destination type.
+
+ With ``partial``, a required field may be missing, as in a patch. Returns a copy of the
+ config with values converted to the declared types. Raises
+ ``contracts.InvalidDestinationConfigError`` when a field is unknown, missing or of the
+ wrong type.
+ """
+ return _import_destination_config().validate_destination_config(destination_type, config, partial=partial)
+
+
+def coerce_destination_config(destination_type: str, config: dict[str, Any]) -> dict[str, Any]:
+ """Return a stored destination config with its values restored to their declared types.
+
+ The stored config reads back with stringified scalars, so ``"False"`` becomes ``False``.
+ """
+ return _service().coerce_config_to_declared_types(destination_type, config)
+
+
def describe_destination_test(destination_type: str) -> contracts.DestinationTest:
"""Return the steps of a destination's connection test, with no step run."""
destination_test = destination_tests.get_destination_test(destination=destination_type)
diff --git a/products/batch_exports/backend/facade/contracts.py b/products/batch_exports/backend/facade/contracts.py
index 3a3a5f44c352..3b82bd06bd73 100644
--- a/products/batch_exports/backend/facade/contracts.py
+++ b/products/batch_exports/backend/facade/contracts.py
@@ -33,6 +33,13 @@ class InvalidBatchExportFilters(ValueError):
"""Raised when event filters are not a list of filters a batch export can apply."""
+class InvalidDestinationConfigError(ValueError):
+ """Raised when a destination config does not match the workflow inputs of its destination type.
+
+ The message says which field is wrong.
+ """
+
+
class UnsupportedDestinationTestError(ValueError):
"""Raised when a destination type has no connection test."""
diff --git a/products/batch_exports/backend/facade/destinations/azure_blob.py b/products/batch_exports/backend/facade/destinations/azure_blob.py
index cb91c1b4b353..7f101a01d466 100644
--- a/products/batch_exports/backend/facade/destinations/azure_blob.py
+++ b/products/batch_exports/backend/facade/destinations/azure_blob.py
@@ -8,13 +8,13 @@ Importing this module loads the destination's vendor SDK, so keep it off the
``django.setup()`` path. See ``posthog/test/repo_invariants/test_startup_import_budget.py``.
"""
+from products.batch_exports.backend.facade.enums import AZURE_BLOB_SUPPORTED_COMPRESSIONS
from products.batch_exports.backend.temporal.destinations.azure_blob_batch_export import (
AzureBlobIntegrationNotFoundError,
MalformedConnectionStringError,
_get_azure_blob_integration as get_azure_blob_integration,
_is_authorization_failure_response_error as is_authorization_failure_response_error,
)
-from products.batch_exports.backend.temporal.destinations.constants import AZURE_BLOB_SUPPORTED_COMPRESSIONS
__all__ = [
"AZURE_BLOB_SUPPORTED_COMPRESSIONS",
diff --git a/products/batch_exports/backend/facade/destinations/s3.py b/products/batch_exports/backend/facade/destinations/s3.py
index e7d2896b8799..df9d726f855c 100644
--- a/products/batch_exports/backend/facade/destinations/s3.py
+++ b/products/batch_exports/backend/facade/destinations/s3.py
@@ -8,7 +8,7 @@ Importing this module loads the destination's vendor SDK, so keep it off the
``django.setup()`` path. See ``posthog/test/repo_invariants/test_startup_import_budget.py``.
"""
-from products.batch_exports.backend.temporal.destinations.constants import S3_SUPPORTED_COMPRESSIONS
+from products.batch_exports.backend.facade.enums import S3_SUPPORTED_COMPRESSIONS
from products.batch_exports.backend.temporal.destinations.s3_batch_export import (
ConcurrentS3Consumer,
ExternalRoleArnNotConfiguredError,
diff --git a/products/batch_exports/backend/facade/enums.py b/products/batch_exports/backend/facade/enums.py
index 276cf17388f4..9847bb2eb06b 100644
--- a/products/batch_exports/backend/facade/enums.py
+++ b/products/batch_exports/backend/facade/enums.py
@@ -1,11 +1,16 @@
-"""Enumerations the batch_exports models use.
+"""Enumerations and fixed value sets the batch_exports models and API use.
A consumer that writes one of these values reads the enum from here and does not need the
model class. The models keep each enum as a class attribute (``BatchExportRun.Status``) so
existing call sites are unchanged. In an annotation the class attribute is not a valid type, so
annotate with the enum from this module directly.
+
+The value sets at the end of the module are the single source for the models, the Temporal
+destinations and the API serializers, so they must stay free of framework imports.
"""
+import pytz
+
from posthog.enums import LabeledStrEnum
@@ -78,3 +83,64 @@ class IncrementalMode(LabeledStrEnum):
APPEND = "APPEND"
MERGE = "MERGE"
# TODO: Add 'FULL_REFRESH': data is re-created on every run.
+
+
+# this is what is used by the Team model
+# (we could use common_timezones instead; this has 433 timezones vs 596 for all_timezones)
+TIMEZONES = [(tz, tz) for tz in pytz.all_timezones]
+
+S3_FAMILY_TYPES: frozenset[str] = frozenset(
+ {BatchExportDestinationType.AWS_S3, BatchExportDestinationType.S3_COMPATIBLE}
+)
+
+# Destinations that write files to object storage, as opposed to the data warehouse destinations
+# (BigQuery, Databricks, Postgres, Redshift, Snowflake) that write rows into a table. File format,
+# compression and object key layout are questions only this half has to answer.
+#
+# Also includes "FileDownload", which writes to a PostHog bucket rather than a customer one.
+# TODO: it probably makes sense to introduce a 'DestinationKind' abstraction in the future,
+# to properly categorize object storage and data warehouse destinations.
+OBJECT_STORAGE_DESTINATIONS: frozenset[str] = frozenset(
+ {
+ BatchExportDestinationType.AWS_S3,
+ BatchExportDestinationType.S3_COMPATIBLE,
+ BatchExportDestinationType.AZURE_BLOB,
+ BatchExportDestinationType.FILE_DOWNLOAD,
+ }
+)
+
+S3_SUPPORTED_COMPRESSIONS: dict[str, list[str]] = {
+ "Parquet": ["zstd", "lz4", "snappy", "gzip", "brotli"],
+ "JSONLines": ["gzip", "brotli"],
+}
+
+AZURE_BLOB_SUPPORTED_COMPRESSIONS: dict[str, list[str]] = {
+ "Parquet": ["zstd", "lz4", "snappy", "gzip", "brotli"],
+ "JSONLines": ["gzip", "brotli"],
+}
+
+# Projected columns a HogQL query can select.
+# NOTE: this is just for the 'old' version of our HogQL support, where a user can define a custom
+# SELECT FROM clause, useful for defining a custom schema. It's not related to the new HogQL model,
+# in which a user can use any arbitrary SELECT query.
+EXPORTABLE_EVENTS_MODEL_FIELDS = frozenset(
+ {
+ "team_id",
+ "timestamp",
+ "event",
+ "distinct_id",
+ "uuid",
+ "created_at",
+ "elements_chain",
+ "person_id",
+ "properties",
+ "person_properties",
+ }
+)
+
+# Single source of truth for the serialized filter `type` values we accept. Enforced at write
+# time by `validate_batch_export_filters` and at query time by `compose_filters_clause`.
+SUPPORTED_FILTER_TYPES = frozenset({"event", "person", "hogql"})
+
+# Quoted, sorted rendering of the supported filter types for user-facing messages.
+SUPPORTED_FILTER_TYPES_DISPLAY = ", ".join(repr(t) for t in sorted(SUPPORTED_FILTER_TYPES))
diff --git a/products/batch_exports/backend/filters.py b/products/batch_exports/backend/filters.py
index 1fee7ec895af..1eeccdb831e9 100644
--- a/products/batch_exports/backend/filters.py
+++ b/products/batch_exports/backend/filters.py
@@ -1,19 +1,13 @@
"""
The event filters a batch export stores, and the checks they must pass on write.
-This module imports no framework and no Temporal code, so the DRF serializer and the facade
-share one set of checks. ``service.py`` imports ``temporalio``, so the checks cannot live
-there: the facade is on the ``django.setup()`` path.
+This module imports no framework and no Temporal code, because the facade imports it and the
+facade is on the ``django.setup()`` path. ``service.py`` imports ``temporalio``, so the checks
+cannot live there.
"""
from products.batch_exports.backend.facade.contracts import InvalidBatchExportFilters
-
-# Single source of truth for the serialized filter `type` values we accept. Enforced at write
-# time by `validate_batch_export_filters` and at query time by `compose_filters_clause`.
-SUPPORTED_FILTER_TYPES = {"event", "person", "hogql"}
-
-# Quoted, sorted rendering of the supported filter types for user-facing messages.
-SUPPORTED_FILTER_TYPES_DISPLAY = ", ".join(repr(t) for t in sorted(SUPPORTED_FILTER_TYPES))
+from products.batch_exports.backend.facade.enums import SUPPORTED_FILTER_TYPES, SUPPORTED_FILTER_TYPES_DISPLAY
def validate_batch_export_filters(filters: object) -> None:
diff --git a/products/batch_exports/backend/logic/destination_config.py b/products/batch_exports/backend/logic/destination_config.py
new file mode 100644
index 000000000000..e42ce78e8d0f
--- /dev/null
+++ b/products/batch_exports/backend/logic/destination_config.py
@@ -0,0 +1,105 @@
+"""The config a batch export destination stores: check it against the workflow inputs.
+
+The facade reaches this module through a call-time import, because the workflow input
+dataclasses live in the service layer, which may not load on the ``django.setup()`` path.
+"""
+
+import typing
+import builtins
+import dataclasses
+from collections.abc import Callable, Mapping
+
+from posthog.utils import str_to_bool
+
+from products.batch_exports.backend.facade import contracts
+from products.batch_exports.backend.service import DESTINATION_WORKFLOWS, BaseBatchExportInputs
+
+
+def _try_convert_to_type(value: typing.Any, target_type: type) -> tuple[typing.Any, bool]:
+ """Attempt to convert value to target type based on well-known casting functions.
+
+ This doesn't raise any exceptions but rather returns a tuple with a bool indicating
+ if the value in the first position was successfully casted to `target_type` or not.
+ If casting fails, the value in the first position is returned unchanged, otherwise a
+ new value of type `target_type` is returned.
+ """
+ current_type = type(value)
+
+ match (current_type, target_type):
+ case (builtins.str, builtins.bool):
+ cast_func: Callable[[typing.Any], typing.Any] = str_to_bool
+ case (builtins.str, builtins.int):
+ cast_func = int
+ case _:
+ return (value, False)
+
+ try:
+ new_value = cast_func(value)
+ except Exception:
+ return (value, False)
+
+ return (new_value, True)
+
+
+def validate_destination_config(
+ destination_type: str, config: Mapping[str, typing.Any], *, partial: bool
+) -> dict[str, typing.Any]:
+ """Check a destination config against the workflow inputs of its destination type.
+
+ Ensure that the config passes the following checks:
+ * Does NOT contain fields that do not exist in workflow inputs.
+ * Contains all required fields as defined by workflow inputs, unless ``partial``.
+ * Provided values match types required by workflow inputs.
+
+ Returns a copy of the config with values converted to the declared types.
+
+ Raises:
+ A `contracts.InvalidDestinationConfigError` if any of these checks fail.
+ """
+ validated = dict(config)
+
+ _, workflow_inputs = DESTINATION_WORKFLOWS[destination_type]
+ base_field_names = {field.name for field in dataclasses.fields(BaseBatchExportInputs)}
+ workflow_fields = dataclasses.fields(workflow_inputs)
+ destination_fields = {field for field in workflow_fields if field.name not in base_field_names}
+
+ extra_fields = validated.keys() - {field.name for field in destination_fields} - base_field_names
+ if extra_fields:
+ str_fields = ", ".join(f"'{extra_field}'" for extra_field in sorted(extra_fields))
+ raise contracts.InvalidDestinationConfigError(f"Configuration has unknown field/s: {str_fields}")
+
+ # Destination config fields without a dataclass default must be provided.
+ for destination_field in destination_fields:
+ is_required = (
+ destination_field.default == dataclasses.MISSING
+ and destination_field.default_factory == dataclasses.MISSING
+ )
+ if destination_field.name not in validated:
+ if is_required and not partial:
+ # When patching we expect a partial configuration. So, we don't
+ # error on missing required fields.
+ raise contracts.InvalidDestinationConfigError(
+ f"Configuration missing required field: '{destination_field.name}'"
+ )
+ else:
+ continue
+
+ config_value = validated[destination_field.name]
+ field_type = destination_field.type
+
+ if not isinstance(field_type, type):
+ # `dataclasses.Field.type` could be something we can't work with.
+ # TODO: Validate these ones too?
+ continue
+
+ if not isinstance(config_value, field_type):
+ config_value, success = _try_convert_to_type(config_value, field_type)
+
+ if not success:
+ raise contracts.InvalidDestinationConfigError(
+ f"Configuration has invalid type: got '{type(config_value).__name__}', expected '{field_type.__name__}'"
+ )
+
+ validated[destination_field.name] = config_value
+
+ return validated
diff --git a/products/batch_exports/backend/models/batch_export.py b/products/batch_exports/backend/models/batch_export.py
index 7076d53ee353..32a93faeabfa 100644
--- a/products/batch_exports/backend/models/batch_export.py
+++ b/products/batch_exports/backend/models/batch_export.py
@@ -9,8 +9,6 @@ from django.contrib.postgres.fields import ArrayField
from django.db import models
from django.db.models import Q
-import pytz
-
# This import prevents a circular import during Django app loading.
# The import chain: apps.py -> tasks -> async_migrations/definition.py -> models.utils
# triggers models/__init__.py which imports batch_exports.models. Without this import,
@@ -22,21 +20,7 @@ from posthog.models.scoping.root_mixin import TeamScopedRootMixin
from posthog.models.utils import UUIDTModel
from products.batch_exports.backend.facade import enums
-
-# this is what is used by the Team model
-# (we could use common_timezones instead; this has 433 timezones vs 596 for all_timezones)
-TIMEZONES = [(tz, tz) for tz in pytz.all_timezones]
-
-S3_FAMILY_TYPES: frozenset[str] = frozenset({"AwsS3", "S3Compatible"})
-
-# Destinations that write files to object storage, as opposed to the data warehouse destinations
-# (BigQuery, Databricks, Postgres, Redshift, Snowflake) that write rows into a table. File format,
-# compression and object key layout are questions only this half has to answer.
-#
-# Also includes "FileDownload", which writes to a PostHog bucket rather than a customer one.
-# TODO: it probably makes sense to introduce a 'DestinationKind' abstraction in the future,
-# to properly categorize object storage and data warehouse destinations.
-OBJECT_STORAGE_DESTINATIONS: frozenset[str] = frozenset({"AwsS3", "S3Compatible", "AzureBlob", "FileDownload"})
+from products.batch_exports.backend.facade.enums import TIMEZONES
class DayOfWeek(IntEnum):
diff --git a/products/batch_exports/backend/presentation/views/batch_export/destinations.py b/products/batch_exports/backend/presentation/views/batch_export/destinations.py
index 24e9ef1b2279..7c9539927a8f 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/destinations.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/destinations.py
@@ -1,8 +1,6 @@
"""Batch export destination serializers: the typed config and request shape of each destination type."""
import typing
-import builtins
-import dataclasses
import collections.abc
from django.db import models
@@ -12,18 +10,16 @@ from rest_framework import serializers
from posthog.api.scoped_related_fields import TeamScopedPrimaryKeyRelatedField
from posthog.models.integration import Integration
-from posthog.utils import str_to_bool
-from products.batch_exports.backend.models.batch_export import OBJECT_STORAGE_DESTINATIONS, BatchExportDestination
-from products.batch_exports.backend.service import (
- DESTINATION_WORKFLOWS,
- BaseBatchExportInputs,
- coerce_config_to_declared_types,
-)
-from products.batch_exports.backend.temporal.destinations.constants import (
+from products.batch_exports.backend.facade import api as batch_exports_api
+from products.batch_exports.backend.facade.contracts import InvalidDestinationConfigError
+from products.batch_exports.backend.facade.enums import (
AZURE_BLOB_SUPPORTED_COMPRESSIONS,
+ OBJECT_STORAGE_DESTINATIONS,
S3_SUPPORTED_COMPRESSIONS,
+ BatchExportDestinationType,
)
+from products.batch_exports.backend.models.batch_export import BatchExportDestination
class DatabricksDestinationConfigSerializer(serializers.Serializer):
@@ -522,8 +518,8 @@ class BatchExportDestinationRequestField(serializers.JSONField):
# the linked integration's kind. Adding a future S3-family destination (e.g. a
# first-class GCS-via-S3 type) is a one-line addition here.
S3_DESTINATION_TO_INTEGRATION_KIND: dict[str, Integration.IntegrationKind] = {
- BatchExportDestination.Destination.AWS_S3: Integration.IntegrationKind.AWS_S3,
- BatchExportDestination.Destination.S3_COMPATIBLE: Integration.IntegrationKind.S3_COMPATIBLE,
+ BatchExportDestinationType.AWS_S3: Integration.IntegrationKind.AWS_S3,
+ BatchExportDestinationType.S3_COMPATIBLE: Integration.IntegrationKind.S3_COMPATIBLE,
}
@@ -535,10 +531,10 @@ def _writes_compressed_parquet(config: dict[str, typing.Any]) -> bool:
def _uses_legacy_parquet_extension(destination_type: str, stored_config: dict[str, typing.Any]) -> bool:
"""Whether an export's stored config already writes the codec into its Parquet file names.
- Reads through `coerce_config_to_declared_types`, because `EncryptedJSONField` stringifies
+ Reads through `coerce_destination_config`, because `EncryptedJSONField` stringifies
scalars on write and the string "False" is truthy.
"""
- coerced = coerce_config_to_declared_types(destination_type, stored_config)
+ coerced = batch_exports_api.coerce_destination_config(destination_type, stored_config)
stored = coerced.get("legacy_parquet_extension")
if stored is None:
# export was created before the `legacy_parquet_extension` field was added
@@ -636,64 +632,17 @@ class BatchExportDestinationSerializer(serializers.ModelSerializer):
export_destination = BatchExportDestination.objects.create(**validated_data)
return export_destination
- def validate(self, attrs: collections.abc.Mapping[str, typing.Any]) -> collections.abc.Mapping[str, typing.Any]:
- """Validate the destination configuration based on workflow inputs.
-
- Ensure that the submitted destination configuration passes the following checks:
- * Does NOT contain fields that do not exist in workflow inputs.
- * Contains all required fields as defined by workflow inputs.
- * Provided values match types required by workflow inputs.
-
- Raises:
- A `serializers.ValidationError` if any of these checks fail.
- """
- export_type, config = attrs["type"], attrs["config"]
+ def validate(self, attrs: dict[str, typing.Any]) -> dict[str, typing.Any]:
+ """Check the config against the destination's workflow inputs, through the facade."""
request = self.context.get("request")
is_patch = request is not None and request.method == "PATCH"
- _, workflow_inputs = DESTINATION_WORKFLOWS[export_type]
- base_field_names = {field.name for field in dataclasses.fields(BaseBatchExportInputs)}
- workflow_fields = dataclasses.fields(workflow_inputs)
- destination_fields = {field for field in workflow_fields if field.name not in base_field_names}
-
- extra_fields = config.keys() - {field.name for field in destination_fields} - base_field_names
- if extra_fields:
- str_fields = ", ".join(f"'{extra_field}'" for extra_field in sorted(extra_fields))
- raise serializers.ValidationError(f"Configuration has unknown field/s: {str_fields}")
-
- # Destination config fields without a dataclass default must be provided.
- for destination_field in destination_fields:
- is_required = (
- destination_field.default == dataclasses.MISSING
- and destination_field.default_factory == dataclasses.MISSING
+ try:
+ attrs["config"] = batch_exports_api.validate_destination_config(
+ attrs["type"], attrs["config"], partial=is_patch
)
- if destination_field.name not in config:
- if is_required and not is_patch:
- # When patching we expect a partial configuration. So, we don't
- # error on missing required fields.
- raise serializers.ValidationError(
- f"Configuration missing required field: '{destination_field.name}'"
- )
- else:
- continue
-
- config_value = config[destination_field.name]
- field_type = destination_field.type
-
- if not isinstance(field_type, type):
- # `dataclasses.Field.type` could be something we can't work with.
- # TODO: Validate these ones too?
- continue
-
- if not isinstance(config_value, field_type):
- config_value, success = try_convert_to_type(config_value, field_type)
-
- if not success:
- raise serializers.ValidationError(
- f"Configuration has invalid type: got '{type(config_value).__name__}', expected '{field_type.__name__}'"
- )
-
- config[destination_field.name] = config_value
+ except InvalidDestinationConfigError as e:
+ raise serializers.ValidationError(str(e))
return attrs
@@ -714,35 +663,6 @@ class BatchExportDestinationSerializer(serializers.ModelSerializer):
return target
config = remove_secret_fields_recursive(data["config"])
- data["config"] = coerce_config_to_declared_types(instance.type, config)
+ data["config"] = batch_exports_api.coerce_destination_config(instance.type, config)
return data
-
-
-Success = bool
-
-
-def try_convert_to_type(value: typing.Any, target_type: type) -> tuple[typing.Any, Success]:
- """Attempt to convert value to target type based on well-known casting functions.
-
- This doesn't raise any exceptions but rather returns a tuple with a bool indicating
- if the value in the first position was successfully casted to `target_type` or not.
- If casting fails, the value in the first position is returned unchanged, otherwise a
- new value of type `target_type` is returned.
- """
- current_type = type(value)
-
- match (current_type, target_type):
- case (builtins.str, builtins.bool):
- cast_func: typing.Callable[[typing.Any], typing.Any] = str_to_bool
- case (builtins.str, builtins.int):
- cast_func = int
- case _:
- return (value, False)
-
- try:
- new_value = cast_func(value)
- except Exception:
- return (value, False)
-
- return (new_value, True)
diff --git a/products/batch_exports/backend/presentation/views/batch_export/exports.py b/products/batch_exports/backend/presentation/views/batch_export/exports.py
index 41a9a2f392e6..2ce3ce178fa6 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/exports.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/exports.py
@@ -44,21 +44,25 @@ from posthog.temporal.common.client import sync_connect
from products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_team
from products.batch_exports.backend.facade import api as batch_exports_api
from products.batch_exports.backend.facade.contracts import InvalidBatchExportFilters, UnsupportedDestinationTestError
-from products.batch_exports.backend.filters import SUPPORTED_FILTER_TYPES_DISPLAY, validate_batch_export_filters
+from products.batch_exports.backend.facade.enums import (
+ AZURE_BLOB_SUPPORTED_COMPRESSIONS,
+ EXPORTABLE_EVENTS_MODEL_FIELDS,
+ S3_FAMILY_TYPES,
+ S3_SUPPORTED_COMPRESSIONS,
+ SUPPORTED_FILTER_TYPES_DISPLAY,
+ TIMEZONES,
+ BatchExportDestinationType,
+ BatchExportInterval,
+ BatchExportModel,
+ IncrementalMode,
+)
from products.batch_exports.backend.hogql_source import (
UnsupportedHogQLQueryError,
load_hogql_modifiers,
serialize_batch_export_query,
validate_hogql_query_for_batch_export,
)
-from products.batch_exports.backend.models.batch_export import (
- BATCH_EXPORT_INTERVALS,
- S3_FAMILY_TYPES,
- TIMEZONES,
- BatchExport,
- BatchExportDestination,
- BatchExportSource,
-)
+from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportDestination, BatchExportSource
from products.batch_exports.backend.presentation.views.utils import (
HOGQL_MODIFIERS_HELP_TEXT,
HogQLModifiersField,
@@ -77,11 +81,6 @@ from products.batch_exports.backend.service import (
sync_batch_export,
unpause_batch_export,
)
-from products.batch_exports.backend.temporal.destinations.constants import (
- AZURE_BLOB_SUPPORTED_COMPRESSIONS,
- S3_SUPPORTED_COMPRESSIONS,
-)
-from products.batch_exports.backend.temporal.sql.events import EXPORTABLE_EVENTS_MODEL_FIELDS
from . import (
destinations as destination_views,
@@ -123,8 +122,8 @@ class BatchExportRequestSerializer(serializers.Serializer):
name = serializers.CharField(help_text="Human-readable name for the batch export.")
model = serializers.ChoiceField(
- choices=BatchExport.Model.choices,
- default=serializers.CreateOnlyDefault(BatchExport.Model.EVENTS), # type: ignore[arg-type]
+ choices=BatchExportModel.choices,
+ default=serializers.CreateOnlyDefault(BatchExportModel.EVENTS), # type: ignore[arg-type]
help_text=(
"Which data model to export: events, persons, sessions, or hogql. "
"The hogql model exports the results of hogql_query."
@@ -134,7 +133,7 @@ class BatchExportRequestSerializer(serializers.Serializer):
help_text="Destination configuration. Required integration_id is enforced per destination type.",
)
interval = serializers.ChoiceField(
- choices=BATCH_EXPORT_INTERVALS,
+ choices=BatchExportInterval.choices,
help_text="How often the batch export should run.",
)
paused = serializers.BooleanField(required=False, help_text="Whether the batch export is paused.")
@@ -163,8 +162,8 @@ class BatchExportRequestSerializer(serializers.Serializer):
help_text="The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.",
)
incremental_mode = serializers.ChoiceField(
- choices=BatchExport.IncrementalMode.choices,
- default=serializers.CreateOnlyDefault(BatchExport.IncrementalMode.APPEND), # type: ignore[arg-type]
+ choices=IncrementalMode.choices,
+ default=serializers.CreateOnlyDefault(IncrementalMode.APPEND), # type: ignore[arg-type]
help_text="How this batch export handles incremental updates.",
)
filters = serializers.JSONField(
@@ -270,8 +269,8 @@ class BatchExportSerializer(serializers.ModelSerializer):
"""Serializer for a BatchExport model."""
model = serializers.ChoiceField(
- choices=BatchExport.Model.choices,
- default=serializers.CreateOnlyDefault(BatchExport.Model.EVENTS), # type: ignore[arg-type]
+ choices=BatchExportModel.choices,
+ default=serializers.CreateOnlyDefault(BatchExportModel.EVENTS), # type: ignore[arg-type]
help_text=(
"Which data model to export: events, persons, sessions, or hogql. "
"The hogql model exports the results of hogql_query."
@@ -286,7 +285,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
help_text="The 10 most recent runs of this batch export, ordered newest first.",
)
interval = serializers.ChoiceField(
- choices=BATCH_EXPORT_INTERVALS,
+ choices=BatchExportInterval.choices,
help_text="How often the batch export should run.",
)
hogql_query = serializers.CharField(
@@ -314,8 +313,8 @@ class BatchExportSerializer(serializers.ModelSerializer):
help_text="The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.",
)
incremental_mode = serializers.ChoiceField(
- choices=BatchExport.IncrementalMode.choices,
- default=serializers.CreateOnlyDefault(BatchExport.IncrementalMode.APPEND), # type: ignore[arg-type]
+ choices=IncrementalMode.choices,
+ default=serializers.CreateOnlyDefault(IncrementalMode.APPEND), # type: ignore[arg-type]
help_text="How this batch export handles incremental updates.",
)
timezone = serializers.ChoiceField(
@@ -378,19 +377,19 @@ class BatchExportSerializer(serializers.ModelSerializer):
model = self._get_model(attrs)
destination = attrs.get("destination")
- if destination and destination.get("type") == BatchExportDestination.Destination.HTTP:
- if model != BatchExport.Model.EVENTS:
+ if destination and destination.get("type") == BatchExportDestinationType.HTTP:
+ if model != BatchExportModel.EVENTS:
raise serializers.ValidationError("HTTP batch exports only support the events model")
- self._validate_attribute_only_supported_for_model(attrs, name="filters", model=BatchExport.Model.EVENTS)
- self._validate_attribute_only_supported_for_model(attrs, name="hogql_modifiers", model=BatchExport.Model.HOGQL)
+ self._validate_attribute_only_supported_for_model(attrs, name="filters", model=BatchExportModel.EVENTS)
+ self._validate_attribute_only_supported_for_model(attrs, name="hogql_modifiers", model=BatchExportModel.HOGQL)
# Pre-defined models have these values hard-coded, only HogQL allows user input
- self._validate_attribute_only_supported_for_model(attrs, name="primary_key", model=BatchExport.Model.HOGQL)
- self._validate_attribute_only_supported_for_model(attrs, name="version_key", model=BatchExport.Model.HOGQL)
+ self._validate_attribute_only_supported_for_model(attrs, name="primary_key", model=BatchExportModel.HOGQL)
+ self._validate_attribute_only_supported_for_model(attrs, name="version_key", model=BatchExportModel.HOGQL)
# TODO: Once these teams are migrated, remove this.
- if model == BatchExport.Model.EVENTS and (hogql_query := attrs.get("hogql_query")) is not None:
+ if model == BatchExportModel.EVENTS and (hogql_query := attrs.get("hogql_query")) is not None:
is_grandfathered_team_id = BatchExport.objects.filter(
schema__isnull=False, team=self.context["get_team"]()
).exists()
@@ -398,12 +397,12 @@ class BatchExportSerializer(serializers.ModelSerializer):
attrs["hogql_query"] = self._validate_events_hogql_query(hogql_query)
else:
self._validate_attribute_only_supported_for_model(
- attrs, name="hogql_query", model=BatchExport.Model.HOGQL
+ attrs, name="hogql_query", model=BatchExportModel.HOGQL
)
else:
- self._validate_attribute_only_supported_for_model(attrs, name="hogql_query", model=BatchExport.Model.HOGQL)
+ self._validate_attribute_only_supported_for_model(attrs, name="hogql_query", model=BatchExportModel.HOGQL)
- if model == BatchExport.Model.HOGQL:
+ if model == BatchExportModel.HOGQL:
self._validate_hogql_query(attrs)
return attrs
@@ -523,7 +522,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
saved_primary_key = source.primary_key if source is not None else None
primary_key = set(attrs.get("primary_key", saved_primary_key) or ())
- if not primary_key and incremental_mode == BatchExport.IncrementalMode.MERGE:
+ if not primary_key and incremental_mode == IncrementalMode.MERGE:
raise serializers.ValidationError(
{"primary_key": "'primary_key' is required when 'incremental_mode' is 'MERGE'"}
)
@@ -566,7 +565,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
def validate_model(self, model: str) -> str:
"""Validate the model is new or not changing when provided."""
- if model == BatchExport.Model.HOGQL:
+ if model == BatchExportModel.HOGQL:
team = self.context["get_team"]()
check_hogql_batch_exports_enabled(team=team)
@@ -660,7 +659,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
def validate_filters(self, filters):
try:
- validate_batch_export_filters(filters)
+ batch_exports_api.validate_batch_export_filters(filters)
except InvalidBatchExportFilters as e:
raise serializers.ValidationError(str(e))
return filters
@@ -703,12 +702,12 @@ class BatchExportSerializer(serializers.ModelSerializer):
merged_config = recursive_dict_merge(existing_config, config)
# SSRF protection for HTTP batch exports
- if destination_type == BatchExportDestination.Destination.HTTP:
+ if destination_type == BatchExportDestinationType.HTTP:
url = merged_config.get("url")
if url and url not in ("https://us.i.posthog.com/batch/", "https://eu.i.posthog.com/batch/"):
raise serializers.ValidationError(f"Invalid destination URL: {url}")
- if destination_type == BatchExportDestination.Destination.SNOWFLAKE:
+ if destination_type == BatchExportDestinationType.SNOWFLAKE:
integration: Integration | None = destination_attrs.get("integration")
if integration is None and "integration" not in destination_attrs and instance is not None:
# A PATCH may send config alone, which keeps the export's existing integration.
@@ -773,7 +772,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
f"Compression {compression} is not supported for file format {file_format}. Supported compressions are {S3_SUPPORTED_COMPRESSIONS[file_format]}"
)
- if destination_type == BatchExportDestination.Destination.DATABRICKS:
+ if destination_type == BatchExportDestinationType.DATABRICKS:
# validate the Integration is valid (this is mandatory for Databricks batch exports)
integration = destination_attrs.get("integration")
if integration is None:
@@ -786,7 +785,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
except DatabricksIntegrationError as e:
raise serializers.ValidationError(str(e))
- if destination_type == BatchExportDestination.Destination.POSTGRES:
+ if destination_type == BatchExportDestinationType.POSTGRES:
integration = destination_attrs.get("integration")
# New Postgres exports must use an Integration for credentials. Exports created before
# integrations existed keep their inline credentials, so only require it on create
@@ -796,14 +795,14 @@ class BatchExportSerializer(serializers.ModelSerializer):
if integration is not None and integration.kind != Integration.IntegrationKind.POSTGRESQL:
raise serializers.ValidationError("Integration is not a PostgreSQL integration.")
- if destination_type == BatchExportDestination.Destination.BIGQUERY:
+ if destination_type == BatchExportDestinationType.BIGQUERY:
integration = destination_attrs.get("integration")
if integration is None:
raise serializers.ValidationError("Integration is required for BigQuery batch exports")
if integration.kind != Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT:
raise serializers.ValidationError("Integration is not a Google Cloud service account integration.")
- if destination_type == BatchExportDestination.Destination.AZURE_BLOB:
+ if destination_type == BatchExportDestinationType.AZURE_BLOB:
# validate the Integration is valid (this is mandatory for Azure Blob batch exports)
integration = destination_attrs.get("integration")
if integration is None:
@@ -828,7 +827,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
f"Compression {compression} is not supported for file format {file_format}. Supported compressions are {AZURE_BLOB_SUPPORTED_COMPRESSIONS[file_format]}"
)
- if destination_type == BatchExportDestination.Destination.REDSHIFT:
+ if destination_type == BatchExportDestinationType.REDSHIFT:
integration = destination_attrs.get("integration")
# Sticky integration: an export that uses one cannot drop back to inline credentials.
@@ -925,7 +924,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
"Authorization for 'COPY' must be an IAM role ARN, AWS credentials, or an integration ID."
)
- if destination_type == BatchExportDestination.Destination.WORKFLOWS:
+ if destination_type == BatchExportDestinationType.WORKFLOWS:
team_id = self.context["team_id"]
team = Team.objects.get(id=team_id)
@@ -944,8 +943,8 @@ class BatchExportSerializer(serializers.ModelSerializer):
raise PermissionDenied("Backfilling Workflows is not enabled for this team.")
if destination_type in (
- BatchExportDestination.Destination.POSTGRES,
- BatchExportDestination.Destination.REDSHIFT,
+ BatchExportDestinationType.POSTGRES,
+ BatchExportDestinationType.REDSHIFT,
):
# PostgreSQL-server integrations (Postgres, plain Redshift) keep the host in the
# linked Integration; AWS Redshift integrations and inline configs keep it in
@@ -977,7 +976,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
version_key = validated_data.pop("version_key", None)
source = None
- if model == BatchExport.Model.HOGQL:
+ if model == BatchExportModel.HOGQL:
source = BatchExportSource(
team_id=team_id,
hogql_query=hogql_query,
@@ -1130,7 +1129,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
integration = destination_data.get("integration", batch_export.destination.integration)
batch_export.destination.integration = integration
- if batch_export.model == BatchExport.Model.HOGQL:
+ if batch_export.model == BatchExportModel.HOGQL:
hogql_attributes = (hogql_query, hogql_modifiers, primary_key, version_key)
if any(attr is not missing for attr in hogql_attributes):
source = batch_export.source or BatchExportSource(team_id=batch_export.team_id)
diff --git a/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py b/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py
index 5a553acf27ac..f36635d3a6d4 100644
--- a/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py
@@ -22,6 +22,7 @@ from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.logger import get_write_only_logger
+from products.batch_exports.backend.facade.enums import AZURE_BLOB_SUPPORTED_COMPRESSIONS as SUPPORTED_COMPRESSIONS
from products.batch_exports.backend.service import (
AzureBlobBatchExportInputs,
BatchExportField,
@@ -36,10 +37,7 @@ from products.batch_exports.backend.temporal.batch_exports import (
is_over_billing_limit_error,
start_batch_export_run,
)
-from products.batch_exports.backend.temporal.destinations.constants import (
- AZURE_BLOB_SUPPORTED_COMPRESSIONS as SUPPORTED_COMPRESSIONS,
- FILE_FORMAT_EXTENSIONS,
-)
+from products.batch_exports.backend.temporal.destinations.constants import FILE_FORMAT_EXTENSIONS
from products.batch_exports.backend.temporal.destinations.utils import EXTERNAL_LOGGER, get_manifest_key, get_object_key
from products.batch_exports.backend.temporal.pipeline.consumer import Consumer, run_consumer_from_stage
from products.batch_exports.backend.temporal.pipeline.entrypoint import execute_batch_export_using_internal_stage
diff --git a/products/batch_exports/backend/temporal/destinations/constants.py b/products/batch_exports/backend/temporal/destinations/constants.py
index 92930847904a..90495eb62f4c 100644
--- a/products/batch_exports/backend/temporal/destinations/constants.py
+++ b/products/batch_exports/backend/temporal/destinations/constants.py
@@ -1,20 +1,10 @@
"""Destination capability tables that must stay import-light.
The destination modules import their vendor SDKs at module scope, so anything that only
-needs these tables — like the batch exports API serializers validating file format and
-compression combinations — should import them from here instead of the destination module.
+needs these tables should import them from here instead of the destination module. The
+compression tables live in ``facade/enums.py``, because the API serializers read them too.
"""
-S3_SUPPORTED_COMPRESSIONS: dict[str, list[str]] = {
- "Parquet": ["zstd", "lz4", "snappy", "gzip", "brotli"],
- "JSONLines": ["gzip", "brotli"],
-}
-
-AZURE_BLOB_SUPPORTED_COMPRESSIONS: dict[str, list[str]] = {
- "Parquet": ["zstd", "lz4", "snappy", "gzip", "brotli"],
- "JSONLines": ["gzip", "brotli"],
-}
-
FILE_FORMAT_EXTENSIONS: dict[str, str] = {
"Parquet": "parquet",
"JSONLines": "jsonl",
diff --git a/products/batch_exports/backend/temporal/destinations/s3_batch_export.py b/products/batch_exports/backend/temporal/destinations/s3_batch_export.py
index 8651c5fe0467..8091c11b6a55 100644
--- a/products/batch_exports/backend/temporal/destinations/s3_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/s3_batch_export.py
@@ -41,6 +41,7 @@ from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.logger import get_logger, get_write_only_logger
+from products.batch_exports.backend.facade.enums import S3_SUPPORTED_COMPRESSIONS as SUPPORTED_COMPRESSIONS
from products.batch_exports.backend.service import (
AWSCredentials,
BatchExportField,
@@ -56,10 +57,7 @@ from products.batch_exports.backend.temporal.batch_exports import (
is_over_billing_limit_error,
start_batch_export_run,
)
-from products.batch_exports.backend.temporal.destinations.constants import (
- FILE_FORMAT_EXTENSIONS,
- S3_SUPPORTED_COMPRESSIONS as SUPPORTED_COMPRESSIONS,
-)
+from products.batch_exports.backend.temporal.destinations.constants import FILE_FORMAT_EXTENSIONS
from products.batch_exports.backend.temporal.destinations.utils import (
get_absolute_key_prefix,
get_manifest_key,
diff --git a/products/batch_exports/backend/temporal/filters.py b/products/batch_exports/backend/temporal/filters.py
index 6fd166cc9a82..5af8efdaf899 100644
--- a/products/batch_exports/backend/temporal/filters.py
+++ b/products/batch_exports/backend/temporal/filters.py
@@ -20,7 +20,7 @@ from posthog.hogql.visitor import TraversingVisitor
from posthog.models import Team
-from products.batch_exports.backend.filters import SUPPORTED_FILTER_TYPES
+from products.batch_exports.backend.facade.enums import SUPPORTED_FILTER_TYPES
class UpdatePropertiesToPersonProperties(TraversingVisitor):
diff --git a/products/batch_exports/backend/temporal/sql/events.py b/products/batch_exports/backend/temporal/sql/events.py
index c21ef9a07710..a440f686b372 100644
--- a/products/batch_exports/backend/temporal/sql/events.py
+++ b/products/batch_exports/backend/temporal/sql/events.py
@@ -4,25 +4,6 @@ from string import Template
from posthog.hogql.printer.events_json_document import event_document_sql, person_document_sql
-# Projected columns a HogQL query can select.
-# NOTE: this is just for the 'old' version of our HogQL support, where a user can define a custom
-# SELECT FROM clause, useful for defining a custom schema. It's not related to the new HogQL model,
-# in which a user can use any arbitrary SELECT query.
-EXPORTABLE_EVENTS_MODEL_FIELDS = frozenset(
- {
- "team_id",
- "timestamp",
- "event",
- "distinct_id",
- "uuid",
- "created_at",
- "elements_chain",
- "person_id",
- "properties",
- "person_properties",
- }
-)
-
SELECT_FROM_EVENTS_VIEW = Template(
"""
SELECT
diff --git a/products/batch_exports/backend/tests/api/test_create_s3.py b/products/batch_exports/backend/tests/api/test_create_s3.py
index 17879a9028f4..ac2683df5ccb 100644
--- a/products/batch_exports/backend/tests/api/test_create_s3.py
+++ b/products/batch_exports/backend/tests/api/test_create_s3.py
@@ -5,7 +5,7 @@ from django.test.client import Client as HttpClient
from rest_framework import status
from temporalio.client import ScheduleActionStartWorkflow
-from products.batch_exports.backend.models.batch_export import S3_FAMILY_TYPES
+from products.batch_exports.backend.facade.enums import S3_FAMILY_TYPES
from products.batch_exports.backend.tests.api.conftest import describe_schedule
from products.batch_exports.backend.tests.api.operations import create_batch_export
diff --git a/pyproject.toml b/pyproject.toml
index 6fcc5d848c24..f86b5bde7088 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -749,14 +749,9 @@ ignore_imports = [
# -> serialize and delete each entry. backend:contract-check stays off until this list is
# empty. Only delete entries here: a new import from these views goes through the facade.
"products.batch_exports.backend.presentation.views.batch_export.destinations -> products.batch_exports.backend.models.batch_export",
- "products.batch_exports.backend.presentation.views.batch_export.destinations -> products.batch_exports.backend.service",
- "products.batch_exports.backend.presentation.views.batch_export.destinations -> products.batch_exports.backend.temporal.destinations.constants",
- "products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.filters",
"products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.hogql_source",
"products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.models.batch_export",
"products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.service",
- "products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.temporal.destinations.constants",
- "products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.temporal.sql.events",
]
# routes.py is the one module core reads to assemble a product's API router, so it is the