Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(batch-exports): move export rules and actions behind the facade (#124715)

Ross7a363fa3c0bf
diff --git a/products/batch_exports/backend/facade/api.py b/products/batch_exports/backend/facade/api.py
index 6e50a2e6a18a..de1897537c37 100644
--- a/products/batch_exports/backend/facade/api.py
+++ b/products/batch_exports/backend/facade/api.py
@@ -11,9 +11,9 @@ is visible in one place.
and ``temporalio`` on the ``django.setup()`` path. Core config-time consumers import this
facade, so it reaches the service only through ``_service()``, which imports it at call
time. For the same reason the Temporal client is built in ``_get_temporal_client()``, and
-the HogQL, destination config, file download, run and backfill modules load through
-``_import_hogql_source()``, ``_import_destination_config()``, ``_import_file_downloads()``,
-``_import_runs()`` and ``_import_backfills()``.
+the HogQL, destination config, export, file download, run and backfill modules load through
+``_import_hogql_source()``, ``_import_destination_config()``, ``_import_exports()``,
+``_import_file_downloads()``, ``_import_runs()`` and ``_import_backfills()``.
Temporal workflow and activity registration crosses as objects, not data, so it lives in
``facade/temporal.py``. The destination clients the warehouse writers reuse live in
@@ -76,6 +76,7 @@ __all__ = [
"get_batch_export_backfill",
"get_batch_export_by_name",
"get_batch_export_run",
+ "get_destination_test_config",
"get_file_download_run",
"get_file_download_url",
"get_latest_completed_run",
@@ -90,9 +91,14 @@ __all__ = [
"list_file_download_runs",
"list_latest_failed_runs",
"list_supported_intervals",
+ "merge_destination_config",
+ "pause_batch_export",
+ "prepare_new_destination_config",
"retry_batch_export_run",
"run_destination_test_step",
"start_file_download",
+ "unpause_batch_export",
+ "validate_batch_export_destination",
"validate_batch_export_filters",
"validate_destination_config",
"validate_hogql_query",
@@ -150,6 +156,13 @@ def _import_destination_config() -> ModuleType:
return destination_config
+def _import_exports() -> ModuleType:
+ """Import the export logic at call time. See the module docstring for why."""
+ from products.batch_exports.backend.logic import exports
+
+ return exports
+
+
def _import_file_downloads() -> ModuleType:
"""Import the file download module at call time. See the module docstring for why."""
from products.batch_exports.backend.logic import file_downloads
@@ -479,10 +492,31 @@ def create_workflows_backfill_export(
def delete_batch_export(export_id: UUID, team_id: int) -> None:
- """Delete a batch export: pause it, cancel its work, then drop its Temporal schedule."""
- service = _service()
- batch_export = BatchExport.objects.select_related("destination").get(id=export_id, team_id=team_id)
- service.delete_batch_export(batch_export)
+ """Delete a batch export: pause it, cancel its work, then drop its Temporal schedule.
+
+ Raises ``contracts.BatchExportNotFoundError`` when the team has no such live export.
+ """
+ _import_exports().delete_batch_export(team_id, export_id)
+
+
+def pause_batch_export(team_id: int, export_id: UUID, *, note: str) -> None:
+ """Pause a batch export, so its schedule starts no new runs.
+
+ ``note`` records who asked, on the Temporal schedule. Raises
+ ``contracts.BatchExportNotFoundError`` when the team has no such export, and
+ ``contracts.BatchExportScheduleError`` when Temporal rejects the change.
+ """
+ _import_exports().pause_batch_export(team_id, export_id, note=note)
+
+
+def unpause_batch_export(team_id: int, export_id: UUID, *, note: str, backfill: bool) -> None:
+ """Unpause a batch export. With ``backfill``, also backfill the runs it missed while paused.
+
+ ``note`` records who asked, on the Temporal schedule. Raises
+ ``contracts.BatchExportNotFoundError`` when the team has no such export, and
+ ``contracts.BatchExportScheduleError`` when Temporal rejects the change.
+ """
+ _import_exports().unpause_batch_export(team_id, export_id, note=note, backfill=backfill)
def backfill_batch_export(
@@ -521,6 +555,62 @@ def coerce_destination_config(destination_type: str, config: dict[str, Any]) ->
return _service().coerce_config_to_declared_types(destination_type, config)
+def validate_batch_export_destination(
+ team_id: int,
+ *,
+ export_id: UUID | None,
+ destination_type: str,
+ config: Mapping[str, Any],
+ integration_id: int | None,
+ integration_sent: bool,
+) -> None:
+ """Check that a destination can be saved, for a new export or as a change to ``export_id``.
+
+ ``config`` already passed ``validate_destination_config``. ``integration_sent`` is false when
+ the request left the integration out, which keeps the existing integration of the export.
+ Raises ``contracts.InvalidBatchExportDestinationError`` with the failed check as the message.
+ """
+ _import_exports().validate_destination(
+ team_id,
+ export_id=export_id,
+ destination_type=destination_type,
+ config=config,
+ integration_id=integration_id,
+ integration_sent=integration_sent,
+ )
+
+
+def prepare_new_destination_config(destination_type: str, config: Mapping[str, Any]) -> dict[str, Any]:
+ """Return the config a new destination saves, with the defaults that a new export opts into.
+
+ Temporary: only the export serializer calls this, because it still saves the destination
+ through the ORM. Do not add callers. The function goes away when export creation moves
+ behind the facade.
+ """
+ return _import_exports().prepare_new_destination_config(destination_type, config)
+
+
+def merge_destination_config(
+ destination_type: str, stored_config: Mapping[str, Any], patch: Mapping[str, Any]
+) -> dict[str, Any]:
+ """Return the config an export saves after a patch, merged key by key into its stored config.
+
+ Temporary: only the export serializer calls this, because it still saves the destination
+ through the ORM. Do not add callers. The function goes away when export updates move
+ behind the facade.
+ """
+ return _import_exports().merge_destination_config(destination_type, stored_config, patch)
+
+
+def get_destination_test_config(team_id: int, export_id: UUID) -> dict[str, Any]:
+ """Return the stored destination config of an export, for a connection test of that export.
+
+ Only the fields that the workflow inputs of the destination declare are kept. The config holds
+ the credentials of the destination, so it must not leave the product.
+ """
+ return _import_exports().get_destination_test_config(team_id, export_id)
+
+
def describe_destination_test(destination_type: str) -> contracts.DestinationTest:
"""Return the steps of a destination's connection test, with no step run."""
destination_test = destination_tests.get_destination_test(destination=destination_type)
diff --git a/products/batch_exports/backend/facade/contracts.py b/products/batch_exports/backend/facade/contracts.py
index 3b82bd06bd73..cb35fd37712a 100644
--- a/products/batch_exports/backend/facade/contracts.py
+++ b/products/batch_exports/backend/facade/contracts.py
@@ -40,6 +40,17 @@ class InvalidDestinationConfigError(ValueError):
"""
+class InvalidBatchExportDestinationError(ValueError):
+ """Raised when a destination cannot be saved, for a new export or as a change to an existing one.
+
+ The message says which check failed.
+ """
+
+
+class BatchExportScheduleError(Exception):
+ """Raised when Temporal rejects a change to the schedule of a batch export."""
+
+
class UnsupportedDestinationTestError(ValueError):
"""Raised when a destination type has no connection test."""
diff --git a/products/batch_exports/backend/logic/exports.py b/products/batch_exports/backend/logic/exports.py
new file mode 100644
index 000000000000..9a1e365e4e85
--- /dev/null
+++ b/products/batch_exports/backend/logic/exports.py
@@ -0,0 +1,523 @@
+"""Scheduled batch exports: check a destination before it is saved, pause or unpause an export.
+
+The facade reaches this module through a call-time import, because it imports the service layer
+and the Temporal client, neither of which may load on the ``django.setup()`` path.
+"""
+
+import typing
+import contextlib
+import dataclasses
+from uuid import UUID
+
+from posthog.models.integration import (
+ AzureBlobIntegration,
+ AzureBlobIntegrationError,
+ DatabricksIntegration,
+ DatabricksIntegrationError,
+ Integration,
+ SnowflakeIntegration,
+ SnowflakeIntegrationError,
+)
+from posthog.security.url_validation import (
+ INVALID_HOST_MESSAGE,
+ UNREACHABLE_HOST_MESSAGE,
+ ShapeError,
+ validate_external_host,
+)
+from posthog.temporal.common.client import sync_connect
+
+from products.batch_exports.backend import service
+from products.batch_exports.backend.facade import contracts
+from products.batch_exports.backend.facade.enums import (
+ AZURE_BLOB_SUPPORTED_COMPRESSIONS,
+ OBJECT_STORAGE_DESTINATIONS,
+ S3_FAMILY_TYPES,
+ S3_SUPPORTED_COMPRESSIONS,
+ BatchExportDestinationType,
+)
+from products.batch_exports.backend.models.batch_export import BatchExport
+
+# S3-family destinations that may authenticate via an Integration, mapped to
+# the linked integration's kind. Adding a future S3-family destination (e.g. a
+# first-class GCS-via-S3 type) is a one-line addition here.
+_S3_DESTINATION_TO_INTEGRATION_KIND: dict[str, Integration.IntegrationKind] = {
+ BatchExportDestinationType.AWS_S3: Integration.IntegrationKind.AWS_S3,
+ BatchExportDestinationType.S3_COMPATIBLE: Integration.IntegrationKind.S3_COMPATIBLE,
+}
+
+# The only URLs an HTTP destination may send to, as SSRF protection.
+_ALLOWED_HTTP_DESTINATION_URLS = ("https://us.i.posthog.com/batch/", "https://eu.i.posthog.com/batch/")
+
+
+def _get_batch_export(team_id: int, export_id: UUID, *related: str) -> BatchExport:
+ """Load a live export of the team, joining only the ``related`` fields the caller reads."""
+ batch_exports = BatchExport.objects.exclude(deleted=True)
+ # `select_related()` with no fields joins every non-null foreign key, so only call it with some.
+ if related:
+ batch_exports = batch_exports.select_related(*related)
+ try:
+ return batch_exports.get(id=export_id, team_id=team_id)
+ except BatchExport.DoesNotExist as e:
+ raise contracts.BatchExportNotFoundError() from e
+
+
+def _get_integration(team_id: int, integration_id: int | None) -> Integration | None:
+ if integration_id is None:
+ return None
+ try:
+ return Integration.objects.get(id=integration_id, team_id=team_id)
+ except Integration.DoesNotExist as e:
+ raise contracts.InvalidBatchExportDestinationError(
+ "The integration for this destination no longer exists. Choose another integration and try again."
+ ) from e
+
+
+def _recursive_dict_merge(
+ old: typing.Mapping[str, typing.Any],
+ new: typing.Mapping[str, typing.Any],
+) -> dict[str, typing.Any]:
+ """Merge two dictionaries, and recursively merge any dictionaries in them."""
+ merged = {**old, **new}
+
+ for key, value in merged.items():
+ if not isinstance(value, dict):
+ continue
+
+ # Recurse only when both sides are dicts: a dict replacing a scalar (e.g. inline
+ # credentials replacing an IAM role or an integration id) is an overwrite, not a merge.
+ old_value = old.get(key)
+ new_value = new.get(key)
+ merged[key] = _recursive_dict_merge(
+ old_value if isinstance(old_value, dict) else {},
+ new_value if isinstance(new_value, dict) else {},
+ )
+
+ return merged
+
+
+def _writes_compressed_parquet(config: dict[str, typing.Any]) -> bool:
+ """Whether a config produces Parquet file names that carry a compression codec."""
+ return config.get("file_format") == "Parquet" and config.get("compression") is not None
+
+
+def _uses_legacy_parquet_extension(destination_type: str, stored_config: dict[str, typing.Any]) -> bool:
+ """Whether an export's stored config already writes the codec into its Parquet file names.
+
+ Reads through `coerce_config_to_declared_types`, because `EncryptedJSONField` stringifies
+ scalars on write and the string "False" is truthy.
+ """
+ coerced = service.coerce_config_to_declared_types(destination_type, stored_config)
+ stored = coerced.get("legacy_parquet_extension")
+ if stored is None:
+ # export was created before the `legacy_parquet_extension` field was added
+ # so will use the legacy extension if it writes compressed Parquet files
+ return _writes_compressed_parquet(coerced)
+ return bool(stored)
+
+
+def _set_default_parquet_extension(destination_type: str, config: dict[str, typing.Any]) -> None:
+ """Opt a newly created destination into the standard `.parquet` extension.
+
+ The workflow input dataclasses default this to `True`, so that an export whose Temporal
+ schedule predates the field keeps the same file extension as before in order to maintain
+ compatibility.
+ """
+ if destination_type in OBJECT_STORAGE_DESTINATIONS:
+ config.setdefault("legacy_parquet_extension", False)
+
+
+def _pin_existing_parquet_extension(destination_type: str, stored_config: dict[str, typing.Any]) -> None:
+ """Record what an export's file names already look like, before a patch can change its format.
+
+ An export that predates this setting has no value for it, and a missing value reads as the
+ legacy naming. That is correct only for an export that already writes names carrying a codec,
+ which means Parquet with a compression codec set. An export on JSON Lines, or on Parquet with
+ no compression, has no such names to keep, so moving it to compressed Parquet has to produce
+ `.parquet` rather than `.parquet.zst`.
+
+ Takes the config as stored, before the incoming patch merges into it, so the value reflects
+ what the export has been running rather than what it is moving to. An explicit value in the
+ patch still wins, because the merge applies afterwards.
+ """
+ if destination_type not in OBJECT_STORAGE_DESTINATIONS:
+ return
+
+ stored_config.setdefault("legacy_parquet_extension", _writes_compressed_parquet(stored_config))
+
+
+def _coerce_integration_id(value: typing.Any) -> int | None:
+ """Return the integration id encoded in a Redshift COPY credential value, if any.
+
+ `BatchExportDestination.config` is an `EncryptedJSONField`, which stringifies scalar
+ leaves on the decrypt round trip, so an id may arrive as an int or a numeric string.
+ """
+ if isinstance(value, bool):
+ return None
+ if isinstance(value, int):
+ return value
+ if isinstance(value, str) and value.isdigit():
+ return int(value)
+ return None
+
+
+def _check_file_format(merged_config: dict[str, typing.Any], supported_compressions: dict[str, list[str]]) -> None:
+ # JSONLines is the default file format for S3 exports for legacy reasons
+ file_format = merged_config.get("file_format", "JSONLines")
+ supported_file_formats = supported_compressions.keys()
+ if file_format not in supported_file_formats:
+ raise contracts.InvalidBatchExportDestinationError(
+ f"File format {file_format} is not supported. Supported file formats are {list(supported_file_formats)}"
+ )
+ compression = merged_config.get("compression", None)
+ if compression and compression not in supported_compressions[file_format]:
+ raise contracts.InvalidBatchExportDestinationError(
+ f"Compression {compression} is not supported for file format {file_format}. Supported compressions are {supported_compressions[file_format]}"
+ )
+
+
+def _check_redshift_copy_inputs(
+ team_id: int,
+ instance: BatchExport | None,
+ merged_config: dict[str, typing.Any],
+ existing_config: dict[str, typing.Any],
+) -> None:
+ copy_inputs = merged_config.get("copy_inputs", {})
+ if not copy_inputs:
+ raise contracts.InvalidBatchExportDestinationError("Missing required inputs for 'COPY'")
+
+ required_inputs = {"s3_bucket", "region_name", "authorization", "bucket_credentials"}
+ if required_inputs - copy_inputs.keys():
+ raise contracts.InvalidBatchExportDestinationError("Missing required input for 'COPY'")
+
+ credential_keys = {"aws_access_key_id", "aws_secret_access_key"}
+
+ bucket_credentials = copy_inputs["bucket_credentials"]
+ bucket_integration_id = _coerce_integration_id(bucket_credentials)
+ authorization = copy_inputs.get("authorization")
+ authorization_integration_id = _coerce_integration_id(authorization)
+
+ existing_copy_inputs = existing_config.get("copy_inputs") or {}
+
+ for field_name, copy_integration_id in (
+ ("bucket_credentials", bucket_integration_id),
+ ("authorization", authorization_integration_id),
+ ):
+ # Sticky like the top-level integration: COPY credentials that reference an
+ # integration cannot drop back to inline values.
+ # TODO: remove this guard once integrations are mandatory for Redshift and
+ # inline credentials are gone.
+ if (
+ instance is not None
+ and copy_integration_id is None
+ and _coerce_integration_id(existing_copy_inputs.get(field_name)) is not None
+ ):
+ raise contracts.InvalidBatchExportDestinationError(
+ f"Cannot switch '{field_name}' from an integration to inline credentials. "
+ "Send a different integration ID to swap, or omit the field to keep the current one."
+ )
+ if copy_integration_id is None:
+ continue
+ # These ids live inside `config` rather than the team-scoped `integration`
+ # field, so team ownership must be checked here.
+ if not Integration.objects.filter(
+ id=copy_integration_id,
+ team_id=team_id,
+ kind=Integration.IntegrationKind.AWS_S3,
+ ).exists():
+ raise contracts.InvalidBatchExportDestinationError(
+ f"'{field_name}' does not reference an AWS S3 integration of this project."
+ )
+
+ if bucket_integration_id is None and (
+ not isinstance(bucket_credentials, dict) or credential_keys - bucket_credentials.keys()
+ ):
+ raise contracts.InvalidBatchExportDestinationError("Missing required bucket credentials for 'COPY'")
+
+ if authorization_integration_id is None:
+ if isinstance(authorization, dict):
+ if credential_keys - authorization.keys():
+ raise contracts.InvalidBatchExportDestinationError("Missing required credentials for 'COPY'")
+ elif isinstance(authorization, str):
+ if not authorization.strip():
+ raise contracts.InvalidBatchExportDestinationError("Missing required IAM role for 'COPY'")
+ else:
+ raise contracts.InvalidBatchExportDestinationError(
+ "Authorization for 'COPY' must be an IAM role ARN, AWS credentials, or an integration ID."
+ )
+
+
+def validate_destination(
+ team_id: int,
+ *,
+ export_id: UUID | None,
+ destination_type: str,
+ config: typing.Mapping[str, typing.Any],
+ integration_id: int | None,
+ integration_sent: bool,
+) -> None:
+ """Check that a destination can be saved, for a new export or as a change to ``export_id``.
+
+ ``integration_sent`` is false when the request left the integration out, which keeps the
+ export's existing integration. An explicit null sends ``integration_id=None``.
+
+ Raises:
+ contracts.InvalidBatchExportDestinationError: If the destination fails a check. The
+ message says which one.
+ contracts.BatchExportNotFoundError: If ``export_id`` names no export of the team.
+ """
+ instance = (
+ _get_batch_export(team_id, export_id, "destination", "destination__integration")
+ if export_id is not None
+ else None
+ )
+ existing_config: dict[str, typing.Any] = instance.destination.config if instance is not None else {}
+
+ if instance is not None and destination_type != instance.destination.type:
+ raise contracts.InvalidBatchExportDestinationError(
+ f"Cannot change destination type from '{instance.destination.type}' to '{destination_type}'. "
+ "Delete this batch export and create a new one with the new destination type."
+ )
+
+ # This setting is used for grandfathered exports that used the legacy Parquet file extension,
+ # and is a one-way migration; once exports use the new `.parquet` extension it is not
+ # possible to go back to using the legacy extension.
+ if config.get("legacy_parquet_extension") is True and not _uses_legacy_parquet_extension(
+ destination_type, existing_config
+ ):
+ raise contracts.InvalidBatchExportDestinationError(
+ "'legacy_parquet_extension' can only stay on for an export that already writes the "
+ "compression codec into its Parquet file names. It cannot be turned on for a new "
+ "export, or turned back on once an export moved to the standard '.parquet' extension."
+ )
+
+ merged_config = _recursive_dict_merge(existing_config, config)
+
+ # The integration the request sent. None when it sent none, or sent an explicit null.
+ if instance is not None and integration_id is not None and instance.destination.integration_id == integration_id:
+ # A PATCH usually sends the export's current integration again, which is already loaded.
+ requested_integration = instance.destination.integration
+ else:
+ requested_integration = _get_integration(team_id, integration_id)
+
+ # A PATCH may send config alone, which keeps the export's existing integration.
+ # An explicit `integration: null` is a removal, and is rejected below.
+ kept_integration = requested_integration
+ if requested_integration is None and not integration_sent and instance is not None:
+ kept_integration = instance.destination.integration
+
+ # SSRF protection for HTTP batch exports
+ if destination_type == BatchExportDestinationType.HTTP:
+ url = merged_config.get("url")
+ if url and url not in _ALLOWED_HTTP_DESTINATION_URLS:
+ raise contracts.InvalidBatchExportDestinationError(f"Invalid destination URL: {url}")
+
+ if destination_type == BatchExportDestinationType.SNOWFLAKE:
+ integration = kept_integration
+ if integration is None:
+ raise contracts.InvalidBatchExportDestinationError("Integration is required for Snowflake batch exports")
+
+ # Also rejects an integration missing its account, user or authentication type, so a
+ # half-built one fails here rather than on the export's first run.
+ try:
+ SnowflakeIntegration(integration)
+ except SnowflakeIntegrationError as e:
+ raise contracts.InvalidBatchExportDestinationError(str(e))
+
+ if destination_type in S3_FAMILY_TYPES:
+ integration = kept_integration
+ if integration is None:
+ raise contracts.InvalidBatchExportDestinationError(
+ f"Integration is required for {destination_type} batch exports"
+ )
+
+ # Credentials and the provider endpoint are not declared on the input dataclasses, so
+ # `validate_destination_config` already rejects them as unknown fields.
+
+ # we already validate the required inputs in `validate_destination_config`
+ # so here we just ensure that the inputs are not empty
+ required_non_empty_inputs = ["bucket_name", "region", "prefix"]
+
+ empty_inputs = []
+
+ for required_input in required_non_empty_inputs:
+ value = config.get(required_input)
+ if value is not None and isinstance(value, str) and value.strip() == "":
+ empty_inputs.append(required_input)
+
+ if empty_inputs:
+ raise contracts.InvalidBatchExportDestinationError(f"The following inputs are empty: {empty_inputs}")
+
+ # The integration must match the destination kind. (Team ownership is already enforced
+ # by the team-scoped `integration` field, which can only resolve integrations belonging
+ # to the request's team.)
+ if integration.kind != _S3_DESTINATION_TO_INTEGRATION_KIND[destination_type]:
+ raise contracts.InvalidBatchExportDestinationError(
+ f"Integration provided is not an AWS S3 integration (got kind='{integration.kind}')"
+ )
+
+ _check_file_format(merged_config, S3_SUPPORTED_COMPRESSIONS)
+
+ if destination_type == BatchExportDestinationType.DATABRICKS:
+ # validate the Integration is valid (this is mandatory for Databricks batch exports)
+ if requested_integration is None:
+ raise contracts.InvalidBatchExportDestinationError("Integration is required for Databricks batch exports")
+ if requested_integration.kind != Integration.IntegrationKind.DATABRICKS:
+ raise contracts.InvalidBatchExportDestinationError("Integration is not a Databricks integration.")
+ # try instantiate the integration to check if it's valid
+ try:
+ DatabricksIntegration(requested_integration)
+ except DatabricksIntegrationError as e:
+ raise contracts.InvalidBatchExportDestinationError(str(e))
+
+ if destination_type == BatchExportDestinationType.POSTGRES:
+ # New Postgres exports must use an Integration for credentials. Exports created before
+ # integrations existed keep their inline credentials, so only require it on create
+ # (`instance is None`); existing inline-credential exports stay valid when edited.
+ if requested_integration is None and instance is None:
+ raise contracts.InvalidBatchExportDestinationError("Integration is required for Postgres batch exports")
+ if requested_integration is not None and requested_integration.kind != Integration.IntegrationKind.POSTGRESQL:
+ raise contracts.InvalidBatchExportDestinationError("Integration is not a PostgreSQL integration.")
+
+ if destination_type == BatchExportDestinationType.BIGQUERY:
+ if requested_integration is None:
+ raise contracts.InvalidBatchExportDestinationError("Integration is required for BigQuery batch exports")
+ if requested_integration.kind != Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT:
+ raise contracts.InvalidBatchExportDestinationError(
+ "Integration is not a Google Cloud service account integration."
+ )
+
+ if destination_type == BatchExportDestinationType.AZURE_BLOB:
+ # validate the Integration is valid (this is mandatory for Azure Blob batch exports)
+ if requested_integration is None:
+ raise contracts.InvalidBatchExportDestinationError("Integration is required for Azure Blob batch exports")
+ if requested_integration.kind != Integration.IntegrationKind.AZURE_BLOB:
+ raise contracts.InvalidBatchExportDestinationError("Integration is not an Azure Blob integration.")
+ # try instantiate the integration to check if it's valid
+ try:
+ AzureBlobIntegration(requested_integration)
+ except AzureBlobIntegrationError as e:
+ raise contracts.InvalidBatchExportDestinationError(str(e))
+
+ _check_file_format(merged_config, AZURE_BLOB_SUPPORTED_COMPRESSIONS)
+
+ if destination_type == BatchExportDestinationType.REDSHIFT:
+ # Sticky integration: an export that uses one cannot drop back to inline credentials.
+ # TODO: remove this guard once inline credentials are gone.
+ if instance is not None and instance.destination.integration is not None and requested_integration is None:
+ raise contracts.InvalidBatchExportDestinationError(
+ "Cannot remove the integration from a Redshift batch export that uses one. "
+ "Re-send its `integration` to keep it (or a different one to swap)."
+ )
+
+ # New Redshift exports must use an Integration for credentials. Exports created before
+ # integrations existed keep their inline credentials, so only require it on create
+ # (`instance is None`); existing inline-credential exports stay valid when edited.
+ if requested_integration is None and instance is None:
+ raise contracts.InvalidBatchExportDestinationError("Integration is required for Redshift batch exports")
+
+ if requested_integration is not None:
+ # (Team ownership is already enforced by the team-scoped `integration` field.)
+ if requested_integration.kind != Integration.IntegrationKind.AWS_REDSHIFT:
+ raise contracts.InvalidBatchExportDestinationError("Integration is not a Redshift integration.")
+
+ # Only plain Redshift integrations store a host; AWS-flavor ones obtain
+ # temporary credentials for a cluster endpoint that stays in the export config.
+ if "host" not in requested_integration.config and not merged_config.get("host"):
+ raise contracts.InvalidBatchExportDestinationError(
+ "A 'host' is required when using an AWS Redshift integration: "
+ "set it to the cluster or workgroup endpoint."
+ )
+
+ if merged_config.get("mode") == "COPY":
+ _check_redshift_copy_inputs(team_id, instance, merged_config, existing_config)
+
+ if destination_type in (
+ BatchExportDestinationType.POSTGRES,
+ BatchExportDestinationType.REDSHIFT,
+ ):
+ # PostgreSQL-server integrations (Postgres, plain Redshift) keep the host in the
+ # linked Integration; AWS Redshift integrations and inline configs keep it in
+ # `config`. Prefer the Integration's host when it has one so we don't skip
+ # SSRF validation (and don't `KeyError` on a `config` that has no `host`).
+ host = requested_integration.config.get("host") if requested_integration is not None else None
+ if host is None:
+ host = merged_config.get("host")
+
+ if host is not None:
+ try:
+ validate_external_host(host)
+ except ShapeError:
+ raise contracts.InvalidBatchExportDestinationError(INVALID_HOST_MESSAGE)
+ except ValueError:
+ raise contracts.InvalidBatchExportDestinationError(UNREACHABLE_HOST_MESSAGE)
+
+
+def prepare_new_destination_config(
+ destination_type: str, config: typing.Mapping[str, typing.Any]
+) -> dict[str, typing.Any]:
+ """Return the config a new destination saves, with the defaults a new export opts into."""
+ prepared = dict(config)
+ _set_default_parquet_extension(destination_type, prepared)
+ return prepared
+
+
+def merge_destination_config(
+ destination_type: str, stored_config: typing.Mapping[str, typing.Any], patch: typing.Mapping[str, typing.Any]
+) -> dict[str, typing.Any]:
+ """Return the config an export saves after a patch merges into its stored config."""
+ pinned = dict(stored_config)
+ _pin_existing_parquet_extension(destination_type, pinned)
+ return _recursive_dict_merge(pinned, patch)
+
+
+def get_destination_test_config(team_id: int, export_id: UUID) -> dict[str, typing.Any]:
+ """Return an export's stored destination config, limited to the fields its workflow inputs declare."""
+ batch_export = _get_batch_export(team_id, export_id, "destination")
+
+ # Remove any additional fields from stored configuration
+ _, workflow_inputs = service.DESTINATION_WORKFLOWS[batch_export.destination.type]
+ workflow_fields = {field.name for field in dataclasses.fields(workflow_inputs)}
+ return {k: v for k, v in batch_export.destination.config.items() if k in workflow_fields}
+
+
+def delete_batch_export(team_id: int, export_id: UUID) -> None:
+ """Delete an export: pause it, cancel its running work, then drop its Temporal schedule.
+
+ Raises:
+ contracts.BatchExportNotFoundError: If the team has no such live export.
+ """
+ # The activity log reads the destination when the delete saves the export.
+ batch_export = _get_batch_export(team_id, export_id, "destination")
+ service.delete_batch_export(batch_export)
+
+
+def _map_schedule_errors() -> typing.Iterator[None]:
+ try:
+ yield
+ except service.BatchExportIdError as e:
+ raise contracts.BatchExportNotFoundError() from e
+ except service.BatchExportServiceRPCError as e:
+ raise contracts.BatchExportScheduleError() from e
+
+
+def pause_batch_export(team_id: int, export_id: UUID, *, note: str) -> None:
+ """Pause an export's Temporal schedule.
+
+ Raises:
+ contracts.BatchExportNotFoundError: If the team has no such export.
+ contracts.BatchExportScheduleError: If Temporal rejects the change.
+ """
+ with _map_schedule_errors():
+ service.pause_batch_export(sync_connect(), str(export_id), note=note, team_id=team_id)
+
+
+def unpause_batch_export(team_id: int, export_id: UUID, *, note: str, backfill: bool) -> None:
+ """Unpause an export's Temporal schedule, and with ``backfill``, backfill the runs missed while paused.
+
+ Raises:
+ contracts.BatchExportNotFoundError: If the team has no such export.
+ contracts.BatchExportScheduleError: If Temporal rejects the change.
+ """
+ with _map_schedule_errors():
+ service.unpause_batch_export(sync_connect(), str(export_id), note=note, backfill=backfill, team_id=team_id)
diff --git a/products/batch_exports/backend/presentation/views/batch_export/destinations.py b/products/batch_exports/backend/presentation/views/batch_export/destinations.py
index 7c9539927a8f..35739e24a37c 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/destinations.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/destinations.py
@@ -13,12 +13,7 @@ from posthog.models.integration import Integration
from products.batch_exports.backend.facade import api as batch_exports_api
from products.batch_exports.backend.facade.contracts import InvalidDestinationConfigError
-from products.batch_exports.backend.facade.enums import (
- AZURE_BLOB_SUPPORTED_COMPRESSIONS,
- OBJECT_STORAGE_DESTINATIONS,
- S3_SUPPORTED_COMPRESSIONS,
- BatchExportDestinationType,
-)
+from products.batch_exports.backend.facade.enums import AZURE_BLOB_SUPPORTED_COMPRESSIONS, S3_SUPPORTED_COMPRESSIONS
from products.batch_exports.backend.models.batch_export import BatchExportDestination
@@ -514,80 +509,6 @@ class BatchExportDestinationRequestField(serializers.JSONField):
pass
-# S3-family destinations that may authenticate via an Integration, mapped to
-# the linked integration's kind. Adding a future S3-family destination (e.g. a
-# first-class GCS-via-S3 type) is a one-line addition here.
-S3_DESTINATION_TO_INTEGRATION_KIND: dict[str, Integration.IntegrationKind] = {
- BatchExportDestinationType.AWS_S3: Integration.IntegrationKind.AWS_S3,
- BatchExportDestinationType.S3_COMPATIBLE: Integration.IntegrationKind.S3_COMPATIBLE,
-}
-
-
-def _writes_compressed_parquet(config: dict[str, typing.Any]) -> bool:
- """Whether a config produces Parquet file names that carry a compression codec."""
- return config.get("file_format") == "Parquet" and config.get("compression") is not None
-
-
-def _uses_legacy_parquet_extension(destination_type: str, stored_config: dict[str, typing.Any]) -> bool:
- """Whether an export's stored config already writes the codec into its Parquet file names.
-
- Reads through `coerce_destination_config`, because `EncryptedJSONField` stringifies
- scalars on write and the string "False" is truthy.
- """
- coerced = batch_exports_api.coerce_destination_config(destination_type, stored_config)
- stored = coerced.get("legacy_parquet_extension")
- if stored is None:
- # export was created before the `legacy_parquet_extension` field was added
- # so will use the legacy extension if it writes compressed Parquet files
- return _writes_compressed_parquet(coerced)
- return bool(stored)
-
-
-def _set_default_parquet_extension(destination_type: str, config: dict[str, typing.Any]) -> None:
- """Opt a newly created destination into the standard `.parquet` extension.
-
- The workflow input dataclasses default this to `True`, so that an export whose Temporal
- schedule predates the field keeps the same file extension as before in order to maintain
- compatibility.
- """
- if destination_type in OBJECT_STORAGE_DESTINATIONS:
- config.setdefault("legacy_parquet_extension", False)
-
-
-def _pin_existing_parquet_extension(destination_type: str, stored_config: dict[str, typing.Any]) -> None:
- """Record what an export's file names already look like, before a patch can change its format.
-
- An export that predates this setting has no value for it, and a missing value reads as the
- legacy naming. That is correct only for an export that already writes names carrying a codec,
- which means Parquet with a compression codec set. An export on JSON Lines, or on Parquet with
- no compression, has no such names to keep, so moving it to compressed Parquet has to produce
- `.parquet` rather than `.parquet.zst`.
-
- Takes the config as stored, before the incoming patch merges into it, so the value reflects
- what the export has been running rather than what it is moving to. An explicit value in the
- patch still wins, because the merge applies afterwards.
- """
- if destination_type not in OBJECT_STORAGE_DESTINATIONS:
- return
-
- stored_config.setdefault("legacy_parquet_extension", _writes_compressed_parquet(stored_config))
-
-
-def _coerce_integration_id(value: typing.Any) -> int | None:
- """Return the integration id encoded in a Redshift COPY credential value, if any.
-
- `BatchExportDestination.config` is an `EncryptedJSONField`, which stringifies scalar
- leaves on the decrypt round trip, so an id may arrive as an int or a numeric string.
- """
- if isinstance(value, bool):
- return None
- if isinstance(value, int):
- return value
- if isinstance(value, str) and value.isdigit():
- return int(value)
- return None
-
-
class BatchExportDestinationSerializer(serializers.ModelSerializer):
"""Serializer for an BatchExportDestination model.
diff --git a/products/batch_exports/backend/presentation/views/batch_export/exports.py b/products/batch_exports/backend/presentation/views/batch_export/exports.py
index 2ce3ce178fa6..07faa66e0d17 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/exports.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/exports.py
@@ -1,5 +1,6 @@
"""Batch export endpoints: create, update, pause, unpause, delete and destination tests."""
+import uuid
import typing
import dataclasses
from typing import cast
@@ -24,31 +25,19 @@ from posthog.api.log_entries import LogEntryMixin
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.utils import action
from posthog.models import Team, User
-from posthog.models.integration import (
- AzureBlobIntegration,
- AzureBlobIntegrationError,
- DatabricksIntegration,
- DatabricksIntegrationError,
- Integration,
- SnowflakeIntegration,
- SnowflakeIntegrationError,
-)
-from posthog.security.url_validation import (
- INVALID_HOST_MESSAGE,
- UNREACHABLE_HOST_MESSAGE,
- ShapeError,
- validate_external_host,
-)
-from posthog.temporal.common.client import sync_connect
+from posthog.models.integration import Integration
from products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_team
from products.batch_exports.backend.facade import api as batch_exports_api
-from products.batch_exports.backend.facade.contracts import InvalidBatchExportFilters, UnsupportedDestinationTestError
+from products.batch_exports.backend.facade.contracts import (
+ BatchExportNotFoundError,
+ BatchExportScheduleError,
+ InvalidBatchExportDestinationError,
+ InvalidBatchExportFilters,
+ UnsupportedDestinationTestError,
+)
from products.batch_exports.backend.facade.enums import (
- AZURE_BLOB_SUPPORTED_COMPRESSIONS,
EXPORTABLE_EVENTS_MODEL_FIELDS,
- S3_FAMILY_TYPES,
- S3_SUPPORTED_COMPRESSIONS,
SUPPORTED_FILTER_TYPES_DISPLAY,
TIMEZONES,
BatchExportDestinationType,
@@ -69,18 +58,7 @@ from products.batch_exports.backend.presentation.views.utils import (
check_high_frequency_batch_exports_enabled,
check_hogql_batch_exports_enabled,
)
-from products.batch_exports.backend.service import (
- DESTINATION_WORKFLOWS,
- BaseBatchExportInputs,
- BatchExportIdError,
- BatchExportSchema,
- BatchExportServiceError,
- BatchExportServiceRPCError,
- delete_batch_export,
- pause_batch_export,
- sync_batch_export,
- unpause_batch_export,
-)
+from products.batch_exports.backend.service import BatchExportSchema, sync_batch_export
from . import (
destinations as destination_views,
@@ -667,262 +645,29 @@ class BatchExportSerializer(serializers.ModelSerializer):
# TODO: could this be moved inside BatchExportDestinationSerializer::validate?
def validate_destination(self, destination_attrs: dict):
destination_type = destination_attrs["type"]
- config = destination_attrs["config"]
view = self.context.get("view")
- instance = None
+ export_id: uuid.UUID | None = None
if self.instance is not None:
- instance = self.instance
- existing_config = instance.destination.config
+ export_id = self.instance.id
elif view is not None and "pk" in view.kwargs:
# Running validation for a `detail=True` action.
- instance = view.get_object()
- existing_config = instance.destination.config
- else:
- existing_config = {}
+ export_id = uuid.UUID(str(view.kwargs["pk"]))
- if instance is not None and destination_type != instance.destination.type:
- raise serializers.ValidationError(
- f"Cannot change destination type from '{instance.destination.type}' to '{destination_type}'. "
- "Delete this batch export and create a new one with the new destination type."
- )
-
- # This setting is used for grandfathered exports that used the legacy Parquet file extension,
- # and is a one-way migration; once exports use the new `.parquet` extension it is not
- # possible to go back to using the legacy extension.
- if config.get("legacy_parquet_extension") is True and not destination_views._uses_legacy_parquet_extension(
- destination_type, existing_config
- ):
- raise serializers.ValidationError(
- "'legacy_parquet_extension' can only stay on for an export that already writes the "
- "compression codec into its Parquet file names. It cannot be turned on for a new "
- "export, or turned back on once an export moved to the standard '.parquet' extension."
+ integration: Integration | None = destination_attrs.get("integration")
+ try:
+ batch_exports_api.validate_batch_export_destination(
+ self.context["team_id"],
+ export_id=export_id,
+ destination_type=destination_type,
+ config=destination_attrs["config"],
+ integration_id=integration.id if integration is not None else None,
+ integration_sent="integration" in destination_attrs,
)
-
- merged_config = recursive_dict_merge(existing_config, config)
-
- # SSRF protection for HTTP batch exports
- if destination_type == BatchExportDestinationType.HTTP:
- url = merged_config.get("url")
- if url and url not in ("https://us.i.posthog.com/batch/", "https://eu.i.posthog.com/batch/"):
- raise serializers.ValidationError(f"Invalid destination URL: {url}")
-
- if destination_type == BatchExportDestinationType.SNOWFLAKE:
- integration: Integration | None = destination_attrs.get("integration")
- if integration is None and "integration" not in destination_attrs and instance is not None:
- # A PATCH may send config alone, which keeps the export's existing integration.
- # An explicit `integration: null` is a removal, and is rejected below.
- integration = instance.destination.integration
-
- if integration is None:
- raise serializers.ValidationError("Integration is required for Snowflake batch exports")
-
- # Also rejects an integration missing its account, user or authentication type, so a
- # half-built one fails here rather than on the export's first run.
- try:
- SnowflakeIntegration(integration)
- except SnowflakeIntegrationError as e:
- raise serializers.ValidationError(str(e))
-
- if destination_type in S3_FAMILY_TYPES:
- integration = destination_attrs.get("integration")
- if integration is None and "integration" not in destination_attrs and instance is not None:
- # A PATCH may send config alone, which keeps the export's existing integration.
- # An explicit `integration: null` is a removal, and is rejected below.
- integration = instance.destination.integration
-
- if integration is None:
- raise serializers.ValidationError(f"Integration is required for {destination_type} batch exports")
-
- # Credentials and the provider endpoint are not declared on the input dataclasses, so
- # `BatchExportDestinationSerializer.validate` already rejects them as unknown fields.
-
- # we already validate the required inputs in BatchExportDestinationSerializer::validate
- # so here we just ensure that the inputs are not empty
- required_non_empty_inputs = ["bucket_name", "region", "prefix"]
-
- empty_inputs = []
-
- for required_input in required_non_empty_inputs:
- value = config.get(required_input)
- if value is not None and isinstance(value, str) and value.strip() == "":
- empty_inputs.append(required_input)
-
- if empty_inputs:
- raise serializers.ValidationError(f"The following inputs are empty: {empty_inputs}")
-
- # The integration must match the destination kind. (Team ownership is already enforced
- # by the team-scoped `integration` field, which can only resolve integrations belonging
- # to the request's team.)
- if integration.kind != destination_views.S3_DESTINATION_TO_INTEGRATION_KIND[destination_type]:
- raise serializers.ValidationError(
- f"Integration provided is not an AWS S3 integration (got kind='{integration.kind}')"
- )
-
- # JSONLines is the default file format for S3 exports for legacy reasons
- file_format = merged_config.get("file_format", "JSONLines")
- supported_file_formats = S3_SUPPORTED_COMPRESSIONS.keys()
- if file_format not in supported_file_formats:
- raise serializers.ValidationError(
- f"File format {file_format} is not supported. Supported file formats are {list(supported_file_formats)}"
- )
- compression = merged_config.get("compression", None)
- if compression and compression not in S3_SUPPORTED_COMPRESSIONS[file_format]:
- raise serializers.ValidationError(
- f"Compression {compression} is not supported for file format {file_format}. Supported compressions are {S3_SUPPORTED_COMPRESSIONS[file_format]}"
- )
-
- if destination_type == BatchExportDestinationType.DATABRICKS:
- # validate the Integration is valid (this is mandatory for Databricks batch exports)
- integration = destination_attrs.get("integration")
- if integration is None:
- raise serializers.ValidationError("Integration is required for Databricks batch exports")
- if integration.kind != Integration.IntegrationKind.DATABRICKS:
- raise serializers.ValidationError("Integration is not a Databricks integration.")
- # try instantiate the integration to check if it's valid
- try:
- DatabricksIntegration(integration)
- except DatabricksIntegrationError as e:
- raise serializers.ValidationError(str(e))
-
- if destination_type == BatchExportDestinationType.POSTGRES:
- integration = destination_attrs.get("integration")
- # New Postgres exports must use an Integration for credentials. Exports created before
- # integrations existed keep their inline credentials, so only require it on create
- # (`instance is None`); existing inline-credential exports stay valid when edited.
- if integration is None and instance is None:
- raise serializers.ValidationError("Integration is required for Postgres batch exports")
- if integration is not None and integration.kind != Integration.IntegrationKind.POSTGRESQL:
- raise serializers.ValidationError("Integration is not a PostgreSQL integration.")
-
- if destination_type == BatchExportDestinationType.BIGQUERY:
- integration = destination_attrs.get("integration")
- if integration is None:
- raise serializers.ValidationError("Integration is required for BigQuery batch exports")
- if integration.kind != Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT:
- raise serializers.ValidationError("Integration is not a Google Cloud service account integration.")
-
- if destination_type == BatchExportDestinationType.AZURE_BLOB:
- # validate the Integration is valid (this is mandatory for Azure Blob batch exports)
- integration = destination_attrs.get("integration")
- if integration is None:
- raise serializers.ValidationError("Integration is required for Azure Blob batch exports")
- if integration.kind != Integration.IntegrationKind.AZURE_BLOB:
- raise serializers.ValidationError("Integration is not an Azure Blob integration.")
- # try instantiate the integration to check if it's valid
- try:
- AzureBlobIntegration(integration)
- except AzureBlobIntegrationError as e:
- raise serializers.ValidationError(str(e))
-
- file_format = merged_config.get("file_format", "JSONLines")
- supported_file_formats = AZURE_BLOB_SUPPORTED_COMPRESSIONS.keys()
- if file_format not in supported_file_formats:
- raise serializers.ValidationError(
- f"File format {file_format} is not supported. Supported file formats are {list(supported_file_formats)}"
- )
- compression = merged_config.get("compression", None)
- if compression and compression not in AZURE_BLOB_SUPPORTED_COMPRESSIONS[file_format]:
- raise serializers.ValidationError(
- f"Compression {compression} is not supported for file format {file_format}. Supported compressions are {AZURE_BLOB_SUPPORTED_COMPRESSIONS[file_format]}"
- )
-
- if destination_type == BatchExportDestinationType.REDSHIFT:
- integration = destination_attrs.get("integration")
-
- # Sticky integration: an export that uses one cannot drop back to inline credentials.
- # TODO: remove this guard once inline credentials are gone.
- if instance is not None and instance.destination.integration is not None and integration is None:
- raise serializers.ValidationError(
- "Cannot remove the integration from a Redshift batch export that uses one. "
- "Re-send its `integration` to keep it (or a different one to swap)."
- )
-
- # New Redshift exports must use an Integration for credentials. Exports created before
- # integrations existed keep their inline credentials, so only require it on create
- # (`instance is None`); existing inline-credential exports stay valid when edited.
- if integration is None and instance is None:
- raise serializers.ValidationError("Integration is required for Redshift batch exports")
-
- if integration is not None:
- # (Team ownership is already enforced by the team-scoped `integration` field.)
- if integration.kind != Integration.IntegrationKind.AWS_REDSHIFT:
- raise serializers.ValidationError("Integration is not a Redshift integration.")
-
- # Only plain Redshift integrations store a host; AWS-flavor ones obtain
- # temporary credentials for a cluster endpoint that stays in the export config.
- if "host" not in integration.config and not merged_config.get("host"):
- raise serializers.ValidationError(
- "A 'host' is required when using an AWS Redshift integration: "
- "set it to the cluster or workgroup endpoint."
- )
-
- mode = merged_config.get("mode")
-
- if mode == "COPY":
- copy_inputs = merged_config.get("copy_inputs", {})
- if not copy_inputs:
- raise serializers.ValidationError("Missing required inputs for 'COPY'")
-
- required_inputs = {"s3_bucket", "region_name", "authorization", "bucket_credentials"}
- if required_inputs - copy_inputs.keys():
- raise serializers.ValidationError("Missing required input for 'COPY'")
-
- credential_keys = {"aws_access_key_id", "aws_secret_access_key"}
-
- bucket_credentials = copy_inputs["bucket_credentials"]
- bucket_integration_id = destination_views._coerce_integration_id(bucket_credentials)
- authorization = copy_inputs.get("authorization")
- authorization_integration_id = destination_views._coerce_integration_id(authorization)
-
- existing_copy_inputs = existing_config.get("copy_inputs") or {}
-
- for field_name, copy_integration_id in (
- ("bucket_credentials", bucket_integration_id),
- ("authorization", authorization_integration_id),
- ):
- # Sticky like the top-level integration: COPY credentials that reference an
- # integration cannot drop back to inline values.
- # TODO: remove this guard once integrations are mandatory for Redshift and
- # inline credentials are gone.
- if (
- instance is not None
- and copy_integration_id is None
- and destination_views._coerce_integration_id(existing_copy_inputs.get(field_name)) is not None
- ):
- raise serializers.ValidationError(
- f"Cannot switch '{field_name}' from an integration to inline credentials. "
- "Send a different integration ID to swap, or omit the field to keep the current one."
- )
- if copy_integration_id is None:
- continue
- # These ids live inside `config` rather than the team-scoped `integration`
- # field, so team ownership must be checked here.
- if not Integration.objects.filter(
- id=copy_integration_id,
- team_id=self.context["team_id"],
- kind=Integration.IntegrationKind.AWS_S3,
- ).exists():
- raise serializers.ValidationError(
- f"'{field_name}' does not reference an AWS S3 integration of this project."
- )
-
- if bucket_integration_id is None and (
- not isinstance(bucket_credentials, dict) or credential_keys - bucket_credentials.keys()
- ):
- raise serializers.ValidationError("Missing required bucket credentials for 'COPY'")
-
- if authorization_integration_id is None:
- if isinstance(authorization, dict):
- if credential_keys - authorization.keys():
- raise serializers.ValidationError("Missing required credentials for 'COPY'")
- elif isinstance(authorization, str):
- if not authorization.strip():
- raise serializers.ValidationError("Missing required IAM role for 'COPY'")
- else:
- raise serializers.ValidationError(
- "Authorization for 'COPY' must be an IAM role ARN, AWS credentials, or an integration ID."
- )
+ except InvalidBatchExportDestinationError as e:
+ raise serializers.ValidationError(str(e))
+ except BatchExportNotFoundError:
+ raise NotFound(f"BatchExport ID '{export_id}' not found.")
if destination_type == BatchExportDestinationType.WORKFLOWS:
team_id = self.context["team_id"]
@@ -942,27 +687,6 @@ class BatchExportSerializer(serializers.ModelSerializer):
):
raise PermissionDenied("Backfilling Workflows is not enabled for this team.")
- if destination_type in (
- BatchExportDestinationType.POSTGRES,
- BatchExportDestinationType.REDSHIFT,
- ):
- # PostgreSQL-server integrations (Postgres, plain Redshift) keep the host in the
- # linked Integration; AWS Redshift integrations and inline configs keep it in
- # `config`. Prefer the Integration's host when it has one so we don't skip
- # SSRF validation (and don't `KeyError` on a `config` that has no `host`).
- integration = destination_attrs.get("integration")
- host = integration.config.get("host") if integration is not None else None
- if host is None:
- host = merged_config.get("host")
-
- if host is not None:
- try:
- validate_external_host(host)
- except ShapeError:
- raise serializers.ValidationError(INVALID_HOST_MESSAGE)
- except ValueError:
- raise serializers.ValidationError(UNREACHABLE_HOST_MESSAGE)
-
return destination_attrs
def create(self, validated_data: dict) -> BatchExport:
@@ -988,7 +712,9 @@ class BatchExportSerializer(serializers.ModelSerializer):
# TODO: Migrate batch exports using a HogQL query to HogQL model.
validated_data["schema"] = self.serialize_hogql_query_to_batch_export_schema(hogql_query)
- destination_views._set_default_parquet_extension(destination_data["type"], destination_data["config"])
+ destination_data["config"] = batch_exports_api.prepare_new_destination_config(
+ destination_data["type"], destination_data["config"]
+ )
destination = BatchExportDestination(**destination_data)
user = self.context["request"].user
@@ -1119,10 +845,8 @@ class BatchExportSerializer(serializers.ModelSerializer):
if destination_data:
# Type changes are rejected by `validate_destination` — the incoming `type`
# (if any) always equals the existing type by the time we get here.
- destination_views._pin_existing_parquet_extension(
- batch_export.destination.type, batch_export.destination.config
- )
- batch_export.destination.config = recursive_dict_merge(
+ batch_export.destination.config = batch_exports_api.merge_destination_config(
+ batch_export.destination.type,
batch_export.destination.config,
destination_data.get("config", {}),
)
@@ -1159,29 +883,6 @@ class BatchExportSerializer(serializers.ModelSerializer):
return batch_export
-def recursive_dict_merge(
- old: dict[str, typing.Any],
- new: dict[str, typing.Any],
-) -> dict[str, typing.Any]:
- """Merge two dictionaries, and recursively merge any dictionaries in them."""
- merged = {**old, **new}
-
- for key, value in merged.items():
- if not isinstance(value, dict):
- continue
-
- # Recurse only when both sides are dicts: a dict replacing a scalar (e.g. inline
- # credentials replacing an IAM role or an integration id) is an overwrite, not a merge.
- old_value = old.get(key)
- new_value = new.get(key)
- merged[key] = recursive_dict_merge(
- old_value if isinstance(old_value, dict) else {},
- new_value if isinstance(new_value, dict) else {},
- )
-
- return merged
-
-
@extend_schema(tags=["batch_exports"])
@extend_schema_view(
# Request bodies use a polymorphic destination schema so that integration-backed types
@@ -1222,16 +923,12 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVi
team_id = batch_export.team_id
note = f"Pause requested by user {user_id} from team {team_id}"
- temporal = sync_connect()
-
try:
- pause_batch_export(temporal, str(batch_export.id), note=note)
- except BatchExportIdError:
- raise NotFound(f"BatchExport ID '{str(batch_export.id)}' not found.")
- except BatchExportServiceRPCError:
+ batch_exports_api.pause_batch_export(team_id, batch_export.id, note=note)
+ except BatchExportNotFoundError:
+ raise NotFound(f"BatchExport ID '{batch_export.id}' not found.")
+ except BatchExportScheduleError:
raise ValidationError("Invalid request to pause a BatchExport could not be carried out")
- except BatchExportServiceError:
- raise
return response.Response({"paused": True})
@@ -1251,16 +948,13 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVi
note = f"Unpause requested by user {user_id} from team {team_id}"
batch_export = self.get_object()
- temporal = sync_connect()
try:
- unpause_batch_export(temporal, str(batch_export.id), note=note, backfill=backfill)
- except BatchExportIdError:
- raise NotFound(f"BatchExport ID '{str(batch_export.id)}' not found.")
- except BatchExportServiceRPCError:
+ batch_exports_api.unpause_batch_export(batch_export.team_id, batch_export.id, note=note, backfill=backfill)
+ except BatchExportNotFoundError:
+ raise NotFound(f"BatchExport ID '{batch_export.id}' not found.")
+ except BatchExportScheduleError:
raise ValidationError("Invalid request to unpause a BatchExport could not be carried out")
- except BatchExportServiceError:
- raise
return response.Response({"paused": False})
@@ -1272,7 +966,10 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVi
since we are deleting, we assume that we can recover from this state by finishing the delete operation by calling
instance.save().
"""
- delete_batch_export(instance)
+ try:
+ batch_exports_api.delete_batch_export(instance.id, instance.team_id)
+ except BatchExportNotFoundError:
+ raise NotFound(f"BatchExport ID '{instance.id}' not found.")
def _parse_destination_test_step(self, request: request.Request) -> int:
step_serializer = DestinationTestStepRequestSerializer(data=request.data)
@@ -1322,13 +1019,10 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVi
test_step = self._parse_destination_test_step(request)
batch_export = self.get_object()
-
- # Remove any additional fields from stored configuration
- _, workflow_inputs = DESTINATION_WORKFLOWS[batch_export.destination.type]
- workflow_fields = {field.name for field in dataclasses.fields(BaseBatchExportInputs)} | {
- field.name for field in dataclasses.fields(workflow_inputs)
- }
- stored_config = {k: v for k, v in batch_export.destination.config.items() if k in workflow_fields}
+ try:
+ stored_config = batch_exports_api.get_destination_test_config(batch_export.team_id, batch_export.id)
+ except BatchExportNotFoundError:
+ raise NotFound(f"BatchExport ID '{batch_export.id}' not found.")
data = request.data
data["destination"]["config"] = {**stored_config, **data["destination"]["config"]}
diff --git a/products/batch_exports/backend/service.py b/products/batch_exports/backend/service.py
index 8151809a5924..9c7434d5e9ea 100644
--- a/products/batch_exports/backend/service.py
+++ b/products/batch_exports/backend/service.py
@@ -806,19 +806,37 @@ class BatchExportServiceScheduleNotFound(BatchExportServiceRPCError):
super().__init__(f"The Temporal Schedule {schedule_id} was not found (maybe it was deleted?)")
-def pause_batch_export(temporal: Client, batch_export_id: str, note: str | None = None) -> bool:
+def _get_batch_export_for_schedule_change(batch_export_id: str, team_id: int | None) -> BatchExport:
+ """Load the export that a pause or unpause changes.
+
+ With ``team_id``, only a live export of that team matches, so a request cannot reach another
+ team's export. Internal callers (Temporal, deletes) pass no team and match any export.
+ """
+ # nosemgrep: idor-lookup-without-team (scoped below when the caller passes team_id)
+ batch_exports = BatchExport.objects.filter(id=batch_export_id)
+ if team_id is not None:
+ batch_exports = batch_exports.filter(team_id=team_id, deleted=False)
+
+ try:
+ return batch_exports.get()
+ except BatchExport.DoesNotExist:
+ raise BatchExportIdError(batch_export_id)
+
+
+def pause_batch_export(
+ temporal: Client, batch_export_id: str, note: str | None = None, *, team_id: int | None = None
+) -> bool:
"""Pause this BatchExport.
We pass the call to the underlying Temporal Schedule.
Returns:
`True` if the batch export was paused, `False` if it was already paused.
+
+ Raises:
+ BatchExportIdError: If no export matches ``batch_export_id`` (and ``team_id``, when given).
"""
- try:
- # nosemgrep: idor-lookup-without-team (internal service, team_id passed as parameter)
- batch_export = BatchExport.objects.get(id=batch_export_id)
- except BatchExport.DoesNotExist:
- raise BatchExportIdError(batch_export_id)
+ batch_export = _get_batch_export_for_schedule_change(batch_export_id, team_id)
if batch_export.paused is True:
return False
@@ -869,6 +887,8 @@ def unpause_batch_export(
batch_export_id: str,
note: str | None = None,
backfill: bool = False,
+ *,
+ team_id: int | None = None,
) -> None:
"""Unpause this BatchExport.
@@ -880,15 +900,12 @@ def unpause_batch_export(
batch_export_id: The ID of the BatchExport to unpause.
note: An optional note to include in the Schedule when unpausing.
backfill: If True, a backfill will be triggered since the BatchExport's last_paused_at.
+ team_id: If set, only a live export of this team matches.
Raises:
- BatchExportIdError: If the provided batch_export_id doesn't point to an existing BatchExport.
+ BatchExportIdError: If no export matches ``batch_export_id`` (and ``team_id``, when given).
"""
- try:
- # nosemgrep: idor-lookup-without-team (internal service, team_id passed as parameter)
- batch_export = BatchExport.objects.get(id=batch_export_id)
- except BatchExport.DoesNotExist:
- raise BatchExportIdError(batch_export_id)
+ batch_export = _get_batch_export_for_schedule_change(batch_export_id, team_id)
if batch_export.paused is False:
return
diff --git a/products/batch_exports/backend/tests/test_facade.py b/products/batch_exports/backend/tests/test_facade.py
index cd914a424555..8316a65438c3 100644
--- a/products/batch_exports/backend/tests/test_facade.py
+++ b/products/batch_exports/backend/tests/test_facade.py
@@ -300,6 +300,27 @@ def test_runs_and_backfills_are_scoped_to_the_team_and_export(team, organization
api.create_batch_export_backfill(team_id, export_id, start_at=None, end_at=None)
[email protected]("scope", ["other_team", "deleted"])
+def test_pause_unpause_and_delete_refuse_another_teams_or_a_deleted_export(team, organization, scope):
+ export_id = _create_export(team, paused=True)
+ if scope == "other_team":
+ team_id = create_team(organization=organization).pk
+ else:
+ team_id = team.pk
+ testing.update_batch_export(export_id, team_id=team.pk, deleted=True)
+
+ # A leaked lookup would pause silently, reach Temporal to unpause, or run the delete.
+ with pytest.raises(contracts.BatchExportNotFoundError):
+ api.pause_batch_export(team_id, export_id, note="pause")
+ with pytest.raises(contracts.BatchExportNotFoundError):
+ api.unpause_batch_export(team_id, export_id, note="unpause", backfill=False)
+ with pytest.raises(contracts.BatchExportNotFoundError):
+ api.delete_batch_export(export_id, team_id)
+
+ export = BatchExport.objects.get(id=export_id)
+ assert (export.paused, export.deleted) == (True, scope == "deleted")
+
+
def test_deleting_team_batch_exports_continues_past_a_missing_schedule(team):
first = _create_export(team, name="first")
second = _create_export(team, name="second")