:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(batch-exports): move export rules and actions behind the facade (#124715)
Ross7a363fa3c0bf
diff --git a/products/batch_exports/backend/facade/api.py b/products/batch_exports/backend/facade/api.pyindex 6e50a2e6a18a..de1897537c37 100644--- a/products/batch_exports/backend/facade/api.py+++ b/products/batch_exports/backend/facade/api.py@@ -11,9 +11,9 @@ is visible in one place.and ``temporalio`` on the ``django.setup()`` path. Core config-time consumers import thisfacade, so it reaches the service only through ``_service()``, which imports it at calltime. For the same reason the Temporal client is built in ``_get_temporal_client()``, and-the HogQL, destination config, file download, run and backfill modules load through-``_import_hogql_source()``, ``_import_destination_config()``, ``_import_file_downloads()``,-``_import_runs()`` and ``_import_backfills()``.+the HogQL, destination config, export, file download, run and backfill modules load through+``_import_hogql_source()``, ``_import_destination_config()``, ``_import_exports()``,+``_import_file_downloads()``, ``_import_runs()`` and ``_import_backfills()``.Temporal workflow and activity registration crosses as objects, not data, so it lives in``facade/temporal.py``. The destination clients the warehouse writers reuse live in@@ -76,6 +76,7 @@ __all__ = ["get_batch_export_backfill","get_batch_export_by_name","get_batch_export_run",+ "get_destination_test_config","get_file_download_run","get_file_download_url","get_latest_completed_run",@@ -90,9 +91,14 @@ __all__ = ["list_file_download_runs","list_latest_failed_runs","list_supported_intervals",+ "merge_destination_config",+ "pause_batch_export",+ "prepare_new_destination_config","retry_batch_export_run","run_destination_test_step","start_file_download",+ "unpause_batch_export",+ "validate_batch_export_destination","validate_batch_export_filters","validate_destination_config","validate_hogql_query",@@ -150,6 +156,13 @@ def _import_destination_config() -> ModuleType:return destination_config+def _import_exports() -> ModuleType:+ """Import the export logic at call time. See the module docstring for why."""+ from products.batch_exports.backend.logic import exports++ return exports++def _import_file_downloads() -> ModuleType:"""Import the file download module at call time. See the module docstring for why."""from products.batch_exports.backend.logic import file_downloads@@ -479,10 +492,31 @@ def create_workflows_backfill_export(def delete_batch_export(export_id: UUID, team_id: int) -> None:- """Delete a batch export: pause it, cancel its work, then drop its Temporal schedule."""- service = _service()- batch_export = BatchExport.objects.select_related("destination").get(id=export_id, team_id=team_id)- service.delete_batch_export(batch_export)+ """Delete a batch export: pause it, cancel its work, then drop its Temporal schedule.++ Raises ``contracts.BatchExportNotFoundError`` when the team has no such live export.+ """+ _import_exports().delete_batch_export(team_id, export_id)+++def pause_batch_export(team_id: int, export_id: UUID, *, note: str) -> None:+ """Pause a batch export, so its schedule starts no new runs.++ ``note`` records who asked, on the Temporal schedule. Raises+ ``contracts.BatchExportNotFoundError`` when the team has no such export, and+ ``contracts.BatchExportScheduleError`` when Temporal rejects the change.+ """+ _import_exports().pause_batch_export(team_id, export_id, note=note)+++def unpause_batch_export(team_id: int, export_id: UUID, *, note: str, backfill: bool) -> None:+ """Unpause a batch export. With ``backfill``, also backfill the runs it missed while paused.++ ``note`` records who asked, on the Temporal schedule. Raises+ ``contracts.BatchExportNotFoundError`` when the team has no such export, and+ ``contracts.BatchExportScheduleError`` when Temporal rejects the change.+ """+ _import_exports().unpause_batch_export(team_id, export_id, note=note, backfill=backfill)def backfill_batch_export(@@ -521,6 +555,62 @@ def coerce_destination_config(destination_type: str, config: dict[str, Any]) ->return _service().coerce_config_to_declared_types(destination_type, config)+def validate_batch_export_destination(+ team_id: int,+ *,+ export_id: UUID | None,+ destination_type: str,+ config: Mapping[str, Any],+ integration_id: int | None,+ integration_sent: bool,+) -> None:+ """Check that a destination can be saved, for a new export or as a change to ``export_id``.++ ``config`` already passed ``validate_destination_config``. ``integration_sent`` is false when+ the request left the integration out, which keeps the existing integration of the export.+ Raises ``contracts.InvalidBatchExportDestinationError`` with the failed check as the message.+ """+ _import_exports().validate_destination(+ team_id,+ export_id=export_id,+ destination_type=destination_type,+ config=config,+ integration_id=integration_id,+ integration_sent=integration_sent,+ )+++def prepare_new_destination_config(destination_type: str, config: Mapping[str, Any]) -> dict[str, Any]:+ """Return the config a new destination saves, with the defaults that a new export opts into.++ Temporary: only the export serializer calls this, because it still saves the destination+ through the ORM. Do not add callers. The function goes away when export creation moves+ behind the facade.+ """+ return _import_exports().prepare_new_destination_config(destination_type, config)+++def merge_destination_config(+ destination_type: str, stored_config: Mapping[str, Any], patch: Mapping[str, Any]+) -> dict[str, Any]:+ """Return the config an export saves after a patch, merged key by key into its stored config.++ Temporary: only the export serializer calls this, because it still saves the destination+ through the ORM. Do not add callers. The function goes away when export updates move+ behind the facade.+ """+ return _import_exports().merge_destination_config(destination_type, stored_config, patch)+++def get_destination_test_config(team_id: int, export_id: UUID) -> dict[str, Any]:+ """Return the stored destination config of an export, for a connection test of that export.++ Only the fields that the workflow inputs of the destination declare are kept. The config holds+ the credentials of the destination, so it must not leave the product.+ """+ return _import_exports().get_destination_test_config(team_id, export_id)++def describe_destination_test(destination_type: str) -> contracts.DestinationTest:"""Return the steps of a destination's connection test, with no step run."""destination_test = destination_tests.get_destination_test(destination=destination_type)diff --git a/products/batch_exports/backend/facade/contracts.py b/products/batch_exports/backend/facade/contracts.pyindex 3b82bd06bd73..cb35fd37712a 100644--- a/products/batch_exports/backend/facade/contracts.py+++ b/products/batch_exports/backend/facade/contracts.py@@ -40,6 +40,17 @@ class InvalidDestinationConfigError(ValueError):"""+class InvalidBatchExportDestinationError(ValueError):+ """Raised when a destination cannot be saved, for a new export or as a change to an existing one.++ The message says which check failed.+ """+++class BatchExportScheduleError(Exception):+ """Raised when Temporal rejects a change to the schedule of a batch export."""++class UnsupportedDestinationTestError(ValueError):"""Raised when a destination type has no connection test."""diff --git a/products/batch_exports/backend/logic/exports.py b/products/batch_exports/backend/logic/exports.pynew file mode 100644index 000000000000..9a1e365e4e85--- /dev/null+++ b/products/batch_exports/backend/logic/exports.py@@ -0,0 +1,523 @@+"""Scheduled batch exports: check a destination before it is saved, pause or unpause an export.++The facade reaches this module through a call-time import, because it imports the service layer+and the Temporal client, neither of which may load on the ``django.setup()`` path.+"""++import typing+import contextlib+import dataclasses+from uuid import UUID++from posthog.models.integration import (+ AzureBlobIntegration,+ AzureBlobIntegrationError,+ DatabricksIntegration,+ DatabricksIntegrationError,+ Integration,+ SnowflakeIntegration,+ SnowflakeIntegrationError,+)+from posthog.security.url_validation import (+ INVALID_HOST_MESSAGE,+ UNREACHABLE_HOST_MESSAGE,+ ShapeError,+ validate_external_host,+)+from posthog.temporal.common.client import sync_connect++from products.batch_exports.backend import service+from products.batch_exports.backend.facade import contracts+from products.batch_exports.backend.facade.enums import (+ AZURE_BLOB_SUPPORTED_COMPRESSIONS,+ OBJECT_STORAGE_DESTINATIONS,+ S3_FAMILY_TYPES,+ S3_SUPPORTED_COMPRESSIONS,+ BatchExportDestinationType,+)+from products.batch_exports.backend.models.batch_export import BatchExport++# S3-family destinations that may authenticate via an Integration, mapped to+# the linked integration's kind. Adding a future S3-family destination (e.g. a+# first-class GCS-via-S3 type) is a one-line addition here.+_S3_DESTINATION_TO_INTEGRATION_KIND: dict[str, Integration.IntegrationKind] = {+ BatchExportDestinationType.AWS_S3: Integration.IntegrationKind.AWS_S3,+ BatchExportDestinationType.S3_COMPATIBLE: Integration.IntegrationKind.S3_COMPATIBLE,+}++# The only URLs an HTTP destination may send to, as SSRF protection.+_ALLOWED_HTTP_DESTINATION_URLS = ("https://us.i.posthog.com/batch/", "https://eu.i.posthog.com/batch/")+++def _get_batch_export(team_id: int, export_id: UUID, *related: str) -> BatchExport:+ """Load a live export of the team, joining only the ``related`` fields the caller reads."""+ batch_exports = BatchExport.objects.exclude(deleted=True)+ # `select_related()` with no fields joins every non-null foreign key, so only call it with some.+ if related:+ batch_exports = batch_exports.select_related(*related)+ try:+ return batch_exports.get(id=export_id, team_id=team_id)+ except BatchExport.DoesNotExist as e:+ raise contracts.BatchExportNotFoundError() from e+++def _get_integration(team_id: int, integration_id: int | None) -> Integration | None:+ if integration_id is None:+ return None+ try:+ return Integration.objects.get(id=integration_id, team_id=team_id)+ except Integration.DoesNotExist as e:+ raise contracts.InvalidBatchExportDestinationError(+ "The integration for this destination no longer exists. Choose another integration and try again."+ ) from e+++def _recursive_dict_merge(+ old: typing.Mapping[str, typing.Any],+ new: typing.Mapping[str, typing.Any],+) -> dict[str, typing.Any]:+ """Merge two dictionaries, and recursively merge any dictionaries in them."""+ merged = {**old, **new}++ for key, value in merged.items():+ if not isinstance(value, dict):+ continue++ # Recurse only when both sides are dicts: a dict replacing a scalar (e.g. inline+ # credentials replacing an IAM role or an integration id) is an overwrite, not a merge.+ old_value = old.get(key)+ new_value = new.get(key)+ merged[key] = _recursive_dict_merge(+ old_value if isinstance(old_value, dict) else {},+ new_value if isinstance(new_value, dict) else {},+ )++ return merged+++def _writes_compressed_parquet(config: dict[str, typing.Any]) -> bool:+ """Whether a config produces Parquet file names that carry a compression codec."""+ return config.get("file_format") == "Parquet" and config.get("compression") is not None+++def _uses_legacy_parquet_extension(destination_type: str, stored_config: dict[str, typing.Any]) -> bool:+ """Whether an export's stored config already writes the codec into its Parquet file names.++ Reads through `coerce_config_to_declared_types`, because `EncryptedJSONField` stringifies+ scalars on write and the string "False" is truthy.+ """+ coerced = service.coerce_config_to_declared_types(destination_type, stored_config)+ stored = coerced.get("legacy_parquet_extension")+ if stored is None:+ # export was created before the `legacy_parquet_extension` field was added+ # so will use the legacy extension if it writes compressed Parquet files+ return _writes_compressed_parquet(coerced)+ return bool(stored)+++def _set_default_parquet_extension(destination_type: str, config: dict[str, typing.Any]) -> None:+ """Opt a newly created destination into the standard `.parquet` extension.++ The workflow input dataclasses default this to `True`, so that an export whose Temporal+ schedule predates the field keeps the same file extension as before in order to maintain+ compatibility.+ """+ if destination_type in OBJECT_STORAGE_DESTINATIONS:+ config.setdefault("legacy_parquet_extension", False)+++def _pin_existing_parquet_extension(destination_type: str, stored_config: dict[str, typing.Any]) -> None:+ """Record what an export's file names already look like, before a patch can change its format.++ An export that predates this setting has no value for it, and a missing value reads as the+ legacy naming. That is correct only for an export that already writes names carrying a codec,+ which means Parquet with a compression codec set. An export on JSON Lines, or on Parquet with+ no compression, has no such names to keep, so moving it to compressed Parquet has to produce+ `.parquet` rather than `.parquet.zst`.++ Takes the config as stored, before the incoming patch merges into it, so the value reflects+ what the export has been running rather than what it is moving to. An explicit value in the+ patch still wins, because the merge applies afterwards.+ """+ if destination_type not in OBJECT_STORAGE_DESTINATIONS:+ return++ stored_config.setdefault("legacy_parquet_extension", _writes_compressed_parquet(stored_config))+++def _coerce_integration_id(value: typing.Any) -> int | None:+ """Return the integration id encoded in a Redshift COPY credential value, if any.++ `BatchExportDestination.config` is an `EncryptedJSONField`, which stringifies scalar+ leaves on the decrypt round trip, so an id may arrive as an int or a numeric string.+ """+ if isinstance(value, bool):+ return None+ if isinstance(value, int):+ return value+ if isinstance(value, str) and value.isdigit():+ return int(value)+ return None+++def _check_file_format(merged_config: dict[str, typing.Any], supported_compressions: dict[str, list[str]]) -> None:+ # JSONLines is the default file format for S3 exports for legacy reasons+ file_format = merged_config.get("file_format", "JSONLines")+ supported_file_formats = supported_compressions.keys()+ if file_format not in supported_file_formats:+ raise contracts.InvalidBatchExportDestinationError(+ f"File format {file_format} is not supported. Supported file formats are {list(supported_file_formats)}"+ )+ compression = merged_config.get("compression", None)+ if compression and compression not in supported_compressions[file_format]:+ raise contracts.InvalidBatchExportDestinationError(+ f"Compression {compression} is not supported for file format {file_format}. Supported compressions are {supported_compressions[file_format]}"+ )+++def _check_redshift_copy_inputs(+ team_id: int,+ instance: BatchExport | None,+ merged_config: dict[str, typing.Any],+ existing_config: dict[str, typing.Any],+) -> None:+ copy_inputs = merged_config.get("copy_inputs", {})+ if not copy_inputs:+ raise contracts.InvalidBatchExportDestinationError("Missing required inputs for 'COPY'")++ required_inputs = {"s3_bucket", "region_name", "authorization", "bucket_credentials"}+ if required_inputs - copy_inputs.keys():+ raise contracts.InvalidBatchExportDestinationError("Missing required input for 'COPY'")++ credential_keys = {"aws_access_key_id", "aws_secret_access_key"}++ bucket_credentials = copy_inputs["bucket_credentials"]+ bucket_integration_id = _coerce_integration_id(bucket_credentials)+ authorization = copy_inputs.get("authorization")+ authorization_integration_id = _coerce_integration_id(authorization)++ existing_copy_inputs = existing_config.get("copy_inputs") or {}++ for field_name, copy_integration_id in (+ ("bucket_credentials", bucket_integration_id),+ ("authorization", authorization_integration_id),+ ):+ # Sticky like the top-level integration: COPY credentials that reference an+ # integration cannot drop back to inline values.+ # TODO: remove this guard once integrations are mandatory for Redshift and+ # inline credentials are gone.+ if (+ instance is not None+ and copy_integration_id is None+ and _coerce_integration_id(existing_copy_inputs.get(field_name)) is not None+ ):+ raise contracts.InvalidBatchExportDestinationError(+ f"Cannot switch '{field_name}' from an integration to inline credentials. "+ "Send a different integration ID to swap, or omit the field to keep the current one."+ )+ if copy_integration_id is None:+ continue+ # These ids live inside `config` rather than the team-scoped `integration`+ # field, so team ownership must be checked here.+ if not Integration.objects.filter(+ id=copy_integration_id,+ team_id=team_id,+ kind=Integration.IntegrationKind.AWS_S3,+ ).exists():+ raise contracts.InvalidBatchExportDestinationError(+ f"'{field_name}' does not reference an AWS S3 integration of this project."+ )++ if bucket_integration_id is None and (+ not isinstance(bucket_credentials, dict) or credential_keys - bucket_credentials.keys()+ ):+ raise contracts.InvalidBatchExportDestinationError("Missing required bucket credentials for 'COPY'")++ if authorization_integration_id is None:+ if isinstance(authorization, dict):+ if credential_keys - authorization.keys():+ raise contracts.InvalidBatchExportDestinationError("Missing required credentials for 'COPY'")+ elif isinstance(authorization, str):+ if not authorization.strip():+ raise contracts.InvalidBatchExportDestinationError("Missing required IAM role for 'COPY'")+ else:+ raise contracts.InvalidBatchExportDestinationError(+ "Authorization for 'COPY' must be an IAM role ARN, AWS credentials, or an integration ID."+ )+++def validate_destination(+ team_id: int,+ *,+ export_id: UUID | None,+ destination_type: str,+ config: typing.Mapping[str, typing.Any],+ integration_id: int | None,+ integration_sent: bool,+) -> None:+ """Check that a destination can be saved, for a new export or as a change to ``export_id``.++ ``integration_sent`` is false when the request left the integration out, which keeps the+ export's existing integration. An explicit null sends ``integration_id=None``.++ Raises:+ contracts.InvalidBatchExportDestinationError: If the destination fails a check. The+ message says which one.+ contracts.BatchExportNotFoundError: If ``export_id`` names no export of the team.+ """+ instance = (+ _get_batch_export(team_id, export_id, "destination", "destination__integration")+ if export_id is not None+ else None+ )+ existing_config: dict[str, typing.Any] = instance.destination.config if instance is not None else {}++ if instance is not None and destination_type != instance.destination.type:+ raise contracts.InvalidBatchExportDestinationError(+ f"Cannot change destination type from '{instance.destination.type}' to '{destination_type}'. "+ "Delete this batch export and create a new one with the new destination type."+ )++ # This setting is used for grandfathered exports that used the legacy Parquet file extension,+ # and is a one-way migration; once exports use the new `.parquet` extension it is not+ # possible to go back to using the legacy extension.+ if config.get("legacy_parquet_extension") is True and not _uses_legacy_parquet_extension(+ destination_type, existing_config+ ):+ raise contracts.InvalidBatchExportDestinationError(+ "'legacy_parquet_extension' can only stay on for an export that already writes the "+ "compression codec into its Parquet file names. It cannot be turned on for a new "+ "export, or turned back on once an export moved to the standard '.parquet' extension."+ )++ merged_config = _recursive_dict_merge(existing_config, config)++ # The integration the request sent. None when it sent none, or sent an explicit null.+ if instance is not None and integration_id is not None and instance.destination.integration_id == integration_id:+ # A PATCH usually sends the export's current integration again, which is already loaded.+ requested_integration = instance.destination.integration+ else:+ requested_integration = _get_integration(team_id, integration_id)++ # A PATCH may send config alone, which keeps the export's existing integration.+ # An explicit `integration: null` is a removal, and is rejected below.+ kept_integration = requested_integration+ if requested_integration is None and not integration_sent and instance is not None:+ kept_integration = instance.destination.integration++ # SSRF protection for HTTP batch exports+ if destination_type == BatchExportDestinationType.HTTP:+ url = merged_config.get("url")+ if url and url not in _ALLOWED_HTTP_DESTINATION_URLS:+ raise contracts.InvalidBatchExportDestinationError(f"Invalid destination URL: {url}")++ if destination_type == BatchExportDestinationType.SNOWFLAKE:+ integration = kept_integration+ if integration is None:+ raise contracts.InvalidBatchExportDestinationError("Integration is required for Snowflake batch exports")++ # Also rejects an integration missing its account, user or authentication type, so a+ # half-built one fails here rather than on the export's first run.+ try:+ SnowflakeIntegration(integration)+ except SnowflakeIntegrationError as e:+ raise contracts.InvalidBatchExportDestinationError(str(e))++ if destination_type in S3_FAMILY_TYPES:+ integration = kept_integration+ if integration is None:+ raise contracts.InvalidBatchExportDestinationError(+ f"Integration is required for {destination_type} batch exports"+ )++ # Credentials and the provider endpoint are not declared on the input dataclasses, so+ # `validate_destination_config` already rejects them as unknown fields.++ # we already validate the required inputs in `validate_destination_config`+ # so here we just ensure that the inputs are not empty+ required_non_empty_inputs = ["bucket_name", "region", "prefix"]++ empty_inputs = []++ for required_input in required_non_empty_inputs:+ value = config.get(required_input)+ if value is not None and isinstance(value, str) and value.strip() == "":+ empty_inputs.append(required_input)++ if empty_inputs:+ raise contracts.InvalidBatchExportDestinationError(f"The following inputs are empty: {empty_inputs}")++ # The integration must match the destination kind. (Team ownership is already enforced+ # by the team-scoped `integration` field, which can only resolve integrations belonging+ # to the request's team.)+ if integration.kind != _S3_DESTINATION_TO_INTEGRATION_KIND[destination_type]:+ raise contracts.InvalidBatchExportDestinationError(+ f"Integration provided is not an AWS S3 integration (got kind='{integration.kind}')"+ )++ _check_file_format(merged_config, S3_SUPPORTED_COMPRESSIONS)++ if destination_type == BatchExportDestinationType.DATABRICKS:+ # validate the Integration is valid (this is mandatory for Databricks batch exports)+ if requested_integration is None:+ raise contracts.InvalidBatchExportDestinationError("Integration is required for Databricks batch exports")+ if requested_integration.kind != Integration.IntegrationKind.DATABRICKS:+ raise contracts.InvalidBatchExportDestinationError("Integration is not a Databricks integration.")+ # try instantiate the integration to check if it's valid+ try:+ DatabricksIntegration(requested_integration)+ except DatabricksIntegrationError as e:+ raise contracts.InvalidBatchExportDestinationError(str(e))++ if destination_type == BatchExportDestinationType.POSTGRES:+ # New Postgres exports must use an Integration for credentials. Exports created before+ # integrations existed keep their inline credentials, so only require it on create+ # (`instance is None`); existing inline-credential exports stay valid when edited.+ if requested_integration is None and instance is None:+ raise contracts.InvalidBatchExportDestinationError("Integration is required for Postgres batch exports")+ if requested_integration is not None and requested_integration.kind != Integration.IntegrationKind.POSTGRESQL:+ raise contracts.InvalidBatchExportDestinationError("Integration is not a PostgreSQL integration.")++ if destination_type == BatchExportDestinationType.BIGQUERY:+ if requested_integration is None:+ raise contracts.InvalidBatchExportDestinationError("Integration is required for BigQuery batch exports")+ if requested_integration.kind != Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT:+ raise contracts.InvalidBatchExportDestinationError(+ "Integration is not a Google Cloud service account integration."+ )++ if destination_type == BatchExportDestinationType.AZURE_BLOB:+ # validate the Integration is valid (this is mandatory for Azure Blob batch exports)+ if requested_integration is None:+ raise contracts.InvalidBatchExportDestinationError("Integration is required for Azure Blob batch exports")+ if requested_integration.kind != Integration.IntegrationKind.AZURE_BLOB:+ raise contracts.InvalidBatchExportDestinationError("Integration is not an Azure Blob integration.")+ # try instantiate the integration to check if it's valid+ try:+ AzureBlobIntegration(requested_integration)+ except AzureBlobIntegrationError as e:+ raise contracts.InvalidBatchExportDestinationError(str(e))++ _check_file_format(merged_config, AZURE_BLOB_SUPPORTED_COMPRESSIONS)++ if destination_type == BatchExportDestinationType.REDSHIFT:+ # Sticky integration: an export that uses one cannot drop back to inline credentials.+ # TODO: remove this guard once inline credentials are gone.+ if instance is not None and instance.destination.integration is not None and requested_integration is None:+ raise contracts.InvalidBatchExportDestinationError(+ "Cannot remove the integration from a Redshift batch export that uses one. "+ "Re-send its `integration` to keep it (or a different one to swap)."+ )++ # New Redshift exports must use an Integration for credentials. Exports created before+ # integrations existed keep their inline credentials, so only require it on create+ # (`instance is None`); existing inline-credential exports stay valid when edited.+ if requested_integration is None and instance is None:+ raise contracts.InvalidBatchExportDestinationError("Integration is required for Redshift batch exports")++ if requested_integration is not None:+ # (Team ownership is already enforced by the team-scoped `integration` field.)+ if requested_integration.kind != Integration.IntegrationKind.AWS_REDSHIFT:+ raise contracts.InvalidBatchExportDestinationError("Integration is not a Redshift integration.")++ # Only plain Redshift integrations store a host; AWS-flavor ones obtain+ # temporary credentials for a cluster endpoint that stays in the export config.+ if "host" not in requested_integration.config and not merged_config.get("host"):+ raise contracts.InvalidBatchExportDestinationError(+ "A 'host' is required when using an AWS Redshift integration: "+ "set it to the cluster or workgroup endpoint."+ )++ if merged_config.get("mode") == "COPY":+ _check_redshift_copy_inputs(team_id, instance, merged_config, existing_config)++ if destination_type in (+ BatchExportDestinationType.POSTGRES,+ BatchExportDestinationType.REDSHIFT,+ ):+ # PostgreSQL-server integrations (Postgres, plain Redshift) keep the host in the+ # linked Integration; AWS Redshift integrations and inline configs keep it in+ # `config`. Prefer the Integration's host when it has one so we don't skip+ # SSRF validation (and don't `KeyError` on a `config` that has no `host`).+ host = requested_integration.config.get("host") if requested_integration is not None else None+ if host is None:+ host = merged_config.get("host")++ if host is not None:+ try:+ validate_external_host(host)+ except ShapeError:+ raise contracts.InvalidBatchExportDestinationError(INVALID_HOST_MESSAGE)+ except ValueError:+ raise contracts.InvalidBatchExportDestinationError(UNREACHABLE_HOST_MESSAGE)+++def prepare_new_destination_config(+ destination_type: str, config: typing.Mapping[str, typing.Any]+) -> dict[str, typing.Any]:+ """Return the config a new destination saves, with the defaults a new export opts into."""+ prepared = dict(config)+ _set_default_parquet_extension(destination_type, prepared)+ return prepared+++def merge_destination_config(+ destination_type: str, stored_config: typing.Mapping[str, typing.Any], patch: typing.Mapping[str, typing.Any]+) -> dict[str, typing.Any]:+ """Return the config an export saves after a patch merges into its stored config."""+ pinned = dict(stored_config)+ _pin_existing_parquet_extension(destination_type, pinned)+ return _recursive_dict_merge(pinned, patch)+++def get_destination_test_config(team_id: int, export_id: UUID) -> dict[str, typing.Any]:+ """Return an export's stored destination config, limited to the fields its workflow inputs declare."""+ batch_export = _get_batch_export(team_id, export_id, "destination")++ # Remove any additional fields from stored configuration+ _, workflow_inputs = service.DESTINATION_WORKFLOWS[batch_export.destination.type]+ workflow_fields = {field.name for field in dataclasses.fields(workflow_inputs)}+ return {k: v for k, v in batch_export.destination.config.items() if k in workflow_fields}+++def delete_batch_export(team_id: int, export_id: UUID) -> None:+ """Delete an export: pause it, cancel its running work, then drop its Temporal schedule.++ Raises:+ contracts.BatchExportNotFoundError: If the team has no such live export.+ """+ # The activity log reads the destination when the delete saves the export.+ batch_export = _get_batch_export(team_id, export_id, "destination")+ service.delete_batch_export(batch_export)+++def _map_schedule_errors() -> typing.Iterator[None]:+ try:+ yield+ except service.BatchExportIdError as e:+ raise contracts.BatchExportNotFoundError() from e+ except service.BatchExportServiceRPCError as e:+ raise contracts.BatchExportScheduleError() from e+++def pause_batch_export(team_id: int, export_id: UUID, *, note: str) -> None:+ """Pause an export's Temporal schedule.++ Raises:+ contracts.BatchExportNotFoundError: If the team has no such export.+ contracts.BatchExportScheduleError: If Temporal rejects the change.+ """+ with _map_schedule_errors():+ service.pause_batch_export(sync_connect(), str(export_id), note=note, team_id=team_id)+++def unpause_batch_export(team_id: int, export_id: UUID, *, note: str, backfill: bool) -> None:+ """Unpause an export's Temporal schedule, and with ``backfill``, backfill the runs missed while paused.++ Raises:+ contracts.BatchExportNotFoundError: If the team has no such export.+ contracts.BatchExportScheduleError: If Temporal rejects the change.+ """+ with _map_schedule_errors():+ service.unpause_batch_export(sync_connect(), str(export_id), note=note, backfill=backfill, team_id=team_id)diff --git a/products/batch_exports/backend/presentation/views/batch_export/destinations.py b/products/batch_exports/backend/presentation/views/batch_export/destinations.pyindex 7c9539927a8f..35739e24a37c 100644--- a/products/batch_exports/backend/presentation/views/batch_export/destinations.py+++ b/products/batch_exports/backend/presentation/views/batch_export/destinations.py@@ -13,12 +13,7 @@ from posthog.models.integration import Integrationfrom products.batch_exports.backend.facade import api as batch_exports_apifrom products.batch_exports.backend.facade.contracts import InvalidDestinationConfigError-from products.batch_exports.backend.facade.enums import (- AZURE_BLOB_SUPPORTED_COMPRESSIONS,- OBJECT_STORAGE_DESTINATIONS,- S3_SUPPORTED_COMPRESSIONS,- BatchExportDestinationType,-)+from products.batch_exports.backend.facade.enums import AZURE_BLOB_SUPPORTED_COMPRESSIONS, S3_SUPPORTED_COMPRESSIONSfrom products.batch_exports.backend.models.batch_export import BatchExportDestination@@ -514,80 +509,6 @@ class BatchExportDestinationRequestField(serializers.JSONField):pass-# S3-family destinations that may authenticate via an Integration, mapped to-# the linked integration's kind. Adding a future S3-family destination (e.g. a-# first-class GCS-via-S3 type) is a one-line addition here.-S3_DESTINATION_TO_INTEGRATION_KIND: dict[str, Integration.IntegrationKind] = {- BatchExportDestinationType.AWS_S3: Integration.IntegrationKind.AWS_S3,- BatchExportDestinationType.S3_COMPATIBLE: Integration.IntegrationKind.S3_COMPATIBLE,-}---def _writes_compressed_parquet(config: dict[str, typing.Any]) -> bool:- """Whether a config produces Parquet file names that carry a compression codec."""- return config.get("file_format") == "Parquet" and config.get("compression") is not None---def _uses_legacy_parquet_extension(destination_type: str, stored_config: dict[str, typing.Any]) -> bool:- """Whether an export's stored config already writes the codec into its Parquet file names.-- Reads through `coerce_destination_config`, because `EncryptedJSONField` stringifies- scalars on write and the string "False" is truthy.- """- coerced = batch_exports_api.coerce_destination_config(destination_type, stored_config)- stored = coerced.get("legacy_parquet_extension")- if stored is None:- # export was created before the `legacy_parquet_extension` field was added- # so will use the legacy extension if it writes compressed Parquet files- return _writes_compressed_parquet(coerced)- return bool(stored)---def _set_default_parquet_extension(destination_type: str, config: dict[str, typing.Any]) -> None:- """Opt a newly created destination into the standard `.parquet` extension.-- The workflow input dataclasses default this to `True`, so that an export whose Temporal- schedule predates the field keeps the same file extension as before in order to maintain- compatibility.- """- if destination_type in OBJECT_STORAGE_DESTINATIONS:- config.setdefault("legacy_parquet_extension", False)---def _pin_existing_parquet_extension(destination_type: str, stored_config: dict[str, typing.Any]) -> None:- """Record what an export's file names already look like, before a patch can change its format.-- An export that predates this setting has no value for it, and a missing value reads as the- legacy naming. That is correct only for an export that already writes names carrying a codec,- which means Parquet with a compression codec set. An export on JSON Lines, or on Parquet with- no compression, has no such names to keep, so moving it to compressed Parquet has to produce- `.parquet` rather than `.parquet.zst`.-- Takes the config as stored, before the incoming patch merges into it, so the value reflects- what the export has been running rather than what it is moving to. An explicit value in the- patch still wins, because the merge applies afterwards.- """- if destination_type not in OBJECT_STORAGE_DESTINATIONS:- return-- stored_config.setdefault("legacy_parquet_extension", _writes_compressed_parquet(stored_config))---def _coerce_integration_id(value: typing.Any) -> int | None:- """Return the integration id encoded in a Redshift COPY credential value, if any.-- `BatchExportDestination.config` is an `EncryptedJSONField`, which stringifies scalar- leaves on the decrypt round trip, so an id may arrive as an int or a numeric string.- """- if isinstance(value, bool):- return None- if isinstance(value, int):- return value- if isinstance(value, str) and value.isdigit():- return int(value)- return None--class BatchExportDestinationSerializer(serializers.ModelSerializer):"""Serializer for an BatchExportDestination model.diff --git a/products/batch_exports/backend/presentation/views/batch_export/exports.py b/products/batch_exports/backend/presentation/views/batch_export/exports.pyindex 2ce3ce178fa6..07faa66e0d17 100644--- a/products/batch_exports/backend/presentation/views/batch_export/exports.py+++ b/products/batch_exports/backend/presentation/views/batch_export/exports.py@@ -1,5 +1,6 @@"""Batch export endpoints: create, update, pause, unpause, delete and destination tests."""+import uuidimport typingimport dataclassesfrom typing import cast@@ -24,31 +25,19 @@ from posthog.api.log_entries import LogEntryMixinfrom posthog.api.routing import TeamAndOrgViewSetMixinfrom posthog.api.utils import actionfrom posthog.models import Team, User-from posthog.models.integration import (- AzureBlobIntegration,- AzureBlobIntegrationError,- DatabricksIntegration,- DatabricksIntegrationError,- Integration,- SnowflakeIntegration,- SnowflakeIntegrationError,-)-from posthog.security.url_validation import (- INVALID_HOST_MESSAGE,- UNREACHABLE_HOST_MESSAGE,- ShapeError,- validate_external_host,-)-from posthog.temporal.common.client import sync_connect+from posthog.models.integration import Integrationfrom products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_teamfrom products.batch_exports.backend.facade import api as batch_exports_api-from products.batch_exports.backend.facade.contracts import InvalidBatchExportFilters, UnsupportedDestinationTestError+from products.batch_exports.backend.facade.contracts import (+ BatchExportNotFoundError,+ BatchExportScheduleError,+ InvalidBatchExportDestinationError,+ InvalidBatchExportFilters,+ UnsupportedDestinationTestError,+)from products.batch_exports.backend.facade.enums import (- AZURE_BLOB_SUPPORTED_COMPRESSIONS,EXPORTABLE_EVENTS_MODEL_FIELDS,- S3_FAMILY_TYPES,- S3_SUPPORTED_COMPRESSIONS,SUPPORTED_FILTER_TYPES_DISPLAY,TIMEZONES,BatchExportDestinationType,@@ -69,18 +58,7 @@ from products.batch_exports.backend.presentation.views.utils import (check_high_frequency_batch_exports_enabled,check_hogql_batch_exports_enabled,)-from products.batch_exports.backend.service import (- DESTINATION_WORKFLOWS,- BaseBatchExportInputs,- BatchExportIdError,- BatchExportSchema,- BatchExportServiceError,- BatchExportServiceRPCError,- delete_batch_export,- pause_batch_export,- sync_batch_export,- unpause_batch_export,-)+from products.batch_exports.backend.service import BatchExportSchema, sync_batch_exportfrom . import (destinations as destination_views,@@ -667,262 +645,29 @@ class BatchExportSerializer(serializers.ModelSerializer):# TODO: could this be moved inside BatchExportDestinationSerializer::validate?def validate_destination(self, destination_attrs: dict):destination_type = destination_attrs["type"]- config = destination_attrs["config"]view = self.context.get("view")- instance = None+ export_id: uuid.UUID | None = Noneif self.instance is not None:- instance = self.instance- existing_config = instance.destination.config+ export_id = self.instance.idelif view is not None and "pk" in view.kwargs:# Running validation for a `detail=True` action.- instance = view.get_object()- existing_config = instance.destination.config- else:- existing_config = {}+ export_id = uuid.UUID(str(view.kwargs["pk"]))- if instance is not None and destination_type != instance.destination.type:- raise serializers.ValidationError(- f"Cannot change destination type from '{instance.destination.type}' to '{destination_type}'. "- "Delete this batch export and create a new one with the new destination type."- )-- # This setting is used for grandfathered exports that used the legacy Parquet file extension,- # and is a one-way migration; once exports use the new `.parquet` extension it is not- # possible to go back to using the legacy extension.- if config.get("legacy_parquet_extension") is True and not destination_views._uses_legacy_parquet_extension(- destination_type, existing_config- ):- raise serializers.ValidationError(- "'legacy_parquet_extension' can only stay on for an export that already writes the "- "compression codec into its Parquet file names. It cannot be turned on for a new "- "export, or turned back on once an export moved to the standard '.parquet' extension."+ integration: Integration | None = destination_attrs.get("integration")+ try:+ batch_exports_api.validate_batch_export_destination(+ self.context["team_id"],+ export_id=export_id,+ destination_type=destination_type,+ config=destination_attrs["config"],+ integration_id=integration.id if integration is not None else None,+ integration_sent="integration" in destination_attrs,)-- merged_config = recursive_dict_merge(existing_config, config)-- # SSRF protection for HTTP batch exports- if destination_type == BatchExportDestinationType.HTTP:- url = merged_config.get("url")- if url and url not in ("https://us.i.posthog.com/batch/", "https://eu.i.posthog.com/batch/"):- raise serializers.ValidationError(f"Invalid destination URL: {url}")-- if destination_type == BatchExportDestinationType.SNOWFLAKE:- integration: Integration | None = destination_attrs.get("integration")- if integration is None and "integration" not in destination_attrs and instance is not None:- # A PATCH may send config alone, which keeps the export's existing integration.- # An explicit `integration: null` is a removal, and is rejected below.- integration = instance.destination.integration-- if integration is None:- raise serializers.ValidationError("Integration is required for Snowflake batch exports")-- # Also rejects an integration missing its account, user or authentication type, so a- # half-built one fails here rather than on the export's first run.- try:- SnowflakeIntegration(integration)- except SnowflakeIntegrationError as e:- raise serializers.ValidationError(str(e))-- if destination_type in S3_FAMILY_TYPES:- integration = destination_attrs.get("integration")- if integration is None and "integration" not in destination_attrs and instance is not None:- # A PATCH may send config alone, which keeps the export's existing integration.- # An explicit `integration: null` is a removal, and is rejected below.- integration = instance.destination.integration-- if integration is None:- raise serializers.ValidationError(f"Integration is required for {destination_type} batch exports")-- # Credentials and the provider endpoint are not declared on the input dataclasses, so- # `BatchExportDestinationSerializer.validate` already rejects them as unknown fields.-- # we already validate the required inputs in BatchExportDestinationSerializer::validate- # so here we just ensure that the inputs are not empty- required_non_empty_inputs = ["bucket_name", "region", "prefix"]-- empty_inputs = []-- for required_input in required_non_empty_inputs:- value = config.get(required_input)- if value is not None and isinstance(value, str) and value.strip() == "":- empty_inputs.append(required_input)-- if empty_inputs:- raise serializers.ValidationError(f"The following inputs are empty: {empty_inputs}")-- # The integration must match the destination kind. (Team ownership is already enforced- # by the team-scoped `integration` field, which can only resolve integrations belonging- # to the request's team.)- if integration.kind != destination_views.S3_DESTINATION_TO_INTEGRATION_KIND[destination_type]:- raise serializers.ValidationError(- f"Integration provided is not an AWS S3 integration (got kind='{integration.kind}')"- )-- # JSONLines is the default file format for S3 exports for legacy reasons- file_format = merged_config.get("file_format", "JSONLines")- supported_file_formats = S3_SUPPORTED_COMPRESSIONS.keys()- if file_format not in supported_file_formats:- raise serializers.ValidationError(- f"File format {file_format} is not supported. Supported file formats are {list(supported_file_formats)}"- )- compression = merged_config.get("compression", None)- if compression and compression not in S3_SUPPORTED_COMPRESSIONS[file_format]:- raise serializers.ValidationError(- f"Compression {compression} is not supported for file format {file_format}. Supported compressions are {S3_SUPPORTED_COMPRESSIONS[file_format]}"- )-- if destination_type == BatchExportDestinationType.DATABRICKS:- # validate the Integration is valid (this is mandatory for Databricks batch exports)- integration = destination_attrs.get("integration")- if integration is None:- raise serializers.ValidationError("Integration is required for Databricks batch exports")- if integration.kind != Integration.IntegrationKind.DATABRICKS:- raise serializers.ValidationError("Integration is not a Databricks integration.")- # try instantiate the integration to check if it's valid- try:- DatabricksIntegration(integration)- except DatabricksIntegrationError as e:- raise serializers.ValidationError(str(e))-- if destination_type == BatchExportDestinationType.POSTGRES:- integration = destination_attrs.get("integration")- # New Postgres exports must use an Integration for credentials. Exports created before- # integrations existed keep their inline credentials, so only require it on create- # (`instance is None`); existing inline-credential exports stay valid when edited.- if integration is None and instance is None:- raise serializers.ValidationError("Integration is required for Postgres batch exports")- if integration is not None and integration.kind != Integration.IntegrationKind.POSTGRESQL:- raise serializers.ValidationError("Integration is not a PostgreSQL integration.")-- if destination_type == BatchExportDestinationType.BIGQUERY:- integration = destination_attrs.get("integration")- if integration is None:- raise serializers.ValidationError("Integration is required for BigQuery batch exports")- if integration.kind != Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT:- raise serializers.ValidationError("Integration is not a Google Cloud service account integration.")-- if destination_type == BatchExportDestinationType.AZURE_BLOB:- # validate the Integration is valid (this is mandatory for Azure Blob batch exports)- integration = destination_attrs.get("integration")- if integration is None:- raise serializers.ValidationError("Integration is required for Azure Blob batch exports")- if integration.kind != Integration.IntegrationKind.AZURE_BLOB:- raise serializers.ValidationError("Integration is not an Azure Blob integration.")- # try instantiate the integration to check if it's valid- try:- AzureBlobIntegration(integration)- except AzureBlobIntegrationError as e:- raise serializers.ValidationError(str(e))-- file_format = merged_config.get("file_format", "JSONLines")- supported_file_formats = AZURE_BLOB_SUPPORTED_COMPRESSIONS.keys()- if file_format not in supported_file_formats:- raise serializers.ValidationError(- f"File format {file_format} is not supported. Supported file formats are {list(supported_file_formats)}"- )- compression = merged_config.get("compression", None)- if compression and compression not in AZURE_BLOB_SUPPORTED_COMPRESSIONS[file_format]:- raise serializers.ValidationError(- f"Compression {compression} is not supported for file format {file_format}. Supported compressions are {AZURE_BLOB_SUPPORTED_COMPRESSIONS[file_format]}"- )-- if destination_type == BatchExportDestinationType.REDSHIFT:- integration = destination_attrs.get("integration")-- # Sticky integration: an export that uses one cannot drop back to inline credentials.- # TODO: remove this guard once inline credentials are gone.- if instance is not None and instance.destination.integration is not None and integration is None:- raise serializers.ValidationError(- "Cannot remove the integration from a Redshift batch export that uses one. "- "Re-send its `integration` to keep it (or a different one to swap)."- )-- # New Redshift exports must use an Integration for credentials. Exports created before- # integrations existed keep their inline credentials, so only require it on create- # (`instance is None`); existing inline-credential exports stay valid when edited.- if integration is None and instance is None:- raise serializers.ValidationError("Integration is required for Redshift batch exports")-- if integration is not None:- # (Team ownership is already enforced by the team-scoped `integration` field.)- if integration.kind != Integration.IntegrationKind.AWS_REDSHIFT:- raise serializers.ValidationError("Integration is not a Redshift integration.")-- # Only plain Redshift integrations store a host; AWS-flavor ones obtain- # temporary credentials for a cluster endpoint that stays in the export config.- if "host" not in integration.config and not merged_config.get("host"):- raise serializers.ValidationError(- "A 'host' is required when using an AWS Redshift integration: "- "set it to the cluster or workgroup endpoint."- )-- mode = merged_config.get("mode")-- if mode == "COPY":- copy_inputs = merged_config.get("copy_inputs", {})- if not copy_inputs:- raise serializers.ValidationError("Missing required inputs for 'COPY'")-- required_inputs = {"s3_bucket", "region_name", "authorization", "bucket_credentials"}- if required_inputs - copy_inputs.keys():- raise serializers.ValidationError("Missing required input for 'COPY'")-- credential_keys = {"aws_access_key_id", "aws_secret_access_key"}-- bucket_credentials = copy_inputs["bucket_credentials"]- bucket_integration_id = destination_views._coerce_integration_id(bucket_credentials)- authorization = copy_inputs.get("authorization")- authorization_integration_id = destination_views._coerce_integration_id(authorization)-- existing_copy_inputs = existing_config.get("copy_inputs") or {}-- for field_name, copy_integration_id in (- ("bucket_credentials", bucket_integration_id),- ("authorization", authorization_integration_id),- ):- # Sticky like the top-level integration: COPY credentials that reference an- # integration cannot drop back to inline values.- # TODO: remove this guard once integrations are mandatory for Redshift and- # inline credentials are gone.- if (- instance is not None- and copy_integration_id is None- and destination_views._coerce_integration_id(existing_copy_inputs.get(field_name)) is not None- ):- raise serializers.ValidationError(- f"Cannot switch '{field_name}' from an integration to inline credentials. "- "Send a different integration ID to swap, or omit the field to keep the current one."- )- if copy_integration_id is None:- continue- # These ids live inside `config` rather than the team-scoped `integration`- # field, so team ownership must be checked here.- if not Integration.objects.filter(- id=copy_integration_id,- team_id=self.context["team_id"],- kind=Integration.IntegrationKind.AWS_S3,- ).exists():- raise serializers.ValidationError(- f"'{field_name}' does not reference an AWS S3 integration of this project."- )-- if bucket_integration_id is None and (- not isinstance(bucket_credentials, dict) or credential_keys - bucket_credentials.keys()- ):- raise serializers.ValidationError("Missing required bucket credentials for 'COPY'")-- if authorization_integration_id is None:- if isinstance(authorization, dict):- if credential_keys - authorization.keys():- raise serializers.ValidationError("Missing required credentials for 'COPY'")- elif isinstance(authorization, str):- if not authorization.strip():- raise serializers.ValidationError("Missing required IAM role for 'COPY'")- else:- raise serializers.ValidationError(- "Authorization for 'COPY' must be an IAM role ARN, AWS credentials, or an integration ID."- )+ except InvalidBatchExportDestinationError as e:+ raise serializers.ValidationError(str(e))+ except BatchExportNotFoundError:+ raise NotFound(f"BatchExport ID '{export_id}' not found.")if destination_type == BatchExportDestinationType.WORKFLOWS:team_id = self.context["team_id"]@@ -942,27 +687,6 @@ class BatchExportSerializer(serializers.ModelSerializer):):raise PermissionDenied("Backfilling Workflows is not enabled for this team.")- if destination_type in (- BatchExportDestinationType.POSTGRES,- BatchExportDestinationType.REDSHIFT,- ):- # PostgreSQL-server integrations (Postgres, plain Redshift) keep the host in the- # linked Integration; AWS Redshift integrations and inline configs keep it in- # `config`. Prefer the Integration's host when it has one so we don't skip- # SSRF validation (and don't `KeyError` on a `config` that has no `host`).- integration = destination_attrs.get("integration")- host = integration.config.get("host") if integration is not None else None- if host is None:- host = merged_config.get("host")-- if host is not None:- try:- validate_external_host(host)- except ShapeError:- raise serializers.ValidationError(INVALID_HOST_MESSAGE)- except ValueError:- raise serializers.ValidationError(UNREACHABLE_HOST_MESSAGE)-return destination_attrsdef create(self, validated_data: dict) -> BatchExport:@@ -988,7 +712,9 @@ class BatchExportSerializer(serializers.ModelSerializer):# TODO: Migrate batch exports using a HogQL query to HogQL model.validated_data["schema"] = self.serialize_hogql_query_to_batch_export_schema(hogql_query)- destination_views._set_default_parquet_extension(destination_data["type"], destination_data["config"])+ destination_data["config"] = batch_exports_api.prepare_new_destination_config(+ destination_data["type"], destination_data["config"]+ )destination = BatchExportDestination(**destination_data)user = self.context["request"].user@@ -1119,10 +845,8 @@ class BatchExportSerializer(serializers.ModelSerializer):if destination_data:# Type changes are rejected by `validate_destination` — the incoming `type`# (if any) always equals the existing type by the time we get here.- destination_views._pin_existing_parquet_extension(- batch_export.destination.type, batch_export.destination.config- )- batch_export.destination.config = recursive_dict_merge(+ batch_export.destination.config = batch_exports_api.merge_destination_config(+ batch_export.destination.type,batch_export.destination.config,destination_data.get("config", {}),)@@ -1159,29 +883,6 @@ class BatchExportSerializer(serializers.ModelSerializer):return batch_export-def recursive_dict_merge(- old: dict[str, typing.Any],- new: dict[str, typing.Any],-) -> dict[str, typing.Any]:- """Merge two dictionaries, and recursively merge any dictionaries in them."""- merged = {**old, **new}-- for key, value in merged.items():- if not isinstance(value, dict):- continue-- # Recurse only when both sides are dicts: a dict replacing a scalar (e.g. inline- # credentials replacing an IAM role or an integration id) is an overwrite, not a merge.- old_value = old.get(key)- new_value = new.get(key)- merged[key] = recursive_dict_merge(- old_value if isinstance(old_value, dict) else {},- new_value if isinstance(new_value, dict) else {},- )-- return merged--@extend_schema(tags=["batch_exports"])@extend_schema_view(# Request bodies use a polymorphic destination schema so that integration-backed types@@ -1222,16 +923,12 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelViteam_id = batch_export.team_idnote = f"Pause requested by user {user_id} from team {team_id}"- temporal = sync_connect()-try:- pause_batch_export(temporal, str(batch_export.id), note=note)- except BatchExportIdError:- raise NotFound(f"BatchExport ID '{str(batch_export.id)}' not found.")- except BatchExportServiceRPCError:+ batch_exports_api.pause_batch_export(team_id, batch_export.id, note=note)+ except BatchExportNotFoundError:+ raise NotFound(f"BatchExport ID '{batch_export.id}' not found.")+ except BatchExportScheduleError:raise ValidationError("Invalid request to pause a BatchExport could not be carried out")- except BatchExportServiceError:- raisereturn response.Response({"paused": True})@@ -1251,16 +948,13 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVinote = f"Unpause requested by user {user_id} from team {team_id}"batch_export = self.get_object()- temporal = sync_connect()try:- unpause_batch_export(temporal, str(batch_export.id), note=note, backfill=backfill)- except BatchExportIdError:- raise NotFound(f"BatchExport ID '{str(batch_export.id)}' not found.")- except BatchExportServiceRPCError:+ batch_exports_api.unpause_batch_export(batch_export.team_id, batch_export.id, note=note, backfill=backfill)+ except BatchExportNotFoundError:+ raise NotFound(f"BatchExport ID '{batch_export.id}' not found.")+ except BatchExportScheduleError:raise ValidationError("Invalid request to unpause a BatchExport could not be carried out")- except BatchExportServiceError:- raisereturn response.Response({"paused": False})@@ -1272,7 +966,10 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVisince we are deleting, we assume that we can recover from this state by finishing the delete operation by callinginstance.save()."""- delete_batch_export(instance)+ try:+ batch_exports_api.delete_batch_export(instance.id, instance.team_id)+ except BatchExportNotFoundError:+ raise NotFound(f"BatchExport ID '{instance.id}' not found.")def _parse_destination_test_step(self, request: request.Request) -> int:step_serializer = DestinationTestStepRequestSerializer(data=request.data)@@ -1322,13 +1019,10 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVitest_step = self._parse_destination_test_step(request)batch_export = self.get_object()-- # Remove any additional fields from stored configuration- _, workflow_inputs = DESTINATION_WORKFLOWS[batch_export.destination.type]- workflow_fields = {field.name for field in dataclasses.fields(BaseBatchExportInputs)} | {- field.name for field in dataclasses.fields(workflow_inputs)- }- stored_config = {k: v for k, v in batch_export.destination.config.items() if k in workflow_fields}+ try:+ stored_config = batch_exports_api.get_destination_test_config(batch_export.team_id, batch_export.id)+ except BatchExportNotFoundError:+ raise NotFound(f"BatchExport ID '{batch_export.id}' not found.")data = request.datadata["destination"]["config"] = {**stored_config, **data["destination"]["config"]}diff --git a/products/batch_exports/backend/service.py b/products/batch_exports/backend/service.pyindex 8151809a5924..9c7434d5e9ea 100644--- a/products/batch_exports/backend/service.py+++ b/products/batch_exports/backend/service.py@@ -806,19 +806,37 @@ class BatchExportServiceScheduleNotFound(BatchExportServiceRPCError):super().__init__(f"The Temporal Schedule {schedule_id} was not found (maybe it was deleted?)")-def pause_batch_export(temporal: Client, batch_export_id: str, note: str | None = None) -> bool:+def _get_batch_export_for_schedule_change(batch_export_id: str, team_id: int | None) -> BatchExport:+ """Load the export that a pause or unpause changes.++ With ``team_id``, only a live export of that team matches, so a request cannot reach another+ team's export. Internal callers (Temporal, deletes) pass no team and match any export.+ """+ # nosemgrep: idor-lookup-without-team (scoped below when the caller passes team_id)+ batch_exports = BatchExport.objects.filter(id=batch_export_id)+ if team_id is not None:+ batch_exports = batch_exports.filter(team_id=team_id, deleted=False)++ try:+ return batch_exports.get()+ except BatchExport.DoesNotExist:+ raise BatchExportIdError(batch_export_id)+++def pause_batch_export(+ temporal: Client, batch_export_id: str, note: str | None = None, *, team_id: int | None = None+) -> bool:"""Pause this BatchExport.We pass the call to the underlying Temporal Schedule.Returns:`True` if the batch export was paused, `False` if it was already paused.++ Raises:+ BatchExportIdError: If no export matches ``batch_export_id`` (and ``team_id``, when given)."""- try:- # nosemgrep: idor-lookup-without-team (internal service, team_id passed as parameter)- batch_export = BatchExport.objects.get(id=batch_export_id)- except BatchExport.DoesNotExist:- raise BatchExportIdError(batch_export_id)+ batch_export = _get_batch_export_for_schedule_change(batch_export_id, team_id)if batch_export.paused is True:return False@@ -869,6 +887,8 @@ def unpause_batch_export(batch_export_id: str,note: str | None = None,backfill: bool = False,+ *,+ team_id: int | None = None,) -> None:"""Unpause this BatchExport.@@ -880,15 +900,12 @@ def unpause_batch_export(batch_export_id: The ID of the BatchExport to unpause.note: An optional note to include in the Schedule when unpausing.backfill: If True, a backfill will be triggered since the BatchExport's last_paused_at.+ team_id: If set, only a live export of this team matches.Raises:- BatchExportIdError: If the provided batch_export_id doesn't point to an existing BatchExport.+ BatchExportIdError: If no export matches ``batch_export_id`` (and ``team_id``, when given)."""- try:- # nosemgrep: idor-lookup-without-team (internal service, team_id passed as parameter)- batch_export = BatchExport.objects.get(id=batch_export_id)- except BatchExport.DoesNotExist:- raise BatchExportIdError(batch_export_id)+ batch_export = _get_batch_export_for_schedule_change(batch_export_id, team_id)if batch_export.paused is False:returndiff --git a/products/batch_exports/backend/tests/test_facade.py b/products/batch_exports/backend/tests/test_facade.pyindex cd914a424555..8316a65438c3 100644--- a/products/batch_exports/backend/tests/test_facade.py+++ b/products/batch_exports/backend/tests/test_facade.py@@ -300,6 +300,27 @@ def test_runs_and_backfills_are_scoped_to_the_team_and_export(team, organizationapi.create_batch_export_backfill(team_id, export_id, start_at=None, end_at=None)[email protected]("scope", ["other_team", "deleted"])+def test_pause_unpause_and_delete_refuse_another_teams_or_a_deleted_export(team, organization, scope):+ export_id = _create_export(team, paused=True)+ if scope == "other_team":+ team_id = create_team(organization=organization).pk+ else:+ team_id = team.pk+ testing.update_batch_export(export_id, team_id=team.pk, deleted=True)++ # A leaked lookup would pause silently, reach Temporal to unpause, or run the delete.+ with pytest.raises(contracts.BatchExportNotFoundError):+ api.pause_batch_export(team_id, export_id, note="pause")+ with pytest.raises(contracts.BatchExportNotFoundError):+ api.unpause_batch_export(team_id, export_id, note="unpause", backfill=False)+ with pytest.raises(contracts.BatchExportNotFoundError):+ api.delete_batch_export(export_id, team_id)++ export = BatchExport.objects.get(id=export_id)+ assert (export.paused, export.deleted) == (True, scope == "deleted")++def test_deleting_team_batch_exports_continues_past_a_missing_schedule(team):first = _create_export(team, name="first")second = _create_export(team, name="second")