Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

refactor(batch-exports): serve export CRUD through the facade (#125658)

Rossee8984cb0e3c
diff --git a/.github/new-events-schema-targets.txt b/.github/new-events-schema-targets.txt
index 7f544d587b9a..23dcad17a61b 100644
--- a/.github/new-events-schema-targets.txt
+++ b/.github/new-events-schema-targets.txt
@@ -72,6 +72,7 @@ products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
products/batch_exports/backend/tests/temporal/test_batch_exports.py
products/batch_exports/backend/tests/temporal/test_filters.py
products/batch_exports/backend/tests/temporal/test_record_batch_model.py
+products/batch_exports/backend/tests/test_custom_events_schema.py
products/batch_exports/backend/tests/test_hogql_source.py
products/feature_flags/backend/test/test_request_usage.py
diff --git a/posthog/test/repo_invariants/test_pagination_contract.py b/posthog/test/repo_invariants/test_pagination_contract.py
index ec1078394677..9636da3590cb 100644
--- a/posthog/test/repo_invariants/test_pagination_contract.py
+++ b/posthog/test/repo_invariants/test_pagination_contract.py
@@ -112,6 +112,7 @@ EXISTING_CUSTOM_LIST_VIEWSETS = {
"products.autoresearch.backend.presentation.views.views.AutoresearchSuggestionViewSet",
"products.autoresearch.backend.presentation.views.views.AutoresearchTrainingRunViewSet",
"products.batch_exports.backend.presentation.views.batch_export.backfills.BatchExportBackfillViewSet",
+ "products.batch_exports.backend.presentation.views.batch_export.exports.BatchExportViewSet",
"products.batch_exports.backend.presentation.views.batch_export.runs.BatchExportRunViewSet",
"products.batch_exports.backend.presentation.views.file_download.FileDownloadBatchExportOnDemandViewSet",
# Lists at most MAX_LIBRARIES_PER_TEAM (100) libraries, so it has no pages.
diff --git a/posthog/test/repo_invariants/test_serializer_timestamp_guard.py b/posthog/test/repo_invariants/test_serializer_timestamp_guard.py
index 76ebf6fc4025..d9a603530e00 100644
--- a/posthog/test/repo_invariants/test_serializer_timestamp_guard.py
+++ b/posthog/test/repo_invariants/test_serializer_timestamp_guard.py
@@ -97,9 +97,6 @@ REVIEWED_WRITABLE: dict[str, str] = {
"products.actions.backend.api.action.ActionSerializer.pinned_at": "TODO: server-derived on pin. update() discards the supplied value and stores its own now(); only the null that unpins is the client's",
"products.alerts.backend.presentation.views.alert.AlertSerializer.snoozed_until": "TODO: server-derived. The client sends a relative date string and update() resolves it to the stored instant",
"products.annotations.backend.api.annotation.AnnotationSerializer.date_marker": "Client sets when the annotation happened, to place it on a chart",
- "products.batch_exports.backend.presentation.views.batch_export.exports.BatchExportSerializer.end_at": "Client-supplied bound, runs after it are not triggered",
- "products.batch_exports.backend.presentation.views.batch_export.exports.BatchExportSerializer.last_paused_at": "TODO: written by the pause action, make read-only",
- "products.batch_exports.backend.presentation.views.batch_export.exports.BatchExportSerializer.start_at": "Client-supplied bound, runs before it are not triggered",
"products.billing_alerts.backend.presentation.serializers.BillingAlertConfigurationSerializer.snoozed_until": "Client-set instant: a plain DateTimeField that ModelSerializer writes to the column unchanged, with no relative-duration parsing on this path",
"products.conversations.backend.api.tickets.TicketSerializer.sla_due_at": "Client or a workflow sets the SLA deadline, and null clears it",
"products.conversations.backend.api.tickets.TicketSerializer.snoozed_until": "Client-set instant: the ticket action assigns validated_data['snoozed_until'] to ticket.snoozed_until, with no relative-duration parsing on this path",
diff --git a/products/batch_exports/backend/custom_events_schema.py b/products/batch_exports/backend/custom_events_schema.py
new file mode 100644
index 000000000000..6776ede02e0a
--- /dev/null
+++ b/products/batch_exports/backend/custom_events_schema.py
@@ -0,0 +1,297 @@
+"""The legacy custom events schema: an events model export that selects its fields with a limited HogQL query.
+
+The API checks the query and stores it as a schema of ClickHouse expressions. Runs recompile the
+stored query when they read the native events table. New exports use the HogQL model in
+`hogql_source` instead. This module must stay importable by both the API layer and the Temporal
+worker, so it should not import from `products.batch_exports.backend.temporal` or any DRF code.
+"""
+
+import typing
+
+from posthog.schema import HogQLQueryModifiers, MaterializationMode, PersonsOnEventsMode
+
+from posthog.hogql import ast
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.schema.util.where_clause_extractor import contains_subquery
+from posthog.hogql.errors import ExposedHogQLError, QueryError
+from posthog.hogql.escape_sql import escape_clickhouse_identifier
+from posthog.hogql.parser import parse_select
+from posthog.hogql.printer import prepare_ast_for_printing, print_prepared_ast
+from posthog.hogql.resolver import resolve_types
+from posthog.hogql.visitor import CloningVisitor, TraversingVisitor, clone_expr
+
+from posthog.clickhouse.events_json import UNPARSEABLE_PROPERTIES_KEY
+
+from products.batch_exports.backend.facade.contracts import UnsupportedHogQLQueryError
+from products.batch_exports.backend.facade.enums import EXPORTABLE_EVENTS_MODEL_FIELDS
+
+if typing.TYPE_CHECKING:
+ from posthog.models import User
+
+ from products.batch_exports.backend.service import BatchExportField, BatchExportSchema
+
+
+class SerializedExportProperties(CloningVisitor):
+ """Rebind event fields while preserving property access restrictions."""
+
+ def __init__(self, table_alias: str, context: HogQLContext) -> None:
+ super().__init__()
+ self.table_alias = table_alias
+
+ from products.access_control.backend.facade.api import ( # noqa: PLC0415 — keeps Django access-control imports off the batch worker import path
+ split_restricted_property_names,
+ )
+
+ restrictions = context.restricted_properties or set()
+ restricted_names = split_restricted_property_names(restrictions)
+ self.event_restrictions = set(restricted_names.event)
+ self.person_restrictions = set(restricted_names.person)
+ if restrictions and context.uses_new_events_schema():
+ self.event_restrictions.add(UNPARSEABLE_PROPERTIES_KEY)
+ self.person_restrictions.add(UNPARSEABLE_PROPERTIES_KEY)
+
+ def visit_field(self, node: ast.Field) -> ast.Expr:
+ node = super().visit_field(node)
+ if node.chain[0] == self.table_alias:
+ node.chain[0] = "events"
+ index = 1 if node.chain[0] == "events" else 0
+ if len(node.chain) <= index:
+ return node
+ if node.chain[index : index + 2] == ["person", "properties"]:
+ # `poe.properties` is the events table's own copy of the person properties.
+ node.chain[index : index + 2] = ["poe", "properties"]
+ if node.chain[index : index + 2] == ["poe", "properties"]:
+ restrictions, property_chain = self.person_restrictions, node.chain[index + 2 :]
+ elif str(node.chain[index]) == "properties":
+ restrictions, property_chain = self.event_restrictions, node.chain[index + 1 :]
+ else:
+ return node
+ if restrictions:
+ property_path = ".".join(str(part) for part in property_chain)
+ if not property_path:
+ raise QueryError("Batch export queries cannot select a restricted properties object")
+ if any(
+ property_path == key or property_path.startswith(key + ".") or key.startswith(property_path + ".")
+ for key in restrictions
+ ):
+ return ast.Constant(value=None)
+ return node
+
+
+def prepare_serialized_export_query(query: ast.SelectQuery, context: HogQLContext) -> ast.SelectQuery:
+ """Resolve a HogQL query the way the export SQL reads it."""
+ assert query.select_from is not None
+ query = SerializedExportProperties(query.select_from.alias or "events", context).visit(query)
+ assert query.select_from is not None
+ # The export SQL names its source `events`, so a user alias must not reach the printed fields.
+ query.select_from.alias = None
+ # Every export template hands the fields a String `properties`, the native source included, so
+ # the legacy shape is the one form valid on both. Resolving against the events table rather than
+ # a subquery is also what lets the swapper keep the cast a typed property needs.
+ context.use_new_events_schema = False
+ return typing.cast(ast.SelectQuery, prepare_ast_for_printing(query, context=context, dialect="clickhouse"))
+
+
+def serialize_batch_export_query(query: ast.SelectQuery, context: HogQLContext) -> "BatchExportSchema":
+ """Compile a HogQL query into stable ClickHouse expressions and aliases."""
+ if context.uses_new_events_schema():
+ hogql = print_prepared_ast(query, context=context, dialect="hogql")
+ query = prepare_serialized_export_query(typing.cast(ast.SelectQuery, parse_select(hogql)), context)
+ stack = [query]
+ else:
+ print_prepared_ast(query, context=context, dialect="clickhouse")
+ context = HogQLContext(
+ team_id=context.team_id,
+ enable_select_queries=True,
+ limit_top_select=False,
+ use_new_events_schema=False,
+ )
+ stack = []
+ hogql = print_prepared_ast(query, context=context, dialect="hogql")
+ fields: list[BatchExportField] = []
+ for field in query.select:
+ if isinstance(field, ast.Alias):
+ expression = print_prepared_ast(field.expr, context=context, dialect="clickhouse", stack=stack)
+ alias = escape_clickhouse_identifier(field.alias)
+ else:
+ expression = print_prepared_ast(field, context=context, dialect="clickhouse", stack=stack)
+ # String constants get parameterized by the ClickHouse printer (e.g., 'hello' becomes
+ # %(hogql_val_0)s), which escape_clickhouse_identifier rejects. Use the raw value instead.
+ alias = escape_clickhouse_identifier(
+ field.value if isinstance(field, ast.Constant) and isinstance(field.value, str) else expression
+ )
+ fields.append({"expression": expression, "alias": alias})
+ return {"fields": fields, "values": context.values, "hogql_query": hogql}
+
+
+def recompile_stored_schema(
+ team_id: int, schema: "BatchExportSchema", hogql_query: str
+) -> tuple[list["BatchExportField"], dict[str, typing.Any]]:
+ """Recompile the stored query of a schema for the native events table, keeping the stored aliases.
+
+ Returns the fields and the query values a run sends with them.
+ """
+ context = HogQLContext(
+ team_id=team_id,
+ enable_select_queries=True,
+ limit_top_select=False,
+ modifiers=HogQLQueryModifiers(
+ materializationMode=MaterializationMode.DISABLED,
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,
+ ),
+ )
+ compiled = serialize_batch_export_query(typing.cast(ast.SelectQuery, parse_select(hogql_query)), context)
+ fields: list[BatchExportField] = [
+ {"expression": compiled_field["expression"], "alias": stored_field["alias"]}
+ for stored_field, compiled_field in zip(schema["fields"], compiled["fields"], strict=True)
+ ]
+ return fields, compiled["values"]
+
+
+def parse_events_hogql_query(
+ hogql_query: str, team_id: int, user: "User | None"
+) -> ast.SelectQuery | ast.SelectSetQuery:
+ """Parse a HogQL SelectQuery from a string query."""
+ from products.access_control.backend.facade.api import ( # noqa: PLC0415 — keeps Django access-control imports off the batch worker import path
+ get_restricted_properties_with_group_type_index_for_team,
+ )
+
+ try:
+ parsed_query = parse_select(hogql_query)
+ except Exception:
+ raise UnsupportedHogQLQueryError("Failed to parse query")
+
+ try:
+ restricted_properties = get_restricted_properties_with_group_type_index_for_team(user=user, team_id=team_id)
+ context = HogQLContext(
+ team_id=team_id,
+ user=user,
+ enable_select_queries=True,
+ restricted_properties=restricted_properties,
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS
+ ),
+ )
+ if restricted_properties:
+ # A restricted query is stored without its HogQL text, so no run can recompile it for
+ # the native source, where a materialized column does not exist.
+ context.modifiers.materializationMode = MaterializationMode.DISABLED
+ use_native_schema = context.uses_new_events_schema()
+ prepared_select_query = typing.cast(
+ ast.SelectQuery,
+ prepare_ast_for_printing(
+ parsed_query, context=context, dialect="hogql" if use_native_schema else "clickhouse"
+ ),
+ )
+ if use_native_schema:
+ resolve_types(clone_expr(parsed_query, clear_types=True), context=context, dialect="clickhouse")
+ except ExposedHogQLError as e:
+ raise UnsupportedHogQLQueryError(f"Invalid HogQL query: {e}")
+
+ return prepared_select_query
+
+
+class _DatabaseFieldFinder(TraversingVisitor):
+ """Walk an AST subtree and collect the names of the database fields it reads."""
+
+ def __init__(self, context: HogQLContext):
+ super().__init__()
+ self.context = context
+ self.names: set[str] = set()
+
+ def visit_field(self, node: ast.Field):
+ resolve = getattr(node.type, "resolve_database_field", None)
+ if resolve is not None:
+ database_field = resolve(self.context)
+ name = getattr(database_field, "name", None)
+ if name is not None:
+ self.names.add(name)
+ super().visit_field(node)
+
+
+def _check_events_hogql_query(team_id: int, user: "User | None", hogql_query: str) -> ast.SelectQuery:
+ """Validate a HogQL query being used for events batch exports.
+
+ This function checks that a query is supported by batch exports:
+ 1. UNION ALL is not supported.
+ 2. Any JOINs are not supported.
+ 3. Query must SELECT FROM events, and only from events.
+ 4. Subqueries in SELECT expressions are not supported.
+ 5. Query must select only from those fields we expose from the events table.
+ """
+ parsed = parse_events_hogql_query(hogql_query, team_id=team_id, user=user)
+
+ if isinstance(parsed, ast.SelectSetQuery):
+ raise UnsupportedHogQLQueryError("UNIONs are not supported")
+
+ parsed = typing.cast(ast.SelectQuery, parsed)
+
+ if parsed.select_from is None:
+ raise UnsupportedHogQLQueryError("Query must SELECT FROM events")
+
+ if parsed.ctes:
+ raise UnsupportedHogQLQueryError("CTEs are not supported")
+
+ if isinstance(parsed.select_from.table, (ast.SelectQuery, ast.SelectSetQuery)):
+ raise UnsupportedHogQLQueryError("Subqueries are not supported")
+
+ # Not sure how to make mypy understand this works, hence the ignore comment.
+ # And if it doesn't, it's still okay as it could mean an unsupported query.
+ # We would come back with the example to properly type this.
+ if parsed.select_from.table.chain != ["events"]: # type: ignore
+ raise UnsupportedHogQLQueryError("Query must only SELECT FROM events")
+
+ if parsed.select_from.next_join is not None:
+ raise UnsupportedHogQLQueryError("JOINs are not supported")
+
+ if any(contains_subquery(field) for field in parsed.select):
+ raise UnsupportedHogQLQueryError("Subqueries in SELECT expressions are not supported")
+
+ # Check that the query only selects from those fields we expose from the events table.
+ field_finder = _DatabaseFieldFinder(HogQLContext(team_id=team_id, enable_select_queries=True))
+ for field in parsed.select:
+ field_finder.visit(field)
+
+ unsupported = sorted(field_finder.names - EXPORTABLE_EVENTS_MODEL_FIELDS)
+ if unsupported:
+ raise UnsupportedHogQLQueryError(
+ f"Batch exports cannot read these fields: {', '.join(unsupported)}. "
+ f"Supported fields are: {', '.join(sorted(EXPORTABLE_EVENTS_MODEL_FIELDS))}."
+ )
+
+ return parsed
+
+
+def serialize_events_query_schema(
+ team_id: int, user: "User | None", hogql_query: ast.SelectQuery
+) -> "BatchExportSchema":
+ """Return a batch export schema from a HogQL query ast."""
+ from products.access_control.backend.facade.api import ( # noqa: PLC0415 — keeps Django access-control imports off the batch worker import path
+ get_restricted_properties_with_group_type_index_for_team,
+ )
+
+ restricted_properties = get_restricted_properties_with_group_type_index_for_team(user=user, team_id=team_id)
+ context = HogQLContext(
+ team_id=team_id,
+ user=user,
+ enable_select_queries=True,
+ limit_top_select=False,
+ restricted_properties=restricted_properties,
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,
+ ),
+ )
+ if context.uses_new_events_schema():
+ context.modifiers.materializationMode = MaterializationMode.DISABLED
+ try:
+ schema = serialize_batch_export_query(hogql_query, context)
+ except ExposedHogQLError:
+ raise UnsupportedHogQLQueryError("Unsupported HogQL query")
+ if context.restricted_properties:
+ schema.pop("hogql_query", None)
+ return schema
+
+
+def build_events_query_schema(team_id: int, user: "User | None", hogql_query: str) -> "BatchExportSchema":
+ """Check a legacy events model query, and return the schema an export stores for it."""
+ return serialize_events_query_schema(team_id, user, _check_events_hogql_query(team_id, user, hogql_query))
diff --git a/products/batch_exports/backend/facade/api.py b/products/batch_exports/backend/facade/api.py
index b59ac56816c1..0f2d687838da 100644
--- a/products/batch_exports/backend/facade/api.py
+++ b/products/batch_exports/backend/facade/api.py
@@ -27,14 +27,12 @@ from types import ModuleType
from typing import TYPE_CHECKING, Any
from uuid import UUID
-from django.db import transaction
from django.db.models import Count, F, Q, Sum
from django.db.models.functions import Coalesce
import structlog
from posthog.integrations import IntegrationManager
-from posthog.models import Team, User
from products.batch_exports.backend import (
destination_tests,
@@ -63,21 +61,21 @@ __all__ = [
"cancel_batch_export_backfill",
"cancel_batch_export_run",
"cancel_file_download",
- "coerce_destination_config",
"count_batch_exports_for_teams",
"count_hogql_rows",
"create_batch_export",
"create_batch_export_backfill",
+ "create_batch_export_from_input",
"create_workflows_backfill_export",
"delete_batch_export",
"delete_batch_exports_for_teams",
"describe_destination_test",
"drop_unknown_hogql_modifiers",
"find_hogql_interval_placeholders",
+ "get_batch_export",
"get_batch_export_backfill",
"get_batch_export_by_name",
"get_batch_export_run",
- "get_destination_test_config",
"get_file_download_run",
"get_file_download_url",
"get_latest_completed_run",
@@ -88,17 +86,18 @@ __all__ = [
"list_backfills_for_export",
"list_batch_export_backfills",
"list_batch_export_runs",
+ "list_batch_exports",
"list_batch_exports_using_integration",
"list_file_download_runs",
"list_latest_failed_runs",
"list_supported_intervals",
- "merge_destination_config",
+ "organization_has_custom_events_schema_exports",
"pause_batch_export",
- "prepare_new_destination_config",
"retry_batch_export_run",
"run_destination_test_step",
"start_file_download",
"unpause_batch_export",
+ "update_batch_export",
"validate_batch_export_destination",
"validate_batch_export_filters",
"validate_destination_config",
@@ -414,20 +413,6 @@ def get_latest_completed_run(export_id: UUID, team_id: int) -> contracts.BatchEx
return _to_run_summary(run) if run is not None else None
-def _schedule_and_save(batch_export: BatchExport) -> None:
- """Create the Temporal schedule of a new export, then save the export and its destination."""
- service = _service()
-
- # Schedule first, then the rows, which is the order every caller uses today. It trades one
- # failure for the other: a failed save leaves an orphaned schedule, where the reverse order
- # would leave a row pointing at no schedule. Neither is compensated yet.
- service.sync_batch_export(batch_export, created=True)
-
- with transaction.atomic():
- batch_export.destination.save()
- batch_export.save()
-
-
def create_batch_export(
team_id: int,
*,
@@ -447,7 +432,7 @@ def create_batch_export(
paused=paused,
end_at=end_at,
)
- _schedule_and_save(batch_export)
+ _import_exports().schedule_and_save(batch_export)
return _to_detail(batch_export)
@@ -488,7 +473,7 @@ def create_workflows_backfill_export(
filters=event_filters,
last_modified_by_id=last_modified_by_id,
)
- _schedule_and_save(batch_export)
+ _import_exports().schedule_and_save(batch_export)
return _to_ref(batch_export)
@@ -500,6 +485,72 @@ def delete_batch_export(export_id: UUID, team_id: int) -> None:
_import_exports().delete_batch_export(team_id, export_id)
+def get_batch_export(
+ team_id: int, export_id: UUID, *, include_latest_runs: bool = True
+) -> contracts.BatchExportConfiguration:
+ """Return a scheduled batch export of the team, with its destination and, unless left out, its latest runs.
+
+ Without ``include_latest_runs``, ``latest_runs`` is empty and no run is read. Raises
+ ``contracts.BatchExportNotFoundError`` when the team has no such export, or it is deleted.
+ """
+ return _import_exports().get_batch_export(team_id, export_id, include_latest_runs=include_latest_runs)
+
+
+def list_batch_exports(
+ team_id: int, *, offset: int, limit: int
+) -> tuple[list[contracts.BatchExportConfiguration], int]:
+ """Return one page of the team's scheduled batch exports, newest first, and how many there are in total.
+
+ Deleted exports, and the exports that back Workflows backfills, are left out.
+ """
+ return _import_exports().list_batch_exports(team_id, offset=offset, limit=limit)
+
+
+def create_batch_export_from_input(
+ team_id: int, *, user_id: int, export: contracts.NewBatchExport
+) -> contracts.BatchExportConfiguration:
+ """Create a scheduled batch export and its Temporal schedule, as ``user_id``.
+
+ ``export`` already passed the checks of ``validate_batch_export_destination`` and
+ ``validate_destination_config``. Raises ``contracts.UnsupportedHogQLQueryError`` when a
+ legacy events model query cannot be stored as a schema.
+ """
+ return _import_exports().create_batch_export(team_id, user_id, export)
+
+
+def update_batch_export(
+ team_id: int, export_id: UUID, *, user_id: int, update: contracts.BatchExportUpdate
+) -> contracts.BatchExportConfiguration:
+ """Change a scheduled batch export as ``user_id``, and sync its Temporal schedule.
+
+ Raises ``contracts.BatchExportNotFoundError`` when the team has no such export, and
+ ``contracts.UnsupportedHogQLQueryError`` when a legacy events model query cannot be stored
+ as a schema.
+ """
+ return _import_exports().update_batch_export(team_id, export_id, user_id, update)
+
+
+def organization_has_custom_events_schema_exports(organization_id: UUID) -> bool:
+ """Whether the organization is grandfathered into sending a HogQL query with the events model.
+
+ Only organizations where a team already has an events model export with a stored query schema are.
+ """
+ return _import_exports().organization_has_custom_events_schema_exports(organization_id)
+
+
+def validate_hogql_query(
+ team_id: int, *, user_id: int, hogql_query: str, hogql_modifiers: Mapping[str, object] | None
+) -> tuple[str, ...]:
+ """Check that a HogQL query can power a batch export, with the user's table and property permissions.
+
+ Returns the names of the columns the query outputs. Raises ``contracts.UnsupportedHogQLQueryError``
+ when the query cannot power a batch export.
+ """
+ return _import_exports().validate_hogql_query(
+ team_id, user_id, hogql_query, dict(hogql_modifiers) if hogql_modifiers is not None else None
+ )
+
+
def pause_batch_export(team_id: int, export_id: UUID, *, note: str) -> None:
"""Pause a batch export, so its schedule starts no new runs.
@@ -548,70 +599,29 @@ def validate_destination_config(destination_type: str, config: Mapping[str, Any]
return _import_destination_config().validate_destination_config(destination_type, config, partial=partial)
-def coerce_destination_config(destination_type: str, config: dict[str, Any]) -> dict[str, Any]:
- """Return a stored destination config with its values restored to their declared types.
-
- The stored config reads back with stringified scalars, so ``"False"`` becomes ``False``.
- """
- return _service().coerce_config_to_declared_types(destination_type, config)
-
-
def validate_batch_export_destination(
team_id: int,
*,
export_id: UUID | None,
destination_type: str,
config: Mapping[str, Any],
- integration_id: int | None,
- integration_sent: bool,
+ integration: contracts.IntegrationChange | None,
) -> None:
"""Check that a destination can be saved, for a new export or as a change to ``export_id``.
- ``config`` already passed ``validate_destination_config``. ``integration_sent`` is false when
- the request left the integration out, which keeps the existing integration of the export.
- Raises ``contracts.InvalidBatchExportDestinationError`` with the failed check as the message.
+ ``config`` already passed ``validate_destination_config``. ``integration`` is None
+ when the request left the integration out, which keeps the existing integration of the export. Raises
+ ``contracts.InvalidBatchExportDestinationError`` with the failed check as the message.
"""
_import_exports().validate_destination(
team_id,
export_id=export_id,
destination_type=destination_type,
config=config,
- integration_id=integration_id,
- integration_sent=integration_sent,
+ integration_change=integration,
)
-def prepare_new_destination_config(destination_type: str, config: Mapping[str, Any]) -> dict[str, Any]:
- """Return the config a new destination saves, with the defaults that a new export opts into.
-
- Temporary: only the export serializer calls this, because it still saves the destination
- through the ORM. Do not add callers. The function goes away when export creation moves
- behind the facade.
- """
- return _import_exports().prepare_new_destination_config(destination_type, config)
-
-
-def merge_destination_config(
- destination_type: str, stored_config: Mapping[str, Any], patch: Mapping[str, Any]
-) -> dict[str, Any]:
- """Return the config an export saves after a patch, merged key by key into its stored config.
-
- Temporary: only the export serializer calls this, because it still saves the destination
- through the ORM. Do not add callers. The function goes away when export updates move
- behind the facade.
- """
- return _import_exports().merge_destination_config(destination_type, stored_config, patch)
-
-
-def get_destination_test_config(team_id: int, export_id: UUID) -> dict[str, Any]:
- """Return the stored destination config of an export, for a connection test of that export.
-
- Only the fields that the workflow inputs of the destination declare are kept. The config holds
- the credentials of the destination, so it must not leave the product.
- """
- return _import_exports().get_destination_test_config(team_id, export_id)
-
-
def describe_destination_test(destination_type: str) -> contracts.DestinationTest:
"""Return the steps of a destination's connection test, with no step run."""
destination_test = destination_tests.get_destination_test(destination=destination_type)
@@ -625,16 +635,28 @@ def run_destination_test_step(
config: Mapping[str, object],
integration_id: int | None,
step: int,
+ export_id: UUID | None = None,
) -> contracts.DestinationTestStep:
"""Run one step of a destination's connection test, and return that step with its result.
- ``config`` is a destination config that already passed validation. The step reaches the
- destination over the network. When the destination authenticates through an integration,
- the integration's config and secrets override any key of the same name in ``config``.
+ ``config`` is a destination config that already passed validation. With ``export_id``, the
+ test runs against that export: ``config`` may be partial, and the export's stored config
+ fills the fields it leaves out. The step reaches the destination over the network. When the
+ destination authenticates through an integration, the integration's config and secrets
+ override any key of the same name in ``config``. Raises ``contracts.BatchExportNotFoundError``
+ when the team has no such live export, and ``contracts.InvalidDestinationConfigError`` when
+ the merged config is not complete.
"""
destination_test = destination_tests.get_destination_test(destination=destination_type)
test_configuration = dict(config)
+ if export_id is not None:
+ # The stored config holds the destination's credentials, so it is merged here and never
+ # leaves the product.
+ # TODO: once inline credentials are gone, a request can send the full config. Then remove
+ # this merge, the `export_id` parameter and `get_destination_test_config`.
+ stored_config = _import_exports().get_destination_test_config(team_id, export_id)
+ test_configuration = validate_destination_config(destination_type, {**stored_config, **config}, partial=False)
if integration_id is not None:
integration = IntegrationManager.kinds(*BATCH_EXPORT_INTEGRATION_KINDS).get(team_id, integration_id)
test_configuration = {
@@ -670,22 +692,6 @@ def find_hogql_interval_placeholders(hogql_query: str) -> frozenset[str]:
)
-def validate_hogql_query(
- team_id: int, *, user_id: int, hogql_query: str, hogql_modifiers: Mapping[str, object] | None
-) -> None:
- """Check that a HogQL query can power a batch export, with the user's table and property permissions.
-
- Raises ``contracts.UnsupportedHogQLQueryError`` when it cannot.
- """
- hogql_source = _import_hogql_source()
- hogql_source.validate_hogql_query_for_batch_export(
- hogql_query,
- Team.objects.get(id=team_id),
- user=User.objects.get(id=user_id),
- modifiers=hogql_source.load_hogql_modifiers(dict(hogql_modifiers) if hogql_modifiers is not None else None),
- )
-
-
def count_hogql_rows(
team_id: int,
*,
diff --git a/products/batch_exports/backend/facade/contracts.py b/products/batch_exports/backend/facade/contracts.py
index cb35fd37712a..dbda75ba208c 100644
--- a/products/batch_exports/backend/facade/contracts.py
+++ b/products/batch_exports/backend/facade/contracts.py
@@ -21,6 +21,7 @@ from dataclasses import (
dataclass as stdlib_dataclass,
field,
)
+from typing import Any
from uuid import UUID
from pydantic.dataclasses import dataclass
@@ -369,6 +370,134 @@ class BatchExportBackfillStart:
destination_type: str
+@dataclass(frozen=True)
+class BatchExportDestinationConfiguration:
+ """The destination of a scheduled batch export, as the API shows it.
+
+ ``config`` has its secret fields removed and its values restored to the types the
+ destination's workflow inputs declare.
+ """
+
+ type: str
+ config: dict[str, Any]
+ integration: int | None
+
+
+@dataclass(frozen=True)
+class BatchExportConfiguration:
+ """A scheduled batch export with every field the API shows.
+
+ ``offset_day`` and ``offset_hour`` are the stored interval offset in the units the API takes.
+ """
+
+ id: UUID
+ team_id: int
+ name: str
+ model: str | None
+ destination: BatchExportDestinationConfiguration
+ interval: str
+ paused: bool
+ created_at: dt.datetime
+ last_updated_at: dt.datetime
+ last_paused_at: dt.datetime | None
+ start_at: dt.datetime | None
+ end_at: dt.datetime | None
+ latest_runs: tuple[BatchExportRunDetail, ...]
+ hogql_query: str | None
+ hogql_modifiers: dict[str, Any] | None
+ primary_key: tuple[str, ...] | None
+ version_key: tuple[str, ...] | None
+ incremental_mode: str | None
+ schema: dict[str, Any] | None
+ filters: Any
+ timezone: str
+ offset_day: int | None
+ offset_hour: int | None
+
+
+@dataclass(frozen=True)
+class BatchExportDestinationInput:
+ """The destination a request saves for a scheduled batch export.
+
+ ``config`` can hold inline credentials, so it stays out of ``repr``.
+ """
+
+ type: str
+ config: dict[str, Any] = field(repr=False)
+ integration_id: int | None = None
+
+
+@dataclass(frozen=True)
+class NewBatchExport:
+ """A scheduled batch export to create. The defaults match the defaults of a saved export.
+
+ ``hogql_query`` is the query of a ``hogql`` model export. For the ``events`` model it is a
+ legacy query that selects the exported fields, which only some teams may still send.
+ """
+
+ name: str
+ model: str
+ interval: str
+ destination: BatchExportDestinationInput
+ incremental_mode: str
+ paused: bool = False
+ last_paused_at: dt.datetime | None = None
+ start_at: dt.datetime | None = None
+ end_at: dt.datetime | None = None
+ filters: Any = None
+ timezone: str = "UTC"
+ interval_offset: int | None = None
+ hogql_query: str | None = None
+ hogql_modifiers: dict[str, Any] | None = None
+ primary_key: tuple[str, ...] | None = None
+ version_key: tuple[str, ...] | None = None
+
+
+@dataclass(frozen=True)
+class IntegrationChange:
+ """A request to set a destination's integration. ``integration_id`` is None to remove it."""
+
+ integration_id: int | None
+
+
+@dataclass(frozen=True)
+class BatchExportDestinationPatch:
+ """A change to the destination of a scheduled batch export.
+
+ ``config`` merges key by key into the stored config. ``integration`` is None when the
+ request left the integration out, which keeps the stored integration.
+ """
+
+ config: dict[str, Any] = field(repr=False)
+ integration: IntegrationChange | None = None
+
+
+@dataclass(frozen=True)
+class BatchExportUpdate:
+ """A change to a scheduled batch export.
+
+ Only the fields named in ``changed`` apply, so a field can be set to None. A field that
+ ``changed`` does not name keeps its stored value, whatever value it has here.
+ """
+
+ changed: frozenset[str]
+ name: str | None = None
+ interval: str | None = None
+ paused: bool | None = None
+ last_paused_at: dt.datetime | None = None
+ start_at: dt.datetime | None = None
+ end_at: dt.datetime | None = None
+ filters: Any = None
+ timezone: str | None = None
+ interval_offset: int | None = None
+ incremental_mode: str | None = None
+ destination: BatchExportDestinationPatch | None = None
+ hogql_query: str | None = None
+ hogql_modifiers: dict[str, Any] | None = None
+ primary_key: tuple[str, ...] | None = None
+ version_key: tuple[str, ...] | None = None
+
+
@stdlib_dataclass(frozen=True)
class AWSCredentials:
"""An AWS key pair, optionally temporary.
diff --git a/products/batch_exports/backend/hogql_source.py b/products/batch_exports/backend/hogql_source.py
index 1cbe0b968d29..881f7f4409a8 100644
--- a/products/batch_exports/backend/hogql_source.py
+++ b/products/batch_exports/backend/hogql_source.py
@@ -15,22 +15,17 @@ from posthog.schema import HogQLQueryModifiers
from posthog.hogql import ast
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import Database
-from posthog.hogql.errors import ExposedHogQLError, QueryError
-from posthog.hogql.escape_sql import escape_clickhouse_identifier
+from posthog.hogql.errors import ExposedHogQLError
from posthog.hogql.modifiers import create_default_modifiers_for_team
from posthog.hogql.parser import parse_select
from posthog.hogql.placeholders import find_placeholders, replace_placeholders
from posthog.hogql.printer import prepare_ast_for_printing, print_prepared_ast
-from posthog.hogql.visitor import CloningVisitor
-
-from posthog.clickhouse.events_json import UNPARSEABLE_PROPERTIES_KEY
from products.batch_exports.backend.facade.contracts import UnsupportedHogQLQueryError
if typing.TYPE_CHECKING:
from posthog.models import Team, User
- from products.batch_exports.backend.service import BatchExportField, BatchExportSchema
DATA_INTERVAL_START_PLACEHOLDER = "data_interval_start"
DATA_INTERVAL_END_PLACEHOLDER = "data_interval_end"
@@ -268,96 +263,3 @@ def load_hogql_modifiers(stored: dict[str, typing.Any] | None) -> HogQLQueryModi
return HogQLQueryModifiers.model_validate(known)
except ValidationError as e:
raise UnsupportedHogQLQueryError(f"Invalid HogQL modifiers: {e}") from e
-
-
-class SerializedExportProperties(CloningVisitor):
- """Rebind event fields while preserving property access restrictions."""
-
- def __init__(self, table_alias: str, context: HogQLContext) -> None:
- super().__init__()
- self.table_alias = table_alias
-
- from products.access_control.backend.facade.api import ( # noqa: PLC0415 — keeps Django access-control imports off the batch worker import path
- split_restricted_property_names,
- )
-
- restrictions = context.restricted_properties or set()
- restricted_names = split_restricted_property_names(restrictions)
- self.event_restrictions = set(restricted_names.event)
- self.person_restrictions = set(restricted_names.person)
- if restrictions and context.uses_new_events_schema():
- self.event_restrictions.add(UNPARSEABLE_PROPERTIES_KEY)
- self.person_restrictions.add(UNPARSEABLE_PROPERTIES_KEY)
-
- def visit_field(self, node: ast.Field) -> ast.Expr:
- node = super().visit_field(node)
- if node.chain[0] == self.table_alias:
- node.chain[0] = "events"
- index = 1 if node.chain[0] == "events" else 0
- if len(node.chain) <= index:
- return node
- if node.chain[index : index + 2] == ["person", "properties"]:
- # `poe.properties` is the events table's own copy of the person properties.
- node.chain[index : index + 2] = ["poe", "properties"]
- if node.chain[index : index + 2] == ["poe", "properties"]:
- restrictions, property_chain = self.person_restrictions, node.chain[index + 2 :]
- elif str(node.chain[index]) == "properties":
- restrictions, property_chain = self.event_restrictions, node.chain[index + 1 :]
- else:
- return node
- if restrictions:
- property_path = ".".join(str(part) for part in property_chain)
- if not property_path:
- raise QueryError("Batch export queries cannot select a restricted properties object")
- if any(
- property_path == key or property_path.startswith(key + ".") or key.startswith(property_path + ".")
- for key in restrictions
- ):
- return ast.Constant(value=None)
- return node
-
-
-def prepare_serialized_export_query(query: ast.SelectQuery, context: HogQLContext) -> ast.SelectQuery:
- """Resolve a HogQL query the way the export SQL reads it."""
- assert query.select_from is not None
- query = SerializedExportProperties(query.select_from.alias or "events", context).visit(query)
- assert query.select_from is not None
- # The export SQL names its source `events`, so a user alias must not reach the printed fields.
- query.select_from.alias = None
- # Every export template hands the fields a String `properties`, the native source included, so
- # the legacy shape is the one form valid on both. Resolving against the events table rather than
- # a subquery is also what lets the swapper keep the cast a typed property needs.
- context.use_new_events_schema = False
- return typing.cast(ast.SelectQuery, prepare_ast_for_printing(query, context=context, dialect="clickhouse"))
-
-
-def serialize_batch_export_query(query: ast.SelectQuery, context: HogQLContext) -> "BatchExportSchema":
- """Compile a HogQL query into stable ClickHouse expressions and aliases."""
- if context.uses_new_events_schema():
- hogql = print_prepared_ast(query, context=context, dialect="hogql")
- query = prepare_serialized_export_query(typing.cast(ast.SelectQuery, parse_select(hogql)), context)
- stack = [query]
- else:
- print_prepared_ast(query, context=context, dialect="clickhouse")
- context = HogQLContext(
- team_id=context.team_id,
- enable_select_queries=True,
- limit_top_select=False,
- use_new_events_schema=False,
- )
- stack = []
- hogql = print_prepared_ast(query, context=context, dialect="hogql")
- fields: list[BatchExportField] = []
- for field in query.select:
- if isinstance(field, ast.Alias):
- expression = print_prepared_ast(field.expr, context=context, dialect="clickhouse", stack=stack)
- alias = escape_clickhouse_identifier(field.alias)
- else:
- expression = print_prepared_ast(field, context=context, dialect="clickhouse", stack=stack)
- # String constants get parameterized by the ClickHouse printer (e.g., 'hello' becomes
- # %(hogql_val_0)s), which escape_clickhouse_identifier rejects. Use the raw value instead.
- alias = escape_clickhouse_identifier(
- field.value if isinstance(field, ast.Constant) and isinstance(field.value, str) else expression
- )
- fields.append({"expression": expression, "alias": alias})
- return {"fields": fields, "values": context.values, "hogql_query": hogql}
diff --git a/products/batch_exports/backend/logic/backfills.py b/products/batch_exports/backend/logic/backfills.py
index b8039c9696f2..9af8902a4b97 100644
--- a/products/batch_exports/backend/logic/backfills.py
+++ b/products/batch_exports/backend/logic/backfills.py
@@ -25,6 +25,7 @@ from products.batch_exports.backend.hogql_source import (
find_interval_placeholders,
parse_hogql_select_for_batch_export,
)
+from products.batch_exports.backend.logic.exports import load_batch_export
from products.batch_exports.backend.logic.pagination import select_page
from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportBackfill
from products.batch_exports.backend.service import (
@@ -222,10 +223,7 @@ def create_backfill(
start_at_input: ISO formatted datetime string for backfill start
end_at_input: ISO formatted datetime string for backfill end
"""
- try:
- batch_export = BatchExport.objects.select_related("destination").get(id=export_id, team_id=team_id)
- except BatchExport.DoesNotExist as e:
- raise contracts.BatchExportNotFoundError() from e
+ batch_export = load_batch_export(team_id, export_id, "destination")
if start_at_input is None:
_check_earliest_backfill_allowed(team_id, batch_export)
diff --git a/products/batch_exports/backend/logic/exports.py b/products/batch_exports/backend/logic/exports.py
index 7af55e4a36ee..b111b857618c 100644
--- a/products/batch_exports/backend/logic/exports.py
+++ b/products/batch_exports/backend/logic/exports.py
@@ -1,7 +1,7 @@
-"""Scheduled batch exports: check a destination before it is saved, pause or unpause an export.
+"""Scheduled batch exports: read, create and change them, check what they save, pause or unpause them.
-The facade reaches this module through a call-time import, because it imports the service layer
-and the Temporal client, neither of which may load on the ``django.setup()`` path.
+The facade reaches this module through a call-time import, because it imports the service layer,
+the Temporal client and the HogQL parser, none of which may load on the ``django.setup()`` path.
"""
import typing
@@ -9,7 +9,12 @@ import contextlib
import dataclasses
from uuid import UUID
+from django.db import transaction
+
+from posthog.hogql import ast
+
from posthog.integrations import IntegrationKind, IntegrationManager, IntegrationNotFound, IntegrationRecord
+from posthog.models import Team, User
from posthog.models.integration import (
AzureBlobIntegration,
AzureBlobIntegrationError,
@@ -27,6 +32,7 @@ from posthog.security.url_validation import (
from posthog.temporal.common.client import sync_connect
from products.batch_exports.backend import service
+from products.batch_exports.backend.custom_events_schema import build_events_query_schema
from products.batch_exports.backend.facade import contracts
from products.batch_exports.backend.facade.enums import (
AZURE_BLOB_SUPPORTED_COMPRESSIONS,
@@ -35,8 +41,11 @@ from products.batch_exports.backend.facade.enums import (
S3_FAMILY_TYPES,
S3_SUPPORTED_COMPRESSIONS,
BatchExportDestinationType,
+ BatchExportModel,
)
-from products.batch_exports.backend.models.batch_export import BatchExport
+from products.batch_exports.backend.hogql_source import load_hogql_modifiers, validate_hogql_query_for_batch_export
+from products.batch_exports.backend.logic.runs import to_run_detail
+from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportDestination, BatchExportSource
# S3-family destinations that may authenticate via an Integration, mapped to
# the linked integration's kind. Adding a future S3-family destination (e.g. a
@@ -50,7 +59,7 @@ _S3_DESTINATION_TO_INTEGRATION_KIND: dict[str, IntegrationKind] = {
_ALLOWED_HTTP_DESTINATION_URLS = ("https://us.i.posthog.com/batch/", "https://eu.i.posthog.com/batch/")
-def _get_batch_export(team_id: int, export_id: UUID, *related: str) -> BatchExport:
+def load_batch_export(team_id: int, export_id: UUID, *related: str) -> BatchExport:
"""Load a live export of the team, joining only the ``related`` fields the caller reads."""
batch_exports = BatchExport.objects.exclude(deleted=True)
# `select_related()` with no fields joins every non-null foreign key, so only call it with some.
@@ -249,21 +258,23 @@ def validate_destination(
export_id: UUID | None,
destination_type: str,
config: typing.Mapping[str, typing.Any],
- integration_id: int | None,
- integration_sent: bool,
+ integration_change: contracts.IntegrationChange | None,
) -> None:
"""Check that a destination can be saved, for a new export or as a change to ``export_id``.
- ``integration_sent`` is false when the request left the integration out, which keeps the
- export's existing integration. An explicit null sends ``integration_id=None``.
+ ``integration_change`` is None when the request left the integration out, which keeps the
+ export's existing integration. An explicit null sends ``IntegrationChange(integration_id=None)``.
Raises:
contracts.InvalidBatchExportDestinationError: If the destination fails a check. The
message says which one.
contracts.BatchExportNotFoundError: If ``export_id`` names no export of the team.
"""
+ # The caller's copy of the export has its secrets stripped, and the Redshift COPY check reads the
+ # stored inline credentials, so the export is read again here.
+ # TODO: once inline credentials are gone, take the existing destination from the caller instead.
instance = (
- _get_batch_export(team_id, export_id, "destination", "destination__integration")
+ load_batch_export(team_id, export_id, "destination", "destination__integration")
if export_id is not None
else None
)
@@ -289,19 +300,26 @@ def validate_destination(
merged_config = _recursive_dict_merge(existing_config, config)
- # The integration the request sent. None when it sent none, or sent an explicit null.
+ # `requested_integration` is the integration the request sent: None when it sent none, or sent
+ # an explicit null. `kept_integration` is the one the export has after the change: a PATCH may
+ # send config alone, which keeps the export's existing integration. An explicit
+ # `integration: null` is a removal, and is rejected below.
requested_integration: IntegrationRecord | None
- if instance is not None and integration_id is not None and instance.destination.integration_id == integration_id:
- # A PATCH usually sends the export's current integration again, which is already loaded.
- requested_integration = instance.destination.integration
+ kept_integration: IntegrationRecord | None
+ if integration_change is None:
+ requested_integration = None
+ kept_integration = instance.destination.integration if instance is not None else None
else:
- requested_integration = _get_integration(team_id, integration_id)
-
- # A PATCH may send config alone, which keeps the export's existing integration.
- # An explicit `integration: null` is a removal, and is rejected below.
- kept_integration = requested_integration
- if requested_integration is None and not integration_sent and instance is not None:
- kept_integration = instance.destination.integration
+ if (
+ instance is not None
+ and integration_change.integration_id is not None
+ and instance.destination.integration_id == integration_change.integration_id
+ ):
+ # A PATCH usually sends the export's current integration again, which is already loaded.
+ requested_integration = instance.destination.integration
+ else:
+ requested_integration = _get_integration(team_id, integration_change.integration_id)
+ kept_integration = requested_integration
# SSRF protection for HTTP batch exports
if destination_type == BatchExportDestinationType.HTTP:
@@ -469,8 +487,12 @@ def merge_destination_config(
def get_destination_test_config(team_id: int, export_id: UUID) -> dict[str, typing.Any]:
- """Return an export's stored destination config, limited to the fields its workflow inputs declare."""
- batch_export = _get_batch_export(team_id, export_id, "destination")
+ """Return an export's stored destination config, limited to the fields its workflow inputs declare.
+
+ Raises:
+ contracts.BatchExportNotFoundError: If the team has no such live export.
+ """
+ batch_export = load_batch_export(team_id, export_id, "destination")
# Remove any additional fields from stored configuration
_, workflow_inputs = service.DESTINATION_WORKFLOWS[batch_export.destination.type]
@@ -485,7 +507,7 @@ def delete_batch_export(team_id: int, export_id: UUID) -> None:
contracts.BatchExportNotFoundError: If the team has no such live export.
"""
# The activity log reads the destination when the delete saves the export.
- batch_export = _get_batch_export(team_id, export_id, "destination")
+ batch_export = load_batch_export(team_id, export_id, "destination")
service.delete_batch_export(batch_export)
@@ -519,3 +541,239 @@ def unpause_batch_export(team_id: int, export_id: UUID, *, note: str, backfill:
"""
with _map_schedule_errors():
service.unpause_batch_export(sync_connect(), str(export_id), note=note, backfill=backfill, team_id=team_id)
+
+
+def _get_user(user_id: int) -> User:
+ return User.objects.get(id=user_id)
+
+
+def validate_hogql_query(
+ team_id: int, user_id: int, hogql_query: str, hogql_modifiers: dict[str, typing.Any] | None
+) -> tuple[str, ...]:
+ """Check that a HogQL query can power a batch export, with the user's permissions, and return its output columns.
+
+ Raises:
+ contracts.UnsupportedHogQLQueryError: If the query cannot power a batch export.
+ """
+ parsed = validate_hogql_query_for_batch_export(
+ hogql_query,
+ Team.objects.get(id=team_id),
+ user=_get_user(user_id),
+ modifiers=load_hogql_modifiers(hogql_modifiers),
+ )
+ # NOTE: Should be populated after a successful preparation
+ assert isinstance(parsed.type, (ast.SelectQueryType, ast.SelectSetQueryType))
+ return tuple(parsed.type.columns)
+
+
+def organization_has_custom_events_schema_exports(organization_id: UUID) -> bool:
+ """Whether a team of the organization has an events model export with a stored query schema.
+
+ Only those organizations may still send a HogQL query with the events model.
+ """
+ return BatchExport.objects.filter(schema__isnull=False, team__organization_id=organization_id).exists()
+
+
+# TODO: remove this and `BatchExportDestination.secret_fields` once inline credentials are gone.
+def _remove_secret_fields(destination_type: str, config: dict[str, typing.Any]) -> dict[str, typing.Any]:
+ target = {}
+
+ for k, v in config.items():
+ if k in BatchExportDestination.secret_fields[destination_type]:
+ continue
+ elif isinstance(v, dict):
+ target[k] = _remove_secret_fields(destination_type, v)
+ else:
+ target[k] = v
+
+ return target
+
+
+def _to_configuration(batch_export: BatchExport, *, include_latest_runs: bool) -> contracts.BatchExportConfiguration:
+ destination = batch_export.destination
+ return contracts.BatchExportConfiguration(
+ id=batch_export.id,
+ team_id=batch_export.team_id,
+ name=batch_export.name,
+ model=batch_export.model,
+ destination=contracts.BatchExportDestinationConfiguration(
+ type=destination.type,
+ config=service.coerce_config_to_declared_types(
+ destination.type, _remove_secret_fields(destination.type, destination.config)
+ ),
+ integration=destination.integration_id,
+ ),
+ interval=batch_export.interval,
+ paused=batch_export.paused,
+ created_at=batch_export.created_at,
+ last_updated_at=batch_export.last_updated_at,
+ last_paused_at=batch_export.last_paused_at,
+ start_at=batch_export.start_at,
+ end_at=batch_export.end_at,
+ latest_runs=tuple(to_run_detail(run) for run in batch_export.latest_runs) if include_latest_runs else (),
+ hogql_query=batch_export.hogql_query,
+ hogql_modifiers=batch_export.hogql_modifiers,
+ primary_key=tuple(batch_export.primary_key) if batch_export.primary_key is not None else None,
+ version_key=tuple(batch_export.version_key) if batch_export.version_key is not None else None,
+ incremental_mode=batch_export.incremental_mode,
+ schema=batch_export.schema,
+ filters=batch_export.filters,
+ timezone=batch_export.timezone,
+ offset_day=batch_export.offset_day,
+ offset_hour=batch_export.offset_hour,
+ )
+
+
+def get_batch_export(team_id: int, export_id: UUID, *, include_latest_runs: bool) -> contracts.BatchExportConfiguration:
+ batch_export = load_batch_export(team_id, export_id, "destination", "source")
+ return _to_configuration(batch_export, include_latest_runs=include_latest_runs)
+
+
+def list_batch_exports(
+ team_id: int, *, offset: int, limit: int
+) -> tuple[list[contracts.BatchExportConfiguration], int]:
+ batch_exports = (
+ BatchExport.objects.exclude(deleted=True)
+ .filter(team_id=team_id)
+ # Workflows exports are backfills that the Workflows product starts, so the list leaves them out.
+ .exclude(destination__type=BatchExportDestinationType.WORKFLOWS)
+ .order_by("-created_at", "-id")
+ .select_related("destination", "source")
+ )
+ page = batch_exports[offset : offset + limit]
+ return [_to_configuration(batch_export, include_latest_runs=True) for batch_export in page], batch_exports.count()
+
+
+def schedule_and_save(batch_export: BatchExport) -> None:
+ """Create the Temporal schedule of a new export, then save the export, its destination and its source."""
+ # Schedule first, then the rows. This trades one failure for the other: a failed save leaves
+ # an orphaned schedule, where the reverse order would leave a row pointing at no schedule.
+ # Neither failure is compensated.
+ service.sync_batch_export(batch_export, created=True)
+
+ with transaction.atomic():
+ batch_export.destination.save()
+ if batch_export.source is not None:
+ batch_export.source.save()
+ batch_export.save()
+
+
+def create_batch_export(
+ team_id: int, user_id: int, new: contracts.NewBatchExport
+) -> contracts.BatchExportConfiguration:
+ """Create a scheduled batch export, then its Temporal schedule, then save it."""
+ user = _get_user(user_id)
+
+ source = None
+ schema = None
+ if new.model == BatchExportModel.HOGQL:
+ source = BatchExportSource(
+ team_id=team_id,
+ hogql_query=new.hogql_query,
+ hogql_modifiers=new.hogql_modifiers,
+ primary_key=list(new.primary_key) if new.primary_key is not None else None,
+ version_key=list(new.version_key) if new.version_key is not None else None,
+ )
+ elif new.hogql_query is not None:
+ # TODO: Migrate batch exports using a HogQL query to HogQL model.
+ schema = build_events_query_schema(team_id, user, new.hogql_query)
+
+ destination = BatchExportDestination(
+ type=new.destination.type,
+ config=prepare_new_destination_config(new.destination.type, new.destination.config),
+ integration_id=new.destination.integration_id,
+ )
+
+ batch_export = BatchExport(
+ team_id=team_id,
+ destination=destination,
+ source=source,
+ last_modified_by=user,
+ name=new.name,
+ model=new.model,
+ interval=new.interval,
+ paused=new.paused,
+ last_paused_at=new.last_paused_at,
+ start_at=new.start_at,
+ end_at=new.end_at,
+ filters=new.filters,
+ timezone=new.timezone,
+ interval_offset=new.interval_offset,
+ incremental_mode=new.incremental_mode,
+ schema=schema,
+ )
+
+ schedule_and_save(batch_export)
+
+ # A new export has no runs yet.
+ return _to_configuration(batch_export, include_latest_runs=False)
+
+
+# The fields of `BatchExportUpdate` that set the model field of the same name.
+_UPDATABLE_FIELDS = (
+ "name",
+ "interval",
+ "paused",
+ "last_paused_at",
+ "start_at",
+ "end_at",
+ "filters",
+ "timezone",
+ "interval_offset",
+ "incremental_mode",
+)
+_SOURCE_FIELDS = ("hogql_query", "hogql_modifiers", "primary_key", "version_key")
+
+
+def update_batch_export(
+ team_id: int, export_id: UUID, user_id: int, update: contracts.BatchExportUpdate
+) -> contracts.BatchExportConfiguration:
+ """Change a scheduled batch export, and sync its Temporal schedule in the same transaction."""
+ user = _get_user(user_id)
+ batch_export = load_batch_export(team_id, export_id, "destination", "source")
+
+ events_query_changed = batch_export.model != BatchExportModel.HOGQL and "hogql_query" in update.changed
+ schema = (
+ build_events_query_schema(team_id, user, update.hogql_query)
+ if events_query_changed and update.hogql_query is not None
+ else None
+ )
+
+ with transaction.atomic():
+ if update.destination is not None:
+ destination = batch_export.destination
+ # Type changes are rejected by `validate_destination`, so the incoming `type`
+ # (if any) always equals the existing type by the time we get here.
+ destination.config = merge_destination_config(
+ destination.type, destination.config, update.destination.config
+ )
+ if update.destination.integration is not None:
+ destination.integration_id = update.destination.integration.integration_id
+
+ if batch_export.model == BatchExportModel.HOGQL:
+ if any(name in update.changed for name in _SOURCE_FIELDS):
+ source = batch_export.source or BatchExportSource(team_id=batch_export.team_id)
+
+ for name in _SOURCE_FIELDS:
+ if name in update.changed:
+ value = getattr(update, name)
+ # The model stores the key columns as lists, the contract as tuples.
+ setattr(source, name, list(value) if isinstance(value, tuple) else value)
+
+ source.save()
+ batch_export.source = source
+
+ elif events_query_changed:
+ batch_export.schema = schema
+
+ batch_export.destination.save()
+
+ for name in _UPDATABLE_FIELDS:
+ if name in update.changed:
+ setattr(batch_export, name, getattr(update, name))
+ batch_export.last_modified_by = user
+ batch_export.save()
+
+ service.sync_batch_export(batch_export, created=False)
+
+ return _to_configuration(batch_export, include_latest_runs=True)
diff --git a/products/batch_exports/backend/logic/runs.py b/products/batch_exports/backend/logic/runs.py
index 6e5a41d2418f..b6bac57e2c3d 100644
--- a/products/batch_exports/backend/logic/runs.py
+++ b/products/batch_exports/backend/logic/runs.py
@@ -21,7 +21,7 @@ LOGGER = structlog.get_logger(__name__)
_CANCELLABLE_STATUSES = (BatchExportRun.Status.RUNNING, BatchExportRun.Status.STARTING)
-def _to_run_detail(run: BatchExportRun) -> contracts.BatchExportRunDetail:
+def to_run_detail(run: BatchExportRun) -> contracts.BatchExportRunDetail:
return contracts.BatchExportRunDetail(
id=run.id,
status=run.status,
@@ -68,13 +68,13 @@ def list_runs(
if filters.statuses:
runs = runs.filter(status__in=filters.statuses)
return [
- _to_run_detail(run)
+ to_run_detail(run)
for run in select_page(runs, page, ordering_fields=contracts.BATCH_EXPORT_RUN_ORDERING_FIELDS)
]
def get_run(team_id: int, export_id: UUID, run_id: UUID) -> contracts.BatchExportRunDetail:
- return _to_run_detail(_get_run(team_id, export_id, run_id))
+ return to_run_detail(_get_run(team_id, export_id, run_id))
def retry_run(team_id: int, export_id: UUID, run_id: UUID) -> str:
diff --git a/products/batch_exports/backend/presentation/views/batch_export/destinations.py b/products/batch_exports/backend/presentation/views/batch_export/destinations.py
index 2b8e5d3d98e7..518ad584e930 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/destinations.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/destinations.py
@@ -1,7 +1,6 @@
"""Batch export destination serializers: the typed config and request shape of each destination type."""
import typing
-import collections.abc
from django.db import models
@@ -16,8 +15,8 @@ from products.batch_exports.backend.facade.enums import (
AZURE_BLOB_SUPPORTED_COMPRESSIONS,
BATCH_EXPORT_INTEGRATION_KINDS,
S3_SUPPORTED_COMPRESSIONS,
+ BatchExportDestinationType,
)
-from products.batch_exports.backend.models.batch_export import BatchExportDestination
class DatabricksDestinationConfigSerializer(serializers.Serializer):
@@ -512,7 +511,7 @@ class BatchExportDestinationRequestField(serializers.JSONField):
pass
-class BatchExportDestinationSerializer(serializers.ModelSerializer):
+class BatchExportDestinationSerializer(serializers.Serializer):
"""Serializer for an BatchExportDestination model.
The `config` field is polymorphic and typed only for destinations that keep
@@ -521,6 +520,9 @@ class BatchExportDestinationSerializer(serializers.ModelSerializer):
but without a typed OpenAPI schema. Secret fields are stripped from `config` on read.
"""
+ type = serializers.ChoiceField(
+ choices=BatchExportDestinationType.choices, help_text="A choice of supported BatchExportDestination types."
+ )
config = TypedBatchExportDestinationConfigField(
help_text=(
"Destination-specific configuration. Fields depend on `type`. Credentials for "
@@ -547,46 +549,19 @@ class BatchExportDestinationSerializer(serializers.ModelSerializer):
),
)
- class Meta:
- model = BatchExportDestination
- fields = ["type", "config", "integration", "integration_id"]
-
- def create(self, validated_data: collections.abc.Mapping[str, typing.Any]) -> BatchExportDestination:
- """Create a BatchExportDestination."""
- export_destination = BatchExportDestination.objects.create(**validated_data)
- return export_destination
-
def validate(self, attrs: dict[str, typing.Any]) -> dict[str, typing.Any]:
"""Check the config against the destination's workflow inputs, through the facade."""
request = self.context.get("request")
is_patch = request is not None and request.method == "PATCH"
+ # A connection test of a saved export may leave out fields, because its stored config fills them.
+ # TODO: remove this and the context key that `run_test_step` sets once inline credentials are gone.
+ partial = is_patch or self.context.get("partial_destination_config", False)
try:
attrs["config"] = batch_exports_api.validate_destination_config(
- attrs["type"], attrs["config"], partial=is_patch
+ attrs["type"], attrs["config"], partial=partial
)
except InvalidDestinationConfigError as e:
raise serializers.ValidationError(str(e))
return attrs
-
- def to_representation(self, instance: BatchExportDestination) -> dict:
- data = super().to_representation(instance)
-
- def remove_secret_fields_recursive(d: dict[str, typing.Any]):
- target = {}
-
- for k, v in d.items():
- if k in BatchExportDestination.secret_fields[instance.type]:
- continue
- elif isinstance(v, dict):
- target[k] = remove_secret_fields_recursive(v)
- else:
- target[k] = v
-
- return target
-
- config = remove_secret_fields_recursive(data["config"])
- data["config"] = batch_exports_api.coerce_destination_config(instance.type, config)
-
- return data
diff --git a/products/batch_exports/backend/presentation/views/batch_export/exports.py b/products/batch_exports/backend/presentation/views/batch_export/exports.py
index e32164b3cb1b..14458b1e4148 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/exports.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/exports.py
@@ -5,39 +5,33 @@ import typing
import dataclasses
from typing import cast
-from django.db import transaction
-
import posthoganalytics
-from drf_spectacular.utils import extend_schema, extend_schema_view
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view
from rest_framework import request, response, serializers, status, viewsets
from rest_framework.exceptions import NotAuthenticated, NotFound, PermissionDenied, ValidationError
-
-from posthog.schema import HogQLQueryModifiers, MaterializationMode, PersonsOnEventsMode
-
-from posthog.hogql import ast, errors
-from posthog.hogql.hogql import HogQLContext
-from posthog.hogql.parser import parse_select
-from posthog.hogql.printer import prepare_ast_for_printing
-from posthog.hogql.resolver import resolve_types
-from posthog.hogql.visitor import TraversingVisitor, clone_expr
+from rest_framework.pagination import LimitOffsetPagination
from posthog.api.log_entries import LogEntryMixin
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.utils import action
from posthog.integrations import IntegrationRecord
-from posthog.models import Team, User
+from posthog.models import User
-from products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_team
-from products.batch_exports.backend.facade import api as batch_exports_api
+from products.batch_exports.backend.facade import (
+ api as batch_exports_api,
+ contracts,
+)
from products.batch_exports.backend.facade.contracts import (
BatchExportNotFoundError,
BatchExportScheduleError,
InvalidBatchExportDestinationError,
InvalidBatchExportFilters,
+ InvalidDestinationConfigError,
UnsupportedDestinationTestError,
+ UnsupportedHogQLQueryError,
)
from products.batch_exports.backend.facade.enums import (
- EXPORTABLE_EVENTS_MODEL_FIELDS,
SUPPORTED_FILTER_TYPES_DISPLAY,
TIMEZONES,
BatchExportDestinationType,
@@ -45,20 +39,14 @@ from products.batch_exports.backend.facade.enums import (
BatchExportModel,
IncrementalMode,
)
-from products.batch_exports.backend.hogql_source import (
- UnsupportedHogQLQueryError,
- load_hogql_modifiers,
- serialize_batch_export_query,
- validate_hogql_query_for_batch_export,
-)
-from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportDestination, BatchExportSource
from products.batch_exports.backend.presentation.views.utils import (
HOGQL_MODIFIERS_HELP_TEXT,
HogQLModifiersField,
check_high_frequency_batch_exports_enabled,
check_hogql_batch_exports_enabled,
+ fetch_facade_page,
+ parse_path_uuid,
)
-from products.batch_exports.backend.service import BatchExportSchema, sync_batch_export
from . import (
destinations as destination_views,
@@ -174,78 +162,27 @@ class BatchExportRequestSerializer(serializers.Serializer):
)
-def parse_events_hogql_query(hogql_query: str, team_id: int, user: User | None) -> ast.SelectQuery | ast.SelectSetQuery:
- """Parse a HogQL SelectQuery from a string query."""
- try:
- parsed_query = parse_select(hogql_query)
- except Exception:
- raise serializers.ValidationError("Failed to parse query")
-
- try:
- restricted_properties = get_restricted_properties_with_group_type_index_for_team(user=user, team_id=team_id)
- context = HogQLContext(
- team_id=team_id,
- user=user,
- enable_select_queries=True,
- restricted_properties=restricted_properties,
- modifiers=HogQLQueryModifiers(
- personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS
- ),
- )
- if restricted_properties:
- # A restricted query is stored without its HogQL text, so no run can recompile it for
- # the native source, where a materialized column does not exist.
- context.modifiers.materializationMode = MaterializationMode.DISABLED
- use_native_schema = context.uses_new_events_schema()
- prepared_select_query = cast(
- ast.SelectQuery,
- prepare_ast_for_printing(
- parsed_query, context=context, dialect="hogql" if use_native_schema else "clickhouse"
- ),
- )
- if use_native_schema:
- resolve_types(clone_expr(parsed_query, clear_types=True), context=context, dialect="clickhouse")
- except errors.ExposedHogQLError as e:
- raise serializers.ValidationError(f"Invalid HogQL query: {e}")
-
- return prepared_select_query
-
-
-class _SubqueryFinder(TraversingVisitor):
- """Walk an AST subtree and flag if any SelectQuery or SelectSetQuery is found."""
-
- def __init__(self):
- super().__init__()
- self.found = False
-
- def visit_select_query(self, node: ast.SelectQuery):
- self.found = True
+def _get_integration_id(destination_attrs: dict[str, typing.Any]) -> int | None:
+ """Return the id of the integration a request sent, or None when it sent none or null."""
+ integration: IntegrationRecord | None = destination_attrs.get("integration")
+ return integration.id if integration is not None else None
- def visit_select_set_query(self, node: ast.SelectSetQuery):
- self.found = True
+def _get_integration_change(destination_attrs: dict[str, typing.Any]) -> contracts.IntegrationChange | None:
+ """Return the integration change a request asks for, or None when it left the integration out."""
+ if "integration" not in destination_attrs:
+ return None
+ return contracts.IntegrationChange(integration_id=_get_integration_id(destination_attrs))
-class _DatabaseFieldFinder(TraversingVisitor):
- """Walk an AST subtree and collect the names of the database fields it reads."""
- def __init__(self, context: HogQLContext):
- super().__init__()
- self.context = context
- self.names: set[str] = set()
-
- def visit_field(self, node: ast.Field):
- resolve = getattr(node.type, "resolve_database_field", None)
- if resolve is not None:
- database_field = resolve(self.context)
- name = getattr(database_field, "name", None)
- if name is not None:
- self.names.add(name)
- super().visit_field(node)
-
-
-class BatchExportSerializer(serializers.ModelSerializer):
+class BatchExportSerializer(serializers.Serializer):
"""Serializer for a BatchExport model."""
+ id = serializers.UUIDField(read_only=True)
+ team_id = serializers.IntegerField(read_only=True, help_text="The team this belongs to.")
+ name = serializers.CharField(
+ help_text="A human-readable name for this BatchExport.", style={"base_template": "textarea.html"}
+ )
model = serializers.ChoiceField(
choices=BatchExportModel.choices,
default=serializers.CreateOnlyDefault(BatchExportModel.EVENTS), # type: ignore[arg-type]
@@ -257,15 +194,31 @@ class BatchExportSerializer(serializers.ModelSerializer):
destination = destination_views.BatchExportDestinationSerializer(
help_text="Destination configuration (type, config, and optional integration)."
)
+ interval = serializers.ChoiceField(
+ choices=BatchExportInterval.choices,
+ help_text="How often the batch export should run.",
+ )
+ paused = serializers.BooleanField(required=False, help_text="Whether this BatchExport is paused or not.")
+ created_at = serializers.DateTimeField(
+ read_only=True, help_text="The timestamp at which this BatchExport was created."
+ )
+ last_updated_at = serializers.DateTimeField(
+ read_only=True, help_text="The timestamp at which this BatchExport was last updated."
+ )
+ last_paused_at = serializers.DateTimeField(
+ required=False, allow_null=True, help_text="The timestamp at which this BatchExport was last paused."
+ )
+ start_at = serializers.DateTimeField(
+ required=False, allow_null=True, help_text="Time before which any Batch Export runs won't be triggered."
+ )
+ end_at = serializers.DateTimeField(
+ required=False, allow_null=True, help_text="Time after which any Batch Export runs won't be triggered."
+ )
latest_runs = run_views.BatchExportRunSerializer(
many=True,
read_only=True,
help_text="The 10 most recent runs of this batch export, ordered newest first.",
)
- interval = serializers.ChoiceField(
- choices=BatchExportInterval.choices,
- help_text="How often the batch export should run.",
- )
hogql_query = serializers.CharField(
required=False,
allow_null=True,
@@ -295,6 +248,8 @@ class BatchExportSerializer(serializers.ModelSerializer):
default=serializers.CreateOnlyDefault(IncrementalMode.APPEND), # type: ignore[arg-type]
help_text="How this batch export handles incremental updates.",
)
+ schema = serializers.JSONField(read_only=True, help_text="A schema of custom fields to select when exporting data.")
+ filters = serializers.JSONField(required=False, allow_null=True)
timezone = serializers.ChoiceField(
choices=TIMEZONES,
required=False,
@@ -316,35 +271,6 @@ class BatchExportSerializer(serializers.ModelSerializer):
help_text="Hour-of-day offset (0-23) for daily and weekly intervals. Only valid when interval is 'day' or 'week'.",
)
- class Meta:
- model = BatchExport
- fields = [
- "id",
- "team_id",
- "name",
- "model",
- "destination",
- "interval",
- "paused",
- "created_at",
- "last_updated_at",
- "last_paused_at",
- "start_at",
- "end_at",
- "latest_runs",
- "hogql_query",
- "hogql_modifiers",
- "primary_key",
- "version_key",
- "incremental_mode",
- "schema",
- "filters",
- "timezone",
- "offset_day",
- "offset_hour",
- ]
- read_only_fields = ["id", "team_id", "created_at", "last_updated_at", "latest_runs", "schema"]
-
def validate(self, attrs: dict) -> dict:
"""Validate this batch export.
@@ -367,13 +293,12 @@ class BatchExportSerializer(serializers.ModelSerializer):
self._validate_attribute_only_supported_for_model(attrs, name="version_key", model=BatchExportModel.HOGQL)
# TODO: Once these teams are migrated, remove this.
- if model == BatchExportModel.EVENTS and (hogql_query := attrs.get("hogql_query")) is not None:
- is_grandfathered_organization = BatchExport.objects.filter(
- schema__isnull=False, team__organization_id=self.context["get_team"]().organization_id
- ).exists()
- if is_grandfathered_organization:
- attrs["hogql_query"] = self._validate_events_hogql_query(hogql_query)
- else:
+ # Saving the export checks the query, when it builds the schema the export stores.
+ if model == BatchExportModel.EVENTS and attrs.get("hogql_query") is not None:
+ is_grandfathered_organization = batch_exports_api.organization_has_custom_events_schema_exports(
+ self.context["get_team"]().organization_id
+ )
+ if not is_grandfathered_organization:
self._validate_attribute_only_supported_for_model(
attrs, name="hogql_query", model=BatchExportModel.HOGQL
)
@@ -386,17 +311,23 @@ class BatchExportSerializer(serializers.ModelSerializer):
return attrs
def _validate_interval_offset(self, attrs: dict) -> dict:
- interval = attrs.get("interval")
-
- if interval is None:
- return attrs
-
- # Convert offset_day and offset_hour to interval_offset
# Check if offset fields are in the request (for PATCH, distinguish between absent and None)
offset_day_provided = "offset_day" in attrs
offset_hour_provided = "offset_hour" in attrs
- offset_day = attrs.pop("offset_day", None)
- offset_hour = attrs.pop("offset_hour", None)
+
+ # Convert offset_day and offset_hour to interval_offset
+ if "interval" in attrs:
+ interval = attrs["interval"]
+ offset_day = attrs.pop("offset_day", None)
+ offset_hour = attrs.pop("offset_hour", None)
+ elif self.instance is not None and (offset_day_provided or offset_hour_provided):
+ # A PATCH may send offsets alone. They apply to the saved interval, and an offset the
+ # PATCH leaves out keeps its saved value.
+ interval = self.instance.interval
+ offset_day = attrs.pop("offset_day", self.instance.offset_day)
+ offset_hour = attrs.pop("offset_hour", self.instance.offset_hour)
+ else:
+ return attrs
if interval == "day":
# For daily exports, only offset_hour is used
@@ -475,30 +406,31 @@ class BatchExportSerializer(serializers.ModelSerializer):
1. No hogql_query was passed for a HogQL model.
2. When a non-None hogql_query for a HogQL model is unsupported.
"""
- source = self.instance.source if self.instance is not None else None
- saved_query = source.hogql_query if source is not None else None
- hogql_query = attrs.get("hogql_query", saved_query)
+ def get_value(name: str) -> typing.Any:
+ """Return the value the request sent, or else the saved one."""
+ return attrs.get(name, getattr(self.instance, name, None))
+
+ hogql_query = get_value("hogql_query")
if hogql_query is None:
raise serializers.ValidationError({"hogql_query": "'hogql_query' is required when 'model' is 'hogql'"})
- user = self.context["request"].user
- team = self.context["get_team"]()
- hogql_modifiers = self._get_hogql_modifiers(attrs)
+ # Saved modifiers are valid, even if the query check re-validates them anyways.
+ hogql_modifiers = get_value("hogql_modifiers")
try:
- parsed = validate_hogql_query_for_batch_export(hogql_query, team, user=user, modifiers=hogql_modifiers)
+ output_columns = set(
+ batch_exports_api.validate_hogql_query(
+ self.context["team_id"],
+ user_id=self._get_user().pk,
+ hogql_query=hogql_query,
+ hogql_modifiers=hogql_modifiers,
+ )
+ )
except UnsupportedHogQLQueryError as e:
raise serializers.ValidationError({"hogql_query": str(e)}) from e
- # NOTE: Should be populated after a successful preparation
- assert isinstance(parsed.type, (ast.SelectQueryType, ast.SelectSetQueryType))
- output_columns = set(parsed.type.columns)
-
- saved_incremental_mode = self.instance.incremental_mode if self.instance is not None else None
- incremental_mode = attrs.get("incremental_mode", saved_incremental_mode)
-
- saved_primary_key = source.primary_key if source is not None else None
- primary_key = set(attrs.get("primary_key", saved_primary_key) or ())
+ incremental_mode = get_value("incremental_mode")
+ primary_key = set(get_value("primary_key") or ())
if not primary_key and incremental_mode == IncrementalMode.MERGE:
raise serializers.ValidationError(
@@ -514,8 +446,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
}
)
- saved_version_key = source.version_key if source is not None else None
- version_key = set(attrs.get("version_key", saved_version_key) or ())
+ version_key = set(get_value("version_key") or ())
missing_version_key = version_key - output_columns
if missing_version_key:
@@ -525,22 +456,6 @@ class BatchExportSerializer(serializers.ModelSerializer):
}
)
- def _get_hogql_modifiers(self, attrs: dict[str, typing.Any]) -> HogQLQueryModifiers | None:
- if "hogql_modifiers" in attrs:
- if attrs["hogql_modifiers"] is None:
- modifiers: HogQLQueryModifiers | None = None
- else:
- # Modifiers were already validated and loaded by the field
- field = typing.cast(HogQLModifiersField, self.fields["hogql_modifiers"])
- modifiers = field.loaded
- else:
- source = self.instance.source if self.instance is not None else None
- # Modifiers are valid if present in the db, even if the function
- # re-validates them anyways.
- modifiers = load_hogql_modifiers(source.hogql_modifiers if source is not None else None)
-
- return modifiers
-
def validate_model(self, model: str) -> str:
"""Validate the model is new or not changing when provided."""
if model == BatchExportModel.HOGQL:
@@ -654,15 +569,13 @@ class BatchExportSerializer(serializers.ModelSerializer):
# Running validation for a `detail=True` action.
export_id = uuid.UUID(str(view.kwargs["pk"]))
- integration: IntegrationRecord | None = destination_attrs.get("integration")
try:
batch_exports_api.validate_batch_export_destination(
self.context["team_id"],
export_id=export_id,
destination_type=destination_type,
config=destination_attrs["config"],
- integration_id=integration.id if integration is not None else None,
- integration_sent="integration" in destination_attrs,
+ integration=_get_integration_change(destination_attrs),
)
except InvalidBatchExportDestinationError as e:
raise serializers.ValidationError(str(e))
@@ -670,8 +583,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
raise NotFound(f"BatchExport ID '{export_id}' not found.")
if destination_type == BatchExportDestinationType.WORKFLOWS:
- team_id = self.context["team_id"]
- team = Team.objects.get(id=team_id)
+ team = self.context["get_team"]()
if not posthoganalytics.feature_enabled(
"backfill-workflows-destination",
@@ -689,198 +601,63 @@ class BatchExportSerializer(serializers.ModelSerializer):
return destination_attrs
- def create(self, validated_data: dict) -> BatchExport:
- """Create a BatchExport."""
- destination_data = validated_data.pop("destination")
- team_id = self.context["team_id"]
- model = validated_data["model"]
- hogql_query = validated_data.pop("hogql_query", None)
- hogql_modifiers = validated_data.pop("hogql_modifiers", None)
- primary_key = validated_data.pop("primary_key", None)
- version_key = validated_data.pop("version_key", None)
-
- source = None
- if model == BatchExportModel.HOGQL:
- source = BatchExportSource(
- team_id=team_id,
- hogql_query=hogql_query,
- hogql_modifiers=hogql_modifiers,
- primary_key=primary_key,
- version_key=version_key,
- )
- elif hogql_query is not None:
- # TODO: Migrate batch exports using a HogQL query to HogQL model.
- validated_data["schema"] = self.serialize_hogql_query_to_batch_export_schema(hogql_query)
-
- destination_data["config"] = batch_exports_api.prepare_new_destination_config(
- destination_data["type"], destination_data["config"]
- )
-
- destination = BatchExportDestination(**destination_data)
+ def _get_user(self) -> User:
user = self.context["request"].user
if not isinstance(user, User):
raise NotAuthenticated()
+ return user
- batch_export = BatchExport(
- team_id=team_id,
- destination=destination,
- source=source,
- last_modified_by=user,
- **validated_data,
- )
-
- sync_batch_export(batch_export, created=True)
-
- with transaction.atomic():
- destination.save()
-
- if source is not None:
- source.save()
-
- batch_export.save()
-
- return batch_export
-
- def serialize_hogql_query_to_batch_export_schema(self, hogql_query: ast.SelectQuery) -> BatchExportSchema:
- """Return a batch export schema from a HogQL query ast."""
- request = self.context.get("request")
- user = request.user if request is not None else None
- restricted_properties = get_restricted_properties_with_group_type_index_for_team(
- user=user, team_id=self.context["team_id"]
- )
- context = HogQLContext(
- team_id=self.context["team_id"],
- user=user,
- enable_select_queries=True,
- limit_top_select=False,
- restricted_properties=restricted_properties,
- modifiers=HogQLQueryModifiers(
- personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,
+ def create(self, validated_data: dict) -> contracts.BatchExportConfiguration:
+ """Create a BatchExport."""
+ destination_data = validated_data["destination"]
+ export = contracts.NewBatchExport(
+ destination=contracts.BatchExportDestinationInput(
+ type=destination_data["type"],
+ config=destination_data["config"],
+ integration_id=_get_integration_id(destination_data),
),
+ **{name: value for name, value in validated_data.items() if name != "destination"},
)
- if context.uses_new_events_schema():
- context.modifiers.materializationMode = MaterializationMode.DISABLED
try:
- schema = serialize_batch_export_query(hogql_query, context)
- except errors.ExposedHogQLError:
- raise serializers.ValidationError("Unsupported HogQL query")
- if context.restricted_properties:
- schema.pop("hogql_query", None)
- return schema
-
- def _validate_events_hogql_query(self, hogql_query: str) -> ast.SelectQuery:
- """Validate a HogQL query being used for events batch exports.
-
- This method essentially checks that a query is supported by batch exports:
- 1. UNION ALL is not supported.
- 2. Any JOINs are not supported.
- 3. Query must SELECT FROM events, and only from events.
- 4. Subqueries in SELECT expressions are not supported.
- 5. Query must select only from those fields we expose from the events table.
- """
- parsed = parse_events_hogql_query(
- hogql_query, team_id=self.context["team_id"], user=self.context["request"].user
- )
-
- if isinstance(parsed, ast.SelectSetQuery):
- raise serializers.ValidationError("UNIONs are not supported")
-
- parsed = cast(ast.SelectQuery, parsed)
-
- if parsed.select_from is None:
- raise serializers.ValidationError("Query must SELECT FROM events")
-
- if parsed.ctes:
- raise serializers.ValidationError("CTEs are not supported")
-
- if isinstance(parsed.select_from.table, (ast.SelectQuery, ast.SelectSetQuery)):
- raise serializers.ValidationError("Subqueries are not supported")
-
- # Not sure how to make mypy understand this works, hence the ignore comment.
- # And if it doesn't, it's still okay as it could mean an unsupported query.
- # We would come back with the example to properly type this.
- if parsed.select_from.table.chain != ["events"]: # type: ignore
- raise serializers.ValidationError("Query must only SELECT FROM events")
-
- if parsed.select_from.next_join is not None:
- raise serializers.ValidationError("JOINs are not supported")
-
- subquery_finder = _SubqueryFinder()
- for field in parsed.select:
- subquery_finder.visit(field)
- if subquery_finder.found:
- raise serializers.ValidationError("Subqueries in SELECT expressions are not supported")
-
- # Check that the query only selects from those fields we expose from the events table.
- field_finder = _DatabaseFieldFinder(HogQLContext(team_id=self.context["team_id"], enable_select_queries=True))
- for field in parsed.select:
- field_finder.visit(field)
-
- unsupported = sorted(field_finder.names - EXPORTABLE_EVENTS_MODEL_FIELDS)
- if unsupported:
- raise serializers.ValidationError(
- f"Batch exports cannot read these fields: {', '.join(unsupported)}. "
- f"Supported fields are: {', '.join(sorted(EXPORTABLE_EVENTS_MODEL_FIELDS))}."
+ return batch_exports_api.create_batch_export_from_input(
+ self.context["team_id"], user_id=self._get_user().pk, export=export
)
+ except UnsupportedHogQLQueryError as e:
+ raise serializers.ValidationError(str(e))
- return parsed
-
- def update(self, batch_export: BatchExport, validated_data: dict) -> BatchExport:
+ def update(
+ self, instance: contracts.BatchExportConfiguration, validated_data: dict
+ ) -> contracts.BatchExportConfiguration:
"""Update a BatchExport."""
+ user = self._get_user()
destination_data = validated_data.pop("destination", None)
+ # The model cannot change, as `validate_model` checks.
+ validated_data.pop("model", None)
+
+ destination = None
+ if destination_data:
+ destination = contracts.BatchExportDestinationPatch(
+ config=destination_data.get("config", {}),
+ integration=_get_integration_change(destination_data),
+ )
- missing = object()
- hogql_query = validated_data.pop("hogql_query", missing)
- hogql_modifiers = validated_data.pop("hogql_modifiers", missing)
- primary_key = validated_data.pop("primary_key", missing)
- version_key = validated_data.pop("version_key", missing)
-
- user = self.context["request"].user
- if not isinstance(user, User):
- raise NotAuthenticated()
-
- validated_data["last_modified_by"] = user
-
- with transaction.atomic():
- if destination_data:
- # Type changes are rejected by `validate_destination` — the incoming `type`
- # (if any) always equals the existing type by the time we get here.
- batch_export.destination.config = batch_exports_api.merge_destination_config(
- batch_export.destination.type,
- batch_export.destination.config,
- destination_data.get("config", {}),
- )
- integration = destination_data.get("integration", batch_export.destination.integration)
- batch_export.destination.integration = integration
-
- if batch_export.model == BatchExportModel.HOGQL:
- hogql_attributes = (hogql_query, hogql_modifiers, primary_key, version_key)
- if any(attr is not missing for attr in hogql_attributes):
- source = batch_export.source or BatchExportSource(team_id=batch_export.team_id)
-
- if hogql_query is not missing:
- source.hogql_query = hogql_query
- if hogql_modifiers is not missing:
- source.hogql_modifiers = hogql_modifiers
- if primary_key is not missing:
- source.primary_key = primary_key
- if version_key is not missing:
- source.version_key = version_key
-
- source.save()
- batch_export.source = source
-
- elif hogql_query is not missing:
- validated_data["schema"] = (
- self.serialize_hogql_query_to_batch_export_schema(hogql_query) if hogql_query is not None else None
- )
-
- batch_export.destination.save()
- batch_export = super().update(batch_export, validated_data)
+ update = contracts.BatchExportUpdate(
+ changed=frozenset(validated_data),
+ destination=destination,
+ **validated_data,
+ )
+ try:
+ return batch_exports_api.update_batch_export(
+ self.context["team_id"], instance.id, user_id=user.pk, update=update
+ )
+ except UnsupportedHogQLQueryError as e:
+ raise serializers.ValidationError(str(e))
- sync_batch_export(batch_export, created=False)
- return batch_export
+# The viewset has no queryset to type the `id` path parameter from, so each detail action declares it.
+BATCH_EXPORT_ID_PARAMETER = OpenApiParameter(
+ "id", OpenApiTypes.UUID, OpenApiParameter.PATH, description="A UUID string identifying this batch export."
+)
@extend_schema(tags=["batch_exports"])
@@ -891,27 +668,42 @@ class BatchExportSerializer(serializers.ModelSerializer):
# created before integrations keep their inline credentials on update.
# Responses continue to use BatchExportSerializer.
create=extend_schema(request=BatchExportRequestSerializer),
- update=extend_schema(request=BatchExportRequestSerializer),
- partial_update=extend_schema(request=BatchExportRequestSerializer),
+ update=extend_schema(request=BatchExportRequestSerializer, parameters=[BATCH_EXPORT_ID_PARAMETER]),
+ partial_update=extend_schema(request=BatchExportRequestSerializer, parameters=[BATCH_EXPORT_ID_PARAMETER]),
+ retrieve=extend_schema(parameters=[BATCH_EXPORT_ID_PARAMETER]),
+ destroy=extend_schema(parameters=[BATCH_EXPORT_ID_PARAMETER]),
+ logs=extend_schema(parameters=[BATCH_EXPORT_ID_PARAMETER]),
)
class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelViewSet):
scope_object = "batch_export"
- queryset = (
- BatchExport.objects.exclude(deleted=True)
- .order_by("-created_at")
- .prefetch_related("destination", "source")
- .all()
- )
serializer_class = BatchExportSerializer
log_source = "batch_exports"
- def safely_get_queryset(self, queryset):
- """Filter out batch exports with Workflows destination type if action is list."""
- if self.action == "list":
- return queryset.exclude(destination__type="Workflows")
- return queryset
+ def dangerously_get_object(self) -> contracts.BatchExportConfiguration:
+ # The mixin's get_object() reads through a queryset, which this view does not have, so the
+ # export is read through the facade instead. That skips the mixin's team filter and object
+ # permission check, so both happen here: the facade filters by team, and
+ # check_object_permissions runs on the contract. `logs` reads the export through here too.
+ export_id = parse_path_uuid(self.kwargs["pk"])
+ try:
+ batch_export = batch_exports_api.get_batch_export(
+ self.team_id, export_id, include_latest_runs=self.action == "retrieve"
+ )
+ except BatchExportNotFoundError as e:
+ raise NotFound(*e.args) from e
+ self.check_object_permissions(self.request, batch_export)
+ return batch_export
+
+ def list(self, request: request.Request, *args: typing.Any, **kwargs: typing.Any) -> response.Response:
+ paginator = cast(LimitOffsetPagination, self.paginator)
+ batch_exports = fetch_facade_page(
+ paginator,
+ request,
+ lambda offset, limit: batch_exports_api.list_batch_exports(self.team_id, offset=offset, limit=limit),
+ )
+ return paginator.get_paginated_response(self.get_serializer(batch_exports, many=True).data)
- @extend_schema(request=None)
+ @extend_schema(request=None, parameters=[BATCH_EXPORT_ID_PARAMETER])
@action(methods=["POST"], detail=True, required_scopes=["batch_export:write"])
def pause(self, request: request.Request, *args, **kwargs) -> response.Response:
"""Pause a BatchExport."""
@@ -932,7 +724,7 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVi
return response.Response({"paused": True})
- @extend_schema(request=BatchExportUnpauseRequestSerializer)
+ @extend_schema(request=BatchExportUnpauseRequestSerializer, parameters=[BATCH_EXPORT_ID_PARAMETER])
@action(methods=["POST"], detail=True, required_scopes=["batch_export:write"])
def unpause(self, request: request.Request, *args, **kwargs) -> response.Response:
"""Unpause a BatchExport."""
@@ -958,13 +750,12 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVi
return response.Response({"paused": False})
- def perform_destroy(self, instance: BatchExport):
+ def perform_destroy(self, instance: contracts.BatchExportConfiguration) -> None:
"""Perform a BatchExport destroy by clearing Temporal and Django state.
If the underlying Temporal Schedule doesn't exist, we ignore the error and proceed with the delete anyways.
The Schedule could have been manually deleted causing Django and Temporal to go out of sync. For whatever reason,
- since we are deleting, we assume that we can recover from this state by finishing the delete operation by calling
- instance.save().
+ since we are deleting, we assume that we can recover from this state by finishing the delete operation.
"""
try:
batch_exports_api.delete_batch_export(instance.id, instance.team_id)
@@ -977,19 +768,25 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVi
request.data.pop("step", None)
return step_serializer.validated_data["step"]
- def _run_destination_test_step(self, serializer: serializers.BaseSerializer, step: int) -> response.Response:
+ def _run_destination_test_step(
+ self, serializer: serializers.BaseSerializer, step: int, *, export_id: uuid.UUID | None
+ ) -> response.Response:
destination = serializer.validated_data["destination"]
- integration: IntegrationRecord | None = destination.get("integration")
try:
result = batch_exports_api.run_destination_test_step(
self.team_id,
+ export_id=export_id,
destination_type=destination["type"],
config=destination["config"],
- integration_id=integration.id if integration else None,
+ integration_id=_get_integration_id(destination),
step=step,
)
except UnsupportedDestinationTestError:
raise ValidationError(f"Connection tests aren't available for {destination['type']} destinations.")
+ except InvalidDestinationConfigError as e:
+ raise ValidationError(str(e))
+ except BatchExportNotFoundError:
+ raise NotFound(f"BatchExport ID '{export_id}' not found.")
return response.Response(dataclasses.asdict(result))
@action(methods=["GET"], detail=False, required_scopes=["batch_export:read"])
@@ -1012,22 +809,17 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVi
serializer = self.get_serializer(data=request.data)
_ = serializer.is_valid(raise_exception=True)
- return self._run_destination_test_step(serializer, test_step)
+ return self._run_destination_test_step(serializer, test_step, export_id=None)
+ @extend_schema(parameters=[BATCH_EXPORT_ID_PARAMETER])
@action(methods=["POST"], detail=True, required_scopes=["batch_export:write"])
def run_test_step(self, request: request.Request, *args, **kwargs) -> response.Response:
test_step = self._parse_destination_test_step(request)
batch_export = self.get_object()
- try:
- stored_config = batch_exports_api.get_destination_test_config(batch_export.team_id, batch_export.id)
- except BatchExportNotFoundError:
- raise NotFound(f"BatchExport ID '{batch_export.id}' not found.")
-
- data = request.data
- data["destination"]["config"] = {**stored_config, **data["destination"]["config"]}
-
- serializer = self.get_serializer(data=data)
+ serializer = self.get_serializer(
+ data=request.data, context={**self.get_serializer_context(), "partial_destination_config": True}
+ )
_ = serializer.is_valid(raise_exception=True)
- return self._run_destination_test_step(serializer, test_step)
+ return self._run_destination_test_step(serializer, test_step, export_id=batch_export.id)
diff --git a/products/batch_exports/backend/presentation/views/file_download.py b/products/batch_exports/backend/presentation/views/file_download.py
index 25d3fbbd941c..20d95ccbf22c 100644
--- a/products/batch_exports/backend/presentation/views/file_download.py
+++ b/products/batch_exports/backend/presentation/views/file_download.py
@@ -37,6 +37,7 @@ from products.batch_exports.backend.presentation.views.utils import (
HOGQL_MODIFIERS_HELP_TEXT,
HogQLModifiersField,
check_hogql_batch_exports_enabled,
+ fetch_facade_page,
parse_path_uuid,
)
@@ -411,17 +412,12 @@ class FileDownloadBatchExportOnDemandViewSet(TeamAndOrgViewSetMixin, LogEntryMix
return run
def list(self, request, *args, **kwargs) -> response.Response:
- # The facade slices the page, so the paginator gets its state set directly instead of a queryset.
paginator = cast(LimitOffsetPagination, self.paginator)
- limit = paginator.get_limit(request)
- offset = paginator.get_offset(request)
- # PAGE_SIZE gives every request a limit.
- assert limit is not None
- runs, count = batch_exports_api.list_file_download_runs(self.team_id, offset=offset, limit=limit)
- paginator.request = request
- paginator.limit = limit
- paginator.offset = offset
- paginator.count = count
+ runs = fetch_facade_page(
+ paginator,
+ request,
+ lambda offset, limit: batch_exports_api.list_file_download_runs(self.team_id, offset=offset, limit=limit),
+ )
return paginator.get_paginated_response(ListOutputSerializer(runs, many=True).data)
@extend_schema(
diff --git a/products/batch_exports/backend/presentation/views/utils.py b/products/batch_exports/backend/presentation/views/utils.py
index 7393e8a57378..ddbc247133c8 100644
--- a/products/batch_exports/backend/presentation/views/utils.py
+++ b/products/batch_exports/backend/presentation/views/utils.py
@@ -1,11 +1,14 @@
import uuid
import typing
+from collections.abc import Callable
import pydantic
import posthoganalytics
from drf_spectacular.utils import extend_schema_field
from rest_framework import serializers
from rest_framework.exceptions import NotFound, PermissionDenied
+from rest_framework.pagination import LimitOffsetPagination
+from rest_framework.request import Request
from posthog.schema import HogQLQueryModifiers
@@ -77,8 +80,6 @@ def check_high_frequency_batch_exports_enabled(team: Team) -> None:
class HogQLModifiersField(serializers.JSONField):
"""HogQL modifiers, validated against `HogQLQueryModifiers` and stored as a plain dict."""
- loaded: HogQLQueryModifiers | None = None
-
def to_internal_value(self, data: typing.Any) -> dict[str, typing.Any]:
value = super().to_internal_value(data)
try:
@@ -89,8 +90,6 @@ class HogQLModifiersField(serializers.JSONField):
location = ".".join(str(part) for part in error["loc"])
messages.append(f"{location}: {error['msg']}" if location else error["msg"])
raise serializers.ValidationError(messages) from e
- else:
- self.loaded = modifiers
return modifiers.model_dump(mode="json", exclude_none=True)
def to_representation(self, value: typing.Any) -> typing.Any:
@@ -99,3 +98,22 @@ class HogQLModifiersField(serializers.JSONField):
if isinstance(value, dict):
value = drop_unknown_hogql_modifiers(value)
return super().to_representation(value)
+
+
+def fetch_facade_page[T](
+ paginator: LimitOffsetPagination, request: Request, fetch_page: Callable[[int, int], tuple[list[T], int]]
+) -> list[T]:
+ """Fetch one page of a list through the facade with ``fetch_page(offset, limit)``, and set up the paginator for it.
+
+ The facade slices the page, so the paginator gets its state set directly instead of a queryset.
+ """
+ limit = paginator.get_limit(request)
+ offset = paginator.get_offset(request)
+ # PAGE_SIZE gives every request a limit.
+ assert limit is not None
+ items, count = fetch_page(offset, limit)
+ paginator.request = request
+ paginator.limit = limit
+ paginator.offset = offset
+ paginator.count = count
+ return items
diff --git a/products/batch_exports/backend/temporal/record_batch_model.py b/products/batch_exports/backend/temporal/record_batch_model.py
index 026bd38dac04..0683d56bc7de 100644
--- a/products/batch_exports/backend/temporal/record_batch_model.py
+++ b/products/batch_exports/backend/temporal/record_batch_model.py
@@ -3,14 +3,12 @@ import uuid
import typing
import datetime as dt
-from posthog.schema import HogQLQueryModifiers, MaterializationMode, PersonsOnEventsMode
-
from posthog.hogql.constants import HogQLQuerySettings
from posthog.hogql.context import HogQLContext
from posthog.hogql.errors import ExposedHogQLError
from posthog.hogql.hogql import ast
from posthog.hogql.metadata import get_table_names
-from posthog.hogql.parser import parse_expr, parse_select
+from posthog.hogql.parser import parse_expr
from posthog.hogql.printer import prepare_ast_for_printing, print_prepared_ast
from posthog.hogql.visitor import clone_expr
@@ -23,6 +21,7 @@ from posthog.sync import database_sync_to_async
from posthog.temporal.common.clickhouse import get_client
from posthog.temporal.common.logger import get_write_only_logger
+from products.batch_exports.backend.custom_events_schema import recompile_stored_schema
from products.batch_exports.backend.hogql_source import (
DATA_INTERVAL_END_PLACEHOLDER,
UnsupportedHogQLQueryError,
@@ -32,7 +31,6 @@ from products.batch_exports.backend.hogql_source import (
load_hogql_modifiers,
parse_hogql_select_for_batch_export,
replace_interval_placeholders,
- serialize_batch_export_query,
validate_hogql_batch_export_user,
)
from products.batch_exports.backend.service import (
@@ -483,20 +481,6 @@ def resolve_batch_exports_model(
)
if schema is not None and (query := schema.get("hogql_query")) and use_new_events_schema(team_id):
- context = HogQLContext(
- team_id=team_id,
- enable_select_queries=True,
- limit_top_select=False,
- modifiers=HogQLQueryModifiers(
- materializationMode=MaterializationMode.DISABLED,
- personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,
- ),
- )
- compiled = serialize_batch_export_query(typing.cast(ast.SelectQuery, parse_select(query)), context)
- fields = [
- {"expression": compiled_field["expression"], "alias": stored_field["alias"]}
- for stored_field, compiled_field in zip(schema["fields"], compiled["fields"], strict=True)
- ]
- extra_query_parameters = compiled["values"]
+ fields, extra_query_parameters = recompile_stored_schema(team_id, schema, query)
return model, record_batch_model, model_name, fields, filters, extra_query_parameters
diff --git a/products/batch_exports/backend/tests/api/backfills/test_create.py b/products/batch_exports/backend/tests/api/backfills/test_create.py
index e17bef20a69d..abbfdea0437c 100644
--- a/products/batch_exports/backend/tests/api/backfills/test_create.py
+++ b/products/batch_exports/backend/tests/api/backfills/test_create.py
@@ -359,19 +359,24 @@ def test_cannot_trigger_backfill_for_another_organization(
assert response.status_code == status.HTTP_403_FORBIDDEN, response.json()
-def test_backfill_is_partitioned_by_team_id(client: HttpClient, temporal, organization, team, user, aws_s3_integration):
- from posthog.api.test.test_team import create_team
-
- other_team = create_team(organization)
-
[email protected]("scope", ["other_team", "deleted"])
+def test_backfill_is_partitioned_by_team_id(
+ client: HttpClient, temporal, organization, team, user, aws_s3_integration, scope
+):
client.force_login(user)
batch_export = _create_batch_export_ok(client, team, aws_s3_integration.id, "events")
batch_export_id = batch_export["id"]
+ if scope == "other_team":
+ team_id = create_team(organization).pk
+ else:
+ team_id = team.pk
+ BatchExport.objects.filter(id=batch_export_id).update(deleted=True)
+
response = backfill_batch_export(
client,
- other_team.pk,
+ team_id,
batch_export_id,
"2021-01-01T00:00:00+00:00",
"2021-01-01T01:00:00+00:00",
diff --git a/products/batch_exports/backend/tests/api/test_destination_tests.py b/products/batch_exports/backend/tests/api/test_destination_tests.py
index 7563d7e086d2..c25acbbb9bfb 100644
--- a/products/batch_exports/backend/tests/api/test_destination_tests.py
+++ b/products/batch_exports/backend/tests/api/test_destination_tests.py
@@ -177,6 +177,7 @@ def test_can_run_s3_test_step_for_new_destination(
assert destination_test["result"]["message"] is None
[email protected]("sends_config", [True, False])
def test_can_run_s3_test_step_for_destination(
client: HttpClient,
bucket_name,
@@ -186,6 +187,7 @@ def test_can_run_s3_test_step_for_destination(
team,
user,
object_storage_integration,
+ sends_config,
):
destination_data = {
"type": "S3Compatible",
@@ -211,9 +213,11 @@ def test_can_run_s3_test_step_for_destination(
batch_export_data,
)
+ # The stored config fills the fields a request leaves out, as the app leaves out secrets.
+ request_destination = destination_data if sends_config else {**destination_data, "config": {}}
response = client.post(
f"/api/projects/{team.pk}/batch_exports/{batch_export['id']}/run_test_step",
- {**{"step": 0}, **batch_export_data},
+ {"step": 0, **batch_export_data, "destination": request_destination},
content_type="application/json",
)
diff --git a/products/batch_exports/backend/tests/api/test_list.py b/products/batch_exports/backend/tests/api/test_list.py
index 6f9d683704e7..5d02241d980d 100644
--- a/products/batch_exports/backend/tests/api/test_list.py
+++ b/products/batch_exports/backend/tests/api/test_list.py
@@ -3,6 +3,8 @@ from unittest import mock
from django.test.client import Client as HttpClient
+from rest_framework import status
+
from posthog.api.test.test_team import create_team
from posthog.api.test.test_user import create_user
@@ -31,6 +33,18 @@ def test_list_batch_exports(client: HttpClient, organization, team, user, s3_bat
response = list_batch_exports_ok(client, team.pk)
assert len(response["results"]) == 2
+ pages: list[str] = []
+ for offset in range(2):
+ response = client.get(f"/api/projects/{team.pk}/batch_exports?limit=1&offset={offset}")
+ assert response.status_code == status.HTTP_200_OK, response.json()
+ page = response.json()
+ assert page["count"] == 2
+ pages.extend(result["id"] for result in page["results"])
+
+ assert page["next"] is None
+ assert page["previous"] is not None
+ assert pages == [second_export["id"], first_export["id"]]
+
# Deleted batch exports should not be returned.
delete_batch_export_ok(client, team.pk, first_export["id"])
response = list_batch_exports_ok(client, team.pk)
diff --git a/products/batch_exports/backend/tests/api/test_serializers.py b/products/batch_exports/backend/tests/api/test_serializers.py
index fd1dd0fdf662..f8825a7ccc3f 100644
--- a/products/batch_exports/backend/tests/api/test_serializers.py
+++ b/products/batch_exports/backend/tests/api/test_serializers.py
@@ -1,29 +1,58 @@
+import uuid
+import datetime as dt
+import dataclasses
from types import SimpleNamespace
-from typing import cast
+from typing import Any
-from posthog.test.base import BaseTest, materialized
+from posthog.test.base import BaseTest
from unittest.mock import patch
-from django.test import SimpleTestCase, override_settings
+from django.test import SimpleTestCase
from parameterized import parameterized
-from rest_framework import serializers
-from posthog.hogql import ast
-from posthog.hogql.property_access_types import RestrictedProperty
-
-from posthog.models import Organization, PropertyDefinition, Team
+from posthog.models import Organization, Team
from posthog.models.integration import Integration
from posthog.models.scoping import team_scope
-from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportSource
+from products.batch_exports.backend.facade import (
+ api as batch_exports_api,
+ contracts,
+)
from products.batch_exports.backend.presentation.views.batch_export.destinations import BatchExportDestinationSerializer
-from products.batch_exports.backend.presentation.views.batch_export.exports import (
- BatchExportSerializer,
- parse_events_hogql_query,
+from products.batch_exports.backend.presentation.views.batch_export.exports import BatchExportSerializer
+
+_CONFIGURATION = contracts.BatchExportConfiguration(
+ id=uuid.uuid4(),
+ team_id=1,
+ name="An export",
+ model="events",
+ destination=contracts.BatchExportDestinationConfiguration(type="NoOp", config={}, integration=None),
+ interval="hour",
+ paused=False,
+ created_at=dt.datetime(2026, 1, 1, tzinfo=dt.UTC),
+ last_updated_at=dt.datetime(2026, 1, 1, tzinfo=dt.UTC),
+ last_paused_at=None,
+ start_at=None,
+ end_at=None,
+ latest_runs=(),
+ hogql_query=None,
+ hogql_modifiers=None,
+ primary_key=None,
+ version_key=None,
+ incremental_mode="APPEND",
+ schema=None,
+ filters=None,
+ timezone="UTC",
+ offset_day=None,
+ offset_hour=None,
)
+def make_configuration(**overrides: Any) -> contracts.BatchExportConfiguration:
+ return dataclasses.replace(_CONFIGURATION, **overrides)
+
+
class TestBatchExportModelValidation(SimpleTestCase):
@parameterized.expand(
[
@@ -36,249 +65,19 @@ class TestBatchExportModelValidation(SimpleTestCase):
]
)
def test_rejects_unsupported_fields(self, model: str, field: str, value: object) -> None:
- serializer = BatchExportSerializer(BatchExport(model=model), data={field: value}, partial=True)
+ serializer = BatchExportSerializer(make_configuration(model=model), data={field: value}, partial=True)
assert not serializer.is_valid()
assert field in serializer.errors
@parameterized.expand([("events",), ("persons",), ("sessions",)])
def test_patch_without_model_specific_fields(self, model: str) -> None:
- serializer = BatchExportSerializer(BatchExport(model=model), data={"name": "Renamed"}, partial=True)
+ serializer = BatchExportSerializer(make_configuration(model=model), data={"name": "Renamed"}, partial=True)
assert serializer.is_valid(), serializer.errors
assert serializer.validated_data == {"name": "Renamed"}
-def prepare_query(query: str, team_id: int) -> ast.SelectQuery:
- """Parse and resolve a HogQL query string into a prepared AST."""
- return cast(ast.SelectQuery, parse_events_hogql_query(query, team_id=team_id, user=None))
-
-
-class TestSerializeHogQLQueryToBatchExportSchema(BaseTest):
- def _make_serializer(self) -> BatchExportSerializer:
- return BatchExportSerializer(context={"team_id": self.team.pk})
-
- @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False)
- def test_resaving_legacy_query_preserves_types_and_column_names(self):
- PropertyDefinition.objects.create(
- team=self.team, name="amount", type=PropertyDefinition.Type.EVENT, property_type="Numeric"
- )
- query = "SELECT round(e.properties.amount) AS rounded, lower(e.event) FROM events AS e"
- serializer = self._make_serializer()
-
- schema = serializer.serialize_hogql_query_to_batch_export_schema(prepare_query(query, self.team.pk))
- resaved = serializer.serialize_hogql_query_to_batch_export_schema(prepare_query(query, self.team.pk))
-
- assert schema == resaved
- assert [field["alias"] for field in schema["fields"]] == ["rounded", "`lower(e.event)`"]
- assert schema["fields"][0]["expression"].startswith("round(accurateCastOrNull(")
- assert schema["values"] == {"hogql_val_0": "amount", "hogql_val_1": "Float64"}
- assert "toFloat(" in schema["hogql_query"]
-
- @parameterized.expand(
- [
- (
- "simple_fields_use_original_name_as_alias",
- "SELECT event, person_id FROM events",
- [
- {"expression": "events.event", "alias": "event"},
- {"expression": "events.person_id", "alias": "person_id"},
- ],
- {},
- ),
- (
- "aliased_fields",
- "SELECT event AS my_event, team_id AS my_team FROM events",
- [
- {"expression": "events.event", "alias": "my_event"},
- {"expression": "events.team_id", "alias": "my_team"},
- ],
- {},
- ),
- (
- "property_access_populates_values",
- "SELECT properties.$browser AS browser FROM events",
- [
- {
- "expression": "replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^\"|\"$', '')",
- "alias": "browser",
- },
- ],
- {"hogql_val_0": "$browser"},
- ),
- (
- "mixed_simple_and_property_fields",
- "SELECT event, properties.$browser AS browser, properties.custom AS custom, person_id FROM events",
- [
- {"expression": "events.event", "alias": "event"},
- {
- "expression": "replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^\"|\"$', '')",
- "alias": "browser",
- },
- {
- "expression": "replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^\"|\"$', '')",
- "alias": "custom",
- },
- {"expression": "events.person_id", "alias": "person_id"},
- ],
- {"hogql_val_0": "$browser", "hogql_val_1": "custom"},
- ),
- ],
- )
- def test_serialize_hogql_query(self, _name, query, expected_fields, expected_values):
- prepared = prepare_query(query, self.team.pk)
- serializer = self._make_serializer()
-
- result = serializer.serialize_hogql_query_to_batch_export_schema(prepared)
-
- assert result["fields"] == expected_fields
- assert result["values"] == expected_values
- assert "hogql_query" in result
-
- @parameterized.expand(
- [
- ("integer", "SELECT 1 FROM events", "1", "`1`"),
- ("string", "SELECT 'hello' FROM events", "%(hogql_val_0)s", "hello"),
- ("float", "SELECT 3.14 FROM events", "3.14", "`3.14`"),
- ("null", "SELECT null FROM events", "NULL", "NULL"),
- ("boolean", "SELECT true FROM events", "1", "`1`"),
- ],
- )
- def test_serialize_hogql_query_with_bare_constant(self, _name, query, expected_expression, expected_alias):
- prepared = prepare_query(query, self.team.pk)
- serializer = self._make_serializer()
-
- result = serializer.serialize_hogql_query_to_batch_export_schema(prepared)
-
- assert len(result["fields"]) == 1
- field = result["fields"][0]
- assert field["expression"] == expected_expression
- assert field["alias"] == expected_alias
-
- def test_serialize_hogql_query_escapes_injected_alias(self):
- """An alias containing SQL injection attempts is escaped with backticks."""
- query = "SELECT uuid AS `x, (SELECT query FROM another_table LIMIT 100) AS leaked` FROM events"
- prepared = prepare_query(query, self.team.pk)
- serializer = self._make_serializer()
-
- result = serializer.serialize_hogql_query_to_batch_export_schema(prepared)
-
- assert len(result["fields"]) == 1
- field = result["fields"][0]
- # The alias must be wrapped in backticks, keeping the malicious string as a single identifier
- assert field["alias"] == "`x, (SELECT query FROM another_table LIMIT 100) AS leaked`"
- assert field["expression"] == "events.uuid"
-
- @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)
- def test_native_query_keeps_property_type_casts(self):
- PropertyDefinition.objects.create(
- team=self.team, name="amount", type=PropertyDefinition.Type.EVENT, property_type="Numeric"
- )
- query = "SELECT e.properties.amount AS amount FROM events AS e"
-
- schema = self._make_serializer().serialize_hogql_query_to_batch_export_schema(
- prepare_query(query, self.team.pk)
- )
-
- assert schema["fields"][0]["expression"].startswith("accurateCastOrNull(")
- assert schema["values"]["hogql_val_1"] == "Float64"
-
- @parameterized.expand([("unrestricted", False), ("restricted", True)])
- @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False)
- def test_materialized_columns_are_kept_out_of_a_query_that_cannot_be_recompiled(self, _name: str, restricted: bool):
- restrictions = (
- {RestrictedProperty(name="secret", property_type=PropertyDefinition.Type.EVENT)} if restricted else set()
- )
- with (
- patch(
- "products.batch_exports.backend.presentation.views.batch_export.exports."
- "get_restricted_properties_with_group_type_index_for_team",
- return_value=restrictions,
- ),
- materialized("events", "$browser"),
- ):
- query = prepare_query("SELECT properties.$browser AS browser FROM events", self.team.pk)
- serializer = BatchExportSerializer(
- context={"team_id": self.team.pk, "request": SimpleNamespace(user=self.user)}
- )
-
- schema = serializer.serialize_hogql_query_to_batch_export_schema(query)
-
- expression = schema["fields"][0]["expression"]
- if restricted:
- assert "hogql_query" not in schema
- assert "mat_" not in expression
- else:
- assert "mat_$browser" in expression
-
- @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)
- @patch(
- "products.batch_exports.backend.presentation.views.batch_export.exports.get_restricted_properties_with_group_type_index_for_team"
- )
- @patch("posthog.models.event.new_events_schema.use_new_events_schema", return_value=True)
- def test_restricted_property_query_is_not_recompiled_without_user(
- self, _use_new_events_schema, get_restricted_properties
- ):
- get_restricted_properties.return_value = {
- RestrictedProperty(name="secret", property_type=PropertyDefinition.Type.EVENT)
- }
- query = prepare_query("SELECT properties.secret AS secret FROM events", self.team.pk)
- serializer = BatchExportSerializer(
- context={"team_id": self.team.pk, "request": SimpleNamespace(user=self.user)}
- )
-
- schema = serializer.serialize_hogql_query_to_batch_export_schema(query)
-
- assert schema["fields"] == [{"expression": "NULL", "alias": "secret"}]
- assert "hogql_query" not in schema
-
- with self.assertRaises(serializers.ValidationError):
- serializer.serialize_hogql_query_to_batch_export_schema(
- prepare_query("SELECT properties FROM events", self.team.pk)
- )
-
- @parameterized.expand(
- [
- ("restricted", "SELECT properties.`$feature/secret` AS flag FROM events", "NULL"),
- ("visible", "SELECT properties.`$feature/checkout` AS flag FROM events", "JSONExtractRaw("),
- ]
- )
- @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)
- @patch(
- "products.batch_exports.backend.presentation.views.batch_export.exports.get_restricted_properties_with_group_type_index_for_team"
- )
- @patch("posthog.models.event.new_events_schema.use_new_events_schema", return_value=True)
- def test_native_export_hides_a_restricted_flag(
- self, _name: str, query: str, expected: str, _use_new_events_schema, get_restricted_properties
- ):
- get_restricted_properties.return_value = {
- RestrictedProperty(name="$feature/secret", property_type=PropertyDefinition.Type.EVENT)
- }
- serializer = BatchExportSerializer(
- context={"team_id": self.team.pk, "request": SimpleNamespace(user=self.user)}
- )
-
- schema = serializer.serialize_hogql_query_to_batch_export_schema(prepare_query(query, self.team.pk))
-
- expression = schema["fields"][0]["expression"]
- assert expression == expected if expected == "NULL" else expected in expression, expression
-
- @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)
- @patch("posthog.models.event.new_events_schema.use_new_events_schema", return_value=True)
- def test_native_export_names_a_bare_flag_column(self, _use_new_events_schema):
- serializer = BatchExportSerializer(
- context={"team_id": self.team.pk, "request": SimpleNamespace(user=self.user)}
- )
-
- schema = serializer.serialize_hogql_query_to_batch_export_schema(
- prepare_query("SELECT properties.`$feature/checkout` FROM events", self.team.pk)
- )
-
- field = schema["fields"][0]
- assert field["alias"] == "`$feature/checkout`", field
- assert "JSONExtractRaw(" in field["expression"], field
-
-
class TestBatchExportIncrementalMode(BaseTest):
@parameterized.expand([("events",), ("hogql",)])
def test_create_and_patch_incremental_mode(self, model: str) -> None:
@@ -299,7 +98,7 @@ class TestBatchExportIncrementalMode(BaseTest):
with (
team_scope(self.team.pk),
- patch("products.batch_exports.backend.presentation.views.batch_export.exports.sync_batch_export"),
+ patch("products.batch_exports.backend.service.sync_batch_export"),
patch(
"products.batch_exports.backend.presentation.views.utils.posthoganalytics.feature_enabled",
return_value=True,
@@ -308,39 +107,42 @@ class TestBatchExportIncrementalMode(BaseTest):
serializer = BatchExportSerializer(data=data, context=context)
assert serializer.is_valid(), serializer.errors
batch_export = serializer.save()
- batch_export.refresh_from_db()
- assert batch_export.incremental_mode == BatchExport.IncrementalMode.MERGE
+ assert batch_exports_api.get_batch_export(self.team.pk, batch_export.id).incremental_mode == "MERGE"
assert BatchExportSerializer(batch_export).data["incremental_mode"] == "MERGE"
serializer = BatchExportSerializer(
batch_export, data={"name": "Renamed export"}, partial=True, context=context
)
assert serializer.is_valid(), serializer.errors
- serializer.save()
- batch_export.refresh_from_db()
- assert batch_export.incremental_mode == BatchExport.IncrementalMode.MERGE
+ batch_export = serializer.save()
+ assert batch_exports_api.get_batch_export(self.team.pk, batch_export.id).incremental_mode == "MERGE"
serializer = BatchExportSerializer(
batch_export, data={"incremental_mode": "APPEND"}, partial=True, context=context
)
assert serializer.is_valid(), serializer.errors
- serializer.save()
- batch_export.refresh_from_db()
- assert batch_export.incremental_mode == BatchExport.IncrementalMode.APPEND
+ batch_export = serializer.save()
+ assert batch_exports_api.get_batch_export(self.team.pk, batch_export.id).incremental_mode == "APPEND"
assert BatchExportSerializer(batch_export).data["incremental_mode"] == "APPEND"
@parameterized.expand([("MERGE", False), ("APPEND", True)])
def test_patch_validates_keys_against_saved_export_mode(self, mode: str, valid: bool) -> None:
- batch_export = BatchExport(
- model=BatchExport.Model.HOGQL,
+ batch_export = make_configuration(
+ team_id=self.team.pk,
+ model="hogql",
incremental_mode=mode,
- source=BatchExportSource(team=self.team, hogql_query="SELECT uuid FROM events", primary_key=["uuid"]),
+ hogql_query="SELECT uuid FROM events",
+ primary_key=("uuid",),
)
serializer = BatchExportSerializer(
batch_export,
data={"primary_key": None},
partial=True,
- context={"get_team": lambda: self.team, "request": SimpleNamespace(user=self.user)},
+ context={
+ "team_id": self.team.pk,
+ "get_team": lambda: self.team,
+ "request": SimpleNamespace(user=self.user),
+ },
)
assert serializer.is_valid() is valid, serializer.errors
diff --git a/products/batch_exports/backend/tests/api/test_update.py b/products/batch_exports/backend/tests/api/test_update.py
index b29b6e6bdd6b..69d988e02c42 100644
--- a/products/batch_exports/backend/tests/api/test_update.py
+++ b/products/batch_exports/backend/tests/api/test_update.py
@@ -423,6 +423,42 @@ def test_can_patch_config(
"Higher frequency batch exports are not enabled for this team.",
id="Cannot update to high frequency batch exports without feature flag",
),
+ pytest.param(
+ {
+ "interval": "day",
+ "timezone": "Europe/Berlin",
+ "offset_day": None,
+ "offset_hour": 1,
+ },
+ {"offset_hour": 5},
+ {
+ "interval": "day",
+ "timezone": "Europe/Berlin",
+ "offset_day": None,
+ "offset_hour": 5,
+ "interval_offset": 5 * 3600,
+ },
+ None,
+ id="Patching only the offset of a daily export",
+ ),
+ pytest.param(
+ {
+ "interval": "week",
+ "timezone": "UTC",
+ "offset_day": 3,
+ "offset_hour": 2,
+ },
+ {"offset_hour": 5},
+ {
+ "interval": "week",
+ "timezone": "UTC",
+ "offset_day": 3, # the saved day stays
+ "offset_hour": 5,
+ "interval_offset": 3 * 86400 + 5 * 3600,
+ },
+ None,
+ id="Patching only the hour offset of a weekly export",
+ ),
],
)
def test_can_patch_schedule_configuration(
diff --git a/products/batch_exports/backend/tests/test_custom_events_schema.py b/products/batch_exports/backend/tests/test_custom_events_schema.py
new file mode 100644
index 000000000000..63413cfb299c
--- /dev/null
+++ b/products/batch_exports/backend/tests/test_custom_events_schema.py
@@ -0,0 +1,220 @@
+from typing import cast
+
+from posthog.test.base import BaseTest, materialized
+from unittest.mock import patch
+
+from django.test import override_settings
+
+from parameterized import parameterized
+
+from posthog.hogql import ast
+from posthog.hogql.property_access_types import RestrictedProperty
+
+from posthog.models import PropertyDefinition
+
+from products.batch_exports.backend.custom_events_schema import parse_events_hogql_query, serialize_events_query_schema
+from products.batch_exports.backend.facade.contracts import UnsupportedHogQLQueryError
+
+
+def prepare_query(query: str, team_id: int) -> ast.SelectQuery:
+ """Parse and resolve a HogQL query string into a prepared AST."""
+ return cast(ast.SelectQuery, parse_events_hogql_query(query, team_id=team_id, user=None))
+
+
+class TestSerializeEventsQuerySchema(BaseTest):
+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False)
+ def test_resaving_legacy_query_preserves_types_and_column_names(self):
+ PropertyDefinition.objects.create(
+ team=self.team, name="amount", type=PropertyDefinition.Type.EVENT, property_type="Numeric"
+ )
+ query = "SELECT round(e.properties.amount) AS rounded, lower(e.event) FROM events AS e"
+
+ schema = serialize_events_query_schema(self.team.pk, None, prepare_query(query, self.team.pk))
+ resaved = serialize_events_query_schema(self.team.pk, None, prepare_query(query, self.team.pk))
+
+ assert schema == resaved
+ assert [field["alias"] for field in schema["fields"]] == ["rounded", "`lower(e.event)`"]
+ assert schema["fields"][0]["expression"].startswith("round(accurateCastOrNull(")
+ assert schema["values"] == {"hogql_val_0": "amount", "hogql_val_1": "Float64"}
+ assert "toFloat(" in schema["hogql_query"]
+
+ @parameterized.expand(
+ [
+ (
+ "simple_fields_use_original_name_as_alias",
+ "SELECT event, person_id FROM events",
+ [
+ {"expression": "events.event", "alias": "event"},
+ {"expression": "events.person_id", "alias": "person_id"},
+ ],
+ {},
+ ),
+ (
+ "aliased_fields",
+ "SELECT event AS my_event, team_id AS my_team FROM events",
+ [
+ {"expression": "events.event", "alias": "my_event"},
+ {"expression": "events.team_id", "alias": "my_team"},
+ ],
+ {},
+ ),
+ (
+ "property_access_populates_values",
+ "SELECT properties.$browser AS browser FROM events",
+ [
+ {
+ "expression": "replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^\"|\"$', '')",
+ "alias": "browser",
+ },
+ ],
+ {"hogql_val_0": "$browser"},
+ ),
+ (
+ "mixed_simple_and_property_fields",
+ "SELECT event, properties.$browser AS browser, properties.custom AS custom, person_id FROM events",
+ [
+ {"expression": "events.event", "alias": "event"},
+ {
+ "expression": "replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^\"|\"$', '')",
+ "alias": "browser",
+ },
+ {
+ "expression": "replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^\"|\"$', '')",
+ "alias": "custom",
+ },
+ {"expression": "events.person_id", "alias": "person_id"},
+ ],
+ {"hogql_val_0": "$browser", "hogql_val_1": "custom"},
+ ),
+ ],
+ )
+ def test_serialize_hogql_query(self, _name, query, expected_fields, expected_values):
+ prepared = prepare_query(query, self.team.pk)
+
+ result = serialize_events_query_schema(self.team.pk, None, prepared)
+
+ assert result["fields"] == expected_fields
+ assert result["values"] == expected_values
+ assert "hogql_query" in result
+
+ @parameterized.expand(
+ [
+ ("integer", "SELECT 1 FROM events", "1", "`1`"),
+ ("string", "SELECT 'hello' FROM events", "%(hogql_val_0)s", "hello"),
+ ("float", "SELECT 3.14 FROM events", "3.14", "`3.14`"),
+ ("null", "SELECT null FROM events", "NULL", "NULL"),
+ ("boolean", "SELECT true FROM events", "1", "`1`"),
+ ],
+ )
+ def test_serialize_hogql_query_with_bare_constant(self, _name, query, expected_expression, expected_alias):
+ prepared = prepare_query(query, self.team.pk)
+
+ result = serialize_events_query_schema(self.team.pk, None, prepared)
+
+ assert len(result["fields"]) == 1
+ field = result["fields"][0]
+ assert field["expression"] == expected_expression
+ assert field["alias"] == expected_alias
+
+ def test_serialize_hogql_query_escapes_injected_alias(self):
+ """An alias containing SQL injection attempts is escaped with backticks."""
+ query = "SELECT uuid AS `x, (SELECT query FROM another_table LIMIT 100) AS leaked` FROM events"
+ prepared = prepare_query(query, self.team.pk)
+
+ result = serialize_events_query_schema(self.team.pk, None, prepared)
+
+ assert len(result["fields"]) == 1
+ field = result["fields"][0]
+ # The alias must be wrapped in backticks, keeping the malicious string as a single identifier
+ assert field["alias"] == "`x, (SELECT query FROM another_table LIMIT 100) AS leaked`"
+ assert field["expression"] == "events.uuid"
+
+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)
+ def test_native_query_keeps_property_type_casts(self):
+ PropertyDefinition.objects.create(
+ team=self.team, name="amount", type=PropertyDefinition.Type.EVENT, property_type="Numeric"
+ )
+ query = "SELECT e.properties.amount AS amount FROM events AS e"
+
+ schema = serialize_events_query_schema(self.team.pk, None, prepare_query(query, self.team.pk))
+
+ assert schema["fields"][0]["expression"].startswith("accurateCastOrNull(")
+ assert schema["values"]["hogql_val_1"] == "Float64"
+
+ @parameterized.expand([("unrestricted", False), ("restricted", True)])
+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False)
+ def test_materialized_columns_are_kept_out_of_a_query_that_cannot_be_recompiled(self, _name: str, restricted: bool):
+ restrictions = (
+ {RestrictedProperty(name="secret", property_type=PropertyDefinition.Type.EVENT)} if restricted else set()
+ )
+ with (
+ patch(
+ "products.access_control.backend.facade.api.get_restricted_properties_with_group_type_index_for_team",
+ return_value=restrictions,
+ ),
+ materialized("events", "$browser"),
+ ):
+ query = prepare_query("SELECT properties.$browser AS browser FROM events", self.team.pk)
+
+ schema = serialize_events_query_schema(self.team.pk, self.user, query)
+
+ expression = schema["fields"][0]["expression"]
+ if restricted:
+ assert "hogql_query" not in schema
+ assert "mat_" not in expression
+ else:
+ assert "mat_$browser" in expression
+
+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)
+ @patch("products.access_control.backend.facade.api.get_restricted_properties_with_group_type_index_for_team")
+ @patch("posthog.models.event.new_events_schema.use_new_events_schema", return_value=True)
+ def test_restricted_property_query_is_not_recompiled_without_user(
+ self, _use_new_events_schema, get_restricted_properties
+ ):
+ get_restricted_properties.return_value = {
+ RestrictedProperty(name="secret", property_type=PropertyDefinition.Type.EVENT)
+ }
+ query = prepare_query("SELECT properties.secret AS secret FROM events", self.team.pk)
+
+ schema = serialize_events_query_schema(self.team.pk, self.user, query)
+
+ assert schema["fields"] == [{"expression": "NULL", "alias": "secret"}]
+ assert "hogql_query" not in schema
+
+ with self.assertRaises(UnsupportedHogQLQueryError):
+ serialize_events_query_schema(
+ self.team.pk, self.user, prepare_query("SELECT properties FROM events", self.team.pk)
+ )
+
+ @parameterized.expand(
+ [
+ ("restricted", "SELECT properties.`$feature/secret` AS flag FROM events", "NULL"),
+ ("visible", "SELECT properties.`$feature/checkout` AS flag FROM events", "JSONExtractRaw("),
+ ]
+ )
+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)
+ @patch("products.access_control.backend.facade.api.get_restricted_properties_with_group_type_index_for_team")
+ @patch("posthog.models.event.new_events_schema.use_new_events_schema", return_value=True)
+ def test_native_export_hides_a_restricted_flag(
+ self, _name: str, query: str, expected: str, _use_new_events_schema, get_restricted_properties
+ ):
+ get_restricted_properties.return_value = {
+ RestrictedProperty(name="$feature/secret", property_type=PropertyDefinition.Type.EVENT)
+ }
+
+ schema = serialize_events_query_schema(self.team.pk, self.user, prepare_query(query, self.team.pk))
+
+ expression = schema["fields"][0]["expression"]
+ assert expression == expected if expected == "NULL" else expected in expression, expression
+
+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)
+ @patch("posthog.models.event.new_events_schema.use_new_events_schema", return_value=True)
+ def test_native_export_names_a_bare_flag_column(self, _use_new_events_schema):
+
+ schema = serialize_events_query_schema(
+ self.team.pk, self.user, prepare_query("SELECT properties.`$feature/checkout` FROM events", self.team.pk)
+ )
+
+ field = schema["fields"][0]
+ assert field["alias"] == "`$feature/checkout`", field
+ assert "JSONExtractRaw(" in field["expression"], field
diff --git a/pyproject.toml b/pyproject.toml
index 937494955c87..9943c8dfaaa6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -762,14 +762,6 @@ ignore_imports = [
"products.access_control.backend.presentation.access_control_settings -> products.access_control.backend.models.access_control",
"products.access_control.backend.presentation.access_control_settings -> products.access_control.backend.models.role",
"products.access_control.backend.presentation.role -> products.access_control.backend.models.role",
- # TODO: batch_exports presentation wave — the moved viewsets, serializers and destination
- # test helpers still reach in-product internals directly. Thin each one to parse -> facade
- # -> serialize and delete each entry. backend:contract-check stays off until this list is
- # empty. Only delete entries here: a new import from these views goes through the facade.
- "products.batch_exports.backend.presentation.views.batch_export.destinations -> products.batch_exports.backend.models.batch_export",
- "products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.hogql_source",
- "products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.models.batch_export",
- "products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.service",
]
# routes.py is the one module core reads to assemble a product's API router, so it is the