Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(batch-exports): serve runs and backfills through the facade (#118663)

Ross0d340a050773
diff --git a/posthog/test/repo_invariants/test_pagination_contract.py b/posthog/test/repo_invariants/test_pagination_contract.py
index e1350df82abf..0c685aaacd28 100644
--- a/posthog/test/repo_invariants/test_pagination_contract.py
+++ b/posthog/test/repo_invariants/test_pagination_contract.py
@@ -102,6 +102,7 @@ EXISTING_CUSTOM_LIST_VIEWSETS = {
"products.autoresearch.backend.presentation.views.views.AutoresearchRunViewSet",
"products.autoresearch.backend.presentation.views.views.AutoresearchSuggestionViewSet",
"products.autoresearch.backend.presentation.views.views.AutoresearchTrainingRunViewSet",
+ "products.batch_exports.backend.presentation.views.batch_export.backfills.BatchExportBackfillViewSet",
"products.batch_exports.backend.presentation.views.batch_export.runs.BatchExportRunViewSet",
"products.batch_exports.backend.presentation.views.file_download.FileDownloadBatchExportOnDemandViewSet",
# Lists at most MAX_LIBRARIES_PER_TEAM (100) libraries, so it has no pages.
diff --git a/posthog/test/repo_invariants/test_serializer_timestamp_guard.py b/posthog/test/repo_invariants/test_serializer_timestamp_guard.py
index ee8c08b65146..8e0ba0ee5921 100644
--- a/posthog/test/repo_invariants/test_serializer_timestamp_guard.py
+++ b/posthog/test/repo_invariants/test_serializer_timestamp_guard.py
@@ -100,16 +100,9 @@ REVIEWED_WRITABLE: dict[str, str] = {
"products.actions.backend.api.action.ActionSerializer.pinned_at": "TODO: server-derived on pin. update() discards the supplied value and stores its own now(); only the null that unpins is the client's",
"products.alerts.backend.presentation.views.alert.AlertSerializer.snoozed_until": "TODO: server-derived. The client sends a relative date string and update() resolves it to the stored instant",
"products.annotations.backend.api.annotation.AnnotationSerializer.date_marker": "Client sets when the annotation happened, to place it on a chart",
- "products.batch_exports.backend.presentation.views.batch_export.backfills.BatchExportBackfillSerializer.adjusted_start_at": "TODO: server-owned, the backfill derives it from the earliest available data",
- "products.batch_exports.backend.presentation.views.batch_export.backfills.BatchExportBackfillSerializer.end_at": "Client requests the data interval to backfill",
- "products.batch_exports.backend.presentation.views.batch_export.backfills.BatchExportBackfillSerializer.finished_at": "TODO: server-owned, written by the backfill workflow when it ends",
- "products.batch_exports.backend.presentation.views.batch_export.backfills.BatchExportBackfillSerializer.start_at": "Client requests the data interval to backfill",
"products.batch_exports.backend.presentation.views.batch_export.exports.BatchExportSerializer.end_at": "Client-supplied bound, runs after it are not triggered",
"products.batch_exports.backend.presentation.views.batch_export.exports.BatchExportSerializer.last_paused_at": "TODO: written by the pause action, make read-only",
"products.batch_exports.backend.presentation.views.batch_export.exports.BatchExportSerializer.start_at": "Client-supplied bound, runs before it are not triggered",
- "products.batch_exports.backend.presentation.views.batch_export.runs.BatchExportRunSerializer.data_interval_end": "Response shape for the POST cancel and retry actions, which take no request body",
- "products.batch_exports.backend.presentation.views.batch_export.runs.BatchExportRunSerializer.data_interval_start": "Response shape for the POST cancel and retry actions, which take no request body",
- "products.batch_exports.backend.presentation.views.batch_export.runs.BatchExportRunSerializer.finished_at": "Response shape for the POST cancel and retry actions, which take no request body",
"products.billing_alerts.backend.presentation.serializers.BillingAlertConfigurationSerializer.snoozed_until": "Client-set instant: a plain DateTimeField that ModelSerializer writes to the column unchanged, with no relative-duration parsing on this path",
"products.conversations.backend.api.tickets.TicketSerializer.sla_due_at": "Client or a workflow sets the SLA deadline, and null clears it",
"products.conversations.backend.api.tickets.TicketSerializer.snoozed_until": "Client-set instant: the ticket action assigns validated_data['snoozed_until'] to ticket.snoozed_until, with no relative-duration parsing on this path",
diff --git a/products/batch_exports/backend/facade/api.py b/products/batch_exports/backend/facade/api.py
index c845e4947277..1d3c4ef824ef 100644
--- a/products/batch_exports/backend/facade/api.py
+++ b/products/batch_exports/backend/facade/api.py
@@ -11,8 +11,8 @@ is visible in one place.
and ``temporalio`` on the ``django.setup()`` path. Core config-time consumers import this
facade, so it reaches the service only through ``_service()``, which imports it at call
time. For the same reason the Temporal client is built in ``_get_temporal_client()``, and
-the HogQL and file download modules load through ``_import_hogql_source()`` and
-``_import_file_downloads()``.
+the HogQL, file download, run and backfill modules load through ``_import_hogql_source()``,
+``_import_file_downloads()``, ``_import_runs()`` and ``_import_backfills()``.
Temporal workflow and activity registration crosses as objects, not data, so it lives in
``facade/temporal.py``. The destination clients the warehouse writers reuse live in
@@ -56,17 +56,22 @@ __all__ = [
"FAILED_BACKFILL_STATUSES",
"MultipleBatchExportsError",
"backfill_batch_export",
+ "cancel_batch_export_backfill",
+ "cancel_batch_export_run",
"cancel_file_download",
"count_batch_exports_for_teams",
"count_hogql_rows",
"create_batch_export",
+ "create_batch_export_backfill",
"create_workflows_backfill_export",
"delete_batch_export",
"delete_batch_exports_for_teams",
"describe_destination_test",
"drop_unknown_hogql_modifiers",
"find_hogql_interval_placeholders",
+ "get_batch_export_backfill",
"get_batch_export_by_name",
+ "get_batch_export_run",
"get_file_download_run",
"get_file_download_url",
"get_latest_completed_run",
@@ -75,10 +80,13 @@ __all__ = [
"get_teams_with_active_batch_exports",
"get_teams_with_billable_rows_exported",
"list_backfills_for_export",
+ "list_batch_export_backfills",
+ "list_batch_export_runs",
"list_batch_exports_using_integration",
"list_file_download_runs",
"list_latest_failed_runs",
"list_supported_intervals",
+ "retry_batch_export_run",
"run_destination_test_step",
"start_file_download",
"validate_hogql_query",
@@ -134,6 +142,20 @@ def _import_file_downloads() -> ModuleType:
return file_downloads
+def _import_runs() -> ModuleType:
+ """Import the run module at call time. See the module docstring for why."""
+ from products.batch_exports.backend.logic import runs
+
+ return runs
+
+
+def _import_backfills() -> ModuleType:
+ """Import the backfill module at call time. See the module docstring for why."""
+ from products.batch_exports.backend.logic import backfills
+
+ return backfills
+
+
def _get_temporal_client() -> "Client":
"""Connect to Temporal at call time. See the module docstring for why."""
from posthog.temporal.common.client import sync_connect
@@ -597,3 +619,63 @@ def get_file_download_url(team_id: int, run_id: UUID, part: str | None) -> str:
``contracts.FileDownloadNotFoundError`` when the run or the file does not exist.
"""
return _import_file_downloads().get_file_download_url(team_id, run_id, part)
+
+
+def list_batch_export_runs(
+ team_id: int, export_id: UUID, *, filters: contracts.BatchExportRunFilters, page: contracts.ListPage
+) -> list[contracts.BatchExportRunDetail]:
+ """Return one page of an export's runs that match ``filters``, ordered by ``created_at`` or ``data_interval_start``."""
+ return _import_runs().list_runs(team_id, export_id, filters, page)
+
+
+def get_batch_export_run(
+ team_id: int, export_id: UUID, run_id: UUID, *, filters: contracts.BatchExportRunFilters
+) -> contracts.BatchExportRunDetail:
+ """Raises ``contracts.BatchExportRunNotFoundError`` unless the team has this run and it matches ``filters``."""
+ return _import_runs().get_run(team_id, export_id, run_id, filters)
+
+
+def retry_batch_export_run(
+ team_id: int, export_id: UUID, run_id: UUID, *, filters: contracts.BatchExportRunFilters
+) -> str:
+ """Backfill a run's data interval again and return the backfill id. Raises as ``get_batch_export_run`` does."""
+ return _import_runs().retry_run(team_id, export_id, run_id, filters)
+
+
+def cancel_batch_export_run(
+ team_id: int, export_id: UUID, run_id: UUID, *, filters: contracts.BatchExportRunFilters
+) -> None:
+ """Cancel a starting or running run. Raises ``contracts.InvalidBatchExportRunRequestError`` in any other status."""
+ _import_runs().cancel_run(team_id, export_id, run_id, filters)
+
+
+def list_batch_export_backfills(
+ team_id: int, export_id: UUID, *, page: contracts.ListPage
+) -> list[contracts.BatchExportBackfillDetail]:
+ """Return one page of an export's backfills, ordered by ``created_at`` or ``start_at``."""
+ return _import_backfills().list_backfills(team_id, export_id, page)
+
+
+def get_batch_export_backfill(team_id: int, export_id: UUID, backfill_id: UUID) -> contracts.BatchExportBackfillDetail:
+ """Raises ``contracts.BatchExportBackfillNotFoundError`` unless the team has this backfill."""
+ return _import_backfills().get_backfill(team_id, export_id, backfill_id)
+
+
+def create_batch_export_backfill(
+ team_id: int, export_id: UUID, *, start_at: object, end_at: object
+) -> contracts.BatchExportBackfillStart:
+ """Start a backfill. ``start_at`` and ``end_at`` arrive as the request sent them.
+
+ Daily and weekly exports take ISO 8601 dates, other intervals ISO 8601 datetimes with a time zone.
+ Raises ``contracts.BatchExportNotFoundError``, ``contracts.InvalidBackfillRequestError`` or
+ ``contracts.TooManyConcurrentBackfillsError``.
+ """
+ return _import_backfills().create_backfill(team_id, export_id, start_at, end_at)
+
+
+def cancel_batch_export_backfill(team_id: int, export_id: UUID, backfill_id: UUID) -> None:
+ """Cancel a starting or running backfill and its runs in progress.
+
+ Raises ``contracts.InvalidBackfillRequestError`` in any other status.
+ """
+ _import_backfills().cancel_backfill(team_id, export_id, backfill_id)
diff --git a/products/batch_exports/backend/facade/contracts.py b/products/batch_exports/backend/facade/contracts.py
index 43f6ba3dd7cc..3a3a5f44c352 100644
--- a/products/batch_exports/backend/facade/contracts.py
+++ b/products/batch_exports/backend/facade/contracts.py
@@ -60,6 +60,33 @@ class FileDownloadStartError(Exception):
"""Raised when the workflow of a new file download does not start. The run is saved as failed."""
+class BatchExportNotFoundError(LookupError):
+ """Raised when the team has no batch export with the id a request names."""
+
+
+class BatchExportRunNotFoundError(LookupError):
+ """Raised when a batch export of the team has no run with the id a request names."""
+
+
+class BatchExportBackfillNotFoundError(LookupError):
+ """Raised when a batch export of the team has no backfill with the id a request names."""
+
+
+class InvalidBatchExportRunRequestError(ValueError):
+ """Raised when a run cannot do what was asked, for example cancel a run that has finished."""
+
+
+class InvalidBackfillRequestError(ValueError):
+ """Raised when a backfill cannot do what was asked, for example start over a range that is not valid.
+
+ The message says what is wrong with the request.
+ """
+
+
+class TooManyConcurrentBackfillsError(Exception):
+ """Raised when a team already has the maximum number of backfills starting or running."""
+
+
@dataclass(frozen=True)
class BatchExportRef:
"""The bare identity of a batch export, for listing it by name."""
@@ -208,6 +235,122 @@ class FileDownloadRun:
records_completed: int | None
+# The fields a page of runs or backfills can be ordered by, as the API's `ordering` parameter names them.
+BATCH_EXPORT_RUN_ORDERING_FIELDS = ("created_at", "data_interval_start")
+BATCH_EXPORT_BACKFILL_ORDERING_FIELDS = ("created_at", "start_at")
+
+
+@dataclass(frozen=True)
+class ListPageBound:
+ """Where a page of an ordered list starts.
+
+ Only the items whose first ordering field sorts before ``value`` (when ``before``) or after
+ it (otherwise) are on the page. ``value`` is the field's value as text, as an API cursor carries it.
+ """
+
+ value: str
+ before: bool
+
+
+@dataclass(frozen=True)
+class ListPage:
+ """One page of an ordered list.
+
+ ``ordering`` names the fields to sort by, most significant first. A leading ``-`` sorts that
+ field in descending order. The page skips ``offset`` items past ``bound`` and holds at most
+ ``limit`` items.
+ """
+
+ ordering: tuple[str, ...]
+ bound: ListPageBound | None
+ offset: int
+ limit: int
+
+
+@dataclass(frozen=True)
+class DatetimeRange:
+ """A range of time that includes both of its ends."""
+
+ start: dt.datetime
+ end: dt.datetime
+
+
+@dataclass(frozen=True)
+class BatchExportRunFilters:
+ """Which runs of a batch export to list. A filter left empty does not apply.
+
+ ``data_interval`` keeps the runs whose whole data interval is inside the range.
+ """
+
+ statuses: tuple[str, ...] = ()
+ created: DatetimeRange | None = None
+ data_interval: DatetimeRange | None = None
+
+
+@dataclass(frozen=True)
+class BatchExportRunDetail:
+ """One run of a batch export, with every field the API shows.
+
+ A run belongs to a scheduled batch export or to an on-demand one, never to both.
+ """
+
+ id: UUID
+ status: str
+ records_completed: int | None
+ records_failed: int | None
+ latest_error: str | None
+ data_interval_start: dt.datetime | None
+ data_interval_end: dt.datetime | None
+ cursor: str | None
+ created_at: dt.datetime
+ finished_at: dt.datetime | None
+ last_updated_at: dt.datetime
+ records_total_count: int | None
+ bytes_exported: int | None
+ batch_export_id: UUID | None
+ batch_export_on_demand_id: UUID | None
+ backfill_id: UUID | None
+
+
+@dataclass(frozen=True)
+class BatchExportBackfillProgress:
+ """How far a backfill has got. ``progress`` is the finished share of the runs, from 0 to 1."""
+
+ total_runs: int | None
+ finished_runs: int | None
+ progress: float | None
+
+
+@dataclass(frozen=True)
+class BatchExportBackfillDetail:
+ """One backfill of a batch export, with every field the API shows.
+
+ ``progress`` is None when it has no meaning: the backfill did not complete and is not
+ running, or it is one run over all data.
+ """
+
+ id: UUID
+ team_id: int
+ batch_export_id: UUID
+ start_at: dt.datetime | None
+ end_at: dt.datetime | None
+ status: str
+ created_at: dt.datetime
+ finished_at: dt.datetime | None
+ last_updated_at: dt.datetime
+ total_records_count: int | None
+ adjusted_start_at: dt.datetime | None
+ progress: BatchExportBackfillProgress | None
+
+
+@dataclass(frozen=True)
+class BatchExportBackfillStart:
+ """A backfill that started, and the destination type of the batch export it fills."""
+
+ backfill_id: str
+ destination_type: str
+
+
@stdlib_dataclass(frozen=True)
class AWSCredentials:
"""An AWS key pair, optionally temporary.
diff --git a/products/batch_exports/backend/logic/backfills.py b/products/batch_exports/backend/logic/backfills.py
new file mode 100644
index 000000000000..b8039c9696f2
--- /dev/null
+++ b/products/batch_exports/backend/logic/backfills.py
@@ -0,0 +1,304 @@
+"""Backfills of a scheduled batch export: start one, list them, look one up or cancel it.
+
+The facade reaches this module through a call-time import, because it imports the service layer,
+the Temporal client and the HogQL parser, none of which may load on the ``django.setup()`` path.
+"""
+
+import uuid
+import datetime as dt
+from typing import Any
+from uuid import UUID
+
+from django.conf import settings
+from django.db.models import QuerySet
+
+import structlog
+import posthoganalytics
+
+from posthog.models import Team
+from posthog.temporal.common.client import sync_connect
+
+from products.batch_exports.backend.facade import contracts
+from products.batch_exports.backend.hogql_source import (
+ DATA_INTERVAL_START_PLACEHOLDER,
+ UnsupportedHogQLQueryError,
+ find_interval_placeholders,
+ parse_hogql_select_for_batch_export,
+)
+from products.batch_exports.backend.logic.pagination import select_page
+from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportBackfill
+from products.batch_exports.backend.service import (
+ BatchExportWithNoEndNotAllowedError,
+ backfill_export,
+ sync_cancel_running_batch_export_backfill,
+)
+
+LOGGER = structlog.get_logger(__name__)
+_ACTIVE_STATUSES = (BatchExportBackfill.Status.STARTING, BatchExportBackfill.Status.RUNNING)
+
+
+def validate_date_input(date_input: Any, batch_export: BatchExport) -> dt.datetime:
+ """Validate and parse a date/datetime input as a proper dt.datetime.
+
+ If the interval is daily or weekly, we expect the input to be an ISO formatted date string.
+ We then need to convert it to a datetime using the batch export's timezone and offset.
+
+ For all other intervals, we expect the input to be an ISO formatted datetime string.
+
+ Args:
+ date_input: The datetime input to parse.
+
+ Raises:
+ InvalidBackfillRequestError: If the input cannot be parsed.
+
+ Returns:
+ The parsed dt.datetime.
+ """
+ if batch_export.interval == "day" or batch_export.interval == "week":
+ try:
+ parsed_date = dt.date.fromisoformat(date_input)
+ except (TypeError, ValueError):
+ # Try to parse as a datetime string so we can give a more helpful error message
+ try:
+ dt.datetime.fromisoformat(date_input)
+ except (TypeError, ValueError):
+ raise contracts.InvalidBackfillRequestError(f"Input '{date_input}' is not a valid ISO formatted date.")
+ raise contracts.InvalidBackfillRequestError(
+ f"Input '{date_input}' is not a valid ISO formatted date. "
+ "Daily or weekly batch export backfills expect only the date component, but a time was included."
+ )
+
+ if batch_export.interval == "week":
+ # Validate that the provided date is on the correct day of the week, according to the batch export's day offset
+ # Python's date.isoweekday() returns 1-7 for Monday-Sunday, so we need to convert it to 0-6 for Sunday-Saturday
+ normalized_day = parsed_date.isoweekday() % 7
+ if normalized_day != batch_export.offset_day:
+ # get day of week as string
+ day_of_week = parsed_date.strftime("%A")
+ expected_day_of_week = batch_export.offset_day_name
+ assert expected_day_of_week is not None
+ raise contracts.InvalidBackfillRequestError(
+ f"Input {date_input} is not on the correct day of the week for this batch export. "
+ f"{date_input} is a {day_of_week} but this batch export is configured to run "
+ f"weekly on {expected_day_of_week}."
+ )
+
+ # If we have an offset hour, add it to the parsed datetime
+ # Also, apply the timezone to the parsed datetime
+ time_of_day = dt.time.min if batch_export.offset_hour is None else dt.time(hour=batch_export.offset_hour)
+ parsed = dt.datetime.combine(parsed_date, time_of_day).replace(tzinfo=batch_export.timezone_info)
+
+ else:
+ try:
+ parsed = dt.datetime.fromisoformat(date_input)
+ except (TypeError, ValueError):
+ raise contracts.InvalidBackfillRequestError(f"Input {date_input} is not a valid ISO formatted datetime.")
+
+ if parsed.tzinfo is None:
+ raise contracts.InvalidBackfillRequestError(f"Input {date_input} is naive.")
+
+ return parsed
+
+
+def _get_progress(backfill: BatchExportBackfill) -> contracts.BatchExportBackfillProgress | None:
+ """Return progress information containing total runs, finished runs, and progress percentage.
+
+ To reduce the number of database calls we make (which could be expensive when fetching a list of backfills) we
+ only get the list of completed runs from the DB if the backfill is still running.
+ """
+ if backfill.status == BatchExportBackfill.Status.COMPLETED:
+ return contracts.BatchExportBackfillProgress(
+ total_runs=backfill.total_expected_runs, finished_runs=backfill.total_expected_runs, progress=1.0
+ )
+ elif backfill.status not in _ACTIVE_STATUSES:
+ # if backfill finished in some other state then progress info may not be meaningful
+ return None
+
+ total_runs = backfill.total_expected_runs
+ if not total_runs:
+ return None
+
+ if backfill.start_at is None and backfill.adjusted_start_at is None:
+ # if it's just a single run, backfilling from the beginning of time, we can't calculate progress based on
+ # the number of completed runs so better to return None
+ return None
+
+ finished_runs = backfill.get_finished_runs()
+ # just make sure we never return a progress > 1
+ total_runs = max(total_runs, finished_runs)
+ return contracts.BatchExportBackfillProgress(
+ total_runs=total_runs, finished_runs=finished_runs, progress=round(finished_runs / total_runs, ndigits=1)
+ )
+
+
+def _to_backfill_detail(backfill: BatchExportBackfill) -> contracts.BatchExportBackfillDetail:
+ return contracts.BatchExportBackfillDetail(
+ id=backfill.id,
+ team_id=backfill.team_id,
+ batch_export_id=backfill.batch_export_id,
+ start_at=backfill.start_at,
+ end_at=backfill.end_at,
+ status=backfill.status,
+ created_at=backfill.created_at,
+ finished_at=backfill.finished_at,
+ last_updated_at=backfill.last_updated_at,
+ total_records_count=backfill.total_records_count,
+ adjusted_start_at=backfill.adjusted_start_at,
+ progress=_get_progress(backfill),
+ )
+
+
+def _query_export_backfills(team_id: int, export_id: UUID) -> QuerySet[BatchExportBackfill]:
+ # The progress of a backfill reads the interval of its batch export.
+ return BatchExportBackfill.objects.filter(batch_export_id=export_id, batch_export__team_id=team_id).select_related(
+ "batch_export"
+ )
+
+
+def _get_backfill(team_id: int, export_id: UUID, backfill_id: UUID) -> BatchExportBackfill:
+ try:
+ return _query_export_backfills(team_id, export_id).get(id=backfill_id)
+ except BatchExportBackfill.DoesNotExist as e:
+ raise contracts.BatchExportBackfillNotFoundError() from e
+
+
+def list_backfills(
+ team_id: int, export_id: UUID, page: contracts.ListPage
+) -> list[contracts.BatchExportBackfillDetail]:
+ backfills = select_page(
+ _query_export_backfills(team_id, export_id),
+ page,
+ ordering_fields=contracts.BATCH_EXPORT_BACKFILL_ORDERING_FIELDS,
+ )
+ return [_to_backfill_detail(backfill) for backfill in backfills]
+
+
+def get_backfill(team_id: int, export_id: UUID, backfill_id: UUID) -> contracts.BatchExportBackfillDetail:
+ return _to_backfill_detail(_get_backfill(team_id, export_id, backfill_id))
+
+
+def _check_earliest_backfill_allowed(team_id: int, batch_export: BatchExport) -> None:
+ """Check that a backfill from the beginning of time can run for this team and batch export."""
+ # Currently, backfills from the beginning of time usually fail due to us hitting ClickHouse memory limits.
+ # Therefore, this feature is behind a feature flag while we improve backfilling behavior.
+ team = Team.objects.select_related("organization").get(id=team_id)
+ if not posthoganalytics.feature_enabled(
+ "batch-export-earliest-backfill",
+ str(team.uuid),
+ groups={"organization": str(team.organization.id)},
+ group_properties={
+ "organization": {
+ "id": str(team.organization.id),
+ "created_at": team.organization.created_at,
+ }
+ },
+ send_feature_flag_events=False,
+ ):
+ raise contracts.InvalidBackfillRequestError(
+ "Backfilling from the beginning of time is not enabled for this team."
+ )
+
+ if batch_export.model == BatchExport.Model.HOGQL and (hogql_query := batch_export.hogql_query) is not None:
+ try:
+ parsed = parse_hogql_select_for_batch_export(hogql_query)
+ except UnsupportedHogQLQueryError as e:
+ raise contracts.InvalidBackfillRequestError(str(e)) from e
+ if DATA_INTERVAL_START_PLACEHOLDER in find_interval_placeholders(parsed):
+ # TODO: We should maybe support beginning-of-time backfills with this placeholder.
+ raise contracts.InvalidBackfillRequestError(
+ "This query references {data_interval_start}, which is unavailable when backfilling from the "
+ "beginning of time. Provide 'start_at' or remove {data_interval_start} from the query."
+ )
+
+
+def create_backfill(
+ team_id: int, export_id: UUID, start_at_input: Any, end_at_input: Any
+) -> contracts.BatchExportBackfillStart:
+ """Start a backfill of a batch export.
+
+ Args:
+ team_id: The team creating the backfill
+ export_id: The batch export to backfill
+ start_at_input: ISO formatted datetime string for backfill start
+ end_at_input: ISO formatted datetime string for backfill end
+ """
+ try:
+ batch_export = BatchExport.objects.select_related("destination").get(id=export_id, team_id=team_id)
+ except BatchExport.DoesNotExist as e:
+ raise contracts.BatchExportNotFoundError() from e
+
+ if start_at_input is None:
+ _check_earliest_backfill_allowed(team_id, batch_export)
+
+ concurrency_limit = settings.BATCH_EXPORT_MAX_CONCURRENT_BACKFILLS_PER_TEAM
+ active_backfills = BatchExportBackfill.objects.filter(team_id=team_id, status__in=_ACTIVE_STATUSES).count()
+ if active_backfills >= concurrency_limit:
+ raise contracts.TooManyConcurrentBackfillsError(
+ f"This team already has {concurrency_limit} batch export backfills running. "
+ f"Wait for some to finish or cancel them before creating more."
+ )
+
+ temporal = sync_connect()
+
+ if start_at_input is not None:
+ start_at = validate_date_input(start_at_input, batch_export)
+ else:
+ start_at = None
+
+ if end_at_input is not None:
+ end_at = validate_date_input(end_at_input, batch_export)
+ else:
+ end_at = None
+
+ # Note: earliest backfill date validation and adjustment is now done in the Temporal workflow
+ # via the get_backfill_info activity. This allows the potentially slow ClickHouse query to run
+ # asynchronously rather than blocking the HTTP request.
+
+ backfill_id = str(uuid.uuid4())
+ started = contracts.BatchExportBackfillStart(
+ backfill_id=backfill_id, destination_type=batch_export.destination.type
+ )
+
+ if start_at is None or end_at is None:
+ backfill_export(
+ temporal=temporal,
+ batch_export_id=str(batch_export.pk),
+ team_id=team_id,
+ start_at=start_at,
+ end_at=end_at,
+ backfill_id=backfill_id,
+ )
+ return started
+
+ if start_at >= end_at:
+ raise contracts.InvalidBackfillRequestError("The initial backfill datetime 'start_at' must be before 'end_at'")
+ if end_at > dt.datetime.now(dt.UTC):
+ raise contracts.InvalidBackfillRequestError(f"The provided 'end_at' ({end_at.isoformat()}) is in the future")
+
+ try:
+ backfill_export(
+ temporal=temporal,
+ batch_export_id=str(batch_export.pk),
+ team_id=team_id,
+ start_at=start_at,
+ end_at=end_at,
+ backfill_id=backfill_id,
+ )
+ return started
+ except BatchExportWithNoEndNotAllowedError:
+ raise contracts.InvalidBackfillRequestError("Backfilling a BatchExport with no end date is not allowed")
+
+
+def cancel_backfill(team_id: int, export_id: UUID, backfill_id: UUID) -> None:
+ backfill = _get_backfill(team_id, export_id, backfill_id)
+ if backfill.status not in _ACTIVE_STATUSES:
+ raise contracts.InvalidBackfillRequestError(f"Cannot cancel a backfill that is in '{backfill.status}' status")
+
+ temporal = sync_connect()
+ try:
+ sync_cancel_running_batch_export_backfill(temporal, backfill)
+ except Exception as e:
+ # It could be the case that the backfill is already cancelled but our database hasn't been updated yet.
+ # In this case, we can just ignore the error but log it for visibility (in case there is an actual
+ # issue).
+ LOGGER.warning("Error cancelling batch export backfill: %s", e)
diff --git a/products/batch_exports/backend/logic/pagination.py b/products/batch_exports/backend/logic/pagination.py
new file mode 100644
index 000000000000..8ebdedebb536
--- /dev/null
+++ b/products/batch_exports/backend/logic/pagination.py
@@ -0,0 +1,35 @@
+"""Read one page of an ordered list of rows, as ``contracts.ListPage`` asks for it."""
+
+from collections.abc import Collection
+
+from django.db.models import Model, QuerySet
+
+from posthog.api.pagination import stable_queryset_ordering
+
+from products.batch_exports.backend.facade import contracts
+
+
+# TODO: Nothing here is specific to batch exports. Move it to posthog/api/pagination.py with the cursor
+# page source in presentation/views/batch_export/pagination.py, and a core page type to replace
+# contracts.ListPage, so other facade-backed products can page with a cursor too.
+def select_page[M: Model](
+ queryset: QuerySet[M], page: contracts.ListPage, *, ordering_fields: Collection[str]
+) -> list[M]:
+ """Return the rows of one page of ``queryset``.
+
+ The rows are also sorted by primary key, in the direction of the first ordering field. Without
+ it, rows that share a value can change places between two queries, so a page boundary between
+ them can skip or repeat a row.
+ """
+ for term in page.ordering:
+ if term.lstrip("-") not in ordering_fields:
+ raise ValueError(f"Cannot order by {term!r}")
+
+ first = page.ordering[0]
+ queryset = stable_queryset_ordering(queryset.order_by(*page.ordering))
+
+ if page.bound is not None:
+ comparison = "lt" if page.bound.before else "gt"
+ queryset = queryset.filter(**{f"{first.lstrip('-')}__{comparison}": page.bound.value})
+
+ return list(queryset[page.offset : page.offset + page.limit])
diff --git a/products/batch_exports/backend/logic/runs.py b/products/batch_exports/backend/logic/runs.py
new file mode 100644
index 000000000000..dce1acd651cf
--- /dev/null
+++ b/products/batch_exports/backend/logic/runs.py
@@ -0,0 +1,108 @@
+"""Runs of a scheduled batch export: list them, look one up, retry it or cancel it.
+
+The facade reaches this module through a call-time import, because it imports the service layer
+and the Temporal client, neither of which may load on the ``django.setup()`` path.
+"""
+
+from uuid import UUID
+
+from django.db.models import QuerySet
+
+import structlog
+
+from posthog.temporal.common.client import sync_connect
+
+from products.batch_exports.backend.facade import contracts
+from products.batch_exports.backend.logic.pagination import select_page
+from products.batch_exports.backend.models.batch_export import BatchExportRun
+from products.batch_exports.backend.service import backfill_export, cancel_running_batch_export_run
+
+LOGGER = structlog.get_logger(__name__)
+_CANCELLABLE_STATUSES = (BatchExportRun.Status.RUNNING, BatchExportRun.Status.STARTING)
+
+
+def _to_run_detail(run: BatchExportRun) -> contracts.BatchExportRunDetail:
+ return contracts.BatchExportRunDetail(
+ id=run.id,
+ status=run.status,
+ records_completed=run.records_completed,
+ records_failed=run.records_failed,
+ latest_error=run.latest_error,
+ data_interval_start=run.data_interval_start,
+ data_interval_end=run.data_interval_end,
+ cursor=run.cursor,
+ created_at=run.created_at,
+ finished_at=run.finished_at,
+ last_updated_at=run.last_updated_at,
+ records_total_count=run.records_total_count,
+ bytes_exported=run.bytes_exported,
+ batch_export_id=run.batch_export_id,
+ batch_export_on_demand_id=run.batch_export_on_demand_id,
+ backfill_id=run.backfill_id,
+ )
+
+
+def _query_export_runs(team_id: int, export_id: UUID) -> QuerySet[BatchExportRun]:
+ return BatchExportRun.objects.filter(batch_export_id=export_id, batch_export__team_id=team_id)
+
+
+def _filter_runs(runs: QuerySet[BatchExportRun], filters: contracts.BatchExportRunFilters) -> QuerySet[BatchExportRun]:
+ if filters.created is not None:
+ runs = runs.filter(created_at__range=(filters.created.start, filters.created.end))
+ if filters.data_interval is not None:
+ runs = runs.filter(
+ data_interval_start__gte=filters.data_interval.start, data_interval_end__lte=filters.data_interval.end
+ )
+ if filters.statuses:
+ runs = runs.filter(status__in=filters.statuses)
+ return runs
+
+
+def _get_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.BatchExportRunFilters) -> BatchExportRun:
+ runs = _filter_runs(_query_export_runs(team_id, export_id), filters)
+ try:
+ return runs.select_related("batch_export").get(id=run_id)
+ except BatchExportRun.DoesNotExist as e:
+ raise contracts.BatchExportRunNotFoundError() from e
+
+
+def list_runs(
+ team_id: int, export_id: UUID, filters: contracts.BatchExportRunFilters, page: contracts.ListPage
+) -> list[contracts.BatchExportRunDetail]:
+ runs = _filter_runs(_query_export_runs(team_id, export_id), filters)
+ return [
+ _to_run_detail(run)
+ for run in select_page(runs, page, ordering_fields=contracts.BATCH_EXPORT_RUN_ORDERING_FIELDS)
+ ]
+
+
+def get_run(
+ team_id: int, export_id: UUID, run_id: UUID, filters: contracts.BatchExportRunFilters
+) -> contracts.BatchExportRunDetail:
+ return _to_run_detail(_get_run(team_id, export_id, run_id, filters))
+
+
+def retry_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.BatchExportRunFilters) -> str:
+ """Backfill the data interval of a run, and return the backfill id."""
+ run = _get_run(team_id, export_id, run_id, filters)
+ return backfill_export(
+ sync_connect(),
+ str(run.batch_export_id),
+ team_id,
+ run.data_interval_start,
+ run.data_interval_end,
+ )
+
+
+def cancel_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.BatchExportRunFilters) -> None:
+ run = _get_run(team_id, export_id, run_id, filters)
+ if run.status not in _CANCELLABLE_STATUSES:
+ raise contracts.InvalidBatchExportRunRequestError(f"Cannot cancel a run that is in '{run.status}' status")
+
+ temporal = sync_connect()
+ try:
+ cancel_running_batch_export_run(temporal, run)
+ except Exception as e:
+ # It could be the case that the run is already cancelled but our database hasn't been updated yet. In
+ # this case, we can just ignore the error but log it for visibility (in case there is an actual issue).
+ LOGGER.warning("Error cancelling batch export run: %s", e)
diff --git a/products/batch_exports/backend/presentation/views/batch_export/backfills.py b/products/batch_exports/backend/presentation/views/batch_export/backfills.py
index f9ea7305cd8a..b2bc00a9db88 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/backfills.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/backfills.py
@@ -1,120 +1,76 @@
"""Batch export backfill endpoints: create, list, retrieve and cancel."""
import uuid
-import datetime as dt
-from dataclasses import dataclass
-from typing import Any
-
-from django.conf import settings
+import dataclasses
+from typing import Any, cast
import structlog
import posthoganalytics
-from drf_spectacular.utils import extend_schema_field
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_field, extend_schema_view
from rest_framework import filters, mixins, request, response, serializers, status, viewsets
from rest_framework.exceptions import APIException, NotFound, ValidationError
-from rest_framework.pagination import CursorPagination
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.utils import action
from posthog.event_usage import groups
-from posthog.models import Team, User
-from posthog.temporal.common.client import sync_connect
+from posthog.models import User
-from products.batch_exports.backend.hogql_source import (
- DATA_INTERVAL_START_PLACEHOLDER,
- UnsupportedHogQLQueryError,
- find_interval_placeholders,
- parse_hogql_select_for_batch_export,
-)
-from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportBackfill
-from products.batch_exports.backend.service import (
- BatchExportWithNoEndNotAllowedError,
- backfill_export,
- sync_cancel_running_batch_export_backfill,
+from products.batch_exports.backend.facade import (
+ api as batch_exports_api,
+ contracts,
)
+from products.batch_exports.backend.facade.enums import BatchExportBackfillStatus
+from products.batch_exports.backend.presentation.views.batch_export.pagination import FacadeCursorPagination
+from products.batch_exports.backend.presentation.views.utils import BIG_INTEGER_MAX, BIG_INTEGER_MIN, parse_path_uuid
logger = structlog.get_logger(__name__)
-def validate_date_input(date_input: Any, batch_export: BatchExport) -> dt.datetime:
- """Validate and parse a date/datetime input as a proper dt.datetime.
-
- If the interval is daily or weekly, we expect the input to be an ISO formatted date string.
- We then need to convert it to a datetime using the batch export's timezone and offset.
-
- For all other intervals, we expect the input to be an ISO formatted datetime string.
-
- Args:
- date_input: The datetime input to parse.
-
- Raises:
- ValidationError: If the input cannot be parsed.
-
- Returns:
- The parsed dt.datetime.
- """
- if batch_export.interval == "day" or batch_export.interval == "week":
- try:
- parsed_date = dt.date.fromisoformat(date_input)
- except (TypeError, ValueError):
- # Try to parse as a datetime string so we can give a more helpful error message
- try:
- parsed = dt.datetime.fromisoformat(date_input)
- raise ValidationError(
- f"Input '{date_input}' is not a valid ISO formatted date. "
- "Daily or weekly batch export backfills expect only the date component, but a time was included."
- )
- except (TypeError, ValueError):
- pass
- raise ValidationError(f"Input '{date_input}' is not a valid ISO formatted date.")
-
- if batch_export.interval == "week":
- # Validate that the provided date is on the correct day of the week, according to the batch export's day offset
- # Python's date.isoweekday() returns 1-7 for Monday-Sunday, so we need to convert it to 0-6 for Sunday-Saturday
- normalized_day = parsed_date.isoweekday() % 7
- if normalized_day != batch_export.offset_day:
- # get day of week as string
- day_of_week = parsed_date.strftime("%A")
- expected_day_of_week = batch_export.offset_day_name
- assert expected_day_of_week is not None
- raise ValidationError(
- f"Input {date_input} is not on the correct day of the week for this batch export. "
- f"{date_input} is a {day_of_week} but this batch export is configured to run "
- f"weekly on {expected_day_of_week}."
- )
-
- # If we have an offset hour, add it to the parsed datetime
- # Also, apply the timezone to the parsed datetime
- time_of_day = dt.time.min if batch_export.offset_hour is None else dt.time(hour=batch_export.offset_hour)
- parsed = dt.datetime.combine(parsed_date, time_of_day).replace(tzinfo=batch_export.timezone_info)
-
- else:
- try:
- parsed = dt.datetime.fromisoformat(date_input)
- except (TypeError, ValueError):
- raise ValidationError(f"Input {date_input} is not a valid ISO formatted datetime.")
-
- if parsed.tzinfo is None:
- raise ValidationError(f"Input {date_input} is naive.")
-
- return parsed
-
-
-@dataclass(frozen=False)
-class BatchExportBackfillProgress:
- """Progress information for a batch export backfill."""
-
- total_runs: int | None
- finished_runs: int | None
- progress: float | None
+# The viewset declares no queryset, so drf-spectacular cannot read the type of the ids from a model.
+BACKFILL_ID_PARAMETER = OpenApiParameter(
+ "id",
+ OpenApiTypes.UUID,
+ OpenApiParameter.PATH,
+ description="A UUID string identifying this batch export backfill.",
+)
-class BatchExportBackfillSerializer(serializers.ModelSerializer):
+class BatchExportBackfillSerializer(serializers.Serializer):
+ id = serializers.UUIDField(read_only=True)
progress = serializers.SerializerMethodField(read_only=True)
-
- class Meta:
- model = BatchExportBackfill
- fields = "__all__"
+ start_at = serializers.DateTimeField(required=False, allow_null=True, help_text="The start of the data interval.")
+ end_at = serializers.DateTimeField(required=False, allow_null=True, help_text="The end of the data interval.")
+ status = serializers.ChoiceField(
+ choices=BatchExportBackfillStatus.choices, help_text="The status of this backfill."
+ )
+ created_at = serializers.DateTimeField(
+ read_only=True, help_text="The timestamp at which this BatchExportBackfill was created."
+ )
+ finished_at = serializers.DateTimeField(
+ required=False,
+ allow_null=True,
+ help_text="The timestamp at which this BatchExportBackfill finished, successfully or not.",
+ )
+ last_updated_at = serializers.DateTimeField(
+ read_only=True, help_text="The timestamp at which this BatchExportBackfill was last updated."
+ )
+ total_records_count = serializers.IntegerField(
+ required=False,
+ allow_null=True,
+ min_value=BIG_INTEGER_MIN,
+ max_value=BIG_INTEGER_MAX,
+ help_text="The total number of records to export. Initially estimated, updated with actual count after completion.",
+ )
+ adjusted_start_at = serializers.DateTimeField(
+ required=False,
+ allow_null=True,
+ help_text="The actual start time after adjustment for earliest available data. May differ from start_at if user requested a date before data exists.",
+ )
+ team = serializers.IntegerField(source="team_id", help_text="The team this belongs to.")
+ batch_export = serializers.UUIDField(
+ source="batch_export_id", help_text="The BatchExport this backfill belongs to."
+ )
@extend_schema_field(
{
@@ -127,38 +83,14 @@ class BatchExportBackfillSerializer(serializers.ModelSerializer):
},
}
)
- def get_progress(self, obj: BatchExportBackfill) -> BatchExportBackfillProgress | None:
- """Return progress information containing total runs, finished runs, and progress percentage.
-
- To reduce the number of database calls we make (which could be expensive when fetching a list of backfills) we
- only get the list of completed runs from the DB if the backfill is still running.
- """
- if obj.status == obj.Status.COMPLETED:
- return BatchExportBackfillProgress(
- total_runs=obj.total_expected_runs, finished_runs=obj.total_expected_runs, progress=1.0
- )
- elif obj.status not in (obj.Status.RUNNING, obj.Status.STARTING):
- # if backfill finished in some other state then progress info may not be meaningful
- return None
-
- total_runs = obj.total_expected_runs
- if not total_runs:
- return None
-
- if obj.start_at is None and obj.adjusted_start_at is None:
- # if it's just a single run, backfilling from the beginning of time, we can't calculate progress based on
- # the number of completed runs so better to return None
+ def get_progress(self, obj: contracts.BatchExportBackfillDetail) -> dict[str, Any] | None:
+ """Return progress information containing total runs, finished runs, and progress percentage."""
+ if obj.progress is None:
return None
-
- finished_runs = obj.get_finished_runs()
- # just make sure we never return a progress > 1
- total_runs = max(total_runs, finished_runs)
- return BatchExportBackfillProgress(
- total_runs=total_runs, finished_runs=finished_runs, progress=round(finished_runs / total_runs, ndigits=1)
- )
+ return dataclasses.asdict(obj.progress)
-class BackfillsCursorPagination(CursorPagination):
+class BackfillsCursorPagination(FacadeCursorPagination):
page_size = 50
@@ -168,156 +100,71 @@ class TooManyConcurrentBackfills(APIException):
default_detail = "Too many concurrent batch export backfills for this team."
-def create_backfill(
- team: Team,
- batch_export: BatchExport,
- start_at_input: str | None,
- end_at_input: str | None,
-) -> str:
- """Create a new backfill for a BatchExport.
-
- Args:
- team: The team creating the backfill
- batch_export: The batch export to backfill
- start_at_input: ISO formatted datetime string for backfill start
- end_at_input: ISO formatted datetime string for backfill end
-
- Returns:
- The pre-generated backfill ID.
- """
- # Currently, backfills from the beginning of time usually fail due to us hitting ClickHouse memory limits.
- # Therefore, this feature is behind a feature flag while we improve backfilling behavior.
- if start_at_input is None:
- if not posthoganalytics.feature_enabled(
- "batch-export-earliest-backfill",
- str(team.uuid),
- groups={"organization": str(team.organization.id)},
- group_properties={
- "organization": {
- "id": str(team.organization.id),
- "created_at": team.organization.created_at,
- }
- },
- send_feature_flag_events=False,
- ):
- raise ValidationError("Backfilling from the beginning of time is not enabled for this team.")
-
- if batch_export.model == BatchExport.Model.HOGQL and (hogql_query := batch_export.hogql_query) is not None:
- try:
- parsed = parse_hogql_select_for_batch_export(hogql_query)
- except UnsupportedHogQLQueryError as e:
- raise ValidationError(str(e)) from e
- if DATA_INTERVAL_START_PLACEHOLDER in find_interval_placeholders(parsed):
- # TODO: We should maybe support beginning-of-time backfills with this placeholder.
- raise ValidationError(
- "This query references {data_interval_start}, which is unavailable when backfilling from the "
- "beginning of time. Provide 'start_at' or remove {data_interval_start} from the query."
- )
-
- concurrency_limit = settings.BATCH_EXPORT_MAX_CONCURRENT_BACKFILLS_PER_TEAM
- active_backfills = BatchExportBackfill.objects.filter(
- team_id=team.pk,
- status__in=[
- BatchExportBackfill.Status.STARTING,
- BatchExportBackfill.Status.RUNNING,
- ],
- ).count()
- if active_backfills >= concurrency_limit:
- raise TooManyConcurrentBackfills(
- f"This team already has {concurrency_limit} batch export backfills running. "
- f"Wait for some to finish or cancel them before creating more."
- )
-
- temporal = sync_connect()
-
- if start_at_input is not None:
- start_at = validate_date_input(start_at_input, batch_export)
- else:
- start_at = None
-
- if end_at_input is not None:
- end_at = validate_date_input(end_at_input, batch_export)
- else:
- end_at = None
-
- # Note: earliest backfill date validation and adjustment is now done in the Temporal workflow
- # via the get_backfill_info activity. This allows the potentially slow ClickHouse query to run
- # asynchronously rather than blocking the HTTP request.
-
- backfill_id = str(uuid.uuid4())
-
- if start_at is None or end_at is None:
- backfill_export(
- temporal=temporal,
- batch_export_id=str(batch_export.pk),
- team_id=team.pk,
- start_at=start_at,
- end_at=end_at,
- backfill_id=backfill_id,
+@extend_schema(
+ parameters=[
+ OpenApiParameter(
+ "batch_export_id",
+ {"type": "string", "format": "uuid", "description": "The BatchExport this backfill belongs to."},
+ OpenApiParameter.PATH,
)
- return backfill_id
-
- if start_at >= end_at:
- raise ValidationError("The initial backfill datetime 'start_at' must be before 'end_at'")
- if end_at > dt.datetime.now(dt.UTC):
- raise ValidationError(f"The provided 'end_at' ({end_at.isoformat()}) is in the future")
-
- try:
- backfill_export(
- temporal=temporal,
- batch_export_id=str(batch_export.pk),
- team_id=team.pk,
- start_at=start_at,
- end_at=end_at,
- backfill_id=backfill_id,
- )
- return backfill_id
- except BatchExportWithNoEndNotAllowedError:
- raise ValidationError("Backfilling a BatchExport with no end date is not allowed")
-
-
-class BatchExportBackfillViewSet(
- TeamAndOrgViewSetMixin,
- mixins.CreateModelMixin,
- mixins.ListModelMixin,
- mixins.RetrieveModelMixin,
- viewsets.GenericViewSet,
-):
+ ]
+)
+@extend_schema_view(retrieve=extend_schema(parameters=[BACKFILL_ID_PARAMETER]))
+class BatchExportBackfillViewSet(TeamAndOrgViewSetMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
"""ViewSet for BatchExportBackfill models.
Allows creating and reading backfills, but not updating or deleting them.
"""
scope_object = "batch_export"
- queryset = BatchExportBackfill.objects.all()
serializer_class = BatchExportBackfillSerializer
pagination_class = BackfillsCursorPagination
- filter_rewrite_rules = {"team_id": "batch_export__team_id"}
+ # The paginator reads the ordering through this filter, and the facade applies it.
filter_backends = [filters.OrderingFilter]
- ordering_fields = ["created_at", "start_at"]
+ ordering_fields = list(contracts.BATCH_EXPORT_BACKFILL_ORDERING_FIELDS)
ordering = "-created_at"
- def safely_get_queryset(self, queryset):
- return queryset.filter(batch_export_id=self.kwargs["parent_lookup_batch_export_id"])
+ def _get_export_id(self, detail: str | None = None) -> uuid.UUID:
+ return parse_path_uuid(self.kwargs["parent_lookup_batch_export_id"], detail)
- def create(self, request: request.Request, *args, **kwargs) -> response.Response:
- """Create a new backfill for a BatchExport."""
+ def dangerously_get_object(self) -> contracts.BatchExportBackfillDetail:
+ # The mixin's get_object() reads through a queryset, which this view no longer has, so the
+ # backfill is read through the facade instead. That skips the mixin's team filter and object
+ # permission check, so both happen here: the facade filters by team and export, and
+ # check_object_permissions runs on the contract.
try:
- batch_export = BatchExport.objects.select_related("destination").get(
- id=self.kwargs["parent_lookup_batch_export_id"], team_id=self.team_id
+ backfill = batch_exports_api.get_batch_export_backfill(
+ self.team_id, self._get_export_id(), parse_path_uuid(self.kwargs["pk"])
)
- except BatchExport.DoesNotExist:
- raise NotFound("BatchExport not found.")
+ except contracts.BatchExportBackfillNotFoundError as e:
+ raise NotFound(*e.args) from e
+ self.check_object_permissions(self.request, backfill)
+ return backfill
+
+ def list(self, request: request.Request, *args: Any, **kwargs: Any) -> response.Response:
+ export_id = self._get_export_id()
+ return cast(FacadeCursorPagination, self.paginator).get_paginated_facade_response(
+ request,
+ self,
+ lambda page: batch_exports_api.list_batch_export_backfills(self.team_id, export_id, page=page),
+ )
+ def create(self, request: request.Request, *args, **kwargs) -> response.Response:
+ """Create a new backfill for a BatchExport."""
+ export_id = self._get_export_id("BatchExport not found.")
start_at = request.data.get("start_at")
end_at = request.data.get("end_at")
- backfill_id = create_backfill(
- self.team,
- batch_export,
- start_at,
- end_at,
- )
+ try:
+ backfill = batch_exports_api.create_batch_export_backfill(
+ self.team_id, export_id, start_at=start_at, end_at=end_at
+ )
+ except contracts.BatchExportNotFoundError as e:
+ raise NotFound("BatchExport not found.") from e
+ except contracts.InvalidBackfillRequestError as e:
+ raise ValidationError(str(e)) from e
+ except contracts.TooManyConcurrentBackfillsError as e:
+ raise TooManyConcurrentBackfills(str(e)) from e
if isinstance(request.user, User):
try:
@@ -325,9 +172,9 @@ class BatchExportBackfillViewSet(
distinct_id=str(request.user.distinct_id),
event="batch export backfill created",
properties={
- "backfill_id": backfill_id,
- "batch_export_id": str(batch_export.pk),
- "destination_type": batch_export.destination.type,
+ "backfill_id": backfill.backfill_id,
+ "batch_export_id": str(export_id),
+ "destination_type": backfill.destination_type,
"has_start_at": start_at is not None,
"has_end_at": end_at is not None,
"team_id": self.team_id,
@@ -337,27 +184,19 @@ class BatchExportBackfillViewSet(
except Exception:
logger.exception("Failed to capture batch export backfill created event")
- return response.Response({"backfill_id": backfill_id}, status=status.HTTP_201_CREATED)
+ return response.Response({"backfill_id": backfill.backfill_id}, status=status.HTTP_201_CREATED)
+ @extend_schema(parameters=[BACKFILL_ID_PARAMETER])
@action(methods=["POST"], detail=True, required_scopes=["batch_export:write"])
def cancel(self, *args, **kwargs) -> response.Response:
"""Cancel a batch export backfill."""
+ backfill = self.get_object()
- batch_export_backfill: BatchExportBackfill = self.get_object()
-
- if (
- batch_export_backfill.status == BatchExportBackfill.Status.RUNNING
- or batch_export_backfill.status == BatchExportBackfill.Status.STARTING
- ):
- temporal = sync_connect()
- try:
- sync_cancel_running_batch_export_backfill(temporal, batch_export_backfill)
- except Exception as e:
- # It could be the case that the backfill is already cancelled but our database hasn't been updated yet.
- # In this case, we can just ignore the error but log it for visibility (in case there is an actual
- # issue).
- logger.warning("Error cancelling batch export backfill: %s", e)
- else:
- raise ValidationError(f"Cannot cancel a backfill that is in '{batch_export_backfill.status}' status")
+ try:
+ batch_exports_api.cancel_batch_export_backfill(self.team_id, self._get_export_id(), backfill.id)
+ except contracts.InvalidBackfillRequestError as e:
+ raise ValidationError(str(e)) from e
+ except contracts.BatchExportBackfillNotFoundError as e:
+ raise NotFound(*e.args) from e
return response.Response({"cancelled": True})
diff --git a/products/batch_exports/backend/presentation/views/batch_export/pagination.py b/products/batch_exports/backend/presentation/views/batch_export/pagination.py
new file mode 100644
index 000000000000..81f001bbe1f8
--- /dev/null
+++ b/products/batch_exports/backend/presentation/views/batch_export/pagination.py
@@ -0,0 +1,59 @@
+"""Cursor pagination over lists that the facade reads."""
+
+from collections.abc import Callable
+from typing import Any, cast
+
+from rest_framework.generics import GenericAPIView
+from rest_framework.pagination import CursorPagination
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+from products.batch_exports.backend.facade import contracts
+
+
+# TODO: Nothing here is specific to batch exports. Move it to posthog/api/pagination.py with
+# select_page in logic/pagination.py, and a core page type to replace contracts.ListPage, so other
+# facade-backed products can page with a cursor too.
+class _FacadePageSource[T]:
+ """Stands in for the queryset that ``CursorPagination.paginate_queryset`` pages through.
+
+ The paginator only orders the queryset, filters it on the cursor position and slices it.
+ This records the ordering and the position, and the slice reads that page through the
+ facade. So DRF still decodes and encodes every cursor, and the cursors stay the same.
+ """
+
+ def __init__(
+ self,
+ fetch: Callable[[contracts.ListPage], list[T]],
+ ordering: tuple[str, ...] = (),
+ bound: contracts.ListPageBound | None = None,
+ ) -> None:
+ self._fetch = fetch
+ self._ordering = ordering
+ self._bound = bound
+
+ def order_by(self, *ordering: str) -> "_FacadePageSource[T]":
+ return _FacadePageSource(self._fetch, ordering, self._bound)
+
+ def filter(self, **lookup: str) -> "_FacadePageSource[T]":
+ # The only filter is the position, as `<first ordering field>__lt` or `__gt`.
+ ((key, value),) = lookup.items()
+ bound = contracts.ListPageBound(value=value, before=key.endswith("__lt"))
+ return _FacadePageSource(self._fetch, self._ordering, bound)
+
+ def __getitem__(self, items: slice) -> list[T]:
+ page = contracts.ListPage(
+ ordering=self._ordering, bound=self._bound, offset=items.start, limit=items.stop - items.start
+ )
+ return self._fetch(page)
+
+
+class FacadeCursorPagination(CursorPagination):
+ def get_paginated_facade_response[T](
+ self, request: Request, view: GenericAPIView, fetch: Callable[[contracts.ListPage], list[T]]
+ ) -> Response:
+ """Return the page the request asks for, serialized by the view. ``fetch`` reads a page through the facade."""
+ page = self.paginate_queryset(cast(Any, _FacadePageSource(fetch)), request, view)
+ # The page size is a class attribute, so the paginator always returns a page.
+ assert page is not None
+ return self.get_paginated_response(view.get_serializer(page, many=True).data)
diff --git a/products/batch_exports/backend/presentation/views/batch_export/runs.py b/products/batch_exports/backend/presentation/views/batch_export/runs.py
index 5fe5b8efa453..4980a85325e9 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/runs.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/runs.py
@@ -1,30 +1,45 @@
"""Batch export run endpoints: list, retrieve, retry and cancel."""
-from typing import cast
+import uuid
+from typing import Any, cast
from django.utils.timezone import now
-import structlog
-from drf_spectacular.utils import extend_schema, extend_schema_view
-from rest_framework import filters, response, serializers, status, viewsets
-from rest_framework.exceptions import ValidationError
-from rest_framework.pagination import CursorPagination
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view
+from rest_framework import filters, mixins, request, response, serializers, status, viewsets
+from rest_framework.exceptions import NotFound, ValidationError
from posthog.api.log_entries import LogEntryMixin
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.utils import action
-from posthog.temporal.common.client import sync_connect
from posthog.utils import relative_date_parse
-from products.batch_exports.backend.models.batch_export import BatchExportRun
-from products.batch_exports.backend.service import backfill_export, cancel_running_batch_export_run
+from products.batch_exports.backend.facade import (
+ api as batch_exports_api,
+ contracts,
+)
+from products.batch_exports.backend.facade.enums import BatchExportRunStatus
+from products.batch_exports.backend.presentation.views.batch_export.pagination import FacadeCursorPagination
+from products.batch_exports.backend.presentation.views.utils import (
+ BIG_INTEGER_MAX,
+ BIG_INTEGER_MIN,
+ INTEGER_MAX,
+ INTEGER_MIN,
+ parse_path_uuid,
+)
-logger = structlog.get_logger(__name__)
+# The viewset declares no queryset, so drf-spectacular cannot read the type of the ids from a model.
+RUN_ID_PARAMETER = OpenApiParameter(
+ "id", OpenApiTypes.UUID, OpenApiParameter.PATH, description="A UUID string identifying this batch export run."
+)
-class BatchExportRunSerializer(serializers.ModelSerializer):
+class BatchExportRunSerializer(serializers.Serializer):
"""Serializer for a BatchExportRun model."""
+ # TODO: Why aren't all these read only?
+ id = serializers.UUIDField(read_only=True)
# Underlying model can be null for on demand batch exports. But scheduled
# batch exports always have a data_interval_end given by the schedule
# itself (even if that isn't used by the underlying HogQL query). This
@@ -33,12 +48,70 @@ class BatchExportRunSerializer(serializers.ModelSerializer):
data_interval_end = serializers.DateTimeField(
required=True, allow_null=False, help_text="The end of the data interval."
)
-
- class Meta:
- model = BatchExportRun
- fields = "__all__"
- # TODO: Why aren't all these read only?
- read_only_fields = ["batch_export"]
+ status = serializers.ChoiceField(choices=BatchExportRunStatus.choices, help_text="The status of this run.")
+ records_completed = serializers.IntegerField(
+ required=False,
+ allow_null=True,
+ min_value=INTEGER_MIN,
+ max_value=INTEGER_MAX,
+ help_text="The number of records that have been exported.",
+ )
+ records_failed = serializers.IntegerField(
+ required=False,
+ allow_null=True,
+ min_value=INTEGER_MIN,
+ max_value=INTEGER_MAX,
+ help_text="The number of records that failed downstream processing (e.g. hog function execution errors).",
+ )
+ latest_error = serializers.CharField(
+ required=False, allow_null=True, help_text="The latest error that occurred during this run."
+ )
+ data_interval_start = serializers.DateTimeField(
+ required=False, allow_null=True, help_text="The start of the data interval."
+ )
+ cursor = serializers.CharField(
+ required=False, allow_null=True, help_text="An opaque cursor that may be used to resume."
+ )
+ created_at = serializers.DateTimeField(
+ read_only=True, help_text="The timestamp at which this BatchExportRun was created."
+ )
+ finished_at = serializers.DateTimeField(
+ required=False,
+ allow_null=True,
+ help_text="The timestamp at which this BatchExportRun finished, successfully or not.",
+ )
+ last_updated_at = serializers.DateTimeField(
+ read_only=True, help_text="The timestamp at which this BatchExportRun was last updated."
+ )
+ records_total_count = serializers.IntegerField(
+ required=False,
+ allow_null=True,
+ min_value=INTEGER_MIN,
+ max_value=INTEGER_MAX,
+ help_text="The total count of records that should be exported in this BatchExportRun.",
+ )
+ bytes_exported = serializers.IntegerField(
+ required=False,
+ allow_null=True,
+ min_value=BIG_INTEGER_MIN,
+ max_value=BIG_INTEGER_MAX,
+ help_text="The number of bytes that have been exported in this BatchExportRun.",
+ )
+ batch_export = serializers.UUIDField(
+ source="batch_export_id",
+ read_only=True,
+ allow_null=True,
+ help_text="The `BatchExport` this run belongs to.",
+ )
+ batch_export_on_demand = serializers.UUIDField(
+ source="batch_export_on_demand_id",
+ required=False,
+ allow_null=True,
+ help_text="The `BatchExportOnDemand` this run belongs to.",
+ )
+ backfill = serializers.UUIDField(
+ source="backfill_id", required=False, allow_null=True, help_text="The backfill this run belongs to."
+ )
class BatchExportRunListQuerySerializer(serializers.Serializer):
@@ -46,7 +119,7 @@ class BatchExportRunListQuerySerializer(serializers.Serializer):
status = serializers.ListField(
required=False,
- child=serializers.ChoiceField(choices=BatchExportRun.Status.choices),
+ child=serializers.ChoiceField(choices=BatchExportRunStatus.choices),
help_text="Only return runs in these statuses. Repeat the parameter to pass more than one status.",
)
after = serializers.CharField(
@@ -87,27 +160,42 @@ class BatchExportRunListQuerySerializer(serializers.Serializer):
)
-class RunsCursorPagination(CursorPagination):
+class RunsCursorPagination(FacadeCursorPagination):
page_size = 100
-@extend_schema(tags=["batch_exports"])
-@extend_schema_view(list=extend_schema(parameters=[BatchExportRunListQuerySerializer]))
-class BatchExportRunViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ReadOnlyModelViewSet):
+@extend_schema(
+ tags=["batch_exports"],
+ parameters=[
+ OpenApiParameter(
+ "batch_export_id",
+ {"type": "string", "format": "uuid", "description": "The `BatchExport` this run belongs to."},
+ OpenApiParameter.PATH,
+ )
+ ],
+)
+@extend_schema_view(
+ list=extend_schema(parameters=[BatchExportRunListQuerySerializer]),
+ retrieve=extend_schema(parameters=[RUN_ID_PARAMETER]),
+ logs=extend_schema(parameters=[RUN_ID_PARAMETER]),
+)
+class BatchExportRunViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
scope_object = "batch_export"
- queryset = BatchExportRun.objects.select_related("batch_export__destination").all()
serializer_class = BatchExportRunSerializer
pagination_class = RunsCursorPagination
- filter_rewrite_rules = {"team_id": "batch_export__team_id"}
+ # The paginator reads the ordering through this filter, and the facade applies it.
filter_backends = [filters.OrderingFilter]
- ordering_fields = ["created_at", "data_interval_start"]
+ ordering_fields = list(contracts.BATCH_EXPORT_RUN_ORDERING_FIELDS)
ordering = "-created_at"
log_source = "batch_exports"
def get_log_entry_instance_id(self) -> str:
return cast(str, self.parents_query_dict["run_id"])
- def safely_get_queryset(self, queryset):
+ def _get_export_id(self) -> uuid.UUID:
+ return parse_path_uuid(self.kwargs["parent_lookup_batch_export_id"])
+
+ def _get_run_filters(self) -> contracts.BatchExportRunFilters:
query = BatchExportRunListQuerySerializer(data=self.request.GET)
query.is_valid(raise_exception=True)
params = query.validated_data
@@ -116,6 +204,7 @@ class BatchExportRunViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.Read
before = params.get("before")
start = params.get("start")
end = params.get("end")
+ statuses = tuple(params.get("status") or ())
# OrderingFilter applies the sort and declares this parameter, so it is not on the serializer.
ordering = self.request.GET.get("ordering", None)
@@ -123,19 +212,41 @@ class BatchExportRunViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.Read
if ordering == "data_interval_start" or ordering == "-data_interval_start":
start_timestamp = relative_date_parse(start if start else "-7d", self.team.timezone_info)
end_timestamp = relative_date_parse(end, self.team.timezone_info) if end else now()
- queryset = queryset.filter(data_interval_start__gte=start_timestamp, data_interval_end__lte=end_timestamp)
- else:
- after_datetime = relative_date_parse(after if after else "-7d", self.team.timezone_info)
- before_datetime = relative_date_parse(before, self.team.timezone_info) if before else now()
- date_range = (after_datetime, before_datetime)
- queryset = queryset.filter(created_at__range=date_range)
+ return contracts.BatchExportRunFilters(
+ statuses=statuses, data_interval=contracts.DatetimeRange(start=start_timestamp, end=end_timestamp)
+ )
+
+ after_datetime = relative_date_parse(after if after else "-7d", self.team.timezone_info)
+ before_datetime = relative_date_parse(before, self.team.timezone_info) if before else now()
+ return contracts.BatchExportRunFilters(
+ statuses=statuses, created=contracts.DatetimeRange(start=after_datetime, end=before_datetime)
+ )
- if statuses := params.get("status"):
- queryset = queryset.filter(status__in=statuses)
+ def dangerously_get_object(self) -> contracts.BatchExportRunDetail:
+ # The mixin's get_object() reads through a queryset, which this view no longer has, so the
+ # run is read through the facade instead. That skips the mixin's team filter and object
+ # permission check, so both happen here: the facade filters by team and export, and
+ # check_object_permissions runs on the contract. `logs` reads the run through here too.
+ # Like the list, a lookup only finds a run that matches the list filters.
+ try:
+ run = batch_exports_api.get_batch_export_run(
+ self.team_id, self._get_export_id(), parse_path_uuid(self.kwargs["pk"]), filters=self._get_run_filters()
+ )
+ except contracts.BatchExportRunNotFoundError as e:
+ raise NotFound(*e.args) from e
+ self.check_object_permissions(self.request, run)
+ return run
- queryset = queryset.filter(batch_export_id=self.kwargs["parent_lookup_batch_export_id"])
- return queryset
+ def list(self, request: request.Request, *args: Any, **kwargs: Any) -> response.Response:
+ export_id = self._get_export_id()
+ filters = self._get_run_filters()
+ return cast(FacadeCursorPagination, self.paginator).get_paginated_facade_response(
+ request,
+ self,
+ lambda page: batch_exports_api.list_batch_export_runs(self.team_id, export_id, filters=filters, page=page),
+ )
+ @extend_schema(parameters=[RUN_ID_PARAMETER])
@action(methods=["POST"], detail=True, required_scopes=["batch_export:write"])
def retry(self, *args, **kwargs) -> response.Response:
"""Retry a batch export run.
@@ -143,37 +254,30 @@ class BatchExportRunViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.Read
We use the same underlying mechanism as when backfilling a batch export, as retrying
a run is the same as backfilling one run.
"""
- batch_export_run = self.get_object()
-
- temporal = sync_connect()
- backfill_id = backfill_export(
- temporal,
- str(batch_export_run.batch_export.id),
- self.team_id,
- batch_export_run.data_interval_start,
- batch_export_run.data_interval_end,
- )
+ run = self.get_object()
+
+ try:
+ backfill_id = batch_exports_api.retry_batch_export_run(
+ self.team_id, self._get_export_id(), run.id, filters=self._get_run_filters()
+ )
+ except contracts.BatchExportRunNotFoundError as e:
+ raise NotFound(*e.args) from e
return response.Response({"backfill_id": backfill_id}, status=status.HTTP_201_CREATED)
+ @extend_schema(parameters=[RUN_ID_PARAMETER])
@action(methods=["POST"], detail=True, required_scopes=["batch_export:write"])
def cancel(self, *args, **kwargs) -> response.Response:
"""Cancel a batch export run."""
+ run = self.get_object()
- batch_export_run: BatchExportRun = self.get_object()
-
- if (
- batch_export_run.status == BatchExportRun.Status.RUNNING
- or batch_export_run.status == BatchExportRun.Status.STARTING
- ):
- temporal = sync_connect()
- try:
- cancel_running_batch_export_run(temporal, batch_export_run)
- except Exception as e:
- # It could be the case that the run is already cancelled but our database hasn't been updated yet. In
- # this case, we can just ignore the error but log it for visibility (in case there is an actual issue).
- logger.warning("Error cancelling batch export run: %s", e)
- else:
- raise ValidationError(f"Cannot cancel a run that is in '{batch_export_run.status}' status")
+ try:
+ batch_exports_api.cancel_batch_export_run(
+ self.team_id, self._get_export_id(), run.id, filters=self._get_run_filters()
+ )
+ except contracts.InvalidBatchExportRunRequestError as e:
+ raise ValidationError(str(e)) from e
+ except contracts.BatchExportRunNotFoundError as e:
+ raise NotFound(*e.args) from e
return response.Response({"cancelled": True})
diff --git a/products/batch_exports/backend/presentation/views/file_download.py b/products/batch_exports/backend/presentation/views/file_download.py
index 387ac43143c8..25d3fbbd941c 100644
--- a/products/batch_exports/backend/presentation/views/file_download.py
+++ b/products/batch_exports/backend/presentation/views/file_download.py
@@ -1,4 +1,3 @@
-import uuid
import datetime as dt
from typing import Any, cast
@@ -38,6 +37,7 @@ from products.batch_exports.backend.presentation.views.utils import (
HOGQL_MODIFIERS_HELP_TEXT,
HogQLModifiersField,
check_hogql_batch_exports_enabled,
+ parse_path_uuid,
)
FILE_DOWNLOAD_MAX_RANGE = dt.timedelta(weeks=1)
@@ -398,12 +398,11 @@ class FileDownloadBatchExportOnDemandViewSet(TeamAndOrgViewSetMixin, LogEntryMix
return super().get_throttles()
def dangerously_get_object(self) -> contracts.FileDownloadRun:
- # This replaces the queryset read only. The facade scopes the read to the team, and the
- # object permissions run on the contract as they ran on the model. `logs` reads the run here.
- try:
- run_id = uuid.UUID(str(self.kwargs["pk"]))
- except ValueError:
- raise NotFound()
+ # The mixin's get_object() reads through a queryset, which this view does not have, so the
+ # run is read through the facade instead. That skips the mixin's team filter and object
+ # permission check, so both happen here: the facade filters by team, and
+ # check_object_permissions runs on the contract. `logs` reads the run through here too.
+ run_id = parse_path_uuid(self.kwargs["pk"])
try:
run = batch_exports_api.get_file_download_run(self.team_id, run_id)
except contracts.FileDownloadNotFoundError as e:
diff --git a/products/batch_exports/backend/presentation/views/utils.py b/products/batch_exports/backend/presentation/views/utils.py
index 91dde38998ed..74a3bc080fa6 100644
--- a/products/batch_exports/backend/presentation/views/utils.py
+++ b/products/batch_exports/backend/presentation/views/utils.py
@@ -1,10 +1,11 @@
+import uuid
import typing
import pydantic
import posthoganalytics
from drf_spectacular.utils import extend_schema_field
from rest_framework import serializers
-from rest_framework.exceptions import PermissionDenied
+from rest_framework.exceptions import NotFound, PermissionDenied
from posthog.schema import HogQLQueryModifiers
@@ -12,6 +13,12 @@ from posthog.models import Team
from products.batch_exports.backend.facade.api import drop_unknown_hogql_modifiers
+# The bounds of the integer columns, which a model serializer reads off the model.
+# TODO: Nothing here is specific to batch exports, and other products hard-code the same numbers.
+# Move these to core, for example derived from django.db.connection.ops.integer_field_range.
+INTEGER_MIN, INTEGER_MAX = -(2**31), 2**31 - 1
+BIG_INTEGER_MIN, BIG_INTEGER_MAX = -(2**63), 2**63 - 1
+
HOGQL_MODIFIERS_HELP_TEXT = (
"HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example "
"{\"convertToProjectTimezone\": true}. Only supported when 'model' is 'hogql'. Each modifier set here "
@@ -22,6 +29,16 @@ HOGQL_MODIFIERS_HELP_TEXT = (
)
+# TODO: Nothing here is specific to batch exports. Move it to posthog/api/utils.py and replace the
+# copies in cross_project_dashboards, review_hog, visual_review and the web_analytics heatmaps views.
+def parse_path_uuid(value: object, detail: str | None = None) -> uuid.UUID:
+ """Return a UUID path parameter. A value that is not a UUID cannot name anything, so it is a 404."""
+ try:
+ return uuid.UUID(str(value))
+ except ValueError:
+ raise NotFound(detail)
+
+
def check_hogql_batch_exports_enabled(team: Team) -> None:
"""Raise if HogQL-powered batch exports are not enabled for the team."""
if not posthoganalytics.feature_enabled(
diff --git a/products/batch_exports/backend/tests/api/backfills/test_create.py b/products/batch_exports/backend/tests/api/backfills/test_create.py
index b1f3f8250a00..e17bef20a69d 100644
--- a/products/batch_exports/backend/tests/api/backfills/test_create.py
+++ b/products/batch_exports/backend/tests/api/backfills/test_create.py
@@ -424,7 +424,7 @@ def test_batch_export_earliest_backfill_rejected_without_feature_flag(
batch_export_id = batch_export["id"]
with patch(
- "products.batch_exports.backend.presentation.views.batch_export.backfills.posthoganalytics.feature_enabled",
+ "products.batch_exports.backend.logic.backfills.posthoganalytics.feature_enabled",
return_value=False,
):
response = backfill_batch_export(
@@ -570,7 +570,7 @@ def test_batch_export_earliest_backfill_allowed_with_feature_flag(
batch_export_id = batch_export["id"]
with patch(
- "products.batch_exports.backend.presentation.views.batch_export.backfills.posthoganalytics.feature_enabled",
+ "products.batch_exports.backend.logic.backfills.posthoganalytics.feature_enabled",
return_value=True,
):
response = backfill_batch_export(
@@ -660,11 +660,11 @@ def test_batch_export_backfill_hogql_interval_validation(
with (
patch(
- "products.batch_exports.backend.presentation.views.batch_export.backfills.posthoganalytics.feature_enabled",
+ "products.batch_exports.backend.logic.backfills.posthoganalytics.feature_enabled",
return_value=True,
),
- patch("products.batch_exports.backend.presentation.views.batch_export.backfills.sync_connect") as connect,
- patch("products.batch_exports.backend.presentation.views.batch_export.backfills.backfill_export") as backfill,
+ patch("products.batch_exports.backend.logic.backfills.sync_connect") as connect,
+ patch("products.batch_exports.backend.logic.backfills.backfill_export") as backfill,
):
response = backfill_batch_export(client, team.pk, str(batch_export.pk), start_at, "2021-01-01T01:00:00+00:00")
diff --git a/products/batch_exports/backend/tests/api/backfills/test_list.py b/products/batch_exports/backend/tests/api/backfills/test_list.py
index 15eba685c0a9..7be66b71c49b 100644
--- a/products/batch_exports/backend/tests/api/backfills/test_list.py
+++ b/products/batch_exports/backend/tests/api/backfills/test_list.py
@@ -1,10 +1,12 @@
import datetime as dt
import pytest
+from unittest.mock import patch
from django.test.client import Client as HttpClient
from products.batch_exports.backend.models.batch_export import BatchExportBackfill
+from products.batch_exports.backend.presentation.views.batch_export.backfills import BackfillsCursorPagination
from products.batch_exports.backend.tests.api.fixtures import create_backfill, create_batch_export, create_destination
from products.batch_exports.backend.tests.api.operations import list_batch_export_backfills_ok
@@ -37,6 +39,32 @@ def test_list_batch_export_backfills(client: HttpClient, organization, team, use
assert len(response["results"]) == 2
[email protected]("ordering", ["start_at", "-start_at"])
+def test_list_batch_export_backfills_pages_through_tied_values(client: HttpClient, team, user, ordering):
+ batch_export = create_batch_export(team, create_destination())
+ early = dt.datetime(2021, 1, 1, tzinfo=dt.UTC)
+ late = dt.datetime(2021, 1, 2, tzinfo=dt.UTC)
+ backfill_ids = {
+ str(create_backfill(team, batch_export, start_at, None, BatchExportBackfill.Status.COMPLETED, None).id)
+ for start_at in [early, early, early, late, late]
+ }
+
+ client.force_login(user)
+ pages = []
+ with patch.object(BackfillsCursorPagination, "page_size", 2):
+ page = client.get(
+ f"/api/projects/{team.pk}/batch_exports/{batch_export.id}/backfills/?ordering={ordering}"
+ ).json()
+ pages.append(page)
+ # Bounded, so a cursor that never advances fails instead of hanging.
+ while page["next"] and len(pages) <= len(backfill_ids):
+ page = client.get(page["next"]).json()
+ pages.append(page)
+
+ listed = [backfill["id"] for page in pages for backfill in page["results"]]
+ assert sorted(listed) == sorted(backfill_ids)
+
+
def test_cannot_list_batch_export_backfills_for_other_organizations(client: HttpClient, organization, team, user):
"""
Should not be able to list batch export backfills for other organizations.
diff --git a/products/batch_exports/backend/tests/test_facade.py b/products/batch_exports/backend/tests/test_facade.py
index 2b35e3b27500..cd914a424555 100644
--- a/products/batch_exports/backend/tests/test_facade.py
+++ b/products/batch_exports/backend/tests/test_facade.py
@@ -269,6 +269,37 @@ def test_backfills_for_export_lists_every_backfill_newest_first_within_the_team(
assert api.list_backfills_for_export(export_id, other_team.pk) == []
[email protected]("scope", ["other_team", "other_export"])
+def test_runs_and_backfills_are_scoped_to_the_team_and_export(team, organization, scope):
+ export_id = _create_export(team)
+ run_id = _create_run(batch_export_id=export_id, finished_at=IN_WINDOW, status=BatchExportRun.Status.RUNNING)
+ backfill_id = testing.create_backfill(export_id, team_id=team.pk, status=BatchExportBackfill.Status.RUNNING)
+ if scope == "other_team":
+ team_id, scoped_export_id = create_team(organization=organization).pk, export_id
+ else:
+ team_id, scoped_export_id = team.pk, _create_export(team, name="other")
+ every_run = contracts.BatchExportRunFilters(
+ created=contracts.DatetimeRange(
+ start=dt.datetime(2000, 1, 1, tzinfo=dt.UTC), end=dt.datetime.now(dt.UTC) + dt.timedelta(days=1)
+ )
+ )
+ page = contracts.ListPage(ordering=("-created_at",), bound=None, offset=0, limit=10)
+
+ assert [run.id for run in api.list_batch_export_runs(team.pk, export_id, filters=every_run, page=page)] == [run_id]
+ assert api.list_batch_export_runs(team_id, scoped_export_id, filters=every_run, page=page) == []
+ assert api.list_batch_export_backfills(team_id, scoped_export_id, page=page) == []
+ # A leaked run or backfill would get past the lookup and reach Temporal instead of raising.
+ for read_run in (api.get_batch_export_run, api.retry_batch_export_run, api.cancel_batch_export_run):
+ with pytest.raises(contracts.BatchExportRunNotFoundError):
+ read_run(team_id, scoped_export_id, run_id, filters=every_run)
+ for read_backfill in (api.get_batch_export_backfill, api.cancel_batch_export_backfill):
+ with pytest.raises(contracts.BatchExportBackfillNotFoundError):
+ read_backfill(team_id, scoped_export_id, backfill_id)
+ if scope == "other_team":
+ with pytest.raises(contracts.BatchExportNotFoundError):
+ api.create_batch_export_backfill(team_id, export_id, start_at=None, end_at=None)
+
+
def test_deleting_team_batch_exports_continues_past_a_missing_schedule(team):
first = _create_export(team, name="first")
second = _create_export(team, name="second")
diff --git a/pyproject.toml b/pyproject.toml
index d5729ffd6ad6..600a244a4404 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -748,9 +748,6 @@ ignore_imports = [
# test helpers still reach in-product internals directly. Thin each one to parse -> facade
# -> serialize and delete each entry. backend:contract-check stays off until this list is
# empty. Only delete entries here: a new import from these views goes through the facade.
- "products.batch_exports.backend.presentation.views.batch_export.backfills -> products.batch_exports.backend.hogql_source",
- "products.batch_exports.backend.presentation.views.batch_export.backfills -> products.batch_exports.backend.models.batch_export",
- "products.batch_exports.backend.presentation.views.batch_export.backfills -> products.batch_exports.backend.service",
"products.batch_exports.backend.presentation.views.batch_export.destinations -> products.batch_exports.backend.models.batch_export",
"products.batch_exports.backend.presentation.views.batch_export.destinations -> products.batch_exports.backend.service",
"products.batch_exports.backend.presentation.views.batch_export.destinations -> products.batch_exports.backend.temporal.destinations.constants",
@@ -760,8 +757,6 @@ ignore_imports = [
"products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.service",
"products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.temporal.destinations.constants",
"products.batch_exports.backend.presentation.views.batch_export.exports -> products.batch_exports.backend.temporal.sql.events",
- "products.batch_exports.backend.presentation.views.batch_export.runs -> products.batch_exports.backend.models.batch_export",
- "products.batch_exports.backend.presentation.views.batch_export.runs -> products.batch_exports.backend.service",
# TODO: subscriptions presentation wave. The viewsets and model serializers moved from ee/
# with their ORM access: the product's own subscription models and the dashboard models
# behind dashboard and tile selection. Thin them to parse -> facade -> serialize and delete