:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(batch-exports): find older runs when retrying or cancelling (#118665)
Ross93d68e62e772
diff --git a/products/batch_exports/backend/facade/api.py b/products/batch_exports/backend/facade/api.pyindex de1897537c37..df64a26e8fba 100644--- a/products/batch_exports/backend/facade/api.py+++ b/products/batch_exports/backend/facade/api.py@@ -757,29 +757,26 @@ def get_file_download_url(team_id: int, run_id: UUID, part: str | None) -> str:def list_batch_export_runs(team_id: int, export_id: UUID, *, filters: contracts.BatchExportRunFilters, page: contracts.ListPage) -> list[contracts.BatchExportRunDetail]:- """Return one page of an export's runs that match ``filters``, ordered by ``created_at`` or ``data_interval_start``."""+ """Return one page of an export's runs that match ``filters``, ordered by ``created_at`` or ``data_interval_start``.++ A deleted export has no runs.+ """return _import_runs().list_runs(team_id, export_id, filters, page)-def get_batch_export_run(- team_id: int, export_id: UUID, run_id: UUID, *, filters: contracts.BatchExportRunFilters-) -> contracts.BatchExportRunDetail:- """Raises ``contracts.BatchExportRunNotFoundError`` unless the team has this run and it matches ``filters``."""- return _import_runs().get_run(team_id, export_id, run_id, filters)+def get_batch_export_run(team_id: int, export_id: UUID, run_id: UUID) -> contracts.BatchExportRunDetail:+ """Raises ``contracts.BatchExportRunNotFoundError`` unless the team has this run under this export, and the export is not deleted."""+ return _import_runs().get_run(team_id, export_id, run_id)-def retry_batch_export_run(- team_id: int, export_id: UUID, run_id: UUID, *, filters: contracts.BatchExportRunFilters-) -> str:+def retry_batch_export_run(team_id: int, export_id: UUID, run_id: UUID) -> str:"""Backfill a run's data interval again and return the backfill id. Raises as ``get_batch_export_run`` does."""- return _import_runs().retry_run(team_id, export_id, run_id, filters)+ return _import_runs().retry_run(team_id, export_id, run_id)-def cancel_batch_export_run(- team_id: int, export_id: UUID, run_id: UUID, *, filters: contracts.BatchExportRunFilters-) -> None:+def cancel_batch_export_run(team_id: int, export_id: UUID, run_id: UUID) -> None:"""Cancel a starting or running run. Raises ``contracts.InvalidBatchExportRunRequestError`` in any other status."""- _import_runs().cancel_run(team_id, export_id, run_id, filters)+ _import_runs().cancel_run(team_id, export_id, run_id)def list_batch_export_backfills(diff --git a/products/batch_exports/backend/logic/runs.py b/products/batch_exports/backend/logic/runs.pyindex dce1acd651cf..6e5a41d2418f 100644--- a/products/batch_exports/backend/logic/runs.py+++ b/products/batch_exports/backend/logic/runs.py@@ -43,25 +43,14 @@ def _to_run_detail(run: BatchExportRun) -> contracts.BatchExportRunDetail:def _query_export_runs(team_id: int, export_id: UUID) -> QuerySet[BatchExportRun]:- return BatchExportRun.objects.filter(batch_export_id=export_id, batch_export__team_id=team_id)---def _filter_runs(runs: QuerySet[BatchExportRun], filters: contracts.BatchExportRunFilters) -> QuerySet[BatchExportRun]:- if filters.created is not None:- runs = runs.filter(created_at__range=(filters.created.start, filters.created.end))- if filters.data_interval is not None:- runs = runs.filter(- data_interval_start__gte=filters.data_interval.start, data_interval_end__lte=filters.data_interval.end- )- if filters.statuses:- runs = runs.filter(status__in=filters.statuses)- return runs+ return BatchExportRun.objects.filter(+ batch_export_id=export_id, batch_export__team_id=team_id, batch_export__deleted=False+ )-def _get_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.BatchExportRunFilters) -> BatchExportRun:- runs = _filter_runs(_query_export_runs(team_id, export_id), filters)+def _get_run(team_id: int, export_id: UUID, run_id: UUID) -> BatchExportRun:try:- return runs.select_related("batch_export").get(id=run_id)+ return _query_export_runs(team_id, export_id).select_related("batch_export").get(id=run_id)except BatchExportRun.DoesNotExist as e:raise contracts.BatchExportRunNotFoundError() from e@@ -69,22 +58,28 @@ def _get_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.Batdef list_runs(team_id: int, export_id: UUID, filters: contracts.BatchExportRunFilters, page: contracts.ListPage) -> list[contracts.BatchExportRunDetail]:- runs = _filter_runs(_query_export_runs(team_id, export_id), filters)+ runs = _query_export_runs(team_id, export_id)+ if filters.created is not None:+ runs = runs.filter(created_at__range=(filters.created.start, filters.created.end))+ if filters.data_interval is not None:+ runs = runs.filter(+ data_interval_start__gte=filters.data_interval.start, data_interval_end__lte=filters.data_interval.end+ )+ if filters.statuses:+ runs = runs.filter(status__in=filters.statuses)return [_to_run_detail(run)for run in select_page(runs, page, ordering_fields=contracts.BATCH_EXPORT_RUN_ORDERING_FIELDS)]-def get_run(- team_id: int, export_id: UUID, run_id: UUID, filters: contracts.BatchExportRunFilters-) -> contracts.BatchExportRunDetail:- return _to_run_detail(_get_run(team_id, export_id, run_id, filters))+def get_run(team_id: int, export_id: UUID, run_id: UUID) -> contracts.BatchExportRunDetail:+ return _to_run_detail(_get_run(team_id, export_id, run_id))-def retry_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.BatchExportRunFilters) -> str:+def retry_run(team_id: int, export_id: UUID, run_id: UUID) -> str:"""Backfill the data interval of a run, and return the backfill id."""- run = _get_run(team_id, export_id, run_id, filters)+ run = _get_run(team_id, export_id, run_id)return backfill_export(sync_connect(),str(run.batch_export_id),@@ -94,8 +89,8 @@ def retry_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.Ba)-def cancel_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.BatchExportRunFilters) -> None:- run = _get_run(team_id, export_id, run_id, filters)+def cancel_run(team_id: int, export_id: UUID, run_id: UUID) -> None:+ run = _get_run(team_id, export_id, run_id)if run.status not in _CANCELLABLE_STATUSES:raise contracts.InvalidBatchExportRunRequestError(f"Cannot cancel a run that is in '{run.status}' status")diff --git a/products/batch_exports/backend/presentation/views/batch_export/runs.py b/products/batch_exports/backend/presentation/views/batch_export/runs.pyindex 4980a85325e9..849018839500 100644--- a/products/batch_exports/backend/presentation/views/batch_export/runs.py+++ b/products/batch_exports/backend/presentation/views/batch_export/runs.py@@ -227,10 +227,9 @@ class BatchExportRunViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, mixins.Retrie# run is read through the facade instead. That skips the mixin's team filter and object# permission check, so both happen here: the facade filters by team and export, and# check_object_permissions runs on the contract. `logs` reads the run through here too.- # Like the list, a lookup only finds a run that matches the list filters.try:run = batch_exports_api.get_batch_export_run(- self.team_id, self._get_export_id(), parse_path_uuid(self.kwargs["pk"]), filters=self._get_run_filters()+ self.team_id, self._get_export_id(), parse_path_uuid(self.kwargs["pk"]))except contracts.BatchExportRunNotFoundError as e:raise NotFound(*e.args) from e@@ -257,9 +256,7 @@ class BatchExportRunViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, mixins.Retrierun = self.get_object()try:- backfill_id = batch_exports_api.retry_batch_export_run(- self.team_id, self._get_export_id(), run.id, filters=self._get_run_filters()- )+ backfill_id = batch_exports_api.retry_batch_export_run(self.team_id, self._get_export_id(), run.id)except contracts.BatchExportRunNotFoundError as e:raise NotFound(*e.args) from e@@ -272,9 +269,7 @@ class BatchExportRunViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, mixins.Retrierun = self.get_object()try:- batch_exports_api.cancel_batch_export_run(- self.team_id, self._get_export_id(), run.id, filters=self._get_run_filters()- )+ batch_exports_api.cancel_batch_export_run(self.team_id, self._get_export_id(), run.id)except contracts.InvalidBatchExportRunRequestError as e:raise ValidationError(str(e)) from eexcept contracts.BatchExportRunNotFoundError as e:diff --git a/products/batch_exports/backend/tests/api/test_runs.py b/products/batch_exports/backend/tests/api/test_runs.pyindex feca9dd397c4..803d6004117c 100644--- a/products/batch_exports/backend/tests/api/test_runs.py+++ b/products/batch_exports/backend/tests/api/test_runs.py@@ -11,7 +11,7 @@ from rest_framework import statusfrom posthog.models.integration import Integration-from products.batch_exports.backend.models.batch_export import BatchExportRun+from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportRunfrom products.batch_exports.backend.tests.api.fixtures import (create_batch_export,create_destination,@@ -22,7 +22,6 @@ from products.batch_exports.backend.tests.api.fixtures import ()from products.batch_exports.backend.tests.api.operations import (backfill_batch_export_ok,- cancel_batch_export_run,cancel_batch_export_run_ok,create_batch_export_ok,get_batch_export,@@ -262,7 +261,22 @@ def test_get_batch_export_runs_date_filter_depends_on_ordering(client: HttpClienassert [run["id"] for run in data["results"]] == [str(run_recent_interval.id), str(run_mid_interval.id)]-def test_cannot_cancel_completed_batch_export_run(client: HttpClient, team, user):+ "created_days_ago,query,export_deleted,expected_status",+ [+ pytest.param(0, "", False, status.HTTP_400_BAD_REQUEST, id="recent"),+ pytest.param(30, "", False, status.HTTP_400_BAD_REQUEST, id="older-than-list-window"),+ pytest.param(0, "?status=Running", False, status.HTTP_400_BAD_REQUEST, id="outside-status-query"),+ pytest.param(0, "", True, status.HTTP_404_NOT_FOUND, id="deleted-export"),+ ],+)+def test_cannot_cancel_completed_batch_export_run(+ client: HttpClient, team, user, created_days_ago, query, export_deleted, expected_status+):+ """Cancel finds a run outside the list's filters, because the UI retries and cancels older runs.++ A run of a deleted export is not found and not listed, as the export itself is not found.+ """destination = create_destination()batch_export = create_batch_export(team, destination)run = create_run(@@ -271,12 +285,20 @@ def test_cannot_cancel_completed_batch_export_run(client: HttpClient, team, userdata_interval_start=dt.datetime(2023, 10, 23, tzinfo=dt.UTC),data_interval_end=dt.datetime(2023, 10, 24, tzinfo=dt.UTC),)+ BatchExportRun.objects.filter(id=run.id).update(+ created_at=dt.datetime.now(dt.UTC) - dt.timedelta(days=created_days_ago)+ )+ if export_deleted:+ BatchExport.objects.filter(id=batch_export.id).update(deleted=True)client.force_login(user)- response = cancel_batch_export_run(client, team.pk, batch_export.id, run.id)+ response = client.post(f"/api/projects/{team.pk}/batch_exports/{batch_export.id}/runs/{run.id}/cancel{query}")- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()- assert response.json()["detail"] == "Cannot cancel a run that is in 'Completed' status"+ assert response.status_code == expected_status, response.json()+ if expected_status == status.HTTP_400_BAD_REQUEST:+ assert response.json()["detail"] == "Cannot cancel a run that is in 'Completed' status"+ if export_deleted:+ assert get_batch_export_runs_ok(client, team.pk, batch_export.id)["results"] == []run.refresh_from_db()assert run.status == BatchExportRun.Status.COMPLETEDdiff --git a/products/batch_exports/backend/tests/test_facade.py b/products/batch_exports/backend/tests/test_facade.pyindex 8316a65438c3..6ed666e0dc3a 100644--- a/products/batch_exports/backend/tests/test_facade.py+++ b/products/batch_exports/backend/tests/test_facade.py@@ -291,7 +291,7 @@ def test_runs_and_backfills_are_scoped_to_the_team_and_export(team, organization# A leaked run or backfill would get past the lookup and reach Temporal instead of raising.for read_run in (api.get_batch_export_run, api.retry_batch_export_run, api.cancel_batch_export_run):with pytest.raises(contracts.BatchExportRunNotFoundError):- read_run(team_id, scoped_export_id, run_id, filters=every_run)+ read_run(team_id, scoped_export_id, run_id)for read_backfill in (api.get_batch_export_backfill, api.cancel_batch_export_backfill):with pytest.raises(contracts.BatchExportBackfillNotFoundError):read_backfill(team_id, scoped_export_id, backfill_id)