Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(batch-exports): find older runs when retrying or cancelling (#118665)

Ross93d68e62e772
diff --git a/products/batch_exports/backend/facade/api.py b/products/batch_exports/backend/facade/api.py
index de1897537c37..df64a26e8fba 100644
--- a/products/batch_exports/backend/facade/api.py
+++ b/products/batch_exports/backend/facade/api.py
@@ -757,29 +757,26 @@ def get_file_download_url(team_id: int, run_id: UUID, part: str | None) -> str:
def list_batch_export_runs(
team_id: int, export_id: UUID, *, filters: contracts.BatchExportRunFilters, page: contracts.ListPage
) -> list[contracts.BatchExportRunDetail]:
- """Return one page of an export's runs that match ``filters``, ordered by ``created_at`` or ``data_interval_start``."""
+ """Return one page of an export's runs that match ``filters``, ordered by ``created_at`` or ``data_interval_start``.
+
+ A deleted export has no runs.
+ """
return _import_runs().list_runs(team_id, export_id, filters, page)
-def get_batch_export_run(
- team_id: int, export_id: UUID, run_id: UUID, *, filters: contracts.BatchExportRunFilters
-) -> contracts.BatchExportRunDetail:
- """Raises ``contracts.BatchExportRunNotFoundError`` unless the team has this run and it matches ``filters``."""
- return _import_runs().get_run(team_id, export_id, run_id, filters)
+def get_batch_export_run(team_id: int, export_id: UUID, run_id: UUID) -> contracts.BatchExportRunDetail:
+ """Raises ``contracts.BatchExportRunNotFoundError`` unless the team has this run under this export, and the export is not deleted."""
+ return _import_runs().get_run(team_id, export_id, run_id)
-def retry_batch_export_run(
- team_id: int, export_id: UUID, run_id: UUID, *, filters: contracts.BatchExportRunFilters
-) -> str:
+def retry_batch_export_run(team_id: int, export_id: UUID, run_id: UUID) -> str:
"""Backfill a run's data interval again and return the backfill id. Raises as ``get_batch_export_run`` does."""
- return _import_runs().retry_run(team_id, export_id, run_id, filters)
+ return _import_runs().retry_run(team_id, export_id, run_id)
-def cancel_batch_export_run(
- team_id: int, export_id: UUID, run_id: UUID, *, filters: contracts.BatchExportRunFilters
-) -> None:
+def cancel_batch_export_run(team_id: int, export_id: UUID, run_id: UUID) -> None:
"""Cancel a starting or running run. Raises ``contracts.InvalidBatchExportRunRequestError`` in any other status."""
- _import_runs().cancel_run(team_id, export_id, run_id, filters)
+ _import_runs().cancel_run(team_id, export_id, run_id)
def list_batch_export_backfills(
diff --git a/products/batch_exports/backend/logic/runs.py b/products/batch_exports/backend/logic/runs.py
index dce1acd651cf..6e5a41d2418f 100644
--- a/products/batch_exports/backend/logic/runs.py
+++ b/products/batch_exports/backend/logic/runs.py
@@ -43,25 +43,14 @@ def _to_run_detail(run: BatchExportRun) -> contracts.BatchExportRunDetail:
def _query_export_runs(team_id: int, export_id: UUID) -> QuerySet[BatchExportRun]:
- return BatchExportRun.objects.filter(batch_export_id=export_id, batch_export__team_id=team_id)
-
-
-def _filter_runs(runs: QuerySet[BatchExportRun], filters: contracts.BatchExportRunFilters) -> QuerySet[BatchExportRun]:
- if filters.created is not None:
- runs = runs.filter(created_at__range=(filters.created.start, filters.created.end))
- if filters.data_interval is not None:
- runs = runs.filter(
- data_interval_start__gte=filters.data_interval.start, data_interval_end__lte=filters.data_interval.end
- )
- if filters.statuses:
- runs = runs.filter(status__in=filters.statuses)
- return runs
+ return BatchExportRun.objects.filter(
+ batch_export_id=export_id, batch_export__team_id=team_id, batch_export__deleted=False
+ )
-def _get_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.BatchExportRunFilters) -> BatchExportRun:
- runs = _filter_runs(_query_export_runs(team_id, export_id), filters)
+def _get_run(team_id: int, export_id: UUID, run_id: UUID) -> BatchExportRun:
try:
- return runs.select_related("batch_export").get(id=run_id)
+ return _query_export_runs(team_id, export_id).select_related("batch_export").get(id=run_id)
except BatchExportRun.DoesNotExist as e:
raise contracts.BatchExportRunNotFoundError() from e
@@ -69,22 +58,28 @@ def _get_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.Bat
def list_runs(
team_id: int, export_id: UUID, filters: contracts.BatchExportRunFilters, page: contracts.ListPage
) -> list[contracts.BatchExportRunDetail]:
- runs = _filter_runs(_query_export_runs(team_id, export_id), filters)
+ runs = _query_export_runs(team_id, export_id)
+ if filters.created is not None:
+ runs = runs.filter(created_at__range=(filters.created.start, filters.created.end))
+ if filters.data_interval is not None:
+ runs = runs.filter(
+ data_interval_start__gte=filters.data_interval.start, data_interval_end__lte=filters.data_interval.end
+ )
+ if filters.statuses:
+ runs = runs.filter(status__in=filters.statuses)
return [
_to_run_detail(run)
for run in select_page(runs, page, ordering_fields=contracts.BATCH_EXPORT_RUN_ORDERING_FIELDS)
]
-def get_run(
- team_id: int, export_id: UUID, run_id: UUID, filters: contracts.BatchExportRunFilters
-) -> contracts.BatchExportRunDetail:
- return _to_run_detail(_get_run(team_id, export_id, run_id, filters))
+def get_run(team_id: int, export_id: UUID, run_id: UUID) -> contracts.BatchExportRunDetail:
+ return _to_run_detail(_get_run(team_id, export_id, run_id))
-def retry_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.BatchExportRunFilters) -> str:
+def retry_run(team_id: int, export_id: UUID, run_id: UUID) -> str:
"""Backfill the data interval of a run, and return the backfill id."""
- run = _get_run(team_id, export_id, run_id, filters)
+ run = _get_run(team_id, export_id, run_id)
return backfill_export(
sync_connect(),
str(run.batch_export_id),
@@ -94,8 +89,8 @@ def retry_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.Ba
)
-def cancel_run(team_id: int, export_id: UUID, run_id: UUID, filters: contracts.BatchExportRunFilters) -> None:
- run = _get_run(team_id, export_id, run_id, filters)
+def cancel_run(team_id: int, export_id: UUID, run_id: UUID) -> None:
+ run = _get_run(team_id, export_id, run_id)
if run.status not in _CANCELLABLE_STATUSES:
raise contracts.InvalidBatchExportRunRequestError(f"Cannot cancel a run that is in '{run.status}' status")
diff --git a/products/batch_exports/backend/presentation/views/batch_export/runs.py b/products/batch_exports/backend/presentation/views/batch_export/runs.py
index 4980a85325e9..849018839500 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/runs.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/runs.py
@@ -227,10 +227,9 @@ class BatchExportRunViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, mixins.Retrie
# run is read through the facade instead. That skips the mixin's team filter and object
# permission check, so both happen here: the facade filters by team and export, and
# check_object_permissions runs on the contract. `logs` reads the run through here too.
- # Like the list, a lookup only finds a run that matches the list filters.
try:
run = batch_exports_api.get_batch_export_run(
- self.team_id, self._get_export_id(), parse_path_uuid(self.kwargs["pk"]), filters=self._get_run_filters()
+ self.team_id, self._get_export_id(), parse_path_uuid(self.kwargs["pk"])
)
except contracts.BatchExportRunNotFoundError as e:
raise NotFound(*e.args) from e
@@ -257,9 +256,7 @@ class BatchExportRunViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, mixins.Retrie
run = self.get_object()
try:
- backfill_id = batch_exports_api.retry_batch_export_run(
- self.team_id, self._get_export_id(), run.id, filters=self._get_run_filters()
- )
+ backfill_id = batch_exports_api.retry_batch_export_run(self.team_id, self._get_export_id(), run.id)
except contracts.BatchExportRunNotFoundError as e:
raise NotFound(*e.args) from e
@@ -272,9 +269,7 @@ class BatchExportRunViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, mixins.Retrie
run = self.get_object()
try:
- batch_exports_api.cancel_batch_export_run(
- self.team_id, self._get_export_id(), run.id, filters=self._get_run_filters()
- )
+ batch_exports_api.cancel_batch_export_run(self.team_id, self._get_export_id(), run.id)
except contracts.InvalidBatchExportRunRequestError as e:
raise ValidationError(str(e)) from e
except contracts.BatchExportRunNotFoundError as e:
diff --git a/products/batch_exports/backend/tests/api/test_runs.py b/products/batch_exports/backend/tests/api/test_runs.py
index feca9dd397c4..803d6004117c 100644
--- a/products/batch_exports/backend/tests/api/test_runs.py
+++ b/products/batch_exports/backend/tests/api/test_runs.py
@@ -11,7 +11,7 @@ from rest_framework import status
from posthog.models.integration import Integration
-from products.batch_exports.backend.models.batch_export import BatchExportRun
+from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportRun
from products.batch_exports.backend.tests.api.fixtures import (
create_batch_export,
create_destination,
@@ -22,7 +22,6 @@ from products.batch_exports.backend.tests.api.fixtures import (
)
from products.batch_exports.backend.tests.api.operations import (
backfill_batch_export_ok,
- cancel_batch_export_run,
cancel_batch_export_run_ok,
create_batch_export_ok,
get_batch_export,
@@ -262,7 +261,22 @@ def test_get_batch_export_runs_date_filter_depends_on_ordering(client: HttpClien
assert [run["id"] for run in data["results"]] == [str(run_recent_interval.id), str(run_mid_interval.id)]
-def test_cannot_cancel_completed_batch_export_run(client: HttpClient, team, user):
+ "created_days_ago,query,export_deleted,expected_status",
+ [
+ pytest.param(0, "", False, status.HTTP_400_BAD_REQUEST, id="recent"),
+ pytest.param(30, "", False, status.HTTP_400_BAD_REQUEST, id="older-than-list-window"),
+ pytest.param(0, "?status=Running", False, status.HTTP_400_BAD_REQUEST, id="outside-status-query"),
+ pytest.param(0, "", True, status.HTTP_404_NOT_FOUND, id="deleted-export"),
+ ],
+)
+def test_cannot_cancel_completed_batch_export_run(
+ client: HttpClient, team, user, created_days_ago, query, export_deleted, expected_status
+):
+ """Cancel finds a run outside the list's filters, because the UI retries and cancels older runs.
+
+ A run of a deleted export is not found and not listed, as the export itself is not found.
+ """
destination = create_destination()
batch_export = create_batch_export(team, destination)
run = create_run(
@@ -271,12 +285,20 @@ def test_cannot_cancel_completed_batch_export_run(client: HttpClient, team, user
data_interval_start=dt.datetime(2023, 10, 23, tzinfo=dt.UTC),
data_interval_end=dt.datetime(2023, 10, 24, tzinfo=dt.UTC),
)
+ BatchExportRun.objects.filter(id=run.id).update(
+ created_at=dt.datetime.now(dt.UTC) - dt.timedelta(days=created_days_ago)
+ )
+ if export_deleted:
+ BatchExport.objects.filter(id=batch_export.id).update(deleted=True)
client.force_login(user)
- response = cancel_batch_export_run(client, team.pk, batch_export.id, run.id)
+ response = client.post(f"/api/projects/{team.pk}/batch_exports/{batch_export.id}/runs/{run.id}/cancel{query}")
- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
- assert response.json()["detail"] == "Cannot cancel a run that is in 'Completed' status"
+ assert response.status_code == expected_status, response.json()
+ if expected_status == status.HTTP_400_BAD_REQUEST:
+ assert response.json()["detail"] == "Cannot cancel a run that is in 'Completed' status"
+ if export_deleted:
+ assert get_batch_export_runs_ok(client, team.pk, batch_export.id)["results"] == []
run.refresh_from_db()
assert run.status == BatchExportRun.Status.COMPLETED
diff --git a/products/batch_exports/backend/tests/test_facade.py b/products/batch_exports/backend/tests/test_facade.py
index 8316a65438c3..6ed666e0dc3a 100644
--- a/products/batch_exports/backend/tests/test_facade.py
+++ b/products/batch_exports/backend/tests/test_facade.py
@@ -291,7 +291,7 @@ def test_runs_and_backfills_are_scoped_to_the_team_and_export(team, organization
# A leaked run or backfill would get past the lookup and reach Temporal instead of raising.
for read_run in (api.get_batch_export_run, api.retry_batch_export_run, api.cancel_batch_export_run):
with pytest.raises(contracts.BatchExportRunNotFoundError):
- read_run(team_id, scoped_export_id, run_id, filters=every_run)
+ read_run(team_id, scoped_export_id, run_id)
for read_backfill in (api.get_batch_export_backfill, api.cancel_batch_export_backfill):
with pytest.raises(contracts.BatchExportBackfillNotFoundError):
read_backfill(team_id, scoped_export_id, backfill_id)