Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(devex): page batch exports and subscriptions with the shared kit (#126449)

Julian Bezb21ad7021687
diff --git a/products/batch_exports/backend/facade/api.py b/products/batch_exports/backend/facade/api.py
index 0f2d687838da..59535bf28c2e 100644
--- a/products/batch_exports/backend/facade/api.py
+++ b/products/batch_exports/backend/facade/api.py
@@ -33,6 +33,7 @@ from django.db.models.functions import Coalesce
import structlog
from posthog.integrations import IntegrationManager
+from posthog.paging import CursorPageRequest, Page, PageRequest
from products.batch_exports.backend import (
destination_tests,
@@ -496,14 +497,12 @@ def get_batch_export(
return _import_exports().get_batch_export(team_id, export_id, include_latest_runs=include_latest_runs)
-def list_batch_exports(
- team_id: int, *, offset: int, limit: int
-) -> tuple[list[contracts.BatchExportConfiguration], int]:
+def list_batch_exports(team_id: int, *, page: PageRequest) -> Page[contracts.BatchExportConfiguration]:
"""Return one page of the team's scheduled batch exports, newest first, and how many there are in total.
Deleted exports, and the exports that back Workflows backfills, are left out.
"""
- return _import_exports().list_batch_exports(team_id, offset=offset, limit=limit)
+ return _import_exports().list_batch_exports(team_id, page=page)
def create_batch_export_from_input(
@@ -718,12 +717,9 @@ def count_hogql_rows(
)
-def list_file_download_runs(
- team_id: int, *, offset: int, limit: int
-) -> tuple[list[contracts.BatchExportRunSummary], int]:
+def list_file_download_runs(team_id: int, *, page: PageRequest) -> Page[contracts.BatchExportRunSummary]:
"""Return one page of the team's file download runs, newest first, and how many there are in total."""
- runs, count = _import_file_downloads().list_file_download_runs(team_id, offset=offset, limit=limit)
- return [_to_run_summary(run) for run in runs], count
+ return _import_file_downloads().list_file_download_runs(team_id, page=page).map(_to_run_summary)
def get_file_download_run(team_id: int, run_id: UUID) -> contracts.FileDownloadRun:
@@ -762,8 +758,8 @@ def get_file_download_url(team_id: int, run_id: UUID, part: str | None) -> str:
def list_batch_export_runs(
- team_id: int, export_id: UUID, *, filters: contracts.BatchExportRunFilters, page: contracts.ListPage
-) -> list[contracts.BatchExportRunDetail]:
+ team_id: int, export_id: UUID, *, filters: contracts.BatchExportRunFilters, page: CursorPageRequest
+) -> Page[contracts.BatchExportRunDetail]:
"""Return one page of an export's runs that match ``filters``, ordered by ``created_at`` or ``data_interval_start``.
A deleted export has no runs.
@@ -787,8 +783,8 @@ def cancel_batch_export_run(team_id: int, export_id: UUID, run_id: UUID) -> None
def list_batch_export_backfills(
- team_id: int, export_id: UUID, *, page: contracts.ListPage
-) -> list[contracts.BatchExportBackfillDetail]:
+ team_id: int, export_id: UUID, *, page: CursorPageRequest
+) -> Page[contracts.BatchExportBackfillDetail]:
"""Return one page of an export's backfills, ordered by ``created_at`` or ``start_at``."""
return _import_backfills().list_backfills(team_id, export_id, page)
diff --git a/products/batch_exports/backend/facade/contracts.py b/products/batch_exports/backend/facade/contracts.py
index dbda75ba208c..961f7a073865 100644
--- a/products/batch_exports/backend/facade/contracts.py
+++ b/products/batch_exports/backend/facade/contracts.py
@@ -259,33 +259,6 @@ BATCH_EXPORT_RUN_ORDERING_FIELDS = ("created_at", "data_interval_start")
BATCH_EXPORT_BACKFILL_ORDERING_FIELDS = ("created_at", "start_at")
-@dataclass(frozen=True)
-class ListPageBound:
- """Where a page of an ordered list starts.
-
- Only the items whose first ordering field sorts before ``value`` (when ``before``) or after
- it (otherwise) are on the page. ``value`` is the field's value as text, as an API cursor carries it.
- """
-
- value: str
- before: bool
-
-
-@dataclass(frozen=True)
-class ListPage:
- """One page of an ordered list.
-
- ``ordering`` names the fields to sort by, most significant first. A leading ``-`` sorts that
- field in descending order. The page skips ``offset`` items past ``bound`` and holds at most
- ``limit`` items.
- """
-
- ordering: tuple[str, ...]
- bound: ListPageBound | None
- offset: int
- limit: int
-
-
@dataclass(frozen=True)
class DatetimeRange:
"""A range of time that includes both of its ends."""
diff --git a/products/batch_exports/backend/logic/backfills.py b/products/batch_exports/backend/logic/backfills.py
index 9af8902a4b97..ee9d3fd1d6e9 100644
--- a/products/batch_exports/backend/logic/backfills.py
+++ b/products/batch_exports/backend/logic/backfills.py
@@ -16,6 +16,8 @@ import structlog
import posthoganalytics
from posthog.models import Team
+from posthog.paging import CursorPageRequest, Page
+from posthog.queryset_paging import select_cursor_page
from posthog.temporal.common.client import sync_connect
from products.batch_exports.backend.facade import contracts
@@ -26,7 +28,6 @@ from products.batch_exports.backend.hogql_source import (
parse_hogql_select_for_batch_export,
)
from products.batch_exports.backend.logic.exports import load_batch_export
-from products.batch_exports.backend.logic.pagination import select_page
from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportBackfill
from products.batch_exports.backend.service import (
BatchExportWithNoEndNotAllowedError,
@@ -163,15 +164,12 @@ def _get_backfill(team_id: int, export_id: UUID, backfill_id: UUID) -> BatchExpo
raise contracts.BatchExportBackfillNotFoundError() from e
-def list_backfills(
- team_id: int, export_id: UUID, page: contracts.ListPage
-) -> list[contracts.BatchExportBackfillDetail]:
- backfills = select_page(
+def list_backfills(team_id: int, export_id: UUID, page: CursorPageRequest) -> Page[contracts.BatchExportBackfillDetail]:
+ return select_cursor_page(
_query_export_backfills(team_id, export_id),
page,
ordering_fields=contracts.BATCH_EXPORT_BACKFILL_ORDERING_FIELDS,
- )
- return [_to_backfill_detail(backfill) for backfill in backfills]
+ ).map(_to_backfill_detail)
def get_backfill(team_id: int, export_id: UUID, backfill_id: UUID) -> contracts.BatchExportBackfillDetail:
diff --git a/products/batch_exports/backend/logic/exports.py b/products/batch_exports/backend/logic/exports.py
index b111b857618c..9e5ad784ff06 100644
--- a/products/batch_exports/backend/logic/exports.py
+++ b/products/batch_exports/backend/logic/exports.py
@@ -23,6 +23,8 @@ from posthog.models.integration import (
SnowflakeIntegration,
SnowflakeIntegrationError,
)
+from posthog.paging import Page, PageRequest
+from posthog.queryset_paging import select_page
from posthog.security.url_validation import (
INVALID_HOST_MESSAGE,
UNREACHABLE_HOST_MESSAGE,
@@ -629,9 +631,7 @@ def get_batch_export(team_id: int, export_id: UUID, *, include_latest_runs: bool
return _to_configuration(batch_export, include_latest_runs=include_latest_runs)
-def list_batch_exports(
- team_id: int, *, offset: int, limit: int
-) -> tuple[list[contracts.BatchExportConfiguration], int]:
+def list_batch_exports(team_id: int, *, page: PageRequest) -> Page[contracts.BatchExportConfiguration]:
batch_exports = (
BatchExport.objects.exclude(deleted=True)
.filter(team_id=team_id)
@@ -640,8 +640,9 @@ def list_batch_exports(
.order_by("-created_at", "-id")
.select_related("destination", "source")
)
- page = batch_exports[offset : offset + limit]
- return [_to_configuration(batch_export, include_latest_runs=True) for batch_export in page], batch_exports.count()
+ return select_page(batch_exports, page).map(
+ lambda batch_export: _to_configuration(batch_export, include_latest_runs=True)
+ )
def schedule_and_save(batch_export: BatchExport) -> None:
diff --git a/products/batch_exports/backend/logic/file_downloads.py b/products/batch_exports/backend/logic/file_downloads.py
index c5826c1dc845..aca0d9d14919 100644
--- a/products/batch_exports/backend/logic/file_downloads.py
+++ b/products/batch_exports/backend/logic/file_downloads.py
@@ -23,6 +23,8 @@ from botocore.exceptions import ClientError
from posthog.hogql.query import execute_hogql_query
from posthog.models import Team, User
+from posthog.paging import Page, PageRequest
+from posthog.queryset_paging import select_page
from posthog.temporal.common.client import sync_connect
from products.batch_exports.backend.facade import contracts
@@ -116,9 +118,8 @@ def _to_file_download_run(batch_export_run: BatchExportRun) -> contracts.FileDow
)
-def list_file_download_runs(team_id: int, *, offset: int, limit: int) -> tuple[list[BatchExportRun], int]:
- runs = _file_download_runs(team_id)
- return list(runs[offset : offset + limit]), runs.count()
+def list_file_download_runs(team_id: int, *, page: PageRequest) -> Page[BatchExportRun]:
+ return select_page(_file_download_runs(team_id), page)
def get_file_download_run(team_id: int, run_id: UUID) -> contracts.FileDownloadRun:
diff --git a/products/batch_exports/backend/logic/pagination.py b/products/batch_exports/backend/logic/pagination.py
deleted file mode 100644
index 2ac917a34657..000000000000
--- a/products/batch_exports/backend/logic/pagination.py
+++ /dev/null
@@ -1,35 +0,0 @@
-"""Read one page of an ordered list of rows, as ``contracts.ListPage`` asks for it."""
-
-from collections.abc import Collection
-
-from django.db.models import Model, QuerySet
-
-from posthog.queryset_paging import stable_queryset_ordering
-
-from products.batch_exports.backend.facade import contracts
-
-
-# TODO: Nothing here is specific to batch exports. Move it to posthog/api/pagination.py with the cursor
-# page source in presentation/views/batch_export/pagination.py, and a core page type to replace
-# contracts.ListPage, so other facade-backed products can page with a cursor too.
-def select_page[M: Model](
- queryset: QuerySet[M], page: contracts.ListPage, *, ordering_fields: Collection[str]
-) -> list[M]:
- """Return the rows of one page of ``queryset``.
-
- The rows are also sorted by primary key, in the direction of the first ordering field. Without
- it, rows that share a value can change places between two queries, so a page boundary between
- them can skip or repeat a row.
- """
- for term in page.ordering:
- if term.lstrip("-") not in ordering_fields:
- raise ValueError(f"Cannot order by {term!r}")
-
- first = page.ordering[0]
- queryset = stable_queryset_ordering(queryset.order_by(*page.ordering))
-
- if page.bound is not None:
- comparison = "lt" if page.bound.before else "gt"
- queryset = queryset.filter(**{f"{first.lstrip('-')}__{comparison}": page.bound.value})
-
- return list(queryset[page.offset : page.offset + page.limit])
diff --git a/products/batch_exports/backend/logic/runs.py b/products/batch_exports/backend/logic/runs.py
index b6bac57e2c3d..222770957c9f 100644
--- a/products/batch_exports/backend/logic/runs.py
+++ b/products/batch_exports/backend/logic/runs.py
@@ -10,10 +10,11 @@ from django.db.models import QuerySet
import structlog
+from posthog.paging import CursorPageRequest, Page
+from posthog.queryset_paging import select_cursor_page
from posthog.temporal.common.client import sync_connect
from products.batch_exports.backend.facade import contracts
-from products.batch_exports.backend.logic.pagination import select_page
from products.batch_exports.backend.models.batch_export import BatchExportRun
from products.batch_exports.backend.service import backfill_export, cancel_running_batch_export_run
@@ -56,8 +57,8 @@ def _get_run(team_id: int, export_id: UUID, run_id: UUID) -> BatchExportRun:
def list_runs(
- team_id: int, export_id: UUID, filters: contracts.BatchExportRunFilters, page: contracts.ListPage
-) -> list[contracts.BatchExportRunDetail]:
+ team_id: int, export_id: UUID, filters: contracts.BatchExportRunFilters, page: CursorPageRequest
+) -> Page[contracts.BatchExportRunDetail]:
runs = _query_export_runs(team_id, export_id)
if filters.created is not None:
runs = runs.filter(created_at__range=(filters.created.start, filters.created.end))
@@ -67,10 +68,7 @@ def list_runs(
)
if filters.statuses:
runs = runs.filter(status__in=filters.statuses)
- return [
- to_run_detail(run)
- for run in select_page(runs, page, ordering_fields=contracts.BATCH_EXPORT_RUN_ORDERING_FIELDS)
- ]
+ return select_cursor_page(runs, page, ordering_fields=contracts.BATCH_EXPORT_RUN_ORDERING_FIELDS).map(to_run_detail)
def get_run(team_id: int, export_id: UUID, run_id: UUID) -> contracts.BatchExportRunDetail:
diff --git a/products/batch_exports/backend/presentation/views/batch_export/backfills.py b/products/batch_exports/backend/presentation/views/batch_export/backfills.py
index 4c7bc85efe09..b8a382a5b24c 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/backfills.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/backfills.py
@@ -2,7 +2,7 @@
import uuid
import dataclasses
-from typing import Any, cast
+from typing import Any
import structlog
import posthoganalytics
@@ -11,6 +11,7 @@ from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema
from rest_framework import filters, mixins, request, response, serializers, status, viewsets
from rest_framework.exceptions import APIException, NotFound, ValidationError
+from posthog.api.pagination import FacadeCursorPagination, facade_cursor_page_response
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.utils import action
from posthog.event_usage import groups
@@ -22,7 +23,6 @@ from products.batch_exports.backend.facade import (
contracts,
)
from products.batch_exports.backend.facade.enums import BatchExportBackfillStatus
-from products.batch_exports.backend.presentation.views.batch_export.pagination import FacadeCursorPagination
from products.batch_exports.backend.presentation.views.utils import parse_path_uuid
logger = structlog.get_logger(__name__)
@@ -144,8 +144,7 @@ class BatchExportBackfillViewSet(TeamAndOrgViewSetMixin, mixins.RetrieveModelMix
def list(self, request: request.Request, *args: Any, **kwargs: Any) -> response.Response:
export_id = self._get_export_id()
- return cast(FacadeCursorPagination, self.paginator).get_paginated_facade_response(
- request,
+ return facade_cursor_page_response(
self,
lambda page: batch_exports_api.list_batch_export_backfills(self.team_id, export_id, page=page),
)
diff --git a/products/batch_exports/backend/presentation/views/batch_export/exports.py b/products/batch_exports/backend/presentation/views/batch_export/exports.py
index 14458b1e4148..6567f8e45a0a 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/exports.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/exports.py
@@ -3,16 +3,15 @@
import uuid
import typing
import dataclasses
-from typing import cast
import posthoganalytics
from drf_spectacular.types import OpenApiTypes
from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view
from rest_framework import request, response, serializers, status, viewsets
from rest_framework.exceptions import NotAuthenticated, NotFound, PermissionDenied, ValidationError
-from rest_framework.pagination import LimitOffsetPagination
from posthog.api.log_entries import LogEntryMixin
+from posthog.api.pagination import FacadeLimitOffsetPagination, facade_page_response
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.utils import action
from posthog.integrations import IntegrationRecord
@@ -44,7 +43,6 @@ from products.batch_exports.backend.presentation.views.utils import (
HogQLModifiersField,
check_high_frequency_batch_exports_enabled,
check_hogql_batch_exports_enabled,
- fetch_facade_page,
parse_path_uuid,
)
@@ -677,6 +675,7 @@ BATCH_EXPORT_ID_PARAMETER = OpenApiParameter(
class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelViewSet):
scope_object = "batch_export"
serializer_class = BatchExportSerializer
+ pagination_class = FacadeLimitOffsetPagination
log_source = "batch_exports"
def dangerously_get_object(self) -> contracts.BatchExportConfiguration:
@@ -695,13 +694,10 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVi
return batch_export
def list(self, request: request.Request, *args: typing.Any, **kwargs: typing.Any) -> response.Response:
- paginator = cast(LimitOffsetPagination, self.paginator)
- batch_exports = fetch_facade_page(
- paginator,
- request,
- lambda offset, limit: batch_exports_api.list_batch_exports(self.team_id, offset=offset, limit=limit),
+ return facade_page_response(
+ self,
+ lambda page: batch_exports_api.list_batch_exports(self.team_id, page=page),
)
- return paginator.get_paginated_response(self.get_serializer(batch_exports, many=True).data)
@extend_schema(request=None, parameters=[BATCH_EXPORT_ID_PARAMETER])
@action(methods=["POST"], detail=True, required_scopes=["batch_export:write"])
diff --git a/products/batch_exports/backend/presentation/views/batch_export/pagination.py b/products/batch_exports/backend/presentation/views/batch_export/pagination.py
deleted file mode 100644
index 81f001bbe1f8..000000000000
--- a/products/batch_exports/backend/presentation/views/batch_export/pagination.py
+++ /dev/null
@@ -1,59 +0,0 @@
-"""Cursor pagination over lists that the facade reads."""
-
-from collections.abc import Callable
-from typing import Any, cast
-
-from rest_framework.generics import GenericAPIView
-from rest_framework.pagination import CursorPagination
-from rest_framework.request import Request
-from rest_framework.response import Response
-
-from products.batch_exports.backend.facade import contracts
-
-
-# TODO: Nothing here is specific to batch exports. Move it to posthog/api/pagination.py with
-# select_page in logic/pagination.py, and a core page type to replace contracts.ListPage, so other
-# facade-backed products can page with a cursor too.
-class _FacadePageSource[T]:
- """Stands in for the queryset that ``CursorPagination.paginate_queryset`` pages through.
-
- The paginator only orders the queryset, filters it on the cursor position and slices it.
- This records the ordering and the position, and the slice reads that page through the
- facade. So DRF still decodes and encodes every cursor, and the cursors stay the same.
- """
-
- def __init__(
- self,
- fetch: Callable[[contracts.ListPage], list[T]],
- ordering: tuple[str, ...] = (),
- bound: contracts.ListPageBound | None = None,
- ) -> None:
- self._fetch = fetch
- self._ordering = ordering
- self._bound = bound
-
- def order_by(self, *ordering: str) -> "_FacadePageSource[T]":
- return _FacadePageSource(self._fetch, ordering, self._bound)
-
- def filter(self, **lookup: str) -> "_FacadePageSource[T]":
- # The only filter is the position, as `<first ordering field>__lt` or `__gt`.
- ((key, value),) = lookup.items()
- bound = contracts.ListPageBound(value=value, before=key.endswith("__lt"))
- return _FacadePageSource(self._fetch, self._ordering, bound)
-
- def __getitem__(self, items: slice) -> list[T]:
- page = contracts.ListPage(
- ordering=self._ordering, bound=self._bound, offset=items.start, limit=items.stop - items.start
- )
- return self._fetch(page)
-
-
-class FacadeCursorPagination(CursorPagination):
- def get_paginated_facade_response[T](
- self, request: Request, view: GenericAPIView, fetch: Callable[[contracts.ListPage], list[T]]
- ) -> Response:
- """Return the page the request asks for, serialized by the view. ``fetch`` reads a page through the facade."""
- page = self.paginate_queryset(cast(Any, _FacadePageSource(fetch)), request, view)
- # The page size is a class attribute, so the paginator always returns a page.
- assert page is not None
- return self.get_paginated_response(view.get_serializer(page, many=True).data)
diff --git a/products/batch_exports/backend/presentation/views/batch_export/runs.py b/products/batch_exports/backend/presentation/views/batch_export/runs.py
index 2acf780fe8bc..e0dbc9fbe074 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/runs.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/runs.py
@@ -11,6 +11,7 @@ from rest_framework import filters, mixins, request, response, serializers, stat
from rest_framework.exceptions import NotFound, ValidationError
from posthog.api.log_entries import LogEntryMixin
+from posthog.api.pagination import FacadeCursorPagination, facade_cursor_page_response
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.utils import action
from posthog.models.integer_field_bounds import (
@@ -26,7 +27,6 @@ from products.batch_exports.backend.facade import (
contracts,
)
from products.batch_exports.backend.facade.enums import BatchExportRunStatus
-from products.batch_exports.backend.presentation.views.batch_export.pagination import FacadeCursorPagination
from products.batch_exports.backend.presentation.views.utils import parse_path_uuid
# The viewset declares no queryset, so drf-spectacular cannot read the type of the ids from a model.
@@ -239,8 +239,7 @@ class BatchExportRunViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, mixins.Retrie
def list(self, request: request.Request, *args: Any, **kwargs: Any) -> response.Response:
export_id = self._get_export_id()
filters = self._get_run_filters()
- return cast(FacadeCursorPagination, self.paginator).get_paginated_facade_response(
- request,
+ return facade_cursor_page_response(
self,
lambda page: batch_exports_api.list_batch_export_runs(self.team_id, export_id, filters=filters, page=page),
)
diff --git a/products/batch_exports/backend/presentation/views/file_download.py b/products/batch_exports/backend/presentation/views/file_download.py
index 20d95ccbf22c..2d3c2b10cb64 100644
--- a/products/batch_exports/backend/presentation/views/file_download.py
+++ b/products/batch_exports/backend/presentation/views/file_download.py
@@ -1,5 +1,5 @@
import datetime as dt
-from typing import Any, cast
+from typing import Any
from django.db import models
from django.http import HttpResponseRedirect
@@ -14,13 +14,13 @@ from drf_spectacular.utils import (
)
from rest_framework import response, serializers, status, viewsets
from rest_framework.exceptions import APIException, NotAuthenticated, NotFound, ValidationError
-from rest_framework.pagination import LimitOffsetPagination
from rest_framework.throttling import BaseThrottle
from posthog.hogql.errors import ExposedHogQLError
from posthog.api.log_entries import LogEntryMixin
from posthog.api.mixins import ValidatedRequest, validated_request
+from posthog.api.pagination import FacadeLimitOffsetPagination, facade_page_response
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.utils import action
from posthog.errors import ExposedCHQueryError
@@ -37,7 +37,6 @@ from products.batch_exports.backend.presentation.views.utils import (
HOGQL_MODIFIERS_HELP_TEXT,
HogQLModifiersField,
check_hogql_batch_exports_enabled,
- fetch_facade_page,
parse_path_uuid,
)
@@ -383,6 +382,7 @@ def _to_file_download_request(validated_data: dict[str, Any]) -> contracts.FileD
class FileDownloadBatchExportOnDemandViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.GenericViewSet):
scope_object = "batch_export"
serializer_class = FileDownloadBatchExportOnDemandSerializer
+ pagination_class = FacadeLimitOffsetPagination
log_source = "batch_exports"
def get_serializer_class(self) -> type[serializers.Serializer]:
@@ -412,13 +412,10 @@ class FileDownloadBatchExportOnDemandViewSet(TeamAndOrgViewSetMixin, LogEntryMix
return run
def list(self, request, *args, **kwargs) -> response.Response:
- paginator = cast(LimitOffsetPagination, self.paginator)
- runs = fetch_facade_page(
- paginator,
- request,
- lambda offset, limit: batch_exports_api.list_file_download_runs(self.team_id, offset=offset, limit=limit),
+ return facade_page_response(
+ self,
+ lambda page: batch_exports_api.list_file_download_runs(self.team_id, page=page),
)
- return paginator.get_paginated_response(ListOutputSerializer(runs, many=True).data)
@extend_schema(
request=PolymorphicProxySerializer(
diff --git a/products/batch_exports/backend/presentation/views/utils.py b/products/batch_exports/backend/presentation/views/utils.py
index 3b880d603f5d..23702babc9b9 100644
--- a/products/batch_exports/backend/presentation/views/utils.py
+++ b/products/batch_exports/backend/presentation/views/utils.py
@@ -1,14 +1,11 @@
import uuid
import typing
-from collections.abc import Callable
import pydantic
import posthoganalytics
from drf_spectacular.utils import extend_schema_field
from rest_framework import serializers
from rest_framework.exceptions import NotFound, PermissionDenied
-from rest_framework.pagination import LimitOffsetPagination
-from rest_framework.request import Request
from posthog.schema import HogQLQueryModifiers
@@ -92,22 +89,3 @@ class HogQLModifiersField(serializers.JSONField):
if isinstance(value, dict):
value = drop_unknown_hogql_modifiers(value)
return super().to_representation(value)
-
-
-def fetch_facade_page[T](
- paginator: LimitOffsetPagination, request: Request, fetch_page: Callable[[int, int], tuple[list[T], int]]
-) -> list[T]:
- """Fetch one page of a list through the facade with ``fetch_page(offset, limit)``, and set up the paginator for it.
-
- The facade slices the page, so the paginator gets its state set directly instead of a queryset.
- """
- limit = paginator.get_limit(request)
- offset = paginator.get_offset(request)
- # PAGE_SIZE gives every request a limit.
- assert limit is not None
- items, count = fetch_page(offset, limit)
- paginator.request = request
- paginator.limit = limit
- paginator.offset = offset
- paginator.count = count
- return items
diff --git a/products/batch_exports/backend/tests/api/backfills/test_list.py b/products/batch_exports/backend/tests/api/backfills/test_list.py
index 7be66b71c49b..1d06d439247c 100644
--- a/products/batch_exports/backend/tests/api/backfills/test_list.py
+++ b/products/batch_exports/backend/tests/api/backfills/test_list.py
@@ -63,6 +63,7 @@ def test_list_batch_export_backfills_pages_through_tied_values(client: HttpClien
listed = [backfill["id"] for page in pages for backfill in page["results"]]
assert sorted(listed) == sorted(backfill_ids)
+ assert [page["count"] for page in pages] == [len(backfill_ids)] * len(pages)
def test_cannot_list_batch_export_backfills_for_other_organizations(client: HttpClient, organization, team, user):
diff --git a/products/batch_exports/backend/tests/api/test_runs.py b/products/batch_exports/backend/tests/api/test_runs.py
index 803d6004117c..a3aba49343a3 100644
--- a/products/batch_exports/backend/tests/api/test_runs.py
+++ b/products/batch_exports/backend/tests/api/test_runs.py
@@ -194,6 +194,7 @@ def test_get_batch_export_runs_filtered_by_status(client: HttpClient, team, user
str(runs[BatchExportRun.Status.FAILED].id),
str(runs[BatchExportRun.Status.FAILED_RETRYABLE].id),
}
+ assert data["count"] == 2
# List all runs, without any status filter, asserting that all runs are returned
data = get_batch_export_runs_ok(client, team.pk, batch_export.id, **query_params)
diff --git a/products/batch_exports/backend/tests/test_facade.py b/products/batch_exports/backend/tests/test_facade.py
index 31115f8d5a8e..ed24970ed255 100644
--- a/products/batch_exports/backend/tests/test_facade.py
+++ b/products/batch_exports/backend/tests/test_facade.py
@@ -12,6 +12,7 @@ from posthog.api.test.test_organization import create_organization
from posthog.api.test.test_team import create_team
from posthog.integrations import IntegrationNotFound
from posthog.models import Integration, User
+from posthog.paging import CursorPageRequest
from posthog.temporal.common.client import sync_connect
from products.batch_exports.backend.facade import api, contracts, testing
@@ -284,11 +285,12 @@ def test_runs_and_backfills_are_scoped_to_the_team_and_export(team, organization
start=dt.datetime(2000, 1, 1, tzinfo=dt.UTC), end=dt.datetime.now(dt.UTC) + dt.timedelta(days=1)
)
)
- page = contracts.ListPage(ordering=("-created_at",), bound=None, offset=0, limit=10)
+ page = CursorPageRequest(ordering=("-created_at",), bound=None, offset=0, limit=10)
- assert [run.id for run in api.list_batch_export_runs(team.pk, export_id, filters=every_run, page=page)] == [run_id]
- assert api.list_batch_export_runs(team_id, scoped_export_id, filters=every_run, page=page) == []
- assert api.list_batch_export_backfills(team_id, scoped_export_id, page=page) == []
+ own_runs = api.list_batch_export_runs(team.pk, export_id, filters=every_run, page=page)
+ assert [run.id for run in own_runs.results] == [run_id]
+ assert api.list_batch_export_runs(team_id, scoped_export_id, filters=every_run, page=page).results == []
+ assert api.list_batch_export_backfills(team_id, scoped_export_id, page=page).results == []
# A leaked run or backfill would get past the lookup and reach Temporal instead of raising.
for read_run in (api.get_batch_export_run, api.retry_batch_export_run, api.cancel_batch_export_run):
with pytest.raises(contracts.BatchExportRunNotFoundError):
diff --git a/products/batch_exports/frontend/generated/api.schemas.ts b/products/batch_exports/frontend/generated/api.schemas.ts
index 6fd0dc5141e6..16fd8e4ac10a 100644
--- a/products/batch_exports/frontend/generated/api.schemas.ts
+++ b/products/batch_exports/frontend/generated/api.schemas.ts
@@ -1894,6 +1894,7 @@ export interface BatchExportBackfillApi {
}
export interface PaginatedBatchExportBackfillListApi {
+ count: number
/** @nullable */
next?: string | null
/** @nullable */
@@ -1902,6 +1903,7 @@ export interface PaginatedBatchExportBackfillListApi {
}
export interface PaginatedBatchExportRunListApi {
+ count: number
/** @nullable */
next?: string | null
/** @nullable */
@@ -2386,6 +2388,7 @@ export const RedshiftDestinationRequestTypeEnumApi = {
export type BatchExportsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
@@ -2541,6 +2544,7 @@ export type BatchExportsLogsRetrieveParams = {
export type FileDownloadBatchExportsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
diff --git a/products/subscriptions/backend/facade/api.py b/products/subscriptions/backend/facade/api.py
index 8efe6ab3a840..1c619100f2da 100644
--- a/products/subscriptions/backend/facade/api.py
+++ b/products/subscriptions/backend/facade/api.py
@@ -12,6 +12,8 @@ from collections.abc import Collection
from datetime import datetime
from typing import TYPE_CHECKING
+from posthog.paging import CursorPageRequest, Page, PageRequest
+
from products.subscriptions.backend.logic import (
auto_disable,
queries,
@@ -217,15 +219,13 @@ def dashboard_has_active_full_subscription(dashboard_id: int) -> bool:
def list_subscriptions(
- *, team_id: int, query: contracts.SubscriptionListQuery, viewer: UserAccessControl, offset: int, limit: int
-) -> contracts.SubscriptionPage:
+ *, team_id: int, query: contracts.SubscriptionListQuery, viewer: UserAccessControl, page: PageRequest
+) -> Page[contracts.SubscriptionRecord]:
"""One page of the team's subscriptions that the viewer may see, in `query.ordering`.
A subscription is hidden when the viewer cannot view an insight or a dashboard it delivers.
"""
- return subscription_reads.list_subscriptions(
- team_id=team_id, query=query, viewer=viewer, offset=offset, limit=limit
- )
+ return subscription_reads.list_subscriptions(team_id=team_id, query=query, viewer=viewer, page=page)
def get_subscription_ref(
@@ -328,17 +328,17 @@ def list_subscription_deliveries(
subscription_id: str | None,
status: str | None,
viewer: UserAccessControl,
- window: contracts.CreatedAtCursorWindow,
+ page: CursorPageRequest,
filter_by_object_access: bool,
include_all_if_admin: bool,
-) -> list[contracts.SubscriptionDeliveryRecord]:
- """One window of a subscription's delivery history that the viewer may see."""
+) -> Page[contracts.SubscriptionDeliveryRecord]:
+ """One page of a subscription's delivery history that the viewer may see, ordered by `created_at`."""
return subscription_reads.list_subscription_deliveries(
team_id=team_id,
subscription_id=subscription_id,
status=status,
viewer=viewer,
- window=window,
+ page=page,
filter_by_object_access=filter_by_object_access,
include_all_if_admin=include_all_if_admin,
)
@@ -368,12 +368,12 @@ def list_subscription_summaries(
insight_id: int | None,
dashboard_id: int | None,
viewer: UserAccessControl,
- window: contracts.CreatedAtCursorWindow,
-) -> list[contracts.SubscriptionSummaryRecord]:
- """One window of the AI summaries that live subscriptions on one insight or one dashboard delivered.
+ page: CursorPageRequest,
+) -> Page[contracts.SubscriptionSummaryRecord]:
+ """One page, ordered by `created_at`, of the AI summaries that live subscriptions on one insight or one dashboard delivered.
Pass exactly one of `insight_id` and `dashboard_id`, after checking that the viewer may view it.
"""
return subscription_reads.list_subscription_summaries(
- team_id=team_id, insight_id=insight_id, dashboard_id=dashboard_id, viewer=viewer, window=window
+ team_id=team_id, insight_id=insight_id, dashboard_id=dashboard_id, viewer=viewer, page=page
)
diff --git a/products/subscriptions/backend/facade/contracts.py b/products/subscriptions/backend/facade/contracts.py
index 39dff06bc149..5b4d9a7e4575 100644
--- a/products/subscriptions/backend/facade/contracts.py
+++ b/products/subscriptions/backend/facade/contracts.py
@@ -203,14 +203,6 @@ class SubscriptionRef:
dashboard_id: int | None
[email protected](frozen=True, kw_only=True)
-class SubscriptionPage:
- """One page of subscriptions, and how many match in total."""
-
- count: int
- results: tuple[SubscriptionRecord, ...]
-
-
@dataclasses.dataclass(frozen=True, kw_only=True)
class SubscriptionFilters:
"""Narrows the subscriptions a read can see. A None filter does not filter.
@@ -243,21 +235,6 @@ class SubscriptionListQuery:
include_all_if_admin: bool = False
[email protected](frozen=True, kw_only=True)
-class CreatedAtCursorWindow:
- """One window of a list ordered by creation time, as cursor pagination asks for it.
-
- `created_before` and `created_after` are the cursor position as the cursor carries it, a
- timestamp string. `limit` rows are returned from `offset` on.
- """
-
- newest_first: bool
- created_before: str | None = None
- created_after: str | None = None
- offset: int
- limit: int
-
-
@dataclasses.dataclass(frozen=True, kw_only=True)
class SubscriptionDeliveryRecord:
"""One delivery run of a subscription, as stored."""
diff --git a/products/subscriptions/backend/logic/subscription_reads.py b/products/subscriptions/backend/logic/subscription_reads.py
index 62f5f112a9e7..ace770021653 100644
--- a/products/subscriptions/backend/logic/subscription_reads.py
+++ b/products/subscriptions/backend/logic/subscription_reads.py
@@ -8,12 +8,12 @@ from django.core.exceptions import ValidationError as DjangoValidationError
from django.db.models import OuterRef, Prefetch, Q, QuerySet, Subquery
from posthog.models.user import User
-from posthog.queryset_paging import stable_queryset_ordering
+from posthog.paging import CursorPageRequest, Page, PageRequest
+from posthog.queryset_paging import select_cursor_page, select_page
from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.product_analytics.backend.facade.api import insights_including_soft_deleted_for_team
from products.subscriptions.backend.facade.contracts import (
- CreatedAtCursorWindow,
MalformedIdError,
SubscriptionCreator,
SubscriptionDashboardContext,
@@ -22,7 +22,6 @@ from products.subscriptions.backend.facade.contracts import (
SubscriptionInsightContext,
SubscriptionListQuery,
SubscriptionNotFoundError,
- SubscriptionPage,
SubscriptionPermissionDeniedError,
SubscriptionRecord,
SubscriptionRef,
@@ -34,6 +33,8 @@ from products.subscriptions.backend.temporal.ai_subscription.spec_generator impo
from .visibility import viewable_delivery_filter, viewable_subscription_filter
+DELIVERY_ORDERING_FIELDS = ("created_at",)
+
# Each search term must match one of these, ignoring case.
SEARCH_LOOKUPS = (
"title__icontains",
@@ -190,8 +191,8 @@ def _filter_subscriptions(queryset: QuerySet, filters: SubscriptionFilters) -> Q
def list_subscriptions(
- *, team_id: int, query: SubscriptionListQuery, viewer: UserAccessControl, offset: int, limit: int
-) -> SubscriptionPage:
+ *, team_id: int, query: SubscriptionListQuery, viewer: UserAccessControl, page: PageRequest
+) -> Page[SubscriptionRecord]:
context_queryset = SubscriptionContext.objects.for_team(team_id).select_related("dashboard", "insight")
queryset = Subscription.objects.prefetch_related(
"dashboard_export_insights",
@@ -218,14 +219,7 @@ def list_subscriptions(
queryset = viewer.filter_queryset_by_access_level(queryset, include_all_if_admin=query.include_all_if_admin)
if query.ordering:
queryset = queryset.order_by(*query.ordering)
- queryset = stable_queryset_ordering(queryset)
-
- count = queryset.count()
- if count == 0 or offset > count:
- return SubscriptionPage(count=count, results=())
- return SubscriptionPage(
- count=count, results=tuple(subscription_record(obj) for obj in queryset[offset : offset + limit])
- )
+ return select_page(queryset, page).map(subscription_record)
def get_subscription_row(
@@ -280,25 +274,19 @@ def is_ai_prompt_subscription(*, team_id: int, subscription_id: str | int) -> bo
)
-def _window(queryset: QuerySet, window: CreatedAtCursorWindow) -> list:
- queryset = queryset.order_by("-created_at" if window.newest_first else "created_at")
- if window.created_before is not None:
- queryset = queryset.filter(created_at__lt=window.created_before)
- if window.created_after is not None:
- queryset = queryset.filter(created_at__gt=window.created_after)
- return list(queryset[window.offset : window.offset + window.limit])
-
-
def _deliveries(
*, team_id: int, subscription_id: str | None, status: str | None, viewer: UserAccessControl
) -> QuerySet[SubscriptionDelivery]:
- queryset = SubscriptionDelivery.objects.all()
+ queryset = SubscriptionDelivery.objects.filter(team_id=team_id)
if subscription_id:
queryset = queryset.filter(subscription_id=subscription_id)
if status:
queryset = queryset.filter(status=status)
- queryset = queryset.filter(viewable_delivery_filter(viewer, team_id)).distinct()
- return queryset.filter(team_id=team_id)
+ # The visibility joins can repeat a row. An id subquery removes the repeats without DISTINCT over
+ # content_snapshot, which holds full query results and makes the page and count queries slow.
+ # The subquery reuses the scoped queryset, so the visibility checks run only on this team's matching rows.
+ visible_ids = queryset.filter(viewable_delivery_filter(viewer, team_id)).values("id")
+ return queryset.filter(id__in=visible_ids)
def list_subscription_deliveries(
@@ -307,14 +295,14 @@ def list_subscription_deliveries(
subscription_id: str | None,
status: str | None,
viewer: UserAccessControl,
- window: CreatedAtCursorWindow,
+ page: CursorPageRequest,
filter_by_object_access: bool,
include_all_if_admin: bool,
-) -> list[SubscriptionDeliveryRecord]:
+) -> Page[SubscriptionDeliveryRecord]:
queryset = _deliveries(team_id=team_id, subscription_id=subscription_id, status=status, viewer=viewer)
if filter_by_object_access:
queryset = viewer.filter_queryset_by_access_level(queryset, include_all_if_admin=include_all_if_admin)
- return [delivery_record(delivery) for delivery in _window(queryset, window)]
+ return select_cursor_page(queryset, page, ordering_fields=DELIVERY_ORDERING_FIELDS).map(delivery_record)
def get_subscription_delivery(
@@ -344,14 +332,26 @@ def viewable_live_insight_id(*, team_id: int, insight_id: int, viewer: UserAcces
return insight.pk
+def _summary_record(delivery: SubscriptionDelivery) -> SubscriptionSummaryRecord:
+ return SubscriptionSummaryRecord(
+ id=delivery.id,
+ subscription_id=delivery.subscription_id,
+ subscription_title=delivery.subscription.title,
+ target_type=delivery.target_type,
+ change_summary=delivery.change_summary,
+ period_start=delivery.period_start, # type: ignore[attr-defined]
+ created_at=delivery.created_at,
+ )
+
+
def list_subscription_summaries(
*,
team_id: int,
insight_id: int | None,
dashboard_id: int | None,
viewer: UserAccessControl,
- window: CreatedAtCursorWindow,
-) -> list[SubscriptionSummaryRecord]:
+ page: CursorPageRequest,
+) -> Page[SubscriptionSummaryRecord]:
# A subscription can change its source, so also match the source in the delivery snapshot.
if insight_id is not None:
source_filter = Q(subscription__insight_id=insight_id, content_snapshot__insights__0__id=insight_id)
@@ -388,15 +388,4 @@ def list_subscription_summaries(
.only("id", "subscription_id", "subscription__title", "target_type", "change_summary", "created_at")
.annotate(period_start=Subquery(previous_delivery_at))
)
- return [
- SubscriptionSummaryRecord(
- id=delivery.id,
- subscription_id=delivery.subscription_id,
- subscription_title=delivery.subscription.title,
- target_type=delivery.target_type,
- change_summary=delivery.change_summary,
- period_start=delivery.period_start,
- created_at=delivery.created_at,
- )
- for delivery in _window(deliveries, window)
- ]
+ return select_cursor_page(deliveries, page, ordering_fields=DELIVERY_ORDERING_FIELDS).map(_summary_record)
diff --git a/products/subscriptions/backend/presentation/views.py b/products/subscriptions/backend/presentation/views.py
index 67778321cebc..3aa115de6aa5 100644
--- a/products/subscriptions/backend/presentation/views.py
+++ b/products/subscriptions/backend/presentation/views.py
@@ -1,9 +1,8 @@
import uuid
import dataclasses
from collections.abc import Callable, Sequence
-from typing import ClassVar, Generic, Optional, Self, TypeVar, cast
+from typing import ClassVar, Optional, cast
-from django.db.models import QuerySet
from django.http import Http404
import posthoganalytics
@@ -19,7 +18,6 @@ from drf_spectacular.utils import (
from rest_framework import exceptions, filters, serializers, status, viewsets
from rest_framework.decorators import action
from rest_framework.exceptions import ValidationError
-from rest_framework.pagination import CursorPagination
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.serializers import as_serializer_error
@@ -28,7 +26,12 @@ from rest_framework.views import APIView
from posthog.api.forbid_destroy_model import ForbidDestroyModel
from posthog.api.ordering import StableOrderingFilter
-from posthog.api.pagination import PrecountedLimitOffsetPagination
+from posthog.api.pagination import (
+ FacadeCursorPagination,
+ FacadeLimitOffsetPagination,
+ facade_cursor_page_response,
+ facade_page_response,
+)
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.shared import UserBasicSerializer
from posthog.event_usage import get_request_analytics_properties, groups
@@ -75,8 +78,6 @@ _AI_DISPLAY_FIELD_SCOPE_NOTE = (
"summary_enabled and summary_prompt_guide for that."
)
-_Row = TypeVar("_Row")
-
def _insight_exists(insight_id: int) -> bool:
# The facade compares the insight's team with the request's team right after this check.
@@ -619,62 +620,6 @@ def _parse_int_list_param(value: str, param: str) -> list[int]:
return ids
-class _CreatedAtCursorSource(Generic[_Row]):
- """Stands in for a queryset under `CursorPagination`, which orders by `created_at`, filters by the
- cursor position, and slices. The slice reads one window through the facade, so DRF still decodes
- the cursor and builds the links."""
-
- def __init__(
- self,
- read_window: Callable[[contracts.CreatedAtCursorWindow], list[_Row]],
- *,
- newest_first: bool = True,
- created_before: str | None = None,
- created_after: str | None = None,
- ) -> None:
- self._read_window = read_window
- self._newest_first = newest_first
- self._created_before = created_before
- self._created_after = created_after
-
- def order_by(self, *ordering: str) -> Self:
- if ordering not in (("-created_at",), ("created_at",)):
- raise ValueError(f"Unsupported ordering: {ordering}")
- return type(self)(
- self._read_window,
- newest_first=ordering == ("-created_at",),
- created_before=self._created_before,
- created_after=self._created_after,
- )
-
- def filter(self, *, created_at__lt: str | None = None, created_at__gt: str | None = None) -> Self:
- return type(self)(
- self._read_window,
- newest_first=self._newest_first,
- created_before=created_at__lt,
- created_after=created_at__gt,
- )
-
- def __getitem__(self, rows: slice) -> list[_Row]:
- offset = rows.start or 0
- return self._read_window(
- contracts.CreatedAtCursorWindow(
- newest_first=self._newest_first,
- created_before=self._created_before,
- created_after=self._created_after,
- offset=offset,
- limit=rows.stop - offset,
- )
- )
-
-
-def _paginate_by_created_at(view: viewsets.GenericViewSet, source: _CreatedAtCursorSource[_Row]) -> list[_Row]:
- # CursorPagination reads the source through the three queryset operations it implements.
- page = view.paginate_queryset(cast(QuerySet, source))
- assert page is not None # both cursor paginations set a page size
- return list(page)
-
-
def _raise_for_refusal(call: Callable[[], contracts.SubscriptionRecord]) -> contracts.SubscriptionRecord:
"""Run a subscription write, and answer each refusal the way a serializer's validate() would."""
try:
@@ -734,7 +679,7 @@ class SubscriptionSummarySerializer(serializers.Serializer):
)
-class SubscriptionSummaryCursorPagination(CursorPagination):
+class SubscriptionSummaryCursorPagination(FacadeCursorPagination):
page_size = 20
ordering = "-created_at"
@@ -828,7 +773,7 @@ _SUBSCRIPTION_ID_PARAMETER = OpenApiParameter(
class SubscriptionViewSet(TeamAndOrgViewSetMixin, ForbidDestroyModel, viewsets.GenericViewSet):
scope_object = "subscription"
serializer_class = SubscriptionSerializer
- pagination_class = PrecountedLimitOffsetPagination
+ pagination_class = FacadeLimitOffsetPagination
filter_backends = [filters.SearchFilter, StableOrderingFilter]
ordering_fields = [
"created_at",
@@ -977,20 +922,12 @@ class SubscriptionViewSet(TeamAndOrgViewSetMixin, ForbidDestroyModel, viewsets.G
filter_by_object_access=not is_service_auth(request),
include_all_if_admin=request.GET.get("admin_include_all") == "true",
)
- paginator = cast(PrecountedLimitOffsetPagination, self.paginator)
- limit = paginator.get_limit(request)
- # A missing or invalid limit falls back to the PAGE_SIZE setting, so there is always one.
- assert limit is not None
- page = api.list_subscriptions(
- team_id=self.team_id,
- query=query,
- viewer=self.user_access_control,
- offset=paginator.get_offset(request),
- limit=limit,
+ return facade_page_response(
+ self,
+ lambda page: api.list_subscriptions(
+ team_id=self.team_id, query=query, viewer=self.user_access_control, page=page
+ ),
)
- paginator.set_count(page.count)
- results = self.paginate_queryset(list(page.results))
- return self.get_paginated_response(self.get_serializer(results, many=True).data)
def retrieve(self, request: Request, **kwargs: object) -> Response:
subscription = self._get_subscription()
@@ -1108,19 +1045,17 @@ class SubscriptionViewSet(TeamAndOrgViewSetMixin, ForbidDestroyModel, viewsets.G
except dashboards_api.DashboardNotViewable:
raise exceptions.PermissionDenied("You do not have access to this resource.")
- page = _paginate_by_created_at(
+ return facade_cursor_page_response(
self,
- _CreatedAtCursorSource(
- lambda window: api.list_subscription_summaries(
- team_id=self.team_id,
- insight_id=insight_id,
- dashboard_id=dashboard_id,
- viewer=self.user_access_control,
- window=window,
- )
+ lambda page: api.list_subscription_summaries(
+ team_id=self.team_id,
+ insight_id=insight_id,
+ dashboard_id=dashboard_id,
+ viewer=self.user_access_control,
+ page=page,
),
+ serializer_class=SubscriptionSummarySerializer,
)
- return self.get_paginated_response(SubscriptionSummarySerializer(page, many=True).data)
@extend_schema(
extensions={"x-product": "subscriptions"},
@@ -1386,7 +1321,7 @@ class SubscriptionDeliverySerializer(serializers.Serializer):
return data
-class SubscriptionDeliveryCursorPagination(CursorPagination):
+class SubscriptionDeliveryCursorPagination(FacadeCursorPagination):
page_size = 50
ordering = "-created_at"
@@ -1455,21 +1390,18 @@ class SubscriptionDeliveryViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSe
raise ValidationError(
{"status": [f"Must be one of: {', '.join(sorted(_DELIVERY_STATUS_VALUES))}."]},
)
- page = _paginate_by_created_at(
+ return facade_cursor_page_response(
self,
- _CreatedAtCursorSource(
- lambda window: api.list_subscription_deliveries(
- team_id=self.team_id,
- subscription_id=self.kwargs.get("parent_lookup_subscription_id"),
- status=status_param or None,
- viewer=self.user_access_control,
- window=window,
- filter_by_object_access=not is_service_auth(request),
- include_all_if_admin=request.GET.get("admin_include_all") == "true",
- )
+ lambda page: api.list_subscription_deliveries(
+ team_id=self.team_id,
+ subscription_id=self.kwargs.get("parent_lookup_subscription_id"),
+ status=status_param or None,
+ viewer=self.user_access_control,
+ page=page,
+ filter_by_object_access=not is_service_auth(request),
+ include_all_if_admin=request.GET.get("admin_include_all") == "true",
),
)
- return self.get_paginated_response(self.get_serializer(page, many=True).data)
def retrieve(self, request: Request, **kwargs: object) -> Response:
try:
diff --git a/products/subscriptions/backend/tests/test_subscription.py b/products/subscriptions/backend/tests/test_subscription.py
index cc8ac711e71f..11fe624d0e87 100644
--- a/products/subscriptions/backend/tests/test_subscription.py
+++ b/products/subscriptions/backend/tests/test_subscription.py
@@ -2821,6 +2821,22 @@ class TestSubscriptionDeliveryAPI(APILicensedTest):
assert back.status_code == status.HTTP_200_OK
assert [row["id"] for row in back.json()["results"]] == [row["id"] for row in body["results"]]
+ # Rows that share the cursor value need a stable order, or a page boundary skips or repeats them.
+ self._create_delivery(idempotency_key="page-d")
+ self._create_delivery(idempotency_key="page-e")
+ SubscriptionDelivery.objects.filter(subscription=self.subscription).update(
+ created_at=datetime(2024, 1, 1, tzinfo=UTC)
+ )
+ pages = [self.client.get(base).json()]
+ # Bounded, so a cursor that never advances fails instead of hanging.
+ while pages[-1]["next"] and len(pages) <= 5:
+ pages.append(self.client.get(pages[-1]["next"]).json())
+ assert [page["count"] for page in pages] == [5, 5, 5]
+ listed = [row["id"] for page in pages for row in page["results"]]
+ assert sorted(listed) == sorted(
+ str(d.id) for d in SubscriptionDelivery.objects.filter(subscription=self.subscription)
+ )
+
def test_summaries_list_completed_summaries_across_subscriptions_on_the_source(self):
second_subscription = Subscription.objects.create(
team=self.team,
diff --git a/products/subscriptions/frontend/components/SubscriptionSummaries/SubscriptionSummaries.stories.tsx b/products/subscriptions/frontend/components/SubscriptionSummaries/SubscriptionSummaries.stories.tsx
index 0391ddf523d8..7364cbc2d1b1 100644
--- a/products/subscriptions/frontend/components/SubscriptionSummaries/SubscriptionSummaries.stories.tsx
+++ b/products/subscriptions/frontend/components/SubscriptionSummaries/SubscriptionSummaries.stories.tsx
@@ -9,6 +9,7 @@ import type { PaginatedSubscriptionSummaryListApi } from 'products/subscriptions
import { SubscriptionSummaries } from './SubscriptionSummaries'
const SUMMARIES: PaginatedSubscriptionSummaryListApi = {
+ count: 3,
next: null,
previous: null,
results: [
diff --git a/products/subscriptions/frontend/generated/api.schemas.ts b/products/subscriptions/frontend/generated/api.schemas.ts
index 8d064936b10a..5ac82d8fbe88 100644
--- a/products/subscriptions/frontend/generated/api.schemas.ts
+++ b/products/subscriptions/frontend/generated/api.schemas.ts
@@ -793,6 +793,7 @@ export interface SubscriptionDeliveryApi {
}
export interface PaginatedSubscriptionDeliveryListApi {
+ count: number
/** @nullable */
next?: string | null
/** @nullable */
@@ -827,6 +828,7 @@ export interface SubscriptionSummaryApi {
}
export interface PaginatedSubscriptionSummaryListApi {
+ count: number
/** @nullable */
next?: string | null
/** @nullable */
@@ -857,6 +859,7 @@ export type SubscriptionsListParams = {
insights?: string
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
diff --git a/products/subscriptions/frontend/scenes/components/SubscriptionDeliveryHistory.stories.tsx b/products/subscriptions/frontend/scenes/components/SubscriptionDeliveryHistory.stories.tsx
index f5cf959afa37..34956ba1493b 100644
--- a/products/subscriptions/frontend/scenes/components/SubscriptionDeliveryHistory.stories.tsx
+++ b/products/subscriptions/frontend/scenes/components/SubscriptionDeliveryHistory.stories.tsx
@@ -25,6 +25,7 @@ export default meta
type Story = StoryObj<typeof SubscriptionDeliveryHistory>
const pageWithRows: PaginatedSubscriptionDeliveryListApi = {
+ count: MOCK_SUBSCRIPTION_DELIVERIES.length,
results: MOCK_SUBSCRIPTION_DELIVERIES,
next: null,
previous: null,
@@ -82,7 +83,7 @@ export const WithExpandedAiReport: Story = {
export const Empty: Story = {
args: {
- deliveriesPage: { results: [], next: null, previous: null },
+ deliveriesPage: { count: 0, results: [], next: null, previous: null },
deliveriesPageLoading: false,
loadDeliveriesPage: noopLoadPage,
onTestDelivery: noopTestDelivery,
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 43a1e2d1bd01..ca05bc1f1367 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -77691,6 +77691,7 @@ export namespace Schemas {
}
export interface PaginatedBatchExportBackfillList {
+ count: number;
/** @nullable */
next?: string | null;
/** @nullable */
@@ -77708,6 +77709,7 @@ export namespace Schemas {
}
export interface PaginatedBatchExportRunList {
+ count: number;
/** @nullable */
next?: string | null;
/** @nullable */
@@ -82179,6 +82181,7 @@ export namespace Schemas {
}
export interface PaginatedSubscriptionDeliveryList {
+ count: number;
/** @nullable */
next?: string | null;
/** @nullable */
@@ -82414,6 +82417,7 @@ export namespace Schemas {
}
export interface PaginatedSubscriptionSummaryList {
+ count: number;
/** @nullable */
next?: string | null;
/** @nullable */
@@ -132484,6 +132488,7 @@ export namespace Schemas {
export type BatchExportsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -137782,6 +137787,7 @@ export namespace Schemas {
export type FileDownloadBatchExportsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -143617,6 +143623,7 @@ export namespace Schemas {
insights?: string;
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
diff --git a/services/mcp/src/generated/batch_exports/api.ts b/services/mcp/src/generated/batch_exports/api.ts
index 73c2f9b1406f..7fcd79741785 100644
--- a/services/mcp/src/generated/batch_exports/api.ts
+++ b/services/mcp/src/generated/batch_exports/api.ts
@@ -16,8 +16,10 @@ export const BatchExportsListParams = () => zod.object({
),
})
+export const batchExportsListQueryLimitMax = 1000
+
export const BatchExportsListQueryParams = () => zod.object({
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod.number().max(batchExportsListQueryLimitMax).optional().describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
})
diff --git a/services/mcp/src/generated/subscriptions/api.ts b/services/mcp/src/generated/subscriptions/api.ts
index a3b607ad778d..5450e9207264 100644
--- a/services/mcp/src/generated/subscriptions/api.ts
+++ b/services/mcp/src/generated/subscriptions/api.ts
@@ -16,6 +16,8 @@ export const SubscriptionsListParams = () => zod.object({
),
})
+export const subscriptionsListQueryLimitMax = 1000
+
export const SubscriptionsListQueryParams = () => zod.object({
created_by: zod.string().optional().describe('Filter by creator user UUID.'),
dashboard: zod.number().optional().describe('Filter by dashboard ID.'),
@@ -25,7 +27,11 @@ export const SubscriptionsListQueryParams = () => zod.object({
.describe('Filter to subscriptions on insights that are tiles of the given dashboard ID.'),
insight: zod.number().optional().describe('Filter by insight ID.'),
insights: zod.string().optional().describe('Filter by a comma-separated list of insight IDs.'),
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod
+ .number()
+ .max(subscriptionsListQueryLimitMax)
+ .optional()
+ .describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
ordering: zod.string().optional().describe('Which field to use when ordering the results.'),
resource_type: zod
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-exports-list.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-exports-list.json
index 8331b796ac73..2fbd7fbe281a 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-exports-list.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-exports-list.json
@@ -3,6 +3,7 @@
"properties": {
"limit": {
"description": "Number of results to return per page.",
+ "maximum": 1000,
"type": "number"
},
"offset": {
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/subscriptions-list.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/subscriptions-list.json
index ac86c089caad..0e05d3af358c 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/subscriptions-list.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/subscriptions-list.json
@@ -23,6 +23,7 @@
},
"limit": {
"description": "Number of results to return per page.",
+ "maximum": 1000,
"type": "number"
},
"offset": {