Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(devex): add a shared paging kit for facade-backed lists (#126403)

Julian Bez6537899e0110
diff --git a/.agents/skills/isolating-product-facade-contracts/SKILL.md b/.agents/skills/isolating-product-facade-contracts/SKILL.md
index 835ca5fdd8a8..c06f41df66f4 100644
--- a/.agents/skills/isolating-product-facade-contracts/SKILL.md
+++ b/.agents/skills/isolating-product-facade-contracts/SKILL.md
@@ -272,6 +272,7 @@ records the decrease. See `products/architecture.md` § Wiring couplings.
- Transaction boundaries belong in the facade (or logic), not in views.
- A view that serves a contract checks object RBAC through an `ObjectAccessRef`: return one from `safely_get_object()`, or pass one to `self.check_object_permissions(request, ref)`. Never check by hand or in the facade. See `products/architecture.md` § Who owns RBAC?
- Never return ORM models across product boundaries.
+- A list view pages through the facade: the facade takes a `PageRequest` and returns a `Page`, and the view uses `FacadeLimitOffsetPagination` or `FacadeCursorPagination`. Never set paginator state by hand. See `products/architecture.md` § How do lists page?
- `hogli product:lint` reads the facade signatures, in the kinds `facade-returns`,
`facade-accepts(<parameter>)` and `facade-logic`. The matching lines of
`products/model_crossing_uses_baseline.txt` record what the facades do today and only shrink, so a
diff --git a/posthog/api/log_entries.py b/posthog/api/log_entries.py
index 3b12a002d01d..94f03fdca0c9 100644
--- a/posthog/api/log_entries.py
+++ b/posthog/api/log_entries.py
@@ -9,9 +9,11 @@ from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework_dataclasses.serializers import DataclassSerializer
+from posthog.api.pagination import FacadeLimitOffsetPagination, facade_page_response
from posthog.api.utils import action
from posthog.clickhouse.client.execute import sync_execute
from posthog.clickhouse.query_tagging import Feature, tag_queries
+from posthog.paging import Page, PageRequest
from posthog.schema_enums import ProductKey
# Default lookback window applied when the caller does not pass `after`. Bounding the
@@ -49,6 +51,12 @@ class LogEntryRequestSerializer(serializers.Serializer):
min_value=1,
help_text="Maximum number of log entries to return (1-500, default 50).",
)
+ offset = serializers.IntegerField(
+ required=False,
+ default=0,
+ min_value=0,
+ help_text="Number of log entries to skip before the first returned entry (default 0).",
+ )
after = serializers.DateTimeField(
required=False,
help_text=(
@@ -74,18 +82,23 @@ class LogEntryRequestSerializer(serializers.Serializer):
)
-def fetch_log_entries(
+class LogEntryPagination(FacadeLimitOffsetPagination):
+ # Matches `LogEntryRequestSerializer.limit`, which rejects a limit out of range before the paginator reads it.
+ default_limit = 50
+ max_limit = 500
+
+
+def _log_entries_conditions(
+ *,
team_id: int,
log_source: str,
log_source_id: str,
- limit: int,
- instance_id: Optional[str] = None,
- after: Optional[datetime] = None,
- before: Optional[datetime] = None,
- search: Optional[str] = None,
- level: Optional[list[str]] = None,
-) -> list[Any]:
- """Fetch a list of batch export log entries from ClickHouse."""
+ instance_id: Optional[str],
+ after: Optional[datetime],
+ before: Optional[datetime],
+ search: Optional[str],
+ level: Optional[list[str]],
+) -> tuple[str, dict[str, Any]]:
if level is None:
level = []
clickhouse_where_parts: list[str] = []
@@ -121,14 +134,104 @@ def fetch_log_entries(
clickhouse_where_parts.append("upper(level) in %(levels)s")
clickhouse_kwargs["levels"] = [lev.upper() for lev in level]
+ return " AND ".join(clickhouse_where_parts), clickhouse_kwargs
+
+
+def fetch_log_entries(
+ team_id: int,
+ log_source: str,
+ log_source_id: str,
+ limit: int,
+ offset: int = 0,
+ instance_id: Optional[str] = None,
+ after: Optional[datetime] = None,
+ before: Optional[datetime] = None,
+ search: Optional[str] = None,
+ level: Optional[list[str]] = None,
+) -> list[Any]:
+ """Fetch a list of batch export log entries from ClickHouse."""
+ where, clickhouse_kwargs = _log_entries_conditions(
+ team_id=team_id,
+ log_source=log_source,
+ log_source_id=log_source_id,
+ instance_id=instance_id,
+ after=after,
+ before=before,
+ search=search,
+ level=level,
+ )
+ clickhouse_kwargs = {**clickhouse_kwargs, "limit": limit, "offset": offset}
+ # Entries can share a timestamp. The extra sort terms give them the same order in every query,
+ # so a page boundary between them cannot skip or repeat an entry.
clickhouse_query = f"""
SELECT log_source_id, instance_id, timestamp, upper(level) as level, message FROM log_entries
- WHERE {" AND ".join(clickhouse_where_parts)} ORDER BY timestamp DESC {f"LIMIT {limit}"}
+ WHERE {where} ORDER BY timestamp DESC, instance_id, message, level LIMIT %(limit)s OFFSET %(offset)s
"""
return [LogEntry(*result) for result in cast(list, sync_execute(clickhouse_query, clickhouse_kwargs))]
+def count_log_entries(
+ team_id: int,
+ log_source: str,
+ log_source_id: str,
+ instance_id: Optional[str] = None,
+ after: Optional[datetime] = None,
+ before: Optional[datetime] = None,
+ search: Optional[str] = None,
+ level: Optional[list[str]] = None,
+) -> int:
+ where, clickhouse_kwargs = _log_entries_conditions(
+ team_id=team_id,
+ log_source=log_source,
+ log_source_id=log_source_id,
+ instance_id=instance_id,
+ after=after,
+ before=before,
+ search=search,
+ level=level,
+ )
+ clickhouse_query = f"SELECT count() FROM log_entries WHERE {where}"
+ rows = cast(list, sync_execute(clickhouse_query, clickhouse_kwargs))
+ return int(rows[0][0])
+
+
+def log_entries_response(
+ view: viewsets.GenericViewSet,
+ request: Request,
+ *,
+ log_source: str,
+ log_source_id: str,
+ instance_id: Optional[str] = None,
+) -> Response:
+ """Return one limit/offset page of an object's log entries. The view must page with `LogEntryPagination`."""
+ param_serializer = LogEntryRequestSerializer(data=request.query_params)
+ if not param_serializer.is_valid():
+ raise ValidationError(param_serializer.errors)
+ params = param_serializer.validated_data
+
+ filters: dict[str, Any] = {
+ "team_id": view.team_id, # type: ignore[attr-defined]
+ "log_source": log_source,
+ "log_source_id": log_source_id,
+ "instance_id": instance_id or params.get("instance_id"),
+ "after": params.get("after"),
+ "before": params.get("before"),
+ "search": params.get("search"),
+ "level": params["level"].split(",") if params.get("level") else None,
+ }
+
+ def fetch(page: PageRequest) -> Page[LogEntry]:
+ rows = fetch_log_entries(**filters, limit=page.limit, offset=page.offset)
+ # A short page is the last page, so it gives the total without a second scan. An empty page
+ # past the first one gives no total, because the offset can be past the end.
+ if len(rows) < page.limit and (rows or page.offset == 0):
+ return Page(results=rows, count=page.offset + len(rows))
+ return Page(results=rows, count=count_log_entries(**filters))
+
+ return facade_page_response(view, fetch, serializer_class=LogEntrySerializer)
+
+
class LogEntryMixin(viewsets.GenericViewSet):
log_source: str # Should be set by the inheriting class
@@ -140,45 +243,21 @@ class LogEntryMixin(viewsets.GenericViewSet):
raise NotImplementedError()
@extend_schema(parameters=[LogEntryRequestSerializer])
- @action(detail=True, methods=["GET"], filter_backends=[])
+ @action(detail=True, methods=["GET"], filter_backends=[], pagination_class=LogEntryPagination)
def logs(self, request: Request, *args, **kwargs):
obj = self.get_object()
- param_serializer = LogEntryRequestSerializer(data=request.query_params)
-
if not self.log_source:
raise ValidationError("log_source not set on the viewset")
product_key = LOG_SOURCE_TO_PRODUCT_KEY.get(self.log_source, ProductKey.PIPELINE_DESTINATIONS)
tag_queries(product=product_key, feature=Feature.QUERY)
- if not param_serializer.is_valid():
- raise ValidationError(param_serializer.errors)
-
- params = param_serializer.validated_data
-
try:
instance_id = self.get_log_entry_instance_id()
except NotImplementedError:
- instance_id = params.get("instance_id")
-
- data = fetch_log_entries(
- team_id=self.team_id, # type: ignore
- log_source=self.log_source,
- log_source_id=str(obj.id),
- limit=params["limit"],
- # From request params
- instance_id=instance_id,
- after=params.get("after"),
- before=params.get("before"),
- search=params.get("search"),
- level=params["level"].split(",") if params.get("level") else None,
- )
-
- page = self.paginate_queryset(data)
- if page is not None:
- serializer = LogEntrySerializer(page, many=True)
- return self.get_paginated_response(serializer.data)
+ instance_id = None
- serializer = LogEntrySerializer(data, many=True)
- return Response({"status": "not implemented"})
+ return log_entries_response(
+ self, request, log_source=self.log_source, log_source_id=str(obj.id), instance_id=instance_id
+ )
diff --git a/posthog/api/pagination.py b/posthog/api/pagination.py
index cf8967b3dbeb..2245b351138b 100644
--- a/posthog/api/pagination.py
+++ b/posthog/api/pagination.py
@@ -1,27 +1,19 @@
-from typing import Any, Optional
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any, Optional, cast
-from django.db.models import QuerySet
+from django.core.exceptions import ImproperlyConfigured
-from rest_framework.pagination import LimitOffsetPagination
+from rest_framework.pagination import CursorPagination, LimitOffsetPagination
+from rest_framework.request import Request
from rest_framework.response import Response
+from rest_framework.serializers import BaseSerializer
from rest_framework.utils.urls import replace_query_param
+from posthog.paging import CursorBound, CursorPageRequest, Page, PageRequest
-def stable_queryset_ordering(queryset: QuerySet) -> QuerySet:
- """Add the primary key as a final ordering term for a paginated queryset."""
- if queryset.query.is_sliced or queryset.query.group_by is not None:
- return queryset
-
- ordering = queryset.query.order_by or queryset.query.extra_order_by or queryset.model._meta.ordering
- if not ordering:
- return queryset.order_by("pk")
-
- primary_key = queryset.model._meta.pk.name
- if any(str(term).lstrip("-") in {"pk", primary_key} for term in ordering):
- return queryset
-
- direction = "-" if str(ordering[0]).startswith("-") else ""
- return queryset.order_by(*ordering, f"{direction}pk")
+if TYPE_CHECKING:
+ # DRF's generics module reads DEFAULT_PAGINATION_CLASS while it loads, which can point at this module.
+ from rest_framework.generics import GenericAPIView
class PrecountedLimitOffsetPagination(LimitOffsetPagination):
@@ -99,3 +91,188 @@ class CappedCountLimitOffsetPagination(PrecountedLimitOffsetPagination):
"description": "True when `count` stopped at a cap, so it is a lower bound and `next` keeps paging past it.",
}
return paginated
+
+
+class BoundedLimitOffsetPagination(LimitOffsetPagination):
+ """Limit/offset pages of at most `max_limit` rows.
+
+ DRF has no upper bound by default, so `?limit=` could ask one request to read a whole table.
+ A larger limit is cut to `max_limit`, and the schema states the maximum.
+ """
+
+ max_limit = 1000
+
+ def get_schema_operation_parameters(self, view: Any) -> list[dict[str, Any]]:
+ parameters = super().get_schema_operation_parameters(view)
+ for parameter in parameters:
+ if parameter["name"] == self.limit_query_param:
+ parameter["schema"]["maximum"] = self.max_limit
+ return parameters
+
+
+class FacadeLimitOffsetPagination(BoundedLimitOffsetPagination):
+ """Builds the standard limit/offset response from a page that a facade read in SQL.
+
+ A facade returns contracts, not a queryset, so it cannot hand its rows to `paginate_queryset`
+ without loading the whole list. The view passes a `fetch` function instead. This paginator reads
+ the limit and offset from the request, asks `fetch` for that page, and keeps the response shape
+ and links of `LimitOffsetPagination`.
+ """
+
+ def read_facade_page[T](self, request: Request, fetch: Callable[[PageRequest], Page[T]]) -> Page[T]:
+ """Read the page the request asks for, and keep the state `get_paginated_response` builds links from."""
+ limit = self.get_limit(request)
+ if limit is None:
+ raise ImproperlyConfigured(f"{type(self).__name__} needs a default_limit to page a facade list.")
+ self.request = request
+ self.limit = limit
+ self.offset = self.get_offset(request)
+ page = fetch(PageRequest(limit=limit, offset=self.offset))
+ self.count = page.count
+ return page
+
+ def get_paginated_facade_response[T](
+ self,
+ request: Request,
+ view: "GenericAPIView",
+ fetch: Callable[[PageRequest], Page[T]],
+ serializer_class: type[BaseSerializer] | None = None,
+ ) -> Response:
+ page = self.read_facade_page(request, fetch)
+ serializer_class = serializer_class or view.get_serializer_class()
+ serializer = serializer_class(page.results, many=True, context=view.get_serializer_context())
+ return self.get_paginated_response(serializer.data)
+
+
+class StableCursorPagination(CursorPagination):
+ """Cursor pages with a stable row order, and a `count` like the limit/offset response.
+
+ A cursor stores the value of the first ordering field. When rows share that value, their order
+ must be the same in every query, so the primary key is added as the last ordering term, in the
+ direction of the first term. Paging forward then never skips or repeats a row. A `previous` link
+ can still land on the wrong rows when more rows share one value than fit on a page, because the
+ cursor does not store the primary key.
+ """
+
+ def get_ordering(self, request: Request, queryset: Any, view: Any) -> tuple[str, ...]:
+ ordering = super().get_ordering(request, queryset, view)
+ model = getattr(queryset, "model", None)
+ primary_key = model._meta.pk.name if model is not None else "pk"
+ if any(term.lstrip("-") in {"pk", primary_key} for term in ordering):
+ return ordering
+ direction = "-" if ordering[0].startswith("-") else ""
+ return (*ordering, f"{direction}pk")
+
+ def paginate_queryset(self, queryset: Any, request: Request, view: Any = None) -> Optional[list[Any]]:
+ self.counted_rows = queryset
+ return super().paginate_queryset(queryset, request, view)
+
+ def get_paginated_response(self, data: Any) -> Response:
+ return Response(
+ {
+ "count": self.counted_rows.count(),
+ "next": self.get_next_link(),
+ "previous": self.get_previous_link(),
+ "results": data,
+ }
+ )
+
+ def get_paginated_response_schema(self, schema: dict) -> dict:
+ paginated = super().get_paginated_response_schema(schema)
+ paginated["required"] = ["count", "results"]
+ paginated["properties"] = {
+ "count": {"type": "integer", "example": 123},
+ **paginated["properties"],
+ }
+ return paginated
+
+
+class _FacadePageSource[T]:
+ """Stands in for the queryset that `CursorPagination.paginate_queryset` pages through.
+
+ The paginator only orders the queryset, filters it on the cursor position and slices it. This
+ records the ordering and the position, and the slice reads that page through the facade. So DRF
+ still decodes and encodes every cursor.
+ """
+
+ def __init__(
+ self,
+ fetch: Callable[[CursorPageRequest], Page[T]],
+ ordering: tuple[str, ...] = (),
+ bound: CursorBound | None = None,
+ fetched: list[Page[T]] | None = None,
+ ) -> None:
+ self._fetch = fetch
+ self._ordering = ordering
+ self._bound = bound
+ # Every copy shares one list, so the source the paginator received can report the count of
+ # the page that a later copy fetched.
+ self._fetched: list[Page[T]] = fetched if fetched is not None else []
+
+ def order_by(self, *ordering: str) -> "_FacadePageSource[T]":
+ return _FacadePageSource(self._fetch, ordering, self._bound, self._fetched)
+
+ def filter(self, **lookup: str) -> "_FacadePageSource[T]":
+ # The only filter is the position, as `<first ordering field>__lt` or `__gt`.
+ ((key, value),) = lookup.items()
+ bound = CursorBound(value=value, before=key.endswith("__lt"))
+ return _FacadePageSource(self._fetch, self._ordering, bound, self._fetched)
+
+ def __getitem__(self, items: slice) -> list[T]:
+ page = self._fetch(
+ CursorPageRequest(
+ ordering=self._ordering, bound=self._bound, offset=items.start, limit=items.stop - items.start
+ )
+ )
+ self._fetched.append(page)
+ return page.results
+
+ def count(self) -> int:
+ return self._fetched[-1].count
+
+
+class FacadeCursorPagination(StableCursorPagination):
+ """Builds the cursor response from pages that a facade reads in SQL.
+
+ The facade answers a `CursorPageRequest` with `select_cursor_page`, so cursors keep the format
+ DRF gives a queryset-backed view.
+ """
+
+ def get_paginated_facade_response[T](
+ self,
+ request: Request,
+ view: "GenericAPIView",
+ fetch: Callable[[CursorPageRequest], Page[T]],
+ serializer_class: type[BaseSerializer] | None = None,
+ ) -> Response:
+ rows = self.paginate_queryset(cast(Any, _FacadePageSource(fetch)), request, view)
+ # The page size is a class attribute, so the paginator always returns a page.
+ assert rows is not None
+
+ serializer_class = serializer_class or view.get_serializer_class()
+ serializer = serializer_class(rows, many=True, context=view.get_serializer_context())
+ return self.get_paginated_response(serializer.data)
+
+
+def facade_page_response[T](
+ view: "GenericAPIView",
+ fetch: Callable[[PageRequest], Page[T]],
+ serializer_class: type[BaseSerializer] | None = None,
+) -> Response:
+ """Return the limit/offset response for one page that `fetch` reads through a facade."""
+ paginator = view.paginator
+ if not isinstance(paginator, FacadeLimitOffsetPagination):
+ raise ImproperlyConfigured(f"{type(view).__name__} needs a FacadeLimitOffsetPagination pagination_class.")
+ return paginator.get_paginated_facade_response(view.request, view, fetch, serializer_class)
+
+
+def facade_cursor_page_response[T](
+ view: "GenericAPIView",
+ fetch: Callable[[CursorPageRequest], Page[T]],
+ serializer_class: type[BaseSerializer] | None = None,
+) -> Response:
+ """Return the cursor response for one page that `fetch` reads through a facade."""
+ paginator = view.paginator
+ if not isinstance(paginator, FacadeCursorPagination):
+ raise ImproperlyConfigured(f"{type(view).__name__} needs a FacadeCursorPagination pagination_class.")
+ return paginator.get_paginated_facade_response(view.request, view, fetch, serializer_class)
diff --git a/posthog/api/routing.py b/posthog/api/routing.py
index 8e9f7b91164c..c1b3bbc761ae 100644
--- a/posthog/api/routing.py
+++ b/posthog/api/routing.py
@@ -12,7 +12,6 @@ from rest_framework.permissions import IsAuthenticated
from rest_framework.routers import DefaultRouter
from rest_framework.viewsets import GenericViewSet
-from posthog.api.pagination import stable_queryset_ordering
from posthog.api.utils import get_token
from posthog.auth import (
DelegatedOAuthAccessTokenAuthentication,
@@ -44,6 +43,7 @@ from posthog.permissions import (
is_service_auth,
)
from posthog.products import is_product_module
+from posthog.queryset_paging import stable_queryset_ordering
from posthog.scopes import APIScopeObjectOrNotSupported
from posthog.user_permissions import UserPermissions
diff --git a/posthog/api/test/test_log_entries.py b/posthog/api/test/test_log_entries.py
index 6d586808f5aa..fabbdab3fc27 100644
--- a/posthog/api/test/test_log_entries.py
+++ b/posthog/api/test/test_log_entries.py
@@ -2,9 +2,11 @@ from datetime import UTC, datetime, timedelta
import time_machine
from posthog.test.base import APIBaseTest, ClickhouseTestMixin, QueryMatchingTest
+from unittest.mock import patch
from rest_framework import status
+from posthog.api.log_entries import count_log_entries
from posthog.clickhouse.client.execute import sync_execute
from products.cdp.backend.models.hog_functions.hog_function import HogFunction
@@ -89,6 +91,26 @@ class TestLogEntries(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):
}
]
+ @time_machine.travel("2023-09-23 12:00:00", tick=False)
+ def test_pages_log_entries_by_offset(self):
+ for hour in range(1, 4):
+ self.create_log_for_function(level="info", message=f"entry {hour}", timestamp=f"2023-09-22 0{hour}:00:00")
+
+ first_page = self.get_log_entries({"limit": 2}).json()
+ with patch("posthog.api.log_entries.count_log_entries", wraps=count_log_entries) as count_query:
+ second_page = self.get_log_entries({"limit": 2, "offset": 2}).json()
+ past_the_end = self.get_log_entries({"limit": 2, "offset": 5}).json()
+
+ assert first_page["count"] == 3
+ assert [entry["message"] for entry in first_page["results"]] == ["entry 3", "entry 2"]
+ assert first_page["next"] is not None
+ assert second_page["count"] == 3
+ assert [entry["message"] for entry in second_page["results"]] == ["entry 1"]
+ assert second_page["next"] is None
+ count_query.assert_not_called()
+ assert past_the_end["count"] == 3
+ assert past_the_end["results"] == []
+
def test_default_lookback_excludes_old_entries(self):
"""Without an explicit `after`, only entries within the default lookback window are returned."""
old_timestamp = (datetime.now(UTC) - timedelta(days=30)).strftime("%Y-%m-%d %H:%M:%S.%f")
diff --git a/posthog/api/test/test_pagination.py b/posthog/api/test/test_pagination.py
new file mode 100644
index 000000000000..6f7579988b6e
--- /dev/null
+++ b/posthog/api/test/test_pagination.py
@@ -0,0 +1,72 @@
+from urllib.parse import parse_qs, urlparse
+
+from posthog.test.base import BaseTest
+
+from parameterized import parameterized
+from rest_framework import serializers
+from rest_framework.generics import GenericAPIView
+from rest_framework.request import Request
+from rest_framework.test import APIRequestFactory
+
+from posthog.api.pagination import FacadeCursorPagination, StableCursorPagination
+from posthog.paging import CursorPageRequest, Page
+from posthog.queryset_paging import select_cursor_page
+
+from products.annotations.backend.models.annotation import Annotation
+
+
+class _AnnotationIdSerializer(serializers.Serializer):
+ id = serializers.IntegerField()
+
+
+class _AnnotationView(GenericAPIView):
+ serializer_class = _AnnotationIdSerializer
+
+
+class _ByDateMarker(FacadeCursorPagination):
+ ordering = "-date_marker"
+ page_size = 2
+
+
+class _ByDateMarkerQueryset(StableCursorPagination):
+ ordering = "-date_marker"
+ page_size = 2
+
+
+class TestCursorPagination(BaseTest):
+ def _page(self, mode: str, cursor: str | None) -> dict:
+ request = Request(APIRequestFactory().get("/annotations/", {"cursor": cursor} if cursor else {}))
+ view = _AnnotationView()
+ view.request = request
+ view.format_kwarg = None
+ queryset = Annotation.objects.filter(team=self.team)
+
+ if mode == "facade":
+
+ def fetch(page: CursorPageRequest) -> Page[Annotation]:
+ return select_cursor_page(queryset, page, ordering_fields={"date_marker"})
+
+ return _ByDateMarker().get_paginated_facade_response(request, view, fetch).data
+
+ paginator = _ByDateMarkerQueryset()
+ rows = paginator.paginate_queryset(queryset, request, view)
+ return paginator.get_paginated_response(_AnnotationIdSerializer(rows, many=True).data).data
+
+ @parameterized.expand([("queryset",), ("facade",)])
+ def test_pages_through_rows_that_share_the_cursor_field_once(self, mode: str) -> None:
+ shared_marker = Annotation.objects.create(team=self.team, content="first").date_marker
+ for content in ["second", "third", "fourth", "fifth"]:
+ Annotation.objects.create(team=self.team, content=content, date_marker=shared_marker)
+ expected_ids = set(Annotation.objects.filter(team=self.team).values_list("id", flat=True))
+
+ seen_ids: list[int] = []
+ cursor = None
+ while True:
+ page = self._page(mode, cursor)
+ assert page["count"] == 5
+ seen_ids.extend(row["id"] for row in page["results"])
+ if page["next"] is None:
+ break
+ cursor = parse_qs(urlparse(page["next"]).query)["cursor"][0]
+
+ assert sorted(seen_ids) == sorted(expected_ids)
diff --git a/posthog/api/test/test_routing.py b/posthog/api/test/test_routing.py
index 9b3034298ef5..0516d488ce62 100644
--- a/posthog/api/test/test_routing.py
+++ b/posthog/api/test/test_routing.py
@@ -20,7 +20,6 @@ from rest_framework import viewsets
from rest_framework.decorators import action
from rest_framework.response import Response
-from posthog.api.pagination import stable_queryset_ordering
from posthog.api.routing import DefaultRouterPlusPlus, RouterRegistry, TeamAndOrgViewSetMixin
from posthog.auth import ProjectSecretAPIKeyAuthentication
from posthog.models.file_system.file_system import FileSystem
@@ -33,6 +32,7 @@ from posthog.models.team.team import Team
from posthog.models.utils import generate_random_token_personal, hash_key_value
from posthog.permissions import APIScopePermission
from posthog.products import product_app_names
+from posthog.queryset_paging import stable_queryset_ordering
from products.annotations.backend.api.annotation import AnnotationSerializer
from products.annotations.backend.models.annotation import Annotation
@@ -69,8 +69,12 @@ def test_stable_queryset_ordering_adds_a_primary_key_tiebreaker() -> None:
assert expression_queryset.query.order_by[-1] == "pk"
+ annotated_queryset = stable_queryset_ordering(Annotation.objects.annotate(count=Count("id")).order_by("content"))
-def test_stable_queryset_ordering_leaves_sliced_and_grouped_querysets_unchanged() -> None:
+ assert annotated_queryset.query.order_by == ("content", "pk")
+
+
+def test_stable_queryset_ordering_leaves_sliced_and_values_grouped_querysets_unchanged() -> None:
sliced_queryset = Annotation.objects.order_by("date_marker")[:1]
assert stable_queryset_ordering(sliced_queryset) is sliced_queryset
diff --git a/posthog/paging.py b/posthog/paging.py
new file mode 100644
index 000000000000..423a97f4af94
--- /dev/null
+++ b/posthog/paging.py
@@ -0,0 +1,70 @@
+"""Framework-free page types that facades take and return.
+
+A facade cannot hand a queryset to a DRF paginator, because contracts stay free of Django and DRF.
+It takes a page request, pages in SQL and returns the page with the total, so presentation can build
+the standard `count`/`next`/`previous`/`results` response from it.
+"""
+
+from collections.abc import Callable
+
+from posthog.dataclasses import frozen
+
+
+@frozen
+class PageRequest:
+ """One page of a list that pages by offset."""
+
+ limit: int
+ offset: int = 0
+
+ def __post_init__(self) -> None:
+ if self.limit < 1:
+ raise ValueError(f"A page holds at least one item, got limit={self.limit}.")
+ if self.offset < 0:
+ raise ValueError(f"A page cannot start before the first item, got offset={self.offset}.")
+
+
+@frozen
+class CursorBound:
+ """Where a cursor page starts.
+
+ Only the items whose first ordering field sorts before `value` (when `before`) or after it
+ (otherwise) are on the page. `value` is the field's value as text, as an API cursor carries it.
+ """
+
+ value: str
+ before: bool
+
+
+@frozen
+class CursorPageRequest:
+ """One page of a list that pages by cursor.
+
+ `ordering` names the fields to sort by, most significant first. A leading `-` sorts that field
+ in descending order. The page skips `offset` items past `bound` and holds at most `limit` items.
+ """
+
+ ordering: tuple[str, ...]
+ bound: CursorBound | None
+ offset: int
+ limit: int
+
+ def __post_init__(self) -> None:
+ if not self.ordering:
+ raise ValueError("A cursor page needs at least one ordering field.")
+ if self.limit < 1:
+ raise ValueError(f"A page holds at least one item, got limit={self.limit}.")
+ if self.offset < 0:
+ raise ValueError(f"A page cannot start before the bound, got offset={self.offset}.")
+
+
+@frozen
+class Page[T]:
+ """The items of one page, and how many items the whole list holds."""
+
+ results: list[T]
+ count: int
+
+ def map[U](self, convert: Callable[[T], U]) -> "Page[U]":
+ """Return the same page with each item converted, for example from a model to a contract."""
+ return Page(results=[convert(item) for item in self.results], count=self.count)
diff --git a/posthog/queryset_paging.py b/posthog/queryset_paging.py
new file mode 100644
index 000000000000..e23e0b456bd2
--- /dev/null
+++ b/posthog/queryset_paging.py
@@ -0,0 +1,67 @@
+"""Pages a Django queryset into the `posthog.paging` types.
+
+Logic and facade modules import it, so it imports Django only and never DRF."""
+
+from collections.abc import Collection
+
+from django.db.models import Model, QuerySet
+
+from posthog.paging import CursorPageRequest, Page, PageRequest
+
+
+def stable_queryset_ordering(queryset: QuerySet) -> QuerySet:
+ """Add the primary key as a final ordering term for a paginated queryset."""
+ if queryset.query.is_sliced:
+ return queryset
+ # `group_by is True` means Django groups by every selected column of the model, primary key
+ # included, so a primary key term cannot change the groups. A tuple comes from
+ # `values(...).annotate(...)`, where a primary key term would split the groups.
+ if queryset.query.group_by is not None and queryset.query.group_by is not True:
+ return queryset
+
+ ordering = queryset.query.order_by or queryset.query.extra_order_by or queryset.model._meta.ordering
+ if not ordering:
+ return queryset.order_by("pk")
+
+ primary_key = queryset.model._meta.pk.name
+ if any(str(term).lstrip("-") in {"pk", primary_key} for term in ordering):
+ return queryset
+
+ direction = "-" if str(ordering[0]).startswith("-") else ""
+ return queryset.order_by(*ordering, f"{direction}pk")
+
+
+def select_page[M: Model](queryset: QuerySet[M], page: PageRequest) -> Page[M]:
+ """Return one page of `queryset` and the number of rows in all of it.
+
+ The rows are also sorted by primary key, so a page boundary between rows that share a sort value
+ cannot skip or repeat a row.
+ """
+ count = queryset.count()
+ if count == 0 or page.offset >= count:
+ return Page(results=[], count=count)
+ rows = stable_queryset_ordering(queryset)[page.offset : page.offset + page.limit]
+ return Page(results=list(rows), count=count)
+
+
+def select_cursor_page[M: Model](
+ queryset: QuerySet[M], page: CursorPageRequest, *, ordering_fields: Collection[str]
+) -> Page[M]:
+ """Return the rows of one cursor page of `queryset` and the number of rows in all of it.
+
+ `ordering_fields` names the fields a client may sort by. The primary key is always allowed,
+ because `StableCursorPagination` adds it as the last ordering term.
+ """
+ primary_key_names = {"pk", queryset.model._meta.pk.name}
+ for term in page.ordering:
+ if term.lstrip("-") not in ordering_fields and term.lstrip("-") not in primary_key_names:
+ raise ValueError(f"Cannot order by {term!r}")
+
+ count = queryset.count()
+ first = page.ordering[0]
+ queryset = stable_queryset_ordering(queryset.order_by(*page.ordering))
+ if page.bound is not None:
+ comparison = "lt" if page.bound.before else "gt"
+ queryset = queryset.filter(**{f"{first.lstrip('-')}__{comparison}": page.bound.value})
+
+ return Page(results=list(queryset[page.offset : page.offset + page.limit]), count=count)
diff --git a/products/architecture.md b/products/architecture.md
index 7fe5594fa99b..ca1258db3f25 100644
--- a/products/architecture.md
+++ b/products/architecture.md
@@ -463,6 +463,26 @@ A resource that inherits access from a parent object (`RESOURCE_FALLBACK_MAP`) n
The facade owns **tenant scoping** (`team_id` enforced via `for_team(team_id)` / a `ProductTeamModel` fail-closed manager) and **domain invariants** (state machines, idempotency) — these must hold for every caller, so they live below the HTTP boundary; user RBAC must not. Keeping RBAC in the shared DRF stack also lets cross-cutting permission tests enforce it consistently across products.
+### How do lists page?
+
+A facade returns contracts, not a queryset, so a view cannot hand its rows to a DRF paginator.
+The facade pages in SQL instead, and the view builds the standard response from that page.
+`posthog/paging.py` holds the page types, `posthog/queryset_paging.py` pages a queryset into them, and `posthog/api/pagination.py` holds the view helpers.
+
+- A facade list function takes a `PageRequest` (or a `CursorPageRequest`) and returns `Page[Contract]`.
+ Page the queryset with `select_page` (or `select_cursor_page`), then convert the rows with `page.map(...)`.
+ Both helpers count the whole list and add the primary key as the last sort term, so a page boundary never skips or repeats a row.
+- The view sets `pagination_class = FacadeLimitOffsetPagination` and returns `facade_page_response(self, fetch)`, where `fetch` calls the facade with the page.
+ For a cursor list, subclass `FacadeCursorPagination`, set `ordering`, and return `facade_cursor_page_response(self, fetch)`.
+- Use limit/offset for a list that people browse with a total.
+ Use a cursor for a list that grows at the top while people read it, such as batch export runs or deliveries, because offsets shift as rows arrive.
+ Do not add page-number lists.
+- Every list returns `count`, `next`, `previous` and `results`, so clients and MCP tools page every product the same way.
+ Add extra fields beside them when a list needs them, and do not replace them.
+- Do not set paginator attributes by hand, build a fake queryset for a paginator, or return a whole list for the view to slice.
+
+`products/visual_review/backend/presentation/views.py` shows the limit/offset form.
+
### Why not mix with the facade?
- Keeps HTTP concerns decoupled
diff --git a/products/batch_exports/backend/logic/pagination.py b/products/batch_exports/backend/logic/pagination.py
index 8ebdedebb536..2ac917a34657 100644
--- a/products/batch_exports/backend/logic/pagination.py
+++ b/products/batch_exports/backend/logic/pagination.py
@@ -4,7 +4,7 @@ from collections.abc import Collection
from django.db.models import Model, QuerySet
-from posthog.api.pagination import stable_queryset_ordering
+from posthog.queryset_paging import stable_queryset_ordering
from products.batch_exports.backend.facade import contracts
diff --git a/products/batch_exports/frontend/generated/api.schemas.ts b/products/batch_exports/frontend/generated/api.schemas.ts
index 2b655e77ecd6..6fd0dc5141e6 100644
--- a/products/batch_exports/frontend/generated/api.schemas.ts
+++ b/products/batch_exports/frontend/generated/api.schemas.ts
@@ -2489,6 +2489,11 @@ export type BatchExportsRunsLogsRetrieveParams = {
* @maximum 500
*/
limit?: number
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number
/**
* Case-insensitive substring search across log messages.
* @minLength 1
@@ -2521,6 +2526,11 @@ export type BatchExportsLogsRetrieveParams = {
* @maximum 500
*/
limit?: number
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number
/**
* Case-insensitive substring search across log messages.
* @minLength 1
@@ -2564,6 +2574,11 @@ export type FileDownloadBatchExportsLogsRetrieveParams = {
* @maximum 500
*/
limit?: number
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number
/**
* Case-insensitive substring search across log messages.
* @minLength 1
diff --git a/products/cdp/frontend/generated/api.schemas.ts b/products/cdp/frontend/generated/api.schemas.ts
index 43850476dd33..623b47846c49 100644
--- a/products/cdp/frontend/generated/api.schemas.ts
+++ b/products/cdp/frontend/generated/api.schemas.ts
@@ -907,6 +907,11 @@ export type HogFunctionsLogsRetrieveParams = {
* @maximum 500
*/
limit?: number
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number
/**
* Case-insensitive substring search across log messages.
* @minLength 1
diff --git a/products/endpoints/frontend/generated/api.schemas.ts b/products/endpoints/frontend/generated/api.schemas.ts
index 5f5fa4541ab0..b9e5752d4695 100644
--- a/products/endpoints/frontend/generated/api.schemas.ts
+++ b/products/endpoints/frontend/generated/api.schemas.ts
@@ -1222,6 +1222,11 @@ export type EndpointsLogsRetrieveParams = {
* @maximum 500
*/
limit?: number
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number
/**
* Case-insensitive substring search across log messages.
* @minLength 1
diff --git a/products/logs/backend/presentation/views/alerts_api.py b/products/logs/backend/presentation/views/alerts_api.py
index c804770cb860..565b7f9f0152 100644
--- a/products/logs/backend/presentation/views/alerts_api.py
+++ b/products/logs/backend/presentation/views/alerts_api.py
@@ -22,7 +22,6 @@ from rest_framework.response import Response
from posthog.schema import LogsAlertFilters
from posthog.api.fields import RepeatedOrCommaSeparatedListField
-from posthog.api.pagination import stable_queryset_ordering
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.shared import UserBasicSerializer
from posthog.dataclasses import frozen
@@ -32,6 +31,7 @@ from posthog.exceptions_capture import capture_exception
from posthog.helpers.impersonation import is_impersonated
from posthog.models.activity_logging.activity_log import Change, Detail, log_activity
from posthog.models.user import User
+from posthog.queryset_paging import stable_queryset_ordering
from posthog.utils import relative_date_parse
from products.alerts.backend.facade.destinations import (
diff --git a/products/subscriptions/backend/logic/subscription_reads.py b/products/subscriptions/backend/logic/subscription_reads.py
index f73757a662b5..62f5f112a9e7 100644
--- a/products/subscriptions/backend/logic/subscription_reads.py
+++ b/products/subscriptions/backend/logic/subscription_reads.py
@@ -7,8 +7,8 @@ from functools import reduce
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db.models import OuterRef, Prefetch, Q, QuerySet, Subquery
-from posthog.api.pagination import stable_queryset_ordering
from posthog.models.user import User
+from posthog.queryset_paging import stable_queryset_ordering
from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.product_analytics.backend.facade.api import insights_including_soft_deleted_for_team
diff --git a/products/visual_review/backend/facade/api.py b/products/visual_review/backend/facade/api.py
index e26e0c7fafdb..1d03fd86096b 100644
--- a/products/visual_review/backend/facade/api.py
+++ b/products/visual_review/backend/facade/api.py
@@ -25,6 +25,8 @@ from django.utils import timezone
from posthog.dataclasses import frozen
from posthog.egress.github.transport import GitHubRateLimitError
from posthog.helpers.trigram_search import search_match_type_from_instance
+from posthog.paging import Page, PageRequest
+from posthog.queryset_paging import select_page
from ..diff_metadata import (
DiffMetadata,
@@ -282,9 +284,8 @@ def get_repo(repo_id: UUID, team_id: int) -> contracts.Repo:
return _to_repo(repo)
-def list_repos(team_id: int) -> list[contracts.Repo]:
- projects = repos.list_repos_for_team(team_id)
- return [_to_repo(p) for p in projects]
+def list_repos(team_id: int, page: PageRequest) -> Page[contracts.Repo]:
+ return select_page(repos.list_repos_for_team(team_id), page).map(_to_repo)
def create_repo(team_id: int, repo_external_id: int, repo_full_name: str) -> contracts.Repo:
@@ -527,7 +528,9 @@ def list_runs(
commit_sha: str | None = None,
branch: str | None = None,
search: str | None = None,
-) -> list[contracts.Run]:
+ *,
+ page: PageRequest,
+) -> Page[contracts.Run]:
runs = run_queries.list_runs_for_team(
team_id,
review_state=review_state,
@@ -537,7 +540,7 @@ def list_runs(
branch=branch,
search=search,
)
- return [_to_run(r) for r in runs]
+ return select_page(runs, page).map(_to_run)
def get_review_state_counts(team_id: int, repo_id: UUID | None = None) -> dict[str, int]:
@@ -849,12 +852,19 @@ def _to_baseline_quarantine_summary(
def list_quarantined(
- repo_id: UUID, team_id: int, identifier: str | None = None, run_type: str | None = None
-) -> list[contracts.QuarantinedIdentifierEntry]:
- entries = quarantine.list_quarantined_identifiers(repo_id, team_id, identifier=identifier, run_type=run_type)
- user_ids = {e.created_by_id for e in entries if e.created_by_id}
+ repo_id: UUID,
+ team_id: int,
+ identifier: str | None = None,
+ run_type: str | None = None,
+ *,
+ page: PageRequest,
+) -> Page[contracts.QuarantinedIdentifierEntry]:
+ entries = select_page(
+ quarantine.list_quarantined_identifiers(repo_id, team_id, identifier=identifier, run_type=run_type), page
+ )
+ user_ids = {e.created_by_id for e in entries.results if e.created_by_id}
user_basic_infos = _fetch_user_basic_infos(user_ids)
- return [_to_quarantined_entry(q, user_basic_infos) for q in entries]
+ return entries.map(lambda entry: _to_quarantined_entry(entry, user_basic_infos))
def quarantine_identifier(
diff --git a/products/visual_review/backend/logic/quarantine.py b/products/visual_review/backend/logic/quarantine.py
index 3788533d6dd6..06b6ab6ba594 100644
--- a/products/visual_review/backend/logic/quarantine.py
+++ b/products/visual_review/backend/logic/quarantine.py
@@ -24,7 +24,7 @@ LIFT_SCOPE_DAYS = 30
def list_quarantined_identifiers(
repo_id: UUID, team_id: int, identifier: str | None = None, run_type: str | None = None
-) -> list[QuarantinedIdentifier]:
+) -> QuerySet[QuarantinedIdentifier]:
qs = (
QuarantinedIdentifier.objects.using(WRITER_DB)
.filter(repo_id=repo_id, team_id=team_id)
@@ -42,7 +42,7 @@ def list_quarantined_identifiers(
else:
now = timezone.now()
qs = qs.filter(Q(expires_at__isnull=True) | Q(expires_at__gt=now))
- return list(qs.order_by("-created_at"))
+ return qs.order_by("-created_at")
def active_quarantined_identifiers(
diff --git a/products/visual_review/backend/logic/repos.py b/products/visual_review/backend/logic/repos.py
index 08afff5fddab..a3d696d92613 100644
--- a/products/visual_review/backend/logic/repos.py
+++ b/products/visual_review/backend/logic/repos.py
@@ -4,6 +4,8 @@ from __future__ import annotations
from uuid import UUID
+from django.db.models import QuerySet
+
from ..facade.contracts import UpdateRepoInput
from ..models import Repo
from . import errors
@@ -16,8 +18,8 @@ def get_repo(repo_id: UUID, team_id: int) -> Repo:
raise errors.RepoNotFoundError(f"Repo {repo_id} not found") from e
-def list_repos_for_team(team_id: int) -> list[Repo]:
- return list(Repo.objects.filter(team_id=team_id).order_by("-created_at"))
+def list_repos_for_team(team_id: int) -> QuerySet[Repo]:
+ return Repo.objects.filter(team_id=team_id).order_by("-created_at")
def create_repo(team_id: int, repo_external_id: int, repo_full_name: str) -> Repo:
diff --git a/products/visual_review/backend/presentation/views.py b/products/visual_review/backend/presentation/views.py
index 0661c9678904..d95b5e0395a2 100644
--- a/products/visual_review/backend/presentation/views.py
+++ b/products/visual_review/backend/presentation/views.py
@@ -25,7 +25,7 @@ from rest_framework.request import Request
from rest_framework.response import Response
from posthog.api.mixins import TypedRequest, validated_request
-from posthog.api.pagination import PrecountedLimitOffsetPagination
+from posthog.api.pagination import FacadeLimitOffsetPagination, PrecountedLimitOffsetPagination, facade_page_response
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.auth import is_mcp_request
from posthog.helpers.trigram_search import MAX_SEARCH_LENGTH
@@ -190,6 +190,7 @@ class RepoViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
"""
scope_object = "visual_review"
+ pagination_class = FacadeLimitOffsetPagination
scope_object_write_actions = ["create", "partial_update", "quarantine", "unquarantine"]
scope_object_read_actions = [
"list",
@@ -204,12 +205,9 @@ class RepoViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
@extend_schema(responses={200: RepoSerializer(many=True)})
def list(self, request: Request, **kwargs) -> Response:
"""List all projects for the team."""
- projects = api.list_repos(self.team_id)
- page = self.paginate_queryset(projects)
- if page is not None:
- serializer = RepoSerializer(instance=page, many=True)
- return self.get_paginated_response(serializer.data)
- return Response(RepoSerializer(instance=projects, many=True).data)
+ return facade_page_response(
+ self, lambda page: api.list_repos(self.team_id, page), serializer_class=RepoSerializer
+ )
@validated_request(
request_serializer=CreateRepoInputSerializer,
@@ -332,12 +330,14 @@ class RepoViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
"""List quarantined identifiers. Without filter: active only. With identifier: full history."""
identifier = request.query_params.get("identifier")
run_type = request.query_params.get("run_type")
- entries = api.list_quarantined(_parse_uuid(pk), team_id=self.team_id, identifier=identifier, run_type=run_type)
- page = self.paginate_queryset(entries)
- if page is not None:
- serializer = QuarantinedIdentifierEntrySerializer(instance=page, many=True)
- return self.get_paginated_response(serializer.data)
- return Response(QuarantinedIdentifierEntrySerializer(instance=entries, many=True).data)
+ repo_id = _parse_uuid(pk)
+ return facade_page_response(
+ self,
+ lambda page: api.list_quarantined(
+ repo_id, team_id=self.team_id, identifier=identifier, run_type=run_type, page=page
+ ),
+ serializer_class=QuarantinedIdentifierEntrySerializer,
+ )
@validated_request(
request_serializer=QuarantineInputSerializer,
@@ -543,6 +543,7 @@ class RepoRunsViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
"""
scope_object = "visual_review"
+ pagination_class = FacadeLimitOffsetPagination
scope_object_read_actions = ["list", "counts"]
serializer_class = RunSerializer
@@ -568,12 +569,13 @@ class RepoRunsViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
status=status.HTTP_400_BAD_REQUEST,
)
repo_id = _parse_uuid(self.parents_query_dict["repo_id"], field="repo_id")
- runs = api.list_runs(self.team_id, review_state=review_state, repo_id=repo_id, search=search)
- page = self.paginate_queryset(runs)
- if page is not None:
- serializer = RunSerializer(instance=page, many=True)
- return self.get_paginated_response(serializer.data)
- return Response(RunSerializer(instance=runs, many=True).data)
+ return facade_page_response(
+ self,
+ lambda page: api.list_runs(
+ self.team_id, review_state=review_state, repo_id=repo_id, search=search, page=page
+ ),
+ serializer_class=RunSerializer,
+ )
@extend_schema(responses={200: ReviewStateCountsSerializer})
@action(detail=False, methods=["get"])
@@ -591,6 +593,7 @@ class RunViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
"""
scope_object = "visual_review"
+ pagination_class = FacadeLimitOffsetPagination
scope_object_write_actions = [
"create",
"complete",
@@ -642,19 +645,20 @@ class RunViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
{"detail": f"search must be at most {MAX_SEARCH_LENGTH} characters"},
status=status.HTTP_400_BAD_REQUEST,
)
- runs = api.list_runs(
- self.team_id,
- review_state=_parse_review_state(request),
- pr_number=pr_number,
- commit_sha=request.query_params.get("commit_sha"),
- branch=request.query_params.get("branch"),
- search=search,
+ review_state = _parse_review_state(request)
+ return facade_page_response(
+ self,
+ lambda page: api.list_runs(
+ self.team_id,
+ review_state=review_state,
+ pr_number=pr_number,
+ commit_sha=request.query_params.get("commit_sha"),
+ branch=request.query_params.get("branch"),
+ search=search,
+ page=page,
+ ),
+ serializer_class=RunSerializer,
)
- page = self.paginate_queryset(runs)
- if page is not None:
- serializer = RunSerializer(instance=page, many=True)
- return self.get_paginated_response(serializer.data)
- return Response(RunSerializer(instance=runs, many=True).data)
@extend_schema(responses={200: ReviewStateCountsSerializer})
@action(detail=False, methods=["get"])
diff --git a/products/visual_review/backend/tests/test_api.py b/products/visual_review/backend/tests/test_api.py
index 575d8e2a6c8f..999b0073aa05 100644
--- a/products/visual_review/backend/tests/test_api.py
+++ b/products/visual_review/backend/tests/test_api.py
@@ -5,6 +5,8 @@ from uuid import UUID, uuid4
import pytest
from unittest.mock import patch
+from posthog.paging import PageRequest
+
from products.visual_review.backend.facade import api
from products.visual_review.backend.facade.contracts import (
ApproveRunInput,
@@ -47,10 +49,11 @@ class TestProjectAPI:
api.create_repo(team_id=team.id, repo_external_id=111, repo_full_name="org/first")
api.create_repo(team_id=team.id, repo_external_id=222, repo_full_name="org/second")
- result = api.list_repos(team.id)
+ first_page = api.list_repos(team.id, PageRequest(limit=1))
+ second_page = api.list_repos(team.id, PageRequest(limit=1, offset=1))
- assert len(result) == 2
- names = {p.repo_full_name for p in result}
+ assert first_page.count == 2
+ names = {p.repo_full_name for p in first_page.results + second_page.results}
assert names == {"org/first", "org/second"}
def test_update_repo_sets_the_fields_it_is_given(self, team):
@@ -279,9 +282,9 @@ class TestRunAPI:
run_type=RunType.PLAYWRIGHT,
)
- runs = api.list_runs(repo.team_id, **filters)
+ runs = api.list_runs(repo.team_id, **filters, page=PageRequest(limit=100))
- assert {r.commit_sha for r in runs} == expected_commits
+ assert {r.commit_sha for r in runs.results} == expected_commits
@patch("products.visual_review.backend.tasks.tasks.emit_run_processing_metrics.delay")
@patch("products.visual_review.backend.tasks.tasks.process_run_diffs.delay")
diff --git a/products/visual_review/frontend/generated/api.schemas.ts b/products/visual_review/frontend/generated/api.schemas.ts
index a37708aa2fe5..5df77640c676 100644
--- a/products/visual_review/frontend/generated/api.schemas.ts
+++ b/products/visual_review/frontend/generated/api.schemas.ts
@@ -769,6 +769,7 @@ export interface PaginatedToleratedHashEntryListApi {
export type VisualReviewReposListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
@@ -784,6 +785,7 @@ export type VisualReviewReposQuarantineListParams = {
identifier?: string
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
@@ -843,6 +845,7 @@ export type VisualReviewReposTolerationPileupsRetrieveParams = {
export type VisualReviewReposRunsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
@@ -891,6 +894,7 @@ export type VisualReviewRunsListParams = {
commit_sha?: string
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
@@ -928,6 +932,7 @@ export type VisualReviewRunsSnapshotHistoryListParams = {
identifier: string
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
@@ -970,6 +975,7 @@ export type VisualReviewRunsToleratedHashesListParams = {
identifier: string
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.ts
index 293684d16cf4..16d7c5b16ecc 100644
--- a/products/warehouse_sources/frontend/generated/api.schemas.ts
+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts
@@ -15047,6 +15047,11 @@ export type ExternalDataSchemasLogsRetrieveParams = {
* @maximum 500
*/
limit?: number
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number
/**
* Case-insensitive substring search across log messages.
* @minLength 1
diff --git a/products/workflows/frontend/generated/api.schemas.ts b/products/workflows/frontend/generated/api.schemas.ts
index 2911cd2e319c..91dfa741f4ee 100644
--- a/products/workflows/frontend/generated/api.schemas.ts
+++ b/products/workflows/frontend/generated/api.schemas.ts
@@ -3087,6 +3087,11 @@ export type HogFlowTemplatesLogsRetrieveParams = {
* @maximum 500
*/
limit?: number
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number
/**
* Case-insensitive substring search across log messages.
* @minLength 1
@@ -3323,6 +3328,11 @@ export type HogFlowsLogsRetrieveParams = {
* @maximum 500
*/
limit?: number
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number
/**
* Case-insensitive substring search across log messages.
* @minLength 1
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 4246b3bee824..df1253ed3179 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -132386,6 +132386,11 @@ export namespace Schemas {
* @maximum 500
*/
limit?: number;
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number;
/**
* Case-insensitive substring search across log messages.
* @minLength 1
@@ -132418,6 +132423,11 @@ export namespace Schemas {
* @maximum 500
*/
limit?: number;
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number;
/**
* Case-insensitive substring search across log messages.
* @minLength 1
@@ -135080,6 +135090,11 @@ export namespace Schemas {
* @maximum 500
*/
limit?: number;
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number;
/**
* Case-insensitive substring search across log messages.
* @minLength 1
@@ -137107,6 +137122,11 @@ export namespace Schemas {
* @maximum 500
*/
limit?: number;
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number;
/**
* Case-insensitive substring search across log messages.
* @minLength 1
@@ -137576,6 +137596,11 @@ export namespace Schemas {
* @maximum 500
*/
limit?: number;
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number;
/**
* Case-insensitive substring search across log messages.
* @minLength 1
@@ -138021,6 +138046,11 @@ export namespace Schemas {
* @maximum 500
*/
limit?: number;
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number;
/**
* Case-insensitive substring search across log messages.
* @minLength 1
@@ -138259,6 +138289,11 @@ export namespace Schemas {
* @maximum 500
*/
limit?: number;
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number;
/**
* Case-insensitive substring search across log messages.
* @minLength 1
@@ -138591,6 +138626,11 @@ export namespace Schemas {
* @maximum 500
*/
limit?: number;
+ /**
+ * Number of log entries to skip before the first returned entry (default 0).
+ * @minimum 0
+ */
+ offset?: number;
/**
* Case-insensitive substring search across log messages.
* @minLength 1
@@ -145381,6 +145421,7 @@ export namespace Schemas {
export type VisualReviewReposListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -145396,6 +145437,7 @@ export namespace Schemas {
identifier?: string;
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -145455,6 +145497,7 @@ export namespace Schemas {
export type VisualReviewReposRunsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -145503,6 +145546,7 @@ export namespace Schemas {
commit_sha?: string;
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -145540,6 +145584,7 @@ export namespace Schemas {
identifier: string;
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -145582,6 +145627,7 @@ export namespace Schemas {
identifier: string;
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
diff --git a/services/mcp/src/generated/cdp_functions/api.ts b/services/mcp/src/generated/cdp_functions/api.ts
index 58328b0cd075..d9c66abbc809 100644
--- a/services/mcp/src/generated/cdp_functions/api.ts
+++ b/services/mcp/src/generated/cdp_functions/api.ts
@@ -1098,6 +1098,9 @@ export const HogFunctionsLogsRetrieveParams = () => zod.object({
export const hogFunctionsLogsRetrieveQueryLimitDefault = 50
export const hogFunctionsLogsRetrieveQueryLimitMax = 500
+export const hogFunctionsLogsRetrieveQueryOffsetDefault = 0
+export const hogFunctionsLogsRetrieveQueryOffsetMin = 0
+
export const HogFunctionsLogsRetrieveQueryParams = () => zod.object({
after: zod.iso
.datetime({ offset: true })
@@ -1123,6 +1126,11 @@ export const HogFunctionsLogsRetrieveQueryParams = () => zod.object({
.max(hogFunctionsLogsRetrieveQueryLimitMax)
.default(hogFunctionsLogsRetrieveQueryLimitDefault)
.describe('Maximum number of log entries to return (1-500, default 50).'),
+ offset: zod
+ .number()
+ .min(hogFunctionsLogsRetrieveQueryOffsetMin)
+ .default(hogFunctionsLogsRetrieveQueryOffsetDefault)
+ .describe('Number of log entries to skip before the first returned entry (default 0).'),
search: zod.string().min(1).optional().describe('Case-insensitive substring search across log messages.'),
})
diff --git a/services/mcp/src/generated/endpoints/api.ts b/services/mcp/src/generated/endpoints/api.ts
index c826f7cd07df..f2bc40dfa37d 100644
--- a/services/mcp/src/generated/endpoints/api.ts
+++ b/services/mcp/src/generated/endpoints/api.ts
@@ -187,6 +187,9 @@ export const EndpointsLogsRetrieveParams = () => zod.object({
export const endpointsLogsRetrieveQueryLimitDefault = 50
export const endpointsLogsRetrieveQueryLimitMax = 500
+export const endpointsLogsRetrieveQueryOffsetDefault = 0
+export const endpointsLogsRetrieveQueryOffsetMin = 0
+
export const EndpointsLogsRetrieveQueryParams = () => zod.object({
after: zod.iso
.datetime({ offset: true })
@@ -212,6 +215,11 @@ export const EndpointsLogsRetrieveQueryParams = () => zod.object({
.max(endpointsLogsRetrieveQueryLimitMax)
.default(endpointsLogsRetrieveQueryLimitDefault)
.describe('Maximum number of log entries to return (1-500, default 50).'),
+ offset: zod
+ .number()
+ .min(endpointsLogsRetrieveQueryOffsetMin)
+ .default(endpointsLogsRetrieveQueryOffsetDefault)
+ .describe('Number of log entries to skip before the first returned entry (default 0).'),
search: zod.string().min(1).optional().describe('Case-insensitive substring search across log messages.'),
})
diff --git a/services/mcp/src/generated/visual_review/api.ts b/services/mcp/src/generated/visual_review/api.ts
index 1c9495ef9d1f..cf337857dff2 100644
--- a/services/mcp/src/generated/visual_review/api.ts
+++ b/services/mcp/src/generated/visual_review/api.ts
@@ -19,8 +19,14 @@ export const VisualReviewReposListParams = () => zod.object({
),
})
+export const visualReviewReposListQueryLimitMax = 1000
+
export const VisualReviewReposListQueryParams = () => zod.object({
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod
+ .number()
+ .max(visualReviewReposListQueryLimitMax)
+ .optional()
+ .describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
})
@@ -86,9 +92,15 @@ export const VisualReviewReposQuarantineListParams = () => zod.object({
),
})
+export const visualReviewReposQuarantineListQueryLimitMax = 1000
+
export const VisualReviewReposQuarantineListQueryParams = () => zod.object({
identifier: zod.string().optional().describe('Filter by identifier (returns full history)'),
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod
+ .number()
+ .max(visualReviewReposQuarantineListQueryLimitMax)
+ .optional()
+ .describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
run_type: zod.string().optional().describe('Filter by run type'),
})
@@ -248,8 +260,14 @@ export const VisualReviewReposRunsListParams = () => zod.object({
repo_id: zod.string(),
})
+export const visualReviewReposRunsListQueryLimitMax = 1000
+
export const VisualReviewReposRunsListQueryParams = () => zod.object({
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod
+ .number()
+ .max(visualReviewReposRunsListQueryLimitMax)
+ .optional()
+ .describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
review_state: zod
.enum(['clean', 'needs_review', 'processing', 'stale'])
@@ -283,10 +301,16 @@ export const VisualReviewRunsListParams = () => zod.object({
),
})
+export const visualReviewRunsListQueryLimitMax = 1000
+
export const VisualReviewRunsListQueryParams = () => zod.object({
branch: zod.string().optional().describe('Filter by branch name'),
commit_sha: zod.string().optional().describe('Filter by full commit SHA'),
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod
+ .number()
+ .max(visualReviewRunsListQueryLimitMax)
+ .optional()
+ .describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
pr_number: zod.number().optional().describe('Filter by GitHub PR number'),
review_state: zod
@@ -464,13 +488,19 @@ export const VisualReviewRunsSnapshotHistoryListParams = () => zod.object({
),
})
+export const visualReviewRunsSnapshotHistoryListQueryLimitMax = 1000
+
export const VisualReviewRunsSnapshotHistoryListQueryParams = () => zod.object({
identifier: zod
.string()
.describe(
'Identifier of the snapshot to look up, for example a Storybook story id plus theme. Read it from the `identifier` field of a snapshot in the run. It is a name rather than a UUID, and it is required in addition to the run id in the path.'
),
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod
+ .number()
+ .max(visualReviewRunsSnapshotHistoryListQueryLimitMax)
+ .optional()
+ .describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
})
@@ -555,13 +585,19 @@ export const VisualReviewRunsToleratedHashesListParams = () => zod.object({
),
})
+export const visualReviewRunsToleratedHashesListQueryLimitMax = 1000
+
export const VisualReviewRunsToleratedHashesListQueryParams = () => zod.object({
identifier: zod
.string()
.describe(
'Identifier of the snapshot to look up, for example a Storybook story id plus theme. Read it from the `identifier` field of a snapshot in the run. It is a name rather than a UUID, and it is required in addition to the run id in the path.'
),
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod
+ .number()
+ .max(visualReviewRunsToleratedHashesListQueryLimitMax)
+ .optional()
+ .describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
})
diff --git a/services/mcp/src/generated/workflows/api.ts b/services/mcp/src/generated/workflows/api.ts
index 9a53ddc9ca40..c1f4e3bb9840 100644
--- a/services/mcp/src/generated/workflows/api.ts
+++ b/services/mcp/src/generated/workflows/api.ts
@@ -1034,6 +1034,9 @@ export const HogFlowsLogsRetrieveParams = () => zod.object({
export const hogFlowsLogsRetrieveQueryLimitDefault = 50
export const hogFlowsLogsRetrieveQueryLimitMax = 500
+export const hogFlowsLogsRetrieveQueryOffsetDefault = 0
+export const hogFlowsLogsRetrieveQueryOffsetMin = 0
+
export const HogFlowsLogsRetrieveQueryParams = () => zod.object({
after: zod.iso
.datetime({ offset: true })
@@ -1059,6 +1062,11 @@ export const HogFlowsLogsRetrieveQueryParams = () => zod.object({
.max(hogFlowsLogsRetrieveQueryLimitMax)
.default(hogFlowsLogsRetrieveQueryLimitDefault)
.describe('Maximum number of log entries to return (1-500, default 50).'),
+ offset: zod
+ .number()
+ .min(hogFlowsLogsRetrieveQueryOffsetMin)
+ .default(hogFlowsLogsRetrieveQueryOffsetDefault)
+ .describe('Number of log entries to skip before the first returned entry (default 0).'),
search: zod.string().min(1).optional().describe('Case-insensitive substring search across log messages.'),
})
diff --git a/services/mcp/src/tools/generated/cdp_functions.ts b/services/mcp/src/tools/generated/cdp_functions.ts
index 18375b04bf2a..42a59fc2cc55 100644
--- a/services/mcp/src/tools/generated/cdp_functions.ts
+++ b/services/mcp/src/tools/generated/cdp_functions.ts
@@ -284,6 +284,7 @@ const cdpFunctionsLogsRetrieve = (): ToolBase<ReturnType<typeof CdpFunctionsLogs
instance_id: params.instance_id,
level: params.level,
limit: params.limit,
+ offset: params.offset,
search: params.search,
},
})
diff --git a/services/mcp/src/tools/generated/endpoints.ts b/services/mcp/src/tools/generated/endpoints.ts
index ba36f7155c04..b1aa043e5d13 100644
--- a/services/mcp/src/tools/generated/endpoints.ts
+++ b/services/mcp/src/tools/generated/endpoints.ts
@@ -118,6 +118,7 @@ const endpointLogs = (): ToolBase<ReturnType<typeof EndpointLogsSchema>, unknown
instance_id: params.instance_id,
level: params.level,
limit: params.limit,
+ offset: params.offset,
search: params.search,
},
})
diff --git a/services/mcp/src/tools/generated/workflows.ts b/services/mcp/src/tools/generated/workflows.ts
index 6639564f8b2d..d5b5fda35367 100644
--- a/services/mcp/src/tools/generated/workflows.ts
+++ b/services/mcp/src/tools/generated/workflows.ts
@@ -673,6 +673,7 @@ const workflowsLogs = (): ToolBase<ReturnType<typeof WorkflowsLogsSchema>, unkno
instance_id: params.instance_id,
level: params.level,
limit: params.limit,
+ offset: params.offset,
search: params.search,
},
})
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/cdp-functions-logs-retrieve.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/cdp-functions-logs-retrieve.json
index 1cd51fc11620..9dc35c611d7a 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/cdp-functions-logs-retrieve.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/cdp-functions-logs-retrieve.json
@@ -34,6 +34,12 @@
"minimum": 1,
"type": "number"
},
+ "offset": {
+ "default": 0,
+ "description": "Number of log entries to skip before the first returned entry (default 0).",
+ "minimum": 0,
+ "type": "number"
+ },
"search": {
"description": "Case-insensitive substring search across log messages.",
"minLength": 1,
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/endpoint-logs.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/endpoint-logs.json
index 27bcb7bf6e6e..1f5ab59b30a3 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/endpoint-logs.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/endpoint-logs.json
@@ -33,6 +33,12 @@
"name": {
"type": "string"
},
+ "offset": {
+ "default": 0,
+ "description": "Number of log entries to skip before the first returned entry (default 0).",
+ "minimum": 0,
+ "type": "number"
+ },
"search": {
"description": "Case-insensitive substring search across log messages.",
"minLength": 1,
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/workflows-logs.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/workflows-logs.json
index 01e38bad4976..acb51365cc6d 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/workflows-logs.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/workflows-logs.json
@@ -34,6 +34,12 @@
"minimum": 1,
"type": "number"
},
+ "offset": {
+ "default": 0,
+ "description": "Number of log entries to skip before the first returned entry (default 0).",
+ "minimum": 0,
+ "type": "number"
+ },
"search": {
"description": "Case-insensitive substring search across log messages.",
"minLength": 1,