Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(devex): store every ordering field in paging cursors (#126694)

Julian Bez600e0eac260f
diff --git a/posthog/api/advanced_activity_logs/test_pagination_ordering.py b/posthog/api/advanced_activity_logs/test_pagination_ordering.py
index fad5ef41723e..f3c23dcf74bb 100644
--- a/posthog/api/advanced_activity_logs/test_pagination_ordering.py
+++ b/posthog/api/advanced_activity_logs/test_pagination_ordering.py
@@ -1,4 +1,6 @@
from datetime import timedelta
+from urllib.parse import parse_qs, quote, urlparse
+from uuid import UUID
import time_machine
from posthog.test.base import APIBaseTest
@@ -56,6 +58,17 @@ class TestActivityLogCursorOrdering(APIBaseTest):
assert idle["results"] == []
assert idle["next"] is not None
+ last_seen = ActivityLog.objects.filter(team_id=self.team.id).order_by("-created_at").first()
+ assert last_seen is not None
+ ActivityLog.objects.create(
+ id=UUID("ffffffff-ffff-ffff-ffff-ffffffffffff"),
+ team_id=self.team.id,
+ organization_id=self.organization.id,
+ scope="FeatureFlag",
+ activity="updated",
+ item_id="same-timestamp",
+ created_at=last_seen.created_at,
+ )
ActivityLog.objects.create(
team_id=self.team.id,
organization_id=self.organization.id,
@@ -65,7 +78,7 @@ class TestActivityLogCursorOrdering(APIBaseTest):
created_at=timezone.now() + timedelta(hours=1),
)
- assert [row["item_id"] for row in self.client.get(tail).json()["results"]] == ["brand-new"]
+ assert [row["item_id"] for row in self.client.get(tail).json()["results"]] == ["same-timestamp", "brand-new"]
@parameterized.expand(
[
@@ -100,6 +113,16 @@ class TestActivityLogCursorOrdering(APIBaseTest):
default = self.client.get(base)
assert [row["item_id"] for row in default.json()["results"]] == ["2", "1", "0"]
+ def test_cursor_from_one_ordering_is_rejected_under_another(self):
+ self._create_logs(3)
+ base = f"/api/projects/{self.team.id}/advanced_activity_logs/?page_size=2"
+ next_link = self.client.get(base).json()["next"]
+ cursor = parse_qs(urlparse(next_link).query)["cursor"][0]
+
+ response = self.client.get(f"{base}&ordering=created_at&cursor={quote(cursor)}")
+
+ assert response.status_code == 404
+
def test_unknown_ordering_value_is_rejected(self):
response = self.client.get(f"/api/projects/{self.team.id}/advanced_activity_logs/?ordering=scope")
diff --git a/posthog/api/advanced_activity_logs/viewset.py b/posthog/api/advanced_activity_logs/viewset.py
index a3a39c47dc33..9f4548126f14 100644
--- a/posthog/api/advanced_activity_logs/viewset.py
+++ b/posthog/api/advanced_activity_logs/viewset.py
@@ -11,7 +11,6 @@ from django.db.models import Q, QuerySet
from drf_spectacular.utils import extend_schema
from rest_framework import mixins, serializers, viewsets
from rest_framework.decorators import action
-from rest_framework.pagination import Cursor
from rest_framework.permissions import BasePermission
from rest_framework.request import Request
from rest_framework.response import Response
@@ -169,9 +168,9 @@ class TailFollowingCursorPagination(StableCursorPagination):
condition: a follower stops when `results` is empty, not when `next` is null. Leaving that on
by default would make the obvious `while next: ...` loop run forever.
- Known gap: the cursor encodes `created_at` and filters strictly past it, so an entry written
- later but sharing the last-seen timestamp is not returned. That is inherent to a
- timestamp-positioned cursor and matches DRF's behavior mid-stream.
+ Known gap: the cursor stores the last-seen `created_at` and `id`, and the next poll reads the
+ rows that sort after both. An entry written later with the same `created_at` comes back only
+ when its `id` sorts after the last-seen one.
"""
# A follower polls this on every request, so a count would scan the activity log each time.
@@ -187,11 +186,8 @@ class TailFollowingCursorPagination(StableCursorPagination):
return None
if not self.page:
# Nothing new since the caller's position, so hand the same cursor back.
- return self.encode_cursor(self.cursor) if self.cursor else None
- position = self._get_position_from_instance(self.page[-1], self.ordering)
- # DRF's stub types Cursor.position as int, but at runtime it holds the string that
- # _get_position_from_instance returns.
- return self.encode_cursor(Cursor(offset=0, reverse=False, position=position)) # type: ignore[arg-type]
+ return self.cursor_link(self.current_cursor) if self.current_cursor else None
+ return self.cursor_link(self.next_cursor(self.page))
class ActivityLogPageNumberPagination(BoundedPageNumberPagination):
@@ -205,9 +201,7 @@ class ActivityLogPagination(TailFollowingCursorPagination):
page_size = 100
page_size_query_param = "page_size"
max_page_size = 1000
- # `created_at` is not unique, and DRF encodes only the first ordering field in the cursor,
- # resolving ties with an offset. That offset is reproducible only when rows sharing a
- # timestamp come back in a stable order, which Postgres does not otherwise guarantee.
+ # `created_at` is not unique, so `id` breaks ties and the cursor stores both.
ordering = ("-created_at", "-id")
def __init__(self) -> None:
diff --git a/posthog/api/comments.py b/posthog/api/comments.py
index c4bcd89c32e8..30db638eb85c 100644
--- a/posthog/api/comments.py
+++ b/posthog/api/comments.py
@@ -665,7 +665,7 @@ class CommentPagination(StableCursorPagination):
if self.is_first_page and len(data) < self.page_size:
count = len(data)
else:
- count = self.counted_rows.count()
+ count = self.count_rows()
return Response(
{
"count": count,
diff --git a/posthog/api/pagination.py b/posthog/api/pagination.py
index f374fbdb0ff4..dc2efe8d49f6 100644
--- a/posthog/api/pagination.py
+++ b/posthog/api/pagination.py
@@ -1,16 +1,26 @@
+import json
+import hashlib
+from base64 import b64decode, b64encode
from collections.abc import Callable
-from typing import TYPE_CHECKING, Any, Optional, cast
+from typing import TYPE_CHECKING, Any, Optional
+from urllib.parse import parse_qs, urlencode
-from django.core.exceptions import ImproperlyConfigured
+from django.core.exceptions import (
+ ImproperlyConfigured,
+ ValidationError as DjangoValidationError,
+)
from rest_framework import serializers
+from rest_framework.exceptions import NotFound, ValidationError
from rest_framework.pagination import CursorPagination, LimitOffsetPagination, PageNumberPagination
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.serializers import BaseSerializer
from rest_framework.utils.urls import remove_query_param, replace_query_param
-from posthog.paging import CursorBound, CursorPageRequest, Page, PageRequest
+from posthog.dataclasses import frozen
+from posthog.paging import CursorPageRequest, Page, PageRequest
+from posthog.queryset_paging import select_cursor_rows
if TYPE_CHECKING:
# DRF's generics module reads DEFAULT_PAGINATION_CLASS while it loads, which can point at this module.
@@ -205,6 +215,14 @@ class CappedCountLimitOffsetPagination(PrecountedLimitOffsetPagination):
return paginated
+def _serialize_rows(view: "GenericAPIView", rows: list[Any], serializer_class: type[BaseSerializer] | None) -> Any:
+ """Serialize a facade page the way the view serializes its own rows, unless the caller names a serializer."""
+ if serializer_class is None:
+ # `get_serializer` keeps a view's own override of how it builds the serializer.
+ return view.get_serializer(rows, many=True).data
+ return serializer_class(rows, many=True, context=view.get_serializer_context()).data
+
+
class FacadeLimitOffsetPagination(BoundedLimitOffsetPagination):
"""Builds the standard limit/offset response from a page that a facade read in SQL.
@@ -234,43 +252,241 @@ class FacadeLimitOffsetPagination(BoundedLimitOffsetPagination):
serializer_class: type[BaseSerializer] | None = None,
) -> Response:
page = self.read_facade_page(request, fetch)
- serializer_class = serializer_class or view.get_serializer_class()
- serializer = serializer_class(page.results, many=True, context=view.get_serializer_context())
- return self.get_paginated_response(serializer.data)
+ return self.get_paginated_response(_serialize_rows(view, page.results, serializer_class))
+
+
+def _reversed_ordering(ordering: tuple[str, ...]) -> tuple[str, ...]:
+ return tuple(term[1:] if term.startswith("-") else f"-{term}" for term in ordering)
+
+
+@frozen
+class PageCursor:
+ """A decoded cursor.
+
+ The page reads the rows that sort strictly after `after` in the direction it pages, and skips
+ the first `offset` of them. `reverse` pages toward the start of the list.
+
+ `drf_position` and `drf_offset` point at the same row the way DRF's cursor does: the rows whose
+ first ordering field sorts strictly after `drf_position`, minus the first `drf_offset` of them.
+ Only a server that still runs DRF's paginator during a deploy reads them.
+ """
+
+ reverse: bool
+ after: tuple[str | None, ...]
+ offset: int = 0
+ drf_position: str | None = None
+ drf_offset: int = 0
+
+
+def _first_term_per_field(ordering: tuple[str, ...] | list[str]) -> tuple[str, ...]:
+ """Keep the first term for each field. A later term on a field that is already sorted cannot change the order."""
+ seen: set[str] = set()
+ kept: list[str] = []
+ for term in ordering:
+ field = term.lstrip("-")
+ if field in seen:
+ continue
+ seen.add(field)
+ kept.append(term)
+ return tuple(kept)
class StableCursorPagination(CursorPagination):
- """Cursor pages with a stable row order, and a `count` like the limit/offset response.
+ """Cursor pages with a total row order, and a `count` like the limit/offset response.
+
+ The primary key is added as the last ordering term, in the direction of the first term, so no
+ two rows sort equal. A cursor stores the value of every ordering field of the row at the edge
+ of the page, and the next page starts strictly past that row. So rows that share a sort value
+ cannot move a page boundary, in either direction, and no offset is needed.
- A cursor stores the value of the first ordering field. When rows share that value, their order
- must be the same in every query, so the primary key is added as the last ordering term, in the
- direction of the first term. Paging forward then never skips or repeats a row. A `previous` link
- can still land on the wrong rows when more rows share one value than fit on a page, because the
- cursor does not store the primary key.
+ DRF's own cursor stores only the first ordering field and skips ties with an offset, which sends
+ a `previous` link to the wrong rows when more rows share one value than fit on a page. Cursors
+ in that format still decode, so links issued before keep working.
"""
# A subclass can leave `count` out, because counting costs a full scan on large tables.
include_count: bool = True
+ # Breaks ties when the list has no queryset model, as for a facade list. Contracts name the primary key `id`.
+ tiebreak_field = "id"
+ max_ordering_fields = 4
+ request: Optional[Request] = None
+ # The URL of the current page, which every link starts from.
+ page_url = ""
def get_ordering(self, request: Request, queryset: Any, view: Any) -> tuple[str, ...]:
- ordering = super().get_ordering(request, queryset, view)
+ ordering = _first_term_per_field(super().get_ordering(request, queryset, view))
+ # The cursor filter grows with the square of the field count, so a request cannot ask for many fields.
+ if len(ordering) > self.max_ordering_fields:
+ raise ValidationError({"ordering": f"Order by at most {self.max_ordering_fields} fields."})
model = getattr(queryset, "model", None)
- primary_key = model._meta.pk.name if model is not None else "pk"
+ primary_key = model._meta.pk.name if model is not None else self.tiebreak_field
if any(term.lstrip("-") in {"pk", primary_key} for term in ordering):
return ordering
direction = "-" if ordering[0].startswith("-") else ""
- return (*ordering, f"{direction}pk")
+ return (*ordering, f"{direction}{primary_key}")
+
+ def position_of(self, item: Any) -> tuple[str | None, ...]:
+ """Return the value of every ordering field of `item` as text, the way a cursor stores it."""
+ values = []
+ for term in self.ordering:
+ field = term.lstrip("-")
+ value = item[field] if isinstance(item, dict) else getattr(item, field)
+ values.append(None if value is None else str(value))
+ return tuple(values)
+
+ def ordering_key(self) -> str:
+ """Return a short token for the list's ordering, so a cursor cannot be read under another ordering.
+
+ It hashes the unreversed ordering, so the next and previous cursors of one list share it.
+ """
+ return hashlib.sha256(",".join(self.ordering).encode("utf-8")).hexdigest()[:8]
+
+ def cursor_link(self, cursor: PageCursor) -> str:
+ tokens: dict[str, str] = {}
+ if cursor.drf_offset:
+ tokens["o"] = str(cursor.drf_offset)
+ if cursor.reverse:
+ tokens["r"] = "1"
+ # A cursor that skips rows came in DRF's format, which `p` and `o` already describe in full.
+ if cursor.after and not cursor.offset:
+ tokens["v"] = json.dumps(list(cursor.after), separators=(",", ":"))
+ tokens["k"] = self.ordering_key()
+ if cursor.drf_position is not None:
+ tokens["p"] = cursor.drf_position
+ encoded = b64encode(urlencode(tokens).encode("ascii")).decode("ascii")
+ return replace_query_param(self.page_url, self.cursor_query_param, encoded)
+
+ def _decode_cursor(self, request: Request) -> PageCursor | None:
+ encoded = request.query_params.get(self.cursor_query_param)
+ if encoded is None:
+ return None
+ try:
+ tokens = parse_qs(b64decode(encoded.encode("ascii")).decode("ascii"), keep_blank_values=True)
+ drf_offset = int(tokens.get("o", ["0"])[0])
+ reverse = bool(int(tokens.get("r", ["0"])[0]))
+ drf_position = tokens["p"][0] if "p" in tokens else None
+ if "v" in tokens:
+ # `v` holds values of the ordering the cursor came from, which another ordering reads as other rows.
+ if "k" in tokens and tokens["k"][0] != self.ordering_key():
+ raise NotFound(self.invalid_cursor_message)
+ # `v` names the edge row exactly, so no rows are skipped. `o` is there for DRF only.
+ after = json.loads(tokens["v"][0])
+ offset = 0
+ else:
+ # A cursor that DRF encoded holds only the first ordering field, as `p`.
+ after = [] if drf_position is None else [drf_position]
+ offset = drf_offset
+ # Deeply nested JSON in `v` makes the parser give up with a RecursionError.
+ except (TypeError, ValueError, RecursionError):
+ raise NotFound(self.invalid_cursor_message)
+ if drf_offset < 0 or not isinstance(after, list) or not all(v is None or isinstance(v, str) for v in after):
+ raise NotFound(self.invalid_cursor_message)
+ return PageCursor(
+ reverse=reverse,
+ after=tuple(after),
+ offset=min(offset, self.offset_cutoff),
+ drf_position=drf_position,
+ drf_offset=min(drf_offset, self.offset_cutoff),
+ )
+
+ def _drf_marker(self, rows_from_edge: list[Any], reverse: bool) -> tuple[str | None, int]:
+ """Return DRF's position and offset for a cursor that starts just past the first row of `rows_from_edge`.
+
+ This copies how DRF's `get_next_link` and `get_previous_link` build a cursor: the position is
+ the nearest row whose first ordering field differs from the edge row, and the offset counts
+ the rows between them.
+ """
+ edge_value = self.position_of(rows_from_edge[0])[0]
+ tied = 0
+ for row in rows_from_edge:
+ value = self.position_of(row)[0]
+ if value != edge_value:
+ return value, tied
+ tied += 1
+
+ # Every row on the page shares the edge value, so the marker has to come from outside the page.
+ incoming = self.current_cursor
+ if incoming is None:
+ return None, tied
+ if incoming.reverse == reverse:
+ return incoming.drf_position, incoming.drf_offset + tied
+ if self.row_past_page is None:
+ return None, tied
+ # DRF does the same on a change of direction. It is exact unless the tie group also covers that row.
+ # During a rolling deploy, an old pod can mis-page such a `previous` link, and any tie group larger than `offset_cutoff`.
+ return self.position_of(self.row_past_page)[0], 0
+
+ def next_cursor(self, page: list[Any]) -> PageCursor:
+ """Return the cursor of the page after the last row of `page`, which must have rows."""
+ position, offset = self._drf_marker(list(reversed(page)), reverse=False)
+ return PageCursor(reverse=False, after=self.position_of(page[-1]), drf_position=position, drf_offset=offset)
+
+ def _read_page[T](
+ self, read: Callable[[CursorPageRequest], list[T]], queryset: Any, request: Request, view: Any
+ ) -> list[T] | None:
+ self.request = request
+ self.page_size = self.get_page_size(request)
+ if not self.page_size:
+ return None
+
+ self.page_url = request.build_absolute_uri()
+ self.ordering = self.get_ordering(request, queryset, view)
+ self.current_cursor = self._decode_cursor(request)
+ start = self.current_cursor or PageCursor(reverse=False, after=())
+ if len(start.after) > len(self.ordering):
+ raise NotFound(self.invalid_cursor_message)
+
+ ordering = _reversed_ordering(self.ordering) if start.reverse else self.ordering
+ try:
+ rows = read(
+ CursorPageRequest(ordering=ordering, after=start.after, offset=start.offset, limit=self.page_size + 1)
+ )
+ except DjangoValidationError:
+ # The cursor carries a value the ordering field cannot hold, such as a malformed UUID.
+ raise NotFound(self.invalid_cursor_message)
+ self.page = rows[: self.page_size]
+ has_more = len(rows) > self.page_size
+ self.row_past_page = rows[self.page_size] if has_more else None
+ has_rows_before = bool(start.after) or start.offset > 0
+ if start.reverse:
+ self.page.reverse()
+ self.has_next, self.has_previous = has_rows_before, has_more
+ else:
+ self.has_next, self.has_previous = has_more, has_rows_before
+
+ if (self.has_next or self.has_previous) and self.template is not None:
+ self.display_page_controls = True
+ return self.page
def paginate_queryset(self, queryset: Any, request: Request, view: Any = None) -> Optional[list[Any]]:
- self.counted_rows = queryset
- return super().paginate_queryset(queryset, request, view)
+ self.count_rows: Callable[[], int] = queryset.count
+ return self._read_page(lambda page: select_cursor_rows(queryset, page), queryset, request, view)
+
+ def get_next_link(self) -> Optional[str]:
+ if not self.has_next:
+ return None
+ if not self.page:
+ # No row sorts before the cursor any more, so the next page is the first page.
+ return remove_query_param(self.page_url, self.cursor_query_param)
+ return self.cursor_link(self.next_cursor(self.page))
+
+ def get_previous_link(self) -> Optional[str]:
+ if not self.has_previous:
+ return None
+ if not self.page:
+ # No row sorts after the cursor any more, so the previous page is the last page.
+ return self.cursor_link(PageCursor(reverse=True, after=()))
+ position, offset = self._drf_marker(self.page, reverse=True)
+ return self.cursor_link(
+ PageCursor(reverse=True, after=self.position_of(self.page[0]), drf_position=position, drf_offset=offset)
+ )
def get_paginated_response(self, data: Any) -> Response:
if not self.include_count:
return super().get_paginated_response(data)
return Response(
{
- "count": self.counted_rows.count(),
+ "count": self.count_rows(),
"next": self.get_next_link(),
"previous": self.get_previous_link(),
"results": data,
@@ -289,55 +505,12 @@ class StableCursorPagination(CursorPagination):
return paginated
-class _FacadePageSource[T]:
- """Stands in for the queryset that `CursorPagination.paginate_queryset` pages through.
-
- The paginator only orders the queryset, filters it on the cursor position and slices it. This
- records the ordering and the position, and the slice reads that page through the facade. So DRF
- still decodes and encodes every cursor.
- """
-
- def __init__(
- self,
- fetch: Callable[[CursorPageRequest], Page[T]],
- ordering: tuple[str, ...] = (),
- bound: CursorBound | None = None,
- fetched: list[Page[T]] | None = None,
- ) -> None:
- self._fetch = fetch
- self._ordering = ordering
- self._bound = bound
- # Every copy shares one list, so the source the paginator received can report the count of
- # the page that a later copy fetched.
- self._fetched: list[Page[T]] = fetched if fetched is not None else []
-
- def order_by(self, *ordering: str) -> "_FacadePageSource[T]":
- return _FacadePageSource(self._fetch, ordering, self._bound, self._fetched)
-
- def filter(self, **lookup: str) -> "_FacadePageSource[T]":
- # The only filter is the position, as `<first ordering field>__lt` or `__gt`.
- ((key, value),) = lookup.items()
- bound = CursorBound(value=value, before=key.endswith("__lt"))
- return _FacadePageSource(self._fetch, self._ordering, bound, self._fetched)
-
- def __getitem__(self, items: slice) -> list[T]:
- page = self._fetch(
- CursorPageRequest(
- ordering=self._ordering, bound=self._bound, offset=items.start, limit=items.stop - items.start
- )
- )
- self._fetched.append(page)
- return page.results
-
- def count(self) -> int:
- return self._fetched[-1].count
-
-
class FacadeCursorPagination(StableCursorPagination):
"""Builds the cursor response from pages that a facade reads in SQL.
- The facade answers a `CursorPageRequest` with `select_cursor_page`, so cursors keep the format
- DRF gives a queryset-backed view.
+ The facade answers a `CursorPageRequest` with `select_cursor_page`, so its cursors work the
+ same as those of a queryset-backed view. The list has no queryset model here, so the tiebreak
+ is `tiebreak_field`, and every ordering field must be readable on the facade's contracts.
"""
def get_paginated_facade_response[T](
@@ -347,13 +520,18 @@ class FacadeCursorPagination(StableCursorPagination):
fetch: Callable[[CursorPageRequest], Page[T]],
serializer_class: type[BaseSerializer] | None = None,
) -> Response:
- rows = self.paginate_queryset(cast(Any, _FacadePageSource(fetch)), request, view)
+ fetched: list[Page[T]] = []
+
+ def read(page: CursorPageRequest) -> list[T]:
+ fetched.append(fetch(page))
+ return fetched[-1].results
+
+ rows = self._read_page(read, None, request, view)
# The page size is a class attribute, so the paginator always returns a page.
assert rows is not None
+ self.count_rows = lambda: fetched[-1].count
- serializer_class = serializer_class or view.get_serializer_class()
- serializer = serializer_class(rows, many=True, context=view.get_serializer_context())
- return self.get_paginated_response(serializer.data)
+ return self.get_paginated_response(_serialize_rows(view, rows, serializer_class))
def facade_page_response[T](
diff --git a/posthog/api/test/test_pagination.py b/posthog/api/test/test_pagination.py
index 6f7579988b6e..604d36644959 100644
--- a/posthog/api/test/test_pagination.py
+++ b/posthog/api/test/test_pagination.py
@@ -1,10 +1,15 @@
-from urllib.parse import parse_qs, urlparse
+from base64 import b64encode
+from datetime import UTC, datetime, timedelta
+from urllib.parse import parse_qs, urlencode, urlparse
from posthog.test.base import BaseTest
from parameterized import parameterized
from rest_framework import serializers
+from rest_framework.exceptions import NotFound, ValidationError
+from rest_framework.filters import OrderingFilter
from rest_framework.generics import GenericAPIView
+from rest_framework.pagination import CursorPagination
from rest_framework.request import Request
from rest_framework.test import APIRequestFactory
@@ -23,6 +28,11 @@ class _AnnotationView(GenericAPIView):
serializer_class = _AnnotationIdSerializer
+class _OrderableAnnotationView(_AnnotationView):
+ filter_backends = [OrderingFilter]
+ ordering_fields = ["date_marker", "created_at", "updated_at", "content", "id"]
+
+
class _ByDateMarker(FacadeCursorPagination):
ordering = "-date_marker"
page_size = 2
@@ -33,6 +43,11 @@ class _ByDateMarkerQueryset(StableCursorPagination):
page_size = 2
+class _ByDateMarkerOnDrf(CursorPagination):
+ ordering = ("-date_marker", "-id")
+ page_size = 2
+
+
class TestCursorPagination(BaseTest):
def _page(self, mode: str, cursor: str | None) -> dict:
request = Request(APIRequestFactory().get("/annotations/", {"cursor": cursor} if cursor else {}))
@@ -52,21 +67,100 @@ class TestCursorPagination(BaseTest):
rows = paginator.paginate_queryset(queryset, request, view)
return paginator.get_paginated_response(_AnnotationIdSerializer(rows, many=True).data).data
+ def _drf_page_ids(self, cursor: str) -> list[int]:
+ request = Request(APIRequestFactory().get("/annotations/", {"cursor": cursor}))
+ rows = _ByDateMarkerOnDrf().paginate_queryset(Annotation.objects.filter(team=self.team), request)
+ assert rows is not None
+ return [row.id for row in rows]
+
+ def _cursor(self, link: str) -> str:
+ return parse_qs(urlparse(link).query)["cursor"][0]
+
@parameterized.expand([("queryset",), ("facade",)])
- def test_pages_through_rows_that_share_the_cursor_field_once(self, mode: str) -> None:
- shared_marker = Annotation.objects.create(team=self.team, content="first").date_marker
- for content in ["second", "third", "fourth", "fifth"]:
- Annotation.objects.create(team=self.team, content=content, date_marker=shared_marker)
+ def test_pages_both_ways_through_rows_that_share_the_cursor_field(self, mode: str) -> None:
+ # A NULL sorts first in descending order, so the walk crosses from NULL rows into a tie group.
+ shared_marker = datetime(2024, 1, 1, tzinfo=UTC)
+ for marker in [None, None, shared_marker, shared_marker, shared_marker]:
+ Annotation.objects.create(team=self.team, content="note", date_marker=marker)
expected_ids = set(Annotation.objects.filter(team=self.team).values_list("id", flat=True))
- seen_ids: list[int] = []
- cursor = None
+ forward_pages: list[list[int]] = []
+ page = self._page(mode, None)
while True:
- page = self._page(mode, cursor)
assert page["count"] == 5
- seen_ids.extend(row["id"] for row in page["results"])
+ forward_pages.append([row["id"] for row in page["results"]])
if page["next"] is None:
break
- cursor = parse_qs(urlparse(page["next"]).query)["cursor"][0]
+ page = self._page(mode, self._cursor(page["next"]))
+
+ assert sorted(id for ids in forward_pages for id in ids) == sorted(expected_ids)
- assert sorted(seen_ids) == sorted(expected_ids)
+ backward_pages = [[row["id"] for row in page["results"]]]
+ while page["previous"] is not None:
+ page = self._page(mode, self._cursor(page["previous"]))
+ backward_pages.append([row["id"] for row in page["results"]])
+
+ assert backward_pages == list(reversed(forward_pages))
+
+ @parameterized.expand([("queryset",), ("facade",)])
+ def test_reads_a_cursor_in_drfs_format(self, mode: str) -> None:
+ shared_marker = Annotation.objects.create(team=self.team, content="first").date_marker
+ for content in ["second", "third"]:
+ Annotation.objects.create(team=self.team, content=content, date_marker=shared_marker)
+ ids_newest_first = list(
+ Annotation.objects.filter(team=self.team).order_by("-date_marker", "-id").values_list("id", flat=True)
+ )
+
+ # DRF wrote "skip two rows from the start" as `o=2`; clients store such links.
+ page = self._page(mode, b64encode(b"o=2").decode("ascii"))
+
+ assert [row["id"] for row in page["results"]] == ids_newest_first[2:]
+
+ @parameterized.expand([("queryset",), ("facade",)])
+ def test_drf_paginator_reads_every_cursor_to_the_same_rows(self, mode: str) -> None:
+ newest = datetime(2024, 1, 3, tzinfo=UTC)
+ shared = newest - timedelta(days=1)
+ for marker in [newest, shared, shared, shared, shared, shared, shared - timedelta(days=1)]:
+ Annotation.objects.create(team=self.team, content="note", date_marker=marker)
+
+ page = self._page(mode, None)
+ cursors: list[str] = []
+ while page["next"] is not None:
+ cursors.append(self._cursor(page["next"]))
+ page = self._page(mode, cursors[-1])
+ while page["previous"] is not None:
+ cursors.append(self._cursor(page["previous"]))
+ page = self._page(mode, cursors[-1])
+
+ assert len(cursors) == 6
+ for cursor in cursors:
+ assert self._drf_page_ids(cursor) == [row["id"] for row in self._page(mode, cursor)["results"]]
+
+ def test_ordering_keeps_one_term_per_field_and_caps_the_field_count(self) -> None:
+ def ordering_for(param: str) -> tuple[str, ...]:
+ request = Request(APIRequestFactory().get("/annotations/", {"ordering": param}))
+ return _ByDateMarkerQueryset().get_ordering(request, Annotation.objects.all(), _OrderableAnnotationView())
+
+ assert ordering_for("date_marker,-date_marker,date_marker") == ("date_marker", "id")
+ with self.assertRaises(ValidationError):
+ ordering_for("date_marker,created_at,updated_at,content,-id")
+
+ @parameterized.expand(
+ [
+ ("negative_offset", b64encode(b"o=-1").decode("ascii")),
+ ("deeply_nested_position", b64encode(urlencode({"v": "[" * 100_000}).encode("ascii")).decode("ascii")),
+ (
+ "value_the_field_cannot_hold",
+ b64encode(urlencode({"v": '["not-a-date","1"]'}).encode("ascii")).decode("ascii"),
+ ),
+ (
+ "id_the_field_cannot_hold",
+ b64encode(urlencode({"v": '["2024-01-01 00:00:00+00:00","not-an-integer"]'}).encode("ascii")).decode(
+ "ascii"
+ ),
+ ),
+ ]
+ )
+ def test_rejects_a_malformed_cursor(self, _name: str, cursor: str) -> None:
+ with self.assertRaises(NotFound):
+ self._page("queryset", cursor)
diff --git a/posthog/paging.py b/posthog/paging.py
index 423a97f4af94..f0e6d883baf8 100644
--- a/posthog/paging.py
+++ b/posthog/paging.py
@@ -24,38 +24,31 @@ class PageRequest:
raise ValueError(f"A page cannot start before the first item, got offset={self.offset}.")
-@frozen
-class CursorBound:
- """Where a cursor page starts.
-
- Only the items whose first ordering field sorts before `value` (when `before`) or after it
- (otherwise) are on the page. `value` is the field's value as text, as an API cursor carries it.
- """
-
- value: str
- before: bool
-
-
@frozen
class CursorPageRequest:
"""One page of a list that pages by cursor.
`ordering` names the fields to sort by, most significant first. A leading `-` sorts that field
- in descending order. The page skips `offset` items past `bound` and holds at most `limit` items.
+ in descending order. `after` holds the values of the leading ordering fields of the item just
+ before the page, as text the way an API cursor carries them, with None for a NULL. The page
+ holds at most `limit` of the items that sort strictly after that item, past the first `offset`
+ of them. An empty `after` starts at the first item.
"""
ordering: tuple[str, ...]
- bound: CursorBound | None
+ after: tuple[str | None, ...]
offset: int
limit: int
def __post_init__(self) -> None:
if not self.ordering:
raise ValueError("A cursor page needs at least one ordering field.")
+ if len(self.after) > len(self.ordering):
+ raise ValueError(f"A cursor holds one value per ordering field, got {len(self.after)} values.")
if self.limit < 1:
raise ValueError(f"A page holds at least one item, got limit={self.limit}.")
if self.offset < 0:
- raise ValueError(f"A page cannot start before the bound, got offset={self.offset}.")
+ raise ValueError(f"A page cannot start before the cursor, got offset={self.offset}.")
@frozen
diff --git a/posthog/queryset_paging.py b/posthog/queryset_paging.py
index e23e0b456bd2..ffef30385b5c 100644
--- a/posthog/queryset_paging.py
+++ b/posthog/queryset_paging.py
@@ -2,9 +2,10 @@
Logic and facade modules import it, so it imports Django only and never DRF."""
-from collections.abc import Collection
+from collections.abc import Collection, Sequence
-from django.db.models import Model, QuerySet
+from django.core.exceptions import FieldDoesNotExist
+from django.db.models import Field, Model, Q, QuerySet
from posthog.paging import CursorPageRequest, Page, PageRequest
@@ -44,6 +45,85 @@ def select_page[M: Model](queryset: QuerySet[M], page: PageRequest) -> Page[M]:
return Page(results=list(rows), count=count)
+def _model_field(model: type[Model], field: str) -> Field | None:
+ """Return the column field that `field` names, or None for an annotation or a relation without a column."""
+ if field == "pk":
+ return model._meta.pk
+ try:
+ model_field = model._meta.get_field(field)
+ except FieldDoesNotExist:
+ return None
+ return model_field if isinstance(model_field, Field) else None
+
+
+def _is_nullable(model: type[Model], field: str) -> bool:
+ model_field = _model_field(model, field)
+ # An annotation, a reverse relation or a generic key can hold NULL, and nothing here tells otherwise.
+ return True if model_field is None else model_field.null
+
+
+def _cursor_value(model: type[Model], field: str, value: str | None) -> object:
+ """Convert a cursor value from text to the type of its field.
+
+ A value the field cannot hold, such as "abc" for an integer, raises Django's ValidationError.
+ """
+ if value is None:
+ return None
+ model_field = _model_field(model, field)
+ if model_field is None:
+ return value
+ return model_field.to_python(value)
+
+
+def _sorts_after(field: str, value: object, *, descending: bool, nullable: bool) -> Q | None:
+ """Match the rows whose `field` alone sorts after `value`, or return None when no row can.
+
+ Postgres sorts NULL after every value in ascending order and before every value in descending
+ order, and `value` is None for a NULL.
+ """
+ if value is None:
+ return Q(**{f"{field}__isnull": False}) if descending else None
+ if descending:
+ return Q(**{f"{field}__lt": value})
+ if nullable:
+ return Q(**{f"{field}__gt": value}) | Q(**{f"{field}__isnull": True})
+ return Q(**{f"{field}__gt": value})
+
+
+def _rows_after(model: type[Model], ordering: Sequence[str], after: Sequence[str | None]) -> Q:
+ """Match the rows that sort strictly after the row whose leading ordering fields hold `after`.
+
+ For the ordering `(a, -b)` this is `a > x OR (a = x AND b < y)`. A cursor that DRF encoded
+ holds only the first field, so the match then compares that field alone.
+ """
+ # Starts as a match for no row, and each field adds one more way to sort after the cursor row.
+ rows_after = Q(pk__in=[])
+ ties = Q()
+ values = [_cursor_value(model, term.lstrip("-"), value) for term, value in zip(ordering, after)]
+ for term, value in zip(ordering, values):
+ field = term.lstrip("-")
+ descending = term.startswith("-")
+ sorts_after = _sorts_after(field, value, descending=descending, nullable=_is_nullable(model, field))
+ if sorts_after is not None:
+ rows_after |= ties & sorts_after
+ ties &= Q(**{f"{field}__isnull": True}) if value is None else Q(**{field: value})
+
+ first = ordering[0].lstrip("-")
+ if len(after) < 2 or values[0] is None or _is_nullable(model, first):
+ return rows_after
+ # The first field's own bound repeats part of the match, so that an index on that field can limit the scan.
+ bound = "lte" if ordering[0].startswith("-") else "gte"
+ return Q(**{f"{first}__{bound}": values[0]}) & rows_after
+
+
+def select_cursor_rows[M: Model](queryset: QuerySet[M], page: CursorPageRequest) -> list[M]:
+ """Return the rows of one cursor page of `queryset`, without counting the whole list."""
+ queryset = stable_queryset_ordering(queryset.order_by(*page.ordering))
+ if page.after:
+ queryset = queryset.filter(_rows_after(queryset.model, page.ordering, page.after))
+ return list(queryset[page.offset : page.offset + page.limit])
+
+
def select_cursor_page[M: Model](
queryset: QuerySet[M], page: CursorPageRequest, *, ordering_fields: Collection[str]
) -> Page[M]:
@@ -58,10 +138,4 @@ def select_cursor_page[M: Model](
raise ValueError(f"Cannot order by {term!r}")
count = queryset.count()
- first = page.ordering[0]
- queryset = stable_queryset_ordering(queryset.order_by(*page.ordering))
- if page.bound is not None:
- comparison = "lt" if page.bound.before else "gt"
- queryset = queryset.filter(**{f"{first.lstrip('-')}__{comparison}": page.bound.value})
-
- return Page(results=list(queryset[page.offset : page.offset + page.limit]), count=count)
+ return Page(results=select_cursor_rows(queryset, page), count=count)
diff --git a/products/architecture.md b/products/architecture.md
index 449fe6e12e4c..ae437d3dc1bd 100644
--- a/products/architecture.md
+++ b/products/architecture.md
@@ -483,9 +483,13 @@ The facade pages in SQL instead, and the view builds the standard response from
- A facade list function takes a `PageRequest` (or a `CursorPageRequest`) and returns `Page[Contract]`.
Page the queryset with `select_page` (or `select_cursor_page`), then convert the rows with `page.map(...)`.
- Both helpers count the whole list and add the primary key as the last sort term, so a page boundary never skips or repeats a row.
+ Both helpers count the whole list and add the primary key as the last sort term, so the page order is deterministic.
+ An offset page can still shift when rows are added or deleted between requests.
+ A cursor page does not skip or repeat a row, unless the ordering values of a row change between requests.
+ The queryset must hold one row per item, so a join that duplicates rows breaks both forms.
- The view sets `pagination_class = FacadeLimitOffsetPagination` and returns `facade_page_response(self, fetch)`, where `fetch` calls the facade with the page.
For a cursor list, subclass `FacadeCursorPagination`, set `ordering`, and return `facade_cursor_page_response(self, fetch)`.
+ A cursor stores every ordering field of the row at the page edge, so the contract must expose those fields and `id`.
- Use limit/offset for a list that people browse with a total.
Use a cursor for a list that grows at the top while people read it, such as batch export runs or deliveries, because offsets shift as rows arrive.
Do not add page-number lists.
diff --git a/products/batch_exports/backend/tests/test_facade.py b/products/batch_exports/backend/tests/test_facade.py
index ed24970ed255..c75d4a90c3a0 100644
--- a/products/batch_exports/backend/tests/test_facade.py
+++ b/products/batch_exports/backend/tests/test_facade.py
@@ -285,7 +285,7 @@ def test_runs_and_backfills_are_scoped_to_the_team_and_export(team, organization
start=dt.datetime(2000, 1, 1, tzinfo=dt.UTC), end=dt.datetime.now(dt.UTC) + dt.timedelta(days=1)
)
)
- page = CursorPageRequest(ordering=("-created_at",), bound=None, offset=0, limit=10)
+ page = CursorPageRequest(ordering=("-created_at",), after=(), offset=0, limit=10)
own_runs = api.list_batch_export_runs(team.pk, export_id, filters=every_run, page=page)
assert [run.id for run in own_runs.results] == [run_id]