:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(devex): store every ordering field in paging cursors (#126694)
Julian Bez600e0eac260f
diff --git a/posthog/api/advanced_activity_logs/test_pagination_ordering.py b/posthog/api/advanced_activity_logs/test_pagination_ordering.pyindex fad5ef41723e..f3c23dcf74bb 100644--- a/posthog/api/advanced_activity_logs/test_pagination_ordering.py+++ b/posthog/api/advanced_activity_logs/test_pagination_ordering.py@@ -1,4 +1,6 @@from datetime import timedelta+from urllib.parse import parse_qs, quote, urlparse+from uuid import UUIDimport time_machinefrom posthog.test.base import APIBaseTest@@ -56,6 +58,17 @@ class TestActivityLogCursorOrdering(APIBaseTest):assert idle["results"] == []assert idle["next"] is not None+ last_seen = ActivityLog.objects.filter(team_id=self.team.id).order_by("-created_at").first()+ assert last_seen is not None+ ActivityLog.objects.create(+ id=UUID("ffffffff-ffff-ffff-ffff-ffffffffffff"),+ team_id=self.team.id,+ organization_id=self.organization.id,+ scope="FeatureFlag",+ activity="updated",+ item_id="same-timestamp",+ created_at=last_seen.created_at,+ )ActivityLog.objects.create(team_id=self.team.id,organization_id=self.organization.id,@@ -65,7 +78,7 @@ class TestActivityLogCursorOrdering(APIBaseTest):created_at=timezone.now() + timedelta(hours=1),)- assert [row["item_id"] for row in self.client.get(tail).json()["results"]] == ["brand-new"]+ assert [row["item_id"] for row in self.client.get(tail).json()["results"]] == ["same-timestamp", "brand-new"]@parameterized.expand([@@ -100,6 +113,16 @@ class TestActivityLogCursorOrdering(APIBaseTest):default = self.client.get(base)assert [row["item_id"] for row in default.json()["results"]] == ["2", "1", "0"]+ def test_cursor_from_one_ordering_is_rejected_under_another(self):+ self._create_logs(3)+ base = f"/api/projects/{self.team.id}/advanced_activity_logs/?page_size=2"+ next_link = self.client.get(base).json()["next"]+ cursor = parse_qs(urlparse(next_link).query)["cursor"][0]++ response = self.client.get(f"{base}&ordering=created_at&cursor={quote(cursor)}")++ assert response.status_code == 404+def test_unknown_ordering_value_is_rejected(self):response = self.client.get(f"/api/projects/{self.team.id}/advanced_activity_logs/?ordering=scope")diff --git a/posthog/api/advanced_activity_logs/viewset.py b/posthog/api/advanced_activity_logs/viewset.pyindex a3a39c47dc33..9f4548126f14 100644--- a/posthog/api/advanced_activity_logs/viewset.py+++ b/posthog/api/advanced_activity_logs/viewset.py@@ -11,7 +11,6 @@ from django.db.models import Q, QuerySetfrom drf_spectacular.utils import extend_schemafrom rest_framework import mixins, serializers, viewsetsfrom rest_framework.decorators import action-from rest_framework.pagination import Cursorfrom rest_framework.permissions import BasePermissionfrom rest_framework.request import Requestfrom rest_framework.response import Response@@ -169,9 +168,9 @@ class TailFollowingCursorPagination(StableCursorPagination):condition: a follower stops when `results` is empty, not when `next` is null. Leaving that onby default would make the obvious `while next: ...` loop run forever.- Known gap: the cursor encodes `created_at` and filters strictly past it, so an entry written- later but sharing the last-seen timestamp is not returned. That is inherent to a- timestamp-positioned cursor and matches DRF's behavior mid-stream.+ Known gap: the cursor stores the last-seen `created_at` and `id`, and the next poll reads the+ rows that sort after both. An entry written later with the same `created_at` comes back only+ when its `id` sorts after the last-seen one."""# A follower polls this on every request, so a count would scan the activity log each time.@@ -187,11 +186,8 @@ class TailFollowingCursorPagination(StableCursorPagination):return Noneif not self.page:# Nothing new since the caller's position, so hand the same cursor back.- return self.encode_cursor(self.cursor) if self.cursor else None- position = self._get_position_from_instance(self.page[-1], self.ordering)- # DRF's stub types Cursor.position as int, but at runtime it holds the string that- # _get_position_from_instance returns.- return self.encode_cursor(Cursor(offset=0, reverse=False, position=position)) # type: ignore[arg-type]+ return self.cursor_link(self.current_cursor) if self.current_cursor else None+ return self.cursor_link(self.next_cursor(self.page))class ActivityLogPageNumberPagination(BoundedPageNumberPagination):@@ -205,9 +201,7 @@ class ActivityLogPagination(TailFollowingCursorPagination):page_size = 100page_size_query_param = "page_size"max_page_size = 1000- # `created_at` is not unique, and DRF encodes only the first ordering field in the cursor,- # resolving ties with an offset. That offset is reproducible only when rows sharing a- # timestamp come back in a stable order, which Postgres does not otherwise guarantee.+ # `created_at` is not unique, so `id` breaks ties and the cursor stores both.ordering = ("-created_at", "-id")def __init__(self) -> None:diff --git a/posthog/api/comments.py b/posthog/api/comments.pyindex c4bcd89c32e8..30db638eb85c 100644--- a/posthog/api/comments.py+++ b/posthog/api/comments.py@@ -665,7 +665,7 @@ class CommentPagination(StableCursorPagination):if self.is_first_page and len(data) < self.page_size:count = len(data)else:- count = self.counted_rows.count()+ count = self.count_rows()return Response({"count": count,diff --git a/posthog/api/pagination.py b/posthog/api/pagination.pyindex f374fbdb0ff4..dc2efe8d49f6 100644--- a/posthog/api/pagination.py+++ b/posthog/api/pagination.py@@ -1,16 +1,26 @@+import json+import hashlib+from base64 import b64decode, b64encodefrom collections.abc import Callable-from typing import TYPE_CHECKING, Any, Optional, cast+from typing import TYPE_CHECKING, Any, Optional+from urllib.parse import parse_qs, urlencode-from django.core.exceptions import ImproperlyConfigured+from django.core.exceptions import (+ ImproperlyConfigured,+ ValidationError as DjangoValidationError,+)from rest_framework import serializers+from rest_framework.exceptions import NotFound, ValidationErrorfrom rest_framework.pagination import CursorPagination, LimitOffsetPagination, PageNumberPaginationfrom rest_framework.request import Requestfrom rest_framework.response import Responsefrom rest_framework.serializers import BaseSerializerfrom rest_framework.utils.urls import remove_query_param, replace_query_param-from posthog.paging import CursorBound, CursorPageRequest, Page, PageRequest+from posthog.dataclasses import frozen+from posthog.paging import CursorPageRequest, Page, PageRequest+from posthog.queryset_paging import select_cursor_rowsif TYPE_CHECKING:# DRF's generics module reads DEFAULT_PAGINATION_CLASS while it loads, which can point at this module.@@ -205,6 +215,14 @@ class CappedCountLimitOffsetPagination(PrecountedLimitOffsetPagination):return paginated+def _serialize_rows(view: "GenericAPIView", rows: list[Any], serializer_class: type[BaseSerializer] | None) -> Any:+ """Serialize a facade page the way the view serializes its own rows, unless the caller names a serializer."""+ if serializer_class is None:+ # `get_serializer` keeps a view's own override of how it builds the serializer.+ return view.get_serializer(rows, many=True).data+ return serializer_class(rows, many=True, context=view.get_serializer_context()).data++class FacadeLimitOffsetPagination(BoundedLimitOffsetPagination):"""Builds the standard limit/offset response from a page that a facade read in SQL.@@ -234,43 +252,241 @@ class FacadeLimitOffsetPagination(BoundedLimitOffsetPagination):serializer_class: type[BaseSerializer] | None = None,) -> Response:page = self.read_facade_page(request, fetch)- serializer_class = serializer_class or view.get_serializer_class()- serializer = serializer_class(page.results, many=True, context=view.get_serializer_context())- return self.get_paginated_response(serializer.data)+ return self.get_paginated_response(_serialize_rows(view, page.results, serializer_class))+++def _reversed_ordering(ordering: tuple[str, ...]) -> tuple[str, ...]:+ return tuple(term[1:] if term.startswith("-") else f"-{term}" for term in ordering)+++@frozen+class PageCursor:+ """A decoded cursor.++ The page reads the rows that sort strictly after `after` in the direction it pages, and skips+ the first `offset` of them. `reverse` pages toward the start of the list.++ `drf_position` and `drf_offset` point at the same row the way DRF's cursor does: the rows whose+ first ordering field sorts strictly after `drf_position`, minus the first `drf_offset` of them.+ Only a server that still runs DRF's paginator during a deploy reads them.+ """++ reverse: bool+ after: tuple[str | None, ...]+ offset: int = 0+ drf_position: str | None = None+ drf_offset: int = 0+++def _first_term_per_field(ordering: tuple[str, ...] | list[str]) -> tuple[str, ...]:+ """Keep the first term for each field. A later term on a field that is already sorted cannot change the order."""+ seen: set[str] = set()+ kept: list[str] = []+ for term in ordering:+ field = term.lstrip("-")+ if field in seen:+ continue+ seen.add(field)+ kept.append(term)+ return tuple(kept)class StableCursorPagination(CursorPagination):- """Cursor pages with a stable row order, and a `count` like the limit/offset response.+ """Cursor pages with a total row order, and a `count` like the limit/offset response.++ The primary key is added as the last ordering term, in the direction of the first term, so no+ two rows sort equal. A cursor stores the value of every ordering field of the row at the edge+ of the page, and the next page starts strictly past that row. So rows that share a sort value+ cannot move a page boundary, in either direction, and no offset is needed.- A cursor stores the value of the first ordering field. When rows share that value, their order- must be the same in every query, so the primary key is added as the last ordering term, in the- direction of the first term. Paging forward then never skips or repeats a row. A `previous` link- can still land on the wrong rows when more rows share one value than fit on a page, because the- cursor does not store the primary key.+ DRF's own cursor stores only the first ordering field and skips ties with an offset, which sends+ a `previous` link to the wrong rows when more rows share one value than fit on a page. Cursors+ in that format still decode, so links issued before keep working."""# A subclass can leave `count` out, because counting costs a full scan on large tables.include_count: bool = True+ # Breaks ties when the list has no queryset model, as for a facade list. Contracts name the primary key `id`.+ tiebreak_field = "id"+ max_ordering_fields = 4+ request: Optional[Request] = None+ # The URL of the current page, which every link starts from.+ page_url = ""def get_ordering(self, request: Request, queryset: Any, view: Any) -> tuple[str, ...]:- ordering = super().get_ordering(request, queryset, view)+ ordering = _first_term_per_field(super().get_ordering(request, queryset, view))+ # The cursor filter grows with the square of the field count, so a request cannot ask for many fields.+ if len(ordering) > self.max_ordering_fields:+ raise ValidationError({"ordering": f"Order by at most {self.max_ordering_fields} fields."})model = getattr(queryset, "model", None)- primary_key = model._meta.pk.name if model is not None else "pk"+ primary_key = model._meta.pk.name if model is not None else self.tiebreak_fieldif any(term.lstrip("-") in {"pk", primary_key} for term in ordering):return orderingdirection = "-" if ordering[0].startswith("-") else ""- return (*ordering, f"{direction}pk")+ return (*ordering, f"{direction}{primary_key}")++ def position_of(self, item: Any) -> tuple[str | None, ...]:+ """Return the value of every ordering field of `item` as text, the way a cursor stores it."""+ values = []+ for term in self.ordering:+ field = term.lstrip("-")+ value = item[field] if isinstance(item, dict) else getattr(item, field)+ values.append(None if value is None else str(value))+ return tuple(values)++ def ordering_key(self) -> str:+ """Return a short token for the list's ordering, so a cursor cannot be read under another ordering.++ It hashes the unreversed ordering, so the next and previous cursors of one list share it.+ """+ return hashlib.sha256(",".join(self.ordering).encode("utf-8")).hexdigest()[:8]++ def cursor_link(self, cursor: PageCursor) -> str:+ tokens: dict[str, str] = {}+ if cursor.drf_offset:+ tokens["o"] = str(cursor.drf_offset)+ if cursor.reverse:+ tokens["r"] = "1"+ # A cursor that skips rows came in DRF's format, which `p` and `o` already describe in full.+ if cursor.after and not cursor.offset:+ tokens["v"] = json.dumps(list(cursor.after), separators=(",", ":"))+ tokens["k"] = self.ordering_key()+ if cursor.drf_position is not None:+ tokens["p"] = cursor.drf_position+ encoded = b64encode(urlencode(tokens).encode("ascii")).decode("ascii")+ return replace_query_param(self.page_url, self.cursor_query_param, encoded)++ def _decode_cursor(self, request: Request) -> PageCursor | None:+ encoded = request.query_params.get(self.cursor_query_param)+ if encoded is None:+ return None+ try:+ tokens = parse_qs(b64decode(encoded.encode("ascii")).decode("ascii"), keep_blank_values=True)+ drf_offset = int(tokens.get("o", ["0"])[0])+ reverse = bool(int(tokens.get("r", ["0"])[0]))+ drf_position = tokens["p"][0] if "p" in tokens else None+ if "v" in tokens:+ # `v` holds values of the ordering the cursor came from, which another ordering reads as other rows.+ if "k" in tokens and tokens["k"][0] != self.ordering_key():+ raise NotFound(self.invalid_cursor_message)+ # `v` names the edge row exactly, so no rows are skipped. `o` is there for DRF only.+ after = json.loads(tokens["v"][0])+ offset = 0+ else:+ # A cursor that DRF encoded holds only the first ordering field, as `p`.+ after = [] if drf_position is None else [drf_position]+ offset = drf_offset+ # Deeply nested JSON in `v` makes the parser give up with a RecursionError.+ except (TypeError, ValueError, RecursionError):+ raise NotFound(self.invalid_cursor_message)+ if drf_offset < 0 or not isinstance(after, list) or not all(v is None or isinstance(v, str) for v in after):+ raise NotFound(self.invalid_cursor_message)+ return PageCursor(+ reverse=reverse,+ after=tuple(after),+ offset=min(offset, self.offset_cutoff),+ drf_position=drf_position,+ drf_offset=min(drf_offset, self.offset_cutoff),+ )++ def _drf_marker(self, rows_from_edge: list[Any], reverse: bool) -> tuple[str | None, int]:+ """Return DRF's position and offset for a cursor that starts just past the first row of `rows_from_edge`.++ This copies how DRF's `get_next_link` and `get_previous_link` build a cursor: the position is+ the nearest row whose first ordering field differs from the edge row, and the offset counts+ the rows between them.+ """+ edge_value = self.position_of(rows_from_edge[0])[0]+ tied = 0+ for row in rows_from_edge:+ value = self.position_of(row)[0]+ if value != edge_value:+ return value, tied+ tied += 1++ # Every row on the page shares the edge value, so the marker has to come from outside the page.+ incoming = self.current_cursor+ if incoming is None:+ return None, tied+ if incoming.reverse == reverse:+ return incoming.drf_position, incoming.drf_offset + tied+ if self.row_past_page is None:+ return None, tied+ # DRF does the same on a change of direction. It is exact unless the tie group also covers that row.+ # During a rolling deploy, an old pod can mis-page such a `previous` link, and any tie group larger than `offset_cutoff`.+ return self.position_of(self.row_past_page)[0], 0++ def next_cursor(self, page: list[Any]) -> PageCursor:+ """Return the cursor of the page after the last row of `page`, which must have rows."""+ position, offset = self._drf_marker(list(reversed(page)), reverse=False)+ return PageCursor(reverse=False, after=self.position_of(page[-1]), drf_position=position, drf_offset=offset)++ def _read_page[T](+ self, read: Callable[[CursorPageRequest], list[T]], queryset: Any, request: Request, view: Any+ ) -> list[T] | None:+ self.request = request+ self.page_size = self.get_page_size(request)+ if not self.page_size:+ return None++ self.page_url = request.build_absolute_uri()+ self.ordering = self.get_ordering(request, queryset, view)+ self.current_cursor = self._decode_cursor(request)+ start = self.current_cursor or PageCursor(reverse=False, after=())+ if len(start.after) > len(self.ordering):+ raise NotFound(self.invalid_cursor_message)++ ordering = _reversed_ordering(self.ordering) if start.reverse else self.ordering+ try:+ rows = read(+ CursorPageRequest(ordering=ordering, after=start.after, offset=start.offset, limit=self.page_size + 1)+ )+ except DjangoValidationError:+ # The cursor carries a value the ordering field cannot hold, such as a malformed UUID.+ raise NotFound(self.invalid_cursor_message)+ self.page = rows[: self.page_size]+ has_more = len(rows) > self.page_size+ self.row_past_page = rows[self.page_size] if has_more else None+ has_rows_before = bool(start.after) or start.offset > 0+ if start.reverse:+ self.page.reverse()+ self.has_next, self.has_previous = has_rows_before, has_more+ else:+ self.has_next, self.has_previous = has_more, has_rows_before++ if (self.has_next or self.has_previous) and self.template is not None:+ self.display_page_controls = True+ return self.pagedef paginate_queryset(self, queryset: Any, request: Request, view: Any = None) -> Optional[list[Any]]:- self.counted_rows = queryset- return super().paginate_queryset(queryset, request, view)+ self.count_rows: Callable[[], int] = queryset.count+ return self._read_page(lambda page: select_cursor_rows(queryset, page), queryset, request, view)++ def get_next_link(self) -> Optional[str]:+ if not self.has_next:+ return None+ if not self.page:+ # No row sorts before the cursor any more, so the next page is the first page.+ return remove_query_param(self.page_url, self.cursor_query_param)+ return self.cursor_link(self.next_cursor(self.page))++ def get_previous_link(self) -> Optional[str]:+ if not self.has_previous:+ return None+ if not self.page:+ # No row sorts after the cursor any more, so the previous page is the last page.+ return self.cursor_link(PageCursor(reverse=True, after=()))+ position, offset = self._drf_marker(self.page, reverse=True)+ return self.cursor_link(+ PageCursor(reverse=True, after=self.position_of(self.page[0]), drf_position=position, drf_offset=offset)+ )def get_paginated_response(self, data: Any) -> Response:if not self.include_count:return super().get_paginated_response(data)return Response({- "count": self.counted_rows.count(),+ "count": self.count_rows(),"next": self.get_next_link(),"previous": self.get_previous_link(),"results": data,@@ -289,55 +505,12 @@ class StableCursorPagination(CursorPagination):return paginated-class _FacadePageSource[T]:- """Stands in for the queryset that `CursorPagination.paginate_queryset` pages through.-- The paginator only orders the queryset, filters it on the cursor position and slices it. This- records the ordering and the position, and the slice reads that page through the facade. So DRF- still decodes and encodes every cursor.- """-- def __init__(- self,- fetch: Callable[[CursorPageRequest], Page[T]],- ordering: tuple[str, ...] = (),- bound: CursorBound | None = None,- fetched: list[Page[T]] | None = None,- ) -> None:- self._fetch = fetch- self._ordering = ordering- self._bound = bound- # Every copy shares one list, so the source the paginator received can report the count of- # the page that a later copy fetched.- self._fetched: list[Page[T]] = fetched if fetched is not None else []-- def order_by(self, *ordering: str) -> "_FacadePageSource[T]":- return _FacadePageSource(self._fetch, ordering, self._bound, self._fetched)-- def filter(self, **lookup: str) -> "_FacadePageSource[T]":- # The only filter is the position, as `<first ordering field>__lt` or `__gt`.- ((key, value),) = lookup.items()- bound = CursorBound(value=value, before=key.endswith("__lt"))- return _FacadePageSource(self._fetch, self._ordering, bound, self._fetched)-- def __getitem__(self, items: slice) -> list[T]:- page = self._fetch(- CursorPageRequest(- ordering=self._ordering, bound=self._bound, offset=items.start, limit=items.stop - items.start- )- )- self._fetched.append(page)- return page.results-- def count(self) -> int:- return self._fetched[-1].count--class FacadeCursorPagination(StableCursorPagination):"""Builds the cursor response from pages that a facade reads in SQL.- The facade answers a `CursorPageRequest` with `select_cursor_page`, so cursors keep the format- DRF gives a queryset-backed view.+ The facade answers a `CursorPageRequest` with `select_cursor_page`, so its cursors work the+ same as those of a queryset-backed view. The list has no queryset model here, so the tiebreak+ is `tiebreak_field`, and every ordering field must be readable on the facade's contracts."""def get_paginated_facade_response[T](@@ -347,13 +520,18 @@ class FacadeCursorPagination(StableCursorPagination):fetch: Callable[[CursorPageRequest], Page[T]],serializer_class: type[BaseSerializer] | None = None,) -> Response:- rows = self.paginate_queryset(cast(Any, _FacadePageSource(fetch)), request, view)+ fetched: list[Page[T]] = []++ def read(page: CursorPageRequest) -> list[T]:+ fetched.append(fetch(page))+ return fetched[-1].results++ rows = self._read_page(read, None, request, view)# The page size is a class attribute, so the paginator always returns a page.assert rows is not None+ self.count_rows = lambda: fetched[-1].count- serializer_class = serializer_class or view.get_serializer_class()- serializer = serializer_class(rows, many=True, context=view.get_serializer_context())- return self.get_paginated_response(serializer.data)+ return self.get_paginated_response(_serialize_rows(view, rows, serializer_class))def facade_page_response[T](diff --git a/posthog/api/test/test_pagination.py b/posthog/api/test/test_pagination.pyindex 6f7579988b6e..604d36644959 100644--- a/posthog/api/test/test_pagination.py+++ b/posthog/api/test/test_pagination.py@@ -1,10 +1,15 @@-from urllib.parse import parse_qs, urlparse+from base64 import b64encode+from datetime import UTC, datetime, timedelta+from urllib.parse import parse_qs, urlencode, urlparsefrom posthog.test.base import BaseTestfrom parameterized import parameterizedfrom rest_framework import serializers+from rest_framework.exceptions import NotFound, ValidationError+from rest_framework.filters import OrderingFilterfrom rest_framework.generics import GenericAPIView+from rest_framework.pagination import CursorPaginationfrom rest_framework.request import Requestfrom rest_framework.test import APIRequestFactory@@ -23,6 +28,11 @@ class _AnnotationView(GenericAPIView):serializer_class = _AnnotationIdSerializer+class _OrderableAnnotationView(_AnnotationView):+ filter_backends = [OrderingFilter]+ ordering_fields = ["date_marker", "created_at", "updated_at", "content", "id"]++class _ByDateMarker(FacadeCursorPagination):ordering = "-date_marker"page_size = 2@@ -33,6 +43,11 @@ class _ByDateMarkerQueryset(StableCursorPagination):page_size = 2+class _ByDateMarkerOnDrf(CursorPagination):+ ordering = ("-date_marker", "-id")+ page_size = 2++class TestCursorPagination(BaseTest):def _page(self, mode: str, cursor: str | None) -> dict:request = Request(APIRequestFactory().get("/annotations/", {"cursor": cursor} if cursor else {}))@@ -52,21 +67,100 @@ class TestCursorPagination(BaseTest):rows = paginator.paginate_queryset(queryset, request, view)return paginator.get_paginated_response(_AnnotationIdSerializer(rows, many=True).data).data+ def _drf_page_ids(self, cursor: str) -> list[int]:+ request = Request(APIRequestFactory().get("/annotations/", {"cursor": cursor}))+ rows = _ByDateMarkerOnDrf().paginate_queryset(Annotation.objects.filter(team=self.team), request)+ assert rows is not None+ return [row.id for row in rows]++ def _cursor(self, link: str) -> str:+ return parse_qs(urlparse(link).query)["cursor"][0]+@parameterized.expand([("queryset",), ("facade",)])- def test_pages_through_rows_that_share_the_cursor_field_once(self, mode: str) -> None:- shared_marker = Annotation.objects.create(team=self.team, content="first").date_marker- for content in ["second", "third", "fourth", "fifth"]:- Annotation.objects.create(team=self.team, content=content, date_marker=shared_marker)+ def test_pages_both_ways_through_rows_that_share_the_cursor_field(self, mode: str) -> None:+ # A NULL sorts first in descending order, so the walk crosses from NULL rows into a tie group.+ shared_marker = datetime(2024, 1, 1, tzinfo=UTC)+ for marker in [None, None, shared_marker, shared_marker, shared_marker]:+ Annotation.objects.create(team=self.team, content="note", date_marker=marker)expected_ids = set(Annotation.objects.filter(team=self.team).values_list("id", flat=True))- seen_ids: list[int] = []- cursor = None+ forward_pages: list[list[int]] = []+ page = self._page(mode, None)while True:- page = self._page(mode, cursor)assert page["count"] == 5- seen_ids.extend(row["id"] for row in page["results"])+ forward_pages.append([row["id"] for row in page["results"]])if page["next"] is None:break- cursor = parse_qs(urlparse(page["next"]).query)["cursor"][0]+ page = self._page(mode, self._cursor(page["next"]))++ assert sorted(id for ids in forward_pages for id in ids) == sorted(expected_ids)- assert sorted(seen_ids) == sorted(expected_ids)+ backward_pages = [[row["id"] for row in page["results"]]]+ while page["previous"] is not None:+ page = self._page(mode, self._cursor(page["previous"]))+ backward_pages.append([row["id"] for row in page["results"]])++ assert backward_pages == list(reversed(forward_pages))++ @parameterized.expand([("queryset",), ("facade",)])+ def test_reads_a_cursor_in_drfs_format(self, mode: str) -> None:+ shared_marker = Annotation.objects.create(team=self.team, content="first").date_marker+ for content in ["second", "third"]:+ Annotation.objects.create(team=self.team, content=content, date_marker=shared_marker)+ ids_newest_first = list(+ Annotation.objects.filter(team=self.team).order_by("-date_marker", "-id").values_list("id", flat=True)+ )++ # DRF wrote "skip two rows from the start" as `o=2`; clients store such links.+ page = self._page(mode, b64encode(b"o=2").decode("ascii"))++ assert [row["id"] for row in page["results"]] == ids_newest_first[2:]++ @parameterized.expand([("queryset",), ("facade",)])+ def test_drf_paginator_reads_every_cursor_to_the_same_rows(self, mode: str) -> None:+ newest = datetime(2024, 1, 3, tzinfo=UTC)+ shared = newest - timedelta(days=1)+ for marker in [newest, shared, shared, shared, shared, shared, shared - timedelta(days=1)]:+ Annotation.objects.create(team=self.team, content="note", date_marker=marker)++ page = self._page(mode, None)+ cursors: list[str] = []+ while page["next"] is not None:+ cursors.append(self._cursor(page["next"]))+ page = self._page(mode, cursors[-1])+ while page["previous"] is not None:+ cursors.append(self._cursor(page["previous"]))+ page = self._page(mode, cursors[-1])++ assert len(cursors) == 6+ for cursor in cursors:+ assert self._drf_page_ids(cursor) == [row["id"] for row in self._page(mode, cursor)["results"]]++ def test_ordering_keeps_one_term_per_field_and_caps_the_field_count(self) -> None:+ def ordering_for(param: str) -> tuple[str, ...]:+ request = Request(APIRequestFactory().get("/annotations/", {"ordering": param}))+ return _ByDateMarkerQueryset().get_ordering(request, Annotation.objects.all(), _OrderableAnnotationView())++ assert ordering_for("date_marker,-date_marker,date_marker") == ("date_marker", "id")+ with self.assertRaises(ValidationError):+ ordering_for("date_marker,created_at,updated_at,content,-id")++ @parameterized.expand(+ [+ ("negative_offset", b64encode(b"o=-1").decode("ascii")),+ ("deeply_nested_position", b64encode(urlencode({"v": "[" * 100_000}).encode("ascii")).decode("ascii")),+ (+ "value_the_field_cannot_hold",+ b64encode(urlencode({"v": '["not-a-date","1"]'}).encode("ascii")).decode("ascii"),+ ),+ (+ "id_the_field_cannot_hold",+ b64encode(urlencode({"v": '["2024-01-01 00:00:00+00:00","not-an-integer"]'}).encode("ascii")).decode(+ "ascii"+ ),+ ),+ ]+ )+ def test_rejects_a_malformed_cursor(self, _name: str, cursor: str) -> None:+ with self.assertRaises(NotFound):+ self._page("queryset", cursor)diff --git a/posthog/paging.py b/posthog/paging.pyindex 423a97f4af94..f0e6d883baf8 100644--- a/posthog/paging.py+++ b/posthog/paging.py@@ -24,38 +24,31 @@ class PageRequest:raise ValueError(f"A page cannot start before the first item, got offset={self.offset}.")-@frozen-class CursorBound:- """Where a cursor page starts.-- Only the items whose first ordering field sorts before `value` (when `before`) or after it- (otherwise) are on the page. `value` is the field's value as text, as an API cursor carries it.- """-- value: str- before: bool--@frozenclass CursorPageRequest:"""One page of a list that pages by cursor.`ordering` names the fields to sort by, most significant first. A leading `-` sorts that field- in descending order. The page skips `offset` items past `bound` and holds at most `limit` items.+ in descending order. `after` holds the values of the leading ordering fields of the item just+ before the page, as text the way an API cursor carries them, with None for a NULL. The page+ holds at most `limit` of the items that sort strictly after that item, past the first `offset`+ of them. An empty `after` starts at the first item."""ordering: tuple[str, ...]- bound: CursorBound | None+ after: tuple[str | None, ...]offset: intlimit: intdef __post_init__(self) -> None:if not self.ordering:raise ValueError("A cursor page needs at least one ordering field.")+ if len(self.after) > len(self.ordering):+ raise ValueError(f"A cursor holds one value per ordering field, got {len(self.after)} values.")if self.limit < 1:raise ValueError(f"A page holds at least one item, got limit={self.limit}.")if self.offset < 0:- raise ValueError(f"A page cannot start before the bound, got offset={self.offset}.")+ raise ValueError(f"A page cannot start before the cursor, got offset={self.offset}.")@frozendiff --git a/posthog/queryset_paging.py b/posthog/queryset_paging.pyindex e23e0b456bd2..ffef30385b5c 100644--- a/posthog/queryset_paging.py+++ b/posthog/queryset_paging.py@@ -2,9 +2,10 @@Logic and facade modules import it, so it imports Django only and never DRF."""-from collections.abc import Collection+from collections.abc import Collection, Sequence-from django.db.models import Model, QuerySet+from django.core.exceptions import FieldDoesNotExist+from django.db.models import Field, Model, Q, QuerySetfrom posthog.paging import CursorPageRequest, Page, PageRequest@@ -44,6 +45,85 @@ def select_page[M: Model](queryset: QuerySet[M], page: PageRequest) -> Page[M]:return Page(results=list(rows), count=count)+def _model_field(model: type[Model], field: str) -> Field | None:+ """Return the column field that `field` names, or None for an annotation or a relation without a column."""+ if field == "pk":+ return model._meta.pk+ try:+ model_field = model._meta.get_field(field)+ except FieldDoesNotExist:+ return None+ return model_field if isinstance(model_field, Field) else None+++def _is_nullable(model: type[Model], field: str) -> bool:+ model_field = _model_field(model, field)+ # An annotation, a reverse relation or a generic key can hold NULL, and nothing here tells otherwise.+ return True if model_field is None else model_field.null+++def _cursor_value(model: type[Model], field: str, value: str | None) -> object:+ """Convert a cursor value from text to the type of its field.++ A value the field cannot hold, such as "abc" for an integer, raises Django's ValidationError.+ """+ if value is None:+ return None+ model_field = _model_field(model, field)+ if model_field is None:+ return value+ return model_field.to_python(value)+++def _sorts_after(field: str, value: object, *, descending: bool, nullable: bool) -> Q | None:+ """Match the rows whose `field` alone sorts after `value`, or return None when no row can.++ Postgres sorts NULL after every value in ascending order and before every value in descending+ order, and `value` is None for a NULL.+ """+ if value is None:+ return Q(**{f"{field}__isnull": False}) if descending else None+ if descending:+ return Q(**{f"{field}__lt": value})+ if nullable:+ return Q(**{f"{field}__gt": value}) | Q(**{f"{field}__isnull": True})+ return Q(**{f"{field}__gt": value})+++def _rows_after(model: type[Model], ordering: Sequence[str], after: Sequence[str | None]) -> Q:+ """Match the rows that sort strictly after the row whose leading ordering fields hold `after`.++ For the ordering `(a, -b)` this is `a > x OR (a = x AND b < y)`. A cursor that DRF encoded+ holds only the first field, so the match then compares that field alone.+ """+ # Starts as a match for no row, and each field adds one more way to sort after the cursor row.+ rows_after = Q(pk__in=[])+ ties = Q()+ values = [_cursor_value(model, term.lstrip("-"), value) for term, value in zip(ordering, after)]+ for term, value in zip(ordering, values):+ field = term.lstrip("-")+ descending = term.startswith("-")+ sorts_after = _sorts_after(field, value, descending=descending, nullable=_is_nullable(model, field))+ if sorts_after is not None:+ rows_after |= ties & sorts_after+ ties &= Q(**{f"{field}__isnull": True}) if value is None else Q(**{field: value})++ first = ordering[0].lstrip("-")+ if len(after) < 2 or values[0] is None or _is_nullable(model, first):+ return rows_after+ # The first field's own bound repeats part of the match, so that an index on that field can limit the scan.+ bound = "lte" if ordering[0].startswith("-") else "gte"+ return Q(**{f"{first}__{bound}": values[0]}) & rows_after+++def select_cursor_rows[M: Model](queryset: QuerySet[M], page: CursorPageRequest) -> list[M]:+ """Return the rows of one cursor page of `queryset`, without counting the whole list."""+ queryset = stable_queryset_ordering(queryset.order_by(*page.ordering))+ if page.after:+ queryset = queryset.filter(_rows_after(queryset.model, page.ordering, page.after))+ return list(queryset[page.offset : page.offset + page.limit])++def select_cursor_page[M: Model](queryset: QuerySet[M], page: CursorPageRequest, *, ordering_fields: Collection[str]) -> Page[M]:@@ -58,10 +138,4 @@ def select_cursor_page[M: Model](raise ValueError(f"Cannot order by {term!r}")count = queryset.count()- first = page.ordering[0]- queryset = stable_queryset_ordering(queryset.order_by(*page.ordering))- if page.bound is not None:- comparison = "lt" if page.bound.before else "gt"- queryset = queryset.filter(**{f"{first.lstrip('-')}__{comparison}": page.bound.value})-- return Page(results=list(queryset[page.offset : page.offset + page.limit]), count=count)+ return Page(results=select_cursor_rows(queryset, page), count=count)diff --git a/products/architecture.md b/products/architecture.mdindex 449fe6e12e4c..ae437d3dc1bd 100644--- a/products/architecture.md+++ b/products/architecture.md@@ -483,9 +483,13 @@ The facade pages in SQL instead, and the view builds the standard response from- A facade list function takes a `PageRequest` (or a `CursorPageRequest`) and returns `Page[Contract]`.Page the queryset with `select_page` (or `select_cursor_page`), then convert the rows with `page.map(...)`.- Both helpers count the whole list and add the primary key as the last sort term, so a page boundary never skips or repeats a row.+ Both helpers count the whole list and add the primary key as the last sort term, so the page order is deterministic.+ An offset page can still shift when rows are added or deleted between requests.+ A cursor page does not skip or repeat a row, unless the ordering values of a row change between requests.+ The queryset must hold one row per item, so a join that duplicates rows breaks both forms.- The view sets `pagination_class = FacadeLimitOffsetPagination` and returns `facade_page_response(self, fetch)`, where `fetch` calls the facade with the page.For a cursor list, subclass `FacadeCursorPagination`, set `ordering`, and return `facade_cursor_page_response(self, fetch)`.+ A cursor stores every ordering field of the row at the page edge, so the contract must expose those fields and `id`.- Use limit/offset for a list that people browse with a total.Use a cursor for a list that grows at the top while people read it, such as batch export runs or deliveries, because offsets shift as rows arrive.Do not add page-number lists.diff --git a/products/batch_exports/backend/tests/test_facade.py b/products/batch_exports/backend/tests/test_facade.pyindex ed24970ed255..c75d4a90c3a0 100644--- a/products/batch_exports/backend/tests/test_facade.py+++ b/products/batch_exports/backend/tests/test_facade.py@@ -285,7 +285,7 @@ def test_runs_and_backfills_are_scoped_to_the_team_and_export(team, organizationstart=dt.datetime(2000, 1, 1, tzinfo=dt.UTC), end=dt.datetime.now(dt.UTC) + dt.timedelta(days=1)))- page = CursorPageRequest(ordering=("-created_at",), bound=None, offset=0, limit=10)+ page = CursorPageRequest(ordering=("-created_at",), after=(), offset=0, limit=10)own_runs = api.list_batch_export_runs(team.pk, export_id, filters=every_run, page=page)assert [run.id for run in own_runs.results] == [run_id]