Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(devex): page autoresearch, signals and canvas lists with the shared kit (#126425)

Julian Bez8fd78b674efc
diff --git a/products/autoresearch/backend/facade/api.py b/products/autoresearch/backend/facade/api.py
index 760a310275fc..5a69645d62cc 100644
--- a/products/autoresearch/backend/facade/api.py
+++ b/products/autoresearch/backend/facade/api.py
@@ -28,6 +28,8 @@ import structlog
from posthog.dataclasses import frozen
from posthog.models.team import Team
from posthog.models.user import User
+from posthog.paging import Page, PageRequest
+from posthog.queryset_paging import select_page
from posthog.storage.object_storage import ObjectStorageError
from products.actions.backend.models.action import Action
@@ -623,7 +625,7 @@ def _training_run_row(
# ── Pipelines ──────────────────────────────────────────────────────────────
-def list_pipelines(team_id: int, *, offset: int, limit: int) -> tuple[list[Pipeline], int]:
+def list_pipelines(team_id: int, page: PageRequest) -> Page[Pipeline]:
"""One page of the team's non-archived pipelines, newest first, plus the total count."""
qs = (
AutoresearchPipeline.objects.for_team(team_id)
@@ -631,24 +633,16 @@ def list_pipelines(team_id: int, *, offset: int, limit: int) -> tuple[list[Pipel
.select_related("created_by")
.order_by("-created_at")
)
- count = qs.count()
- rows = list(
- qs[offset : offset + limit].prefetch_related(
- Prefetch(
- "models",
- queryset=AutoresearchModel.objects.for_team(team_id)
- .filter(role=AutoresearchModel.Role.CHAMPION)
- .order_by("-created_at"),
- to_attr="prefetched_champions",
- )
- )
- )
- champions: dict[UUID, AutoresearchModel] = {
- row.id: row.prefetched_champions[0] for row in rows if row.prefetched_champions
- }
- activity = _pipeline_activity(team_id, [row.id for row in rows], champions)
- pipelines = [_pipeline_to_contract(row, champion=champions.get(row.id), activity=activity[row.id]) for row in rows]
- return pipelines, count
+ rows = select_page(qs, page)
+ champions: dict[UUID, AutoresearchModel] = {}
+ for model in (
+ AutoresearchModel.objects.for_team(team_id)
+ .filter(pipeline_id__in=[row.id for row in rows.results], role=AutoresearchModel.Role.CHAMPION)
+ .order_by("-created_at")
+ ):
+ champions.setdefault(model.pipeline_id, model)
+ activity = _pipeline_activity(team_id, [row.id for row in rows.results], champions)
+ return rows.map(lambda row: _pipeline_to_contract(row, champion=champions.get(row.id), activity=activity[row.id]))
def get_pipeline(team_id: int, pipeline_id: str | UUID) -> Pipeline:
@@ -1121,14 +1115,13 @@ def validate_features_sql(features_sql: str) -> None:
# ── Models ─────────────────────────────────────────────────────────────────
-def list_models(team_id: int, *, pipeline_id: str | UUID | None, offset: int, limit: int) -> tuple[list[Model], int]:
+def list_models(team_id: int, *, pipeline_id: str | UUID | None, page: PageRequest) -> Page[Model]:
qs = AutoresearchModel.objects.for_team(team_id).order_by("-created_at")
if pipeline_id:
qs = qs.filter(pipeline_id=_as_uuid(pipeline_id))
- count = qs.count()
- rows = list(qs[offset : offset + limit])
- in_shadow = shadow_set_ids(team_id, {row.pipeline_id for row in rows})
- return [_model_to_contract(row, in_shadow_set=row.pk in in_shadow) for row in rows], count
+ rows = select_page(qs, page)
+ in_shadow = shadow_set_ids(team_id, {row.pipeline_id for row in rows.results})
+ return rows.map(lambda row: _model_to_contract(row, in_shadow_set=row.pk in in_shadow))
def get_model(team_id: int, model_id: str | UUID, *, pipeline_id: str | UUID | None = None) -> Model | None:
@@ -1147,12 +1140,11 @@ def get_model(team_id: int, model_id: str | UUID, *, pipeline_id: str | UUID | N
# ── Operational runs ───────────────────────────────────────────────────────
-def list_runs(team_id: int, *, pipeline_id: str | UUID | None, offset: int, limit: int) -> tuple[list[Run], int]:
+def list_runs(team_id: int, *, pipeline_id: str | UUID | None, page: PageRequest) -> Page[Run]:
qs = AutoresearchRun.objects.for_team(team_id).order_by("-created_at")
if pipeline_id:
qs = qs.filter(pipeline_id=_as_uuid(pipeline_id))
- count = qs.count()
- return [_run_to_contract(row) for row in qs[offset : offset + limit]], count
+ return select_page(qs, page).map(_run_to_contract)
def get_run(team_id: int, run_id: str | UUID, *, pipeline_id: str | UUID | None = None) -> Run | None:
@@ -1419,14 +1411,11 @@ def _segment_thresholds(team_id: int, pipeline: AutoresearchPipeline) -> Predict
# ── Training runs ──────────────────────────────────────────────────────────
-def list_training_runs(
- team_id: int, *, pipeline_id: str | UUID | None, offset: int, limit: int
-) -> tuple[list[TrainingRun], int]:
+def list_training_runs(team_id: int, *, pipeline_id: str | UUID | None, page: PageRequest) -> Page[TrainingRun]:
qs = AutoresearchTrainingRun.objects.for_team(team_id).prefetch_related("iterations").order_by("-created_at")
if pipeline_id:
qs = qs.filter(pipeline_id=_as_uuid(pipeline_id))
- count = qs.count()
- return [_training_run_to_contract(row) for row in qs[offset : offset + limit]], count
+ return select_page(qs, page).map(_training_run_to_contract)
def get_training_run(
@@ -1997,12 +1986,9 @@ def delete_artifact(
# ── Suggestions ────────────────────────────────────────────────────────────
-def list_suggestions(
- team_id: int, *, pipeline_id: str | UUID | None, offset: int, limit: int
-) -> tuple[list[Suggestion], int]:
+def list_suggestions(team_id: int, *, pipeline_id: str | UUID | None, page: PageRequest) -> Page[Suggestion]:
qs = _suggestion_rows(team_id, pipeline_id=pipeline_id).order_by("-created_at")
- count = qs.count()
- return [_suggestion_to_contract(row) for row in qs[offset : offset + limit]], count
+ return select_page(qs, page).map(_suggestion_to_contract)
def _suggestion_rows(team_id: int, *, pipeline_id: str | UUID | None) -> Any:
diff --git a/products/autoresearch/backend/presentation/views/views.py b/products/autoresearch/backend/presentation/views/views.py
index 7b6f25f18dc8..36b35d5b8efe 100644
--- a/products/autoresearch/backend/presentation/views/views.py
+++ b/products/autoresearch/backend/presentation/views/views.py
@@ -29,6 +29,7 @@ from rest_framework.views import APIView
from posthog.api.documentation import PostHogAutoSchema
from posthog.api.mixins import validated_request
+from posthog.api.pagination import FacadeLimitOffsetPagination, facade_page_response
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.models.user import User
from posthog.oauth_provenance import get_oauth_access_token, is_sandbox_origin_request
@@ -132,24 +133,6 @@ class AutoresearchAccessPermission(BasePermission):
return has_autoresearch_access(request.user, team_id=team_id)
-class _FacadePaginationMixin:
- # Drives the standard ``LimitOffsetPagination`` envelope from a facade ``(page, count)``
- # result. The facade does the slicing, so the paginator's state is set directly rather than
- # handed a queryset — keeping the param names, default page size, and ``count`` / ``next`` /
- # ``previous`` shape identical to the model-backed viewsets.
- def _paginate_via_facade(self, request: Request, fetch: Any, serializer_class: Any) -> Response:
- paginator = self.paginator # type: ignore[attr-defined]
- limit = paginator.get_limit(request)
- offset = paginator.get_offset(request)
- page, count = fetch(offset=offset, limit=limit)
- paginator.request = request
- paginator.limit = limit
- paginator.offset = offset
- paginator.count = count
- serializer = serializer_class(instance=page, many=True)
- return paginator.get_paginated_response(serializer.data)
-
-
class ArtifactStorageUnavailableError(APIException):
status_code = 503
default_detail = "Object storage is unavailable, so the artifact could not be stored. Try again in a moment."
@@ -202,7 +185,7 @@ def _validated_with_sentinels(serializer: drf_serializers.BaseSerializer) -> Any
@extend_schema(tags=["autoresearch"])
-class AutoresearchPipelineViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMixin, viewsets.ModelViewSet):
+class AutoresearchPipelineViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
"""
Manage autoresearch prediction pipelines.
@@ -239,6 +222,7 @@ class AutoresearchPipelineViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMixin
permission_classes = [AutoresearchAccessPermission]
serializer_class = AutoresearchPipelineSerializer
queryset = None # data is reached through the facade; declared for router/schema only
+ pagination_class = FacadeLimitOffsetPagination
def initial(self, request: Request, *args: Any, **kwargs: Any) -> None:
super().initial(request, *args, **kwargs)
@@ -282,10 +266,10 @@ class AutoresearchPipelineViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMixin
return AutoresearchPipelineSerializer
def list(self, request: Request, *args: Any, **kwargs: Any) -> Response:
- return self._paginate_via_facade(
- request,
- lambda offset, limit: api.list_pipelines(self.team_id, offset=offset, limit=limit),
- AutoresearchPipelineSerializer,
+ return facade_page_response(
+ self,
+ lambda page: api.list_pipelines(self.team_id, page),
+ serializer_class=AutoresearchPipelineSerializer,
)
def retrieve(self, request: Request, *args: Any, **kwargs: Any) -> Response:
@@ -731,7 +715,7 @@ class AutoresearchPipelineViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMixin
@extend_schema(tags=["autoresearch"])
-class AutoresearchModelViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMixin, viewsets.ReadOnlyModelViewSet):
+class AutoresearchModelViewSet(TeamAndOrgViewSetMixin, viewsets.ReadOnlyModelViewSet):
"""
List and retrieve champion/challenger models for a pipeline.
@@ -748,17 +732,16 @@ class AutoresearchModelViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMixin, v
permission_classes = [AutoresearchAccessPermission]
serializer_class = AutoresearchModelSerializer
queryset = None # data is reached through the facade; declared for router/schema only
+ pagination_class = FacadeLimitOffsetPagination
def _should_skip_parents_filter(self) -> bool:
return True
def list(self, request: Request, *args: Any, **kwargs: Any) -> Response:
- return self._paginate_via_facade(
- request,
- lambda offset, limit: api.list_models(
- self.team_id, pipeline_id=_parent_pipeline_id(self), offset=offset, limit=limit
- ),
- AutoresearchModelSerializer,
+ return facade_page_response(
+ self,
+ lambda page: api.list_models(self.team_id, pipeline_id=_parent_pipeline_id(self), page=page),
+ serializer_class=AutoresearchModelSerializer,
)
def retrieve(self, request: Request, *args: Any, **kwargs: Any) -> Response:
@@ -769,7 +752,7 @@ class AutoresearchModelViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMixin, v
@extend_schema(tags=["autoresearch"])
-class AutoresearchRunViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMixin, viewsets.ReadOnlyModelViewSet):
+class AutoresearchRunViewSet(TeamAndOrgViewSetMixin, viewsets.ReadOnlyModelViewSet):
"""
List and retrieve inference and validation runs for a pipeline.
"""
@@ -782,17 +765,16 @@ class AutoresearchRunViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMixin, vie
permission_classes = [AutoresearchAccessPermission]
serializer_class = AutoresearchRunSerializer
queryset = None # data is reached through the facade; declared for router/schema only
+ pagination_class = FacadeLimitOffsetPagination
def _should_skip_parents_filter(self) -> bool:
return True
def list(self, request: Request, *args: Any, **kwargs: Any) -> Response:
- return self._paginate_via_facade(
- request,
- lambda offset, limit: api.list_runs(
- self.team_id, pipeline_id=_parent_pipeline_id(self), offset=offset, limit=limit
- ),
- AutoresearchRunSerializer,
+ return facade_page_response(
+ self,
+ lambda page: api.list_runs(self.team_id, pipeline_id=_parent_pipeline_id(self), page=page),
+ serializer_class=AutoresearchRunSerializer,
)
def retrieve(self, request: Request, *args: Any, **kwargs: Any) -> Response:
@@ -803,7 +785,7 @@ class AutoresearchRunViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMixin, vie
@extend_schema(tags=["autoresearch"])
-class AutoresearchTrainingRunViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMixin, viewsets.ModelViewSet):
+class AutoresearchTrainingRunViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
"""
List and retrieve training runs for a pipeline.
@@ -826,6 +808,7 @@ class AutoresearchTrainingRunViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMi
permission_classes = [AutoresearchAccessPermission]
serializer_class = AutoresearchTrainingRunSerializer
queryset = None # data is reached through the facade; declared for router/schema only
+ pagination_class = FacadeLimitOffsetPagination
def get_throttles(self) -> list[BaseThrottle]:
# The feature query and the anchor count are unsampled ClickHouse scans, so a personal API
@@ -841,12 +824,10 @@ class AutoresearchTrainingRunViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMi
return True
def list(self, request: Request, *args: Any, **kwargs: Any) -> Response:
- return self._paginate_via_facade(
- request,
- lambda offset, limit: api.list_training_runs(
- self.team_id, pipeline_id=_parent_pipeline_id(self), offset=offset, limit=limit
- ),
- AutoresearchTrainingRunSerializer,
+ return facade_page_response(
+ self,
+ lambda page: api.list_training_runs(self.team_id, pipeline_id=_parent_pipeline_id(self), page=page),
+ serializer_class=AutoresearchTrainingRunSerializer,
)
def retrieve(self, request: Request, *args: Any, **kwargs: Any) -> Response:
@@ -1178,7 +1159,7 @@ class AutoresearchTrainingRunViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMi
@extend_schema(tags=["autoresearch"])
-class AutoresearchSuggestionViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMixin, viewsets.GenericViewSet):
+class AutoresearchSuggestionViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
"""
Submit and list steering suggestions for a running pipeline.
@@ -1196,6 +1177,7 @@ class AutoresearchSuggestionViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMix
permission_classes = [AutoresearchAccessPermission]
serializer_class = AutoresearchSuggestionSerializer
queryset = None # data is reached through the facade; declared for router/schema only
+ pagination_class = FacadeLimitOffsetPagination
def _should_skip_parents_filter(self) -> bool:
return True
@@ -1209,12 +1191,10 @@ class AutoresearchSuggestionViewSet(TeamAndOrgViewSetMixin, _FacadePaginationMix
),
)
def list(self, request: Request, *args: Any, **kwargs: Any) -> Response:
- return self._paginate_via_facade(
- request,
- lambda offset, limit: api.list_suggestions(
- self.team_id, pipeline_id=_parent_pipeline_id(self), offset=offset, limit=limit
- ),
- AutoresearchSuggestionSerializer,
+ return facade_page_response(
+ self,
+ lambda page: api.list_suggestions(self.team_id, pipeline_id=_parent_pipeline_id(self), page=page),
+ serializer_class=AutoresearchSuggestionSerializer,
)
@extend_schema(
diff --git a/products/autoresearch/frontend/generated/api.schemas.ts b/products/autoresearch/frontend/generated/api.schemas.ts
index 53b4aa5b65ae..fdc97a14be5c 100644
--- a/products/autoresearch/frontend/generated/api.schemas.ts
+++ b/products/autoresearch/frontend/generated/api.schemas.ts
@@ -2178,6 +2178,7 @@ export interface ValidatePipelineResponseApi {
export type AutoresearchListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
@@ -2189,6 +2190,7 @@ export type AutoresearchListParams = {
export type AutoresearchModelsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
@@ -2200,6 +2202,7 @@ export type AutoresearchModelsListParams = {
export type AutoresearchRunsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
@@ -2211,6 +2214,7 @@ export type AutoresearchRunsListParams = {
export type AutoresearchSuggestionsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
@@ -2222,6 +2226,7 @@ export type AutoresearchSuggestionsListParams = {
export type AutoresearchTrainingRunsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
diff --git a/products/canvas/backend/facade/api.py b/products/canvas/backend/facade/api.py
index 938e09efd8bd..5c82ce3beebd 100644
--- a/products/canvas/backend/facade/api.py
+++ b/products/canvas/backend/facade/api.py
@@ -46,7 +46,6 @@ from products.canvas.backend.layout import (
validate_layout_references as validate_layout_references,
)
from products.canvas.backend.logic.canvases import (
- count_canvases as count_canvases,
create_canvas as create_canvas,
delete_canvas as delete_canvas,
get_canvas as get_canvas,
diff --git a/products/canvas/backend/logic/canvases.py b/products/canvas/backend/logic/canvases.py
index 6943551a0111..6285d7c8493c 100644
--- a/products/canvas/backend/logic/canvases.py
+++ b/products/canvas/backend/logic/canvases.py
@@ -11,6 +11,8 @@ from django.utils import timezone
import structlog
from posthog.models.user import User
+from posthog.paging import Page, PageRequest
+from posthog.queryset_paging import select_page
from products.canvas.backend.facade.contracts import (
CanvasFieldChange,
@@ -79,27 +81,6 @@ def _listing(
return queryset
-def count_canvases(
- viewer: CanvasViewer,
- *,
- channel_id: str | None,
- kind: str | None,
- search: str | None,
- created_by: str | None = None,
- user_access_control: "UserAccessControl | None",
- include_all_if_admin: bool,
-) -> int:
- return _listing(
- viewer,
- channel_id=channel_id,
- kind=kind,
- search=search,
- created_by=created_by,
- user_access_control=user_access_control,
- include_all_if_admin=include_all_if_admin,
- ).count()
-
-
def list_canvases(
viewer: CanvasViewer,
*,
@@ -109,10 +90,9 @@ def list_canvases(
created_by: str | None = None,
user_access_control: "UserAccessControl | None",
include_all_if_admin: bool,
- offset: int,
- limit: int,
+ page: PageRequest,
ordering: str | None = None,
-) -> list[CanvasRecord]:
+) -> Page[CanvasRecord]:
"""`ordering` is a field to sort by, newest first by default. The caller validates it."""
rows = _listing(
viewer,
@@ -125,7 +105,7 @@ def list_canvases(
).select_related(*_RECORD_RELATIONS)
if ordering is not None:
rows = rows.order_by(ordering, "-id")
- return [canvas_record(canvas) for canvas in rows[offset : offset + limit]]
+ return select_page(rows, page).map(canvas_record)
def get_canvas(viewer: CanvasViewer, access: CanvasAccess, canvas_id: UUID | str) -> CanvasRecord:
diff --git a/products/canvas/backend/presentation/serializers.py b/products/canvas/backend/presentation/serializers.py
index 4295e9aa6aa7..754de7681754 100644
--- a/products/canvas/backend/presentation/serializers.py
+++ b/products/canvas/backend/presentation/serializers.py
@@ -1415,6 +1415,18 @@ class CanvasStateResponseSerializer(serializers.Serializer):
many=True,
help_text="The canvas's shared entries plus the caller's own user-scoped entries.",
)
+ count = serializers.IntegerField(help_text="Number of entries in the whole selection, across all pages.")
+ next = serializers.URLField(
+ allow_null=True,
+ help_text=(
+ "URL of the next page, or null when complete. It pages the same way as this request: by cursor "
+ "when the request used a cursor, otherwise by offset."
+ ),
+ )
+ previous = serializers.URLField(
+ allow_null=True,
+ help_text="URL of the previous page by offset, or null on the first page and for a cursor request.",
+ )
next_offset = serializers.IntegerField(
allow_null=True,
help_text="Next entry offset, or null when complete. Null when the request used a cursor.",
diff --git a/products/canvas/backend/presentation/views.py b/products/canvas/backend/presentation/views.py
index 853208d2feb5..800f43afd3f5 100644
--- a/products/canvas/backend/presentation/views.py
+++ b/products/canvas/backend/presentation/views.py
@@ -1,7 +1,7 @@
import json
import hashlib
from collections import Counter
-from collections.abc import Callable, Sequence
+from collections.abc import Sequence
from typing import Any, cast
from uuid import UUID
@@ -15,11 +15,12 @@ from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_sche
from rest_framework import status, viewsets
from rest_framework.decorators import action
from rest_framework.exceptions import NotFound, PermissionDenied, ValidationError
-from rest_framework.pagination import LimitOffsetPagination
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.throttling import BaseThrottle, SimpleRateThrottle
+from rest_framework.utils.urls import replace_query_param
+from posthog.api.pagination import FacadeLimitOffsetPagination, facade_page_response
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.auth import OAuthAccessTokenAuthentication
from posthog.event_usage import report_user_action
@@ -344,6 +345,22 @@ class CanvasAccessMixin(TeamAndOrgViewSetMixin):
CANVAS_LIST_ORDERINGS = ["-created_at", "-updated_at"]
+def _state_page_links(request: Request, query: dict[str, Any], result: dict[str, Any]) -> dict[str, str | None]:
+ """The `next` and `previous` URLs of one canvas state page."""
+ url = request.build_absolute_uri()
+ next_link = None
+ if query.get("cursor") is not None and result["next_cursor"] is not None:
+ next_link = replace_query_param(url, "cursor", result["next_cursor"])
+ elif result["next_offset"] is not None:
+ next_link = replace_query_param(url, "offset", result["next_offset"])
+ limit = query.get("limit")
+ offset = query["offset"]
+ previous_link = None
+ if query.get("cursor") is None and limit is not None and offset > 0:
+ previous_link = replace_query_param(url, "offset", max(0, offset - limit))
+ return {"next": next_link, "previous": previous_link}
+
+
class CanvasViewSet(CanvasAccessMixin, viewsets.GenericViewSet):
"""Canvases: agent-built sandboxed browser apps, filed into channels.
@@ -355,6 +372,7 @@ class CanvasViewSet(CanvasAccessMixin, viewsets.GenericViewSet):
# The facade reads canvases; the viewset holds no queryset.
queryset = None
serializer_class = CanvasSerializer
+ pagination_class = FacadeLimitOffsetPagination
http_method_names = ["get", "post", "patch", "delete", "head", "options"]
scope_object_read_actions = [
"list",
@@ -469,21 +487,6 @@ class CanvasViewSet(CanvasAccessMixin, viewsets.GenericViewSet):
self._check_object_access(canvas)
return canvas
- def _paginated(self, request: Request, fetch: Callable[[int, int], Any], count: int) -> Response:
- """The standard limit/offset envelope around one facade page."""
- paginator = cast(LimitOffsetPagination, self.paginator)
- limit = paginator.get_limit(request)
- # The default page size always applies, so there is always a limit.
- if limit is None:
- limit = count
- offset = paginator.get_offset(request)
- paginator.request = request
- paginator.limit = limit
- paginator.offset = offset
- paginator.count = count
- page = [] if count == 0 or offset > count else fetch(offset, limit)
- return paginator.get_paginated_response(page)
-
@extend_schema(
parameters=[
OpenApiParameter(
@@ -531,15 +534,10 @@ class CanvasViewSet(CanvasAccessMixin, viewsets.GenericViewSet):
}
requested_ordering = request.query_params.get("ordering")
ordering = requested_ordering if requested_ordering in CANVAS_LIST_ORDERINGS else None
- return self._paginated(
- request,
- lambda offset, limit: (
- CanvasSerializer(
- canvas_api.list_canvases(viewer, offset=offset, limit=limit, ordering=ordering, **filters),
- many=True,
- ).data
- ),
- canvas_api.count_canvases(viewer, **filters),
+ return facade_page_response(
+ self,
+ lambda page: canvas_api.list_canvases(viewer, page=page, ordering=ordering, **filters),
+ serializer_class=CanvasSerializer,
)
def retrieve(self, request: Request, *args: Any, **kwargs: Any) -> Response:
@@ -2073,7 +2071,8 @@ class CanvasViewSet(CanvasAccessMixin, viewsets.GenericViewSet):
)
except CanvasRequestRejected as rejection:
return Response(rejection.body, status=rejection.status_code)
- return Response(CanvasStateResponseSerializer(result).data)
+ links = _state_page_links(request, query.validated_data, result)
+ return Response(CanvasStateResponseSerializer({**result, **links}).data)
@extend_schema(
operation_id="canvases_state_value_retrieve",
diff --git a/products/canvas/backend/state_reads.py b/products/canvas/backend/state_reads.py
index dbb4bda7c39c..4eb67bbbbd72 100644
--- a/products/canvas/backend/state_reads.py
+++ b/products/canvas/backend/state_reads.py
@@ -56,16 +56,17 @@ class CanvasStateReader:
between pages can make it skip an entry that existed all along. A
cursor takes precedence over `offset`.
"""
- if cursor is not None:
- after = StateCursor.decode(cursor)
- queryset = queryset.filter(Q(scope__gt=after.scope) | Q(scope=after.scope, key__gt=after.key))
- offset = 0
if scope:
queryset = queryset.filter(scope=scope)
if key is not None:
queryset = queryset.filter(key=key)
if key_prefix:
queryset = queryset.filter(key__startswith=key_prefix)
+ selection = queryset
+ if cursor is not None:
+ after = StateCursor.decode(cursor)
+ queryset = queryset.filter(Q(scope__gt=after.scope) | Q(scope=after.scope, key__gt=after.key))
+ offset = 0
# A scope holds up to 256 keys of 64 KB, so a whole-scope read can exceed what the caller can hold.
fields = ["scope", "key", "updated_at"]
if not keys_only:
@@ -74,8 +75,15 @@ class CanvasStateReader:
rows = list(selected[offset : offset + limit + 1] if limit is not None else selected[offset:])
has_more = limit is not None and len(rows) > limit
page = rows[:limit] if limit is not None else rows
+ # The count is of the whole selection, so it stays the same on every page of one read.
+ # A last page reached by offset already gives it, which spares canvas apps that poll a COUNT.
+ if cursor is None and not has_more and (page or offset == 0):
+ count = offset + len(page)
+ else:
+ count = selection.count()
return {
"entries": page,
+ "count": count,
"next_offset": offset + limit if has_more and limit is not None and cursor is None else None,
"next_cursor": StateCursor(scope=page[-1]["scope"], key=page[-1]["key"]).encode() if has_more else None,
"complete": not has_more,
diff --git a/products/canvas/backend/tests/test_canvas_api.py b/products/canvas/backend/tests/test_canvas_api.py
index 3b2bbcdf8fc5..06ef0b537a40 100644
--- a/products/canvas/backend/tests/test_canvas_api.py
+++ b/products/canvas/backend/tests/test_canvas_api.py
@@ -1850,10 +1850,13 @@ class TestCanvasState(CanvasAPIBaseTest):
assert "value" not in first["entries"][0]
assert first["next_offset"] == 1
assert first["complete"] is False
- second = self.client.get(url, {"keys_only": "true", "limit": "1", "offset": "1"}).json()
+ assert (first["count"], first["previous"]) == (2, None)
+ second = self.client.get(first["next"]).json()
assert [entry["key"] for entry in second["entries"]] == ["policy"]
assert second["next_offset"] is None
assert second["complete"] is True
+ assert (second["count"], second["next"]) == (2, None)
+ assert [entry["key"] for entry in self.client.get(second["previous"]).json()["entries"]] == ["board"]
selected = self.client.get(url, {"key": "policy"}).json()
assert [(entry["key"], entry["value"]) for entry in selected["entries"]] == [("policy", "a long policy")]
diff --git a/products/canvas/frontend/generated/api.schemas.ts b/products/canvas/frontend/generated/api.schemas.ts
index fababe3aee4e..f73f4749cc0c 100644
--- a/products/canvas/frontend/generated/api.schemas.ts
+++ b/products/canvas/frontend/generated/api.schemas.ts
@@ -1613,6 +1613,18 @@ export interface CanvasStateEntryApi {
export interface CanvasStateResponseApi {
/** The canvas's shared entries plus the caller's own user-scoped entries. */
entries: CanvasStateEntryApi[]
+ /** Number of entries in the whole selection, across all pages. */
+ count: number
+ /**
+ * URL of the next page, or null when complete. It pages the same way as this request: by cursor when the request used a cursor, otherwise by offset.
+ * @nullable
+ */
+ next: string | null
+ /**
+ * URL of the previous page by offset, or null on the first page and for a cursor request.
+ * @nullable
+ */
+ previous: string | null
/**
* Next entry offset, or null when complete. Null when the request used a cursor.
* @nullable
@@ -1911,6 +1923,7 @@ export type CanvasesListParams = {
kind?: CanvasesListKind
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
@@ -2109,6 +2122,7 @@ export const CanvasesStateValueRetrieveScope = {
export type CanvasesVersionsRetrieveParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
diff --git a/products/signals/backend/scout_report/trial_inbox.py b/products/signals/backend/scout_report/trial_inbox.py
index 74c45e1f3246..ef99138b886f 100644
--- a/products/signals/backend/scout_report/trial_inbox.py
+++ b/products/signals/backend/scout_report/trial_inbox.py
@@ -10,10 +10,11 @@ from django.db.models import QuerySet
from pydantic import JsonValue
from rest_framework import exceptions, serializers
-from rest_framework.pagination import LimitOffsetPagination
from rest_framework.response import Response
+from posthog.api.pagination import FacadeLimitOffsetPagination
from posthog.models import User
+from posthog.paging import Page, PageRequest
from products.signals.backend.artefact_schemas import ActionabilityAssessment
from products.signals.backend.models import MAX_SCOUT_REPORT_NOTES, SignalReport, SignalReportArtefact
@@ -346,26 +347,26 @@ class TrialInboxReads:
return value if isinstance(value, (int, float)) else 0
return str(value or "")
- def list(self, *, count_only: bool = False, include_source_metadata: bool = True) -> Response:
- self._validate_parameters()
- queryset = self.view.filter_queryset(self.view.get_queryset())
- private = self.store.reports()
- documents = self._private_documents(private, include_source_metadata=include_source_metadata and not count_only)
- queryset = queryset.exclude(id__in=[report.id for report in private])
- count = queryset.count() + len(documents)
- if count_only:
- return Response({"count": count, "next": None, "previous": None, "results": []})
- paginator = self.view.paginator
- if not isinstance(paginator, LimitOffsetPagination):
- raise exceptions.ValidationError({"detail": "This scout run requires limit/offset inbox pagination."})
- paginator.request = self.view.request
- paginator.limit = paginator.get_limit(self.view.request)
- paginator.offset = paginator.get_offset(self.view.request)
- paginator.count = count
- limit = paginator.limit or count
+ def _merged_count(self, queryset: QuerySet[SignalReport], documents: Sequence[Mapping[str, object]]) -> int:
+ return queryset.count() + len(documents)
+
+ def _merged_page(
+ self,
+ queryset: QuerySet[SignalReport],
+ documents: Sequence[Mapping[str, object]],
+ page: PageRequest,
+ *,
+ include_source_metadata: bool,
+ ) -> Page[Mapping[str, object]]:
+ """One page of production rows and this run's private reports, sorted together.
+
+ The private reports are not in Postgres, so the sort happens in Python. It sorts a window of
+ production rows that is just wide enough to hold every row of the page.
+ """
+ count = self._merged_count(queryset, documents)
# At most this run's private reports can shift a production row across the requested offset.
- start = max(0, paginator.offset - len(documents))
- production = list(queryset[start : paginator.offset + limit])
+ start = max(0, page.offset - len(documents))
+ production = list(queryset[start : page.offset + page.limit])
merged: list[Mapping[str, object]] = [
*self.view._render_report_rows(production, include_source_metadata=include_source_metadata),
*documents,
@@ -375,8 +376,29 @@ class TrialInboxReads:
clauses.append("id")
for clause in reversed(clauses):
merged.sort(key=lambda row: self._sort_value(row, clause.lstrip("-")), reverse=clause.startswith("-"))
- offset = paginator.offset - start
- return paginator.get_paginated_response(merged[offset : offset + limit])
+ offset = page.offset - start
+ return Page(results=merged[offset : offset + page.limit], count=count)
+
+ def list(self, *, count_only: bool = False, include_source_metadata: bool = True) -> Response:
+ self._validate_parameters()
+ queryset = self.view.filter_queryset(self.view.get_queryset())
+ private = self.store.reports()
+ documents = self._private_documents(private, include_source_metadata=include_source_metadata and not count_only)
+ queryset = queryset.exclude(id__in=[report.id for report in private])
+ if count_only:
+ count = self._merged_count(queryset, documents)
+ return Response({"count": count, "next": None, "previous": None, "results": []})
+ # The view's own paginator pages this list too, so both paths share one limit cap.
+ paginator = self.view.paginator
+ if not isinstance(paginator, FacadeLimitOffsetPagination):
+ raise exceptions.ValidationError({"detail": "This scout run requires limit/offset inbox pagination."})
+ page = paginator.read_facade_page(
+ self.view.request,
+ lambda page_request: self._merged_page(
+ queryset, documents, page_request, include_source_metadata=include_source_metadata
+ ),
+ )
+ return paginator.get_paginated_response(page.results)
def private_report_artefacts(store: ScoutTrialStore, report_id: str) -> list[SignalReportArtefact]:
diff --git a/products/signals/backend/views.py b/products/signals/backend/views.py
index c52b065789a9..9f920f1ff0d3 100644
--- a/products/signals/backend/views.py
+++ b/products/signals/backend/views.py
@@ -43,7 +43,6 @@ from pydantic import ValidationError as PydanticValidationError
from rest_framework import exceptions, mixins, serializers, status, viewsets
from rest_framework.decorators import action
from rest_framework.exceptions import NotFound
-from rest_framework.pagination import LimitOffsetPagination
from rest_framework.permissions import BasePermission, IsAuthenticated
from rest_framework.request import Request
from rest_framework.response import Response
@@ -53,6 +52,7 @@ from temporalio.exceptions import WorkflowAlreadyStartedError
from posthog.api.integration import github_rate_limited_response
from posthog.api.mixins import ValidatedRequest, validated_request
+from posthog.api.pagination import FacadeLimitOffsetPagination
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.auth import (
InternalAPIAuthentication,
@@ -1300,7 +1300,7 @@ class SignalReportReingestionStatusSerializer(serializers.Serializer):
report_id = serializers.UUIDField(help_text="Report being re-ingested.")
-class SignalReportListPagination(LimitOffsetPagination):
+class SignalReportListPagination(FacadeLimitOffsetPagination):
def paginate_queryset(self, queryset, request, view=None):
# Fetch the page before the count. A page that ends before the limit already gives the
# total, so the second scan is not needed. A `search` filter rechecks the text of every
diff --git a/products/signals/frontend/generated/api.schemas.ts b/products/signals/frontend/generated/api.schemas.ts
index 5f9627ac09f5..289700031e8f 100644
--- a/products/signals/frontend/generated/api.schemas.ts
+++ b/products/signals/frontend/generated/api.schemas.ts
@@ -10961,6 +10961,7 @@ export type SignalsReportsListParams = {
include_unlisted?: boolean
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number
/**
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 02d57d5c56ab..6a107e66c69f 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -23353,6 +23353,18 @@ export namespace Schemas {
export interface CanvasStateResponse {
/** The canvas's shared entries plus the caller's own user-scoped entries. */
entries: CanvasStateEntry[];
+ /** Number of entries in the whole selection, across all pages. */
+ count: number;
+ /**
+ * URL of the next page, or null when complete. It pages the same way as this request: by cursor when the request used a cursor, otherwise by offset.
+ * @nullable
+ */
+ next: string | null;
+ /**
+ * URL of the previous page by offset, or null on the first page and for a cursor request.
+ * @nullable
+ */
+ previous: string | null;
/**
* Next entry offset, or null when complete. Null when the request used a cursor.
* @nullable
@@ -132472,6 +132484,7 @@ export namespace Schemas {
export type AutoresearchListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -132483,6 +132496,7 @@ export namespace Schemas {
export type AutoresearchModelsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -132494,6 +132508,7 @@ export namespace Schemas {
export type AutoresearchRunsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -132505,6 +132520,7 @@ export namespace Schemas {
export type AutoresearchSuggestionsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -132516,6 +132532,7 @@ export namespace Schemas {
export type AutoresearchTrainingRunsListParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -132961,6 +132978,7 @@ export namespace Schemas {
kind?: CanvasesListKind;
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -133162,6 +133180,7 @@ export namespace Schemas {
export type CanvasesVersionsRetrieveParams = {
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
@@ -142896,6 +142915,7 @@ export namespace Schemas {
include_unlisted?: boolean;
/**
* Number of results to return per page.
+ * @maximum 1000
*/
limit?: number;
/**
diff --git a/services/mcp/src/generated/autoresearch/api.ts b/services/mcp/src/generated/autoresearch/api.ts
index 8e63e401cde9..c1b316a0e118 100644
--- a/services/mcp/src/generated/autoresearch/api.ts
+++ b/services/mcp/src/generated/autoresearch/api.ts
@@ -23,8 +23,10 @@ export const AutoresearchListParams = () => zod.object({
),
})
+export const autoresearchListQueryLimitMax = 1000
+
export const AutoresearchListQueryParams = () => zod.object({
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod.number().max(autoresearchListQueryLimitMax).optional().describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
})
@@ -209,8 +211,14 @@ export const AutoresearchModelsListParams = () => zod.object({
),
})
+export const autoresearchModelsListQueryLimitMax = 1000
+
export const AutoresearchModelsListQueryParams = () => zod.object({
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod
+ .number()
+ .max(autoresearchModelsListQueryLimitMax)
+ .optional()
+ .describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
})
@@ -243,8 +251,14 @@ export const AutoresearchRunsListParams = () => zod.object({
),
})
+export const autoresearchRunsListQueryLimitMax = 1000
+
export const AutoresearchRunsListQueryParams = () => zod.object({
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod
+ .number()
+ .max(autoresearchRunsListQueryLimitMax)
+ .optional()
+ .describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
})
@@ -261,8 +275,14 @@ export const AutoresearchSuggestionsListParams = () => zod.object({
),
})
+export const autoresearchSuggestionsListQueryLimitMax = 1000
+
export const AutoresearchSuggestionsListQueryParams = () => zod.object({
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod
+ .number()
+ .max(autoresearchSuggestionsListQueryLimitMax)
+ .optional()
+ .describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
})
@@ -348,8 +368,14 @@ export const AutoresearchTrainingRunsListParams = () => zod.object({
),
})
+export const autoresearchTrainingRunsListQueryLimitMax = 1000
+
export const AutoresearchTrainingRunsListQueryParams = () => zod.object({
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod
+ .number()
+ .max(autoresearchTrainingRunsListQueryLimitMax)
+ .optional()
+ .describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
})
diff --git a/services/mcp/src/generated/canvas/api.ts b/services/mcp/src/generated/canvas/api.ts
index 8ef1fd0a8917..45c829def3b8 100644
--- a/services/mcp/src/generated/canvas/api.ts
+++ b/services/mcp/src/generated/canvas/api.ts
@@ -22,6 +22,8 @@ export const CanvasesListParams = () => zod.object({
),
})
+export const canvasesListQueryLimitMax = 1000
+
export const CanvasesListQueryParams = () => zod.object({
channel: zod.string().optional().describe('Only return canvases in this channel.'),
created_by: zod.string().optional().describe('Only return canvases created by the user with this UUID.'),
@@ -29,7 +31,7 @@ export const CanvasesListQueryParams = () => zod.object({
.enum(['component', 'freeform', 'grid'])
.optional()
.describe('Only return canvases of this kind. kind=component lists the component store.'),
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod.number().max(canvasesListQueryLimitMax).optional().describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
ordering: zod
.enum(['-created_at', '-updated_at'])
diff --git a/services/mcp/src/generated/signals/api.ts b/services/mcp/src/generated/signals/api.ts
index 04f87835d8c2..b0714f6702e5 100644
--- a/services/mcp/src/generated/signals/api.ts
+++ b/services/mcp/src/generated/signals/api.ts
@@ -321,6 +321,7 @@ export const signalsReportsListQueryExcludeMax = 100
export const signalsReportsListQueryIncludeSourceMetadataDefault = true
export const signalsReportsListQueryIncludeUnlistedDefault = false
+export const signalsReportsListQueryLimitMax = 1000
export const SignalsReportsListQueryParams = () => zod.object({
actionability: zod
@@ -388,7 +389,11 @@ export const SignalsReportsListQueryParams = () => zod.object({
.describe(
'Include reports kept out of the shared inbox. A report is kept out when the scout that authored it turned off its `inbox` output destination. Defaults to false. Fetching a report by id always works, whatever this value is.'
),
- limit: zod.number().optional().describe('Number of results to return per page.'),
+ limit: zod
+ .number()
+ .max(signalsReportsListQueryLimitMax)
+ .optional()
+ .describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
ordering: zod
.string()
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/canvas-list.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/canvas-list.json
index 2e21e42962a1..efe9f6a99e30 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/canvas-list.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/canvas-list.json
@@ -16,6 +16,7 @@
},
"limit": {
"description": "Number of results to return per page.",
+ "maximum": 1000,
"type": "number"
},
"offset": {
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-reports-list.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-reports-list.json
index 7195ec6b8eb7..d79314830a71 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-reports-list.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-reports-list.json
@@ -57,6 +57,7 @@
},
"limit": {
"description": "Number of results to return per page.",
+ "maximum": 1000,
"type": "number"
},
"offset": {