:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(devex): cap list page size at 1000 by default (#126411)
Julian Bezd3057e591393
diff --git a/frontend/src/models/cohortsModel.test.ts b/frontend/src/models/cohortsModel.test.tsindex 77b35cff6af4..caea9c418f0f 100644--- a/frontend/src/models/cohortsModel.test.ts+++ b/frontend/src/models/cohortsModel.test.ts@@ -80,6 +80,28 @@ describe('cohortsModel', () => {expect(logic.values.allCohorts.results).toHaveLength(2)})+ it('follows next pages until the cohort list is complete', async () => {+ await expectLogic(logic).toFinishAllListeners()+ const [firstCohort, secondCohort] = MOCK_COHORTS.results+ useMocks({+ get: {+ '/api/projects/:team/cohorts/': ({ request }) => {+ const offset = Number(new URL(request.url).searchParams.get('offset') ?? 0)+ if (offset === 0) {+ return { count: 2, next: 'http://localhost/next-page', results: [firstCohort] }+ }+ return { count: 2, next: null, results: [secondCohort] }+ },+ },+ })++ await expectLogic(logic, () => logic.actions.loadAllCohorts()).toDispatchActions(['loadAllCohortsSuccess'])+ expect(logic.values.allCohorts.results.map(({ id }) => id)).toEqual([1, 2])++ await expectLogic(logic, () => logic.actions.loadCohorts()).toDispatchActions(['loadCohortsSuccess'])+ expect(logic.values.cohorts.results.map(({ id }) => id)).toEqual([1, 2])+ })+it('sets polling timeout for calculating cohorts when on cohorts page', async () => {// Set the current location to the cohorts pagerouter.actions.push(urls.cohorts())diff --git a/frontend/src/models/cohortsModel.ts b/frontend/src/models/cohortsModel.tsindex 815ded295a7e..748175ad8351 100644--- a/frontend/src/models/cohortsModel.ts+++ b/frontend/src/models/cohortsModel.ts@@ -82,7 +82,8 @@ export function isIndividualInsightPath(pathname: string): boolean {export const COHORTS_PER_PAGE = 100-export const MAX_COHORTS_FOR_FULL_LIST = 2000+// The cohort list returns at most 1000 rows per request, so a full-list load reads every page.+const COHORTS_FULL_LIST_PAGE_SIZE = 1000export interface CohortFilters {search?: string@@ -94,6 +95,24 @@ export const DEFAULT_COHORT_FILTERS: CohortFilters = {page: 1,}+async function loadAllCohortPages<T>(+ loadPage: (params: {+ limit: number+ offset: number+ }) => Promise<{ count: number; next?: string | null; results: T[] }>+): Promise<CountedPaginatedResponse<T>> {+ const results: T[] = []+ let count = 0+ let hasNextPage = true+ while (hasNextPage) {+ const page = await loadPage({ limit: COHORTS_FULL_LIST_PAGE_SIZE, offset: results.length })+ results.push(...page.results)+ count = page.count+ hasNextPage = !!page.next && page.results.length > 0+ }+ return { count, results }+}+export function processCohort(cohort: CohortType): CohortType {return {...cohort,@@ -236,16 +255,10 @@ export interface cohortsModelActions {errorObject?: any}loadAllCohortsSuccess: (- allCohorts: {- count: number- results: CohortType[]- },+ allCohorts: CountedPaginatedResponse<CohortType>,payload?: any) => {- allCohorts: {- count: number- results: CohortType[]- }+ allCohorts: CountedPaginatedResponse<CohortType>payload?: any}loadCohorts: () => any@@ -278,16 +291,10 @@ export interface cohortsModelActions {errorObject?: any}loadCohortsSuccess: (- cohorts: {- count: number- results: CohortType[]- },+ cohorts: CountedPaginatedResponse<CohortType>,payload?: any) => {- cohorts: {- count: number- results: CohortType[]- }+ cohorts: CountedPaginatedResponse<CohortType>payload?: any}setPollTimeout: (pollTimeout: number | null) => {@@ -331,13 +338,16 @@ export const cohortsModel = kea<cohortsModelType>([cohorts: {__default: { count: 0, results: [] } as CountedPaginatedResponse<CohortType>,loadCohorts: async () => {- const response = await api.cohorts.listPaginated({- limit: MAX_COHORTS_FOR_FULL_LIST,- })+ if (!isAuthenticatedTeam(values.currentTeam)) {+ return values.cohorts+ }+ const projectId = String(values.currentTeam.project_id)+ const response = await loadAllCohortPages((params) => cohortsApi.cohortsList(projectId, params))personsLogic.findMounted({ syncWithUrl: true })?.actions.loadCohorts()return {count: response.count,- results: response.results.map((cohort) => processCohort(cohort)),+ // The shared cache still exposes the legacy shape to its consumers.+ results: response.results.map((cohort) => processCohort(cohort as unknown as CohortType)),}},},@@ -385,21 +395,25 @@ export const cohortsModel = kea<cohortsModelType>([cache.fullListRequested = true// `allCohorts` feeds `cohortsById`, mostly id + name lookups, so request the// basic payload that drops the legacy `groups`/`query` JSON. Fetching the full- // payload for up to 2,000 cohorts on this hot path was slow enough to hit the+ // payload for every cohort on this hot path was slow enough to hit the// gateway timeout on cohort-heavy teams. `filters` stays in the basic payload:// the feature-flag intent warning reads it off `cohortsById` (see// featureFlagIntentWarningLogic.hasBehavioralCriteria), and its raw shape is// what that check expects, so no `processCohort` normalization is needed here.- const response = await api.cohorts.listBasic({- limit: MAX_COHORTS_FOR_FULL_LIST,- })- const listedIds = new Set(response.results.map(({ id }) => id))+ if (!isAuthenticatedTeam(values.currentTeam)) {+ return values.allCohorts+ }+ const projectId = String(values.currentTeam.project_id)+ const response = await loadAllCohortPages((params) =>+ cohortsApi.cohortsList(projectId, { ...params, basic: true })+ )+ const listedIds = new Set<CohortType['id']>(response.results.map(({ id }) => id))return {count: response.count,// `groups` is dropped from the basic payload but required on CohortType, so// add the empty default back — nothing reading `cohortsById` uses `groups`.results: [- ...response.results.map((cohort): CohortType => ({ ...cohort, groups: [] })),+ ...response.results.map((cohort) => ({ ...(cohort as unknown as CohortType), groups: [] })),...values.allCohorts.results.filter(({ id }) => !listedIds.has(id) && cache.referencedCohortIds?.has(id)),diff --git a/frontend/src/models/dashboardsModel.tsx b/frontend/src/models/dashboardsModel.tsxindex 2f8bb93f0430..e59fe1e6b673 100644--- a/frontend/src/models/dashboardsModel.tsx+++ b/frontend/src/models/dashboardsModel.tsx@@ -525,7 +525,7 @@ export const dashboardsModel = kea<dashboardsModelType>([let hasNextPage = truewhile (hasNextPage) {const page = await dashboardsList(String(teamId), {- limit: 2000,+ limit: 1000,offset: results.length,exclude_generated: true,})@@ -561,7 +561,7 @@ export const dashboardsModel = kea<dashboardsModelType>([let apiUrl =url ||- `api/projects/${teamLogic.values.currentTeamId}/dashboards/?limit=2000&exclude_generated=true`+ `api/projects/${teamLogic.values.currentTeamId}/dashboards/?limit=1000&exclude_generated=true`// nosemgrep: prefer-codegen-api -- Legacy raw API call with a URL built at runtime and an unchecked response type. Use a generated function if one covers this endpoint.const dashboards: PaginatedResponse<DashboardType> = await api.get(apiUrl)diff --git a/posthog/api/pagination.py b/posthog/api/pagination.pyindex 54ac69a86598..e0ffc2283df0 100644--- a/posthog/api/pagination.py+++ b/posthog/api/pagination.py@@ -176,6 +176,17 @@ class BoundedLimitOffsetPagination(LimitOffsetPagination):return parameters+class DefaultLimitOffsetPagination(BoundedLimitOffsetPagination):+ """The project-wide default paginator: it caps `limit` at `max_limit` but leaves the schema as DRF writes it.++ Clients generated from the schema keep accepting a larger limit. The server answers with at most+ `max_limit` rows and a `next` link, so a client that follows `next` still reads every row.+ """++ def get_schema_operation_parameters(self, view: Any) -> list[dict[str, Any]]:+ return LimitOffsetPagination.get_schema_operation_parameters(self, view)++class FacadeLimitOffsetPagination(BoundedLimitOffsetPagination):"""Builds the standard limit/offset response from a page that a facade read in SQL.diff --git a/posthog/settings/web.py b/posthog/settings/web.pyindex 059614260606..93b3cdb14651 100644--- a/posthog/settings/web.py+++ b/posthog/settings/web.py@@ -559,7 +559,7 @@ EMAIL_VERIFICATION_SKIP_FOR_DOMAINS = [REST_FRAMEWORK = {"DEFAULT_AUTHENTICATION_CLASSES": ["posthog.auth.SessionAuthentication"],- "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination",+ "DEFAULT_PAGINATION_CLASS": "posthog.api.pagination.DefaultLimitOffsetPagination","DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"],"DEFAULT_RENDERER_CLASSES": ["posthog.renderers.SafeJSONRenderer"],"PAGE_SIZE": 100,diff --git a/products/business_knowledge/backend/api/views.py b/products/business_knowledge/backend/api/views.pyindex cefa33e614a5..cf73062f6dc5 100644--- a/products/business_knowledge/backend/api/views.py+++ b/products/business_knowledge/backend/api/views.py@@ -19,6 +19,7 @@ from rest_framework.permissions import IsAuthenticatedfrom rest_framework.request import Requestfrom rest_framework.response import Response+from posthog.api.pagination import BoundedLimitOffsetPaginationfrom posthog.api.routing import TeamAndOrgViewSetMixinfrom posthog.event_usage import groupsfrom posthog.models.user import User@@ -32,6 +33,7 @@ from ..constants import (BK_DRILLDOWN_MAX_RADIUS,BK_SEARCH_DEFAULT_LIMIT,BK_SEARCH_MAX_LIMIT,+ MAX_URLS_PER_SOURCE,)from ..file_parse import FileParseErrorfrom ..github_repos import connection_status@@ -95,6 +97,11 @@ def _requested_library_ids(team_id: int, request: Request) -> list[UUID] | None:return parsed+class SourceDocumentsPagination(BoundedLimitOffsetPagination):+ # The frontend reads a source's whole document list in one request, and a crawl stops at this many URLs.+ max_limit = MAX_URLS_PER_SOURCE++class _ConflictError(exceptions.APIException):# 409 is the right semantics for "resource is currently busy / in a# state that conflicts with the request". DRF has no first-class helper@@ -498,6 +505,7 @@ class KnowledgeSourceViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):methods=["get"],url_path="documents",required_scopes=["business_knowledge:read"],+ pagination_class=SourceDocumentsPagination,)def documents(self, request: Request, pk: str, **kwargs) -> Response:try:diff --git a/products/business_knowledge/frontend/generated/api.schemas.ts b/products/business_knowledge/frontend/generated/api.schemas.tsindex 8368597f9445..f876a726f4b0 100644--- a/products/business_knowledge/frontend/generated/api.schemas.ts+++ b/products/business_knowledge/frontend/generated/api.schemas.ts@@ -750,6 +750,7 @@ export const BusinessKnowledgeSourcesListSourceType = {export type BusinessKnowledgeSourcesDocumentsListParams = {/*** Number of results to return per page.+ * @maximum 5000*/limit?: number/**diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 938d8d6532ca..a6bd9ebd8ad6 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -132961,6 +132961,7 @@ export namespace Schemas {export type BusinessKnowledgeSourcesDocumentsListParams = {/*** Number of results to return per page.+ * @maximum 5000*/limit?: number;/**