Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(devex): cap list page size at 1000 by default (#126411)

Julian Bezd3057e591393
diff --git a/frontend/src/models/cohortsModel.test.ts b/frontend/src/models/cohortsModel.test.ts
index 77b35cff6af4..caea9c418f0f 100644
--- a/frontend/src/models/cohortsModel.test.ts
+++ b/frontend/src/models/cohortsModel.test.ts
@@ -80,6 +80,28 @@ describe('cohortsModel', () => {
expect(logic.values.allCohorts.results).toHaveLength(2)
})
+ it('follows next pages until the cohort list is complete', async () => {
+ await expectLogic(logic).toFinishAllListeners()
+ const [firstCohort, secondCohort] = MOCK_COHORTS.results
+ useMocks({
+ get: {
+ '/api/projects/:team/cohorts/': ({ request }) => {
+ const offset = Number(new URL(request.url).searchParams.get('offset') ?? 0)
+ if (offset === 0) {
+ return { count: 2, next: 'http://localhost/next-page', results: [firstCohort] }
+ }
+ return { count: 2, next: null, results: [secondCohort] }
+ },
+ },
+ })
+
+ await expectLogic(logic, () => logic.actions.loadAllCohorts()).toDispatchActions(['loadAllCohortsSuccess'])
+ expect(logic.values.allCohorts.results.map(({ id }) => id)).toEqual([1, 2])
+
+ await expectLogic(logic, () => logic.actions.loadCohorts()).toDispatchActions(['loadCohortsSuccess'])
+ expect(logic.values.cohorts.results.map(({ id }) => id)).toEqual([1, 2])
+ })
+
it('sets polling timeout for calculating cohorts when on cohorts page', async () => {
// Set the current location to the cohorts page
router.actions.push(urls.cohorts())
diff --git a/frontend/src/models/cohortsModel.ts b/frontend/src/models/cohortsModel.ts
index 815ded295a7e..748175ad8351 100644
--- a/frontend/src/models/cohortsModel.ts
+++ b/frontend/src/models/cohortsModel.ts
@@ -82,7 +82,8 @@ export function isIndividualInsightPath(pathname: string): boolean {
export const COHORTS_PER_PAGE = 100
-export const MAX_COHORTS_FOR_FULL_LIST = 2000
+// The cohort list returns at most 1000 rows per request, so a full-list load reads every page.
+const COHORTS_FULL_LIST_PAGE_SIZE = 1000
export interface CohortFilters {
search?: string
@@ -94,6 +95,24 @@ export const DEFAULT_COHORT_FILTERS: CohortFilters = {
page: 1,
}
+async function loadAllCohortPages<T>(
+ loadPage: (params: {
+ limit: number
+ offset: number
+ }) => Promise<{ count: number; next?: string | null; results: T[] }>
+): Promise<CountedPaginatedResponse<T>> {
+ const results: T[] = []
+ let count = 0
+ let hasNextPage = true
+ while (hasNextPage) {
+ const page = await loadPage({ limit: COHORTS_FULL_LIST_PAGE_SIZE, offset: results.length })
+ results.push(...page.results)
+ count = page.count
+ hasNextPage = !!page.next && page.results.length > 0
+ }
+ return { count, results }
+}
+
export function processCohort(cohort: CohortType): CohortType {
return {
...cohort,
@@ -236,16 +255,10 @@ export interface cohortsModelActions {
errorObject?: any
}
loadAllCohortsSuccess: (
- allCohorts: {
- count: number
- results: CohortType[]
- },
+ allCohorts: CountedPaginatedResponse<CohortType>,
payload?: any
) => {
- allCohorts: {
- count: number
- results: CohortType[]
- }
+ allCohorts: CountedPaginatedResponse<CohortType>
payload?: any
}
loadCohorts: () => any
@@ -278,16 +291,10 @@ export interface cohortsModelActions {
errorObject?: any
}
loadCohortsSuccess: (
- cohorts: {
- count: number
- results: CohortType[]
- },
+ cohorts: CountedPaginatedResponse<CohortType>,
payload?: any
) => {
- cohorts: {
- count: number
- results: CohortType[]
- }
+ cohorts: CountedPaginatedResponse<CohortType>
payload?: any
}
setPollTimeout: (pollTimeout: number | null) => {
@@ -331,13 +338,16 @@ export const cohortsModel = kea<cohortsModelType>([
cohorts: {
__default: { count: 0, results: [] } as CountedPaginatedResponse<CohortType>,
loadCohorts: async () => {
- const response = await api.cohorts.listPaginated({
- limit: MAX_COHORTS_FOR_FULL_LIST,
- })
+ if (!isAuthenticatedTeam(values.currentTeam)) {
+ return values.cohorts
+ }
+ const projectId = String(values.currentTeam.project_id)
+ const response = await loadAllCohortPages((params) => cohortsApi.cohortsList(projectId, params))
personsLogic.findMounted({ syncWithUrl: true })?.actions.loadCohorts()
return {
count: response.count,
- results: response.results.map((cohort) => processCohort(cohort)),
+ // The shared cache still exposes the legacy shape to its consumers.
+ results: response.results.map((cohort) => processCohort(cohort as unknown as CohortType)),
}
},
},
@@ -385,21 +395,25 @@ export const cohortsModel = kea<cohortsModelType>([
cache.fullListRequested = true
// `allCohorts` feeds `cohortsById`, mostly id + name lookups, so request the
// basic payload that drops the legacy `groups`/`query` JSON. Fetching the full
- // payload for up to 2,000 cohorts on this hot path was slow enough to hit the
+ // payload for every cohort on this hot path was slow enough to hit the
// gateway timeout on cohort-heavy teams. `filters` stays in the basic payload:
// the feature-flag intent warning reads it off `cohortsById` (see
// featureFlagIntentWarningLogic.hasBehavioralCriteria), and its raw shape is
// what that check expects, so no `processCohort` normalization is needed here.
- const response = await api.cohorts.listBasic({
- limit: MAX_COHORTS_FOR_FULL_LIST,
- })
- const listedIds = new Set(response.results.map(({ id }) => id))
+ if (!isAuthenticatedTeam(values.currentTeam)) {
+ return values.allCohorts
+ }
+ const projectId = String(values.currentTeam.project_id)
+ const response = await loadAllCohortPages((params) =>
+ cohortsApi.cohortsList(projectId, { ...params, basic: true })
+ )
+ const listedIds = new Set<CohortType['id']>(response.results.map(({ id }) => id))
return {
count: response.count,
// `groups` is dropped from the basic payload but required on CohortType, so
// add the empty default back — nothing reading `cohortsById` uses `groups`.
results: [
- ...response.results.map((cohort): CohortType => ({ ...cohort, groups: [] })),
+ ...response.results.map((cohort) => ({ ...(cohort as unknown as CohortType), groups: [] })),
...values.allCohorts.results.filter(
({ id }) => !listedIds.has(id) && cache.referencedCohortIds?.has(id)
),
diff --git a/frontend/src/models/dashboardsModel.tsx b/frontend/src/models/dashboardsModel.tsx
index 2f8bb93f0430..e59fe1e6b673 100644
--- a/frontend/src/models/dashboardsModel.tsx
+++ b/frontend/src/models/dashboardsModel.tsx
@@ -525,7 +525,7 @@ export const dashboardsModel = kea<dashboardsModelType>([
let hasNextPage = true
while (hasNextPage) {
const page = await dashboardsList(String(teamId), {
- limit: 2000,
+ limit: 1000,
offset: results.length,
exclude_generated: true,
})
@@ -561,7 +561,7 @@ export const dashboardsModel = kea<dashboardsModelType>([
let apiUrl =
url ||
- `api/projects/${teamLogic.values.currentTeamId}/dashboards/?limit=2000&exclude_generated=true`
+ `api/projects/${teamLogic.values.currentTeamId}/dashboards/?limit=1000&exclude_generated=true`
// nosemgrep: prefer-codegen-api -- Legacy raw API call with a URL built at runtime and an unchecked response type. Use a generated function if one covers this endpoint.
const dashboards: PaginatedResponse<DashboardType> = await api.get(apiUrl)
diff --git a/posthog/api/pagination.py b/posthog/api/pagination.py
index 54ac69a86598..e0ffc2283df0 100644
--- a/posthog/api/pagination.py
+++ b/posthog/api/pagination.py
@@ -176,6 +176,17 @@ class BoundedLimitOffsetPagination(LimitOffsetPagination):
return parameters
+class DefaultLimitOffsetPagination(BoundedLimitOffsetPagination):
+ """The project-wide default paginator: it caps `limit` at `max_limit` but leaves the schema as DRF writes it.
+
+ Clients generated from the schema keep accepting a larger limit. The server answers with at most
+ `max_limit` rows and a `next` link, so a client that follows `next` still reads every row.
+ """
+
+ def get_schema_operation_parameters(self, view: Any) -> list[dict[str, Any]]:
+ return LimitOffsetPagination.get_schema_operation_parameters(self, view)
+
+
class FacadeLimitOffsetPagination(BoundedLimitOffsetPagination):
"""Builds the standard limit/offset response from a page that a facade read in SQL.
diff --git a/posthog/settings/web.py b/posthog/settings/web.py
index 059614260606..93b3cdb14651 100644
--- a/posthog/settings/web.py
+++ b/posthog/settings/web.py
@@ -559,7 +559,7 @@ EMAIL_VERIFICATION_SKIP_FOR_DOMAINS = [
REST_FRAMEWORK = {
"DEFAULT_AUTHENTICATION_CLASSES": ["posthog.auth.SessionAuthentication"],
- "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination",
+ "DEFAULT_PAGINATION_CLASS": "posthog.api.pagination.DefaultLimitOffsetPagination",
"DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"],
"DEFAULT_RENDERER_CLASSES": ["posthog.renderers.SafeJSONRenderer"],
"PAGE_SIZE": 100,
diff --git a/products/business_knowledge/backend/api/views.py b/products/business_knowledge/backend/api/views.py
index cefa33e614a5..cf73062f6dc5 100644
--- a/products/business_knowledge/backend/api/views.py
+++ b/products/business_knowledge/backend/api/views.py
@@ -19,6 +19,7 @@ from rest_framework.permissions import IsAuthenticated
from rest_framework.request import Request
from rest_framework.response import Response
+from posthog.api.pagination import BoundedLimitOffsetPagination
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.event_usage import groups
from posthog.models.user import User
@@ -32,6 +33,7 @@ from ..constants import (
BK_DRILLDOWN_MAX_RADIUS,
BK_SEARCH_DEFAULT_LIMIT,
BK_SEARCH_MAX_LIMIT,
+ MAX_URLS_PER_SOURCE,
)
from ..file_parse import FileParseError
from ..github_repos import connection_status
@@ -95,6 +97,11 @@ def _requested_library_ids(team_id: int, request: Request) -> list[UUID] | None:
return parsed
+class SourceDocumentsPagination(BoundedLimitOffsetPagination):
+ # The frontend reads a source's whole document list in one request, and a crawl stops at this many URLs.
+ max_limit = MAX_URLS_PER_SOURCE
+
+
class _ConflictError(exceptions.APIException):
# 409 is the right semantics for "resource is currently busy / in a
# state that conflicts with the request". DRF has no first-class helper
@@ -498,6 +505,7 @@ class KnowledgeSourceViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
methods=["get"],
url_path="documents",
required_scopes=["business_knowledge:read"],
+ pagination_class=SourceDocumentsPagination,
)
def documents(self, request: Request, pk: str, **kwargs) -> Response:
try:
diff --git a/products/business_knowledge/frontend/generated/api.schemas.ts b/products/business_knowledge/frontend/generated/api.schemas.ts
index 8368597f9445..f876a726f4b0 100644
--- a/products/business_knowledge/frontend/generated/api.schemas.ts
+++ b/products/business_knowledge/frontend/generated/api.schemas.ts
@@ -750,6 +750,7 @@ export const BusinessKnowledgeSourcesListSourceType = {
export type BusinessKnowledgeSourcesDocumentsListParams = {
/**
* Number of results to return per page.
+ * @maximum 5000
*/
limit?: number
/**
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 938d8d6532ca..a6bd9ebd8ad6 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -132961,6 +132961,7 @@ export namespace Schemas {
export type BusinessKnowledgeSourcesDocumentsListParams = {
/**
* Number of results to return per page.
+ * @maximum 5000
*/
limit?: number;
/**