:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(cohorts): scope the cohort list by team ids instead of a team join (#118325)
posthog[bot]a32384de7ffb
diff --git a/posthog/api/cohort.py b/posthog/api/cohort.pyindex 2d282c92ba86..21bbd6eae9b0 100644--- a/posthog/api/cohort.py+++ b/posthog/api/cohort.py@@ -2177,6 +2177,20 @@ class CohortViewSet(TeamAndOrgViewSetMixin, ForbidDestroyModel, viewsets.ModelVicontext["basic_cohort_list"] = self._is_basic_list_request()return context+ def _filter_queryset_by_parents_lookups(self, queryset: QuerySet) -> QuerySet:+ # The default project scoping filters on `team__project_id`, a JOIN to `posthog_team`.+ # With that JOIN, Postgres reads and sorts every cohort of the project before it applies+ # the page limit. A filter on `team_id` lets `cohort_team_created_idx` serve the+ # `-created_at` page directly. A project and its only team share an id. This filter+ # therefore selects the same rows as the default. The API cannot add a second team to a+ # project, because `ProjectEnvironmentsViewSet` rejects every request.+ if set(self.parents_query_dict) != {"project_id"}:+ return super()._filter_queryset_by_parents_lookups(queryset)+ # `RootTeamQuerySet.filter` rewrites a `team_id=` keyword into an `OR` over the parent team.+ # That `OR` JOINs `posthog_team`. A `Q` object skips the rewrite, as the default+ # `team__project_id` filter does.+ return queryset.filter(Q(team_id=self.team_id))+def _filter_request(self, request: Request, queryset: QuerySet) -> tuple[QuerySet, bool]:# Returns (queryset, search_ordered). `search_ordered` is True only when a non-blank# search applied trigram relevance ordering, so the caller knows not to re-impose the@@ -2246,13 +2260,11 @@ class CohortViewSet(TeamAndOrgViewSetMixin, ForbidDestroyModel, viewsets.ModelVi# `created_by` and `team` are forward FKs, so `select_related` JOINs them in one query# instead of the extra round-trips `prefetch_related` costs. The list serializer never- # reads `cohort.team`, so the list path only `select_related`s `created_by`. Project- # scoping still JOINs `posthog_team` to filter on `project_id`, but without hydration that- # JOIN reads only the id — not the full team payload (heavy JSON and array columns) each- # cohort row would otherwise carry. Detail and write actions do read `cohort.team`, so they- # keep the hydrating JOIN but re-apply `.defer(*DEPRECATED_ATTRS)` — mirroring- # `TeamManager`'s lazy-load defer — so the deprecated taxonomy columns, which TOAST out to- # megabytes per team, stay off it.+ # reads `cohort.team`, so the list path only `select_related`s `created_by`, and the cohort+ # SELECT does not read the full team payload (heavy JSON and array columns) per row. Detail+ # and write actions do read `cohort.team`, so they keep the hydrating JOIN but re-apply+ # `.defer(*DEPRECATED_ATTRS)` — mirroring `TeamManager`'s lazy-load defer — so the deprecated+ # taxonomy columns, which TOAST out to megabytes per team, stay off it.if self.action == "list":queryset = queryset.select_related("created_by")else:diff --git a/posthog/api/test/test_cohort.py b/posthog/api/test/test_cohort.pyindex 410858260cba..df765e6b33fa 100644--- a/posthog/api/test/test_cohort.py+++ b/posthog/api/test/test_cohort.py@@ -443,13 +443,11 @@ class TestCohort(TestExportMixin, ClickhouseTestMixin, APIBaseTest, QueryMatchin@patch("posthog.api.cohort.report_user_action")@patch("posthog.tasks.calculate_cohort.calculate_cohort_ch.delay")- def test_list_cohorts_does_not_hydrate_team(self, patch_calculate_cohort, patch_capture):- # The list serializer never reads `cohort.team`, so the cohort SELECT must not hydrate the- # team payload — the heavy JSON and array columns each row would otherwise carry. Project- # scoping still JOINs `posthog_team` to filter on `project_id`, so guard the payload columns- # specifically rather than the JOIN. The nplus1 query-count guard would not catch a re-added- # `select_related("team")`, since hydrating columns onto that existing JOIN adds no query,- # so assert the SQL directly.+ def test_list_cohorts_does_not_join_team(self, patch_calculate_cohort, patch_capture):+ # The list serializer never reads `cohort.team`, and project scoping filters on the team ID,+ # so the cohort SELECT must not JOIN `posthog_team`. A JOIN hydrates the heavy team payload,+ # or, through a `team__project_id` filter, stops `cohort_team_created_idx` from serving the+ # ordered page. The query count does not change in either case, so assert the SQL directly.self.client.post(f"/api/projects/{self.team.id}/cohorts",data={"name": "whatever", "groups": [{"properties": {"team_id": 5}}]},@@ -461,11 +459,20 @@ class TestCohort(TestExportMixin, ClickhouseTestMixin, APIBaseTest, QueryMatchincohort_queries = [q["sql"] for q in context.captured_queries if 'FROM "posthog_cohort"' in q["sql"]]assert cohort_queries, "expected the list to run a query against posthog_cohort"- # These team payload columns are never part of the cohort filter, so their presence would- # mean the team row is hydrated onto each cohort row.for sql in cohort_queries:- for column in ("test_account_filters", "session_replay_config"):- assert f'posthog_team"."{column}"' not in sql+ assert 'JOIN "posthog_team"' not in sql++ @patch("posthog.api.cohort.report_user_action")+ @patch("posthog.tasks.calculate_cohort.calculate_cohort_ch.delay")+ def test_list_cohorts_excludes_other_projects(self, patch_calculate_cohort, patch_capture):+ other_project_team = Team.objects.create(organization=self.organization)+ own = Cohort.objects.create(team=self.team, name="own")+ Cohort.objects.create(team=other_project_team, name="other project")++ response = self.client.get(f"/api/projects/{self.team.id}/cohorts")++ assert response.status_code == status.HTTP_200_OK+ assert {c["id"] for c in response.json()["results"]} == {own.id}@patch("posthog.api.cohort.report_user_action")@patch("posthog.tasks.calculate_cohort.calculate_cohort_ch.delay")@@ -473,8 +480,8 @@ class TestCohort(TestExportMixin, ClickhouseTestMixin, APIBaseTest, QueryMatchin# Detail and write actions read `cohort.team`, so they keep the hydrating `select_related("team")`# JOIN. They must re-apply `.defer(*DEPRECATED_ATTRS)` the way `TeamManager` does on lazy loads, or# every retrieve re-reads the deprecated taxonomy columns (`event_names` and siblings) that TOAST- # out to megabytes per team. `test_list_cohorts_does_not_hydrate_team` guards only the list query- # and only non-deprecated columns, so it cannot catch a dropped defer here. Assert the SQL directly.+ # out to megabytes per team. `test_list_cohorts_does_not_join_team` guards only the list query,+ # so it cannot catch a dropped defer here. Assert the SQL directly.cohort_id = self.client.post(f"/api/projects/{self.team.id}/cohorts",data={"name": "whatever", "groups": [{"properties": {"team_id": 5}}]},diff --git a/posthog/management/commands/test/test_resave_cohorts.py b/posthog/management/commands/test/test_resave_cohorts.pyindex 8cd78820854a..97dcc8a99b86 100644--- a/posthog/management/commands/test/test_resave_cohorts.py+++ b/posthog/management/commands/test/test_resave_cohorts.py@@ -209,7 +209,7 @@ class TestResaveCohortsCommandSingleTeam(BaseTest):# cohort would pull the deprecated taxonomy columns (which TOAST out to megabytes per team)# back once per cohort. The command already holds the team object, so the cohort SELECT must# not JOIN and hydrate `posthog_team`. A query-count guard would miss a re-added- # `select_related("team")`, so assert the SQL directly, as `test_list_cohorts_does_not_hydrate_team`+ # `select_related("team")`, so assert the SQL directly, as `test_list_cohorts_does_not_join_team`# does.team: Team = self.teamfor i in range(3):