:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(scoping): batch lazy related reads on team-scoped models (#126463)
Paul D'Ambra7f74b15d4707
diff --git a/posthog/models/scoping/README.md b/posthog/models/scoping/README.mdindex de13bf3e3147..0855956a2ff6 100644--- a/posthog/models/scoping/README.md+++ b/posthog/models/scoping/README.md@@ -54,6 +54,23 @@ def my_task(team_id: int):Repo.objects.all()```+## Fetch mode++Scoped querysets use `FETCH_RELATION_PEERS`, a variant of Django's `FETCH_PEERS` fetch mode.+When code reads a relation that a row has not loaded, Django loads it for every row from the same query, in one query.+A loop that reads `run.repo` on each of N rows costs 2 queries, not N + 1.+The batch covers every row the query returned, so a loop that reads one row and stops still loads the relation for all of them.+Slice the queryset when you only need the first rows.++Deferred fields from `.only()` and `.defer()` still load one row at a time.+Django's batched load of a deferred field overwrites values already assigned on other rows, and fails when another row has been deleted.++`.iterator()` and `.aiterator()` use `FETCH_ONE`.+A streamed query shares one peer list across all its rows, so `FETCH_PEERS` would scan a growing list on each read.+Use `select_related()` in a streamed loop that reads a relation.++To opt out, call `.fetch_mode(models.FETCH_ONE)` on the queryset.+## Adoption### New main-DB modeldiff --git a/posthog/models/scoping/manager.py b/posthog/models/scoping/manager.pyindex 3cffbb8720d0..0ebabf7ff301 100644--- a/posthog/models/scoping/manager.py+++ b/posthog/models/scoping/manager.py@@ -16,15 +16,38 @@ raise on framework code that has no team context. Raw SQL bypasses theORM entirely. Use this alongside explicit team checks at the API layer."""-from typing import TypeVar, cast+from collections.abc import AsyncIterator, Iterator+from typing import Any, TypeVar, castfrom django.db import models+from django.db.models.fetch_modes import FetchPeers+from django.db.models.query_utils import DeferredAttributefrom posthog.models.scoping import get_current_team_idT = TypeVar("T", bound=models.Model)+class FetchRelationPeers(FetchPeers):+ """`FETCH_PEERS` for relations, `FETCH_ONE` for deferred fields."""++ __slots__ = ()++ def fetch(self, fetcher: Any, instance: models.Model) -> None:+ # Django's DeferredAttribute.fetch_many overwrites a value the caller already+ # assigned on a peer, and raises KeyError when a peer row is gone.+ if isinstance(fetcher, DeferredAttribute):+ fetcher.fetch_one(instance)+ else:+ super().fetch(fetcher, instance)++ def __reduce__(self) -> str:+ return "FETCH_RELATION_PEERS"+++FETCH_RELATION_PEERS = FetchRelationPeers()++class TeamScopeError(Exception):"""Raised when a team-scoped model is queried without team context.@@ -65,8 +88,29 @@ class TeamScopedQuerySet(models.QuerySet[T]):Provides an `unscoped()` method to bypass automatic filtering when youneed to explicitly query across teams.++ Defaults to `FETCH_RELATION_PEERS`: a lazy related read on one row loads+ it for every row the same query returned, in one query."""+ def __init__(self, *args: Any, **kwargs: Any) -> None:+ super().__init__(*args, **kwargs)+ self._fetch_mode = FETCH_RELATION_PEERS++ def iterator(self, chunk_size: int | None = None) -> Iterator[T]:+ return super(TeamScopedQuerySet, self._without_streamed_peers()).iterator(chunk_size)++ def aiterator(self, chunk_size: int = 2000) -> AsyncIterator[T]:+ return super(TeamScopedQuerySet, self._without_streamed_peers()).aiterator(chunk_size)++ def _without_streamed_peers(self) -> "TeamScopedQuerySet[T]":+ # A streamed query shares one peer list across all its rows, and FETCH_PEERS+ # scans that list on each lazy read, so a loop that reads a relation per row+ # costs O(rows²). The rows a loop has dropped are not batched anyway.+ if self._fetch_mode is FETCH_RELATION_PEERS:+ return self.fetch_mode(models.FETCH_ONE)+ return self+def _apply_team_filter(self, team_id: int) -> "TeamScopedQuerySet[T]":"""Apply team filtering. Caller passes the canonical team_id; wetrust it and filter directly. No DB resolution at read time —diff --git a/posthog/models/scoping/test_manager.py b/posthog/models/scoping/test_manager.pyindex 7d437642826b..8da8172430aa 100644--- a/posthog/models/scoping/test_manager.py+++ b/posthog/models/scoping/test_manager.py@@ -1,6 +1,10 @@+from collections.abc import Callable, Iterable+import pytestfrom posthog.test.base import BaseTest+from parameterized import parameterized+from posthog.models.scoping import team_scope, unscopedfrom posthog.models.scoping.manager import TeamScopedManager, TeamScopeErrorfrom posthog.models.team import Team@@ -71,6 +75,49 @@ class TestTeamScopedManager(BaseTest):assert flag is not Noneassert flag.key == "active-flag"+ @parameterized.expand(+ [+ ("ambient_scope", lambda manager, team_id: manager.get_queryset().filter(active=True), 2),+ ("unscoped", lambda manager, team_id: manager.unscoped().filter(team_id=team_id), 2),+ ("for_team", lambda manager, team_id: manager.for_team(team_id, canonical=True), 2),+ ("queryset_unscoped", lambda manager, team_id: manager.unscoped().unscoped().filter(team_id=team_id), 2),+ ("iterator", lambda manager, team_id: manager.for_team(team_id, canonical=True).iterator(), 4),+ ]+ )+ def test_related_read_on_every_row_batches_across_peers(+ self,+ _name: str,+ build: Callable[[TeamScopedManager[FeatureFlag], int], Iterable[FeatureFlag]],+ expected_queries: int,+ ) -> None:+ for key in ("flag-1", "flag-2", "flag-3"):+ FeatureFlag.objects.create(team=self.team, key=key, active=True, created_by=self.user)++ with team_scope(self.team.id):+ rows = build(self._make_manager(), self.team.id)+ with self.assertNumQueries(expected_queries):+ flags = list(rows)+ assert [flag.created_by for flag in flags] == [self.user] * 3++ def test_deferred_read_keeps_value_assigned_on_peer(self) -> None:+ for key in ("flag-1", "flag-2"):+ FeatureFlag.objects.create(team=self.team, key=key, name=key, created_by=self.user)++ first, second = list(self._make_manager().for_team(self.team.id, canonical=True).only("id").order_by("key"))+ first.name = "edited"++ assert second.name == "flag-2"+ assert first.name == "edited"++ def test_deferred_read_survives_deleted_peer(self) -> None:+ for key in ("flag-1", "flag-2"):+ FeatureFlag.objects.create(team=self.team, key=key, name=key, created_by=self.user)++ first, second = list(self._make_manager().for_team(self.team.id, canonical=True).only("id").order_by("key"))+ FeatureFlag.objects.filter(pk=second.pk).delete()++ assert first.name == "flag-1"+def test_unscoped_context_clears_team(self):with team_scope(self.team.id):from posthog.models.scoping import get_current_team_id