Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(scoping): batch lazy related reads on team-scoped models (#126463)

Paul D'Ambra7f74b15d4707
diff --git a/posthog/models/scoping/README.md b/posthog/models/scoping/README.md
index de13bf3e3147..0855956a2ff6 100644
--- a/posthog/models/scoping/README.md
+++ b/posthog/models/scoping/README.md
@@ -54,6 +54,23 @@ def my_task(team_id: int):
Repo.objects.all()
```
+## Fetch mode
+
+Scoped querysets use `FETCH_RELATION_PEERS`, a variant of Django's `FETCH_PEERS` fetch mode.
+When code reads a relation that a row has not loaded, Django loads it for every row from the same query, in one query.
+A loop that reads `run.repo` on each of N rows costs 2 queries, not N + 1.
+The batch covers every row the query returned, so a loop that reads one row and stops still loads the relation for all of them.
+Slice the queryset when you only need the first rows.
+
+Deferred fields from `.only()` and `.defer()` still load one row at a time.
+Django's batched load of a deferred field overwrites values already assigned on other rows, and fails when another row has been deleted.
+
+`.iterator()` and `.aiterator()` use `FETCH_ONE`.
+A streamed query shares one peer list across all its rows, so `FETCH_PEERS` would scan a growing list on each read.
+Use `select_related()` in a streamed loop that reads a relation.
+
+To opt out, call `.fetch_mode(models.FETCH_ONE)` on the queryset.
+
## Adoption
### New main-DB model
diff --git a/posthog/models/scoping/manager.py b/posthog/models/scoping/manager.py
index 3cffbb8720d0..0ebabf7ff301 100644
--- a/posthog/models/scoping/manager.py
+++ b/posthog/models/scoping/manager.py
@@ -16,15 +16,38 @@ raise on framework code that has no team context. Raw SQL bypasses the
ORM entirely. Use this alongside explicit team checks at the API layer.
"""
-from typing import TypeVar, cast
+from collections.abc import AsyncIterator, Iterator
+from typing import Any, TypeVar, cast
from django.db import models
+from django.db.models.fetch_modes import FetchPeers
+from django.db.models.query_utils import DeferredAttribute
from posthog.models.scoping import get_current_team_id
T = TypeVar("T", bound=models.Model)
+class FetchRelationPeers(FetchPeers):
+ """`FETCH_PEERS` for relations, `FETCH_ONE` for deferred fields."""
+
+ __slots__ = ()
+
+ def fetch(self, fetcher: Any, instance: models.Model) -> None:
+ # Django's DeferredAttribute.fetch_many overwrites a value the caller already
+ # assigned on a peer, and raises KeyError when a peer row is gone.
+ if isinstance(fetcher, DeferredAttribute):
+ fetcher.fetch_one(instance)
+ else:
+ super().fetch(fetcher, instance)
+
+ def __reduce__(self) -> str:
+ return "FETCH_RELATION_PEERS"
+
+
+FETCH_RELATION_PEERS = FetchRelationPeers()
+
+
class TeamScopeError(Exception):
"""Raised when a team-scoped model is queried without team context.
@@ -65,8 +88,29 @@ class TeamScopedQuerySet(models.QuerySet[T]):
Provides an `unscoped()` method to bypass automatic filtering when you
need to explicitly query across teams.
+
+ Defaults to `FETCH_RELATION_PEERS`: a lazy related read on one row loads
+ it for every row the same query returned, in one query.
"""
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ super().__init__(*args, **kwargs)
+ self._fetch_mode = FETCH_RELATION_PEERS
+
+ def iterator(self, chunk_size: int | None = None) -> Iterator[T]:
+ return super(TeamScopedQuerySet, self._without_streamed_peers()).iterator(chunk_size)
+
+ def aiterator(self, chunk_size: int = 2000) -> AsyncIterator[T]:
+ return super(TeamScopedQuerySet, self._without_streamed_peers()).aiterator(chunk_size)
+
+ def _without_streamed_peers(self) -> "TeamScopedQuerySet[T]":
+ # A streamed query shares one peer list across all its rows, and FETCH_PEERS
+ # scans that list on each lazy read, so a loop that reads a relation per row
+ # costs O(rows²). The rows a loop has dropped are not batched anyway.
+ if self._fetch_mode is FETCH_RELATION_PEERS:
+ return self.fetch_mode(models.FETCH_ONE)
+ return self
+
def _apply_team_filter(self, team_id: int) -> "TeamScopedQuerySet[T]":
"""Apply team filtering. Caller passes the canonical team_id; we
trust it and filter directly. No DB resolution at read time —
diff --git a/posthog/models/scoping/test_manager.py b/posthog/models/scoping/test_manager.py
index 7d437642826b..8da8172430aa 100644
--- a/posthog/models/scoping/test_manager.py
+++ b/posthog/models/scoping/test_manager.py
@@ -1,6 +1,10 @@
+from collections.abc import Callable, Iterable
+
import pytest
from posthog.test.base import BaseTest
+from parameterized import parameterized
+
from posthog.models.scoping import team_scope, unscoped
from posthog.models.scoping.manager import TeamScopedManager, TeamScopeError
from posthog.models.team import Team
@@ -71,6 +75,49 @@ class TestTeamScopedManager(BaseTest):
assert flag is not None
assert flag.key == "active-flag"
+ @parameterized.expand(
+ [
+ ("ambient_scope", lambda manager, team_id: manager.get_queryset().filter(active=True), 2),
+ ("unscoped", lambda manager, team_id: manager.unscoped().filter(team_id=team_id), 2),
+ ("for_team", lambda manager, team_id: manager.for_team(team_id, canonical=True), 2),
+ ("queryset_unscoped", lambda manager, team_id: manager.unscoped().unscoped().filter(team_id=team_id), 2),
+ ("iterator", lambda manager, team_id: manager.for_team(team_id, canonical=True).iterator(), 4),
+ ]
+ )
+ def test_related_read_on_every_row_batches_across_peers(
+ self,
+ _name: str,
+ build: Callable[[TeamScopedManager[FeatureFlag], int], Iterable[FeatureFlag]],
+ expected_queries: int,
+ ) -> None:
+ for key in ("flag-1", "flag-2", "flag-3"):
+ FeatureFlag.objects.create(team=self.team, key=key, active=True, created_by=self.user)
+
+ with team_scope(self.team.id):
+ rows = build(self._make_manager(), self.team.id)
+ with self.assertNumQueries(expected_queries):
+ flags = list(rows)
+ assert [flag.created_by for flag in flags] == [self.user] * 3
+
+ def test_deferred_read_keeps_value_assigned_on_peer(self) -> None:
+ for key in ("flag-1", "flag-2"):
+ FeatureFlag.objects.create(team=self.team, key=key, name=key, created_by=self.user)
+
+ first, second = list(self._make_manager().for_team(self.team.id, canonical=True).only("id").order_by("key"))
+ first.name = "edited"
+
+ assert second.name == "flag-2"
+ assert first.name == "edited"
+
+ def test_deferred_read_survives_deleted_peer(self) -> None:
+ for key in ("flag-1", "flag-2"):
+ FeatureFlag.objects.create(team=self.team, key=key, name=key, created_by=self.user)
+
+ first, second = list(self._make_manager().for_team(self.team.id, canonical=True).only("id").order_by("key"))
+ FeatureFlag.objects.filter(pk=second.pk).delete()
+
+ assert first.name == "flag-1"
+
def test_unscoped_context_clears_team(self):
with team_scope(self.team.id):
from posthog.models.scoping import get_current_team_id