:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(devex): keep org access cache pickles per Django release (#124998)
Julian Bezf11215beffa7
diff --git a/posthog/models/test/test_organization_model.py b/posthog/models/test/test_organization_model.pyindex b28c7dc25142..1bdc536a0016 100644--- a/posthog/models/test/test_organization_model.py+++ b/posthog/models/test/test_organization_model.py@@ -272,22 +272,30 @@ class TestOrganization(BaseTest):@parameterized.expand([- ("one_organization_cold", 1, False),- ("many_organizations_cold", 5, False),- ("one_organization_warm", 1, True),- ("many_organizations_warm", 5, True),+ ("memberships_one_organization_cold", 1, False, False, 4),+ ("memberships_many_organizations_cold", 5, False, False, 4),+ ("memberships_one_organization_warm", 1, True, False, 4),+ ("memberships_many_organizations_warm", 5, True, False, 4),+ ("membership_cold", 1, False, True, 1),+ ("membership_warm", 1, True, True, 1),])- def test_access_cache_reads_memberships_in_constant_round_trips(self, _name, organization_count, warm):+ def test_access_cache_reads_in_constant_round_trips(self, _name, organization_count, warm, single, round_trips):organizations = [self.organization]for index in range(organization_count - 1):organization = Organization.objects.create(name=f"Extra organization {index}")OrganizationMembership.objects.create(organization=organization, user=self.user)organizations.append(organization)+ def lookup() -> list[OrganizationMembership]:+ if not single:+ return get_cached_organization_memberships(self.user)+ membership = get_cached_organization_membership(self.organization.id, self.user)+ return [] if membership is None else [membership]+with self.settings(ORGANIZATION_ACCESS_CACHE_ENABLED=True):if warm:- get_cached_organization_memberships(self.user)+ lookup()# The local memory test cache implements get_many with one get per key, unlike Redis MGET.unpatched_get = access_cache.get@@ -302,10 +310,29 @@ class TestOrganization(BaseTest):},) as get_many,):- memberships = get_cached_organization_memberships(self.user)+ memberships = lookup()- assert get.call_count + get_many.call_count == 4- assert {membership.organization for membership in memberships} == set(organizations)+ assert get.call_count + get_many.call_count == round_trips+ expected = {self.organization} if single else set(organizations)+ assert {membership.organization for membership in memberships} == expected++ def test_access_cache_invalidation_reaches_values_cached_by_another_django_release(self):+ with self.settings(ORGANIZATION_ACCESS_CACHE_ENABLED=True):+ with patch("posthog.organization_caching._PICKLE_NAMESPACE", "django5.2"):+ assert get_cached_organization_membership(self.organization.id, self.user) is not None++ with patch("posthog.organization_caching._PICKLE_NAMESPACE", "django6.1"):+ with self.assertNumQueries(1):+ membership = get_cached_organization_membership(self.organization.id, self.user)+ assert membership is not None+ membership.level = OrganizationMembership.Level.ADMIN+ with self.captureOnCommitCallbacks(execute=True):+ membership.save()++ with patch("posthog.organization_caching._PICKLE_NAMESPACE", "django5.2"):+ cached = get_cached_organization_membership(self.organization.id, self.user)+ assert cached is not None+ assert cached.level == OrganizationMembership.Level.ADMINdef test_access_cache_is_invalidated_when_organization_or_membership_changes(self):with self.settings(ORGANIZATION_ACCESS_CACHE_ENABLED=True):diff --git a/posthog/organization_caching.py b/posthog/organization_caching.pyindex 4b0c736a5181..212c2f177004 100644--- a/posthog/organization_caching.py+++ b/posthog/organization_caching.py@@ -3,6 +3,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Any, castfrom uuid import UUID, uuid4+import djangofrom django.apps import appsfrom django.conf import settingsfrom django.core.cache import caches@@ -23,6 +24,11 @@ access_cache = caches["organization_access"]ORGANIZATION_ACCESS_CACHE_TTL_SECONDS = 60_ORGANIZATION_ACCESS_CACHE_MISS = "missing"+# Pickled models do not load across Django releases (6.1 pickles import a module 5.2 lacks), so value+# keys carry the release for deploys and rollbacks. Version keys must stay shared so that an+# invalidation on one release also makes the cached values of the other release stale.+_PICKLE_NAMESPACE = f"django{django.VERSION[0]}.{django.VERSION[1]}"+def _organization_model() -> type[Organization]:return cast("type[Organization]", apps.get_model("posthog", "Organization"))@@ -48,6 +54,10 @@ def _access_cache_version_key(key: str) -> str:return f"{key}:version"+def _access_cache_value_key(key: str) -> str:+ return f"{key}:{_PICKLE_NAMESPACE}"++def _get_access_cache_value(key: str) -> Any:try:return access_cache.get(key)@@ -99,7 +109,7 @@ def _get_cache_versions(keys: list[str]) -> dict[str, str | None]:def _set_versioned_access_cache_value(key: str, value: Any, version: str | None) -> None:- _set_access_cache_value(key, (version, value))+ _set_access_cache_value(_access_cache_value_key(key), (version, value))def _unwrap_versioned_value(cached: Any, version: str | None) -> Any | None:@@ -110,7 +120,7 @@ def _unwrap_versioned_value(cached: Any, version: str | None) -> Any | None:def _get_versioned_access_cache_value(key: str, version: str | None) -> Any | None:- return _unwrap_versioned_value(_get_access_cache_value(key), version)+ return _unwrap_versioned_value(_get_access_cache_value(_access_cache_value_key(key)), version)def _cache_organization(organization: Organization, version: str | None) -> None:@@ -123,11 +133,11 @@ def _get_cached_organizations(organization_ids: list[UUID]) -> dict[UUID, Organiorganization_model = _organization_model()keys = {organization_id: _organization_cache_key(organization_id) for organization_id in organization_ids}versions = _get_cache_versions(list(keys.values()))- cached = _get_access_cache_values(list(keys.values()))+ cached = _get_access_cache_values([_access_cache_value_key(key) for key in keys.values()])organizations: dict[UUID, Organization] = {}for organization_id, key in keys.items():- value = _unwrap_versioned_value(cached.get(key), versions[key])+ value = _unwrap_versioned_value(cached.get(_access_cache_value_key(key)), versions[key])if isinstance(value, organization_model):organizations[organization_id] = value@@ -135,7 +145,10 @@ def _get_cached_organizations(organization_ids: list[UUID]) -> dict[UUID, Organiif missing_ids:loaded = list(organization_model.objects.filter(id__in=missing_ids))_set_access_cache_values(- {keys[organization.id]: (versions[keys[organization.id]], organization) for organization in loaded}+ {+ _access_cache_value_key(keys[organization.id]): (versions[keys[organization.id]], organization)+ for organization in loaded+ })organizations.update({organization.id: organization for organization in loaded})return organizations@@ -181,11 +194,11 @@ def _prepare_cached_membership(return membership-def _cache_membership(membership: OrganizationMembership, version: str | None = None) -> None:+def _cache_membership(membership: OrganizationMembership, version: str | None) -> None:organization = membership._state.fields_cache.pop("organization", None)user = membership._state.fields_cache.pop("user", None)key = _organization_membership_cache_key(membership.organization_id, membership.user_id)- _set_versioned_access_cache_value(key, membership, _get_cache_version(key) if version is None else version)+ _set_versioned_access_cache_value(key, membership, version)if organization is not None:membership._state.fields_cache["organization"] = organizationif user is not None:@@ -203,13 +216,27 @@ def get_cached_organization_membership(organization_id: str | UUID, user: User)return Nonekey = _organization_membership_cache_key(organization_id, user.id)- version = _get_cache_version(key)- organization_version = _get_cache_version(_organization_cache_key(organization_id))- cached = _get_versioned_access_cache_value(key, version)+ organization_key = _organization_cache_key(organization_id)+ cached_values = _get_access_cache_values(+ [+ _access_cache_version_key(key),+ _access_cache_version_key(organization_key),+ _access_cache_value_key(key),+ _access_cache_value_key(organization_key),+ ]+ )+ version = _as_cache_version(cached_values.get(_access_cache_version_key(key)))+ organization_version = _as_cache_version(cached_values.get(_access_cache_version_key(organization_key)))+ cached = _unwrap_versioned_value(cached_values.get(_access_cache_value_key(key)), version)if cached == _ORGANIZATION_ACCESS_CACHE_MISS:return Noneif isinstance(cached, organization_membership_model):- return _prepare_cached_membership(cached, user)+ organization = _unwrap_versioned_value(+ cached_values.get(_access_cache_value_key(organization_key)), organization_version+ )+ if not isinstance(organization, _organization_model()):+ organization = None+ return _prepare_cached_membership(cached, user, organization)try:membership = organization_membership_model.objects.select_related("organization").get(@@ -250,7 +277,7 @@ def invalidate_organization_access_cache(organization_id: str | UUID) -> None:key = _organization_cache_key(organization_id)_set_access_cache_value(_access_cache_version_key(key), uuid4().hex)- _delete_access_cache_value(key)+ _delete_access_cache_value(_access_cache_value_key(key))def invalidate_organization_membership_access_cache(organization_id: str | UUID, user_id: int) -> None:@@ -265,7 +292,7 @@ def invalidate_organization_membership_access_cache(organization_id: str | UUID,def invalidate_after_commit() -> None:for key in keys:_set_access_cache_value(_access_cache_version_key(key), uuid4().hex)- _delete_access_cache_value(key)+ _delete_access_cache_value(_access_cache_value_key(key))transaction.on_commit(invalidate_after_commit)