:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(billing): make billing org users backfill report failures (#125020)
Daniel Snitkovskiy333d2295ab7d
diff --git a/ee/billing/billing_manager.py b/ee/billing/billing_manager.pyindex 3c83a39eb1d9..885e3e6f8a7b 100644--- a/ee/billing/billing_manager.py+++ b/ee/billing/billing_manager.py@@ -10,7 +10,9 @@ from uuid import UUIDfrom django.conf import settingsfrom django.core.cache import cache-from django.db.models import F+from django.core.exceptions import ValidationError+from django.core.validators import validate_email+from django.db.models import F, Qfrom django.utils import timezonefrom django.utils.dateparse import parse_datetime@@ -93,6 +95,11 @@ class BillingAPIErrorCodes(Enum):OPEN_INVOICES_ERROR = "open_invoices_error"+class OrganizationHasNoOwnerError(Exception):+ def __init__(self) -> None:+ super().__init__("No owner membership found for organization")++class BillingServiceOpenInvoicesError(Exception):def __init__(self, message: str):self.message = message@@ -140,6 +147,23 @@ def _free_trial_active(customer: CustomerInfo) -> bool:return expires > timezone.now()+def _is_valid_email(email: str) -> bool:+ try:+ validate_email(email)+ except ValidationError:+ return False+ return True+++def get_most_recent_owner(organization: Organization) -> User | None:+ membership = (+ OrganizationMembership.objects.filter(organization=organization, level=OrganizationMembership.Level.OWNER)+ .order_by("-joined_at")+ .first()+ )+ return membership.user if membership else None++def _get_user_organization_role(user: User, organization: Organization) -> Optional[str]:"""Get a user role display string in a given organization, if membership doesn't exist return None.@@ -447,7 +471,7 @@ class BillingManager:def update_billing(self, organization: Organization, data: dict[str, Any], authorizer_actor: Optional[User] = None- ) -> None:+ ) -> requests.Response:res = http_session.patch(f"{BILLING_SERVICE_URL}/api/billing/",headers=self.get_auth_headers(organization, authorizer_actor=authorizer_actor),@@ -455,6 +479,7 @@ class BillingManager:)handle_billing_service_error(res)+ return resdef update_available_product_features(self, organization: Organization) -> list[dict[str, Any]]:res = http_session.get(@@ -492,56 +517,62 @@ class BillingManager:def update_billing_organization_users(self, organization: Organization) -> None:"""- Updates the register of users in the Billing service.- Since this can be called with users that are not ADMINs and update_billing requires- an ADMIN role, we do a privilege escalation using the owner.+ Updates the register of users in the Billing service, reporting any failure to error+ tracking instead of raising."""try:- distinct_ids = list(organization.members.values_list("distinct_id", flat=True))-- first_owner_membership = (- OrganizationMembership.objects.filter(organization=organization, level=15)- .order_by("-joined_at")- .first()- )- if not first_owner_membership:- capture_exception(- Exception(f"No owner membership found for organization"),- {"organization_id": organization.id},- )- return- first_owner = first_owner_membership.user-- admin_emails = list(- organization.members.filter(- organization_membership__level__gte=OrganizationMembership.Level.ADMIN- ).values_list("email", flat=True)- )+ self.sync_organization_users(organization)+ except Exception as e:+ capture_exception(e, {"organization_id": organization.id})- org_users = list(- organization.members.values("email", "distinct_id", "organization_membership__level")- .order_by("email") # Deterministic order for tests- .annotate(role=F("organization_membership__level"))- .filter(role__gte=OrganizationMembership.Level.ADMIN)- .values(- "email",- "distinct_id",- "role",- )+ def sync_organization_users(self, organization: Organization) -> requests.Response:+ """+ Sends the organization's members to the Billing service and returns its response.+ A 401 or 404 from billing is returned rather than raised, so a caller that needs to know+ whether the sync landed checks the status code.+ Since this can be called with users that are not ADMINs and update_billing requires+ an ADMIN role, we do a privilege escalation using the owner.+ """+ first_owner = get_most_recent_owner(organization)+ if not first_owner:+ raise OrganizationHasNoOwnerError()++ # Billing rejects the whole update when one value fails its validation, so values it would+ # reject are left out: a NULL or empty distinct_id, and an admin email that is not valid.+ members_with_distinct_id = organization.members.exclude(Q(distinct_id__isnull=True) | Q(distinct_id=""))++ distinct_ids = list(members_with_distinct_id.values_list("distinct_id", flat=True))++ admin_emails = [+ for email in organization.members.filter(+ organization_membership__level__gte=OrganizationMembership.Level.ADMIN+ ).values_list("email", flat=True)+ if _is_valid_email(email)+ ]++ org_users = list(+ members_with_distinct_id.values("email", "distinct_id", "organization_membership__level")+ .order_by("email") # Deterministic order for tests+ .annotate(role=F("organization_membership__level"))+ .filter(role__gte=OrganizationMembership.Level.ADMIN)+ .values(+ "email",+ "distinct_id",+ "role",)+ )- self.update_billing(- organization,- {- "distinct_ids": distinct_ids,- "org_customer_email": first_owner.email,- "org_admin_emails": admin_emails,- "org_users": org_users,- },- authorizer_actor=first_owner,- )- except Exception as e:- capture_exception(e, {"organization_id": organization.id})+ return self.update_billing(+ organization,+ {+ "distinct_ids": distinct_ids,+ "org_customer_email": first_owner.email,+ "org_admin_emails": admin_emails,+ "org_users": org_users,+ },+ authorizer_actor=first_owner,+ )def activate_subscription(self, organization: Organization, data: dict[str, Any]) -> dict[str, Any]:res = http_session.post(diff --git a/ee/billing/test/test_billing_manager.py b/ee/billing/test/test_billing_manager.pyindex b743d7c598e4..fedc29a1b18b 100644--- a/ee/billing/test/test_billing_manager.py+++ b/ee/billing/test/test_billing_manager.py@@ -32,8 +32,10 @@ from ee.billing.billing_manager import (BILLING_PROVIDER_WEBHOOK_SIGNATURE_VERSION,BILLING_PROVIDER_WEBHOOK_TIMESTAMP_HEADER,BillingManager,+ BillingServiceResponseError,FundingStatusUnavailable,OrganizationFundingStatus,+ OrganizationHasNoOwnerError,PrepaidCreditState,_get_user_organization_role,_parse_funding_status,@@ -1564,8 +1566,116 @@ class TestUpdateBillingOrganizationUsersPrivilegeEscalation(BaseTest):# Should capture exception about no ownermock_capture_exception.assert_called_once()- exception_call = mock_capture_exception.call_args- assert "No owner membership found" in str(exception_call[0][0])+ exception, properties = mock_capture_exception.call_args[0]+ assert str(exception) == "No owner membership found for organization"+ assert properties == {"organization_id": self.organization.id}++ @parameterized.expand(+ [+ ("unauthorized", 401, None, False),+ ("not_found", 404, None, False),+ ("server_error", 500, None, True),+ ("connection_error", None, requests.ConnectionError("boom"), True),+ ]+ )+ @patch("ee.billing.billing_manager.capture_exception")+ @patch("ee.billing.billing_manager.http_session.patch")+ def test_update_billing_org_users_never_raises_and_captures_only_errors(+ self, _name, status_code, side_effect, expect_capture, mock_patch, mock_capture_exception+ ):+ User.objects.create_and_join(+ organization=self.organization,+ email="[email protected]",+ password=None,+ level=OrganizationMembership.Level.OWNER,+ )+ mock_patch.return_value = MagicMock(status_code=status_code, json=MagicMock(return_value={}))+ mock_patch.side_effect = side_effect++ BillingManager(self.license, user=self.user).update_billing_organization_users(self.organization)++ mock_patch.assert_called_once()+ if expect_capture:+ mock_capture_exception.assert_called_once()+ assert mock_capture_exception.call_args[0][1] == {"organization_id": self.organization.id}+ else:+ mock_capture_exception.assert_not_called()++ @parameterized.expand([("ok", 200), ("unauthorized", 401), ("not_found", 404)])+ @patch("ee.billing.billing_manager.http_session.patch")+ def test_sync_organization_users_returns_billing_response(self, _name, status_code, mock_patch):+ User.objects.create_and_join(+ organization=self.organization,+ email="[email protected]",+ password=None,+ level=OrganizationMembership.Level.OWNER,+ )+ mock_patch.return_value = MagicMock(status_code=status_code, json=MagicMock(return_value={}))++ response = BillingManager(self.license, user=self.user).sync_organization_users(self.organization)++ assert response.status_code == status_code++ @patch("ee.billing.billing_manager.http_session.patch")+ def test_sync_organization_users_raises_on_billing_error(self, mock_patch):+ User.objects.create_and_join(+ organization=self.organization,+ email="[email protected]",+ password=None,+ level=OrganizationMembership.Level.OWNER,+ )+ mock_patch.return_value = MagicMock(status_code=500, json=MagicMock(return_value={}))++ with self.assertRaises(BillingServiceResponseError) as ctx:+ BillingManager(self.license, user=self.user).sync_organization_users(self.organization)++ assert ctx.exception.status_code == 500++ @patch("ee.billing.billing_manager.http_session.patch")+ def test_sync_organization_users_raises_without_owner(self, mock_patch):+ with self.assertRaises(OrganizationHasNoOwnerError):+ BillingManager(self.license, user=self.user).sync_organization_users(self.organization)++ mock_patch.assert_not_called()++ @patch("ee.billing.billing_manager.http_session.patch")+ def test_sync_organization_users_leaves_out_values_billing_rejects(self, mock_patch):+ mock_patch.return_value = MagicMock(status_code=200, json=MagicMock(return_value={}))+ owner = User.objects.create_and_join(+ organization=self.organization,+ email="[email protected]",+ password=None,+ level=OrganizationMembership.Level.OWNER,+ )+ admin_without_distinct_id = User.objects.create_and_join(+ organization=self.organization,+ email="[email protected]",+ password=None,+ level=OrganizationMembership.Level.ADMIN,+ )+ User.objects.filter(id=admin_without_distinct_id.id).update(distinct_id=None)+ admin_with_invalid_email = User.objects.create_and_join(+ organization=self.organization,+ email="not-an-email",+ password=None,+ level=OrganizationMembership.Level.ADMIN,+ )++ BillingManager(self.license, user=owner).sync_organization_users(self.organization)++ payload = mock_patch.call_args.kwargs["json"]+ assert len(payload["distinct_ids"]) == 3+ assert set(payload["distinct_ids"]) == {+ self.user.distinct_id,+ owner.distinct_id,+ admin_with_invalid_email.distinct_id,+ }+ assert payload["org_customer_email"] == "[email protected]"+ assert sorted(payload["org_admin_emails"]) == ["[email protected]", "[email protected]"]+ assert payload["org_users"] == [+ {"email": "not-an-email", "distinct_id": admin_with_invalid_email.distinct_id, "role": 8},+ {"email": "[email protected]", "distinct_id": owner.distinct_id, "role": 15},+ ]@patch("ee.billing.billing_manager.http_session.patch")@patch("posthog.event_usage.posthoganalytics.capture")diff --git a/posthog/management/commands/backfill_billing_organization_users.py b/posthog/management/commands/backfill_billing_organization_users.pyindex 560ca8591f79..992a69390ad2 100644--- a/posthog/management/commands/backfill_billing_organization_users.py+++ b/posthog/management/commands/backfill_billing_organization_users.py@@ -1,66 +1,89 @@import csv+from contextlib import ExitStack+from typing import Any-from django.core.management import BaseCommand+from django.core.exceptions import ValidationError+from django.core.management import BaseCommand, CommandParser-import structlog+from posthog.models.organization import Organization-from posthog.models.organization import Organization, OrganizationMembership+from ee.billing.billing_manager import BillingManager, BillingServiceResponseError, get_most_recent_owner-from ee.billing.billing_manager import BillingManager+PROGRESS_EVERY = 100-logger = structlog.get_logger(__name__)+def sync_organization(org_id: str) -> str | None:+ """Syncs one organization's members to billing. Returns why it failed, or None when billing accepted it."""+ try:+ return _sync_organization(org_id)+ except BillingServiceResponseError as e:+ return f"billing returned {e.status_code}"+ except Exception as e:+ return f"{type(e).__name__}: {e}"-class Command(BaseCommand):- """- Used to sync organization_users with billing- """- def add_arguments(self, parser):- parser.add_argument("csv_file", type=str)- parser.add_argument("--offset", type=int, default=0)+def _sync_organization(org_id: str) -> str | None:+ try:+ organization = Organization.objects.get(id=org_id)+ except (Organization.DoesNotExist, ValidationError):+ return "organization not found"++ owner = get_most_recent_owner(organization)+ if owner is None:+ return "organization has no owner"++ response = BillingManager(license=None, user=owner).sync_organization_users(organization)+ if not 200 <= response.status_code < 300:+ return f"billing returned {response.status_code}"+ return None- def handle(self, *args, **options):++class Command(BaseCommand):+ help = (+ "Sync organization members to billing for each org_id in a CSV file. "+ "Run it away from billing's 03:45 UTC usage run, since each synced org with a subscription queues a usage sync."+ )++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument("csv_file", type=str, help="CSV file with an org_id column")+ parser.add_argument("--offset", type=int, default=0, help="Skip this many rows of the CSV")+ parser.add_argument(+ "--failed-csv",+ type=str,+ default=None,+ help="Also write failed org_id,reason rows to this path. It can be passed back in as csv_file.",+ )++ def handle(self, *args: Any, **options: Any) -> None:with open(options["csv_file"], encoding="utf-8") as f:- reader = csv.DictReader(f)-- entries = list(reader)[options["offset"] :]- total_count = len(entries)- completed_count = 0- failed_count = 0- failed_org_ids = []-- logger.info(f"Total Backfill Entries: {total_count}")-- for row in entries:- org_id = row["org_id"]- try:- org = Organization.objects.get(id=org_id)-- owner = (- OrganizationMembership.objects.filter( # type: ignore- organization=org, level=OrganizationMembership.Level.OWNER- )- .first()- .user- )-- billing_manager = BillingManager(license=None, user=owner)-- try:- billing_manager.update_billing_organization_users(org)- except Exception as e:- logger.exception(f"Failed calling update_billing_organization_users: {e}")- raise-- completed_count += 1- except Exception:- failed_count += 1+ org_ids = [row["org_id"] for row in csv.DictReader(f)][options["offset"] :]++ total = len(org_ids)+ failed_org_ids: list[str] = []+ self.stdout.write(f"Syncing {total} organizations with billing")++ with ExitStack() as stack:+ failed_file = (+ stack.enter_context(open(options["failed_csv"], "w", encoding="utf-8", newline=""))+ if options["failed_csv"]+ else None+ )+ failed_writer = csv.writer(failed_file) if failed_file else None+ if failed_writer:+ failed_writer.writerow(["org_id", "reason"])++ for done, org_id in enumerate(org_ids, start=1):+ reason = sync_organization(org_id)+ if reason is not None:failed_org_ids.append(org_id)+ self.stderr.write(f"FAILED {org_id}: {reason}")+ if failed_file and failed_writer:+ failed_writer.writerow([org_id, reason])+ failed_file.flush()- logger.info(f"{completed_count}/{total_count} ({failed_count} failed)")-- logger.info(f"Finished backfilling {total_count} organizations")+ if done % PROGRESS_EVERY == 0:+ self.stdout.write(f"Progress: {done}/{total} ({len(failed_org_ids)} failed)")- if failed_org_ids:- logger.info(f"Failed updating {failed_org_ids}")+ self.stdout.write(f"Finished: {total - len(failed_org_ids)} ok, {len(failed_org_ids)} failed")+ if failed_org_ids:+ self.stdout.write(f"Failed org ids: {', '.join(failed_org_ids)}")diff --git a/posthog/management/commands/test/test_backfill_billing_organization_users.py b/posthog/management/commands/test/test_backfill_billing_organization_users.pynew file mode 100644index 000000000000..62e07ef9ba4a--- /dev/null+++ b/posthog/management/commands/test/test_backfill_billing_organization_users.py@@ -0,0 +1,128 @@+import csv+import uuid+import datetime+import tempfile+from io import StringIO+from pathlib import Path+from typing import cast++from posthog.test.base import BaseTest+from unittest.mock import MagicMock, patch++from django.core.management import call_command+from django.db import OperationalError++import requests+from parameterized import parameterized++from posthog.cloud_utils import TEST_clear_instance_license_cache+from posthog.models.organization import Organization, OrganizationMembership+from posthog.models.user import User++from ee.models.license import License, LicenseManager+++class TestBackfillBillingOrganizationUsers(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ TEST_clear_instance_license_cache()+ super(LicenseManager, cast(LicenseManager, License.objects)).create(+ key="license_id::license_secret",+ plan="enterprise",+ valid_until=datetime.datetime(2038, 1, 19, 3, 14, 7),+ )+ tmp_dir = tempfile.TemporaryDirectory()+ self.addCleanup(tmp_dir.cleanup)+ self.tmp_dir = Path(tmp_dir.name)++ def _org_with_owner(self, name: str) -> Organization:+ organization = Organization.objects.create(name=name)+ User.objects.create_and_join(+ organization=organization,+ email=f"owner-{organization.id}@example.com",+ password=None,+ level=OrganizationMembership.Level.OWNER,+ )+ return organization++ def _run(self, org_ids: list[str], **options: str | int) -> tuple[str, str]:+ csv_path = self.tmp_dir / "orgs.csv"+ csv_path.write_text("org_id\n" + "".join(f"{org_id}\n" for org_id in org_ids))+ out, err = StringIO(), StringIO()+ call_command("backfill_billing_organization_users", str(csv_path), stdout=out, stderr=err, **options)+ return out.getvalue(), err.getvalue()++ @parameterized.expand(+ [+ ("ok", 200, None, None),+ ("unauthorized", 401, None, "billing returned 401"),+ ("not_found", 404, None, "billing returned 404"),+ ("server_error", 500, None, "billing returned 500"),+ ("raised", None, requests.ConnectionError("boom"), "ConnectionError: boom"),+ ]+ )+ @patch("ee.billing.billing_manager.http_session.patch")+ def test_reports_billing_outcome(self, _name, status_code, side_effect, expected_reason, mock_patch):+ organization = self._org_with_owner("Synced org")+ mock_patch.return_value = MagicMock(status_code=status_code, json=MagicMock(return_value={}))+ mock_patch.side_effect = side_effect++ out, err = self._run([str(organization.id)])++ mock_patch.assert_called_once()+ assert "Syncing 1 organizations with billing" in out+ if expected_reason is None:+ assert err == ""+ assert "Finished: 1 ok, 0 failed" in out+ else:+ assert f"FAILED {organization.id}: {expected_reason}" in err+ assert "Finished: 0 ok, 1 failed" in out+ assert f"Failed org ids: {organization.id}" in out++ @patch("posthog.management.commands.backfill_billing_organization_users.PROGRESS_EVERY", 2)+ @patch("ee.billing.billing_manager.http_session.patch")+ def test_offset_missing_org_no_owner_and_failed_csv(self, mock_patch):+ mock_patch.return_value = MagicMock(status_code=200, json=MagicMock(return_value={}))+ skipped = self._org_with_owner("Skipped by offset")+ synced = self._org_with_owner("Synced")+ ownerless = Organization.objects.create(name="No owner")+ missing_id = str(uuid.uuid4())+ failed_csv = self.tmp_dir / "failed.csv"++ out, err = self._run(+ [str(skipped.id), str(synced.id), missing_id, str(ownerless.id)],+ offset=1,+ failed_csv=str(failed_csv),+ )++ mock_patch.assert_called_once()+ assert mock_patch.call_args.kwargs["json"]["org_customer_email"] == f"owner-{synced.id}@example.com"+ assert "Syncing 3 organizations with billing" in out+ assert f"FAILED {missing_id}: organization not found" in err+ assert f"FAILED {ownerless.id}: organization has no owner" in err+ assert "Progress: 2/3 (1 failed)" in out+ assert "Finished: 1 ok, 2 failed" in out+ assert f"Failed org ids: {missing_id}, {ownerless.id}" in out+ with failed_csv.open() as f:+ assert list(csv.reader(f)) == [+ ["org_id", "reason"],+ [missing_id, "organization not found"],+ [str(ownerless.id), "organization has no owner"],+ ]++ @patch("ee.billing.billing_manager.http_session.patch")+ def test_database_error_fails_the_org_and_continues(self, mock_patch):+ mock_patch.return_value = MagicMock(status_code=200, json=MagicMock(return_value={}))+ first = self._org_with_owner("Lookup fails")+ second = self._org_with_owner("Synced")+ second_owner = User.objects.get(email=f"owner-{second.id}@example.com")++ with patch(+ "posthog.management.commands.backfill_billing_organization_users.get_most_recent_owner",+ side_effect=[OperationalError("statement timeout"), second_owner],+ ):+ out, err = self._run([str(first.id), str(second.id)])++ assert f"FAILED {first.id}: OperationalError: statement timeout" in err+ mock_patch.assert_called_once()+ assert "Finished: 1 ok, 1 failed" in out