Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(billing): make billing org users backfill report failures (#125020)

Daniel Snitkovskiy333d2295ab7d
diff --git a/ee/billing/billing_manager.py b/ee/billing/billing_manager.py
index 3c83a39eb1d9..885e3e6f8a7b 100644
--- a/ee/billing/billing_manager.py
+++ b/ee/billing/billing_manager.py
@@ -10,7 +10,9 @@ from uuid import UUID
from django.conf import settings
from django.core.cache import cache
-from django.db.models import F
+from django.core.exceptions import ValidationError
+from django.core.validators import validate_email
+from django.db.models import F, Q
from django.utils import timezone
from django.utils.dateparse import parse_datetime
@@ -93,6 +95,11 @@ class BillingAPIErrorCodes(Enum):
OPEN_INVOICES_ERROR = "open_invoices_error"
+class OrganizationHasNoOwnerError(Exception):
+ def __init__(self) -> None:
+ super().__init__("No owner membership found for organization")
+
+
class BillingServiceOpenInvoicesError(Exception):
def __init__(self, message: str):
self.message = message
@@ -140,6 +147,23 @@ def _free_trial_active(customer: CustomerInfo) -> bool:
return expires > timezone.now()
+def _is_valid_email(email: str) -> bool:
+ try:
+ validate_email(email)
+ except ValidationError:
+ return False
+ return True
+
+
+def get_most_recent_owner(organization: Organization) -> User | None:
+ membership = (
+ OrganizationMembership.objects.filter(organization=organization, level=OrganizationMembership.Level.OWNER)
+ .order_by("-joined_at")
+ .first()
+ )
+ return membership.user if membership else None
+
+
def _get_user_organization_role(user: User, organization: Organization) -> Optional[str]:
"""
Get a user role display string in a given organization, if membership doesn't exist return None.
@@ -447,7 +471,7 @@ class BillingManager:
def update_billing(
self, organization: Organization, data: dict[str, Any], authorizer_actor: Optional[User] = None
- ) -> None:
+ ) -> requests.Response:
res = http_session.patch(
f"{BILLING_SERVICE_URL}/api/billing/",
headers=self.get_auth_headers(organization, authorizer_actor=authorizer_actor),
@@ -455,6 +479,7 @@ class BillingManager:
)
handle_billing_service_error(res)
+ return res
def update_available_product_features(self, organization: Organization) -> list[dict[str, Any]]:
res = http_session.get(
@@ -492,56 +517,62 @@ class BillingManager:
def update_billing_organization_users(self, organization: Organization) -> None:
"""
- Updates the register of users in the Billing service.
- Since this can be called with users that are not ADMINs and update_billing requires
- an ADMIN role, we do a privilege escalation using the owner.
+ Updates the register of users in the Billing service, reporting any failure to error
+ tracking instead of raising.
"""
try:
- distinct_ids = list(organization.members.values_list("distinct_id", flat=True))
-
- first_owner_membership = (
- OrganizationMembership.objects.filter(organization=organization, level=15)
- .order_by("-joined_at")
- .first()
- )
- if not first_owner_membership:
- capture_exception(
- Exception(f"No owner membership found for organization"),
- {"organization_id": organization.id},
- )
- return
- first_owner = first_owner_membership.user
-
- admin_emails = list(
- organization.members.filter(
- organization_membership__level__gte=OrganizationMembership.Level.ADMIN
- ).values_list("email", flat=True)
- )
+ self.sync_organization_users(organization)
+ except Exception as e:
+ capture_exception(e, {"organization_id": organization.id})
- org_users = list(
- organization.members.values("email", "distinct_id", "organization_membership__level")
- .order_by("email") # Deterministic order for tests
- .annotate(role=F("organization_membership__level"))
- .filter(role__gte=OrganizationMembership.Level.ADMIN)
- .values(
- "email",
- "distinct_id",
- "role",
- )
+ def sync_organization_users(self, organization: Organization) -> requests.Response:
+ """
+ Sends the organization's members to the Billing service and returns its response.
+ A 401 or 404 from billing is returned rather than raised, so a caller that needs to know
+ whether the sync landed checks the status code.
+ Since this can be called with users that are not ADMINs and update_billing requires
+ an ADMIN role, we do a privilege escalation using the owner.
+ """
+ first_owner = get_most_recent_owner(organization)
+ if not first_owner:
+ raise OrganizationHasNoOwnerError()
+
+ # Billing rejects the whole update when one value fails its validation, so values it would
+ # reject are left out: a NULL or empty distinct_id, and an admin email that is not valid.
+ members_with_distinct_id = organization.members.exclude(Q(distinct_id__isnull=True) | Q(distinct_id=""))
+
+ distinct_ids = list(members_with_distinct_id.values_list("distinct_id", flat=True))
+
+ admin_emails = [
+ email
+ for email in organization.members.filter(
+ organization_membership__level__gte=OrganizationMembership.Level.ADMIN
+ ).values_list("email", flat=True)
+ if _is_valid_email(email)
+ ]
+
+ org_users = list(
+ members_with_distinct_id.values("email", "distinct_id", "organization_membership__level")
+ .order_by("email") # Deterministic order for tests
+ .annotate(role=F("organization_membership__level"))
+ .filter(role__gte=OrganizationMembership.Level.ADMIN)
+ .values(
+ "email",
+ "distinct_id",
+ "role",
)
+ )
- self.update_billing(
- organization,
- {
- "distinct_ids": distinct_ids,
- "org_customer_email": first_owner.email,
- "org_admin_emails": admin_emails,
- "org_users": org_users,
- },
- authorizer_actor=first_owner,
- )
- except Exception as e:
- capture_exception(e, {"organization_id": organization.id})
+ return self.update_billing(
+ organization,
+ {
+ "distinct_ids": distinct_ids,
+ "org_customer_email": first_owner.email,
+ "org_admin_emails": admin_emails,
+ "org_users": org_users,
+ },
+ authorizer_actor=first_owner,
+ )
def activate_subscription(self, organization: Organization, data: dict[str, Any]) -> dict[str, Any]:
res = http_session.post(
diff --git a/ee/billing/test/test_billing_manager.py b/ee/billing/test/test_billing_manager.py
index b743d7c598e4..fedc29a1b18b 100644
--- a/ee/billing/test/test_billing_manager.py
+++ b/ee/billing/test/test_billing_manager.py
@@ -32,8 +32,10 @@ from ee.billing.billing_manager import (
BILLING_PROVIDER_WEBHOOK_SIGNATURE_VERSION,
BILLING_PROVIDER_WEBHOOK_TIMESTAMP_HEADER,
BillingManager,
+ BillingServiceResponseError,
FundingStatusUnavailable,
OrganizationFundingStatus,
+ OrganizationHasNoOwnerError,
PrepaidCreditState,
_get_user_organization_role,
_parse_funding_status,
@@ -1564,8 +1566,116 @@ class TestUpdateBillingOrganizationUsersPrivilegeEscalation(BaseTest):
# Should capture exception about no owner
mock_capture_exception.assert_called_once()
- exception_call = mock_capture_exception.call_args
- assert "No owner membership found" in str(exception_call[0][0])
+ exception, properties = mock_capture_exception.call_args[0]
+ assert str(exception) == "No owner membership found for organization"
+ assert properties == {"organization_id": self.organization.id}
+
+ @parameterized.expand(
+ [
+ ("unauthorized", 401, None, False),
+ ("not_found", 404, None, False),
+ ("server_error", 500, None, True),
+ ("connection_error", None, requests.ConnectionError("boom"), True),
+ ]
+ )
+ @patch("ee.billing.billing_manager.capture_exception")
+ @patch("ee.billing.billing_manager.http_session.patch")
+ def test_update_billing_org_users_never_raises_and_captures_only_errors(
+ self, _name, status_code, side_effect, expect_capture, mock_patch, mock_capture_exception
+ ):
+ User.objects.create_and_join(
+ organization=self.organization,
+ password=None,
+ level=OrganizationMembership.Level.OWNER,
+ )
+ mock_patch.return_value = MagicMock(status_code=status_code, json=MagicMock(return_value={}))
+ mock_patch.side_effect = side_effect
+
+ BillingManager(self.license, user=self.user).update_billing_organization_users(self.organization)
+
+ mock_patch.assert_called_once()
+ if expect_capture:
+ mock_capture_exception.assert_called_once()
+ assert mock_capture_exception.call_args[0][1] == {"organization_id": self.organization.id}
+ else:
+ mock_capture_exception.assert_not_called()
+
+ @parameterized.expand([("ok", 200), ("unauthorized", 401), ("not_found", 404)])
+ @patch("ee.billing.billing_manager.http_session.patch")
+ def test_sync_organization_users_returns_billing_response(self, _name, status_code, mock_patch):
+ User.objects.create_and_join(
+ organization=self.organization,
+ password=None,
+ level=OrganizationMembership.Level.OWNER,
+ )
+ mock_patch.return_value = MagicMock(status_code=status_code, json=MagicMock(return_value={}))
+
+ response = BillingManager(self.license, user=self.user).sync_organization_users(self.organization)
+
+ assert response.status_code == status_code
+
+ @patch("ee.billing.billing_manager.http_session.patch")
+ def test_sync_organization_users_raises_on_billing_error(self, mock_patch):
+ User.objects.create_and_join(
+ organization=self.organization,
+ password=None,
+ level=OrganizationMembership.Level.OWNER,
+ )
+ mock_patch.return_value = MagicMock(status_code=500, json=MagicMock(return_value={}))
+
+ with self.assertRaises(BillingServiceResponseError) as ctx:
+ BillingManager(self.license, user=self.user).sync_organization_users(self.organization)
+
+ assert ctx.exception.status_code == 500
+
+ @patch("ee.billing.billing_manager.http_session.patch")
+ def test_sync_organization_users_raises_without_owner(self, mock_patch):
+ with self.assertRaises(OrganizationHasNoOwnerError):
+ BillingManager(self.license, user=self.user).sync_organization_users(self.organization)
+
+ mock_patch.assert_not_called()
+
+ @patch("ee.billing.billing_manager.http_session.patch")
+ def test_sync_organization_users_leaves_out_values_billing_rejects(self, mock_patch):
+ mock_patch.return_value = MagicMock(status_code=200, json=MagicMock(return_value={}))
+ owner = User.objects.create_and_join(
+ organization=self.organization,
+ password=None,
+ level=OrganizationMembership.Level.OWNER,
+ )
+ admin_without_distinct_id = User.objects.create_and_join(
+ organization=self.organization,
+ password=None,
+ level=OrganizationMembership.Level.ADMIN,
+ )
+ User.objects.filter(id=admin_without_distinct_id.id).update(distinct_id=None)
+ admin_with_invalid_email = User.objects.create_and_join(
+ organization=self.organization,
+ email="not-an-email",
+ password=None,
+ level=OrganizationMembership.Level.ADMIN,
+ )
+
+ BillingManager(self.license, user=owner).sync_organization_users(self.organization)
+
+ payload = mock_patch.call_args.kwargs["json"]
+ assert len(payload["distinct_ids"]) == 3
+ assert set(payload["distinct_ids"]) == {
+ self.user.distinct_id,
+ owner.distinct_id,
+ admin_with_invalid_email.distinct_id,
+ }
+ assert payload["org_customer_email"] == "[email protected]"
+ assert sorted(payload["org_admin_emails"]) == ["[email protected]", "[email protected]"]
+ assert payload["org_users"] == [
+ {"email": "not-an-email", "distinct_id": admin_with_invalid_email.distinct_id, "role": 8},
+ {"email": "[email protected]", "distinct_id": owner.distinct_id, "role": 15},
+ ]
@patch("ee.billing.billing_manager.http_session.patch")
@patch("posthog.event_usage.posthoganalytics.capture")
diff --git a/posthog/management/commands/backfill_billing_organization_users.py b/posthog/management/commands/backfill_billing_organization_users.py
index 560ca8591f79..992a69390ad2 100644
--- a/posthog/management/commands/backfill_billing_organization_users.py
+++ b/posthog/management/commands/backfill_billing_organization_users.py
@@ -1,66 +1,89 @@
import csv
+from contextlib import ExitStack
+from typing import Any
-from django.core.management import BaseCommand
+from django.core.exceptions import ValidationError
+from django.core.management import BaseCommand, CommandParser
-import structlog
+from posthog.models.organization import Organization
-from posthog.models.organization import Organization, OrganizationMembership
+from ee.billing.billing_manager import BillingManager, BillingServiceResponseError, get_most_recent_owner
-from ee.billing.billing_manager import BillingManager
+PROGRESS_EVERY = 100
-logger = structlog.get_logger(__name__)
+def sync_organization(org_id: str) -> str | None:
+ """Syncs one organization's members to billing. Returns why it failed, or None when billing accepted it."""
+ try:
+ return _sync_organization(org_id)
+ except BillingServiceResponseError as e:
+ return f"billing returned {e.status_code}"
+ except Exception as e:
+ return f"{type(e).__name__}: {e}"
-class Command(BaseCommand):
- """
- Used to sync organization_users with billing
- """
- def add_arguments(self, parser):
- parser.add_argument("csv_file", type=str)
- parser.add_argument("--offset", type=int, default=0)
+def _sync_organization(org_id: str) -> str | None:
+ try:
+ organization = Organization.objects.get(id=org_id)
+ except (Organization.DoesNotExist, ValidationError):
+ return "organization not found"
+
+ owner = get_most_recent_owner(organization)
+ if owner is None:
+ return "organization has no owner"
+
+ response = BillingManager(license=None, user=owner).sync_organization_users(organization)
+ if not 200 <= response.status_code < 300:
+ return f"billing returned {response.status_code}"
+ return None
- def handle(self, *args, **options):
+
+class Command(BaseCommand):
+ help = (
+ "Sync organization members to billing for each org_id in a CSV file. "
+ "Run it away from billing's 03:45 UTC usage run, since each synced org with a subscription queues a usage sync."
+ )
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument("csv_file", type=str, help="CSV file with an org_id column")
+ parser.add_argument("--offset", type=int, default=0, help="Skip this many rows of the CSV")
+ parser.add_argument(
+ "--failed-csv",
+ type=str,
+ default=None,
+ help="Also write failed org_id,reason rows to this path. It can be passed back in as csv_file.",
+ )
+
+ def handle(self, *args: Any, **options: Any) -> None:
with open(options["csv_file"], encoding="utf-8") as f:
- reader = csv.DictReader(f)
-
- entries = list(reader)[options["offset"] :]
- total_count = len(entries)
- completed_count = 0
- failed_count = 0
- failed_org_ids = []
-
- logger.info(f"Total Backfill Entries: {total_count}")
-
- for row in entries:
- org_id = row["org_id"]
- try:
- org = Organization.objects.get(id=org_id)
-
- owner = (
- OrganizationMembership.objects.filter( # type: ignore
- organization=org, level=OrganizationMembership.Level.OWNER
- )
- .first()
- .user
- )
-
- billing_manager = BillingManager(license=None, user=owner)
-
- try:
- billing_manager.update_billing_organization_users(org)
- except Exception as e:
- logger.exception(f"Failed calling update_billing_organization_users: {e}")
- raise
-
- completed_count += 1
- except Exception:
- failed_count += 1
+ org_ids = [row["org_id"] for row in csv.DictReader(f)][options["offset"] :]
+
+ total = len(org_ids)
+ failed_org_ids: list[str] = []
+ self.stdout.write(f"Syncing {total} organizations with billing")
+
+ with ExitStack() as stack:
+ failed_file = (
+ stack.enter_context(open(options["failed_csv"], "w", encoding="utf-8", newline=""))
+ if options["failed_csv"]
+ else None
+ )
+ failed_writer = csv.writer(failed_file) if failed_file else None
+ if failed_writer:
+ failed_writer.writerow(["org_id", "reason"])
+
+ for done, org_id in enumerate(org_ids, start=1):
+ reason = sync_organization(org_id)
+ if reason is not None:
failed_org_ids.append(org_id)
+ self.stderr.write(f"FAILED {org_id}: {reason}")
+ if failed_file and failed_writer:
+ failed_writer.writerow([org_id, reason])
+ failed_file.flush()
- logger.info(f"{completed_count}/{total_count} ({failed_count} failed)")
-
- logger.info(f"Finished backfilling {total_count} organizations")
+ if done % PROGRESS_EVERY == 0:
+ self.stdout.write(f"Progress: {done}/{total} ({len(failed_org_ids)} failed)")
- if failed_org_ids:
- logger.info(f"Failed updating {failed_org_ids}")
+ self.stdout.write(f"Finished: {total - len(failed_org_ids)} ok, {len(failed_org_ids)} failed")
+ if failed_org_ids:
+ self.stdout.write(f"Failed org ids: {', '.join(failed_org_ids)}")
diff --git a/posthog/management/commands/test/test_backfill_billing_organization_users.py b/posthog/management/commands/test/test_backfill_billing_organization_users.py
new file mode 100644
index 000000000000..62e07ef9ba4a
--- /dev/null
+++ b/posthog/management/commands/test/test_backfill_billing_organization_users.py
@@ -0,0 +1,128 @@
+import csv
+import uuid
+import datetime
+import tempfile
+from io import StringIO
+from pathlib import Path
+from typing import cast
+
+from posthog.test.base import BaseTest
+from unittest.mock import MagicMock, patch
+
+from django.core.management import call_command
+from django.db import OperationalError
+
+import requests
+from parameterized import parameterized
+
+from posthog.cloud_utils import TEST_clear_instance_license_cache
+from posthog.models.organization import Organization, OrganizationMembership
+from posthog.models.user import User
+
+from ee.models.license import License, LicenseManager
+
+
+class TestBackfillBillingOrganizationUsers(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ TEST_clear_instance_license_cache()
+ super(LicenseManager, cast(LicenseManager, License.objects)).create(
+ key="license_id::license_secret",
+ plan="enterprise",
+ valid_until=datetime.datetime(2038, 1, 19, 3, 14, 7),
+ )
+ tmp_dir = tempfile.TemporaryDirectory()
+ self.addCleanup(tmp_dir.cleanup)
+ self.tmp_dir = Path(tmp_dir.name)
+
+ def _org_with_owner(self, name: str) -> Organization:
+ organization = Organization.objects.create(name=name)
+ User.objects.create_and_join(
+ organization=organization,
+ email=f"owner-{organization.id}@example.com",
+ password=None,
+ level=OrganizationMembership.Level.OWNER,
+ )
+ return organization
+
+ def _run(self, org_ids: list[str], **options: str | int) -> tuple[str, str]:
+ csv_path = self.tmp_dir / "orgs.csv"
+ csv_path.write_text("org_id\n" + "".join(f"{org_id}\n" for org_id in org_ids))
+ out, err = StringIO(), StringIO()
+ call_command("backfill_billing_organization_users", str(csv_path), stdout=out, stderr=err, **options)
+ return out.getvalue(), err.getvalue()
+
+ @parameterized.expand(
+ [
+ ("ok", 200, None, None),
+ ("unauthorized", 401, None, "billing returned 401"),
+ ("not_found", 404, None, "billing returned 404"),
+ ("server_error", 500, None, "billing returned 500"),
+ ("raised", None, requests.ConnectionError("boom"), "ConnectionError: boom"),
+ ]
+ )
+ @patch("ee.billing.billing_manager.http_session.patch")
+ def test_reports_billing_outcome(self, _name, status_code, side_effect, expected_reason, mock_patch):
+ organization = self._org_with_owner("Synced org")
+ mock_patch.return_value = MagicMock(status_code=status_code, json=MagicMock(return_value={}))
+ mock_patch.side_effect = side_effect
+
+ out, err = self._run([str(organization.id)])
+
+ mock_patch.assert_called_once()
+ assert "Syncing 1 organizations with billing" in out
+ if expected_reason is None:
+ assert err == ""
+ assert "Finished: 1 ok, 0 failed" in out
+ else:
+ assert f"FAILED {organization.id}: {expected_reason}" in err
+ assert "Finished: 0 ok, 1 failed" in out
+ assert f"Failed org ids: {organization.id}" in out
+
+ @patch("posthog.management.commands.backfill_billing_organization_users.PROGRESS_EVERY", 2)
+ @patch("ee.billing.billing_manager.http_session.patch")
+ def test_offset_missing_org_no_owner_and_failed_csv(self, mock_patch):
+ mock_patch.return_value = MagicMock(status_code=200, json=MagicMock(return_value={}))
+ skipped = self._org_with_owner("Skipped by offset")
+ synced = self._org_with_owner("Synced")
+ ownerless = Organization.objects.create(name="No owner")
+ missing_id = str(uuid.uuid4())
+ failed_csv = self.tmp_dir / "failed.csv"
+
+ out, err = self._run(
+ [str(skipped.id), str(synced.id), missing_id, str(ownerless.id)],
+ offset=1,
+ failed_csv=str(failed_csv),
+ )
+
+ mock_patch.assert_called_once()
+ assert mock_patch.call_args.kwargs["json"]["org_customer_email"] == f"owner-{synced.id}@example.com"
+ assert "Syncing 3 organizations with billing" in out
+ assert f"FAILED {missing_id}: organization not found" in err
+ assert f"FAILED {ownerless.id}: organization has no owner" in err
+ assert "Progress: 2/3 (1 failed)" in out
+ assert "Finished: 1 ok, 2 failed" in out
+ assert f"Failed org ids: {missing_id}, {ownerless.id}" in out
+ with failed_csv.open() as f:
+ assert list(csv.reader(f)) == [
+ ["org_id", "reason"],
+ [missing_id, "organization not found"],
+ [str(ownerless.id), "organization has no owner"],
+ ]
+
+ @patch("ee.billing.billing_manager.http_session.patch")
+ def test_database_error_fails_the_org_and_continues(self, mock_patch):
+ mock_patch.return_value = MagicMock(status_code=200, json=MagicMock(return_value={}))
+ first = self._org_with_owner("Lookup fails")
+ second = self._org_with_owner("Synced")
+ second_owner = User.objects.get(email=f"owner-{second.id}@example.com")
+
+ with patch(
+ "posthog.management.commands.backfill_billing_organization_users.get_most_recent_owner",
+ side_effect=[OperationalError("statement timeout"), second_owner],
+ ):
+ out, err = self._run([str(first.id), str(second.id)])
+
+ assert f"FAILED {first.id}: OperationalError: statement timeout" in err
+ mock_patch.assert_called_once()
+ assert "Finished: 1 ok, 1 failed" in out