:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import datetime
from contextlib import nullcontext
from typing import Any, cast
from uuid import uuid4
import time_machine
from posthog.test.base import BaseTest, FuzzyInt, _create_event, flush_persons_and_events
from unittest.mock import patch
from django.apps import apps
from django.db import DatabaseError
from django.test import SimpleTestCase, override_settings
from django.utils import timezone
from django.utils.timezone import now
from dateutil.relativedelta import relativedelta
from parameterized import parameterized
from posthog.api.test.test_team import create_team
from posthog.constants import FlagRequestType
from posthog.models.organization import Organization
from posthog.models.team.team import Team
from posthog.redis import get_client
from posthog.test.fixtures import create_app_metric2
from posthog.usage_counters import UsageCounterService
from products.ai_observability.backend.models.offline_evaluations import OfflineEvaluationUsageLedger
from ee.billing.quota_limiting import (
BILLING_STATUS_STAMP_KEYS,
INFORMATIONAL_USAGE_RESOURCES,
QUOTA_LIMIT_DATA_RETENTION_FLAG,
YESTERDAY_FINISHED_KEY_PREFIX,
OrganizationUsageInfo,
QuotaLimitingCaches,
QuotaLimitingRunResult,
QuotaResource,
UsageCounters,
_identify_refresh_candidates,
_patch_todays_usage,
add_limited_team_tokens,
get_team_attribute_by_quota_resource,
list_limited_team_attributes,
org_quota_limited_until,
refresh_org_self_driving_quota,
remove_limited_team_tokens,
replace_limited_team_tokens,
set_org_usage_summary,
update_all_orgs_billing_quotas,
update_org_billing_quotas,
update_organization_usage_fields,
)
from ee.billing.quota_usage import QuotaUsageBatch
from ee.clickhouse.materialized_columns.columns import materialize
def zero_trust_scores():
return {resource.value: 0 for resource in QuotaResource}
@override_settings(CLOUD_DEPLOYMENT="US") # As PostHog Cloud
class TestQuotaLimiting(BaseTest):
CLASS_DATA_LEVEL_SETUP = False
def setUp(self) -> None:
super().setUp()
self.redis_client = get_client()
self.redis_client.delete(f"@posthog/quota-limits/events")
self.redis_client.delete(f"@posthog/quota-limits/exceptions")
self.redis_client.delete(f"@posthog/quota-limits/recordings")
self.redis_client.delete(f"@posthog/quota-limits/rows_synced")
self.redis_client.delete(f"@posthog/quota-limits/api_queries_read_bytes")
self.redis_client.delete(f"@posthog/quota-limits/survey_responses")
self.redis_client.delete(f"@posthog/quota-limits/rows_exported")
self.redis_client.delete(f"@posthog/quota-limits/llm_events")
self.redis_client.delete(f"@posthog/quota-limits/mcp_events")
self.redis_client.delete(f"@posthog/quota-limits/cdp_trigger_events")
self.redis_client.delete(f"@posthog/quota-limits/ai_credits")
self.redis_client.delete(f"@posthog/quota-limits/logs_mb_ingested")
self.redis_client.delete(f"@posthog/quota-limiting-suspended/events")
self.redis_client.delete(f"@posthog/quota-limiting-suspended/exceptions")
self.redis_client.delete(f"@posthog/quota-limiting-suspended/recordings")
self.redis_client.delete(f"@posthog/quota-limiting-suspended/rows_synced")
self.redis_client.delete(f"@posthog/quota-limiting-suspended/api_queries_read_bytes")
self.redis_client.delete(f"@posthog/quota-limiting-suspended/survey_responses")
self.redis_client.delete(f"@posthog/quota-limiting-suspended/rows_exported")
self.redis_client.delete(f"@posthog/quota-limiting-suspended/llm_events")
self.redis_client.delete(f"@posthog/quota-limiting-suspended/mcp_events")
self.redis_client.delete(f"@posthog/quota-limiting-suspended/cdp_trigger_events")
self.redis_client.delete(f"@posthog/quota-limiting-suspended/ai_credits")
for key in self.redis_client.scan_iter(f"{YESTERDAY_FINISHED_KEY_PREFIX}*"):
self.redis_client.delete(key)
materialize("events", "$exception_values")
def _run_quota_limiting(self) -> QuotaLimitingRunResult:
with self.captureOnCommitCallbacks(execute=True):
return update_all_orgs_billing_quotas()
@parameterized.expand([(False,), (True,)])
@time_machine.travel("2026-10-31T23:59:59.999999Z", tick=False)
def test_offline_usage_updates_today_without_recounting_billing_history(self, deleted_team: bool) -> None:
self.organization.usage = {
"period": ["2026-10-01T00:00:00Z", "2026-11-01T00:00:00Z"],
"llm_events": {"usage": 50, "todays_usage": 0, "limit": 1000},
}
self.organization.save(update_fields=["usage"])
for accepted_at, quantity in ((now() - datetime.timedelta(days=1), 50), (now(), 20), (now(), 10)):
OfflineEvaluationUsageLedger.objects.for_team(self.team.id).create(
team_id=self.team.id,
organization_id=self.organization.id,
experiment_id=uuid4(),
accepted_at=accepted_at,
quantity=quantity,
)
if deleted_team:
self.team.delete()
for _ in range(2):
self._run_quota_limiting()
self.organization.refresh_from_db()
self.assertEqual(self.organization.usage["llm_events"]["usage"], 50)
self.assertEqual(self.organization.usage["llm_events"]["todays_usage"], 0 if deleted_team else 30)
@parameterized.expand(
[
("billing_total_lacks_yesterday", 0, "2026-10-30", "2026-10-01T00:00:00Z", True, False),
("no_report_yet_this_period", 0, None, "2026-10-01T00:00:00Z", True, False),
("billing_total_has_yesterday", 0, "2026-10-31", "2026-10-01T00:00:00Z", False, False),
("yesterday_is_in_the_previous_period", 0, "2026-10-30", "2026-11-01T00:00:00Z", False, False),
("first_stamp_after_the_first_run", 1, "2026-10-30", "2026-10-01T00:00:00Z", True, False),
("no_stamp", None, None, "2026-10-01T00:00:00Z", False, False),
("first_run_write_skipped", 0, "2026-10-30", "2026-10-01T00:00:00Z", True, True),
]
)
@patch("posthoganalytics.capture")
@time_machine.travel("2026-11-01T00:30:00Z", tick=False)
def test_quota_run_counts_yesterday_until_billings_total_has_it(
self,
_name: str,
stamped_from_run: int | None,
latest_usage_report_date: str | None,
period_start: str,
expect_limited: bool,
first_write_skipped: bool,
_capture,
) -> None:
self.organization.usage = {
"events": {"usage": 90, "limit": 100, "todays_usage": 0},
"period": [period_start, "2026-12-01T00:00:00Z"],
}
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
for timestamp, count in (("2026-10-31T12:00:00Z", 15), ("2026-11-01T00:10:00Z", 1)):
for _ in range(count):
_create_event(
distinct_id=str(uuid4()),
event="$event1",
properties={"$lib": "$web"},
timestamp=timestamp,
team=self.team,
)
flush_persons_and_events()
token = self.team.api_token.encode("UTF-8")
def billing_status_lands_first(organization: Organization, *args: Any, **kwargs: Any) -> bool:
Organization.objects.filter(id=organization.id).update(
usage={**(organization.usage or {}), "billing_status_computed_at": "2026-11-01T00:20:00+00:00"}
)
return _patch_todays_usage(organization, *args, **kwargs)
with patch.object(
UsageCounterService, "fetch_report", autospec=True, side_effect=UsageCounterService.fetch_report
) as fetch_report:
for run in range(2):
if run == stamped_from_run:
self.organization.refresh_from_db()
stamped_usage: dict[str, Any] = {
**(self.organization.usage or {}),
"billing_status_computed_at": "2026-10-31T09:00:00+00:00",
"billing_latest_usage_report_date": latest_usage_report_date,
}
self.organization.usage = stamped_usage
self.organization.save()
with (
patch("ee.billing.quota_limiting._patch_todays_usage", side_effect=billing_status_lands_first)
if first_write_skipped and run == 0
else nullcontext()
):
self._run_quota_limiting()
limited = (
expect_limited and stamped_from_run is not None and run >= stamped_from_run + first_write_skipped
)
assert self.redis_client.zrange("@posthog/quota-limits/events", 0, -1) == ([token] if limited else [])
self.organization.refresh_from_db()
assert self.organization.usage["events"]["todays_usage"] == 1
update_org_billing_quotas(self.organization)
assert self.redis_client.zrange("@posthog/quota-limits/events", 0, -1) == ([token] if expect_limited else [])
today, yesterday = datetime.date(2026, 11, 1), datetime.date(2026, 10, 31)
assert [call.args[1].start.date() for call in fetch_report.call_args_list] == (
[today, yesterday, today, yesterday] if first_write_skipped else [today, yesterday, today]
)
@patch("posthoganalytics.capture")
@patch("posthoganalytics.feature_enabled", return_value=True)
@time_machine.travel("2021-01-25T23:59:59Z", tick=False)
def test_quota_limiting_feature_flag_enabled(self, patch_feature_enabled, patch_capture) -> None:
with self.settings(USE_TZ=False):
self.organization.usage = {
"events": {"usage": 99, "limit": 100},
"exceptions": {"usage": 10, "limit": 100},
"recordings": {"usage": 1, "limit": 100},
"rows_synced": {"usage": 5, "limit": 100},
"feature_flag_requests": {"usage": 5, "limit": 100},
"api_queries_read_bytes": {"usage": 10, "limit": 100},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 10, "limit": 100},
}
self.organization.save()
distinct_id = str(uuid4())
# add a bunch of events so that the organization is over the limit
for _ in range(0, 10):
_create_event(
distinct_id=distinct_id,
event="$event1",
properties={"$lib": "$web"},
timestamp=now() - relativedelta(hours=1),
team=self.team,
)
org_id = str(self.organization.id)
flush_persons_and_events()
with (
self.settings(USAGE_COUNTER_REALTIME_MODES="events:realtime"),
patch("posthog.tasks.usage_report.get_usage_records_in_period") as records,
):
result = self._run_quota_limiting()
records.assert_not_called()
# feature_enabled will be called for AI billing check and then for data retention flag
patch_feature_enabled.assert_any_call(
QUOTA_LIMIT_DATA_RETENTION_FLAG,
str(self.organization.id),
groups={"organization": org_id},
group_properties={"organization": {"id": str(org_id)}},
)
# Check out many times it was called
assert patch_capture.call_count == 1 # 1 org event from org_quota_limited_until
# Find the org action call
org_action_call = next(
call for call in patch_capture.call_args_list if call.kwargs.get("event") == "org_quota_limited_until"
)
assert org_action_call.kwargs.get("properties") == {
"event": "ignored",
"current_usage": 109,
"resource": "events",
"feature_flag": QUOTA_LIMIT_DATA_RETENTION_FLAG,
}
assert org_action_call.kwargs.get("groups") == {
"instance": "http://localhost:8010",
"organization": org_id,
}
# Feature flag is enabled so they won't be limited.
assert result.quota_limited_orgs["events"] == {}
assert result.quota_limited_orgs["exceptions"] == {}
assert result.quota_limited_orgs["recordings"] == {}
assert result.quota_limited_orgs["rows_synced"] == {}
assert result.quota_limited_orgs["feature_flag_requests"] == {}
assert result.quota_limited_orgs["api_queries_read_bytes"] == {}
assert result.quota_limited_orgs["survey_responses"] == {}
assert result.quota_limited_orgs["rows_exported"] == {}
assert result.quota_limiting_suspended_orgs["events"] == {}
assert result.quota_limiting_suspended_orgs["exceptions"] == {}
assert result.quota_limiting_suspended_orgs["recordings"] == {}
assert result.quota_limiting_suspended_orgs["rows_synced"] == {}
assert result.quota_limiting_suspended_orgs["feature_flag_requests"] == {}
assert result.quota_limiting_suspended_orgs["api_queries_read_bytes"] == {}
assert result.quota_limiting_suspended_orgs["survey_responses"] == {}
assert result.quota_limiting_suspended_orgs["rows_exported"] == {}
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/exceptions", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/recordings", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/rows_synced", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/feature_flag_requests", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/api_queries_read_bytes", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/survey_responses", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/rows_exported", 0, -1) == []
patch_capture.reset_mock()
# Add this org to the redis cache.
team_tokens = get_team_attribute_by_quota_resource(self.organization)
add_limited_team_tokens(
QuotaResource.EVENTS,
dict.fromkeys(team_tokens, 1612137599),
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
result = self._run_quota_limiting()
# Check out many times it was called
assert patch_capture.call_count == 1 # 1 org event from org_quota_limited_until
# Find the org action call
org_action_call = next(
call for call in patch_capture.call_args_list if call.kwargs.get("event") == "org_quota_limited_until"
)
assert org_action_call.kwargs.get("properties") == {
"event": "already limited",
"current_usage": 109,
"resource": "events",
"quota_limited_until": 1612137599,
"quota_limiting_suspended_until": None,
}
assert org_action_call.kwargs.get("groups") == {
"instance": "http://localhost:8010",
"organization": org_id,
}
# Feature flag is on but we only suspend limiting for orgs that were not previously limited. This org should still be in the set.
# NOTE on the asserted dict: org_id is a variable (see above), not a string key, and the value is the timestamp at which
# quota_limiting should end or quota_limiting_suspension should end.
assert result.quota_limited_orgs["events"] == {org_id: 1612137599}
assert result.quota_limited_orgs["exceptions"] == {}
assert result.quota_limited_orgs["recordings"] == {}
assert result.quota_limited_orgs["rows_synced"] == {}
assert result.quota_limited_orgs["feature_flag_requests"] == {}
assert result.quota_limited_orgs["api_queries_read_bytes"] == {}
assert result.quota_limited_orgs["survey_responses"] == {}
assert result.quota_limited_orgs["rows_exported"] == {}
assert result.quota_limiting_suspended_orgs["events"] == {}
assert result.quota_limiting_suspended_orgs["exceptions"] == {}
assert result.quota_limiting_suspended_orgs["recordings"] == {}
assert result.quota_limiting_suspended_orgs["rows_synced"] == {}
assert result.quota_limiting_suspended_orgs["feature_flag_requests"] == {}
assert result.quota_limiting_suspended_orgs["api_queries_read_bytes"] == {}
assert result.quota_limiting_suspended_orgs["survey_responses"] == {}
assert result.quota_limiting_suspended_orgs["rows_exported"] == {}
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == [self.team.api_token.encode("UTF-8")]
assert self.redis_client.zrange(f"@posthog/quota-limits/exceptions", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/recordings", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/rows_synced", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/feature_flag_requests", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/api_queries_read_bytes", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/survey_responses", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/rows_exported", 0, -1) == []
@patch("posthoganalytics.capture")
def test_quota_limit_feature_flag_not_on(self, patch_capture) -> None:
# Confirm that we don't send an event if they weren't going to be limited.
self.organization.usage = {
"events": {"usage": 99, "limit": 100, "todays_usage": 0},
"exceptions": {"usage": 10, "limit": 100, "todays_usage": 0},
"recordings": {"usage": 1, "limit": 100, "todays_usage": 0},
"rows_synced": {"usage": 5, "limit": 100, "todays_usage": 0},
"feature_flag_requests": {"usage": 5, "limit": 100, "todays_usage": 0},
"api_queries_read_bytes": {"usage": 1000, "limit": 1000000, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 10, "limit": 100, "todays_usage": 0},
}
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
flush_persons_and_events()
with self.assertNumQueries(FuzzyInt(4, 20)):
result = self._run_quota_limiting()
assert patch_capture.call_count == 0 # No events should be captured since org won't be limited
assert result.quota_limited_orgs["events"] == {}
assert result.quota_limited_orgs["exceptions"] == {}
assert result.quota_limited_orgs["recordings"] == {}
assert result.quota_limited_orgs["rows_synced"] == {}
assert result.quota_limited_orgs["feature_flag_requests"] == {}
assert result.quota_limited_orgs["api_queries_read_bytes"] == {}
assert result.quota_limited_orgs["survey_responses"] == {}
assert result.quota_limited_orgs["rows_exported"] == {}
assert result.quota_limiting_suspended_orgs["events"] == {}
assert result.quota_limiting_suspended_orgs["exceptions"] == {}
assert result.quota_limiting_suspended_orgs["recordings"] == {}
assert result.quota_limiting_suspended_orgs["rows_synced"] == {}
assert result.quota_limiting_suspended_orgs["feature_flag_requests"] == {}
assert result.quota_limiting_suspended_orgs["api_queries_read_bytes"] == {}
assert result.quota_limiting_suspended_orgs["survey_responses"] == {}
assert result.quota_limiting_suspended_orgs["rows_exported"] == {}
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/exceptions", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/recordings", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/rows_synced", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/feature_flag_requests", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/api_queries_read_bytes", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/survey_responses", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/rows_exported", 0, -1) == []
def test_billing_rate_limit_not_set_if_missing_org_usage(self) -> None:
with self.settings(USE_TZ=False):
self.organization.usage = {}
self.organization.save()
distinct_id = str(uuid4())
# we add a bunch of events, so that the organization is over the limit, for events only, but not for recordings
# however, since the organization has no usage set, we should not rate limit
for _ in range(0, 10):
_create_event(
distinct_id=distinct_id,
event="$event1",
properties={"$lib": "$web"},
timestamp=now() - relativedelta(hours=1), # current day
team=self.team,
)
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["events"] == {}
assert result.quota_limited_orgs["exceptions"] == {}
assert result.quota_limited_orgs["recordings"] == {}
assert result.quota_limited_orgs["rows_synced"] == {}
assert result.quota_limited_orgs["feature_flag_requests"] == {}
assert result.quota_limited_orgs["survey_responses"] == {}
assert result.quota_limiting_suspended_orgs["events"] == {}
assert result.quota_limiting_suspended_orgs["exceptions"] == {}
assert result.quota_limiting_suspended_orgs["recordings"] == {}
assert result.quota_limiting_suspended_orgs["rows_synced"] == {}
assert result.quota_limiting_suspended_orgs["feature_flag_requests"] == {}
assert result.quota_limiting_suspended_orgs["survey_responses"] == {}
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/exceptions", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/recordings", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/rows_synced", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/feature_flag_requests", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/api_queries_read_bytes", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/survey_responses", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/rows_exported", 0, -1) == []
@patch("posthoganalytics.capture")
def test_todays_usage_reads_every_legacy_row_shape(self, _capture) -> None:
def flag_requests(_begin, _end, request_type):
counts = {
FlagRequestType.DECIDE: 7,
FlagRequestType.LOCAL_EVALUATION: 2,
FlagRequestType.LOCAL_EVALUATION_NOT_MODIFIED: 3,
}
return [(str(self.team.id), counts[request_type])]
with (
self.settings(USE_TZ=False),
time_machine.travel("2021-01-25T00:00:00Z", tick=False),
patch(
"posthog.tasks.usage_report.get_teams_with_feature_flag_requests_count_in_period",
side_effect=flag_requests,
),
patch(
"posthog.tasks.usage_report.get_teams_with_rows_synced_in_period",
return_value=[{"team_id": self.team.id, "total": 5}],
),
patch(
"posthog.tasks.usage_report.get_teams_with_rows_exported_in_period",
return_value=[{"team_id": self.team.id, "total": 3}],
),
):
self.organization.usage = {
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"feature_flag_requests": {"usage": 0, "limit": None, "todays_usage": 0},
"rows_synced": {"usage": 0, "limit": None, "todays_usage": 0},
"rows_exported": {"usage": 0, "limit": None, "todays_usage": 0},
}
self.organization.save()
update_all_orgs_billing_quotas()
self.organization.refresh_from_db()
assert self.organization.usage is not None
assert self.organization.usage["feature_flag_requests"]["todays_usage"] == 7 + 2 * 10 + 3
assert self.organization.usage["rows_synced"]["todays_usage"] == 5
assert self.organization.usage["rows_exported"]["todays_usage"] == 3
@patch("posthoganalytics.capture")
def test_billing_rate_limit(self, patch_capture) -> None:
def create_usage_summary(**kwargs) -> dict[str, Any]:
data: dict[str, Any] = {
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
for resource in QuotaResource:
data[resource.value] = {"limit": 100, "usage": 10, "todays_usage": 0}
data.update(kwargs)
return data
def assert_other_resources_not_limited(limited_orgs, suspended_orgs):
for resource in QuotaResource:
if resource != QuotaResource.EVENTS:
assert limited_orgs[resource.value] == {}
assert suspended_orgs[resource.value] == {}
assert self.redis_client.zrange(f"@posthog/quota-limits/{resource.value}", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limiting-suspended/{resource.value}", 0, -1) == []
with self.settings(USE_TZ=False), time_machine.travel("2021-01-25T00:00:00Z", tick=False):
self.organization.usage = create_usage_summary(
events={"usage": 99, "limit": 100},
)
self.organization.save()
distinct_id = str(uuid4())
# we add a bunch of events, so that the organization is over the limit
# in this case the org has usage limits, so we want to rate limit for events, but not for recordings
for _ in range(0, 10):
_create_event(
distinct_id=distinct_id,
event="$event1",
properties={"$lib": "$web"},
timestamp=now(),
team=self.team,
)
flush_persons_and_events()
result = self._run_quota_limiting()
# Will be immediately rate limited as trust score was unset.
org_id = str(self.organization.id)
assert result.quota_limited_orgs["events"] == {org_id: 1612137599}
assert result.quota_limiting_suspended_orgs["events"] == {}
assert_other_resources_not_limited(result.quota_limited_orgs, result.quota_limiting_suspended_orgs)
# Check out many times it was called
assert (
patch_capture.call_count == 2
) # 1 org_quota_limited_until event + 1 organization quota limits changed event
# Find the org action call
org_action_call = next(
call
for call in patch_capture.call_args_list
if call.kwargs.get("event") == "organization quota limits changed"
)
assert org_action_call.kwargs.get("properties") == {
"quota_limited_events": 1612137599,
"quota_limited_exceptions": None,
"quota_limited_recordings": None,
"quota_limited_api_queries": None,
"quota_limited_rows_synced": None,
"quota_limited_feature_flags": None,
"quota_limited_survey_responses": None,
"quota_limited_llm_events": None,
"quota_limited_mcp_events": None,
"quota_limited_cdp_trigger_events": None,
"quota_limited_rows_exported": None,
}
assert org_action_call.kwargs.get("groups") == {
"instance": "http://localhost:8010",
"organization": org_id,
}
for resource in QuotaResource:
if resource == QuotaResource.EVENTS:
assert self.redis_client.zrange(f"@posthog/quota-limits/{resource.value}", 0, -1) == [
self.team.api_token.encode("UTF-8")
]
else:
assert self.redis_client.zrange(f"@posthog/quota-limits/{resource.value}", 0, -1) == []
self.organization.refresh_from_db()
assert self.organization.usage == create_usage_summary(
events={
"usage": 99,
"limit": 100,
"todays_usage": 10,
"daily_usage": {"2021-01-25": 10},
"quota_limited_until": 1612137599,
}
)
# # Increase the trust score. They are already being limited, so we will not suspend their limiting.
self.organization.customer_trust_scores = {
"events": 7,
"exceptions": 0,
"recordings": 0,
"rows_synced": 0,
"feature_flag_requests": 0,
QuotaResource.API_QUERIES.value: 0,
"survey_responses": 0,
"rows_exported": 0,
}
self.organization.save()
result = self._run_quota_limiting()
assert result.quota_limited_orgs["events"] == {org_id: 1612137599}
assert result.quota_limiting_suspended_orgs["events"] == {}
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == [
self.team.api_token.encode("UTF-8")
]
assert_other_resources_not_limited(result.quota_limited_orgs, result.quota_limiting_suspended_orgs)
# Reset the event limiting set so their limiting will be suspended for 1 day.
self.redis_client.delete(f"@posthog/quota-limits/events")
self.organization.usage = create_usage_summary(
events={"usage": 99, "limit": 100, "todays_usage": 0},
)
self.organization.save()
result = self._run_quota_limiting()
assert result.quota_limited_orgs["events"] == {}
assert result.quota_limiting_suspended_orgs["events"] == {org_id: 1611705600}
assert self.redis_client.zrange(f"@posthog/quota-limiting-suspended/events", 0, -1) == [
self.team.api_token.encode("UTF-8")
]
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == []
assert_other_resources_not_limited(result.quota_limited_orgs, result.quota_limiting_suspended_orgs)
# Check that limiting still suspended 23 hrs later
with time_machine.travel("2021-01-25T23:00:00Z", tick=False):
result = self._run_quota_limiting()
assert result.quota_limited_orgs["events"] == {}
assert result.quota_limiting_suspended_orgs["events"] == {org_id: 1611705600}
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limiting-suspended/events", 0, -1) == [
self.team.api_token.encode("UTF-8")
]
assert_other_resources_not_limited(result.quota_limited_orgs, result.quota_limiting_suspended_orgs)
self.organization.refresh_from_db()
assert self.organization.usage == create_usage_summary(
events={
"usage": 99,
"limit": 100,
"todays_usage": 10,
"daily_usage": {"2021-01-25": 10},
"quota_limiting_suspended_until": 1611705600,
}
)
# Check that org is being limited after suspension expired
with time_machine.travel("2021-01-27T03:00:00Z", tick=False):
self.organization.usage = create_usage_summary(
events={"usage": 109, "limit": 100, "quota_limiting_suspended_until": 1611705600},
)
self.organization.save()
result = self._run_quota_limiting()
assert result.quota_limited_orgs["events"] == {org_id: 1612137599}
assert result.quota_limiting_suspended_orgs["events"] == {}
self.organization.refresh_from_db()
assert_other_resources_not_limited(result.quota_limited_orgs, result.quota_limiting_suspended_orgs)
assert self.organization.usage == create_usage_summary(
events={"usage": 109, "limit": 100, "todays_usage": 0, "quota_limited_until": 1612137599}
)
assert self.redis_client.zrange(f"@posthog/quota-limiting-suspended/events", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == [
self.team.api_token.encode("UTF-8")
]
# Increase trust score. Their quota limiting suspension expiration should not update.
with time_machine.travel("2021-01-25T00:00:00Z", tick=False):
assert self.redis_client.delete(f"@posthog/quota-limits/events")
self.organization.customer_trust_scores = {
"events": 10,
"exceptions": 0,
"recordings": 0,
"rows_synced": 0,
"feature_flag_requests": 0,
QuotaResource.API_QUERIES.value: 0,
"survey_responses": 0,
}
self.organization.usage = create_usage_summary(
events={"usage": 109, "limit": 100, "quota_limiting_suspended_until": 1611705600},
)
self.organization.save()
result = self._run_quota_limiting()
assert_other_resources_not_limited(result.quota_limited_orgs, result.quota_limiting_suspended_orgs)
assert result.quota_limited_orgs["events"] == {}
assert result.quota_limiting_suspended_orgs["events"] == {org_id: 1611705600}
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limiting-suspended/events", 0, -1) == [
self.team.api_token.encode("UTF-8")
]
# Reset, quota limiting should be suspended for 3 days.
self.organization.customer_trust_scores = {
"events": 10,
"exceptions": 0,
"recordings": 0,
"rows_synced": 0,
"feature_flag_requests": 0,
QuotaResource.API_QUERIES.value: 0,
"survey_responses": 0,
}
self.organization.usage = create_usage_summary(
events={"usage": 109, "limit": 100},
)
self.organization.save()
result = self._run_quota_limiting()
assert_other_resources_not_limited(result.quota_limited_orgs, result.quota_limiting_suspended_orgs)
assert result.quota_limited_orgs["events"] == {}
assert result.quota_limiting_suspended_orgs["events"] == {org_id: 1611878400}
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limiting-suspended/events", 0, -1) == [
self.team.api_token.encode("UTF-8")
]
# Decrease the trust score to 0. Quota limiting should immediately take effect.
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.usage = create_usage_summary(
events={"usage": 109, "limit": 100, "quota_limiting_suspended_until": 1611705600},
)
self.organization.save()
result = self._run_quota_limiting()
assert_other_resources_not_limited(result.quota_limited_orgs, result.quota_limiting_suspended_orgs)
assert result.quota_limited_orgs["events"] == {org_id: 1612137599}
assert result.quota_limiting_suspended_orgs["events"] == {}
assert self.redis_client.zrange(f"@posthog/quota-limiting-suspended/events", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == [
self.team.api_token.encode("UTF-8")
]
with time_machine.travel("2021-01-28T00:00:00Z", tick=False):
self.redis_client.delete(f"@posthog/quota-limits/events")
# Quota limiting suspension date set in previous billing period, update to new suspension expiration
self.organization.customer_trust_scores = {
"events": 10,
"exceptions": 0,
"recordings": 0,
"rows_synced": 0,
"feature_flag_requests": 0,
QuotaResource.API_QUERIES.value: 0,
"survey_responses": 0,
}
self.organization.usage = create_usage_summary(
events={"usage": 109, "limit": 100, "quota_limiting_suspended_until": 1611705600},
period=["2021-01-27T00:00:00Z", "2021-01-31T23:59:59Z"],
)
self.organization.save()
result = self._run_quota_limiting()
assert_other_resources_not_limited(result.quota_limited_orgs, result.quota_limiting_suspended_orgs)
assert result.quota_limited_orgs["events"] == {}
assert result.quota_limiting_suspended_orgs["events"] == {org_id: 1612137600}
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limiting-suspended/events", 0, -1) == [
self.team.api_token.encode("UTF-8")
]
def test_set_org_usage_summary_updates_correctly(self):
self.organization.usage = {
"events": {"usage": 99, "limit": 100},
"exceptions": {"usage": 10, "limit": 100},
"recordings": {"usage": 1, "limit": 100},
"rows_synced": {"usage": 5, "limit": 100},
"feature_flag_requests": {"usage": 5, "limit": 100},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 10, "limit": 100},
}
self.organization.save()
new_usage = {
"events": {"usage": 100, "limit": 100},
"exceptions": {"usage": 20, "limit": 100},
"recordings": {"usage": 2, "limit": 100},
"rows_synced": {"usage": 6, "limit": 100},
"feature_flag_requests": {"usage": 6, "limit": 100},
"period": [
"2021-01-01T00:00:00Z",
"2021-01-31T23:59:59Z",
],
"survey_responses": {"usage": 20, "limit": 100},
}
assert set_org_usage_summary(self.organization, new_usage=cast(OrganizationUsageInfo, new_usage))
assert self.organization.usage == {
"events": {"usage": 100, "limit": 100, "todays_usage": 0},
"exceptions": {"usage": 20, "limit": 100, "todays_usage": 0},
"recordings": {"usage": 2, "limit": 100, "todays_usage": 0},
"rows_synced": {"usage": 6, "limit": 100, "todays_usage": 0},
"feature_flag_requests": {"usage": 6, "limit": 100, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 20, "limit": 100, "todays_usage": 0},
}
def test_set_org_usage_summary_does_nothing_if_the_same(self):
self.organization.usage = {
"events": {"usage": 99, "limit": 100, "todays_usage": 10},
"exceptions": {"usage": 10, "limit": 100, "todays_usage": 50},
"recordings": {"usage": 1, "limit": 100, "todays_usage": 11},
"rows_synced": {"usage": 5, "limit": 100, "todays_usage": 11},
"feature_flag_requests": {"usage": 5, "limit": 100, "todays_usage": 11},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 10, "limit": 100, "todays_usage": 50},
}
self.organization.save()
new_usage = {
"events": {"usage": 99, "limit": 100},
"exceptions": {"usage": 10, "limit": 100},
"recordings": {"usage": 1, "limit": 100},
"rows_synced": {"usage": 5, "limit": 100},
"feature_flag_requests": {"usage": 5, "limit": 100},
"period": [
"2021-01-01T00:00:00Z",
"2021-01-31T23:59:59Z",
],
"survey_responses": {"usage": 10, "limit": 100},
}
assert not set_org_usage_summary(self.organization, new_usage=cast(OrganizationUsageInfo, new_usage))
assert self.organization.usage == {
"events": {"usage": 99, "limit": 100, "todays_usage": 10},
"exceptions": {"usage": 10, "limit": 100, "todays_usage": 50},
"recordings": {"usage": 1, "limit": 100, "todays_usage": 11},
"rows_synced": {"usage": 5, "limit": 100, "todays_usage": 11},
"feature_flag_requests": {"usage": 5, "limit": 100, "todays_usage": 11},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 10, "limit": 100, "todays_usage": 50},
}
def test_set_org_usage_summary_updates_todays_usage(self):
self.organization.usage = {
"events": {"usage": 99, "limit": 100, "todays_usage": 10},
"exceptions": {"usage": 10, "limit": 100, "todays_usage": 50},
"recordings": {"usage": 1, "limit": 100, "todays_usage": 11},
"rows_synced": {"usage": 5, "limit": 100, "todays_usage": 11},
"feature_flag_requests": {"usage": 5, "limit": 100, "todays_usage": 11},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 10, "limit": 100, "todays_usage": 50},
}
self.organization.save()
assert set_org_usage_summary(
self.organization,
todays_usage=cast(
UsageCounters,
{
"events": 20,
"exceptions": 51,
"recordings": 21,
"rows_synced": 21,
"feature_flag_requests": 21,
"survey_responses": 21,
},
),
)
assert self.organization.usage == {
"events": {"usage": 99, "limit": 100, "todays_usage": 20},
"exceptions": {"usage": 10, "limit": 100, "todays_usage": 51},
"recordings": {"usage": 1, "limit": 100, "todays_usage": 21},
"rows_synced": {"usage": 5, "limit": 100, "todays_usage": 21},
"feature_flag_requests": {"usage": 5, "limit": 100, "todays_usage": 21},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 10, "limit": 100, "todays_usage": 21},
}
def test_org_quota_limited_until(self):
self.organization.usage = None
self.organization.customer_trust_scores = zero_trust_scores()
previously_quota_limited_team_tokens_events = list_limited_team_attributes(
QuotaResource.EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
previously_quota_limited_team_tokens_exceptions = list_limited_team_attributes(
QuotaResource.EXCEPTIONS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
previously_quota_limited_team_tokens_recordings = list_limited_team_attributes(
QuotaResource.RECORDINGS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
previously_quota_limited_team_tokens_rows_synced = list_limited_team_attributes(
QuotaResource.ROWS_SYNCED, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
previously_quota_limited_team_tokens_surveys = list_limited_team_attributes(
QuotaResource.SURVEY_RESPONSES, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
previously_quota_limited_team_tokens_ai_credits = list_limited_team_attributes(
QuotaResource.AI_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
assert (
org_quota_limited_until(
self.organization, QuotaResource.EVENTS, previously_quota_limited_team_tokens_events
)
is None
)
self.organization.usage = {
"events": {"usage": 99, "limit": 100},
"exceptions": {"usage": 10, "limit": 100},
"recordings": {"usage": 1, "limit": 100},
"rows_synced": {"usage": 99, "limit": 100},
"feature_flag_requests": {"usage": 99, "limit": 100},
"api_queries_read_bytes": {"usage": 99, "limit": 100},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 10, "limit": 100},
"ai_credits": {"usage": 10, "limit": 100},
}
# Not over quota
assert (
org_quota_limited_until(
self.organization, QuotaResource.EVENTS, previously_quota_limited_team_tokens_events
)
is None
)
# Over quota
self.organization.usage["events"]["usage"] = 120
assert org_quota_limited_until(
self.organization, QuotaResource.EVENTS, previously_quota_limited_team_tokens_events
) == {
"quota_limited_until": 1612137599,
"quota_limiting_suspended_until": None,
}
# At quota limit with today's usage
self.organization.usage["events"]["usage"] = 90
self.organization.usage["events"]["todays_usage"] = 10
assert org_quota_limited_until(
self.organization, QuotaResource.EVENTS, previously_quota_limited_team_tokens_events
) == {
"quota_limited_until": 1612137599,
"quota_limiting_suspended_until": None,
}
# No limit clears quota_limited_until
self.organization.usage["events"]["limit"] = None
assert (
org_quota_limited_until(
self.organization, QuotaResource.EVENTS, previously_quota_limited_team_tokens_events
)
is None
)
assert self.organization.usage["events"].get("quota_limited_until") is None
assert self.organization.usage["events"].get("quota_limiting_suspended_until") is None
# Not over quota
self.organization.usage["exceptions"]["usage"] = 99
assert (
org_quota_limited_until(
self.organization, QuotaResource.EXCEPTIONS, previously_quota_limited_team_tokens_exceptions
)
is None
)
# Over quota
self.organization.usage["exceptions"]["usage"] = 101
assert org_quota_limited_until(
self.organization, QuotaResource.EXCEPTIONS, previously_quota_limited_team_tokens_exceptions
) == {
"quota_limited_until": 1612137599,
"quota_limiting_suspended_until": None,
}
# Under limit + buffer
self.organization.usage["recordings"]["usage"] = 1099
assert (
org_quota_limited_until(
self.organization, QuotaResource.RECORDINGS, previously_quota_limited_team_tokens_recordings
)
is None
)
# Over limit + buffer
self.organization.usage["recordings"]["usage"] = 1100
assert org_quota_limited_until(
self.organization, QuotaResource.RECORDINGS, previously_quota_limited_team_tokens_recordings
) == {
"quota_limited_until": 1612137599,
"quota_limiting_suspended_until": None,
}
# Not over quota
assert (
org_quota_limited_until(
self.organization, QuotaResource.ROWS_SYNCED, previously_quota_limited_team_tokens_rows_synced
)
is None
)
# Over quota
self.organization.usage["rows_synced"]["usage"] = 101
assert org_quota_limited_until(
self.organization, QuotaResource.ROWS_SYNCED, previously_quota_limited_team_tokens_rows_synced
) == {
"quota_limited_until": 1612137599,
"quota_limiting_suspended_until": None,
}
# Not over quota
self.organization.usage["survey_responses"]["usage"] = 99
assert (
org_quota_limited_until(
self.organization, QuotaResource.SURVEY_RESPONSES, previously_quota_limited_team_tokens_surveys
)
is None
)
# Over quota
self.organization.usage["survey_responses"]["usage"] = 101
assert org_quota_limited_until(
self.organization, QuotaResource.SURVEY_RESPONSES, previously_quota_limited_team_tokens_surveys
) == {
"quota_limited_until": 1612137599,
"quota_limiting_suspended_until": None,
}
# Not over quota
assert (
org_quota_limited_until(
self.organization, QuotaResource.AI_CREDITS, previously_quota_limited_team_tokens_ai_credits
)
is None
)
# Over quota
self.organization.usage["ai_credits"]["usage"] = 101
assert org_quota_limited_until(
self.organization, QuotaResource.AI_CREDITS, previously_quota_limited_team_tokens_ai_credits
) == {
"quota_limited_until": 1612137599,
"quota_limiting_suspended_until": None,
}
with time_machine.travel("2021-01-25T00:00:00Z", tick=False):
# Different trust scores so different grace periods
self.organization.customer_trust_scores = {
QuotaResource.EVENTS.value: 7,
QuotaResource.EXCEPTIONS.value: 7,
QuotaResource.RECORDINGS.value: 3,
QuotaResource.ROWS_SYNCED.value: 10,
QuotaResource.FEATURE_FLAG_REQUESTS.value: 10,
QuotaResource.API_QUERIES.value: 10,
QuotaResource.SURVEY_RESPONSES.value: 7,
QuotaResource.AI_CREDITS.value: 10,
}
# Update to be over quota on all resources
self.organization.usage = {
"events": {"usage": 101, "limit": 100},
"exceptions": {"usage": 101, "limit": 100},
"recordings": {"usage": 1101, "limit": 100}, # overage buffer of 1000
"rows_synced": {"usage": 101, "limit": 100},
"feature_flag_requests": {"usage": 101, "limit": 100},
"api_queries_read_bytes": {"usage": 101, "limit": 100},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 101, "limit": 100},
"ai_credits": {"usage": 101, "limit": 100},
}
# All resources over quota
assert org_quota_limited_until(
self.organization, QuotaResource.ROWS_SYNCED, previously_quota_limited_team_tokens_rows_synced
) == {
"quota_limited_until": None,
"quota_limiting_suspended_until": 1611878400, # grace period 3 days
}
assert org_quota_limited_until(
self.organization, QuotaResource.EVENTS, previously_quota_limited_team_tokens_events
) == {
"quota_limited_until": None,
"quota_limiting_suspended_until": 1611705600, # grace period 1 day
}
assert org_quota_limited_until(
self.organization, QuotaResource.EXCEPTIONS, previously_quota_limited_team_tokens_exceptions
) == {
"quota_limited_until": None,
"quota_limiting_suspended_until": 1611705600, # grace period 1 day
}
assert org_quota_limited_until(
self.organization, QuotaResource.RECORDINGS, previously_quota_limited_team_tokens_recordings
) == {
"quota_limited_until": 1612137599, # no grace period
"quota_limiting_suspended_until": None,
}
assert org_quota_limited_until(
self.organization, QuotaResource.API_QUERIES, previously_quota_limited_team_tokens_recordings
) == {
"quota_limited_until": None,
"quota_limiting_suspended_until": 1611878400, # grace period 3 days
}
assert org_quota_limited_until(
self.organization, QuotaResource.SURVEY_RESPONSES, previously_quota_limited_team_tokens_surveys
) == {
"quota_limited_until": None,
"quota_limiting_suspended_until": 1611705600, # grace period 1 day
}
# AI credits have NO grace period - immediately limit
assert org_quota_limited_until(
self.organization, QuotaResource.AI_CREDITS, previously_quota_limited_team_tokens_ai_credits
) == {
"quota_limited_until": 1612137599,
"quota_limiting_suspended_until": None,
}
def test_over_quota_but_not_dropped_org(self):
self.organization.usage = None
previously_quota_limited_team_tokens_events = list_limited_team_attributes(
QuotaResource.EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
previously_quota_limited_team_tokens_exceptions = list_limited_team_attributes(
QuotaResource.EXCEPTIONS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
previously_quota_limited_team_tokens_recordings = list_limited_team_attributes(
QuotaResource.RECORDINGS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
previously_quota_limited_team_tokens_rows_synced = list_limited_team_attributes(
QuotaResource.ROWS_SYNCED, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
previously_quota_limited_team_tokens_surveys = list_limited_team_attributes(
QuotaResource.SURVEY_RESPONSES, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
previously_quota_limited_team_tokens_ai_credits = list_limited_team_attributes(
QuotaResource.AI_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
assert (
org_quota_limited_until(
self.organization, QuotaResource.EVENTS, previously_quota_limited_team_tokens_events
)
is None
)
self.organization.usage = {
"events": {"usage": 100, "limit": 90},
"exceptions": {"usage": 100, "limit": 90},
"recordings": {"usage": 100, "limit": 90},
"rows_synced": {"usage": 100, "limit": 90},
"feature_flag_requests": {"usage": 100, "limit": 90},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 10, "limit": 100},
"ai_credits": {"usage": 100, "limit": 90},
}
self.organization.never_drop_data = True
assert (
org_quota_limited_until(
self.organization, QuotaResource.EVENTS, previously_quota_limited_team_tokens_events
)
is None
)
assert (
org_quota_limited_until(
self.organization, QuotaResource.EXCEPTIONS, previously_quota_limited_team_tokens_exceptions
)
is None
)
assert (
org_quota_limited_until(
self.organization, QuotaResource.RECORDINGS, previously_quota_limited_team_tokens_recordings
)
is None
)
assert (
org_quota_limited_until(
self.organization, QuotaResource.ROWS_SYNCED, previously_quota_limited_team_tokens_rows_synced
)
is None
)
assert (
org_quota_limited_until(
self.organization, QuotaResource.SURVEY_RESPONSES, previously_quota_limited_team_tokens_surveys
)
is None
)
# AI credits ignore never_drop_data - should still be limited
assert org_quota_limited_until(
self.organization, QuotaResource.AI_CREDITS, previously_quota_limited_team_tokens_ai_credits
) == {
"quota_limited_until": 1612137599,
"quota_limiting_suspended_until": None,
}
# reset for subsequent tests
self.organization.never_drop_data = False
def test_update_org_billing_quotas_invalidates_llm_gateway_quota_cache(self) -> None:
gateway_redis_url = "redis://llm-gateway-redis-test/"
# Per-resource entries carry the gateway's credential-fingerprint suffix; the
# billing bit is keyed per team (see _redis_key/_billing_key in quota_resolver).
billing_key = f"quota:code_usage_billing:team:{self.team.id}"
generation_key = f"quota:generation:team:{self.team.id}"
other_generation_key = f"quota:generation:team:{self.team.id + 1}"
with self.settings(LLM_GATEWAY_REDIS_URL=gateway_redis_url):
gateway_redis = get_client(gateway_redis_url)
gateway_redis.set(billing_key, "stale")
gateway_redis.set(other_generation_key, 4)
# Seed the central Redis too: eviction must target the gateway's own
# instance, not the default client (which would silently no-op in prod).
self.redis_client.set(billing_key, "central")
self.organization.usage = {
"events": {"usage": 1, "limit": 100},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
update_org_billing_quotas(self.organization)
assert gateway_redis.get(billing_key) is None
assert gateway_redis.get(generation_key) == b"1"
assert gateway_redis.get(other_generation_key) == b"4"
assert self.redis_client.get(billing_key) == b"central"
self.redis_client.delete(billing_key)
def test_update_org_billing_quotas(self):
with time_machine.travel("2021-01-01T12:59:59Z", tick=False):
other_team = create_team(organization=self.organization)
now = timezone.now().timestamp()
replace_limited_team_tokens(
QuotaResource.EVENTS,
{"1234": int(now + 10000)},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
replace_limited_team_tokens(
QuotaResource.EXCEPTIONS,
{"5678": int(now + 10000)},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
replace_limited_team_tokens(
QuotaResource.ROWS_SYNCED,
{"1337": int(now + 10000)},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
replace_limited_team_tokens(
QuotaResource.SURVEY_RESPONSES,
{"5678": int(now + 10000)},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
self.organization.usage = {
"events": {"usage": 99, "limit": 100},
"exceptions": {"usage": 10, "limit": 100},
"recordings": {"usage": 1, "limit": 100},
"rows_synced": {"usage": 35, "limit": 100},
"feature_flag_requests": {"usage": 5, "limit": 100},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 10, "limit": 100},
}
update_org_billing_quotas(self.organization)
assert list_limited_team_attributes(QuotaResource.EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY) == [
"1234"
]
assert list_limited_team_attributes(
QuotaResource.EXCEPTIONS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
) == ["5678"]
assert list_limited_team_attributes(
QuotaResource.ROWS_SYNCED, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
) == ["1337"]
assert list_limited_team_attributes(
QuotaResource.SURVEY_RESPONSES, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
) == ["5678"]
self.organization.usage["events"]["usage"] = 120
self.organization.usage["exceptions"]["usage"] = 120
self.organization.usage["rows_synced"]["usage"] = 120
self.organization.usage["survey_responses"]["usage"] = 120
update_org_billing_quotas(self.organization)
assert sorted(
list_limited_team_attributes(QuotaResource.EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == sorted(["1234", self.team.api_token, other_team.api_token])
assert sorted(
list_limited_team_attributes(QuotaResource.EXCEPTIONS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == sorted(["5678", self.team.api_token, other_team.api_token])
# rows_synced uses teams, not tokens
assert sorted(
list_limited_team_attributes(QuotaResource.ROWS_SYNCED, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == sorted(["1337", str(self.team.api_token), str(other_team.api_token)])
assert sorted(
list_limited_team_attributes(
QuotaResource.SURVEY_RESPONSES, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
) == sorted(["5678", self.team.api_token, other_team.api_token])
self.organization.usage["events"]["usage"] = 80
self.organization.usage["exceptions"]["usage"] = 80
self.organization.usage["rows_synced"]["usage"] = 36
self.organization.usage["survey_responses"]["usage"] = 80
update_org_billing_quotas(self.organization)
assert sorted(
list_limited_team_attributes(QuotaResource.EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == sorted(["1234"])
assert sorted(
list_limited_team_attributes(QuotaResource.EXCEPTIONS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == sorted(["5678"])
assert sorted(
list_limited_team_attributes(QuotaResource.ROWS_SYNCED, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == sorted(["1337"])
assert sorted(
list_limited_team_attributes(
QuotaResource.SURVEY_RESPONSES, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
) == sorted(["5678"])
self.organization.customer_trust_scores = {
"events": 10,
"exceptions": 10,
"recordings": 0,
"rows_synced": 7,
"feature_flag_requests": 10,
QuotaResource.API_QUERIES.value: 10,
"survey_responses": 10,
}
self.organization.save()
self.organization.usage["events"]["usage"] = 120
self.organization.usage["exceptions"]["usage"] = 120
self.organization.usage["rows_synced"]["usage"] = 120
self.organization.usage["survey_responses"]["usage"] = 120
update_org_billing_quotas(self.organization)
assert sorted(
list_limited_team_attributes(QuotaResource.EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == sorted(["1234"])
assert sorted(
list_limited_team_attributes(QuotaResource.EVENTS, QuotaLimitingCaches.QUOTA_LIMITING_SUSPENDED_KEY)
) == sorted([self.team.api_token, other_team.api_token])
assert sorted(
list_limited_team_attributes(QuotaResource.EXCEPTIONS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == sorted(["5678"])
assert sorted(
list_limited_team_attributes(QuotaResource.EXCEPTIONS, QuotaLimitingCaches.QUOTA_LIMITING_SUSPENDED_KEY)
) == sorted([str(self.team.api_token), str(other_team.api_token)])
# rows_synced uses teams, not tokens
assert sorted(
list_limited_team_attributes(QuotaResource.ROWS_SYNCED, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == sorted(["1337"])
assert sorted(
list_limited_team_attributes(
QuotaResource.ROWS_SYNCED, QuotaLimitingCaches.QUOTA_LIMITING_SUSPENDED_KEY
)
) == sorted([str(self.team.api_token), str(other_team.api_token)])
assert sorted(
list_limited_team_attributes(
QuotaResource.SURVEY_RESPONSES, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
) == sorted(["5678"])
assert sorted(
list_limited_team_attributes(
QuotaResource.SURVEY_RESPONSES, QuotaLimitingCaches.QUOTA_LIMITING_SUSPENDED_KEY
)
) == sorted([str(self.team.api_token), str(other_team.api_token)])
self.organization.usage["events"]["usage"] = 80
self.organization.usage["exceptions"]["usage"] = 80
self.organization.usage["rows_synced"]["usage"] = 36
self.organization.usage["survey_responses"]["usage"] = 80
update_org_billing_quotas(self.organization)
assert sorted(
list_limited_team_attributes(QuotaResource.EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == sorted(["1234"])
assert sorted(
list_limited_team_attributes(QuotaResource.EXCEPTIONS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == sorted(["5678"])
assert sorted(
list_limited_team_attributes(QuotaResource.ROWS_SYNCED, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == sorted(["1337"])
assert sorted(
list_limited_team_attributes(
QuotaResource.SURVEY_RESPONSES, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
) == sorted(["5678"])
@patch("ee.billing.quota_limiting.capture_exception")
def test_get_team_attribute_by_quota_resource(self, mock_capture):
Team.objects.all().delete()
team1 = Team.objects.create(organization=self.organization, api_token="token1")
team2 = Team.objects.create(organization=self.organization, api_token="token2")
tokens = get_team_attribute_by_quota_resource(self.organization)
self.assertEqual(set(tokens), {"token1", "token2"})
team1.delete()
team2.delete()
Team.objects.create(organization=self.organization, api_token="")
tokens = get_team_attribute_by_quota_resource(self.organization)
self.assertEqual(tokens, [])
mock_capture.assert_called_once()
call_args = mock_capture.call_args
self.assertEqual(str(call_args[0][0]), "quota_limiting: No team tokens found for organization")
self.assertEqual(call_args[0][1], {"organization_id": self.organization.id})
def test_feature_flags_quota_limiting(self):
with self.settings(USE_TZ=False), time_machine.travel("2021-01-25T00:00:00Z", tick=False):
self.organization.usage = {
"events": {"usage": 10, "limit": 100},
"exceptions": {"usage": 10, "limit": 100},
"recordings": {"usage": 10, "limit": 100},
"rows_synced": {"usage": 10, "limit": 100},
"feature_flag_requests": {"usage": 110, "limit": 100},
"api_queries_read_bytes": {"usage": 10, "limit": 100},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 10, "limit": 100},
}
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
# Test that feature flags always get 2-day grace period even with trust score 0
result = update_all_orgs_billing_quotas()
org_id = str(self.organization.id)
assert result.quota_limited_orgs["feature_flag_requests"] == {}
assert result.quota_limiting_suspended_orgs["feature_flag_requests"] == {
org_id: 1611792000
} # 2 day suspension
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limiting-suspended/feature_flag_requests", 0, -1
)
# Test medium trust score (7) - should still get 2 day suspension due to special case
self.organization.customer_trust_scores["feature_flag_requests"] = 7
self.organization.usage["feature_flag_requests"] = {"usage": 110, "limit": 100}
self.organization.save()
self.redis_client.delete(f"@posthog/quota-limiting-suspended/feature_flag_requests")
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["feature_flag_requests"] == {}
assert result.quota_limiting_suspended_orgs["feature_flag_requests"] == {
org_id: 1611792000
} # 2 day suspension
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limiting-suspended/feature_flag_requests", 0, -1
)
# Test suspension expiry leads to limiting after 2 days
with time_machine.travel("2021-01-28T00:00:00Z", tick=False): # 3 days later
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["feature_flag_requests"] == {org_id: 1612137599}
assert result.quota_limiting_suspended_orgs["feature_flag_requests"] == {}
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limits/feature_flag_requests", 0, -1
)
# Test medium-high trust score (10) - should get 3 day suspension
with time_machine.travel("2021-01-25T00:00:00Z", tick=False):
self.organization.customer_trust_scores["feature_flag_requests"] = 10
self.organization.usage["feature_flag_requests"] = {"usage": 110, "limit": 100}
self.organization.save()
self.redis_client.delete(f"@posthog/quota-limits/feature_flag_requests")
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["feature_flag_requests"] == {}
assert result.quota_limiting_suspended_orgs["feature_flag_requests"] == {
org_id: 1611878400
} # 3 day suspension
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limiting-suspended/feature_flag_requests", 0, -1
)
# Test never_drop_data organization is not limited
self.organization.customer_trust_scores["feature_flag_requests"] = 0
self.organization.never_drop_data = True
self.organization.save()
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["feature_flag_requests"] == {}
assert result.quota_limiting_suspended_orgs["feature_flag_requests"] == {}
assert self.redis_client.zrange(f"@posthog/quota-limits/feature_flag_requests", 0, -1) == []
assert self.redis_client.zrange(f"@posthog/quota-limiting-suspended/feature_flag_requests", 0, -1) == []
def test_feature_flags_always_get_2_day_grace_period(self):
"""Test that feature flags always get at least a 2-day grace period, or their trust score grace period if higher"""
with self.settings(USE_TZ=False), time_machine.travel("2021-01-25T00:00:00Z", tick=False):
self.organization.usage = {
"events": {"usage": 10, "limit": 100},
"exceptions": {"usage": 10, "limit": 100},
"recordings": {"usage": 10, "limit": 100},
"rows_synced": {"usage": 10, "limit": 100},
"feature_flag_requests": {"usage": 110, "limit": 100},
"api_queries_read_bytes": {"usage": 10, "limit": 100},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"survey_responses": {"usage": 10, "limit": 100},
}
org_id = str(self.organization.id)
# Test different trust scores get at least 2-day grace period for feature flags
test_cases = [
(0, 1611792000), # Trust score 0 -> 2 days (1611792000 = 2021-01-27T00:00:00Z)
(3, 1611792000), # Trust score 3 -> 2 days (normally 0, but minimum is 2)
(7, 1611792000), # Trust score 7 -> 2 days (normally 1, but minimum is 2)
(10, 1611878400), # Trust score 10 -> 3 days (normally 3, which is > 2)
(15, 1612051200), # Trust score 15 -> 5 days (normally 5, which is > 2)
]
for trust_score, expected_timestamp in test_cases:
self.organization.customer_trust_scores = {"feature_flag_requests": trust_score}
self.organization.save()
# Clear any existing Redis state
self.redis_client.delete(f"@posthog/quota-limits/feature_flag_requests")
self.redis_client.delete(f"@posthog/quota-limiting-suspended/feature_flag_requests")
result = update_all_orgs_billing_quotas()
# Should get at least 2-day grace period, or more if trust score allows
assert result.quota_limited_orgs["feature_flag_requests"] == {}, (
f"Trust score {trust_score} should not immediately limit"
)
assert result.quota_limiting_suspended_orgs["feature_flag_requests"] == {org_id: expected_timestamp}, (
f"Trust score {trust_score} should get appropriate grace period"
)
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limiting-suspended/feature_flag_requests", 0, -1
)
def test_api_queries_quota_limiting(self):
with self.settings(USE_TZ=False), time_machine.travel("2021-01-25T00:00:00Z", tick=False):
self.organization.usage = {
"events": {"usage": 10, "limit": 100},
"recordings": {"usage": 10, "limit": 100},
"rows_synced": {"usage": 10, "limit": 100},
"feature_flag_requests": {"usage": 10, "limit": 100},
"api_queries_read_bytes": {"usage": 1100, "limit": 1000},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
trust_key = QuotaResource.API_QUERIES.value
self.organization.customer_trust_scores = {
"events": 0,
"recordings": 0,
"rows_synced": 0,
"feature_flag_requests": 0,
trust_key: 0,
}
self.organization.save()
# Test immediate limiting with trust score 0
result = update_all_orgs_billing_quotas()
org_id = str(self.organization.id)
assert result.quota_limited_orgs["api_queries_read_bytes"] == {org_id: 1612137599}
assert result.quota_limiting_suspended_orgs["api_queries_read_bytes"] == {}
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limits/api_queries_read_bytes", 0, -1
)
# Test medium trust score (7) - should get 1 day suspension
self.organization.customer_trust_scores[trust_key] = 7
self.organization.usage["api_queries_read_bytes"] = {"usage": 1100, "limit": 1000}
self.organization.save()
self.redis_client.delete(f"@posthog/quota-limits/api_queries_read_bytes")
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["api_queries_read_bytes"] == {}
assert result.quota_limiting_suspended_orgs["api_queries_read_bytes"] == {
org_id: 1611705600
} # 1 day suspension
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limiting-suspended/api_queries_read_bytes", 0, -1
)
# Test suspension expiry leads to limiting
with time_machine.travel("2021-01-27T00:00:00Z", tick=False): # 2 days later
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["api_queries_read_bytes"] == {org_id: 1612137599}
assert result.quota_limiting_suspended_orgs["api_queries_read_bytes"] == {}
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limits/api_queries_read_bytes", 0, -1
)
# Test medium-high trust score (10) - should get 3 day suspension
with time_machine.travel("2021-01-25T00:00:00Z", tick=False):
self.organization.customer_trust_scores[trust_key] = 10
self.organization.usage["api_queries_read_bytes"] = {"usage": 110, "limit": 100}
self.organization.save()
self.redis_client.delete(f"@posthog/quota-limits/api_queries_read_bytes")
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["api_queries_read_bytes"] == {}
assert result.quota_limiting_suspended_orgs["api_queries_read_bytes"] == {
org_id: 1611878400
} # 3 day suspension
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limiting-suspended/api_queries_read_bytes", 0, -1
)
# Test high trust score (15) - should get 5 day suspension
with time_machine.travel("2021-01-25T00:00:00Z", tick=False):
self.organization.customer_trust_scores[trust_key] = 15
self.organization.usage["api_queries_read_bytes"] = {"usage": 110, "limit": 100}
self.organization.save()
self.redis_client.delete(f"@posthog/quota-limits/api_queries_read_bytes")
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["api_queries_read_bytes"] == {}
assert result.quota_limiting_suspended_orgs["api_queries_read_bytes"] == {
org_id: 1612051200
} # 5 day suspension
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limiting-suspended/api_queries_read_bytes", 0, -1
)
# Test never_drop_data organization is not limited
self.organization.customer_trust_scores[trust_key] = 0
self.organization.never_drop_data = True
self.organization.save()
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["api_queries_read_bytes"] == {}
assert result.quota_limiting_suspended_orgs["api_queries_read_bytes"] == {}
assert self.redis_client.zrange(f"@posthog/quota-limits/api_queries_read_bytes", 0, -1) == []
def test_ai_credits_quota_limiting(self):
"""Test that AI credits have no grace period and immediately limit regardless of trust score."""
with self.settings(USE_TZ=False), time_machine.travel("2021-01-25T00:00:00Z", tick=False):
self.organization.usage = {
"events": {"usage": 10, "limit": 100},
"recordings": {"usage": 10, "limit": 100},
"rows_synced": {"usage": 10, "limit": 100},
"feature_flag_requests": {"usage": 10, "limit": 100},
"ai_credits": {"usage": 1100, "limit": 1000},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
trust_key = QuotaResource.AI_CREDITS.value
org_id = str(self.organization.id)
# Test immediate limiting with trust score 0 - NO grace period
self.organization.customer_trust_scores = {
"events": 0,
"recordings": 0,
"rows_synced": 0,
"feature_flags": 0,
trust_key: 0,
}
self.organization.save()
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["ai_credits"] == {org_id: 1612137599}
assert result.quota_limiting_suspended_orgs["ai_credits"] == {}
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limits/ai_credits", 0, -1
)
# Test medium trust score (7) - should STILL immediately limit, NO grace period
self.organization.customer_trust_scores[trust_key] = 7
self.organization.usage["ai_credits"] = {"usage": 1100, "limit": 1000}
self.organization.save()
self.redis_client.delete(f"@posthog/quota-limits/ai_credits")
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["ai_credits"] == {org_id: 1612137599}
assert result.quota_limiting_suspended_orgs["ai_credits"] == {}
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limits/ai_credits", 0, -1
)
# Test medium-high trust score (10) - should STILL immediately limit, NO grace period
self.organization.customer_trust_scores[trust_key] = 10
self.organization.usage["ai_credits"] = {"usage": 1100, "limit": 1000}
self.organization.save()
self.redis_client.delete(f"@posthog/quota-limits/ai_credits")
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["ai_credits"] == {org_id: 1612137599}
assert result.quota_limiting_suspended_orgs["ai_credits"] == {}
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limits/ai_credits", 0, -1
)
# Test high trust score (15) - should STILL immediately limit, NO grace period
self.organization.customer_trust_scores[trust_key] = 15
self.organization.usage["ai_credits"] = {"usage": 1100, "limit": 1000}
self.organization.save()
self.redis_client.delete(f"@posthog/quota-limits/ai_credits")
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["ai_credits"] == {org_id: 1612137599}
assert result.quota_limiting_suspended_orgs["ai_credits"] == {}
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limits/ai_credits", 0, -1
)
# Test that never_drop_data does NOT apply to AI credits - still gets limited
self.organization.customer_trust_scores[trust_key] = 0
self.organization.never_drop_data = True
self.organization.save()
self.redis_client.delete(f"@posthog/quota-limits/ai_credits")
result = update_all_orgs_billing_quotas()
assert result.quota_limited_orgs["ai_credits"] == {org_id: 1612137599}
assert result.quota_limiting_suspended_orgs["ai_credits"] == {}
assert self.team.api_token.encode("UTF-8") in self.redis_client.zrange(
f"@posthog/quota-limits/ai_credits", 0, -1
)
@patch("posthoganalytics.capture")
@time_machine.travel("2021-01-25T00:00:00Z", tick=False)
def test_quota_limited_until_but_not_over_limit(self, mock_capture) -> None:
"""Test that when a customer is not over the limit but has quota_limited_until set, the suspension is removed."""
with self.settings(USE_TZ=False):
# Set up organization with usage below the limit but with quota_limited_until set
self.organization.usage = {
"events": {
"usage": 80,
"limit": 100,
"quota_limited_until": 1612137599, # End of billing period
},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
# Add the team token to the quota-limits list
team_token = self.team.api_token.encode("UTF-8")
self.redis_client.zadd(f"@posthog/quota-limits/events", {team_token: 1612137599})
# Run the quota limiting check
result = self._run_quota_limiting()
# Verify the organization is no longer quota limited
assert "events" in result.quota_limited_orgs
assert "events" in result.quota_limiting_suspended_orgs
assert result.quota_limited_orgs["events"] == {}
assert result.quota_limiting_suspended_orgs["events"] == {}
# Verify the team token was removed from the quota-limits list
assert self.redis_client.zrange(f"@posthog/quota-limits/events", 0, -1) == []
# Verify the organization usage was updated to remove quota_limited_until
self.organization.refresh_from_db()
assert self.organization.usage["events"].get("quota_limited_until") is None
# Find the specific call for org_quota_limited_until with suspension removed
event = None
for call in mock_capture.call_args_list:
if len(call) >= 2 and call[1]["event"] == "org_quota_limited_until":
event = call
break
# Verify the correct event was reported
assert event is not None, "Could not find org_quota_limited_until call with suspension removed"
assert event[1]["properties"]["current_usage"] == 80
assert event[1]["properties"]["resource"] == "events"
assert event[1]["properties"]["quota_limiting_suspended_until"] is None
assert "organization" in event[1]["groups"]
assert event[1]["groups"]["organization"] == str(self.organization.id)
@patch("posthoganalytics.capture")
@time_machine.travel("2021-01-25T23:59:59Z", tick=False)
def test_quota_limiting_surveys(self, mock_capture) -> None:
"""Test that surveys quota limiting works correctly"""
with self.settings(USE_TZ=False):
# Set up usage data with surveys over the limit
self.organization.usage = {
"survey_responses": {"usage": 95, "limit": 100, "todays_usage": 10}, # 105 total, over limit of 100
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.customer_trust_scores = {"survey_responses": 0} # Low trust score
self.organization.save()
# Run quota limiting update
update_org_billing_quotas(self.organization)
# Verify team token was added to quota limited list
limited_tokens = list_limited_team_attributes(
QuotaResource.SURVEY_RESPONSES, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
assert self.team.api_token in limited_tokens
# Verify org usage was updated with quota_limited_until
self.organization.refresh_from_db()
assert self.organization.usage["survey_responses"].get("quota_limited_until") is not None
# Verify analytics event was captured
mock_capture.assert_called()
# Check that the correct properties were logged
org_action_call = None
for call in mock_capture.call_args_list:
if len(call) >= 2 and call[1]["event"] == "org_quota_limited_until":
org_action_call = call
break
assert org_action_call is not None
assert org_action_call[1]["properties"]["resource"] == "survey_responses"
assert org_action_call[1]["properties"]["current_usage"] == 105
assert org_action_call[1]["properties"]["event"] == "suspended"
@time_machine.travel("2021-01-25T23:59:59Z", tick=False)
def test_quota_limiting_surveys_under_limit(self) -> None:
"""Test that surveys under quota limit are not restricted"""
with self.settings(USE_TZ=False):
# Set up usage data with surveys under the limit
self.organization.usage = {
"survey_responses": {"usage": 80, "limit": 100, "todays_usage": 5}, # 85 total, under limit
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
# Run quota limiting update
update_org_billing_quotas(self.organization)
# Verify team token was NOT added to quota limited list
limited_tokens = list_limited_team_attributes(
QuotaResource.SURVEY_RESPONSES, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
assert self.team.api_token not in limited_tokens
# Verify org usage does not have quota_limited_until
self.organization.refresh_from_db()
assert self.organization.usage["survey_responses"].get("quota_limited_until") is None
@time_machine.travel("2021-01-25T23:59:59Z", tick=False)
@patch("posthoganalytics.capture")
def test_quota_limiting_ai_events(self, mock_capture) -> None:
"""Test that AI events are properly limited when quota exceeded"""
with self.settings(USE_TZ=False):
# Set up usage data with AI events over the limit
self.organization.usage = {
"llm_events": {"usage": 100, "limit": 100, "todays_usage": 5}, # 105 total, over limit
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
# Run quota limiting update
update_org_billing_quotas(self.organization)
# Verify team token was added to quota limited list
limited_tokens = list_limited_team_attributes(
QuotaResource.LLM_EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
assert self.team.api_token in limited_tokens
# Verify org usage was updated with quota_limited_until
self.organization.refresh_from_db()
assert self.organization.usage["llm_events"].get("quota_limited_until") is not None
# Verify analytics event was captured
mock_capture.assert_called()
# Check that the correct properties were logged
org_action_call = None
for call in mock_capture.call_args_list:
if len(call) >= 2 and call[1]["event"] == "org_quota_limited_until":
org_action_call = call
break
assert org_action_call is not None
assert org_action_call[1]["properties"]["resource"] == "llm_events"
assert org_action_call[1]["properties"]["current_usage"] == 105
assert org_action_call[1]["properties"]["event"] == "suspended"
@time_machine.travel("2021-01-25T23:59:59Z", tick=False)
def test_quota_limiting_ai_events_under_limit(self) -> None:
"""Test that AI events under quota limit are not restricted"""
with self.settings(USE_TZ=False):
# Set up usage data with AI events under the limit
self.organization.usage = {
"llm_events": {"usage": 80, "limit": 100, "todays_usage": 5}, # 85 total, under limit
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
# Run quota limiting update
update_org_billing_quotas(self.organization)
# Verify team token was NOT added to quota limited list
limited_tokens = list_limited_team_attributes(
QuotaResource.LLM_EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
assert self.team.api_token not in limited_tokens
# Verify org usage does not have quota_limited_until
self.organization.refresh_from_db()
assert self.organization.usage["llm_events"].get("quota_limited_until") is None
@time_machine.travel("2021-01-25T23:59:59Z", tick=False)
def test_ai_events_trust_score_tracking(self) -> None:
"""Test trust score updates for AI event limiting"""
with self.settings(USE_TZ=False):
# Set up initial trust scores
self.organization.customer_trust_scores = {
"events": 0,
"exceptions": 0,
"recordings": 0,
"rows_synced": 0,
"feature_flag_requests": 0,
"api_queries_read_bytes": 0,
"survey_responses": 0,
"llm_events": 0,
}
self.organization.usage = {
"llm_events": {"usage": 100, "limit": 100, "todays_usage": 5}, # Over limit
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
# First run should limit immediately due to trust score 0
update_org_billing_quotas(self.organization)
limited_tokens = list_limited_team_attributes(
QuotaResource.LLM_EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
assert self.team.api_token in limited_tokens
# Increase trust score
self.organization.customer_trust_scores["llm_events"] = 7
self.organization.save()
# Clear the limiting to test suspension behavior
self.redis_client.delete(f"@posthog/quota-limits/llm_events")
# Also clear the quota_limited_until field to reset state
self.organization.usage["llm_events"]["quota_limited_until"] = None
# Keep over the limit
self.organization.usage["llm_events"]["todays_usage"] = 5
self.organization.save()
# Run again - should be suspended this time
update_org_billing_quotas(self.organization)
# Should be in suspended list, not limited list
limited_tokens = list_limited_team_attributes(
QuotaResource.LLM_EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
assert self.team.api_token not in limited_tokens
suspended_tokens = list_limited_team_attributes(
QuotaResource.LLM_EVENTS, QuotaLimitingCaches.QUOTA_LIMITING_SUSPENDED_KEY
)
assert self.team.api_token in suspended_tokens
@time_machine.travel("2021-01-25T23:59:59Z", tick=False)
def test_ai_events_quota_with_overage_buffer(self) -> None:
"""Test overage buffer behavior for AI events"""
with self.settings(USE_TZ=False):
# AI events have 0 overage buffer according to OVERAGE_BUFFER
self.organization.usage = {
"llm_events": {"usage": 100, "limit": 100, "todays_usage": 0}, # Exactly at limit
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
# Run quota limiting update
update_org_billing_quotas(self.organization)
# Should be limited when exactly at limit (>= comparison with 0 buffer means at limit = over limit)
limited_tokens = list_limited_team_attributes(
QuotaResource.LLM_EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
assert self.team.api_token in limited_tokens
# Reduce usage to be under limit
self.organization.usage["llm_events"]["usage"] = 99
self.organization.save()
update_org_billing_quotas(self.organization)
# Now should not be limited
limited_tokens = list_limited_team_attributes(
QuotaResource.LLM_EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
assert self.team.api_token not in limited_tokens
@time_machine.travel("2021-01-25T23:59:59Z", tick=False)
@patch("posthoganalytics.capture")
def test_ai_events_quota_suspension_and_resume(self, mock_capture) -> None:
"""Test suspension and resume mechanics for AI quotas"""
with self.settings(USE_TZ=False):
# Start with trust score that allows suspension
self.organization.customer_trust_scores = {
"events": 0,
"exceptions": 0,
"recordings": 0,
"rows_synced": 0,
"feature_flag_requests": 0,
"api_queries_read_bytes": 0,
"survey_responses": 0,
"llm_events": 7, # High trust score
}
self.organization.usage = {
"llm_events": {"usage": 100, "limit": 100, "todays_usage": 5}, # Over limit
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
# First run should suspend (trust score 7 gets suspension, not immediate limiting)
update_org_billing_quotas(self.organization)
assert self.team.api_token in list_limited_team_attributes(
QuotaResource.LLM_EVENTS, QuotaLimitingCaches.QUOTA_LIMITING_SUSPENDED_KEY
)
# Now reduce usage below limit to test resume
self.organization.usage["llm_events"]["todays_usage"] = 0
self.organization.usage["llm_events"]["usage"] = 90 # Under limit
self.organization.save()
# Run again - should clear suspension since under limit
update_org_billing_quotas(self.organization)
assert self.team.api_token not in list_limited_team_attributes(
QuotaResource.LLM_EVENTS, QuotaLimitingCaches.QUOTA_LIMITING_SUSPENDED_KEY
)
assert self.team.api_token not in list_limited_team_attributes(
QuotaResource.LLM_EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
# Verify analytics events were captured
events_captured = [call[1]["event"] for call in mock_capture.call_args_list if len(call) >= 2]
assert "org_quota_limited_until" in events_captured # Should have suspension and removal events
@patch("posthoganalytics.capture")
def test_logs_and_traces_count_against_one_shared_quota(self, patch_capture) -> None:
with self.settings(USE_TZ=False), time_machine.travel("2021-01-25T00:00:00Z", tick=False):
self.organization.usage = {
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"logs_mb_ingested": {"usage": 0, "limit": 2, "todays_usage": 0},
}
self.organization.save()
# Each signal stays under the shared limit alone. Together they exceed it.
for app_source in ("logs", "traces"):
create_app_metric2(
team_id=self.team.id,
app_source=app_source,
metric_name="bytes_ingested",
count=1_500_000,
timestamp=now(),
)
result = update_all_orgs_billing_quotas()
org_id = str(self.organization.id)
assert result.quota_limited_orgs["logs_mb_ingested"] == {org_id: 1612137599}
assert self.redis_client.zrange("@posthog/quota-limits/logs_mb_ingested", 0, -1) == [
self.team.api_token.encode("UTF-8")
]
@parameterized.expand(
[
("over_the_mcp_limit", {"usage": 0, "limit": 2, "todays_usage": 0}, True),
("no_mcp_limit_from_billing", None, False),
]
)
@patch("posthoganalytics.capture")
def test_mcp_events_count_against_the_mcp_events_quota(
self, _name: str, mcp_events_usage: dict[str, int] | None, expect_limited: bool, patch_capture
) -> None:
with self.settings(USE_TZ=False), time_machine.travel("2021-01-25T00:00:00Z", tick=False):
self.organization.usage = {"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"]}
if mcp_events_usage is not None:
self.organization.usage["mcp_events"] = mcp_events_usage
self.organization.save()
for _ in range(3):
_create_event(event="$mcp_tool_call", team=self.team, distinct_id="mcp_user", timestamp=now())
flush_persons_and_events()
result = update_all_orgs_billing_quotas()
org_id = str(self.organization.id)
limited_tokens = [self.team.api_token.encode("UTF-8")] if expect_limited else []
assert result.quota_limited_orgs["mcp_events"] == ({org_id: 1612137599} if expect_limited else {})
assert self.redis_client.zrange("@posthog/quota-limits/mcp_events", 0, -1) == limited_tokens
@patch("posthoganalytics.capture")
def test_traces_alone_can_exceed_the_shared_quota(self, patch_capture) -> None:
with self.settings(USE_TZ=False), time_machine.travel("2021-01-25T00:00:00Z", tick=False):
self.organization.usage = {
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
"logs_mb_ingested": {"usage": 0, "limit": 2, "todays_usage": 0},
}
self.organization.save()
create_app_metric2(
team_id=self.team.id,
app_source="traces",
metric_name="bytes_ingested",
count=2_500_000,
timestamp=now(),
)
result = update_all_orgs_billing_quotas()
org_id = str(self.organization.id)
assert result.quota_limited_orgs["logs_mb_ingested"] == {org_id: 1612137599}
assert self.redis_client.zrange("@posthog/quota-limits/logs_mb_ingested", 0, -1) == [
self.team.api_token.encode("UTF-8")
]
def test_usage_keys_stay_in_sync(self):
"""
Ensure QuotaResource, UsageCounters, and OrganizationUsageInfo all use the same keys
(except for `period` and the billing status stamps).
"""
from posthog.models.organization import OrganizationUsageInfo
org_usage_keys = set(OrganizationUsageInfo.__annotations__.keys()) - {"period", *BILLING_STATUS_STAMP_KEYS}
quota_resource_keys = {resource.value for resource in QuotaResource}
usage_counter_keys = set(UsageCounters.__annotations__.keys())
# Check QuotaResource matches OrganizationUsageInfo, except the informational
# component fields, which are deliberately never quota-enforced
informational_keys = set(INFORMATIONAL_USAGE_RESOURCES)
assert informational_keys <= org_usage_keys
assert not informational_keys & quota_resource_keys, "informational resources must not be QuotaResources"
missing_from_quota = org_usage_keys - quota_resource_keys - informational_keys
extra_in_quota = quota_resource_keys - org_usage_keys
assert not missing_from_quota, f"QuotaResource is missing keys from OrganizationUsageInfo: {missing_from_quota}"
assert not extra_in_quota, f"QuotaResource has extra keys not in OrganizationUsageInfo: {extra_in_quota}"
# Check UsageCounters matches OrganizationUsageInfo
missing_from_counters = org_usage_keys - usage_counter_keys
extra_in_counters = usage_counter_keys - org_usage_keys
assert not missing_from_counters, (
f"UsageCounters is missing keys from OrganizationUsageInfo: {missing_from_counters}"
)
assert not extra_in_counters, f"UsageCounters has extra keys not in OrganizationUsageInfo: {extra_in_counters}"
@patch("posthoganalytics.capture")
@time_machine.travel("2021-01-25T12:00:00Z", tick=False)
def test_update_all_orgs_billing_quotas_refreshes_candidate_orgs_before_decision(self, patch_capture) -> None:
"""
End-to-end: an over-limit-looking org is identified as a refresh candidate, so a
concurrent billing webhook that lands after the queries phase but before the org
loop reaches that row is observed by the per-iteration refresh. The decision and
targeted writes that follow must not clobber the fresh `usage` / `limit` /
`period` written by billing.
This is the customer incident shape: free-tier upgrade fires before the cron's
per-org iteration would have written the row.
"""
from posthog.models.organization import Organization
with self.settings(USE_TZ=False):
stale_usage = {
"events": {"usage": 9_999_999, "limit": 10_000, "todays_usage": 0},
"recordings": {"usage": 0, "limit": 10_000, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.usage = stale_usage
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
# Create events so the ClickHouse rollup produces a non-zero todays_usage,
# exercising the targeted patch path. Without events the loop short-circuits
# before any write.
distinct_id = str(uuid4())
for _ in range(5):
_create_event(
distinct_id=distinct_id,
event="$event1",
properties={"$lib": "$web"},
timestamp=now() - relativedelta(hours=1),
team=self.team,
)
flush_persons_and_events()
fresh_usage = {
"events": {"usage": 0, "limit": 10_000_000, "todays_usage": 0},
"recordings": {"usage": 0, "limit": 10_000_000, "todays_usage": 0},
"period": ["2021-01-15T00:00:00Z", "2021-02-14T23:59:59Z"],
}
# Hook the simulated webhook write between the queries phase and the org
# loop. `list_limited_team_attributes` is called once per resource right
# before the candidate-detection pass, so wrapping it lands the write at the
# right moment without patching anything inside the loop itself.
org_id = self.organization.id
simulated_webhook_write_count = {"count": 0}
real_list_limited_team_attributes = list_limited_team_attributes
def list_then_simulate_webhook(*args, **kwargs):
if simulated_webhook_write_count["count"] == 0:
Organization.objects.filter(id=org_id).update(usage=fresh_usage)
simulated_webhook_write_count["count"] += 1
return real_list_limited_team_attributes(*args, **kwargs)
with patch(
"ee.billing.quota_limiting.list_limited_team_attributes",
side_effect=list_then_simulate_webhook,
):
update_all_orgs_billing_quotas()
assert simulated_webhook_write_count["count"] == 1, (
"The simulated webhook hook did not fire — the test no longer covers the "
"queries-phase-to-loop-start window. Re-anchor the hook to a call site "
"between the queries phase and the org loop."
)
final_org = Organization.objects.get(id=org_id)
assert final_org.usage["events"]["usage"] == 0, (
"The per-iteration refresh must observe the webhook's fresh `usage` and "
"the targeted patch must not clobber it back to the stale snapshot "
"loaded at the start of the job (Calmio regression)."
)
assert final_org.usage["events"]["limit"] == 10_000_000, (
"Fresh `limit` from the webhook must survive the targeted patch."
)
assert final_org.usage["period"] == [
"2021-01-15T00:00:00Z",
"2021-02-14T23:59:59Z",
], "Fresh `period` from the webhook must survive the targeted patch."
@patch("posthoganalytics.capture")
@time_machine.travel("2021-01-25T12:00:00Z", tick=False)
def test_update_all_orgs_billing_quotas_targeted_patch_preserves_concurrent_billing_writes(
self, patch_capture
) -> None:
"""
End-to-end: even for orgs that are NOT refresh candidates (under limit, no quota
markers), a billing webhook that lands mid-iteration must survive the per-org
write. The targeted `jsonb_set` patch only touches `usage[resource][todays_usage]`,
so billing-owned `usage` / `limit` / `period` are not clobbered.
"""
from posthog.models.organization import Organization
with self.settings(USE_TZ=False):
cached_snapshot = {
"events": {"usage": 0, "limit": 10_000_000, "todays_usage": 0},
"recordings": {"usage": 0, "limit": 10_000_000, "todays_usage": 0},
"period": ["2021-01-15T00:00:00Z", "2021-02-14T23:59:59Z"],
}
self.organization.usage = cached_snapshot
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
distinct_id = str(uuid4())
for _ in range(5):
_create_event(
distinct_id=distinct_id,
event="$event1",
properties={"$lib": "$web"},
timestamp=now() - relativedelta(hours=1),
team=self.team,
)
flush_persons_and_events()
mid_loop_webhook = {
"events": {"usage": 0, "limit": 50_000_000, "todays_usage": 0},
"recordings": {"usage": 0, "limit": 50_000_000, "todays_usage": 0},
"period": ["2021-01-20T00:00:00Z", "2021-02-19T23:59:59Z"],
}
org_id = self.organization.id
mid_loop_write_count = {"count": 0}
real_patch = _patch_todays_usage
def webhook_before_per_org_patch(organization, *args, **kwargs):
# Land the webhook write between candidate detection (already done) and
# the targeted per-org patch (about to happen). Subsequent calls pass
# through unchanged.
if organization.id == org_id and mid_loop_write_count["count"] == 0:
Organization.objects.filter(id=org_id).update(usage=mid_loop_webhook)
mid_loop_write_count["count"] += 1
return real_patch(organization, *args, **kwargs)
with patch(
"ee.billing.quota_limiting._patch_todays_usage",
side_effect=webhook_before_per_org_patch,
):
update_all_orgs_billing_quotas()
assert mid_loop_write_count["count"] == 1
final_org = Organization.objects.get(id=org_id)
assert final_org.usage["period"] == [
"2021-01-20T00:00:00Z",
"2021-02-19T23:59:59Z",
], "Fresh `period` from the webhook must survive the targeted patch."
assert final_org.usage["events"]["limit"] == 50_000_000, (
"Fresh `limit` from the webhook must survive the targeted patch."
)
assert final_org.usage["events"]["todays_usage"] == 5, (
"Targeted patch must still write the cron's `todays_usage` for the resource."
)
@patch("posthoganalytics.capture")
@time_machine.travel("2021-01-25T12:00:00Z", tick=False)
def test_candidate_refresh_changes_quota_decision(self, patch_capture) -> None:
"""
End-to-end: the per-iteration refresh must actually change the decision when
the DB row has moved. Without the refresh, the cached over-limit snapshot
would cause us to add the team's token to Redis and report the org as
quota-limited. With the refresh, fresh DB state shows the org is comfortably
under the (now much higher) limit, so no Redis token is added.
"""
with self.settings(USE_TZ=False):
stale_usage = {
"events": {"usage": 9_999_999, "limit": 10_000, "todays_usage": 0},
"recordings": {"usage": 0, "limit": 10_000, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.usage = stale_usage
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
distinct_id = str(uuid4())
for _ in range(5):
_create_event(
distinct_id=distinct_id,
event="$event1",
properties={"$lib": "$web"},
timestamp=now() - relativedelta(hours=1),
team=self.team,
)
flush_persons_and_events()
# A billing webhook lands a fresh paid plan with much higher limits before
# the org loop reaches this org.
fresh_usage = {
"events": {"usage": 0, "limit": 10_000_000, "todays_usage": 0},
"recordings": {"usage": 0, "limit": 10_000_000, "todays_usage": 0},
"period": ["2021-01-15T00:00:00Z", "2021-02-14T23:59:59Z"],
}
org_id = self.organization.id
simulated_webhook_write_count = {"count": 0}
real_list_limited_team_attributes = list_limited_team_attributes
def list_then_simulate_webhook(*args, **kwargs):
if simulated_webhook_write_count["count"] == 0:
Organization.objects.filter(id=org_id).update(usage=fresh_usage)
simulated_webhook_write_count["count"] += 1
return real_list_limited_team_attributes(*args, **kwargs)
with patch(
"ee.billing.quota_limiting.list_limited_team_attributes",
side_effect=list_then_simulate_webhook,
):
result = update_all_orgs_billing_quotas()
# The decision flipped from "limit" (cached) to "no limit" (refreshed).
assert str(org_id) not in result.quota_limited_orgs["events"]
assert str(org_id) not in result.quota_limiting_suspended_orgs["events"]
# And no Redis token was added — the team is free to ingest.
assert self.redis_client.zrange("@posthog/quota-limits/events", 0, -1) == []
@patch("posthoganalytics.capture")
@time_machine.travel("2021-01-25T12:00:00Z", tick=False)
def test_limit_decrease_for_non_candidate_uses_stale_snapshot(self, patch_capture) -> None:
"""
Documents the residual race window the targeted refresh does NOT close: a
billing-driven *limit decrease* (e.g. a downgrade) that lands after the
candidate set is computed is not caught this run. The cached `limit` still
looks high, so the org is not flagged as a candidate, no per-iteration refresh
happens, and the decision is made against the stale (high-limit) snapshot. The
next cron run picks it up within ~30 minutes.
This asserts the current contract directly. When a future change closes the
race, this test fails loudly with a meaningful diff and the assertion below
gets flipped in the same PR.
"""
with self.settings(USE_TZ=False):
cached_high_limit = {
"events": {"usage": 0, "limit": 10_000_000, "todays_usage": 0},
"recordings": {"usage": 0, "limit": 10_000_000, "todays_usage": 0},
"period": ["2021-01-15T00:00:00Z", "2021-02-14T23:59:59Z"],
}
self.organization.usage = cached_high_limit
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
distinct_id = str(uuid4())
for _ in range(50):
_create_event(
distinct_id=distinct_id,
event="$event1",
properties={"$lib": "$web"},
timestamp=now() - relativedelta(hours=1),
team=self.team,
)
flush_persons_and_events()
# Mid-loop downgrade lands a much lower limit; the 50 events from
# ClickHouse now exceed it. A candidate-refresh would catch this; a
# non-candidate path doesn't.
mid_loop_downgrade = {
"events": {"usage": 0, "limit": 10, "todays_usage": 0},
"recordings": {"usage": 0, "limit": 10, "todays_usage": 0},
"period": ["2021-01-15T00:00:00Z", "2021-02-14T23:59:59Z"],
}
org_id = self.organization.id
mid_loop_write_count = {"count": 0}
real_patch = _patch_todays_usage
def webhook_before_per_org_patch(organization, *args, **kwargs):
if organization.id == org_id and mid_loop_write_count["count"] == 0:
Organization.objects.filter(id=org_id).update(usage=mid_loop_downgrade)
mid_loop_write_count["count"] += 1
return real_patch(organization, *args, **kwargs)
with patch(
"ee.billing.quota_limiting._patch_todays_usage",
side_effect=webhook_before_per_org_patch,
):
result = update_all_orgs_billing_quotas()
assert mid_loop_write_count["count"] == 1
# Current contract: the same-run downgrade is missed because the org was
# never flagged as a candidate (cached limit was high), so no refresh
# happened and `org_quota_limited_until` ran against the stale snapshot.
# If this assertion ever fails, the residual race has been closed —
# flip it to `in result.quota_limited_orgs["events"]` in the same PR.
assert str(org_id) not in result.quota_limited_orgs["events"]
# `update_team_remote_config` is lazy-imported inside `dispatch_recordings_remote_config_sync`,
# so patches target `posthog.tasks.remote_config.update_team_remote_config` directly.
@parameterized.expand(
[
(
"newly_limited",
None,
{"events": {"usage": 1, "limit": 100}, "recordings": {"usage": 6_000, "limit": 5_000}},
1,
),
(
"active_member_becomes_unlimited",
10_000,
{"events": {"usage": 1, "limit": 100}, "recordings": {"usage": 1, "limit": 5_000}},
1,
),
(
"expired_member_becomes_unlimited",
-10_000,
{"events": {"usage": 1, "limit": 100}, "recordings": {"usage": 1, "limit": 5_000}},
1,
),
(
"still_limited_no_transition",
10_000,
{"events": {"usage": 1, "limit": 100}, "recordings": {"usage": 6_000, "limit": 5_000}},
0,
),
(
"non_recordings_resource_over_limit",
None,
{"events": {"usage": 1_000_000, "limit": 100}, "recordings": {"usage": 0, "limit": 5_000}},
0,
),
]
)
@time_machine.travel("2021-01-25T00:00:00Z", tick=False)
@patch("posthog.tasks.remote_config.update_team_remote_config")
def test_update_org_billing_quotas_recordings_remote_config_dispatch(
self,
_name,
prepopulate_zset_score_offset,
usage_overrides,
expected_dispatch_count,
mock_update_remote_config,
) -> None:
if prepopulate_zset_score_offset is not None:
score = int(timezone.now().timestamp()) + prepopulate_zset_score_offset
replace_limited_team_tokens(
QuotaResource.RECORDINGS,
{self.team.api_token: score},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
self.organization.usage = {
**usage_overrides,
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
update_org_billing_quotas(self.organization)
assert mock_update_remote_config.apply_async.call_count == expected_dispatch_count
if expected_dispatch_count > 0:
mock_update_remote_config.apply_async.assert_called_with(
args=[self.team.id],
kwargs={"bypass_recordings_quota_cache": True},
countdown=35,
)
@time_machine.travel("2021-01-25T00:00:00Z", tick=False)
@patch("posthog.tasks.remote_config.update_team_remote_config")
def test_update_org_billing_quotas_no_teams_no_dispatch(self, mock_update_remote_config) -> None:
"""An org with no teams (or only teams with empty api_tokens) takes the observe-only
branch — no Redis mutations, no dispatch, no crash."""
self.organization.teams.all().delete()
self.organization.usage = {
"events": {"usage": 1, "limit": 100},
"recordings": {"usage": 6_000, "limit": 5_000},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
update_org_billing_quotas(self.organization)
mock_update_remote_config.apply_async.assert_not_called()
@parameterized.expand(
[
("active_member_becomes_unlimited", 10_000, 1, 1, False, 1, False),
("expired_member_becomes_unlimited", -10_000, 1, 1, False, 1, False),
("still_limited_no_dispatch", 10_000, 6_000, 1, False, 0, True),
("non_recordings_over_limit_no_dispatch", None, 1, 1_000_000, False, 0, False),
("dry_run_skips_dispatch_and_mutation", 10_000, 1, 1, True, 0, True),
]
)
@patch("posthog.tasks.remote_config.update_team_remote_config")
@patch("posthoganalytics.capture")
@time_machine.travel("2021-01-25T00:00:00Z", tick=False)
def test_update_all_orgs_billing_quotas_recordings_remote_config_dispatch(
self,
_name,
prepopulate_zset_score_offset,
usage_recordings,
usage_events,
dry_run,
expected_dispatch_count,
expected_token_in_recordings_zset_after,
patch_capture,
mock_update_remote_config,
) -> None:
recordings_zset_key = "@posthog/quota-limits/recordings"
if prepopulate_zset_score_offset is not None:
score = int(timezone.now().timestamp()) + prepopulate_zset_score_offset
replace_limited_team_tokens(
QuotaResource.RECORDINGS,
{self.team.api_token: score},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
with self.settings(USE_TZ=False):
self.organization.usage = {
"events": {"usage": usage_events, "limit": 100, "todays_usage": 0},
"recordings": {"usage": usage_recordings, "limit": 5_000, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
update_all_orgs_billing_quotas(dry_run=dry_run)
assert mock_update_remote_config.apply_async.call_count == expected_dispatch_count
if expected_dispatch_count > 0:
mock_update_remote_config.apply_async.assert_called_with(
args=[self.team.id],
kwargs={"bypass_recordings_quota_cache": True},
countdown=35,
)
members_after = {member.decode("utf-8") for member in self.redis_client.zrange(recordings_zset_key, 0, -1)}
assert (self.team.api_token in members_after) is expected_token_in_recordings_zset_after
@patch("posthog.tasks.remote_config.update_team_remote_config")
def test_update_org_billing_quotas_recordings_limit_removed_clears_usage_marker(
self, mock_update_remote_config
) -> None:
recordings_zset_key = "@posthog/quota-limits/recordings"
score = int(timezone.now().timestamp()) + 10_000
replace_limited_team_tokens(
QuotaResource.RECORDINGS,
{self.team.api_token: score},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
self.organization.usage = {
"events": {"usage": 0, "limit": 100, "todays_usage": 0},
"recordings": {
# Legacy/partial usage blobs can store explicit JSON null counters.
"usage": None,
"limit": None,
"todays_usage": None,
"quota_limited_until": score,
},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
update_org_billing_quotas(self.organization)
assert self.organization.usage["recordings"].get("quota_limited_until") is None
assert self.organization.usage["recordings"].get("quota_limiting_suspended_until") is None
members_after = {member.decode("utf-8") for member in self.redis_client.zrange(recordings_zset_key, 0, -1)}
assert self.team.api_token not in members_after
mock_update_remote_config.apply_async.assert_called_once_with(
args=[self.team.id],
kwargs={"bypass_recordings_quota_cache": True},
countdown=35,
)
@patch("posthog.tasks.remote_config.update_team_remote_config")
def test_update_org_billing_quotas_recordings_free_allocation_keeps_limited_marker(
self, mock_update_remote_config
) -> None:
recordings_zset_key = "@posthog/quota-limits/recordings"
score = int(timezone.now().timestamp()) + 10_000
replace_limited_team_tokens(
QuotaResource.RECORDINGS,
{self.team.api_token: score},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
self.organization.usage = {
"events": {"usage": 0, "limit": 100, "todays_usage": 0},
"recordings": {
"usage": 6_000,
"limit": 5_000,
"todays_usage": 0,
"quota_limited_until": score,
},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
update_org_billing_quotas(self.organization)
assert self.organization.usage["recordings"].get("quota_limited_until") is not None
members_after = {member.decode("utf-8") for member in self.redis_client.zrange(recordings_zset_key, 0, -1)}
assert self.team.api_token in members_after
mock_update_remote_config.apply_async.assert_not_called()
def _full_usage_counters(**overrides: int) -> UsageCounters:
base = UsageCounters(
events=0,
exceptions=0,
recordings=0,
rows_synced=0,
feature_flag_requests=0,
api_queries_read_bytes=0,
survey_responses=0,
llm_events=0,
mcp_events=0,
ai_credits=0,
signals_credits=0,
signals_spend_credits=0,
posthog_code_credits=0,
posthog_code_token_credits=0,
sandbox_compute_credits=0,
sandbox_compute_cpu_millicore_seconds=0,
sandbox_compute_memory_mib_seconds=0,
cdp_trigger_events=0,
rows_exported=0,
workflow_emails=0,
workflow_push=0,
workflow_destinations_dispatched=0,
logs_mb_ingested=0,
replay_vision_credits=0,
)
base.update(overrides) # type: ignore[typeddict-item]
return base
def _empty_previously_limited() -> dict[str, list[str]]:
return {resource.value: [] for resource in QuotaResource}
def _fake_org(usage: dict | None) -> Any:
org = type("FakeOrg", (), {})()
org.id = uuid4()
org.usage = usage
return org
_PERIOD = ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"]
_PATCH_DAY = datetime.date(2021, 1, 15)
@override_settings(CLOUD_DEPLOYMENT="US")
class TestIdentifyRefreshCandidates(SimpleTestCase):
"""Isolated unit coverage for `_identify_refresh_candidates`. The cron-loop
integration tests above also exercise this function but conflate it with the rest
of the run; these cases cover each candidate-selection branch in isolation."""
@parameterized.expand(
[
(
"skips_orgs_with_no_usage",
None,
_full_usage_counters(),
[],
{},
{},
False,
),
(
"skips_orgs_with_usage_but_no_period",
{"events": {"usage": 5, "limit": 10}},
_full_usage_counters(),
[],
{},
{},
False,
),
(
"flags_orgs_with_existing_quota_marker",
{
"events": {"usage": 0, "limit": 1_000, "quota_limited_until": 1_700_000_000},
"period": _PERIOD,
},
_full_usage_counters(),
[],
{},
{},
True,
),
(
"flags_orgs_with_existing_suspension_marker",
{
"events": {"usage": 0, "limit": 1_000, "quota_limiting_suspended_until": 1_700_000_000},
"period": _PERIOD,
},
_full_usage_counters(),
[],
{},
{},
True,
),
(
"flags_orgs_appearing_newly_over_limit",
{
"events": {"usage": 950, "limit": 1_000, "todays_usage": 0},
"period": _PERIOD,
},
# 950 cached + 100 today = 1050 >= 1000 + 0 buffer → candidate
_full_usage_counters(events=100),
[],
{},
{},
True,
),
(
"flags_orgs_over_limit_only_with_yesterdays_count",
{
"events": {"usage": 900, "limit": 1_000, "todays_usage": 0},
"period": _PERIOD,
"billing_latest_usage_report_date": "2021-01-13",
},
_full_usage_counters(events=50),
[],
{},
{},
True,
{"events": 100},
),
(
"skips_under_limit_orgs_without_markers_or_redis",
{
"events": {"usage": 100, "limit": 1_000, "todays_usage": 0},
"period": _PERIOD,
},
_full_usage_counters(events=10),
[],
{},
{},
False,
),
(
"skips_resources_with_no_limit_set",
# `limit=None` means the org has no cap on this resource.
{
"events": {"usage": 999_999, "limit": None, "todays_usage": 0},
"period": _PERIOD,
},
_full_usage_counters(events=999_999),
[],
{},
{},
False,
),
(
"flags_orgs_with_team_token_in_redis_limiter_set",
{
"events": {"usage": 10, "limit": 1_000, "todays_usage": 0},
"period": _PERIOD,
},
_full_usage_counters(),
["phc_xyz"],
{"events": ["phc_xyz"]},
{},
True,
),
(
# Regression: stale entry in the Redis limiter set must flag the org
# even when `usage[resource]` is missing — otherwise we'd never refresh
# and never get the chance to clear the stale entry.
"redis_token_flags_org_with_empty_resource_dict",
{"period": _PERIOD},
_full_usage_counters(),
["phc_stale"],
{"events": ["phc_stale"]},
{},
True,
),
(
# Symmetric regression: a stale entry in the suspension Redis set must
# also flag the org. Without this, a team only present in the suspended
# set (no overage, no usage marker) would never be re-evaluated and the
# suspension entry could persist past its grace period.
"flags_orgs_with_team_token_in_redis_suspended_set",
{
"events": {"usage": 10, "limit": 1_000, "todays_usage": 0},
"period": _PERIOD,
},
_full_usage_counters(),
["phc_susp"],
{},
{"events": ["phc_susp"]},
True,
),
(
"suspended_redis_token_flags_org_with_empty_resource_dict",
{"period": _PERIOD},
_full_usage_counters(),
["phc_susp_stale"],
{},
{"events": ["phc_susp_stale"]},
True,
),
(
# Only `events` is over limit; other resources are well under. The org
# should still be flagged from the single over-limit resource.
"multi_resource_partial_overage_flags_from_one_resource",
{
"events": {"usage": 990, "limit": 1_000, "todays_usage": 0},
"recordings": {"usage": 0, "limit": 100_000, "todays_usage": 0},
"rows_synced": {"usage": 0, "limit": 100_000, "todays_usage": 0},
"period": _PERIOD,
},
_full_usage_counters(events=20, recordings=5, rows_synced=5),
[],
{},
{},
True,
),
(
# Some orgs land with `usage=None` on a resource (legacy/partial init).
# The check must coerce to 0, not raise or flag on falsy comparisons.
"usage_value_none_treated_as_zero",
{
"events": {"usage": None, "limit": 1_000, "todays_usage": 0},
"period": _PERIOD,
},
_full_usage_counters(events=10),
[],
{},
{},
False,
),
(
# Recordings get a 1000-event buffer per `OVERAGE_BUFFER`. Cached usage
# right at the limit is not yet a candidate.
"recordings_overage_buffer_is_respected",
{
"recordings": {"usage": 1_000, "limit": 1_000, "todays_usage": 0},
"period": _PERIOD,
},
# 1000 + 999 = 1999, threshold = 1000 + 1000 = 2000
_full_usage_counters(recordings=999),
[],
{},
{},
False,
),
(
"zero_limit_block_is_not_a_candidate",
{
"posthog_code_credits": {
"usage": 0,
"limit": 0,
"todays_usage": 0,
"quota_limited_until": 1_700_000_000,
},
"period": _PERIOD,
},
_full_usage_counters(),
["phc_blocked"],
{"posthog_code_credits": ["phc_blocked"]},
{},
False,
),
(
"zero_limit_block_does_not_hide_another_resource",
{
"posthog_code_credits": {
"usage": 0,
"limit": 0,
"todays_usage": 0,
"quota_limited_until": 1_700_000_000,
},
"events": {"usage": 0, "limit": 1_000, "quota_limited_until": 1_700_000_000},
"period": _PERIOD,
},
_full_usage_counters(),
[],
{},
{},
True,
),
(
"zero_limit_with_overage_buffer_is_a_candidate",
{
"recordings": {"usage": 0, "limit": 0, "todays_usage": 0, "quota_limited_until": 1_700_000_000},
"period": _PERIOD,
},
_full_usage_counters(),
[],
{},
{},
True,
),
(
"zero_signals_limit_is_a_candidate",
{
"signals_credits": {
"usage": 0,
"limit": 0,
"todays_usage": 0,
"quota_limited_until": 1_700_000_000,
},
"period": _PERIOD,
},
_full_usage_counters(),
[],
{},
{},
True,
),
]
)
def test_candidate_selection(
self,
_name: str,
usage: dict | None,
todays: UsageCounters,
team_tokens: list[str],
previously_limited_overrides: dict[str, list[str]],
previously_suspended_overrides: dict[str, list[str]],
expected_candidate: bool,
finished_yesterday: dict[str, int] | None = None,
) -> None:
org = _fake_org(usage=usage)
org_id = str(org.id)
orgs_by_id = {org_id: org}
todays_usage_report = {org_id: todays}
teams_by_org = {org_id: team_tokens} if team_tokens else {}
previously_limited = _empty_previously_limited()
previously_limited.update(previously_limited_overrides)
previously_suspended = _empty_previously_limited()
previously_suspended.update(previously_suspended_overrides)
candidates = _identify_refresh_candidates(
orgs_by_id,
todays_usage_report,
teams_by_org,
previously_limited,
previously_suspended,
today=_PATCH_DAY,
finished_yesterday_by_org=None if finished_yesterday is None else {org_id: finished_yesterday},
)
assert candidates == ({org_id} if expected_candidate else set())
@override_settings(CLOUD_DEPLOYMENT="US")
class TestPatchTodaysUsage(BaseTest):
"""Isolated unit coverage for `_patch_todays_usage`. Verifies the targeted-patch
semantics that protect billing-owned `usage` / `limit` / `period` from being
clobbered by stale snapshots."""
def test_returns_false_when_organization_usage_is_none(self) -> None:
self.organization.usage = None
self.organization.save()
changed = _patch_todays_usage(
self.organization, _full_usage_counters(events=42), today=_PATCH_DAY, finished_yesterday=None
)
assert changed is False
def test_patches_desktop_components_without_changing_units_or_double_counting(self) -> None:
self.organization.usage = {
"posthog_code_credits": {"usage": 100, "limit": 2_000, "todays_usage": 0},
"posthog_code_token_credits": {"usage": 80, "limit": None, "todays_usage": 0},
"sandbox_compute_credits": {"usage": 20, "limit": None, "todays_usage": 0},
"sandbox_compute_cpu_millicore_seconds": {
"usage": 1_500,
"limit": None,
"todays_usage": 0,
},
"sandbox_compute_memory_mib_seconds": {
"usage": 4_608,
"limit": None,
"todays_usage": 0,
},
"period": _PERIOD,
}
self.organization.save()
todays_usage = _full_usage_counters(
posthog_code_credits=15,
posthog_code_token_credits=12,
sandbox_compute_credits=3,
sandbox_compute_cpu_millicore_seconds=9_876_543_210,
sandbox_compute_memory_mib_seconds=7_654_321_098,
)
assert _patch_todays_usage(self.organization, todays_usage, today=_PATCH_DAY, finished_yesterday=None) is True
assert _patch_todays_usage(self.organization, todays_usage, today=_PATCH_DAY, finished_yesterday=None) is False
self.organization.refresh_from_db()
assert self.organization.usage["posthog_code_credits"]["todays_usage"] == 15
assert self.organization.usage["posthog_code_token_credits"]["todays_usage"] == 12
assert self.organization.usage["sandbox_compute_credits"]["todays_usage"] == 3
assert self.organization.usage["sandbox_compute_cpu_millicore_seconds"]["todays_usage"] == 9_876_543_210
assert self.organization.usage["sandbox_compute_memory_mib_seconds"]["todays_usage"] == 7_654_321_098
def test_missing_billing_components_preserve_the_last_known_breakdown(self) -> None:
existing_component = {"usage": 10, "limit": None, "todays_usage": 3}
self.organization.usage = {
"events": {"usage": 100, "limit": 1_000, "todays_usage": 7},
"sandbox_compute_credits": existing_component,
"posthog_code_token_credits": {
"usage": 20,
"limit": None,
"todays_usage": 4,
"quota_limited_until": 123,
},
"period": _PERIOD,
}
new_usage = cast(
OrganizationUsageInfo,
{
"events": {"usage": 101, "limit": 1_000},
"sandbox_compute_credits": {},
"posthog_code_token_credits": {"usage": 21, "limit": None},
"period": _PERIOD,
},
)
assert set_org_usage_summary(self.organization, new_usage=new_usage) is True
assert self.organization.usage["sandbox_compute_credits"] == existing_component
assert self.organization.usage["posthog_code_token_credits"] == {
"usage": 21,
"limit": None,
"todays_usage": 0,
}
def test_omitted_components_without_prior_data_leave_no_empty_placeholders(self) -> None:
# A billing payload that predates component reporting arrives at an org that has
# never had component data (today's prod state until billing ships components).
# The OrganizationUsageInfo construction defaults each component to {}; those
# placeholders must not be written into the org's usage dict.
self.organization.usage = {
"events": {"usage": 100, "limit": 1_000, "todays_usage": 7},
"period": _PERIOD,
}
new_usage = cast(
OrganizationUsageInfo,
{
"events": {"usage": 101, "limit": 1_000},
"posthog_code_token_credits": {},
"sandbox_compute_credits": {},
"sandbox_compute_cpu_millicore_seconds": {},
"sandbox_compute_memory_mib_seconds": {},
"period": _PERIOD,
},
)
assert set_org_usage_summary(self.organization, new_usage=new_usage) is True
for field in INFORMATIONAL_USAGE_RESOURCES:
assert field not in self.organization.usage, field
def test_returns_false_when_no_resources_to_patch(self) -> None:
# Org has only `period` — no per-resource dicts to patch.
self.organization.usage = {"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"]}
self.organization.save()
changed = _patch_todays_usage(
self.organization, _full_usage_counters(events=42), today=_PATCH_DAY, finished_yesterday=None
)
assert changed is False
@parameterized.expand(
[
("first_run_of_the_day", {"2021-01-14": 9}, {"events": 12}),
("later_run", {"2021-01-13": 3, "2021-01-14": 12, "2021-01-15": 2}, None),
]
)
def test_daily_usage_holds_yesterday_and_today(self, _name, stored, finished_yesterday) -> None:
self.organization.usage = {
"events": {"usage": 100, "limit": 1_000, "todays_usage": 2, "daily_usage": stored},
"period": _PERIOD,
}
self.organization.save()
_patch_todays_usage(
self.organization, _full_usage_counters(events=4), today=_PATCH_DAY, finished_yesterday=finished_yesterday
)
self.organization.refresh_from_db()
assert self.organization.usage["events"]["daily_usage"] == {"2021-01-14": 12, "2021-01-15": 4}
def test_returns_false_when_value_is_unchanged(self) -> None:
self.organization.usage = {
"events": {"usage": 100, "limit": 1_000, "todays_usage": 7, "daily_usage": {"2021-01-15": 7}},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
before_updated_at = self.organization.updated_at
changed = _patch_todays_usage(
self.organization, _full_usage_counters(events=7), today=_PATCH_DAY, finished_yesterday=None
)
assert changed is False
# No SQL UPDATE means `updated_at` should not move.
self.organization.refresh_from_db()
assert self.organization.updated_at == before_updated_at
def test_writes_zero_when_todays_usage_missing_and_report_is_zero(self) -> None:
# Edge case: a freshly-billed resource arrives with no `todays_usage` key at
# all, and ClickHouse reports 0 for the period. `existing.get("todays_usage")`
# returns `None`, which compares unequal to `0`, so the patch must write `0`
# explicitly — initializing the key on the row instead of leaving it absent.
self.organization.usage = {
"events": {"usage": 100, "limit": 1_000}, # no `todays_usage` key
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
changed = _patch_todays_usage(
self.organization, _full_usage_counters(events=0), today=_PATCH_DAY, finished_yesterday=None
)
assert changed is True
self.organization.refresh_from_db()
assert self.organization.usage["events"]["todays_usage"] == 0
assert "daily_usage" not in self.organization.usage["events"]
# Sibling fields untouched.
assert self.organization.usage["events"]["usage"] == 100
assert self.organization.usage["events"]["limit"] == 1_000
def test_partial_patch_writes_only_changed_resources(self) -> None:
self.organization.usage = {
"events": {"usage": 100, "limit": 1_000, "todays_usage": 5},
"recordings": {"usage": 200, "limit": 1_000, "todays_usage": 5},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
before_updated_at = self.organization.updated_at
# Only `events` changes; `recordings` stays identical.
changed = _patch_todays_usage(
self.organization, _full_usage_counters(events=42, recordings=5), today=_PATCH_DAY, finished_yesterday=None
)
assert changed is True
# In-memory mirror reflects the change.
assert self.organization.usage["events"]["todays_usage"] == 42
assert self.organization.usage["recordings"]["todays_usage"] == 5
self.organization.refresh_from_db()
assert self.organization.usage["events"]["todays_usage"] == 42
# Sibling fields untouched.
assert self.organization.usage["events"]["usage"] == 100
assert self.organization.usage["events"]["limit"] == 1_000
assert self.organization.usage["recordings"]["todays_usage"] == 5
assert self.organization.usage["period"] == ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"]
# Matches prior `save(update_fields=["usage"])` behavior: `auto_now=True` only
# fires for fields actually in `update_fields`, so `updated_at` wasn't bumped
# before and isn't bumped now.
assert self.organization.updated_at == before_updated_at
@parameterized.expand([("individual", False), ("batch", True)])
def test_targeted_patch_does_not_clobber_concurrent_billing_write(self, _name, batched_write) -> None:
batch = QuotaUsageBatch() if batched_write else None
self.organization.usage = {
"events": {"usage": 100, "limit": 1_000, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
# Concurrent billing write lands fresh `usage` / `limit` / `period` after the
# in-memory snapshot was loaded.
Organization.objects.filter(id=self.organization.id).update(
usage={
"events": {"usage": 0, "limit": 50_000_000, "todays_usage": 0},
"period": ["2021-02-01T00:00:00Z", "2021-02-28T23:59:59Z"],
}
)
# The stale in-memory org still believes limit=1000; the targeted patch must
# only touch `usage[events][todays_usage]` and leave billing-owned fields alone.
changed = _patch_todays_usage(
self.organization, _full_usage_counters(events=99), today=_PATCH_DAY, finished_yesterday=None, batch=batch
)
if batch is not None:
batch.write()
assert changed is True
fresh = Organization.objects.get(id=self.organization.id)
assert fresh.usage["events"]["todays_usage"] == 99
assert fresh.usage["events"]["usage"] == 0
assert fresh.usage["events"]["limit"] == 50_000_000
assert fresh.usage["period"] == ["2021-02-01T00:00:00Z", "2021-02-28T23:59:59Z"]
@override_settings(CLOUD_DEPLOYMENT="US")
class TestUpdateOrganizationUsageFields(BaseTest):
"""Isolated unit coverage for the partial-write helper that writes the
quota-limiting-owned keys (`quota_limited_until`, `quota_limiting_suspended_until`)
inside `usage[resource]`."""
def test_empty_fields_skips_db_write(self) -> None:
self.organization.usage = {
"events": {"usage": 100, "limit": 1_000, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
before_updated_at = self.organization.updated_at
update_organization_usage_fields(self.organization, QuotaResource.EVENTS, {})
self.organization.refresh_from_db()
assert self.organization.updated_at == before_updated_at
@parameterized.expand([("individual", False), ("batch", True)])
def test_set_and_delete_keys_in_one_call(self, _name, batched_write) -> None:
batch = QuotaUsageBatch() if batched_write else None
self.organization.usage = {
"events": {
"usage": 100,
"limit": 1_000,
"todays_usage": 0,
"quota_limiting_suspended_until": 1_700_000_000,
},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
update_organization_usage_fields(
self.organization,
QuotaResource.EVENTS,
{"quota_limited_until": 1_800_000_000, "quota_limiting_suspended_until": None},
batch=batch,
)
if batch is not None:
batch.write()
# In-memory mirror.
assert self.organization.usage["events"]["quota_limited_until"] == 1_800_000_000
assert "quota_limiting_suspended_until" not in self.organization.usage["events"]
# Persisted state.
self.organization.refresh_from_db()
assert self.organization.usage["events"]["quota_limited_until"] == 1_800_000_000
assert "quota_limiting_suspended_until" not in self.organization.usage["events"]
# Billing-owned fields untouched.
assert self.organization.usage["events"]["usage"] == 100
assert self.organization.usage["events"]["limit"] == 1_000
def test_all_none_deletes_on_missing_keys_is_a_noop(self) -> None:
# Clearing markers that aren't present should not raise and should leave the
# row's `usage` shape untouched (the SQL UPDATE still runs but produces an
# equivalent jsonb). Mirrors the common cron path where `org_quota_limited_until`
# clears markers on every "not over limit" pass even when there were none to
# begin with.
self.organization.usage = {
"events": {"usage": 100, "limit": 1_000, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
before_usage = dict(self.organization.usage["events"])
update_organization_usage_fields(
self.organization,
QuotaResource.EVENTS,
{"quota_limited_until": None, "quota_limiting_suspended_until": None},
)
self.organization.refresh_from_db()
# Same shape — no spurious null keys introduced, billing-owned fields untouched.
assert self.organization.usage["events"] == before_usage
def test_concurrent_billing_period_change_is_preserved(self) -> None:
self.organization.usage = {
"events": {"usage": 100, "limit": 1_000, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.save()
# Billing rewrites `usage` and `period` after we loaded our in-memory copy.
Organization.objects.filter(id=self.organization.id).update(
usage={
"events": {"usage": 0, "limit": 10_000_000, "todays_usage": 0},
"period": ["2021-02-01T00:00:00Z", "2021-02-28T23:59:59Z"],
}
)
update_organization_usage_fields(
self.organization, QuotaResource.EVENTS, {"quota_limited_until": 1_800_000_000}
)
fresh = Organization.objects.get(id=self.organization.id)
assert fresh.usage["events"]["quota_limited_until"] == 1_800_000_000
assert fresh.usage["events"]["usage"] == 0
assert fresh.usage["events"]["limit"] == 10_000_000
assert fresh.usage["period"] == ["2021-02-01T00:00:00Z", "2021-02-28T23:59:59Z"]
class TestSignalsRefundQuotaOffset(BaseTest):
# A free-plan signals org: limit == the 4500-credit free allocation (3 PRs).
def _set_signals_usage(self, usage: int, *, todays_usage: int = 0) -> None:
self.organization.usage = {
"signals_credits": {"usage": usage, "todays_usage": todays_usage, "limit": 4500},
"period": ["2026-06-01T00:00:00Z", "2026-07-01T00:00:00Z"],
}
self.organization.customer_trust_scores = {}
self.organization.save()
def _credited_refund(self, *, pr_run_created_at: datetime.datetime) -> None:
# `ee` may not import `products.signals` (module boundaries), so reach its models via the
# app registry at runtime — same pattern the signals tests use for the tasks product.
SignalReport = apps.get_model("signals", "SignalReport")
SignalReportRefund = apps.get_model("signals", "SignalReportRefund")
report = SignalReport.objects.create(team=self.team, status=SignalReport.Status.SUPPRESSED)
SignalReportRefund.all_teams.create(
team=self.team,
report=report,
reason=SignalReportRefund.Reason.PR_INCORRECT,
billing_path=SignalReportRefund.BillingPath.CREDITED,
credits=1500,
pr_url="https://github.com/x/y/pull/1",
pr_run_created_at=pr_run_created_at,
)
@parameterized.expand(
[
# At the free-tier limit (3 PRs billed), a credited refund of one PR frees the slot.
("in_period_refund_unlimits", datetime.datetime(2026, 6, 10, tzinfo=datetime.UTC), False),
# A refund whose PR run predates the period must not grant this period a free slot.
("out_of_period_refund_keeps_limit", datetime.datetime(2026, 5, 10, tzinfo=datetime.UTC), True),
]
)
@patch("posthoganalytics.capture")
@patch("posthoganalytics.feature_enabled", return_value=False)
@time_machine.travel("2026-06-15T12:00:00Z", tick=False)
def test_credited_refund_offsets_quota_decision(
self, _name, pr_run_created_at, expect_limited, _feature_enabled, _capture
) -> None:
self._set_signals_usage(4500)
self._credited_refund(pr_run_created_at=pr_run_created_at)
result = org_quota_limited_until(self.organization, QuotaResource.SIGNALS_CREDITS, [], [])
if expect_limited:
assert result is not None
assert result["quota_limited_until"] is not None
else:
assert result is None
# The corruption regression: the offset lives only in the boolean decision — the stored
# usage summary (what billing syncs and the UI reads) must stay untouched.
self.organization.refresh_from_db()
stored_usage = self.organization.usage
assert stored_usage is not None
assert stored_usage["signals_credits"]["usage"] == 4500
assert stored_usage["signals_credits"]["todays_usage"] == 0
@patch("posthoganalytics.capture")
@patch("posthoganalytics.feature_enabled", return_value=False)
@time_machine.travel("2026-06-15T12:00:00Z", tick=False)
def test_refund_lookup_skipped_when_under_limit(self, _feature_enabled, _capture) -> None:
# The offset query must not run once per org in the all-orgs cron — only for orgs that
# would otherwise be limited.
self._set_signals_usage(1000)
with patch("ee.billing.quota_limiting.get_signals_credited_refund_credits_for_org") as mock_offset:
result = org_quota_limited_until(self.organization, QuotaResource.SIGNALS_CREDITS, [], [])
assert result is None
mock_offset.assert_not_called()
@patch("posthoganalytics.capture")
@patch("posthoganalytics.feature_enabled", return_value=False)
@time_machine.travel("2026-06-15T12:00:00Z", tick=False)
def test_offset_ignores_other_resources(self, _feature_enabled, _capture) -> None:
# An over-limit events org must never trigger the signals refund lookup.
self.organization.usage = {
"events": {"usage": 200, "todays_usage": 0, "limit": 100},
"period": ["2026-06-01T00:00:00Z", "2026-07-01T00:00:00Z"],
}
self.organization.customer_trust_scores = {}
self.organization.save()
with patch("ee.billing.quota_limiting.get_signals_credited_refund_credits_for_org") as mock_offset:
org_quota_limited_until(self.organization, QuotaResource.EVENTS, [], [])
mock_offset.assert_not_called()
class TestRefreshOrgSelfDrivingQuota(BaseTest):
def _set_self_driving_usage(self, usage: int, *, todays_usage: int = 0, limit: int = 4500) -> None:
self.organization.usage = {
"signals_credits": {"usage": usage, "todays_usage": todays_usage, "limit": limit},
"period": ["2026-06-01T00:00:00Z", "2026-07-01T00:00:00Z"],
}
self.organization.customer_trust_scores = {}
self.organization.save()
@parameterized.expand(
[
("midnight", True, 1500, False, False, True, False),
("midnight_cron", True, 1500, False, False, True, True),
("redis_only", False, 1500, False, False, True, False),
("raised_limit", True, 3000, False, False, False, False),
("removed_limit", True, None, False, False, False, False),
("credited_refund", True, 1500, True, False, False, False),
("new_period", True, 1500, False, True, False, False),
]
)
@time_machine.travel("2026-06-15T00:30:00Z", tick=False)
def test_quota_release_checks_prs_missing_from_billing_usage(
self,
_name: str,
persisted_marker: bool,
limit: int | None,
refunded: bool,
new_period: bool,
expect_limited: bool,
via_cron: bool,
) -> None:
self._set_self_driving_usage(0, limit=1500)
assert self.organization.usage is not None
self.organization.usage["signals_credits"]["limit"] = limit
if persisted_marker:
self.organization.usage["signals_credits"]["quota_limited_until"] = 1782864000
if new_period:
self.organization.usage["period"] = ["2026-06-15T00:00:00Z", "2026-07-15T00:00:00Z"]
self.organization.save()
report = apps.get_model("signals", "SignalReport").objects.create(team=self.team)
task = apps.get_model("tasks", "Task").objects.create(team=self.team, title="Quota fixture")
apps.get_model("signals", "SignalReportTask").objects.create(
team=self.team, report=report, task=task, relationship="implementation"
)
pr_run_at = datetime.datetime(2026, 6, 14, 23, 45, tzinfo=datetime.UTC)
apps.get_model("tasks", "TaskRun").objects.create(
team=self.team,
task=task,
created_at=pr_run_at,
output={"pr_url": "https://github.com/example/quota-fixture/pull/1"},
)
if refunded:
apps.get_model("signals", "SignalReportRefund").objects.for_team(self.team.id).create(
team_id=self.team.id,
report=report,
reason="pr_incorrect",
billing_path="credited",
credits=1500,
pr_url="https://github.com/example/quota-fixture/pull/1",
pr_run_created_at=pr_run_at,
)
add_limited_team_tokens(
QuotaResource.SIGNALS_CREDITS,
{self.team.api_token: 1782864000},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
if via_cron:
update_all_orgs_billing_quotas()
else:
update_org_billing_quotas(self.organization)
assert (
self.team.api_token
in list_limited_team_attributes(QuotaResource.SIGNALS_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
) == expect_limited
self.organization.refresh_from_db()
assert self.organization.usage is not None
assert self.organization.usage["signals_credits"]["usage"] == 0
assert self.organization.usage["signals_credits"]["todays_usage"] == 0
@parameterized.expand(
[
("active", "2026-07-01", "2026-07-01", "2026-07-01"),
("active_earlier_expiry", "2026-06-16", "2026-06-16", "2026-06-16"),
("expired", "2026-06-01", "2026-06-01", None),
("expired_with_active_redis", "2026-06-01", "2026-07-01", "2026-07-01"),
("redis_only", None, "2026-07-01", "2026-07-01"),
]
)
@time_machine.travel("2026-06-15T00:30:00Z", tick=False)
def test_failed_period_recount_preserves_existing_block(
self, _name: str, persisted_expiry: str | None, redis_expiry: str, expected_expiry: str | None
) -> None:
self._set_self_driving_usage(0)
assert self.organization.usage is not None
if persisted_expiry:
self.organization.usage["signals_credits"]["quota_limited_until"] = round(
datetime.datetime.fromisoformat(persisted_expiry).replace(tzinfo=datetime.UTC).timestamp()
)
self.organization.save()
add_limited_team_tokens(
QuotaResource.SIGNALS_CREDITS,
{
self.team.api_token: round(
datetime.datetime.fromisoformat(redis_expiry).replace(tzinfo=datetime.UTC).timestamp()
)
},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
def count_credits(organization_id: str, begin: datetime.datetime, end: datetime.datetime) -> int:
if begin.day == 1:
raise DatabaseError("Period usage unavailable")
return 0
with patch(
"ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_org", side_effect=count_credits
):
update_all_orgs_billing_quotas()
expected_timestamp = (
datetime.datetime.fromisoformat(expected_expiry).replace(tzinfo=datetime.UTC).timestamp()
if expected_expiry
else None
)
assert (
get_client().zscore(
f"{QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY.value}{QuotaResource.SIGNALS_CREDITS.value}",
self.team.api_token,
)
== expected_timestamp
)
@parameterized.expand([("without_stamp", False), ("with_stamp", True)])
@patch("posthoganalytics.capture")
@patch("posthoganalytics.feature_enabled", return_value=False)
@time_machine.travel("2026-06-15T12:00:00Z", tick=False)
def test_refresh_patches_todays_usage_and_flips_the_limiter(
self, _name, stamped, _feature_enabled, _capture
) -> None:
# The event-driven path exists because the cron-patched todays_usage predates the PR that
# just landed: a stale value here means the flag doesn't flip until the next cron tick.
self._set_self_driving_usage(3000, todays_usage=0)
if stamped:
assert self.organization.usage is not None
self.organization.usage["billing_latest_usage_report_date"] = "2026-06-14"
self.organization.usage["signals_credits"]["daily_usage"] = {}
self.organization.save()
with patch(
"ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_org", return_value=1500
) as live_mock:
refresh_org_self_driving_quota(str(self.organization.id))
assert live_mock.call_args.args[0] == self.organization.id
self.organization.refresh_from_db()
assert self.organization.usage is not None
assert self.organization.usage["signals_credits"]["todays_usage"] == 1500
# 3000 recorded + 1500 live == the 4500 limit: the team must now be in the limited zset.
assert self.team.api_token in list_limited_team_attributes(
QuotaResource.SIGNALS_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
@patch("posthoganalytics.capture")
@patch("posthoganalytics.feature_enabled", return_value=False)
@time_machine.travel("2026-06-15T12:00:00Z", tick=False)
def test_stale_concurrent_refresh_cannot_lower_usage_or_unpause(self, _feature_enabled, _capture) -> None:
# Two refreshes can overlap when two PRs land close together; the one that counted usage
# before the newer PR can finish last. Its stale, lower count must not overwrite the
# fresher value and momentarily reopen the gates for an over-limit org.
self._set_self_driving_usage(3000, todays_usage=1500)
replace_limited_team_tokens(
QuotaResource.SIGNALS_CREDITS,
{self.team.api_token: 1750323600},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
with patch("ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_org", return_value=100):
refresh_org_self_driving_quota(str(self.organization.id))
self.organization.refresh_from_db()
assert self.organization.usage is not None
assert self.organization.usage["signals_credits"]["todays_usage"] == 1500
assert self.team.api_token in list_limited_team_attributes(
QuotaResource.SIGNALS_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
@parameterized.expand([("recount_succeeds", False), ("recount_fails", True)])
@patch("posthoganalytics.capture")
@patch("posthoganalytics.feature_enabled", return_value=False)
@time_machine.travel("2026-06-15T12:00:00Z", tick=False)
def test_quota_cron_recounts_live_instead_of_writing_its_stale_snapshot(
self, _name, recount_fails, _feature_enabled, _capture
) -> None:
# The cron snapshots fleet usage before its org loop, so a push-refresh that fires while
# the loop runs (a PR landing) writes a fresher, higher todays_usage than the snapshot.
# Writing the snapshot back would unpause the over-limit org until the next tick; the
# cron must recount candidates live at decision time instead.
self._set_self_driving_usage(3000, todays_usage=1500)
# Score must be in the frozen clock's future: candidate selection reads the zset
# through `list_limited_team_attributes`, which drops expired entries.
replace_limited_team_tokens(
QuotaResource.SIGNALS_CREDITS,
{self.team.api_token: 1798761600},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
with (
patch(
"posthog.tasks.usage_report.get_teams_with_signals_credits_used_in_period",
return_value=[(self.team.id, 100)],
),
patch(
"ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_orgs",
return_value={str(self.organization.id): 1500},
side_effect=DatabaseError("Signals usage unavailable") if recount_fails else None,
) as live_mock,
):
update_all_orgs_billing_quotas()
assert live_mock.call_args.args[0] == [str(self.organization.id)]
self.organization.refresh_from_db()
assert self.organization.usage is not None
assert self.organization.usage["signals_credits"]["todays_usage"] == 1500
assert self.team.api_token in list_limited_team_attributes(
QuotaResource.SIGNALS_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
@patch("posthoganalytics.capture")
@patch("posthoganalytics.feature_enabled", return_value=False)
@time_machine.travel("2026-06-15T12:00:00Z", tick=False)
def test_quota_cron_keeps_zero_limit_block_without_recounting(self, _feature_enabled, _capture) -> None:
self._set_self_driving_usage(0)
assert self.organization.usage is not None
self.organization.usage["posthog_code_credits"] = {
"usage": 0,
"todays_usage": 0,
"limit": 0,
"quota_limited_until": 1782864000,
}
self.organization.save()
replace_limited_team_tokens(
QuotaResource.POSTHOG_CODE_CREDITS,
{self.team.api_token: 1782864000},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
with patch(
"ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_orgs", return_value={}
) as live_mock:
update_all_orgs_billing_quotas()
assert all(str(self.organization.id) not in call.args[0] for call in live_mock.call_args_list)
assert self.team.api_token in list_limited_team_attributes(
QuotaResource.POSTHOG_CODE_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
@parameterized.expand(
[
("kept_when_added_after_snapshot", False, 1798761600, True),
("removed_when_snapshot_org_is_under_limit", True, 1798761600, False),
("expired_entry_purged", False, 1750000000, False),
]
)
@patch("posthoganalytics.capture")
@patch("posthoganalytics.feature_enabled", return_value=False)
@time_machine.travel("2026-06-15T12:00:00Z", tick=False)
def test_quota_cron_reconciles_limiter_instead_of_replacing_it(
self, _name, in_snapshot, score, expect_present, _feature_enabled, _capture
) -> None:
# A PR that crosses the limit while the cron runs is written to Redis after the cron's
# snapshot, so a wholesale replace would wipe it and unblock the org until the next tick.
self._set_self_driving_usage(1500)
add_limited_team_tokens(
QuotaResource.SIGNALS_CREDITS,
{self.team.api_token: score},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
def snapshot(resource: QuotaResource, cache_key: QuotaLimitingCaches, use_cache: bool = True) -> list[str]:
if (
in_snapshot
and resource == QuotaResource.SIGNALS_CREDITS
and cache_key == QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
):
return [self.team.api_token]
return []
with (
patch("ee.billing.quota_limiting.list_limited_team_attributes", side_effect=snapshot) as snapshot_mock,
patch("posthog.tasks.usage_report.get_teams_with_signals_credits_used_in_period", return_value=[]),
patch("ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_org", return_value=0),
):
update_all_orgs_billing_quotas()
# The cache is off under TEST, so only this assertion catches a refactor that drops the kwarg
# and hands the cron the previous run's snapshot in production.
assert snapshot_mock.call_args_list
assert all(call.kwargs.get("use_cache") is False for call in snapshot_mock.call_args_list)
zset_score = get_client().zscore(
f"{QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY.value}{QuotaResource.SIGNALS_CREDITS.value}",
self.team.api_token,
)
assert (zset_score is not None) == expect_present
def test_refresh_is_a_noop_without_self_driving_usage(self) -> None:
self.organization.usage = {"events": {"usage": 1, "todays_usage": 0, "limit": None}}
self.organization.save()
with patch("ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_org") as live_mock:
refresh_org_self_driving_quota(str(self.organization.id))
live_mock.assert_not_called()
@override_settings(CLOUD_DEPLOYMENT="US", AI_GATEWAY_REDIS_URL="redis://localhost:6379/15")
class TestLLMGatewayQuotaProjectionHooks(BaseTest):
_GATEWAY_REDIS_URL = "redis://localhost:6379/15"
def setUp(self):
super().setUp()
from posthog.storage.team_llm_gateway_quota_cache import team_llm_gateway_quota_hypercache
team_llm_gateway_quota_hypercache.cache_client.clear()
for resource in (QuotaResource.AI_CREDITS, QuotaResource.POSTHOG_CODE_CREDITS):
remove_limited_team_tokens(resource, [self.team.api_token], QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
def tearDown(self):
for resource in (QuotaResource.AI_CREDITS, QuotaResource.POSTHOG_CODE_CREDITS):
remove_limited_team_tokens(resource, [self.team.api_token], QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)
super().tearDown()
@patch("posthog.tasks.team_llm_gateway_quota.settings")
@patch("posthog.storage.team_llm_gateway_quota_cache.settings")
@time_machine.travel("2021-01-25T00:00:00Z", tick=False)
def test_update_org_billing_quotas_projects_then_lifts_the_gateway_quota_blob(
self, mock_settings, task_settings
) -> None:
from posthog.storage.team_llm_gateway_quota_cache import get_team_quota_blob
mock_settings.AI_GATEWAY_REDIS_URL = self._GATEWAY_REDIS_URL
task_settings.AI_GATEWAY_REDIS_URL = self._GATEWAY_REDIS_URL
other_team = create_team(organization=self.organization)
with self.settings(USE_TZ=False):
self.organization.usage = {
"events": {"usage": 1, "limit": 100, "todays_usage": 0},
"ai_credits": {"usage": 1_000, "limit": 100, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
update_org_billing_quotas(self.organization)
for team in (self.team, other_team):
blob = get_team_quota_blob(team)
assert blob is not None, team.id
assert blob["team_id"] == team.id
assert blob["buckets"]["ai_credits"]["limited"] is True
assert "posthog_code_credits" not in blob["buckets"]
assert blob["org_deactivated"] is False
self.organization.usage["ai_credits"]["limit"] = 10_000
self.organization.save()
update_org_billing_quotas(self.organization)
for team in (self.team, other_team):
assert get_team_quota_blob(team) is None, team.id
@patch("posthog.storage.team_llm_gateway_quota_cache.settings")
def test_update_org_billing_quotas_projection_failure_does_not_fail_billing(self, mock_settings) -> None:
mock_settings.AI_GATEWAY_REDIS_URL = self._GATEWAY_REDIS_URL
self.organization.usage = {
"events": {"usage": 1, "limit": 100, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
with (
patch(
"posthog.tasks.team_llm_gateway_quota.project_org_llm_gateway_quota_task.delay",
side_effect=RuntimeError("broker down"),
) as delay,
patch("ee.billing.quota_limiting.capture_exception") as capture,
):
update_org_billing_quotas(self.organization)
delay.assert_called_once_with(str(self.organization.id))
capture.assert_called_once()
@parameterized.expand([("live_run_projects", False, True), ("dry_run_projects_nothing", True, False)])
@patch("ee.billing.quota_limiting._project_llm_gateway_quota_for_tokens")
@patch("posthoganalytics.capture")
@time_machine.travel("2021-01-25T00:00:00Z", tick=False)
def test_update_all_orgs_billing_quotas_projects_the_ai_zset_diff(
self, _name, dry_run, expect_projection, _capture, project
) -> None:
with self.settings(USE_TZ=False):
self.organization.usage = {
"events": {"usage": 1, "limit": 100, "todays_usage": 0},
"posthog_code_credits": {"usage": 5_000, "limit": 100, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
update_all_orgs_billing_quotas(dry_run=dry_run)
if not expect_projection:
project.assert_not_called()
return
project.assert_called_once()
(tokens,) = project.call_args.args
assert self.team.api_token in tokens
@patch("ee.billing.quota_limiting._project_llm_gateway_quota_for_tokens")
@patch("posthoganalytics.capture")
@time_machine.travel("2021-01-25T00:00:00Z", tick=False)
def test_update_all_orgs_billing_quotas_reprojects_a_lifted_team(self, _capture, project) -> None:
add_limited_team_tokens(
QuotaResource.POSTHOG_CODE_CREDITS,
{self.team.api_token: 1611900000},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
with self.settings(USE_TZ=False):
self.organization.usage = {
"events": {"usage": 1, "limit": 100, "todays_usage": 0},
"posthog_code_credits": {"usage": 1, "limit": 100, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
update_all_orgs_billing_quotas()
project.assert_called_once()
(tokens,) = project.call_args.args
assert self.team.api_token in tokens
def test_the_all_orgs_projection_is_enqueued_in_batches_not_run_inline(self) -> None:
from ee.billing.quota_limiting import _project_llm_gateway_quota_for_tokens
tokens = [f"phc_{i:04d}" for i in range(1_001)] + ["", "phc_0000"]
with (
patch("posthog.tasks.team_llm_gateway_quota.project_teams_llm_gateway_quota_task.delay") as delay,
patch("posthog.storage.team_llm_gateway_quota_cache.project_teams_quota_by_token") as inline,
):
_project_llm_gateway_quota_for_tokens(tokens)
inline.assert_not_called()
batches = [call.args[0] for call in delay.call_args_list]
assert [len(batch) for batch in batches] == [500, 500, 1]
assert sorted(token for batch in batches for token in batch) == sorted(set(tokens) - {""})
def test_nothing_is_enqueued_without_gateway_redis_url(self) -> None:
from ee.billing.quota_limiting import _project_llm_gateway_quota_for_org, _project_llm_gateway_quota_for_tokens
with (
self.settings(AI_GATEWAY_REDIS_URL=None),
patch("posthog.tasks.team_llm_gateway_quota.project_teams_llm_gateway_quota_task.delay") as teams_delay,
patch("posthog.tasks.team_llm_gateway_quota.project_org_llm_gateway_quota_task.delay") as org_delay,
):
_project_llm_gateway_quota_for_tokens([self.team.api_token])
_project_llm_gateway_quota_for_org(self.organization)
teams_delay.assert_not_called()
org_delay.assert_not_called()
def test_an_enqueue_failure_is_captured_not_raised(self) -> None:
from ee.billing.quota_limiting import _project_llm_gateway_quota_for_tokens
with (
patch(
"posthog.tasks.team_llm_gateway_quota.project_teams_llm_gateway_quota_task.delay",
side_effect=RuntimeError("broker down"),
),
patch("ee.billing.quota_limiting.capture_exception") as capture,
):
_project_llm_gateway_quota_for_tokens([self.team.api_token])
capture.assert_called_once()
def test_one_failed_enqueue_does_not_skip_the_later_batches(self) -> None:
from ee.billing.quota_limiting import _project_llm_gateway_quota_for_tokens
with (
patch(
"posthog.tasks.team_llm_gateway_quota.project_teams_llm_gateway_quota_task.delay",
side_effect=[RuntimeError("broker blip"), None, None],
) as delay,
patch("ee.billing.quota_limiting.capture_exception") as capture,
):
_project_llm_gateway_quota_for_tokens([f"phc_{i:04d}" for i in range(1_001)])
assert delay.call_count == 3
capture.assert_called_once()
@parameterized.expand(
[
("ai_credits", QuotaResource.AI_CREDITS, True),
("posthog_code_credits", QuotaResource.POSTHOG_CODE_CREDITS, True),
("events", QuotaResource.EVENTS, False),
]
)
def test_staff_limit_and_unlimit_reproject_only_ai_buckets(self, _name, resource, expect_projection) -> None:
self.organization.usage = {"period": ["2021-01-01T00:00:00Z", "2099-01-31T23:59:59Z"]}
self.organization.save()
with patch("posthog.tasks.team_llm_gateway_quota.project_org_llm_gateway_quota_task.delay") as delay:
self.organization.limit_product_until_end_of_billing_cycle(resource)
self.organization.unlimit_product(resource)
assert delay.call_count == (2 if expect_projection else 0)
@patch("ee.billing.quota_limiting._project_llm_gateway_quota_for_tokens")
@patch("posthoganalytics.capture")
@time_machine.travel("2021-01-25T00:00:00Z", tick=False)
def test_update_all_orgs_billing_quotas_reprojects_a_team_that_stays_limited(self, _capture, project) -> None:
add_limited_team_tokens(
QuotaResource.POSTHOG_CODE_CREDITS,
{self.team.api_token: 1611900000},
QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
)
with self.settings(USE_TZ=False):
self.organization.usage = {
"events": {"usage": 1, "limit": 100, "todays_usage": 0},
"posthog_code_credits": {"usage": 5_000, "limit": 100, "todays_usage": 0},
"period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
}
self.organization.customer_trust_scores = zero_trust_scores()
self.organization.save()
update_all_orgs_billing_quotas()
project.assert_called_once()
(tokens,) = project.call_args.args
assert self.team.api_token in tokens
members = {m.decode("utf-8") for m in get_client().zrange("@posthog/quota-limits/posthog_code_credits", 0, -1)}
assert self.team.api_token in members