:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(billing): cut per-organization work in the quota limiting loop (#121806)
Alex Mays26e91e75f541
diff --git a/ee/billing/quota_limiting.py b/ee/billing/quota_limiting.pyindex 6821a853ca44..fa4c0d1b1dfc 100644--- a/ee/billing/quota_limiting.py+++ b/ee/billing/quota_limiting.py@@ -3,6 +3,7 @@ import jsonfrom collections.abc import Callable, Collection, Iterable, Mapping, Sequencefrom datetime import datetime, timedeltafrom enum import Enum+from functools import lru_cache, partialfrom itertools import batched, groupbyfrom time import timefrom typing import Any, Optional, TypedDict, cast@@ -31,6 +32,7 @@ from posthog.tasks.usage_report import (combine_posthog_code_credits,convert_team_usage_rows_to_dict,get_self_driving_credits_used_in_period_for_org,+ get_self_driving_credits_used_in_period_for_orgs,get_signals_credited_refund_credits_for_org,get_teams_with_api_queries_metrics,get_teams_with_apm_tracing_usage_in_period,@@ -431,6 +433,11 @@ def _signals_credited_refund_offset(organization: Organization, resource: QuotaRreturn get_signals_credited_refund_credits_for_org(organization.id, period_start, period_end)+@lru_cache(maxsize=8192)+def _period_timestamp(value: str) -> int:+ return round(dateutil.parser.isoparse(value).timestamp())++def org_quota_limited_until(organization: Organization,resource: QuotaResource,@@ -514,8 +521,8 @@ def org_quota_limited_until(refund_offset_properties = {"signals_refund_offset": refund_offset} if refund_offset else {}is_over_limit = current_usage - refund_offset >= limit + OVERAGE_BUFFER[resource]- billing_period_start = round(dateutil.parser.isoparse(organization.usage["period"][0]).timestamp())- billing_period_end = round(dateutil.parser.isoparse(organization.usage["period"][1]).timestamp())+ billing_period_start = _period_timestamp(organization.usage["period"][0])+ billing_period_end = _period_timestamp(organization.usage["period"][1])# - customer_trust_scores can be empty {} for orgs not yet synced from billing. Default to 0 (no grace period)# - customer_trust_scores in posthog_organization use usage_key values (matching QuotaResource values)@@ -1154,6 +1161,10 @@ def _identify_refresh_candidates(`quota_limiting_suspended_until` marker;3. it appears newly limitable from cached state — `usage + todays_usage >= limit + buffer`for any resource.++ A resource with a zero limit and no overage buffer never makes an org a candidate. That+ limit blocks the resource at any usage, so the cached state gives the same decision.+ Signals credits are the exception, because a refund offset can lift a signals block."""# Either Redis set is reason enough to refresh, so union them once up front and# do a single membership check per resource below.@@ -1175,6 +1186,10 @@ def _identify_refresh_candidates(for resource in QuotaResource:field = resource.value+ usage_for_resource = org.usage.get(field)+ limit = usage_for_resource.get("limit") if usage_for_resource else None+ if limit == 0 and OVERAGE_BUFFER[resource] == 0 and resource != QuotaResource.SIGNALS_CREDITS:+ continue# Redis check first: an org with a stale entry in either Redis set must be# refreshed and re-evaluated even when its `usage[resource]` is empty or@@ -1184,7 +1199,6 @@ def _identify_refresh_candidates(is_candidate = Truebreak- usage_for_resource = org.usage.get(field)if not usage_for_resource:continue@@ -1194,7 +1208,6 @@ def _identify_refresh_candidates(is_candidate = Truebreak- limit = usage_for_resource.get("limit")if limit is None:continueusage_value = usage_for_resource.get("usage") or 0@@ -1483,12 +1496,31 @@ def update_all_orgs_billing_quotas(.in_bulk).values()}+ signals_recount_ids = [+ org_id+ for org_id, fresh in fresh_orgs.items()+ if (fresh.usage or {}).get(QuotaResource.SIGNALS_CREDITS.value)+ ]+ try:+ signals_recounts = retry_quota_database_operation(+ partial(get_self_driving_credits_used_in_period_for_orgs, signals_recount_ids, period.start, period.end)+ )+ except Exception as error:+ if isinstance(error, (DatabaseError, InterfaceError)) and not settings.TEST:+ close_old_connections()+ capture_exception(error, {"organization_ids": signals_recount_ids})+ # These organizations keep their existing limits. The loop still decides the rest of the chunk.+ for org_id in signals_recount_ids:+ discard_decision(org_id, teams_by_org.get(org_id, []))+ signals_recounts = {}refresh_count += len(fresh_orgs)refresh_total_seconds += time() - refresh_startusage_batch = QuotaUsageBatch()for org_id, todays_report in reports.usage.items():org = orgs_by_id[org_id]try:+ if org_id in failed_org_ids:+ continueif org.usage and org.usage.get("period"):if org_id in refresh_candidates:fresh = fresh_orgs.get(org_id)@@ -1501,9 +1533,7 @@ def update_all_orgs_billing_quotas(usage_batch.require_unchanged_usage(org, fresh.usage_md5)if (org.usage or {}).get(QuotaResource.SIGNALS_CREDITS.value):# Push refreshes can raise signals usage after the source snapshot. Recount before writing it.- todays_report["signals_credits"] = get_self_driving_credits_used_in_period_for_org(- org_id, period.start, period.end- )+ todays_report["signals_credits"] = signals_recounts.get(org_id, 0)_patch_todays_usage(org, todays_report, batch=usage_batch)org_is_limited = org_is_suspended = Falsefor resource in QuotaResource:diff --git a/ee/billing/quota_usage.py b/ee/billing/quota_usage.pyindex a96ae8a9a2b1..ed3b01041cf0 100644--- a/ee/billing/quota_usage.py+++ b/ee/billing/quota_usage.py@@ -1,6 +1,5 @@import copyimport json-from collections import defaultdictfrom collections.abc import Callable, Sequencefrom time import sleepfrom typing import Any, TypeVar@@ -17,7 +16,6 @@ from posthog.models.organization import Organizationlogger = structlog.get_logger(__name__)UsagePatch = tuple[list[str], Any]-_PatchRow = tuple[UUID, str | None, list[Any]]T = TypeVar("T")@@ -34,6 +32,25 @@ def retry_quota_database_operation(operation: Callable[[], T]) -> T:return operation()+def _resource_patch(operations: Sequence[UsagePatch]) -> dict[str, dict[str, Any]]:+ """Folds [resource, key] operations into one set map and one delete list for each resource.++ The last operation on a key wins.+ """+ patch: dict[str, dict[str, Any]] = {}+ for (resource, key), value in operations:+ entry = patch.setdefault(resource, {"set": {}, "delete": []})+ if value is None:+ entry["set"].pop(key, None)+ if key not in entry["delete"]:+ entry["delete"].append(key)+ else:+ entry["set"][key] = value+ if key in entry["delete"]:+ entry["delete"].remove(key)+ return patch++@frozenclass _QuotaAction:organization: Organization@@ -52,6 +69,8 @@ class QuotaUsageBatch:self._actions: dict[UUID, list[_QuotaAction]] = {}def patch(self, organization: Organization, operations: Sequence[UsagePatch]) -> None:+ if any(len(path) != 2 for path, _ in operations):+ raise ValueError("QuotaUsageBatch patches only [resource, key] paths")self._patches.setdefault(organization.id, []).extend(operations)def require_unchanged_usage(self, organization: Organization, usage_md5: str) -> None:@@ -96,48 +115,32 @@ class QuotaUsageBatch:transaction.on_commit(action.report)def _write(self) -> set[UUID]:- groups: dict[tuple[tuple[tuple[str, ...], bool], ...], list[_PatchRow]] = defaultdict(list)- for org_id, operations in self._patches.items():- if operations:- shape = tuple((tuple(path), value is None) for path, value in operations)- values = [value for _, value in operations if value is not None]- groups[shape].append((org_id, self._usage_md5.get(org_id), values))-- skipped: set[UUID] = set()- if not groups:- return skipped-- # Each shape can need a separate statement; all of an organization's patches commit together.- with transaction.atomic():- for shape, rows in groups.items():- expression = "o.usage"- parameters: list[Any] = []- value_index = 0- for path, delete in shape:- if delete:- expression = f"({expression}) #- %s::text[]"- else:- expression = f"jsonb_set({expression}, %s::text[], v.v{value_index})"- value_index += 1- parameters.append(list(path))-- array_sql = "%s::uuid[], %s::text[]" + ", %s::jsonb[]" * value_index- parameters.append([org_id for org_id, _, _ in rows])- parameters.append([usage_md5 for _, usage_md5, _ in rows])- parameters.extend([json.dumps(values[index]) for _, _, values in rows] for index in range(value_index))- columns = ", ".join(["id", "usage_md5", *(f"v{i}" for i in range(value_index))])- table = Organization._meta.db_table- sql = (- f"UPDATE {table} AS o SET usage = {expression} "- f"FROM UNNEST({array_sql}) AS v({columns}) "- "WHERE o.id = v.id AND (v.usage_md5 IS NULL OR md5(o.usage::text) = v.usage_md5) "- "RETURNING o.id"- )- with connection.cursor() as cursor:- # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (identifiers come from model metadata and fixed templates; paths and values are parameters)- cursor.execute(sql, parameters)- written = {row[0] for row in cursor.fetchall()}- skipped.update(- org_id for org_id, usage_md5, _ in rows if usage_md5 is not None and org_id not in written- )- return skipped+ rows = [+ (org_id, self._usage_md5.get(org_id), _resource_patch(operations))+ for org_id, operations in self._patches.items()+ if operations+ ]+ if not rows:+ return set()++ # Skip a resource the row lacks instead of creating a partial resource object without its limit and usage.+ sql = (+ f"UPDATE {Organization._meta.db_table} AS o SET usage = o.usage || coalesce(("+ "SELECT jsonb_object_agg(r.key, ((o.usage -> r.key) || (r.value -> 'set')) "+ "- ARRAY(SELECT jsonb_array_elements_text(r.value -> 'delete'))) "+ "FROM jsonb_each(v.patch) AS r WHERE jsonb_typeof(o.usage -> r.key) = 'object'"+ "), '{}'::jsonb) "+ "FROM UNNEST(%s::uuid[], %s::text[], %s::jsonb[]) AS v(id, usage_md5, patch) "+ "WHERE o.id = v.id AND (v.usage_md5 IS NULL OR md5(o.usage::text) = v.usage_md5) "+ "RETURNING o.id"+ )+ parameters: list[Any] = [+ [org_id for org_id, _, _ in rows],+ [usage_md5 for _, usage_md5, _ in rows],+ [json.dumps(patch) for _, _, patch in rows],+ ]+ with connection.cursor() as cursor:+ # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (identifiers come from model metadata and fixed templates; resources, keys and values are parameters)+ cursor.execute(sql, parameters)+ written = {row[0] for row in cursor.fetchall()}+ return {org_id for org_id, usage_md5, _ in rows if usage_md5 is not None and org_id not in written}diff --git a/ee/billing/test/test_quota_limiting.py b/ee/billing/test/test_quota_limiting.pyindex d07e4bd6084c..2776c77ef9ff 100644--- a/ee/billing/test/test_quota_limiting.py+++ b/ee/billing/test/test_quota_limiting.py@@ -2685,6 +2685,70 @@ class TestIdentifyRefreshCandidates(SimpleTestCase):{},False,),+ (+ "zero_limit_block_is_not_a_candidate",+ {+ "posthog_code_credits": {+ "usage": 0,+ "limit": 0,+ "todays_usage": 0,+ "quota_limited_until": 1_700_000_000,+ },+ "period": _PERIOD,+ },+ _full_usage_counters(),+ ["phc_blocked"],+ {"posthog_code_credits": ["phc_blocked"]},+ {},+ False,+ ),+ (+ "zero_limit_block_does_not_hide_another_resource",+ {+ "posthog_code_credits": {+ "usage": 0,+ "limit": 0,+ "todays_usage": 0,+ "quota_limited_until": 1_700_000_000,+ },+ "events": {"usage": 0, "limit": 1_000, "quota_limited_until": 1_700_000_000},+ "period": _PERIOD,+ },+ _full_usage_counters(),+ [],+ {},+ {},+ True,+ ),+ (+ "zero_limit_with_overage_buffer_is_a_candidate",+ {+ "recordings": {"usage": 0, "limit": 0, "todays_usage": 0, "quota_limited_until": 1_700_000_000},+ "period": _PERIOD,+ },+ _full_usage_counters(),+ [],+ {},+ {},+ True,+ ),+ (+ "zero_signals_limit_is_a_candidate",+ {+ "signals_credits": {+ "usage": 0,+ "limit": 0,+ "todays_usage": 0,+ "quota_limited_until": 1_700_000_000,+ },+ "period": _PERIOD,+ },+ _full_usage_counters(),+ [],+ {},+ {},+ True,+ ),])def test_candidate_selection(@@ -3296,10 +3360,13 @@ class TestRefreshOrgSelfDrivingQuota(BaseTest):QuotaResource.SIGNALS_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)+ @parameterized.expand([("recount_succeeds", False), ("recount_fails", True)])@patch("posthoganalytics.capture")@patch("posthoganalytics.feature_enabled", return_value=False)@time_machine.travel("2026-06-15T12:00:00Z", tick=False)- def test_quota_cron_recounts_live_instead_of_writing_its_stale_snapshot(self, _feature_enabled, _capture) -> None:+ def test_quota_cron_recounts_live_instead_of_writing_its_stale_snapshot(+ self, _name, recount_fails, _feature_enabled, _capture+ ) -> None:# The cron snapshots fleet usage before its org loop, so a push-refresh that fires while# the loop runs (a PR landing) writes a fresher, higher todays_usage than the snapshot.# Writing the snapshot back would unpause the over-limit org until the next tick; the@@ -3318,12 +3385,14 @@ class TestRefreshOrgSelfDrivingQuota(BaseTest):return_value=[(self.team.id, 100)],),patch(- "ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_org", return_value=1500+ "ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_orgs",+ return_value={str(self.organization.id): 1500},+ side_effect=DatabaseError("Signals usage unavailable") if recount_fails else None,) as live_mock,):update_all_orgs_billing_quotas()- assert live_mock.call_args.args[0] == str(self.organization.id)+ assert live_mock.call_args.args[0] == [str(self.organization.id)]self.organization.refresh_from_db()assert self.organization.usage is not Noneassert self.organization.usage["signals_credits"]["todays_usage"] == 1500@@ -3331,6 +3400,34 @@ class TestRefreshOrgSelfDrivingQuota(BaseTest):QuotaResource.SIGNALS_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY)+ @patch("posthoganalytics.capture")+ @patch("posthoganalytics.feature_enabled", return_value=False)+ @time_machine.travel("2026-06-15T12:00:00Z", tick=False)+ def test_quota_cron_keeps_zero_limit_block_without_recounting(self, _feature_enabled, _capture) -> None:+ self._set_self_driving_usage(0)+ assert self.organization.usage is not None+ self.organization.usage["posthog_code_credits"] = {+ "usage": 0,+ "todays_usage": 0,+ "limit": 0,+ "quota_limited_until": 1782864000,+ }+ self.organization.save()+ replace_limited_team_tokens(+ QuotaResource.POSTHOG_CODE_CREDITS,+ {self.team.api_token: 1782864000},+ QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,+ )+ with patch(+ "ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_orgs", return_value={}+ ) as live_mock:+ update_all_orgs_billing_quotas()++ assert all(str(self.organization.id) not in call.args[0] for call in live_mock.call_args_list)+ assert self.team.api_token in list_limited_team_attributes(+ QuotaResource.POSTHOG_CODE_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY+ )+@parameterized.expand([("kept_when_added_after_snapshot", False, 1798761600, True),diff --git a/ee/billing/test/test_quota_usage.py b/ee/billing/test/test_quota_usage.pyindex 5e9d806783f4..77020d181e74 100644--- a/ee/billing/test/test_quota_usage.py+++ b/ee/billing/test/test_quota_usage.py@@ -21,7 +21,7 @@ from ee.billing.quota_usage import QuotaUsageBatchclass TestQuotaUsageBatch(BaseTest):@parameterized.expand([("set", False), ("delete_only", True)])- def test_sql_shape_is_independent_of_batch_size(self, _name, delete_only) -> None:+ def test_one_statement_for_any_batch_size_or_patch_shape(self, _name, delete_only) -> None:other = Organization.objects.create(name="Other organization", slug="other-quota-organization")organizations = [self.organization, other]usage = {"events": {"usage": 10, "todays_usage": 0, "quota_limited_until": 1234}}@@ -37,23 +37,21 @@ class TestQuotaUsageBatch(BaseTest):for batch_size in (1, 2):batch = QuotaUsageBatch()for index, org in enumerate(organizations[:batch_size], start=1):- batch.patch(- org,- [- (["events", "todays_usage"], None if delete_only else index * 20),- (["events", "quota_limited_until"], None if delete_only else index * 100),- ],- )+ operations = [(["events", "todays_usage"], None if delete_only else index * 20)]+ if index == 2:+ operations.append((["events", "quota_limited_until"], None if delete_only else index * 100))+ batch.patch(org, operations)batch.write()assert len(statements) == 2assert statements[0] == statements[1]- for index, org in enumerate(organizations, start=1):- org.refresh_from_db()- expected = {"usage": 10}- if not delete_only:- expected.update(todays_usage=index * 20, quota_limited_until=index * 100)- assert org.usage == {"events": expected}+ first, second = (Organization.objects.get(id=org.id).usage for org in organizations)+ if delete_only:+ assert first == {"events": {"usage": 10, "quota_limited_until": 1234}}+ assert second == {"events": {"usage": 10}}+ else:+ assert first == {"events": {"usage": 10, "todays_usage": 20, "quota_limited_until": 1234}}+ assert second == {"events": {"usage": 10, "todays_usage": 40, "quota_limited_until": 200}}@parameterized.expand([("all_present", False), ("deleted_organization", True)])def test_multiple_patches_preserve_other_organizations_and_fields(self, _name, delete_second) -> None:@@ -63,6 +61,7 @@ class TestQuotaUsageBatch(BaseTest):batch = QuotaUsageBatch()batch.patch(self.organization, [(["events", "todays_usage"], 20)])batch.patch(self.organization, [(["events", "quota_limited_until"], 1234)])+ batch.patch(self.organization, [(["recordings", "todays_usage"], 5)])batch.patch(other, [(["events", "todays_usage"], 30)])if delete_second:other.delete()@@ -83,7 +82,7 @@ class TestQuotaUsageBatch(BaseTest):)@patch("posthoganalytics.capture")@patch("ee.billing.quota_usage.sleep")- def test_failed_statement_rolls_back_batch_before_retry_or_events(+ def test_failed_write_leaves_rows_unchanged_and_sends_no_events(self, _name, error_type, failures, _sleep, capture) -> None:other = Organization.objects.create(name="Other organization", slug="other-quota-organization")@@ -95,23 +94,20 @@ class TestQuotaUsageBatch(BaseTest):properties = {"usage": {"events": 20}}batch.report_action(self.organization, "organization quota limits changed", properties)properties["usage"]["events"] = 999- writes = 0errors = 0- def fail_second_statement(execute, sql, params, many, context):- nonlocal writes, errors+ def fail_first_attempts(execute, sql, params, many, context):+ nonlocal errorsif sql.startswith("UPDATE posthog_organization AS o"):- writes += 1- if writes % 2 == 1:- assert Organization.objects.get(id=self.organization.id).usage == usage- assert Organization.objects.get(id=other.id).usage == usage- elif errors < failures:+ assert Organization.objects.get(id=self.organization.id).usage == usage+ assert Organization.objects.get(id=other.id).usage == usage+ if errors < failures:errors += 1raise error_type("synthetic write failure")return execute(sql, params, many, context)succeeds = error_type is OperationalError and failures == 1- with connection.execute_wrapper(fail_second_statement), self.captureOnCommitCallbacks(execute=True):+ with connection.execute_wrapper(fail_first_attempts), self.captureOnCommitCallbacks(execute=True):if succeeds:batch.write()batch.publish()diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.pyindex 3ff9928703ab..d8f5c8b2410d 100644--- a/posthog/management/commands/start_temporal_worker.py+++ b/posthog/management/commands/start_temporal_worker.py@@ -55,7 +55,7 @@ from posthog.temporal.common.interceptor import is_task_queue_supportedfrom posthog.temporal.common.liveness_tracker import LivenessInterceptor, get_liveness_trackerfrom posthog.temporal.common.logger import configure_logger, get_loggerfrom posthog.temporal.common.shutdown import ShutdownSignalListener-from posthog.temporal.common.worker import ActivityWorkerConfig, ManagedWorker, create_worker+from posthog.temporal.common.worker import ManagedWorker, create_workerfrom posthog.temporal.common.zombie_exit import ZombieActivityExitfrom posthog.temporal.data_modeling import (ACTIVITIES as DATA_MODELING_ACTIVITIES,@@ -527,6 +527,7 @@ _task_queue_specs = [+ BILLING_ALERTS_ACTIVITIES+ BILLING_ACTIVE_SYNC_ACTIVITIES,),+ (settings.USAGE_REPORT_AGGREGATION_TASK_QUEUE, [], USAGE_REPORTS_AGGREGATION_ACTIVITIES),(settings.VIDEO_EXPORT_TASK_QUEUE,SIGNALS_PRODUCT_WORKFLOWS@@ -672,15 +673,6 @@ for task_queue_name, workflows_for_queue, activities_for_queue in _task_queue_spWORKFLOWS_DICT = _workflowsACTIVITIES_DICT = _activities-ACTIVITY_WORKER_CONFIGS = {- settings.BILLING_TASK_QUEUE: (- ActivityWorkerConfig(- task_queue=settings.USAGE_REPORT_AGGREGATION_TASK_QUEUE,- activities=USAGE_REPORTS_AGGREGATION_ACTIVITIES,- max_concurrent_activities=1,- ),- ),-}def workflows_include_data_import_syncs(workflows: collections.abc.Iterable[type]) -> bool:@@ -990,7 +982,6 @@ class Command(BaseCommand):),enable_combined_metrics_server=not disable_combined_metrics_server,enable_open_telemetry_plugin=enable_otel,- activity_worker_configs=ACTIVITY_WORKER_CONFIGS.get(task_queue, ()),))diff --git a/posthog/management/commands/test/test_start_temporal_worker.py b/posthog/management/commands/test/test_start_temporal_worker.pyindex e7e3e4be983d..a9c4c353e211 100644--- a/posthog/management/commands/test/test_start_temporal_worker.py+++ b/posthog/management/commands/test/test_start_temporal_worker.py@@ -6,7 +6,6 @@ from django.conf import settingsfrom posthog.management.commands.start_temporal_worker import (ACTIVITIES_DICT,- ACTIVITY_WORKER_CONFIGS,AI_WORKFLOWS,ALERT_AI_QUEUE_ACTIVITIES,DATA_SYNC_WORKFLOWS,@@ -19,6 +18,7 @@ from posthog.management.commands.start_temporal_worker import (workflows_include_data_import_syncs,)from posthog.temporal.usage_report.activities import aggregate_and_chunk_org_reports+from posthog.temporal.usage_report.workflow import RunUsageReportsWorkflowfrom products.alerts_platform.backend.facade.temporal import (DELIVERY_ACTIVITIES,@@ -43,14 +43,19 @@ class _NotADataSyncWorkflow:pass-def test_billing_registers_aggregation_with_separate_capacity() -> None:- (aggregation_worker,) = ACTIVITY_WORKER_CONFIGS[settings.BILLING_TASK_QUEUE]- assert aggregation_worker.task_queue == settings.USAGE_REPORT_AGGREGATION_TASK_QUEUE- assert aggregation_worker.task_queue != settings.BILLING_TASK_QUEUE- assert aggregation_worker.max_concurrent_activities == 1- assert aggregation_worker.activities == [aggregate_and_chunk_org_reports]- # A workflow may have scheduled aggregation before the queue change deployed.- assert aggregate_and_chunk_org_reports in ACTIVITIES_DICT[settings.BILLING_TASK_QUEUE]+# Queue settings collapse to a single dev queue under DEBUG, so match on the spec entries rather than on queue names.+def test_aggregation_runs_on_its_own_queue_and_the_billing_queue() -> None:+ aggregation_entries = [+ activities+ for task_queue, workflows, activities in _task_queue_specs+ if task_queue == settings.USAGE_REPORT_AGGREGATION_TASK_QUEUE and not workflows+ ]+ assert any(aggregate_and_chunk_org_reports in activities for activities in aggregation_entries)+ # A workflow may have scheduled aggregation on the billing queue before the queue change deployed.+ (billing_activities,) = [+ activities for _, workflows, activities in _task_queue_specs if RunUsageReportsWorkflow in workflows+ ]+ assert aggregate_and_chunk_org_reports in billing_activities@pytest.mark.parametrize(diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.pyindex 3054c2371eff..4b52f0104dda 100644--- a/posthog/settings/temporal.py+++ b/posthog/settings/temporal.py@@ -372,8 +372,8 @@ TASKS_WORKFLOW_DISPATCH_OUTBOX_ENABLED: bool = get_from_env(STAMPHOG_TASK_QUEUE = _set_temporal_task_queue("stamphog-task-queue")TEST_TASK_QUEUE = _set_temporal_task_queue("test-task-queue")BILLING_TASK_QUEUE = _set_temporal_task_queue("billing-task-queue")-# Keep aggregation separate even when local development combines the other queues.-USAGE_REPORT_AGGREGATION_TASK_QUEUE = f"{BILLING_TASK_QUEUE}-usage-report-aggregation"+# A dedicated worker deployment polls this queue, so aggregation does not share a process with quota limiting.+USAGE_REPORT_AGGREGATION_TASK_QUEUE = _set_temporal_task_queue("billing-task-queue-usage-report-aggregation")VIDEO_EXPORT_TASK_QUEUE = _set_temporal_task_queue("video-export-task-queue")ANALYTICS_PLATFORM_TASK_QUEUE = _set_temporal_task_queue("analytics-platform-task-queue")# Keep the smoke fleets separate in local development as well as deployed environments.diff --git a/posthog/tasks/test/test_usage_report.py b/posthog/tasks/test/test_usage_report.pyindex c26f168d1fce..61538a07a9f9 100644--- a/posthog/tasks/test/test_usage_report.py+++ b/posthog/tasks/test/test_usage_report.py@@ -70,6 +70,7 @@ from posthog.tasks.usage_report import (capture_report,get_all_event_metrics_in_period,get_instance_metadata,+ get_self_driving_credits_used_in_period_for_orgs,get_teams_with_ai_credits_used_in_period,get_teams_with_billable_event_count_in_period,get_teams_with_event_count_with_groups_in_period,@@ -5086,6 +5087,24 @@ class TestAIEventsUsageReport(ClickhouseDestroyTablesMixin, TestCase, Clickhouseself.assertEqual(result, [(team.id, 1500)])+ def test_self_driving_credits_for_orgs_sum_each_orgs_teams(self) -> None:+ org_a = Organization.objects.create(name="Org A")+ org_b = Organization.objects.create(name="Org B")+ team_a1 = Team.objects.create(organization=org_a)+ team_a2 = Team.objects.create(organization=org_a)+ team_b = Team.objects.create(organization=org_b)+ period = get_previous_day(at=now() + relativedelta(days=1))+ org_ids = [str(org_a.id), str(org_b.id)]++ with patch(+ "posthog.tasks.usage_report.get_signals_billing_credits_by_team",+ return_value=[(team_a1.id, 1500), (team_a2.id, 1500), (team_b.id, 1500)],+ ) as signals_credits:+ totals = get_self_driving_credits_used_in_period_for_orgs(org_ids, period.start, period.end)++ assert totals == {str(org_a.id): 3000, str(org_b.id): 1500}+ assert signals_credits.call_args.kwargs["organization_ids"] == org_ids+@patch("posthog.tasks.usage_report.get_instance_region")def test_ai_credits_counts_billable_generation_with_no_trace(self, mock_region: MagicMock) -> None:"""A billable generation with no matching $ai_trace bills via the empty-trace fallback.diff --git a/posthog/tasks/usage_report.py b/posthog/tasks/usage_report.pyindex a80f7530b03a..f6663ee58106 100644--- a/posthog/tasks/usage_report.py+++ b/posthog/tasks/usage_report.py@@ -6,7 +6,7 @@ import base64import loggingimport dataclassesfrom collections import Counter, defaultdict-from collections.abc import Callable, Iterable, Iterator, Sequence+from collections.abc import Callable, Collection, Iterable, Iterator, Sequencefrom datetime import datetime, timedeltafrom typing import Any, Literal, Optional, TypedDict, Union@@ -2090,6 +2090,23 @@ def get_signals_credited_refund_credits_for_org(return credited_refund_credits_for_org(organization_id, begin, end)+def get_self_driving_credits_used_in_period_for_orgs(+ organization_ids: Collection[str], begin: datetime, end: datetime+) -> dict[str, int]:+ """Live self-driving credits used in `[begin, end)` for each org, from one read. Orgs with no usage are absent."""+ if not organization_ids:+ return {}+ rows = get_signals_billing_credits_by_team(begin, end, organization_ids=list(organization_ids))+ if not rows:+ return {}+ team_orgs = dict(Team.objects.filter(id__in=[team_id for team_id, _ in rows]).values_list("id", "organization_id"))+ totals: dict[str, int] = defaultdict(int)+ for team_id, credits in rows:+ if (organization_id := team_orgs.get(team_id)) is not None:+ totals[str(organization_id)] += credits+ return dict(totals)++def get_self_driving_credits_used_in_period_for_org(organization_id: str | uuid.UUID, begin: datetime, end: datetime) -> int:diff --git a/products/signals/backend/billing.py b/products/signals/backend/billing.pyindex 56e228c480f3..99bb4e56895d 100644--- a/products/signals/backend/billing.py+++ b/products/signals/backend/billing.py@@ -37,7 +37,7 @@ would otherwise be repeatable free work.import uuidfrom collections import defaultdict-from collections.abc import Sequence+from collections.abc import Collection, Sequencefrom datetime import datetimefrom typing import TYPE_CHECKING, NamedTuple@@ -327,7 +327,10 @@ def current_billing_period_bounds(organization: "Organization") -> BillingPerioddef get_signals_billing_credits_by_team(- begin: datetime, end: datetime, organization_id: str | uuid.UUID | None = None+ begin: datetime,+ end: datetime,+ organization_id: str | uuid.UUID | None = None,+ organization_ids: Collection[str | uuid.UUID] | None = None,) -> list[tuple[int, int]]:"""Signals credits used per team in `[begin, end)`.@@ -336,7 +339,8 @@ def get_signals_billing_credits_by_team(such run exists before `begin`. Each billable report is charged a flat`SIGNALS_CREDITS_PER_REPORT_WITH_PR`. Returns `[(team_id, credits), ...]` for teams withnon-zero usage only. `organization_id` narrows the scan to one org (the widget's live-count- path); usage reporting passes nothing and stays fleet-wide.+ path), and `organization_ids` to a set of orgs (the quota run's recount). Usage reporting passes+ neither and stays fleet-wide.The entry scan is bounded by PRs shipped in the period (via the `created_at` +`output__pr_url` indexes), not by the total number of reports, task runs, or teams.@@ -350,6 +354,8 @@ def get_signals_billing_credits_by_team()if organization_id is not None:bridges = bridges.filter(team__organization_id=organization_id)+ if organization_ids is not None:+ bridges = bridges.filter(team__organization_id__in=organization_ids)report_team: dict[uuid.UUID, int] = {}for report_id, team_id in bridges.values_list("report_id", "team_id"):report_team.setdefault(report_id, team_id)diff --git a/products/signals/backend/test/test_billing.py b/products/signals/backend/test/test_billing.pyindex 7552c9b381f8..c1ee2caadb00 100644--- a/products/signals/backend/test/test_billing.py+++ b/products/signals/backend/test/test_billing.py@@ -7,7 +7,7 @@ from django.apps import appsfrom parameterized import parameterized-from posthog.models import Team+from posthog.models import Organization, Teamfrom products.signals.backend.artefact_schemas import TASK_RUN_TYPE_IMPLEMENTATION, TASK_RUN_TYPE_RESEARCHfrom products.signals.backend.billing import (@@ -129,6 +129,19 @@ class TestSignalsBilling(BaseTest):# 1 credit = $0.01, so $15 == 1500 credits.self.assertEqual(SIGNALS_CREDITS_PER_REPORT_WITH_PR, 1500)+ @parameterized.expand([("organization_id", False), ("organization_ids", True)])+ def test_organization_filter_excludes_other_organizations(self, _name: str, as_list: bool) -> None:+ other_team = Team.objects.create(organization=Organization.objects.create(name="Other org"))+ for team in (self.team, other_team):+ self._pr_run(self._report(team=team), created_at=_at(10), team=team)++ org_filter = (+ {"organization_ids": [self.organization.id]} if as_list else {"organization_id": self.organization.id}+ )+ self.assertEqual(+ dict(get_signals_billing_credits_by_team(PERIOD_START, PERIOD_END, **org_filter)), {self.team.id: 1500}+ )+def test_report_with_pr_billed_flat(self) -> None:report = self._report()self._pr_run(report, created_at=_at(10))