Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(billing): cut per-organization work in the quota limiting loop (#121806)

Alex Mays26e91e75f541
diff --git a/ee/billing/quota_limiting.py b/ee/billing/quota_limiting.py
index 6821a853ca44..fa4c0d1b1dfc 100644
--- a/ee/billing/quota_limiting.py
+++ b/ee/billing/quota_limiting.py
@@ -3,6 +3,7 @@ import json
from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
from datetime import datetime, timedelta
from enum import Enum
+from functools import lru_cache, partial
from itertools import batched, groupby
from time import time
from typing import Any, Optional, TypedDict, cast
@@ -31,6 +32,7 @@ from posthog.tasks.usage_report import (
combine_posthog_code_credits,
convert_team_usage_rows_to_dict,
get_self_driving_credits_used_in_period_for_org,
+ get_self_driving_credits_used_in_period_for_orgs,
get_signals_credited_refund_credits_for_org,
get_teams_with_api_queries_metrics,
get_teams_with_apm_tracing_usage_in_period,
@@ -431,6 +433,11 @@ def _signals_credited_refund_offset(organization: Organization, resource: QuotaR
return get_signals_credited_refund_credits_for_org(organization.id, period_start, period_end)
+@lru_cache(maxsize=8192)
+def _period_timestamp(value: str) -> int:
+ return round(dateutil.parser.isoparse(value).timestamp())
+
+
def org_quota_limited_until(
organization: Organization,
resource: QuotaResource,
@@ -514,8 +521,8 @@ def org_quota_limited_until(
refund_offset_properties = {"signals_refund_offset": refund_offset} if refund_offset else {}
is_over_limit = current_usage - refund_offset >= limit + OVERAGE_BUFFER[resource]
- billing_period_start = round(dateutil.parser.isoparse(organization.usage["period"][0]).timestamp())
- billing_period_end = round(dateutil.parser.isoparse(organization.usage["period"][1]).timestamp())
+ billing_period_start = _period_timestamp(organization.usage["period"][0])
+ billing_period_end = _period_timestamp(organization.usage["period"][1])
# - customer_trust_scores can be empty {} for orgs not yet synced from billing. Default to 0 (no grace period)
# - customer_trust_scores in posthog_organization use usage_key values (matching QuotaResource values)
@@ -1154,6 +1161,10 @@ def _identify_refresh_candidates(
`quota_limiting_suspended_until` marker;
3. it appears newly limitable from cached state — `usage + todays_usage >= limit + buffer`
for any resource.
+
+ A resource with a zero limit and no overage buffer never makes an org a candidate. That
+ limit blocks the resource at any usage, so the cached state gives the same decision.
+ Signals credits are the exception, because a refund offset can lift a signals block.
"""
# Either Redis set is reason enough to refresh, so union them once up front and
# do a single membership check per resource below.
@@ -1175,6 +1186,10 @@ def _identify_refresh_candidates(
for resource in QuotaResource:
field = resource.value
+ usage_for_resource = org.usage.get(field)
+ limit = usage_for_resource.get("limit") if usage_for_resource else None
+ if limit == 0 and OVERAGE_BUFFER[resource] == 0 and resource != QuotaResource.SIGNALS_CREDITS:
+ continue
# Redis check first: an org with a stale entry in either Redis set must be
# refreshed and re-evaluated even when its `usage[resource]` is empty or
@@ -1184,7 +1199,6 @@ def _identify_refresh_candidates(
is_candidate = True
break
- usage_for_resource = org.usage.get(field)
if not usage_for_resource:
continue
@@ -1194,7 +1208,6 @@ def _identify_refresh_candidates(
is_candidate = True
break
- limit = usage_for_resource.get("limit")
if limit is None:
continue
usage_value = usage_for_resource.get("usage") or 0
@@ -1483,12 +1496,31 @@ def update_all_orgs_billing_quotas(
.in_bulk
).values()
}
+ signals_recount_ids = [
+ org_id
+ for org_id, fresh in fresh_orgs.items()
+ if (fresh.usage or {}).get(QuotaResource.SIGNALS_CREDITS.value)
+ ]
+ try:
+ signals_recounts = retry_quota_database_operation(
+ partial(get_self_driving_credits_used_in_period_for_orgs, signals_recount_ids, period.start, period.end)
+ )
+ except Exception as error:
+ if isinstance(error, (DatabaseError, InterfaceError)) and not settings.TEST:
+ close_old_connections()
+ capture_exception(error, {"organization_ids": signals_recount_ids})
+ # These organizations keep their existing limits. The loop still decides the rest of the chunk.
+ for org_id in signals_recount_ids:
+ discard_decision(org_id, teams_by_org.get(org_id, []))
+ signals_recounts = {}
refresh_count += len(fresh_orgs)
refresh_total_seconds += time() - refresh_start
usage_batch = QuotaUsageBatch()
for org_id, todays_report in reports.usage.items():
org = orgs_by_id[org_id]
try:
+ if org_id in failed_org_ids:
+ continue
if org.usage and org.usage.get("period"):
if org_id in refresh_candidates:
fresh = fresh_orgs.get(org_id)
@@ -1501,9 +1533,7 @@ def update_all_orgs_billing_quotas(
usage_batch.require_unchanged_usage(org, fresh.usage_md5)
if (org.usage or {}).get(QuotaResource.SIGNALS_CREDITS.value):
# Push refreshes can raise signals usage after the source snapshot. Recount before writing it.
- todays_report["signals_credits"] = get_self_driving_credits_used_in_period_for_org(
- org_id, period.start, period.end
- )
+ todays_report["signals_credits"] = signals_recounts.get(org_id, 0)
_patch_todays_usage(org, todays_report, batch=usage_batch)
org_is_limited = org_is_suspended = False
for resource in QuotaResource:
diff --git a/ee/billing/quota_usage.py b/ee/billing/quota_usage.py
index a96ae8a9a2b1..ed3b01041cf0 100644
--- a/ee/billing/quota_usage.py
+++ b/ee/billing/quota_usage.py
@@ -1,6 +1,5 @@
import copy
import json
-from collections import defaultdict
from collections.abc import Callable, Sequence
from time import sleep
from typing import Any, TypeVar
@@ -17,7 +16,6 @@ from posthog.models.organization import Organization
logger = structlog.get_logger(__name__)
UsagePatch = tuple[list[str], Any]
-_PatchRow = tuple[UUID, str | None, list[Any]]
T = TypeVar("T")
@@ -34,6 +32,25 @@ def retry_quota_database_operation(operation: Callable[[], T]) -> T:
return operation()
+def _resource_patch(operations: Sequence[UsagePatch]) -> dict[str, dict[str, Any]]:
+ """Folds [resource, key] operations into one set map and one delete list for each resource.
+
+ The last operation on a key wins.
+ """
+ patch: dict[str, dict[str, Any]] = {}
+ for (resource, key), value in operations:
+ entry = patch.setdefault(resource, {"set": {}, "delete": []})
+ if value is None:
+ entry["set"].pop(key, None)
+ if key not in entry["delete"]:
+ entry["delete"].append(key)
+ else:
+ entry["set"][key] = value
+ if key in entry["delete"]:
+ entry["delete"].remove(key)
+ return patch
+
+
@frozen
class _QuotaAction:
organization: Organization
@@ -52,6 +69,8 @@ class QuotaUsageBatch:
self._actions: dict[UUID, list[_QuotaAction]] = {}
def patch(self, organization: Organization, operations: Sequence[UsagePatch]) -> None:
+ if any(len(path) != 2 for path, _ in operations):
+ raise ValueError("QuotaUsageBatch patches only [resource, key] paths")
self._patches.setdefault(organization.id, []).extend(operations)
def require_unchanged_usage(self, organization: Organization, usage_md5: str) -> None:
@@ -96,48 +115,32 @@ class QuotaUsageBatch:
transaction.on_commit(action.report)
def _write(self) -> set[UUID]:
- groups: dict[tuple[tuple[tuple[str, ...], bool], ...], list[_PatchRow]] = defaultdict(list)
- for org_id, operations in self._patches.items():
- if operations:
- shape = tuple((tuple(path), value is None) for path, value in operations)
- values = [value for _, value in operations if value is not None]
- groups[shape].append((org_id, self._usage_md5.get(org_id), values))
-
- skipped: set[UUID] = set()
- if not groups:
- return skipped
-
- # Each shape can need a separate statement; all of an organization's patches commit together.
- with transaction.atomic():
- for shape, rows in groups.items():
- expression = "o.usage"
- parameters: list[Any] = []
- value_index = 0
- for path, delete in shape:
- if delete:
- expression = f"({expression}) #- %s::text[]"
- else:
- expression = f"jsonb_set({expression}, %s::text[], v.v{value_index})"
- value_index += 1
- parameters.append(list(path))
-
- array_sql = "%s::uuid[], %s::text[]" + ", %s::jsonb[]" * value_index
- parameters.append([org_id for org_id, _, _ in rows])
- parameters.append([usage_md5 for _, usage_md5, _ in rows])
- parameters.extend([json.dumps(values[index]) for _, _, values in rows] for index in range(value_index))
- columns = ", ".join(["id", "usage_md5", *(f"v{i}" for i in range(value_index))])
- table = Organization._meta.db_table
- sql = (
- f"UPDATE {table} AS o SET usage = {expression} "
- f"FROM UNNEST({array_sql}) AS v({columns}) "
- "WHERE o.id = v.id AND (v.usage_md5 IS NULL OR md5(o.usage::text) = v.usage_md5) "
- "RETURNING o.id"
- )
- with connection.cursor() as cursor:
- # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (identifiers come from model metadata and fixed templates; paths and values are parameters)
- cursor.execute(sql, parameters)
- written = {row[0] for row in cursor.fetchall()}
- skipped.update(
- org_id for org_id, usage_md5, _ in rows if usage_md5 is not None and org_id not in written
- )
- return skipped
+ rows = [
+ (org_id, self._usage_md5.get(org_id), _resource_patch(operations))
+ for org_id, operations in self._patches.items()
+ if operations
+ ]
+ if not rows:
+ return set()
+
+ # Skip a resource the row lacks instead of creating a partial resource object without its limit and usage.
+ sql = (
+ f"UPDATE {Organization._meta.db_table} AS o SET usage = o.usage || coalesce(("
+ "SELECT jsonb_object_agg(r.key, ((o.usage -> r.key) || (r.value -> 'set')) "
+ "- ARRAY(SELECT jsonb_array_elements_text(r.value -> 'delete'))) "
+ "FROM jsonb_each(v.patch) AS r WHERE jsonb_typeof(o.usage -> r.key) = 'object'"
+ "), '{}'::jsonb) "
+ "FROM UNNEST(%s::uuid[], %s::text[], %s::jsonb[]) AS v(id, usage_md5, patch) "
+ "WHERE o.id = v.id AND (v.usage_md5 IS NULL OR md5(o.usage::text) = v.usage_md5) "
+ "RETURNING o.id"
+ )
+ parameters: list[Any] = [
+ [org_id for org_id, _, _ in rows],
+ [usage_md5 for _, usage_md5, _ in rows],
+ [json.dumps(patch) for _, _, patch in rows],
+ ]
+ with connection.cursor() as cursor:
+ # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (identifiers come from model metadata and fixed templates; resources, keys and values are parameters)
+ cursor.execute(sql, parameters)
+ written = {row[0] for row in cursor.fetchall()}
+ return {org_id for org_id, usage_md5, _ in rows if usage_md5 is not None and org_id not in written}
diff --git a/ee/billing/test/test_quota_limiting.py b/ee/billing/test/test_quota_limiting.py
index d07e4bd6084c..2776c77ef9ff 100644
--- a/ee/billing/test/test_quota_limiting.py
+++ b/ee/billing/test/test_quota_limiting.py
@@ -2685,6 +2685,70 @@ class TestIdentifyRefreshCandidates(SimpleTestCase):
{},
False,
),
+ (
+ "zero_limit_block_is_not_a_candidate",
+ {
+ "posthog_code_credits": {
+ "usage": 0,
+ "limit": 0,
+ "todays_usage": 0,
+ "quota_limited_until": 1_700_000_000,
+ },
+ "period": _PERIOD,
+ },
+ _full_usage_counters(),
+ ["phc_blocked"],
+ {"posthog_code_credits": ["phc_blocked"]},
+ {},
+ False,
+ ),
+ (
+ "zero_limit_block_does_not_hide_another_resource",
+ {
+ "posthog_code_credits": {
+ "usage": 0,
+ "limit": 0,
+ "todays_usage": 0,
+ "quota_limited_until": 1_700_000_000,
+ },
+ "events": {"usage": 0, "limit": 1_000, "quota_limited_until": 1_700_000_000},
+ "period": _PERIOD,
+ },
+ _full_usage_counters(),
+ [],
+ {},
+ {},
+ True,
+ ),
+ (
+ "zero_limit_with_overage_buffer_is_a_candidate",
+ {
+ "recordings": {"usage": 0, "limit": 0, "todays_usage": 0, "quota_limited_until": 1_700_000_000},
+ "period": _PERIOD,
+ },
+ _full_usage_counters(),
+ [],
+ {},
+ {},
+ True,
+ ),
+ (
+ "zero_signals_limit_is_a_candidate",
+ {
+ "signals_credits": {
+ "usage": 0,
+ "limit": 0,
+ "todays_usage": 0,
+ "quota_limited_until": 1_700_000_000,
+ },
+ "period": _PERIOD,
+ },
+ _full_usage_counters(),
+ [],
+ {},
+ {},
+ True,
+ ),
]
)
def test_candidate_selection(
@@ -3296,10 +3360,13 @@ class TestRefreshOrgSelfDrivingQuota(BaseTest):
QuotaResource.SIGNALS_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
+ @parameterized.expand([("recount_succeeds", False), ("recount_fails", True)])
@patch("posthoganalytics.capture")
@patch("posthoganalytics.feature_enabled", return_value=False)
@time_machine.travel("2026-06-15T12:00:00Z", tick=False)
- def test_quota_cron_recounts_live_instead_of_writing_its_stale_snapshot(self, _feature_enabled, _capture) -> None:
+ def test_quota_cron_recounts_live_instead_of_writing_its_stale_snapshot(
+ self, _name, recount_fails, _feature_enabled, _capture
+ ) -> None:
# The cron snapshots fleet usage before its org loop, so a push-refresh that fires while
# the loop runs (a PR landing) writes a fresher, higher todays_usage than the snapshot.
# Writing the snapshot back would unpause the over-limit org until the next tick; the
@@ -3318,12 +3385,14 @@ class TestRefreshOrgSelfDrivingQuota(BaseTest):
return_value=[(self.team.id, 100)],
),
patch(
- "ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_org", return_value=1500
+ "ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_orgs",
+ return_value={str(self.organization.id): 1500},
+ side_effect=DatabaseError("Signals usage unavailable") if recount_fails else None,
) as live_mock,
):
update_all_orgs_billing_quotas()
- assert live_mock.call_args.args[0] == str(self.organization.id)
+ assert live_mock.call_args.args[0] == [str(self.organization.id)]
self.organization.refresh_from_db()
assert self.organization.usage is not None
assert self.organization.usage["signals_credits"]["todays_usage"] == 1500
@@ -3331,6 +3400,34 @@ class TestRefreshOrgSelfDrivingQuota(BaseTest):
QuotaResource.SIGNALS_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
)
+ @patch("posthoganalytics.capture")
+ @patch("posthoganalytics.feature_enabled", return_value=False)
+ @time_machine.travel("2026-06-15T12:00:00Z", tick=False)
+ def test_quota_cron_keeps_zero_limit_block_without_recounting(self, _feature_enabled, _capture) -> None:
+ self._set_self_driving_usage(0)
+ assert self.organization.usage is not None
+ self.organization.usage["posthog_code_credits"] = {
+ "usage": 0,
+ "todays_usage": 0,
+ "limit": 0,
+ "quota_limited_until": 1782864000,
+ }
+ self.organization.save()
+ replace_limited_team_tokens(
+ QuotaResource.POSTHOG_CODE_CREDITS,
+ {self.team.api_token: 1782864000},
+ QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY,
+ )
+ with patch(
+ "ee.billing.quota_limiting.get_self_driving_credits_used_in_period_for_orgs", return_value={}
+ ) as live_mock:
+ update_all_orgs_billing_quotas()
+
+ assert all(str(self.organization.id) not in call.args[0] for call in live_mock.call_args_list)
+ assert self.team.api_token in list_limited_team_attributes(
+ QuotaResource.POSTHOG_CODE_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY
+ )
+
@parameterized.expand(
[
("kept_when_added_after_snapshot", False, 1798761600, True),
diff --git a/ee/billing/test/test_quota_usage.py b/ee/billing/test/test_quota_usage.py
index 5e9d806783f4..77020d181e74 100644
--- a/ee/billing/test/test_quota_usage.py
+++ b/ee/billing/test/test_quota_usage.py
@@ -21,7 +21,7 @@ from ee.billing.quota_usage import QuotaUsageBatch
class TestQuotaUsageBatch(BaseTest):
@parameterized.expand([("set", False), ("delete_only", True)])
- def test_sql_shape_is_independent_of_batch_size(self, _name, delete_only) -> None:
+ def test_one_statement_for_any_batch_size_or_patch_shape(self, _name, delete_only) -> None:
other = Organization.objects.create(name="Other organization", slug="other-quota-organization")
organizations = [self.organization, other]
usage = {"events": {"usage": 10, "todays_usage": 0, "quota_limited_until": 1234}}
@@ -37,23 +37,21 @@ class TestQuotaUsageBatch(BaseTest):
for batch_size in (1, 2):
batch = QuotaUsageBatch()
for index, org in enumerate(organizations[:batch_size], start=1):
- batch.patch(
- org,
- [
- (["events", "todays_usage"], None if delete_only else index * 20),
- (["events", "quota_limited_until"], None if delete_only else index * 100),
- ],
- )
+ operations = [(["events", "todays_usage"], None if delete_only else index * 20)]
+ if index == 2:
+ operations.append((["events", "quota_limited_until"], None if delete_only else index * 100))
+ batch.patch(org, operations)
batch.write()
assert len(statements) == 2
assert statements[0] == statements[1]
- for index, org in enumerate(organizations, start=1):
- org.refresh_from_db()
- expected = {"usage": 10}
- if not delete_only:
- expected.update(todays_usage=index * 20, quota_limited_until=index * 100)
- assert org.usage == {"events": expected}
+ first, second = (Organization.objects.get(id=org.id).usage for org in organizations)
+ if delete_only:
+ assert first == {"events": {"usage": 10, "quota_limited_until": 1234}}
+ assert second == {"events": {"usage": 10}}
+ else:
+ assert first == {"events": {"usage": 10, "todays_usage": 20, "quota_limited_until": 1234}}
+ assert second == {"events": {"usage": 10, "todays_usage": 40, "quota_limited_until": 200}}
@parameterized.expand([("all_present", False), ("deleted_organization", True)])
def test_multiple_patches_preserve_other_organizations_and_fields(self, _name, delete_second) -> None:
@@ -63,6 +61,7 @@ class TestQuotaUsageBatch(BaseTest):
batch = QuotaUsageBatch()
batch.patch(self.organization, [(["events", "todays_usage"], 20)])
batch.patch(self.organization, [(["events", "quota_limited_until"], 1234)])
+ batch.patch(self.organization, [(["recordings", "todays_usage"], 5)])
batch.patch(other, [(["events", "todays_usage"], 30)])
if delete_second:
other.delete()
@@ -83,7 +82,7 @@ class TestQuotaUsageBatch(BaseTest):
)
@patch("posthoganalytics.capture")
@patch("ee.billing.quota_usage.sleep")
- def test_failed_statement_rolls_back_batch_before_retry_or_events(
+ def test_failed_write_leaves_rows_unchanged_and_sends_no_events(
self, _name, error_type, failures, _sleep, capture
) -> None:
other = Organization.objects.create(name="Other organization", slug="other-quota-organization")
@@ -95,23 +94,20 @@ class TestQuotaUsageBatch(BaseTest):
properties = {"usage": {"events": 20}}
batch.report_action(self.organization, "organization quota limits changed", properties)
properties["usage"]["events"] = 999
- writes = 0
errors = 0
- def fail_second_statement(execute, sql, params, many, context):
- nonlocal writes, errors
+ def fail_first_attempts(execute, sql, params, many, context):
+ nonlocal errors
if sql.startswith("UPDATE posthog_organization AS o"):
- writes += 1
- if writes % 2 == 1:
- assert Organization.objects.get(id=self.organization.id).usage == usage
- assert Organization.objects.get(id=other.id).usage == usage
- elif errors < failures:
+ assert Organization.objects.get(id=self.organization.id).usage == usage
+ assert Organization.objects.get(id=other.id).usage == usage
+ if errors < failures:
errors += 1
raise error_type("synthetic write failure")
return execute(sql, params, many, context)
succeeds = error_type is OperationalError and failures == 1
- with connection.execute_wrapper(fail_second_statement), self.captureOnCommitCallbacks(execute=True):
+ with connection.execute_wrapper(fail_first_attempts), self.captureOnCommitCallbacks(execute=True):
if succeeds:
batch.write()
batch.publish()
diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.py
index 3ff9928703ab..d8f5c8b2410d 100644
--- a/posthog/management/commands/start_temporal_worker.py
+++ b/posthog/management/commands/start_temporal_worker.py
@@ -55,7 +55,7 @@ from posthog.temporal.common.interceptor import is_task_queue_supported
from posthog.temporal.common.liveness_tracker import LivenessInterceptor, get_liveness_tracker
from posthog.temporal.common.logger import configure_logger, get_logger
from posthog.temporal.common.shutdown import ShutdownSignalListener
-from posthog.temporal.common.worker import ActivityWorkerConfig, ManagedWorker, create_worker
+from posthog.temporal.common.worker import ManagedWorker, create_worker
from posthog.temporal.common.zombie_exit import ZombieActivityExit
from posthog.temporal.data_modeling import (
ACTIVITIES as DATA_MODELING_ACTIVITIES,
@@ -527,6 +527,7 @@ _task_queue_specs = [
+ BILLING_ALERTS_ACTIVITIES
+ BILLING_ACTIVE_SYNC_ACTIVITIES,
),
+ (settings.USAGE_REPORT_AGGREGATION_TASK_QUEUE, [], USAGE_REPORTS_AGGREGATION_ACTIVITIES),
(
settings.VIDEO_EXPORT_TASK_QUEUE,
SIGNALS_PRODUCT_WORKFLOWS
@@ -672,15 +673,6 @@ for task_queue_name, workflows_for_queue, activities_for_queue in _task_queue_sp
WORKFLOWS_DICT = _workflows
ACTIVITIES_DICT = _activities
-ACTIVITY_WORKER_CONFIGS = {
- settings.BILLING_TASK_QUEUE: (
- ActivityWorkerConfig(
- task_queue=settings.USAGE_REPORT_AGGREGATION_TASK_QUEUE,
- activities=USAGE_REPORTS_AGGREGATION_ACTIVITIES,
- max_concurrent_activities=1,
- ),
- ),
-}
def workflows_include_data_import_syncs(workflows: collections.abc.Iterable[type]) -> bool:
@@ -990,7 +982,6 @@ class Command(BaseCommand):
),
enable_combined_metrics_server=not disable_combined_metrics_server,
enable_open_telemetry_plugin=enable_otel,
- activity_worker_configs=ACTIVITY_WORKER_CONFIGS.get(task_queue, ()),
)
)
diff --git a/posthog/management/commands/test/test_start_temporal_worker.py b/posthog/management/commands/test/test_start_temporal_worker.py
index e7e3e4be983d..a9c4c353e211 100644
--- a/posthog/management/commands/test/test_start_temporal_worker.py
+++ b/posthog/management/commands/test/test_start_temporal_worker.py
@@ -6,7 +6,6 @@ from django.conf import settings
from posthog.management.commands.start_temporal_worker import (
ACTIVITIES_DICT,
- ACTIVITY_WORKER_CONFIGS,
AI_WORKFLOWS,
ALERT_AI_QUEUE_ACTIVITIES,
DATA_SYNC_WORKFLOWS,
@@ -19,6 +18,7 @@ from posthog.management.commands.start_temporal_worker import (
workflows_include_data_import_syncs,
)
from posthog.temporal.usage_report.activities import aggregate_and_chunk_org_reports
+from posthog.temporal.usage_report.workflow import RunUsageReportsWorkflow
from products.alerts_platform.backend.facade.temporal import (
DELIVERY_ACTIVITIES,
@@ -43,14 +43,19 @@ class _NotADataSyncWorkflow:
pass
-def test_billing_registers_aggregation_with_separate_capacity() -> None:
- (aggregation_worker,) = ACTIVITY_WORKER_CONFIGS[settings.BILLING_TASK_QUEUE]
- assert aggregation_worker.task_queue == settings.USAGE_REPORT_AGGREGATION_TASK_QUEUE
- assert aggregation_worker.task_queue != settings.BILLING_TASK_QUEUE
- assert aggregation_worker.max_concurrent_activities == 1
- assert aggregation_worker.activities == [aggregate_and_chunk_org_reports]
- # A workflow may have scheduled aggregation before the queue change deployed.
- assert aggregate_and_chunk_org_reports in ACTIVITIES_DICT[settings.BILLING_TASK_QUEUE]
+# Queue settings collapse to a single dev queue under DEBUG, so match on the spec entries rather than on queue names.
+def test_aggregation_runs_on_its_own_queue_and_the_billing_queue() -> None:
+ aggregation_entries = [
+ activities
+ for task_queue, workflows, activities in _task_queue_specs
+ if task_queue == settings.USAGE_REPORT_AGGREGATION_TASK_QUEUE and not workflows
+ ]
+ assert any(aggregate_and_chunk_org_reports in activities for activities in aggregation_entries)
+ # A workflow may have scheduled aggregation on the billing queue before the queue change deployed.
+ (billing_activities,) = [
+ activities for _, workflows, activities in _task_queue_specs if RunUsageReportsWorkflow in workflows
+ ]
+ assert aggregate_and_chunk_org_reports in billing_activities
@pytest.mark.parametrize(
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py
index 3054c2371eff..4b52f0104dda 100644
--- a/posthog/settings/temporal.py
+++ b/posthog/settings/temporal.py
@@ -372,8 +372,8 @@ TASKS_WORKFLOW_DISPATCH_OUTBOX_ENABLED: bool = get_from_env(
STAMPHOG_TASK_QUEUE = _set_temporal_task_queue("stamphog-task-queue")
TEST_TASK_QUEUE = _set_temporal_task_queue("test-task-queue")
BILLING_TASK_QUEUE = _set_temporal_task_queue("billing-task-queue")
-# Keep aggregation separate even when local development combines the other queues.
-USAGE_REPORT_AGGREGATION_TASK_QUEUE = f"{BILLING_TASK_QUEUE}-usage-report-aggregation"
+# A dedicated worker deployment polls this queue, so aggregation does not share a process with quota limiting.
+USAGE_REPORT_AGGREGATION_TASK_QUEUE = _set_temporal_task_queue("billing-task-queue-usage-report-aggregation")
VIDEO_EXPORT_TASK_QUEUE = _set_temporal_task_queue("video-export-task-queue")
ANALYTICS_PLATFORM_TASK_QUEUE = _set_temporal_task_queue("analytics-platform-task-queue")
# Keep the smoke fleets separate in local development as well as deployed environments.
diff --git a/posthog/tasks/test/test_usage_report.py b/posthog/tasks/test/test_usage_report.py
index c26f168d1fce..61538a07a9f9 100644
--- a/posthog/tasks/test/test_usage_report.py
+++ b/posthog/tasks/test/test_usage_report.py
@@ -70,6 +70,7 @@ from posthog.tasks.usage_report import (
capture_report,
get_all_event_metrics_in_period,
get_instance_metadata,
+ get_self_driving_credits_used_in_period_for_orgs,
get_teams_with_ai_credits_used_in_period,
get_teams_with_billable_event_count_in_period,
get_teams_with_event_count_with_groups_in_period,
@@ -5086,6 +5087,24 @@ class TestAIEventsUsageReport(ClickhouseDestroyTablesMixin, TestCase, Clickhouse
self.assertEqual(result, [(team.id, 1500)])
+ def test_self_driving_credits_for_orgs_sum_each_orgs_teams(self) -> None:
+ org_a = Organization.objects.create(name="Org A")
+ org_b = Organization.objects.create(name="Org B")
+ team_a1 = Team.objects.create(organization=org_a)
+ team_a2 = Team.objects.create(organization=org_a)
+ team_b = Team.objects.create(organization=org_b)
+ period = get_previous_day(at=now() + relativedelta(days=1))
+ org_ids = [str(org_a.id), str(org_b.id)]
+
+ with patch(
+ "posthog.tasks.usage_report.get_signals_billing_credits_by_team",
+ return_value=[(team_a1.id, 1500), (team_a2.id, 1500), (team_b.id, 1500)],
+ ) as signals_credits:
+ totals = get_self_driving_credits_used_in_period_for_orgs(org_ids, period.start, period.end)
+
+ assert totals == {str(org_a.id): 3000, str(org_b.id): 1500}
+ assert signals_credits.call_args.kwargs["organization_ids"] == org_ids
+
@patch("posthog.tasks.usage_report.get_instance_region")
def test_ai_credits_counts_billable_generation_with_no_trace(self, mock_region: MagicMock) -> None:
"""A billable generation with no matching $ai_trace bills via the empty-trace fallback.
diff --git a/posthog/tasks/usage_report.py b/posthog/tasks/usage_report.py
index a80f7530b03a..f6663ee58106 100644
--- a/posthog/tasks/usage_report.py
+++ b/posthog/tasks/usage_report.py
@@ -6,7 +6,7 @@ import base64
import logging
import dataclasses
from collections import Counter, defaultdict
-from collections.abc import Callable, Iterable, Iterator, Sequence
+from collections.abc import Callable, Collection, Iterable, Iterator, Sequence
from datetime import datetime, timedelta
from typing import Any, Literal, Optional, TypedDict, Union
@@ -2090,6 +2090,23 @@ def get_signals_credited_refund_credits_for_org(
return credited_refund_credits_for_org(organization_id, begin, end)
+def get_self_driving_credits_used_in_period_for_orgs(
+ organization_ids: Collection[str], begin: datetime, end: datetime
+) -> dict[str, int]:
+ """Live self-driving credits used in `[begin, end)` for each org, from one read. Orgs with no usage are absent."""
+ if not organization_ids:
+ return {}
+ rows = get_signals_billing_credits_by_team(begin, end, organization_ids=list(organization_ids))
+ if not rows:
+ return {}
+ team_orgs = dict(Team.objects.filter(id__in=[team_id for team_id, _ in rows]).values_list("id", "organization_id"))
+ totals: dict[str, int] = defaultdict(int)
+ for team_id, credits in rows:
+ if (organization_id := team_orgs.get(team_id)) is not None:
+ totals[str(organization_id)] += credits
+ return dict(totals)
+
+
def get_self_driving_credits_used_in_period_for_org(
organization_id: str | uuid.UUID, begin: datetime, end: datetime
) -> int:
diff --git a/products/signals/backend/billing.py b/products/signals/backend/billing.py
index 56e228c480f3..99bb4e56895d 100644
--- a/products/signals/backend/billing.py
+++ b/products/signals/backend/billing.py
@@ -37,7 +37,7 @@ would otherwise be repeatable free work.
import uuid
from collections import defaultdict
-from collections.abc import Sequence
+from collections.abc import Collection, Sequence
from datetime import datetime
from typing import TYPE_CHECKING, NamedTuple
@@ -327,7 +327,10 @@ def current_billing_period_bounds(organization: "Organization") -> BillingPeriod
def get_signals_billing_credits_by_team(
- begin: datetime, end: datetime, organization_id: str | uuid.UUID | None = None
+ begin: datetime,
+ end: datetime,
+ organization_id: str | uuid.UUID | None = None,
+ organization_ids: Collection[str | uuid.UUID] | None = None,
) -> list[tuple[int, int]]:
"""Signals credits used per team in `[begin, end)`.
@@ -336,7 +339,8 @@ def get_signals_billing_credits_by_team(
such run exists before `begin`. Each billable report is charged a flat
`SIGNALS_CREDITS_PER_REPORT_WITH_PR`. Returns `[(team_id, credits), ...]` for teams with
non-zero usage only. `organization_id` narrows the scan to one org (the widget's live-count
- path); usage reporting passes nothing and stays fleet-wide.
+ path), and `organization_ids` to a set of orgs (the quota run's recount). Usage reporting passes
+ neither and stays fleet-wide.
The entry scan is bounded by PRs shipped in the period (via the `created_at` +
`output__pr_url` indexes), not by the total number of reports, task runs, or teams.
@@ -350,6 +354,8 @@ def get_signals_billing_credits_by_team(
)
if organization_id is not None:
bridges = bridges.filter(team__organization_id=organization_id)
+ if organization_ids is not None:
+ bridges = bridges.filter(team__organization_id__in=organization_ids)
report_team: dict[uuid.UUID, int] = {}
for report_id, team_id in bridges.values_list("report_id", "team_id"):
report_team.setdefault(report_id, team_id)
diff --git a/products/signals/backend/test/test_billing.py b/products/signals/backend/test/test_billing.py
index 7552c9b381f8..c1ee2caadb00 100644
--- a/products/signals/backend/test/test_billing.py
+++ b/products/signals/backend/test/test_billing.py
@@ -7,7 +7,7 @@ from django.apps import apps
from parameterized import parameterized
-from posthog.models import Team
+from posthog.models import Organization, Team
from products.signals.backend.artefact_schemas import TASK_RUN_TYPE_IMPLEMENTATION, TASK_RUN_TYPE_RESEARCH
from products.signals.backend.billing import (
@@ -129,6 +129,19 @@ class TestSignalsBilling(BaseTest):
# 1 credit = $0.01, so $15 == 1500 credits.
self.assertEqual(SIGNALS_CREDITS_PER_REPORT_WITH_PR, 1500)
+ @parameterized.expand([("organization_id", False), ("organization_ids", True)])
+ def test_organization_filter_excludes_other_organizations(self, _name: str, as_list: bool) -> None:
+ other_team = Team.objects.create(organization=Organization.objects.create(name="Other org"))
+ for team in (self.team, other_team):
+ self._pr_run(self._report(team=team), created_at=_at(10), team=team)
+
+ org_filter = (
+ {"organization_ids": [self.organization.id]} if as_list else {"organization_id": self.organization.id}
+ )
+ self.assertEqual(
+ dict(get_signals_billing_credits_by_team(PERIOD_START, PERIOD_END, **org_filter)), {self.team.id: 1500}
+ )
+
def test_report_with_pr_billed_flat(self) -> None:
report = self._report()
self._pr_run(report, created_at=_at(10))