:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(posthog-ai): rate-limit only eligible context selections and record every outcome (#130937)
Adam Bowkere12900be001d
diff --git a/docs/published/handbook/engineering/ai/sandboxed-agents.md b/docs/published/handbook/engineering/ai/sandboxed-agents.mdindex e84a49807bd4..fdd13c384924 100644--- a/docs/published/handbook/engineering/ai/sandboxed-agents.md+++ b/docs/published/handbook/engineering/ai/sandboxed-agents.md@@ -761,7 +761,10 @@ Before a human prompt reaches Claude or Codex, the sandbox calls the task-boundFollow-ups delivered through the user-message command, including automated CI and peer messages, can trigger selection.Internal retry continuations, steering, and slash commands do not trigger selection.Shadow and treatment require the organization's existing AI data processing consent before retrieval or model calls.-Sandbox OAuth requests share project rate limits of 30 preparations per minute and 300 per hour.+Shadow and treatment selections share project rate limits of 30 preparations per minute and 300 per hour.+The limits apply after the eligibility, variant, and consent checks, so ineligible runs, such as scouts, never consume them.+A selection over the limit returns `rate_limited` and skips retrieval and model calls.+Each sandbox task can also call the selection endpoint at most 30 times per minute, whatever its eligibility.System One first checks whether organizational context could help, using the request and bounded conversation history.When the gate passes, the selector calls the existing skill search, catalog list/search, and Business Knowledge search APIs with the sandbox bearer credential.@@ -782,7 +785,7 @@ The agent is instructed to use these references silently, including during progrIt can cite the underlying sources and explain verification failures, but must not mention selection, suggestions, injection, or the hidden block's metadata.Control skips retrieval. Shadow computes the selected bundle without injecting it. Treatment injects the bundle.-Selection has a three-second budget by default, starting before worker submission and run/actor lookups. Expired lookups do not start source or model requests.+Selection has an eight-second budget by default, starting before worker submission and run/actor lookups. Expired lookups do not start source or model requests.Saturation, timeout, and selection failures leave the ordinary prompt flow available.Source requests run concurrently and use at most half the remaining budget, leaving time to score ready results.A denied, failed, or slow source does not suppress results from other sources.diff --git a/packages/agent/packages/agent/src/posthog-api.ts b/packages/agent/packages/agent/src/posthog-api.tsindex ac9181868e6e..f24f6add4cbc 100644--- a/packages/agent/packages/agent/src/posthog-api.ts+++ b/packages/agent/packages/agent/src/posthog-api.ts@@ -116,7 +116,7 @@ export class PostHogAPIClient {{method: "POST",body: JSON.stringify(input),- signal: AbortSignal.timeout(5_000),+ signal: AbortSignal.timeout(9_000),},);return contextSelectionResponseSchema.parse(response);diff --git a/posthog/rate_limit.py b/posthog/rate_limit.pyindex 1e0ac6127b16..020d32dcbd65 100644--- a/posthog/rate_limit.py+++ b/posthog/rate_limit.py@@ -1839,6 +1839,22 @@ class ContextSelectionSustainedThrottle(_TeamBucketRateThrottle):rate = "300/hour"+class ContextSelectionTaskThrottle(PersonalApiKeyOrUserRateThrottle):+ """Caps prepare calls per sandbox task, whether or not the task is eligible for selection.++ The project-wide selection limits count only eligible selections, so this per-task ceiling is+ what stops one looping sandbox from holding the shared request slots or flooding outcome events.+ """++ scope = "context_selection_task"+ rate = "30/minute"++ def get_cache_key(self, request: "Request", view: "APIView") -> str:+ access_token = getattr(getattr(request, "successful_authenticator", None), "access_token", None)+ ident = getattr(access_token, "sandbox_task_id", None) or request.user.pk or self.get_ident(request)+ return self.cache_format % {"scope": self.scope, "ident": ident}++class TaskRunChartRenderThrottle(PersonalApiKeyOrUserRateThrottle):# A chart render holds a worker for up to 90s, so it needs a far lower ceiling than the# ClickHouse throttles. Keyed per team so rotating API keys doesn't split the bucket.diff --git a/posthog/settings/web.py b/posthog/settings/web.pyindex 9efeadc0f288..e02447007d3c 100644--- a/posthog/settings/web.py+++ b/posthog/settings/web.py@@ -1638,4 +1638,4 @@ WEB_ANALYTICS_SESSION_ID_SET_TEAM_IDS: list[int] = [# 0 rejects every stream (emergency lever).SSE_MAX_CONCURRENT_STREAMS_PER_PROCESS = get_from_env("SSE_MAX_CONCURRENT_STREAMS_PER_PROCESS", 500, type_cast=int)-CONTEXT_SELECTION_TIMEOUT_SECONDS = 3.0+CONTEXT_SELECTION_TIMEOUT_SECONDS = 8.0diff --git a/products/context_layer/backend/presentation/selection.py b/products/context_layer/backend/presentation/selection.pyindex 0c3404d99ea4..28beffa8a95b 100644--- a/products/context_layer/backend/presentation/selection.py+++ b/products/context_layer/backend/presentation/selection.py@@ -16,7 +16,11 @@ from posthog.api.routing import TeamAndOrgViewSetMixinfrom posthog.models.user import Userfrom posthog.oauth_provenance import get_oauth_access_token, is_sandbox_oauth_requestfrom posthog.permissions import APIScopePermission-from posthog.rate_limit import ContextSelectionBurstThrottle, ContextSelectionSustainedThrottle+from posthog.rate_limit import (+ ContextSelectionBurstThrottle,+ ContextSelectionSustainedThrottle,+ ContextSelectionTaskThrottle,+)from products.context_layer.backend.facade import selection@@ -70,7 +74,7 @@ class ContextSelectionViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):@action(detail=False,methods=["post"],- throttle_classes=[ContextSelectionBurstThrottle, ContextSelectionSustainedThrottle],+ throttle_classes=[ContextSelectionTaskThrottle],)def prepare(self, request: Request, **kwargs: object) -> Response:serializer = PrepareSerializer(data=request.data)@@ -84,6 +88,13 @@ class ContextSelectionViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):data = dict(serializer.validated_data)run_id = data.pop("run_id")authorization = get_authorization_header(request).decode("ascii")++ def admit() -> bool:+ return all(+ throttle.allow_request(request, self)+ for throttle in (ContextSelectionBurstThrottle(), ContextSelectionSustainedThrottle())+ )+try:result = selection.prepare_context_selection(self.team_id,@@ -92,6 +103,7 @@ class ContextSelectionViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):request.user,selection.SelectionInput(**data),authorization,+ admit,)except selection.SelectionDenied as error:raise PermissionDenied(str(error)) from errordiff --git a/products/context_layer/backend/selection.py b/products/context_layer/backend/selection.pyindex 4ad17c3d55c3..8ee5245aeb6c 100644--- a/products/context_layer/backend/selection.py+++ b/products/context_layer/backend/selection.py@@ -104,6 +104,19 @@ class SelectionUnavailable(RuntimeError):pass+def elapsed_ms(started: float) -> int:+ return round((time.monotonic() - started) * 1000)+++def is_timeout(error: BaseException) -> bool:+ current: BaseException | None = error+ while current is not None:+ if isinstance(current, (TimeoutError, httpx.TimeoutException)):+ return True+ current = current.__cause__+ return False++def time_remaining(deadline: float) -> float:remaining = deadline - time.monotonic()if remaining <= 0:@@ -158,8 +171,21 @@ def selection_mode(run: TaskRun, actor: User) -> str:class ContextSelector:- def __init__(self, run: TaskRun, actor: User, selection: SelectionInput, authorization: str) -> None:+ def __init__(+ self,+ run: TaskRun,+ actor: User,+ selection: SelectionInput,+ authorization: str,+ admit: Callable[[], bool] = lambda: True,+ ) -> None:self.run = run+ self.admit = admit+ self.deadline: float | None = None+ self.gate_probability: float | None = None+ self.step_ms: dict[str, int] = {}+ self.source_ms: dict[str, int] = {}+ self.source_errors: dict[str, str] = {}self.actor = actorself.selection = selectionself._authorization = authorization@@ -173,6 +199,13 @@ class ContextSelector:self.query += "\n" + history[-(MAX_SEARCH_LENGTH - len(self.query) - 1) :]def search(self, kind: SourceKind, deadline: float) -> list[Candidate]:+ started = time.monotonic()+ try:+ return self.search_source(kind, deadline)+ finally:+ self.source_ms[kind] = elapsed_ms(started)++ def search_source(self, kind: SourceKind, deadline: float) -> list[Candidate]:if not self.query:return []path, limit = SOURCE_ENDPOINTS[kind]@@ -262,10 +295,11 @@ class ContextSelector:try:candidates.extend(future.result())except Exception as error:- errors[pending[future]] = type(error).__name__+ errors[pending[future]] = "TimeoutError" if is_timeout(error) else type(error).__name__for future in unfinished:future.cancel()errors[pending[future]] = "TimeoutError"+ self.source_errors = errorsif errors:logger.info("context_selection_source_errors", selection_id=self.selection_id, errors=errors)return candidates@@ -326,17 +360,25 @@ class ContextSelector:scored.append((candidate, answer.probability))return scored+ def timed[T](self, step: str, operation: Callable[[], T]) -> T:+ started = time.monotonic()+ try:+ return operation()+ finally:+ self.step_ms[step] = elapsed_ms(started)+def select(self, deadline: float) -> PreparedContext:- gate = self.gate(deadline)+ gate = self.timed("gate", lambda: self.gate(deadline))+ self.gate_probability = gateif gate <= 0.3:- self.capture_outcome("gate_skipped", gate, [])+ self.capture_outcome("gate_skipped")return PreparedContext(selection_id=self.selection_id, mode=self.mode, reason="gate_skipped")remaining = time_remaining(deadline)- candidates = self.retrieve(deadline - remaining / 2)- scored = self.rerank(candidates, deadline)+ candidates = self.timed("retrieval", lambda: self.retrieve(deadline - remaining / 2))+ scored = self.timed("rerank", lambda: self.rerank(candidates, deadline))context = self.render(scored)reason = "selected" if context else "empty"- self.capture_outcome(reason, gate, scored)+ self.capture_outcome(reason, scored)return PreparedContext(selection_id=self.selection_id,mode=self.mode,@@ -344,10 +386,13 @@ class ContextSelector:reason=reason,)- def capture_outcome(self, reason: str, gate: float, scored: list[tuple[Candidate, float]]) -> None:+ def capture_outcome(+ self, reason: str, scored: list[tuple[Candidate, float]] | None = None, error_type: str | None = None+ ) -> None:+ scored = scored or []# The gateway drops System One inputs and answers from its events, so this is the only record of decisions.# The event goes to PostHog's own analytics, so it carries no customer-written names.- privacy_mode = is_privacy_mode_enabled(self.run.team)+ privacy_mode = is_privacy_mode_enabled(self.run.team) if scored else Nonecandidates: list[dict[str, object]] = []for candidate, score in sorted(scored, key=lambda pair: -pair[1]):entry: dict[str, object] = {"kind": candidate.kind, "score": round(score, 4)}@@ -364,14 +409,30 @@ class ContextSelector:"config_version": CONFIG_VERSION,"runtime_version": self.selection.runtime_version,"reason": reason,- "gate_probability": round(gate, 4),+ "error_type": error_type,+ "gate_probability": None if self.gate_probability is None else round(self.gate_probability, 4),"candidate_count": len(scored),+ "budget_ms": round(settings.CONTEXT_SELECTION_TIMEOUT_SECONDS * 1000),+ "duration_ms": self.duration_ms(),+ "gate_ms": self.step_ms.get("gate"),+ "retrieval_ms": self.step_ms.get("retrieval"),+ "rerank_ms": self.step_ms.get("rerank"),+ "source_ms": dict(self.source_ms),+ "source_errors": dict(self.source_errors),+ "source_timeout_count": sum(error == "TimeoutError" for error in self.source_errors.values()),+ "over_budget": reason == "timeout" or "TimeoutError" in self.source_errors.values(),"privacy_mode": privacy_mode,"candidates": candidates,},groups=groups(self.run.team.organization, self.run.team),)+ def duration_ms(self) -> int | None:+ if self.deadline is None:+ return None+ started = self.deadline - settings.CONTEXT_SELECTION_TIMEOUT_SECONDS+ return elapsed_ms(started)+def candidate_ref(self, candidate: Candidate) -> str:if candidate.kind != "skill":return candidate.id@@ -414,27 +475,43 @@ class ContextSelector:return header + body + footer if selected_count else ""def prepare(self, deadline: float) -> PreparedContext:+ self.deadline = deadlinetry:time_remaining(deadline)self.mode = selection_mode(self.run, self.actor)if self.mode == "disabled":return PreparedContext()- if not self.selection.prompt.strip() or self.mode == "control":- return PreparedContext(- selection_id=self.selection_id,- mode=self.mode,- reason="no_text" if not self.selection.prompt.strip() else "control",- )+ if not self.selection.prompt.strip():+ return PreparedContext(selection_id=self.selection_id, mode=self.mode, reason="no_text")+ if self.mode == "control":+ return self.finish("control")if self.run.team.organization.is_ai_data_processing_approved is not True:- return PreparedContext(selection_id=self.selection_id, mode=self.mode, reason="ai_consent_required")+ return self.finish("ai_consent_required")+ if not self.admit():+ return self.finish("rate_limited")return self.select(deadline)except Exception as error:- logger.warning("context_selection_failed", selection_id=self.selection_id, error_type=type(error).__name__)- return PreparedContext(selection_id=self.selection_id, mode=self.mode, reason="error")+ error_type = type(error).__name__+ logger.warning("context_selection_failed", selection_id=self.selection_id, error_type=error_type)+ try:+ return self.finish("timeout" if is_timeout(error) else "error", error_type)+ except Exception:+ logger.exception("context_selection_outcome_capture_failed", selection_id=self.selection_id)+ return PreparedContext(selection_id=self.selection_id, mode=self.mode, reason="error")++ def finish(self, reason: str, error_type: str | None = None) -> PreparedContext:+ self.capture_outcome(reason, error_type=error_type)+ return PreparedContext(selection_id=self.selection_id, mode=self.mode, reason=reason)def prepare_context_selection(- team_id: int, run_id: UUID, task_id: UUID, actor: User, selection: SelectionInput, authorization: str+ team_id: int,+ run_id: UUID,+ task_id: UUID,+ actor: User,+ selection: SelectionInput,+ authorization: str,+ admit: Callable[[], bool] = lambda: True,) -> PreparedContext:def execute(deadline: float) -> PreparedContext:time_remaining(deadline)@@ -450,7 +527,7 @@ def prepare_context_selection(if not is_current_task_run_actor(team_id, run_id, actor.id):raise SelectionDenied("The credential no longer belongs to the current actor.")time_remaining(deadline)- result = ContextSelector(run, actor, selection, authorization).prepare(deadline)+ result = ContextSelector(run, actor, selection, authorization, admit).prepare(deadline)if not is_current_task_run_actor(team_id, run_id, actor.id):raise SelectionDenied("The credential no longer belongs to the current actor.")return resultdiff --git a/products/context_layer/backend/test/test_api.py b/products/context_layer/backend/test/test_api.pyindex 021fd262ecb8..91627bcf436c 100644--- a/products/context_layer/backend/test/test_api.py+++ b/products/context_layer/backend/test/test_api.py@@ -20,7 +20,11 @@ from posthog.models.organization import Organizationfrom posthog.models.scoping import team_scopefrom posthog.models.team.team import Teamfrom posthog.models.user import User-from posthog.rate_limit import ContextSelectionBurstThrottle, ContextSelectionSustainedThrottle+from posthog.rate_limit import (+ ContextSelectionBurstThrottle,+ ContextSelectionSustainedThrottle,+ ContextSelectionTaskThrottle,+)from posthog.storage.object_storage import UnavailableStoragefrom posthog.temporal.oauth import POSTHOG_AI_APP_CLIENT_ID_DEVfrom posthog.utils import safe_cache_set@@ -1260,15 +1264,19 @@ class TestSelectionThrottleAPI(NonAtomicAPIBaseTest):@parameterized.expand([- ("burst", ContextSelectionBurstThrottle, "1/minute"),- ("sustained", ContextSelectionSustainedThrottle, "1/hour"),+ ("burst", ContextSelectionBurstThrottle, "1/minute", 200),+ ("sustained", ContextSelectionSustainedThrottle, "1/hour", 200),+ ("task", ContextSelectionTaskThrottle, "1/minute", 429),])def test_selection_rate_limit_covers_sandbox_oauth(self,name: str,- throttle: type[ContextSelectionBurstThrottle | ContextSelectionSustainedThrottle],+ throttle: type[+ ContextSelectionBurstThrottle | ContextSelectionSustainedThrottle | ContextSelectionTaskThrottle+ ],rate: str,+ limited_status: int,) -> None:task = apps.get_model("tasks", "Task").objects.create(team=self.team, created_by=self.user, title="Question", origin_product="posthog_ai"@@ -1318,6 +1326,8 @@ class TestSelectionThrottleAPI(NonAtomicAPIBaseTest):assert allowed.status_code == 200, allowed.contentassert model.call_count == 1limited = self.client.post(url, payload, format="json", HTTP_AUTHORIZATION=f"Bearer {tokens[1]}")- assert limited.status_code == 429, limited.content+ assert limited.status_code == limited_status, limited.content+ if limited_status == 200:+ assert limited.json()["reason"] == "rate_limited"assert model.call_count == 1source.assert_not_called()diff --git a/products/context_layer/backend/test/test_selection.py b/products/context_layer/backend/test/test_selection.pyindex 7727c5f301a2..77653303d32b 100644--- a/products/context_layer/backend/test/test_selection.py+++ b/products/context_layer/backend/test/test_selection.py@@ -5,7 +5,7 @@ from dataclasses import replacefrom threading import BoundedSemaphore, Eventfrom uuid import uuid4-from unittest.mock import patch+from unittest.mock import Mock, patchfrom django.test import SimpleTestCase, override_settings@@ -16,10 +16,17 @@ from rest_framework.parsers import JSONParserfrom rest_framework.request import Requestfrom rest_framework.test import APIRequestFactory-from posthog.llm.system_one import JsonValue, Question, SystemOneResult, parse_system_one_response+from posthog.llm.system_one import (+ JsonValue,+ Question,+ SystemOneRequestFailed,+ SystemOneResult,+ parse_system_one_response,+)from posthog.models.organization import Organizationfrom posthog.models.team.team import Teamfrom posthog.models.user import User+from posthog.rate_limit import ContextSelectionTaskThrottlefrom products.context_layer.backend.presentation.selection import ContextSelectionViewSetfrom products.context_layer.backend.selection import (@@ -219,14 +226,30 @@ class TestSelection(SimpleTestCase):self.assertEqual(result.reason, "selected")self.assertEqual(len(self.requests), 5)- @parameterized.expand([("control", False), ("disabled", False), ("shadow", False), ("treatment", True)])- def test_variants_control_retrieval_injection_and_call_metadata(self, mode: str, inject: bool) -> None:+ @parameterized.expand(+ [+ ("control", True, False),+ ("disabled", True, False),+ ("shadow", True, False),+ ("treatment", True, True),+ ("treatment", False, False),+ ]+ )+ def test_variants_control_retrieval_injection_and_call_metadata(+ self, mode: str, admitted: bool, inject: bool+ ) -> None:self.flag.return_value = mode+ self.selector.admit = Mock(return_value=admitted)result = self.selector.prepare(time.monotonic() + 60)+ selects = mode in ("shadow", "treatment")self.assertEqual(result.mode, mode)self.assertEqual(bool(result.context), inject)- self.assertEqual(bool(self.requests), mode in ("shadow", "treatment"))- if mode in ("shadow", "treatment"):+ self.assertEqual(self.selector.admit.called, selects)+ self.assertEqual(bool(self.requests), selects and admitted)+ if not admitted:+ self.assertEqual(result.reason, "rate_limited")+ self.client_factory.assert_not_called()+ elif selects:labels = self.client_factory.call_args.kwargs["properties"]self.assertEqual(labels["selection_id"], result.selection_id)self.assertEqual(labels["mode"], mode)@@ -284,8 +307,17 @@ class TestSelection(SimpleTestCase):self.assertEqual(result.context, "")self.assertEqual(result.reason, "error")- @parameterized.expand([("timeout", "error"), ("incomplete", "error"), ("refusal", "error"), ("empty", "empty")])- def test_failed_or_empty_rerank_preserves_delivery(self, outcome: str, reason: str) -> None:+ @parameterized.expand(+ [+ ("timeout", "timeout", "ReadTimeout"),+ ("gateway_timeout", "timeout", "SystemOneRequestFailed"),+ ("deadline", "timeout", "TimeoutError"),+ ("incomplete", "error", "SystemOneRequestFailed"),+ ("refusal", "error", "ValueError"),+ ("empty", "empty", None),+ ]+ )+ def test_failed_or_empty_rerank_preserves_delivery(self, outcome: str, reason: str, error_type: str | None) -> None:self.rows["llm_skills/search/"] = [self.skill, {**self.skill, "name": "slow-guide"}]if outcome == "empty":self.rows = {}@@ -295,6 +327,10 @@ class TestSelection(SimpleTestCase):return self.decide(state=state, questions=questions)if outcome == "timeout":raise httpx.ReadTimeout("Timed out")+ if outcome == "gateway_timeout":+ raise SystemOneRequestFailed("The ai-gateway was not reached") from httpx.ReadTimeout("Timed out")+ if outcome == "deadline":+ raise TimeoutError("selector_deadline")answers: dict[str, JsonValue] = {key: {"noul": 0.95} for key in questions}key = next(iter(answers))if outcome == "incomplete":@@ -304,10 +340,20 @@ class TestSelection(SimpleTestCase):return parse_system_one_response({"model": "test", "answers": answers}, questions)self.model.decide.side_effect = decide- result = self.selector.prepare(time.monotonic() + 60)+ with (+ patch("products.context_layer.backend.selection.posthoganalytics.capture") as capture,+ patch("products.context_layer.backend.selection.is_privacy_mode_enabled") as privacy_mode,+ ):+ result = self.selector.prepare(time.monotonic() + 60)+ privacy_mode.assert_not_called()self.assertEqual(result.context, "")self.assertEqual(result.mode, "treatment")self.assertEqual(result.reason, reason)+ properties = capture.call_args.kwargs["properties"]+ self.assertEqual((properties["reason"], properties["error_type"]), (reason, error_type))+ self.assertEqual(properties["over_budget"], reason == "timeout")+ self.assertIsNotNone(properties["gate_ms"])+ self.assertIsNotNone(properties["rerank_ms"])if outcome != "empty":self.assertEqual(self.client_factory.call_args.kwargs["properties"]["selection_step"], "rerank")@@ -349,7 +395,8 @@ class TestSelection(SimpleTestCase):patch("products.context_layer.backend.selection.is_privacy_mode_enabled", return_value=privacy_mode),patch("products.context_layer.backend.selection.posthoganalytics.capture") as capture,):- self.selector.capture_outcome("selected", 0.8, scored)+ self.selector.gate_probability = 0.8+ self.selector.capture_outcome("selected", scored)properties = capture.call_args.kwargs["properties"]self.assertEqual(properties["reason"], "selected")self.assertEqual([c["score"] for c in properties["candidates"]], [0.9, 0.2])@@ -394,6 +441,18 @@ class TestSelectionPermissions(SimpleTestCase):)+class TestSelectionRequestThrottle(SimpleTestCase):+ def test_request_ceiling_is_per_sandbox_task_not_per_project(self) -> None:+ def key(task_id: object) -> str:+ request = Mock(user=User(id=5))+ request.successful_authenticator.access_token.sandbox_task_id = task_id+ return ContextSelectionTaskThrottle().get_cache_key(request, Mock())++ first, second = uuid4(), uuid4()+ self.assertEqual(key(first), key(first))+ self.assertNotEqual(key(first), key(second))++class TestSelectionDeadline(SimpleTestCase):@parameterized.expand([("lookup",), ("actor",)])@override_settings(CONTEXT_SELECTION_TIMEOUT_SECONDS=60)