:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(slack-app): report out-of-credits instead of a repo picker (#118568)
Vojta Bartoše1ae82b5c844
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 25d3fbec05e3..d941d7383b5f 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -1475,7 +1475,7 @@6 posthog/hogql/base.py6 posthog/hogql/scripts/build_grammar_strategies.py6 posthog/local_bootstrap/config.py-6 products/slack_app/backend/temporal/types.py+5 products/slack_app/backend/temporal/types.py6 posthog/temporal/ai/sync_vectors.py6 posthog/temporal/salesforce_enrichment/stripe_workflow.py6 products/batch_exports/backend/temporal/destinations/redshift_batch_export.pydiff --git a/products/slack_app/backend/temporal/activities/repo_selection.py b/products/slack_app/backend/temporal/activities/repo_selection.pyindex 4bb493b777a8..4865574269b2 100644--- a/products/slack_app/backend/temporal/activities/repo_selection.py+++ b/products/slack_app/backend/temporal/activities/repo_selection.py@@ -128,10 +128,12 @@ async def discover_posthog_code_repository_via_agent_activity("""from posthog.models.integration import Integration, SlackIntegration+ from products.slack_app.backend.api import post_quota_exhausted_denialfrom products.tasks.backend.facade import api as tasks_facadefrom products.tasks.backend.facade.repo_selection import (RepoSelectionRejectedError,RepoSelectionUnavailableError,+ RepoSelectionUsageLimitError,select_repository,)@@ -190,6 +192,25 @@ async def discover_posthog_code_repository_via_agent_activity(repo_research_task_id=research_ids.get("task_id"),repo_research_run_id=research_ids.get("run_id"),)+ except RepoSelectionUsageLimitError:+ logger.warning("slack_app_repo_selection_usage_limited", channel=channel)+ await asyncio.to_thread(+ post_quota_exhausted_denial,+ integration=integration,+ slack=SlackIntegration(integration),+ channel=channel,+ thread_ts=event.get("thread_ts") or event.get("ts") or "",+ slack_user_id=event.get("user") or "",+ context="repo_selection",+ )+ return SlackRepoSelectionOutcome(+ status="failed",+ repository=None,+ reason="Team is out of AI credits.",+ repo_research_task_id=research_ids.get("task_id"),+ repo_research_run_id=research_ids.get("run_id"),+ usage_limited=True,+ )except RepoSelectionUnavailableError as exc:logger.warning("posthog_code_repo_selection_unavailable",diff --git a/products/slack_app/backend/temporal/slack_app_mention_processing.py b/products/slack_app/backend/temporal/slack_app_mention_processing.pyindex 459f62057d5a..7d18c0417868 100644--- a/products/slack_app/backend/temporal/slack_app_mention_processing.py+++ b/products/slack_app/backend/temporal/slack_app_mention_processing.py@@ -53,6 +53,8 @@ POSTHOG_CODE_SLACK_PICKER_TIMEOUT_MINUTES = 15_PATCH_ID_MODEL_ROUTER = "slack-app-model-router-v1"# Histories recorded without this marker replay `_prework_in_sequence`._PATCH_ID_PARALLEL_PREWORK = "slack-app-parallel-prework-v1"+# Histories recorded without this marker offer the picker after a usage-limited discovery.+_PATCH_ID_REPO_SELECTION_USAGE_LIMIT = "slack-app-repo-selection-usage-limit-v1"@workflow.defn(name="slack-app-mention-processing")@@ -168,6 +170,9 @@ class SlackAppMentionProcessingWorkflow(PostHogWorkflow):repository = outcome.repositoryelif outcome.status == "no_match":repository = None+ elif workflow.patched(_PATCH_ID_REPO_SELECTION_USAGE_LIMIT) and outcome.usage_limited:+ # The activity told the thread the team is out of credits. A picked repo would fail the same way.+ returnelse:# Agent crashed/timed out/hallucinated — italicize its reason# above the picker guidance so the user sees why.diff --git a/products/slack_app/backend/temporal/tests/activities/test_discover_repository_activity.py b/products/slack_app/backend/temporal/tests/activities/test_discover_repository_activity.pynew file mode 100644index 000000000000..288a19961e91--- /dev/null+++ b/products/slack_app/backend/temporal/tests/activities/test_discover_repository_activity.py@@ -0,0 +1,34 @@+import contextlib++import pytest+from unittest.mock import AsyncMock, MagicMock, patch++from products.slack_app.backend.temporal.activities import repo_selection+from products.slack_app.backend.temporal.types import PostHogCodeSlackMentionWorkflowInputs+from products.tasks.backend.facade.repo_selection import RepoSelectionUsageLimitError+++async def test_usage_limited_selection_posts_the_credits_message_instead_of_a_picker() -> None:+ event = {"channel": "C1", "ts": "1.2", "thread_ts": "1.0", "user": "U1", "text": "fix the bug"}+ inputs = PostHogCodeSlackMentionWorkflowInputs(event=event, integration_id=1, slack_team_id="T1", user_id=42)+ integration = MagicMock(team_id=7)+ objects = MagicMock()+ objects.select_related.return_value.aget = AsyncMock(return_value=integration)++ with (+ patch("posthog.models.integration.Integration.objects", objects),+ patch("posthog.models.integration.SlackIntegration"),+ patch.object(repo_selection, "Heartbeater", return_value=contextlib.nullcontext()),+ patch(+ "products.tasks.backend.facade.repo_selection.select_repository",+ AsyncMock(side_effect=RepoSelectionUsageLimitError("API Error: 402 out of credits")),+ ),+ patch("products.slack_app.backend.api.post_quota_exhausted_denial") as denial,+ ):+ outcome = await repo_selection.discover_posthog_code_repository_via_agent_activity(inputs, "C1", event, [], 42)++ assert outcome.usage_limited is True+ assert outcome.repository is None+ assert denial.call_args.kwargs["thread_ts"] == "1.0"+ assert denial.call_args.kwargs["slack_user_id"] == "U1"diff --git a/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py b/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.pyindex 3f08585c5daa..fa1f8ac93027 100644--- a/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py+++ b/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py@@ -98,6 +98,8 @@ class _Recorder:self.picker_workflow_id: str | None = None# True holds untagged replies back on the thread creator's `ask` mode.self.awaiting_confirmation = False+ # True makes discovery report that the team is out of AI credits.+ self.discover_usage_limited = Falsedef _fake_activities(rec: _Recorder) -> list:@@ -191,7 +193,9 @@ def _fake_activities(rec: _Recorder) -> list:thread_messages: list[SlackThreadMessage],user_id: int,) -> SlackRepoSelectionOutcome:- return SlackRepoSelectionOutcome(status="failed", repository=None, reason="agent crashed")+ return SlackRepoSelectionOutcome(+ status="failed", repository=None, reason="agent crashed", usage_limited=rec.discover_usage_limited+ )@activity.defn(name="post_posthog_code_repo_picker_activity")async def post_picker(@@ -594,6 +598,21 @@ async def test_repo_picker_signal_resolves_and_queue_continues():assert rec.created == [("1.1", "org/picked"), ("1.2", "org/auto-repo")]+async def test_usage_limited_discovery_ends_without_a_picker_or_a_task():+ rec = _Recorder()+ rec.cascade_modes["1.1"] = "agent_needed"+ rec.discover_usage_limited = True++ async with _Harness(rec) as h:+ handle = await _signal_with_start(h.env, h.task_queue, f"wf-{uuid.uuid4()}", _message("1.1"))+ await asyncio.wait_for(handle.result(), timeout=30)++ assert not rec.picker_posted.is_set()+ assert rec.created == []+ assert rec.internal_errors == []++@pytest.mark.asyncioasync def test_hung_child_times_out_and_queue_continues(monkeypatch):# A child that never finishes used to stall the conversation forever: the queuediff --git a/products/slack_app/backend/temporal/types.py b/products/slack_app/backend/temporal/types.pyindex f109491a5000..57b47ab68f89 100644--- a/products/slack_app/backend/temporal/types.py+++ b/products/slack_app/backend/temporal/types.py@@ -300,7 +300,7 @@ class PostHogCodeRepoCascadeOutcome:reason: str-@dataclass+@dataclass(frozen=True)class SlackRepoSelectionOutcome:"""Discovery-agent result wrapped at the activity boundary.@@ -310,6 +310,11 @@ class SlackRepoSelectionOutcome:`repo_research_task_id`/`repo_research_run_id` point at the internal sandboxrun the repo discovery agent spun up to make this call.++ `usage_limited` marks a `failed` outcome where the team has no AI credits left.+ The activity already told the thread, so the workflow stops instead of offering+ the picker. It defaults to False, so histories recorded before it existed decode+ to the picker path."""status: Literal["found", "no_match", "failed"]@@ -317,6 +322,7 @@ class SlackRepoSelectionOutcome:reason: strrepo_research_task_id: str | None = Nonerepo_research_run_id: str | None = None+ usage_limited: bool = False@dataclassdiff --git a/products/tasks/backend/facade/repo_selection.py b/products/tasks/backend/facade/repo_selection.pyindex 89408b003432..40b8e2da6db9 100644--- a/products/tasks/backend/facade/repo_selection.py+++ b/products/tasks/backend/facade/repo_selection.py@@ -12,6 +12,7 @@ from products.tasks.backend.logic.repo_selection import (RepoSelectionRejectedError,RepoSelectionResult,RepoSelectionUnavailableError,+ RepoSelectionUsageLimitError,list_team_connected_repositories,resolve_team_github_integration,select_repository,@@ -33,6 +34,7 @@ __all__ = ["RepoSelectionRejectedError","RepoSelectionResult","RepoSelectionUnavailableError",+ "RepoSelectionUsageLimitError","get_github_token","list_team_connected_repositories","repository_accessible_via_integration",diff --git a/products/tasks/backend/logic/repo_selection/__init__.py b/products/tasks/backend/logic/repo_selection/__init__.pyindex bbc181ca6a5f..9ba3e28dacfa 100644--- a/products/tasks/backend/logic/repo_selection/__init__.py+++ b/products/tasks/backend/logic/repo_selection/__init__.py@@ -10,6 +10,7 @@ if TYPE_CHECKING:REPO_SELECTION_DUMMY_REPOSITORY,RepoSelectionRejectedError,RepoSelectionUnavailableError,+ RepoSelectionUsageLimitError,list_team_connected_repositories,resolve_team_github_integration,select_repository,@@ -21,6 +22,7 @@ __all__ = ["RepoSelectionRejectedError","RepoSelectionResult","RepoSelectionUnavailableError",+ "RepoSelectionUsageLimitError","list_team_connected_repositories","resolve_team_github_integration","select_repository",@@ -35,6 +37,7 @@ _LAZY_FROM_AGENT = frozenset("REPO_SELECTION_DUMMY_REPOSITORY","RepoSelectionRejectedError","RepoSelectionUnavailableError",+ "RepoSelectionUsageLimitError","list_team_connected_repositories","resolve_team_github_integration","select_repository",diff --git a/products/tasks/backend/logic/repo_selection/agent.py b/products/tasks/backend/logic/repo_selection/agent.pyindex 37c945b10d67..c0b7ce4b1add 100644--- a/products/tasks/backend/logic/repo_selection/agent.py+++ b/products/tasks/backend/logic/repo_selection/agent.py@@ -73,6 +73,24 @@ class RepoSelectionUnavailableError(Exception):super().__init__(reason)+class RepoSelectionUsageLimitError(Exception):+ """Raised when the LLM gateway refused the agent's model calls with a 402, because the team+ has no credits left for this billing period.++ A task on any repository would hit the same refusal, so callers should report the limit+ instead of falling back to a repository picker.+ """++ def __init__(self, message: str) -> None:+ self.message = message+ super().__init__(message)+++# The agent prints a refused model call as `API Error: <status> <body>`. The gateway answers a+# team with no credits left with a 402.+_USAGE_LIMIT_REPLY = re.compile(r"\s*API Error: 402\b")++def _first_user_github_integration(user_ids: Iterable[int]) -> UserIntegration | None:"""Pick one GitHub ``UserIntegration`` among ``user_ids`` to act as a fallback source.@@ -699,21 +717,36 @@ async def select_repository(service_tier=service_tier,)- session, result = await MultiTurnSession.start(- prompt=prompt,- context=sandbox_context,- model=RepoSelectionResult,- step_name=step_name,- verbose=verbose,- output_fn=output_fn,- origin_product=origin_product,- signal_report_id=signal_report_id,- ai_stage="repo_selection",- internal=True,- # An unreadable end turn used to fail the whole selection, and the caller then dropped the- # user into a manual repository picker. Read the pick out of the raw reply instead.- fallback_from_text=lambda text: _salvage_repo_selection(text, candidate_repos),- )+ usage_limit_reply: str | None = None++ def _salvage(text: str) -> RepoSelectionResult:+ nonlocal usage_limit_reply+ if _USAGE_LIMIT_REPLY.match(text):+ # The runner raises its own parse error when this raises, so the refusal is kept here.+ usage_limit_reply = text+ raise ValueError("The end turn is a usage limit refusal, not a selection")+ return _salvage_repo_selection(text, candidate_repos)++ try:+ session, result = await MultiTurnSession.start(+ prompt=prompt,+ context=sandbox_context,+ model=RepoSelectionResult,+ step_name=step_name,+ verbose=verbose,+ output_fn=output_fn,+ origin_product=origin_product,+ signal_report_id=signal_report_id,+ ai_stage="repo_selection",+ internal=True,+ # An unreadable end turn used to fail the whole selection, and the caller then dropped the+ # user into a manual repository picker. Read the pick out of the raw reply instead.+ fallback_from_text=_salvage,+ )+ except Exception as e:+ if usage_limit_reply is not None:+ raise RepoSelectionUsageLimitError(usage_limit_reply) from e+ raise# Track repo discovery execution (for example, for Slack)if on_research_session is not None:on_research_session(str(session.task.id), str(session.task_run.id))diff --git a/products/tasks/backend/tests/test_repo_selection_prompt.py b/products/tasks/backend/tests/test_repo_selection_prompt.pyindex 0cbcae9daf1e..a1748037fd69 100644--- a/products/tasks/backend/tests/test_repo_selection_prompt.py+++ b/products/tasks/backend/tests/test_repo_selection_prompt.py@@ -9,6 +9,7 @@ from posthog.models.repo_routing_rule import RepoRoutingRulefrom products.tasks.backend.logic.repo_selection.agent import (PINNED_REPOSITORY_REASON,+ RepoSelectionUsageLimitError,_build_repo_selection_prompt,_routing_rules_block,_salvage_repo_selection,@@ -210,6 +211,35 @@ def test_select_repository_salvages_an_unreadable_end_turn() -> None:assert selected.repository == "acme/b"+def test_select_repository_reports_a_usage_limit_refusal() -> None:+ github = MagicMock()+ github.list_all_cached_repositories.return_value = [{"full_name": "acme/a"}, {"full_name": "acme/b"}]++ async def start(**kwargs):+ # The runner swallows a raising fallback and raises its own parse error instead.+ with pytest.raises(ValueError):+ kwargs["fallback_from_text"]("API Error: 402 Your team has used its monthly PostHog AI credits.")+ raise ValueError("No JSON in initial turn")++ with (+ patch(f"{_AGENT}.GitHubRepositoryFullCache") as cache,+ patch(f"{_AGENT}._list_eligible_full_names", return_value={"acme/a", "acme/b"}),+ patch(f"{_AGENT}._routing_rules_block", return_value=None),+ patch(f"{_AGENT}.MultiTurnSession.start", AsyncMock(side_effect=start)),+ pytest.raises(RepoSelectionUsageLimitError),+ ):+ cache.return_value.sync_full_cache = AsyncMock()+ async_to_sync(select_repository)(+ 1,+ 1,+ "which repo?",+ origin_product=Task.OriginProduct.SLACK,+ priority=Priority.NORMAL,+ github=github,+ candidate_repos=["acme/a", "acme/b"],+ )++@pytest.mark.parametrize("pinned,expected_repository",[("Acme/B", "acme/b"), (" acme/b ", "acme/b"), ("acme/gone", None)],