Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(slack-app): report out-of-credits instead of a repo picker (#118568)

Vojta Bartoše1ae82b5c844
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 25d3fbec05e3..d941d7383b5f 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -1475,7 +1475,7 @@
6 posthog/hogql/base.py
6 posthog/hogql/scripts/build_grammar_strategies.py
6 posthog/local_bootstrap/config.py
-6 products/slack_app/backend/temporal/types.py
+5 products/slack_app/backend/temporal/types.py
6 posthog/temporal/ai/sync_vectors.py
6 posthog/temporal/salesforce_enrichment/stripe_workflow.py
6 products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
diff --git a/products/slack_app/backend/temporal/activities/repo_selection.py b/products/slack_app/backend/temporal/activities/repo_selection.py
index 4bb493b777a8..4865574269b2 100644
--- a/products/slack_app/backend/temporal/activities/repo_selection.py
+++ b/products/slack_app/backend/temporal/activities/repo_selection.py
@@ -128,10 +128,12 @@ async def discover_posthog_code_repository_via_agent_activity(
"""
from posthog.models.integration import Integration, SlackIntegration
+ from products.slack_app.backend.api import post_quota_exhausted_denial
from products.tasks.backend.facade import api as tasks_facade
from products.tasks.backend.facade.repo_selection import (
RepoSelectionRejectedError,
RepoSelectionUnavailableError,
+ RepoSelectionUsageLimitError,
select_repository,
)
@@ -190,6 +192,25 @@ async def discover_posthog_code_repository_via_agent_activity(
repo_research_task_id=research_ids.get("task_id"),
repo_research_run_id=research_ids.get("run_id"),
)
+ except RepoSelectionUsageLimitError:
+ logger.warning("slack_app_repo_selection_usage_limited", channel=channel)
+ await asyncio.to_thread(
+ post_quota_exhausted_denial,
+ integration=integration,
+ slack=SlackIntegration(integration),
+ channel=channel,
+ thread_ts=event.get("thread_ts") or event.get("ts") or "",
+ slack_user_id=event.get("user") or "",
+ context="repo_selection",
+ )
+ return SlackRepoSelectionOutcome(
+ status="failed",
+ repository=None,
+ reason="Team is out of AI credits.",
+ repo_research_task_id=research_ids.get("task_id"),
+ repo_research_run_id=research_ids.get("run_id"),
+ usage_limited=True,
+ )
except RepoSelectionUnavailableError as exc:
logger.warning(
"posthog_code_repo_selection_unavailable",
diff --git a/products/slack_app/backend/temporal/slack_app_mention_processing.py b/products/slack_app/backend/temporal/slack_app_mention_processing.py
index 459f62057d5a..7d18c0417868 100644
--- a/products/slack_app/backend/temporal/slack_app_mention_processing.py
+++ b/products/slack_app/backend/temporal/slack_app_mention_processing.py
@@ -53,6 +53,8 @@ POSTHOG_CODE_SLACK_PICKER_TIMEOUT_MINUTES = 15
_PATCH_ID_MODEL_ROUTER = "slack-app-model-router-v1"
# Histories recorded without this marker replay `_prework_in_sequence`.
_PATCH_ID_PARALLEL_PREWORK = "slack-app-parallel-prework-v1"
+# Histories recorded without this marker offer the picker after a usage-limited discovery.
+_PATCH_ID_REPO_SELECTION_USAGE_LIMIT = "slack-app-repo-selection-usage-limit-v1"
@workflow.defn(name="slack-app-mention-processing")
@@ -168,6 +170,9 @@ class SlackAppMentionProcessingWorkflow(PostHogWorkflow):
repository = outcome.repository
elif outcome.status == "no_match":
repository = None
+ elif workflow.patched(_PATCH_ID_REPO_SELECTION_USAGE_LIMIT) and outcome.usage_limited:
+ # The activity told the thread the team is out of credits. A picked repo would fail the same way.
+ return
else:
# Agent crashed/timed out/hallucinated — italicize its reason
# above the picker guidance so the user sees why.
diff --git a/products/slack_app/backend/temporal/tests/activities/test_discover_repository_activity.py b/products/slack_app/backend/temporal/tests/activities/test_discover_repository_activity.py
new file mode 100644
index 000000000000..288a19961e91
--- /dev/null
+++ b/products/slack_app/backend/temporal/tests/activities/test_discover_repository_activity.py
@@ -0,0 +1,34 @@
+import contextlib
+
+import pytest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from products.slack_app.backend.temporal.activities import repo_selection
+from products.slack_app.backend.temporal.types import PostHogCodeSlackMentionWorkflowInputs
+from products.tasks.backend.facade.repo_selection import RepoSelectionUsageLimitError
+
+
+async def test_usage_limited_selection_posts_the_credits_message_instead_of_a_picker() -> None:
+ event = {"channel": "C1", "ts": "1.2", "thread_ts": "1.0", "user": "U1", "text": "fix the bug"}
+ inputs = PostHogCodeSlackMentionWorkflowInputs(event=event, integration_id=1, slack_team_id="T1", user_id=42)
+ integration = MagicMock(team_id=7)
+ objects = MagicMock()
+ objects.select_related.return_value.aget = AsyncMock(return_value=integration)
+
+ with (
+ patch("posthog.models.integration.Integration.objects", objects),
+ patch("posthog.models.integration.SlackIntegration"),
+ patch.object(repo_selection, "Heartbeater", return_value=contextlib.nullcontext()),
+ patch(
+ "products.tasks.backend.facade.repo_selection.select_repository",
+ AsyncMock(side_effect=RepoSelectionUsageLimitError("API Error: 402 out of credits")),
+ ),
+ patch("products.slack_app.backend.api.post_quota_exhausted_denial") as denial,
+ ):
+ outcome = await repo_selection.discover_posthog_code_repository_via_agent_activity(inputs, "C1", event, [], 42)
+
+ assert outcome.usage_limited is True
+ assert outcome.repository is None
+ assert denial.call_args.kwargs["thread_ts"] == "1.0"
+ assert denial.call_args.kwargs["slack_user_id"] == "U1"
diff --git a/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py b/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py
index 3f08585c5daa..fa1f8ac93027 100644
--- a/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py
+++ b/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py
@@ -98,6 +98,8 @@ class _Recorder:
self.picker_workflow_id: str | None = None
# True holds untagged replies back on the thread creator's `ask` mode.
self.awaiting_confirmation = False
+ # True makes discovery report that the team is out of AI credits.
+ self.discover_usage_limited = False
def _fake_activities(rec: _Recorder) -> list:
@@ -191,7 +193,9 @@ def _fake_activities(rec: _Recorder) -> list:
thread_messages: list[SlackThreadMessage],
user_id: int,
) -> SlackRepoSelectionOutcome:
- return SlackRepoSelectionOutcome(status="failed", repository=None, reason="agent crashed")
+ return SlackRepoSelectionOutcome(
+ status="failed", repository=None, reason="agent crashed", usage_limited=rec.discover_usage_limited
+ )
@activity.defn(name="post_posthog_code_repo_picker_activity")
async def post_picker(
@@ -594,6 +598,21 @@ async def test_repo_picker_signal_resolves_and_queue_continues():
assert rec.created == [("1.1", "org/picked"), ("1.2", "org/auto-repo")]
+async def test_usage_limited_discovery_ends_without_a_picker_or_a_task():
+ rec = _Recorder()
+ rec.cascade_modes["1.1"] = "agent_needed"
+ rec.discover_usage_limited = True
+
+ async with _Harness(rec) as h:
+ handle = await _signal_with_start(h.env, h.task_queue, f"wf-{uuid.uuid4()}", _message("1.1"))
+ await asyncio.wait_for(handle.result(), timeout=30)
+
+ assert not rec.picker_posted.is_set()
+ assert rec.created == []
+ assert rec.internal_errors == []
+
+
@pytest.mark.asyncio
async def test_hung_child_times_out_and_queue_continues(monkeypatch):
# A child that never finishes used to stall the conversation forever: the queue
diff --git a/products/slack_app/backend/temporal/types.py b/products/slack_app/backend/temporal/types.py
index f109491a5000..57b47ab68f89 100644
--- a/products/slack_app/backend/temporal/types.py
+++ b/products/slack_app/backend/temporal/types.py
@@ -300,7 +300,7 @@ class PostHogCodeRepoCascadeOutcome:
reason: str
-@dataclass
+@dataclass(frozen=True)
class SlackRepoSelectionOutcome:
"""Discovery-agent result wrapped at the activity boundary.
@@ -310,6 +310,11 @@ class SlackRepoSelectionOutcome:
`repo_research_task_id`/`repo_research_run_id` point at the internal sandbox
run the repo discovery agent spun up to make this call.
+
+ `usage_limited` marks a `failed` outcome where the team has no AI credits left.
+ The activity already told the thread, so the workflow stops instead of offering
+ the picker. It defaults to False, so histories recorded before it existed decode
+ to the picker path.
"""
status: Literal["found", "no_match", "failed"]
@@ -317,6 +322,7 @@ class SlackRepoSelectionOutcome:
reason: str
repo_research_task_id: str | None = None
repo_research_run_id: str | None = None
+ usage_limited: bool = False
@dataclass
diff --git a/products/tasks/backend/facade/repo_selection.py b/products/tasks/backend/facade/repo_selection.py
index 89408b003432..40b8e2da6db9 100644
--- a/products/tasks/backend/facade/repo_selection.py
+++ b/products/tasks/backend/facade/repo_selection.py
@@ -12,6 +12,7 @@ from products.tasks.backend.logic.repo_selection import (
RepoSelectionRejectedError,
RepoSelectionResult,
RepoSelectionUnavailableError,
+ RepoSelectionUsageLimitError,
list_team_connected_repositories,
resolve_team_github_integration,
select_repository,
@@ -33,6 +34,7 @@ __all__ = [
"RepoSelectionRejectedError",
"RepoSelectionResult",
"RepoSelectionUnavailableError",
+ "RepoSelectionUsageLimitError",
"get_github_token",
"list_team_connected_repositories",
"repository_accessible_via_integration",
diff --git a/products/tasks/backend/logic/repo_selection/__init__.py b/products/tasks/backend/logic/repo_selection/__init__.py
index bbc181ca6a5f..9ba3e28dacfa 100644
--- a/products/tasks/backend/logic/repo_selection/__init__.py
+++ b/products/tasks/backend/logic/repo_selection/__init__.py
@@ -10,6 +10,7 @@ if TYPE_CHECKING:
REPO_SELECTION_DUMMY_REPOSITORY,
RepoSelectionRejectedError,
RepoSelectionUnavailableError,
+ RepoSelectionUsageLimitError,
list_team_connected_repositories,
resolve_team_github_integration,
select_repository,
@@ -21,6 +22,7 @@ __all__ = [
"RepoSelectionRejectedError",
"RepoSelectionResult",
"RepoSelectionUnavailableError",
+ "RepoSelectionUsageLimitError",
"list_team_connected_repositories",
"resolve_team_github_integration",
"select_repository",
@@ -35,6 +37,7 @@ _LAZY_FROM_AGENT = frozenset(
"REPO_SELECTION_DUMMY_REPOSITORY",
"RepoSelectionRejectedError",
"RepoSelectionUnavailableError",
+ "RepoSelectionUsageLimitError",
"list_team_connected_repositories",
"resolve_team_github_integration",
"select_repository",
diff --git a/products/tasks/backend/logic/repo_selection/agent.py b/products/tasks/backend/logic/repo_selection/agent.py
index 37c945b10d67..c0b7ce4b1add 100644
--- a/products/tasks/backend/logic/repo_selection/agent.py
+++ b/products/tasks/backend/logic/repo_selection/agent.py
@@ -73,6 +73,24 @@ class RepoSelectionUnavailableError(Exception):
super().__init__(reason)
+class RepoSelectionUsageLimitError(Exception):
+ """Raised when the LLM gateway refused the agent's model calls with a 402, because the team
+ has no credits left for this billing period.
+
+ A task on any repository would hit the same refusal, so callers should report the limit
+ instead of falling back to a repository picker.
+ """
+
+ def __init__(self, message: str) -> None:
+ self.message = message
+ super().__init__(message)
+
+
+# The agent prints a refused model call as `API Error: <status> <body>`. The gateway answers a
+# team with no credits left with a 402.
+_USAGE_LIMIT_REPLY = re.compile(r"\s*API Error: 402\b")
+
+
def _first_user_github_integration(user_ids: Iterable[int]) -> UserIntegration | None:
"""Pick one GitHub ``UserIntegration`` among ``user_ids`` to act as a fallback source.
@@ -699,21 +717,36 @@ async def select_repository(
service_tier=service_tier,
)
- session, result = await MultiTurnSession.start(
- prompt=prompt,
- context=sandbox_context,
- model=RepoSelectionResult,
- step_name=step_name,
- verbose=verbose,
- output_fn=output_fn,
- origin_product=origin_product,
- signal_report_id=signal_report_id,
- ai_stage="repo_selection",
- internal=True,
- # An unreadable end turn used to fail the whole selection, and the caller then dropped the
- # user into a manual repository picker. Read the pick out of the raw reply instead.
- fallback_from_text=lambda text: _salvage_repo_selection(text, candidate_repos),
- )
+ usage_limit_reply: str | None = None
+
+ def _salvage(text: str) -> RepoSelectionResult:
+ nonlocal usage_limit_reply
+ if _USAGE_LIMIT_REPLY.match(text):
+ # The runner raises its own parse error when this raises, so the refusal is kept here.
+ usage_limit_reply = text
+ raise ValueError("The end turn is a usage limit refusal, not a selection")
+ return _salvage_repo_selection(text, candidate_repos)
+
+ try:
+ session, result = await MultiTurnSession.start(
+ prompt=prompt,
+ context=sandbox_context,
+ model=RepoSelectionResult,
+ step_name=step_name,
+ verbose=verbose,
+ output_fn=output_fn,
+ origin_product=origin_product,
+ signal_report_id=signal_report_id,
+ ai_stage="repo_selection",
+ internal=True,
+ # An unreadable end turn used to fail the whole selection, and the caller then dropped the
+ # user into a manual repository picker. Read the pick out of the raw reply instead.
+ fallback_from_text=_salvage,
+ )
+ except Exception as e:
+ if usage_limit_reply is not None:
+ raise RepoSelectionUsageLimitError(usage_limit_reply) from e
+ raise
# Track repo discovery execution (for example, for Slack)
if on_research_session is not None:
on_research_session(str(session.task.id), str(session.task_run.id))
diff --git a/products/tasks/backend/tests/test_repo_selection_prompt.py b/products/tasks/backend/tests/test_repo_selection_prompt.py
index 0cbcae9daf1e..a1748037fd69 100644
--- a/products/tasks/backend/tests/test_repo_selection_prompt.py
+++ b/products/tasks/backend/tests/test_repo_selection_prompt.py
@@ -9,6 +9,7 @@ from posthog.models.repo_routing_rule import RepoRoutingRule
from products.tasks.backend.logic.repo_selection.agent import (
PINNED_REPOSITORY_REASON,
+ RepoSelectionUsageLimitError,
_build_repo_selection_prompt,
_routing_rules_block,
_salvage_repo_selection,
@@ -210,6 +211,35 @@ def test_select_repository_salvages_an_unreadable_end_turn() -> None:
assert selected.repository == "acme/b"
+def test_select_repository_reports_a_usage_limit_refusal() -> None:
+ github = MagicMock()
+ github.list_all_cached_repositories.return_value = [{"full_name": "acme/a"}, {"full_name": "acme/b"}]
+
+ async def start(**kwargs):
+ # The runner swallows a raising fallback and raises its own parse error instead.
+ with pytest.raises(ValueError):
+ kwargs["fallback_from_text"]("API Error: 402 Your team has used its monthly PostHog AI credits.")
+ raise ValueError("No JSON in initial turn")
+
+ with (
+ patch(f"{_AGENT}.GitHubRepositoryFullCache") as cache,
+ patch(f"{_AGENT}._list_eligible_full_names", return_value={"acme/a", "acme/b"}),
+ patch(f"{_AGENT}._routing_rules_block", return_value=None),
+ patch(f"{_AGENT}.MultiTurnSession.start", AsyncMock(side_effect=start)),
+ pytest.raises(RepoSelectionUsageLimitError),
+ ):
+ cache.return_value.sync_full_cache = AsyncMock()
+ async_to_sync(select_repository)(
+ 1,
+ 1,
+ "which repo?",
+ origin_product=Task.OriginProduct.SLACK,
+ priority=Priority.NORMAL,
+ github=github,
+ candidate_repos=["acme/a", "acme/b"],
+ )
+
+
@pytest.mark.parametrize(
"pinned,expected_repository",
[("Acme/B", "acme/b"), (" acme/b ", "acme/b"), ("acme/gone", None)],