Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(slack): bill classifiers as slack_app, rename model router product (#118524)

Vojta Bartošddb00c53a33e
diff --git a/products/slack_app/backend/temporal/activities/classifiers.py b/products/slack_app/backend/temporal/activities/classifiers.py
index 06ec877077df..da166bc15f58 100644
--- a/products/slack_app/backend/temporal/activities/classifiers.py
+++ b/products/slack_app/backend/temporal/activities/classifiers.py
@@ -6,7 +6,7 @@ from openai.types.shared_params import ResponseFormatJSONSchema
from pydantic import BaseModel, ValidationError
from temporalio import activity
-from posthog.llm.gateway_client import build_anthropic_client, build_openai_client, trace_id_from_seed
+from posthog.llm.gateway_client import Product, build_anthropic_client, build_openai_client, trace_id_from_seed
from posthog.llm.semantic_enrichment import extract_json_object
from posthog.models.integration import Integration, SlackIntegration
from posthog.models.repo_routing_rule import RepoRoutingRule
@@ -39,12 +39,28 @@ prompts = PromptTemplates(Path(__file__).parent / "prompts")
CLASSIFIER_THREAD_HISTORY_MESSAGES = 10
CLASSIFIER_MODEL = "claude-haiku-4-5-20251001"
-# Every classifier here shares `ai_product="slack_app_routing"`, so the captured generation
-# carries this property to say which one of them made the call. An online evaluation scopes
-# itself with it; without it a judge would grade all four.
+# The classifiers bill the customer's PostHog AI credits, the same as the Slack agent's runs,
+# so they share its product.
+CLASSIFIER_AI_PRODUCT = "slack_app"
+# The Python gateway's `slack_app` route refuses API keys, so its fallback keeps the
+# `slack_app_routing` route, which accepts them. The usage report does not bill that route.
+CLASSIFIER_FALLBACK_ROUTE: Product = "slack_app_routing"
+
+# Every classifier here and every Slack agent run share `ai_product="slack_app"`, so the captured
+# generation carries this property to say which classifier made the call. An online evaluation
+# scopes itself with it; without it a judge would grade every Slack generation.
CLASSIFIER_PROPERTY = "slack_app_classifier"
+def _classifier_properties(classifier: str, team_id: int | None) -> dict[str, str]:
+ properties = {CLASSIFIER_PROPERTY: classifier}
+ # The gateway and the usage report charge the team named here. Without it, the call
+ # settles on the gateway's relay team and bills no customer.
+ if team_id is not None:
+ properties["team_id"] = str(team_id)
+ return properties
+
+
def _thread_trace_id(slack_team_id: str | None, thread_ts: str | None) -> str | None:
"""Group one Slack thread's routing calls into a single trace.
@@ -110,6 +126,7 @@ def classify_task_needs_repo(
thread_messages: list[SlackThreadMessage],
routing_rules: list[str] | None = None,
trace_id: str | None = None,
+ team_id: int | None = None,
) -> bool:
"""Classify whether a Slack conversation requires code repository access.
@@ -209,10 +226,10 @@ def classify_task_needs_repo(
try:
# The Go gateway refuses a Claude model on chat completions.
client = build_anthropic_client(
- product="slack_app_routing",
- ai_product="slack_app_routing",
+ product=CLASSIFIER_FALLBACK_ROUTE,
+ ai_product=CLASSIFIER_AI_PRODUCT,
trace_id=trace_id,
- properties={CLASSIFIER_PROPERTY: "task_needs_repo"},
+ properties=_classifier_properties("task_needs_repo", team_id),
)
response = client.messages.create(
model=CLASSIFIER_MODEL,
@@ -276,6 +293,7 @@ def classify_posthog_code_task_needs_repo_activity(
thread_messages,
routing_rules=routing_rules,
trace_id=_thread_trace_id(inputs.slack_team_id, thread_ts),
+ team_id=integration.team_id,
)
@@ -305,6 +323,7 @@ def classify_message_is_agent_directed(
task_title: str,
thread_history: list[SlackThreadMessage],
trace_id: str | None = None,
+ team_id: int | None = None,
) -> bool:
"""Classify whether an untagged Slack thread reply is an instruction to the running
PostHog Slack App, or people talking to each other.
@@ -339,10 +358,10 @@ def classify_message_is_agent_directed(
)
try:
client = build_openai_client(
- product="slack_app_routing",
- ai_product="slack_app_routing",
+ product=CLASSIFIER_FALLBACK_ROUTE,
+ ai_product=CLASSIFIER_AI_PRODUCT,
trace_id=trace_id,
- properties={CLASSIFIER_PROPERTY: "agent_directed"},
+ properties=_classifier_properties("agent_directed", team_id),
).with_options(timeout=AGENT_DIRECTED_TIMEOUT_SECONDS, max_retries=AGENT_DIRECTED_MAX_RETRIES)
response = client.chat.completions.create(
model=AGENT_DIRECTED_CLASSIFIER_MODEL,
@@ -430,6 +449,7 @@ def classify_untagged_followup_activity(
task_title,
thread_history,
trace_id=_thread_trace_id(inputs.slack_team_id, thread_ts),
+ team_id=integration.team_id,
):
return True
@@ -487,6 +507,7 @@ def classify_slack_app_model_override(
event_text: str,
choices: tuple[ModelChoice, ...],
trace_id: str | None = None,
+ team_id: int | None = None,
) -> SlackAppModelOverride | None:
"""Read a per-task model or reasoning-effort request out of a Slack mention.
@@ -516,10 +537,10 @@ def classify_slack_app_model_override(
try:
client = build_openai_client(
- product="slack_app_routing",
- ai_product="slack_app_routing",
+ product=CLASSIFIER_FALLBACK_ROUTE,
+ ai_product=CLASSIFIER_AI_PRODUCT,
trace_id=trace_id,
- properties={CLASSIFIER_PROPERTY: "model_override"},
+ properties=_classifier_properties("model_override", team_id),
).with_options(timeout=MODEL_OVERRIDE_TIMEOUT_SECONDS, max_retries=MODEL_OVERRIDE_MAX_RETRIES)
response = client.chat.completions.create(
model=MODEL_OVERRIDE_CLASSIFIER_MODEL,
@@ -592,6 +613,7 @@ def classify_slack_app_model_override_activity(input: SlackAppModelOverrideInput
input.event_text,
choices,
trace_id=_thread_trace_id(input.slack_team_id, input.thread_ts),
+ team_id=integration.team_id,
)
if override is None:
return None
@@ -638,6 +660,7 @@ def classify_slack_app_project_route(
projects: list[Integration],
default: Integration | None = None,
trace_id: str | None = None,
+ team_id: int | None = None,
) -> Integration | None:
"""Read the project a mention asked to be answered from, out of its text.
@@ -671,10 +694,10 @@ def classify_slack_app_project_route(
try:
client = build_openai_client(
- product="slack_app_routing",
- ai_product="slack_app_routing",
+ product=CLASSIFIER_FALLBACK_ROUTE,
+ ai_product=CLASSIFIER_AI_PRODUCT,
trace_id=trace_id,
- properties={CLASSIFIER_PROPERTY: "project_route"},
+ properties=_classifier_properties("project_route", team_id),
).with_options(timeout=PROJECT_ROUTE_TIMEOUT_SECONDS, max_retries=PROJECT_ROUTE_MAX_RETRIES)
response = client.chat.completions.create(
model=PROJECT_ROUTE_CLASSIFIER_MODEL,
@@ -739,6 +762,7 @@ def classify_slack_app_project_route_activity(input: SlackAppProjectRouteInput)
projects,
default=integration,
trace_id=_thread_trace_id(input.slack_team_id, input.thread_ts),
+ team_id=integration.team_id,
)
except Exception:
# The fallback boundary: a mention we cannot classify stays on the project
diff --git a/products/slack_app/backend/temporal/activities/model_router.py b/products/slack_app/backend/temporal/activities/model_router.py
index 39c005a70a48..33d380fb17d3 100644
--- a/products/slack_app/backend/temporal/activities/model_router.py
+++ b/products/slack_app/backend/temporal/activities/model_router.py
@@ -36,6 +36,7 @@ MODEL_ROUTER_TIMEOUT_SECONDS = 5.0
_QUESTION_ID = "model"
DECISION_SPAN_NAME = "slack_model_router_decision"
+MODEL_ROUTER_AI_PRODUCT = "slack_app_model_router"
@frozen
@@ -63,7 +64,7 @@ def classify_slack_app_model_router(
# No TypeSafe fallback: the request is customer text, and TypeSafe is a third party.
client = build_system_one_client(
model=MODEL_ROUTER_DECISION_MODEL,
- ai_product="slack_app_routing",
+ ai_product=MODEL_ROUTER_AI_PRODUCT,
team_id=team_id,
distinct_id=distinct_id,
trace_id=trace_id,
@@ -120,7 +121,7 @@ def _capture_decision(
}
],
CLASSIFIER_PROPERTY: "slack_model_router",
- "ai_product": "slack_app_routing",
+ "ai_product": MODEL_ROUTER_AI_PRODUCT,
},
)
diff --git a/products/slack_app/backend/temporal/tests/activities/test_classifier_gateway_routing.py b/products/slack_app/backend/temporal/tests/activities/test_classifier_gateway_routing.py
index d4d7d64e151e..172b7daf1c09 100644
--- a/products/slack_app/backend/temporal/tests/activities/test_classifier_gateway_routing.py
+++ b/products/slack_app/backend/temporal/tests/activities/test_classifier_gateway_routing.py
@@ -25,6 +25,7 @@ AI_GATEWAY_KEY = "phs_go"
LLM_GATEWAY_URL = "http://llm-gateway:8080"
LLM_GATEWAY_KEY = "phx_legacy"
TRACE_ID = "1b9d6bcd-bbfd-5b1d-9c2f-9f0a4b4a1b2c"
+TEAM_ID = 41
# Both modes set every setting the builders read, so the wrong route cannot pass by accident.
GO_GATEWAY = {
@@ -48,19 +49,24 @@ def _classify_needs_repo():
NEEDS_REPO_TEXT,
[SlackThreadMessage(user="Alessandro", text=NEEDS_REPO_TEXT)],
trace_id=TRACE_ID,
+ team_id=TEAM_ID,
)
def _classify_agent_directed():
- return classify_message_is_agent_directed("also check the mobile breakpoint", "Fix checkout", [], trace_id=TRACE_ID)
+ return classify_message_is_agent_directed(
+ "also check the mobile breakpoint", "Fix checkout", [], trace_id=TRACE_ID, team_id=TEAM_ID
+ )
def _classify_model_override():
- return classify_slack_app_model_override("use fable for this", CHOICES, trace_id=TRACE_ID)
+ return classify_slack_app_model_override("use fable for this", CHOICES, trace_id=TRACE_ID, team_id=TEAM_ID)
def _classify_project_route():
- return classify_slack_app_project_route("check the error rate on staging", PROJECTS, trace_id=TRACE_ID)
+ return classify_slack_app_project_route(
+ "check the error rate on staging", PROJECTS, trace_id=TRACE_ID, team_id=TEAM_ID
+ )
OPENAI_CLASSIFIERS = [
@@ -84,11 +90,13 @@ def _chat_reply(mock_openai: MagicMock, text: str) -> None:
def _assert_routing_product(headers: dict[str, str], classifier: str) -> None:
# `slack_app` bills the customer.
- assert headers["X-PostHog-Product"] == "slack_app_routing"
+ assert headers["X-PostHog-Product"] == "slack_app"
properties = json.loads(headers["X-PostHog-Properties"])
- assert properties["ai_product"] == "slack_app_routing"
- # All four classifiers share that product, so an online evaluation can only reach one
- # of them through this label. A call that loses it is graded as the wrong classifier.
+ assert properties["ai_product"] == "slack_app"
+ # The gateway charges this team. A call that loses it bills no customer.
+ assert properties["team_id"] == str(TEAM_ID)
+ # Every Slack generation shares that product, so an online evaluation can only reach one
+ # classifier through this label. A call that loses it is graded as the wrong classifier.
assert properties[CLASSIFIER_PROPERTY] == classifier
# Without this header the gateway stamps a fresh id per request, so a classifier that drops it
# stops grouping with its thread.
diff --git a/products/slack_app/backend/temporal/tests/activities/test_model_override_activity.py b/products/slack_app/backend/temporal/tests/activities/test_model_override_activity.py
index a04a5c1f6789..5f8a8c207cca 100644
--- a/products/slack_app/backend/temporal/tests/activities/test_model_override_activity.py
+++ b/products/slack_app/backend/temporal/tests/activities/test_model_override_activity.py
@@ -81,7 +81,7 @@ class TestClassifySlackAppModelOverrideActivity:
):
text = "fix the flaky checkout test"
assert classify_slack_app_model_override_activity(_input(integration, text)) == classified
- classify.assert_called_once_with(text, CATALOGUE, trace_id=None)
+ classify.assert_called_once_with(text, CATALOGUE, trace_id=None, team_id=integration.team_id)
@pytest.mark.parametrize(
"thread_ts,groups_with_its_thread",
diff --git a/products/slack_app/backend/temporal/tests/test_classify_task_needs_repo.py b/products/slack_app/backend/temporal/tests/test_classify_task_needs_repo.py
index 6b47c5b9bf92..eeb87cc7facf 100644
--- a/products/slack_app/backend/temporal/tests/test_classify_task_needs_repo.py
+++ b/products/slack_app/backend/temporal/tests/test_classify_task_needs_repo.py
@@ -198,7 +198,7 @@ class TestClassifyTaskNeedsRepo:
build_client.assert_called_once_with(
product="slack_app_routing",
- ai_product="slack_app_routing",
+ ai_product="slack_app",
trace_id=None,
properties={CLASSIFIER_PROPERTY: "task_needs_repo"},
)