:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(slack): bill classifiers as slack_app, rename model router product (#118524)
Vojta Bartošddb00c53a33e
diff --git a/products/slack_app/backend/temporal/activities/classifiers.py b/products/slack_app/backend/temporal/activities/classifiers.pyindex 06ec877077df..da166bc15f58 100644--- a/products/slack_app/backend/temporal/activities/classifiers.py+++ b/products/slack_app/backend/temporal/activities/classifiers.py@@ -6,7 +6,7 @@ from openai.types.shared_params import ResponseFormatJSONSchemafrom pydantic import BaseModel, ValidationErrorfrom temporalio import activity-from posthog.llm.gateway_client import build_anthropic_client, build_openai_client, trace_id_from_seed+from posthog.llm.gateway_client import Product, build_anthropic_client, build_openai_client, trace_id_from_seedfrom posthog.llm.semantic_enrichment import extract_json_objectfrom posthog.models.integration import Integration, SlackIntegrationfrom posthog.models.repo_routing_rule import RepoRoutingRule@@ -39,12 +39,28 @@ prompts = PromptTemplates(Path(__file__).parent / "prompts")CLASSIFIER_THREAD_HISTORY_MESSAGES = 10CLASSIFIER_MODEL = "claude-haiku-4-5-20251001"-# Every classifier here shares `ai_product="slack_app_routing"`, so the captured generation-# carries this property to say which one of them made the call. An online evaluation scopes-# itself with it; without it a judge would grade all four.+# The classifiers bill the customer's PostHog AI credits, the same as the Slack agent's runs,+# so they share its product.+CLASSIFIER_AI_PRODUCT = "slack_app"+# The Python gateway's `slack_app` route refuses API keys, so its fallback keeps the+# `slack_app_routing` route, which accepts them. The usage report does not bill that route.+CLASSIFIER_FALLBACK_ROUTE: Product = "slack_app_routing"++# Every classifier here and every Slack agent run share `ai_product="slack_app"`, so the captured+# generation carries this property to say which classifier made the call. An online evaluation+# scopes itself with it; without it a judge would grade every Slack generation.CLASSIFIER_PROPERTY = "slack_app_classifier"+def _classifier_properties(classifier: str, team_id: int | None) -> dict[str, str]:+ properties = {CLASSIFIER_PROPERTY: classifier}+ # The gateway and the usage report charge the team named here. Without it, the call+ # settles on the gateway's relay team and bills no customer.+ if team_id is not None:+ properties["team_id"] = str(team_id)+ return properties++def _thread_trace_id(slack_team_id: str | None, thread_ts: str | None) -> str | None:"""Group one Slack thread's routing calls into a single trace.@@ -110,6 +126,7 @@ def classify_task_needs_repo(thread_messages: list[SlackThreadMessage],routing_rules: list[str] | None = None,trace_id: str | None = None,+ team_id: int | None = None,) -> bool:"""Classify whether a Slack conversation requires code repository access.@@ -209,10 +226,10 @@ def classify_task_needs_repo(try:# The Go gateway refuses a Claude model on chat completions.client = build_anthropic_client(- product="slack_app_routing",- ai_product="slack_app_routing",+ product=CLASSIFIER_FALLBACK_ROUTE,+ ai_product=CLASSIFIER_AI_PRODUCT,trace_id=trace_id,- properties={CLASSIFIER_PROPERTY: "task_needs_repo"},+ properties=_classifier_properties("task_needs_repo", team_id),)response = client.messages.create(model=CLASSIFIER_MODEL,@@ -276,6 +293,7 @@ def classify_posthog_code_task_needs_repo_activity(thread_messages,routing_rules=routing_rules,trace_id=_thread_trace_id(inputs.slack_team_id, thread_ts),+ team_id=integration.team_id,)@@ -305,6 +323,7 @@ def classify_message_is_agent_directed(task_title: str,thread_history: list[SlackThreadMessage],trace_id: str | None = None,+ team_id: int | None = None,) -> bool:"""Classify whether an untagged Slack thread reply is an instruction to the runningPostHog Slack App, or people talking to each other.@@ -339,10 +358,10 @@ def classify_message_is_agent_directed()try:client = build_openai_client(- product="slack_app_routing",- ai_product="slack_app_routing",+ product=CLASSIFIER_FALLBACK_ROUTE,+ ai_product=CLASSIFIER_AI_PRODUCT,trace_id=trace_id,- properties={CLASSIFIER_PROPERTY: "agent_directed"},+ properties=_classifier_properties("agent_directed", team_id),).with_options(timeout=AGENT_DIRECTED_TIMEOUT_SECONDS, max_retries=AGENT_DIRECTED_MAX_RETRIES)response = client.chat.completions.create(model=AGENT_DIRECTED_CLASSIFIER_MODEL,@@ -430,6 +449,7 @@ def classify_untagged_followup_activity(task_title,thread_history,trace_id=_thread_trace_id(inputs.slack_team_id, thread_ts),+ team_id=integration.team_id,):return True@@ -487,6 +507,7 @@ def classify_slack_app_model_override(event_text: str,choices: tuple[ModelChoice, ...],trace_id: str | None = None,+ team_id: int | None = None,) -> SlackAppModelOverride | None:"""Read a per-task model or reasoning-effort request out of a Slack mention.@@ -516,10 +537,10 @@ def classify_slack_app_model_override(try:client = build_openai_client(- product="slack_app_routing",- ai_product="slack_app_routing",+ product=CLASSIFIER_FALLBACK_ROUTE,+ ai_product=CLASSIFIER_AI_PRODUCT,trace_id=trace_id,- properties={CLASSIFIER_PROPERTY: "model_override"},+ properties=_classifier_properties("model_override", team_id),).with_options(timeout=MODEL_OVERRIDE_TIMEOUT_SECONDS, max_retries=MODEL_OVERRIDE_MAX_RETRIES)response = client.chat.completions.create(model=MODEL_OVERRIDE_CLASSIFIER_MODEL,@@ -592,6 +613,7 @@ def classify_slack_app_model_override_activity(input: SlackAppModelOverrideInputinput.event_text,choices,trace_id=_thread_trace_id(input.slack_team_id, input.thread_ts),+ team_id=integration.team_id,)if override is None:return None@@ -638,6 +660,7 @@ def classify_slack_app_project_route(projects: list[Integration],default: Integration | None = None,trace_id: str | None = None,+ team_id: int | None = None,) -> Integration | None:"""Read the project a mention asked to be answered from, out of its text.@@ -671,10 +694,10 @@ def classify_slack_app_project_route(try:client = build_openai_client(- product="slack_app_routing",- ai_product="slack_app_routing",+ product=CLASSIFIER_FALLBACK_ROUTE,+ ai_product=CLASSIFIER_AI_PRODUCT,trace_id=trace_id,- properties={CLASSIFIER_PROPERTY: "project_route"},+ properties=_classifier_properties("project_route", team_id),).with_options(timeout=PROJECT_ROUTE_TIMEOUT_SECONDS, max_retries=PROJECT_ROUTE_MAX_RETRIES)response = client.chat.completions.create(model=PROJECT_ROUTE_CLASSIFIER_MODEL,@@ -739,6 +762,7 @@ def classify_slack_app_project_route_activity(input: SlackAppProjectRouteInput)projects,default=integration,trace_id=_thread_trace_id(input.slack_team_id, input.thread_ts),+ team_id=integration.team_id,)except Exception:# The fallback boundary: a mention we cannot classify stays on the projectdiff --git a/products/slack_app/backend/temporal/activities/model_router.py b/products/slack_app/backend/temporal/activities/model_router.pyindex 39c005a70a48..33d380fb17d3 100644--- a/products/slack_app/backend/temporal/activities/model_router.py+++ b/products/slack_app/backend/temporal/activities/model_router.py@@ -36,6 +36,7 @@ MODEL_ROUTER_TIMEOUT_SECONDS = 5.0_QUESTION_ID = "model"DECISION_SPAN_NAME = "slack_model_router_decision"+MODEL_ROUTER_AI_PRODUCT = "slack_app_model_router"@frozen@@ -63,7 +64,7 @@ def classify_slack_app_model_router(# No TypeSafe fallback: the request is customer text, and TypeSafe is a third party.client = build_system_one_client(model=MODEL_ROUTER_DECISION_MODEL,- ai_product="slack_app_routing",+ ai_product=MODEL_ROUTER_AI_PRODUCT,team_id=team_id,distinct_id=distinct_id,trace_id=trace_id,@@ -120,7 +121,7 @@ def _capture_decision(}],CLASSIFIER_PROPERTY: "slack_model_router",- "ai_product": "slack_app_routing",+ "ai_product": MODEL_ROUTER_AI_PRODUCT,},)diff --git a/products/slack_app/backend/temporal/tests/activities/test_classifier_gateway_routing.py b/products/slack_app/backend/temporal/tests/activities/test_classifier_gateway_routing.pyindex d4d7d64e151e..172b7daf1c09 100644--- a/products/slack_app/backend/temporal/tests/activities/test_classifier_gateway_routing.py+++ b/products/slack_app/backend/temporal/tests/activities/test_classifier_gateway_routing.py@@ -25,6 +25,7 @@ AI_GATEWAY_KEY = "phs_go"LLM_GATEWAY_URL = "http://llm-gateway:8080"LLM_GATEWAY_KEY = "phx_legacy"TRACE_ID = "1b9d6bcd-bbfd-5b1d-9c2f-9f0a4b4a1b2c"+TEAM_ID = 41# Both modes set every setting the builders read, so the wrong route cannot pass by accident.GO_GATEWAY = {@@ -48,19 +49,24 @@ def _classify_needs_repo():NEEDS_REPO_TEXT,[SlackThreadMessage(user="Alessandro", text=NEEDS_REPO_TEXT)],trace_id=TRACE_ID,+ team_id=TEAM_ID,)def _classify_agent_directed():- return classify_message_is_agent_directed("also check the mobile breakpoint", "Fix checkout", [], trace_id=TRACE_ID)+ return classify_message_is_agent_directed(+ "also check the mobile breakpoint", "Fix checkout", [], trace_id=TRACE_ID, team_id=TEAM_ID+ )def _classify_model_override():- return classify_slack_app_model_override("use fable for this", CHOICES, trace_id=TRACE_ID)+ return classify_slack_app_model_override("use fable for this", CHOICES, trace_id=TRACE_ID, team_id=TEAM_ID)def _classify_project_route():- return classify_slack_app_project_route("check the error rate on staging", PROJECTS, trace_id=TRACE_ID)+ return classify_slack_app_project_route(+ "check the error rate on staging", PROJECTS, trace_id=TRACE_ID, team_id=TEAM_ID+ )OPENAI_CLASSIFIERS = [@@ -84,11 +90,13 @@ def _chat_reply(mock_openai: MagicMock, text: str) -> None:def _assert_routing_product(headers: dict[str, str], classifier: str) -> None:# `slack_app` bills the customer.- assert headers["X-PostHog-Product"] == "slack_app_routing"+ assert headers["X-PostHog-Product"] == "slack_app"properties = json.loads(headers["X-PostHog-Properties"])- assert properties["ai_product"] == "slack_app_routing"- # All four classifiers share that product, so an online evaluation can only reach one- # of them through this label. A call that loses it is graded as the wrong classifier.+ assert properties["ai_product"] == "slack_app"+ # The gateway charges this team. A call that loses it bills no customer.+ assert properties["team_id"] == str(TEAM_ID)+ # Every Slack generation shares that product, so an online evaluation can only reach one+ # classifier through this label. A call that loses it is graded as the wrong classifier.assert properties[CLASSIFIER_PROPERTY] == classifier# Without this header the gateway stamps a fresh id per request, so a classifier that drops it# stops grouping with its thread.diff --git a/products/slack_app/backend/temporal/tests/activities/test_model_override_activity.py b/products/slack_app/backend/temporal/tests/activities/test_model_override_activity.pyindex a04a5c1f6789..5f8a8c207cca 100644--- a/products/slack_app/backend/temporal/tests/activities/test_model_override_activity.py+++ b/products/slack_app/backend/temporal/tests/activities/test_model_override_activity.py@@ -81,7 +81,7 @@ class TestClassifySlackAppModelOverrideActivity:):text = "fix the flaky checkout test"assert classify_slack_app_model_override_activity(_input(integration, text)) == classified- classify.assert_called_once_with(text, CATALOGUE, trace_id=None)+ classify.assert_called_once_with(text, CATALOGUE, trace_id=None, team_id=integration.team_id)@pytest.mark.parametrize("thread_ts,groups_with_its_thread",diff --git a/products/slack_app/backend/temporal/tests/test_classify_task_needs_repo.py b/products/slack_app/backend/temporal/tests/test_classify_task_needs_repo.pyindex 6b47c5b9bf92..eeb87cc7facf 100644--- a/products/slack_app/backend/temporal/tests/test_classify_task_needs_repo.py+++ b/products/slack_app/backend/temporal/tests/test_classify_task_needs_repo.py@@ -198,7 +198,7 @@ class TestClassifyTaskNeedsRepo:build_client.assert_called_once_with(product="slack_app_routing",- ai_product="slack_app_routing",+ ai_product="slack_app",trace_id=None,properties={CLASSIFIER_PROPERTY: "task_needs_repo"},)