:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(slack-app): offer answers to untagged channel questions (#114598)
Vojta Bartošfd2e8ba0a549
diff --git a/products/slack_app/backend/api.py b/products/slack_app/backend/api.pyindex 7136fa913254..9112dcff11c3 100644--- a/products/slack_app/backend/api.py+++ b/products/slack_app/backend/api.py@@ -51,9 +51,11 @@ from products.slack_app.backend.feature_flags import (is_slack_app_assistant_enabled,is_slack_app_oauth_enabled,is_slack_app_project_picker_enabled,+ is_slack_app_unprompted_answers_enabled,)from products.slack_app.backend.helpers import local_dev_slack_emailfrom products.slack_app.backend.logic import reply_suggestions+from products.slack_app.backend.logic.untagged_question import is_untagged_question_candidatefrom products.slack_app.backend.models import (ChannelWelcomeMode,SlackChannel,@@ -87,7 +89,7 @@ from products.slack_app.backend.services.slack_app_home import (handle_app_home_opened as _handle_app_home_opened,handle_app_home_view_submission as _handle_app_home_view_submission,)-from products.slack_app.backend.services.slack_conversations import channel_is_ext_shared+from products.slack_app.backend.services.slack_conversations import channel_ext_shared_state, channel_is_ext_sharedfrom products.slack_app.backend.services.slack_fork_context import clear_pending_fork, get_pending_forkfrom products.slack_app.backend.services.slack_messages import (FORK_THREAD_ACTION_ID,@@ -105,6 +107,8 @@ from products.slack_app.backend.services.slack_scopes import REQUIRED_SLACK_SCOPfrom products.slack_app.backend.services.slack_settings import (resolve_channel_welcome_mode,resolve_untagged_followup_mode,+ resolve_untagged_question_mode,+ set_untagged_question_mode,)from products.slack_app.backend.services.slack_user_info import (clear_workspace_profile_cache,@@ -207,6 +211,11 @@ UNTAGGED_FOLLOWUP_BLOCK_ID_PREFIX = "posthog_code_untagged_followup"UNTAGGED_FOLLOWUP_ACTION_RUN = "posthog_code_untagged_followup_run"UNTAGGED_FOLLOWUP_ACTION_DISMISS = "posthog_code_untagged_followup_dismiss"UNTAGGED_FOLLOWUP_CONTEXT_KIND = "untagged_followup"+UNTAGGED_QUESTION_BLOCK_ID_PREFIX = "posthog_code_untagged_question"+UNTAGGED_QUESTION_ACTION_RUN = "posthog_code_untagged_question_run"+UNTAGGED_QUESTION_ACTION_DISMISS = "posthog_code_untagged_question_dismiss"+UNTAGGED_QUESTION_ACTION_TURN_OFF = "posthog_code_untagged_question_turn_off"+UNTAGGED_QUESTION_CONTEXT_KIND = "untagged_question"_MAX_GITHUB_REPOS = 500# The installation webhook marks snapshots stale without saving rows, so it does not clear this key.@@ -597,6 +606,9 @@ REGION_PROXY_HEADER = "X-PostHog-Region-Proxied"# be connected in both regions; the receiver runs the emit for its own projects and nothing else,# so the sender's pipeline (thread follow-ups, mentions) stays the only one handling the event.EMIT_ONLY_MIRROR_HEADER = "X-PostHog-Slack-Emit-Only"+# Marks a full top-level post the sender already emitted and mirrored. The receiver handles the+# untagged question but skips the emit, which the mirror already covers.+ALREADY_EMITTED_HEADER = "X-PostHog-Slack-Already-Emitted"REGION_PROXY_TIMEOUT_SECONDS = 3# Tight budget: the workspace_claims endpoint is just a DB .exists(), and EU calls it inline# before deciding whether to proxy. Slack's webhook ack deadline is 3s total, so we want this@@ -705,20 +717,25 @@ def send_region_proxy_request(return None-def _proxy_event_to_region(request: HttpRequest, target_domain: str) -> requests.Response | None:+def _proxy_event_to_region(+ request: HttpRequest, target_domain: str, extra_headers: dict[str, str] | None = None+) -> requests.Response | None:"""Forward the original Slack event to the other region, tagged so the receiver does not hop again."""return send_region_proxy_request(method=request.method or "POST",target_url=_proxy_target_url(request, target_domain),- headers=_proxy_request_headers(request),+ headers={**_proxy_request_headers(request), **(extra_headers or {})},params=dict(request.GET.lists()) if request.GET else None,body=request.body or None,)-def _proxy_event_and_return_route(request: HttpRequest, target_domain: str) -> str:+def _proxy_event_and_return_route(+ request: HttpRequest, target_domain: str, extra_headers: dict[str, str] | None = None+) -> str:"""Forward and translate the upstream result into a routing outcome string."""- return ROUTE_PROXIED if _proxy_event_to_region(request, target_domain) is not None else ROUTE_PROXY_FAILED+ upstream = _proxy_event_to_region(request, target_domain, extra_headers)+ return ROUTE_PROXIED if upstream is not None else ROUTE_PROXY_FAILEDdef _is_top_level_channel_post(event: dict[str, Any]) -> bool:@@ -1432,6 +1449,24 @@ def _message_handled_cache_key(slack_team_id: str, event: dict[str, Any]) -> strreturn f"slack_app:message_handled:v1:{slack_team_id}:{channel}:{message_ts}"+def claim_message_handled(slack_team_id: str, event: dict[str, Any], handled_as: str) -> bool:+ """Record that the pipeline acted on this message, unless another path already did.++ Atomic, so an answer to an untagged question and an edit of the same message that adds a tag+ can't both start a run. Returns the marker's current value on a lost claim, so a caller can+ recognise its own earlier claim.+ """+ cache_key = _message_handled_cache_key(slack_team_id, event)+ if cache_key is None:+ return True+ return bool(cache.add(cache_key, handled_as, timeout=MESSAGE_HANDLED_MARKER_TTL_SECONDS))+++def message_handled_by(slack_team_id: str, event: dict[str, Any]) -> str | None:+ cache_key = _message_handled_cache_key(slack_team_id, event)+ return cache.get(cache_key) if cache_key else None++def _mark_message_handled(slack_team_id: str, event: dict[str, Any], handled_as: str) -> None:"""Record that the pipeline acted on this message, so that a later edit of it starts nothing."""cache_key = _message_handled_cache_key(slack_team_id, event)@@ -1459,7 +1494,7 @@ def _edited_mention_ignore_cause(event: dict[str, Any], slack_team_id: str) -> sreturn "no_message_ts"if time.time() - posted_at > EDITED_MENTION_WINDOW_SECONDS:return "too_old"- if not cache.add(cache_key, "edited_mention", timeout=MESSAGE_HANDLED_MARKER_TTL_SECONDS):+ if not claim_message_handled(slack_team_id, event, "edited_mention"):return f"handled_as_{cache.get(cache_key) or 'unknown'}"return None@@ -2489,6 +2524,92 @@ def _handle_app_uninstalled(request: HttpRequest, slack_team_id: str) -> str:return ROUTE_HANDLED_LOCALLY+def _route_untagged_question(+ request: HttpRequest,+ event: dict[str, Any],+ slack_team_id: str,+ event_id: str | None,+ *,+ proxied: bool,+ incoming_host: str,+ other_domain: str,+ can_defer: bool,+ is_ext_shared_channel: bool,+) -> str:+ """Start the classifier workflow for a top-level channel post nobody tagged the app in.++ Every drop here is silent: nobody asked PostHog anything, so nobody waits for an answer.+ An author without a PostHog account is a normal drop, not a failure.+ Externally shared channels are out, because people outside the organization would see the reply.+ """+ if is_ext_shared_channel or not is_untagged_question_candidate(event):+ return ROUTE_HANDLED_LOCALLY+ slack_user_id = str(event.get("user") or "")+ channel = event.get("channel") if isinstance(event.get("channel"), str) else None++ workspace_result = load_integrations(+ slack_team_id=slack_team_id,+ kinds=[SLACK_INTEGRATION_KIND],+ slack_user_id=slack_user_id,+ channel=channel,+ )+ # Without a local connection no mirror went out, so the full event can cross, but only to a+ # region that confirms it holds the workspace. The probe answer is cached per workspace.+ if not workspace_result.candidates:+ if proxied or not cross_region_routing_enabled():+ return ROUTE_HANDLED_LOCALLY+ claimed = does_other_region_claim_workspace(+ slack_team_id=slack_team_id, kinds=[SLACK_INTEGRATION_KIND], incoming_host=incoming_host+ )+ if claimed is not True:+ return ROUTE_HANDLED_LOCALLY+ return _proxy_event_and_return_route(request, other_domain)+ # The same US precedence as mentions, so an author whose account is only in US still gets an+ # answer when EU receives the post. This region already emitted the post and queued a mirror,+ # so the forwarded copy tells the receiver to skip its emit.+ if _us_should_handle_instead(slack_team_id, [SLACK_INTEGRATION_KIND], can_defer, incoming_host):+ return _proxy_event_and_return_route(request, other_domain, {ALREADY_EMITTED_HEADER: "1"})++ # One indexed read, ahead of user resolution, which can call Slack's users.info for every author.+ if resolve_untagged_question_mode(slack_team_id, slack_user_id) == UntaggedFollowupMode.NEVER:+ return ROUTE_HANDLED_LOCALLY++ resolution = resolve_user_for_workspace(+ workspace_result=workspace_result,+ slack_team_id=slack_team_id,+ slack_user_id=slack_user_id,+ event_id=event_id,+ )+ posthog_user = resolution.user+ if posthog_user is None:+ return ROUTE_HANDLED_LOCALLY+ candidates = resolution.candidates+ # No picker: asking which project a question nobody addressed to us belongs to is noise.+ target = resolution.integration or (candidates[0] if len(candidates) == 1 else None)+ if target is None or SlackIntegration(target).missing_scopes(REQUIRED_SLACK_SCOPES):+ return ROUTE_HANDLED_LOCALLY+ # Checked on the project that will answer: a workspace can connect projects from several+ # organizations, and the flag is rolled out per organization.+ if not is_slack_app_unprompted_answers_enabled(target):+ return ROUTE_HANDLED_LOCALLY++ logger.info(+ "slack_app_untagged_question_dispatched",+ slack_team_id=slack_team_id,+ integration_id=target.id,+ channel=channel,+ message_ts=event.get("ts"),+ )+ return _start_mention_workflow(+ event,+ target,+ slack_team_id,+ event_id,+ posthog_user=posthog_user,+ untagged_question=True,+ )++def _dispatch_mention_to_target(event: dict,mention_target: Integration,@@ -2676,7 +2797,7 @@ def route_posthog_code_event_to_relevant_region()return ROUTE_HANDLED_LOCALLY- should_try_other_region = emit_slack_message_event(+ should_try_other_region = request.headers.get(ALREADY_EMITTED_HEADER) != "1" and emit_slack_message_event(event,slack_team_id,event_id=event_id,@@ -2743,9 +2864,20 @@ def route_posthog_code_event_to_relevant_region(message_ts=event.get("ts"),)return ROUTE_HANDLED_LOCALLY- # Top-level channel posts dominate the wire volume; drop before the pipeline's DB hits.+ # Top-level channel posts dominate the wire volume, so they leave the shared pipeline+ # here and face cheap gates before any DB hit.if _is_top_level_channel_post(event):- return ROUTE_HANDLED_LOCALLY+ return _route_untagged_question(+ request,+ event,+ slack_team_id,+ event_id,+ proxied=proxied,+ incoming_host=incoming_host,+ other_domain=other_domain,+ can_defer=can_defer_to_other_region,+ is_ext_shared_channel=is_ext_shared_channel,+ )slack_user_id_str = str(event.get("user") or "")channel_str = event.get("channel") if isinstance(event.get("channel"), str) else None@@ -3822,6 +3954,217 @@ def _post_untagged_followup_prompt(return True+def _post_untagged_question_prompt(slack: SlackIntegration, integration: Integration, event: dict[str, Any]) -> bool:+ """Offer the author, privately, to answer the question they posted without tagging the app.++ Posted when the author's mode is ``ask``. The whole event is stashed behind a one-off+ token so a click dispatches the original message. Not threaded: the post has no thread+ yet, so an ephemeral anchored to one would never be seen.+ """+ channel = event.get("channel") if isinstance(event.get("channel"), str) else None+ slack_user_id = event.get("user") if isinstance(event.get("user"), str) else None+ if not channel or not slack_user_id:+ logger.warning("slack_app_untagged_question_prompt_skipped", integration_id=integration.id)+ return False++ context_token = uuid.uuid4().hex+ cache.set(+ _picker_context_cache_key(context_token),+ {+ "kind": UNTAGGED_QUESTION_CONTEXT_KIND,+ "integration_id": integration.id,+ "slack_user_id": slack_user_id,+ "event": event,+ },+ timeout=PICKER_TOKEN_MAX_AGE_SECONDS,+ )++ home_tab_url = app_home_url(integration)+ home_tab_label = f"<{home_tab_url}|PostHog app Home tab>" if home_tab_url else "PostHog app Home tab"+ blocks: list[dict[str, Any]] = [+ {+ "type": "section",+ "block_id": f"{UNTAGGED_QUESTION_BLOCK_ID_PREFIX}:{context_token}",+ "text": {+ "type": "mrkdwn",+ "text": (+ "I think I can answer your question from your PostHog data. Want me to reply in a thread? "+ "Only you can see this."+ ),+ },+ },+ {+ "type": "actions",+ "block_id": f"{UNTAGGED_QUESTION_BLOCK_ID_PREFIX}_actions:{context_token}",+ "elements": [+ {+ "type": "button",+ "action_id": UNTAGGED_QUESTION_ACTION_RUN,+ "style": "primary",+ "text": {"type": "plain_text", "text": "Yes, answer it"},+ "value": context_token,+ },+ {+ "type": "button",+ "action_id": UNTAGGED_QUESTION_ACTION_DISMISS,+ "text": {"type": "plain_text", "text": "No thanks"},+ "value": context_token,+ },+ {+ "type": "button",+ "action_id": UNTAGGED_QUESTION_ACTION_TURN_OFF,+ "text": {"type": "plain_text", "text": "Only when I tag you"},+ "value": context_token,+ },+ ],+ },+ {+ "type": "context",+ "elements": [{"type": "mrkdwn", "text": f"You can change this in the {home_tab_label}."}],+ },+ ]++ try:+ slack.client.chat_postEphemeral(+ channel=channel,+ user=slack_user_id,+ text="PostHog can answer your question. Want a reply?",+ blocks=blocks,+ )+ except Exception:+ logger.warning(+ "slack_app_untagged_question_prompt_failed",+ integration_id=integration.id,+ slack_channel_id=channel,+ exc_info=True,+ )+ return False+ capture_slack_event(integration, "slack app unprompted answer offered", slack_user_id=slack_user_id)+ return True+++@frozen+class _UntaggedQuestionClick:+ context_token: str+ context: dict[str, Any]+ integration: Integration+++def _untagged_question_click(payload: dict) -> _UntaggedQuestionClick | None:+ """The live prompt context and its integration, or None when the prompt is stale or not the clicker's."""+ context_token = _extract_context_token(payload)+ context = _decode_picker_context(context_token) if context_token else None+ if not context or context.get("kind") != UNTAGGED_QUESTION_CONTEXT_KIND:+ return None+ slack_team_id = payload.get("team", {}).get("id", "")+ integration_id = context.get("integration_id")+ # The prompt is ephemeral, so only its recipient can click it, but the dispatch runs as the clicker.+ if not integration_id or not slack_team_id or payload.get("user", {}).get("id") != context.get("slack_user_id"):+ return None+ integration = (+ Integration.objects.filter( # nosemgrep: idor-lookup-without-team+ id=integration_id, # nosemgrep: idor-taint-user-input-to-model-get+ kind=SLACK_INTEGRATION_KIND,+ integration_id=slack_team_id,+ )+ .select_related("team__organization")+ .first()+ )+ if integration is None:+ return None+ return _UntaggedQuestionClick(context_token=context_token, context=context, integration=integration)+++def _handle_untagged_question_run(payload: dict) -> HttpResponse:+ response_url = payload.get("response_url", "")+ click = _untagged_question_click(payload)+ if click is None:+ _delete_ephemeral_via_response_url(response_url)+ return HttpResponse(status=200)+ context_token, context, integration = click.context_token, click.context, click.integration+ slack_user_id = context["slack_user_id"]+ event = context.get("event")++ # The confirmed run skips the gates the webhook and the workflow already applied, so the+ # ones that can change while the prompt waits are checked again here. The two checks that+ # call Slack run last.+ slack = SlackIntegration(integration)+ if (+ not isinstance(event, dict)+ or slack.missing_scopes(REQUIRED_SLACK_SCOPES)+ or resolve_untagged_question_mode(integration.integration_id, slack_user_id) == UntaggedFollowupMode.NEVER+ or not is_slack_app_unprompted_answers_enabled(integration)+ or (posthog_user := _is_org_member(integration, slack_user_id)) is None+ or not _can_access_team(posthog_user, integration)+ ):+ _delete_ephemeral_via_response_url(response_url)+ return HttpResponse(status=200)++ # A channel shared outside the organization since the offer would show the answer to people+ # the webhook's gate kept it from. When Slack cannot say, the offer stays for a later click.+ ext_shared = channel_ext_shared_state(slack, event.get("channel", ""))+ if ext_shared is None:+ return HttpResponse(status=200)+ if ext_shared:+ _delete_ephemeral_via_response_url(response_url)+ return HttpResponse(status=200)++ _start_mention_workflow(+ event,+ integration,+ integration.integration_id,+ None,+ posthog_user=posthog_user,+ untagged_question=True,+ untagged_question_confirmed=True,+ )+ # Cleared only once the run is dispatched, so a Temporal outage leaves the offer clickable.+ cache.delete(_picker_context_cache_key(context_token))+ capture_slack_event(+ integration, "slack app unprompted answer confirmed", slack_user_id=slack_user_id, posthog_user=posthog_user+ )+ _delete_ephemeral_via_response_url(response_url)+ return HttpResponse(status=200)+++def _handle_untagged_question_dismiss(payload: dict) -> HttpResponse:+ """Drop the offer. The choice covers this one message, so nothing is stored."""+ click = _untagged_question_click(payload)+ _delete_ephemeral_via_response_url(payload.get("response_url", ""))+ if click is not None:+ context_token, context, integration = click.context_token, click.context, click.integration+ cache.delete(_picker_context_cache_key(context_token))+ capture_slack_event(+ integration, "slack app unprompted answer dismissed", slack_user_id=context["slack_user_id"]+ )+ return HttpResponse(status=200)+++def _handle_untagged_question_turn_off(payload: dict) -> HttpResponse:+ """Stop answering the clicker's untagged channel questions, and say where to turn it back on."""+ response_url = payload.get("response_url", "")+ click = _untagged_question_click(payload)+ if click is None:+ _delete_ephemeral_via_response_url(response_url)+ return HttpResponse(status=200)+ context_token, context, integration = click.context_token, click.context, click.integration+ slack_user_id = context["slack_user_id"]+ cache.delete(_picker_context_cache_key(context_token))+ set_untagged_question_mode(integration.integration_id, slack_user_id, UntaggedFollowupMode.NEVER)+ capture_slack_event(integration, "slack app unprompted answers turned off", slack_user_id=slack_user_id)+ inbox_interactivity.post_response_url(+ response_url,+ {+ "replace_original": True,+ "text": (+ "Got it. I'll only answer your questions in channels when you tag me. "+ "You can change this in the PostHog app Home tab."+ ),+ },+ )+ return HttpResponse(status=200)++def _can_access_team(posthog_user: User, integration: Integration) -> bool:"""Whether the user can reach the integration's project.@@ -4291,6 +4634,8 @@ def _start_mention_workflow(posthog_user: User,untagged_followup: bool = False,untagged_followup_confirmed: bool = False,+ untagged_question: bool = False,+ untagged_question_confirmed: bool = False,is_ext_shared_channel: bool = False,awaited_request_reply: bool = False,fork_source_channel: str | None = None,@@ -4312,10 +4657,11 @@ def _start_mention_workflow(The ``fork_source_*`` fields mark a forked run, where the thread being answered(a DM) and the thread supplying the context (the forked channel thread) differ.They suppress the same two side effects: nobody mentioned us, and a reply in a- forked DM can't be resolving a picker.+ forked DM can't be resolving a picker. ``untagged_question`` suppresses them+ for the same reason, and makes the workflow ask its classifier first."""is_fork = bool(fork_source_channel and fork_source_thread_ts)- if not untagged_followup and not is_fork:+ if not untagged_followup and not untagged_question and not is_fork:_report_slack_mention_received(event,integration,@@ -4333,6 +4679,8 @@ def _start_mention_workflow(user_id=posthog_user.id,untagged_followup=untagged_followup,untagged_followup_confirmed=untagged_followup_confirmed,+ untagged_question=untagged_question,+ untagged_question_confirmed=untagged_question_confirmed,is_ext_shared_channel=is_ext_shared_channel,fork_source_channel=fork_source_channel,fork_source_thread_ts=fork_source_thread_ts,@@ -5838,6 +6186,12 @@ def posthog_code_interactivity_handler(request: HttpRequest) -> HttpResponse:return _handle_untagged_followup_run(payload)if action_id == UNTAGGED_FOLLOWUP_ACTION_DISMISS:return _handle_untagged_followup_dismiss(payload)+ if action_id == UNTAGGED_QUESTION_ACTION_RUN:+ return _handle_untagged_question_run(payload)+ if action_id == UNTAGGED_QUESTION_ACTION_DISMISS:+ return _handle_untagged_question_dismiss(payload)+ if action_id == UNTAGGED_QUESTION_ACTION_TURN_OFF:+ return _handle_untagged_question_turn_off(payload)if project_picker.is_project_picker_action(action_id):return _handle_project_picker_pick(payload)if action_id == SIGNALS_DISMISS_REPORT_ACTION_ID:diff --git a/products/slack_app/backend/feature_flags.py b/products/slack_app/backend/feature_flags.pyindex c11b6caef31b..9ed198f65b4d 100644--- a/products/slack_app/backend/feature_flags.py+++ b/products/slack_app/backend/feature_flags.py@@ -49,6 +49,7 @@ SLACK_APP_AGENT_DESIGN_FLAG = "slack-app-agent-design"SLACK_APP_FORKING_FLAG = "slack-app-forking"SLACK_APP_PROJECT_PICKER_FLAG = "slack-app-project-picker"SLACK_APP_REPLY_SUGGESTIONS_FLAG = "slack-app-reply-suggestions"+SLACK_APP_UNPROMPTED_ANSWERS_FLAG = "slack-app-unprompted-answers"# Linking a Slack identity to a PostHog user resolves the Slack profile and its email.@@ -146,6 +147,19 @@ def is_slack_app_reply_suggestions_enabled(integration: Integration) -> bool:)+def is_slack_app_unprompted_answers_enabled(integration: Integration) -> bool:+ """Gate for answering top-level channel messages nobody tagged the app in.++ Keyed on the workspace, not the author, because the webhook checks it before it resolves+ who wrote the message. Roll it out with an organization condition.+ """+ return _workspace_flag_enabled(+ SLACK_APP_UNPROMPTED_ANSWERS_FLAG,+ integration,+ failure_log_key="slack_app_unprompted_answers_feature_flag_check_failed",+ )++def is_slack_app_assistant_enabled(integration: Integration) -> bool:"""Gate for the DM assistant: the install must hold the scopes the DM surfacecalls — ``im:history`` in particular, without which the assistant would answerdiff --git a/products/slack_app/backend/logic/untagged_question.py b/products/slack_app/backend/logic/untagged_question.pynew file mode 100644index 000000000000..816521bf96c4--- /dev/null+++ b/products/slack_app/backend/logic/untagged_question.py@@ -0,0 +1,30 @@+"""Which top-level channel posts, written without tagging the app, are worth a classifier call."""++import re+from typing import Any++UNTAGGED_QUESTION_MIN_CHARS = 12+UNTAGGED_QUESTION_MAX_CHARS = 2000+_QUESTION_OPENER_PATTERN = re.compile(+ r"^(how|what|why|when|where|which|who|whose|is|are|was|were|can|could|does|do|did|should|would|will|has|have|any|anyone|anybody)\b",+ re.IGNORECASE,+)+++def is_untagged_question_candidate(event: dict[str, Any]) -> bool:+ """Whether a top-level post nobody tagged the app in could be a question for PostHog.++ Text-only checks, because every top-level post in every channel the app is in comes through+ here. The classifier makes the real decision later, on the few posts that pass.+ """+ # ``group`` is a private channel. Group DMs (``mpim``) are private conversations between people.+ if event.get("channel_type") not in ("channel", "group"):+ return False+ text = (event.get("text") or "").strip()+ if not UNTAGGED_QUESTION_MIN_CHARS <= len(text) <= UNTAGGED_QUESTION_MAX_CHARS:+ return False+ # A tag of the app arrives as its own ``app_mention``. A tag of anyone else, a user group or+ # the whole channel addresses them, not PostHog.+ if "<@" in text or "<!" in text:+ return False+ return "?" in text or bool(_QUESTION_OPENER_PATTERN.match(text))diff --git a/products/slack_app/backend/migrations/0021_slacksettings_untagged_question_mode.py b/products/slack_app/backend/migrations/0021_slacksettings_untagged_question_mode.pynew file mode 100644index 000000000000..a17bd2a62a58--- /dev/null+++ b/products/slack_app/backend/migrations/0021_slacksettings_untagged_question_mode.py@@ -0,0 +1,27 @@+# Generated by Django 5.2.18 on 2026-10-19 14:02++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("slack_app", "0020_slacksettings_auto_model_choice"),+ ]++ operations = [+ migrations.AddField(+ model_name="slacksettings",+ name="untagged_question_mode",+ field=models.CharField(+ blank=True,+ choices=[+ ("auto", "Always pick it up"),+ ("ask", "Ask before picking it up"),+ ("never", "Never pick it up"),+ ],+ help_text="What PostHog does with this user's top-level channel questions that do not tag the app.",+ max_length=16,+ null=True,+ ),+ ),+ ]diff --git a/products/slack_app/backend/migrations/max_migration.txt b/products/slack_app/backend/migrations/max_migration.txtindex e36f82e9ade6..19664f63b5a9 100644--- a/products/slack_app/backend/migrations/max_migration.txt+++ b/products/slack_app/backend/migrations/max_migration.txt@@ -1 +1 @@-0020_slacksettings_auto_model_choice+0021_slacksettings_untagged_question_modediff --git a/products/slack_app/backend/models.py b/products/slack_app/backend/models.pyindex d1c18b029d72..b3d4f3bc35c4 100644--- a/products/slack_app/backend/models.py+++ b/products/slack_app/backend/models.py@@ -113,11 +113,12 @@ class SlackUserProfileCache(UUIDModel):class UntaggedFollowupMode(models.TextChoices):- """What PostHog does with an untagged reply in a thread it already owns.+ """What PostHog does with a message that does not tag it.- Read from the thread creator's settings row, so it governs everyone- replying in a thread that user started. An unset row resolves to ``ASK``,- so the person who replied can choose whether to send their message.+ Read from the settings row of the person who started the thread, so it governs+ every untagged reply in a thread PostHog owns, and the author's own top-level+ channel question. An unset row resolves to ``ASK``, so the person who wrote the+ message chooses whether to send it."""AUTO = "auto", "Always pick it up"@@ -182,6 +183,14 @@ class SlackSettings(UUIDModel):choices=UntaggedFollowupMode.choices,help_text="What PostHog does with untagged replies in threads this user started.",)+ # NULL means the user has never picked, which resolves to ``ASK``.+ untagged_question_mode = models.CharField(+ max_length=16,+ null=True,+ blank=True,+ choices=UntaggedFollowupMode.choices,+ help_text="What PostHog does with this user's top-level channel questions that do not tag the app.",+ )# Only read on the workspace-wide row. NULL resolves to ``CHANNEL``.channel_welcome_mode = models.CharField(max_length=16,diff --git a/products/slack_app/backend/services/slack_app_home.py b/products/slack_app/backend/services/slack_app_home.pyindex 142b842c4864..a880ec266b63 100644--- a/products/slack_app/backend/services/slack_app_home.py+++ b/products/slack_app/backend/services/slack_app_home.py@@ -33,7 +33,7 @@ from posthog.models.user_integration import UserIntegrationfrom posthog.user_permissions import UserPermissionsfrom products.slack_app.backend.analytics import capture_slack_event-from products.slack_app.backend.feature_flags import is_slack_app_oauth_enabled+from products.slack_app.backend.feature_flags import is_slack_app_oauth_enabled, is_slack_app_unprompted_answers_enabledfrom products.slack_app.backend.models import (ChannelWelcomeMode,SlackSettings,@@ -73,8 +73,11 @@ from products.slack_app.backend.services.slack_settings import (resolve_auto_model_choice,resolve_channel_welcome_mode,resolve_untagged_followup_mode,+ resolve_untagged_question_mode,set_auto_model_choice,set_channel_welcome_mode,+ set_untagged_followup_mode,+ set_untagged_question_mode,)from products.slack_app.backend.services.slack_user_info import is_slack_workspace_adminfrom products.slack_app.backend.services.slack_user_oauth import build_invite_url, find_linked_posthog_user@@ -101,6 +104,7 @@ ACTION_TASKS_PAGE_NEXT = "slack_app_home:tasks_page_next"ACTION_STATS_WINDOW = "slack_app_home:stats_window"ACTION_STATS_REFRESH = "slack_app_home:stats_refresh"ACTION_SET_UNTAGGED_FOLLOWUP_MODE = "slack_app_home:set_untagged_followup_mode"+ACTION_SET_UNTAGGED_QUESTION_MODE = "slack_app_home:set_untagged_question_mode"ACTION_SET_CHANNEL_WELCOME_MODE = "slack_app_home:set_channel_welcome_mode"ACTION_SET_AUTO_MODEL_CHOICE = "slack_app_home:set_auto_model_choice"# URL buttons: Slack opens the link itself and posts a block_actions payload we@@ -128,6 +132,7 @@ HOME_ACTION_IDS: frozenset[str] = frozenset(ACTION_STATS_WINDOW,ACTION_STATS_REFRESH,ACTION_SET_UNTAGGED_FOLLOWUP_MODE,+ ACTION_SET_UNTAGGED_QUESTION_MODE,ACTION_SET_CHANNEL_WELCOME_MODE,ACTION_SET_AUTO_MODEL_CHOICE,ACTION_GITHUB_SETTINGS,@@ -441,6 +446,7 @@ def render_home_view(tasks_state: TasksState | None = None,stats_state: StatsState | None = None,untagged_followup_mode: UntaggedFollowupMode | None = None,+ untagged_question_mode: UntaggedFollowupMode | None = None,channel_welcome_mode: ChannelWelcomeMode | None = None,auto_model_choice: bool | None = None,has_project_access: bool = True,@@ -490,6 +496,11 @@ def render_home_view(blocks.append({"type": "divider"})blocks.extend(_untagged_followups_section_blocks(untagged_followup_mode))+ # Section 4b — channel questions. `None` means the flag is off for this workspace.+ if untagged_question_mode is not None:+ blocks.append({"type": "divider"})+ blocks.extend(_untagged_questions_section_blocks(untagged_question_mode))+# Section 5 — channel welcome: a workspace setting, so only admins see it.if is_admin and channel_welcome_mode is not None:blocks.append({"type": "divider"})@@ -951,36 +962,69 @@ UNTAGGED_FOLLOWUP_MODE_LABELS: dict[str, str] = {UntaggedFollowupMode.NEVER: "Leave them alone",}+UNTAGGED_QUESTION_MODE_LABELS: dict[str, str] = {+ UntaggedFollowupMode.AUTO: "Answer automatically",+ UntaggedFollowupMode.ASK: "Ask me first",+ UntaggedFollowupMode.NEVER: "Never answer",+}-def _untagged_followups_section_blocks(mode: UntaggedFollowupMode) -> list[dict]:- """Picker for how untagged replies land in the threads you started.- Ask until picked. The choice covers every reply in those threads,- including the ones you write yourself.+def _untagged_mode_section_blocks(+ mode: UntaggedFollowupMode,+ *,+ action_id: str,+ labels: dict[str, str],+ title: str,+ subtitle: str,+ note: str,+) -> list[dict]:+ """One picker for what PostHog does with a kind of message nobody tagged it in.++ Both pickers ask until the viewer picks, so nothing reaches the agent or the+ channel on a default."""options = [{"text": {"type": "plain_text", "text": label, "emoji": True}, "value": value}- for value, label in UNTAGGED_FOLLOWUP_MODE_LABELS.items()+ for value, label in labels.items()]select: dict[str, Any] = {"type": "static_select",- "action_id": ACTION_SET_UNTAGGED_FOLLOWUP_MODE,+ "action_id": action_id,"options": options,"initial_option": next(o for o in options if o["value"] == mode.value),}return [- _section_title(- "💬 Thread follow-ups",- "What I do with replies in a thread you started, when nobody tags @PostHog.",- ),+ _section_title(title, subtitle),{"type": "actions", "elements": [select]},- {- "type": "context",- "elements": [{"type": "mrkdwn", "text": "Applies to every reply in those threads, yours included."}],- },+ {"type": "context", "elements": [{"type": "mrkdwn", "text": note}]},]+def _untagged_followups_section_blocks(mode: UntaggedFollowupMode) -> list[dict]:+ return _untagged_mode_section_blocks(+ mode,+ action_id=ACTION_SET_UNTAGGED_FOLLOWUP_MODE,+ labels=UNTAGGED_FOLLOWUP_MODE_LABELS,+ title="💬 Thread follow-ups",+ subtitle="What I do with replies in a thread you started, when nobody tags @PostHog.",+ note="Applies to every reply in those threads, yours included.",+ )+++def _untagged_questions_section_blocks(mode: UntaggedFollowupMode) -> list[dict]:+ return _untagged_mode_section_blocks(+ mode,+ action_id=ACTION_SET_UNTAGGED_QUESTION_MODE,+ labels=UNTAGGED_QUESTION_MODE_LABELS,+ title="❓ Questions in channels",+ subtitle=(+ "What I do when you post a question in a channel without tagging @PostHog, "+ "and your PostHog data can answer it."+ ),+ note="Only your own questions. Thread follow-ups keep the setting above.",+ )++CHANNEL_WELCOME_MODE_LABELS: dict[str, str] = {ChannelWelcomeMode.CHANNEL: "Post it in the channel",ChannelWelcomeMode.INVITER: "Show it only to the person who added me",@@ -1696,7 +1740,14 @@ def handle_ai_preferences_block_action(payload: dict, action: dict) -> HttpResporeturn HttpResponse(status=200)if action_id == ACTION_SET_UNTAGGED_FOLLOWUP_MODE:- _apply_untagged_followup_mode_pick(integration, slack_user_id, action)+ if (mode := _picked_untagged_mode(action)) is not None:+ set_untagged_followup_mode(integration.integration_id, slack_user_id, mode)+ republish()+ return HttpResponse(status=200)++ if action_id == ACTION_SET_UNTAGGED_QUESTION_MODE:+ if (mode := _picked_untagged_mode(action)) is not None and is_slack_app_unprompted_answers_enabled(integration):+ set_untagged_question_mode(integration.integration_id, slack_user_id, mode)republish()return HttpResponse(status=200)@@ -2070,17 +2121,11 @@ def _reset_personal_preferences(integration: Integration, slack_user_id: str) ->)-def _apply_untagged_followup_mode_pick(integration: Integration, slack_user_id: str, action: dict) -> None:- """Persist the picked mode. An unrecognised value is ignored rather than stored."""+def _picked_untagged_mode(action: dict) -> UntaggedFollowupMode | None:+ """The mode a picker click carries. An unrecognised value is ignored rather than stored."""picked = (action.get("selected_option") or {}).get("value")- if picked not in UntaggedFollowupMode.values:- return- SlackSettings.objects.update_or_create(- slack_workspace_id=integration.integration_id,- slack_user_id=slack_user_id,- defaults={"untagged_followup_mode": picked},- )+ return UntaggedFollowupMode(picked) if picked in UntaggedFollowupMode.values else Nonedef _clear_project_personal(integration: Integration, slack_user_id: str) -> None:@@ -2092,7 +2137,7 @@ def _clear_project_personal(integration: Integration, slack_user_id: str) -> Non).first()if row is None:return- if not row.untagged_followup_mode and not row.auto_model_choice:+ if not row.untagged_followup_mode and not row.untagged_question_mode and not row.auto_model_choice:row.delete()returnrow.default_integration = None@@ -2153,6 +2198,11 @@ def _build_home_view(tasks_state=tasks_state,stats_state=stats_state,untagged_followup_mode=resolve_untagged_followup_mode(integration, slack_user_id),+ untagged_question_mode=(+ resolve_untagged_question_mode(integration.integration_id, slack_user_id)+ if is_slack_app_unprompted_answers_enabled(integration)+ else None+ ),channel_welcome_mode=resolve_channel_welcome_mode(integration.integration_id) if is_admin else None,auto_model_choice=(resolve_auto_model_choice(integration.integration_id, slack_user_id)diff --git a/products/slack_app/backend/services/slack_conversations.py b/products/slack_app/backend/services/slack_conversations.pyindex 04a10eaf8fb1..fd5bc4cad16d 100644--- a/products/slack_app/backend/services/slack_conversations.py+++ b/products/slack_app/backend/services/slack_conversations.py@@ -100,10 +100,19 @@ def channel_is_ext_shared(slack: SlackIntegration, channel: str) -> bool:unreadable channel is treated as externally shared, which at worst asks the userto approve a channel that did not need it."""+ return channel_ext_shared_state(slack, channel) is not False+++def channel_ext_shared_state(slack: SlackIntegration, channel: str) -> bool | None:+ """Like ``channel_is_ext_shared``, but ``None`` when Slack could not say.++ For a caller that can wait and ask again, a failed lookup is not the same as a shared+ channel.+ """try:info = slack.client.conversations_info(channel=channel)conversation: dict[str, Any] = info.get("channel") or {}return bool(conversation.get("is_ext_shared") or conversation.get("is_pending_ext_shared"))except Exception:logger.warning("slack_app_channel_info_failed", channel=channel)- return True+ return Nonediff --git a/products/slack_app/backend/services/slack_settings.py b/products/slack_app/backend/services/slack_settings.pyindex 7b6cdc4bdf82..d15b37157be9 100644--- a/products/slack_app/backend/services/slack_settings.py+++ b/products/slack_app/backend/services/slack_settings.py@@ -41,21 +41,32 @@ def resolve_untagged_followup_mode(integration: Integration, slack_user_id: strto `ASK`, so the replier chooses whether to send the message. An unknownvalue resolves to `NEVER`."""+ return _resolve_untagged_mode(integration.integration_id, slack_user_id, "untagged_followup_mode")++def resolve_untagged_question_mode(slack_workspace_id: str, slack_user_id: str | None) -> UntaggedFollowupMode:+ """Resolve what PostHog does with this Slack user's top-level channel questions that do not tag it.++ Read from the author's own row, so one person's choice never changes what happens+ to anyone else's messages. An absent row or empty column resolves to `ASK`: PostHog+ offers privately and answers only after the author confirms. An unknown value+ resolves to `NEVER`.+ """+ return _resolve_untagged_mode(slack_workspace_id, slack_user_id, "untagged_question_mode")+++def _resolve_untagged_mode(slack_workspace_id: str, slack_user_id: str | None, column: str) -> UntaggedFollowupMode:if not slack_user_id:return UntaggedFollowupMode.NEVERfrom products.slack_app.backend.models import SlackSettingsrow = (- SlackSettings.objects.filter(- slack_workspace_id=integration.integration_id,- slack_user_id=slack_user_id,- )- .values("untagged_followup_mode")+ SlackSettings.objects.filter(slack_workspace_id=slack_workspace_id, slack_user_id=slack_user_id)+ .values(column).first())- stored = row["untagged_followup_mode"] if row else None+ stored = row[column] if row else Noneif stored is None:return UntaggedFollowupMode.ASKif stored in UntaggedFollowupMode.values:@@ -63,6 +74,24 @@ def resolve_untagged_followup_mode(integration: Integration, slack_user_id: strreturn UntaggedFollowupMode.NEVER+def set_untagged_followup_mode(slack_workspace_id: str, slack_user_id: str, mode: UntaggedFollowupMode) -> None:+ _set_untagged_mode(slack_workspace_id, slack_user_id, "untagged_followup_mode", mode)+++def set_untagged_question_mode(slack_workspace_id: str, slack_user_id: str, mode: UntaggedFollowupMode) -> None:+ _set_untagged_mode(slack_workspace_id, slack_user_id, "untagged_question_mode", mode)+++def _set_untagged_mode(slack_workspace_id: str, slack_user_id: str, column: str, mode: UntaggedFollowupMode) -> None:+ from products.slack_app.backend.models import SlackSettings++ SlackSettings.objects.update_or_create(+ slack_workspace_id=slack_workspace_id,+ slack_user_id=slack_user_id,+ defaults={column: mode.value},+ )++def resolve_channel_welcome_mode(slack_workspace_id: str) -> ChannelWelcomeMode:"""Resolve where the greeting goes when someone adds the app to a channel.@@ -124,7 +153,10 @@ __all__ = ["AIPreferences","resolve_auto_model_choice","resolve_channel_welcome_mode",+ "resolve_untagged_question_mode","set_auto_model_choice",+ "set_untagged_followup_mode",+ "set_untagged_question_mode","set_channel_welcome_mode","resolve_untagged_followup_mode",]diff --git a/products/slack_app/backend/temporal/__init__.py b/products/slack_app/backend/temporal/__init__.pyindex 2fbe876c7417..a1f8072285d9 100644--- a/products/slack_app/backend/temporal/__init__.py+++ b/products/slack_app/backend/temporal/__init__.py@@ -17,6 +17,7 @@ from products.slack_app.backend.temporal.activities import (classify_slack_app_model_router_activity,classify_slack_app_project_route_activity,classify_untagged_followup_activity,+ classify_untagged_question_activity,collect_posthog_code_thread_messages_activity,create_posthog_code_routing_rule_activity,create_posthog_code_task_for_repo_activity,@@ -36,6 +37,7 @@ from products.slack_app.backend.temporal.activities import (prepare_slack_unfurl_charts_activity,process_slack_app_fork_thread_activity,request_untagged_followup_confirmation_activity,+ request_untagged_question_confirmation_activity,resolve_posthog_code_slack_command_user_activity,run_posthog_slack_inbox_onboarding_activity,)@@ -61,6 +63,8 @@ SLACK_APP_ACTIVITIES = [process_slack_app_fork_thread_activity,classify_untagged_followup_activity,request_untagged_followup_confirmation_activity,+ classify_untagged_question_activity,+ request_untagged_question_confirmation_activity,resolve_posthog_code_slack_command_user_activity,collect_posthog_code_thread_messages_activity,cascade_posthog_code_repository_activity,@@ -114,6 +118,7 @@ __all__ = ["classify_posthog_code_task_needs_repo_activity","classify_slack_app_model_override_activity","classify_slack_app_project_route_activity",+ "classify_untagged_question_activity","classify_untagged_followup_activity","collect_posthog_code_thread_messages_activity","create_posthog_code_routing_rule_activity",@@ -130,6 +135,7 @@ __all__ = ["post_posthog_code_no_repos_activity","post_posthog_code_picker_timeout_activity","post_posthog_code_repo_picker_activity",+ "request_untagged_question_confirmation_activity","request_untagged_followup_confirmation_activity","resolve_posthog_code_slack_command_user_activity","classify_slack_app_model_router_activity",diff --git a/products/slack_app/backend/temporal/activities/__init__.py b/products/slack_app/backend/temporal/activities/__init__.pyindex b4d73a0a3378..3d4a5cd6de8a 100644--- a/products/slack_app/backend/temporal/activities/__init__.py+++ b/products/slack_app/backend/temporal/activities/__init__.py@@ -47,6 +47,10 @@ from products.slack_app.backend.temporal.activities.unfurl_charts import (post_slack_unfurl_charts_activity,prepare_slack_unfurl_charts_activity,)+from products.slack_app.backend.temporal.activities.untagged_question import (+ classify_untagged_question_activity,+ request_untagged_question_confirmation_activity,+)from products.slack_app.backend.temporal.activities.user_resolution import (resolve_posthog_code_slack_command_user_activity,)@@ -64,7 +68,9 @@ __all__ = ["classify_slack_app_project_route","classify_slack_app_project_route_activity","classify_task_needs_repo",+ "classify_untagged_question_activity","classify_untagged_followup_activity",+ "request_untagged_question_confirmation_activity","collect_posthog_code_thread_messages_activity","create_posthog_code_routing_rule_activity","create_posthog_code_task_for_repo_activity",diff --git a/products/slack_app/backend/temporal/activities/billing.py b/products/slack_app/backend/temporal/activities/billing.pyindex 07116a24e85c..94f52d6e32f4 100644--- a/products/slack_app/backend/temporal/activities/billing.py+++ b/products/slack_app/backend/temporal/activities/billing.py@@ -32,13 +32,13 @@ def enforce_posthog_code_billing_quota_activity(integration_id=inputs.slack_team_id,)slack = SlackIntegration(integration)- # An untagged reply has asked PostHog for nothing yet, because both the classifier- # and the confirmation prompt run after this gate. The denial goes into the thread- # where everyone in the channel reads it, so an over-quota team would collect one- # under every reply people write in a thread PostHog owns, chitchat included. Stop- # the run without the message and keep the message for a mention or a confirmed- # reply, which are the turns that asked for work.- unrequested = inputs.untagged_followup and not inputs.untagged_followup_confirmed+ # An untagged reply or an untagged channel question has asked PostHog for nothing+ # yet, because both the classifier and the confirmation prompt run after this gate.+ # The denial goes into the thread where everyone in the channel reads it, so an+ # over-quota team would collect one under every reply people write in a thread+ # PostHog owns, chitchat included. Stop the run without the message and keep the+ # message for a mention or a confirmed reply, which are the turns that asked for work.+ unrequested = (inputs.untagged_followup or inputs.untagged_question) and not inputs.untagged_confirmedblocked = block_if_team_over_quota(integration=integration,slack=slack,@@ -50,7 +50,7 @@ def enforce_posthog_code_billing_quota_activity()if blocked and unrequested:logger.info(- "slack_app_untagged_followup_dropped_over_quota",+ "slack_app_unaddressed_message_dropped_over_quota",team_id=integration.team_id,channel=channel,thread_ts=thread_ts,diff --git a/products/slack_app/backend/temporal/activities/task_creation.py b/products/slack_app/backend/temporal/activities/task_creation.pyindex 8796290d3d97..54254d820005 100644--- a/products/slack_app/backend/temporal/activities/task_creation.py+++ b/products/slack_app/backend/temporal/activities/task_creation.py@@ -550,7 +550,7 @@ def derive_mention_workflow_id(inputs: PostHogCodeSlackMentionWorkflowInputs) ->suffix = inputs.slack_event_idelse:suffix = f"{event.get('channel', '')}:{event.get('ts', '')}"- if inputs.untagged_followup_confirmed:+ if inputs.untagged_confirmed:suffix = f"{suffix}:confirmed"return f"{MENTION_WORKFLOW_ID_PREFIX}{inputs.slack_team_id}:{suffix}"diff --git a/products/slack_app/backend/temporal/activities/untagged_question.py b/products/slack_app/backend/temporal/activities/untagged_question.pynew file mode 100644index 000000000000..8cfab392012e--- /dev/null+++ b/products/slack_app/backend/temporal/activities/untagged_question.py@@ -0,0 +1,265 @@+"""Answering a top-level Slack channel message that did not tag the app.++Nobody asked PostHog anything, so the bar is high on purpose: a wrong answer interrupts a+conversation in public. Jev, a System One decision model, answers two yes/no questions about+the message, and both must clear a strict threshold. A missed question costs the author one+@PostHog, which they can still type.+"""++import structlog+from temporalio import activity++from posthog.dataclasses import frozen+from posthog.llm.managed_decision_model import DEFAULT_DECISION_MODEL+from posthog.llm.system_one import NoulAnswer, NoulQuestion, SystemOneNotConfigured+from posthog.llm.system_one_client import build_system_one_client+from posthog.models.integration import Integration, SlackIntegration+from posthog.models.user import User+from posthog.temporal.common.utils import close_db_connections++from products.slack_app.backend.analytics import capture_slack_event+from products.slack_app.backend.models import UntaggedFollowupMode+from products.slack_app.backend.services.slack_conversations import channel_is_ext_shared+from products.slack_app.backend.services.slack_settings import resolve_untagged_question_mode+from products.slack_app.backend.temporal.activities.classifiers import CLASSIFIER_PROPERTY, _thread_trace_id+from products.slack_app.backend.temporal.types import (+ PostHogCodeSlackMentionWorkflowInputs,+ coerce_mention_workflow_inputs,+)++logger = structlog.get_logger(__name__)++UNTAGGED_QUESTION_DECISION_MODEL = DEFAULT_DECISION_MODEL+# Not the `slack_app` product the other classifiers bill under: this call runs on messages+# nobody sent to PostHog, so it must not spend the team's AI credits.+UNTAGGED_QUESTION_AI_PRODUCT = "slack_app_untagged_question"+# On the path to the first reply. A miss only means no answer, so a slow call is not worth waiting for.+UNTAGGED_QUESTION_TIMEOUT_SECONDS = 5.0+# Both judgments must clear this. Tuned for precision over recall, see+# products/slack_app/evals/eval_untagged_question.py.+UNTAGGED_QUESTION_MIN_PROBABILITY = 0.85++ASKS_QUESTION_ID = "asks_for_information"+ANSWERABLE_QUESTION_ID = "answerable_by_posthog"++ASKS_FOR_INFORMATION_INSTRUCTIONS = (+ "`message` is a new top-level post in a team's Slack channel. Nobody tagged an assistant in it. "+ "Is the author asking for a specific piece of information or a specific analysis that they do not have yet?"+)+ASKS_FOR_INFORMATION_TRUE = (+ "A real question or request for facts, numbers, an explanation, or a lookup. "+ "Examples: how many users signed up last week, why did checkout conversion drop yesterday, "+ "which feature flag controls the new onboarding, are there errors on the billing page since the deploy. "+ "A yes or no question about whether a metric went up, went down or changed asks for that metric, "+ "including when the metric is written as shorthand such as WAU or churn. "+ "A request to find where something lives, such as which file sends a tracking call, is a lookup. "+ "A question about why a number or an error count changed asks for an analysis, even when it names the "+ "change it saw. The question can be in any language."+)+ASKS_FOR_INFORMATION_FALSE = (+ "Anything else: an announcement, a status update, a statement or an opinion, a rhetorical question, "+ "a social message, a joke, a greeting, a thank-you, a call to meet or talk, a request for a person to "+ "do something, or a question only one named person can answer."+)++ANSWERABLE_INSTRUCTIONS = (+ "PostHog is a product analytics platform. Its assistant can read the team's PostHog data: events, "+ "insights, trends, funnels, retention, dashboards, session recordings, feature flags, experiments, "+ "surveys, error tracking, logs, LLM traces and data warehouse tables. It can also search the PostHog "+ "documentation and read the team's code repositories. "+ "Could that assistant answer the question in `message` correctly and fully, with only those sources?"+)+ANSWERABLE_TRUE = (+ "The answer is in product usage data, error or log data, feature flag or experiment state, "+ "the team's code, or the PostHog documentation. Product metrics count as usage data when written "+ "as shorthand: DAU, WAU, MAU, stickiness, churn, retention or conversion rate mean the team's own "+ "numbers, which PostHog holds."+)+ANSWERABLE_FALSE = (+ "The answer needs knowledge those sources do not hold: plans, opinions, decisions, people's availability, "+ "private conversations, customer contracts, finance, HR, or systems PostHog cannot read. Also false when "+ "the question is too vague to answer, or when it is not a question."+)+++@frozen+class UntaggedQuestionVerdict:+ asks_for_information: float+ answerable_by_posthog: float++ @property+ def answerable(self) -> bool:+ return min(self.asks_for_information, self.answerable_by_posthog) >= UNTAGGED_QUESTION_MIN_PROBABILITY+++def classify_untagged_question(+ event_text: str,+ *,+ team_id: int,+ distinct_id: str | None,+ trace_id: str | None = None,+) -> UntaggedQuestionVerdict | None:+ """Ask the decision model whether PostHog can answer this message. ``None`` means no verdict.++ Raises when no System One server is configured or the call fails, so the caller can tell+ an outage from a refusal.+ """+ text = event_text.strip()+ if not text:+ return None+ # No TypeSafe fallback: the message is customer text, and TypeSafe is a third party.+ client = build_system_one_client(+ model=UNTAGGED_QUESTION_DECISION_MODEL,+ ai_product=UNTAGGED_QUESTION_AI_PRODUCT,+ team_id=team_id,+ distinct_id=distinct_id,+ trace_id=trace_id,+ properties={CLASSIFIER_PROPERTY: "untagged_question"},+ timeout=UNTAGGED_QUESTION_TIMEOUT_SECONDS,+ )+ result = client.decide(+ state={"message": text},+ questions={+ ASKS_QUESTION_ID: NoulQuestion(+ instructions=ASKS_FOR_INFORMATION_INSTRUCTIONS,+ criteria_true=ASKS_FOR_INFORMATION_TRUE,+ criteria_false=ASKS_FOR_INFORMATION_FALSE,+ ),+ ANSWERABLE_QUESTION_ID: NoulQuestion(+ instructions=ANSWERABLE_INSTRUCTIONS,+ criteria_true=ANSWERABLE_TRUE,+ criteria_false=ANSWERABLE_FALSE,+ ),+ },+ )+ asks = result.answers.get(ASKS_QUESTION_ID)+ answerable = result.answers.get(ANSWERABLE_QUESTION_ID)+ if not isinstance(asks, NoulAnswer) or not isinstance(answerable, NoulAnswer):+ logger.info("slack_app_untagged_question_refused")+ return None+ return UntaggedQuestionVerdict(asks_for_information=asks.probability, answerable_by_posthog=answerable.probability)+++def _load_integration(inputs: PostHogCodeSlackMentionWorkflowInputs) -> Integration:+ return Integration.objects.select_related("team__organization").get(+ id=inputs.integration_id,+ kind="slack",+ integration_id=inputs.slack_team_id,+ )+++@close_db_connections+def classify_untagged_question_activity(inputs: PostHogCodeSlackMentionWorkflowInputs) -> bool:+ """Whether PostHog should answer, or offer to answer, a top-level message nobody tagged it in.++ Any failure returns ``False``: staying quiet is the safe default when nobody asked.+ """+ inputs = coerce_mention_workflow_inputs(inputs)+ integration = _load_integration(inputs)+ user = User.objects.filter(id=inputs.user_id).first()+ if user is None:+ return False++ verdict: UntaggedQuestionVerdict | None = None+ try:+ verdict = classify_untagged_question(+ inputs.event.get("text") or "",+ team_id=integration.team_id,+ distinct_id=user.distinct_id,+ trace_id=_thread_trace_id(inputs.slack_team_id, inputs.event.get("ts")),+ )+ except SystemOneNotConfigured:+ logger.info("slack_app_untagged_question_system_one_not_configured")+ except Exception:+ logger.exception("slack_app_untagged_question_classifier_failed")++ answerable = verdict is not None and verdict.answerable+ logger.info(+ "slack_app_untagged_question_classified",+ slack_team_id=inputs.slack_team_id,+ message_ts=inputs.event.get("ts"),+ classified=verdict is not None,+ answerable=answerable,+ asks_for_information=verdict.asks_for_information if verdict else None,+ answerable_by_posthog=verdict.answerable_by_posthog if verdict else None,+ threshold=UNTAGGED_QUESTION_MIN_PROBABILITY,+ )+ # The message text stays out of this event: its author never addressed PostHog.+ capture_slack_event(+ integration,+ "slack app unprompted question classified",+ slack_user_id=inputs.event.get("user"),+ posthog_user=user,+ classified=verdict is not None,+ answerable=answerable,+ asks_for_information=verdict.asks_for_information if verdict else None,+ answerable_by_posthog=verdict.answerable_by_posthog if verdict else None,+ threshold=UNTAGGED_QUESTION_MIN_PROBABILITY,+ )+ if not answerable:+ return False++ from products.slack_app.backend.api import (+ claim_message_handled, # noqa: PLC0415 — keeps the webhook module off the worker import path+ message_handled_by,+ )++ # An edit of this message that adds a tag starts its own run. Whichever path claims+ # the message first runs, and the other stops. The run id keeps the claim through a+ # retry of this activity.+ claimant = f"untagged_question:{activity.info().workflow_run_id}"+ if (+ not claim_message_handled(inputs.slack_team_id, inputs.event, claimant)+ and message_handled_by(inputs.slack_team_id, inputs.event) != claimant+ ):+ logger.info("slack_app_untagged_question_already_handled", slack_team_id=inputs.slack_team_id)+ return False+ return True+++@close_db_connections+def request_untagged_question_confirmation_activity(inputs: PostHogCodeSlackMentionWorkflowInputs) -> bool:+ """Apply the author's untagged-message mode to a question the classifier passed.++ Returns ``True`` when the run must stop here: the private offer now waits for the+ author, or the author turned untagged pickups off while this run was in flight.+ """+ from products.slack_app.backend.api import (+ _post_untagged_question_prompt, # noqa: PLC0415 — keeps the webhook module off the worker import path+ )++ inputs = coerce_mention_workflow_inputs(inputs)+ slack_user_id = inputs.event.get("user")+ mode = resolve_untagged_question_mode(inputs.slack_team_id, slack_user_id)+ logger.info(+ "slack_app_untagged_question_mode_resolved",+ slack_team_id=inputs.slack_team_id,+ message_ts=inputs.event.get("ts"),+ mode=mode,+ )+ if mode == UntaggedFollowupMode.NEVER:+ logger.info("slack_app_untagged_questions_switched_off_mid_run", slack_team_id=inputs.slack_team_id)+ return True++ integration = _load_integration(inputs)+ if mode == UntaggedFollowupMode.AUTO:+ # Nobody clicks before an automatic answer, so the channel's sharing is read again here.+ # The webhook checked the event envelope, which can be stale by the time a worker runs.+ if channel_is_ext_shared(SlackIntegration(integration), str(inputs.event.get("channel") or "")):+ logger.info("slack_app_untagged_question_channel_now_shared", slack_team_id=inputs.slack_team_id)+ return True+ return False++ prompted = _post_untagged_question_prompt(SlackIntegration(integration), integration, inputs.event)+ if not prompted:+ # Still held back: answering without the offer the author's setting asks for would break it.+ logger.warning("slack_app_untagged_question_prompt_not_delivered", integration_id=integration.id)+ else:+ logger.info(+ "slack_app_untagged_question_prompt_posted",+ integration_id=integration.id,+ message_ts=inputs.event.get("ts"),+ )+ return Truediff --git a/products/slack_app/backend/temporal/slack_app_mention.py b/products/slack_app/backend/temporal/slack_app_mention.pyindex 10a33b88be39..afbde9ee9c78 100644--- a/products/slack_app/backend/temporal/slack_app_mention.py+++ b/products/slack_app/backend/temporal/slack_app_mention.py@@ -122,12 +122,12 @@ class SlackAppMentionWorkflow(PostHogWorkflow):async def _react(self, message: PostHogCodeSlackMentionWorkflowInputs, reaction_activity: Any) -> None:"""Run one of the reaction activities for a mention or DM. Untagged- thread followups get no reactions — they were never addressed to the- bot, and most are dropped by the chitchat classifier.+ thread followups and untagged questions get no reactions,+ because nobody addressed them to the bot and a classifier drops most of them."""channel = message.event.get("channel")message_ts = message.event.get("ts")- if message.untagged_followup or not channel or not message_ts:+ if message.untagged_followup or message.untagged_question or not channel or not message_ts:returntry:await workflow.execute_activity(@@ -157,7 +157,7 @@ class SlackAppMentionWorkflow(PostHogWorkflow):"""channel = message.event.get("channel")thread_ts = message.event.get("thread_ts") or message.event.get("ts")- if message.untagged_followup or not channel or not thread_ts:+ if message.untagged_followup or message.untagged_question or not channel or not thread_ts:returntry:await workflow.execute_activity(diff --git a/products/slack_app/backend/temporal/slack_app_mention_processing.py b/products/slack_app/backend/temporal/slack_app_mention_processing.pyindex 21941eabfbf7..74b239982489 100644--- a/products/slack_app/backend/temporal/slack_app_mention_processing.py+++ b/products/slack_app/backend/temporal/slack_app_mention_processing.py@@ -25,6 +25,7 @@ from products.slack_app.backend.temporal import (classify_slack_app_model_router_activity,classify_slack_app_project_route_activity,classify_untagged_followup_activity,+ classify_untagged_question_activity,collect_posthog_code_thread_messages_activity,create_posthog_code_task_for_repo_activity,discover_posthog_code_repository_via_agent_activity,@@ -35,6 +36,7 @@ from products.slack_app.backend.temporal import (post_posthog_code_repo_picker_activity,posthog_code_thread_has_task_activity,request_untagged_followup_confirmation_activity,+ request_untagged_question_confirmation_activity,)if TYPE_CHECKING:@@ -87,6 +89,9 @@ class SlackAppMentionProcessingWorkflow(PostHogWorkflow):if not channel or not thread_ts or not slack_user_id:return+ # An untagged question stays silent, error replies included, until its author's+ # mode lets the run answer. Old histories carry no such input, so they never skip.+ silent_on_error = inputs.untagged_question and not inputs.untagged_question_confirmedtry:# Gate every workflow entry on the team's AI-credits quota before any# other activity runs. Webhook-level short-circuit catches the common@@ -104,6 +109,18 @@ class SlackAppMentionProcessingWorkflow(PostHogWorkflow):if blocked:return+ # Both gates run before anything visible, because nobody tagged us in this message.+ if silent_on_error:+ answerable = await _execute_posthog_code_activity(classify_untagged_question_activity, inputs)+ if not answerable:+ return+ awaiting_confirmation = await _execute_posthog_code_activity(+ request_untagged_question_confirmation_activity, inputs+ )+ if awaiting_confirmation:+ return+ silent_on_error = False+workflow.deprecate_patch(_PATCH_ID_PARALLEL_PREWORK)prework = await _prework_in_parallel(inputs, channel, thread_ts, slack_user_id)if prework is None:@@ -236,6 +253,8 @@ class SlackAppMentionProcessingWorkflow(PostHogWorkflow):"error_type": type(exc).__name__,},)+ if silent_on_error:+ returnawait _execute_posthog_code_activity(post_posthog_code_internal_error_activity,inputs,diff --git a/products/slack_app/backend/temporal/tests/activities/test_billing_quota_activity.py b/products/slack_app/backend/temporal/tests/activities/test_billing_quota_activity.pyindex a91ce5ee1365..fa1191a22426 100644--- a/products/slack_app/backend/temporal/tests/activities/test_billing_quota_activity.py+++ b/products/slack_app/backend/temporal/tests/activities/test_billing_quota_activity.py@@ -22,7 +22,14 @@ class TestEnforcePostHogCodeBillingQuotaActivity(TestCase):self.user = User.objects.create(email="[email protected]")self.integration = Integration.objects.create(team=self.team, kind="slack", integration_id="T_SLACK", config={})- def _call(self, *, untagged_followup: bool, confirmed: bool) -> bool:+ def _call(+ self,+ *,+ untagged_followup: bool,+ confirmed: bool,+ untagged_question: bool = False,+ untagged_question_confirmed: bool = False,+ ) -> bool:inputs = PostHogCodeSlackMentionWorkflowInputs(event={},integration_id=self.integration.id,@@ -30,6 +37,8 @@ class TestEnforcePostHogCodeBillingQuotaActivity(TestCase):user_id=self.user.id,untagged_followup=untagged_followup,untagged_followup_confirmed=confirmed,+ untagged_question=untagged_question,+ untagged_question_confirmed=untagged_question_confirmed,)return enforce_posthog_code_billing_quota_activity(inputs, "C001", "1000.0000", "U_BOB")@@ -38,19 +47,28 @@ class TestEnforcePostHogCodeBillingQuotaActivity(TestCase):# The gate runs before the classifier and before the prompt, so an untagged# reply has asked for nothing yet. A denial under it would land in the thread# for every reply people write, chitchat included.- ("unconfirmed_reply", True, False, False),- ("confirmed_reply", True, True, True),- ("mention", False, False, True),+ ("unconfirmed_reply", True, False, False, False, False),+ ("confirmed_reply", True, True, False, False, True),+ ("mention", False, False, False, False, True),+ # A channel question nobody tagged us in is the same: a denial under it would+ # announce in public that PostHog read a message nobody sent it.+ ("unconfirmed_untagged_question", False, False, True, False, False),+ ("confirmed_untagged_question", False, False, True, True, True),])def test_denial_reaches_the_thread_only_for_a_turn_that_asked_for_work(- self, _name, untagged_followup, confirmed, expect_denial+ self, _name, untagged_followup, confirmed, untagged_question, untagged_question_confirmed, expect_denial):with (patch("ee.billing.quota_limiting.is_team_limited", return_value=True),patch("products.slack_app.backend.api.post_quota_exhausted_denial") as mock_denial,):- blocked = self._call(untagged_followup=untagged_followup, confirmed=confirmed)+ blocked = self._call(+ untagged_followup=untagged_followup,+ confirmed=confirmed,+ untagged_question=untagged_question,+ untagged_question_confirmed=untagged_question_confirmed,+ )assert blocked is Trueassert mock_denial.called is expect_denialdiff --git a/products/slack_app/backend/temporal/tests/activities/test_untagged_question.py b/products/slack_app/backend/temporal/tests/activities/test_untagged_question.pynew file mode 100644index 000000000000..2413789012b1--- /dev/null+++ b/products/slack_app/backend/temporal/tests/activities/test_untagged_question.py@@ -0,0 +1,123 @@+from typing import Any++from unittest.mock import MagicMock, patch++from django.core.cache import cache+from django.test import SimpleTestCase, TestCase++from parameterized import parameterized++from posthog.llm.system_one import NoulAnswer, RefusalAnswer, SystemOneResult+from posthog.models.integration import Integration+from posthog.models.organization import Organization+from posthog.models.team.team import Team+from posthog.models.user import User++from products.slack_app.backend.api import claim_message_handled+from products.slack_app.backend.models import SlackSettings, UntaggedFollowupMode+from products.slack_app.backend.temporal.activities.untagged_question import (+ ANSWERABLE_QUESTION_ID,+ ASKS_QUESTION_ID,+ UntaggedQuestionVerdict,+ classify_untagged_question,+ classify_untagged_question_activity,+ request_untagged_question_confirmation_activity,+)+from products.slack_app.backend.temporal.types import PostHogCodeSlackMentionWorkflowInputs++MODULE = "products.slack_app.backend.temporal.activities.untagged_question"+++def _result(asks: Any, answerable: Any) -> SystemOneResult:+ return SystemOneResult(+ model="jev", answers={ASKS_QUESTION_ID: asks, ANSWERABLE_QUESTION_ID: answerable}, input_tokens=None+ )+++class TestClassifyUntaggedQuestion(SimpleTestCase):+ @parameterized.expand(+ [+ ("both confident", NoulAnswer(probability=0.97), NoulAnswer(probability=0.91), True),+ ("a question PostHog cannot answer", NoulAnswer(probability=0.97), NoulAnswer(probability=0.6), False),+ ("answerable but not a question", NoulAnswer(probability=0.5), NoulAnswer(probability=0.95), False),+ ("a refusal", NoulAnswer(probability=0.99), RefusalAnswer(), None),+ ]+ )+ def test_both_judgments_must_clear_the_threshold(self, _name, asks, answerable, expected):+ client = MagicMock()+ client.decide.return_value = _result(asks, answerable)+ with patch(f"{MODULE}.build_system_one_client", return_value=client):+ verdict = classify_untagged_question(+ "How many people signed up last week?", team_id=1, distinct_id="user-1"+ )++ assert (verdict.answerable if verdict else None) == expected+ # The message travels as state, never inside the instructions.+ state = client.decide.call_args.kwargs["state"]+ assert state == {"message": "How many people signed up last week?"}+++class TestRequestUntaggedQuestionConfirmation(TestCase):+ def setUp(self):+ cache.clear()+ organization = Organization.objects.create(name="Org")+ team = Team.objects.create(organization=organization, name="Team")+ self.user = User.objects.create(email="[email protected]", distinct_id="user-1")+ self.integration = Integration.objects.create(+ team=team, kind="slack", integration_id="T_WS", sensitive_config={"access_token": "xoxb"}+ )+ self.inputs = PostHogCodeSlackMentionWorkflowInputs(+ event={"type": "message", "channel": "C1", "user": "U_ALICE", "ts": "1.0", "text": "how many signups?"},+ integration_id=self.integration.id,+ slack_team_id="T_WS",+ user_id=self.user.id,+ untagged_question=True,+ )++ @parameterized.expand(+ [+ ("auto answers", UntaggedFollowupMode.AUTO, False, False),+ ("never stays quiet", UntaggedFollowupMode.NEVER, True, False),+ ("ask offers privately", UntaggedFollowupMode.ASK, True, True),+ ("unset offers privately", None, True, True),+ ]+ )+ def test_the_authors_mode_decides(self, _name, user_mode, expect_stop, expect_prompt):+ if user_mode is not None:+ SlackSettings.objects.create(+ slack_workspace_id="T_WS", slack_user_id="U_ALICE", untagged_question_mode=user_mode+ )+ with (+ patch("products.slack_app.backend.api._post_untagged_question_prompt", return_value=True) as mock_prompt,+ patch(f"{MODULE}.channel_is_ext_shared", return_value=False),+ ):+ stop = request_untagged_question_confirmation_activity(self.inputs)++ assert stop is expect_stop+ assert mock_prompt.called is expect_prompt++ def test_auto_stays_quiet_in_a_channel_shared_since_the_webhook(self):+ SlackSettings.objects.create(+ slack_workspace_id="T_WS", slack_user_id="U_ALICE", untagged_question_mode=UntaggedFollowupMode.AUTO+ )+ with patch(f"{MODULE}.channel_is_ext_shared", return_value=True):+ assert request_untagged_question_confirmation_activity(self.inputs) is True++ @parameterized.expand(+ [+ ("first to claim", None, True),+ ("an edited mention claimed it first", "edited_mention", False),+ # A retry after a crash between the claim and its recorded result keeps the message.+ ("its own earlier attempt claimed it", "untagged_question:run-1", True),+ ]+ )+ def test_an_answerable_question_runs_only_when_it_claims_the_message(self, _name, claimed_by, expected):+ if claimed_by is not None:+ claim_message_handled("T_WS", self.inputs.event, claimed_by)+ verdict = UntaggedQuestionVerdict(asks_for_information=0.99, answerable_by_posthog=0.99)+ info = MagicMock(workflow_run_id="run-1")+ with (+ patch(f"{MODULE}.classify_untagged_question", return_value=verdict),+ patch(f"{MODULE}.activity.info", return_value=info),+ ):+ assert classify_untagged_question_activity(self.inputs) is expecteddiff --git a/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py b/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.pyindex 855f620619f9..0ba6bd1e0b19 100644--- a/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py+++ b/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py@@ -8,6 +8,7 @@ import pytestfrom temporalio import activityfrom temporalio.common import WorkflowIDConflictPolicy, WorkflowIDReusePolicy+from temporalio.exceptions import ApplicationErrorfrom temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Worker@@ -34,15 +35,19 @@ def _message(*,event_id: str | None = None,untagged: bool = False,+ untagged_question: bool = False,text: str = "fix the bug",) -> PostHogCodeSlackMentionWorkflowInputs:+ # A top-level post is its own thread root.+ thread_ts = ts if untagged_question else "100.0"return PostHogCodeSlackMentionWorkflowInputs(- event={"channel": "C1", "ts": ts, "thread_ts": "100.0", "user": "U1", "text": text},+ event={"channel": "C1", "ts": ts, "thread_ts": thread_ts, "user": "U1", "text": text},integration_id=1,slack_team_id="T1",slack_event_id=event_id,user_id=42,untagged_followup=untagged,+ untagged_question=untagged_question,)@@ -100,6 +105,9 @@ class _Recorder:self.awaiting_confirmation = False# True makes discovery report that the team is out of AI credits.self.discover_usage_limited = False+ self.untagged_question_verdict: bool | Literal["raise"] = True+ # True holds an untagged question back on a private offer.+ self.untagged_question_awaiting_confirmation = Falsedef _fake_activities(rec: _Recorder) -> list:@@ -128,6 +136,16 @@ def _fake_activities(rec: _Recorder) -> list:) -> bool:return rec.awaiting_confirmation+ @activity.defn(name="classify_untagged_question_activity")+ async def classify_untagged_question(inputs: PostHogCodeSlackMentionWorkflowInputs) -> bool:+ if rec.untagged_question_verdict == "raise":+ raise ApplicationError("classifier down", non_retryable=True)+ return rec.untagged_question_verdict++ @activity.defn(name="request_untagged_question_confirmation_activity")+ async def request_untagged_question_confirmation(inputs: PostHogCodeSlackMentionWorkflowInputs) -> bool:+ return rec.untagged_question_awaiting_confirmation+@activity.defn(name="forward_posthog_code_followup_activity")async def forward(inputs: PostHogCodeSlackMentionWorkflowInputs,@@ -288,6 +306,8 @@ def _fake_activities(rec: _Recorder) -> list:classify_followup,request_confirmation,has_task,+ classify_untagged_question,+ request_untagged_question_confirmation,forward,collect,cascade,@@ -676,3 +696,33 @@ async def test_deleted_trigger_message_creates_no_task_and_says_nothing():assert rec.created == []assert rec.internal_errors == []+++ "verdict,awaiting_confirmation,expect_created",+ [+ # Nobody asked, so a failure says nothing in the channel either.+ ("raise", False, False),+ (False, False, False),+ (True, True, False),+ (True, False, True),+ ],+)+async def test_untagged_question_stays_silent_until_its_mode_lets_it_answer(+ verdict, awaiting_confirmation, expect_created+):+ rec = _Recorder()+ rec.untagged_question_verdict = verdict+ rec.untagged_question_awaiting_confirmation = awaiting_confirmation++ async with _Harness(rec) as h:+ handle = await _signal_with_start(+ h.env, h.task_queue, f"wf-{uuid.uuid4()}", _message("1.1", untagged_question=True)+ )+ await asyncio.wait_for(handle.result(), timeout=30)++ assert rec.internal_errors == []+ assert rec.queued_marked == []+ assert rec.processing_marked == []+ assert bool(rec.created) is expect_createddiff --git a/products/slack_app/backend/temporal/types.py b/products/slack_app/backend/temporal/types.pyindex 57b47ab68f89..f1a6c6f1c48f 100644--- a/products/slack_app/backend/temporal/types.py+++ b/products/slack_app/backend/temporal/types.py@@ -120,6 +120,13 @@ class PostHogCodeSlackMentionWorkflowInputs:# ephemeral prompt. The classifier ran before the prompt was posted and the# answer is in, so the run skips both on the way back through.untagged_followup_confirmed: bool = False+ # True when the workflow was started for a top-level channel message that did not+ # tag the app. Nothing visible happens until the classifier says PostHog can answer+ # it and the author's untagged-message mode allows it.+ untagged_question: bool = False+ # True when the author already accepted the private offer to answer, so the run+ # skips the classifier and the offer on the way back through.+ untagged_question_confirmed: bool = False# Slack sets this on the event envelope for Slack Connect channels. It is# threaded through to task run state so customer-facing Slack replies remain# approval-gated even when a user's internal-write tier is full-auto.@@ -139,6 +146,10 @@ class PostHogCodeSlackMentionWorkflowInputs:# than the messages.fork_source_task_id: str | None = None+ @property+ def untagged_confirmed(self) -> bool:+ return self.untagged_followup_confirmed or self.untagged_question_confirmed+def coerce_mention_workflow_inputs(inputs: object) -> PostHogCodeSlackMentionWorkflowInputs:"""Normalise an activity's ``inputs`` back into the dataclass.diff --git a/products/slack_app/backend/tests/services/test_slack_app_home.py b/products/slack_app/backend/tests/services/test_slack_app_home.pyindex e04736b858bc..fa6784d0de78 100644--- a/products/slack_app/backend/tests/services/test_slack_app_home.py+++ b/products/slack_app/backend/tests/services/test_slack_app_home.py@@ -473,6 +473,7 @@ class TestRenderHomeView:),stats_state=StatsState(tasks_started=4, tasks_with_pr=2, tasks_merged=1, active_people=2),untagged_followup_mode=UntaggedFollowupMode.AUTO,+ untagged_question_mode=UntaggedFollowupMode.ASK,channel_welcome_mode=ChannelWelcomeMode.CHANNEL,auto_model_choice=auto_model_choice,)@@ -1377,8 +1378,12 @@ class TestResetPersonal:class TestResetProjectPersonal:@pytest.mark.parametrize("other_settings",- [{"untagged_followup_mode": UntaggedFollowupMode.AUTO}, {"auto_model_choice": True}],- ids=["followup_mode", "auto_model_choice"],+ [+ {"untagged_followup_mode": UntaggedFollowupMode.AUTO},+ {"untagged_question_mode": UntaggedFollowupMode.NEVER},+ {"auto_model_choice": True},+ ],+ ids=["followup_mode", "question_mode", "auto_model_choice"],)def test_clears_routing_only_when_other_settings_present(self, slack_integration, mock_slack_client, flag_on, admin_user, other_settingsdiff --git a/products/slack_app/backend/tests/services/test_slack_settings.py b/products/slack_app/backend/tests/services/test_slack_settings.pyindex ad5e7cfb9f27..db4f2940443f 100644--- a/products/slack_app/backend/tests/services/test_slack_settings.py+++ b/products/slack_app/backend/tests/services/test_slack_settings.py@@ -5,7 +5,10 @@ from posthog.models.organization import Organizationfrom posthog.models.team.team import Teamfrom products.slack_app.backend.models import SlackSettings, UntaggedFollowupMode-from products.slack_app.backend.services.slack_settings import resolve_untagged_followup_mode+from products.slack_app.backend.services.slack_settings import (+ resolve_untagged_followup_mode,+ resolve_untagged_question_mode,+)@pytest.fixture@@ -52,3 +55,25 @@ class TestResolveUntaggedFollowupMode:untagged_followup_mode=UntaggedFollowupMode.AUTO,)assert resolve_untagged_followup_mode(integration, "U001") == UntaggedFollowupMode.ASK+++class TestResolveUntaggedQuestionMode:+ def test_unset_resolves_ask(self, slack_setup):+ assert resolve_untagged_question_mode("T_WS", "U001") == UntaggedFollowupMode.ASK++ @pytest.mark.parametrize(+ "followup_mode,question_mode,expected",+ [+ # Each column answers only its own question.+ (UntaggedFollowupMode.AUTO, None, UntaggedFollowupMode.ASK),+ (UntaggedFollowupMode.NEVER, UntaggedFollowupMode.AUTO, UntaggedFollowupMode.AUTO),+ ],+ )+ def test_the_followup_choice_does_not_leak(self, slack_setup, followup_mode, question_mode, expected):+ SlackSettings.objects.create(+ slack_workspace_id="T_WS",+ slack_user_id="U001",+ untagged_followup_mode=followup_mode,+ untagged_question_mode=question_mode,+ )+ assert resolve_untagged_question_mode("T_WS", "U001") == expecteddiff --git a/products/slack_app/backend/tests/test_thread_message_routing.py b/products/slack_app/backend/tests/test_thread_message_routing.pyindex e62bd29ec3f3..768943194ad2 100644--- a/products/slack_app/backend/tests/test_thread_message_routing.py+++ b/products/slack_app/backend/tests/test_thread_message_routing.py@@ -121,18 +121,31 @@ class TestRouteThreadMessage(TestCase):defaults.update(overrides)return defaults- def _route(self, event: dict, slack_team_id: str = "T_SLACK") -> str:+ def _make_question_event(self, **overrides) -> dict:+ defaults = {+ "channel": "C002",+ "channel_type": "channel",+ "ts": "2000.0000",+ "thread_ts": None,+ "text": "How many people signed up last week?",+ }+ return self._make_event(**{**defaults, **overrides})++ def _route(self, event: dict, slack_team_id: str = "T_SLACK", *, is_ext_shared_channel: bool = False) -> str:from products.slack_app.backend.api import route_posthog_code_event_to_relevant_regionrequest = self.factory.post("/slack/event-callback/", HTTP_HOST="us.posthog.com")- return route_posthog_code_event_to_relevant_region(request, event, slack_team_id)+ return route_posthog_code_event_to_relevant_region(+ request, event, slack_team_id, is_ext_shared_channel=is_ext_shared_channel+ )# --- Cheap pre-DB gates ------------------------------------------------def test_top_level_message_dropped_before_db(self):"""A message with no ``thread_ts`` (or where ``thread_ts == ts``) is a- top-level post in the channel, not a thread reply. Drop before- touching the DB — channel chatter dominates wire volume."""+ top-level post in the channel, not a thread reply. One that fails the cheap+ question gates is dropped before touching the DB, because channel chatter+ dominates wire volume."""from products.slack_app.backend.api import ROUTE_HANDLED_LOCALLYevent = self._make_event(thread_ts="1001.0000") # same as ts@@ -145,6 +158,66 @@ class TestRouteThreadMessage(TestCase):mock_filter.assert_not_called()mock_start.assert_not_called()+ @parameterized.expand(+ [+ ("an untagged question", {}, True, None, False, True),+ ("a private channel", {"channel_type": "group"}, True, None, False, True),+ ("a statement", {"text": "Shipped the new export filter today"}, True, None, False, False),+ ("a question to a person", {"text": "<@U_ALICE> how many signups last week?"}, True, None, False, False),+ ("a group DM", {"channel_type": "mpim"}, True, None, False, False),+ ("the flag is off", {}, False, None, False, False),+ ("the author picks up only tagged messages", {}, True, UntaggedFollowupMode.NEVER, False, False),+ ("an author without a PostHog account", {"user": "U_UNKNOWN"}, True, None, False, False),+ ("an externally shared channel", {}, True, None, True, False),+ ]+ )+ def test_top_level_question_starts_the_untagged_question_workflow(+ self, _name, overrides, flag_on, author_mode, ext_shared, expect_dispatch+ ):+ if author_mode is not None:+ SlackSettings.objects.create(+ slack_workspace_id="T_SLACK", slack_user_id="U_BOB", untagged_question_mode=author_mode+ )+ event = self._make_question_event(**overrides)+ with (+ patch("products.slack_app.backend.api.is_slack_app_unprompted_answers_enabled", return_value=flag_on),+ patch("products.slack_app.backend.api._start_mention_workflow") as mock_start,+ patch("products.slack_app.backend.api._post_user_resolution_failure_reply") as mock_failure_reply,+ ):+ self._route(event, is_ext_shared_channel=ext_shared)++ assert mock_start.called is expect_dispatch+ # Nobody asked, so a drop says nothing, an unknown author included.+ mock_failure_reply.assert_not_called()+ if expect_dispatch:+ kwargs = mock_start.call_args.kwargs+ assert kwargs["untagged_question"] is True+ assert kwargs["posthog_user"].id == self.bob.id+ assert mock_start.call_args.args[1].id == self.integration.id++ @parameterized.expand(+ [+ ("the other region holds the workspace", True, True),+ ("nobody confirms it", None, False),+ ]+ )+ def test_top_level_question_for_a_workspace_held_elsewhere_crosses_only_when_claimed(+ self, _name, claimed, expect_proxy+ ):+ event = self._make_question_event()+ with (+ patch("products.slack_app.backend.api.cross_region_routing_enabled", return_value=True),+ patch("products.slack_app.backend.api.emit_slack_message_event", return_value=False),+ patch("products.slack_app.backend.api._mirror_message_event_to_other_region"),+ patch("products.slack_app.backend.api.does_other_region_claim_workspace", return_value=claimed),+ patch("products.slack_app.backend.api._proxy_event_and_return_route", return_value="proxied") as proxy,+ patch("products.slack_app.backend.api._start_mention_workflow") as mock_start,+ ):+ self._route(event, slack_team_id="T_ELSEWHERE")++ assert proxy.called is expect_proxy+ mock_start.assert_not_called()+def test_no_user_dropped(self):from products.slack_app.backend.api import ROUTE_HANDLED_LOCALLYdiff --git a/products/slack_app/backend/tests/test_untagged_followup_interactivity.py b/products/slack_app/backend/tests/test_untagged_followup_interactivity.pyindex b76cf41b16a5..a98c2e5b7cfc 100644--- a/products/slack_app/backend/tests/test_untagged_followup_interactivity.py+++ b/products/slack_app/backend/tests/test_untagged_followup_interactivity.py@@ -20,6 +20,10 @@ from products.slack_app.backend.api import (UNTAGGED_FOLLOWUP_ACTION_RUN,UNTAGGED_FOLLOWUP_BLOCK_ID_PREFIX,UNTAGGED_FOLLOWUP_CONTEXT_KIND,+ UNTAGGED_QUESTION_ACTION_RUN,+ UNTAGGED_QUESTION_ACTION_TURN_OFF,+ UNTAGGED_QUESTION_BLOCK_ID_PREFIX,+ UNTAGGED_QUESTION_CONTEXT_KIND,_picker_context_cache_key,)from products.slack_app.backend.models import SlackSettings, SlackThreadTaskMapping, UntaggedFollowupMode@@ -252,3 +256,159 @@ class TestUntaggedFollowupInteractivity(TestCase):assert response.status_code == 200mock_start.assert_not_called()+++@override_settings(DEBUG=True)+@patch("products.slack_app.backend.api.requests.post")+@patch("products.slack_app.backend.api.SlackIntegration")+class TestUntaggedQuestionInteractivity(TestCase):+ signing_secret = "posthog-code-test-secret"+ slack_team_id = "T12345"+ response_url = "https://hooks.slack.example/response/def"+ context_token = "untagged-question-token-123"++ def setUp(self):+ cache.clear()+ self.client = APIClient()+ self.organization = Organization.objects.create(name="Test Org")+ self.team = Team.objects.create(organization=self.organization, name="Test Team")+ self.member_user = User.objects.create(email="[email protected]", distinct_id="user-member")+ OrganizationMembership.objects.create(user=self.member_user, organization=self.organization)+ self.integration = Integration.objects.create(+ team=self.team,+ kind="slack",+ integration_id=self.slack_team_id,+ sensitive_config={"access_token": "xoxb-test"},+ )+ self.event = {+ "type": "message",+ "channel": "C002",+ "channel_type": "channel",+ "user": "U_BOB",+ "ts": "2000.0000",+ "text": "How many people signed up last week?",+ }+ cache.set(+ _picker_context_cache_key(self.context_token),+ {+ "kind": UNTAGGED_QUESTION_CONTEXT_KIND,+ "integration_id": self.integration.id,+ "slack_user_id": "U_BOB",+ "event": self.event,+ },+ timeout=900,+ )++ def _click(+ self,+ action_id: str,+ slack_user_id: str = "U_BOB",+ *,+ flag_on: bool = True,+ start_fails: bool = False,+ ext_shared: bool | None = False,+ ) -> Any:+ payload = {+ "type": "block_actions",+ "team": {"id": self.slack_team_id},+ "user": {"id": slack_user_id},+ "response_url": self.response_url,+ "actions": [+ {+ "action_id": action_id,+ "block_id": f"{UNTAGGED_QUESTION_BLOCK_ID_PREFIX}_actions:{self.context_token}",+ "value": self.context_token,+ }+ ],+ }+ body_str = f"payload={json.dumps(payload)}"+ signed = sign_slack_request(body_str.encode(), self.signing_secret)+ with (+ patch(+ "products.slack_app.backend.api.get_slack_user_info",+ return_value={"user": {"profile": {"email": self.member_user.email}}},+ ),+ patch(+ "products.slack_app.backend.api._start_mention_workflow",+ side_effect=RuntimeError("temporal down") if start_fails else None,+ ) as mock_start,+ patch("products.slack_app.backend.api.is_slack_app_unprompted_answers_enabled", return_value=flag_on),+ patch("products.slack_app.backend.api.channel_ext_shared_state", return_value=ext_shared),+ ):+ self.client.post(+ "/slack/interactivity-callback/",+ data=body_str,+ content_type="application/x-www-form-urlencoded",+ headers={"x-slack-signature": signed.signature, "x-slack-request-timestamp": signed.timestamp},+ )+ return mock_start++ def _stored_mode(self) -> str | None:+ row = SlackSettings.objects.filter(slack_workspace_id=self.slack_team_id, slack_user_id="U_BOB").first()+ return row.untagged_question_mode if row else None++ def test_confirmation_in_a_channel_shared_since_the_offer_answers_nothing(self, mock_slack_cls, mock_post):+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}+ mock_slack_cls.return_value.missing_scopes.return_value = set()++ assert not self._click(UNTAGGED_QUESTION_ACTION_RUN, ext_shared=True).called++ def test_a_failed_sharing_lookup_keeps_the_offer(self, mock_slack_cls, mock_post):+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}+ mock_slack_cls.return_value.missing_scopes.return_value = set()++ assert not self._click(UNTAGGED_QUESTION_ACTION_RUN, ext_shared=None).called+ # The offer and its cached context survive, so the author can click again.+ assert cache.get(_picker_context_cache_key(self.context_token)) is not None+ assert not any("delete_original" in str(c) for c in mock_post.call_args_list)++ def test_a_failed_dispatch_leaves_the_offer_clickable(self, mock_slack_cls, mock_post):+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}+ mock_slack_cls.return_value.missing_scopes.return_value = set()++ with self.assertRaises(RuntimeError):+ self._click(UNTAGGED_QUESTION_ACTION_RUN, start_fails=True)+ assert cache.get(_picker_context_cache_key(self.context_token)) is not None++ self._click(UNTAGGED_QUESTION_ACTION_RUN).assert_called_once()+ assert cache.get(_picker_context_cache_key(self.context_token)) is None++ def test_confirmation_answers_the_original_message(self, mock_slack_cls, mock_post):+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}+ mock_slack_cls.return_value.missing_scopes.return_value = set()++ mock_start = self._click(UNTAGGED_QUESTION_ACTION_RUN)++ mock_start.assert_called_once()+ assert mock_start.call_args.args[0] == self.event+ assert mock_start.call_args.kwargs["untagged_question"] is True+ # Without this the re-dispatch would classify again and offer again.+ assert mock_start.call_args.kwargs["untagged_question_confirmed"] is True+ assert cache.get(_picker_context_cache_key(self.context_token)) is None++ def test_click_from_anyone_but_the_author_answers_nothing(self, mock_slack_cls, mock_post):+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}++ assert not self._click(UNTAGGED_QUESTION_ACTION_RUN, slack_user_id="U_EVE").called++ def test_stop_offering_turns_answers_off(self, mock_slack_cls, mock_post):+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}++ assert not self._click(UNTAGGED_QUESTION_ACTION_TURN_OFF).called+ assert self._stored_mode() == UntaggedFollowupMode.NEVER+ assert mock_post.call_args.kwargs["json"]["replace_original"] is True++ def test_confirmation_after_the_author_turned_answers_off_answers_nothing(self, mock_slack_cls, mock_post):+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}+ SlackSettings.objects.create(+ slack_workspace_id=self.slack_team_id,+ slack_user_id="U_BOB",+ untagged_question_mode=UntaggedFollowupMode.NEVER,+ )++ assert not self._click(UNTAGGED_QUESTION_ACTION_RUN).called++ def test_confirmation_after_the_flag_was_turned_off_answers_nothing(self, mock_slack_cls, mock_post):+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}++ assert not self._click(UNTAGGED_QUESTION_ACTION_RUN, flag_on=False).calleddiff --git a/products/slack_app/evals/eval_untagged_question.py b/products/slack_app/evals/eval_untagged_question.pynew file mode 100644index 000000000000..66f077c862da--- /dev/null+++ b/products/slack_app/evals/eval_untagged_question.py@@ -0,0 +1,263 @@+"""Which top-level channel messages, posted without tagging the app, can PostHog answer?++Every top-level message in a channel the app is in that looks like a question reaches+`classify_untagged_question`, which asks Jev, a System One decision model, two yes/no+questions. Both must clear `UNTAGGED_QUESTION_MIN_PROBABILITY`.++Nobody asked PostHog anything, so `no_unasked_answer` is the number to watch: a wrong answer+interrupts a conversation in public. A missed question costs the author one @PostHog.++To run:+ hogli evals eval_untagged_question+ hogli evals eval_untagged_question --eval signups_last_week+"""++from __future__ import annotations++import asyncio++from products.posthog_ai.eval_harness.config import BaseEvalCase+from products.posthog_ai.eval_harness.harness.context import EvalContext+from products.posthog_ai.eval_harness.harness.requirements import SuiteKind+from products.posthog_ai.eval_harness.one_shot import OneShotPublicEval+from products.slack_app.backend.temporal.activities import untagged_question+from products.slack_app.evals.scorers import UNTAGGED_QUESTION_KEY, NoUnaskedAnswer, UntaggedQuestionMatch++SUITE_KIND = SuiteKind.ONE_SHOT+++def _answerable(answerable: bool) -> dict:+ return {UNTAGGED_QUESTION_KEY: {"answerable": answerable}}+++ANSWERABLE_CASES = [+ BaseEvalCase(name="signups_last_week", prompt="how many people signed up last week?", expected=_answerable(True)),+ BaseEvalCase(+ name="funnel_drop",+ prompt="Why did conversion from signup to first upload drop so much yesterday?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="flag_rollout_state",+ prompt="is the new onboarding flag rolled out to everyone yet or still 50%?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="errors_since_deploy",+ prompt="Are we seeing more errors on the file upload page since this morning's deploy?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="feature_adoption",+ prompt="what share of weekly active users actually used shared folders this month?",+ expected=_answerable(True),+ ),+ # A metric written as its acronym is the same data question as the metric spelled out.+ BaseEvalCase(+ name="metric_acronym_with_greeting",+ prompt="Hey posthog, what's the DAU for last week?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="metric_acronym_trend",+ prompt="is MAU up or down vs the month before?",+ expected=_answerable(True),+ ),+ # One case per kind of data the assistant can read, beyond the trends and errors above.+ BaseEvalCase(+ name="cohort_retention",+ prompt="what does week 4 retention look like for people who signed up in March?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="session_recordings",+ prompt="can we find recordings of users who rage-clicked the export button?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="experiment_result",+ prompt="did the pricing page experiment reach significance yet?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="survey_answers",+ prompt="what did people say in the NPS survey this month?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="llm_cost",+ prompt="how much did our LLM calls cost yesterday?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="product_how_to",+ prompt="how do I filter out internal users from my insights in PostHog?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="event_in_the_code",+ prompt="where in our code do we fire the checkout_started event?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="error_spike_cause",+ prompt="why did the TypeError on the dashboard page spike overnight?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="churn_after_change",+ prompt="has churn gone up since we shortened the free trial?",+ expected=_answerable(True),+ ),+ BaseEvalCase(+ name="question_in_czech",+ prompt="kolik lidí se včera zaregistrovalo?",+ expected=_answerable(True),+ ),+]++# Messages that look like questions, so they pass the webhook's cheap gate, and that PostHog+# must leave alone.+LEAVE_ALONE_CASES = [+ BaseEvalCase(name="lunch_plan", prompt="lunch at 12:30 today?", expected=_answerable(False)),+ BaseEvalCase(name="who_is_on_call", prompt="who's on call this weekend?", expected=_answerable(False)),+ BaseEvalCase(+ name="meeting_time", prompt="should we move standup to 10am from next week?", expected=_answerable(False)+ ),+ BaseEvalCase(+ name="colleague_availability",+ prompt="does anyone know when Priya is back from leave?",+ expected=_answerable(False),+ ),+ BaseEvalCase(name="opinion_poll", prompt="what do you all think of the new logo?", expected=_answerable(False)),+ BaseEvalCase(+ name="request_for_a_person",+ prompt="can someone review my PR when you have a minute?",+ expected=_answerable(False),+ ),+ BaseEvalCase(name="small_talk", prompt="how's everyone doing after the offsite?", expected=_answerable(False)),+ BaseEvalCase(+ name="contract_question",+ prompt="when does our contract with the payments vendor renew?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="rhetorical",+ prompt="Why is it always the Friday deploys that break? Anyway, fixed now.",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="announcement_with_question",+ prompt="Shipped the new pricing page today! Any feedback, drop it in the thread?",+ expected=_answerable(False),+ ),+ # Acronyms that are not product metrics, so the shorthand rule must not reach them.+ BaseEvalCase(+ name="acronym_that_is_not_a_metric",+ prompt="anyone know the ETA on the new laptops?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="hr_acronym",+ prompt="what's our PTO policy over the holidays?",+ expected=_answerable(False),+ ),+ # A metric named in a question that asks for an opinion, not for the number.+ BaseEvalCase(+ name="metric_in_an_opinion_question",+ prompt="should we use DAU or WAU as our north star metric?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="strategy_question",+ prompt="should we prioritize mobile or web next quarter?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="finance_question",+ prompt="what's our burn rate looking like this quarter?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="sales_deal",+ prompt="has the Northwind deal closed yet?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="it_outage",+ prompt="is the VPN down for anyone else?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="too_vague",+ prompt="any thoughts on this one?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="personal_favor",+ prompt="can someone grab me a coffee on the way in?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="question_for_a_named_person",+ prompt="Sam, what did you think of the demo yesterday?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="document_status",+ prompt="is the Q3 roadmap doc ready for review?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="hiring_question",+ prompt="how many open roles do we have on the growth team?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="company_calendar",+ prompt="when is the next company holiday?",+ expected=_answerable(False),+ ),+ BaseEvalCase(+ name="outside_world",+ prompt="what's the weather going to be like in London tomorrow?",+ expected=_answerable(False),+ ),+]+++async def eval_untagged_question(ctx: EvalContext) -> None:+ async def task(case: BaseEvalCase, task_ctx: EvalContext) -> dict:+ model = untagged_question.UNTAGGED_QUESTION_DECISION_MODEL+ if task_ctx.demo_data is None:+ return {"decision_model": model, "answerable": None, "error": "No demo data team to bill the call to"}+ try:+ # Sync and blocking on the gateway, so off the event loop.+ verdict = await asyncio.to_thread(+ untagged_question.classify_untagged_question,+ case.prompt,+ team_id=task_ctx.demo_data.master_team_id,+ distinct_id=None,+ )+ except Exception as error:+ return {"decision_model": model, "answerable": None, "error": f"{type(error).__name__}: {error}"}+ answerable = verdict is not None and verdict.answerable+ return {+ "decision_model": model,+ "answerable": answerable,+ "verdict": {+ "asks_for_information": verdict.asks_for_information,+ "answerable_by_posthog": verdict.answerable_by_posthog,+ }+ if verdict+ else None,+ "last_message": f"{model}: answerable={answerable}",+ }++ await OneShotPublicEval(+ experiment_name="slack-app-untagged-question",+ cases=[*ANSWERABLE_CASES, *LEAVE_ALONE_CASES],+ scorers=[UntaggedQuestionMatch(), NoUnaskedAnswer()],+ task=task,+ ctx=ctx,+ )diff --git a/products/slack_app/evals/scorers.py b/products/slack_app/evals/scorers.pyindex 6408bae68135..256013109429 100644--- a/products/slack_app/evals/scorers.py+++ b/products/slack_app/evals/scorers.py@@ -464,3 +464,52 @@ Answer `yes` if every reply follows all of these rules. Answer `no` if one replymax_completion_tokens=256,**kwargs,)+++UNTAGGED_QUESTION_KEY = "untagged_question"+++class UntaggedQuestionMatch(Scorer):+ """Did the classifier decide about answering the way a person in the channel would?"""++ def _name(self) -> str:+ return UNTAGGED_QUESTION_KEY++ def _run_eval_sync(self, output: dict | None, expected=None, **kwargs) -> Score:+ want = (expected or {}).get(UNTAGGED_QUESTION_KEY) or {}+ if "answerable" not in want:+ return Score(name=self._name(), score=None, metadata={"reason": "No expectation for this case"})+ if output and output.get("error"):+ return Score(name=self._name(), score=0.0, metadata={"reason": output["error"]})++ got = (output or {}).get("answerable")+ return Score(+ name=self._name(),+ score=1.0 if got == want["answerable"] else 0.0,+ metadata={"expected": want["answerable"], "actual": got, "verdict": (output or {}).get("verdict")},+ )+++class NoUnaskedAnswer(Scorer):+ """The expensive direction: answering a channel message PostHog cannot or should not answer.++ Skips on cases PostHog should answer, so the score reads as a rate over the messages it+ should have left alone.+ """++ def _name(self) -> str:+ return "no_unasked_answer"++ def _run_eval_sync(self, output: dict | None, expected=None, **kwargs) -> Score:+ want = (expected or {}).get(UNTAGGED_QUESTION_KEY) or {}+ if want.get("answerable", True):+ return Score(name=self._name(), score=None, metadata={"reason": "Case is answerable"})+ if output and output.get("error"):+ # A failed call answers nothing, this scorer's passing answer, so it would hide an outage.+ return Score(name=self._name(), score=None, metadata={"reason": output["error"]})++ return Score(+ name=self._name(),+ score=0.0 if (output or {}).get("answerable") else 1.0,+ metadata={"actual": (output or {}).get("answerable"), "verdict": (output or {}).get("verdict")},+ )