Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(slack-app): offer answers to untagged channel questions (#114598)

Vojta Bartošfd2e8ba0a549
diff --git a/products/slack_app/backend/api.py b/products/slack_app/backend/api.py
index 7136fa913254..9112dcff11c3 100644
--- a/products/slack_app/backend/api.py
+++ b/products/slack_app/backend/api.py
@@ -51,9 +51,11 @@ from products.slack_app.backend.feature_flags import (
is_slack_app_assistant_enabled,
is_slack_app_oauth_enabled,
is_slack_app_project_picker_enabled,
+ is_slack_app_unprompted_answers_enabled,
)
from products.slack_app.backend.helpers import local_dev_slack_email
from products.slack_app.backend.logic import reply_suggestions
+from products.slack_app.backend.logic.untagged_question import is_untagged_question_candidate
from products.slack_app.backend.models import (
ChannelWelcomeMode,
SlackChannel,
@@ -87,7 +89,7 @@ from products.slack_app.backend.services.slack_app_home import (
handle_app_home_opened as _handle_app_home_opened,
handle_app_home_view_submission as _handle_app_home_view_submission,
)
-from products.slack_app.backend.services.slack_conversations import channel_is_ext_shared
+from products.slack_app.backend.services.slack_conversations import channel_ext_shared_state, channel_is_ext_shared
from products.slack_app.backend.services.slack_fork_context import clear_pending_fork, get_pending_fork
from products.slack_app.backend.services.slack_messages import (
FORK_THREAD_ACTION_ID,
@@ -105,6 +107,8 @@ from products.slack_app.backend.services.slack_scopes import REQUIRED_SLACK_SCOP
from products.slack_app.backend.services.slack_settings import (
resolve_channel_welcome_mode,
resolve_untagged_followup_mode,
+ resolve_untagged_question_mode,
+ set_untagged_question_mode,
)
from products.slack_app.backend.services.slack_user_info import (
clear_workspace_profile_cache,
@@ -207,6 +211,11 @@ UNTAGGED_FOLLOWUP_BLOCK_ID_PREFIX = "posthog_code_untagged_followup"
UNTAGGED_FOLLOWUP_ACTION_RUN = "posthog_code_untagged_followup_run"
UNTAGGED_FOLLOWUP_ACTION_DISMISS = "posthog_code_untagged_followup_dismiss"
UNTAGGED_FOLLOWUP_CONTEXT_KIND = "untagged_followup"
+UNTAGGED_QUESTION_BLOCK_ID_PREFIX = "posthog_code_untagged_question"
+UNTAGGED_QUESTION_ACTION_RUN = "posthog_code_untagged_question_run"
+UNTAGGED_QUESTION_ACTION_DISMISS = "posthog_code_untagged_question_dismiss"
+UNTAGGED_QUESTION_ACTION_TURN_OFF = "posthog_code_untagged_question_turn_off"
+UNTAGGED_QUESTION_CONTEXT_KIND = "untagged_question"
_MAX_GITHUB_REPOS = 500
# The installation webhook marks snapshots stale without saving rows, so it does not clear this key.
@@ -597,6 +606,9 @@ REGION_PROXY_HEADER = "X-PostHog-Region-Proxied"
# be connected in both regions; the receiver runs the emit for its own projects and nothing else,
# so the sender's pipeline (thread follow-ups, mentions) stays the only one handling the event.
EMIT_ONLY_MIRROR_HEADER = "X-PostHog-Slack-Emit-Only"
+# Marks a full top-level post the sender already emitted and mirrored. The receiver handles the
+# untagged question but skips the emit, which the mirror already covers.
+ALREADY_EMITTED_HEADER = "X-PostHog-Slack-Already-Emitted"
REGION_PROXY_TIMEOUT_SECONDS = 3
# Tight budget: the workspace_claims endpoint is just a DB .exists(), and EU calls it inline
# before deciding whether to proxy. Slack's webhook ack deadline is 3s total, so we want this
@@ -705,20 +717,25 @@ def send_region_proxy_request(
return None
-def _proxy_event_to_region(request: HttpRequest, target_domain: str) -> requests.Response | None:
+def _proxy_event_to_region(
+ request: HttpRequest, target_domain: str, extra_headers: dict[str, str] | None = None
+) -> requests.Response | None:
"""Forward the original Slack event to the other region, tagged so the receiver does not hop again."""
return send_region_proxy_request(
method=request.method or "POST",
target_url=_proxy_target_url(request, target_domain),
- headers=_proxy_request_headers(request),
+ headers={**_proxy_request_headers(request), **(extra_headers or {})},
params=dict(request.GET.lists()) if request.GET else None,
body=request.body or None,
)
-def _proxy_event_and_return_route(request: HttpRequest, target_domain: str) -> str:
+def _proxy_event_and_return_route(
+ request: HttpRequest, target_domain: str, extra_headers: dict[str, str] | None = None
+) -> str:
"""Forward and translate the upstream result into a routing outcome string."""
- return ROUTE_PROXIED if _proxy_event_to_region(request, target_domain) is not None else ROUTE_PROXY_FAILED
+ upstream = _proxy_event_to_region(request, target_domain, extra_headers)
+ return ROUTE_PROXIED if upstream is not None else ROUTE_PROXY_FAILED
def _is_top_level_channel_post(event: dict[str, Any]) -> bool:
@@ -1432,6 +1449,24 @@ def _message_handled_cache_key(slack_team_id: str, event: dict[str, Any]) -> str
return f"slack_app:message_handled:v1:{slack_team_id}:{channel}:{message_ts}"
+def claim_message_handled(slack_team_id: str, event: dict[str, Any], handled_as: str) -> bool:
+ """Record that the pipeline acted on this message, unless another path already did.
+
+ Atomic, so an answer to an untagged question and an edit of the same message that adds a tag
+ can't both start a run. Returns the marker's current value on a lost claim, so a caller can
+ recognise its own earlier claim.
+ """
+ cache_key = _message_handled_cache_key(slack_team_id, event)
+ if cache_key is None:
+ return True
+ return bool(cache.add(cache_key, handled_as, timeout=MESSAGE_HANDLED_MARKER_TTL_SECONDS))
+
+
+def message_handled_by(slack_team_id: str, event: dict[str, Any]) -> str | None:
+ cache_key = _message_handled_cache_key(slack_team_id, event)
+ return cache.get(cache_key) if cache_key else None
+
+
def _mark_message_handled(slack_team_id: str, event: dict[str, Any], handled_as: str) -> None:
"""Record that the pipeline acted on this message, so that a later edit of it starts nothing."""
cache_key = _message_handled_cache_key(slack_team_id, event)
@@ -1459,7 +1494,7 @@ def _edited_mention_ignore_cause(event: dict[str, Any], slack_team_id: str) -> s
return "no_message_ts"
if time.time() - posted_at > EDITED_MENTION_WINDOW_SECONDS:
return "too_old"
- if not cache.add(cache_key, "edited_mention", timeout=MESSAGE_HANDLED_MARKER_TTL_SECONDS):
+ if not claim_message_handled(slack_team_id, event, "edited_mention"):
return f"handled_as_{cache.get(cache_key) or 'unknown'}"
return None
@@ -2489,6 +2524,92 @@ def _handle_app_uninstalled(request: HttpRequest, slack_team_id: str) -> str:
return ROUTE_HANDLED_LOCALLY
+def _route_untagged_question(
+ request: HttpRequest,
+ event: dict[str, Any],
+ slack_team_id: str,
+ event_id: str | None,
+ *,
+ proxied: bool,
+ incoming_host: str,
+ other_domain: str,
+ can_defer: bool,
+ is_ext_shared_channel: bool,
+) -> str:
+ """Start the classifier workflow for a top-level channel post nobody tagged the app in.
+
+ Every drop here is silent: nobody asked PostHog anything, so nobody waits for an answer.
+ An author without a PostHog account is a normal drop, not a failure.
+ Externally shared channels are out, because people outside the organization would see the reply.
+ """
+ if is_ext_shared_channel or not is_untagged_question_candidate(event):
+ return ROUTE_HANDLED_LOCALLY
+ slack_user_id = str(event.get("user") or "")
+ channel = event.get("channel") if isinstance(event.get("channel"), str) else None
+
+ workspace_result = load_integrations(
+ slack_team_id=slack_team_id,
+ kinds=[SLACK_INTEGRATION_KIND],
+ slack_user_id=slack_user_id,
+ channel=channel,
+ )
+ # Without a local connection no mirror went out, so the full event can cross, but only to a
+ # region that confirms it holds the workspace. The probe answer is cached per workspace.
+ if not workspace_result.candidates:
+ if proxied or not cross_region_routing_enabled():
+ return ROUTE_HANDLED_LOCALLY
+ claimed = does_other_region_claim_workspace(
+ slack_team_id=slack_team_id, kinds=[SLACK_INTEGRATION_KIND], incoming_host=incoming_host
+ )
+ if claimed is not True:
+ return ROUTE_HANDLED_LOCALLY
+ return _proxy_event_and_return_route(request, other_domain)
+ # The same US precedence as mentions, so an author whose account is only in US still gets an
+ # answer when EU receives the post. This region already emitted the post and queued a mirror,
+ # so the forwarded copy tells the receiver to skip its emit.
+ if _us_should_handle_instead(slack_team_id, [SLACK_INTEGRATION_KIND], can_defer, incoming_host):
+ return _proxy_event_and_return_route(request, other_domain, {ALREADY_EMITTED_HEADER: "1"})
+
+ # One indexed read, ahead of user resolution, which can call Slack's users.info for every author.
+ if resolve_untagged_question_mode(slack_team_id, slack_user_id) == UntaggedFollowupMode.NEVER:
+ return ROUTE_HANDLED_LOCALLY
+
+ resolution = resolve_user_for_workspace(
+ workspace_result=workspace_result,
+ slack_team_id=slack_team_id,
+ slack_user_id=slack_user_id,
+ event_id=event_id,
+ )
+ posthog_user = resolution.user
+ if posthog_user is None:
+ return ROUTE_HANDLED_LOCALLY
+ candidates = resolution.candidates
+ # No picker: asking which project a question nobody addressed to us belongs to is noise.
+ target = resolution.integration or (candidates[0] if len(candidates) == 1 else None)
+ if target is None or SlackIntegration(target).missing_scopes(REQUIRED_SLACK_SCOPES):
+ return ROUTE_HANDLED_LOCALLY
+ # Checked on the project that will answer: a workspace can connect projects from several
+ # organizations, and the flag is rolled out per organization.
+ if not is_slack_app_unprompted_answers_enabled(target):
+ return ROUTE_HANDLED_LOCALLY
+
+ logger.info(
+ "slack_app_untagged_question_dispatched",
+ slack_team_id=slack_team_id,
+ integration_id=target.id,
+ channel=channel,
+ message_ts=event.get("ts"),
+ )
+ return _start_mention_workflow(
+ event,
+ target,
+ slack_team_id,
+ event_id,
+ posthog_user=posthog_user,
+ untagged_question=True,
+ )
+
+
def _dispatch_mention_to_target(
event: dict,
mention_target: Integration,
@@ -2676,7 +2797,7 @@ def route_posthog_code_event_to_relevant_region(
)
return ROUTE_HANDLED_LOCALLY
- should_try_other_region = emit_slack_message_event(
+ should_try_other_region = request.headers.get(ALREADY_EMITTED_HEADER) != "1" and emit_slack_message_event(
event,
slack_team_id,
event_id=event_id,
@@ -2743,9 +2864,20 @@ def route_posthog_code_event_to_relevant_region(
message_ts=event.get("ts"),
)
return ROUTE_HANDLED_LOCALLY
- # Top-level channel posts dominate the wire volume; drop before the pipeline's DB hits.
+ # Top-level channel posts dominate the wire volume, so they leave the shared pipeline
+ # here and face cheap gates before any DB hit.
if _is_top_level_channel_post(event):
- return ROUTE_HANDLED_LOCALLY
+ return _route_untagged_question(
+ request,
+ event,
+ slack_team_id,
+ event_id,
+ proxied=proxied,
+ incoming_host=incoming_host,
+ other_domain=other_domain,
+ can_defer=can_defer_to_other_region,
+ is_ext_shared_channel=is_ext_shared_channel,
+ )
slack_user_id_str = str(event.get("user") or "")
channel_str = event.get("channel") if isinstance(event.get("channel"), str) else None
@@ -3822,6 +3954,217 @@ def _post_untagged_followup_prompt(
return True
+def _post_untagged_question_prompt(slack: SlackIntegration, integration: Integration, event: dict[str, Any]) -> bool:
+ """Offer the author, privately, to answer the question they posted without tagging the app.
+
+ Posted when the author's mode is ``ask``. The whole event is stashed behind a one-off
+ token so a click dispatches the original message. Not threaded: the post has no thread
+ yet, so an ephemeral anchored to one would never be seen.
+ """
+ channel = event.get("channel") if isinstance(event.get("channel"), str) else None
+ slack_user_id = event.get("user") if isinstance(event.get("user"), str) else None
+ if not channel or not slack_user_id:
+ logger.warning("slack_app_untagged_question_prompt_skipped", integration_id=integration.id)
+ return False
+
+ context_token = uuid.uuid4().hex
+ cache.set(
+ _picker_context_cache_key(context_token),
+ {
+ "kind": UNTAGGED_QUESTION_CONTEXT_KIND,
+ "integration_id": integration.id,
+ "slack_user_id": slack_user_id,
+ "event": event,
+ },
+ timeout=PICKER_TOKEN_MAX_AGE_SECONDS,
+ )
+
+ home_tab_url = app_home_url(integration)
+ home_tab_label = f"<{home_tab_url}|PostHog app Home tab>" if home_tab_url else "PostHog app Home tab"
+ blocks: list[dict[str, Any]] = [
+ {
+ "type": "section",
+ "block_id": f"{UNTAGGED_QUESTION_BLOCK_ID_PREFIX}:{context_token}",
+ "text": {
+ "type": "mrkdwn",
+ "text": (
+ "I think I can answer your question from your PostHog data. Want me to reply in a thread? "
+ "Only you can see this."
+ ),
+ },
+ },
+ {
+ "type": "actions",
+ "block_id": f"{UNTAGGED_QUESTION_BLOCK_ID_PREFIX}_actions:{context_token}",
+ "elements": [
+ {
+ "type": "button",
+ "action_id": UNTAGGED_QUESTION_ACTION_RUN,
+ "style": "primary",
+ "text": {"type": "plain_text", "text": "Yes, answer it"},
+ "value": context_token,
+ },
+ {
+ "type": "button",
+ "action_id": UNTAGGED_QUESTION_ACTION_DISMISS,
+ "text": {"type": "plain_text", "text": "No thanks"},
+ "value": context_token,
+ },
+ {
+ "type": "button",
+ "action_id": UNTAGGED_QUESTION_ACTION_TURN_OFF,
+ "text": {"type": "plain_text", "text": "Only when I tag you"},
+ "value": context_token,
+ },
+ ],
+ },
+ {
+ "type": "context",
+ "elements": [{"type": "mrkdwn", "text": f"You can change this in the {home_tab_label}."}],
+ },
+ ]
+
+ try:
+ slack.client.chat_postEphemeral(
+ channel=channel,
+ user=slack_user_id,
+ text="PostHog can answer your question. Want a reply?",
+ blocks=blocks,
+ )
+ except Exception:
+ logger.warning(
+ "slack_app_untagged_question_prompt_failed",
+ integration_id=integration.id,
+ slack_channel_id=channel,
+ exc_info=True,
+ )
+ return False
+ capture_slack_event(integration, "slack app unprompted answer offered", slack_user_id=slack_user_id)
+ return True
+
+
+@frozen
+class _UntaggedQuestionClick:
+ context_token: str
+ context: dict[str, Any]
+ integration: Integration
+
+
+def _untagged_question_click(payload: dict) -> _UntaggedQuestionClick | None:
+ """The live prompt context and its integration, or None when the prompt is stale or not the clicker's."""
+ context_token = _extract_context_token(payload)
+ context = _decode_picker_context(context_token) if context_token else None
+ if not context or context.get("kind") != UNTAGGED_QUESTION_CONTEXT_KIND:
+ return None
+ slack_team_id = payload.get("team", {}).get("id", "")
+ integration_id = context.get("integration_id")
+ # The prompt is ephemeral, so only its recipient can click it, but the dispatch runs as the clicker.
+ if not integration_id or not slack_team_id or payload.get("user", {}).get("id") != context.get("slack_user_id"):
+ return None
+ integration = (
+ Integration.objects.filter( # nosemgrep: idor-lookup-without-team
+ id=integration_id, # nosemgrep: idor-taint-user-input-to-model-get
+ kind=SLACK_INTEGRATION_KIND,
+ integration_id=slack_team_id,
+ )
+ .select_related("team__organization")
+ .first()
+ )
+ if integration is None:
+ return None
+ return _UntaggedQuestionClick(context_token=context_token, context=context, integration=integration)
+
+
+def _handle_untagged_question_run(payload: dict) -> HttpResponse:
+ response_url = payload.get("response_url", "")
+ click = _untagged_question_click(payload)
+ if click is None:
+ _delete_ephemeral_via_response_url(response_url)
+ return HttpResponse(status=200)
+ context_token, context, integration = click.context_token, click.context, click.integration
+ slack_user_id = context["slack_user_id"]
+ event = context.get("event")
+
+ # The confirmed run skips the gates the webhook and the workflow already applied, so the
+ # ones that can change while the prompt waits are checked again here. The two checks that
+ # call Slack run last.
+ slack = SlackIntegration(integration)
+ if (
+ not isinstance(event, dict)
+ or slack.missing_scopes(REQUIRED_SLACK_SCOPES)
+ or resolve_untagged_question_mode(integration.integration_id, slack_user_id) == UntaggedFollowupMode.NEVER
+ or not is_slack_app_unprompted_answers_enabled(integration)
+ or (posthog_user := _is_org_member(integration, slack_user_id)) is None
+ or not _can_access_team(posthog_user, integration)
+ ):
+ _delete_ephemeral_via_response_url(response_url)
+ return HttpResponse(status=200)
+
+ # A channel shared outside the organization since the offer would show the answer to people
+ # the webhook's gate kept it from. When Slack cannot say, the offer stays for a later click.
+ ext_shared = channel_ext_shared_state(slack, event.get("channel", ""))
+ if ext_shared is None:
+ return HttpResponse(status=200)
+ if ext_shared:
+ _delete_ephemeral_via_response_url(response_url)
+ return HttpResponse(status=200)
+
+ _start_mention_workflow(
+ event,
+ integration,
+ integration.integration_id,
+ None,
+ posthog_user=posthog_user,
+ untagged_question=True,
+ untagged_question_confirmed=True,
+ )
+ # Cleared only once the run is dispatched, so a Temporal outage leaves the offer clickable.
+ cache.delete(_picker_context_cache_key(context_token))
+ capture_slack_event(
+ integration, "slack app unprompted answer confirmed", slack_user_id=slack_user_id, posthog_user=posthog_user
+ )
+ _delete_ephemeral_via_response_url(response_url)
+ return HttpResponse(status=200)
+
+
+def _handle_untagged_question_dismiss(payload: dict) -> HttpResponse:
+ """Drop the offer. The choice covers this one message, so nothing is stored."""
+ click = _untagged_question_click(payload)
+ _delete_ephemeral_via_response_url(payload.get("response_url", ""))
+ if click is not None:
+ context_token, context, integration = click.context_token, click.context, click.integration
+ cache.delete(_picker_context_cache_key(context_token))
+ capture_slack_event(
+ integration, "slack app unprompted answer dismissed", slack_user_id=context["slack_user_id"]
+ )
+ return HttpResponse(status=200)
+
+
+def _handle_untagged_question_turn_off(payload: dict) -> HttpResponse:
+ """Stop answering the clicker's untagged channel questions, and say where to turn it back on."""
+ response_url = payload.get("response_url", "")
+ click = _untagged_question_click(payload)
+ if click is None:
+ _delete_ephemeral_via_response_url(response_url)
+ return HttpResponse(status=200)
+ context_token, context, integration = click.context_token, click.context, click.integration
+ slack_user_id = context["slack_user_id"]
+ cache.delete(_picker_context_cache_key(context_token))
+ set_untagged_question_mode(integration.integration_id, slack_user_id, UntaggedFollowupMode.NEVER)
+ capture_slack_event(integration, "slack app unprompted answers turned off", slack_user_id=slack_user_id)
+ inbox_interactivity.post_response_url(
+ response_url,
+ {
+ "replace_original": True,
+ "text": (
+ "Got it. I'll only answer your questions in channels when you tag me. "
+ "You can change this in the PostHog app Home tab."
+ ),
+ },
+ )
+ return HttpResponse(status=200)
+
+
def _can_access_team(posthog_user: User, integration: Integration) -> bool:
"""Whether the user can reach the integration's project.
@@ -4291,6 +4634,8 @@ def _start_mention_workflow(
posthog_user: User,
untagged_followup: bool = False,
untagged_followup_confirmed: bool = False,
+ untagged_question: bool = False,
+ untagged_question_confirmed: bool = False,
is_ext_shared_channel: bool = False,
awaited_request_reply: bool = False,
fork_source_channel: str | None = None,
@@ -4312,10 +4657,11 @@ def _start_mention_workflow(
The ``fork_source_*`` fields mark a forked run, where the thread being answered
(a DM) and the thread supplying the context (the forked channel thread) differ.
They suppress the same two side effects: nobody mentioned us, and a reply in a
- forked DM can't be resolving a picker.
+ forked DM can't be resolving a picker. ``untagged_question`` suppresses them
+ for the same reason, and makes the workflow ask its classifier first.
"""
is_fork = bool(fork_source_channel and fork_source_thread_ts)
- if not untagged_followup and not is_fork:
+ if not untagged_followup and not untagged_question and not is_fork:
_report_slack_mention_received(
event,
integration,
@@ -4333,6 +4679,8 @@ def _start_mention_workflow(
user_id=posthog_user.id,
untagged_followup=untagged_followup,
untagged_followup_confirmed=untagged_followup_confirmed,
+ untagged_question=untagged_question,
+ untagged_question_confirmed=untagged_question_confirmed,
is_ext_shared_channel=is_ext_shared_channel,
fork_source_channel=fork_source_channel,
fork_source_thread_ts=fork_source_thread_ts,
@@ -5838,6 +6186,12 @@ def posthog_code_interactivity_handler(request: HttpRequest) -> HttpResponse:
return _handle_untagged_followup_run(payload)
if action_id == UNTAGGED_FOLLOWUP_ACTION_DISMISS:
return _handle_untagged_followup_dismiss(payload)
+ if action_id == UNTAGGED_QUESTION_ACTION_RUN:
+ return _handle_untagged_question_run(payload)
+ if action_id == UNTAGGED_QUESTION_ACTION_DISMISS:
+ return _handle_untagged_question_dismiss(payload)
+ if action_id == UNTAGGED_QUESTION_ACTION_TURN_OFF:
+ return _handle_untagged_question_turn_off(payload)
if project_picker.is_project_picker_action(action_id):
return _handle_project_picker_pick(payload)
if action_id == SIGNALS_DISMISS_REPORT_ACTION_ID:
diff --git a/products/slack_app/backend/feature_flags.py b/products/slack_app/backend/feature_flags.py
index c11b6caef31b..9ed198f65b4d 100644
--- a/products/slack_app/backend/feature_flags.py
+++ b/products/slack_app/backend/feature_flags.py
@@ -49,6 +49,7 @@ SLACK_APP_AGENT_DESIGN_FLAG = "slack-app-agent-design"
SLACK_APP_FORKING_FLAG = "slack-app-forking"
SLACK_APP_PROJECT_PICKER_FLAG = "slack-app-project-picker"
SLACK_APP_REPLY_SUGGESTIONS_FLAG = "slack-app-reply-suggestions"
+SLACK_APP_UNPROMPTED_ANSWERS_FLAG = "slack-app-unprompted-answers"
# Linking a Slack identity to a PostHog user resolves the Slack profile and its email.
@@ -146,6 +147,19 @@ def is_slack_app_reply_suggestions_enabled(integration: Integration) -> bool:
)
+def is_slack_app_unprompted_answers_enabled(integration: Integration) -> bool:
+ """Gate for answering top-level channel messages nobody tagged the app in.
+
+ Keyed on the workspace, not the author, because the webhook checks it before it resolves
+ who wrote the message. Roll it out with an organization condition.
+ """
+ return _workspace_flag_enabled(
+ SLACK_APP_UNPROMPTED_ANSWERS_FLAG,
+ integration,
+ failure_log_key="slack_app_unprompted_answers_feature_flag_check_failed",
+ )
+
+
def is_slack_app_assistant_enabled(integration: Integration) -> bool:
"""Gate for the DM assistant: the install must hold the scopes the DM surface
calls — ``im:history`` in particular, without which the assistant would answer
diff --git a/products/slack_app/backend/logic/untagged_question.py b/products/slack_app/backend/logic/untagged_question.py
new file mode 100644
index 000000000000..816521bf96c4
--- /dev/null
+++ b/products/slack_app/backend/logic/untagged_question.py
@@ -0,0 +1,30 @@
+"""Which top-level channel posts, written without tagging the app, are worth a classifier call."""
+
+import re
+from typing import Any
+
+UNTAGGED_QUESTION_MIN_CHARS = 12
+UNTAGGED_QUESTION_MAX_CHARS = 2000
+_QUESTION_OPENER_PATTERN = re.compile(
+ r"^(how|what|why|when|where|which|who|whose|is|are|was|were|can|could|does|do|did|should|would|will|has|have|any|anyone|anybody)\b",
+ re.IGNORECASE,
+)
+
+
+def is_untagged_question_candidate(event: dict[str, Any]) -> bool:
+ """Whether a top-level post nobody tagged the app in could be a question for PostHog.
+
+ Text-only checks, because every top-level post in every channel the app is in comes through
+ here. The classifier makes the real decision later, on the few posts that pass.
+ """
+ # ``group`` is a private channel. Group DMs (``mpim``) are private conversations between people.
+ if event.get("channel_type") not in ("channel", "group"):
+ return False
+ text = (event.get("text") or "").strip()
+ if not UNTAGGED_QUESTION_MIN_CHARS <= len(text) <= UNTAGGED_QUESTION_MAX_CHARS:
+ return False
+ # A tag of the app arrives as its own ``app_mention``. A tag of anyone else, a user group or
+ # the whole channel addresses them, not PostHog.
+ if "<@" in text or "<!" in text:
+ return False
+ return "?" in text or bool(_QUESTION_OPENER_PATTERN.match(text))
diff --git a/products/slack_app/backend/migrations/0021_slacksettings_untagged_question_mode.py b/products/slack_app/backend/migrations/0021_slacksettings_untagged_question_mode.py
new file mode 100644
index 000000000000..a17bd2a62a58
--- /dev/null
+++ b/products/slack_app/backend/migrations/0021_slacksettings_untagged_question_mode.py
@@ -0,0 +1,27 @@
+# Generated by Django 5.2.18 on 2026-10-19 14:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("slack_app", "0020_slacksettings_auto_model_choice"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="slacksettings",
+ name="untagged_question_mode",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("auto", "Always pick it up"),
+ ("ask", "Ask before picking it up"),
+ ("never", "Never pick it up"),
+ ],
+ help_text="What PostHog does with this user's top-level channel questions that do not tag the app.",
+ max_length=16,
+ null=True,
+ ),
+ ),
+ ]
diff --git a/products/slack_app/backend/migrations/max_migration.txt b/products/slack_app/backend/migrations/max_migration.txt
index e36f82e9ade6..19664f63b5a9 100644
--- a/products/slack_app/backend/migrations/max_migration.txt
+++ b/products/slack_app/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0020_slacksettings_auto_model_choice
+0021_slacksettings_untagged_question_mode
diff --git a/products/slack_app/backend/models.py b/products/slack_app/backend/models.py
index d1c18b029d72..b3d4f3bc35c4 100644
--- a/products/slack_app/backend/models.py
+++ b/products/slack_app/backend/models.py
@@ -113,11 +113,12 @@ class SlackUserProfileCache(UUIDModel):
class UntaggedFollowupMode(models.TextChoices):
- """What PostHog does with an untagged reply in a thread it already owns.
+ """What PostHog does with a message that does not tag it.
- Read from the thread creator's settings row, so it governs everyone
- replying in a thread that user started. An unset row resolves to ``ASK``,
- so the person who replied can choose whether to send their message.
+ Read from the settings row of the person who started the thread, so it governs
+ every untagged reply in a thread PostHog owns, and the author's own top-level
+ channel question. An unset row resolves to ``ASK``, so the person who wrote the
+ message chooses whether to send it.
"""
AUTO = "auto", "Always pick it up"
@@ -182,6 +183,14 @@ class SlackSettings(UUIDModel):
choices=UntaggedFollowupMode.choices,
help_text="What PostHog does with untagged replies in threads this user started.",
)
+ # NULL means the user has never picked, which resolves to ``ASK``.
+ untagged_question_mode = models.CharField(
+ max_length=16,
+ null=True,
+ blank=True,
+ choices=UntaggedFollowupMode.choices,
+ help_text="What PostHog does with this user's top-level channel questions that do not tag the app.",
+ )
# Only read on the workspace-wide row. NULL resolves to ``CHANNEL``.
channel_welcome_mode = models.CharField(
max_length=16,
diff --git a/products/slack_app/backend/services/slack_app_home.py b/products/slack_app/backend/services/slack_app_home.py
index 142b842c4864..a880ec266b63 100644
--- a/products/slack_app/backend/services/slack_app_home.py
+++ b/products/slack_app/backend/services/slack_app_home.py
@@ -33,7 +33,7 @@ from posthog.models.user_integration import UserIntegration
from posthog.user_permissions import UserPermissions
from products.slack_app.backend.analytics import capture_slack_event
-from products.slack_app.backend.feature_flags import is_slack_app_oauth_enabled
+from products.slack_app.backend.feature_flags import is_slack_app_oauth_enabled, is_slack_app_unprompted_answers_enabled
from products.slack_app.backend.models import (
ChannelWelcomeMode,
SlackSettings,
@@ -73,8 +73,11 @@ from products.slack_app.backend.services.slack_settings import (
resolve_auto_model_choice,
resolve_channel_welcome_mode,
resolve_untagged_followup_mode,
+ resolve_untagged_question_mode,
set_auto_model_choice,
set_channel_welcome_mode,
+ set_untagged_followup_mode,
+ set_untagged_question_mode,
)
from products.slack_app.backend.services.slack_user_info import is_slack_workspace_admin
from products.slack_app.backend.services.slack_user_oauth import build_invite_url, find_linked_posthog_user
@@ -101,6 +104,7 @@ ACTION_TASKS_PAGE_NEXT = "slack_app_home:tasks_page_next"
ACTION_STATS_WINDOW = "slack_app_home:stats_window"
ACTION_STATS_REFRESH = "slack_app_home:stats_refresh"
ACTION_SET_UNTAGGED_FOLLOWUP_MODE = "slack_app_home:set_untagged_followup_mode"
+ACTION_SET_UNTAGGED_QUESTION_MODE = "slack_app_home:set_untagged_question_mode"
ACTION_SET_CHANNEL_WELCOME_MODE = "slack_app_home:set_channel_welcome_mode"
ACTION_SET_AUTO_MODEL_CHOICE = "slack_app_home:set_auto_model_choice"
# URL buttons: Slack opens the link itself and posts a block_actions payload we
@@ -128,6 +132,7 @@ HOME_ACTION_IDS: frozenset[str] = frozenset(
ACTION_STATS_WINDOW,
ACTION_STATS_REFRESH,
ACTION_SET_UNTAGGED_FOLLOWUP_MODE,
+ ACTION_SET_UNTAGGED_QUESTION_MODE,
ACTION_SET_CHANNEL_WELCOME_MODE,
ACTION_SET_AUTO_MODEL_CHOICE,
ACTION_GITHUB_SETTINGS,
@@ -441,6 +446,7 @@ def render_home_view(
tasks_state: TasksState | None = None,
stats_state: StatsState | None = None,
untagged_followup_mode: UntaggedFollowupMode | None = None,
+ untagged_question_mode: UntaggedFollowupMode | None = None,
channel_welcome_mode: ChannelWelcomeMode | None = None,
auto_model_choice: bool | None = None,
has_project_access: bool = True,
@@ -490,6 +496,11 @@ def render_home_view(
blocks.append({"type": "divider"})
blocks.extend(_untagged_followups_section_blocks(untagged_followup_mode))
+ # Section 4b — channel questions. `None` means the flag is off for this workspace.
+ if untagged_question_mode is not None:
+ blocks.append({"type": "divider"})
+ blocks.extend(_untagged_questions_section_blocks(untagged_question_mode))
+
# Section 5 — channel welcome: a workspace setting, so only admins see it.
if is_admin and channel_welcome_mode is not None:
blocks.append({"type": "divider"})
@@ -951,36 +962,69 @@ UNTAGGED_FOLLOWUP_MODE_LABELS: dict[str, str] = {
UntaggedFollowupMode.NEVER: "Leave them alone",
}
+UNTAGGED_QUESTION_MODE_LABELS: dict[str, str] = {
+ UntaggedFollowupMode.AUTO: "Answer automatically",
+ UntaggedFollowupMode.ASK: "Ask me first",
+ UntaggedFollowupMode.NEVER: "Never answer",
+}
-def _untagged_followups_section_blocks(mode: UntaggedFollowupMode) -> list[dict]:
- """Picker for how untagged replies land in the threads you started.
- Ask until picked. The choice covers every reply in those threads,
- including the ones you write yourself.
+def _untagged_mode_section_blocks(
+ mode: UntaggedFollowupMode,
+ *,
+ action_id: str,
+ labels: dict[str, str],
+ title: str,
+ subtitle: str,
+ note: str,
+) -> list[dict]:
+ """One picker for what PostHog does with a kind of message nobody tagged it in.
+
+ Both pickers ask until the viewer picks, so nothing reaches the agent or the
+ channel on a default.
"""
options = [
{"text": {"type": "plain_text", "text": label, "emoji": True}, "value": value}
- for value, label in UNTAGGED_FOLLOWUP_MODE_LABELS.items()
+ for value, label in labels.items()
]
select: dict[str, Any] = {
"type": "static_select",
- "action_id": ACTION_SET_UNTAGGED_FOLLOWUP_MODE,
+ "action_id": action_id,
"options": options,
"initial_option": next(o for o in options if o["value"] == mode.value),
}
return [
- _section_title(
- "💬 Thread follow-ups",
- "What I do with replies in a thread you started, when nobody tags @PostHog.",
- ),
+ _section_title(title, subtitle),
{"type": "actions", "elements": [select]},
- {
- "type": "context",
- "elements": [{"type": "mrkdwn", "text": "Applies to every reply in those threads, yours included."}],
- },
+ {"type": "context", "elements": [{"type": "mrkdwn", "text": note}]},
]
+def _untagged_followups_section_blocks(mode: UntaggedFollowupMode) -> list[dict]:
+ return _untagged_mode_section_blocks(
+ mode,
+ action_id=ACTION_SET_UNTAGGED_FOLLOWUP_MODE,
+ labels=UNTAGGED_FOLLOWUP_MODE_LABELS,
+ title="💬 Thread follow-ups",
+ subtitle="What I do with replies in a thread you started, when nobody tags @PostHog.",
+ note="Applies to every reply in those threads, yours included.",
+ )
+
+
+def _untagged_questions_section_blocks(mode: UntaggedFollowupMode) -> list[dict]:
+ return _untagged_mode_section_blocks(
+ mode,
+ action_id=ACTION_SET_UNTAGGED_QUESTION_MODE,
+ labels=UNTAGGED_QUESTION_MODE_LABELS,
+ title="❓ Questions in channels",
+ subtitle=(
+ "What I do when you post a question in a channel without tagging @PostHog, "
+ "and your PostHog data can answer it."
+ ),
+ note="Only your own questions. Thread follow-ups keep the setting above.",
+ )
+
+
CHANNEL_WELCOME_MODE_LABELS: dict[str, str] = {
ChannelWelcomeMode.CHANNEL: "Post it in the channel",
ChannelWelcomeMode.INVITER: "Show it only to the person who added me",
@@ -1696,7 +1740,14 @@ def handle_ai_preferences_block_action(payload: dict, action: dict) -> HttpRespo
return HttpResponse(status=200)
if action_id == ACTION_SET_UNTAGGED_FOLLOWUP_MODE:
- _apply_untagged_followup_mode_pick(integration, slack_user_id, action)
+ if (mode := _picked_untagged_mode(action)) is not None:
+ set_untagged_followup_mode(integration.integration_id, slack_user_id, mode)
+ republish()
+ return HttpResponse(status=200)
+
+ if action_id == ACTION_SET_UNTAGGED_QUESTION_MODE:
+ if (mode := _picked_untagged_mode(action)) is not None and is_slack_app_unprompted_answers_enabled(integration):
+ set_untagged_question_mode(integration.integration_id, slack_user_id, mode)
republish()
return HttpResponse(status=200)
@@ -2070,17 +2121,11 @@ def _reset_personal_preferences(integration: Integration, slack_user_id: str) ->
)
-def _apply_untagged_followup_mode_pick(integration: Integration, slack_user_id: str, action: dict) -> None:
- """Persist the picked mode. An unrecognised value is ignored rather than stored."""
+def _picked_untagged_mode(action: dict) -> UntaggedFollowupMode | None:
+ """The mode a picker click carries. An unrecognised value is ignored rather than stored."""
picked = (action.get("selected_option") or {}).get("value")
- if picked not in UntaggedFollowupMode.values:
- return
- SlackSettings.objects.update_or_create(
- slack_workspace_id=integration.integration_id,
- slack_user_id=slack_user_id,
- defaults={"untagged_followup_mode": picked},
- )
+ return UntaggedFollowupMode(picked) if picked in UntaggedFollowupMode.values else None
def _clear_project_personal(integration: Integration, slack_user_id: str) -> None:
@@ -2092,7 +2137,7 @@ def _clear_project_personal(integration: Integration, slack_user_id: str) -> Non
).first()
if row is None:
return
- if not row.untagged_followup_mode and not row.auto_model_choice:
+ if not row.untagged_followup_mode and not row.untagged_question_mode and not row.auto_model_choice:
row.delete()
return
row.default_integration = None
@@ -2153,6 +2198,11 @@ def _build_home_view(
tasks_state=tasks_state,
stats_state=stats_state,
untagged_followup_mode=resolve_untagged_followup_mode(integration, slack_user_id),
+ untagged_question_mode=(
+ resolve_untagged_question_mode(integration.integration_id, slack_user_id)
+ if is_slack_app_unprompted_answers_enabled(integration)
+ else None
+ ),
channel_welcome_mode=resolve_channel_welcome_mode(integration.integration_id) if is_admin else None,
auto_model_choice=(
resolve_auto_model_choice(integration.integration_id, slack_user_id)
diff --git a/products/slack_app/backend/services/slack_conversations.py b/products/slack_app/backend/services/slack_conversations.py
index 04a10eaf8fb1..fd5bc4cad16d 100644
--- a/products/slack_app/backend/services/slack_conversations.py
+++ b/products/slack_app/backend/services/slack_conversations.py
@@ -100,10 +100,19 @@ def channel_is_ext_shared(slack: SlackIntegration, channel: str) -> bool:
unreadable channel is treated as externally shared, which at worst asks the user
to approve a channel that did not need it.
"""
+ return channel_ext_shared_state(slack, channel) is not False
+
+
+def channel_ext_shared_state(slack: SlackIntegration, channel: str) -> bool | None:
+ """Like ``channel_is_ext_shared``, but ``None`` when Slack could not say.
+
+ For a caller that can wait and ask again, a failed lookup is not the same as a shared
+ channel.
+ """
try:
info = slack.client.conversations_info(channel=channel)
conversation: dict[str, Any] = info.get("channel") or {}
return bool(conversation.get("is_ext_shared") or conversation.get("is_pending_ext_shared"))
except Exception:
logger.warning("slack_app_channel_info_failed", channel=channel)
- return True
+ return None
diff --git a/products/slack_app/backend/services/slack_settings.py b/products/slack_app/backend/services/slack_settings.py
index 7b6cdc4bdf82..d15b37157be9 100644
--- a/products/slack_app/backend/services/slack_settings.py
+++ b/products/slack_app/backend/services/slack_settings.py
@@ -41,21 +41,32 @@ def resolve_untagged_followup_mode(integration: Integration, slack_user_id: str
to `ASK`, so the replier chooses whether to send the message. An unknown
value resolves to `NEVER`.
"""
+ return _resolve_untagged_mode(integration.integration_id, slack_user_id, "untagged_followup_mode")
+
+def resolve_untagged_question_mode(slack_workspace_id: str, slack_user_id: str | None) -> UntaggedFollowupMode:
+ """Resolve what PostHog does with this Slack user's top-level channel questions that do not tag it.
+
+ Read from the author's own row, so one person's choice never changes what happens
+ to anyone else's messages. An absent row or empty column resolves to `ASK`: PostHog
+ offers privately and answers only after the author confirms. An unknown value
+ resolves to `NEVER`.
+ """
+ return _resolve_untagged_mode(slack_workspace_id, slack_user_id, "untagged_question_mode")
+
+
+def _resolve_untagged_mode(slack_workspace_id: str, slack_user_id: str | None, column: str) -> UntaggedFollowupMode:
if not slack_user_id:
return UntaggedFollowupMode.NEVER
from products.slack_app.backend.models import SlackSettings
row = (
- SlackSettings.objects.filter(
- slack_workspace_id=integration.integration_id,
- slack_user_id=slack_user_id,
- )
- .values("untagged_followup_mode")
+ SlackSettings.objects.filter(slack_workspace_id=slack_workspace_id, slack_user_id=slack_user_id)
+ .values(column)
.first()
)
- stored = row["untagged_followup_mode"] if row else None
+ stored = row[column] if row else None
if stored is None:
return UntaggedFollowupMode.ASK
if stored in UntaggedFollowupMode.values:
@@ -63,6 +74,24 @@ def resolve_untagged_followup_mode(integration: Integration, slack_user_id: str
return UntaggedFollowupMode.NEVER
+def set_untagged_followup_mode(slack_workspace_id: str, slack_user_id: str, mode: UntaggedFollowupMode) -> None:
+ _set_untagged_mode(slack_workspace_id, slack_user_id, "untagged_followup_mode", mode)
+
+
+def set_untagged_question_mode(slack_workspace_id: str, slack_user_id: str, mode: UntaggedFollowupMode) -> None:
+ _set_untagged_mode(slack_workspace_id, slack_user_id, "untagged_question_mode", mode)
+
+
+def _set_untagged_mode(slack_workspace_id: str, slack_user_id: str, column: str, mode: UntaggedFollowupMode) -> None:
+ from products.slack_app.backend.models import SlackSettings
+
+ SlackSettings.objects.update_or_create(
+ slack_workspace_id=slack_workspace_id,
+ slack_user_id=slack_user_id,
+ defaults={column: mode.value},
+ )
+
+
def resolve_channel_welcome_mode(slack_workspace_id: str) -> ChannelWelcomeMode:
"""Resolve where the greeting goes when someone adds the app to a channel.
@@ -124,7 +153,10 @@ __all__ = [
"AIPreferences",
"resolve_auto_model_choice",
"resolve_channel_welcome_mode",
+ "resolve_untagged_question_mode",
"set_auto_model_choice",
+ "set_untagged_followup_mode",
+ "set_untagged_question_mode",
"set_channel_welcome_mode",
"resolve_untagged_followup_mode",
]
diff --git a/products/slack_app/backend/temporal/__init__.py b/products/slack_app/backend/temporal/__init__.py
index 2fbe876c7417..a1f8072285d9 100644
--- a/products/slack_app/backend/temporal/__init__.py
+++ b/products/slack_app/backend/temporal/__init__.py
@@ -17,6 +17,7 @@ from products.slack_app.backend.temporal.activities import (
classify_slack_app_model_router_activity,
classify_slack_app_project_route_activity,
classify_untagged_followup_activity,
+ classify_untagged_question_activity,
collect_posthog_code_thread_messages_activity,
create_posthog_code_routing_rule_activity,
create_posthog_code_task_for_repo_activity,
@@ -36,6 +37,7 @@ from products.slack_app.backend.temporal.activities import (
prepare_slack_unfurl_charts_activity,
process_slack_app_fork_thread_activity,
request_untagged_followup_confirmation_activity,
+ request_untagged_question_confirmation_activity,
resolve_posthog_code_slack_command_user_activity,
run_posthog_slack_inbox_onboarding_activity,
)
@@ -61,6 +63,8 @@ SLACK_APP_ACTIVITIES = [
process_slack_app_fork_thread_activity,
classify_untagged_followup_activity,
request_untagged_followup_confirmation_activity,
+ classify_untagged_question_activity,
+ request_untagged_question_confirmation_activity,
resolve_posthog_code_slack_command_user_activity,
collect_posthog_code_thread_messages_activity,
cascade_posthog_code_repository_activity,
@@ -114,6 +118,7 @@ __all__ = [
"classify_posthog_code_task_needs_repo_activity",
"classify_slack_app_model_override_activity",
"classify_slack_app_project_route_activity",
+ "classify_untagged_question_activity",
"classify_untagged_followup_activity",
"collect_posthog_code_thread_messages_activity",
"create_posthog_code_routing_rule_activity",
@@ -130,6 +135,7 @@ __all__ = [
"post_posthog_code_no_repos_activity",
"post_posthog_code_picker_timeout_activity",
"post_posthog_code_repo_picker_activity",
+ "request_untagged_question_confirmation_activity",
"request_untagged_followup_confirmation_activity",
"resolve_posthog_code_slack_command_user_activity",
"classify_slack_app_model_router_activity",
diff --git a/products/slack_app/backend/temporal/activities/__init__.py b/products/slack_app/backend/temporal/activities/__init__.py
index b4d73a0a3378..3d4a5cd6de8a 100644
--- a/products/slack_app/backend/temporal/activities/__init__.py
+++ b/products/slack_app/backend/temporal/activities/__init__.py
@@ -47,6 +47,10 @@ from products.slack_app.backend.temporal.activities.unfurl_charts import (
post_slack_unfurl_charts_activity,
prepare_slack_unfurl_charts_activity,
)
+from products.slack_app.backend.temporal.activities.untagged_question import (
+ classify_untagged_question_activity,
+ request_untagged_question_confirmation_activity,
+)
from products.slack_app.backend.temporal.activities.user_resolution import (
resolve_posthog_code_slack_command_user_activity,
)
@@ -64,7 +68,9 @@ __all__ = [
"classify_slack_app_project_route",
"classify_slack_app_project_route_activity",
"classify_task_needs_repo",
+ "classify_untagged_question_activity",
"classify_untagged_followup_activity",
+ "request_untagged_question_confirmation_activity",
"collect_posthog_code_thread_messages_activity",
"create_posthog_code_routing_rule_activity",
"create_posthog_code_task_for_repo_activity",
diff --git a/products/slack_app/backend/temporal/activities/billing.py b/products/slack_app/backend/temporal/activities/billing.py
index 07116a24e85c..94f52d6e32f4 100644
--- a/products/slack_app/backend/temporal/activities/billing.py
+++ b/products/slack_app/backend/temporal/activities/billing.py
@@ -32,13 +32,13 @@ def enforce_posthog_code_billing_quota_activity(
integration_id=inputs.slack_team_id,
)
slack = SlackIntegration(integration)
- # An untagged reply has asked PostHog for nothing yet, because both the classifier
- # and the confirmation prompt run after this gate. The denial goes into the thread
- # where everyone in the channel reads it, so an over-quota team would collect one
- # under every reply people write in a thread PostHog owns, chitchat included. Stop
- # the run without the message and keep the message for a mention or a confirmed
- # reply, which are the turns that asked for work.
- unrequested = inputs.untagged_followup and not inputs.untagged_followup_confirmed
+ # An untagged reply or an untagged channel question has asked PostHog for nothing
+ # yet, because both the classifier and the confirmation prompt run after this gate.
+ # The denial goes into the thread where everyone in the channel reads it, so an
+ # over-quota team would collect one under every reply people write in a thread
+ # PostHog owns, chitchat included. Stop the run without the message and keep the
+ # message for a mention or a confirmed reply, which are the turns that asked for work.
+ unrequested = (inputs.untagged_followup or inputs.untagged_question) and not inputs.untagged_confirmed
blocked = block_if_team_over_quota(
integration=integration,
slack=slack,
@@ -50,7 +50,7 @@ def enforce_posthog_code_billing_quota_activity(
)
if blocked and unrequested:
logger.info(
- "slack_app_untagged_followup_dropped_over_quota",
+ "slack_app_unaddressed_message_dropped_over_quota",
team_id=integration.team_id,
channel=channel,
thread_ts=thread_ts,
diff --git a/products/slack_app/backend/temporal/activities/task_creation.py b/products/slack_app/backend/temporal/activities/task_creation.py
index 8796290d3d97..54254d820005 100644
--- a/products/slack_app/backend/temporal/activities/task_creation.py
+++ b/products/slack_app/backend/temporal/activities/task_creation.py
@@ -550,7 +550,7 @@ def derive_mention_workflow_id(inputs: PostHogCodeSlackMentionWorkflowInputs) ->
suffix = inputs.slack_event_id
else:
suffix = f"{event.get('channel', '')}:{event.get('ts', '')}"
- if inputs.untagged_followup_confirmed:
+ if inputs.untagged_confirmed:
suffix = f"{suffix}:confirmed"
return f"{MENTION_WORKFLOW_ID_PREFIX}{inputs.slack_team_id}:{suffix}"
diff --git a/products/slack_app/backend/temporal/activities/untagged_question.py b/products/slack_app/backend/temporal/activities/untagged_question.py
new file mode 100644
index 000000000000..8cfab392012e
--- /dev/null
+++ b/products/slack_app/backend/temporal/activities/untagged_question.py
@@ -0,0 +1,265 @@
+"""Answering a top-level Slack channel message that did not tag the app.
+
+Nobody asked PostHog anything, so the bar is high on purpose: a wrong answer interrupts a
+conversation in public. Jev, a System One decision model, answers two yes/no questions about
+the message, and both must clear a strict threshold. A missed question costs the author one
+@PostHog, which they can still type.
+"""
+
+import structlog
+from temporalio import activity
+
+from posthog.dataclasses import frozen
+from posthog.llm.managed_decision_model import DEFAULT_DECISION_MODEL
+from posthog.llm.system_one import NoulAnswer, NoulQuestion, SystemOneNotConfigured
+from posthog.llm.system_one_client import build_system_one_client
+from posthog.models.integration import Integration, SlackIntegration
+from posthog.models.user import User
+from posthog.temporal.common.utils import close_db_connections
+
+from products.slack_app.backend.analytics import capture_slack_event
+from products.slack_app.backend.models import UntaggedFollowupMode
+from products.slack_app.backend.services.slack_conversations import channel_is_ext_shared
+from products.slack_app.backend.services.slack_settings import resolve_untagged_question_mode
+from products.slack_app.backend.temporal.activities.classifiers import CLASSIFIER_PROPERTY, _thread_trace_id
+from products.slack_app.backend.temporal.types import (
+ PostHogCodeSlackMentionWorkflowInputs,
+ coerce_mention_workflow_inputs,
+)
+
+logger = structlog.get_logger(__name__)
+
+UNTAGGED_QUESTION_DECISION_MODEL = DEFAULT_DECISION_MODEL
+# Not the `slack_app` product the other classifiers bill under: this call runs on messages
+# nobody sent to PostHog, so it must not spend the team's AI credits.
+UNTAGGED_QUESTION_AI_PRODUCT = "slack_app_untagged_question"
+# On the path to the first reply. A miss only means no answer, so a slow call is not worth waiting for.
+UNTAGGED_QUESTION_TIMEOUT_SECONDS = 5.0
+# Both judgments must clear this. Tuned for precision over recall, see
+# products/slack_app/evals/eval_untagged_question.py.
+UNTAGGED_QUESTION_MIN_PROBABILITY = 0.85
+
+ASKS_QUESTION_ID = "asks_for_information"
+ANSWERABLE_QUESTION_ID = "answerable_by_posthog"
+
+ASKS_FOR_INFORMATION_INSTRUCTIONS = (
+ "`message` is a new top-level post in a team's Slack channel. Nobody tagged an assistant in it. "
+ "Is the author asking for a specific piece of information or a specific analysis that they do not have yet?"
+)
+ASKS_FOR_INFORMATION_TRUE = (
+ "A real question or request for facts, numbers, an explanation, or a lookup. "
+ "Examples: how many users signed up last week, why did checkout conversion drop yesterday, "
+ "which feature flag controls the new onboarding, are there errors on the billing page since the deploy. "
+ "A yes or no question about whether a metric went up, went down or changed asks for that metric, "
+ "including when the metric is written as shorthand such as WAU or churn. "
+ "A request to find where something lives, such as which file sends a tracking call, is a lookup. "
+ "A question about why a number or an error count changed asks for an analysis, even when it names the "
+ "change it saw. The question can be in any language."
+)
+ASKS_FOR_INFORMATION_FALSE = (
+ "Anything else: an announcement, a status update, a statement or an opinion, a rhetorical question, "
+ "a social message, a joke, a greeting, a thank-you, a call to meet or talk, a request for a person to "
+ "do something, or a question only one named person can answer."
+)
+
+ANSWERABLE_INSTRUCTIONS = (
+ "PostHog is a product analytics platform. Its assistant can read the team's PostHog data: events, "
+ "insights, trends, funnels, retention, dashboards, session recordings, feature flags, experiments, "
+ "surveys, error tracking, logs, LLM traces and data warehouse tables. It can also search the PostHog "
+ "documentation and read the team's code repositories. "
+ "Could that assistant answer the question in `message` correctly and fully, with only those sources?"
+)
+ANSWERABLE_TRUE = (
+ "The answer is in product usage data, error or log data, feature flag or experiment state, "
+ "the team's code, or the PostHog documentation. Product metrics count as usage data when written "
+ "as shorthand: DAU, WAU, MAU, stickiness, churn, retention or conversion rate mean the team's own "
+ "numbers, which PostHog holds."
+)
+ANSWERABLE_FALSE = (
+ "The answer needs knowledge those sources do not hold: plans, opinions, decisions, people's availability, "
+ "private conversations, customer contracts, finance, HR, or systems PostHog cannot read. Also false when "
+ "the question is too vague to answer, or when it is not a question."
+)
+
+
+@frozen
+class UntaggedQuestionVerdict:
+ asks_for_information: float
+ answerable_by_posthog: float
+
+ @property
+ def answerable(self) -> bool:
+ return min(self.asks_for_information, self.answerable_by_posthog) >= UNTAGGED_QUESTION_MIN_PROBABILITY
+
+
+def classify_untagged_question(
+ event_text: str,
+ *,
+ team_id: int,
+ distinct_id: str | None,
+ trace_id: str | None = None,
+) -> UntaggedQuestionVerdict | None:
+ """Ask the decision model whether PostHog can answer this message. ``None`` means no verdict.
+
+ Raises when no System One server is configured or the call fails, so the caller can tell
+ an outage from a refusal.
+ """
+ text = event_text.strip()
+ if not text:
+ return None
+ # No TypeSafe fallback: the message is customer text, and TypeSafe is a third party.
+ client = build_system_one_client(
+ model=UNTAGGED_QUESTION_DECISION_MODEL,
+ ai_product=UNTAGGED_QUESTION_AI_PRODUCT,
+ team_id=team_id,
+ distinct_id=distinct_id,
+ trace_id=trace_id,
+ properties={CLASSIFIER_PROPERTY: "untagged_question"},
+ timeout=UNTAGGED_QUESTION_TIMEOUT_SECONDS,
+ )
+ result = client.decide(
+ state={"message": text},
+ questions={
+ ASKS_QUESTION_ID: NoulQuestion(
+ instructions=ASKS_FOR_INFORMATION_INSTRUCTIONS,
+ criteria_true=ASKS_FOR_INFORMATION_TRUE,
+ criteria_false=ASKS_FOR_INFORMATION_FALSE,
+ ),
+ ANSWERABLE_QUESTION_ID: NoulQuestion(
+ instructions=ANSWERABLE_INSTRUCTIONS,
+ criteria_true=ANSWERABLE_TRUE,
+ criteria_false=ANSWERABLE_FALSE,
+ ),
+ },
+ )
+ asks = result.answers.get(ASKS_QUESTION_ID)
+ answerable = result.answers.get(ANSWERABLE_QUESTION_ID)
+ if not isinstance(asks, NoulAnswer) or not isinstance(answerable, NoulAnswer):
+ logger.info("slack_app_untagged_question_refused")
+ return None
+ return UntaggedQuestionVerdict(asks_for_information=asks.probability, answerable_by_posthog=answerable.probability)
+
+
+def _load_integration(inputs: PostHogCodeSlackMentionWorkflowInputs) -> Integration:
+ return Integration.objects.select_related("team__organization").get(
+ id=inputs.integration_id,
+ kind="slack",
+ integration_id=inputs.slack_team_id,
+ )
+
+
+@close_db_connections
+def classify_untagged_question_activity(inputs: PostHogCodeSlackMentionWorkflowInputs) -> bool:
+ """Whether PostHog should answer, or offer to answer, a top-level message nobody tagged it in.
+
+ Any failure returns ``False``: staying quiet is the safe default when nobody asked.
+ """
+ inputs = coerce_mention_workflow_inputs(inputs)
+ integration = _load_integration(inputs)
+ user = User.objects.filter(id=inputs.user_id).first()
+ if user is None:
+ return False
+
+ verdict: UntaggedQuestionVerdict | None = None
+ try:
+ verdict = classify_untagged_question(
+ inputs.event.get("text") or "",
+ team_id=integration.team_id,
+ distinct_id=user.distinct_id,
+ trace_id=_thread_trace_id(inputs.slack_team_id, inputs.event.get("ts")),
+ )
+ except SystemOneNotConfigured:
+ logger.info("slack_app_untagged_question_system_one_not_configured")
+ except Exception:
+ logger.exception("slack_app_untagged_question_classifier_failed")
+
+ answerable = verdict is not None and verdict.answerable
+ logger.info(
+ "slack_app_untagged_question_classified",
+ slack_team_id=inputs.slack_team_id,
+ message_ts=inputs.event.get("ts"),
+ classified=verdict is not None,
+ answerable=answerable,
+ asks_for_information=verdict.asks_for_information if verdict else None,
+ answerable_by_posthog=verdict.answerable_by_posthog if verdict else None,
+ threshold=UNTAGGED_QUESTION_MIN_PROBABILITY,
+ )
+ # The message text stays out of this event: its author never addressed PostHog.
+ capture_slack_event(
+ integration,
+ "slack app unprompted question classified",
+ slack_user_id=inputs.event.get("user"),
+ posthog_user=user,
+ classified=verdict is not None,
+ answerable=answerable,
+ asks_for_information=verdict.asks_for_information if verdict else None,
+ answerable_by_posthog=verdict.answerable_by_posthog if verdict else None,
+ threshold=UNTAGGED_QUESTION_MIN_PROBABILITY,
+ )
+ if not answerable:
+ return False
+
+ from products.slack_app.backend.api import (
+ claim_message_handled, # noqa: PLC0415 — keeps the webhook module off the worker import path
+ message_handled_by,
+ )
+
+ # An edit of this message that adds a tag starts its own run. Whichever path claims
+ # the message first runs, and the other stops. The run id keeps the claim through a
+ # retry of this activity.
+ claimant = f"untagged_question:{activity.info().workflow_run_id}"
+ if (
+ not claim_message_handled(inputs.slack_team_id, inputs.event, claimant)
+ and message_handled_by(inputs.slack_team_id, inputs.event) != claimant
+ ):
+ logger.info("slack_app_untagged_question_already_handled", slack_team_id=inputs.slack_team_id)
+ return False
+ return True
+
+
+@close_db_connections
+def request_untagged_question_confirmation_activity(inputs: PostHogCodeSlackMentionWorkflowInputs) -> bool:
+ """Apply the author's untagged-message mode to a question the classifier passed.
+
+ Returns ``True`` when the run must stop here: the private offer now waits for the
+ author, or the author turned untagged pickups off while this run was in flight.
+ """
+ from products.slack_app.backend.api import (
+ _post_untagged_question_prompt, # noqa: PLC0415 — keeps the webhook module off the worker import path
+ )
+
+ inputs = coerce_mention_workflow_inputs(inputs)
+ slack_user_id = inputs.event.get("user")
+ mode = resolve_untagged_question_mode(inputs.slack_team_id, slack_user_id)
+ logger.info(
+ "slack_app_untagged_question_mode_resolved",
+ slack_team_id=inputs.slack_team_id,
+ message_ts=inputs.event.get("ts"),
+ mode=mode,
+ )
+ if mode == UntaggedFollowupMode.NEVER:
+ logger.info("slack_app_untagged_questions_switched_off_mid_run", slack_team_id=inputs.slack_team_id)
+ return True
+
+ integration = _load_integration(inputs)
+ if mode == UntaggedFollowupMode.AUTO:
+ # Nobody clicks before an automatic answer, so the channel's sharing is read again here.
+ # The webhook checked the event envelope, which can be stale by the time a worker runs.
+ if channel_is_ext_shared(SlackIntegration(integration), str(inputs.event.get("channel") or "")):
+ logger.info("slack_app_untagged_question_channel_now_shared", slack_team_id=inputs.slack_team_id)
+ return True
+ return False
+
+ prompted = _post_untagged_question_prompt(SlackIntegration(integration), integration, inputs.event)
+ if not prompted:
+ # Still held back: answering without the offer the author's setting asks for would break it.
+ logger.warning("slack_app_untagged_question_prompt_not_delivered", integration_id=integration.id)
+ else:
+ logger.info(
+ "slack_app_untagged_question_prompt_posted",
+ integration_id=integration.id,
+ message_ts=inputs.event.get("ts"),
+ )
+ return True
diff --git a/products/slack_app/backend/temporal/slack_app_mention.py b/products/slack_app/backend/temporal/slack_app_mention.py
index 10a33b88be39..afbde9ee9c78 100644
--- a/products/slack_app/backend/temporal/slack_app_mention.py
+++ b/products/slack_app/backend/temporal/slack_app_mention.py
@@ -122,12 +122,12 @@ class SlackAppMentionWorkflow(PostHogWorkflow):
async def _react(self, message: PostHogCodeSlackMentionWorkflowInputs, reaction_activity: Any) -> None:
"""Run one of the reaction activities for a mention or DM. Untagged
- thread followups get no reactions — they were never addressed to the
- bot, and most are dropped by the chitchat classifier.
+ thread followups and untagged questions get no reactions,
+ because nobody addressed them to the bot and a classifier drops most of them.
"""
channel = message.event.get("channel")
message_ts = message.event.get("ts")
- if message.untagged_followup or not channel or not message_ts:
+ if message.untagged_followup or message.untagged_question or not channel or not message_ts:
return
try:
await workflow.execute_activity(
@@ -157,7 +157,7 @@ class SlackAppMentionWorkflow(PostHogWorkflow):
"""
channel = message.event.get("channel")
thread_ts = message.event.get("thread_ts") or message.event.get("ts")
- if message.untagged_followup or not channel or not thread_ts:
+ if message.untagged_followup or message.untagged_question or not channel or not thread_ts:
return
try:
await workflow.execute_activity(
diff --git a/products/slack_app/backend/temporal/slack_app_mention_processing.py b/products/slack_app/backend/temporal/slack_app_mention_processing.py
index 21941eabfbf7..74b239982489 100644
--- a/products/slack_app/backend/temporal/slack_app_mention_processing.py
+++ b/products/slack_app/backend/temporal/slack_app_mention_processing.py
@@ -25,6 +25,7 @@ from products.slack_app.backend.temporal import (
classify_slack_app_model_router_activity,
classify_slack_app_project_route_activity,
classify_untagged_followup_activity,
+ classify_untagged_question_activity,
collect_posthog_code_thread_messages_activity,
create_posthog_code_task_for_repo_activity,
discover_posthog_code_repository_via_agent_activity,
@@ -35,6 +36,7 @@ from products.slack_app.backend.temporal import (
post_posthog_code_repo_picker_activity,
posthog_code_thread_has_task_activity,
request_untagged_followup_confirmation_activity,
+ request_untagged_question_confirmation_activity,
)
if TYPE_CHECKING:
@@ -87,6 +89,9 @@ class SlackAppMentionProcessingWorkflow(PostHogWorkflow):
if not channel or not thread_ts or not slack_user_id:
return
+ # An untagged question stays silent, error replies included, until its author's
+ # mode lets the run answer. Old histories carry no such input, so they never skip.
+ silent_on_error = inputs.untagged_question and not inputs.untagged_question_confirmed
try:
# Gate every workflow entry on the team's AI-credits quota before any
# other activity runs. Webhook-level short-circuit catches the common
@@ -104,6 +109,18 @@ class SlackAppMentionProcessingWorkflow(PostHogWorkflow):
if blocked:
return
+ # Both gates run before anything visible, because nobody tagged us in this message.
+ if silent_on_error:
+ answerable = await _execute_posthog_code_activity(classify_untagged_question_activity, inputs)
+ if not answerable:
+ return
+ awaiting_confirmation = await _execute_posthog_code_activity(
+ request_untagged_question_confirmation_activity, inputs
+ )
+ if awaiting_confirmation:
+ return
+ silent_on_error = False
+
workflow.deprecate_patch(_PATCH_ID_PARALLEL_PREWORK)
prework = await _prework_in_parallel(inputs, channel, thread_ts, slack_user_id)
if prework is None:
@@ -236,6 +253,8 @@ class SlackAppMentionProcessingWorkflow(PostHogWorkflow):
"error_type": type(exc).__name__,
},
)
+ if silent_on_error:
+ return
await _execute_posthog_code_activity(
post_posthog_code_internal_error_activity,
inputs,
diff --git a/products/slack_app/backend/temporal/tests/activities/test_billing_quota_activity.py b/products/slack_app/backend/temporal/tests/activities/test_billing_quota_activity.py
index a91ce5ee1365..fa1191a22426 100644
--- a/products/slack_app/backend/temporal/tests/activities/test_billing_quota_activity.py
+++ b/products/slack_app/backend/temporal/tests/activities/test_billing_quota_activity.py
@@ -22,7 +22,14 @@ class TestEnforcePostHogCodeBillingQuotaActivity(TestCase):
self.user = User.objects.create(email="[email protected]")
self.integration = Integration.objects.create(team=self.team, kind="slack", integration_id="T_SLACK", config={})
- def _call(self, *, untagged_followup: bool, confirmed: bool) -> bool:
+ def _call(
+ self,
+ *,
+ untagged_followup: bool,
+ confirmed: bool,
+ untagged_question: bool = False,
+ untagged_question_confirmed: bool = False,
+ ) -> bool:
inputs = PostHogCodeSlackMentionWorkflowInputs(
event={},
integration_id=self.integration.id,
@@ -30,6 +37,8 @@ class TestEnforcePostHogCodeBillingQuotaActivity(TestCase):
user_id=self.user.id,
untagged_followup=untagged_followup,
untagged_followup_confirmed=confirmed,
+ untagged_question=untagged_question,
+ untagged_question_confirmed=untagged_question_confirmed,
)
return enforce_posthog_code_billing_quota_activity(inputs, "C001", "1000.0000", "U_BOB")
@@ -38,19 +47,28 @@ class TestEnforcePostHogCodeBillingQuotaActivity(TestCase):
# The gate runs before the classifier and before the prompt, so an untagged
# reply has asked for nothing yet. A denial under it would land in the thread
# for every reply people write, chitchat included.
- ("unconfirmed_reply", True, False, False),
- ("confirmed_reply", True, True, True),
- ("mention", False, False, True),
+ ("unconfirmed_reply", True, False, False, False, False),
+ ("confirmed_reply", True, True, False, False, True),
+ ("mention", False, False, False, False, True),
+ # A channel question nobody tagged us in is the same: a denial under it would
+ # announce in public that PostHog read a message nobody sent it.
+ ("unconfirmed_untagged_question", False, False, True, False, False),
+ ("confirmed_untagged_question", False, False, True, True, True),
]
)
def test_denial_reaches_the_thread_only_for_a_turn_that_asked_for_work(
- self, _name, untagged_followup, confirmed, expect_denial
+ self, _name, untagged_followup, confirmed, untagged_question, untagged_question_confirmed, expect_denial
):
with (
patch("ee.billing.quota_limiting.is_team_limited", return_value=True),
patch("products.slack_app.backend.api.post_quota_exhausted_denial") as mock_denial,
):
- blocked = self._call(untagged_followup=untagged_followup, confirmed=confirmed)
+ blocked = self._call(
+ untagged_followup=untagged_followup,
+ confirmed=confirmed,
+ untagged_question=untagged_question,
+ untagged_question_confirmed=untagged_question_confirmed,
+ )
assert blocked is True
assert mock_denial.called is expect_denial
diff --git a/products/slack_app/backend/temporal/tests/activities/test_untagged_question.py b/products/slack_app/backend/temporal/tests/activities/test_untagged_question.py
new file mode 100644
index 000000000000..2413789012b1
--- /dev/null
+++ b/products/slack_app/backend/temporal/tests/activities/test_untagged_question.py
@@ -0,0 +1,123 @@
+from typing import Any
+
+from unittest.mock import MagicMock, patch
+
+from django.core.cache import cache
+from django.test import SimpleTestCase, TestCase
+
+from parameterized import parameterized
+
+from posthog.llm.system_one import NoulAnswer, RefusalAnswer, SystemOneResult
+from posthog.models.integration import Integration
+from posthog.models.organization import Organization
+from posthog.models.team.team import Team
+from posthog.models.user import User
+
+from products.slack_app.backend.api import claim_message_handled
+from products.slack_app.backend.models import SlackSettings, UntaggedFollowupMode
+from products.slack_app.backend.temporal.activities.untagged_question import (
+ ANSWERABLE_QUESTION_ID,
+ ASKS_QUESTION_ID,
+ UntaggedQuestionVerdict,
+ classify_untagged_question,
+ classify_untagged_question_activity,
+ request_untagged_question_confirmation_activity,
+)
+from products.slack_app.backend.temporal.types import PostHogCodeSlackMentionWorkflowInputs
+
+MODULE = "products.slack_app.backend.temporal.activities.untagged_question"
+
+
+def _result(asks: Any, answerable: Any) -> SystemOneResult:
+ return SystemOneResult(
+ model="jev", answers={ASKS_QUESTION_ID: asks, ANSWERABLE_QUESTION_ID: answerable}, input_tokens=None
+ )
+
+
+class TestClassifyUntaggedQuestion(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("both confident", NoulAnswer(probability=0.97), NoulAnswer(probability=0.91), True),
+ ("a question PostHog cannot answer", NoulAnswer(probability=0.97), NoulAnswer(probability=0.6), False),
+ ("answerable but not a question", NoulAnswer(probability=0.5), NoulAnswer(probability=0.95), False),
+ ("a refusal", NoulAnswer(probability=0.99), RefusalAnswer(), None),
+ ]
+ )
+ def test_both_judgments_must_clear_the_threshold(self, _name, asks, answerable, expected):
+ client = MagicMock()
+ client.decide.return_value = _result(asks, answerable)
+ with patch(f"{MODULE}.build_system_one_client", return_value=client):
+ verdict = classify_untagged_question(
+ "How many people signed up last week?", team_id=1, distinct_id="user-1"
+ )
+
+ assert (verdict.answerable if verdict else None) == expected
+ # The message travels as state, never inside the instructions.
+ state = client.decide.call_args.kwargs["state"]
+ assert state == {"message": "How many people signed up last week?"}
+
+
+class TestRequestUntaggedQuestionConfirmation(TestCase):
+ def setUp(self):
+ cache.clear()
+ organization = Organization.objects.create(name="Org")
+ team = Team.objects.create(organization=organization, name="Team")
+ self.user = User.objects.create(email="[email protected]", distinct_id="user-1")
+ self.integration = Integration.objects.create(
+ team=team, kind="slack", integration_id="T_WS", sensitive_config={"access_token": "xoxb"}
+ )
+ self.inputs = PostHogCodeSlackMentionWorkflowInputs(
+ event={"type": "message", "channel": "C1", "user": "U_ALICE", "ts": "1.0", "text": "how many signups?"},
+ integration_id=self.integration.id,
+ slack_team_id="T_WS",
+ user_id=self.user.id,
+ untagged_question=True,
+ )
+
+ @parameterized.expand(
+ [
+ ("auto answers", UntaggedFollowupMode.AUTO, False, False),
+ ("never stays quiet", UntaggedFollowupMode.NEVER, True, False),
+ ("ask offers privately", UntaggedFollowupMode.ASK, True, True),
+ ("unset offers privately", None, True, True),
+ ]
+ )
+ def test_the_authors_mode_decides(self, _name, user_mode, expect_stop, expect_prompt):
+ if user_mode is not None:
+ SlackSettings.objects.create(
+ slack_workspace_id="T_WS", slack_user_id="U_ALICE", untagged_question_mode=user_mode
+ )
+ with (
+ patch("products.slack_app.backend.api._post_untagged_question_prompt", return_value=True) as mock_prompt,
+ patch(f"{MODULE}.channel_is_ext_shared", return_value=False),
+ ):
+ stop = request_untagged_question_confirmation_activity(self.inputs)
+
+ assert stop is expect_stop
+ assert mock_prompt.called is expect_prompt
+
+ def test_auto_stays_quiet_in_a_channel_shared_since_the_webhook(self):
+ SlackSettings.objects.create(
+ slack_workspace_id="T_WS", slack_user_id="U_ALICE", untagged_question_mode=UntaggedFollowupMode.AUTO
+ )
+ with patch(f"{MODULE}.channel_is_ext_shared", return_value=True):
+ assert request_untagged_question_confirmation_activity(self.inputs) is True
+
+ @parameterized.expand(
+ [
+ ("first to claim", None, True),
+ ("an edited mention claimed it first", "edited_mention", False),
+ # A retry after a crash between the claim and its recorded result keeps the message.
+ ("its own earlier attempt claimed it", "untagged_question:run-1", True),
+ ]
+ )
+ def test_an_answerable_question_runs_only_when_it_claims_the_message(self, _name, claimed_by, expected):
+ if claimed_by is not None:
+ claim_message_handled("T_WS", self.inputs.event, claimed_by)
+ verdict = UntaggedQuestionVerdict(asks_for_information=0.99, answerable_by_posthog=0.99)
+ info = MagicMock(workflow_run_id="run-1")
+ with (
+ patch(f"{MODULE}.classify_untagged_question", return_value=verdict),
+ patch(f"{MODULE}.activity.info", return_value=info),
+ ):
+ assert classify_untagged_question_activity(self.inputs) is expected
diff --git a/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py b/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py
index 855f620619f9..0ba6bd1e0b19 100644
--- a/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py
+++ b/products/slack_app/backend/temporal/tests/test_slack_app_mention_workflow.py
@@ -8,6 +8,7 @@ import pytest
from temporalio import activity
from temporalio.common import WorkflowIDConflictPolicy, WorkflowIDReusePolicy
+from temporalio.exceptions import ApplicationError
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
@@ -34,15 +35,19 @@ def _message(
*,
event_id: str | None = None,
untagged: bool = False,
+ untagged_question: bool = False,
text: str = "fix the bug",
) -> PostHogCodeSlackMentionWorkflowInputs:
+ # A top-level post is its own thread root.
+ thread_ts = ts if untagged_question else "100.0"
return PostHogCodeSlackMentionWorkflowInputs(
- event={"channel": "C1", "ts": ts, "thread_ts": "100.0", "user": "U1", "text": text},
+ event={"channel": "C1", "ts": ts, "thread_ts": thread_ts, "user": "U1", "text": text},
integration_id=1,
slack_team_id="T1",
slack_event_id=event_id,
user_id=42,
untagged_followup=untagged,
+ untagged_question=untagged_question,
)
@@ -100,6 +105,9 @@ class _Recorder:
self.awaiting_confirmation = False
# True makes discovery report that the team is out of AI credits.
self.discover_usage_limited = False
+ self.untagged_question_verdict: bool | Literal["raise"] = True
+ # True holds an untagged question back on a private offer.
+ self.untagged_question_awaiting_confirmation = False
def _fake_activities(rec: _Recorder) -> list:
@@ -128,6 +136,16 @@ def _fake_activities(rec: _Recorder) -> list:
) -> bool:
return rec.awaiting_confirmation
+ @activity.defn(name="classify_untagged_question_activity")
+ async def classify_untagged_question(inputs: PostHogCodeSlackMentionWorkflowInputs) -> bool:
+ if rec.untagged_question_verdict == "raise":
+ raise ApplicationError("classifier down", non_retryable=True)
+ return rec.untagged_question_verdict
+
+ @activity.defn(name="request_untagged_question_confirmation_activity")
+ async def request_untagged_question_confirmation(inputs: PostHogCodeSlackMentionWorkflowInputs) -> bool:
+ return rec.untagged_question_awaiting_confirmation
+
@activity.defn(name="forward_posthog_code_followup_activity")
async def forward(
inputs: PostHogCodeSlackMentionWorkflowInputs,
@@ -288,6 +306,8 @@ def _fake_activities(rec: _Recorder) -> list:
classify_followup,
request_confirmation,
has_task,
+ classify_untagged_question,
+ request_untagged_question_confirmation,
forward,
collect,
cascade,
@@ -676,3 +696,33 @@ async def test_deleted_trigger_message_creates_no_task_and_says_nothing():
assert rec.created == []
assert rec.internal_errors == []
+
+
+ "verdict,awaiting_confirmation,expect_created",
+ [
+ # Nobody asked, so a failure says nothing in the channel either.
+ ("raise", False, False),
+ (False, False, False),
+ (True, True, False),
+ (True, False, True),
+ ],
+)
+async def test_untagged_question_stays_silent_until_its_mode_lets_it_answer(
+ verdict, awaiting_confirmation, expect_created
+):
+ rec = _Recorder()
+ rec.untagged_question_verdict = verdict
+ rec.untagged_question_awaiting_confirmation = awaiting_confirmation
+
+ async with _Harness(rec) as h:
+ handle = await _signal_with_start(
+ h.env, h.task_queue, f"wf-{uuid.uuid4()}", _message("1.1", untagged_question=True)
+ )
+ await asyncio.wait_for(handle.result(), timeout=30)
+
+ assert rec.internal_errors == []
+ assert rec.queued_marked == []
+ assert rec.processing_marked == []
+ assert bool(rec.created) is expect_created
diff --git a/products/slack_app/backend/temporal/types.py b/products/slack_app/backend/temporal/types.py
index 57b47ab68f89..f1a6c6f1c48f 100644
--- a/products/slack_app/backend/temporal/types.py
+++ b/products/slack_app/backend/temporal/types.py
@@ -120,6 +120,13 @@ class PostHogCodeSlackMentionWorkflowInputs:
# ephemeral prompt. The classifier ran before the prompt was posted and the
# answer is in, so the run skips both on the way back through.
untagged_followup_confirmed: bool = False
+ # True when the workflow was started for a top-level channel message that did not
+ # tag the app. Nothing visible happens until the classifier says PostHog can answer
+ # it and the author's untagged-message mode allows it.
+ untagged_question: bool = False
+ # True when the author already accepted the private offer to answer, so the run
+ # skips the classifier and the offer on the way back through.
+ untagged_question_confirmed: bool = False
# Slack sets this on the event envelope for Slack Connect channels. It is
# threaded through to task run state so customer-facing Slack replies remain
# approval-gated even when a user's internal-write tier is full-auto.
@@ -139,6 +146,10 @@ class PostHogCodeSlackMentionWorkflowInputs:
# than the messages.
fork_source_task_id: str | None = None
+ @property
+ def untagged_confirmed(self) -> bool:
+ return self.untagged_followup_confirmed or self.untagged_question_confirmed
+
def coerce_mention_workflow_inputs(inputs: object) -> PostHogCodeSlackMentionWorkflowInputs:
"""Normalise an activity's ``inputs`` back into the dataclass.
diff --git a/products/slack_app/backend/tests/services/test_slack_app_home.py b/products/slack_app/backend/tests/services/test_slack_app_home.py
index e04736b858bc..fa6784d0de78 100644
--- a/products/slack_app/backend/tests/services/test_slack_app_home.py
+++ b/products/slack_app/backend/tests/services/test_slack_app_home.py
@@ -473,6 +473,7 @@ class TestRenderHomeView:
),
stats_state=StatsState(tasks_started=4, tasks_with_pr=2, tasks_merged=1, active_people=2),
untagged_followup_mode=UntaggedFollowupMode.AUTO,
+ untagged_question_mode=UntaggedFollowupMode.ASK,
channel_welcome_mode=ChannelWelcomeMode.CHANNEL,
auto_model_choice=auto_model_choice,
)
@@ -1377,8 +1378,12 @@ class TestResetPersonal:
class TestResetProjectPersonal:
@pytest.mark.parametrize(
"other_settings",
- [{"untagged_followup_mode": UntaggedFollowupMode.AUTO}, {"auto_model_choice": True}],
- ids=["followup_mode", "auto_model_choice"],
+ [
+ {"untagged_followup_mode": UntaggedFollowupMode.AUTO},
+ {"untagged_question_mode": UntaggedFollowupMode.NEVER},
+ {"auto_model_choice": True},
+ ],
+ ids=["followup_mode", "question_mode", "auto_model_choice"],
)
def test_clears_routing_only_when_other_settings_present(
self, slack_integration, mock_slack_client, flag_on, admin_user, other_settings
diff --git a/products/slack_app/backend/tests/services/test_slack_settings.py b/products/slack_app/backend/tests/services/test_slack_settings.py
index ad5e7cfb9f27..db4f2940443f 100644
--- a/products/slack_app/backend/tests/services/test_slack_settings.py
+++ b/products/slack_app/backend/tests/services/test_slack_settings.py
@@ -5,7 +5,10 @@ from posthog.models.organization import Organization
from posthog.models.team.team import Team
from products.slack_app.backend.models import SlackSettings, UntaggedFollowupMode
-from products.slack_app.backend.services.slack_settings import resolve_untagged_followup_mode
+from products.slack_app.backend.services.slack_settings import (
+ resolve_untagged_followup_mode,
+ resolve_untagged_question_mode,
+)
@pytest.fixture
@@ -52,3 +55,25 @@ class TestResolveUntaggedFollowupMode:
untagged_followup_mode=UntaggedFollowupMode.AUTO,
)
assert resolve_untagged_followup_mode(integration, "U001") == UntaggedFollowupMode.ASK
+
+
+class TestResolveUntaggedQuestionMode:
+ def test_unset_resolves_ask(self, slack_setup):
+ assert resolve_untagged_question_mode("T_WS", "U001") == UntaggedFollowupMode.ASK
+
+ @pytest.mark.parametrize(
+ "followup_mode,question_mode,expected",
+ [
+ # Each column answers only its own question.
+ (UntaggedFollowupMode.AUTO, None, UntaggedFollowupMode.ASK),
+ (UntaggedFollowupMode.NEVER, UntaggedFollowupMode.AUTO, UntaggedFollowupMode.AUTO),
+ ],
+ )
+ def test_the_followup_choice_does_not_leak(self, slack_setup, followup_mode, question_mode, expected):
+ SlackSettings.objects.create(
+ slack_workspace_id="T_WS",
+ slack_user_id="U001",
+ untagged_followup_mode=followup_mode,
+ untagged_question_mode=question_mode,
+ )
+ assert resolve_untagged_question_mode("T_WS", "U001") == expected
diff --git a/products/slack_app/backend/tests/test_thread_message_routing.py b/products/slack_app/backend/tests/test_thread_message_routing.py
index e62bd29ec3f3..768943194ad2 100644
--- a/products/slack_app/backend/tests/test_thread_message_routing.py
+++ b/products/slack_app/backend/tests/test_thread_message_routing.py
@@ -121,18 +121,31 @@ class TestRouteThreadMessage(TestCase):
defaults.update(overrides)
return defaults
- def _route(self, event: dict, slack_team_id: str = "T_SLACK") -> str:
+ def _make_question_event(self, **overrides) -> dict:
+ defaults = {
+ "channel": "C002",
+ "channel_type": "channel",
+ "ts": "2000.0000",
+ "thread_ts": None,
+ "text": "How many people signed up last week?",
+ }
+ return self._make_event(**{**defaults, **overrides})
+
+ def _route(self, event: dict, slack_team_id: str = "T_SLACK", *, is_ext_shared_channel: bool = False) -> str:
from products.slack_app.backend.api import route_posthog_code_event_to_relevant_region
request = self.factory.post("/slack/event-callback/", HTTP_HOST="us.posthog.com")
- return route_posthog_code_event_to_relevant_region(request, event, slack_team_id)
+ return route_posthog_code_event_to_relevant_region(
+ request, event, slack_team_id, is_ext_shared_channel=is_ext_shared_channel
+ )
# --- Cheap pre-DB gates ------------------------------------------------
def test_top_level_message_dropped_before_db(self):
"""A message with no ``thread_ts`` (or where ``thread_ts == ts``) is a
- top-level post in the channel, not a thread reply. Drop before
- touching the DB — channel chatter dominates wire volume."""
+ top-level post in the channel, not a thread reply. One that fails the cheap
+ question gates is dropped before touching the DB, because channel chatter
+ dominates wire volume."""
from products.slack_app.backend.api import ROUTE_HANDLED_LOCALLY
event = self._make_event(thread_ts="1001.0000") # same as ts
@@ -145,6 +158,66 @@ class TestRouteThreadMessage(TestCase):
mock_filter.assert_not_called()
mock_start.assert_not_called()
+ @parameterized.expand(
+ [
+ ("an untagged question", {}, True, None, False, True),
+ ("a private channel", {"channel_type": "group"}, True, None, False, True),
+ ("a statement", {"text": "Shipped the new export filter today"}, True, None, False, False),
+ ("a question to a person", {"text": "<@U_ALICE> how many signups last week?"}, True, None, False, False),
+ ("a group DM", {"channel_type": "mpim"}, True, None, False, False),
+ ("the flag is off", {}, False, None, False, False),
+ ("the author picks up only tagged messages", {}, True, UntaggedFollowupMode.NEVER, False, False),
+ ("an author without a PostHog account", {"user": "U_UNKNOWN"}, True, None, False, False),
+ ("an externally shared channel", {}, True, None, True, False),
+ ]
+ )
+ def test_top_level_question_starts_the_untagged_question_workflow(
+ self, _name, overrides, flag_on, author_mode, ext_shared, expect_dispatch
+ ):
+ if author_mode is not None:
+ SlackSettings.objects.create(
+ slack_workspace_id="T_SLACK", slack_user_id="U_BOB", untagged_question_mode=author_mode
+ )
+ event = self._make_question_event(**overrides)
+ with (
+ patch("products.slack_app.backend.api.is_slack_app_unprompted_answers_enabled", return_value=flag_on),
+ patch("products.slack_app.backend.api._start_mention_workflow") as mock_start,
+ patch("products.slack_app.backend.api._post_user_resolution_failure_reply") as mock_failure_reply,
+ ):
+ self._route(event, is_ext_shared_channel=ext_shared)
+
+ assert mock_start.called is expect_dispatch
+ # Nobody asked, so a drop says nothing, an unknown author included.
+ mock_failure_reply.assert_not_called()
+ if expect_dispatch:
+ kwargs = mock_start.call_args.kwargs
+ assert kwargs["untagged_question"] is True
+ assert kwargs["posthog_user"].id == self.bob.id
+ assert mock_start.call_args.args[1].id == self.integration.id
+
+ @parameterized.expand(
+ [
+ ("the other region holds the workspace", True, True),
+ ("nobody confirms it", None, False),
+ ]
+ )
+ def test_top_level_question_for_a_workspace_held_elsewhere_crosses_only_when_claimed(
+ self, _name, claimed, expect_proxy
+ ):
+ event = self._make_question_event()
+ with (
+ patch("products.slack_app.backend.api.cross_region_routing_enabled", return_value=True),
+ patch("products.slack_app.backend.api.emit_slack_message_event", return_value=False),
+ patch("products.slack_app.backend.api._mirror_message_event_to_other_region"),
+ patch("products.slack_app.backend.api.does_other_region_claim_workspace", return_value=claimed),
+ patch("products.slack_app.backend.api._proxy_event_and_return_route", return_value="proxied") as proxy,
+ patch("products.slack_app.backend.api._start_mention_workflow") as mock_start,
+ ):
+ self._route(event, slack_team_id="T_ELSEWHERE")
+
+ assert proxy.called is expect_proxy
+ mock_start.assert_not_called()
+
def test_no_user_dropped(self):
from products.slack_app.backend.api import ROUTE_HANDLED_LOCALLY
diff --git a/products/slack_app/backend/tests/test_untagged_followup_interactivity.py b/products/slack_app/backend/tests/test_untagged_followup_interactivity.py
index b76cf41b16a5..a98c2e5b7cfc 100644
--- a/products/slack_app/backend/tests/test_untagged_followup_interactivity.py
+++ b/products/slack_app/backend/tests/test_untagged_followup_interactivity.py
@@ -20,6 +20,10 @@ from products.slack_app.backend.api import (
UNTAGGED_FOLLOWUP_ACTION_RUN,
UNTAGGED_FOLLOWUP_BLOCK_ID_PREFIX,
UNTAGGED_FOLLOWUP_CONTEXT_KIND,
+ UNTAGGED_QUESTION_ACTION_RUN,
+ UNTAGGED_QUESTION_ACTION_TURN_OFF,
+ UNTAGGED_QUESTION_BLOCK_ID_PREFIX,
+ UNTAGGED_QUESTION_CONTEXT_KIND,
_picker_context_cache_key,
)
from products.slack_app.backend.models import SlackSettings, SlackThreadTaskMapping, UntaggedFollowupMode
@@ -252,3 +256,159 @@ class TestUntaggedFollowupInteractivity(TestCase):
assert response.status_code == 200
mock_start.assert_not_called()
+
+
+@override_settings(DEBUG=True)
+@patch("products.slack_app.backend.api.requests.post")
+@patch("products.slack_app.backend.api.SlackIntegration")
+class TestUntaggedQuestionInteractivity(TestCase):
+ signing_secret = "posthog-code-test-secret"
+ slack_team_id = "T12345"
+ response_url = "https://hooks.slack.example/response/def"
+ context_token = "untagged-question-token-123"
+
+ def setUp(self):
+ cache.clear()
+ self.client = APIClient()
+ self.organization = Organization.objects.create(name="Test Org")
+ self.team = Team.objects.create(organization=self.organization, name="Test Team")
+ self.member_user = User.objects.create(email="[email protected]", distinct_id="user-member")
+ OrganizationMembership.objects.create(user=self.member_user, organization=self.organization)
+ self.integration = Integration.objects.create(
+ team=self.team,
+ kind="slack",
+ integration_id=self.slack_team_id,
+ sensitive_config={"access_token": "xoxb-test"},
+ )
+ self.event = {
+ "type": "message",
+ "channel": "C002",
+ "channel_type": "channel",
+ "user": "U_BOB",
+ "ts": "2000.0000",
+ "text": "How many people signed up last week?",
+ }
+ cache.set(
+ _picker_context_cache_key(self.context_token),
+ {
+ "kind": UNTAGGED_QUESTION_CONTEXT_KIND,
+ "integration_id": self.integration.id,
+ "slack_user_id": "U_BOB",
+ "event": self.event,
+ },
+ timeout=900,
+ )
+
+ def _click(
+ self,
+ action_id: str,
+ slack_user_id: str = "U_BOB",
+ *,
+ flag_on: bool = True,
+ start_fails: bool = False,
+ ext_shared: bool | None = False,
+ ) -> Any:
+ payload = {
+ "type": "block_actions",
+ "team": {"id": self.slack_team_id},
+ "user": {"id": slack_user_id},
+ "response_url": self.response_url,
+ "actions": [
+ {
+ "action_id": action_id,
+ "block_id": f"{UNTAGGED_QUESTION_BLOCK_ID_PREFIX}_actions:{self.context_token}",
+ "value": self.context_token,
+ }
+ ],
+ }
+ body_str = f"payload={json.dumps(payload)}"
+ signed = sign_slack_request(body_str.encode(), self.signing_secret)
+ with (
+ patch(
+ "products.slack_app.backend.api.get_slack_user_info",
+ return_value={"user": {"profile": {"email": self.member_user.email}}},
+ ),
+ patch(
+ "products.slack_app.backend.api._start_mention_workflow",
+ side_effect=RuntimeError("temporal down") if start_fails else None,
+ ) as mock_start,
+ patch("products.slack_app.backend.api.is_slack_app_unprompted_answers_enabled", return_value=flag_on),
+ patch("products.slack_app.backend.api.channel_ext_shared_state", return_value=ext_shared),
+ ):
+ self.client.post(
+ "/slack/interactivity-callback/",
+ data=body_str,
+ content_type="application/x-www-form-urlencoded",
+ headers={"x-slack-signature": signed.signature, "x-slack-request-timestamp": signed.timestamp},
+ )
+ return mock_start
+
+ def _stored_mode(self) -> str | None:
+ row = SlackSettings.objects.filter(slack_workspace_id=self.slack_team_id, slack_user_id="U_BOB").first()
+ return row.untagged_question_mode if row else None
+
+ def test_confirmation_in_a_channel_shared_since_the_offer_answers_nothing(self, mock_slack_cls, mock_post):
+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}
+ mock_slack_cls.return_value.missing_scopes.return_value = set()
+
+ assert not self._click(UNTAGGED_QUESTION_ACTION_RUN, ext_shared=True).called
+
+ def test_a_failed_sharing_lookup_keeps_the_offer(self, mock_slack_cls, mock_post):
+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}
+ mock_slack_cls.return_value.missing_scopes.return_value = set()
+
+ assert not self._click(UNTAGGED_QUESTION_ACTION_RUN, ext_shared=None).called
+ # The offer and its cached context survive, so the author can click again.
+ assert cache.get(_picker_context_cache_key(self.context_token)) is not None
+ assert not any("delete_original" in str(c) for c in mock_post.call_args_list)
+
+ def test_a_failed_dispatch_leaves_the_offer_clickable(self, mock_slack_cls, mock_post):
+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}
+ mock_slack_cls.return_value.missing_scopes.return_value = set()
+
+ with self.assertRaises(RuntimeError):
+ self._click(UNTAGGED_QUESTION_ACTION_RUN, start_fails=True)
+ assert cache.get(_picker_context_cache_key(self.context_token)) is not None
+
+ self._click(UNTAGGED_QUESTION_ACTION_RUN).assert_called_once()
+ assert cache.get(_picker_context_cache_key(self.context_token)) is None
+
+ def test_confirmation_answers_the_original_message(self, mock_slack_cls, mock_post):
+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}
+ mock_slack_cls.return_value.missing_scopes.return_value = set()
+
+ mock_start = self._click(UNTAGGED_QUESTION_ACTION_RUN)
+
+ mock_start.assert_called_once()
+ assert mock_start.call_args.args[0] == self.event
+ assert mock_start.call_args.kwargs["untagged_question"] is True
+ # Without this the re-dispatch would classify again and offer again.
+ assert mock_start.call_args.kwargs["untagged_question_confirmed"] is True
+ assert cache.get(_picker_context_cache_key(self.context_token)) is None
+
+ def test_click_from_anyone_but_the_author_answers_nothing(self, mock_slack_cls, mock_post):
+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}
+
+ assert not self._click(UNTAGGED_QUESTION_ACTION_RUN, slack_user_id="U_EVE").called
+
+ def test_stop_offering_turns_answers_off(self, mock_slack_cls, mock_post):
+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}
+
+ assert not self._click(UNTAGGED_QUESTION_ACTION_TURN_OFF).called
+ assert self._stored_mode() == UntaggedFollowupMode.NEVER
+ assert mock_post.call_args.kwargs["json"]["replace_original"] is True
+
+ def test_confirmation_after_the_author_turned_answers_off_answers_nothing(self, mock_slack_cls, mock_post):
+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}
+ SlackSettings.objects.create(
+ slack_workspace_id=self.slack_team_id,
+ slack_user_id="U_BOB",
+ untagged_question_mode=UntaggedFollowupMode.NEVER,
+ )
+
+ assert not self._click(UNTAGGED_QUESTION_ACTION_RUN).called
+
+ def test_confirmation_after_the_flag_was_turned_off_answers_nothing(self, mock_slack_cls, mock_post):
+ mock_slack_cls.slack_config.return_value = {"SLACK_APP_SIGNING_SECRET": self.signing_secret}
+
+ assert not self._click(UNTAGGED_QUESTION_ACTION_RUN, flag_on=False).called
diff --git a/products/slack_app/evals/eval_untagged_question.py b/products/slack_app/evals/eval_untagged_question.py
new file mode 100644
index 000000000000..66f077c862da
--- /dev/null
+++ b/products/slack_app/evals/eval_untagged_question.py
@@ -0,0 +1,263 @@
+"""Which top-level channel messages, posted without tagging the app, can PostHog answer?
+
+Every top-level message in a channel the app is in that looks like a question reaches
+`classify_untagged_question`, which asks Jev, a System One decision model, two yes/no
+questions. Both must clear `UNTAGGED_QUESTION_MIN_PROBABILITY`.
+
+Nobody asked PostHog anything, so `no_unasked_answer` is the number to watch: a wrong answer
+interrupts a conversation in public. A missed question costs the author one @PostHog.
+
+To run:
+ hogli evals eval_untagged_question
+ hogli evals eval_untagged_question --eval signups_last_week
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from products.posthog_ai.eval_harness.config import BaseEvalCase
+from products.posthog_ai.eval_harness.harness.context import EvalContext
+from products.posthog_ai.eval_harness.harness.requirements import SuiteKind
+from products.posthog_ai.eval_harness.one_shot import OneShotPublicEval
+from products.slack_app.backend.temporal.activities import untagged_question
+from products.slack_app.evals.scorers import UNTAGGED_QUESTION_KEY, NoUnaskedAnswer, UntaggedQuestionMatch
+
+SUITE_KIND = SuiteKind.ONE_SHOT
+
+
+def _answerable(answerable: bool) -> dict:
+ return {UNTAGGED_QUESTION_KEY: {"answerable": answerable}}
+
+
+ANSWERABLE_CASES = [
+ BaseEvalCase(name="signups_last_week", prompt="how many people signed up last week?", expected=_answerable(True)),
+ BaseEvalCase(
+ name="funnel_drop",
+ prompt="Why did conversion from signup to first upload drop so much yesterday?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="flag_rollout_state",
+ prompt="is the new onboarding flag rolled out to everyone yet or still 50%?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="errors_since_deploy",
+ prompt="Are we seeing more errors on the file upload page since this morning's deploy?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="feature_adoption",
+ prompt="what share of weekly active users actually used shared folders this month?",
+ expected=_answerable(True),
+ ),
+ # A metric written as its acronym is the same data question as the metric spelled out.
+ BaseEvalCase(
+ name="metric_acronym_with_greeting",
+ prompt="Hey posthog, what's the DAU for last week?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="metric_acronym_trend",
+ prompt="is MAU up or down vs the month before?",
+ expected=_answerable(True),
+ ),
+ # One case per kind of data the assistant can read, beyond the trends and errors above.
+ BaseEvalCase(
+ name="cohort_retention",
+ prompt="what does week 4 retention look like for people who signed up in March?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="session_recordings",
+ prompt="can we find recordings of users who rage-clicked the export button?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="experiment_result",
+ prompt="did the pricing page experiment reach significance yet?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="survey_answers",
+ prompt="what did people say in the NPS survey this month?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="llm_cost",
+ prompt="how much did our LLM calls cost yesterday?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="product_how_to",
+ prompt="how do I filter out internal users from my insights in PostHog?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="event_in_the_code",
+ prompt="where in our code do we fire the checkout_started event?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="error_spike_cause",
+ prompt="why did the TypeError on the dashboard page spike overnight?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="churn_after_change",
+ prompt="has churn gone up since we shortened the free trial?",
+ expected=_answerable(True),
+ ),
+ BaseEvalCase(
+ name="question_in_czech",
+ prompt="kolik lidí se včera zaregistrovalo?",
+ expected=_answerable(True),
+ ),
+]
+
+# Messages that look like questions, so they pass the webhook's cheap gate, and that PostHog
+# must leave alone.
+LEAVE_ALONE_CASES = [
+ BaseEvalCase(name="lunch_plan", prompt="lunch at 12:30 today?", expected=_answerable(False)),
+ BaseEvalCase(name="who_is_on_call", prompt="who's on call this weekend?", expected=_answerable(False)),
+ BaseEvalCase(
+ name="meeting_time", prompt="should we move standup to 10am from next week?", expected=_answerable(False)
+ ),
+ BaseEvalCase(
+ name="colleague_availability",
+ prompt="does anyone know when Priya is back from leave?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(name="opinion_poll", prompt="what do you all think of the new logo?", expected=_answerable(False)),
+ BaseEvalCase(
+ name="request_for_a_person",
+ prompt="can someone review my PR when you have a minute?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(name="small_talk", prompt="how's everyone doing after the offsite?", expected=_answerable(False)),
+ BaseEvalCase(
+ name="contract_question",
+ prompt="when does our contract with the payments vendor renew?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="rhetorical",
+ prompt="Why is it always the Friday deploys that break? Anyway, fixed now.",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="announcement_with_question",
+ prompt="Shipped the new pricing page today! Any feedback, drop it in the thread?",
+ expected=_answerable(False),
+ ),
+ # Acronyms that are not product metrics, so the shorthand rule must not reach them.
+ BaseEvalCase(
+ name="acronym_that_is_not_a_metric",
+ prompt="anyone know the ETA on the new laptops?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="hr_acronym",
+ prompt="what's our PTO policy over the holidays?",
+ expected=_answerable(False),
+ ),
+ # A metric named in a question that asks for an opinion, not for the number.
+ BaseEvalCase(
+ name="metric_in_an_opinion_question",
+ prompt="should we use DAU or WAU as our north star metric?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="strategy_question",
+ prompt="should we prioritize mobile or web next quarter?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="finance_question",
+ prompt="what's our burn rate looking like this quarter?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="sales_deal",
+ prompt="has the Northwind deal closed yet?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="it_outage",
+ prompt="is the VPN down for anyone else?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="too_vague",
+ prompt="any thoughts on this one?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="personal_favor",
+ prompt="can someone grab me a coffee on the way in?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="question_for_a_named_person",
+ prompt="Sam, what did you think of the demo yesterday?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="document_status",
+ prompt="is the Q3 roadmap doc ready for review?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="hiring_question",
+ prompt="how many open roles do we have on the growth team?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="company_calendar",
+ prompt="when is the next company holiday?",
+ expected=_answerable(False),
+ ),
+ BaseEvalCase(
+ name="outside_world",
+ prompt="what's the weather going to be like in London tomorrow?",
+ expected=_answerable(False),
+ ),
+]
+
+
+async def eval_untagged_question(ctx: EvalContext) -> None:
+ async def task(case: BaseEvalCase, task_ctx: EvalContext) -> dict:
+ model = untagged_question.UNTAGGED_QUESTION_DECISION_MODEL
+ if task_ctx.demo_data is None:
+ return {"decision_model": model, "answerable": None, "error": "No demo data team to bill the call to"}
+ try:
+ # Sync and blocking on the gateway, so off the event loop.
+ verdict = await asyncio.to_thread(
+ untagged_question.classify_untagged_question,
+ case.prompt,
+ team_id=task_ctx.demo_data.master_team_id,
+ distinct_id=None,
+ )
+ except Exception as error:
+ return {"decision_model": model, "answerable": None, "error": f"{type(error).__name__}: {error}"}
+ answerable = verdict is not None and verdict.answerable
+ return {
+ "decision_model": model,
+ "answerable": answerable,
+ "verdict": {
+ "asks_for_information": verdict.asks_for_information,
+ "answerable_by_posthog": verdict.answerable_by_posthog,
+ }
+ if verdict
+ else None,
+ "last_message": f"{model}: answerable={answerable}",
+ }
+
+ await OneShotPublicEval(
+ experiment_name="slack-app-untagged-question",
+ cases=[*ANSWERABLE_CASES, *LEAVE_ALONE_CASES],
+ scorers=[UntaggedQuestionMatch(), NoUnaskedAnswer()],
+ task=task,
+ ctx=ctx,
+ )
diff --git a/products/slack_app/evals/scorers.py b/products/slack_app/evals/scorers.py
index 6408bae68135..256013109429 100644
--- a/products/slack_app/evals/scorers.py
+++ b/products/slack_app/evals/scorers.py
@@ -464,3 +464,52 @@ Answer `yes` if every reply follows all of these rules. Answer `no` if one reply
max_completion_tokens=256,
**kwargs,
)
+
+
+UNTAGGED_QUESTION_KEY = "untagged_question"
+
+
+class UntaggedQuestionMatch(Scorer):
+ """Did the classifier decide about answering the way a person in the channel would?"""
+
+ def _name(self) -> str:
+ return UNTAGGED_QUESTION_KEY
+
+ def _run_eval_sync(self, output: dict | None, expected=None, **kwargs) -> Score:
+ want = (expected or {}).get(UNTAGGED_QUESTION_KEY) or {}
+ if "answerable" not in want:
+ return Score(name=self._name(), score=None, metadata={"reason": "No expectation for this case"})
+ if output and output.get("error"):
+ return Score(name=self._name(), score=0.0, metadata={"reason": output["error"]})
+
+ got = (output or {}).get("answerable")
+ return Score(
+ name=self._name(),
+ score=1.0 if got == want["answerable"] else 0.0,
+ metadata={"expected": want["answerable"], "actual": got, "verdict": (output or {}).get("verdict")},
+ )
+
+
+class NoUnaskedAnswer(Scorer):
+ """The expensive direction: answering a channel message PostHog cannot or should not answer.
+
+ Skips on cases PostHog should answer, so the score reads as a rate over the messages it
+ should have left alone.
+ """
+
+ def _name(self) -> str:
+ return "no_unasked_answer"
+
+ def _run_eval_sync(self, output: dict | None, expected=None, **kwargs) -> Score:
+ want = (expected or {}).get(UNTAGGED_QUESTION_KEY) or {}
+ if want.get("answerable", True):
+ return Score(name=self._name(), score=None, metadata={"reason": "Case is answerable"})
+ if output and output.get("error"):
+ # A failed call answers nothing, this scorer's passing answer, so it would hide an outage.
+ return Score(name=self._name(), score=None, metadata={"reason": output["error"]})
+
+ return Score(
+ name=self._name(),
+ score=0.0 if (output or {}).get("answerable") else 1.0,
+ metadata={"actual": (output or {}).get("answerable"), "verdict": (output or {}).get("verdict")},
+ )